diff --git a/app/nullius/execution.py b/app/nullius/execution.py index 6de6d3d9..4998f745 100644 --- a/app/nullius/execution.py +++ b/app/nullius/execution.py @@ -1674,25 +1674,45 @@ def active() -> List[Execution]: return [e for e in EXECUTIONS.values() if e.state in ALIVE] -def for_job(job_id: str) -> Optional[Execution]: +def owners() -> Dict[str, Execution]: + """Each job a plan names -> the newest execution whose script writes it; of + several started at once, the first listed.""" + out: Dict[str, Execution] = {} + for e in EXECUTIONS.values(): + for j in e.plan.jobs: + name = str(j.get("name")) + if name not in out or e.started > out[name].started: + out[name] = e + return out + + +def run_ends(owner: Optional[Dict[str, Execution]] = None) -> Dict[str, Optional[tuple]]: + """Each job a run writes -> its ``job_state`` once the whole run has ended, + None while the run is alive: the ``runs`` of ``workspace.load_jobs``. A + part's end is written down before the run's last copy from its host. + ``owner``: ``owners()``, when the caller holds it.""" + return {name: None if e.state in ALIVE else e.job_state(name) + for name, e in (owners() if owner is None else owner).items()} + + +def for_job(job_id: str, owner: Optional[Dict[str, Execution]] = None) -> Optional[Execution]: """The newest execution whose script writes this job, if any - the one the job page shows beside the job, or instead of it while Harbor has - not written it yet.""" - found = [e for e in EXECUTIONS.values() if any(str(j.get("name")) == job_id for j in e.plan.jobs)] - return max(found, key=lambda e: e.started) if found else None + not written it yet. ``owner``: ``owners()``, when the caller holds it.""" + return (owners() if owner is None else owner).get(job_id) ENDED = ("stopped", "failed", "done") -def ended_for(job_id: str) -> Optional[tuple]: +def ended_for(job_id: str, owner: Optional[Dict[str, Execution]] = None) -> Optional[tuple]: """``(state, finished_at)`` when the run that writes this job has ended - stopped, failed or done - by the runner's own record; None while it is alive or when no run of this application wrote the job. A Harbor killed by a Stop leaves its feed without ``job.end`` and its result record with trials "running", so a page reading either alone would say - the job goes on; this is the fact that says otherwise.""" - ex = for_job(job_id) + the job goes on; this is the fact that says otherwise. ``owner``: as ``for_job``'s.""" + ex = for_job(job_id, owner) return ex.job_state(job_id) if ex is not None else None @@ -1762,6 +1782,37 @@ def size_of(path: str) -> int: return total +# Sizes of ended runs' folders, by folder: (what they were read at, bytes). +_SIZES: Dict[str, tuple] = {} +_SIZES_LOCK = threading.Lock() + + +def run_size(ex: "Execution") -> int: + """``size_of`` the run's folder, as the Launches page lists it: walked at + each ask while the run is alive; once it has ended, again only when its + state, its record, its folder or its jobs folder moved. What Delete asks + (``deletion``) walks the folder afresh. + shortcut: a file changed deep in an ended run alone (a trace view saved on + its trial page) is counted from the next move of those; walk the job folders too if that matters.""" + if ex.state in ALIVE: + return size_of(ex.dir) + + def mtime(path): + try: + return os.stat(path).st_mtime_ns + except OSError: + return None + seen = (ex.state, ex.finished, *map(mtime, (ex.dir, os.path.join(ex.dir, RECORD), ex.jobs))) + with _SIZES_LOCK: + known = _SIZES.get(ex.dir) + if known is not None and known[0] == seen: + return known[1] + size = size_of(ex.dir) + with _SIZES_LOCK: + _SIZES[ex.dir] = (seen, size) + return size + + def _deletable(name: str, root: Optional[str]) -> tuple: """A run's folder and record, when it is a run that has ended; ValueError (alive, or not a run) or OSError (its record unreadable) otherwise.""" @@ -1781,7 +1832,7 @@ def deletable(names: List[str], root: Optional[str] = None) -> None: _deletable(n, root) -def _run_deletion(name: str, home: str, root: Optional[str], holding: Callable) -> dict: +def _run_deletion(name: str, home: str, root: Optional[str], holding: Callable, ends: dict) -> dict: path, rec = _deletable(name, root) plan = rec.get("plan") or {} # a run refused at the name check never made the host folder: one of that name there is not its own @@ -1789,7 +1840,7 @@ def _run_deletion(name: str, home: str, root: Optional[str], holding: Callable) held = [] for ws, roots in holding(path): dirs = parse.find_job_dirs(roots) - ids = [j.job_id for j in workspace.load_jobs(ws, roots)] + ids = [j.job_id for j in workspace.load_jobs(ws, roots, ends)] # a job of the same id from another root shares these judgements: they stay others = {e.get("job_id") for d, e in workspace.job_index(ws).items() if d not in dirs} judged = judgements.job_dirs(ws.dir, [i for i in ids if i.split("~")[0] not in others]) @@ -1807,7 +1858,8 @@ def deletion(names: List[str], home: str, root: Optional[str] = None, holding: O a run: nothing goes then. ``holding``: a run folder's ``workspace.holding``, when the caller has it already.""" holding = holding or (lambda path: workspace.holding(home, path)) - runs = [_run_deletion(n, home, root, holding) for n in dict.fromkeys(names)] + ends = run_ends() + runs = [_run_deletion(n, home, root, holding, ends) for n in dict.fromkeys(names)] copies: Dict[str, int] = {} for r in runs: if r["host"]: diff --git a/app/nullius/workspace.py b/app/nullius/workspace.py index 2020aa5f..a8f9a9e2 100644 --- a/app/nullius/workspace.py +++ b/app/nullius/workspace.py @@ -19,6 +19,16 @@ still going changes its files, the signature moves, the job is parsed again. A cached job that was parsed by an older version of ``parse`` is refreshed too — the signature carries ``parse.SCHEMA`` so a change there invalidates. + +A job that has ended (``_ended``: its result records a finish and nothing +running or pending) is checked by its *head* alone: its own result.json and +config.json, and its folder, whose mtime moves when a trial folder comes or +goes. A Harbor resume rewrites result.json, so it is caught; a file changed +inside a trial of an ended job and nothing else is not. The workspace's +Refresh (``clear_cache``) reads every job again. A job a run of this +application writes is checked trial by trial while that run goes on (a host +run's copy can bring a trial's files after the job's own), and its head +carries the run's end, so the last copy is checked trial by trial once more. """ from __future__ import annotations @@ -311,6 +321,25 @@ def stat(path): return hashlib.sha256("\n".join(parts).encode()).hexdigest(), trials +def _head(job_dir: str) -> str: + """What an ended job's parse can change by without a trial changing in place: + its own records, and its folder's mtime (a trial folder added or removed).""" + parts = [f"schema={SCHEMA}/{parse.SCHEMA}"] + for path in (job_dir, os.path.join(job_dir, "result.json"), os.path.join(job_dir, "config.json")): + try: + st = os.stat(path) + except OSError: + parts.append("-") + continue + parts.append(f"{st.st_size}:{st.st_mtime_ns}") + return "\n".join(parts) + + +def _ended(job: parse.Job) -> bool: + """Its result records a finish and nothing running or pending: its trials are final.""" + return job.finished_at is not None and job.state == "finished" + + def _encode(obj): if isinstance(obj, datetime): return {"$dt": obj.isoformat()} @@ -381,37 +410,70 @@ def _write_index(ws: Workspace, index: dict) -> None: _write_atomically(_index_path(ws), lambda fh: json.dump(index, fh, ensure_ascii=False, indent=1)) -# Jobs already decoded in this process, by cache file: (signature, Job). A page -# that loads the workspace then skips the JSON decode and the rebuild of every -# unchanged job. A Job is never changed once loaded (parse.split_agents and the -# compare page make new ones with dataclasses.replace). +# Jobs already decoded in this process, by cache file: (signature, Job, trial +# signatures, head). A page that loads the workspace then skips the JSON decode +# and the rebuild of every unchanged job. A Job is never changed once loaded +# (parse.split_agents and the compare page make new ones with dataclasses.replace). _DECODED: dict = {} _DECODED_LOCK = threading.Lock() -def load_job(ws: Workspace, job_dir: str, index: dict | None = None) -> tuple: - """(Job, from_cache). Parses only when the signature changed. ``index``: - the rail's index as load_jobs holds it, noted in place.""" - sig, trial_sigs = _signatures(job_dir) +def _read_cache(path: str, job_dir: str) -> tuple | None: + """(signature, head, Job) of the cache file *path* when it is *job_dir*'s, else None.""" + try: + with open(path, encoding="utf-8") as fh: + cached = json.load(fh, object_hook=_decode) + # "agents": a cache from before split_agents knows nothing of a job's several agents + if cached.get("dir") == job_dir and "agents" in cached.get("job", {}): + return cached.get("signature"), cached.get("head"), job_from_dict(cached["job"]) + except (OSError, json.JSONDecodeError, TypeError, KeyError, ValueError, AttributeError): + pass + return None + + +def _write_cache(ws: Workspace, path: str, sig: str, head: str, job: parse.Job) -> None: + os.makedirs(ws.jobs_dir, exist_ok=True) + _write_atomically(path, lambda fh: json.dump({"signature": sig, "head": head, "dir": job.dir, "cached": _now(), + "job": job_to_dict(job)}, fh, default=_encode, ensure_ascii=False)) + + +def load_job(ws: Workspace, job_dir: str, index: dict | None = None, runs: dict | None = None) -> tuple: + """(Job, from_cache). Parses only when the signature changed; an ended job + whose head is unchanged is not stated trial by trial. ``index``: the + rail's index as load_jobs holds it, noted in place. ``runs``: job folder + name -> the end of the run of this application that writes it, None while + that run goes on; a job no run writes has no entry.""" + run = (runs or {}).get(os.path.basename(job_dir), "") + short = run is not None # the head alone may answer + head = _head(job_dir) + (f"\nrun={run}" if run else "") # before anything is read: a change after it is caught next time path = _cache_file(ws, job_dir) with _DECODED_LOCK: known = _DECODED.get(path) + if short and known is not None and known[1].dir == job_dir and known[3] == head and _ended(known[1]): + _note_in_index(ws, job_dir, known[1], index) + return known[1], True + cached = _read_cache(path, job_dir) if known is None else None + if short and cached is not None and cached[1] == head and _ended(cached[2]): + with _DECODED_LOCK: + _DECODED[path] = (cached[0], cached[2], {}, head) + _note_in_index(ws, job_dir, cached[2], index) + return cached[2], True + sig, trial_sigs = _signatures(job_dir) if sig: if known is not None and known[0] == sig and known[1].dir == job_dir: + with _DECODED_LOCK: + _DECODED[path] = (sig, known[1], known[2], head) _note_in_index(ws, job_dir, known[1], index) return known[1], True - try: - with open(path, encoding="utf-8") as fh: - cached = json.load(fh, object_hook=_decode) - # "agents": a cache from before split_agents knows nothing of a job's several agents - if cached.get("signature") == sig and cached.get("dir") == job_dir and "agents" in cached.get("job", {}): - job = job_from_dict(cached["job"]) - with _DECODED_LOCK: - _DECODED[path] = (sig, job, trial_sigs) - _note_in_index(ws, job_dir, job, index) - return job, True - except (OSError, json.JSONDecodeError, TypeError, KeyError, ValueError): - pass + if known is not None: + cached = _read_cache(path, job_dir) + if cached is not None and cached[0] == sig: + if cached[1] != head and _ended(cached[2]): + _write_cache(ws, path, sig, head, cached[2]) # read by its head from the next process on + with _DECODED_LOCK: + _DECODED[path] = (sig, cached[2], trial_sigs, head) + _note_in_index(ws, job_dir, cached[2], index) + return cached[2], True reuse = None if known is not None and known[1].dir == job_dir: before = {t.dir: t for t in known[1].trials} @@ -420,23 +482,22 @@ def load_job(ws: Workspace, job_dir: str, index: dict | None = None) -> tuple: reuse = lambda sub: before.get(sub) if old_sigs.get(sub) and old_sigs.get(sub) == trial_sigs.get(sub) else None job = parse.parse_job(job_dir, reuse=reuse) if sig: - os.makedirs(ws.jobs_dir, exist_ok=True) - _write_atomically(path, lambda fh: json.dump({"signature": sig, "dir": job_dir, "cached": _now(), - "job": job_to_dict(job)}, fh, default=_encode, ensure_ascii=False)) + _write_cache(ws, path, sig, head, job) with _DECODED_LOCK: - _DECODED[path] = (sig, job, trial_sigs) + _DECODED[path] = (sig, job, trial_sigs, head) _note_in_index(ws, job_dir, job, index) return job, False -def load_jobs(ws: Workspace, roots) -> list: +def load_jobs(ws: Workspace, roots, runs: dict | None = None) -> list: """Every job under *roots*, each from the cache when its files are - unchanged. Same order and de-duplication as ``parse.load_all``.""" + unchanged. Same order and de-duplication as ``parse.load_all``. + ``runs``: as ``load_job``'s.""" # the rail's index read once and written once, not once per job: re-reading it for # each made a workspace of n jobs cost n parses of an index of n entries index = job_index(ws) before = dict(index) - jobs = [load_job(ws, d, index)[0] for d in parse.find_job_dirs(roots)] + jobs = [load_job(ws, d, index, runs)[0] for d in parse.find_job_dirs(roots)] if index != before: _write_index(ws, index) return parse.split_agents(jobs) diff --git a/app/templates/compare/compare.html b/app/templates/compare/compare.html index 0dffc29c..0ececc2a 100644 --- a/app/templates/compare/compare.html +++ b/app/templates/compare/compare.html @@ -146,10 +146,10 @@ {% endif %} Outcome mix - {% call(i) each() %}{% set j = jobs[i] %} + {% call(i) each() %}{% set j = jobs[i] %}{% set oc = j.counts %} - {%- for k in ORDER %}{% if j.counts[k] %}{% endif %}{% endfor -%} + {%- for k in ORDER %}{% if oc[k] %}{% endif %}{% endfor -%} {% endcall %} diff --git a/app/templates/runs/feasibility.html b/app/templates/runs/feasibility.html index bf2cc289..c31b463f 100644 --- a/app/templates/runs/feasibility.html +++ b/app/templates/runs/feasibility.html @@ -74,7 +74,7 @@
{{ report.n_feasible }}/{{ report.rows|length }} feasible - {% for k in F_ORDER %}{% if report.counts[k] %}{{ F_LABELS[k] }} {{ report.counts[k] }}{% endif %}{% endfor %} + {% set oc = report.counts %}{% for k in F_ORDER %}{% if oc[k] %}{{ F_LABELS[k] }} {{ oc[k] }}{% endif %}{% endfor %}
{% if report.rows %} diff --git a/app/templates/runs/index.html b/app/templates/runs/index.html index 49316d72..57c9bdf7 100644 --- a/app/templates/runs/index.html +++ b/app/templates/runs/index.html @@ -14,10 +14,10 @@ {#- One row of a launch's table: a task check (`kind` 'control') or an agent; `part` 'fold' for a participant's shards as one row, 'shard' for one of them, '' for a run of one job. #} -{% macro launch_row(r, g, kind, part='', first=False) %}{% set L = g.launch %}{% set j = r.job %}{% set v = r.verdict %} +{% macro launch_row(r, g, kind, part='', first=False) %}{% set L = g.launch %}{% set j = r.job %}{% set v = r.verdict %}{% set oc = j.counts if j else none %} + {%- else %} data-outcomes="{% if j %}{% for k in ORDER %}{% if oc[k] %}{{ k }} {% endif %}{% endfor %}{% if j.n_regressions %}regression{% endif %}{% endif %}"{% endif %}> {%- if part == 'shard' %}
{{ shard_link(r) }}
{%- else %} @@ -45,9 +45,9 @@ {%- elif j and j.adjusted_solve_rate is not none %}
{{ j.adjusted_solve_rate|pct }}
{%- else %}
—
{% endif %} - {%- if j and j.trials and not j.scored_only and j.counts.unknown and j.counts.unknown * 4 >= j.trials|length %}⚠ {{ j.counts.unknown }} of {{ j.trials|length }} have no verdict{% endif %} + {%- if j and j.trials and not j.scored_only and oc.unknown and oc.unknown * 4 >= j.trials|length %}⚠ {{ oc.unknown }} of {{ j.trials|length }} have no verdict{% endif %} {%- endif %} - {% if j and j.trials %}{% for k in ORDER %}{% if j.counts[k] %}{% endif %}{% endfor %}{% else %}{% endif %} + {% if j and j.trials %}{% for k in ORDER %}{% if oc[k] %}{% endif %}{% endfor %}{% else %}{% endif %} {%- if kind == 'control' %} — {%- else %} @@ -178,8 +178,8 @@

{{ g.dataset_title if L.dataset else (L.name or g.dataset_title) }}{% if g.n {% for j in jobs %} {# A control row carries no outcomes to filter by: the chips count the agents' trials, and a nop's rejections are not failed verifications. #} - {% set v = verdicts[j.job_id] %} - {{ check_cell('Compare ' ~ j.job_id, j.job_id) }} @@ -213,8 +213,8 @@

{{ g.dataset_title if L.dataset else (L.name or g.dataset_title) }}{% if g.n {% else %} - - {%- for k in ORDER %}{% if j.counts[k] %}{% endif %}{% endfor -%} + + {%- for k in ORDER %}{% if oc[k] %}{% endif %}{% endfor -%} {% endif %} diff --git a/app/templates/runs/job.html b/app/templates/runs/job.html index 8c9d1570..ae65b060 100644 --- a/app/templates/runs/job.html +++ b/app/templates/runs/job.html @@ -6,7 +6,7 @@ {% block crumb %}/ {{ job.job_id }}{% endblock %} {% block head %} {% endblock %} -{% block body %} +{% block body %}{% set oc = job.counts %} {% set meta %}
@@ -227,14 +227,14 @@
Outcomes
- {% for k in ORDER %}{% if job.counts[k] %}{% endif %}{% endfor %} + {% for k in ORDER %}{% if oc[k] %}{% endif %}{% endfor %}
- {% for k in ORDER %}{% if job.counts[k] %}{% set h = 45 + 30 * ((nlines.get(k) or 1) - 1) %} + {% for k in ORDER %}{% if oc[k] %}{% set h = 45 + 30 * ((nlines.get(k) or 1) - 1) %} {{ job.counts[k] }}{{ LABELS[k] }}{% if k in INFRA %}infra{% endif %} + data-tip="{{ DESCRIPTIONS[k] }}">{{ oc[k] }}{{ LABELS[k] }}{% if k in INFRA %}infra{% endif %} {% endif %}{% endfor %}
{{ job.trials|length }}Trials{% if job.n_planned and job.n_planned != job.trials|length %} of {{ job.n_planned }} planned{% endif %}
@@ -341,7 +341,7 @@

Trials

{{ job.trials|length }}
OutcomeAll▾
- {% for k in ORDER %}{% if job.counts[k] %}{{ LABELS[k] }}{{ job.counts[k] }}{% endif %}{% endfor %} + {% for k in ORDER %}{% if oc[k] %}{{ LABELS[k] }}{{ oc[k] }}{% endif %}{% endfor %}
{% if judgement and judgement.causes %}
Judge causeAll▾ @@ -478,7 +478,7 @@

Trials

Definitionsoutcomes · rates · judge
- {% for k in ORDER %}{% if job.counts[k] %}
{{ LABELS[k] }}{% if k in INFRA %} infra{% endif %}
{{ DESCRIPTIONS[k] }}
{% endif %}{% endfor %} + {% for k in ORDER %}{% if oc[k] %}
{{ LABELS[k] }}{% if k in INFRA %} infra{% endif %}
{{ DESCRIPTIONS[k] }}
{% endif %}{% endfor %}
Adjusted rate
Solves divided by the trials that actually ran: it drops the outcomes marked infra, which failed before the agent could be judged. The raw rate is Harbor's mean over every trial, shown beside it only when the two differ. A run in which no trial carries a verdict shows —, never 0%.
{% if not control %}
Solved meter
One unit per trial that ran, solves filled; an infra trial is a hollow unit after a gap. Past 12 trials the units become one bar. A hovered trial row lights its unit.
{% endif %}
Tokens · Total
Under each trial's value a thin bar from zero, on one scale for the column: the largest trial fills its track. Total is the trial start to finish and includes overhead that Env, Agent and Verify do not.
diff --git a/app/tests/test_execution_stop.py b/app/tests/test_execution_stop.py index a0f8e0e3..8115c052 100644 --- a/app/tests/test_execution_stop.py +++ b/app/tests/test_execution_stop.py @@ -51,6 +51,21 @@ def test_a_stop_as_a_part_ends_is_never_paused_over(self): self.assertEqual(ex.ends["oracle"]["rc"], 3) # it did end, with its own result self.assertFalse((self.tmp / "r" / "run-next.log").exists()) + def test_a_stopped_part_s_job_is_written_until_the_run_ends(self): + """Its part's end is written down before the run's last copy from its + host: until the run itself ends, ``run_ends`` says the job is still written.""" + plan = execution.Plan(name="r", script="", parts=self.PARTS, jobs=[{"name": "r-oracle", "agent": "oracle"}]) + ex = execution.Execution(plan, root=str(self.tmp)) + seen = [] + class Seen(StopAsItEnds): + def __setitem__(self, key, value): + super().__setitem__(key, value) + seen.append((ex.state, execution.run_ends({"r-oracle": ex})["r-oracle"])) + ex.ends = Seen(ex) + self.wait(ex.start(), 5) + self.assertEqual(seen, [("stopping", None)]) + self.assertEqual(execution.run_ends({"r-oracle": ex})["r-oracle"][0], "failed") # the part's own result, once the run ended + def test_a_run_stopped_before_any_part_says_it_was_stopped(self): """Stopped as the target was made ready: no part ran, the run's exit is the one it had (0), and its last line says it was stopped and diff --git a/app/tests/test_run_delete.py b/app/tests/test_run_delete.py index dda38f8b..c710539e 100644 --- a/app/tests/test_run_delete.py +++ b/app/tests/test_run_delete.py @@ -171,6 +171,46 @@ def rmtree(path, *a, **kw): self.assertFalse(os.path.exists(os.path.join(self.root, "s1"))); self.assertTrue(os.path.isdir(s2)) +class RunSize(unittest.TestCase): + setUp, run_ = RunDelete.setUp, RunDelete.run_ + + def _grow(self): + with open(os.path.join(self.jobs, "job-a", "tau3-bench__t0__abc", "result.json"), "a") as fh: + fh.write(" " * 1000) + + def test_an_ended_run_is_walked_again_once_it_moves_and_delete_always_asks_afresh(self): + ex = execution.Execution(execution.Plan(name="r1", script="true"), root=self.root, record={"state": "done"}) + first = execution.run_size(ex) + self.assertEqual(first, execution.size_of(ex.dir)) + self._grow() # deep in a trial: the listing keeps what it read + self.assertEqual(execution.run_size(ex), first) + self.assertEqual(execution.deletion(["r1"], self.home, self.root)["bytes"], first + 1000) + _write_json(os.path.join(ex.dir, "nullius.log"), {}) # its folder moves: walked again + self.assertEqual(execution.run_size(ex), execution.size_of(ex.dir)) + + def test_a_run_still_going_is_walked_at_each_ask(self): + ex = execution.Execution(execution.Plan(name="r1", script="true"), root=self.root, record={"state": "running"}) + first = execution.run_size(ex) + self._grow() + self.assertEqual(execution.run_size(ex), first + 1000) + + +class Owners(unittest.TestCase): + def test_several_records_naming_a_job_give_it_the_newest_and_the_first_listed_of_a_tie(self): + def rec(name, started, *jobs): + plan = execution.Plan(name=name, script="true", jobs=[{"name": j} for j in jobs]) + return execution.Execution(plan, root=tempfile.gettempdir(), record={"id": name, "state": "done", "started": started}) + recs = [rec("old", 1, "a", "b"), rec("tie-1", 5, "b", "c"), rec("tie-2", 5, "b", "c"), rec("new", 9, "a"), rec("older", 3, "a", "c")] + with patch.dict(execution.EXECUTIONS, {e.id: e for e in recs}, clear=True): + owner = execution.owners() + for job in "abcd": + found = [e for e in recs if any(j["name"] == job for j in e.plan.jobs)] + before = max(found, key=lambda e: e.started) if found else None # for_job as it was: one scan per job + self.assertIs(owner.get(job), before) + self.assertIs(execution.for_job(job), before); self.assertIs(execution.for_job(job, owner), before) + self.assertEqual({j: e.id for j, e in owner.items()}, {"a": "new", "b": "tie-1", "c": "tie-1"}) + + class SizeOf(unittest.TestCase): def test_a_file_gone_since_the_walk_counts_nothing(self): d = tempfile.mkdtemp(); self.addCleanup(shutil.rmtree, d) diff --git a/app/tests/test_standard_shard.py b/app/tests/test_standard_shard.py index ebe97fc6..1762cfad 100644 --- a/app/tests/test_standard_shard.py +++ b/app/tests/test_standard_shard.py @@ -315,8 +315,7 @@ def test_a_fold_is_read_again_only_when_the_workspace_parses_a_shard_again(self) first = launches.fold([workspace.load_job(ws, d)[0] for d in dirs]) self.assertEqual(launches.fold([workspace.load_job(ws, d)[0] for d in dirs]), first) self.assertEqual(check.call_count, 1) - trial = os.path.join(dirs[1], sorted(n for n in os.listdir(dirs[1]) if n.startswith("t-"))[0], "result.json") - with open(trial, "a") as f: + with open(os.path.join(dirs[1], "result.json"), "a") as f: # an ended shard is read again by its own record f.write("\n") launches.fold([workspace.load_job(ws, d)[0] for d in dirs]) self.assertEqual(check.call_count, 2) diff --git a/app/tests/test_workspace.py b/app/tests/test_workspace.py index a40589a9..35f792db 100644 --- a/app/tests/test_workspace.py +++ b/app/tests/test_workspace.py @@ -296,6 +296,93 @@ def test_a_running_job_reads_only_the_trials_that_moved(self): self.assertEqual(len(again.trials), 3) self.assertIs(again.trials[0], first.trials[0]); self.assertIs(again.trials[1], first.trials[1]) + def _end(self, **stats): + _write_json(os.path.join(self.job_dir, "result.json"), { + "id": "j1", "started_at": "2026-09-04T01:00:00Z", "finished_at": "2026-09-04T01:05:00Z", + "stats": {"n_running_trials": 0, "n_pending_trials": 0, **stats}}) + + def _rewrite_trial(self, reward): + target = os.path.join(self.job_dir, "tau3-bench__t1__abc", "result.json") + with open(target, encoding="utf-8") as fh: + data = json.load(fh) + data["verifier_result"] = {"rewards": {"reward": reward}} + _write_json(target, data) + + def test_an_ended_job_is_not_stated_trial_by_trial(self): + self._end() + first, _ = workspace.load_job(self.ws, self.job_dir) + with patch.object(workspace, "_signatures", side_effect=AssertionError("trials stated")): + again, from_cache = workspace.load_job(self.ws, self.job_dir) + self.assertIs(again, first); self.assertTrue(from_cache) + workspace._DECODED.clear() # a new process: the file on disk, by its head + again, from_cache = workspace.load_job(self.ws, self.job_dir) + self.assertTrue(from_cache); self.assertEqual(again, first) + + def test_a_cache_from_before_the_head_learns_it_once(self): + self._end() + workspace.load_job(self.ws, self.job_dir) + path = workspace._cache_file(self.ws, self.job_dir) + with open(path, encoding="utf-8") as fh: + data = json.load(fh) + del data["head"] + _write_json(path, data) + workspace._DECODED.clear() + _, from_cache = workspace.load_job(self.ws, self.job_dir) # by its signature, the head written down + self.assertTrue(from_cache) + workspace._DECODED.clear() + with patch.object(workspace, "_signatures", side_effect=AssertionError("trials stated")): + _, from_cache = workspace.load_job(self.ws, self.job_dir) + self.assertTrue(from_cache) + + def test_an_ended_job_a_resume_rewrites_is_parsed_again(self): + self._end() + workspace.load_job(self.ws, self.job_dir) + self._rewrite_trial(1.0) + _write_json(os.path.join(self.job_dir, "result.json"), + {"id": "j1", "started_at": "2026-09-04T01:00:00Z", "stats": {"n_running_trials": 1}}) + job, from_cache = workspace.load_job(self.ws, self.job_dir) + self.assertFalse(from_cache) + self.assertEqual([t.reward for t in job.trials], [1.0, 1.0]) + self.assertTrue(job.is_running) + + def test_an_ended_job_that_gains_a_trial_is_parsed_again(self): + self._end() + workspace.load_job(self.ws, self.job_dir) + time.sleep(0.01) + shutil.copytree(os.path.join(self.job_dir, "tau3-bench__t1__abc"), os.path.join(self.job_dir, "tau3-bench__t9__abc")) + job, from_cache = workspace.load_job(self.ws, self.job_dir) + self.assertFalse(from_cache); self.assertEqual(len(job.trials), 3) + + def test_a_running_job_updates_with_each_trial(self): + self._end(n_running_trials=1) + workspace.load_job(self.ws, self.job_dir) + for reward in (1.0, 0.5): + time.sleep(0.01) + self._rewrite_trial(reward) # its result.json, config.json and folder as they were + job, from_cache = workspace.load_job(self.ws, self.job_dir) + self.assertFalse(from_cache); self.assertEqual(job.trials[1].reward, reward) + + def test_a_host_run_s_trial_copied_after_its_job_is_read_once_the_run_ends(self): + """A host run's copy can bring the job's final result.json with a trial's + file still mid-write, and the next copy correct the trial alone.""" + self._end() + workspace.load_job(self.ws, self.job_dir, runs={"job-a": None}) # the stale trial parsed, its run going on + time.sleep(0.01) + self._rewrite_trial(1.0) + done = {"job-a": ("done", "2026-09-04T01:06:00Z")} # marked done after its last copy + job, from_cache = workspace.load_job(self.ws, self.job_dir, runs=done) + self.assertFalse(from_cache); self.assertEqual(job.trials[1].reward, 1.0) + with patch.object(workspace, "_signatures", side_effect=AssertionError("trials stated")): + again, from_cache = workspace.load_job(self.ws, self.job_dir, runs=done) + self.assertIs(again, job); self.assertTrue(from_cache) + + def test_a_job_no_run_writes_is_read_by_its_head(self): + self._end() + first, _ = workspace.load_job(self.ws, self.job_dir, runs={"job-b": None}) + with patch.object(workspace, "_signatures", side_effect=AssertionError("trials stated")): + again, from_cache = workspace.load_job(self.ws, self.job_dir, runs={"job-b": None}) + self.assertIs(again, first); self.assertTrue(from_cache) + def test_index_lists_runs_without_a_parse(self): self.assertEqual(workspace.job_index(self.ws), {}) workspace.load_job(self.ws, self.job_dir) diff --git a/app/ui_tests/test_compare_kept.py b/app/ui_tests/test_compare_kept.py index 9d2f06fe..bb3a5134 100644 --- a/app/ui_tests/test_compare_kept.py +++ b/app/ui_tests/test_compare_kept.py @@ -47,7 +47,7 @@ def scored(task, groups, **kw): self.stack.enter_context(patch.object(workspace, "list_all", return_value=list(ws.values()))) self.stack.enter_context(patch.object(workspace, "job_index", return_value={})) self.stack.enter_context(patch.object(workspace, "load", side_effect=lambda home, name: ws.get(name))) - self.stack.enter_context(patch.object(workspace, "load_jobs", side_effect=lambda w, roots: jobs[w.name])) + self.stack.enter_context(patch.object(workspace, "load_jobs", side_effect=lambda w, roots, runs=None: jobs[w.name])) self.client = web.app.test_client() def get(self, path="/compare", **query): diff --git a/app/ui_tests/test_comparison_routes.py b/app/ui_tests/test_comparison_routes.py index 88a1539c..75d9893b 100644 --- a/app/ui_tests/test_comparison_routes.py +++ b/app/ui_tests/test_comparison_routes.py @@ -32,7 +32,7 @@ def setUp(self): self.stack.enter_context(patch.object(workspace, "job_index", return_value={})) self.stack.enter_context(patch.object(workspace, "load", side_effect=lambda home, name: {"active": self.active, "foreign": self.foreign}.get(name))) - self.stack.enter_context(patch.object(workspace, "load_jobs", side_effect=lambda ws, roots: self.jobs[ws.name])) + self.stack.enter_context(patch.object(workspace, "load_jobs", side_effect=lambda ws, roots, runs=None: self.jobs[ws.name])) for obj, name in ((web.shared, "open_workspace"), (web.shared, "_import_roots"), (workspace, "touch"), (workspace, "save")): self.stack.enter_context(patch.object(obj, name, side_effect=AssertionError(f"Unexpected workspace mutation: {name}"))) diff --git a/app/ui_tests/test_controls_apart.py b/app/ui_tests/test_controls_apart.py index 35120baf..cb1c33d5 100644 --- a/app/ui_tests/test_controls_apart.py +++ b/app/ui_tests/test_controls_apart.py @@ -76,7 +76,7 @@ def setUp(self): self.stack.enter_context(patch.object(workspace, "job_index", return_value={})) self.stack.enter_context(patch.object(workspace, "load", side_effect=lambda home, name: self.ws)) self.jobs = [self.agent, self.nop, self.oracle] - self.stack.enter_context(patch.object(workspace, "load_jobs", side_effect=lambda ws, roots: self.jobs)) + self.stack.enter_context(patch.object(workspace, "load_jobs", side_effect=lambda ws, roots, runs=None: self.jobs)) self.stack.enter_context(patch.object(web.shared, "_jobs_for_request", side_effect=lambda: self.jobs)) self.client = web.app.test_client() diff --git a/app/ui_tests/test_job_lookup.py b/app/ui_tests/test_job_lookup.py new file mode 100644 index 00000000..6d5d3fdd --- /dev/null +++ b/app/ui_tests/test_job_lookup.py @@ -0,0 +1,60 @@ +"""A single job's page reads that job's folder alone, and the whole workspace +on any doubt (``shared._job_for_request``), finding what ``_find_job`` over +every job finds.""" +from __future__ import annotations + +import os +import shutil +import tempfile +import unittest +from contextlib import ExitStack +from pathlib import Path +from unittest.mock import patch + +from nullius import workspace + +from .test_routes import web + +JOB = Path(__file__).resolve().parents[1] / "tests" / "fixtures" / "harbor-jobs" / "liveboard-oracle-x4" + + +class JobLookup(unittest.TestCase): + def setUp(self): + self.stack = ExitStack() + self.addCleanup(self.stack.close) + tmp = Path(self.stack.enter_context(tempfile.TemporaryDirectory(prefix="nullius-job-lookup-"))) + self.roots = [str(tmp / "a"), str(tmp / "b")] + shutil.copytree(JOB, tmp / "a" / "job-x") + shutil.copytree(JOB, tmp / "b" / "job-y") + self.stack.enter_context(patch.object(web.shared, "WS_HOME", str(tmp / "workspaces"))) + self.stack.enter_context(patch.dict(web.shared.STATE, {"workspace": None, "roots": [], "label": "", "job_roots": []}, clear=True)) + self.ws = workspace.create(web.shared.WS_HOME, self.roots, "lookup") + web.shared.open_workspace(self.ws) + self.whole = patch.object(workspace, "load_jobs", wraps=workspace.load_jobs) + + def lookup(self, job_id, query=""): + with web.app.test_request_context(f"/job/{job_id}{query}"), self.whole as whole: + return web.shared._job_for_request(job_id), whole.called + + def test_one_folder_of_that_name_is_read_alone(self): + job, whole = self.lookup("job-x") + self.assertFalse(whole) + self.assertEqual(job.dir, os.path.join(self.roots[0], "job-x")) + self.assertEqual(web.app.test_client().get("/job/job-x").status_code, 200) + + def test_another_workspace_asked_for_reads_the_whole_of_it(self): + job, whole = self.lookup("job-x", f"?workspace={self.ws.name}") + self.assertTrue(whole); self.assertEqual(job.dir, os.path.join(self.roots[0], "job-x")) + + def test_two_folders_of_that_name_read_the_whole_workspace_and_find_the_first(self): + shutil.copytree(JOB, os.path.join(self.roots[1], "job-x")) + job, whole = self.lookup("job-x") + self.assertTrue(whole); self.assertEqual(job.dir, os.path.join(self.roots[0], "job-x")) + + def test_no_such_folder_reads_the_whole_workspace(self): + job, whole = self.lookup("job-z") + self.assertTrue(whole); self.assertIsNone(job) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/ui_tests/test_launch_shards.py b/app/ui_tests/test_launch_shards.py index a17e82a1..298f9cf9 100644 --- a/app/ui_tests/test_launch_shards.py +++ b/app/ui_tests/test_launch_shards.py @@ -82,7 +82,7 @@ def setUp(self): for name, value in (("list_all", [ws]), ("job_index", {})): self.stack.enter_context(patch.object(workspace, name, return_value=value)) self.stack.enter_context(patch.object(workspace, "load", side_effect=lambda home, name: ws)) - self.stack.enter_context(patch.object(workspace, "load_jobs", side_effect=lambda w, roots: self.jobs)) + self.stack.enter_context(patch.object(workspace, "load_jobs", side_effect=lambda w, roots, runs=None: self.jobs)) self.stack.enter_context(patch.object(web.shared, "_jobs_for_request", side_effect=lambda: self.jobs)) self.stack.enter_context(patch.dict(execution.EXECUTIONS, {}, clear=True)) self.client = web.app.test_client() diff --git a/app/ui_tests/test_live_board.py b/app/ui_tests/test_live_board.py index 6415561b..cafcbf46 100644 --- a/app/ui_tests/test_live_board.py +++ b/app/ui_tests/test_live_board.py @@ -98,7 +98,7 @@ def setUp(self): self.stack.enter_context(patch.object(workspace, "job_index", return_value={})) self.stack.enter_context(patch.object(workspace, "load", side_effect=lambda home, name: self.ws)) self.stack.enter_context(patch.object(workspace, "load_jobs", - side_effect=lambda ws, roots: [self.running, self.ended, self.no_feed])) + side_effect=lambda ws, roots, runs=None: [self.running, self.ended, self.no_feed])) for obj, name in [(web.shared, "open_workspace"), (web.shared, "_import_roots"), (workspace, "save"), (workspace, "touch")]: self.stack.enter_context(patch.object(obj, name, side_effect=AssertionError(f"read-only route called {name}"))) self.client = web.app.test_client() @@ -156,7 +156,7 @@ def test_a_broken_feed_is_reported_not_fatal(self): complete = text[: text.rfind("\n") + 1] (broken / "events.jsonl").write_text(complete + "this is not json\n" + complete.splitlines()[0] + "\n", encoding="utf-8") job = parse.parse_job(str(broken)) - with patch.object(workspace, "load_jobs", side_effect=lambda ws, roots: [job]): + with patch.object(workspace, "load_jobs", side_effect=lambda ws, roots, runs=None: [job]): page = self.client.get("/job/broken-run"); self.addCleanup(page.close) self.assertEqual(page.status_code, 200) html = page.get_data(as_text=True) @@ -172,7 +172,7 @@ def test_live_route_reads_harbors_recorded_run(self): job_with_feed(self.root, "oracle-x4") shutil.copy(FIXTURE / "harbor-oracle-x4" / "events.jsonl", recorded / "events.jsonl") job = parse.parse_job(str(recorded)) - with patch.object(workspace, "load_jobs", side_effect=lambda ws, roots: [job]): + with patch.object(workspace, "load_jobs", side_effect=lambda ws, roots, runs=None: [job]): r = self.client.get("/job/oracle-x4/live"); self.addCleanup(r.close) self.assertEqual(r.status_code, 200) s = r.get_json() @@ -196,7 +196,7 @@ def test_a_launched_job_is_listed_as_preparing_and_its_page_waits_for_the_feed(s "datasets": [{"name": "tau3-demo", "n_tasks": 3}], "n_concurrent_trials": 3}) job = parse.parse_job(str(launched)) self.assertTrue(job.is_preparing) - with patch.object(workspace, "load_jobs", side_effect=lambda ws, roots: [job]): + with patch.object(workspace, "load_jobs", side_effect=lambda ws, roots, runs=None: [job]): runs = self.client.get("/runs"); self.addCleanup(runs.close) html = runs.get_data(as_text=True) self.assertIn(">preparing", html) @@ -257,7 +257,7 @@ def test_a_feed_with_only_job_prepare_puts_the_preparation_window_on_the_board(s (launched / "events.jsonl").write_text("\n".join(text[:2]) + "\n", encoding="utf-8") # as a poller saw it at 08:00:20 job = parse.parse_job(str(launched)) self.assertTrue(job.is_preparing) # no result record yet - with patch.object(workspace, "load_jobs", side_effect=lambda ws, roots: [job]): + with patch.object(workspace, "load_jobs", side_effect=lambda ws, roots, runs=None: [job]): r = self.client.get("/job/prepare-run/live"); self.addCleanup(r.close) snap = r.get_json() self.assertEqual((snap["state"], snap["running"], snap["preparing"]["stage"], snap["trials"]), ("preparing", False, "image", [])) @@ -291,7 +291,7 @@ def test_a_runner_s_arm_is_watched_on_its_board_then_read_as_results(self): (120, 60, 0.0004, 1, True, True)) self.assertIsNotNone(cases["lme-arm"].duration_min) self.assertEqual((cases["lme-arm"].runner_judge, cases["lme-arm"].questions_from), ("official gpt-4o · evaluate_qa.py", "longmemeval_s")) - with patch.object(workspace, "load_jobs", side_effect=lambda ws, roots: list(cases.values())): + with patch.object(workspace, "load_jobs", side_effect=lambda ws, roots, runs=None: list(cases.values())): for job_id, opens, base in (("lme-running", "live", ""), ("lme-arm", "results", "/job/lme-arm")): with self.subTest(job_id): page = self.client.get("/job/" + job_id); self.addCleanup(page.close) @@ -332,7 +332,7 @@ def test_a_task_scored_on_its_parts_shows_its_score_and_no_solve_rate(self): shutil.copytree(NATIVE / "parts-arm", arm) job = parse.parse_job(str(arm)) self.assertEqual(sorted((t.trial_name, t.outcome_label, t.score, t.n_parts) for t in job.trials), [("t1", "Scored", 0.5, 2), ("t2", "Scored", 1.0, 1)]) - with patch.object(workspace, "load_jobs", side_effect=lambda ws, roots: [job]): + with patch.object(workspace, "load_jobs", side_effect=lambda ws, roots, runs=None: [job]): page = self.client.get("/job/parts-arm"); self.addCleanup(page.close) html = page.get_data(as_text=True) self.assertEqual(page.status_code, 200) diff --git a/app/ui_tests/test_routes.py b/app/ui_tests/test_routes.py index 529eb82d..c29124bd 100644 --- a/app/ui_tests/test_routes.py +++ b/app/ui_tests/test_routes.py @@ -137,7 +137,7 @@ def setUp(self): self.stack.enter_context(patch.object(workspace, "load", side_effect=lambda home, name: {"active": self.active, "foreign": self.foreign}.get(name))) self.load_jobs = self.stack.enter_context(patch.object(workspace, "load_jobs", - side_effect=lambda ws, roots: [self.foreign_job if ws.name == "foreign" else self.active_job])) + side_effect=lambda ws, roots, runs=None: [self.foreign_job if ws.name == "foreign" else self.active_job])) for obj, name in [(web.shared, "open_workspace"), (web.shared, "_import_roots"), (workspace, "save"), (workspace, "touch")]: self.stack.enter_context(patch.object(obj, name, side_effect=AssertionError( diff --git a/app/ui_tests/test_runs_by_launch.py b/app/ui_tests/test_runs_by_launch.py index 10ffd688..570c7e53 100644 --- a/app/ui_tests/test_runs_by_launch.py +++ b/app/ui_tests/test_runs_by_launch.py @@ -60,7 +60,7 @@ def setUp(self): self.stack.enter_context(patch.object(workspace, "job_index", return_value={})) self.stack.enter_context(patch.object(workspace, "load", side_effect=lambda home, name: self.ws)) self.jobs = [self.later, self.strong, self.nop, self.weak, self.oracle] - self.stack.enter_context(patch.object(workspace, "load_jobs", side_effect=lambda ws, roots: self.jobs)) + self.stack.enter_context(patch.object(workspace, "load_jobs", side_effect=lambda ws, roots, runs=None: self.jobs)) self.stack.enter_context(patch.object(web.shared, "_jobs_for_request", side_effect=lambda: self.jobs)) self.stack.enter_context(patch.dict(execution.EXECUTIONS, {}, clear=True)) self.client = web.app.test_client() @@ -174,7 +174,7 @@ def test_a_later_paused_shard_has_a_resume_link(self): group = SimpleNamespace(job_ids=[], pending=["first", "second"], source="record", dataset="") first = SimpleNamespace(state="done", plan=SimpleNamespace(recipe={}, jobs=[]), refused=lambda _: None, never_ran=lambda _: False) second = SimpleNamespace(state="paused", plan=SimpleNamespace(recipe={}, jobs=[]), live_state=lambda _: "paused") - with patch.object(execution, "for_job", side_effect=lambda key: {"first": first, "second": second}[key]), \ + with patch("web.shared.run_for", side_effect=lambda key: {"first": first, "second": second}[key]), \ patch("web.runs._control_planned", return_value=""): self.assertEqual(_launch_view(group, {}, {}, {})["paused"], "second") diff --git a/app/web/builder.py b/app/web/builder.py index e190398f..001c5e6a 100644 --- a/app/web/builder.py +++ b/app/web/builder.py @@ -46,7 +46,7 @@ def new_run(): data["loadRecipe"] = ex.plan.recipe data["loadRun"] = run elif like: - job = shared._find_job(shared._jobs_for_request(), like) + job = shared._job_for_request(like) if job is None: abort(404) data["loadLike"] = {"job": job.job_id, "agent": job.agent_name, "launch": job.launch, diff --git a/app/web/judge.py b/app/web/judge.py index b83542d9..19e4601d 100644 --- a/app/web/judge.py +++ b/app/web/judge.py @@ -290,7 +290,7 @@ def judge_trial_route(job_id, trial_name): def judge_job_route(job_id): """Judge the job's unjudged trials with one judge, the form's ``lane`` (Decision unless it names the LLM judge): the click is the request.""" - job = shared._find_job(shared._jobs_for_request(), job_id) + job = shared._job_for_request(job_id) if job is None: abort(404) lane = request.form.get("lane") if request.form.get("lane") in client.LANES else "decision" diff --git a/app/web/judge_tasks.py b/app/web/judge_tasks.py index 786c76ee..960bf985 100644 --- a/app/web/judge_tasks.py +++ b/app/web/judge_tasks.py @@ -16,7 +16,7 @@ def judge_tasks_route(job_id): """Ask the instruction score of every task of the job that has none, on the click of the button.""" from . import judge - job = shared._find_job(shared._jobs_for_request(), job_id) + job = shared._job_for_request(job_id) if job is None: abort(404) todo = judge.tasks_todo(job) diff --git a/app/web/reconcile.py b/app/web/reconcile.py index 4bb71e78..a91ab65e 100644 --- a/app/web/reconcile.py +++ b/app/web/reconcile.py @@ -37,7 +37,7 @@ def view_of(job, threshold: float) -> dict: @app.route("/job//reconcile") def reconcile_page(job_id): - j = shared._find_job(shared._jobs_for_request(), job_id) + j = shared._job_for_request(job_id) if not j: abort(404) return render_template("runs/reconcile.html", job=j, view=view_of(j, threshold_requested()), diff --git a/app/web/runs.py b/app/web/runs.py index a5693077..63188775 100644 --- a/app/web/runs.py +++ b/app/web/runs.py @@ -51,7 +51,7 @@ def index(): workspaces=workspace.list_all(shared.WS_HOME)) # A Harbor killed by a Stop (or dead with the run) leaves its result # record saying trials are running; the runner's record says otherwise. - ended = {j.job_id: execution.ended_for(j.job_id) for j in jobs} + ended = {j.job_id: shared.ended_for(j.job_id) for j in jobs} # The header, the chips and the mix bar describe the agents' trials. A # control (nop, oracle) is a check on the tasks, not a competitor: it is # listed on its own line as behaved or not (`feasibility.summary_of`), @@ -112,12 +112,12 @@ def _grouped(ws=None): shows: the runs grouped by the launch that made them (nullius.runs.launches) - the builder's run record, else the same dataset within half an hour. ``ws``: another workspace's, read from its cache without opening it.""" - jobs = shared._load_jobs() if ws is None else workspace.load_jobs(ws, ws.job_roots or ws.roots) + jobs = shared._load_jobs() if ws is None else workspace.load_jobs(ws, ws.job_roots or ws.roots, shared.run_ends()) have = {j.job_id for j in jobs} # The run records are the app's, not the workspace's: only those launched # into this workspace's roots, or with a job already written here, belong. - roots = None if ws is None else list(ws.roots) + list(ws.job_roots) - mine = [e for e in execution.EXECUTIONS.values() if _launched_here(e, have, roots)] + real = _real_roots(None if ws is None else list(ws.roots) + list(ws.job_roots)) + mine = [e for e in execution.EXECUTIONS.values() if _launched_here(e, have, real)] merged = launches.merged_sources(jobs) jobs = [j for j in jobs if j.job_id not in merged] return jobs, have, mine, launches.group(jobs, {e.id: _record(e, merged) for e in mine}) @@ -151,16 +151,27 @@ def _tally(g, by_id) -> dict: return {"n_runs": len(written) + len(pending) - checks, "n_checks": checks} -def _launched_here(ex, have, roots=None) -> bool: - """A run record belongs to the open workspace (or one of ``roots``) when its - job root is one of the workspace's roots (a Run adopts it there) or one of - its jobs is here.""" +def _real_roots(roots=None) -> set: + """``roots`` (the open workspace's when None), links resolved: what ``_launched_here`` matches against.""" + if roots is None: + roots = list(shared.STATE.get("roots") or []) + list(shared.STATE.get("job_roots") or []) + return {os.path.realpath(r) for r in roots} + + +def _launched_here(ex, have, real) -> bool: + """A run record belongs to a workspace when its job root is one of the + workspace's roots, ``real`` (``_real_roots``; a Run adopts it there), or + one of its jobs is here, ``have``. Each record's root is resolved once per + request, however many workspaces ask.""" if any(str(x.get("name")) in have for x in ex.plan.jobs): return True jobs_dir = getattr(ex, "jobs", None) - if roots is None: - roots = list(shared.STATE.get("roots") or []) + list(shared.STATE.get("job_roots") or []) - return bool(jobs_dir) and os.path.realpath(jobs_dir) in {os.path.realpath(r) for r in roots} + if not jobs_dir: + return False + memo = shared.request_memo("real_jobs") + if jobs_dir not in memo: + memo[jobs_dir] = os.path.realpath(jobs_dir) + return memo[jobs_dir] in real # A launch's runs, in the order the block lists them: finished first, best @@ -180,7 +191,7 @@ def _state_of(job_id, j, ended): "running".""" if j is not None and ended.get(job_id): return _said(ended[job_id][0]) - ex = execution.for_job(job_id) + ex = shared.run_for(job_id) if ex is not None and ex.state in execution.ALIVE: return ex.live_state(job_id) if j is None: @@ -200,7 +211,7 @@ def _control_planned(job_id) -> str: """``nop`` or ``oracle`` when a job not written yet is one of the run's controls by its plan, else ''. A standard plan names a declared control by that word; its setups are ``original`` or ``model``, never one.""" - ex = execution.for_job(job_id) + ex = shared.run_for(job_id) return "" if ex is None else next((_control_of(j) for j in ex.plan.jobs if str(j.get("name")) == job_id), "") @@ -238,7 +249,7 @@ def _most_telling(states) -> str: def _host_of(job_id) -> str: - ex = execution.for_job(job_id) + ex = shared.run_for(job_id) return "" if ex is None else ex.plan.host.label if ex.plan.host else "this machine" @@ -311,7 +322,7 @@ def _launch_view(g, by_id, verdicts, ended) -> dict: rows += [{"job": None, "job_id": i, "state": _state_of(i, None, ended)} for i in g.pending if i not in dict(waiting)] for r in rows: if r["state"] == "refused": - r["why"] = execution.for_job(r["job_id"]).refused(r["job_id"]) + r["why"] = shared.run_for(r["job_id"]).refused(r["job_id"]) rows = _fold(rows, ended) rate = lambda r: r["job"].adjusted_solve_rate if r["job"] is not None and r["job"].adjusted_solve_rate is not None else -1.0 rows.sort(key=lambda r: (_STATE_ORDER.get(r["state"], 9), -rate(r), r["job_id"])) @@ -349,14 +360,14 @@ def _launch_view(g, by_id, verdicts, ended) -> dict: counts = {} for r in leaves(controls + rows): counts[r["state"]] = counts.get(r["state"], 0) + 1 - ex = execution.for_job((g.job_ids + g.pending)[0]) + ex = shared.run_for((g.job_ids + g.pending)[0]) return {"gone": _deletion(_runs_of(g)), "launch": g, "checks": checks, "verdict": verdict, "controls": controls, "checked": checked, "agents": rows, "n_tasks": n_tasks, "compare": finished, **_tally(g, by_id), "paused": next((i for i in g.job_ids + g.pending - if (execution.for_job(i) is not None and execution.for_job(i).state == "paused")), None), + if (shared.run_for(i) is not None and shared.run_for(i).state == "paused")), None), "summary": " · ".join(f"{n} {s}" for s, n in sorted(counts.items(), key=lambda kv: _STATE_ORDER.get(kv[0], 9))), "recipe": (g.job_ids + g.pending)[0] if g.source == "record" and ex is not None and ex.plan.recipe else None, "feasibility": checks[0][0].job_id if checks else None, @@ -372,7 +383,7 @@ def _roots_pulse() -> str: Nothing is parsed; this is a handful of stats per job.""" dirs = parse.find_job_dirs(shared.STATE["job_roots"] or []) have = {os.path.basename(d) for d in dirs} - marks = [(f"run:{e.id}", e.state, 0) for e in execution.EXECUTIONS.values() if _launched_here(e, have)] + marks = [(f"run:{e.id}", e.state, 0) for e in execution.EXECUTIONS.values() if _launched_here(e, have, _real_roots())] for d in dirs: feed = os.path.join(d, "events.jsonl") marks.append((os.path.basename(d), os.path.isfile(os.path.join(d, "result.json")), @@ -544,11 +555,12 @@ def upload(): @app.route("/job/", methods=["GET", "HEAD"]) def job(job_id): + # every job, not the one alone: its Delete (_deletable) walks the workspace anyway, and one folder first costs more j = shared._find_job(shared._jobs_for_request(), job_id) - ex = execution.for_job(job_id) + ex = shared.run_for(job_id) # this job's own state beside the run's: ended, or a part still to come while another runs own = ((execution.NOT_RUN, None) if ex is not None and ex.state not in execution.ALIVE and ex.never_ran(job_id) - else execution.ended_for(job_id) or ((ex.live_state(job_id), None) + else shared.ended_for(job_id) or ((ex.live_state(job_id), None) if ex is not None and ex.live_state(job_id) in ("queued", execution.NOT_RUN) else None)) if not j: # Started from the builder and not written by Harbor yet: the run's @@ -561,13 +573,14 @@ def job(job_id): return render_template("runs/job.html", job=j, live_board=board, view=_view_of(board, j), execution=ex, csrf=shared._csrf_token(), judgement=judge.job_summary(j) if j.trials else None, - control=feasibility.summary_of(j), ended=execution.ended_for(job_id), own=own, deletable=_deletable(job_id, ex), + control=feasibility.summary_of(j), ended=shared.ended_for(job_id), own=own, deletable=_deletable(job_id, ex), trial_rows=attempts.grouped(j), pass_k=attempts.for_job(j)) -def _runs_of(g, owner=execution.for_job) -> list: +def _runs_of(g, owner=None) -> list: """The runs Nullius made that hold every row of the launch *g*, and no job outside it; [] when a row is not one of theirs. ``owner``: a job's run.""" + owner = owner or shared.run_for ids = g.job_ids + g.pending runs = list({id(e): e for e in map(owner, ids)}.values()) if None in runs or any(not {str(x.get("name")) for x in e.plan.jobs} <= set(ids) for e in runs): @@ -641,7 +654,7 @@ def _held_launches() -> tuple: launches from it.""" recs = sorted(execution.EXECUTIONS.values(), key=lambda e: e.started) held = workspace.holding_all(shared.WS_HOME, {os.path.realpath(e.dir) for e in recs}) - owner = {str(x.get("name")): e for e in recs for x in e.plan.jobs} # the newest run naming a job, as for_job + owner = execution.owners() by_id, every = _every() for j in by_id.values(): # a merge beside its shards: their run's owner.setdefault(j.job_id, next((owner[m] for m in getattr(j, "merged_from", []) if m in owner), None)) @@ -655,7 +668,7 @@ def _inventory(g, runs, wss, by_id) -> dict: its Delete would take is asked only when it opens (``launch_ask``).""" return {"launch": g, "runs": runs, "why": _undeletable(runs), "shards": len(runs), "state": _most_telling(map(_run_state, runs)), **_tally(g, by_id), "machines": list(dict.fromkeys(e.plan.host.label if e.plan.host else "this machine" for e in runs)), - "bytes": sum(execution.size_of(e.dir) for e in runs), "workspaces": wss, + "bytes": sum(map(execution.run_size, runs)), "workspaces": wss, # the agents its jobs recorded, and those its records name for jobs not written yet "agents": sorted({by_id[i].agent_label for i in g.job_ids if by_id[i].agent_label} | {str(x["agent"]) for e in runs for x in e.plan.jobs if x.get("agent")})} @@ -799,7 +812,7 @@ def _live_board_state(j): if not live.has_feed(j.dir): return None try: - snap = live.snapshot(j.dir, ended=execution.ended_for(j.job_id), with_timeline=True) + snap = live.snapshot(j.dir, ended=shared.ended_for(j.job_id), with_timeline=True) except live.FeedError as e: # A malformed line in the middle of the feed is a broken writer, not a # race. The page must still render — the settled sections read @@ -820,11 +833,11 @@ def job_live(job_id): the cache on every request or, worse, hit it and show a board that stopped. Folding a few thousand lines is cheap enough to do every second. """ - j = shared._find_job(shared._jobs_for_request(), job_id) + j = shared._job_for_request(job_id) if not j: abort(404) if not live.has_feed(j.dir): - ended = execution.ended_for(job_id) + ended = shared.ended_for(job_id) if ended: # The preparing page must settle even if the runner never wrote a feed. response = jsonify({"state": "finished", "running": False, "ended_by": ended[0]}) @@ -832,7 +845,7 @@ def job_live(job_id): return response abort(404) try: - snap = live.snapshot(j.dir, ended=execution.ended_for(j.job_id), with_timeline=True) + snap = live.snapshot(j.dir, ended=shared.ended_for(j.job_id), with_timeline=True) except live.FeedError as e: response = jsonify({"error": f"events.jsonl is not a feed: {e}"}) response.status_code = 422 diff --git a/app/web/shared.py b/app/web/shared.py index 65521036..fb15f90d 100644 --- a/app/web/shared.py +++ b/app/web/shared.py @@ -12,6 +12,7 @@ from markupsafe import Markup, escape from werkzeug.exceptions import Forbidden from nullius import importers as harnesses +from nullius.runs import parse from nullius import execution from nullius.catalogue import catalog from nullius import workspace @@ -363,14 +364,46 @@ def _load_jobs(): ws = STATE["workspace"] if ws is None: return [] - return workspace.load_jobs(ws, STATE["job_roots"]) + return workspace.load_jobs(ws, STATE["job_roots"], run_ends()) + + +def request_memo(name: str) -> dict: + """A dict kept for the rest of this request under *name*; a new one outside a request.""" + return g.setdefault(f"_memo_{name}", {}) if has_request_context() else {} + + +def _owners() -> dict: + """``execution.owners``, once per request: a page asks it of every job it lists.""" + memo = request_memo("owners") + if "owners" not in memo: + memo["owners"] = execution.owners() + return memo["owners"] + + +def run_for(job_id): + """``execution.for_job`` over this request's ``_owners``.""" + return execution.for_job(job_id, _owners()) + + +def ended_for(job_id): + """``execution.ended_for`` over this request's ``_owners``.""" + return execution.ended_for(job_id, _owners()) + + +def run_ends(): + """``execution.run_ends`` over this request's ``_owners``, once per request: + the Workspaces page reads every workspace.""" + memo = request_memo("run_ends") + if "run_ends" not in memo: + memo["run_ends"] = execution.run_ends(_owners()) + return memo["run_ends"] def _jobs_for_workspace(name=None): """Read an indexed workspace without making it active or converting sources. Once per request and workspace: a page that asks for the jobs several times (the job page's controls, the rail, a summary) walks them once.""" - memo = g.setdefault("_jobs", {}) if has_request_context() else {} + memo = request_memo("jobs") key = name if name and not (STATE["workspace"] and name == STATE["workspace"].name) else "" if key in memo: return memo[key] @@ -380,7 +413,7 @@ def _jobs_for_workspace(name=None): ws = workspace.load(WS_HOME, name) if ws is None: abort(404) - jobs = workspace.load_jobs(ws, ws.job_roots or ws.roots) + jobs = workspace.load_jobs(ws, ws.job_roots or ws.roots, run_ends()) memo[key] = jobs return jobs @@ -414,6 +447,22 @@ def _find_job(jobs, job_id): return next((j for j in jobs if j.job_id == job_id), None) +def _job_for_request(job_id): + """``_find_job(_jobs_for_request(), job_id)``, reading only the job's own + folder when the open workspace has exactly one of that name (a view of one + of a job's agents is ``~``). The whole workspace on any + doubt: another workspace asked for, no such folder, or several.""" + ws = STATE["workspace"] + if ws is not None and not request.args.get("workspace") and "" not in request_memo("jobs"): + names = {job_id, job_id.rsplit("~", 1)[0]} + dirs = [d for d in parse.find_job_dirs(STATE["job_roots"]) if os.path.basename(d) in names] + if len(dirs) == 1: + job = _find_job(parse.split_agents([workspace.load_job(ws, dirs[0], runs=run_ends())[0]]), job_id) + if job is not None: + return job + return _find_job(_jobs_for_request(), job_id) + + def _safe_dest(base: str, rel: str) -> str | None: """Join *rel* onto *base*, refusing anything that escapes *base*.""" dest = os.path.normpath(os.path.join(base, rel)) diff --git a/app/web/traces.py b/app/web/traces.py index e41a4fa9..e490c409 100644 --- a/app/web/traces.py +++ b/app/web/traces.py @@ -111,7 +111,7 @@ def _trial_evidence(job_id, trial_name, tries=False): """Resolve URL identifiers through parsed runs, never as filesystem paths. ``tries``: an attempt's earlier try too, ``.try-N``: evidence to read, never a trial to judge.""" - job = shared._find_job(shared._jobs_for_request(), job_id) + job = shared._job_for_request(job_id) if job is None: abort(404) trial = _find_trial(job, trial_name) or (_find_try(job, trial_name) if tries else None)