diff --git a/app/nullius/execution.py b/app/nullius/execution.py index 6de6d3d9..4998f745 100644 --- a/app/nullius/execution.py +++ b/app/nullius/execution.py @@ -1674,25 +1674,45 @@ def active() -> List[Execution]: return [e for e in EXECUTIONS.values() if e.state in ALIVE] -def for_job(job_id: str) -> Optional[Execution]: +def owners() -> Dict[str, Execution]: + """Each job a plan names -> the newest execution whose script writes it; of + several started at once, the first listed.""" + out: Dict[str, Execution] = {} + for e in EXECUTIONS.values(): + for j in e.plan.jobs: + name = str(j.get("name")) + if name not in out or e.started > out[name].started: + out[name] = e + return out + + +def run_ends(owner: Optional[Dict[str, Execution]] = None) -> Dict[str, Optional[tuple]]: + """Each job a run writes -> its ``job_state`` once the whole run has ended, + None while the run is alive: the ``runs`` of ``workspace.load_jobs``. A + part's end is written down before the run's last copy from its host. + ``owner``: ``owners()``, when the caller holds it.""" + return {name: None if e.state in ALIVE else e.job_state(name) + for name, e in (owners() if owner is None else owner).items()} + + +def for_job(job_id: str, owner: Optional[Dict[str, Execution]] = None) -> Optional[Execution]: """The newest execution whose script writes this job, if any - the one the job page shows beside the job, or instead of it while Harbor has - not written it yet.""" - found = [e for e in EXECUTIONS.values() if any(str(j.get("name")) == job_id for j in e.plan.jobs)] - return max(found, key=lambda e: e.started) if found else None + not written it yet. ``owner``: ``owners()``, when the caller holds it.""" + return (owners() if owner is None else owner).get(job_id) ENDED = ("stopped", "failed", "done") -def ended_for(job_id: str) -> Optional[tuple]: +def ended_for(job_id: str, owner: Optional[Dict[str, Execution]] = None) -> Optional[tuple]: """``(state, finished_at)`` when the run that writes this job has ended - stopped, failed or done - by the runner's own record; None while it is alive or when no run of this application wrote the job. A Harbor killed by a Stop leaves its feed without ``job.end`` and its result record with trials "running", so a page reading either alone would say - the job goes on; this is the fact that says otherwise.""" - ex = for_job(job_id) + the job goes on; this is the fact that says otherwise. ``owner``: as ``for_job``'s.""" + ex = for_job(job_id, owner) return ex.job_state(job_id) if ex is not None else None @@ -1762,6 +1782,37 @@ def size_of(path: str) -> int: return total +# Sizes of ended runs' folders, by folder: (what they were read at, bytes). +_SIZES: Dict[str, tuple] = {} +_SIZES_LOCK = threading.Lock() + + +def run_size(ex: "Execution") -> int: + """``size_of`` the run's folder, as the Launches page lists it: walked at + each ask while the run is alive; once it has ended, again only when its + state, its record, its folder or its jobs folder moved. What Delete asks + (``deletion``) walks the folder afresh. + shortcut: a file changed deep in an ended run alone (a trace view saved on + its trial page) is counted from the next move of those; walk the job folders too if that matters.""" + if ex.state in ALIVE: + return size_of(ex.dir) + + def mtime(path): + try: + return os.stat(path).st_mtime_ns + except OSError: + return None + seen = (ex.state, ex.finished, *map(mtime, (ex.dir, os.path.join(ex.dir, RECORD), ex.jobs))) + with _SIZES_LOCK: + known = _SIZES.get(ex.dir) + if known is not None and known[0] == seen: + return known[1] + size = size_of(ex.dir) + with _SIZES_LOCK: + _SIZES[ex.dir] = (seen, size) + return size + + def _deletable(name: str, root: Optional[str]) -> tuple: """A run's folder and record, when it is a run that has ended; ValueError (alive, or not a run) or OSError (its record unreadable) otherwise.""" @@ -1781,7 +1832,7 @@ def deletable(names: List[str], root: Optional[str] = None) -> None: _deletable(n, root) -def _run_deletion(name: str, home: str, root: Optional[str], holding: Callable) -> dict: +def _run_deletion(name: str, home: str, root: Optional[str], holding: Callable, ends: dict) -> dict: path, rec = _deletable(name, root) plan = rec.get("plan") or {} # a run refused at the name check never made the host folder: one of that name there is not its own @@ -1789,7 +1840,7 @@ def _run_deletion(name: str, home: str, root: Optional[str], holding: Callable) held = [] for ws, roots in holding(path): dirs = parse.find_job_dirs(roots) - ids = [j.job_id for j in workspace.load_jobs(ws, roots)] + ids = [j.job_id for j in workspace.load_jobs(ws, roots, ends)] # a job of the same id from another root shares these judgements: they stay others = {e.get("job_id") for d, e in workspace.job_index(ws).items() if d not in dirs} judged = judgements.job_dirs(ws.dir, [i for i in ids if i.split("~")[0] not in others]) @@ -1807,7 +1858,8 @@ def deletion(names: List[str], home: str, root: Optional[str] = None, holding: O a run: nothing goes then. ``holding``: a run folder's ``workspace.holding``, when the caller has it already.""" holding = holding or (lambda path: workspace.holding(home, path)) - runs = [_run_deletion(n, home, root, holding) for n in dict.fromkeys(names)] + ends = run_ends() + runs = [_run_deletion(n, home, root, holding, ends) for n in dict.fromkeys(names)] copies: Dict[str, int] = {} for r in runs: if r["host"]: diff --git a/app/nullius/workspace.py b/app/nullius/workspace.py index 2020aa5f..a8f9a9e2 100644 --- a/app/nullius/workspace.py +++ b/app/nullius/workspace.py @@ -19,6 +19,16 @@ still going changes its files, the signature moves, the job is parsed again. A cached job that was parsed by an older version of ``parse`` is refreshed too — the signature carries ``parse.SCHEMA`` so a change there invalidates. + +A job that has ended (``_ended``: its result records a finish and nothing +running or pending) is checked by its *head* alone: its own result.json and +config.json, and its folder, whose mtime moves when a trial folder comes or +goes. A Harbor resume rewrites result.json, so it is caught; a file changed +inside a trial of an ended job and nothing else is not. The workspace's +Refresh (``clear_cache``) reads every job again. A job a run of this +application writes is checked trial by trial while that run goes on (a host +run's copy can bring a trial's files after the job's own), and its head +carries the run's end, so the last copy is checked trial by trial once more. """ from __future__ import annotations @@ -311,6 +321,25 @@ def stat(path): return hashlib.sha256("\n".join(parts).encode()).hexdigest(), trials +def _head(job_dir: str) -> str: + """What an ended job's parse can change by without a trial changing in place: + its own records, and its folder's mtime (a trial folder added or removed).""" + parts = [f"schema={SCHEMA}/{parse.SCHEMA}"] + for path in (job_dir, os.path.join(job_dir, "result.json"), os.path.join(job_dir, "config.json")): + try: + st = os.stat(path) + except OSError: + parts.append("-") + continue + parts.append(f"{st.st_size}:{st.st_mtime_ns}") + return "\n".join(parts) + + +def _ended(job: parse.Job) -> bool: + """Its result records a finish and nothing running or pending: its trials are final.""" + return job.finished_at is not None and job.state == "finished" + + def _encode(obj): if isinstance(obj, datetime): return {"$dt": obj.isoformat()} @@ -381,37 +410,70 @@ def _write_index(ws: Workspace, index: dict) -> None: _write_atomically(_index_path(ws), lambda fh: json.dump(index, fh, ensure_ascii=False, indent=1)) -# Jobs already decoded in this process, by cache file: (signature, Job). A page -# that loads the workspace then skips the JSON decode and the rebuild of every -# unchanged job. A Job is never changed once loaded (parse.split_agents and the -# compare page make new ones with dataclasses.replace). +# Jobs already decoded in this process, by cache file: (signature, Job, trial +# signatures, head). A page that loads the workspace then skips the JSON decode +# and the rebuild of every unchanged job. A Job is never changed once loaded +# (parse.split_agents and the compare page make new ones with dataclasses.replace). _DECODED: dict = {} _DECODED_LOCK = threading.Lock() -def load_job(ws: Workspace, job_dir: str, index: dict | None = None) -> tuple: - """(Job, from_cache). Parses only when the signature changed. ``index``: - the rail's index as load_jobs holds it, noted in place.""" - sig, trial_sigs = _signatures(job_dir) +def _read_cache(path: str, job_dir: str) -> tuple | None: + """(signature, head, Job) of the cache file *path* when it is *job_dir*'s, else None.""" + try: + with open(path, encoding="utf-8") as fh: + cached = json.load(fh, object_hook=_decode) + # "agents": a cache from before split_agents knows nothing of a job's several agents + if cached.get("dir") == job_dir and "agents" in cached.get("job", {}): + return cached.get("signature"), cached.get("head"), job_from_dict(cached["job"]) + except (OSError, json.JSONDecodeError, TypeError, KeyError, ValueError, AttributeError): + pass + return None + + +def _write_cache(ws: Workspace, path: str, sig: str, head: str, job: parse.Job) -> None: + os.makedirs(ws.jobs_dir, exist_ok=True) + _write_atomically(path, lambda fh: json.dump({"signature": sig, "head": head, "dir": job.dir, "cached": _now(), + "job": job_to_dict(job)}, fh, default=_encode, ensure_ascii=False)) + + +def load_job(ws: Workspace, job_dir: str, index: dict | None = None, runs: dict | None = None) -> tuple: + """(Job, from_cache). Parses only when the signature changed; an ended job + whose head is unchanged is not stated trial by trial. ``index``: the + rail's index as load_jobs holds it, noted in place. ``runs``: job folder + name -> the end of the run of this application that writes it, None while + that run goes on; a job no run writes has no entry.""" + run = (runs or {}).get(os.path.basename(job_dir), "") + short = run is not None # the head alone may answer + head = _head(job_dir) + (f"\nrun={run}" if run else "") # before anything is read: a change after it is caught next time path = _cache_file(ws, job_dir) with _DECODED_LOCK: known = _DECODED.get(path) + if short and known is not None and known[1].dir == job_dir and known[3] == head and _ended(known[1]): + _note_in_index(ws, job_dir, known[1], index) + return known[1], True + cached = _read_cache(path, job_dir) if known is None else None + if short and cached is not None and cached[1] == head and _ended(cached[2]): + with _DECODED_LOCK: + _DECODED[path] = (cached[0], cached[2], {}, head) + _note_in_index(ws, job_dir, cached[2], index) + return cached[2], True + sig, trial_sigs = _signatures(job_dir) if sig: if known is not None and known[0] == sig and known[1].dir == job_dir: + with _DECODED_LOCK: + _DECODED[path] = (sig, known[1], known[2], head) _note_in_index(ws, job_dir, known[1], index) return known[1], True - try: - with open(path, encoding="utf-8") as fh: - cached = json.load(fh, object_hook=_decode) - # "agents": a cache from before split_agents knows nothing of a job's several agents - if cached.get("signature") == sig and cached.get("dir") == job_dir and "agents" in cached.get("job", {}): - job = job_from_dict(cached["job"]) - with _DECODED_LOCK: - _DECODED[path] = (sig, job, trial_sigs) - _note_in_index(ws, job_dir, job, index) - return job, True - except (OSError, json.JSONDecodeError, TypeError, KeyError, ValueError): - pass + if known is not None: + cached = _read_cache(path, job_dir) + if cached is not None and cached[0] == sig: + if cached[1] != head and _ended(cached[2]): + _write_cache(ws, path, sig, head, cached[2]) # read by its head from the next process on + with _DECODED_LOCK: + _DECODED[path] = (sig, cached[2], trial_sigs, head) + _note_in_index(ws, job_dir, cached[2], index) + return cached[2], True reuse = None if known is not None and known[1].dir == job_dir: before = {t.dir: t for t in known[1].trials} @@ -420,23 +482,22 @@ def load_job(ws: Workspace, job_dir: str, index: dict | None = None) -> tuple: reuse = lambda sub: before.get(sub) if old_sigs.get(sub) and old_sigs.get(sub) == trial_sigs.get(sub) else None job = parse.parse_job(job_dir, reuse=reuse) if sig: - os.makedirs(ws.jobs_dir, exist_ok=True) - _write_atomically(path, lambda fh: json.dump({"signature": sig, "dir": job_dir, "cached": _now(), - "job": job_to_dict(job)}, fh, default=_encode, ensure_ascii=False)) + _write_cache(ws, path, sig, head, job) with _DECODED_LOCK: - _DECODED[path] = (sig, job, trial_sigs) + _DECODED[path] = (sig, job, trial_sigs, head) _note_in_index(ws, job_dir, job, index) return job, False -def load_jobs(ws: Workspace, roots) -> list: +def load_jobs(ws: Workspace, roots, runs: dict | None = None) -> list: """Every job under *roots*, each from the cache when its files are - unchanged. Same order and de-duplication as ``parse.load_all``.""" + unchanged. Same order and de-duplication as ``parse.load_all``. + ``runs``: as ``load_job``'s.""" # the rail's index read once and written once, not once per job: re-reading it for # each made a workspace of n jobs cost n parses of an index of n entries index = job_index(ws) before = dict(index) - jobs = [load_job(ws, d, index)[0] for d in parse.find_job_dirs(roots)] + jobs = [load_job(ws, d, index, runs)[0] for d in parse.find_job_dirs(roots)] if index != before: _write_index(ws, index) return parse.split_agents(jobs) diff --git a/app/templates/compare/compare.html b/app/templates/compare/compare.html index 0dffc29c..0ececc2a 100644 --- a/app/templates/compare/compare.html +++ b/app/templates/compare/compare.html @@ -146,10 +146,10 @@ {% endif %}