Skip to content
72 changes: 62 additions & 10 deletions app/nullius/execution.py
Original file line number Diff line number Diff line change
Expand Up @@ -1674,25 +1674,45 @@ def active() -> List[Execution]:
return [e for e in EXECUTIONS.values() if e.state in ALIVE]


def for_job(job_id: str) -> Optional[Execution]:
def owners() -> Dict[str, Execution]:
"""Each job a plan names -> the newest execution whose script writes it; of
several started at once, the first listed."""
out: Dict[str, Execution] = {}
for e in EXECUTIONS.values():
for j in e.plan.jobs:
name = str(j.get("name"))
if name not in out or e.started > out[name].started:
out[name] = e
return out


def run_ends(owner: Optional[Dict[str, Execution]] = None) -> Dict[str, Optional[tuple]]:
"""Each job a run writes -> its ``job_state`` once the whole run has ended,
None while the run is alive: the ``runs`` of ``workspace.load_jobs``. A
part's end is written down before the run's last copy from its host.
``owner``: ``owners()``, when the caller holds it."""
return {name: None if e.state in ALIVE else e.job_state(name)
for name, e in (owners() if owner is None else owner).items()}


def for_job(job_id: str, owner: Optional[Dict[str, Execution]] = None) -> Optional[Execution]:
"""The newest execution whose script writes this job, if any - the one
the job page shows beside the job, or instead of it while Harbor has
not written it yet."""
found = [e for e in EXECUTIONS.values() if any(str(j.get("name")) == job_id for j in e.plan.jobs)]
return max(found, key=lambda e: e.started) if found else None
not written it yet. ``owner``: ``owners()``, when the caller holds it."""
return (owners() if owner is None else owner).get(job_id)


ENDED = ("stopped", "failed", "done")


def ended_for(job_id: str) -> Optional[tuple]:
def ended_for(job_id: str, owner: Optional[Dict[str, Execution]] = None) -> Optional[tuple]:
"""``(state, finished_at)`` when the run that writes this job has ended
- stopped, failed or done - by the runner's own record; None while it
is alive or when no run of this application wrote the job. A Harbor
killed by a Stop leaves its feed without ``job.end`` and its result
record with trials "running", so a page reading either alone would say
the job goes on; this is the fact that says otherwise."""
ex = for_job(job_id)
the job goes on; this is the fact that says otherwise. ``owner``: as ``for_job``'s."""
ex = for_job(job_id, owner)
return ex.job_state(job_id) if ex is not None else None


Expand Down Expand Up @@ -1762,6 +1782,37 @@ def size_of(path: str) -> int:
return total


# Sizes of ended runs' folders, by folder: (what they were read at, bytes).
_SIZES: Dict[str, tuple] = {}
_SIZES_LOCK = threading.Lock()


def run_size(ex: "Execution") -> int:
"""``size_of`` the run's folder, as the Launches page lists it: walked at
each ask while the run is alive; once it has ended, again only when its
state, its record, its folder or its jobs folder moved. What Delete asks
(``deletion``) walks the folder afresh.
shortcut: a file changed deep in an ended run alone (a trace view saved on
its trial page) is counted from the next move of those; walk the job folders too if that matters."""
if ex.state in ALIVE:
return size_of(ex.dir)

def mtime(path):
try:
return os.stat(path).st_mtime_ns
except OSError:
return None
seen = (ex.state, ex.finished, *map(mtime, (ex.dir, os.path.join(ex.dir, RECORD), ex.jobs)))
with _SIZES_LOCK:
known = _SIZES.get(ex.dir)
if known is not None and known[0] == seen:
return known[1]
size = size_of(ex.dir)
with _SIZES_LOCK:
_SIZES[ex.dir] = (seen, size)
return size


def _deletable(name: str, root: Optional[str]) -> tuple:
"""A run's folder and record, when it is a run that has ended; ValueError
(alive, or not a run) or OSError (its record unreadable) otherwise."""
Expand All @@ -1781,15 +1832,15 @@ def deletable(names: List[str], root: Optional[str] = None) -> None:
_deletable(n, root)


def _run_deletion(name: str, home: str, root: Optional[str], holding: Callable) -> dict:
def _run_deletion(name: str, home: str, root: Optional[str], holding: Callable, ends: dict) -> dict:
path, rec = _deletable(name, root)
plan = rec.get("plan") or {}
# a run refused at the name check never made the host folder: one of that name there is not its own
host = hosts.parse(plan["host"]) if plan.get("host") and (rec.get("launched") or rec.get("step") not in ("connect", "name")) else None
held = []
for ws, roots in holding(path):
dirs = parse.find_job_dirs(roots)
ids = [j.job_id for j in workspace.load_jobs(ws, roots)]
ids = [j.job_id for j in workspace.load_jobs(ws, roots, ends)]
# a job of the same id from another root shares these judgements: they stay
others = {e.get("job_id") for d, e in workspace.job_index(ws).items() if d not in dirs}
judged = judgements.job_dirs(ws.dir, [i for i in ids if i.split("~")[0] not in others])
Expand All @@ -1807,7 +1858,8 @@ def deletion(names: List[str], home: str, root: Optional[str] = None, holding: O
a run: nothing goes then. ``holding``: a run folder's ``workspace.holding``,
when the caller has it already."""
holding = holding or (lambda path: workspace.holding(home, path))
runs = [_run_deletion(n, home, root, holding) for n in dict.fromkeys(names)]
ends = run_ends()
runs = [_run_deletion(n, home, root, holding, ends) for n in dict.fromkeys(names)]
copies: Dict[str, int] = {}
for r in runs:
if r["host"]:
Expand Down
115 changes: 88 additions & 27 deletions app/nullius/workspace.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,16 @@
still going changes its files, the signature moves, the job is parsed again.
A cached job that was parsed by an older version of ``parse`` is refreshed
too — the signature carries ``parse.SCHEMA`` so a change there invalidates.

A job that has ended (``_ended``: its result records a finish and nothing
running or pending) is checked by its *head* alone: its own result.json and
config.json, and its folder, whose mtime moves when a trial folder comes or
goes. A Harbor resume rewrites result.json, so it is caught; a file changed
inside a trial of an ended job and nothing else is not. The workspace's
Refresh (``clear_cache``) reads every job again. A job a run of this
application writes is checked trial by trial while that run goes on (a host
run's copy can bring a trial's files after the job's own), and its head
carries the run's end, so the last copy is checked trial by trial once more.
"""
from __future__ import annotations

Expand Down Expand Up @@ -311,6 +321,25 @@ def stat(path):
return hashlib.sha256("\n".join(parts).encode()).hexdigest(), trials


def _head(job_dir: str) -> str:
"""What an ended job's parse can change by without a trial changing in place:
its own records, and its folder's mtime (a trial folder added or removed)."""
parts = [f"schema={SCHEMA}/{parse.SCHEMA}"]
for path in (job_dir, os.path.join(job_dir, "result.json"), os.path.join(job_dir, "config.json")):
try:
st = os.stat(path)
except OSError:
parts.append("-")
continue
parts.append(f"{st.st_size}:{st.st_mtime_ns}")
return "\n".join(parts)


def _ended(job: parse.Job) -> bool:
"""Its result records a finish and nothing running or pending: its trials are final."""
return job.finished_at is not None and job.state == "finished"


def _encode(obj):
if isinstance(obj, datetime):
return {"$dt": obj.isoformat()}
Expand Down Expand Up @@ -381,37 +410,70 @@ def _write_index(ws: Workspace, index: dict) -> None:
_write_atomically(_index_path(ws), lambda fh: json.dump(index, fh, ensure_ascii=False, indent=1))


# Jobs already decoded in this process, by cache file: (signature, Job). A page
# that loads the workspace then skips the JSON decode and the rebuild of every
# unchanged job. A Job is never changed once loaded (parse.split_agents and the
# compare page make new ones with dataclasses.replace).
# Jobs already decoded in this process, by cache file: (signature, Job, trial
# signatures, head). A page that loads the workspace then skips the JSON decode
# and the rebuild of every unchanged job. A Job is never changed once loaded
# (parse.split_agents and the compare page make new ones with dataclasses.replace).
_DECODED: dict = {}
_DECODED_LOCK = threading.Lock()


def load_job(ws: Workspace, job_dir: str, index: dict | None = None) -> tuple:
"""(Job, from_cache). Parses only when the signature changed. ``index``:
the rail's index as load_jobs holds it, noted in place."""
sig, trial_sigs = _signatures(job_dir)
def _read_cache(path: str, job_dir: str) -> tuple | None:
"""(signature, head, Job) of the cache file *path* when it is *job_dir*'s, else None."""
try:
with open(path, encoding="utf-8") as fh:
cached = json.load(fh, object_hook=_decode)
# "agents": a cache from before split_agents knows nothing of a job's several agents
if cached.get("dir") == job_dir and "agents" in cached.get("job", {}):
return cached.get("signature"), cached.get("head"), job_from_dict(cached["job"])
except (OSError, json.JSONDecodeError, TypeError, KeyError, ValueError, AttributeError):
pass
return None


def _write_cache(ws: Workspace, path: str, sig: str, head: str, job: parse.Job) -> None:
os.makedirs(ws.jobs_dir, exist_ok=True)
_write_atomically(path, lambda fh: json.dump({"signature": sig, "head": head, "dir": job.dir, "cached": _now(),
"job": job_to_dict(job)}, fh, default=_encode, ensure_ascii=False))


def load_job(ws: Workspace, job_dir: str, index: dict | None = None, runs: dict | None = None) -> tuple:
"""(Job, from_cache). Parses only when the signature changed; an ended job
whose head is unchanged is not stated trial by trial. ``index``: the
rail's index as load_jobs holds it, noted in place. ``runs``: job folder
name -> the end of the run of this application that writes it, None while
that run goes on; a job no run writes has no entry."""
run = (runs or {}).get(os.path.basename(job_dir), "")
short = run is not None # the head alone may answer
head = _head(job_dir) + (f"\nrun={run}" if run else "") # before anything is read: a change after it is caught next time
path = _cache_file(ws, job_dir)
with _DECODED_LOCK:
known = _DECODED.get(path)
if short and known is not None and known[1].dir == job_dir and known[3] == head and _ended(known[1]):
_note_in_index(ws, job_dir, known[1], index)
return known[1], True
cached = _read_cache(path, job_dir) if known is None else None
if short and cached is not None and cached[1] == head and _ended(cached[2]):
with _DECODED_LOCK:
_DECODED[path] = (cached[0], cached[2], {}, head)
_note_in_index(ws, job_dir, cached[2], index)
return cached[2], True
sig, trial_sigs = _signatures(job_dir)
if sig:
if known is not None and known[0] == sig and known[1].dir == job_dir:
with _DECODED_LOCK:
_DECODED[path] = (sig, known[1], known[2], head)
_note_in_index(ws, job_dir, known[1], index)
return known[1], True
try:
with open(path, encoding="utf-8") as fh:
cached = json.load(fh, object_hook=_decode)
# "agents": a cache from before split_agents knows nothing of a job's several agents
if cached.get("signature") == sig and cached.get("dir") == job_dir and "agents" in cached.get("job", {}):
job = job_from_dict(cached["job"])
with _DECODED_LOCK:
_DECODED[path] = (sig, job, trial_sigs)
_note_in_index(ws, job_dir, job, index)
return job, True
except (OSError, json.JSONDecodeError, TypeError, KeyError, ValueError):
pass
if known is not None:
cached = _read_cache(path, job_dir)
if cached is not None and cached[0] == sig:
if cached[1] != head and _ended(cached[2]):
_write_cache(ws, path, sig, head, cached[2]) # read by its head from the next process on
with _DECODED_LOCK:
_DECODED[path] = (sig, cached[2], trial_sigs, head)
_note_in_index(ws, job_dir, cached[2], index)
return cached[2], True
reuse = None
if known is not None and known[1].dir == job_dir:
before = {t.dir: t for t in known[1].trials}
Expand All @@ -420,23 +482,22 @@ def load_job(ws: Workspace, job_dir: str, index: dict | None = None) -> tuple:
reuse = lambda sub: before.get(sub) if old_sigs.get(sub) and old_sigs.get(sub) == trial_sigs.get(sub) else None
job = parse.parse_job(job_dir, reuse=reuse)
if sig:
os.makedirs(ws.jobs_dir, exist_ok=True)
_write_atomically(path, lambda fh: json.dump({"signature": sig, "dir": job_dir, "cached": _now(),
"job": job_to_dict(job)}, fh, default=_encode, ensure_ascii=False))
_write_cache(ws, path, sig, head, job)
with _DECODED_LOCK:
_DECODED[path] = (sig, job, trial_sigs)
_DECODED[path] = (sig, job, trial_sigs, head)
_note_in_index(ws, job_dir, job, index)
return job, False


def load_jobs(ws: Workspace, roots) -> list:
def load_jobs(ws: Workspace, roots, runs: dict | None = None) -> list:
"""Every job under *roots*, each from the cache when its files are
unchanged. Same order and de-duplication as ``parse.load_all``."""
unchanged. Same order and de-duplication as ``parse.load_all``.
``runs``: as ``load_job``'s."""
# the rail's index read once and written once, not once per job: re-reading it for
# each made a workspace of n jobs cost n parses of an index of n entries
index = job_index(ws)
before = dict(index)
jobs = [load_job(ws, d, index)[0] for d in parse.find_job_dirs(roots)]
jobs = [load_job(ws, d, index, runs)[0] for d in parse.find_job_dirs(roots)]
if index != before:
_write_index(ws, index)
return parse.split_agents(jobs)
Expand Down
6 changes: 3 additions & 3 deletions app/templates/compare/compare.html
Original file line number Diff line number Diff line change
Expand Up @@ -146,10 +146,10 @@
{% endif %}
<tr>
<th{{ classes(METRIC, true, 'rowhead') }}>Outcome mix</th>
{% call(i) each() %}{% set j = jobs[i] %}
{% call(i) each() %}{% set j = jobs[i] %}{% set oc = j.counts %}
<td><span class="minibar">
{%- for k in ORDER %}{% if j.counts[k] %}<span class="seg oc-{{ k }}" style="flex:{{ j.counts[k] }}"
title="{{ LABELS[k] }}: {{ j.counts[k] }}"></span>{% endif %}{% endfor -%}
{%- for k in ORDER %}{% if oc[k] %}<span class="seg oc-{{ k }}" style="flex:{{ oc[k] }}"
title="{{ LABELS[k] }}: {{ oc[k] }}"></span>{% endif %}{% endfor -%}
</span></td>
{% endcall %}
</tr>
Expand Down
2 changes: 1 addition & 1 deletion app/templates/runs/feasibility.html
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@

<div class="legend feas-summary" role="group" aria-label="Feasibility classes">
<span class="feas-headline" id="feasibility-count"><strong>{{ report.n_feasible }}</strong><span class="faint">/{{ report.rows|length }}</span> feasible</span>
{% for k in F_ORDER %}{% if report.counts[k] %}<span class="chip fv-{{ k }}">{{ F_LABELS[k] }} <b>{{ report.counts[k] }}</b></span>{% endif %}{% endfor %}
{% set oc = report.counts %}{% for k in F_ORDER %}{% if oc[k] %}<span class="chip fv-{{ k }}">{{ F_LABELS[k] }} <b>{{ oc[k] }}</b></span>{% endif %}{% endfor %}
</div>

{% if report.rows %}
Expand Down
Loading
Loading