diff --git a/app/nullius/importers/__init__.py b/app/nullius/importers/__init__.py index 9f8a4be6..6666c055 100644 --- a/app/nullius/importers/__init__.py +++ b/app/nullius/importers/__init__.py @@ -50,6 +50,7 @@ from __future__ import annotations import importlib +import os import pkgutil import shutil import tempfile @@ -136,5 +137,27 @@ def problem(path: str) -> str | None: return None +def write_if_changed(path: str, data: bytes) -> bool: + """Write *data* to *path* unless the file already holds it; True when + written. A conversion runs at each open of a workspace; an unchanged one + keeps its mtimes, and with them the parse cache that keys on them.""" + try: + with open(path, "rb") as fh: + if fh.read() == data: + return False + except OSError: + pass + with open(path, "wb") as fh: + fh.write(data) + return True + + +def mark_changed(job_dir: str, summaries: list) -> None: + """Move the job's own ``result.json`` when one of its trials was written: + an ended job's cached parse is checked by that file, not trial by trial.""" + if any(s["changed"] for s in summaries): + os.utime(os.path.join(job_dir, "result.json")) + + def labels() -> str: return ", ".join(getattr(m, "LABEL", name) for name, m in available().items()) diff --git a/app/nullius/importers/atif.py b/app/nullius/importers/atif.py index 56b2ac7f..f9b21c0c 100644 --- a/app/nullius/importers/atif.py +++ b/app/nullius/importers/atif.py @@ -70,6 +70,7 @@ import shutil from datetime import timezone +from nullius import importers from nullius.redact import redact from nullius.traces.trajectory import parse_ts @@ -273,6 +274,10 @@ def number(key): } +def _dump(path: str, data) -> bool: + return importers.write_if_changed(path, json.dumps(data, indent=2).encode()) + + def _sha256(path: str) -> str: digest = hashlib.sha256() with open(path, "rb") as fh: @@ -316,15 +321,15 @@ def convert_document(path: str, doc: dict, trial: str, job_dir: str) -> dict: "session_id": doc.get("session_id"), }, } - with open(os.path.join(trial_dir, "result.json"), "w", encoding="utf-8") as fh: - json.dump(result, fh, indent=2) + changed = _dump(os.path.join(trial_dir, "result.json"), result) # Byte for byte: the trial page reads what the producer wrote, and a # digest of the copy is a digest of the source. - shutil.copyfile(path, os.path.join(agent_dir, "trajectory.json")) + with open(path, "rb") as fh: + changed |= importers.write_if_changed(os.path.join(agent_dir, "trajectory.json"), fh.read()) return {"trial_dir": trial, "started_at": started, "finished_at": finished, - "agent_info": result["agent_info"]} + "agent_info": result["agent_info"], "changed": changed} @@ -346,30 +351,29 @@ def _write_job(docs: list, root: str, out_root: str, job_name: str, command: str # ordered as instants, not as text: one stamp with a zone and one without sort apart as strings starts = sorted((s["started_at"] for s in summaries if s["started_at"]), key=_instant) ends = sorted((s["finished_at"] for s in summaries if s["finished_at"]), key=_instant) - with open(os.path.join(job_dir, "result.json"), "w", encoding="utf-8") as fh: - json.dump({ - "started_at": starts[0] if starts else None, - "finished_at": ends[-1] if ends else None, - "n_total_trials": len(summaries), - "stats": {"n_errored_trials": 0, "n_running_trials": 0, - "n_pending_trials": 0, "n_retries": 0}, - }, fh, indent=2) + _dump(os.path.join(job_dir, "result.json"), { + "started_at": starts[0] if starts else None, + "finished_at": ends[-1] if ends else None, + "n_total_trials": len(summaries), + "stats": {"n_errored_trials": 0, "n_running_trials": 0, + "n_pending_trials": 0, "n_retries": 0}, + }) + importers.mark_changed(job_dir, summaries) # The first document's producer names the job; a tree that mixes # producers still shows each trial's own under agent_info. agent = next((s["agent_info"] for s in summaries if s["agent_info"]["name"]), summaries[0]["agent_info"]) - with open(os.path.join(job_dir, "config.json"), "w", encoding="utf-8") as fh: - json.dump({ - "job_name": job_name, - "harness": NAME, # provenance; the walker ignores unknown keys - "command": command or ( - f"PYTHONPATH=app python3 -m nullius.importers.cli {shlex.quote(root)} " - f"-o {shlex.quote(out_root)} --job-name {shlex.quote(job_name)}"), - "agents": [{"name": agent["name"], - "model_name": agent["model_info"]["name"]}], - "tasks": [{"name": f"{NAME}/{s['trial_dir']}"} for s in summaries], - }, fh, indent=2) + _dump(os.path.join(job_dir, "config.json"), { + "job_name": job_name, + "harness": NAME, # provenance; the walker ignores unknown keys + "command": command or ( + f"PYTHONPATH=app python3 -m nullius.importers.cli {shlex.quote(root)} " + f"-o {shlex.quote(out_root)} --job-name {shlex.quote(job_name)}"), + "agents": [{"name": agent["name"], + "model_name": agent["model_info"]["name"]}], + "tasks": [{"name": f"{NAME}/{s['trial_dir']}"} for s in summaries], + }) return job_dir diff --git a/app/nullius/importers/otel.py b/app/nullius/importers/otel.py index a7cdd02e..a186b77e 100644 --- a/app/nullius/importers/otel.py +++ b/app/nullius/importers/otel.py @@ -54,6 +54,7 @@ import shutil from datetime import datetime, timezone +from nullius import importers from nullius.redact import redact from nullius.traces import otel as reader from nullius.traces import trace @@ -139,10 +140,8 @@ def _iso(ns: int) -> str: return datetime.fromtimestamp(ns / 1e9, tz=timezone.utc).isoformat().replace("+00:00", "Z") -def _dump(path: str, data) -> None: - with open(path, "w", encoding="utf-8") as fh: - json.dump(data, fh, ensure_ascii=False, indent=1) - fh.write("\n") +def _dump(path: str, data) -> bool: + return importers.write_if_changed(path, (json.dumps(data, ensure_ascii=False, indent=1) + "\n").encode()) def _window(document: dict) -> tuple: @@ -206,10 +205,10 @@ def write_trial(document: dict, job_dir: str) -> dict: "n_cache_tokens": tokens.get("cache_read_tokens", 0), "cost_usd": None, } - _dump(os.path.join(trial_dir, "result.json"), result) - _dump(os.path.join(agent_dir, "trace.json"), document) + changed = _dump(os.path.join(trial_dir, "result.json"), result) + changed |= _dump(os.path.join(agent_dir, "trace.json"), document) return {"trial_dir": trial, "started_at": started, "finished_at": finished, - "model": model, "agent": agent.get("name") or ""} + "model": model, "agent": agent.get("name") or "", "changed": changed} def _write_job(documents: list, out_root: str, job_name: str, command: str, @@ -235,6 +234,7 @@ def _write_job(documents: list, out_root: str, job_name: str, command: str, "stats": {"n_errored_trials": 0, "n_running_trials": 0, "n_pending_trials": 0, "n_retries": 0}, }) + importers.mark_changed(job_dir, summaries) agents = sorted({s["agent"] for s in summaries if s["agent"]}) or [""] models = sorted({s["model"] for s in summaries if s["model"]}) _dump(os.path.join(job_dir, "config.json"), { diff --git a/app/tests/test_import_unchanged_source.py b/app/tests/test_import_unchanged_source.py new file mode 100644 index 00000000..32052b03 --- /dev/null +++ b/app/tests/test_import_unchanged_source.py @@ -0,0 +1,66 @@ +"""Opening a workspace again over an unchanged ATIF or OpenTelemetry source +leaves the converted job as it was, so its cached parse is used; a changed +source is converted and parsed again.""" +from __future__ import annotations + +import os +import shutil +import tempfile +import time +import unittest +from pathlib import Path + +from nullius import importers, workspace +from tests.harnesses.test_atif import minimal, write_doc +from tests.test_cli_contract import otlp_doc + + +def open_again(root, imports): + """What shared._import_roots does for a root on each open_workspace.""" + module = importers.detect(root) + return [d for tree in importers.trees(module, root) for d in module.convert(tree, imports)] + + +def mtimes(job_dir): + return {os.path.relpath(os.path.join(d, f), job_dir): os.stat(os.path.join(d, f)).st_mtime_ns + for d, _, files in os.walk(job_dir) for f in files} + + +class UnchangedSource(unittest.TestCase): + def setUp(self): + self.tmp = Path(tempfile.mkdtemp(prefix="nullius-import-")) + self.addCleanup(shutil.rmtree, self.tmp, True) + self.src = self.tmp / "src" + + def check(self, change): + """Convert, read, open again unchanged, then again after *change*.""" + ws = workspace.create(str(self.tmp / "workspaces"), [str(self.src)], "w") + [job_dir] = open_again(str(self.src), ws.imports_dir) + before = mtimes(job_dir) + self.assertFalse(workspace.load_job(ws, job_dir)[1]) # the first read parses + time.sleep(0.05) + open_again(str(self.src), ws.imports_dir) # opened again; the source unchanged + moved = sorted(f for f, m in mtimes(job_dir).items() if before.get(f) != m) + self.assertEqual((moved, workspace.load_job(ws, job_dir)[1]), ([], True)) + change() + open_again(str(self.src), ws.imports_dir) # opened again; the source changed + self.assertNotEqual(mtimes(job_dir), before) + self.assertFalse(workspace.load_job(ws, job_dir)[1]) + + # Each change leaves the job's own result.json and config.json as they were. + def test_atif(self): + write_doc(self.src, "run-1.json", minimal()) + edited = minimal() + edited["steps"][1]["message"] = "redone" + self.check(lambda: write_doc(self.src, "run-1.json", edited)) + + def test_opentelemetry(self): + write_doc(self.src, "export.json", otlp_doc()) + edited = otlp_doc() + edited["resourceSpans"][0]["scopeSpans"][0]["spans"][0]["attributes"] = [ + {"key": "k", "value": {"stringValue": "v"}}] + self.check(lambda: write_doc(self.src, "export.json", edited)) + + +if __name__ == "__main__": + unittest.main()