Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions app/nullius/runs/parse.py
Original file line number Diff line number Diff line change
Expand Up @@ -679,7 +679,7 @@ def parse_trial(job_id: str, trial_dir: str) -> Trial | None:
source=source,
model_name=(res.get("agent_info", {}).get("model_info", {}) or {}).get("name", ""),
agent_version=(res.get("agent_info") or {}).get("version") or "",
reasoning_effort=str((((res.get("config") or {}).get("agent") or {}).get("kwargs") or {}).get("reasoning_effort", "")),
reasoning_effort=str((((res.get("config") or {}).get("agent") or {}).get("kwargs") or {}).get("reasoning_effort") or ""),
agent_info_name=str((res.get("agent_info") or {}).get("name") or ""),
outcome=outcome,
reward=reward,
Expand Down Expand Up @@ -919,7 +919,7 @@ def parse_job(job_dir: str, reuse=None, trials: bool = True) -> Job:
stats = res.get("stats") or {}

agents = cfg.get("agents") or [{}]
model = agents[0].get("model_name", "")
model = agents[0].get("model_name") or ""

# Three config shapes in the wild: `datasets: [{path, n_tasks}]` for a
# local checkout, `datasets: [{name, version, n_tasks}]` for a registry
Expand Down Expand Up @@ -955,7 +955,7 @@ def parse_job(job_dir: str, reuse=None, trials: bool = True) -> Job:
started_at=_iso(res.get("started_at")),
finished_at=_iso(res.get("finished_at")),
agent_name=agents[0].get("name", ""),
reasoning_effort=str((agents[0].get("kwargs") or {}).get("reasoning_effort", "")),
reasoning_effort=str((agents[0].get("kwargs") or {}).get("reasoning_effort") or ""),
n_total_trials=res.get("n_total_trials") or 0,
n_errored_trials=stats.get("n_errored_trials") or 0,
n_running_trials=stats.get("n_running_trials") or 0,
Expand Down Expand Up @@ -997,7 +997,7 @@ def parse_job(job_dir: str, reuse=None, trials: bool = True) -> Job:
job.launch = launch_of(cfg)
if len(agents) > 1 and not job.runner:
job.agents = [{"name": a.get("name") or a.get("import_path") or "", "model": a.get("model_name") or "",
"effort": str((a.get("kwargs") or {}).get("reasoning_effort", "")),
"effort": str((a.get("kwargs") or {}).get("reasoning_effort") or ""),
"command": redact(harbor_command(dict(cfg, agents=[a]),
next((t.env_type for t in job.trials if t.env_type), "")))}
for a in agents if isinstance(a, dict)]
Expand Down
18 changes: 18 additions & 0 deletions app/tests/test_parse.py
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,24 @@ def test_the_trials_name_the_agent(self):
self.assertEqual((job.agent_name, job.agent_label, job.is_control), ("oracle", "oracle", True))


class NullInConfig(unittest.TestCase):
"""A present JSON null reads as nothing recorded, as an absent key does:
pages sort and print these strings."""

def test_null_model_and_effort_read_as_empty(self):
with tempfile.TemporaryDirectory() as tmp:
_write_json(os.path.join(tmp, "result.json"), {})
_write_json(os.path.join(tmp, "config.json"), {"agents": [
{"name": "oracle", "model_name": None, "kwargs": {"reasoning_effort": None}},
{"name": "nop", "model_name": None, "kwargs": {"reasoning_effort": None}}]})
trial = os.path.join(tmp, "t__x")
_write_json(os.path.join(trial, "result.json"), {"task_name": "t", "trial_name": "t__x",
"config": {"agent": {"kwargs": {"reasoning_effort": None}}}})
job = parse.parse_job(tmp)
self.assertEqual((job.model_name, job.reasoning_effort, job.trials[0].reasoning_effort), ("", "", ""))
self.assertEqual([a["effort"] for a in job.agents], ["", ""])


class Stamps(unittest.TestCase):
"""Harbor's stamps carry no zone and mean UTC; the standard's and the
imports' carry one. Pages order and subtract them together."""
Expand Down
13 changes: 12 additions & 1 deletion app/ui_tests/test_controls_apart.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
"""
from __future__ import annotations

import json
import re
import tempfile
import unittest
Expand All @@ -19,7 +20,7 @@
from unittest.mock import patch

from nullius import workspace
from nullius.runs import feasibility
from nullius.runs import feasibility, parse
from tests.test_feasibility import write_job
from .test_routes import web

Expand Down Expand Up @@ -93,6 +94,16 @@ def row(self, html, job_id):
self.assertEqual(len(rows), 1, job_id)
return rows[0]

def test_a_control_whose_config_has_a_null_model_lists(self):
with tempfile.TemporaryDirectory() as root:
write_job(root, "tau-z-oracle", "oracle", {"a": 1}, n_tasks=1)
path = Path(root) / "tau-z-oracle" / "config.json"
cfg = json.loads(path.read_text()); cfg["agents"][0]["model_name"] = None
path.write_text(json.dumps(cfg))
self.jobs = [self.agent, parse.parse_job(str(path.parent))]
html = self.get("/runs")
self.assertNotIn("None", self.row(html, "tau-x-agent") + self.row(html, "tau-z-oracle"))

def test_a_control_is_read_as_behaved_or_not_never_as_a_score(self):
self.assertIsNone(feasibility.summary_of(self.agent))
nop = feasibility.summary_of(self.nop)
Expand Down
Loading