diff --git a/core/agent_runtime/goal_runtime.py b/core/agent_runtime/goal_runtime.py index 286bb51d..95f38a9e 100644 --- a/core/agent_runtime/goal_runtime.py +++ b/core/agent_runtime/goal_runtime.py @@ -4,7 +4,14 @@ import threading from dataclasses import dataclass -from typing import Any, Protocol +from typing import TYPE_CHECKING, Any, Protocol + +if TYPE_CHECKING: + from core.agent_runtime.goal_verifier import ( + GoalCompletionRequest, + GoalCompletionVerifier, + GoalVerdict, + ) GOAL_TOOL_NAMES = frozenset({"get_goal", "update_goal"}) @@ -45,20 +52,67 @@ def update_goal( reason: str | None, ) -> dict[str, Any]: ... + def completion_request( + self, + context: GoalRuntimeContext, + *, + reason: str | None, + ) -> GoalCompletionRequest: + """Build the claim a configured verifier judges. + + Only consulted while a completion verifier is configured; carries the + real objective and this Turn's existing evidence references. + """ + ... + class GoalRuntimeRouter: - """Route Goal tools only while one attributed Turn owns the runtime.""" + """Route Goal tools only while one attributed Turn owns the runtime. + + When a completion verifier is configured (see + :meth:`configure_completion_verifier`), a ``complete`` request is judged + exactly once, asynchronously, before any durable update. A rejection is + returned to the model as data — with the verifier's justification — and + the goal is not updated. After ``max_rejections`` rejected claims the + router moves the goal to ``blocked`` itself, so a model that keeps + re-claiming completion cannot burn Turns forever. + """ def __init__(self) -> None: self._lock = threading.Lock() self._handler: GoalRuntimeHandler | None = None self._context: GoalRuntimeContext | None = None self._terminal_requested = False + self._verifier: GoalCompletionVerifier | None = None + self._max_rejections = 3 + # Rejected completion claims per goal, so a model cannot burn Turns + # retrying the same claim; cleared on success or auto-block. + self._rejections: dict[str, int] = {} def configure(self, handler: GoalRuntimeHandler) -> None: with self._lock: self._handler = handler + def configure_completion_verifier( + self, + verifier: GoalCompletionVerifier, + *, + max_rejections: int = 3, + ) -> None: + """Arm the single asynchronous completion gate. + + There is deliberately no way to pass a pre-computed verdict into + :meth:`request`; every ``complete`` claim goes through ``verifier`` + exactly once. + """ + + if max_rejections < 1: + raise ValueError("max_rejections must be positive") + with self._lock: + self._verifier = verifier + self._max_rejections = max_rejections + self._rejections.clear() + def activate(self, context: GoalRuntimeContext) -> None: with self._lock: if self._context is not None and self._context.turn_id != context.turn_id: @@ -79,13 +133,33 @@ def read(self) -> dict[str, Any]: handler, context = self._snapshot() return handler.read_goal(context) - def request( + async def request( self, *, status: str, reason: str | None, ) -> dict[str, Any]: + """Record one terminal decision, verifying a ``complete`` claim once.""" + handler, context = self._snapshot() + with self._lock: + verifier = self._verifier + + if status == "complete" and verifier is not None: + from core.agent_runtime.goal_verifier import GoalVerdict + + try: + verdict = await verifier( + handler.completion_request(context, reason=reason) + ) + except Exception as exc: # noqa: BLE001 - fail closed, report to the model + verdict = GoalVerdict( + confirmed=False, + justification=f"Completion verification failed: {exc}", + ) + if not verdict.confirmed: + return self._reject_completion(handler, context, verdict) + result = handler.update_goal( context, status=status, @@ -95,8 +169,60 @@ def request( if self._context != context: raise GoalRuntimeError("Goal Turn changed while recording the decision") self._terminal_requested = True + self._rejections.pop(context.goal_id, None) return result + def _reject_completion( + self, + handler: GoalRuntimeHandler, + context: GoalRuntimeContext, + verdict: GoalVerdict, + ) -> dict[str, Any]: + """Return a rejection to the model; auto-block after N rejections.""" + + with self._lock: + if self._context != context: + raise GoalRuntimeError("Goal Turn changed while recording the decision") + count = self._rejections.get(context.goal_id, 0) + 1 + self._rejections[context.goal_id] = count + max_rejections = self._max_rejections + exhausted = count >= max_rejections + + rejection: dict[str, Any] = { + "goalId": context.goal_id, + "updated": False, + "verdict": { + "confirmed": False, + "justification": verdict.justification, + }, + "rejections": count, + "maxRejections": max_rejections, + } + if not exhausted: + return rejection + + # The claim is exhausted: move the Goal to blocked (a terminal + # decision the model asked for by never producing evidence) instead + # of letting it burn Turns on the same rejected completion claim. + result = handler.update_goal( + context, + status="blocked", + reason=verdict.justification, + ) + with self._lock: + if self._context != context: + raise GoalRuntimeError("Goal Turn changed while recording the decision") + self._terminal_requested = True + self._rejections.pop(context.goal_id, None) + return { + **result, + "updated": True, + "verdict": rejection["verdict"], + "rejections": count, + "maxRejections": max_rejections, + "autoBlocked": True, + } + def visible_tool_names(self, names: tuple[str, ...]) -> tuple[str, ...]: """Narrow tools dynamically without ever adding a capability.""" diff --git a/core/agent_runtime/goal_verifier.py b/core/agent_runtime/goal_verifier.py new file mode 100644 index 00000000..75fc5d16 --- /dev/null +++ b/core/agent_runtime/goal_verifier.py @@ -0,0 +1,132 @@ +"""Independent verification of a Goal completion claim. + +The verifier answers one question: does this Turn's evidence support the +model's claim that the Goal is complete? It is consulted exactly once per +completion request, asynchronously, inside the goal runtime — there is no +synchronous side door and no caller-supplied "already verified" flag. + +Rejections are data, not exceptions. A rejected claim returns its +justification to the model so the model can either produce the missing +evidence or declare the Goal blocked; a verifier that raises or cannot be +asked fails closed (rejects). After a configured number of rejections the +goal runtime moves the Goal to ``blocked`` rather than letting the model +burn Turns retrying the same claim. +""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import Protocol + + +@dataclass(frozen=True, slots=True) +class GoalEvidenceItem: + """One Turn evidence item, reduced to what a verifier judges.""" + + kind: str + status: str + summary: str + + +@dataclass(frozen=True, slots=True) +class GoalCompletionRequest: + """The claim under review: the real objective plus Turn evidence.""" + + objective: str + reason: str + evidence: tuple[GoalEvidenceItem, ...] + + +@dataclass(frozen=True, slots=True) +class GoalVerdict: + """The verifier's answer. ``justification`` always travels to the model.""" + + confirmed: bool + justification: str + + +class GoalCompletionVerifier(Protocol): + """An async callable that judges one completion claim.""" + + def __call__(self, request: GoalCompletionRequest) -> Awaitable[GoalVerdict]: ... + + +class RuleBasedVerifier: + """Fail-closed structural checks over the objective, reason, and evidence. + + This is the built-in verifier behind the ``rule`` setting. It does not + judge whether the work is *good*; it judges whether the claim is + *supported*: a completion claim with no objective, no stated reason, or + no recorded Turn evidence is rejected with a justification the model can + act on. + """ + + async def __call__(self, request: GoalCompletionRequest) -> GoalVerdict: + if not request.objective.strip(): + return GoalVerdict( + confirmed=False, + justification="The Goal has no objective to verify the completion claim against.", + ) + if not request.reason.strip(): + return GoalVerdict( + confirmed=False, + justification="The completion claim states no reason. Explain what was done before marking the Goal complete.", + ) + if not request.evidence: + return GoalVerdict( + confirmed=False, + justification=( + "The Turn recorded no evidence (tool calls, file changes, or test " + "results) to support completion. Do the work first, then claim completion." + ), + ) + failed = [ + item + for item in request.evidence + if item.status.strip().lower() in {"failed", "error"} + ] + if failed: + return GoalVerdict( + confirmed=False, + justification=( + f"{len(failed)} of {len(request.evidence)} evidence item(s) failed " + f"(latest: {failed[-1].summary!r}). Resolve the failures before completing." + ), + ) + return GoalVerdict( + confirmed=True, + justification=( + f"{len(request.evidence)} Turn evidence item(s) support the claim." + ), + ) + + +def model_backed_verifier( + ask: Callable[[GoalCompletionRequest], Awaitable[str]], +) -> GoalCompletionVerifier: + """Adapt a one-shot text answerer into a fail-closed verifier. + + ``ask`` returns the model's answer; anything other than an affirmative + answer rejects, and an ``ask`` that raises rejects too (fail closed). + """ + + async def verify(request: GoalCompletionRequest) -> GoalVerdict: + try: + answer = (await ask(request)).strip().lower() + except Exception as exc: # noqa: BLE001 - fail closed, report to the model + return GoalVerdict( + confirmed=False, + justification=f"The completion verifier failed: {exc}", + ) + confirmed = answer.startswith("yes") + return GoalVerdict( + confirmed=confirmed, + justification=( + "The model verifier confirmed the evidence supports completion." + if confirmed + else "The model verifier rejected the completion claim." + ), + ) + + return verify diff --git a/core/agent_setup.py b/core/agent_setup.py index f1217d6b..784f2685 100644 --- a/core/agent_setup.py +++ b/core/agent_setup.py @@ -339,6 +339,15 @@ def build_agent_session( mode_override=permission_mode_override, profile_override=execution_security_profile, ) + if goal_runtime is not None: + goal_cfg = getattr(active_runtime.config, "goal", None) + if goal_cfg is not None and goal_cfg.completion_verifier == "rule": + from core.agent_runtime.goal_verifier import RuleBasedVerifier + + goal_runtime.configure_completion_verifier( + RuleBasedVerifier(), + max_rejections=goal_cfg.max_rejections, + ) engine = build_permission_engine( security_cfg, cwd=workspace, diff --git a/core/application/goal_extension.py b/core/application/goal_extension.py index 2ae0afc6..f81775a8 100644 --- a/core/application/goal_extension.py +++ b/core/application/goal_extension.py @@ -10,7 +10,7 @@ from enum import StrEnum from importlib.resources import files from pathlib import Path -from typing import Any, Iterator +from typing import TYPE_CHECKING, Any, Iterator from core.agent_runtime.goal_runtime import GoalRuntimeContext, GoalRuntimeHandler from core.application.errors import ( @@ -43,6 +43,9 @@ ) from core.skills.models import MAX_SELECTED_SKILLS, SkillSelection +if TYPE_CHECKING: + from core.agent_runtime.goal_verifier import GoalCompletionRequest + logger = logging.getLogger(__name__) # Without a token budget, only the model's update_goal ends a Goal. A model @@ -642,6 +645,56 @@ def update_goal( "updated": True, } + def completion_request( + self, + context: GoalRuntimeContext, + *, + reason: str | None, + ) -> GoalCompletionRequest: + """Build the verifier's view: the real objective and this Turn's evidence. + + The old branch hardcoded ``objective=""``; here the verifier judges the + actual objective the model is claiming to satisfy, plus the evidence + references this Turn already recorded (same filter as the outcome). + """ + + from core.agent_runtime.goal_verifier import ( + GoalCompletionRequest, + GoalEvidenceItem, + ) + + goal = self.read(context.thread_id) + objective = goal.objective if goal is not None else "" + try: + snapshot = self.turns.read(context.turn_id) + except TurnNotFoundError as exc: + logger.info( + "Goal completion Turn %s is unavailable: %s", + context.turn_id, + exc, + ) + return GoalCompletionRequest( + objective=objective, + reason=reason or "", + evidence=(), + ) + evidence = tuple( + GoalEvidenceItem( + kind=item.kind.value, + status=item.status.value, + summary=(item.summary.strip() or item.kind.value.replace("_", " "))[ + :GOAL_OUTCOME_EVIDENCE_SUMMARY_MAX_CHARS + ], + ) + for item in sorted(snapshot.items, key=lambda candidate: candidate.ordinal) + if item.kind in _EVIDENCE_KINDS + )[:GOAL_OUTCOME_EVIDENCE_MAX_ITEMS] + return GoalCompletionRequest( + objective=objective, + reason=reason or "", + evidence=evidence, + ) + def recover_incomplete(self) -> int: """Replay committed Goal Turns whose ledger settlement may be missing.""" diff --git a/core/config.py b/core/config.py index 0a53023d..c79ed991 100644 --- a/core/config.py +++ b/core/config.py @@ -414,6 +414,22 @@ class WorkspaceConfig(_Base): max_input_mb: int = 100 +class GoalConfig(_Base): + """Thread Goal completion verification policy. + + - ``completion_verifier``: ``"off"`` (default) keeps legacy behavior — the + model's ``update_goal(status="complete")`` is recorded directly. ``"rule"`` + arms the built-in fail-closed verifier: every ``complete`` claim is judged + exactly once, asynchronously, against the real objective plus this Turn's + evidence, and the verifier's justification is returned to the model. + - ``max_rejections``: after this many rejected completion claims the Goal is + moved to ``blocked`` (instead of letting the model burn Turns retrying). + """ + + completion_verifier: Literal["off", "rule"] = "off" + max_rejections: int = 3 + + class DictationConfig(_Base): """Voice input for the prompt box, transcribed by a Parakeet-style server. @@ -576,6 +592,7 @@ class DeepCodeConfig(BaseSettings): skills: SkillsConfig = Field(default_factory=SkillsConfig) security: SecurityConfig = Field(default_factory=SecurityConfig) workspace: WorkspaceConfig = Field(default_factory=WorkspaceConfig) + goal: GoalConfig = Field(default_factory=GoalConfig) # Absent means "voice input is off"; present means the user picked an # endpoint for it. A default *instance* could not tell those apart, and a # disabled microphone that still ships an endpoint would be a config that diff --git a/core/harness/tools/goal.py b/core/harness/tools/goal.py index 4c62b0d4..d6ee3b9d 100644 --- a/core/harness/tools/goal.py +++ b/core/harness/tools/goal.py @@ -85,7 +85,7 @@ async def execute( status: str, reason: str | None = None, ) -> dict[str, Any]: - return self.runtime.request( + return await self.runtime.request( status=status, reason=reason, ) diff --git a/tests/app_server/test_server.py b/tests/app_server/test_server.py index 4254548c..c40940d8 100644 --- a/tests/app_server/test_server.py +++ b/tests/app_server/test_server.py @@ -1423,7 +1423,7 @@ def load_history(self, _messages) -> None: async def run_stream(self, _op): yield Event("1", TurnStarted()) - self.goal_runtime.request( + await self.goal_runtime.request( status="complete", reason="Scheduled repository work and verification completed.", ) diff --git a/tests/application/test_automation_goal_runs.py b/tests/application/test_automation_goal_runs.py index 48676b53..7a27d4f0 100644 --- a/tests/application/test_automation_goal_runs.py +++ b/tests/application/test_automation_goal_runs.py @@ -75,7 +75,7 @@ async def run_stream(self, op): while step.gate is not None and not step.gate.is_set(): await asyncio.sleep(0.005) if step.decision is not None: - self.goal_runtime.request( + await self.goal_runtime.request( status=step.decision, reason=f"Agent requested {step.decision} in scripted Turn {index}.", ) diff --git a/tests/application/test_automation_service.py b/tests/application/test_automation_service.py index 85142063..b632ec06 100644 --- a/tests/application/test_automation_service.py +++ b/tests/application/test_automation_service.py @@ -43,7 +43,7 @@ async def run_stream(self, op): yield Event("1", TurnStarted()) if self.hang: await asyncio.Event().wait() - self.goal_runtime.request( + await self.goal_runtime.request( status="complete", reason="The requested automation and verification completed.", ) diff --git a/tests/test_automation_cli.py b/tests/test_automation_cli.py index 808b8f54..08d3c1a1 100644 --- a/tests/test_automation_cli.py +++ b/tests/test_automation_cli.py @@ -162,7 +162,7 @@ def load_history(self, messages) -> None: async def run_stream(self, op): self.history.append({"role": "user", "content": op.text}) yield Event("start", TurnStarted()) - self.goal_runtime.request( + await self.goal_runtime.request( status="complete", reason="Automation CLI verification completed.", ) diff --git a/tests/test_automation_foreground.py b/tests/test_automation_foreground.py index 980a2019..d1e60432 100644 --- a/tests/test_automation_foreground.py +++ b/tests/test_automation_foreground.py @@ -38,7 +38,7 @@ async def run_stream(self, operation): self._factory.decisions.append(approved) status = "complete" if approved else "blocked" reason = "verified result written" if approved else "approval denied" - self._goal_runtime.request(status=status, reason=reason) + await self._goal_runtime.request(status=status, reason=reason) yield Event("message", AgentMessage(reason)) yield Event("complete", TaskComplete(reason, status)) self.history.append({"role": "assistant", "content": reason}) diff --git a/tests/test_goal_completion_verifier.py b/tests/test_goal_completion_verifier.py new file mode 100644 index 00000000..ac5539dc --- /dev/null +++ b/tests/test_goal_completion_verifier.py @@ -0,0 +1,374 @@ +"""Goal completion verifier: single async gate, justification, blocked-after-N.""" + +from __future__ import annotations + +import asyncio + +import pytest + +from core.agent_runtime.goal_runtime import ( + GoalRuntimeContext, + GoalRuntimeRouter, +) +from core.agent_runtime.goal_verifier import ( + GoalCompletionRequest, + GoalCompletionVerifier, + GoalEvidenceItem, + GoalVerdict, + RuleBasedVerifier, + model_backed_verifier, +) +from core.agent_runtime.tools.registry import ToolRegistry +from core.harness.tools.goal import UpdateGoalTool + + +class FakeGoalHandler: + """Application-owned durable operations, recorded for assertions.""" + + def __init__(self, objective: str = "Ship the feature.") -> None: + self.requests: list[dict[str, object]] = [] + self.objective = objective + + def read_goal(self, context: GoalRuntimeContext): + return {"goalId": context.goal_id, "objective": self.objective} + + def update_goal( + self, + context: GoalRuntimeContext, + *, + status: str, + reason: str | None, + ): + self.requests.append( + {"turnId": context.turn_id, "status": status, "reason": reason} + ) + return {"goalId": context.goal_id, "status": status} + + def completion_request( + self, + context: GoalRuntimeContext, + *, + reason: str | None, + ) -> GoalCompletionRequest: + return GoalCompletionRequest( + objective=self.objective, + reason=reason or "", + evidence=( + GoalEvidenceItem( + kind="test_result", + status="passed", + summary="Focused tests pass.", + ), + ), + ) + + +class RejectingVerifier: + """Async verifier that always rejects; counts calls to prove single-gate.""" + + def __init__(self, justification: str = "No file change evidence.") -> None: + self.calls = 0 + self.justification = justification + + async def __call__(self, request: GoalCompletionRequest) -> GoalVerdict: + self.calls += 1 + return GoalVerdict(confirmed=False, justification=self.justification) + + +class ConfirmingVerifier: + async def __call__(self, request: GoalCompletionRequest) -> GoalVerdict: + return GoalVerdict(confirmed=True, justification="Evidence supports it.") + + +class RaisingVerifier: + def __call__(self, request: GoalCompletionRequest) -> GoalVerdict: + raise RuntimeError("verifier backend unavailable") + + +def _context() -> GoalRuntimeContext: + return GoalRuntimeContext( + thread_id="session", + goal_id="goal_1", + turn_id="turn_1", + ) + + +@pytest.mark.asyncio +async def test_update_goal_tool_rejection_returns_justification_and_records_nothing() -> ( + None +): + """E2E through UpdateGoalTool: async verifier rejects, model sees why.""" + + runtime = GoalRuntimeRouter() + handler = FakeGoalHandler() + runtime.configure(handler) + verifier = RejectingVerifier(justification="No file change evidence.") + runtime.configure_completion_verifier(verifier) + runtime.activate(_context()) + registry = ToolRegistry() + registry.register(UpdateGoalTool(runtime)) + + result = await registry.execute( + "update_goal", + {"status": "complete", "reason": "The tests pass, so it is done."}, + ) + + # The verifier ran exactly once — no sync re-check, no bypass. + assert verifier.calls == 1 + # The justification travels back to the model. + assert result["updated"] is False + assert result["verdict"]["confirmed"] is False + assert result["verdict"]["justification"] == "No file change evidence." + assert result["rejections"] == 1 + # No durable completion was recorded; the goal stays open. + assert handler.requests == [] + # The terminal gate did not freeze: the model may keep working. + assert runtime.visible_tool_names(("read", "update_goal")) == ( + "read", + "update_goal", + ) + + +@pytest.mark.asyncio +async def test_goal_blocks_after_max_rejections_instead_of_burning_turns() -> None: + """N rejections move the goal to blocked and freeze the turn.""" + + runtime = GoalRuntimeRouter() + handler = FakeGoalHandler() + runtime.configure(handler) + verifier = RejectingVerifier() + runtime.configure_completion_verifier(verifier, max_rejections=2) + runtime.activate(_context()) + registry = ToolRegistry() + registry.register(UpdateGoalTool(runtime)) + + first = await registry.execute( + "update_goal", {"status": "complete", "reason": "claim one"} + ) + # First rejection is returned as data; nothing durable yet. + assert first["updated"] is False + assert first["rejections"] == 1 + assert handler.requests == [] + + second = await registry.execute( + "update_goal", {"status": "complete", "reason": "claim two"} + ) + + assert verifier.calls == 2 + # Second (max) rejection auto-blocks the goal and freezes the turn. + assert second["autoBlocked"] is True + assert second["verdict"]["justification"] + assert handler.requests == [ + { + "turnId": "turn_1", + "status": "blocked", + "reason": "No file change evidence.", + } + ] + assert runtime.visible_tool_names(("read", "update_goal")) == () + + +@pytest.mark.asyncio +async def test_confirming_verifier_records_completion_exactly_once() -> None: + """A confirmed claim records completion; the verifier is not called twice.""" + + runtime = GoalRuntimeRouter() + handler = FakeGoalHandler() + runtime.configure(handler) + calls = {"n": 0} + + async def verifier(request: GoalCompletionRequest) -> GoalVerdict: + calls["n"] += 1 + return GoalVerdict(confirmed=True, justification="ok") + + runtime.configure_completion_verifier(verifier) + runtime.activate(_context()) + + result = await runtime.request( + status="complete", reason="Feature shipped with tests." + ) + + assert calls["n"] == 1 + assert result["status"] == "complete" + assert handler.requests == [ + { + "turnId": "turn_1", + "status": "complete", + "reason": "Feature shipped with tests.", + } + ] + + +@pytest.mark.asyncio +async def test_raising_verifier_fails_closed_and_reports_to_model() -> None: + """A verifier that raises rejects (fail closed) and tells the model why.""" + + runtime = GoalRuntimeRouter() + handler = FakeGoalHandler() + runtime.configure(handler) + runtime.configure_completion_verifier(RaisingVerifier()) + runtime.activate(_context()) + + result = await runtime.request(status="complete", reason="Done.") + + assert result["updated"] is False + assert "verifier backend unavailable" in result["verdict"]["justification"] + assert handler.requests == [] + + +@pytest.mark.asyncio +async def test_blocked_status_is_not_verified() -> None: + """Only complete claims are verified; blocked goes straight through.""" + + runtime = GoalRuntimeRouter() + handler = FakeGoalHandler() + runtime.configure(handler) + verifier = RejectingVerifier() + runtime.configure_completion_verifier(verifier) + runtime.activate(_context()) + + result = await runtime.request(status="blocked", reason="Upstream API is down.") + + assert verifier.calls == 0 + assert result["status"] == "blocked" + assert handler.requests == [ + {"turnId": "turn_1", "status": "blocked", "reason": "Upstream API is down."} + ] + + +def test_rule_based_verifier_requires_objective_reason_and_evidence() -> None: + verifier = RuleBasedVerifier() + + no_objective = asyncio.run( + verifier(GoalCompletionRequest(objective=" ", reason="done", evidence=())) + ) + assert no_objective.confirmed is False + assert "no objective" in no_objective.justification.lower() + + no_reason = asyncio.run( + verifier(GoalCompletionRequest(objective="Ship it.", reason=" ", evidence=())) + ) + assert no_reason.confirmed is False + assert "no reason" in no_reason.justification.lower() + + no_evidence = asyncio.run( + verifier( + GoalCompletionRequest( + objective="Ship it.", reason="Looks done.", evidence=() + ) + ) + ) + assert no_evidence.confirmed is False + assert "no evidence" in no_evidence.justification.lower() + + failed = asyncio.run( + verifier( + GoalCompletionRequest( + objective="Ship it.", + reason="Ran the suite.", + evidence=( + GoalEvidenceItem( + kind="test_result", + status="failed", + summary="2 tests failed.", + ), + ), + ) + ) + ) + assert failed.confirmed is False + assert "failed" in failed.justification.lower() + + good = asyncio.run( + verifier( + GoalCompletionRequest( + objective="Ship it.", + reason="Ran the suite.", + evidence=( + GoalEvidenceItem( + kind="test_result", + status="passed", + summary="All tests pass.", + ), + ), + ) + ) + ) + assert good.confirmed is True + + +@pytest.mark.asyncio +async def test_model_backed_verifier_fail_closed_on_ask_error() -> None: + async def ask(request: GoalCompletionRequest) -> str: + raise ConnectionError("model unreachable") + + verifier = model_backed_verifier(ask) + verdict = await verifier( + GoalCompletionRequest(objective="Ship it.", reason="done", evidence=()) + ) + + assert verdict.confirmed is False + assert "model unreachable" in verdict.justification + + +@pytest.mark.asyncio +async def test_model_backed_verifier_treats_non_yes_as_rejection() -> None: + async def ask(request: GoalCompletionRequest) -> str: + return "no, the tests did not run" + + verifier = model_backed_verifier(ask) + verdict = await verifier( + GoalCompletionRequest(objective="Ship it.", reason="done", evidence=()) + ) + + assert verdict.confirmed is False + + +@pytest.mark.asyncio +async def test_verifier_protocol_accepts_plain_async_callable() -> None: + """A bare async function satisfies the verifier contract (no class needed).""" + + async def verifier(request: GoalCompletionRequest) -> GoalVerdict: + return GoalVerdict(confirmed=True, justification="ok") + + runtime = GoalRuntimeRouter() + handler = FakeGoalHandler() + runtime.configure(handler) + runtime.configure_completion_verifier(verifier) # type: ignore[arg-type] + runtime.activate(_context()) + + result = await runtime.request(status="complete", reason="Shipped.") + + assert result["status"] == "complete" + + +def test_completion_request_carries_real_objective_and_evidence() -> None: + """The request handed to the verifier is built by the handler, not hardcoded.""" + + handler = FakeGoalHandler(objective="Fix the off-by-one in pagination.") + request = handler.completion_request(_context(), reason="Tests pass now.") + + assert request.objective == "Fix the off-by-one in pagination." + assert request.reason == "Tests pass now." + assert request.evidence and request.evidence[0].kind == "test_result" + + +def test_configure_completion_verifier_rejects_non_positive_max() -> None: + runtime = GoalRuntimeRouter() + + with pytest.raises(ValueError): + runtime.configure_completion_verifier(ConfirmingVerifier(), max_rejections=0) + + +def test_verifier_contract_is_async_callable_protocol() -> None: + """RuleBasedVerifier and model_backed_verifier both satisfy the Protocol.""" + + async def _accept(verifier: GoalCompletionVerifier) -> None: + await verifier(GoalCompletionRequest(objective="x", reason="y", evidence=())) + + asyncio.run(_accept(RuleBasedVerifier())) + asyncio.run(_accept(model_backed_verifier(_yes))) + + +async def _yes(_request: GoalCompletionRequest) -> str: + return "yes" diff --git a/tests/test_goal_runtime.py b/tests/test_goal_runtime.py index 843a1ab6..0de7ea36 100644 --- a/tests/test_goal_runtime.py +++ b/tests/test_goal_runtime.py @@ -45,7 +45,10 @@ def _context() -> GoalRuntimeContext: ) -def test_goal_tools_are_hidden_outside_an_attempt_and_freeze_after_request() -> None: +@pytest.mark.asyncio +async def test_goal_tools_are_hidden_outside_an_attempt_and_freeze_after_request() -> ( + None +): runtime = GoalRuntimeRouter() handler = FakeGoalHandler() runtime.configure(handler) @@ -57,7 +60,7 @@ def test_goal_tools_are_hidden_outside_an_attempt_and_freeze_after_request() -> assert runtime.visible_tool_names(registered) == registered assert runtime.closure_prompt() is not None assert runtime.closure_prompt(stop_hook_active=True) is None - result = runtime.request( + result = await runtime.request( status="complete", reason="The latest Goal is complete.", )