Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 17 additions & 5 deletions agentdescent/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -2570,7 +2570,7 @@ def cmd_plan(a: argparse.Namespace) -> int:
probe = _probe_if_asked(a, spec) if payload.get("ok", True) else None
# Default next is evolve; a refused probe replaces it with the repair
# action so a reader (or a host following `next`) does not start a run
# that cannot work (#1093). `ok` stays about the plan/spec itself.
# that cannot work (#1093), and flips `ok` too (#1175) -- see below.
next_line = f"agentdescent evolve {a.spec}"
# Keep an ad-hoc `--algorithm` / `--param` override in the suggested
# command: it lives in memory, not in the spec file.
Expand All @@ -2579,10 +2579,19 @@ def cmd_plan(a: argparse.Namespace) -> int:
for param in getattr(a, "param", None) or []:
next_line += f" --param {param}"
then_line = None
probe_refused = False
if probe is not None:
payload["probe"] = probe
payload["ready"] = bool(probe.get("ok", True))
if not payload["ready"]:
# One refusal contract on every surface (#1175): a probe that
# cannot work is not `ok` either, so the CLI and the MCP `plan`
# tool print the same `ok`/`ready` pair and a host reading either
# key reaches the same verdict. The plan itself parsed, but the
# run it describes cannot start; the exit code stays 1, which is
# the config answer `evolve`'s first call would have taken.
payload["ok"] = False
probe_refused = True
next_line = first_failed_next(probe)
payload["next"] = next_line
then_line = f"agentdescent plan {a.spec} --probe"
Expand Down Expand Up @@ -2658,7 +2667,7 @@ def cmd_plan(a: argparse.Namespace) -> int:
probe_lines=(_probe_lines(probe) if probe is not None else None),
then_line=then_line, baseline_lines=baseline_lines,
refused_headroom=refused_headroom)
if not payload.get("ok", True):
if not payload.get("ok", True) and not probe_refused:
return 2
# A saturated seed is a config answer, like a refused probe: plan found the
# run cannot learn anything, before the run store gained an entry (#1113).
Expand Down Expand Up @@ -4850,11 +4859,14 @@ def _cmd_algorithms_run(a: argparse.Namespace) -> int:
# wrote: one call, before the `evolve` it recommends (#1048).
probe = _probe_if_asked(a, load_spec(obj["spec_path"]))
obj["probe"] = probe
# `ok` stays about the written plan; `ready` is whether evolve can
# start. A refused probe also replaces `next` so a host does not
# follow it into a run that cannot work (#1093).
# The one refusal contract (#1175): `ok: false`, `ready: false` and
# `next` as the repair, identical to `plan --probe` and the MCP `plan`
# tool. The written plan is fine, but the run it recommends cannot
# start, so a host reading either key refuses it. The exit code stays
# 1 (a config answer, not the spec error that exits 2).
obj["ready"] = bool(probe.get("ok", True))
if not obj["ready"]:
obj["ok"] = False
obj["next"] = first_failed_next(probe)
then_line = f"agentdescent plan {obj['spec_path']} --probe"
if as_json:
Expand Down
3 changes: 2 additions & 1 deletion agentdescent/integrations/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,8 @@ after `doctor`. Do not invent a keyless path for those cards.
and answers "will start work" before a run exists. If it refuses, give the
user its `next` (renew the plan, fix the model name or key) and do **not**
call `start` -- a refused probe is the run failing in round one, found one
step earlier.
step earlier. A refusal prints `ok: false` and `ready: false` on both
surfaces, so the same check reads the same answer whichever one you drive.
To keep improving across sessions, offer `agentdescent campaign <spec>
--sessions N`; it works on a copy, reports the workspace/final target and
never touches the live target. Run its `--dry-run` first. A campaign has no
Expand Down
18 changes: 11 additions & 7 deletions agentdescent/mcp.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,8 +161,9 @@ def advanced_from_env(environ: Optional[Dict[str, str]] = None) -> bool:
"budget (budget is only checked between rounds). When the spec has an "
"`algorithm`, the resolved name/params are echoed. Pass probe=true to send "
"one minimal call per model-backed agent (the same preflight as "
"`plan --probe`); a refusal returns `probe` and makes this object `ok: false` "
"with `next` naming the fix, so do not call start then. Show the user the "
"`plan --probe`); a refusal returns `probe`, and prints the same refusal "
"contract as the CLI (`ok: false`, `ready: false`) with `next` naming the "
"fix, so do not call start then. Show the user the "
"spec and the estimate and get a yes before calling start."),
"start": (
"Start an evolution run in the background and return its run_id at once. The run "
Expand Down Expand Up @@ -407,16 +408,19 @@ def _with_probe(payload: Dict[str, Any], spec: EvolveSpec) -> Dict[str, Any]:

The body is the same ``probe_payload`` the CLI's ``plan --probe`` runs, so
a host gets the same rows and the same repair sentence instead of a second
implementation. A refusal makes the plan itself ``ok: false`` and replaces
``next`` with that repair: a host following ``next`` must not start a run
that cannot work. ``probe`` is added only when the caller asked for it --
plain ``plan`` stays the free call that sends nothing.
implementation. A refusal prints the one contract both surfaces carry
(#1175): ``ok: false``, ``ready: false`` and ``next`` replaced with that
repair, so a host following ``next`` must not start a run that cannot work
and a host porting its check from the CLI reads the same keys. ``probe`` is
added only when the caller asked for it -- plain ``plan`` stays the free
call that sends nothing, and has no ``ready`` key.
"""
from .probe import first_failed_next, probe_payload

result = probe_payload(spec)
payload["probe"] = result
if not result.get("ok", True):
payload["ready"] = bool(result.get("ok", True))
if not payload["ready"]:
payload["ok"] = False
payload["next"] = first_failed_next(result)
return payload
Expand Down
13 changes: 13 additions & 0 deletions changelog.d/1175-mcp-probe-contract.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
Changed
A refused probe now prints one JSON contract on every surface: `ok: false`,
`ready: false` and `next` as the repair action. The MCP `plan` tool gains
`ready` (it had only `ok: false`), and `plan --probe --json` /
`algorithms run --probe --json` now set `ok: false` instead of `ok: true`, so a
host reads the same keys and reaches the same verdict whichever surface it
drives (#1175).

This supersedes the 0.11.0 entry (#1093), which said: "`--json` sets
`ready: false` while leaving `ok` about the plan itself, and a successful or
omitted probe is unchanged (#1093)." It now sets `ok: false` too, on both
surfaces, because a refusal is not success anywhere; a successful or omitted
probe is still unchanged.
6 changes: 4 additions & 2 deletions docs/algorithms.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,10 @@ subscription that cannot work is refused there instead of inside
`evolve` -- with the same `SubscriptionInactive` / `QuotaExhausted` /
`ProviderConfigError` answer and no run store entry. A refused probe
ends with `next:` naming the repair (not `evolve`), and `--json`
sets `ready: false` while `ok` stays about the plan itself (#1093).
It is off by default: plain `plan` still runs and spends nothing.
sets `ok: false` and `ready: false` -- the same refusal contract the MCP
`plan` tool prints, so a host reads either key the same way on both
surfaces (#1175). It is off by default: plain `plan` still runs and
spends nothing, and has no `ready` key.

Include cases the seed gets wrong: if the seed already scores
the target on held-out, the run stops at once with
Expand Down
8 changes: 7 additions & 1 deletion docs/mcp-path-a.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,15 @@ CLI Path A verbs match: `demo`, `doctor`, `init`/`algorithms run`, `plan`, `evol
`status`/`watch`, `show`, `apply`, `cancel`, `resume`, `algorithms list|show`.

```
demo (optional, zero-key) → doctor → write a spec → plan (show the user) → start → status → show → ask → apply
demo (optional, zero-key) → doctor → write a spec → plan (show the user; --probe on a first model) → start → status → show → ask → apply
```

The first time a spec names a model or endpoint, run the plan with the probe
on: `agentdescent plan <spec> --probe` on the CLI, or `plan(spec, probe=true)`
over MCP. A refused probe prints one contract on both surfaces -- `ok: false`,
`ready: false`, and `next` as the repair action -- so give the user `next` and
do not call `start` (#1175).

## Advanced (opt-in)

Audit, release, platform, capability, ops, experience, tenant, meta_campaign,
Expand Down
3 changes: 2 additions & 1 deletion docs/plugin-quickstart.md
Original file line number Diff line number Diff line change
Expand Up @@ -281,7 +281,8 @@ Two habits worth keeping from the first run:
wrong model name or a dead endpoint is refused before a run store entry is
written. If the probe fails, the last line is the repair action (not
`next: agentdescent evolve`); `then:` is `plan <spec> --probe` to re-test,
and `--json` sets `ready: false` while leaving `ok` about the plan itself.
and `--json` sets `ok: false` and `ready: false` -- the MCP `plan` tool
prints the same pair, so one check works on both surfaces.
CLI agents are reported as `probe: … skipped (cli agent)` because one
call would cost a whole session.
* `--budget 5 --usd-per-call 0.02` on the first real run. A dollar budget
Expand Down
15 changes: 9 additions & 6 deletions docs/plugins.md
Original file line number Diff line number Diff line change
Expand Up @@ -258,12 +258,15 @@ all ~39 tools as the primary path.
the data and builds the policy bundle without running anything, then returns
the composed `evolve()` arguments and an upper bound on agent calls. A wrong
field fails here, by name, not in round one of a background process.
* **The probe is on `plan`, for both surfaces.** `plan --probe` (CLI) and
`plan(spec, probe=true)` (MCP) send one minimal call per model-backed agent,
through the same adapter the run would use, so an expired subscription or a
wrong model name is answered before a run store entry exists. A refusal sets
`ok: false` and `next` to the repair action, never to `start`; the default
`probe` is off, and plain `plan` still sends nothing.
* **The probe is on `plan`, for both surfaces, and one refusal contract comes
back.** `plan --probe` (CLI) and `plan(spec, probe=true)` (MCP) send one
minimal call per model-backed agent, through the same adapter the run would
use, so an expired subscription or a wrong model name is answered before a
run store entry exists. A refusal prints `ok: false`, `ready: false` and
`next` as the repair action -- identical keys and values on both surfaces, so
a host never has to translate its check -- and `next` never says `start`. The
default `probe` is off; plain `plan` still sends nothing and has no `ready`
key.
* **`start` returns in under a second.** The run is a detached process; the host
polls `status`. MCP tool calls time out and hosts restart their servers
between sessions, so nothing blocks on `evolve()`.
Expand Down
3 changes: 2 additions & 1 deletion integrations/claude-code/skills/agentdescent/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,8 @@ after `doctor`. Do not invent a keyless path for those cards.
and answers "will start work" before a run exists. If it refuses, give the
user its `next` (renew the plan, fix the model name or key) and do **not**
call `start` -- a refused probe is the run failing in round one, found one
step earlier.
step earlier. A refusal prints `ok: false` and `ready: false` on both
surfaces, so the same check reads the same answer whichever one you drive.
To keep improving across sessions, offer `agentdescent campaign <spec>
--sessions N`; it works on a copy, reports the workspace/final target and
never touches the live target. Run its `--dry-run` first. A campaign has no
Expand Down
2 changes: 1 addition & 1 deletion integrations/dsh-agentdescent/lib/index.js

Large diffs are not rendered by default.

64 changes: 62 additions & 2 deletions tests/test_mcp_plan_probe.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,22 @@
skill: it teaches `plan` -> user yes -> `start`, and never mentions the probe.

These rows pin the tool contract (`probe` is opt-in, reports the same payload
`probe_payload` returns, and a refusal makes `ok` false with a repair `next`)
and the host template (every generated SKILL.md teaches the probe).
`probe_payload` returns, and a refusal makes `ok` false with a repair `next`),
the one probe verdict both surfaces publish (`ready`, #1175) and the host
template (every generated SKILL.md teaches the probe).
"""
from __future__ import annotations

import io
import json
import threading
from contextlib import redirect_stderr, redirect_stdout
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path

import pytest

from agentdescent import cli
from agentdescent.integrations import HOSTS, host_status, install
from agentdescent.mcp import Tools

Expand Down Expand Up @@ -141,3 +145,59 @@ def test_every_host_skill_teaches_the_probe(tmp_path):
status = host_status(host, home=str(tmp_path))
assert status["state"] == "ok", (host, status)
assert status["skill"] == "ok", (host, status)


def _write_spec(tmp_path):
path = tmp_path / "spec.json"
path.write_text(json.dumps(_spec()), encoding="utf-8")
return str(path)


def _cli_plan(spec_path):
out, err = io.StringIO(), io.StringIO()
with redirect_stdout(out), redirect_stderr(err):
code = cli.main(["plan", spec_path, "--json", "--probe"])
return code, out.getvalue(), err.getvalue()


def test_both_surfaces_publish_the_same_refusal_triple(tmp_path, endpoint):
"""#1175: a refused probe prints one contract on both surfaces.

`(ok, ready, next)` is identical -- ``false``, ``false``, the repair
sentence. `ok` is not "the spec parsed": a run that cannot start is not
success on either surface, and a host that ports its check from the CLI to
MCP (or back) reaches the same verdict from either key.
"""
endpoint(code=400, body=ARK_SUBSCRIPTION)

cli_code, cli_out, cli_err = _cli_plan(_write_spec(tmp_path))
cli_payload = json.loads(cli_out)
mcp_payload = Tools(str(tmp_path / "runs")).plan(_spec(), probe=True)

assert cli_code == 1, (cli_code, cli_err)
cli_triple = (cli_payload["ok"], cli_payload["ready"], cli_payload["next"])
mcp_triple = (mcp_payload["ok"], mcp_payload["ready"], mcp_payload["next"])
assert cli_triple == mcp_triple, (cli_triple, mcp_triple)
assert cli_triple[0] is False and cli_triple[1] is False
assert cli_triple[2].startswith("renew the subscription"), cli_triple[2]
assert cli_payload["probe"]["ok"] is False
assert mcp_payload["probe"]["ok"] is False
assert _Handler.hits == 2 # one probe call per surface


def test_both_surfaces_publish_ok_and_ready_true_for_a_healthy_probe(tmp_path,
endpoint):
endpoint(code=200)

cli_code, cli_out, cli_err = _cli_plan(_write_spec(tmp_path))
cli_payload = json.loads(cli_out)
mcp_payload = Tools(str(tmp_path / "runs")).plan(_spec(), probe=True)

assert cli_code == 0, (cli_code, cli_err)
assert (cli_payload["ok"], cli_payload["ready"]) == (True, True)
assert (mcp_payload["ok"], mcp_payload["ready"]) == (True, True)


def test_the_mcp_plan_without_a_probe_has_no_ready_key(tmp_path):
out = Tools(str(tmp_path / "runs")).plan(_spec())
assert "ready" not in out
13 changes: 9 additions & 4 deletions tests/test_plan_probe.py
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,7 @@ def test_algorithms_run_json_exits_nonzero_on_a_refused_probe(tmp_path, monkeypa

payload = json.loads(out)
assert payload["probe"]["ok"] is False
assert payload["ok"] is True, "the plan itself is fine; only the probe refused"
assert payload["ok"] is False, "a refusal is not success on either surface (#1175)"
assert payload["ready"] is False
assert payload["next"] == (
"renew the subscription at the provider, or point api_key_env / "
Expand Down Expand Up @@ -344,15 +344,20 @@ def fake_claude(**kwargs):
assert seen["max_tokens"] == probe_module.PROBE_MAX_TOKENS
assert seen["timeout"] == probe_module.DEFAULT_PROBE_TIMEOUT

def test_plan_json_marks_a_refused_probe_unready_and_keeps_ok(tmp_path, endpoint):
"""`ok` stays about the plan; `ready` and `next` are how a host gates (#1093)."""
def test_plan_json_marks_a_refused_probe_unready_and_not_ok(tmp_path, endpoint):
"""A refusal is neither `ok` nor `ready`; `next` is the repair (#1175).

The plan file parsed, but the run it describes cannot start, so the same
pair travels to the MCP `plan` tool instead of `ok: true` here and
`ok: false` there.
"""
endpoint(body=ARK_SUBSCRIPTION)

code, out, err = _run(["plan", _write_spec(tmp_path), "--json", "--probe"])

payload = json.loads(out)
assert code == 1, (code, err)
assert payload["ok"] is True
assert payload["ok"] is False
assert payload["ready"] is False
assert payload["probe"]["ok"] is False
assert payload["next"] == (
Expand Down
6 changes: 4 additions & 2 deletions tools/gen_algorithms_docs.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,10 @@ def render() -> str:
"`evolve` -- with the same `SubscriptionInactive` / `QuotaExhausted` /",
"`ProviderConfigError` answer and no run store entry. A refused probe",
"ends with `next:` naming the repair (not `evolve`), and `--json`",
"sets `ready: false` while `ok` stays about the plan itself (#1093).",
"It is off by default: plain `plan` still runs and spends nothing.",
"sets `ok: false` and `ready: false` -- the same refusal contract the MCP",
"`plan` tool prints, so a host reads either key the same way on both",
"surfaces (#1175). It is off by default: plain `plan` still runs and",
"spends nothing, and has no `ready` key.",
"",
"Include cases the seed gets wrong: if the seed already scores",
"the target on held-out, the run stops at once with",
Expand Down
Loading