Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions docs/development/testing-and-quality.md
Original file line number Diff line number Diff line change
Expand Up @@ -1313,6 +1313,25 @@ fields; this does not introduce a new telemetry system or receipt gate.

### Budget Failure Decisions

The thin heartbeat's 4,500-character presentation allowance did not cover the
complete combination of a 320-character scope, all five runtime capability
flags, Codex App transport and an explicit CLI route. The existing public CLI
fixture with `/opt/loopx/qualified-tools/bin/loopx` emits 4,851 normalized body
characters at `a36e01c32`; its instructions and command arguments are complete.
Calibrate only this presentation allowance to 4,900, leaving 49 characters above
that input. Retain the original over-budget observation and add the complete
CLI combination plus an oversized-route negative case. Prompt text, capability
flags, the visible Goal's 4,000-character host limit, existing JSON/body fixture
ceilings, execution authority and frozen qualification results remain unchanged.
This corrects diagnostic coverage; it is not an output reduction or a claim of
model compliance, runtime recovery or lower cost.

薄提示原 4,500 字符展示预算未覆盖完整能力与显式 CLI 路由的组合;同一公开夹具
保留 320 字符 scope、五项能力和完整指令时为 4,851。仅将该展示预算校准为
4,900,保留原超限记录,并加入完整组合及更长路由仍超限的反例。正文、参数、
4,000 字符宿主硬边界、既有 JSON/正文夹具上限及执行权限不变;不据此声称节省
输出、恢复运行或改善模型行为。

The fixed visible-Goal host fixture's 3,200-character regression guard predates
`1d5d2d0f6`, which adds the known Todo choice to the first current guard. On the
same four host inputs at `b201da8ba`, substituting only the pre-change dispatch
Expand Down
5 changes: 4 additions & 1 deletion loopx/control_plane/heartbeat/budget.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,10 @@
# The same generic CLI + 320-char scope with explicit runtime also needs
# 5,715 chars; retain those routes instead of shortening the instructions.
"brief": 5_800,
"thin": 4_500,
# Full runtime capability flags plus an explicit CLI route and 320-char
# scope emit 4,851 normalized chars with unchanged guidance. Keep that
# supported combination; this is not a transport or execution limit.
"thin": 4_900,
"visible_goal": 4_000,
}
NATIVE_GOAL_HOST_MAX_CHARS = INTERFACE_BUDGET_CHARS["visible_goal"]
Expand Down
40 changes: 39 additions & 1 deletion tests/control_plane/test_cli_output_budget.py
Original file line number Diff line number Diff line change
Expand Up @@ -1712,12 +1712,50 @@ def test_thin_json_budget_retains_full_supported_scope(tmp_path: Path) -> None:
)
payload = measurement["payload"]
assert scope in payload["task_body"]
assert payload["interface_budget"]["max_chars"] == 4500
assert payload["interface_budget"]["max_chars"] == 4900
assert payload["interface_budget"]["within_budget"] is True
assert "refresh-state --goal-outcome-update-json" in payload["task_body"]
assert "Local outcomes never replace root continuation, budget or stop criteria." in payload["task_body"]


@pytest.mark.parametrize("route_padding, within_budget", [("", True), ("x" * 48, False)])
def test_thin_budget_covers_complete_capability_and_cli_routes(
tmp_path: Path, route_padding: str, within_budget: bool,
) -> None:
# Exercise the actual CLI composition, not a shortened instruction or an
# estimated sum. An oversized route must still report a budget failure.
with _stable_budget_fixture_root(tmp_path) as root:
project, runtime, registry_path, state_file = _write_fixture(root, SCENARIOS[0])
import_fixture_authority(registry_path, GOAL_ID)
command = _surface_commands(
project=project, runtime=runtime, registry_path=registry_path,
state_file=state_file, output_format="json",
)["heartbeat_prompt_thin"]
scope = "x" * 320
command[command.index("--agent-scope") + 1] = scope
cli_route = f"/opt/loopx/{route_padding}qualified-tools/bin/loopx"
command.extend(["--codex-app", "--cli-bin", cli_route])
capabilities = ("shell", "filesystem", "network", "external_evidence_poll", "browser")
for capability in capabilities:
command.extend(["--available-capability", capability])
exit_code, text = _invoke_cli(command)
assert exit_code == 0, text
payload = json.loads(text)
body, budget = payload["task_body"], payload["interface_budget"]
assert scope in body and body.count(cli_route) >= 2
assert all(f"--available-capability {capability}" in body for capability in capabilities)
assert "--codex-app" in body and "--runtime-root" in body and "--registry" in body
assert "--todo-id <id>" in body and "settlement_plan.ordered_steps" in body
assert "ambiguous effects need readback/recovery, not blind retry" in body
assert "refresh-state --goal-outcome-update-json" in body
assert "Local outcomes never replace root continuation, budget or stop criteria." in body
# Preserve the historical 4,500-character failure as a characterized
# input; the allowance changes, not the complete emitted instructions.
assert budget["budget_char_count"] > 4500
assert budget["max_chars"] == 4900
assert budget["within_budget"] is within_budget


def test_brief_budget_retains_full_commands_on_real_long_paths() -> None:
# A reproducible 128-character absolute root; directory depth can still
# change JSON escaping cost. Keep full routes and this workload;
Expand Down
4 changes: 2 additions & 2 deletions tests/control_plane/test_heartbeat_prompt_support.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ def test_native_profile_long_routes_retain_one_complete_settlement_contract() ->

@pytest.mark.parametrize("mode,limit", [
("full", 13100), ("compact", 8000), ("brief", 5800),
("thin", 4500), ("visible_goal", 4000),
("thin", 4900), ("visible_goal", 4000),
])
def test_prompt_body_limit_remains_independent_of_json_envelope(mode: str, limit: int) -> None:
for size in (limit, limit + 1):
Expand Down Expand Up @@ -356,7 +356,7 @@ def test_brief_diagnostics_keep_only_the_selected_prompt_command() -> None:

@pytest.mark.parametrize(
("mode", "base_budget"),
[("full", 13100), ("compact", 8000), ("brief", 5800), ("thin", 4500)],
[("full", 13100), ("compact", 8000), ("brief", 5800), ("thin", 4900)],
)
@pytest.mark.parametrize("scope_chars", [0, 320])
@pytest.mark.parametrize("runtime_profile", ["codex_app_heartbeat", "generic_cli"])
Expand Down
Loading