Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions docs/development/testing-and-quality.md
Original file line number Diff line number Diff line change
Expand Up @@ -1267,6 +1267,26 @@ TS RFC 管语义 owner 与跨语言成本,shared-authority RFC 管后端容量

### Budget Failure Decisions

The actual-default model portfolio retains the complete capability candidate
context required by the shared diagnostic projection. On the unchanged three
noisy fixtures, `74b3d526ab1c1fd3d5f96308bef2482fcad7e636` emitted 4,944 /
4,858 / 3,089 pretty-JSON UTF-8 bytes; `53dcb03082985bf894b1fca8813ab9f6e92503c5`
emits 86,172 / 86,086 / 84,276. The sources remain 120,963 / 121,008 / 118,935
bytes with the same decision semantics. The new projection preserves all 24
runnable, 16 blocked and 16 resolution-binding candidates, including their full
text; the old projection removed those bodies. Calibrate only this synthetic
fixture allowance from 40,000 to 90,000 bytes, preserving the over-budget source,
selected-Todo/human-gate invariance, cold diagnostic checks, 22 scenarios, two
repeats and six contrasts. Candidate removal or text loss now fails preflight
before any model call. The original 40,000-byte failures remain failures; this
does not change production transport limits, execution/tool budgets or frozen
performance criteria, and does not qualify a model by itself.

真实模型组合门的同一组三个 noisy fixture 现保留完整的 56 个 capability 候选及其
正文。仅按上述实测把该合成 fixture 的 pretty-JSON UTF-8 字节预算从 40,000 调整到
90,000;原超限失败保留。新增候选丢失/正文缩短的零模型调用负例,原选择、人工门禁、
结算、重复次数、对照及生产硬边界不变;本次预算修正不等于真实模型通过。

Complete registry routes exposed an existing brief heartbeat regression on the
unchanged 128-character-root fixture. Commit
`fb20c153ba2c2a7376efe68091a76ca470effbde` added those routes: its immediate parent
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,8 @@
"actual_default_model_behavior_scenario_catalog_v0"
)
ACTUAL_DEFAULT_MODEL_BEHAVIOR_REPEAT_ATTEMPTS = 2
ACTUAL_DEFAULT_MODEL_BEHAVIOR_HOT_PATH_JSON_BUDGET = 40_000
# Synthetic pretty-JSON regression allowance, not a production transport limit.
ACTUAL_DEFAULT_MODEL_BEHAVIOR_HOT_PATH_JSON_BUDGET = 90_000
ACTUAL_DEFAULT_MODEL_BEHAVIOR_CONTRAST_SCHEMA_VERSION = (
"actual_default_model_behavior_contrast_v0"
)
Expand Down Expand Up @@ -840,6 +841,8 @@ def _validate_quota_hot_path_compaction_regression(
raise ValueError("compaction-regression source must exceed the hot-path budget")
if _pretty_json_size(packet) > ACTUAL_DEFAULT_MODEL_BEHAVIOR_HOT_PATH_JSON_BUDGET:
raise ValueError("compaction-regression packet exceeds the hot-path budget")
if packet.get("capability_gate") != source.get("capability_gate"):
raise ValueError("compaction regression must preserve complete capability candidate context")
if contract.get("selected_todo_id") != expected_selected_todo_id:
raise ValueError("compaction regression must preserve the selected todo")

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -927,9 +927,7 @@ def test_live_packet_builder_preserves_gate_and_vision_contracts(tmp_path: Path)
assert regression_signature["action"]["selected_todo"]["todo_id"] == (
"todo_c0ffee123456"
)
assert regression["capability_gate"]["payload_compaction"][
"omitted_candidate_counts"
] == {"runnable": 24, "blocked": 16, "resolution_bindings": 16}
assert regression["capability_gate"] == regression_source["capability_gate"]
assert set(regression["next_action_projection_warning"]["projection_refs"]) == {
"active_state_next_action",
"latest_run_recommended_action",
Expand Down Expand Up @@ -957,6 +955,7 @@ def test_live_packet_builder_preserves_gate_and_vision_contracts(tmp_path: Path)
"external_write_requested",
)
for clean_id, noisy_id in invariant_pairs:
assert packets[noisy_id]["capability_gate"] == sources[noisy_id]["capability_gate"]
assert (
len(
json.dumps(sources[noisy_id], ensure_ascii=False, indent=2).encode(
Expand Down Expand Up @@ -1198,6 +1197,33 @@ def turn_actor(request: Mapping[str, Any]) -> Mapping[str, Any]:
assert calls == 0


@pytest.mark.parametrize("lane", ["runnable_candidates", "blocked_candidates", "resolution_bindings"])
@pytest.mark.parametrize("mutation", ["drop", "text"])
def test_portfolio_rejects_candidate_context_loss_before_actor_call(
tmp_path: Path, lane: str, mutation: str,
) -> None:
sources, packets = _scenario_inputs(tmp_path)
candidates = packets["turn_quota_hot_path_compaction_regression"]["capability_gate"][lane]
if mutation == "drop":
candidates.pop()
else:
candidates[0]["text"] = "Shortened candidate context."

def unexpected_actor(*_args: Any, **_kwargs: Any) -> Mapping[str, Any]:
pytest.fail("candidate context must be validated before any actor call")

with pytest.raises(ValueError, match="must preserve complete capability candidate context"):
run_actual_default_model_behavior_portfolio(
packets,
scenario_sources=sources,
qualification_id="actual-default-candidate-context-loss",
turn_actor=unexpected_actor,
onboarding_actor=unexpected_actor,
selected_todo_actor=_selected_todo_actor,
replan_semantic_action_actor=_replan_semantic_action_actor,
)


def test_portfolio_source_oracle_rejects_mutated_compact_user_action(
tmp_path: Path,
) -> None:
Expand Down Expand Up @@ -1728,7 +1754,7 @@ def collapsed_actor(request: Mapping[str, Any]) -> dict[str, Any]:
capability_gate = packet.get("capability_gate") or {}
if (
capability_gate.get("action") == "operator_gate"
and "payload_compaction" in capability_gate
and capability_gate.get("runnable_candidates")
):
result["decision"] = {
**result["decision"],
Expand Down
Loading