diff --git a/assets/autoresearch/icon.png b/assets/autoresearch/icon.png new file mode 100644 index 0000000..4c98ea0 Binary files /dev/null and b/assets/autoresearch/icon.png differ diff --git a/assets/icon-sources.json b/assets/icon-sources.json index 18b6b04..8d7e5a5 100644 --- a/assets/icon-sources.json +++ b/assets/icon-sources.json @@ -2049,5 +2049,12 @@ "homepage": "https://www.zyte.com", "mimeType": "image/png", "sha256": "8613b1dc383a3b6090a8a81aefc13f4867667fbf1902dc35471f8d554e69a1cd" + }, + { + "name": "autoresearch", + "icon": "autoresearch/icon.png", + "source": "https://github.com/luochang212/zcode-autoresearch/blob/main/assets/icon.svg", + "mimeType": "image/png", + "sha256": "990e83531f283f2a24f456c8dec76fef87fc0970ca3710463ca4af51efbbd686" } ] diff --git a/marketplace.json b/marketplace.json index a6a0269..a327338 100644 --- a/marketplace.json +++ b/marketplace.json @@ -152,6 +152,29 @@ "motion" ] }, + { + "name": "autoresearch", + "source": "./plugins/autoresearch", + "icon": "https://cdn-zcode.z.ai/zcode/official-plugin/assets/autoresearch/icon.png", + "description": "Autonomous experiment loop for ZCode: set a goal, pick a mechanical metric, and let the agent iterate — measure, keep what works, revert what doesn't, repeat. Provides init/run/log experiment tools (MCP), a loop protocol skill, guardrails (frozen benchmark, checks backpressure, memory injection, Stop continuation), and a static dashboard export.", + "description_i18n": { + "en": "Autonomous experiment loop for ZCode: set a goal, pick a mechanical metric, and let the agent iterate — measure, keep what works, revert what doesn't, repeat.", + "zh-CN": "ZCode 自主实验循环：设定目标与机械度量，让 agent 迭代——测量、保留有效改动、回滚无效改动、循环往复。" + }, + "version": "0.1.0", + "author": { + "name": "luochang212" + }, + "category": "developer-tools", + "keywords": [ + "autoresearch", + "experiment", + "optimization", + "loop", + "benchmark", + "mcp" + ] + }, { "name": "accounting-and-reporting", "source": "./plugins/accounting-and-reporting", diff --git a/plugins/autoresearch/.mcp.json b/plugins/autoresearch/.mcp.json new file mode 100644 index 0000000..719b7e9 --- /dev/null +++ b/plugins/autoresearch/.mcp.json @@ -0,0 +1,17 @@ +{ + "mcpServers": { + "autoresearch": { + "type": "stdio", + "command": "node", + "args": ["${ZCODE_PLUGIN_ROOT}/mcp/server.ts"], + "cwd": "${ZCODE_PROJECT_DIR}", + "env": { + "AR_MAX_ITERATIONS": "${user_config.maxIterations}", + "AR_BENCHMARK_TIMEOUT_MS": "${user_config.benchmarkTimeoutMs}", + "AR_CHECKS_TIMEOUT_MS": "${user_config.checksTimeoutMs}" + }, + "enabled": true, + "timeoutMs": 60000 + } + } +} diff --git a/plugins/autoresearch/.zcode-plugin/plugin.json b/plugins/autoresearch/.zcode-plugin/plugin.json new file mode 100644 index 0000000..9fd286e --- /dev/null +++ b/plugins/autoresearch/.zcode-plugin/plugin.json @@ -0,0 +1,40 @@ +{ + "name": "autoresearch", + "description": "Autonomous experiment loop for ZCode: set a goal, pick a mechanical metric, and let the agent iterate. It measures, keeps what works, reverts what doesn't, and repeats. Provides init/run/log experiment tools (MCP), a loop protocol skill, guardrails (frozen benchmark, checks backpressure, memory injection, Stop continuation), and static + live dashboards.", + "description_i18n": { + "en": "Autonomous experiment loop for ZCode: set a goal, pick a mechanical metric, and let the agent iterate. It measures, keeps what works, reverts what doesn't, and repeats.", + "zh-CN": "ZCode 自主实验循环：设定目标与机械度量，让 agent 迭代。每一轮测量、保留有效改动、回滚无效改动，如此往复。" + }, + "version": "0.1.0", + "author": { + "name": "luochang212" + }, + "keywords": [ + "autoresearch", + "experiment", + "optimization", + "loop", + "benchmark", + "mcp" + ], + "userConfig": { + "maxIterations": { + "title": "Max Iterations", + "description": "Default iteration cap per experiment segment. Overridable per session via .auto/config.json.", + "type": "number", + "default": 20 + }, + "benchmarkTimeoutMs": { + "title": "Benchmark Timeout (ms)", + "description": "Wall-clock timeout for run_experiment commands.", + "type": "number", + "default": 600000 + }, + "checksTimeoutMs": { + "title": "Checks Timeout (ms)", + "description": "Wall-clock timeout for the correctness check script (.auto/checks.sh).", + "type": "number", + "default": 300000 + } + } +} diff --git a/plugins/autoresearch/README.md b/plugins/autoresearch/README.md new file mode 100644 index 0000000..076b482 --- /dev/null +++ b/plugins/autoresearch/README.md @@ -0,0 +1,121 @@ +# autoresearch + +[中文文档](./README_CN.md) + +Let the ZCode coding agent iterate autonomously on a fixed, mechanical metric: modify code → run the benchmark → keep improvements, revert regressions → repeat. + +Based on research into [karpathy/autoresearch](https://github.com/karpathy/autoresearch) and [pi-autoresearch](https://github.com/davebcn87/pi-autoresearch) (see `docs/research/autoresearch-survey.md`). Architecture decisions live in `adr/decisions/`. + +## Security and side effects + +This plugin executes code and operates on a git repository. Enabling it grants code-execution trust (official marketplace convention). Specifically, it: + +- **Runs commands**: `run_experiment` executes the benchmark script you author (`.auto/measure.sh`) and, when present, the correctness gate (`.auto/checks.sh`); +- **Runs git operations automatically**: `git commit` on keep, automatic rollback on non-keep (`.auto/` is exempt from rollback); +- **Installs ZCode hooks**: Stop (loop continuation), PreToolUse (frozen-file write protection), PermissionRequest (experiment-tool gating), UserPromptSubmit/SessionStart (ledger memory injection); +- **Serves a local HTTP dashboard** on 127.0.0.1 via `export_dashboard`; +- **Writes session state** to `.auto/` files (`log.jsonl`, `config.json`) in the project directory. + +No third-party npm dependencies: the MCP server and hooks are Node-stdlib TypeScript scripts (Node ≥24, types stripped natively, no build step). + +## Install + +This repository is itself a plugin marketplace (`marketplace.json` points to `./plugin`). In ZCode: + +1. Add the marketplace: this repository's URL (or a local directory). +2. In **Settings → Plugin Management**, install and enable `autoresearch`. +3. The plugin provides: an MCP server (5 tools), the `autoresearch` skill, 5 slash commands, and 5 hooks. + +## Usage + +``` +/autoresearch:autoresearch # enter/resume autoresearch mode (runs setup if there is no session) +/autoresearch:export # export a static dashboard (autoresearch-dashboard.html) +/autoresearch:off # pause loop continuation (sets autoresearchOff: true) +/autoresearch:clear # reset the session ledger +/autoresearch:finalize # organize kept experiments into a clean branch (scripts/finalize.sh) +``` + +Or let the skill trigger on its own (descriptions containing "autoresearch", "autonomous optimization", etc.). A full loop: + +1. **Setup**: pick a mechanical metric → write `.auto/measure.sh` (emits `METRIC name=value` lines) → optionally `.auto/checks.sh` (correctness gate) → write the `.auto/prompt.md` charter → create an experiment branch `git checkout -b autoresearch/`. +2. `init_experiment` (metric_name, direction) → run a baseline. +3. **Loop**: one focused change → `run_experiment` → `log_experiment` (keep auto-commits / non-keep auto-rolls-back, `.auto/` exempt). + +## Tools (MCP) + +| Tool | What it does | +| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `init_experiment` | Start/restart an experiment segment (name, primary metric, direction lower/higher) | +| `run_experiment` | Run the benchmark: times it, parses `METRIC name=value` lines, returns a truncated tail (10 lines / 4KB), kills the process group on timeout, takes the median over `repeat` runs, runs the `before.sh` hook | +| `log_experiment` | Record the outcome: keep auto-commits (`experiment:` prefix); non-keep auto-rolls-back (`.auto/` exempt); returns baseline/best/delta/confidence/plateau plus a next-action hint; runs the `after.sh` hook | +| `export_dashboard` | Serve a live local dashboard (127.0.0.1 + SSE auto-refresh) and write a static HTML fallback | +| `clear_experiments` | Delete `.auto/log.jsonl` and reset the session (keeps measure/checks/prompt) | + +## Guardrails + +- **Benchmark locking**: when `.auto/measure.sh` exists, `run_experiment` only executes that script (validated after stripping env/time/nice wrappers). +- **Correctness backpressure**: when `.auto/checks.sh` exists, it runs automatically after a passing benchmark; a failing gate forbids keep and rolls back. +- **Write protection**: a PreToolUse hook denies writes to `.auto/measure.sh` / `.auto/checks.sh`. +- **Tool gating (approximate)**: a PermissionRequest hook denies experiment tools when there is no session (only covers calls that go through the permission prompt). +- **Auto-resume hint**: SessionStart injects a continuation hint when an active session is detected; `/autoresearch:off` pauses it (`autoresearchOff: true`). +- **Memory injection**: UserPromptSubmit/SessionStart hooks inject an aggregated summary (progress + deduped tried directions + best trajectory + ASI distillation) so progress survives compaction; repeated/oscillating attempts (doom-loop) trigger a hint to switch direction. +- **Loop continuation**: the Stop hook blocks (`decision:block`) while a loop is unfinished (zcode platform limit: 3 consecutive windows). +- **Iteration hooks**: `.auto/hooks/before.sh` (pre-benchmark) and `after.sh` (post-record) run on every experiment (fail-open, 30s timeout, stdout → `*_steer`). +- **Hook ecosystem**: `skills/autoresearch-hooks` tutorial + 6 ready-to-use examples in `hooks/examples/` (anti-thrash, hypothesis reflection, idea rotator, learnings journal, auto-tag winners, macOS notify); copy one to `.auto/hooks/` and go (parsed with Node, no jq dependency). +- **Stop-loss**: after `consecutiveFailures` in a row (default 3, configurable in `.auto/config.json`) the plugin hints you to stop. +- **Ledger audit**: `log_experiment` validates invariants before writing (keep must be a real improvement, a discarded real improvement must have failed the guard, event ordering, commit field); violations are rejected; a crashed segment that wasn't rolled back blocks continuation. `auditBypass: true` in `.auto/config.json` explicitly skips it (not recommended). +- **Benchmark drift detection**: `init_experiment` records hashes of measure.sh/checks.sh; `run_experiment` compares them, and a mid-run benchmark change returns a `benchmark_drift` warning (prevents "faking the metric by editing the benchmark"). +- **Secondary-metric constraints** (opt-in): `log_experiment` supports `constraints: [{name, maxPct}]`. On keep, secondary metrics must stay within maxPct% of the first run's value; anything beyond rejects the keep (prevents reward hacking like "trading memory for speed"). + +## Directory structure + +```text +plugin/ +├── .zcode-plugin/plugin.json # manifest (userConfig: maxIterations / timeouts) +├── .mcp.json # MCP stdio server declaration +├── mcp/ +│ ├── server.ts # JSON-RPC line protocol + tools +│ └── lib/ # pure logic: experiment / ledger / git / validate / dashboard / dashboard-server / html / paths +├── hooks/ +│ ├── hooks.json # Stop / PreToolUse / PermissionRequest / UserPromptSubmit / SessionStart +│ ├── stop-continue.ts # loop unfinished → block +│ ├── guard-frozen.ts # frozen-file write protection → deny +│ ├── permission-gate.ts # experiment-tool gating → deny +│ ├── memory-inject.ts # ledger tail injection +│ ├── session-start.ts # session resume hints +│ └── examples/ # 6 ready-to-use before/after iteration hooks +├── skills/ +│ ├── autoresearch/ # SKILL.md thin router + references/ +│ └── autoresearch-hooks/ # iteration-hook tutorial +├── commands/ # autoresearch / export / off / clear / finalize +├── scripts/finalize.sh # /autoresearch:finalize implementation +└── tests/ # node --test unit tests +``` + +## workingDir + +Setting `"workingDir": "work/"` in `.auto/config.json` separates the research directory from the project directory (ledger/benchmark/git/dashboard all act on work/, config stays in the project). + +## Session state (`.auto/`) + +| File | Purpose | +| ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `log.jsonl` | **append-only single source of truth**: config lines + run lines; segments advance on config lines | +| `prompt.md` | session charter (goal/metric/scope/Off Limits/What's Been Tried) | +| `measure.sh` | benchmark script (frozen) | +| `checks.sh` | optional correctness gate (frozen) | +| `config.json` | optional session overrides: `maxIterations`, `consecutiveFailures`, `workingDir`, `auditBypass`, `autoresearchOff` (server-managed keys like `benchmarkHashes`/`pendingChecksFailed` also live here) | +| `ideas.md` | optional hypothesis list | + +## Known limits (research-backed, see `docs/research/autoresearch-survey.md` §4.1) + +- **No session-injection API**: no overnight unattended runs; rely on the 3-window Stop-hook allowance plus user re-triggering to continue. +- **Headless mode (`--prompt`) does not run hooks**: guardrails take effect in interactive sessions; run autoresearch in an interactive session. +- `git add -A` commits unrelated dirty files together (known pi inheritance); commit a clean baseline during setup. + +## Development + +```bash +cd plugin && node --test tests/*.test.ts # unit tests +``` diff --git a/plugins/autoresearch/README_CN.md b/plugins/autoresearch/README_CN.md new file mode 100644 index 0000000..690ee85 --- /dev/null +++ b/plugins/autoresearch/README_CN.md @@ -0,0 +1,121 @@ +# autoresearch + +[English](./README.md) + +让 ZCode 的 coding agent 在**固定度量**上自主迭代优化：改代码 → 跑基准 → 保留改进、回滚退化 → 循环。 + +基于 [karpathy/autoresearch](https://github.com/karpathy/autoresearch) 与 [pi-autoresearch](https://github.com/davebcn87/pi-autoresearch) 的调研（见仓库根 `docs/research/autoresearch-survey.md`）。架构决策见 `adr/decisions/1-*.md`、`2-*.md`。 + +## 安装 + +本仓库即一个插件市场（`marketplace.json` 指向 `./plugin`）。在 ZCode 中： + +1. 添加市场：本地目录 / 本仓库地址。 +2. 在 **Settings → Plugin Management** 安装并启用 `autoresearch`。 +3. 插件提供：MCP 服务（5 个工具）、skill `autoresearch`、5 个命令、5 个 hook。启用插件即授予代码执行信任（官方约定）。 + +> 无第三方 npm 依赖：MCP server 与 hooks 均为 Node 标准库 TypeScript 脚本（Node ≥24，类型由 Node 原生剥离，无构建步骤）。 + +## 安全与副作用 + +启用本插件即授予代码执行信任（官方市场约定）。插件会： + +- **执行命令**：`run_experiment` 运行你编写的基准脚本 `.auto/measure.sh`，以及存在时的正确性门禁 `.auto/checks.sh`； +- **自动执行 git 操作**：keep 时自动 `git commit`，非 keep 时自动回滚（`.auto/` 豁免回滚）； +- **安装 ZCode hooks**：Stop（循环续跑）、PreToolUse（冻结文件写保护）、PermissionRequest（实验工具门禁）、UserPromptSubmit/SessionStart（账本记忆注入）； +- **启动本地 HTTP dashboard**：`export_dashboard` 监听 127.0.0.1； +- **写入会话状态**：项目目录下的 `.auto/`（`log.jsonl`、`config.json`）。 + +## 用法 + +``` +/autoresearch:autoresearch <目标> # 进入/恢复 autoresearch 模式（无会话则走 setup） +/autoresearch:export # 导出静态 dashboard（autoresearch-dashboard.html） +/autoresearch:off # 暂停循环续跑（autoresearchOff: true） +/autoresearch:clear # 重置会话账本 +/autoresearch:finalize # 把 kept 实验整理为干净分支（scripts/finalize.sh） +``` + +或让 skill 自动触发（描述含 "autoresearch"、"自主优化" 等）。一次完整循环： + +1. **Setup**：定机械度量 → 写 `.auto/measure.sh`（输出 `METRIC name=value` 行）→ 可选 `.auto/checks.sh`（正确性门禁）→ 写 `.auto/prompt.md` 章程 → 建实验分支 `git checkout -b autoresearch/`。 +2. `init_experiment`（metric_name、direction）→ 跑一次 baseline。 +3. **循环**：一次聚焦改动 → `run_experiment` → `log_experiment`（keep 自动 commit / 非 keep 自动回滚，`.auto/` 豁免）。 + +## 工具（MCP） + +| 工具 | 作用 | +| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `init_experiment` | 建立/重启实验 segment（名称、主度量、方向 lower/higher） | +| `run_experiment` | 跑基准：计时、`METRIC name=value` 解析、10 行/4KB 截断回传、超时杀进程组、`repeat` 取中位数、执行 `before.sh` 钩子 | +| `log_experiment` | 记录结果：keep 自动 `git commit`（`experiment:` 前缀）；非 keep 自动回滚（豁免 `.auto/`）；返回 baseline/best/delta/confidence/plateau 与下一步提示、执行 `after.sh` 钩子 | +| `export_dashboard` | 起本地 live dashboard（127.0.0.1 + SSE 自动刷新）并写静态 HTML 兜底 | +| `clear_experiments` | 删除 `.auto/log.jsonl` 重置会话（保留 measure/checks/prompt） | + +## 护栏 + +- **benchmark 锁定**：`.auto/measure.sh` 存在时，`run_experiment` 只执行该脚本（剥 env/time/nice 包装后校验）。 +- **正确性背压**：`.auto/checks.sh` 存在时 benchmark 通过后自动执行；失败禁 keep 并自动回滚。 +- **写保护**：PreToolUse hook deny 对 `.auto/measure.sh` / `.auto/checks.sh` 的写入。 +- **工具门禁（近似）**：无会话时 PermissionRequest hook deny 实验工具（仅覆盖经权限询问的调用）。 +- **自动激活提示**：SessionStart 检测活动会话时注入续跑引导；`/autoresearch:off` 可暂停（`autoresearchOff: true`）。 +- **记忆注入**：UserPromptSubmit/SessionStart hook 注入聚合摘要（进度 + 已尝试方向去重 + best 轨迹 + ASI 提炼），compaction 后不丢进度；检测到重复/震荡尝试（doom-loop）时提示换方向。 +- **循环续跑**：Stop hook 在循环未结束时 `decision:block`（zcode 平台限制连续 3 次窗口）。 +- **迭代钩子**：`.auto/hooks/before.sh`（基准前）与 `after.sh`（记录后）每次实验自动执行（fail-open，30s 超时，stdout→`*_steer`）。 +- **钩子生态**：`skills/autoresearch-hooks` 教学 + `hooks/examples/` 6 个现成示例（防重复失败/换思路/假设反思/学习日志/通知/最优打标），复制到 `.auto/hooks/` 即用（node 解析，无 jq 依赖）。 +- **止损**：连续失败达 `.auto/config.json` 的 `consecutiveFailures`（默认 3，可配）时提示停止。 +- **账本审计**：`log_experiment` 写入前校验不变量（keep 必须真实改进、discard 真改进须 failed guard、事件顺序、commit 字段），违规拒收；crash 未回滚禁止续跑。`.auto/config.json` 的 `auditBypass: true` 可显式跳过（不推荐）。 +- **基准漂移检测**：`init_experiment` 记录 measure.sh/checks.sh 哈希，`run_experiment` 比对；基准中途变更时返回 `benchmark_drift` 警告（防"改基准造假 metric"）。 +- **次级度量约束**（opt-in）：`log_experiment` 支持 `constraints: [{name, maxPct}]`。keep 时校验次级度量不超首轮值的 maxPct%，超界拒收（防"用内存换速度"类 reward hacking）。 + +## 目录结构 + +``` +plugin/ +├── .zcode-plugin/plugin.json # manifest（userConfig: maxIterations / 超时） +├── .mcp.json # MCP stdio server 声明 +├── mcp/ +│ ├── server.ts # JSON-RPC 换行协议 + 5 个工具 +│ └── lib/ # 纯逻辑：experiment / ledger / git / validate / dashboard / dashboard-server / html / paths +├── hooks/ +│ ├── hooks.json # Stop / PreToolUse / PermissionRequest / UserPromptSubmit / SessionStart +│ ├── stop-continue.ts # 循环未结束 → block +│ ├── guard-frozen.ts # 冻结文件写保护 → deny +│ ├── permission-gate.ts # 实验工具门禁 → deny +│ ├── memory-inject.ts # 账本尾行注入 +│ ├── session-start.ts # 会话恢复提示 +│ └── examples/ # 6 个现成的 before/after 迭代钩子示例 +├── skills/ +│ ├── autoresearch/ # SKILL.md 薄路由 + references/ +│ └── autoresearch-hooks/ # 迭代钩子教学 +├── commands/ # autoresearch / export / off / clear / finalize +├── scripts/finalize.sh # /autoresearch:finalize 实现 +└── tests/ # node --test 单元测试 +``` + +## workingDir + +在 `.auto/config.json` 设 `"workingDir": "work/"` 可将研究目录与项目目录分离（账本/基准/git/dashboard 全部作用于 work/，config 留在项目）。 + +## 会话状态（`.auto/`） + +| 文件 | 作用 | +| ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `log.jsonl` | **append-only 单一事实源**：config 行 + run 行；segment 由 config 行推进 | +| `prompt.md` | 会话章程（目标/度量/范围/Off Limits/What's Been Tried） | +| `measure.sh` | 基准脚本（冻结） | +| `checks.sh` | 可选正确性门禁（冻结） | +| `config.json` | 可选会话覆盖项：`maxIterations`、`consecutiveFailures`、`workingDir`、`auditBypass`、`autoresearchOff`（`benchmarkHashes`/`pendingChecksFailed` 等服务端管理键也存于此） | +| `ideas.md` | 可选假设清单 | + +## 已知边界（研究实证，详见报告 §4.1） + +- **无会话注入 API**：无过夜无人值守；靠 Stop hook 3 次窗口 + 用户再触发续跑。 +- **无头模式（`--prompt`）不执行 hooks**：护栏在交互式会话生效；请用交互式会话跑 autoresearch。 +- `git add -A` 会把无关脏文件一起 commit（继承 pi 的已知弱点）；setup 时先提交干净基线。 + +## 开发 + +```bash +cd plugin && node --test tests/*.test.ts # 单元测试 +``` diff --git a/plugins/autoresearch/commands/autoresearch.md b/plugins/autoresearch/commands/autoresearch.md new file mode 100644 index 0000000..85ba472 --- /dev/null +++ b/plugins/autoresearch/commands/autoresearch.md @@ -0,0 +1,14 @@ +--- +description: Enter autoresearch mode. Resumes from .auto/prompt.md when a session exists, otherwise sets one up from your goal. Usage: /autoresearch:autoresearch +--- + +Enter autoresearch mode for this workspace. + +1. If `.auto/log.jsonl` and `.auto/prompt.md` exist, **resume**: read the charter and the ledger, then continue the loop (one focused change → `run_experiment` → `log_experiment`). +2. Otherwise **set up a new session** from the goal in $ARGUMENTS: + - Load the skill `autoresearch` (or read `skills/autoresearch/SKILL.md`). + - Follow `references/setup-guide.md`: pick a mechanical metric, create `.auto/measure.sh` (prints `METRIC name=value`), optional `.auto/checks.sh`, write `.auto/prompt.md` charter. + - `init_experiment` with metric name and direction, run a baseline, then start the loop. +3. Remind the user: keep the benchmark frozen (`.auto/measure.sh` / `.auto/checks.sh` are write-protected by the plugin hook), and use `/autoresearch:export` for a dashboard. + +$ARGUMENTS diff --git a/plugins/autoresearch/commands/clear.md b/plugins/autoresearch/commands/clear.md new file mode 100644 index 0000000..1a82499 --- /dev/null +++ b/plugins/autoresearch/commands/clear.md @@ -0,0 +1,11 @@ +--- +description: Clear the autoresearch session by deleting .auto/log.jsonl, then start fresh. Keeps measure.sh / checks.sh / prompt.md. Usage: /autoresearch:clear +--- + +Clear the current autoresearch session. + +1. Confirm with the user that they want to wipe the experiment history. This cannot be undone: the session state is gone, though the ledger and all `experiment:` commits remain in git history. +2. Call the `clear_experiments` tool. +3. Report the result. A fresh target can now start with `/autoresearch:autoresearch ` or `init_experiment`. + +Note: kept `experiment:` commits remain in git history; only the `.auto/` session ledger is reset. diff --git a/plugins/autoresearch/commands/export.md b/plugins/autoresearch/commands/export.md new file mode 100644 index 0000000..e88a61f --- /dev/null +++ b/plugins/autoresearch/commands/export.md @@ -0,0 +1,11 @@ +--- +description: Render .auto/log.jsonl into autoresearch-dashboard.html. Usage: /autoresearch:export +--- + +Export the autoresearch experiment dashboard. + +1. Call the `export_dashboard` tool (prefer the MCP tool; the same export logic also lives in `${ZCODE_PLUGIN_ROOT}/mcp/server.ts`). +2. If the tool is unavailable, fall back to reading `.auto/log.jsonl` yourself, summarizing experiments (status, metric, delta vs baseline, direction), and writing a self-contained `autoresearch-dashboard.html` in the workspace root. +3. Tell the user the file path (`autoresearch-dashboard.html`) and a 2-3 line summary of progress (experiments run, kept, best metric). + +If there is no `.auto/log.jsonl`, say so and suggest `/autoresearch:autoresearch` to start a session first. diff --git a/plugins/autoresearch/commands/finalize.md b/plugins/autoresearch/commands/finalize.md new file mode 100644 index 0000000..6652daa --- /dev/null +++ b/plugins/autoresearch/commands/finalize.md @@ -0,0 +1,28 @@ +--- +description: Split kept experiments into clean topic branches you can PR. Usage: /autoresearch:finalize +--- + +Finalize the experiment session into clean, PR-able topic branches. + +1. Read `.auto/log.jsonl`, collect the **kept** experiments (status=keep with a commit). +2. Group them by file dependency: two experiments may share a branch only if their changed files overlap; group small, keep order. +3. Write `groups.json` at the project root: + ```json + { + "base": "", + "goal": "", + "groups": [ + { + "title": "perf: sieve", + "body": "...", + "last_commit": "", + "slug": "sieve" + } + ] + } + ``` + `last_commit` must be the full kept commit hash (`git rev-parse `). +4. Run `bash ${ZCODE_PLUGIN_ROOT}/scripts/finalize.sh `. +5. Report: the created branches (`autoresearch//NN-`), the overall metric improvement, and cleanup notes (`git branch -D` + `rm -r .auto` when done). + +If the script reports a file appearing in multiple groups, merge those groups or re-split and rerun. diff --git a/plugins/autoresearch/commands/off.md b/plugins/autoresearch/commands/off.md new file mode 100644 index 0000000..ed06592 --- /dev/null +++ b/plugins/autoresearch/commands/off.md @@ -0,0 +1,10 @@ +--- +description: Stop the auto-resume hints while keeping the session. Resume anytime with /autoresearch:autoresearch. Usage: /autoresearch:off +--- + +Pause autoresearch without wiping the session. + +1. Set `autoresearchOff: true` in `.auto/config.json` (create the file if missing). The SessionStart hook will stop injecting "resume" hints for this workspace. +2. The ledger and all experiment commits stay intact. +3. To resume: run `/autoresearch:autoresearch` (it ignores the off marker), or clear the marker (`autoresearchOff: false`) for hints again. +4. To start completely fresh: `/autoresearch:clear`. diff --git a/plugins/autoresearch/hooks/examples/after/auto-tag-winners.sh b/plugins/autoresearch/hooks/examples/after/auto-tag-winners.sh new file mode 100755 index 0000000..9d4d225 --- /dev/null +++ b/plugins/autoresearch/hooks/examples/after/auto-tag-winners.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# auto-tag-winners: tag every new best with a sortable git tag so +# `git log --tags` reads as a progression record. after hook. +# Pure side effect — no steer output. +set -euo pipefail + +payload="$(cat)" + +node - "$payload" <<'NODE' +const { execFileSync } = require('child_process'); +const p = JSON.parse(process.argv[2]); +const run = p.run_entry; +const session = p.session; +if (run.status !== 'keep') process.exit(0); +const best = session?.best_metric; +if (best == null || run.metric == null) process.exit(0); +if (run.metric !== best) process.exit(0); // not a new best +const tag = `autoresearch/best-run-${run.run}-${run.metric}`; +try { + execFileSync('git', ['-C', p.cwd, 'tag', '-f', tag], { stdio: 'ignore' }); +} catch { + /* not a git repo → silent */ +} +NODE diff --git a/plugins/autoresearch/hooks/examples/after/learnings-journal.sh b/plugins/autoresearch/hooks/examples/after/learnings-journal.sh new file mode 100755 index 0000000..2d66a6c --- /dev/null +++ b/plugins/autoresearch/hooks/examples/after/learnings-journal.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# learnings-journal: append one markdown line per experiment to +# .auto/learnings.md — a human-readable diary that survives the loop. +# after hook. Pure side effect, no steer. +set -euo pipefail + +payload="$(cat)" + +node - "$payload" <<'NODE' +const p = JSON.parse(process.argv[2]); +const fs = require('fs'); +const path = require('path'); +const run = p.run_entry; +const journal = `${p.cwd}/.auto/learnings.md`; +fs.mkdirSync(path.dirname(journal), { recursive: true }); +const line = `- run ${run.run} [${run.status}] metric=${run.metric ?? '—'}: ${run.description ?? ''}`; +fs.appendFileSync(journal, line + '\n'); +NODE diff --git a/plugins/autoresearch/hooks/examples/after/macos-notify.sh b/plugins/autoresearch/hooks/examples/after/macos-notify.sh new file mode 100755 index 0000000..37d68fa --- /dev/null +++ b/plugins/autoresearch/hooks/examples/after/macos-notify.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# macos-notify: post a macOS notification when an experiment completes. +# after hook. macOS only (osascript); silently no-ops elsewhere. +# Pure side effect — no steer output. +set -euo pipefail + +payload="$(cat)" + +node - "$payload" <<'NODE' +const { execFileSync } = require('child_process'); +const p = JSON.parse(process.argv[2]); +const run = p.run_entry; +const session = p.session; +const title = `autoresearch run ${run.run}: ${run.status}`; +const body = `metric=${run.metric ?? '—'} (best=${session?.best_metric ?? '—'}) ${run.description ?? ''}`; +try { + execFileSync('osascript', ['-e', `display notification "${body.replace(/"/g, '\\"')}" with title "${title.replace(/"/g, '\\"')}"`], { stdio: 'ignore' }); +} catch { + /* no osascript (non-macOS) → silent */ +} +NODE diff --git a/plugins/autoresearch/hooks/examples/before/anti-thrash.sh b/plugins/autoresearch/hooks/examples/before/anti-thrash.sh new file mode 100755 index 0000000..bdd3bcf --- /dev/null +++ b/plugins/autoresearch/hooks/examples/before/anti-thrash.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# anti-thrash: after N consecutive discards/crashes, suggest a structural rethink. +# before hook. Reads the ledger tail via stdin payload (.cwd). Silent otherwise. +# +# Contract fields used: event, cwd, last_run, session.run_count +set -euo pipefail + +readonly STREAK_THRESHOLD=3 +readonly WINDOW=5 + +payload="$(cat)" + +node - "$payload" <<'NODE' +const p = JSON.parse(process.argv[2]); +const fs = require('fs'); +const log = `${p.cwd}/.auto/log.jsonl`; +if (!fs.existsSync(log)) process.exit(0); + +const tail = fs.readFileSync(log, 'utf8') + .split('\n').filter(Boolean).map(l => { try { return JSON.parse(l); } catch { return null; } }) + .filter(e => e && e.type === 'run') + .slice(-5); + +let streak = 0; +for (const r of [...tail].reverse()) { + if (r.status === 'keep') break; + streak += 1; +} +if (streak < 3) process.exit(0); + +console.log(`⚠️ ${streak} consecutive non-keep results. Consider:`); +console.log(' - re-reading .auto/prompt.md and the benchmark script'); +console.log(' - something structurally different, not another variation of the same idea'); +console.log(' - measuring where time/space is actually spent before the next change'); +NODE diff --git a/plugins/autoresearch/hooks/examples/before/hypothesis-reflection.sh b/plugins/autoresearch/hooks/examples/before/hypothesis-reflection.sh new file mode 100755 index 0000000..b115354 --- /dev/null +++ b/plugins/autoresearch/hooks/examples/before/hypothesis-reflection.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# hypothesis-reflection: before each run, remind the agent to state a clear +# hypothesis when the previous run recorded none (asi.hypothesis missing). +# before hook. Silent when the last run already had one. +set -euo pipefail + +payload="$(cat)" + +node - "$payload" <<'NODE' +const p = JSON.parse(process.argv[2]); +const last = p.last_run; +if (!last) process.exit(0); +if (last.asi && last.asi.hypothesis) process.exit(0); + +console.log('🧪 The last run had no recorded hypothesis (asi.hypothesis).'); +console.log(' Before this run, state in one line what you are testing and why it should help,'); +console.log(' then pass it as asi.hypothesis in log_experiment.'); +NODE diff --git a/plugins/autoresearch/hooks/examples/before/idea-rotator.sh b/plugins/autoresearch/hooks/examples/before/idea-rotator.sh new file mode 100755 index 0000000..c848b72 --- /dev/null +++ b/plugins/autoresearch/hooks/examples/before/idea-rotator.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# idea-rotator: pick the next untried idea from .auto/ideas.md (one idea per +# line, lines starting with '#' are ignored) and steer the agent to try it. +# before hook. Silent when there is no ideas file or no untried ideas left. +set -euo pipefail + +payload="$(cat)" + +node - "$payload" <<'NODE' +const p = JSON.parse(process.argv[2]); +const fs = require('fs'); +const ideasFile = `${p.cwd}/.auto/ideas.md`; +if (!fs.existsSync(ideasFile)) process.exit(0); + +const lines = fs.readFileSync(ideasFile, 'utf8') + .split('\n') + .map(l => l.trim()) + .filter(l => l && !l.startsWith('#')); + +if (lines.length === 0) process.exit(0); + +// Rotate using the run counter so each experiment surfaces a different idea. +const idx = (p.session?.run_count ?? 0) % lines.length; +console.log(`💡 untried idea to consider: ${lines[idx]}`); +console.log(' (add/remove lines in .auto/ideas.md to control the pool)'); +NODE diff --git a/plugins/autoresearch/hooks/guard-frozen.ts b/plugins/autoresearch/hooks/guard-frozen.ts new file mode 100644 index 0000000..61abb91 --- /dev/null +++ b/plugins/autoresearch/hooks/guard-frozen.ts @@ -0,0 +1,47 @@ +#!/usr/bin/env node +// PreToolUse hook: deny writes to the frozen benchmark scripts +// (.auto/measure.sh, .auto/checks.sh). The matcher limits this to +// Write|Edit|ApplyPatch; path filtering happens here, per zcode docs. +import { resolve, relative } from "node:path"; +import { resolveWorkCwd } from "../mcp/lib/paths.ts"; + +interface PreToolUseInput { + tool_input?: { file_path?: string; path?: string }; + toolInput?: { file_path?: string; path?: string }; +} + +const cwd = resolveWorkCwd(process.argv[2] || process.cwd()); +const FROZEN = new Set([".auto/measure.sh", ".auto/checks.sh"]); + +let raw = ""; +process.stdin.setEncoding("utf8"); +for await (const chunk of process.stdin as AsyncIterable) raw += chunk; + +let input: PreToolUseInput = {}; +try { + input = raw.trim() ? (JSON.parse(raw) as PreToolUseInput) : {}; +} catch { + process.exit(0); // fail open +} + +const ti = input.tool_input || input.toolInput || {}; +const fp = ti.file_path || ti.path || ""; +if (!fp) process.exit(0); + +let rel: string; +try { + rel = relative(cwd, resolve(cwd, fp)); +} catch { + process.exit(0); +} +if (!FROZEN.has(rel)) process.exit(0); + +process.stdout.write( + JSON.stringify({ + hookSpecificOutput: { + hookEventName: "PreToolUse", + permissionDecision: "deny", + permissionDecisionReason: `[autoresearch] ${rel} is frozen: the benchmark metric must not change during the loop. If you really need a new metric, start over: init_experiment with a new target.`, + }, + }), +); diff --git a/plugins/autoresearch/hooks/hooks.json b/plugins/autoresearch/hooks/hooks.json new file mode 100644 index 0000000..a6bf2c4 --- /dev/null +++ b/plugins/autoresearch/hooks/hooks.json @@ -0,0 +1,87 @@ +{ + "description": "autoresearch plugin hooks: loop continuation (Stop), frozen benchmark write protection (PreToolUse), ledger memory injection (UserPromptSubmit/SessionStart). All fail-open.", + "hooks": { + "Stop": [ + { + "hooks": [ + { + "type": "process", + "command": "node", + "args": [ + "${ZCODE_PLUGIN_ROOT}/hooks/stop-continue.ts", + "${ZCODE_PROJECT_DIR}" + ], + "timeoutMs": 5000, + "statusMessage": "autoresearch: checking loop continuation…" + } + ] + } + ], + "PreToolUse": [ + { + "matcher": "Write|Edit|ApplyPatch", + "hooks": [ + { + "type": "process", + "command": "node", + "args": [ + "${ZCODE_PLUGIN_ROOT}/hooks/guard-frozen.ts", + "${ZCODE_PROJECT_DIR}" + ], + "timeoutMs": 5000, + "statusMessage": "autoresearch: checking frozen files…" + } + ] + } + ], + "PermissionRequest": [ + { + "hooks": [ + { + "type": "process", + "command": "node", + "args": [ + "${ZCODE_PLUGIN_ROOT}/hooks/permission-gate.ts", + "${ZCODE_PROJECT_DIR}" + ], + "timeoutMs": 5000, + "statusMessage": "autoresearch: gating experiment tools…" + } + ] + } + ], + "UserPromptSubmit": [ + { + "hooks": [ + { + "type": "process", + "command": "node", + "args": [ + "${ZCODE_PLUGIN_ROOT}/hooks/memory-inject.ts", + "${ZCODE_PROJECT_DIR}" + ], + "timeoutMs": 5000, + "statusMessage": "autoresearch: injecting ledger memory…" + } + ] + } + ], + "SessionStart": [ + { + "matcher": "startup|resume", + "hooks": [ + { + "type": "process", + "command": "node", + "args": [ + "${ZCODE_PLUGIN_ROOT}/hooks/session-start.ts", + "${ZCODE_PROJECT_DIR}" + ], + "timeoutMs": 5000, + "statusMessage": "autoresearch: session context…" + } + ] + } + ] + } +} diff --git a/plugins/autoresearch/hooks/memory-inject.ts b/plugins/autoresearch/hooks/memory-inject.ts new file mode 100644 index 0000000..ef94d6a --- /dev/null +++ b/plugins/autoresearch/hooks/memory-inject.ts @@ -0,0 +1,105 @@ +#!/usr/bin/env node +// UserPromptSubmit hook: inject an aggregated session-memory summary so the +// loop survives compaction and long sessions — progress, deduplicated tried +// directions, best trajectory, recent runs with ASI, and a doom-loop warning. +import { rebuildState, readSessionConfig } from "../mcp/lib/ledger.ts"; +import { resolveWorkCwd } from "../mcp/lib/paths.ts"; +import { + directionLabel, + detectDoomLoop, + normalizeHypothesis, + hypothesesSimilar, +} from "../mcp/lib/experiment.ts"; +import type { SessionState } from "../mcp/lib/types.ts"; + +const projectCwd = process.argv[2] || process.cwd(); +const cwd = resolveWorkCwd(projectCwd); + +function pass(): never { + process.exit(0); +} + +let state: SessionState | undefined; +try { + const cfg = readSessionConfig(projectCwd); + const max = Number(cfg.maxIterations); + state = rebuildState(cwd, { + maxIterations: Number.isFinite(max) && max > 0 ? max : 20, + }); +} catch { + pass(); +} +if (!state || !state.config || state.runs.length === 0) pass(); + +const cfg = state.config; +const lines: string[] = []; + +// Progress line +lines.push( + `[autoresearch 记忆] segment ${state.segment}（metric=${cfg.metricName}，direction=${cfg.direction ?? "lower"}）` + + `已跑 ${state.runs.length}/${state.maxIterations ?? 20} 次，baseline=${state.baseline ?? "—"}，best=${state.best ?? "—"}。`, +); + +// Deduplicated tried directions (similarity-based, most recent label kept) +const tried: string[] = []; +const triedNorm: string[] = []; +for (const r of state.runs) { + const label = directionLabel(r); + const n = normalizeHypothesis(label) ?? label; + if (triedNorm.some((t) => hypothesesSimilar(t, n))) continue; + triedNorm.push(n); + tried.push(label); +} +const triedList = tried.slice(-8); +if (triedList.length > 0) + lines.push(`已尝试方向：${triedList.join("、")}（避免重复尝试）。`); + +// Best trajectory: baseline → improving keeps (≤6 steps) +const kept = state.runs.filter((r) => r.status === "keep" && r.metric != null); +const steps = kept.filter((r) => r.metric !== state.baseline).slice(-6); +if (steps.length > 0) { + const traj = steps.map( + (r) => `${r.metric}(${directionLabel(r).slice(0, 14)})`, + ); + lines.push(`best 轨迹：${state.baseline ?? "—"} → ${traj.join(" → ")}。`); +} + +// Recent runs with ASI extraction +const recent = state.runs + .slice(-3) + .map((r) => { + let line = `#${r.run} ${r.status} metric=${r.metric ?? "—"} ${r.description ?? ""}`; + if (r.asi && typeof r.asi === "object") { + const parts: string[] = []; + if (r.asi.hypothesis) parts.push(`hyp: ${r.asi.hypothesis}`); + if (r.asi.next_action_hint) parts.push(`next: ${r.asi.next_action_hint}`); + if (r.asi.rollback) parts.push(`rollback: ${r.asi.rollback}`); + if (parts.length) line += "\n " + parts.join("\n "); + } + return line; + }) + .join("\n"); +lines.push(`最近记录：\n${recent}`); + +// Doom-loop warning +const doom = detectDoomLoop(state.runs); +if (doom) { + lines.push( + doom.pattern === "oscillate" + ? "⚠️ 检测到 A→B→A→B 震荡尝试：请停止在两个方向上反复，换一个结构性不同的方向。" + : "⚠️ 检测到连续重复尝试：请停止重复同一假设，换一个结构性不同的方向。", + ); +} + +lines.push( + "如果你在运行 autoresearch 循环，请基于上述进度选择下一个假设并继续 run_experiment → log_experiment。", +); + +process.stdout.write( + JSON.stringify({ + hookSpecificOutput: { + hookEventName: "UserPromptSubmit", + additionalContext: lines.join("\n"), + }, + }), +); diff --git a/plugins/autoresearch/hooks/permission-gate.ts b/plugins/autoresearch/hooks/permission-gate.ts new file mode 100644 index 0000000..595170c --- /dev/null +++ b/plugins/autoresearch/hooks/permission-gate.ts @@ -0,0 +1,54 @@ +#!/usr/bin/env node +// PermissionRequest hook: approximate tool gate (pi-gap M3, #1). +// When the workspace has no active experiment session (.auto/log.jsonl), deny +// permission prompts for the experiment tools so the loop cannot be started by +// accident. With a session, allow. This is an approximation — calls that are +// auto-approved never reach PermissionRequest; tool-internal checks and the +// skill remain the backstop. +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { resolveWorkCwd } from "../mcp/lib/paths.ts"; + +interface PermissionRequestInput { + tool_name?: string; + toolName?: string; +} + +const projectCwd = process.argv[2] || process.cwd(); +const cwd = resolveWorkCwd(projectCwd); +const EXPERIMENT_TOOLS = new Set([ + "init_experiment", + "run_experiment", + "log_experiment", + "export_dashboard", + "clear_experiments", +]); + +let raw = ""; +process.stdin.setEncoding("utf8"); +for await (const chunk of process.stdin as AsyncIterable) raw += chunk; + +let input: PermissionRequestInput = {}; +try { + input = raw.trim() ? (JSON.parse(raw) as PermissionRequestInput) : {}; +} catch { + process.exit(0); // fail open +} + +const tool = input.tool_name || input.toolName || ""; +if (!EXPERIMENT_TOOLS.has(tool)) process.exit(0); + +const hasSession = existsSync(join(cwd, ".auto", "log.jsonl")); +if (hasSession) process.exit(0); + +process.stdout.write( + JSON.stringify({ + hookSpecificOutput: { + hookEventName: "PermissionRequest", + decision: { + behavior: "deny", + message: `[autoresearch] 当前工作区没有实验会话（.auto/log.jsonl 不存在），${tool} 已被拦截。请先通过 /autoresearch:autoresearch 建立会话。`, + }, + }, + }), +); diff --git a/plugins/autoresearch/hooks/session-start.ts b/plugins/autoresearch/hooks/session-start.ts new file mode 100644 index 0000000..96c7fee --- /dev/null +++ b/plugins/autoresearch/hooks/session-start.ts @@ -0,0 +1,39 @@ +#!/usr/bin/env node +// SessionStart hook: point the model at an existing autoresearch session +// (auto-activation prompt). Respects an explicit `autoresearchOff: true` +// decision in .auto/config.json — after /autoresearch:off no resume hint is +// injected, though the session can still be entered manually. +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { resolveWorkCwd } from "../mcp/lib/paths.ts"; + +const projectCwd = process.argv[2] || process.cwd(); +const cwd = resolveWorkCwd(projectCwd); +const log = join(cwd, ".auto", "log.jsonl"); +if (!existsSync(log)) process.exit(0); + +let off = false; +try { + // The off switch lives in the PROJECT config (where the server writes it), + // even when the ledger lives in a workingDir research directory. + const cfg = JSON.parse( + readFileSync(join(projectCwd, ".auto", "config.json"), "utf8"), + ) as { autoresearchOff?: unknown }; + off = cfg.autoresearchOff === true; +} catch { + /* no config → treat as active */ +} + +if (off) process.exit(0); + +process.stdout.write( + JSON.stringify({ + hookSpecificOutput: { + hookEventName: "SessionStart", + additionalContext: + `本工作区存在 autoresearch 会话（.auto/log.jsonl）。` + + `可用 /autoresearch:autoresearch 继续循环，或 /autoresearch:export 导出 dashboard；` + + `暂停可用 /autoresearch:off，重置可用 /autoresearch:clear。`, + }, + }), +); diff --git a/plugins/autoresearch/hooks/stop-continue.ts b/plugins/autoresearch/hooks/stop-continue.ts new file mode 100644 index 0000000..35267c2 --- /dev/null +++ b/plugins/autoresearch/hooks/stop-continue.ts @@ -0,0 +1,77 @@ +#!/usr/bin/env node +// Stop hook: keep the autoresearch loop running while it is not finished. +// zcode grants at most 3 consecutive Stop continuations per window, so this +// hook only fires when the ledger shows the loop should continue. +import { rebuildState, readSessionConfig } from "../mcp/lib/ledger.ts"; +import { resolveWorkCwd } from "../mcp/lib/paths.ts"; +import { isStopReached, detectDoomLoop } from "../mcp/lib/experiment.ts"; +import type { SessionState } from "../mcp/lib/types.ts"; + +const projectCwd = process.argv[2] || process.cwd(); +const cwd = resolveWorkCwd(projectCwd); + +function failOpen(): never { + process.exit(0); +} + +let state: SessionState | undefined; +try { + const cfg = readSessionConfig(projectCwd); + const max = Number(cfg.maxIterations); + const fails = Number(cfg.consecutiveFailures); + state = rebuildState(cwd, { + maxIterations: Number.isFinite(max) && max > 0 ? max : 20, + consecutiveFailures: Number.isFinite(fails) && fails > 0 ? fails : 3, + }); +} catch { + failOpen(); +} + +// No active session → let the model finish normally. +if (!state || !state.config || state.runs.length === 0) failOpen(); + +const finished = isStopReached( + state.runs, + state.maxIterations ?? 20, + state.failureThreshold, +); +if (finished) failOpen(); + +// Plateau convergence: recent runs improved < 1% → let the model wrap up. +// The Stop schema has no allow-with-reason shape, so the advisory goes to +// stderr (hook log) while the hook exits 0 to allow the stop. +if (state.plateau) { + process.stderr.write( + `[autoresearch] 循环已进入平台期（最近 5 轮改善 < 1%，best=${state.best ?? "—"}）。` + + `建议：用 run_experiment repeat:3 复测确认，或 init_experiment 开启新 segment，或就此收尾总结。\n`, + ); + process.exit(0); +} else { + const dir = state.config?.direction ?? "lower"; + const tail = state.runs + .slice(-3) + .map((r) => { + let line = `#${r.run} ${r.status} metric=${r.metric ?? "—"} ${r.description ?? ""}`; + if (r.asi && typeof r.asi === "object") { + const parts: string[] = []; + if (r.asi.hypothesis) parts.push(`hyp: ${r.asi.hypothesis}`); + if (r.asi.next_action_hint) + parts.push(`next: ${r.asi.next_action_hint}`); + if (r.asi.rollback) parts.push(`rollback: ${r.asi.rollback}`); + if (parts.length) line += "\n " + parts.join("\n "); + } + return line; + }) + .join("\n"); + + const reason = + `[autoresearch] 实验循环未结束：segment ${state.segment} 已跑 ${state.runs.length}/${state.maxIterations ?? 20} 次，` + + `direction=${dir}，baseline=${state.baseline ?? "—"}，best=${state.best ?? "—"}。` + + `最近记录：\n${tail}\n` + + (detectDoomLoop(state.runs) + ? `⚠️ 检测到重复/震荡尝试：请停止重复同一假设，换一个结构性不同的方向。\n` + : "") + + `请继续下一个假设：修改代码 → run_experiment → log_experiment（keep/discard）。`; + + process.stdout.write(JSON.stringify({ decision: "block", reason })); +} diff --git a/plugins/autoresearch/mcp/lib/dashboard-server.ts b/plugins/autoresearch/mcp/lib/dashboard-server.ts new file mode 100644 index 0000000..3f1ac2a --- /dev/null +++ b/plugins/autoresearch/mcp/lib/dashboard-server.ts @@ -0,0 +1,104 @@ +// Local HTTP + SSE dashboard server, hosted inside the MCP server process. +// Routes: / (live HTML), /autoresearch.jsonl (ledger raw), /events (SSE). +import { createServer } from "node:http"; +import type { Server, ServerResponse } from "node:http"; +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { rebuildState, readSessionConfig } from "./ledger.ts"; +import { renderLiveDashboard } from "./dashboard.ts"; + +const clients = new Set(); +let server: Server | null = null; +let boundPort: number | null = null; + +function broadcast(): void { + // The MCP server process hosts this HTTP server: a single dead client must + // never take the whole process down. Skip and evict broken connections. + for (const res of clients) { + if (res.destroyed) { + clients.delete(res); + continue; + } + try { + res.write(`event: jsonl-updated\ndata: ${Date.now()}\n\n`); + } catch { + clients.delete(res); + } + } +} + +function start(workCwd: string): Promise<{ port: number; url: string }> { + if (server) { + return Promise.resolve({ + port: boundPort ?? 0, + url: `http://127.0.0.1:${boundPort ?? 0}`, + }); + } + const srv = createServer((req, res) => { + const url = new URL(req.url ?? "/", "http://127.0.0.1"); + if (url.pathname === "/events") { + res.writeHead(200, { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + Connection: "keep-alive", + }); + res.write("retry: 2000\n\n"); + clients.add(res); + req.on("close", () => clients.delete(res)); + return; + } + if (url.pathname === "/autoresearch.jsonl") { + const log = join(workCwd, ".auto", "log.jsonl"); + // The ledger legitimately disappears (clear_experiments); respond 404 + // instead of throwing ENOENT inside the MCP server process. + if (!existsSync(log)) { + res.writeHead(404, { "Content-Type": "text/plain; charset=utf-8" }); + res.end("ledger not found (no active session)"); + return; + } + res.writeHead(200, { + "Content-Type": "application/x-ndjson; charset=utf-8", + }); + res.end(readFileSync(log, "utf8")); + return; + } + if (url.pathname === "/" || url.pathname === "") { + const state = rebuildState(workCwd, { + maxIterations: Number(readSessionConfig(workCwd).maxIterations) || 20, + }); + res.writeHead(200, { "Content-Type": "text/html; charset=utf-8" }); + res.end(renderLiveDashboard(state)); + return; + } + res.writeHead(404); + res.end("not found"); + }); + return new Promise((resolve, reject) => { + srv.on("error", reject); + srv.listen(0, "127.0.0.1", () => { + const addr = srv.address(); + if (addr && typeof addr === "object") { + boundPort = addr.port; + server = srv; + resolve({ port: boundPort, url: `http://127.0.0.1:${boundPort}` }); + } else { + reject(new Error("dashboard server failed to bind")); + } + }); + }); +} + +export function ensureDashboardServer(workCwd: string): Promise<{ + port: number; + url: string; +}> { + return start(workCwd); +} + +export function broadcastDashboardUpdate(): void { + if (server) broadcast(); +} + +export function dashboardServerInfo(): { url: string } | null { + return server ? { url: `http://127.0.0.1:${boundPort ?? 0}` } : null; +} diff --git a/plugins/autoresearch/mcp/lib/dashboard.ts b/plugins/autoresearch/mcp/lib/dashboard.ts new file mode 100644 index 0000000..f844daf --- /dev/null +++ b/plugins/autoresearch/mcp/lib/dashboard.ts @@ -0,0 +1,268 @@ +// Static dashboard renderer: pure function from ledger state to self-contained HTML. +import { escapeHtml } from "./html.ts"; +import { deltaFor } from "./ledger.ts"; +import type { LedgerRun, SessionState } from "./types.ts"; + +const STATUS_LABEL: Record = { + keep: "keep", + discard: "discard", + crash: "crash", + checks_failed: "checks_failed", + noop: "no-op", +}; + +/** Compact metric number: 4-decimal cap with trailing zeros stripped (42.0000 → 42). */ +function fmtMetric(v: number | null | undefined): string { + if (v == null || !Number.isFinite(v)) return "—"; + return String(Number(v.toFixed(4))); +} + +/** + * Nice grid-tick values for [lo, hi]: step is 1/2/2.5/5 × 10^k, so tick values + * stay short and all share the same decimal precision — no 61.6667-style noise. + */ +function niceTicks( + lo: number, + hi: number, + target = 3, +): Array<{ value: number; decimals: number }> { + const raw = (hi - lo) / target; + const pow = 10 ** Math.floor(Math.log10(raw)); + const norm = raw / pow; + const step = + pow * + (norm <= 1 ? 1 : norm <= 2 ? 2 : norm <= 2.5 ? 2.5 : norm <= 5 ? 5 : 10); + const ds = String(step).split(".")[1]; + const decimals = ds ? ds.replace(/0+$/, "").length : 0; + const out: Array<{ value: number; decimals: number }> = []; + const first = Math.ceil((lo - 1e-9) / step); + const last = Math.floor((hi + 1e-9) / step); + for (let k = first; k <= last; k++) out.push({ value: k * step, decimals }); + return out; +} + +/** + * Inline-SVG metric trend: valid metric points in run order (keep filled, + * others hollow), baseline as a dashed reference, light horizontal grid ticks + * with values on the left. Fewer than 2 points → "". Zero external resources — + * the dashboard must stay self-contained. + */ +function renderTrendSvg(state: SessionState): string { + const pts = state.runs.filter( + (r): r is LedgerRun & { metric: number } => + r.metric != null && Number.isFinite(r.metric), + ); + if (pts.length < 2) return ""; + const W = 860; + const H = 120; + const padX = 44; // left gutter carries the grid-tick values + const padY = 10; + const vals = pts.map((r) => r.metric); + let lo = Math.min(...vals, state.baseline ?? Infinity); + let hi = Math.max(...vals, state.baseline ?? -Infinity); + if (hi === lo) { + lo -= 1; // degenerate flat line → pad it so points sit mid-chart + hi += 1; + } + const span = hi - lo; + const x = (i: number) => padX + (i * (W - 2 * padX)) / (pts.length - 1); + const y = (v: number) => padY + ((hi - v) * (H - 2 * padY)) / span; + // Light horizontal grid ticks at "nice" values covering [lo, hi]. + const ticks = niceTicks(lo, hi) + .map(({ value: v, decimals }) => { + const yy = y(v); + return ( + `` + + `${v.toFixed(decimals)}` + ); + }) + .join(""); + const poly = pts.map( + (r, i) => `${x(i).toFixed(1)},${y(r.metric).toFixed(1)}`, + ); + const circleCls = (st: string) => + st === "keep" + ? "c-keep" + : st === "discard" + ? "c-discard" + : st === "noop" + ? "c-noop" + : "c-fail"; + const circles = pts + .map( + (r, i) => + ``, + ) + .join(""); + const baseline = state.baseline; + const baseLine = + baseline != null && + Number.isFinite(baseline) && + baseline >= lo && + baseline <= hi + ? `` + + `baseline ${escapeHtml(fmtMetric(baseline))}` + : ""; + return ` +${ticks} + +${baseLine} +${circles} +`; +} + +export function renderDashboard(state: SessionState): string { + return renderBody(state, false); +} + +/** Live variant: same body plus an SSE client that auto-reloads on updates. */ +export function renderLiveDashboard(state: SessionState): string { + return renderBody(state, true); +} + +function renderBody(state: SessionState, live: boolean): string { + const cfg = state.config; + const direction = cfg?.direction ?? "lower"; + const rows = state.runs + .map((r, i) => { + const delta = deltaFor(state, r.metric); + const deltaText = + delta == null ? "—" : (delta >= 0 ? "+" : "") + fmtMetric(delta); + const improved = delta != null && delta > 0; + const cls = + r.status === "keep" + ? "keep" + : r.status === "discard" + ? "discard" + : r.status === "noop" + ? "noop" + : "crash"; + return { i: i + 1, run: r, deltaText, improved, cls }; + }) + .reverse(); // newest first + + const kept = state.runs.filter((r) => r.status === "keep"); + const failures = state.runs.filter((r) => r.status !== "keep"); + const conf = state.confidence; + const trend = renderTrendSvg(state); + + return ` + + + + +autoresearch: ${escapeHtml(cfg?.name ?? "session")} + + + +

autoresearch · ${escapeHtml(cfg?.name ?? "session")}

+
+ ${cfg ? `metric ${escapeHtml(cfg.metricName ?? "")} · direction ${escapeHtml(direction)}` : "no session yet"} + ${cfg?.metricUnit ? ` · unit ${escapeHtml(cfg.metricUnit)}` : ""} +
+
+
${state.runs.length}
experiments
+
${kept.length}
kept
+
${failures.length}
reverted
+
${fmtMetric(state.baseline)}
baseline
+
${fmtMetric(state.best)}
best
+ ${conf ? `
${escapeHtml(conf.level)}
confidence ${conf.confidence.toFixed(2)}
` : ""} +
+${trend ? `
${trend}
` : ""} +${ + rows.length === 0 + ? "

暂无实验记录。

" + : ` +
+ + + +${rows + .map( + (r) => ` + + + + + + +`, + ) + .join("\n")} + +
#statusmetricΔ vs baselinecommitdescription
${r.i}${STATUS_LABEL[r.run.status] ?? escapeHtml(r.run.status)}${fmtMetric(r.run.metric)}${r.improved ? "▲" : "▼"} ${escapeHtml(r.deltaText)}${r.run.commit ? `${escapeHtml(r.run.commit)}` : "—"}${escapeHtml(r.run.description ?? "")}
+
` +} +${ + live + ? `` + : "" +} + +`; +} diff --git a/plugins/autoresearch/mcp/lib/experiment.ts b/plugins/autoresearch/mcp/lib/experiment.ts new file mode 100644 index 0000000..0698102 --- /dev/null +++ b/plugins/autoresearch/mcp/lib/experiment.ts @@ -0,0 +1,273 @@ +// Pure functions for the autoresearch experiment loop. +// No I/O here so they are unit-testable without a workspace. +import type { Confidence, Direction, RunLike } from "./types.ts"; + +export const METRIC_RE = /^METRIC\s+([\w.µ]+)=(\S+)$/; + +const FORBIDDEN_KEYS = new Set(["__proto__", "constructor", "prototype"]); + +/** + * Parse `METRIC name=value` lines out of command output. + * Returns { metrics, primary } where primary is metrics[metricName] if present. + * Same-name keys: last wins. Dangerous key names are rejected. + */ +export function parseMetricLines( + output: unknown, + metricName?: string, +): { metrics: Record; primary: number | undefined } { + const metrics: Record = {}; + let primary: number | undefined; + for (const line of String(output ?? "").split("\n")) { + const m = METRIC_RE.exec(line.trim()); + if (!m) continue; + const [, name, rawValue] = m; + if (FORBIDDEN_KEYS.has(name)) continue; + const value = Number(rawValue); + if (!Number.isFinite(value)) continue; + metrics[name] = value; + if (name === metricName) primary = value; + } + return { metrics, primary }; +} + +/** + * Direction-aware improvement test. + * direction: "lower" (default) or "higher". + */ +export function isBetter( + current: number | null | undefined, + best: number | null | undefined, + direction: Direction = "lower", +): boolean { + if (current == null || best == null) return false; + return direction === "higher" ? current > best : current < best; +} + +/** + * MAD-based noise floor for the current segment. + * Returns null when there are fewer than 3 data points or MAD is 0. + */ +export function median(values: number[]): number | null { + if (values.length === 0) return null; + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 + ? (sorted[mid - 1] + sorted[mid]) / 2 + : sorted[mid]; +} + +export function computeConfidence({ + values, + baseline, + best, +}: { + values: number[]; + baseline: number | null; + best: number | null; +}): Confidence | null { + if (values.length < 3 || baseline == null || best == null) return null; + const med = median(values); + if (med == null || med === 0) return null; + const deviations = values.map((v) => Math.abs(v - med)); + const mad = median(deviations); + if (mad == null || mad === 0) return null; + const ratio = Math.abs(best - baseline) / mad; + let level: Confidence["level"] = "red"; + if (ratio >= 2.0) level = "green"; + else if (ratio >= 1.0) level = "yellow"; + return { confidence: ratio, level }; +} + +/** + * Enforce that a run_experiment command is (a wrapper around) the benchmark + * script. Strips leading `FOO=bar` assignments and `env/time/nice/nohup` + * wrappers, then requires the core command to start with the measure script + * path. Returns the unwrapped command string, or null when the command is + * not the benchmark script. Prevents `evil; ./measure.sh` chained injection. + */ +const WRAP_RE = /^(env|time|nice|nohup)\s+/; +// leading env assignment including its value: `FOO=1 ` or `FOO="a b" ` +const ASSIGN_RE = /^[A-Za-z_][A-Za-z0-9_]*=\S*\s*/; + +export function unwrapMeasureCommand( + command: unknown, + measureScript: string, +): string | null { + let cmd = String(command ?? "").trim(); + if (!cmd) return null; + // Strip leading env assignments and wrapper prefixes, alternating, until + // stable (`env X=1 bash .auto/measure.sh` needs env→assignment→bash). + let prev: string; + do { + prev = cmd; + cmd = cmd.replace(ASSIGN_RE, "").trim(); + cmd = cmd.replace(WRAP_RE, "").trim(); + } while (cmd !== prev); + if (!cmd) return null; + // 3) core must be the measure script itself (optional ./ and .auto/ prefix, + // optional bash wrapper) + const variants = [ + measureScript, + `./${measureScript}`, + `.auto/${measureScript}`, + `./.auto/${measureScript}`, + `bash ${measureScript}`, + `bash ./${measureScript}`, + `bash .auto/${measureScript}`, + `bash ./.auto/${measureScript}`, + ]; + const match = variants.find((v) => cmd === v || cmd.startsWith(v + " ")); + if (!match) return null; + // 4) args after the script must be plain tokens — whitelist characters only. + // A blacklist cannot enumerate the shell injection surface (newlines, CR, + // redirection, quotes, backticks, $, globs, ...), so anything outside + // [word chars . / : = + - space tab] is rejected. + const rest = cmd.slice(match.length); + if (!/^[\w./:=+ \t-]*$/.test(rest)) return null; + return cmd; +} + +/** + * Decide whether the loop has reached a stop condition. + * Stop when: current segment runs >= maxIterations, or the trailing run of + * real failures (discard/crash/checks_failed) reaches consecutiveFailures. + * noop neither counts as a failure nor keeps the streak alive — it breaks + * the chain, like keep does. + */ +export function isStopReached( + runs: RunLike[], + maxIterations: number | undefined, + consecutiveFailures = 3, +): boolean { + if (maxIterations != null && runs.length >= maxIterations) return true; + if (runs.length === 0) return false; + let streak = 0; + for (let i = runs.length - 1; i >= 0; i--) { + const s = runs[i].status; + if (s === "discard" || s === "crash" || s === "checks_failed") streak += 1; + else break; + } + return streak >= consecutiveFailures; +} + +/** + * Normalize a hypothesis/description for comparison: lowercase, strip + * non-alphanumerics, sort tokens. Returns null when there is no signal. + */ +export function normalizeHypothesis(text: unknown): string | null { + const tokens = String(text ?? "") + .toLowerCase() + .replace(/[^a-z0-9\u4e00-\u9fff]+/g, " ") + .split(/\s+/) + .filter((t) => t.length >= 2); + if (tokens.length === 0) return null; + return [...tokens].sort().join(" "); +} + +/** Jaccard similarity of two normalized hypotheses (token sets), or subset. */ +export function hypothesesSimilar( + a: string | null | undefined, + b: string | null | undefined, +): boolean { + if (a === b) return true; + if (!a || !b) return false; + const A = new Set(a.split(" ")); + const B = new Set(b.split(" ")); + const inter = [...A].filter((t) => B.has(t)).length; + const union = new Set([...A, ...B]).size; + if (inter === Math.min(A.size, B.size)) return true; // one is a subset + return union > 0 && inter / union >= 0.5; +} + +/** + * Direction label for a run: prefer asi.hypothesis, else description; take the + * leading clause (up to first comma/period/semicolon), capped at 40 chars. + */ +export function directionLabel(run: RunLike): string { + const raw = run?.asi?.hypothesis || run?.description || ""; + const clause = + String(raw) + .split(/[,.;，。；]/)[0] + ?.trim() || ""; + return clause.length > 40 + ? clause.slice(0, 40) + "…" + : clause || (run?.status ?? "?"); +} + +/** + * Doom-loop detection (ml-intern idea, text-layer): repeated or oscillating + * hypotheses. Returns { doomLoop, pattern } or null. + * - repeat: last 3 runs have similar normalized hypotheses. + * - oscillate: last 4 runs are [X, Y, X, Y] (X~X, Y~Y, X!~Y). + */ +export function detectDoomLoop( + runs: RunLike[], + { window = 6 } = {}, +): { doomLoop: boolean; pattern: "repeat" | "oscillate" } | null { + const norm = runs + .filter((r) => r.description || r?.asi?.hypothesis) + .slice(-window) + .map((r) => normalizeHypothesis(r.asi?.hypothesis || r.description)) + .filter((n): n is string => n != null); + if (norm.length < 3) return null; + + // 3+ consecutive repeats (needs 3) + const last3 = norm.slice(-3); + if ( + hypothesesSimilar(last3[0], last3[1]) && + hypothesesSimilar(last3[1], last3[2]) + ) { + return { doomLoop: true, pattern: "repeat" }; + } + + // A→B→A→B oscillation (needs 4) + if (norm.length >= 4) { + const [a, b, c, d] = norm.slice(-4); + if ( + hypothesesSimilar(a, c) && + hypothesesSimilar(b, d) && + !hypothesesSimilar(a, b) + ) { + return { doomLoop: true, pattern: "oscillate" }; + } + } + + return null; +} + +/** + * Plateau detection: within the last `window` runs with a valid metric, the + * best direction-aware improvement relative to the window's first metric is + * below `minImprovement`. Returns false when there are fewer than `window` + * valid records (not enough data to judge). + */ +export function detectPlateau( + runs: RunLike[], + { + window = 5, + minImprovement = 0.01, + direction = "lower", + }: { + window?: number; + minImprovement?: number; + direction?: Direction; + } = {}, +): boolean { + const valid = runs + .filter((r) => r.metric != null && Number.isFinite(r.metric)) + .slice(-window); + if (valid.length < window) return false; + const first = valid[0].metric; + if (first == null) return false; + let best = first; + for (const r of valid) { + if (r.metric == null) continue; + if (direction === "higher" ? r.metric > best : r.metric < best) + best = r.metric; + } + const improvement = + first === 0 + ? Math.abs(best - first) + : Math.abs(best - first) / Math.abs(first); + return improvement < minImprovement; +} diff --git a/plugins/autoresearch/mcp/lib/git.ts b/plugins/autoresearch/mcp/lib/git.ts new file mode 100644 index 0000000..d161b85 --- /dev/null +++ b/plugins/autoresearch/mcp/lib/git.ts @@ -0,0 +1,103 @@ +// Git operations for the experiment loop (ADR-2 semantics): +// keep → commit with `experiment:` prefix + structured Result JSON. +// discard/crash/checks_failed → drop working-tree changes, exempt `.auto/`. +import { execFileSync } from "node:child_process"; + +export function git(cwd: string, args: string[]): string { + return execFileSync("git", args, { cwd, encoding: "utf8" }).trim(); +} + +export function isGitRepo(cwd: string): boolean { + try { + execFileSync("git", ["rev-parse", "--git-dir"], { cwd, stdio: "ignore" }); + return true; + } catch { + return false; + } +} + +/** + * Whether the working tree has real (non-`.auto/`) changes. Session-file + * writes (ledger, config) must not count: e.g. the crash-unresolved gate + * would otherwise block forever right after logging the crash itself. + */ +export function isDirty(cwd: string): boolean { + try { + return ( + git(cwd, ["status", "--porcelain", "--", ".", ":(exclude).auto"]).length > + 0 + ); + } catch { + return false; + } +} + +export function shortHash(cwd: string): string { + return git(cwd, ["rev-parse", "--short=7", "HEAD"]); +} + +/** + * Commit all tracked+untracked changes as one experiment, excluding the + * `.auto/` session dir (ledger noise must not ride along, and a keep with + * only session-file changes must hit the "nothing to commit" path). + * Returns the short hash, or null when there is nothing to commit. + */ +export function commitExperiment( + cwd: string, + { description, result }: { description: string; result: unknown }, +): string | null { + git(cwd, ["add", "-A", "--", ".", ":(exclude).auto"]); + // git diff --cached --quiet exits 0 when there are no staged changes. + const hasStaged = (() => { + try { + execFileSync("git", ["diff", "--cached", "--quiet"], { + cwd, + stdio: "ignore", + }); + return false; + } catch { + return true; + } + })(); + if (!hasStaged) return null; + const body = `experiment: ${description}\n\nResult: ${JSON.stringify(result)}`; + execFileSync("git", ["commit", "-m", body], { cwd, stdio: "ignore" }); + return shortHash(cwd); +} + +/** + * Discard every working-tree + index change while keeping the `.auto/` + * session directory intact. Uses `checkout HEAD` (not `checkout --`) so + * staged-but-uncommitted experiment changes are reverted to HEAD too. + */ +export function rollbackWorkingTree(cwd: string): void { + execFileSync( + "git", + ["checkout", "HEAD", "--", ".", ":(exclude,glob)**/.auto/**"], + { cwd, stdio: "ignore" }, + ); + // Unstage anything (e.g. accidentally staged .auto content) without touching files. + execFileSync("git", ["reset", "-q"], { cwd, stdio: "ignore" }); + execFileSync( + "git", + [ + "clean", + "-fd", + "-e", + ".auto", + "-e", + ".auto/", + "-e", + "autoresearch-dashboard.html", + ], + { cwd, stdio: "ignore" }, + ); +} + +export function currentBranch(cwd: string): string { + try { + return git(cwd, ["branch", "--show-current"]); + } catch { + return ""; + } +} diff --git a/plugins/autoresearch/mcp/lib/html.ts b/plugins/autoresearch/mcp/lib/html.ts new file mode 100644 index 0000000..d05e354 --- /dev/null +++ b/plugins/autoresearch/mcp/lib/html.ts @@ -0,0 +1,8 @@ +export function escapeHtml(s: unknown): string { + return String(s ?? "") + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); +} diff --git a/plugins/autoresearch/mcp/lib/ledger.ts b/plugins/autoresearch/mcp/lib/ledger.ts new file mode 100644 index 0000000..3666ad8 --- /dev/null +++ b/plugins/autoresearch/mcp/lib/ledger.ts @@ -0,0 +1,196 @@ +// `.auto/` ledger: append-only JSONL source of truth + segment rebuild. +import { + readFileSync, + writeFileSync, + appendFileSync, + existsSync, + mkdirSync, +} from "node:fs"; +import { join } from "node:path"; +import { computeConfidence, isBetter, detectPlateau } from "./experiment.ts"; +import type { + LedgerEntry, + LedgerRun, + SessionConfig, + SessionState, +} from "./types.ts"; + +export const AUTO_DIR = ".auto"; +export const LOG_FILE = join(AUTO_DIR, "log.jsonl"); +export const PROMPT_FILE = join(AUTO_DIR, "prompt.md"); +export const MEASURE_FILE = join(AUTO_DIR, "measure.sh"); +export const CHECKS_FILE = join(AUTO_DIR, "checks.sh"); +export const CONFIG_FILE = join(AUTO_DIR, "config.json"); +export const IDEAS_FILE = join(AUTO_DIR, "ideas.md"); +export const DASHBOARD_FILE = "autoresearch-dashboard.html"; + +export function autoPaths(cwd: string): { + root: string; + log: string; + prompt: string; + measure: string; + checks: string; + config: string; + ideas: string; + dashboard: string; +} { + return { + root: join(cwd, AUTO_DIR), + log: join(cwd, LOG_FILE), + prompt: join(cwd, PROMPT_FILE), + measure: join(cwd, MEASURE_FILE), + checks: join(cwd, CHECKS_FILE), + config: join(cwd, CONFIG_FILE), + ideas: join(cwd, IDEAS_FILE), + dashboard: join(cwd, DASHBOARD_FILE), + }; +} + +export function ensureAutoDir(cwd: string): void { + mkdirSync(join(cwd, AUTO_DIR), { recursive: true }); +} + +export function appendLedgerEntry(cwd: string, entry: LedgerEntry): void { + ensureAutoDir(cwd); + appendFileSync(join(cwd, LOG_FILE), JSON.stringify(entry) + "\n", "utf8"); +} + +export function readLedger(cwd: string): LedgerEntry[] { + const log = join(cwd, LOG_FILE); + if (!existsSync(log)) return []; + return readFileSync(log, "utf8") + .split("\n") + .filter((l) => l.trim()) + .map((l) => { + try { + return JSON.parse(l) as LedgerEntry; + } catch { + return null; + } + }) + .filter((e): e is LedgerEntry => e != null); +} + +/** + * Rebuild the session state from the ledger file. + * - segment advances on every `config` entry. + * - runs after a config entry belong to that config's segment. + * - baseline = first run's primary metric in the segment. + * - best = best kept run's metric in the segment (direction-aware). + */ +export function rebuildState( + cwd: string, + options: { + maxIterations?: number; + consecutiveFailures?: number; + plateauWindow?: number; + plateauMinImprovement?: number; + } = {}, +): SessionState { + const entries = readLedger(cwd); + const state: SessionState = { + config: null, + segment: 0, + runs: [], + baseline: null, + best: null, + lastRunChecksFailed: false, + lastRun: null, + totalExperiments: 0, + consecutiveFailures: 0, + confidence: null, + plateau: false, + failureThreshold: 3, + }; + for (const e of entries) { + if (e.type === "config") { + state.config = e; + state.segment = e.segment ?? state.segment + 1; + state.runs = []; + state.baseline = null; + state.best = null; + } else if (e.type === "run") { + const run: LedgerRun = { + ...e, + segment: state.segment, + config: state.config, + }; + state.runs.push(run); + state.totalExperiments += 1; + if (state.baseline == null && run.metric != null) + state.baseline = run.metric; + if (run.status === "keep" && run.metric != null) { + const dir = state.config?.direction ?? "lower"; + if (state.best == null || isBetter(run.metric, state.best, dir)) + state.best = run.metric; + } + // Consecutive-failure streak (guardrails spec): only real failures + // count; keep and noop both break the chain. + if (run.status === "keep" || run.status === "noop") + state.consecutiveFailures = 0; + else state.consecutiveFailures += 1; + // Per-run overwrite (no one-way latch): the flag always describes the + // latest ledger run. Status alone marks checks failure; the explicit + // field covers hand-written/legacy rows. + state.lastRunChecksFailed = + run.checksFailed === true || run.status === "checks_failed"; + state.lastRun = run; + } + } + // Confidence over the current segment's values. + const values = state.runs + .map((r) => r.metric) + .filter((v): v is number => v != null && Number.isFinite(v)); + if (state.config && values.length > 0) { + state.confidence = computeConfidence({ + values, + baseline: state.baseline, + best: state.best, + }); + } else { + state.confidence = null; + } + if (options.maxIterations != null) + state.maxIterations = options.maxIterations; + state.failureThreshold = options.consecutiveFailures ?? 3; + // Plateau over the current segment's recent runs. + if (state.config && state.runs.length >= (options.plateauWindow ?? 5)) { + state.plateau = detectPlateau(state.runs, { + window: options.plateauWindow ?? 5, + minImprovement: options.plateauMinImprovement ?? 0.01, + direction: state.config.direction ?? "lower", + }); + } else { + state.plateau = false; + } + return state; +} + +/** + * Delta of a run's metric against the segment baseline, direction-aware: + * positive = improvement (lower metric + baseline was higher, or higher metric). + */ +export function deltaFor( + state: SessionState, + metric: number | null | undefined, +): number | null { + if (metric == null || state.baseline == null) return null; + const dir = state.config?.direction ?? "lower"; + const raw = metric - state.baseline; + return dir === "higher" ? raw : -raw; +} + +export function readSessionConfig(cwd: string): SessionConfig { + const cfg = join(cwd, CONFIG_FILE); + if (!existsSync(cfg)) return {}; + try { + return JSON.parse(readFileSync(cfg, "utf8")) as SessionConfig; + } catch { + return {}; + } +} + +export function writeDashboard(cwd: string, html: string): string { + writeFileSync(join(cwd, DASHBOARD_FILE), html, "utf8"); + return join(cwd, DASHBOARD_FILE); +} diff --git a/plugins/autoresearch/mcp/lib/paths.ts b/plugins/autoresearch/mcp/lib/paths.ts new file mode 100644 index 0000000..adaf36c --- /dev/null +++ b/plugins/autoresearch/mcp/lib/paths.ts @@ -0,0 +1,21 @@ +// Resolve the effective research directory. `.auto/config.json` in the project +// dir may set `workingDir` (relative to the project or absolute); when it +// exists, all experiment operations happen there (config stays in the project). +import { readFileSync, statSync } from "node:fs"; +import { resolve, isAbsolute, join } from "node:path"; + +export function resolveWorkCwd(projectCwd: string): string { + try { + const cfg = JSON.parse( + readFileSync(join(projectCwd, ".auto", "config.json"), "utf8"), + ) as { workingDir?: unknown }; + const wd = cfg.workingDir; + if (typeof wd === "string" && wd.trim()) { + const target = isAbsolute(wd) ? wd : resolve(projectCwd, wd); + if (statSync(target).isDirectory()) return target; + } + } catch { + /* no config or no workingDir → project dir */ + } + return projectCwd; +} diff --git a/plugins/autoresearch/mcp/lib/types.ts b/plugins/autoresearch/mcp/lib/types.ts new file mode 100644 index 0000000..fd33406 --- /dev/null +++ b/plugins/autoresearch/mcp/lib/types.ts @@ -0,0 +1,107 @@ +// Shared domain types for the autoresearch plugin (server, hooks, lib, tests). +// Erasable-only syntax: no enums/namespaces so Node can strip types natively. + +export type Direction = "lower" | "higher"; + +/** `config` row written by init_experiment — starts a new segment. */ +export interface LedgerConfig { + type: "config"; + segment: number; + name: string; + metricName: string; + metricUnit?: string; + direction: Direction; + createdAt?: string; +} + +export type RunStatus = "keep" | "discard" | "crash" | "checks_failed" | "noop"; + +/** Actionable Side Information — survives rollback. */ +export interface Asi { + hypothesis?: string; + next_action_hint?: string; + rollback?: string; + [key: string]: unknown; +} + +/** `run` row written by log_experiment. */ +export interface LedgerRun { + type: "run"; + run: number; + /** segment is re-derived by rebuildState; kept optional for hand-written rows. */ + segment?: number; + status: RunStatus; + metric: number | null; + metrics?: Record; + description?: string; + commit?: string | null; + checksFailed?: boolean; + failedGuard?: boolean; + asi?: Asi | null; + timestamp?: string; + /** merged in by rebuildState: the config entry this run belongs to. */ + config?: LedgerConfig | null; +} + +/** `hook` row written by the iteration-hook logger. */ +export interface LedgerHook { + type: "hook"; + stage: "before" | "after"; + exit_code: number | null; + duration_ms: number; + stdout_bytes: number; + timed_out: boolean; +} + +export type LedgerEntry = LedgerConfig | LedgerRun | LedgerHook; + +/** + * Loose run shape consumed by analysis functions (isStopReached, + * detectDoomLoop, detectPlateau, directionLabel, validateLedger). Callers and + * tests may pass partial rows; only the fields actually read must be present. + */ +export interface RunLike { + type?: "run"; + run?: number; + segment?: number; + status?: string; + metric?: number | null; + description?: string; + commit?: string | null; + asi?: Asi | null; +} + +export interface Confidence { + confidence: number; + level: "red" | "yellow" | "green"; +} + +/** Session state rebuilt from the ledger (see ledger.rebuildState). */ +export interface SessionState { + config: LedgerConfig | null; + segment: number; + runs: LedgerRun[]; + baseline: number | null; + best: number | null; + lastRunChecksFailed: boolean; + lastRun: LedgerRun | null; + totalExperiments: number; + consecutiveFailures: number; + confidence: Confidence | null; + plateau: boolean; + maxIterations?: number; + failureThreshold: number; +} + +/** `.auto/config.json` — optional per-session overrides. */ +export interface SessionConfig { + maxIterations?: number; + consecutiveFailures?: number; + workingDir?: string; + auditBypass?: boolean; + autoresearchOff?: boolean; + benchmarkHashes?: { measure: string | null; checks: string | null } | null; + /** Server-managed: checks outcome of the latest run_experiment (keep gate). */ + pendingChecksFailed?: boolean; + [key: string]: unknown; +} diff --git a/plugins/autoresearch/mcp/lib/validate.ts b/plugins/autoresearch/mcp/lib/validate.ts new file mode 100644 index 0000000..f098dd1 --- /dev/null +++ b/plugins/autoresearch/mcp/lib/validate.ts @@ -0,0 +1,136 @@ +// Ledger audit invariants (leo-inspired): the ledger must be a replayable +// state machine. Pure function, zero I/O, unit-testable. +import { isBetter } from "./experiment.ts"; +import type { LedgerConfig } from "./types.ts"; + +const VALID_STATUS = new Set([ + "keep", + "discard", + "crash", + "checks_failed", + "noop", +]); + +/** + * A ledger row as the auditor sees it. The audit's whole job is to accept + * possibly-malformed rows, so fields are loose (string status/type allowed). + */ +export interface LedgerRow { + type?: string; + run?: number; + segment?: number; + status?: string; + metric?: number | null; + commit?: string | null; +} + +export interface LedgerViolation { + code: string; + run?: number; + message: string; +} + +/** + * Validate a run sequence against the session config. Returns a list of + * { code, run, message } violations; empty list means the ledger is sound. + * + * Invariants: + * - event order: run numbers contiguous, segment matches config, status valid + * - baseline first: config must precede any run + * - keep must improve: every keep after the baseline must beat the current + * retained metric (direction-aware) + * - a discarded improvement needs a failed guard: if a non-keep run's metric + * beats the retained value, only `checks_failed` may discard it + * - commit field: keep rows must carry a commit, non-keep rows must not + */ +export function validateLedger( + runs: Array, + config: Pick | null | undefined, +): LedgerViolation[] { + const violations: LedgerViolation[] = []; + const direction = config?.direction ?? "lower"; + let retained: number | null = null; // current best kept metric (null until first keep) + let expectedRun = 1; + + const push = (code: string, run: number | undefined, message: string) => + violations.push({ code, run, message }); + + for (const r of runs) { + if (r.type === "config") { + expectedRun = 1; // a new segment restarts run numbering + continue; + } + + // ---- event order / baseline ---- + if (r.type !== "run") { + push("event_order", undefined, `unknown row type ${r.type}`); + continue; + } + if (!VALID_STATUS.has(r.status ?? "")) { + push("event_order", r.run, `invalid status ${r.status}`); + } + if (r.run !== expectedRun) { + push( + "event_order", + r.run, + `run number ${r.run} != expected ${expectedRun}`, + ); + } + expectedRun += 1; + if (r.segment !== config?.segment) { + push( + "event_order", + r.run, + `segment ${r.segment} != config segment ${config?.segment}`, + ); + } + + // ---- commit field consistency (non-keep rows must not carry a commit; + // keep rows' commit is generated by the tool after the git step) ---- + if (r.status !== "keep" && r.commit) { + push("commit_field", r.run, "non-keep row must not carry a commit"); + } + + // ---- keep must improve / discard needs failed guard ---- + const metric = r.metric; + if (metric == null || !Number.isFinite(metric)) { + if (r.status !== "crash") + push("event_order", r.run, "non-crash row missing metric"); + continue; + } + + if (r.status === "keep") { + if (retained == null) { + retained = metric; // baseline / first keep + } else if (!isBetter(metric, retained, direction)) { + push( + "keep_without_improvement", + r.run, + `keep metric ${metric} does not beat retained ${retained} (${direction})`, + ); + } else { + retained = metric; + } + } else if (r.status === "noop") { + // noop does not change the retained value and needs no commit + } else if (r.status === "crash") { + // crash metric is null — a crash measured nothing, so it must not + // touch the retained value (legacy rows carrying 0 are tolerated) + } else { + // discard / checks_failed + if ( + retained != null && + isBetter(metric, retained, direction) && + r.status !== "checks_failed" + ) { + push( + "discarded_improvement", + r.run, + `metric ${metric} beats retained ${retained} but status is ${r.status}, not checks_failed`, + ); + } + } + } + + return violations; +} diff --git a/plugins/autoresearch/mcp/server.ts b/plugins/autoresearch/mcp/server.ts new file mode 100644 index 0000000..d298116 --- /dev/null +++ b/plugins/autoresearch/mcp/server.ts @@ -0,0 +1,1157 @@ +#!/usr/bin/env node +// zcode-autoresearch MCP server (stdio, newline-delimited JSON-RPC). +// Tools: init_experiment / run_experiment / log_experiment / export_dashboard. +// Design: experiment/autoresearch + ADR-1 (MCP tools carry mechanism). +import { spawn } from "node:child_process"; +import { + appendFileSync, + existsSync, + readFileSync, + statSync, + writeFileSync, +} from "node:fs"; +import { createHash } from "node:crypto"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + parseMetricLines, + unwrapMeasureCommand, + median, + detectDoomLoop, +} from "./lib/experiment.ts"; +import { + autoPaths, + appendLedgerEntry, + rebuildState, + readSessionConfig, + writeDashboard, +} from "./lib/ledger.ts"; +import { + commitExperiment, + rollbackWorkingTree, + isGitRepo, + isDirty, + currentBranch, +} from "./lib/git.ts"; +import { renderDashboard } from "./lib/dashboard.ts"; +import { resolveWorkCwd } from "./lib/paths.ts"; +import { validateLedger } from "./lib/validate.ts"; +import { + ensureDashboardServer, + broadcastDashboardUpdate, +} from "./lib/dashboard-server.ts"; +import type { + Direction, + LedgerRun, + RunStatus, + SessionState, +} from "./lib/types.ts"; + +interface JsonRpcRequest { + jsonrpc?: string; + id?: unknown; + method?: string; + params?: { name?: string; arguments?: Record }; +} + +interface RunOutcome { + exitCode: number | null; + signal: NodeJS.Signals | null; + durationMs: number; + /** Metric-parseable text: the full output, or just the METRIC lines when the output spilled to a file. */ + output: string; + /** Display-tail source: the full output, or a bounded tail when spilled. */ + outputTail: string; + logFile: string | null; + timedOut: boolean; +} + +interface HookOutcome { + exitCode: number | null; + timedOut: boolean; + stdout: string; + stderr: string; + durationMs: number; + spawnError?: string; +} + +interface InitToolArgs { + name?: unknown; + metric_name?: string; + metricName?: string; + metric_unit?: string; + metricUnit?: string; + direction?: string; +} + +interface RunToolArgs { + command?: unknown; + timeout_seconds?: number; + repeat?: number; +} + +interface LogToolArgs { + status?: string; + description?: unknown; + metric?: unknown; + metrics?: Record; + asi?: Record; + constraints?: Array<{ name: string; maxPct: number }>; + commit?: string; +} + +const projectCwd = process.cwd(); +// Effective research directory: `.auto/config.json` may set workingDir. +const cwd = resolveWorkCwd(projectCwd); +const paths = autoPaths(cwd); + +const envMax = Number(process.env.AR_MAX_ITERATIONS); +const DEFAULT_MAX_ITERATIONS = + Number.isFinite(envMax) && envMax > 0 ? envMax : 20; +const BENCHMARK_TIMEOUT_MS = + Number(process.env.AR_BENCHMARK_TIMEOUT_MS) || 600_000; +const CHECKS_TIMEOUT_MS = Number(process.env.AR_CHECKS_TIMEOUT_MS) || 300_000; + +// LLM-facing output budget (mirrors pi-autoresearch): tight truncation. +const LLM_MAX_LINES = 10; +const LLM_MAX_BYTES = 4096; + +// --------------------------------------------------------------------------- +// JSON-RPC transport (newline-delimited on stdout; logs on stderr) +// --------------------------------------------------------------------------- + +function send(msg: unknown) { + process.stdout.write(JSON.stringify(msg) + "\n"); +} + +function result(id: unknown, result: unknown) { + send({ jsonrpc: "2.0", id, result }); +} + +function error(id: unknown, code: number, message: string) { + send({ jsonrpc: "2.0", id, error: { code, message } }); +} + +function log(...args: unknown[]) { + process.stderr.write(`[autoresearch] ${args.join(" ")}\n`); +} + +// --------------------------------------------------------------------------- +// Shared helpers +// --------------------------------------------------------------------------- + +function maxIterations() { + const cfg = readSessionConfig(projectCwd); + const v = Number(cfg.maxIterations); + return Number.isFinite(v) && v > 0 ? v : DEFAULT_MAX_ITERATIONS; +} + +function consecutiveFailures() { + const cfg = readSessionConfig(projectCwd); + const v = Number(cfg.consecutiveFailures); + return Number.isFinite(v) && v > 0 ? v : 3; +} + +function sessionState() { + return rebuildState(cwd, { + maxIterations: maxIterations(), + consecutiveFailures: consecutiveFailures(), + }); +} + +function truncateTail( + text: unknown, + maxLines = LLM_MAX_LINES, + maxBytes = LLM_MAX_BYTES, +): string { + if (text == null) return ""; + const lines = String(text).split("\n"); + let out = lines.slice(-maxLines).join("\n"); + if (Buffer.byteLength(out, "utf8") > maxBytes) { + out = Buffer.from(out, "utf8").subarray(0, maxBytes).toString("utf8"); + } + return out; +} + +// Output accounting: under the spill threshold the full output stays in +// memory; once it spills, data streams straight to the spill file and only +// the METRIC lines (scanned incrementally, position-independent) plus a +// bounded tail survive in memory. +const SPILL_THRESHOLD_BYTES = 2 * 1024 * 1024; +const TAIL_CAP_BYTES = 64 * 1024; +const MAX_METRIC_LINES = 1000; +const KILL_GRACE_MS = 5_000; + +function runCommand(command: string, timeoutMs: number): Promise { + return new Promise((resolve) => { + const started = Date.now(); + const proc = spawn("bash", ["-c", command], { + cwd, + detached: true, // own process group so we can kill the tree + stdio: ["ignore", "pipe", "pipe"], + }); + const chunks: Buffer[] = []; // full output, only while under the threshold + let totalBytes = 0; + let logFile: string | null = null; + const tail: Buffer[] = []; + let tailBytes = 0; + const metricLines: string[] = []; + let carry = ""; // partial line carried between data events + const onData = (d: Buffer) => { + totalBytes += d.length; + if (!logFile) { + chunks.push(d); + if (totalBytes > SPILL_THRESHOLD_BYTES) { + logFile = join( + tmpdir(), + `pi-experiment-${process.pid}-${Date.now()}.log`, + ); + writeFileSync(logFile, Buffer.concat(chunks)); + chunks.length = 0; + log("output overflowed, spilling to", logFile); + } + } else { + try { + appendFileSync(logFile, d); + } catch { + /* ignore */ + } + } + tail.push(d); + tailBytes += d.length; + while (tailBytes > TAIL_CAP_BYTES && tail.length > 0) { + const excess = tailBytes - TAIL_CAP_BYTES; + const first = tail[0]; + if (first.length <= excess) { + tailBytes -= first.length; + tail.shift(); + } else { + tail[0] = first.subarray(excess); + tailBytes -= excess; + } + } + carry += d.toString("utf8"); + const lines = carry.split("\n"); + carry = lines.pop() ?? ""; + if (carry.length > TAIL_CAP_BYTES) carry = carry.slice(-TAIL_CAP_BYTES); + for (const line of lines) { + if (metricLines.length < MAX_METRIC_LINES && line.startsWith("METRIC ")) + metricLines.push(line); + } + }; + proc.stdout.on("data", onData); + proc.stderr.on("data", onData); + let didTimeout = false; + let killTimer: NodeJS.Timeout | null = null; + const kill = () => { + didTimeout = true; + try { + if (proc.pid != null) process.kill(-proc.pid, "SIGTERM"); + } catch { + /* already gone */ + } + // A benchmark that traps/ignores SIGTERM must not hang the tool call: + // escalate to SIGKILL (uncatchable) on the whole process group. + killTimer = setTimeout(() => { + try { + if (proc.pid != null) process.kill(-proc.pid, "SIGKILL"); + } catch { + /* already gone */ + } + }, KILL_GRACE_MS); + }; + const timer = setTimeout(kill, timeoutMs); + proc.on("close", (code, signal) => { + clearTimeout(timer); + if (killTimer) clearTimeout(killTimer); + const elapsed = Date.now() - started; + const full = Buffer.concat(chunks).toString("utf8"); + resolve({ + exitCode: code, + signal, + durationMs: elapsed, + output: logFile ? metricLines.join("\n") : full, + outputTail: logFile ? Buffer.concat(tail).toString("utf8") : full, + logFile, + timedOut: didTimeout, + }); + }); + }); +} + +async function runChecks(checksFile: string) { + const res = await runCommand(`bash ${checksFile}`, CHECKS_TIMEOUT_MS); + return { + failed: res.exitCode !== 0, + exitCode: res.exitCode, + durationMs: res.durationMs, + outputTail: truncateTail(res.outputTail, 80, 4096), + }; +} + +// --------------------------------------------------------------------------- +// Iteration hooks (.auto/hooks/before.sh / after.sh) — pi-gap M1 (#23) +// --------------------------------------------------------------------------- + +const HOOK_TIMEOUT_MS = 30_000; +const HOOK_MAX_BYTES = 8 * 1024; + +function isExecutable(file: string): boolean { + if (!existsSync(file)) return false; + try { + return (statSync(file).mode & 0o111) !== 0; + } catch { + return false; + } +} + +// --- benchmark drift detection (frozen-file hashes) ------------------------- +function sha256File(file: string): string | null { + if (!existsSync(file)) return null; + return createHash("sha256").update(readFileSync(file)).digest("hex"); +} + +function currentBenchmarkHashes(): { + measure: string | null; + checks: string | null; +} { + return { + measure: sha256File(paths.measure), + checks: sha256File(paths.checks), + }; +} + +function readBenchmarkHashes() { + try { + return readSessionConfig(projectCwd).benchmarkHashes ?? null; + } catch { + return null; + } +} + +/** Merge a patch into the project's `.auto/config.json` (creates if missing). */ +function patchSessionConfig(patch: Record): void { + const cfgPath = join(projectCwd, ".auto", "config.json"); + let cfg: Record = {}; + try { + cfg = JSON.parse(readFileSync(cfgPath, "utf8")) as Record; + } catch { + /* start fresh */ + } + Object.assign(cfg, patch); + writeFileSync(cfgPath, JSON.stringify(cfg, null, 2)); +} + +function writeBenchmarkHashes(hashes: { + measure: string | null; + checks: string | null; +}): void { + patchSessionConfig({ benchmarkHashes: hashes }); +} + +/** + * Persist the checks outcome of the latest run_experiment so log_experiment's + * keep gate works even across an MCP server restart (pi `runtime.lastRunChecks` + * equivalent, but on disk). `failed` is true only when checks ran and failed. + */ +function setPendingChecksFailed(failed: boolean): void { + patchSessionConfig({ pendingChecksFailed: failed }); +} + +function pendingChecksFailed(): boolean { + return readSessionConfig(projectCwd).pendingChecksFailed === true; +} + +/** + * Compare current frozen-file hashes against the session's recorded ones. + * Returns { drift, reason, deleted } where drift is true when a recorded hash + * changed (deleted=false) or a recorded file was deleted (deleted=true). + * First sighting (recorded null but file exists) records the hash without + * warning. + */ +function checkBenchmarkDrift() { + const recorded = readBenchmarkHashes(); + const current = currentBenchmarkHashes(); + if (!recorded) { + writeBenchmarkHashes(current); + return { + drift: false, + reason: "recorded", + deleted: false, + hashes: current, + }; + } + const merged = { ...recorded }; + let firstSeen = false; + for (const key of ["measure", "checks"] as const) { + if (recorded[key] == null && current[key] != null) { + merged[key] = current[key]; // first sighting: record, no warning + firstSeen = true; + continue; + } + // changed (hash mismatch) or deleted (current null) → drift + if (recorded[key] != null && current[key] == null) { + return { drift: true, reason: key, deleted: true, hashes: current }; + } + if (recorded[key] != null && recorded[key] !== current[key]) { + return { drift: true, reason: key, deleted: false, hashes: current }; + } + } + if (firstSeen) writeBenchmarkHashes(merged); + return { + drift: false, + reason: firstSeen ? "recorded" : null, + deleted: false, + hashes: current, + }; +} + +/** + * Run an iteration hook: bash