From 2738275e508b416fae75f065cd162d940c45be66 Mon Sep 17 00:00:00 2001 From: Farnam Date: Sun, 18 Oct 2026 16:35:12 +0330 Subject: [PATCH 01/14] Pin structural TinyTools fork for reproducible OpenHuman builds --- .gitmodules | 2 +- vendor/tinytools | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 3c1225634..5c74d2ac4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,7 +3,7 @@ url = https://github.com/tinyhumansai/tinyagents.wiki.git [submodule "vendor/tinytools"] path = vendor/tinytools - url = https://github.com/tinyhumansai/tinytools.git + url = https://github.com/AmirrezaFarnamTaheri/tinytools.git [submodule "vendor/tinyinference"] path = vendor/tinyinference url = https://github.com/tinyhumansai/tinyinference.git diff --git a/vendor/tinytools b/vendor/tinytools index 9e1156141..daec6c7a5 160000 --- a/vendor/tinytools +++ b/vendor/tinytools @@ -1 +1 @@ -Subproject commit 9e11561413374bf0865a3ff5377a0b4de451f10e +Subproject commit daec6c7a563336dd83d2e65ae38b2b169179aada From 69eb675a64ab0317531ce37a9a117c0e09dc42cc Mon Sep 17 00:00:00 2001 From: Farnam Date: Sun, 18 Oct 2026 16:55:40 +0330 Subject: [PATCH 02/14] Isolate transcript turn-lock test destinations --- .../src/transcript/turn_lock_tests.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/crates/tinyagents-session/src/transcript/turn_lock_tests.rs b/crates/tinyagents-session/src/transcript/turn_lock_tests.rs index cfbe61969..d5bc2e9b1 100644 --- a/crates/tinyagents-session/src/transcript/turn_lock_tests.rs +++ b/crates/tinyagents-session/src/transcript/turn_lock_tests.rs @@ -15,6 +15,7 @@ async fn is_held(locator: &dyn TranscriptLocator, session: &SessionRef) -> bool /// A locator naming a fixed destination, so separate instances stand for /// separate handles over one store. +/// Each test needs its own destination because the lock registry is process-wide. struct Keyed(&'static str); impl TranscriptLocator for Keyed { @@ -34,7 +35,7 @@ impl TranscriptLocator for Keyed { #[tokio::test] async fn a_held_turn_lock_blocks_the_same_session_until_released() { - let locator = Keyed("store-a"); + let locator = Keyed("turn-lock-held"); let session = SessionRef::scoped("thread-1", "agent"); let guard = lock_session_turn(&locator, &session).await.unwrap(); @@ -47,25 +48,25 @@ async fn a_held_turn_lock_blocks_the_same_session_until_released() { #[tokio::test] async fn every_generation_and_every_locator_over_one_destination_share_the_lock() { let session = SessionRef::scoped("thread-1", "agent"); - let _guard = lock_session_turn(&Keyed("store-a"), &session) + let _guard = lock_session_turn(&Keyed("turn-lock-generations"), &session) .await .unwrap(); // A compaction moves the head on; it is still the same conversation. A // host also rebuilds its locator per turn rather than sharing one. - let rebuilt = Keyed("store-a"); + let rebuilt = Keyed("turn-lock-generations"); assert!(is_held(&rebuilt, &session.next_generation().next_generation()).await); } #[tokio::test] async fn other_sessions_and_other_destinations_are_independent() { - let locator = Keyed("store-a"); + let locator = Keyed("turn-lock-independent-a"); let session = SessionRef::scoped("thread-1", "agent"); let _guard = lock_session_turn(&locator, &session).await.unwrap(); assert!(!is_held(&locator, &SessionRef::scoped("thread-2", "agent")).await); assert!(!is_held(&locator, &SessionRef::scoped("thread-1", "other-agent")).await); - assert!(!is_held(&Keyed("store-b"), &session).await); + assert!(!is_held(&Keyed("turn-lock-independent-b"), &session).await); } #[tokio::test] From 4a470a4102d746f8158637fe56389b95a7ce1029 Mon Sep 17 00:00:00 2001 From: Farnam Date: Sun, 18 Oct 2026 22:54:22 +0330 Subject: [PATCH 03/14] ci: audit TinyAgents upstream convergence for OpenHuman PR 14 --- ...penhuman-upstream-tinyagents-sync-once.yml | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 .github/workflows/openhuman-upstream-tinyagents-sync-once.yml diff --git a/.github/workflows/openhuman-upstream-tinyagents-sync-once.yml b/.github/workflows/openhuman-upstream-tinyagents-sync-once.yml new file mode 100644 index 000000000..076084314 --- /dev/null +++ b/.github/workflows/openhuman-upstream-tinyagents-sync-once.yml @@ -0,0 +1,59 @@ +name: OpenHuman TinyAgents fork upstream sync (one-shot) +on: + push: + branches: + - integration/openhuman-structural-20261018 + paths: + - .github/workflows/openhuman-upstream-tinyagents-sync-once.yml +permissions: + contents: write +jobs: + integrate: + runs-on: ubuntu-latest + timeout-minutes: 40 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: false + persist-credentials: true + - name: Audit and merge TinyAgents canonical main + shell: bash + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + branch="integration/openhuman-structural-20261018" + expected="$GITHUB_SHA" + current="$(git ls-remote origin "refs/heads/$branch" | cut -f1)" + if [ "$current" != "$expected" ]; then + echo "::error::Branch changed since this workflow started; stop before updating" + exit 3 + fi + git remote add canonical https://github.com/tinyhumansai/tinyagents.git + git fetch --no-tags canonical +refs/heads/main:refs/remotes/canonical/main + upstream="$(git rev-parse refs/remotes/canonical/main)" + echo "Upstream TinyAgents $upstream" + echo "Merge base $(git merge-base HEAD "$upstream")" + mkdir -p sync-report + echo "$upstream" > sync-report/upstream.txt + if git merge --no-commit --no-ff "$upstream"; then + git rm -- .github/workflows/openhuman-upstream-tinyagents-sync-once.yml + git diff --cached --check + git commit -m "merge: onboard canonical TinyAgents main ($upstream)" + git push origin "HEAD:refs/heads/$branch" + else + git diff --name-only --diff-filter=U | tee sync-report/conflicts.txt + git ls-files -u > sync-report/unmerged-stages.txt + git diff --cc > sync-report/combined.diff + git status --short > sync-report/status.txt + echo "::error::TinyAgents source merge conflicts ($(wc -l < sync-report/conflicts.txt))" + exit 2 + fi + - name: Publish conflict audit + if: failure() + uses: actions/upload-artifact@v4 + with: + name: openhuman-tinyagents-sync-conflicts + path: sync-report/ + if-no-files-found: warn From a3db1a7a5f7e07a58efcc36602a16c943c5a3cbe Mon Sep 17 00:00:00 2001 From: Farnam Date: Sun, 18 Oct 2026 23:02:21 +0330 Subject: [PATCH 04/14] ci: resolve TinyAgents merge by retaining structural TinyTools and canonical test scopes --- ...penhuman-upstream-tinyagents-sync-once.yml | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/.github/workflows/openhuman-upstream-tinyagents-sync-once.yml b/.github/workflows/openhuman-upstream-tinyagents-sync-once.yml index 076084314..a8fc368ae 100644 --- a/.github/workflows/openhuman-upstream-tinyagents-sync-once.yml +++ b/.github/workflows/openhuman-upstream-tinyagents-sync-once.yml @@ -37,19 +37,37 @@ jobs: echo "Merge base $(git merge-base HEAD "$upstream")" mkdir -p sync-report echo "$upstream" > sync-report/upstream.txt - if git merge --no-commit --no-ff "$upstream"; then - git rm -- .github/workflows/openhuman-upstream-tinyagents-sync-once.yml - git diff --cached --check - git commit -m "merge: onboard canonical TinyAgents main ($upstream)" - git push origin "HEAD:refs/heads/$branch" - else + status=0 + git merge --no-commit --no-ff "$upstream" || status=$? + if [ "$status" -ne 0 ]; then git diff --name-only --diff-filter=U | tee sync-report/conflicts.txt git ls-files -u > sync-report/unmerged-stages.txt git diff --cc > sync-report/combined.diff git status --short > sync-report/status.txt - echo "::error::TinyAgents source merge conflicts ($(wc -l < sync-report/conflicts.txt))" + expected="$(printf '%s\n' 'crates/tinyagents-session/src/transcript/turn_lock_tests.rs' 'vendor/tinytools' | sort)" + actual="$(sort sync-report/conflicts.txt)" + if [ "$actual" != "$expected" ]; then + echo "::error::Unreviewed TinyAgents conflicts; refusing an automatic resolution" + exit 2 + fi + # Only test-fixture naming differs; prefer upstream's globally + # unique store/thread identifiers to prevent parallel-test bleed. + git checkout --theirs -- crates/tinyagents-session/src/transcript/turn_lock_tests.rs + git add -- crates/tinyagents-session/src/transcript/turn_lock_tests.rs + # Preserve the structural-code additions in the fork of TinyTools, + # now rebased by a real upstream merge and verified against its main. + git update-index --add --cacheinfo 160000,0d349716deafc4d91a595bf85400e8445cd1a6e8,vendor/tinytools + fi + # GITHUB_TOKEN has contents:write but not workflows permission. + # Sync upstream YAML through the GitHub connector after source merge. + git restore --source=HEAD --staged --worktree -- .github/workflows + if git diff --name-only --diff-filter=U | grep .; then + echo "::error::Still contains unmerged paths" exit 2 fi + git diff --cached --check + git commit -m "merge: onboard canonical TinyAgents main ($upstream) with TinyTools structural fork" + git push origin "HEAD:refs/heads/$branch" - name: Publish conflict audit if: failure() uses: actions/upload-artifact@v4 From d2e17bebe5bc9f9dd5ce2c7c09c33e674b865cbb Mon Sep 17 00:00:00 2001 From: Farnam Date: Sun, 18 Oct 2026 23:03:38 +0330 Subject: [PATCH 05/14] chore: remove completed one-shot TinyAgents integration workflow --- ...penhuman-upstream-tinyagents-sync-once.yml | 77 ------------------- 1 file changed, 77 deletions(-) delete mode 100644 .github/workflows/openhuman-upstream-tinyagents-sync-once.yml diff --git a/.github/workflows/openhuman-upstream-tinyagents-sync-once.yml b/.github/workflows/openhuman-upstream-tinyagents-sync-once.yml deleted file mode 100644 index a8fc368ae..000000000 --- a/.github/workflows/openhuman-upstream-tinyagents-sync-once.yml +++ /dev/null @@ -1,77 +0,0 @@ -name: OpenHuman TinyAgents fork upstream sync (one-shot) -on: - push: - branches: - - integration/openhuman-structural-20261018 - paths: - - .github/workflows/openhuman-upstream-tinyagents-sync-once.yml -permissions: - contents: write -jobs: - integrate: - runs-on: ubuntu-latest - timeout-minutes: 40 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: false - persist-credentials: true - - name: Audit and merge TinyAgents canonical main - shell: bash - run: | - set -euo pipefail - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - branch="integration/openhuman-structural-20261018" - expected="$GITHUB_SHA" - current="$(git ls-remote origin "refs/heads/$branch" | cut -f1)" - if [ "$current" != "$expected" ]; then - echo "::error::Branch changed since this workflow started; stop before updating" - exit 3 - fi - git remote add canonical https://github.com/tinyhumansai/tinyagents.git - git fetch --no-tags canonical +refs/heads/main:refs/remotes/canonical/main - upstream="$(git rev-parse refs/remotes/canonical/main)" - echo "Upstream TinyAgents $upstream" - echo "Merge base $(git merge-base HEAD "$upstream")" - mkdir -p sync-report - echo "$upstream" > sync-report/upstream.txt - status=0 - git merge --no-commit --no-ff "$upstream" || status=$? - if [ "$status" -ne 0 ]; then - git diff --name-only --diff-filter=U | tee sync-report/conflicts.txt - git ls-files -u > sync-report/unmerged-stages.txt - git diff --cc > sync-report/combined.diff - git status --short > sync-report/status.txt - expected="$(printf '%s\n' 'crates/tinyagents-session/src/transcript/turn_lock_tests.rs' 'vendor/tinytools' | sort)" - actual="$(sort sync-report/conflicts.txt)" - if [ "$actual" != "$expected" ]; then - echo "::error::Unreviewed TinyAgents conflicts; refusing an automatic resolution" - exit 2 - fi - # Only test-fixture naming differs; prefer upstream's globally - # unique store/thread identifiers to prevent parallel-test bleed. - git checkout --theirs -- crates/tinyagents-session/src/transcript/turn_lock_tests.rs - git add -- crates/tinyagents-session/src/transcript/turn_lock_tests.rs - # Preserve the structural-code additions in the fork of TinyTools, - # now rebased by a real upstream merge and verified against its main. - git update-index --add --cacheinfo 160000,0d349716deafc4d91a595bf85400e8445cd1a6e8,vendor/tinytools - fi - # GITHUB_TOKEN has contents:write but not workflows permission. - # Sync upstream YAML through the GitHub connector after source merge. - git restore --source=HEAD --staged --worktree -- .github/workflows - if git diff --name-only --diff-filter=U | grep .; then - echo "::error::Still contains unmerged paths" - exit 2 - fi - git diff --cached --check - git commit -m "merge: onboard canonical TinyAgents main ($upstream) with TinyTools structural fork" - git push origin "HEAD:refs/heads/$branch" - - name: Publish conflict audit - if: failure() - uses: actions/upload-artifact@v4 - with: - name: openhuman-tinyagents-sync-conflicts - path: sync-report/ - if-no-files-found: warn From 01256cfddd668ef25579950392cad944c1b35cbb Mon Sep 17 00:00:00 2001 From: Farnam Date: Sun, 18 Oct 2026 23:18:01 +0330 Subject: [PATCH 06/14] fix(harness): match struct-shaped packed disclosure targets Update the pinned selection matcher for PackTarget::Answer's permission-bearing struct variant. Add a regression asserting that disclosure cannot expose an executable tool or elevated permission. --- .../src/tool/packs/mod_tests.rs | 19 +++++++++++++++++++ .../tinyagents-harness/src/tool/packs/tool.rs | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/crates/tinyagents-harness/src/tool/packs/mod_tests.rs b/crates/tinyagents-harness/src/tool/packs/mod_tests.rs index 3fd899f3e..c41bbd300 100644 --- a/crates/tinyagents-harness/src/tool/packs/mod_tests.rs +++ b/crates/tinyagents-harness/src/tool/packs/mod_tests.rs @@ -107,6 +107,25 @@ fn find<'a>(tools: &'a [Box], name: &str) -> &'a dyn Tool { .unwrap_or_else(|| panic!("{name} missing")) } +#[test] +fn a_pinned_disclosure_has_no_executable_selection() { + // A pinned disclosure is an Answer struct variant, never a packed + // execution target. Only Execute may expose an admitted inner tool. + let pinned = super::types::PinnedPackCall { + outer: Arc::new(FakeTool::plain("use_skill")), + target: super::types::PackTarget::Answer { + result: ToolResult::success("pack schema"), + permission: PermissionLevel::ReadOnly, + }, + }; + assert!(pinned.selection().is_none()); + assert!(pinned.admission_target().is_none()); + assert_eq!( + pinned.permission_level_with_args(&Value::Null), + PermissionLevel::ReadOnly, + ); +} + #[test] fn the_use_skill_declaration_is_byte_stable() { let tools = bound(vec![]); diff --git a/crates/tinyagents-harness/src/tool/packs/tool.rs b/crates/tinyagents-harness/src/tool/packs/tool.rs index 7c2e9b218..89ce0d339 100644 --- a/crates/tinyagents-harness/src/tool/packs/tool.rs +++ b/crates/tinyagents-harness/src/tool/packs/tool.rs @@ -85,7 +85,7 @@ impl PinnedPackCall { index: *index, arguments: arguments.clone(), }), - PackTarget::Answer(_) => None, + PackTarget::Answer { .. } => None, } } From 7583b96b0a2361137cd4e55e187237ad7baf7eee Mon Sep 17 00:00:00 2001 From: Farnam Date: Fri, 23 Oct 2026 13:49:38 +0330 Subject: [PATCH 07/14] Add atomic edits and lane moves for pending run messages --- .../src/run_queue/README.md | 10 ++- .../tinyagents-harness/src/run_queue/mod.rs | 79 +++++++++++++++++-- .../src/run_queue/mod_tests.rs | 57 +++++++++++++ 3 files changed, 140 insertions(+), 6 deletions(-) diff --git a/crates/tinyagents-harness/src/run_queue/README.md b/crates/tinyagents-harness/src/run_queue/README.md index c2c6ae0fc..0f23dd849 100644 --- a/crates/tinyagents-harness/src/run_queue/README.md +++ b/crates/tinyagents-harness/src/run_queue/README.md @@ -33,6 +33,11 @@ stops, pauses, and deferrals leave the queue untouched. Details: `QueueMode`-aware form; `status()` snapshots per-lane depth; `clear()` empties every lane and returns how many items were dropped. + `edit_first_where(predicate, edit)` changes one pending payload in place; + `move_first_where(predicate, lane, index)` relocates one pending item to + a destination lane and position. Both return `false` if delivery already + consumed the item. Indices address the destination after removal and clamp + to its length. Other items keep their relative order. - [`QueueLane`] — which lane an item belongs to: `Steer` (inject at the next safe boundary as an instruction), `Followup` (dispatch as a fresh turn once the active run completes), `Collect` (handed back to the host at run end as @@ -54,11 +59,14 @@ stops, pauses, and deferrals leave the queue untouched. Details: ## Operational constraints - Every method serializes through a single `std::sync::Mutex` guarding all - three lanes together. It is held only for one vector operation and never + three lanes together. It is held only for in-memory operations and never across an `.await`, so the `async` methods never block a runtime thread for long and the synchronous `push_now`/`drain_now` are safe anywhere. The queue is meant to hold a handful of pending items, not a high-throughput channel. +- Edit callbacks run under that lock. Validate replacement data before calling, + keep callbacks brief, and never call back into the queue. A move to `Steer` + still waits for a safe runtime boundary; it does not cancel an active tool. - `SteeringForwarderGuard` drains and delivers under its shutdown gate and requeues residual (delivered-but-unapplied) steers synchronously in `Drop`, so once the guard has dropped, every steer the run did not apply is back in diff --git a/crates/tinyagents-harness/src/run_queue/mod.rs b/crates/tinyagents-harness/src/run_queue/mod.rs index f1aed8b0f..f7d9783cb 100644 --- a/crates/tinyagents-harness/src/run_queue/mod.rs +++ b/crates/tinyagents-harness/src/run_queue/mod.rs @@ -56,8 +56,7 @@ pub struct RunQueue { inner: Mutex>, } -/// The three lanes behind [`RunQueue`]'s lock, each an append-only FIFO until -/// drained. +/// The three FIFO lanes behind [`RunQueue`]'s lock. #[derive(Debug)] struct RunQueueInner { /// Immediate steering messages, consumed as soon as the loop reaches a @@ -81,9 +80,9 @@ impl RunQueue { } } - /// Locks the lanes. A panic while the lock was held cannot leave a lane - /// half-updated (every operation is a single vector call), so a poisoned - /// lock is recovered rather than propagated. + /// Locks the lanes, recovering a poisoned lock. Vector invariants remain + /// valid after unwinding; an editing callback must keep its payload valid + /// even when it panics because callback changes are not rolled back. fn lock(&self) -> MutexGuard<'_, RunQueueInner> { self.inner.lock().unwrap_or_else(PoisonError::into_inner) } @@ -259,6 +258,76 @@ impl RunQueue { let after = inner.steers.len() + inner.followups.len() + inner.collects.len(); before - after } + + /// Edits the first matching pending item in delivery order, atomically + /// with respect to delivery, removal, and other queue mutations. + /// + /// Returns `false` if the item has already been consumed. `edit` runs + /// under the queue lock: it must be brief and must not call this queue. + /// Hosts retain responsibility for validating the replacement payload. + pub async fn edit_first_where( + &self, + pred: impl Fn(&T) -> bool, + edit: impl FnOnce(&mut T), + ) -> bool { + let mut inner = self.lock(); + let RunQueueInner { + steers, + followups, + collects, + } = &mut *inner; + let item = steers + .iter_mut() + .chain(followups.iter_mut()) + .chain(collects.iter_mut()) + .find(|item| pred(item)); + match item { + Some(item) => { + edit(item); + true + } + None => false, + } + } + + /// Moves the first matching pending item to `lane` at `index`, atomically + /// with respect to delivery. `index` addresses the destination after the + /// item is removed and is clamped to that lane's length. + /// + /// Moving a follow-up into the steer lane requests delivery at the next + /// safe boundary; it never interrupts a tool batch. Returns `false` when + /// the item has already been consumed. All other items retain their order. + pub async fn move_first_where( + &self, + pred: impl Fn(&T) -> bool, + lane: QueueLane, + index: usize, + ) -> bool { + let mut inner = self.lock(); + let item = { + let RunQueueInner { + steers, + followups, + collects, + } = &mut *inner; + [steers, followups, collects].into_iter().find_map(|items| { + items + .iter() + .position(&pred) + .map(|position| items.remove(position)) + }) + }; + let Some(item) = item else { + return false; + }; + let target = match lane { + QueueLane::Steer => &mut inner.steers, + QueueLane::Followup => &mut inner.followups, + QueueLane::Collect => &mut inner.collects, + }; + target.insert(index.min(target.len()), item); + true + } } impl Default for RunQueue { diff --git a/crates/tinyagents-harness/src/run_queue/mod_tests.rs b/crates/tinyagents-harness/src/run_queue/mod_tests.rs index d27735230..b1abf543e 100644 --- a/crates/tinyagents-harness/src/run_queue/mod_tests.rs +++ b/crates/tinyagents-harness/src/run_queue/mod_tests.rs @@ -164,3 +164,60 @@ async fn remove_where_with_no_match_removes_nothing() { assert_eq!(removed, 0); assert_eq!(queue.status().await.total, 2); } + +#[tokio::test] +async fn editing_changes_only_the_first_pending_match_in_delivery_order() { + let queue = RunQueue::new(); + queue.push(QueueLane::Steer, 1).await; + queue.push(QueueLane::Followup, 1).await; + assert!( + queue + .edit_first_where(|item| *item == 1, |item| *item = 2) + .await + ); + assert_eq!( + queue.snapshot().await, + vec![(QueueLane::Steer, 2), (QueueLane::Followup, 1)] + ); + queue.drain(QueueLane::Steer).await; + assert!( + !queue + .edit_first_where( + |item| *item == 2, + |_| panic!("consumed item must not be edited") + ) + .await + ); +} + +#[tokio::test] +async fn moving_can_reorder_and_promote_a_followup_without_dropping_other_items() { + let queue = RunQueue::new(); + for item in [1, 2, 3] { + queue.push(QueueLane::Followup, item).await; + } + assert!( + queue + .move_first_where(|item| *item == 3, QueueLane::Followup, 0) + .await + ); + assert!( + queue + .move_first_where(|item| *item == 2, QueueLane::Steer, usize::MAX) + .await + ); + assert_eq!( + queue.snapshot().await, + vec![ + (QueueLane::Steer, 2), + (QueueLane::Followup, 3), + (QueueLane::Followup, 1) + ] + ); + assert!( + !queue + .move_first_where(|item| *item == 99, QueueLane::Collect, 0) + .await + ); + assert_eq!(queue.status().await.total, 3); +} From 2db1d19c073bd203744610f42b3e80e3b2e72327 Mon Sep 17 00:00:00 2001 From: Farnam Date: Fri, 23 Oct 2026 13:49:56 +0330 Subject: [PATCH 08/14] Persist addressed run inputs with atomic deduplication --- .../src/run_ledger/command_center/README.md | 16 ++ .../src/run_ledger/drivers/events.rs | 8 +- .../src/run_ledger/drivers/inbox.rs | 193 ++++++++++++++++++ .../src/run_ledger/drivers/inbox_tests.rs | 120 +++++++++++ .../src/run_ledger/drivers/mod.rs | 1 + .../src/run_ledger/drivers/schema.rs | 11 +- .../src/run_ledger/drivers/schema_tests.rs | 1 + .../src/run_ledger/drivers/storage.rs | 4 + .../src/run_ledger/inbox.rs | 63 ++++++ .../tinyagents-session/src/run_ledger/mod.rs | 1 + 10 files changed, 412 insertions(+), 6 deletions(-) create mode 100644 crates/tinyagents-session/src/run_ledger/drivers/inbox.rs create mode 100644 crates/tinyagents-session/src/run_ledger/drivers/inbox_tests.rs create mode 100644 crates/tinyagents-session/src/run_ledger/inbox.rs diff --git a/crates/tinyagents-session/src/run_ledger/command_center/README.md b/crates/tinyagents-session/src/run_ledger/command_center/README.md index b8a01ecf5..5efff064d 100644 --- a/crates/tinyagents-session/src/run_ledger/command_center/README.md +++ b/crates/tinyagents-session/src/run_ledger/command_center/README.md @@ -38,3 +38,19 @@ the host passes an agent-id to display-name closure and owns the RPC surface. returns `InvalidTransition` when the verb no longer applies. - `stop` stamps `completed_at` and stores the optional reason as `error`; `retry` and `continue` clear both; `follow_up` leaves the status unchanged. +# Durable addressed input draft + +`run_ledger::inbox::RunInboxStore` is a separate optional capability implemented +by `DriverRunLedger`. `enqueue_input` validates the exact persisted recipient +lineage and commits the queued record, deduplication identity and `input_queued` +event in one checked batch. Reusing the root/submission id for different content +returns a conflict. Receipts survive process restart and list in event-sequence +order. A backend without checked atomic batches returns its native error. + +This is an admission draft, not a complete delivery implementation. No operation +marks input delivered, changes a run to running, or launches a worker. Recipient +integration must atomically commit admitted input into the durable session and +record consumption before exposing delivery/resumption controls. Live job +request-id deduplication alone is not sufficient for this contract. Entries have +no implicit expiry; retention must preserve receipt identities while retries are +possible. Core host consumption awaits upstream availability. diff --git a/crates/tinyagents-session/src/run_ledger/drivers/events.rs b/crates/tinyagents-session/src/run_ledger/drivers/events.rs index 543acd4f4..e276d2382 100644 --- a/crates/tinyagents-session/src/run_ledger/drivers/events.rs +++ b/crates/tinyagents-session/src/run_ledger/drivers/events.rs @@ -22,13 +22,13 @@ pub(super) fn head_after(highest: u64) -> Option { (highest < i64::MAX as u64).then_some(highest + 1) } -struct EventCommit { - event: RunEvent, - writes: Vec, +pub(super) struct EventCommit { + pub(super) event: RunEvent, + pub(super) writes: Vec, } impl DriverRunLedger { - async fn event_commit(&self, input: &RunEventAppend) -> Result { + pub(super) async fn event_commit(&self, input: &RunEventAppend) -> Result { let head = self.read::(EVENT_HEADS, &input.run_id).await?; let sequence = match &head { // A run migrated from v1 has events but no head (the v1 head's id diff --git a/crates/tinyagents-session/src/run_ledger/drivers/inbox.rs b/crates/tinyagents-session/src/run_ledger/drivers/inbox.rs new file mode 100644 index 000000000..29a58c9c9 --- /dev/null +++ b/crates/tinyagents-session/src/run_ledger/drivers/inbox.rs @@ -0,0 +1,193 @@ +//! Atomic durable input admission with persisted parent-lineage fencing. + +use super::dto::{from_ms, stored_as_is}; +use super::storage::{AGENTS, CAS_ATTEMPTS, INPUTS, key, now, put_op}; +use super::*; +use crate::run_ledger::inbox::{AddressedInput, QueuedInput, RunInboxStore}; +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; +use tinystoragedrivers_core::{ErrorKind, ExpectedDocument, Precondition, ReadCheck}; + +#[derive(Clone, Serialize, Deserialize)] +struct StoredInput { + #[serde(flatten)] + input: AddressedInput, + sequence: u64, + created_ms: i64, +} +stored_as_is!(StoredInput); + +impl StoredInput { + fn receipt(self) -> Result { + Ok(QueuedInput { + input: self.input, + sequence: self.sequence, + created_at: from_ms(self.created_ms)?, + }) + } +} + +fn validate(input: &AddressedInput) -> Result<()> { + for id in [ + &input.submission_id, + &input.root_thread_id, + &input.recipient_run_id, + ] { + if id.trim().is_empty() || id.len() > 512 || id.contains('\0') { + return Err(StorageError::serialization("invalid addressed input identity").into()); + } + } + if input.text.trim().is_empty() || input.text.len() > 1024 * 1024 { + return Err(StorageError::serialization( + "addressed input must contain 1..=1048576 bytes of nonblank text", + ) + .into()); + } + Ok(()) +} + +impl DriverRunLedger { + // Walk exact persisted run links, never infer ownership from a display name + // or an in-memory worker. Every observed link is fenced in the admission. + async fn input_lineage(&self, root: &str, recipient: &str) -> Result> { + let mut current = recipient.to_owned(); + let mut visited = HashSet::new(); + let mut checks = Vec::new(); + for _ in 0..128 { + if !visited.insert(current.clone()) { + return Err(StorageError::serialization("cyclic input recipient lineage").into()); + } + let row = self + .read::(AGENTS, ¤t) + .await? + .ok_or_else(|| StorageError::serialization("input recipient lineage is missing"))?; + checks.push(ReadCheck { + collection: AGENTS.into(), + id: key(&[¤t])?, + expected: ExpectedDocument::Version(row.version), + }); + match row.doc.parent_run_id { + Some(parent) => current = parent, + None if row.doc.parent_thread_id.as_deref() == Some(root) => return Ok(checks), + None => { + return Err(StorageError::serialization( + "input recipient belongs to another root", + ) + .into()); + } + } + } + Err(StorageError::serialization("input recipient lineage exceeds 128 runs").into()) + } +} + +#[async_trait::async_trait] +impl RunInboxStore for DriverRunLedger { + async fn enqueue_input(&self, input: AddressedInput) -> Result { + validate(&input)?; + self.declare().await?; + let id = key(&[&input.root_thread_id, &input.submission_id])?; + for _ in 0..CAS_ATTEMPTS { + let mut checks = self + .input_lineage(&input.root_thread_id, &input.recipient_run_id) + .await?; + let previous = self.scoped.documents().get(INPUTS, &id).await?; + if let Some(previous) = previous { + let stored: StoredInput = super::storage::decode(previous.doc)?; + if stored.input != input { + return Err(StorageError::conflict( + "submission id already addresses different input", + ) + .into()); + } + checks.push(ReadCheck { + collection: INPUTS.into(), + id: id.clone(), + expected: ExpectedDocument::Version(previous.version), + }); + match self + .scoped + .documents() + .atomic_batch_checked(checks, vec![]) + .await + { + Ok(_) => return stored.receipt(), + Err(error) if error.kind() == ErrorKind::Conflict => continue, + Err(error) => return Err(error.into()), + } + } + let commit = self.event_commit(&RunEventAppend { + run_id: input.recipient_run_id.clone(), + event_type: "input_queued".into(), + payload: serde_json::json!({"submission_id": input.submission_id, "mode": input.mode}), + }).await?; + let stored = StoredInput { + input: input.clone(), + sequence: commit.event.sequence, + created_ms: now().timestamp_millis(), + }; + let mut writes = commit.writes; + writes.push(put_op(INPUTS, id.clone(), &stored, Precondition::Absent)?); + match self + .scoped + .documents() + .atomic_batch_checked(checks, writes) + .await + { + Ok(_) => return stored.receipt(), + Err(error) if error.kind() == ErrorKind::Conflict => continue, + Err(error) => return Err(error.into()), + } + } + Err(StorageError::conflict("input admission contention budget exhausted").into()) + } + + async fn pending_inputs( + &self, + root: &str, + recipient: &str, + limit: u32, + ) -> Result> { + if root.trim().is_empty() || recipient.trim().is_empty() { + return Err( + StorageError::serialization("input root and recipient are required").into(), + ); + } + for _ in 0..CAS_ATTEMPTS { + let checks = self.input_lineage(root, recipient).await?; + let page = self + .scoped + .documents() + .query( + INPUTS, + &Query::filter( + Filter::eq("root_thread_id", root) + .and(Filter::eq("recipient_run_id", recipient)), + ) + .sort(Sort::asc("sequence")) + .limit(limit.clamp(1, 500) as usize), + ) + .await?; + let inputs = page + .items + .into_iter() + .map(|row| super::storage::decode::(row.doc)?.receipt()) + .collect::>>()?; + match self + .scoped + .documents() + .atomic_batch_checked(checks, vec![]) + .await + { + Ok(_) => return Ok(inputs), + Err(error) if error.kind() == ErrorKind::Conflict => continue, + Err(error) => return Err(error.into()), + } + } + Err(StorageError::conflict("input read contention budget exhausted").into()) + } +} + +#[cfg(test)] +#[path = "inbox_tests.rs"] +mod tests; diff --git a/crates/tinyagents-session/src/run_ledger/drivers/inbox_tests.rs b/crates/tinyagents-session/src/run_ledger/drivers/inbox_tests.rs new file mode 100644 index 000000000..d4cdd3472 --- /dev/null +++ b/crates/tinyagents-session/src/run_ledger/drivers/inbox_tests.rs @@ -0,0 +1,120 @@ +use super::*; +use crate::run_ledger::inbox::InputMode; +use tinystoragedrivers_core::{MemoryStorage, StorageBackend}; + +fn ledger() -> DriverRunLedger { + DriverRunLedger::new( + MemoryStorage::new() + .for_scope(&Scope::new("inbox-tests").unwrap()) + .unwrap(), + ) +} + +fn run(id: &str, parent: Option<&str>, thread: Option<&str>) -> AgentRunUpsert { + AgentRunUpsert { + id: id.into(), + kind: AgentRunKind::Subagent, + parent_run_id: parent.map(str::to_owned), + parent_thread_id: thread.map(str::to_owned), + agent_id: None, + status: AgentRunStatus::AwaitingUser, + prompt_ref: None, + worker_thread_id: None, + checkpoint_path: None, + checkpoint: None, + summary: None, + error: None, + metadata: serde_json::json!({}), + started_at: None, + completed_at: None, + } +} + +fn input(id: &str) -> AddressedInput { + AddressedInput { + submission_id: id.into(), + root_thread_id: "root".into(), + recipient_run_id: "child".into(), + mode: InputMode::Answer, + text: "Keep this exact answer\nwith its line break.".into(), + } +} + +async fn prepare(ledger: &DriverRunLedger) { + ledger + .upsert_agent_run(run("parent", None, Some("root"))) + .await + .unwrap(); + ledger + .upsert_agent_run(run("child", Some("parent"), Some("worker-thread"))) + .await + .unwrap(); +} + +#[tokio::test] +async fn duplicate_admission_survives_handle_recreation_and_records_one_event() { + let ledger = ledger(); + prepare(&ledger).await; + let message = input("submission-1"); + let first = ledger.enqueue_input(message.clone()).await.unwrap(); + let reopened = DriverRunLedger::new(ledger.scoped.clone()); + assert_eq!( + reopened.enqueue_input(message.clone()).await.unwrap(), + first + ); + assert_eq!( + reopened.pending_inputs("root", "child", 20).await.unwrap(), + vec![first] + ); + let events = reopened + .list_recent_run_events(&RunEventListRequest { + run_id: "child".into(), + after_sequence: None, + limit: None, + }) + .await + .unwrap(); + assert_eq!(events.events.len(), 1); + assert_eq!(events.events[0].event_type, "input_queued"); + let mut changed = message; + changed.text = "Different content".into(); + assert!(reopened.enqueue_input(changed).await.is_err()); +} + +#[tokio::test] +async fn concurrent_duplicate_admissions_share_one_receipt_and_preserve_order() { + let ledger = ledger(); + prepare(&ledger).await; + let (left, right) = tokio::join!( + ledger.enqueue_input(input("same")), + ledger.enqueue_input(input("same")) + ); + assert_eq!(left.unwrap(), right.unwrap()); + ledger.enqueue_input(input("second")).await.unwrap(); + let queued = ledger.pending_inputs("root", "child", 20).await.unwrap(); + assert_eq!(queued.len(), 2); + assert!(queued[0].sequence < queued[1].sequence); + assert_eq!(queued[0].input.submission_id, "same"); + assert_eq!(queued[1].input.submission_id, "second"); +} + +#[tokio::test] +async fn cross_root_cyclic_missing_and_blank_inputs_are_rejected() { + let ledger = ledger(); + prepare(&ledger).await; + let mut wrong_root = input("wrong"); + wrong_root.root_thread_id = "other".into(); + assert!(ledger.enqueue_input(wrong_root).await.is_err()); + assert!(ledger.pending_inputs("other", "child", 20).await.is_err()); + let mut missing = input("missing"); + missing.recipient_run_id = "absent".into(); + assert!(ledger.enqueue_input(missing).await.is_err()); + let mut blank = input("blank"); + blank.text = " \n ".into(); + assert!(ledger.enqueue_input(blank).await.is_err()); + ledger + .upsert_agent_run(run("parent", Some("child"), Some("root"))) + .await + .unwrap(); + assert!(ledger.enqueue_input(input("cycle")).await.is_err()); +} diff --git a/crates/tinyagents-session/src/run_ledger/drivers/mod.rs b/crates/tinyagents-session/src/run_ledger/drivers/mod.rs index 3423c6ec5..4b8211bb9 100644 --- a/crates/tinyagents-session/src/run_ledger/drivers/mod.rs +++ b/crates/tinyagents-session/src/run_ledger/drivers/mod.rs @@ -11,6 +11,7 @@ mod effects; mod events; mod history; mod history_retention; +mod inbox; #[cfg(feature = "legacy-import")] mod import; #[cfg(feature = "legacy-import")] diff --git a/crates/tinyagents-session/src/run_ledger/drivers/schema.rs b/crates/tinyagents-session/src/run_ledger/drivers/schema.rs index 3241b4a66..148ca3758 100644 --- a/crates/tinyagents-session/src/run_ledger/drivers/schema.rs +++ b/crates/tinyagents-session/src/run_ledger/drivers/schema.rs @@ -22,7 +22,7 @@ //! and the next sequence of a migrated run is recomputed from its events. use super::storage::{ - EVENT_STREAM, HISTORY_OWNER, LEDGER_OWNER, SCHEMA_VERSION, history_specs, ledger_specs, + EVENT_STREAM, HISTORY_OWNER, INPUTS, LEDGER_OWNER, SCHEMA_VERSION, history_specs, ledger_specs, }; use tinystoragedrivers_core::{Owner, Schema, ScopeClass}; @@ -33,7 +33,14 @@ pub fn ledger_storage_schema() -> Schema { let owner = Owner::new(LEDGER_OWNER).expect("`ledger` is a valid owner"); let mut schema = Schema::new(owner); for spec in ledger_specs() { - schema = schema.collection_with(SCHEMA_VERSION, spec, ScopeClass::Agent); + // Inputs are new, with no legacy v1 envelope to migrate. Their initial + // schema is version one; declaring v2 would demand a nonexistent step. + let version = if spec.name == INPUTS { + 1 + } else { + SCHEMA_VERSION + }; + schema = schema.collection_with(version, spec, ScopeClass::Agent); } schema = schema.stream(EVENT_STREAM, ScopeClass::Agent); for step in super::steps::steps() { diff --git a/crates/tinyagents-session/src/run_ledger/drivers/schema_tests.rs b/crates/tinyagents-session/src/run_ledger/drivers/schema_tests.rs index 8a7d62697..98c500bfe 100644 --- a/crates/tinyagents-session/src/run_ledger/drivers/schema_tests.rs +++ b/crates/tinyagents-session/src/run_ledger/drivers/schema_tests.rs @@ -100,6 +100,7 @@ fn the_schemas_name_what_each_owner_declares() { "ledger.event_heads", "ledger.event_outbox", "ledger.events", + "ledger.inputs", "ledger.members", "ledger.tasks", "ledger.teams", diff --git a/crates/tinyagents-session/src/run_ledger/drivers/storage.rs b/crates/tinyagents-session/src/run_ledger/drivers/storage.rs index 737514cb6..b6e1209f6 100644 --- a/crates/tinyagents-session/src/run_ledger/drivers/storage.rs +++ b/crates/tinyagents-session/src/run_ledger/drivers/storage.rs @@ -33,6 +33,7 @@ pub(super) const EFFECTS: &str = "ledger.tool_effects"; pub(super) const EVENTS: &str = "ledger.event_outbox"; pub(super) const EVENT_HEADS: &str = "ledger.event_heads"; pub(super) const TASKS: &str = "ledger.tasks"; +pub(super) const INPUTS: &str = "ledger.inputs"; /// The per-run event stream family: `ledger.events/`. pub(super) const EVENT_STREAM: &str = "ledger.events"; pub(super) const HIST_SESSIONS: &str = "history.sessions"; @@ -81,6 +82,9 @@ pub(super) fn eq_nonblank(filter: Filter, field: &str, value: Option<&str>) -> F /// metadata, so none declares a TTL. pub(super) fn ledger_specs() -> Vec { vec![ + CollectionSpec::new(INPUTS) + .index(IndexSpec::new("recipient_sequence", ["recipient_run_id", "sequence"]).unique()) + .index(IndexSpec::new("root", ["root_thread_id"])), CollectionSpec::new(EVENT_HEADS), CollectionSpec::new(EVENTS) .index(IndexSpec::new("run_sequence", ["run_id", "sequence"]).unique()), diff --git a/crates/tinyagents-session/src/run_ledger/inbox.rs b/crates/tinyagents-session/src/run_ledger/inbox.rs new file mode 100644 index 000000000..7a582b01f --- /dev/null +++ b/crates/tinyagents-session/src/run_ledger/inbox.rs @@ -0,0 +1,63 @@ +//! Durable addressed input admission, independent of a live worker registry. +//! +//! A receipt means the input is durably queued, not that a model has read it. +//! Recipients must commit input to their own durable session before acknowledging +//! delivery. This admission API deliberately exposes no ledger-only "delivered" +//! mutation. A host must not substitute live steering for durable admission. + +use serde::{Deserialize, Serialize}; + +/// Intended delivery boundary in the recipient's existing session. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum InputMode { + /// Admit at the next safe boundary of running work. + Steer, + /// Queue another turn after the current work. + FollowUp, + /// Answer the recipient's pending question. + Answer, +} + +/// Operator input addressed to one exact durable run under a root conversation. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AddressedInput { + /// Unique within the root conversation; retry the same payload with this id. + pub submission_id: String, + /// Root conversation whose persisted lineage authorizes the target. + pub root_thread_id: String, + /// Exact run-ledger identity, never an agent definition or display name. + pub recipient_run_id: String, + /// Required recipient execution boundary. + pub mode: InputMode, + /// Original user text; not trimmed or rewritten after admission. + pub text: String, +} + +/// Durable queued input. Repeated admission returns the original receipt. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct QueuedInput { + /// Original addressed input. + pub input: AddressedInput, + /// Recipient event sequence allocated atomically with admission. + pub sequence: u64, + /// Admission time at storage millisecond precision. + pub created_at: chrono::DateTime, +} + +/// Optional durable inbox capability of an already scoped ledger handle. +#[async_trait::async_trait] +pub trait RunInboxStore: Send + Sync { + /// Validate persisted root lineage and atomically insert input plus receipt. + /// Reusing an id for different content is an error, never another delivery. + async fn enqueue_input(&self, input: AddressedInput) -> super::LedgerResult; + + /// Read queued input in admission order after validating root ownership. + /// Limit is clamped to 1..=500; this never consumes or acknowledges input. + async fn pending_inputs( + &self, + root_thread_id: &str, + recipient_run_id: &str, + limit: u32, + ) -> super::LedgerResult>; +} diff --git a/crates/tinyagents-session/src/run_ledger/mod.rs b/crates/tinyagents-session/src/run_ledger/mod.rs index 2ff520478..bb30bd32d 100644 --- a/crates/tinyagents-session/src/run_ledger/mod.rs +++ b/crates/tinyagents-session/src/run_ledger/mod.rs @@ -13,6 +13,7 @@ pub mod command_center; pub mod drivers; pub mod error; pub mod history; +pub mod inbox; pub mod port; pub mod recovery; mod status_map; From 266ab91419ed55f111aed97313ba1d0e52f7d010 Mon Sep 17 00:00:00 2001 From: Farnam Date: Fri, 23 Oct 2026 23:00:30 +0330 Subject: [PATCH 09/14] fix(harness): preserve classified stop causes in OpenHuman integration Backport the owner fix proposed in tinyhumansai/tinyagents#571 so host finish_run cleanup cannot erase AgentRun.failure_cause. Keep an existing cause authoritative and retain latch cleanup. Source-commit: 83b46043d02cb491d6fb255dd9fe07544bf0527b Source-PR: https://github.com/tinyhumansai/tinyagents/pull/571 --- .../src/middleware/library/README.md | 3 + .../src/middleware/library/failure_scope.rs | 16 ++- .../library/failure_scope_finish_tests.rs | 98 +++++++++++++++++++ 3 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 crates/tinyagents-harness/src/middleware/library/failure_scope_finish_tests.rs diff --git a/crates/tinyagents-harness/src/middleware/library/README.md b/crates/tinyagents-harness/src/middleware/library/README.md index 06549e233..26df7bcf4 100644 --- a/crates/tinyagents-harness/src/middleware/library/README.md +++ b/crates/tinyagents-harness/src/middleware/library/README.md @@ -116,6 +116,9 @@ Grouped by extension shape: call-id map. It owns per-run no-progress, classified-recovery, and recoverable failure trackers, typed halts, and one-shot request nudges. Host policy also supplies lifecycle callbacks for stop-latch batch/refusal/progress evidence. + Before `finish_run` clears host evidence, `after_agent` preserves a current + classified halt cause on `AgentRun::failure_cause`, retaining any cause the + loop already recorded. Construct a **new driver per run** because its steering handle and halt-summary slot are run specific. - `run_output_ladder` — the shared compact, persist-or-cap, disclose sequence diff --git a/crates/tinyagents-harness/src/middleware/library/failure_scope.rs b/crates/tinyagents-harness/src/middleware/library/failure_scope.rs index f5d6f328c..c263ddd3c 100644 --- a/crates/tinyagents-harness/src/middleware/library/failure_scope.rs +++ b/crates/tinyagents-harness/src/middleware/library/failure_scope.rs @@ -211,7 +211,8 @@ pub trait FailureScopePolicy: FailurePolicy { } /// Called when the run ends or errors so a host stop-latch adapter can - /// discard its run-specific evidence. + /// discard its run-specific evidence. On `after_agent`, the driver copies + /// any current classified cause into the run before calling this hook. fn finish_run(&self, _ctx: &RunContext) {} } @@ -599,8 +600,15 @@ impl Middleware<(), Ctx> for FailureScopeMiddleware { &self, ctx: &mut RunContext, _state: &(), - _run: &mut crate::middleware::AgentRun, + run: &mut crate::middleware::AgentRun, ) -> Result<()> { + // A host callback may have latched the cause after the loop last + // observed it. Capture it while the policy still owns that evidence; + // finish_run is allowed to clear the latch. Keep a cause already + // recorded by the loop or another middleware authoritative. + if run.failure_cause.is_none() { + run.failure_cause = self.policy.current_halt_cause(ctx); + } self.policy.finish_run(ctx); self.clear_run(ctx.instance_id()); Ok(()) @@ -620,3 +628,7 @@ impl Middleware<(), Ctx> for FailureScopeMiddleware { #[cfg(test)] #[path = "failure_scope_tests.rs"] mod tests; + +#[cfg(test)] +#[path = "failure_scope_finish_tests.rs"] +mod finish_tests; diff --git a/crates/tinyagents-harness/src/middleware/library/failure_scope_finish_tests.rs b/crates/tinyagents-harness/src/middleware/library/failure_scope_finish_tests.rs new file mode 100644 index 000000000..a7f60199d --- /dev/null +++ b/crates/tinyagents-harness/src/middleware/library/failure_scope_finish_tests.rs @@ -0,0 +1,98 @@ +use super::*; + +use crate::context::RunConfig; +use crate::middleware::AgentRun; +use crate::no_progress::ClassifiedStopCause; + +#[derive(Default)] +struct LatchPolicy { + cause: Mutex>, +} + +impl FailurePolicy for LatchPolicy { + fn identical_halt_summary(&self, tool: &str, count: u32, _failure_text: &str) -> String { + format!("{tool} failed {count} times") + } + + fn no_progress_halt_summary(&self, count: u32, tool: &str, _failure_text: &str) -> String { + format!("{tool} made no progress in {count} calls") + } +} + +impl FailureScopePolicy<()> for LatchPolicy { + fn assess( + &self, + _ctx: &RunContext<()>, + _invocation: &ToolInvocationIdentity, + _call: &ToolCall, + _scope: &FailureScope, + _result: &ToolResult, + ) -> FailureAssessment { + FailureAssessment::Ignore + } + + fn current_halt_cause(&self, _ctx: &RunContext<()>) -> Option { + self.cause.lock().unwrap().clone() + } + + fn finish_run(&self, _ctx: &RunContext<()>) { + *self.cause.lock().unwrap() = None; + } +} + +fn cause(class: &str) -> ClassifiedStopCause { + ClassifiedStopCause { + class: class.into(), + operation: "send_message".into(), + summary: format!("Stopped after {class}"), + } +} + +async fn finish( + latched: Option, + recorded: Option, +) -> AgentRun { + let policy = Arc::new(LatchPolicy { + cause: Mutex::new(latched), + }); + let middleware = FailureScopeMiddleware::new( + SteeringHandle::allow_all(), + Arc::new(Mutex::new(None)), + policy.clone(), + ); + let mut ctx = RunContext::new(RunConfig::new("finish-cause"), ()); + let mut run = AgentRun { + failure_cause: recorded, + ..AgentRun::default() + }; + + middleware + .after_agent(&mut ctx, &(), &mut run) + .await + .unwrap(); + + assert!( + policy.cause.lock().unwrap().is_none(), + "finish must still discard the host's per-run latch" + ); + run +} + +#[tokio::test] +async fn classified_cause_survives_host_latch_cleanup() { + let expected = cause("uncertain_side_effect"); + let run = finish(Some(expected.clone()), None).await; + assert_eq!(run.failure_cause, Some(expected)); +} + +#[tokio::test] +async fn finish_preserves_an_already_recorded_cause() { + let recorded = cause("provider_out_of_budget"); + let run = finish(Some(cause("uncertain_side_effect")), Some(recorded.clone())).await; + assert_eq!(run.failure_cause, Some(recorded)); +} + +#[tokio::test] +async fn unclassified_finish_does_not_invent_a_cause() { + assert_eq!(finish(None, None).await.failure_cause, None); +} From ca5c45f8c0e2136098df776ea2f964bd1345aecf Mon Sep 17 00:00:00 2001 From: Farnam Date: Mon, 26 Oct 2026 22:39:08 +0330 Subject: [PATCH 10/14] fix(harness): own structural execution and update storage adapters --- Cargo.lock | 157 +++++++++- crates/tinyagents-harness-tools/Cargo.toml | 5 + crates/tinyagents-harness-tools/README.md | 8 + .../src/code/README.md | 161 ++++++++++ .../src/code/arguments.rs | 155 ++++++++++ .../src/code/chunk_tests.rs | 255 ++++++++++++++++ .../src/code/chunks.rs | 95 ++++++ .../src/code/contract_tests.rs | 175 +++++++++++ .../src/code/declaration.rs | 229 ++++++++++++++ .../src/code/inspect.rs | 132 ++++++++ .../src/code/inspection_tests.rs | 282 +++++++++++++++++ .../tinyagents-harness-tools/src/code/io.rs | 126 ++++++++ .../tinyagents-harness-tools/src/code/mod.rs | 85 ++++++ .../src/code/mutate.rs | 248 +++++++++++++++ .../src/code/mutation_tests.rs | 287 ++++++++++++++++++ .../src/code/publication_tests.rs | 225 ++++++++++++++ .../src/code/publish.rs | 133 ++++++++ .../src/code/response.rs | 57 ++++ .../src/code/test_support.rs | 204 +++++++++++++ .../src/code/types.rs | 110 +++++++ crates/tinyagents-harness-tools/src/lib.rs | 5 + .../tinyagents-harness/src/run_queue/mod.rs | 6 +- .../src/run_ledger/drivers/inbox.rs | 2 +- .../src/run_ledger/drivers/inbox_tests.rs | 2 +- vendor/tinytools | 2 +- 25 files changed, 3132 insertions(+), 14 deletions(-) create mode 100644 crates/tinyagents-harness-tools/src/code/README.md create mode 100644 crates/tinyagents-harness-tools/src/code/arguments.rs create mode 100644 crates/tinyagents-harness-tools/src/code/chunk_tests.rs create mode 100644 crates/tinyagents-harness-tools/src/code/chunks.rs create mode 100644 crates/tinyagents-harness-tools/src/code/contract_tests.rs create mode 100644 crates/tinyagents-harness-tools/src/code/declaration.rs create mode 100644 crates/tinyagents-harness-tools/src/code/inspect.rs create mode 100644 crates/tinyagents-harness-tools/src/code/inspection_tests.rs create mode 100644 crates/tinyagents-harness-tools/src/code/io.rs create mode 100644 crates/tinyagents-harness-tools/src/code/mod.rs create mode 100644 crates/tinyagents-harness-tools/src/code/mutate.rs create mode 100644 crates/tinyagents-harness-tools/src/code/mutation_tests.rs create mode 100644 crates/tinyagents-harness-tools/src/code/publication_tests.rs create mode 100644 crates/tinyagents-harness-tools/src/code/publish.rs create mode 100644 crates/tinyagents-harness-tools/src/code/response.rs create mode 100644 crates/tinyagents-harness-tools/src/code/test_support.rs create mode 100644 crates/tinyagents-harness-tools/src/code/types.rs diff --git a/Cargo.lock b/Cargo.lock index 3313c5266..52634fc24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,6 +47,12 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + [[package]] name = "async-trait" version = "0.1.92" @@ -106,6 +112,19 @@ dependencies = [ "wyz", ] +[[package]] +name = "blake3" +version = "1.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" +dependencies = [ + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.1", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -156,7 +175,7 @@ dependencies = [ "maybe-owned", "rustix", "rustix-linux-procfs", - "windows-sys 0.60.2", + "windows-sys 0.61.2", "winx", ] @@ -237,6 +256,12 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -351,7 +376,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -384,7 +409,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -1327,7 +1352,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -1567,7 +1592,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -1679,6 +1704,8 @@ version = "1.0.154" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" dependencies = [ + "foldhash", + "indexmap", "itoa", "memchr", "serde", @@ -1796,7 +1823,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1817,6 +1844,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" + [[package]] name = "subtle" version = "2.6.1" @@ -1892,7 +1925,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2002,6 +2035,7 @@ dependencies = [ "tinyagents-tasks", "tinyinference-llm", "tinytools", + "tinytools-code", "tokio", "tracing", "uuid", @@ -2551,6 +2585,23 @@ dependencies = [ "tracing", ] +[[package]] +name = "tinytools-code" +version = "0.6.0" +dependencies = [ + "blake3", + "serde", + "serde_json", + "tree-sitter", + "tree-sitter-go", + "tree-sitter-javascript", + "tree-sitter-json", + "tree-sitter-python", + "tree-sitter-rust", + "tree-sitter-toml-ng", + "tree-sitter-typescript", +] + [[package]] name = "tinyvec" version = "1.13.3" @@ -2780,6 +2831,96 @@ dependencies = [ "once_cell", ] +[[package]] +name = "tree-sitter" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78f873475d258561b06f1c595d93308a7ed124d9977cb26b148c2084a4a3cc87" +dependencies = [ + "cc", + "regex", + "regex-syntax", + "serde_json", + "streaming-iterator", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-go" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8560a4d2f835cc0d4d2c2e03cbd0dde2f6114b43bc491164238d333e28b16ea" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-javascript" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68204f2abc0627a90bdf06e605f5c470aa26fdcb2081ea553a04bdad756693f5" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-json" +version = "0.24.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d727acca406c0020cffc6cf35516764f36c8e3dc4408e5ebe2cb35a947ec471" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-language" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "009994f150cc0cd50ff54917d5bc8bffe8cad10ca10d81c34da2ec421ae61782" + +[[package]] +name = "tree-sitter-python" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bf85fd39652e740bf60f46f4cda9492c3a9ad75880575bf14960f775cb74a1c" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-rust" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439e577dbe07423ec2582ac62c7531120dbfccfa6e5f92406f93dd271a120e45" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-toml-ng" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9adc2c898ae49730e857d75be403da3f92bb81d8e37a2f918a08dd10de5ebb1" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-typescript" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5f76ed8d947a75cc446d5fccd8b602ebf0cde64ccf2ffa434d873d7a575eff" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -3092,7 +3233,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/crates/tinyagents-harness-tools/Cargo.toml b/crates/tinyagents-harness-tools/Cargo.toml index 73828e783..2e3351663 100644 --- a/crates/tinyagents-harness-tools/Cargo.toml +++ b/crates/tinyagents-harness-tools/Cargo.toml @@ -28,6 +28,10 @@ tinyagents-harness = { path = "../tinyagents-harness", version = "3.0.0", defaul tinyagents-tasks = { path = "../tinyagents-tasks", version = "3.0.0", default-features = false, optional = true } tinyinference-llm = { path = "../../vendor/tinyinference/crates/tinyinference-llm", version = "0.5.0", optional = true } tinytools = { path = "../../vendor/tinytools/crates/tinytools", version = "0.6.0" } +# Pure structural parsing; native code-tool I/O uses this crate's filesystem gate. +tinytools-code = { path = "../../vendor/tinytools/crates/tinytools-code", version = "0.6.0", optional = true } +# Same-directory staging for atomic structural edits. +tempfile = { workspace = true, optional = true } tinyagents-orchestration = { path = "../tinyagents-orchestration", version = "3.0.0", default-features = false, optional = true } tokio = { workspace = true, features = ["sync", "time", "macros", "rt", "rt-multi-thread"] } tracing = { workspace = true } @@ -48,6 +52,7 @@ skills = [] state-block = ["goals", "todos", "dep:tinyinference-llm"] subagents = ["dep:tinyagents-orchestration", "dep:tinyagents-tasks", "dep:tinyinference-llm"] fs = ["dep:base64", "dep:fs2", "dep:glob", "dep:libc", "dep:log", "dep:regex", "dep:walkdir", "tokio/fs", "tokio/io-util"] +code = ["fs", "dep:tinytools-code", "dep:tempfile"] net = ["dep:futures-util", "dep:log", "dep:reqwest", "dep:sha2", "tokio/fs", "tokio/io-util"] exec = ["dep:rustix", "tokio/fs", "tokio/io-util", "tokio/process"] media = [] diff --git a/crates/tinyagents-harness-tools/README.md b/crates/tinyagents-harness-tools/README.md index c8bbc1000..b718e2f80 100644 --- a/crates/tinyagents-harness-tools/README.md +++ b/crates/tinyagents-harness-tools/README.md @@ -20,6 +20,12 @@ the generic artifact-filing wrapper at `tinyagents_harness_tools::media`, and `tinyagents_harness_tools::plan_review`. These features do not change the default set. +The opt-in `code` feature enables `fs` and exposes six deferred structural tools +through `code_tools` / `code_tools_with_options`. They use the pure +`tinytools-code` parser and this crate's existing `fs::filesystem::FsGate` and +`fs::file_state`, so structural writes share the host's filesystem policy and +stale-read tracking. See the [structural tool contract](src/code/README.md). + ## Public modules and tools - `goals` provides `GoalStore`, `GoalTool`, observer notifications, atomic @@ -39,6 +45,8 @@ default set. `tinyagents_orchestration::subagent::SubAgentJobRegistry`. - `thread_locks` provides the public `ThreadLockMap` concurrency primitive. - `fs` provides filesystem tools and file-state operations in `fs`. +- `code` provides bounded syntax inspection and revision-checked structural + edits behind the same filesystem gate. - `net` provides network tools in `net`. - `exec` provides `ExecTool`, `InstallTool`, process timeout utilities, and host-supplied execution ports in `exec`. diff --git a/crates/tinyagents-harness-tools/src/code/README.md b/crates/tinyagents-harness-tools/src/code/README.md new file mode 100644 index 000000000..30b01624d --- /dev/null +++ b/crates/tinyagents-harness-tools/src/code/README.md @@ -0,0 +1,161 @@ +# Structural code tools + +Enable the `code` Cargo feature and call `tinyagents_harness_tools::code_tools(gate)` to +obtain six `Box` values. `code_tools_with_options` accepts +`CodeToolOptions` when a host needs tighter resource limits. Construction is +fallible because options and the bundled catalog are validated. + +The tools use the pure `tinytools-code` engine for parsing and rendering. This +module owns native filesystem mechanics; the supplied `FsGate` owns policy. +Neither module executes generated code, installs declared dependencies, resolves +semantic types, or treats a syntax check as a successful build or test run. + +## Availability and authorization + +All six tools are deferred, belong to family `code`, and carry tags `code`, +`structural`, and `pack:code`. Every static minimum permission is `ReadOnly` so +hosts can retain preview access. `ast_patch` and `chunk_apply` require `Write` +unless the invocation contains the literal JSON boolean `dry_run: true`. +Missing, false, null, string and numeric flags never lower the declaration. +Nonboolean flags fail strict decoding before any filesystem mutation. + +Write calls consult the original gate's autonomy and action budget. The host's +`write_needs_approval` controls external-effect routing. Previews neither prompt +as writes nor consume actions. A trusted `ToolRunContext` workspace descriptor +scopes path validation per call without changing the original gate or moving +its accounting into a separate policy object. + +## Arguments + +Unknown fields are rejected. `language` can explicitly select `rust`, +`typescript`, `tsx`, `javascript`, `python`, `go`, `json`, or `toml`; otherwise +inspection and patch tools infer it from a supported filename extension. + +| Tool | Required arguments | Optional arguments | +| --- | --- | --- | +| `ast_outline` | `path` | `language`, `level`, `offset`, `limit`, `start_line`, `line_count` | +| `ast_symbol` | `path`, `selector` | `language`, `start_line`, `line_count` | +| `ast_patch` | `path`, `expected_revision`, `operation`, `code` | `language`, `selector`, `dry_run`, `start_line`, `line_count` | +| `chunk_search` | None | `query`, `language`, `offset`, `limit` | +| `chunk_inspect` | Exactly one of `id` or `manifest_path` | `offset`, `limit` | +| `chunk_apply` | `path`, `chunk_revision`, exactly one of `id` or `manifest_path`, plus the destination guard below | `bindings`, `selector`, `dry_run`, `start_line`, `line_count` | + +`selector` accepts a revision-bound symbol ID, an unambiguous qualified name, +or an unambiguous local name. A truncated symbol collection only permits IDs. +The patch operations are `replace_body`, `replace_symbol`, `insert_import`, +`append_member`, and `wrap`. All except `insert_import` need a selector; a wrap +uses exactly one `{{node}}` marker. Source revisions are 64 lowercase hexadecimal +digits returned by the engine, not Git commit hashes. + +For `chunk_apply`, edit an existing file with `expected_revision`, or create a +new file with `create_new: true` and no `expected_revision`. Creation refuses any +existing target and requires its parent directory to exist. A complete-file +chunk replaces the complete file and accepts no selector. Function-body and +member chunks require an existing file and a selector, and use the same pure +patch engine as `ast_patch`. `bindings` has two optional objects: `parameters` +for typed values and `slots` for supplied code strings. + +`chunk_search` searches the twelve bundled patterns. To use a custom pattern, +inspect its `manifest_path` and pass that same path to `chunk_apply`. The +manifest's template path must be portable and relative to the manifest's +directory. Both files pass the gate, and the resolved template must remain +inside that directory, including after symlink resolution. A returned +`chunk_revision` binds the exact manifest and template contents. Reformatting a +custom manifest changes this content identity even when its normalized +`manifest_revision` remains the same. + +## Receipts and paging + +Successful results contain a JSON block. List pages use this shape: + +```json +{"items": [], "offset": 0, "total": 0, "next_offset": null} +``` + +`offset` is zero-based. `limit` is positive and bounded by `max_page_items`. +`chunk_search` defaults to five results, or the configured `max_page_items` if +that is smaller; an explicit `limit` can select any positive size up to that +bound. Other record lists default to `max_page_items`. +Outline symbols, references and diagnostics each have their own page object. +Their separate truncation flags disclose parser collection limits; a page's +`total` counts retained records, not records beyond those limits. + +`ast_outline` accepts `level: 0` for metadata only, with no `skeleton` field. +The default `level: 1` includes a folded source page alongside that metadata. +Other levels and noninteger values are rejected before reading the file. + +Source and folded-skeleton pages use this shape: + +```json +{"text": "fn example() {}\n", "start_line": 1, "total_lines": 1, + "next_start_line": null, "truncated": false} +``` + +`start_line` is one-based, relative to the returned source selection: the file +for mutation receipts, the folded source for outlines, or the declaration for +`ast_symbol`. Symbol ranges independently preserve absolute source coordinates. +`line_count` must be positive and no greater than `max_page_lines`. A single +oversized line can exceed the output budget even on a one-line page; that call +returns a bounded error instead of silently dropping content. + +Mutation receipts include `path`, `language`, `before_revision` (null for +creation), `revision`, `changed`, `written`, `dry_run`, and a source page. Chunk +receipts also include `chunk_id`, `chunk_revision`, `rendered_revision`, and +informational `dependencies`. `written: true` means publication succeeded. +An unchanged result reports `written: false` and consumes no action. + +For example, preview a body edit after reading the current revision: + +```json +{"path":"src/lib.rs","expected_revision":"<64 lowercase hex digits>", + "selector":"calculate","operation":"replace_body","code":"42", + "dry_run":true} +``` + +## Resource bounds + +| `CodeToolOptions` field | Default | +| --- | --- | +| `limits` | `CodeLimits::default()`; 1 MiB source, 32 cached entries, 8 MiB retained source | +| `max_argument_bytes` | 1 MiB | +| `max_manifest_bytes` | 64 KiB | +| `max_template_bytes` | 256 KiB | +| `max_output_bytes` | 64 KiB | +| `max_page_items` | 32 | +| `max_page_lines` | 80 | + +Argument validation counts serialized UTF-8 and JSON escape bytes with a writer +that stops at the limit, without allocating a second complete argument buffer. +Native inputs must be regular UTF-8 files. Reads take at most the configured +bound plus one byte so a file growing during a read cannot cause an unbounded +allocation. The output bound measures the actual pretty-printed model-visible +JSON. Errors are truncated only at UTF-8 boundaries. The complete success +receipt is checked before any staging file or action-budget consumption. + +## Publication and concurrency + +The tools share a mutation mutex and participate in `file_state`'s path lock +when that coordinator is enabled. Successful publications record the current +file-state agent's write. Exact source revisions remain required regardless of +whether the optional coordinator is enabled. + +Edits stage a complete file in the destination's validated directory. Existing +permission bits are retained; new files use the temporary-file library's +private creation permissions. Before publication, the adapter reauthorizes the +original destination path and rechecks target, manifest and template revisions. +The original gate's write authority is checked after action reservation and +again after the final asynchronous path and revision checks; the reserved +action is charged only once. +Creation uses no-clobber publication, and replacement uses an atomic rename. +Failed checks and cancellation drop the staged file, preserving the target. +The final atomic publication is a short synchronous operation: cancellation +cannot expose a partially written destination. + +These checks coordinate cooperating tools and detect changes observed before +publication. They do not claim an operating-system compare-and-swap against an +unrelated writer that changes a file between the last revision check and rename, +or power-loss durability of a directory entry. Filesystem confinement remains +the host gate's responsibility. + +See the accepted [specification](../../../../docs/specs/structural-code.md) and +[implementation plan](../../../../docs/plans/structural-code.md). diff --git a/crates/tinyagents-harness-tools/src/code/arguments.rs b/crates/tinyagents-harness-tools/src/code/arguments.rs new file mode 100644 index 000000000..7a0fae31f --- /dev/null +++ b/crates/tinyagents-harness-tools/src/code/arguments.rs @@ -0,0 +1,155 @@ +//! Strict argument packets; malformed preview flags never become writes. + +use anyhow::{Context, ensure}; +use serde::{Deserialize, de::DeserializeOwned}; +use serde_json::Value; +use tinytools_code::{ChunkBindings, Language, PatchOperation}; + +/// Stop serialization at the bound instead of copying an oversized value. +pub(super) fn check_size(args: &Value, maximum: usize) -> anyhow::Result<()> { + serde_json::to_writer(ArgumentBudget { remaining: maximum }, args) + .context("structural tool arguments exceed the byte budget") +} + +struct ArgumentBudget { + remaining: usize, +} + +impl std::io::Write for ArgumentBudget { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.remaining = self + .remaining + .checked_sub(bytes.len()) + .ok_or_else(|| std::io::Error::other("argument byte budget exhausted"))?; + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +pub(super) fn decode(args: Value) -> anyhow::Result { + serde_json::from_value(args).context("invalid structural tool arguments") +} + +pub(super) fn language(path: &str, explicit: Option) -> anyhow::Result { + explicit + .or_else(|| Language::from_path(path)) + .context("unknown filename extension; supply an explicit supported language") +} + +pub(super) fn literal_preview(args: &Value) -> bool { + args.get("dry_run") == Some(&Value::Bool(true)) +} + +pub(super) fn page_size(requested: Option, maximum: usize) -> anyhow::Result { + let size = requested.unwrap_or(maximum); + ensure!( + size > 0 && size <= maximum, + "page size must be between 1 and {maximum}" + ); + Ok(size) +} + +pub(super) fn require_revision(value: &str) -> anyhow::Result<()> { + ensure!( + value.len() == 64 + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)), + "expected revision must contain 64 lowercase hexadecimal digits" + ); + Ok(()) +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Outline { + pub(super) path: String, + pub(super) language: Option, + #[serde(default = "outline_level")] + pub(super) level: u8, + #[serde(default)] + pub(super) offset: usize, + pub(super) limit: Option, + #[serde(default = "first_line")] + pub(super) start_line: usize, + pub(super) line_count: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Symbol { + pub(super) path: String, + pub(super) language: Option, + pub(super) selector: String, + #[serde(default = "first_line")] + pub(super) start_line: usize, + pub(super) line_count: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Patch { + pub(super) path: String, + pub(super) language: Option, + pub(super) expected_revision: String, + pub(super) selector: Option, + pub(super) operation: PatchOperation, + pub(super) code: String, + #[serde(default)] + pub(super) dry_run: bool, + #[serde(default = "first_line")] + pub(super) start_line: usize, + pub(super) line_count: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Search { + #[serde(default)] + pub(super) query: String, + pub(super) language: Option, + #[serde(default)] + pub(super) offset: usize, + pub(super) limit: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Inspect { + pub(super) id: Option, + pub(super) manifest_path: Option, + #[serde(default)] + pub(super) offset: usize, + pub(super) limit: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Apply { + pub(super) path: String, + pub(super) id: Option, + pub(super) manifest_path: Option, + pub(super) chunk_revision: String, + #[serde(default)] + pub(super) bindings: ChunkBindings, + pub(super) selector: Option, + pub(super) expected_revision: Option, + #[serde(default)] + pub(super) create_new: bool, + #[serde(default)] + pub(super) dry_run: bool, + #[serde(default = "first_line")] + pub(super) start_line: usize, + pub(super) line_count: Option, +} + +const fn first_line() -> usize { + 1 +} + +const fn outline_level() -> u8 { + 1 +} diff --git a/crates/tinyagents-harness-tools/src/code/chunk_tests.rs b/crates/tinyagents-harness-tools/src/code/chunk_tests.rs new file mode 100644 index 000000000..1e52c64a4 --- /dev/null +++ b/crates/tinyagents-harness-tools/src/code/chunk_tests.rs @@ -0,0 +1,255 @@ +//! Custom catalog provenance, native bounds and destination regressions. + +#![allow(clippy::unwrap_used)] + +use super::test_support::*; +use super::*; +use serde_json::json; +use std::sync::atomic::Ordering; +use tinytools::{ToolCallOptions, WorkspaceDescriptor}; + +#[tokio::test] +async fn a_custom_chunk_revision_binds_raw_metadata_and_template_content() { + let dir = tempfile::tempdir().unwrap(); + custom_manifest(dir.path(), "file", "fn created() {}\n"); + let tools = code_tools(TestGate::new(dir.path())).unwrap(); + let entry = tool(&tools, "chunk_inspect"); + let original = success(entry, json!({"manifest_path":"manifest.json"})).await; + let path = dir.path().join("manifest.json"); + std::fs::write( + &path, + format!("{}\n", std::fs::read_to_string(&path).unwrap()), + ) + .unwrap(); + let reformatted = success(entry, json!({"manifest_path":"manifest.json"})).await; + assert_ne!(original["chunk_revision"], reformatted["chunk_revision"]); + assert_eq!( + original["manifest_revision"], + reformatted["manifest_revision"] + ); + std::fs::write(dir.path().join("template.rs.txt"), "fn different() {}\n").unwrap(); + let edited = success(entry, json!({"manifest_path":"manifest.json"})).await; + assert_ne!(reformatted["chunk_revision"], edited["chunk_revision"]); + assert_eq!( + reformatted["manifest_revision"], + edited["manifest_revision"] + ); + assert!( + failure( + tool(&tools, "chunk_apply"), + json!({"manifest_path":"manifest.json", "path":"new.rs", "create_new":true, + "chunk_revision":original["chunk_revision"]}) + ) + .await + .contains("chunk revision") + ); + assert!(!dir.path().join("new.rs").exists()); +} + +#[tokio::test] +async fn malformed_manifests_and_nonportable_templates_fail_before_any_effect() { + let dir = tempfile::tempdir().unwrap(); + let gate = TestGate::new(dir.path()); + let tools = code_tools(gate.clone()).unwrap(); + for path in [ + "../outside.txt", + "/absolute.txt", + "dir\\template.txt", + "C:template.txt", + "a//b", + "", + ] { + std::fs::write(dir.path().join("manifest.json"), serde_json::to_vec(&json!({ + "id":"custom.pattern.v1", "summary":"Pattern", "language":"rust", "template_path":path + })).unwrap()).unwrap(); + assert!( + failure( + tool(&tools, "chunk_inspect"), + json!({"manifest_path":"manifest.json"}) + ) + .await + .contains("template_path must be a portable, relative path") + ); + } + std::fs::write(dir.path().join("manifest.json"), "{not json}").unwrap(); + assert!( + failure( + tool(&tools, "chunk_inspect"), + json!({"manifest_path":"manifest.json"}) + ) + .await + .contains("manifest") + ); + assert!( + failure( + tool(&tools, "chunk_inspect"), + json!({"id":"rust.fallible-function.v1","manifest_path":"manifest.json"}) + ) + .await + .contains("exactly one") + ); + assert_eq!(gate.state.used.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn manifest_and_template_reads_each_enforce_their_own_byte_budget() { + let dir = tempfile::tempdir().unwrap(); + custom_manifest(dir.path(), "file", "fn created() {}\n"); + let gate = TestGate::new(dir.path()); + let manifest_limited = code_tools_with_options( + gate.clone(), + CodeToolOptions { + max_manifest_bytes: 8, + ..Default::default() + }, + ) + .unwrap(); + assert!( + failure( + tool(&manifest_limited, "chunk_inspect"), + json!({"manifest_path":"manifest.json"}) + ) + .await + .contains("read budget") + ); + let template_limited = code_tools_with_options( + gate, + CodeToolOptions { + max_template_bytes: 8, + ..Default::default() + }, + ) + .unwrap(); + assert!( + failure( + tool(&template_limited, "chunk_inspect"), + json!({"manifest_path":"manifest.json"}) + ) + .await + .contains("read budget") + ); +} + +#[cfg(unix)] +#[tokio::test] +async fn a_template_symlink_cannot_escape_its_manifest_directory_inside_the_workspace() { + let dir = tempfile::tempdir().unwrap(); + let patterns = dir.path().join("patterns"); + std::fs::create_dir(&patterns).unwrap(); + custom_manifest(&patterns, "file", "fn original() {}\n"); + std::fs::remove_file(patterns.join("template.rs.txt")).unwrap(); + std::fs::write(dir.path().join("outside.rs"), "fn outside() {}\n").unwrap(); + std::os::unix::fs::symlink( + dir.path().join("outside.rs"), + patterns.join("template.rs.txt"), + ) + .unwrap(); + let tools = code_tools(TestGate::new(dir.path())).unwrap(); + assert!( + failure( + tool(&tools, "chunk_inspect"), + json!({"manifest_path":"patterns/manifest.json"}) + ) + .await + .contains("manifest directory") + ); +} + +#[tokio::test] +async fn missing_parent_directories_are_not_created_by_a_failed_chunk_publication() { + let dir = tempfile::tempdir().unwrap(); + let tools = code_tools(TestGate::new(dir.path())).unwrap(); + let inspected = success( + tool(&tools, "chunk_inspect"), + json!({"id":"rust.fallible-function.v1"}), + ) + .await; + assert!( + failure( + tool(&tools, "chunk_apply"), + json!({"id":"rust.fallible-function.v1", "path":"missing/new.rs", "create_new":true, + "chunk_revision":inspected["chunk_revision"]}) + ) + .await + .contains("parent directory") + ); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0); +} + +#[tokio::test] +async fn an_unchanged_patch_has_a_receipt_but_does_not_consume_an_action() { + let dir = tempfile::tempdir().unwrap(); + let source = "fn value() -> i32 { 1 }\n"; + std::fs::write(dir.path().join("source.rs"), source).unwrap(); + let gate = TestGate::new(dir.path()); + let tools = code_tools(gate.clone()).unwrap(); + let mut args = patch_args(source); + args["operation"] = json!("replace_symbol"); + args["code"] = json!(source.trim_end()); + let receipt = success(tool(&tools, "ast_patch"), args).await; + assert_eq!(receipt["changed"], false); + assert_eq!(receipt["written"], false); + assert_eq!(gate.state.used.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn chunk_receipt_overflow_never_creates_a_destination() { + let dir = tempfile::tempdir().unwrap(); + let gate = TestGate::new(dir.path()); + let tools = code_tools(gate.clone()).unwrap(); + let inspected = success( + tool(&tools, "chunk_inspect"), + json!({"id":"rust.fallible-function.v1"}), + ) + .await; + let bounded = code_tools_with_options( + gate.clone(), + CodeToolOptions { + max_output_bytes: 160, + ..Default::default() + }, + ) + .unwrap(); + assert!( + failure( + tool(&bounded, "chunk_apply"), + json!({"id":"rust.fallible-function.v1", "path":"new.rs", "create_new":true, + "chunk_revision":inspected["chunk_revision"]}) + ) + .await + .contains("output budget") + ); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0); + assert_eq!(gate.state.used.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn isolated_workspace_writes_land_in_the_child_and_charge_the_original_gate() { + let parent = tempfile::tempdir().unwrap(); + let child = tempfile::tempdir().unwrap(); + let source = "fn value() -> i32 { 1 }\n"; + std::fs::write(parent.path().join("source.rs"), source).unwrap(); + std::fs::write(child.path().join("source.rs"), source).unwrap(); + let gate = TestGate::new(parent.path()); + let tools = code_tools(gate.clone()).unwrap(); + let context = Workspace(WorkspaceDescriptor::new(child.path().to_path_buf())); + let result = tool(&tools, "ast_patch") + .execute_with_context( + patch_args(source), + ToolCallOptions::default(), + Some(&context), + ) + .await + .unwrap(); + assert!(!result.is_error, "{}", result.output()); + assert_eq!( + std::fs::read_to_string(parent.path().join("source.rs")).unwrap(), + source + ); + assert!( + std::fs::read_to_string(child.path().join("source.rs")) + .unwrap() + .contains('2') + ); + assert_eq!(gate.state.used.load(Ordering::SeqCst), 1); +} diff --git a/crates/tinyagents-harness-tools/src/code/chunks.rs b/crates/tinyagents-harness-tools/src/code/chunks.rs new file mode 100644 index 000000000..9619845de --- /dev/null +++ b/crates/tinyagents-harness-tools/src/code/chunks.rs @@ -0,0 +1,95 @@ +//! Content-bound chunk snapshots, including gated custom templates. + +use std::path::Path; +use std::sync::Arc; + +use anyhow::{Context, ensure}; +use tinytools_code::{ChunkManifest, revision}; + +use super::{io, types::CodeTool}; +use crate::fs::filesystem::FsGate; + +pub(super) struct LoadedChunk { + pub(super) manifest: ChunkManifest, + pub(super) template: String, + pub(super) revision: String, + pub(super) guards: Vec, +} + +impl CodeTool { + pub(super) async fn load_chunk( + &self, + gate: &Arc, + id: Option<&str>, + manifest_path: Option<&str>, + ) -> anyhow::Result { + ensure!( + id.is_some() != manifest_path.is_some(), + "provide exactly one of id or manifest_path" + ); + if let Some(id) = id { + let manifest = self.shared.catalog.manifest(id)?.clone(); + let template = self + .shared + .catalog + .builtin_source(id) + .context("bundled chunk has no template")? + .to_owned(); + let encoded = serde_json::to_string(&manifest)?; + return Ok(LoadedChunk { + manifest, + revision: chunk_revision(&encoded, &template), + template, + guards: Vec::new(), + }); + } + let original = manifest_path.context("missing custom manifest path")?; + let options = self.shared.options; + let snapshot = io::read(gate, original, options.max_manifest_bytes).await?; + let manifest: ChunkManifest = + serde_json::from_str(&snapshot.source).context("invalid custom chunk manifest")?; + manifest.validate()?; + let parent = snapshot + .guard + .resolved + .parent() + .context("manifest has no parent directory")?; + let template_path = relative_template(parent, &manifest.template_path)?; + let template = io::read( + gate, + template_path + .to_str() + .context("template path must be UTF-8")?, + options.max_template_bytes, + ) + .await?; + ensure!( + template.guard.resolved.starts_with(parent), + "custom template must remain within the manifest directory" + ); + Ok(LoadedChunk { + manifest, + revision: chunk_revision(&snapshot.source, &template.source), + template: template.source, + guards: vec![snapshot.guard, template.guard], + }) + } +} + +fn relative_template(parent: &Path, value: &str) -> anyhow::Result { + ensure!( + !value.is_empty() + && !value.contains(['\\', ':']) + && !value + .split('/') + .any(|part| part.is_empty() || part == "." || part == ".."), + "custom template path must be portable and relative to the manifest" + ); + Ok(parent.join(value)) +} + +fn chunk_revision(manifest: &str, template: &str) -> String { + // Hash each bounded component before combining so no delimiter ambiguity + // or accidental cross-component concatenation can preserve the identity. + revision(&format!("{}:{}", revision(manifest), revision(template))) +} diff --git a/crates/tinyagents-harness-tools/src/code/contract_tests.rs b/crates/tinyagents-harness-tools/src/code/contract_tests.rs new file mode 100644 index 000000000..4720549f1 --- /dev/null +++ b/crates/tinyagents-harness-tools/src/code/contract_tests.rs @@ -0,0 +1,175 @@ +//! Permission, exposure, schema and strict argument-boundary regressions. + +#![allow(clippy::unwrap_used)] + +use super::test_support::*; +use super::*; +use serde_json::{Value, json}; +use tinytools::{PermissionLevel, ToolExposure}; + +#[test] +fn deferred_tools_preserve_static_preview_access_and_classify_every_write_argument() { + let dir = tempfile::tempdir().unwrap(); + let tools = code_tools(TestGate::new(dir.path())).unwrap(); + assert_eq!( + tools.iter().map(|tool| tool.name()).collect::>(), + [ + "ast_outline", + "ast_symbol", + "ast_patch", + "chunk_search", + "chunk_inspect", + "chunk_apply" + ] + ); + for entry in &tools { + assert_eq!(entry.permission_level(), PermissionLevel::ReadOnly); + assert_eq!(entry.exposure(), ToolExposure::Deferred); + assert_eq!(entry.family(), Some("code")); + assert_eq!(entry.tags(), ["code", "structural", "pack:code"]); + assert!(entry.policy().classified); + let writes = ["ast_patch", "chunk_apply"].contains(&entry.name()); + assert_eq!(entry.policy().side_effects.writes_files, writes); + assert_eq!(entry.external_effect(), writes); + assert_eq!(entry.parameters_schema()["additionalProperties"], false); + for value in [ + Value::Null, + json!(false), + json!("true"), + json!(1), + json!({}), + ] { + let args = json!({"dry_run":value}); + assert_eq!( + entry.permission_level_with_args(&args), + if writes { + PermissionLevel::Write + } else { + PermissionLevel::ReadOnly + } + ); + assert_eq!(entry.external_effect_with_args(&args), writes); + assert_eq!(entry.is_concurrency_safe(&args), !writes); + } + assert_eq!( + entry.permission_level_with_args(&json!({"dry_run":true})), + PermissionLevel::ReadOnly + ); + assert!(!entry.external_effect_with_args(&json!({"dry_run":true}))); + assert!(entry.is_concurrency_safe(&json!({"dry_run":true}))); + } +} + +#[test] +fn rejects_invalid_host_bounds_before_constructing_tools() { + let dir = tempfile::tempdir().unwrap(); + let invalid = [ + CodeToolOptions { + max_argument_bytes: 0, + ..Default::default() + }, + CodeToolOptions { + max_manifest_bytes: 1_048_577, + ..Default::default() + }, + CodeToolOptions { + max_template_bytes: 0, + ..Default::default() + }, + CodeToolOptions { + max_output_bytes: 16_777_217, + ..Default::default() + }, + CodeToolOptions { + max_page_items: 0, + ..Default::default() + }, + CodeToolOptions { + max_page_lines: 4097, + ..Default::default() + }, + CodeToolOptions { + limits: tinytools_code::CodeLimits { + max_source_bytes: 0, + ..Default::default() + }, + ..Default::default() + }, + ]; + for options in invalid { + assert!(code_tools_with_options(TestGate::new(dir.path()), options).is_err()); + } +} + +#[tokio::test] +async fn malformed_or_oversized_arguments_fail_without_consuming_a_write_budget() { + let dir = tempfile::tempdir().unwrap(); + let gate = TestGate::new(dir.path()); + let tools = code_tools_with_options( + gate.clone(), + CodeToolOptions { + max_argument_bytes: 512, + max_output_bytes: 31, + ..Default::default() + }, + ) + .unwrap(); + for args in [ + json!(null), + json!({"path":"x","dry_run":"true"}), + json!({"unknown":"分析".repeat(300)}), + ] { + let result = tool(&tools, "ast_patch").execute(args).await.unwrap(); + assert!(result.is_error); + assert!(result.output().len() <= 31); + } + assert_eq!(gate.state.used.load(std::sync::atomic::Ordering::SeqCst), 0); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0); +} + +#[test] +fn output_pages_and_error_truncation_preserve_utf8_and_report_continuations() { + let error = response::bounded_error("分析abc", 4); + assert_eq!(error.output(), "分"); + let records = response::records(&[1, 2, 3], usize::MAX, 2); + assert_eq!(records["items"], json!([])); + assert_eq!(records["next_offset"], Value::Null); + let page = response::source_page("one\ntwo\nthree", 2, 1).unwrap(); + assert_eq!(page["text"], "two\n"); + assert_eq!(page["next_start_line"], 3); + assert_eq!(page["truncated"], true); + assert!(response::source_page("", 0, 1).is_err()); +} + +#[tokio::test] +async fn argument_budget_counts_serialized_utf8_and_escape_bytes_at_the_exact_boundary() { + let dir = tempfile::tempdir().unwrap(); + // The compact object contributes twelve bytes around each query value. + for (query, bytes) in [("a", 13), ("\n", 14), ("分", 15), ("\"\\", 16), ("\0", 18)] { + let args = json!({"query":query}); + for (maximum, accepted) in [(bytes, true), (bytes - 1, false)] { + let gate = TestGate::new(dir.path()); + let tools = code_tools_with_options( + gate.clone(), + CodeToolOptions { + max_argument_bytes: maximum, + ..Default::default() + }, + ) + .unwrap(); + let result = tool(&tools, "chunk_search") + .execute(args.clone()) + .await + .unwrap(); + assert_eq!( + !result.is_error, accepted, + "query {query:?}, limit {maximum}" + ); + if !accepted { + assert!(result.output().contains("arguments exceed the byte budget")); + } + assert_eq!(gate.state.used.load(std::sync::atomic::Ordering::SeqCst), 0); + } + } + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0); +} diff --git a/crates/tinyagents-harness-tools/src/code/declaration.rs b/crates/tinyagents-harness-tools/src/code/declaration.rs new file mode 100644 index 000000000..2ad99eb3b --- /dev/null +++ b/crates/tinyagents-harness-tools/src/code/declaration.rs @@ -0,0 +1,229 @@ +//! Tool metadata, strict dispatch and the host's per-call workspace scope. + +use std::sync::Arc; + +use async_trait::async_trait; +use serde_json::{Value, json}; +use tinytools::{ + PermissionLevel, Tool, ToolAccess, ToolCallOptions, ToolExposure, ToolPolicy, ToolResult, + ToolRunContext, ToolRuntime, ToolSideEffects, WorkspaceAccess, +}; + +use super::{ + arguments, response, + types::{CodeTool, Kind}, +}; +use crate::fs::filesystem::FsGate; + +#[async_trait] +impl Tool for CodeTool { + fn name(&self) -> &str { + self.kind.name() + } + + fn description(&self) -> &str { + match self.kind { + Kind::Outline => { + "Inspect a bounded lexical outline and folded source with content revisions and paging. This does not resolve semantic types." + } + Kind::Symbol => { + "Read one unambiguous lexical symbol by revision-bound ID, qualified name or local name; source is paginated." + } + Kind::Patch => { + "Apply one syntax-checked structural edit to an exact expected revision. Set dry_run to the literal boolean true for a read-only preview." + } + Kind::Search => { + "Search bundled typed code patterns by metadata. Dependencies are informational and are never installed." + } + Kind::Inspect => { + "Inspect a bundled pattern or gated custom manifest. Returns the content-bound chunk_revision required by chunk_apply." + } + Kind::Apply => { + "Render a typed pattern into a revision-checked file or selected container. Requires chunk_revision; use create_new for no-clobber creation, or expected_revision for edits. Literal dry_run true previews without writing." + } + } + } + + fn parameters_schema(&self) -> Value { + schema(self.kind, self.shared.options.max_page_items) + } + + fn permission_level(&self) -> PermissionLevel { + PermissionLevel::ReadOnly + } + + fn permission_level_with_args(&self, args: &Value) -> PermissionLevel { + if self.kind.mutates() && !arguments::literal_preview(args) { + PermissionLevel::Write + } else { + PermissionLevel::ReadOnly + } + } + + fn exposure(&self) -> ToolExposure { + ToolExposure::Deferred + } + + fn family(&self) -> Option<&str> { + Some("code") + } + + fn tags(&self) -> Vec { + ["code", "structural", "pack:code"] + .map(str::to_owned) + .to_vec() + } + + fn is_concurrency_safe(&self, args: &Value) -> bool { + !self.kind.mutates() || arguments::literal_preview(args) + } + + fn external_effect(&self) -> bool { + self.kind.mutates() && self.gate.write_needs_approval() + } + + fn external_effect_with_args(&self, args: &Value) -> bool { + self.external_effect() && !arguments::literal_preview(args) + } + + fn policy(&self) -> ToolPolicy { + let writes = self.kind.mutates(); + ToolPolicy::classified() + .with_side_effects(ToolSideEffects { + read_only: !writes, + writes_files: writes, + ..ToolSideEffects::default() + }) + .with_runtime(ToolRuntime { + idempotent: !writes, + cancelable: true, + max_result_bytes: Some(self.shared.options.max_output_bytes), + ..ToolRuntime::default() + }) + .with_access(ToolAccess { + workspace: WorkspaceAccess::Scoped, + trusted_roots: vec![self.gate.action_dir().display().to_string()], + background_safe: !writes, + ..ToolAccess::default() + }) + } + + async fn execute(&self, args: Value) -> anyhow::Result { + self.execute_with_context(args, ToolCallOptions::default(), None) + .await + } + + async fn execute_with_context( + &self, + args: Value, + _options: ToolCallOptions, + context: Option<&dyn ToolRunContext>, + ) -> anyhow::Result { + let gate = context.and_then(ToolRunContext::workspace).map_or_else( + || Arc::clone(&self.gate), + |workspace| self.gate.scoped_to_workspace(&workspace.root), + ); + Ok(match self.dispatch(args, &gate).await { + Ok(result) => result, + Err(error) => { + response::bounded_error(&format!("{error:#}"), self.shared.options.max_output_bytes) + } + }) + } +} + +impl CodeTool { + async fn dispatch(&self, args: Value, gate: &Arc) -> anyhow::Result { + arguments::check_size(&args, self.shared.options.max_argument_bytes)?; + match self.kind { + Kind::Outline => self.outline(arguments::decode(args)?, gate).await, + Kind::Symbol => self.symbol(arguments::decode(args)?, gate).await, + Kind::Patch => self.patch(arguments::decode(args)?, gate).await, + Kind::Search => self.search(&arguments::decode(args)?), + Kind::Inspect => self.inspect(arguments::decode(args)?, gate).await, + Kind::Apply => self.apply(arguments::decode(args)?, gate).await, + } + } +} + +fn schema(kind: Kind, max_page_items: usize) -> Value { + let mut properties = serde_json::Map::new(); + let required = match kind { + Kind::Outline | Kind::Symbol | Kind::Patch => { + properties.insert("path".into(), json!({"type":"string"})); + properties.insert("language".into(), json!({"type":"string","enum":["rust","typescript","tsx","javascript","python","go","json","toml"]})); + if kind == Kind::Outline { + properties.insert( + "level".into(), + json!({"type":"integer","enum":[0,1],"default":1,"description":"0 returns metadata only; 1 also returns the folded source page"}), + ); + add_record_page(&mut properties); + vec!["path"] + } else { + properties.insert("selector".into(), json!({"type":"string"})); + if kind == Kind::Symbol { + vec!["path", "selector"] + } else { + properties.insert("expected_revision".into(), revision_schema()); + properties.insert("operation".into(), json!({"type":"string","enum":["replace_body","replace_symbol","insert_import","append_member","wrap"]})); + properties.insert("code".into(), json!({"type":"string"})); + properties.insert("dry_run".into(), json!({"type":"boolean","default":false})); + vec!["path", "expected_revision", "operation", "code"] + } + } + } + Kind::Search => { + properties.insert("query".into(), json!({"type":"string","default":""})); + properties.insert("language".into(), json!({"type":"string","enum":["rust","typescript","tsx","javascript","python","go","json","toml"]})); + add_record_page(&mut properties); + properties.insert( + "limit".into(), + json!({"type":"integer","minimum":1,"maximum":max_page_items,"default":max_page_items.min(5)}), + ); + vec![] + } + Kind::Inspect | Kind::Apply => { + properties.insert("id".into(), json!({"type":"string","description":"Bundled chunk ID; mutually exclusive with manifest_path"})); + properties.insert("manifest_path".into(), json!({"type":"string","description":"Custom manifest file permitted by FsGate; mutually exclusive with id"})); + if kind == Kind::Inspect { + add_record_page(&mut properties); + vec![] + } else { + properties.insert("path".into(), json!({"type":"string"})); + properties.insert("chunk_revision".into(), revision_schema()); + properties.insert("expected_revision".into(), revision_schema()); + properties.insert("selector".into(), json!({"type":"string"})); + properties.insert( + "create_new".into(), + json!({"type":"boolean","default":false}), + ); + properties.insert("dry_run".into(), json!({"type":"boolean","default":false})); + properties.insert("bindings".into(), json!({"type":"object","properties":{"parameters":{"type":"object"},"slots":{"type":"object","additionalProperties":{"type":"string"}}},"additionalProperties":false})); + vec!["path", "chunk_revision"] + } + } + }; + if matches!( + kind, + Kind::Outline | Kind::Symbol | Kind::Patch | Kind::Apply + ) { + properties.insert( + "start_line".into(), + json!({"type":"integer","minimum":1,"default":1}), + ); + properties.insert("line_count".into(), json!({"type":"integer","minimum":1})); + } + json!({"type":"object","properties":properties,"required":required,"additionalProperties":false}) +} + +fn add_record_page(properties: &mut serde_json::Map) { + properties.insert( + "offset".into(), + json!({"type":"integer","minimum":0,"default":0}), + ); + properties.insert("limit".into(), json!({"type":"integer","minimum":1})); +} + +fn revision_schema() -> Value { + json!({"type":"string","pattern":"^[0-9a-f]{64}$"}) +} diff --git a/crates/tinyagents-harness-tools/src/code/inspect.rs b/crates/tinyagents-harness-tools/src/code/inspect.rs new file mode 100644 index 000000000..6dd5badcd --- /dev/null +++ b/crates/tinyagents-harness-tools/src/code/inspect.rs @@ -0,0 +1,132 @@ +//! Bounded structural and catalog inspection with explicit continuation fields. + +use std::sync::Arc; + +use anyhow::{Context, ensure}; +use serde_json::json; +use tinytools::ToolResult; +use tinytools_code::select_symbol; + +use super::{arguments, io, response, types::CodeTool}; +use crate::fs::filesystem::FsGate; + +impl CodeTool { + pub(super) async fn outline( + &self, + args: arguments::Outline, + gate: &Arc, + ) -> anyhow::Result { + let options = self.shared.options; + ensure!(args.level <= 1, "outline level must be 0 or 1"); + ensure!(args.start_line > 0, "start_line must be at least 1"); + let limit = arguments::page_size(args.limit, options.max_page_items)?; + let lines = arguments::page_size(args.line_count, options.max_page_lines)?; + let language = arguments::language(&args.path, args.language)?; + let snapshot = io::read(gate, &args.path, options.limits.max_source_bytes).await?; + let report = self.shared.engine.lock().analyze( + &snapshot.guard.resolved.to_string_lossy(), + language, + &snapshot.source, + )?; + let mut result = json!({ + "path": args.path, + "language": language, + "revision": report.revision, + "symbols": response::records(&report.symbols, args.offset, limit), + "references": response::records(&report.references, args.offset, limit), + "diagnostics": response::records(&report.diagnostics, args.offset, limit), + "symbols_truncated": report.symbols_truncated, + "references_truncated": report.references_truncated, + "diagnostics_truncated": report.diagnostics_truncated, + "cache_status": report.cache_status + }); + if args.level == 1 { + result["skeleton"] = response::source_page(&report.skeleton, args.start_line, lines)?; + } + response::bounded_json(result, options.max_output_bytes) + } + + pub(super) async fn symbol( + &self, + args: arguments::Symbol, + gate: &Arc, + ) -> anyhow::Result { + let options = self.shared.options; + let lines = arguments::page_size(args.line_count, options.max_page_lines)?; + let language = arguments::language(&args.path, args.language)?; + let snapshot = io::read(gate, &args.path, options.limits.max_source_bytes).await?; + let report = self.shared.engine.lock().analyze( + &snapshot.guard.resolved.to_string_lossy(), + language, + &snapshot.source, + )?; + let symbol = select_symbol(&report, &args.selector)?; + let source = snapshot + .source + .get(symbol.range.start_byte..symbol.range.end_byte) + .context("symbol range is outside its source snapshot")?; + response::bounded_json( + json!({ + "path": args.path, + "language": language, + "revision": report.revision, + "symbol": symbol, + "source": response::source_page(source, args.start_line, lines)? + }), + options.max_output_bytes, + ) + } + + pub(super) fn search(&self, args: &arguments::Search) -> anyhow::Result { + let options = self.shared.options; + let limit = arguments::page_size( + Some(args.limit.unwrap_or(options.max_page_items.min(5))), + options.max_page_items, + )?; + ensure!( + args.query.len() <= 1024, + "chunk search query exceeds 1024 bytes" + ); + // The built-in catalog contains twelve entries; fifty is the pure + // search cap, so the adapter can page every bundled match exactly. + let matches = self.shared.catalog.search(&args.query, args.language, 50); + response::bounded_json( + json!({ + "query": args.query, + "chunks": response::records(&matches, args.offset, limit) + }), + options.max_output_bytes, + ) + } + + pub(super) async fn inspect( + &self, + args: arguments::Inspect, + gate: &Arc, + ) -> anyhow::Result { + let options = self.shared.options; + let limit = arguments::page_size(args.limit, options.max_page_items)?; + let chunk = self + .load_chunk(gate, args.id.as_deref(), args.manifest_path.as_deref()) + .await?; + let manifest = &chunk.manifest; + response::bounded_json( + json!({ + "id": manifest.id, + "summary": manifest.summary, + "language": manifest.language, + "target": manifest.target, + "chunk_revision": chunk.revision, + "manifest_revision": manifest.revision()?, + "template_revision": tinytools_code::revision(&chunk.template), + "template_path": manifest.template_path, + "parameters": response::records(&manifest.parameters, args.offset, limit), + "slots": response::records(&manifest.slots, args.offset, limit), + "tags": response::records(&manifest.tags, args.offset, limit), + "dependencies": response::records(&manifest.dependencies, args.offset, limit), + "forbidden_nodes": response::records(&manifest.forbidden_nodes, args.offset, limit) + }), + options.max_output_bytes, + ) + } +} diff --git a/crates/tinyagents-harness-tools/src/code/inspection_tests.rs b/crates/tinyagents-harness-tools/src/code/inspection_tests.rs new file mode 100644 index 000000000..397ef0567 --- /dev/null +++ b/crates/tinyagents-harness-tools/src/code/inspection_tests.rs @@ -0,0 +1,282 @@ +//! Real filesystem inspection, source paging and catalog-boundary regressions. + +#![allow(clippy::unwrap_used)] + +use super::test_support::*; +use super::*; +use serde_json::json; +use std::sync::atomic::Ordering; +use tinytools::{ToolCallOptions, WorkspaceDescriptor}; + +#[tokio::test] +async fn outline_and_symbol_pages_identify_the_exact_source_snapshot() { + let dir = tempfile::tempdir().unwrap(); + let source = "fn first() -> i32 {\n 1\n}\nfn second() {}\n"; + std::fs::write(dir.path().join("source.rs"), source).unwrap(); + let tools = code_tools(TestGate::new(dir.path())).unwrap(); + let outline = success( + tool(&tools, "ast_outline"), + json!({"path":"source.rs","limit":1,"line_count":1}), + ) + .await; + assert_eq!(outline["revision"], tinytools_code::revision(source)); + assert_eq!(outline["symbols"]["items"][0]["name"], "first"); + assert_eq!(outline["symbols"]["next_offset"], 1); + assert_eq!(outline["cache_status"], "fresh"); + let second = success( + tool(&tools, "ast_outline"), + json!({"path":"source.rs","offset":1,"limit":1}), + ) + .await; + assert_eq!(second["symbols"]["items"][0]["name"], "second"); + assert_eq!(second["cache_status"], "cached"); + let symbol = success(tool(&tools, "ast_symbol"), json!({"path":"source.rs","selector":outline["symbols"]["items"][0]["id"],"start_line":2,"line_count":1})).await; + assert_eq!(symbol["source"]["text"], " 1\n"); + assert_eq!(symbol["source"]["next_start_line"], 3); + assert_eq!(symbol["symbol"]["name"], "first"); + std::fs::write(dir.path().join("source.rs"), source.replace('1', "2")).unwrap(); + let changed = success(tool(&tools, "ast_outline"), json!({"path":"source.rs"})).await; + assert_ne!(changed["revision"], outline["revision"]); + assert_eq!(changed["cache_status"], "incremental"); +} + +#[tokio::test] +async fn outline_levels_preserve_metadata_and_default_to_paged_folded_source() { + let dir = tempfile::tempdir().unwrap(); + let source = "fn first() -> i32 {\n 1\n}\nfn second() {}\n"; + std::fs::write(dir.path().join("source.rs"), source).unwrap(); + let tools = code_tools(TestGate::new(dir.path())).unwrap(); + let entry = tool(&tools, "ast_outline"); + let metadata = success(entry, json!({"path":"source.rs","level":0,"limit":1})).await; + let default = success(entry, json!({"path":"source.rs","limit":1,"line_count":1})).await; + let explicit = success( + entry, + json!({"path":"source.rs","level":1,"limit":1,"line_count":1}), + ) + .await; + assert!(metadata.get("skeleton").is_none()); + assert_eq!(metadata["revision"], tinytools_code::revision(source)); + assert_eq!(metadata["symbols"], default["symbols"]); + assert_eq!(metadata["symbols"]["next_offset"], 1); + assert_eq!(default["skeleton"], explicit["skeleton"]); + assert_eq!(default["skeleton"]["start_line"], 1); + assert_eq!(default["skeleton"]["next_start_line"], 2); + assert_eq!( + entry.parameters_schema()["properties"]["level"]["default"], + 1 + ); + assert_eq!( + entry.parameters_schema()["properties"]["level"]["enum"], + json!([0, 1]) + ); +} + +#[tokio::test] +async fn outline_rejects_unsupported_or_malformed_levels_before_reading_a_path() { + let dir = tempfile::tempdir().unwrap(); + let gate = TestGate::new(dir.path()); + gate.state.denied.store(true, Ordering::SeqCst); + let tools = code_tools(gate.clone()).unwrap(); + for level in [ + json!(2), + json!(255), + json!(-1), + json!(256), + json!(1.5), + json!("1"), + json!(true), + json!(null), + ] { + let error = failure( + tool(&tools, "ast_outline"), + json!({"path":"missing.rs","level":level}), + ) + .await; + assert!( + error.contains("outline level") || error.contains("invalid structural tool arguments") + ); + assert!(!error.contains("policy-blocked")); + } + assert!( + failure( + tool(&tools, "ast_outline"), + json!({"path":"missing.rs","level":0,"start_line":0}) + ) + .await + .contains("start_line") + ); + assert_eq!(gate.state.used.load(Ordering::SeqCst), 0); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0); +} + +#[tokio::test] +async fn chunk_search_defaults_to_five_bounded_results_with_explicit_continuation() { + let dir = tempfile::tempdir().unwrap(); + for maximum in [2, 32] { + let tools = code_tools_with_options( + TestGate::new(dir.path()), + CodeToolOptions { + max_page_items: maximum, + ..Default::default() + }, + ) + .unwrap(); + let entry = tool(&tools, "chunk_search"); + let first = success(entry, json!({})).await; + let default_limit = maximum.min(5); + assert_eq!( + first["chunks"]["items"].as_array().unwrap().len(), + default_limit + ); + assert_eq!(first["chunks"]["total"], 12); + assert_eq!(first["chunks"]["next_offset"], default_limit); + assert_eq!( + entry.parameters_schema()["properties"]["limit"]["default"], + default_limit + ); + let next = success(entry, json!({"offset":10,"limit":maximum})).await; + assert_eq!(next["chunks"]["items"].as_array().unwrap().len(), 2); + assert_eq!(next["chunks"]["next_offset"], json!(null)); + let explicit = success(entry, json!({"limit":maximum})).await; + assert_eq!( + explicit["chunks"]["items"].as_array().unwrap().len(), + maximum.min(12) + ); + for invalid in [0, maximum + 1] { + assert!( + failure(entry, json!({"limit":invalid})) + .await + .contains("page size") + ); + } + } +} + +#[tokio::test] +async fn denied_missing_binary_large_and_non_file_inputs_are_model_visible_errors() { + let dir = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + std::fs::write(outside.path().join("outside.rs"), "fn outside() {}").unwrap(); + std::fs::write(dir.path().join("binary.rs"), [0xff, 0xfe]).unwrap(); + std::fs::write(dir.path().join("large.rs"), " ".repeat(64)).unwrap(); + std::fs::create_dir(dir.path().join("folder.rs")).unwrap(); + let gate = TestGate::new(dir.path()); + let tools = code_tools_with_options( + gate.clone(), + CodeToolOptions { + limits: tinytools_code::CodeLimits { + max_source_bytes: 32, + ..Default::default() + }, + ..Default::default() + }, + ) + .unwrap(); + for path in [ + "missing.rs".to_owned(), + "binary.rs".into(), + "large.rs".into(), + "folder.rs".into(), + outside.path().join("outside.rs").display().to_string(), + "../outside.rs".into(), + ] { + assert_ne!( + failure(tool(&tools, "ast_outline"), json!({"path":path})).await, + "" + ); + } + assert!( + failure(tool(&tools, "ast_outline"), json!({"path":"unknown"})) + .await + .contains("explicit") + ); + assert!( + failure( + tool(&tools, "ast_outline"), + json!({"path":"source.rs","limit":0}) + ) + .await + .contains("page size") + ); + assert!( + failure( + tool(&tools, "ast_symbol"), + json!({"path":"source.rs","selector":"x","line_count":81}) + ) + .await + .contains("page size") + ); + gate.state.denied.store(true, Ordering::SeqCst); + assert!( + failure(tool(&tools, "ast_outline"), json!({"path":"binary.rs"})) + .await + .contains("policy-blocked") + ); + assert_eq!(gate.state.used.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn the_trusted_run_workspace_scopes_reads_without_mutating_the_parent_gate() { + let parent = tempfile::tempdir().unwrap(); + let child = tempfile::tempdir().unwrap(); + std::fs::write(parent.path().join("source.rs"), "fn parent() {}").unwrap(); + std::fs::write(child.path().join("source.rs"), "fn child() {}").unwrap(); + let gate = TestGate::new(parent.path()); + let tools = code_tools(gate.clone()).unwrap(); + let context = Workspace(WorkspaceDescriptor::new(child.path().to_path_buf())); + let result = tool(&tools, "ast_outline") + .execute_with_context( + json!({"path":"source.rs"}), + ToolCallOptions::default(), + Some(&context), + ) + .await + .unwrap(); + assert!(!result.is_error, "{}", result.output()); + assert!(result.output().contains("child")); + let ordinary = success(tool(&tools, "ast_outline"), json!({"path":"source.rs"})).await; + assert_eq!(ordinary["symbols"]["items"][0]["name"], "parent"); + assert_eq!(gate.action_dir(), parent.path()); +} + +#[tokio::test] +async fn bundled_catalog_search_and_inspection_are_paged_metadata_only() { + let dir = tempfile::tempdir().unwrap(); + let gate = TestGate::new(dir.path()); + gate.state.denied.store(true, Ordering::SeqCst); + let tools = code_tools(gate.clone()).unwrap(); + let first = success(tool(&tools, "chunk_search"), json!({"limit":1})).await; + assert_eq!(first["chunks"]["total"], 12); + assert_eq!(first["chunks"]["next_offset"], 1); + let inspected = success( + tool(&tools, "chunk_inspect"), + json!({"id":"rust.fallible-function.v1","limit":1}), + ) + .await; + assert_eq!(inspected["parameters"]["items"][0]["name"], "name"); + assert_eq!(inspected["parameters"]["next_offset"], 1); + assert_eq!(inspected["chunk_revision"].as_str().unwrap().len(), 64); + assert!(inspected.get("source").is_none()); + assert!( + failure(tool(&tools, "chunk_inspect"), json!({})) + .await + .contains("exactly one") + ); + assert!( + failure( + tool(&tools, "chunk_inspect"), + json!({"id":"no-such-pattern"}) + ) + .await + .contains("no-such-pattern") + ); + assert!( + failure( + tool(&tools, "chunk_search"), + json!({"query":"x".repeat(1025)}) + ) + .await + .contains("1024") + ); + assert_eq!(gate.state.used.load(Ordering::SeqCst), 0); +} diff --git a/crates/tinyagents-harness-tools/src/code/io.rs b/crates/tinyagents-harness-tools/src/code/io.rs new file mode 100644 index 000000000..a17d7e874 --- /dev/null +++ b/crates/tinyagents-harness-tools/src/code/io.rs @@ -0,0 +1,126 @@ +//! Bounded regular-file reads and reauthorization of retained source guards. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use anyhow::{Context, anyhow, ensure}; +use tinytools_code::revision; +use tokio::io::AsyncReadExt; + +use crate::fs::filesystem::FsGate; + +#[derive(Debug, Clone)] +pub(super) struct ReadGuard { + pub(super) original: String, + pub(super) resolved: PathBuf, + pub(super) revision: String, + pub(super) maximum: usize, +} + +pub(super) struct Snapshot { + pub(super) guard: ReadGuard, + pub(super) source: String, +} + +pub(super) async fn read( + gate: &Arc, + original: &str, + maximum: usize, +) -> anyhow::Result { + let resolved = gate + .validate_path(original) + .await + .map_err(|message| anyhow!(message))?; + let source = read_bounded(&resolved, maximum).await?; + Ok(Snapshot { + guard: ReadGuard { + original: original.into(), + resolved, + revision: revision(&source), + maximum, + }, + source, + }) +} + +pub(super) async fn read_bounded(path: &Path, maximum: usize) -> anyhow::Result { + let metadata = tokio::fs::metadata(path) + .await + .context("could not inspect structural input")?; + ensure!( + metadata.is_file(), + "structural input must be a regular file" + ); + let mut options = tokio::fs::OpenOptions::new(); + options.read(true); + // The gate already resolved the path. Refuse a later symlink swap, and + // never block on a FIFO substituted between the metadata and open calls. + #[cfg(unix)] + options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); + let file = options + .open(path) + .await + .context("could not open structural input")?; + let metadata = file + .metadata() + .await + .context("could not inspect structural input")?; + ensure!( + metadata.is_file(), + "structural input must be a regular file" + ); + let bound = u64::try_from(maximum)?.saturating_add(1); + let mut bytes = Vec::new(); + file.take(bound) + .read_to_end(&mut bytes) + .await + .context("could not read structural input")?; + ensure!( + bytes.len() <= maximum, + "structural input exceeds the {maximum}-byte read budget" + ); + String::from_utf8(bytes).context("structural input must be valid UTF-8") +} + +pub(super) async fn verify(gate: &Arc, guard: &ReadGuard) -> anyhow::Result<()> { + let current = read(gate, &guard.original, guard.maximum).await?; + ensure!( + current.guard.resolved == guard.resolved, + "structural input path changed during the call; inspect it again" + ); + ensure!( + current.guard.revision == guard.revision, + "structural input revision changed during the call; inspect it again" + ); + Ok(()) +} + +pub(super) async fn resolve_target( + gate: &Arc, + original: &str, +) -> anyhow::Result { + let path = gate + .validate_parent_path(original) + .await + .map_err(|message| anyhow!(message))?; + reject_symlink(&path).await?; + Ok(path) +} + +pub(super) async fn reject_symlink(path: &Path) -> anyhow::Result<()> { + match tokio::fs::symlink_metadata(path).await { + Ok(metadata) => { + ensure!( + !metadata.file_type().is_symlink(), + "refusing to replace a symlink" + ); + ensure!( + metadata.is_file(), + "structural destination must be a regular file" + ); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error).context("could not inspect structural destination"), + } + Ok(()) +} diff --git a/crates/tinyagents-harness-tools/src/code/mod.rs b/crates/tinyagents-harness-tools/src/code/mod.rs new file mode 100644 index 000000000..dbdbd5d66 --- /dev/null +++ b/crates/tinyagents-harness-tools/src/code/mod.rs @@ -0,0 +1,85 @@ +//! Optional structural tools behind the host's filesystem gate. +//! +//! [`code_tools`] builds six deferred tools sharing a bounded parser cache and +//! write lock. Only the literal JSON boolean `dry_run: true` previews edits; +//! real writes require exact content revisions and publish atomically. See the +//! module README for the argument and receipt contracts. + +mod arguments; +mod chunks; +mod declaration; +mod inspect; +mod io; +mod mutate; +mod publish; +mod response; +mod types; + +use std::sync::Arc; + +use parking_lot::Mutex; +use tinytools::Tool; +use tinytools_code::{ChunkCatalog, CodeEngine}; + +use crate::fs::filesystem::FsGate; + +pub use types::CodeToolOptions; +use types::{CodeTool, Kind, Shared}; + +/// Build the six deferred structural tools with their default resource bounds. +/// +/// # Errors +/// +/// Returns an error if the bundled catalog or parser configuration is invalid. +pub fn code_tools(gate: Arc) -> anyhow::Result>> { + code_tools_with_options(gate, CodeToolOptions::default()) +} + +/// Build the structural tools with host-selected parser and I/O budgets. +/// +/// All returned tools share the same parser cache and mutation lock. They do +/// not install dependencies, execute generated code, or grant filesystem access. +/// +/// # Errors +/// +/// Returns an error for invalid resource bounds or an invalid bundled catalog. +pub fn code_tools_with_options( + gate: Arc, + options: CodeToolOptions, +) -> anyhow::Result>> { + options.validate()?; + let shared = Arc::new(Shared { + engine: Mutex::new(CodeEngine::new(options.limits)?), + catalog: ChunkCatalog::builtin()?, + write_lock: tokio::sync::Mutex::new(()), + options, + }); + Ok(Kind::ALL + .into_iter() + .map(move |kind| { + Box::new(CodeTool { + kind, + gate: Arc::clone(&gate), + shared: Arc::clone(&shared), + }) as Box + }) + .collect()) +} + +#[cfg(test)] +#[path = "chunk_tests.rs"] +mod chunk_tests; +#[cfg(test)] +#[path = "contract_tests.rs"] +mod contract_tests; +#[cfg(test)] +#[path = "inspection_tests.rs"] +mod inspection_tests; +#[cfg(test)] +#[path = "mutation_tests.rs"] +mod mutation_tests; +#[cfg(test)] +#[path = "publication_tests.rs"] +mod publication_tests; +#[cfg(test)] +mod test_support; diff --git a/crates/tinyagents-harness-tools/src/code/mutate.rs b/crates/tinyagents-harness-tools/src/code/mutate.rs new file mode 100644 index 000000000..1b127abe3 --- /dev/null +++ b/crates/tinyagents-harness-tools/src/code/mutate.rs @@ -0,0 +1,248 @@ +//! Revision-checked edits and chunk rendering with receipt preflight. + +use std::sync::Arc; + +use anyhow::{Context, ensure}; +use serde_json::{Value, json}; +use tinytools::ToolResult; +use tinytools_code::{ChunkTarget, PatchOperation, PatchRequest, render_chunk, revision}; + +use super::{ + arguments, io, + publish::{self, Destination}, + response, + types::CodeTool, +}; +use crate::fs::{file_state, filesystem::FsGate}; + +struct Change { + destination: Destination, + source: String, + changed: bool, + language: tinytools_code::Language, +} + +impl CodeTool { + pub(super) async fn patch( + &self, + args: arguments::Patch, + gate: &Arc, + ) -> anyhow::Result { + arguments::require_revision(&args.expected_revision)?; + self.permit_write(args.dry_run)?; + let language = arguments::language(&args.path, args.language)?; + let resolved = io::resolve_target(gate, &args.path).await?; + let _group_lock = self.shared.write_lock.lock().await; + let _path_lock = file_state::acquire_path_lock(&resolved).await; + let snapshot = io::read( + gate, + &args.path, + self.shared.options.limits.max_source_bytes, + ) + .await?; + ensure!( + snapshot.guard.resolved == resolved, + "structural destination resolves inconsistently" + ); + let result = self.shared.engine.lock().patch( + &resolved.to_string_lossy(), + language, + &snapshot.source, + &PatchRequest { + expected_revision: args.expected_revision, + selector: args.selector, + operation: args.operation, + code: args.code, + }, + )?; + let change = Change { + destination: Destination { + original: args.path, + resolved, + before: Some(snapshot.guard), + }, + source: result.source, + changed: result.changed, + language, + }; + let receipt = self.receipt( + &change, + args.dry_run, + args.start_line, + args.line_count, + None, + )?; + if change.changed && !args.dry_run { + self.publish(gate, &change.destination, &[], &change.source) + .await?; + } + Ok(receipt) + } + + pub(super) async fn apply( + &self, + args: arguments::Apply, + gate: &Arc, + ) -> anyhow::Result { + arguments::require_revision(&args.chunk_revision)?; + ensure!( + args.create_new != args.expected_revision.is_some(), + "provide expected_revision for edits, or create_new: true without expected_revision" + ); + if let Some(expected) = &args.expected_revision { + arguments::require_revision(expected)?; + } + self.permit_write(args.dry_run)?; + let chunk = self + .load_chunk(gate, args.id.as_deref(), args.manifest_path.as_deref()) + .await?; + ensure!( + chunk.revision == args.chunk_revision, + "chunk revision mismatch; inspect the current manifest and template" + ); + ensure!( + !args.create_new || chunk.manifest.target == ChunkTarget::File, + "create_new requires a complete-file chunk" + ); + let rendered = render_chunk( + &mut self.shared.engine.lock(), + &chunk.manifest, + &chunk.template, + &args.bindings, + )?; + let resolved = io::resolve_target(gate, &args.path).await?; + let _group_lock = self.shared.write_lock.lock().await; + let _path_lock = file_state::acquire_path_lock(&resolved).await; + let snapshot = self.destination_snapshot(gate, &args, &resolved).await?; + let change = self.chunk_change(&args, resolved, snapshot, &rendered)?; + let extra = json!({ + "chunk_id": rendered.id, + "chunk_revision": chunk.revision, + "rendered_revision": rendered.revision, + "dependencies": rendered.dependencies + }); + let receipt = self.receipt( + &change, + args.dry_run, + args.start_line, + args.line_count, + Some(extra), + )?; + if change.changed && !args.dry_run { + self.publish(gate, &change.destination, &chunk.guards, &change.source) + .await?; + } + Ok(receipt) + } + + async fn destination_snapshot( + &self, + gate: &Arc, + args: &arguments::Apply, + resolved: &std::path::Path, + ) -> anyhow::Result> { + if args.create_new { + publish::require_absent(resolved).await?; + return Ok(None); + } + let snapshot = io::read( + gate, + &args.path, + self.shared.options.limits.max_source_bytes, + ) + .await?; + ensure!( + snapshot.guard.resolved == resolved, + "structural destination resolves inconsistently" + ); + ensure!( + Some(&snapshot.guard.revision) == args.expected_revision.as_ref(), + "source revision mismatch; inspect the current destination" + ); + Ok(Some(snapshot)) + } + + fn chunk_change( + &self, + args: &arguments::Apply, + resolved: std::path::PathBuf, + snapshot: Option, + rendered: &tinytools_code::RenderedChunk, + ) -> anyhow::Result { + let (source, changed) = if rendered.target == ChunkTarget::File { + ensure!( + args.selector.is_none(), + "a complete-file chunk does not accept a selector" + ); + if let Some(before) = &snapshot { + self.shared + .engine + .lock() + .validate(rendered.language, &before.source)?; + } + ( + rendered.source.clone(), + snapshot + .as_ref() + .is_none_or(|before| before.source != rendered.source), + ) + } else { + let before = snapshot + .as_ref() + .context("container chunks require an existing destination")?; + let operation = if rendered.target == ChunkTarget::FunctionBody { + PatchOperation::ReplaceBody + } else { + PatchOperation::AppendMember + }; + let patched = self.shared.engine.lock().patch( + &resolved.to_string_lossy(), + rendered.language, + &before.source, + &PatchRequest { + expected_revision: before.guard.revision.clone(), + selector: args.selector.clone(), + operation, + code: rendered.source.clone(), + }, + )?; + (patched.source, patched.changed) + }; + Ok(Change { + destination: Destination { + original: args.path.clone(), + resolved, + before: snapshot.map(|before| before.guard), + }, + source, + changed, + language: rendered.language, + }) + } + + fn receipt( + &self, + change: &Change, + dry_run: bool, + start_line: usize, + line_count: Option, + extra: Option, + ) -> anyhow::Result { + let options = self.shared.options; + let lines = arguments::page_size(line_count, options.max_page_lines)?; + let mut value = json!({ + "path": change.destination.original, + "language": change.language, + "before_revision": change.destination.before.as_ref().map(|before| &before.revision), + "revision": revision(&change.source), + "changed": change.changed, + "written": change.changed && !dry_run, + "dry_run": dry_run, + "source": response::source_page(&change.source, start_line, lines)? + }); + if let (Some(object), Some(Value::Object(extra))) = (value.as_object_mut(), extra) { + object.extend(extra); + } + response::bounded_json(value, options.max_output_bytes) + } +} diff --git a/crates/tinyagents-harness-tools/src/code/mutation_tests.rs b/crates/tinyagents-harness-tools/src/code/mutation_tests.rs new file mode 100644 index 000000000..5ba84ed98 --- /dev/null +++ b/crates/tinyagents-harness-tools/src/code/mutation_tests.rs @@ -0,0 +1,287 @@ +//! Preview, revision, typed-chunk and actual filesystem mutation regressions. + +#![allow(clippy::unwrap_used)] + +use super::test_support::*; +use super::*; +use serde_json::{Value, json}; +use std::sync::atomic::Ordering; +use tinytools_code::revision; + +const SOURCE: &str = "fn value() -> i32 { 1 }\n"; + +#[tokio::test] +async fn literal_previews_work_in_read_only_mode_and_do_not_consume_actions() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("source.rs"); + std::fs::write(&path, SOURCE).unwrap(); + let gate = TestGate::new(dir.path()); + gate.state.read_only.store(true, Ordering::SeqCst); + let tools = code_tools(gate.clone()).unwrap(); + let mut args = patch_args(SOURCE); + args["dry_run"] = json!(true); + let preview = success(tool(&tools, "ast_patch"), args.clone()).await; + assert_eq!(preview["written"], false); + assert_eq!(preview["dry_run"], true); + assert_eq!(preview["changed"], true); + assert_eq!(preview["before_revision"], revision(SOURCE)); + assert_ne!(preview["revision"], revision(SOURCE)); + assert_eq!(std::fs::read_to_string(&path).unwrap(), SOURCE); + args["dry_run"] = json!(false); + assert!( + failure(tool(&tools, "ast_patch"), args.clone()) + .await + .contains("policy-blocked") + ); + for invalid in [json!("true"), Value::Null, json!(1)] { + args["dry_run"] = invalid; + assert!( + failure(tool(&tools, "ast_patch"), args.clone()) + .await + .contains("arguments") + ); + } + assert_eq!(gate.state.used.load(Ordering::SeqCst), 0); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1); +} + +#[tokio::test] +async fn a_real_patch_requires_the_current_revision_and_records_the_actual_write() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("source.rs"); + std::fs::write(&path, SOURCE).unwrap(); + let gate = TestGate::new(dir.path()); + let tools = code_tools(gate.clone()).unwrap(); + crate::fs::file_state::init_global(true); + let resolved = path.canonicalize().unwrap(); + crate::fs::file_state::record_read( + "code-parent", + resolved.clone(), + std::time::SystemTime::now(), + false, + std::time::Instant::now(), + ); + let receipt = crate::fs::file_state::with_file_state_agent_id( + "code-writer".into(), + success(tool(&tools, "ast_patch"), patch_args(SOURCE)), + ) + .await; + let changed = std::fs::read_to_string(&path).unwrap(); + assert!(changed.contains('2')); + assert_eq!(receipt["written"], true); + assert_eq!(receipt["dry_run"], false); + assert_eq!(receipt["revision"], revision(&changed)); + assert_eq!(gate.state.used.load(Ordering::SeqCst), 1); + assert!(crate::fs::file_state::check_stale_read("code-parent", &resolved).is_some()); + assert!( + failure(tool(&tools, "ast_patch"), patch_args(SOURCE)) + .await + .contains("revision") + ); + assert_eq!(std::fs::read_to_string(&path).unwrap(), changed); + assert_eq!(gate.state.used.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn concurrent_edits_from_one_snapshot_cannot_overwrite_each_other() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("source.rs"), SOURCE).unwrap(); + let gate = TestGate::new(dir.path()); + let tools = code_tools(gate.clone()).unwrap(); + let entry = tool(&tools, "ast_patch"); + let mut second = patch_args(SOURCE); + second["code"] = json!("3"); + let (first, second) = tokio::join!(entry.execute(patch_args(SOURCE)), entry.execute(second)); + assert_ne!(first.unwrap().is_error, second.unwrap().is_error); + assert_eq!(gate.state.used.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn chunks_require_inspected_content_and_create_without_clobbering() { + let dir = tempfile::tempdir().unwrap(); + let gate = TestGate::new(dir.path()); + let tools = code_tools(gate.clone()).unwrap(); + let inspected = success( + tool(&tools, "chunk_inspect"), + json!({"id":"rust.fallible-function.v1"}), + ) + .await; + let mut args = json!({"id":"rust.fallible-function.v1", "path":"new.rs", "create_new":true, + "chunk_revision":inspected["chunk_revision"], "bindings":{"parameters":{"name":"created"}}, "dry_run":true}); + let preview = success(tool(&tools, "chunk_apply"), args.clone()).await; + assert_eq!(preview["before_revision"], Value::Null); + assert_eq!(preview["written"], false); + assert!( + preview["source"]["text"] + .as_str() + .unwrap() + .contains("fn created") + ); + assert!(!dir.path().join("new.rs").exists()); + args["dry_run"] = json!(false); + let receipt = success(tool(&tools, "chunk_apply"), args.clone()).await; + assert_eq!(receipt["written"], true); + let original = std::fs::read_to_string(dir.path().join("new.rs")).unwrap(); + assert_eq!(receipt["revision"], revision(&original)); + assert!( + failure(tool(&tools, "chunk_apply"), args.clone()) + .await + .contains("existing") + ); + args["path"] = json!("another.rs"); + args["chunk_revision"] = json!("a".repeat(64)); + assert!( + failure(tool(&tools, "chunk_apply"), args) + .await + .contains("chunk revision") + ); + assert_eq!( + std::fs::read_to_string(dir.path().join("new.rs")).unwrap(), + original + ); + assert_eq!(gate.state.used.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn chunks_edit_current_files_bodies_and_members_through_the_same_revision_guard() { + let dir = tempfile::tempdir().unwrap(); + let tools = code_tools(TestGate::new(dir.path())).unwrap(); + std::fs::write(dir.path().join("source.rs"), SOURCE).unwrap(); + let file = success( + tool(&tools, "chunk_inspect"), + json!({"id":"rust.fallible-function.v1"}), + ) + .await; + let receipt = success( + tool(&tools, "chunk_apply"), + json!({"id":"rust.fallible-function.v1", "path":"source.rs", + "chunk_revision":file["chunk_revision"], "expected_revision":revision(SOURCE)}), + ) + .await; + assert_eq!(receipt["written"], true); + let body_source = "fn sum(left: u32, right: u32) -> Result { Ok(0) }"; + std::fs::write(dir.path().join("source.rs"), body_source).unwrap(); + let body = success( + tool(&tools, "chunk_inspect"), + json!({"id":"rust.checked-add.v1"}), + ) + .await; + let receipt = success( + tool(&tools, "chunk_apply"), + json!({"id":"rust.checked-add.v1", "path":"source.rs", "selector":"sum", + "chunk_revision":body["chunk_revision"], "expected_revision":revision(body_source)}), + ) + .await; + assert!( + receipt["source"]["text"] + .as_str() + .unwrap() + .contains("checked_add") + ); + custom_manifest(dir.path(), "member", "fn added(&self) {}\n"); + let member_source = "struct Thing;\nimpl Thing {}\n"; + std::fs::write(dir.path().join("source.rs"), member_source).unwrap(); + let outline = success(tool(&tools, "ast_outline"), json!({"path":"source.rs"})).await; + let selected = outline["symbols"]["items"] + .as_array() + .unwrap() + .iter() + .find(|item| item["kind"] == "impl_item") + .unwrap()["id"] + .clone(); + let member = success( + tool(&tools, "chunk_inspect"), + json!({"manifest_path":"manifest.json"}), + ) + .await; + let receipt = success( + tool(&tools, "chunk_apply"), + json!({"manifest_path":"manifest.json", "path":"source.rs", "selector":selected, + "chunk_revision":member["chunk_revision"], "expected_revision":revision(member_source)}), + ) + .await; + assert!( + receipt["source"]["text"] + .as_str() + .unwrap() + .contains("fn added") + ); +} + +#[tokio::test] +async fn inconsistent_chunk_destinations_and_invalid_bindings_never_write() { + let dir = tempfile::tempdir().unwrap(); + let gate = TestGate::new(dir.path()); + let tools = code_tools(gate.clone()).unwrap(); + let inspected = success( + tool(&tools, "chunk_inspect"), + json!({"id":"rust.fallible-function.v1"}), + ) + .await; + let base = json!({"id":"rust.fallible-function.v1", "path":"source.rs", "chunk_revision":inspected["chunk_revision"]}); + assert!( + failure(tool(&tools, "chunk_apply"), base.clone()) + .await + .contains("expected_revision") + ); + let mut args = base; + args["create_new"] = json!(true); + args["expected_revision"] = json!(revision(SOURCE)); + assert!( + failure(tool(&tools, "chunk_apply"), args.clone()) + .await + .contains("without expected_revision") + ); + args.as_object_mut().unwrap().remove("expected_revision"); + args["bindings"] = json!({"parameters":{"name":"not an identifier"}}); + assert!( + failure(tool(&tools, "chunk_apply"), args.clone()) + .await + .contains("identifier parameter violates its declared kind") + ); + args["bindings"] = json!({}); + args["selector"] = json!("unnecessary"); + assert!( + failure(tool(&tools, "chunk_apply"), args) + .await + .contains("selector") + ); + let member = success( + tool(&tools, "chunk_inspect"), + json!({"id":"rust.public-field.v1"}), + ) + .await; + assert!( + failure( + tool(&tools, "chunk_apply"), + json!({"id":"rust.public-field.v1", "path":"new.rs", "create_new":true, + "chunk_revision":member["chunk_revision"]}) + ) + .await + .contains("complete-file") + ); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0); + assert_eq!(gate.state.used.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn oversized_receipts_are_refused_before_any_write_or_budget_consumption() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("source.rs"); + std::fs::write(&path, SOURCE).unwrap(); + let gate = TestGate::new(dir.path()); + let tools = code_tools_with_options( + gate.clone(), + CodeToolOptions { + max_output_bytes: 160, + ..Default::default() + }, + ) + .unwrap(); + let error = failure(tool(&tools, "ast_patch"), patch_args(SOURCE)).await; + assert!(error.contains("output budget")); + assert!(error.len() <= 160); + assert_eq!(std::fs::read_to_string(&path).unwrap(), SOURCE); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1); + assert_eq!(gate.state.used.load(Ordering::SeqCst), 0); +} diff --git a/crates/tinyagents-harness-tools/src/code/publication_tests.rs b/crates/tinyagents-harness-tools/src/code/publication_tests.rs new file mode 100644 index 000000000..841035be6 --- /dev/null +++ b/crates/tinyagents-harness-tools/src/code/publication_tests.rs @@ -0,0 +1,225 @@ +//! Revision races, final policy decisions, atomicity and staging cleanup. + +#![allow(clippy::unwrap_used)] + +use super::test_support::*; +use super::*; +use serde_json::json; +use std::sync::atomic::Ordering; + +const SOURCE: &str = "fn value() -> i32 { 1 }\n"; + +#[tokio::test] +async fn an_external_write_after_admission_is_preserved_and_staging_is_removed() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("source.rs"); + std::fs::write(&path, SOURCE).unwrap(); + let gate = TestGate::new(dir.path()); + *gate.state.action.lock() = Some(Action::Write( + path.clone(), + "fn value() -> i32 { 99 }".into(), + )); + let tools = code_tools(gate.clone()).unwrap(); + assert!( + failure(tool(&tools, "ast_patch"), patch_args(SOURCE)) + .await + .contains("revision changed") + ); + assert!(std::fs::read_to_string(path).unwrap().contains("99")); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1); +} + +#[tokio::test] +async fn original_path_reauthorization_and_late_read_only_policy_prevent_publication() { + for revoke_at in ["path", "action", "publication"] { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("source.rs"); + std::fs::write(&path, SOURCE).unwrap(); + let gate = TestGate::new(dir.path()); + if revoke_at == "action" { + *gate.state.action.lock() = Some(Action::Revoke); + } else if revoke_at == "publication" { + gate.state.revoke_at_recheck.store(true, Ordering::SeqCst); + } else { + gate.state.deny_recheck.store(true, Ordering::SeqCst); + } + let tools = code_tools(gate).unwrap(); + assert!( + failure(tool(&tools, "ast_patch"), patch_args(SOURCE)) + .await + .contains("policy-blocked") + ); + assert_eq!(std::fs::read_to_string(path).unwrap(), SOURCE); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1); + } +} + +#[tokio::test] +async fn the_final_available_action_can_publish_after_reserving_its_budget() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("source.rs"); + std::fs::write(&path, SOURCE).unwrap(); + let gate = Arc::new(TestGate { + root: dir.path().to_path_buf(), + state: Arc::new(State::default()), + max_actions: 1, + }); + let tools = code_tools(gate.clone()).unwrap(); + let receipt = success(tool(&tools, "ast_patch"), patch_args(SOURCE)).await; + assert_eq!(receipt["written"], true); + assert!(std::fs::read_to_string(path).unwrap().contains('2')); + assert_eq!(gate.state.used.load(Ordering::SeqCst), 1); + assert!(gate.is_rate_limited()); +} + +#[tokio::test] +async fn depleted_and_racing_action_budgets_refuse_before_staging() { + for predepleted in [false, true] { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("source.rs"), SOURCE).unwrap(); + let gate = TestGate::new(dir.path()); + if predepleted { + gate.state.used.store(100, Ordering::SeqCst); + } else { + gate.state.refuse_record.store(true, Ordering::SeqCst); + } + let tools = code_tools(gate).unwrap(); + assert!( + failure(tool(&tools, "ast_patch"), patch_args(SOURCE)) + .await + .contains("budget") + ); + assert_eq!( + std::fs::read_to_string(dir.path().join("source.rs")).unwrap(), + SOURCE + ); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1); + } +} + +#[tokio::test] +async fn deleted_destinations_and_new_file_collisions_do_not_get_recreated_or_clobbered() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("source.rs"); + std::fs::write(&path, SOURCE).unwrap(); + let gate = TestGate::new(dir.path()); + *gate.state.action.lock() = Some(Action::Remove(path.clone())); + let tools = code_tools(gate.clone()).unwrap(); + assert!( + failure(tool(&tools, "ast_patch"), patch_args(SOURCE)) + .await + .contains("permissions") + ); + assert!(!path.exists()); + let inspected = success( + tool(&tools, "chunk_inspect"), + json!({"id":"rust.fallible-function.v1"}), + ) + .await; + *gate.state.action.lock() = Some(Action::Write(path.clone(), "external writer".into())); + assert!( + failure( + tool(&tools, "chunk_apply"), + json!({"id":"rust.fallible-function.v1","path":"source.rs", "create_new":true, + "chunk_revision":inspected["chunk_revision"]}) + ) + .await + .contains("existing") + ); + assert_eq!(std::fs::read_to_string(path).unwrap(), "external writer"); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1); +} + +#[tokio::test] +async fn a_changed_custom_manifest_or_template_invalidates_publication() { + for manifest_changed in [false, true] { + let dir = tempfile::tempdir().unwrap(); + custom_manifest(dir.path(), "file", "fn created() {}\n"); + let gate = TestGate::new(dir.path()); + let tools = code_tools(gate.clone()).unwrap(); + let inspected = success( + tool(&tools, "chunk_inspect"), + json!({"manifest_path":"manifest.json"}), + ) + .await; + let path = dir.path().join(if manifest_changed { + "manifest.json" + } else { + "template.rs.txt" + }); + let changed = if manifest_changed { + std::fs::read_to_string(&path) + .unwrap() + .replace("Custom pattern", "Changed pattern") + } else { + "fn changed() {}\n".into() + }; + *gate.state.action.lock() = Some(Action::Write(path, changed)); + assert!( + failure( + tool(&tools, "chunk_apply"), + json!({"manifest_path":"manifest.json", "path":"new.rs", "create_new":true, + "chunk_revision":inspected["chunk_revision"]}) + ) + .await + .contains("revision changed") + ); + assert!(!dir.path().join("new.rs").exists()); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 2); + } +} + +#[tokio::test] +async fn cancellation_after_staging_keeps_the_original_and_removes_the_temporary_file() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("source.rs"); + std::fs::write(&path, SOURCE).unwrap(); + let entered = Arc::new(tokio::sync::Notify::new()); + let released = Arc::new(tokio::sync::Notify::new()); + let gate = Arc::new(TestGate { + root: dir.path().to_path_buf(), + max_actions: 100, + state: Arc::new(State { + pause_recheck: Some((entered.clone(), released)), + ..Default::default() + }), + }); + let tools = code_tools(gate).unwrap(); + let mut task = + tokio::spawn(async move { tool(&tools, "ast_patch").execute(patch_args(SOURCE)).await }); + let completed = tokio::select! { + () = entered.notified() => None, + result = &mut task => Some(result), + }; + assert!( + completed.is_none(), + "call did not reach its staging pause: {completed:?}" + ); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + assert_eq!(std::fs::read_to_string(path).unwrap(), SOURCE); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1); +} + +#[cfg(unix)] +#[tokio::test] +async fn replacement_preserves_permissions_and_symlink_destinations_are_refused() { + use std::os::unix::fs::{PermissionsExt, symlink}; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("source.rs"); + std::fs::write(&path, SOURCE).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o751)).unwrap(); + let tools = code_tools(TestGate::new(dir.path())).unwrap(); + success(tool(&tools, "ast_patch"), patch_args(SOURCE)).await; + assert_eq!( + std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o751 + ); + std::fs::rename(&path, dir.path().join("real.rs")).unwrap(); + symlink(dir.path().join("real.rs"), &path).unwrap(); + assert!( + failure(tool(&tools, "ast_patch"), patch_args(SOURCE)) + .await + .contains("symlink") + ); +} diff --git a/crates/tinyagents-harness-tools/src/code/publish.rs b/crates/tinyagents-harness-tools/src/code/publish.rs new file mode 100644 index 000000000..9192e329f --- /dev/null +++ b/crates/tinyagents-harness-tools/src/code/publish.rs @@ -0,0 +1,133 @@ +//! Same-directory staging and atomic publication after final source checks. + +use std::io::Write; +use std::path::PathBuf; +use std::sync::Arc; + +use anyhow::{Context, ensure}; +use tempfile::NamedTempFile; + +use super::{io, types::CodeTool}; +use crate::fs::{file_state, filesystem::FsGate}; + +pub(super) struct Destination { + pub(super) original: String, + pub(super) resolved: PathBuf, + pub(super) before: Option, +} + +impl CodeTool { + pub(super) fn permit_write(&self, preview: bool) -> anyhow::Result<()> { + if !preview { + self.require_write_authority()?; + ensure!( + !self.gate.is_rate_limited(), + "structural action budget exhausted" + ); + } + Ok(()) + } + + fn require_write_authority(&self) -> anyhow::Result<()> { + ensure!( + self.gate.can_act() && !self.gate.is_read_only(), + "[policy-blocked] structural writes are read-only under the host policy" + ); + Ok(()) + } + + pub(super) async fn publish( + &self, + gate: &Arc, + destination: &Destination, + guards: &[io::ReadGuard], + source: &str, + ) -> anyhow::Result<()> { + self.permit_write(false)?; + ensure!( + self.gate.record_action(), + "structural action budget exhausted" + ); + self.require_write_authority()?; + let staged = stage(destination, source).await?; + // Custom metadata and templates can change while parsing or staging. + // Revalidate their original paths as well as their exact content. + for guard in guards { + io::verify(gate, guard).await?; + } + let resolved = io::resolve_target(gate, &destination.original).await?; + ensure!( + resolved == destination.resolved, + "structural destination path changed during the call" + ); + if let Some(before) = &destination.before { + io::verify(gate, before).await?; + } else { + require_absent(&resolved).await?; + } + // Reauthorization can yield while the host revokes write access. + // The action is already reserved: recheck authority, not its budget. + self.require_write_authority()?; + if destination.before.is_some() { + staged + .persist(&resolved) + .map_err(|error| error.error) + .context("atomic structural replacement failed")?; + } else { + staged + .persist_noclobber(&resolved) + .map_err(|error| error.error) + .context("atomic structural creation failed")?; + } + if let Some(agent) = file_state::current_file_state_agent_id() { + file_state::record_write(&agent, resolved); + } + Ok(()) + } +} + +pub(super) async fn require_absent(path: &std::path::Path) -> anyhow::Result<()> { + match tokio::fs::symlink_metadata(path).await { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error).context("could not inspect creation destination"), + Ok(_) => anyhow::bail!("create_new refuses an existing destination"), + } +} + +async fn stage(destination: &Destination, source: &str) -> anyhow::Result { + let parent = destination + .resolved + .parent() + .context("destination has no parent")? + .to_path_buf(); + let permissions = if destination.before.is_some() { + Some( + tokio::fs::metadata(&destination.resolved) + .await + .context("could not inspect destination permissions")? + .permissions(), + ) + } else { + None + }; + let source = source.to_owned(); + tokio::task::spawn_blocking(move || { + let mut file = tempfile::Builder::new() + .prefix(".tinytools-code-") + .tempfile_in(parent) + .context("could not stage structural output; parent directory must exist")?; + file.write_all(source.as_bytes()) + .context("could not stage complete structural output")?; + if let Some(permissions) = permissions { + file.as_file() + .set_permissions(permissions) + .context("could not preserve file permissions")?; + } + file.as_file() + .sync_all() + .context("could not sync staged structural output")?; + Ok::<_, anyhow::Error>(file) + }) + .await + .context("structural staging task failed")? +} diff --git a/crates/tinyagents-harness-tools/src/code/response.rs b/crates/tinyagents-harness-tools/src/code/response.rs new file mode 100644 index 000000000..10d1b52af --- /dev/null +++ b/crates/tinyagents-harness-tools/src/code/response.rs @@ -0,0 +1,57 @@ +//! JSON receipt preflight and byte-safe model-visible paging. + +use anyhow::ensure; +use serde::Serialize; +use serde_json::{Value, json}; +use tinytools::ToolResult; + +pub(super) fn bounded_json(value: Value, maximum: usize) -> anyhow::Result { + let result = ToolResult::json(value); + ensure!( + result.output().len() <= maximum, + "structural result exceeds the {maximum}-byte output budget; reduce the requested page" + ); + Ok(result) +} + +pub(super) fn bounded_error(message: &str, maximum: usize) -> ToolResult { + let mut end = message.len().min(maximum); + while !message.is_char_boundary(end) { + end -= 1; + } + ToolResult::error(&message[..end]) +} + +pub(super) fn records(items: &[T], offset: usize, limit: usize) -> Value { + let start = offset.min(items.len()); + let end = start.saturating_add(limit).min(items.len()); + json!({ + "items": &items[start..end], + "offset": start, + "total": items.len(), + "next_offset": (end < items.len()).then_some(end) + }) +} + +pub(super) fn source_page( + source: &str, + start_line: usize, + line_count: usize, +) -> anyhow::Result { + ensure!(start_line > 0, "start_line must be at least 1"); + let total = source.lines().count().max(1); + let start = start_line.saturating_sub(1).min(total); + let end = start.saturating_add(line_count).min(total); + let text: String = source + .split_inclusive('\n') + .skip(start) + .take(end - start) + .collect(); + Ok(json!({ + "text": text, + "start_line": start + 1, + "total_lines": total, + "next_start_line": (end < total).then_some(end + 1), + "truncated": start > 0 || end < total + })) +} diff --git a/crates/tinyagents-harness-tools/src/code/test_support.rs b/crates/tinyagents-harness-tools/src/code/test_support.rs new file mode 100644 index 000000000..b8ccca59c --- /dev/null +++ b/crates/tinyagents-harness-tools/src/code/test_support.rs @@ -0,0 +1,204 @@ +//! A deterministic host gate and public-tool fixtures for native regressions. + +#![allow(clippy::unwrap_used)] + +use std::path::{Component, Path, PathBuf}; +use std::sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, +}; + +use async_trait::async_trait; +use parking_lot::Mutex; +use serde_json::{Value, json}; +use tinytools::{Tool, ToolRunContext, WorkspaceDescriptor}; + +use super::*; + +#[derive(Debug)] +pub(super) enum Action { + Write(PathBuf, String), + Revoke, + Remove(PathBuf), +} + +#[derive(Debug, Default)] +pub(super) struct State { + pub(super) read_only: AtomicBool, + pub(super) denied: AtomicBool, + pub(super) used: AtomicUsize, + pub(super) refuse_record: AtomicBool, + pub(super) action: Mutex>, + pub(super) parent_calls: AtomicUsize, + pub(super) deny_recheck: AtomicBool, + pub(super) revoke_at_recheck: AtomicBool, + pub(super) pause_recheck: Option<(Arc, Arc)>, +} + +#[derive(Debug)] +pub(super) struct TestGate { + pub(super) root: PathBuf, + pub(super) state: Arc, + pub(super) max_actions: usize, +} + +impl TestGate { + pub(super) fn new(root: &Path) -> Arc { + Arc::new(Self { + root: root.to_path_buf(), + state: Arc::new(State::default()), + max_actions: 100, + }) + } + + fn candidate(&self, path: &str) -> Result { + if self.state.denied.load(Ordering::SeqCst) || !self.is_path_string_allowed(path) { + return Err("[policy-blocked] test host denied path".into()); + } + let path = Path::new(path); + Ok(if path.is_absolute() { + path.to_path_buf() + } else { + self.root.join(path) + }) + } + + fn inside(&self, path: PathBuf) -> Result { + let root = self + .root + .canonicalize() + .map_err(|error| error.to_string())?; + if !path.starts_with(root) { + return Err("[policy-blocked] outside test workspace".into()); + } + Ok(path) + } +} + +#[async_trait] +impl FsGate for TestGate { + fn can_act(&self) -> bool { + !self.state.read_only.load(Ordering::SeqCst) + } + fn is_read_only(&self) -> bool { + !self.can_act() + } + fn is_rate_limited(&self) -> bool { + self.state.used.load(Ordering::SeqCst) >= self.max_actions + } + fn record_action(&self) -> bool { + if self.state.refuse_record.load(Ordering::SeqCst) { + return false; + } + let permitted = self.state.used.fetch_add(1, Ordering::SeqCst) < self.max_actions; + if let Some(action) = self.state.action.lock().take() { + match action { + Action::Write(path, source) => std::fs::write(path, source).unwrap(), + Action::Revoke => self.state.read_only.store(true, Ordering::SeqCst), + Action::Remove(path) => std::fs::remove_file(path).unwrap(), + } + } + permitted + } + fn write_needs_approval(&self) -> bool { + true + } + fn action_dir(&self) -> &Path { + &self.root + } + fn is_path_string_allowed(&self, path: &str) -> bool { + !path.contains('\0') + && !Path::new(path) + .components() + .any(|part| part == Component::ParentDir) + } + async fn validate_path(&self, path: &str) -> Result { + let full = self.candidate(path)?; + self.inside( + tokio::fs::canonicalize(full) + .await + .map_err(|error| error.to_string())?, + ) + } + async fn validate_parent_path(&self, path: &str) -> Result { + let calls = self.state.parent_calls.fetch_add(1, Ordering::SeqCst); + if calls > 0 { + if self.state.deny_recheck.load(Ordering::SeqCst) { + return Err("[policy-blocked] original path revoked".into()); + } + if let Some((entered, released)) = &self.state.pause_recheck { + entered.notify_one(); + released.notified().await; + } + if self.state.revoke_at_recheck.load(Ordering::SeqCst) { + self.state.read_only.store(true, Ordering::SeqCst); + } + } + let full = self.candidate(path)?; + let mut existing = full.parent().ok_or("missing parent")?.to_path_buf(); + while !existing.exists() { + existing = existing.parent().ok_or("missing ancestor")?.to_path_buf(); + } + let canonical = tokio::fs::canonicalize(&existing) + .await + .map_err(|error| error.to_string())?; + self.inside( + canonical.join( + full.strip_prefix(existing) + .map_err(|error| error.to_string())?, + ), + ) + } + fn scoped_to_workspace(&self, root: &Path) -> Arc { + Arc::new(Self { + root: root.to_path_buf(), + state: Arc::clone(&self.state), + max_actions: self.max_actions, + }) + } +} + +pub(super) struct Workspace(pub(super) WorkspaceDescriptor); +impl ToolRunContext for Workspace { + fn workspace(&self) -> Option<&WorkspaceDescriptor> { + Some(&self.0) + } +} + +pub(super) fn tool<'a>(tools: &'a [Box], name: &str) -> &'a dyn Tool { + tools + .iter() + .find(|tool| tool.name() == name) + .unwrap() + .as_ref() +} + +pub(super) async fn success(tool: &dyn Tool, args: Value) -> Value { + let result = tool.execute(args).await.unwrap(); + assert!(!result.is_error, "{}: {}", tool.name(), result.output()); + serde_json::from_str(&result.output()).unwrap() +} + +pub(super) async fn failure(tool: &dyn Tool, args: Value) -> String { + let result = tool.execute(args).await.unwrap(); + assert!(result.is_error, "unexpected success: {}", result.output()); + result.output() +} + +pub(super) fn patch_args(source: &str) -> Value { + json!({"path":"source.rs","expected_revision":tinytools_code::revision(source), + "selector":"value","operation":"replace_body","code":"2"}) +} + +pub(super) fn custom_manifest(root: &Path, target: &str, template: &str) { + std::fs::write(root.join("template.rs.txt"), template).unwrap(); + std::fs::write( + root.join("manifest.json"), + serde_json::to_vec(&json!({ + "id":"custom.pattern.v1", "summary":"Custom pattern", "language":"rust", + "target":target, "template_path":"template.rs.txt", "parameters":[], "slots":[] + })) + .unwrap(), + ) + .unwrap(); +} diff --git a/crates/tinyagents-harness-tools/src/code/types.rs b/crates/tinyagents-harness-tools/src/code/types.rs new file mode 100644 index 000000000..24c66c017 --- /dev/null +++ b/crates/tinyagents-harness-tools/src/code/types.rs @@ -0,0 +1,110 @@ +//! Host-configured budgets and the shared native adapter state. + +use std::sync::Arc; + +use anyhow::ensure; +use parking_lot::Mutex; +use tinytools_code::{ChunkCatalog, CodeEngine, CodeLimits}; + +use crate::fs::filesystem::FsGate; + +/// Resource limits for the optional structural tools. +/// +/// Source and parser-cache limits live in [`CodeLimits`]. The remaining limits +/// cover native argument decoding, metadata, templates and model-visible output. +#[derive(Debug, Clone, Copy)] +pub struct CodeToolOptions { + /// Pure parser, source, cache and record bounds. + pub limits: CodeLimits, + /// Maximum serialized argument bytes accepted by one invocation. + pub max_argument_bytes: usize, + /// Maximum bytes read from a custom manifest. + pub max_manifest_bytes: usize, + /// Maximum bytes read from a custom template. + pub max_template_bytes: usize, + /// Maximum UTF-8 bytes in the model-visible JSON or error text. + pub max_output_bytes: usize, + /// Maximum records returned per list page; also the default page size. + pub max_page_items: usize, + /// Maximum source lines returned per page; also the default page size. + pub max_page_lines: usize, +} + +impl Default for CodeToolOptions { + fn default() -> Self { + Self { + limits: CodeLimits::default(), + max_argument_bytes: 1_048_576, + max_manifest_bytes: 65_536, + max_template_bytes: 262_144, + max_output_bytes: 65_536, + max_page_items: 32, + max_page_lines: 80, + } + } +} + +impl CodeToolOptions { + pub(super) fn validate(self) -> anyhow::Result<()> { + for (name, value, maximum) in [ + ("argument", self.max_argument_bytes, 16_777_216), + ("manifest", self.max_manifest_bytes, 1_048_576), + ("template", self.max_template_bytes, 16_777_216), + ("output", self.max_output_bytes, 16_777_216), + ("page item", self.max_page_items, 1024), + ("page line", self.max_page_lines, 4096), + ] { + ensure!(value > 0 && value <= maximum, "invalid {name} budget"); + } + Ok(()) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Kind { + Outline, + Symbol, + Patch, + Search, + Inspect, + Apply, +} + +impl Kind { + pub(super) const ALL: [Self; 6] = [ + Self::Outline, + Self::Symbol, + Self::Patch, + Self::Search, + Self::Inspect, + Self::Apply, + ]; + + pub(super) fn name(self) -> &'static str { + match self { + Self::Outline => "ast_outline", + Self::Symbol => "ast_symbol", + Self::Patch => "ast_patch", + Self::Search => "chunk_search", + Self::Inspect => "chunk_inspect", + Self::Apply => "chunk_apply", + } + } + + pub(super) fn mutates(self) -> bool { + matches!(self, Self::Patch | Self::Apply) + } +} + +pub(super) struct Shared { + pub(super) engine: Mutex, + pub(super) catalog: ChunkCatalog, + pub(super) write_lock: tokio::sync::Mutex<()>, + pub(super) options: CodeToolOptions, +} + +pub(super) struct CodeTool { + pub(super) kind: Kind, + pub(super) gate: Arc, + pub(super) shared: Arc, +} diff --git a/crates/tinyagents-harness-tools/src/lib.rs b/crates/tinyagents-harness-tools/src/lib.rs index 348ef5d6c..6f2f55b6b 100644 --- a/crates/tinyagents-harness-tools/src/lib.rs +++ b/crates/tinyagents-harness-tools/src/lib.rs @@ -19,6 +19,11 @@ mod time; mod time_parse; mod wait; +#[cfg(feature = "code")] +pub mod code; +#[cfg(feature = "code")] +pub use code::{CodeToolOptions, code_tools, code_tools_with_options}; + #[cfg(feature = "exec")] pub mod exec; #[cfg(feature = "fs")] diff --git a/crates/tinyagents-harness/src/run_queue/mod.rs b/crates/tinyagents-harness/src/run_queue/mod.rs index fd80fe305..5b0b8fbc5 100644 --- a/crates/tinyagents-harness/src/run_queue/mod.rs +++ b/crates/tinyagents-harness/src/run_queue/mod.rs @@ -487,10 +487,10 @@ impl RunQueue { .iter_mut() .chain(followups.iter_mut()) .chain(collects.iter_mut()) - .find(|item| pred(item)); + .find(|item| pred(&item.value)); match item { Some(item) => { - edit(item); + edit(&mut item.value); true } None => false, @@ -520,7 +520,7 @@ impl RunQueue { [steers, followups, collects].into_iter().find_map(|items| { items .iter() - .position(&pred) + .position(|item| pred(&item.value)) .map(|position| items.remove(position)) }) }; diff --git a/crates/tinyagents-session/src/run_ledger/drivers/inbox.rs b/crates/tinyagents-session/src/run_ledger/drivers/inbox.rs index 29a58c9c9..8e5a954ab 100644 --- a/crates/tinyagents-session/src/run_ledger/drivers/inbox.rs +++ b/crates/tinyagents-session/src/run_ledger/drivers/inbox.rs @@ -6,7 +6,7 @@ use super::*; use crate::run_ledger::inbox::{AddressedInput, QueuedInput, RunInboxStore}; use serde::{Deserialize, Serialize}; use std::collections::HashSet; -use tinystoragedrivers_core::{ErrorKind, ExpectedDocument, Precondition, ReadCheck}; +use tinystoragedrivers_types::{ErrorKind, ExpectedDocument, Precondition, ReadCheck}; #[derive(Clone, Serialize, Deserialize)] struct StoredInput { diff --git a/crates/tinyagents-session/src/run_ledger/drivers/inbox_tests.rs b/crates/tinyagents-session/src/run_ledger/drivers/inbox_tests.rs index d4cdd3472..5cdfdba41 100644 --- a/crates/tinyagents-session/src/run_ledger/drivers/inbox_tests.rs +++ b/crates/tinyagents-session/src/run_ledger/drivers/inbox_tests.rs @@ -1,6 +1,6 @@ use super::*; use crate::run_ledger::inbox::InputMode; -use tinystoragedrivers_core::{MemoryStorage, StorageBackend}; +use crate::testkit::MemoryStorage; fn ledger() -> DriverRunLedger { DriverRunLedger::new( diff --git a/vendor/tinytools b/vendor/tinytools index a6a46ba4b..4662b8533 160000 --- a/vendor/tinytools +++ b/vendor/tinytools @@ -1 +1 @@ -Subproject commit a6a46ba4b8818eeaee2142cf6346e16bbd7b5caa +Subproject commit 4662b85336c20a461c355a8b4ef79e08c9bc3292 From 95e7dbea639b1559d1905162e825b46495916b8b Mon Sep 17 00:00:00 2001 From: Farnam Date: Tue, 27 Oct 2026 22:38:15 +0330 Subject: [PATCH 11/14] fix(submodules): use canonical TinyTools source for upstream-reachable structural parser --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 5c74d2ac4..3c1225634 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,7 +3,7 @@ url = https://github.com/tinyhumansai/tinyagents.wiki.git [submodule "vendor/tinytools"] path = vendor/tinytools - url = https://github.com/AmirrezaFarnamTaheri/tinytools.git + url = https://github.com/tinyhumansai/tinytools.git [submodule "vendor/tinyinference"] path = vendor/tinyinference url = https://github.com/tinyhumansai/tinyinference.git From 61b87c4891f5e023aedb9018bc8664f41a38b8c6 Mon Sep 17 00:00:00 2001 From: Farnam Date: Tue, 27 Oct 2026 22:38:21 +0330 Subject: [PATCH 12/14] docs(code): resolve structural specification and plan through vendored parser owner --- crates/tinyagents-harness-tools/src/code/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/tinyagents-harness-tools/src/code/README.md b/crates/tinyagents-harness-tools/src/code/README.md index 30b01624d..586170b67 100644 --- a/crates/tinyagents-harness-tools/src/code/README.md +++ b/crates/tinyagents-harness-tools/src/code/README.md @@ -157,5 +157,5 @@ unrelated writer that changes a file between the last revision check and rename, or power-loss durability of a directory entry. Filesystem confinement remains the host gate's responsibility. -See the accepted [specification](../../../../docs/specs/structural-code.md) and -[implementation plan](../../../../docs/plans/structural-code.md). +See the accepted [specification](../../../../vendor/tinytools/docs/specs/structural-code.md) and +[implementation plan](../../../../vendor/tinytools/docs/plans/structural-code.md). From 931f824371220038e599d9832cafc49a23c836da Mon Sep 17 00:00:00 2001 From: Farnam Date: Tue, 27 Oct 2026 22:41:34 +0330 Subject: [PATCH 13/14] test(queue): reproduce edit and move races during durable admission Retry (RED) --- .../src/run_queue/mod_tests.rs | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/crates/tinyagents-harness/src/run_queue/mod_tests.rs b/crates/tinyagents-harness/src/run_queue/mod_tests.rs index dc7fd439d..d2c1d2784 100644 --- a/crates/tinyagents-harness/src/run_queue/mod_tests.rs +++ b/crates/tinyagents-harness/src/run_queue/mod_tests.rs @@ -30,6 +30,116 @@ impl QueueAdmission for RetryOnce { } } +// Admission may consult a durable lifecycle while the queue mutex is free. +// Concurrent edits and lane moves must not falsely report a Retry item as +// consumed during that temporary absence from the visible lane. +#[derive(Debug)] +struct PausedRetryAdmission { + entered: Arc, + release: Arc, +} + +impl QueueAdmission for PausedRetryAdmission { + fn acquire(&self) -> Result, QueueAdmissionError> { + self.entered.wait(); + self.release.wait(); + Err(QueueAdmissionError::Retry) + } +} + +#[tokio::test] +async fn edit_waits_for_in_flight_admission_before_reporting_consumed() { + use std::future::Future; + use std::task::{Context, Poll, Waker}; + + let queue = Arc::new(RunQueue::new()); + let entered = Arc::new(std::sync::Barrier::new(2)); + let release = Arc::new(std::sync::Barrier::new(2)); + queue + .push_guarded( + QueueLane::Followup, + "pending".to_string(), + Arc::new(PausedRetryAdmission { + entered: Arc::clone(&entered), + release: Arc::clone(&release), + }), + ) + .await; + let worker_queue = Arc::clone(&queue); + let worker = std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .build() + .unwrap() + .block_on(worker_queue.take_admitted(QueueLane::Followup, QueueMode::All)) + .is_empty() + }); + entered.wait(); + let mut edit = std::pin::pin!(queue.edit_first_where( + |item| item == "pending", + |item| *item = "revised".to_string(), + )); + let mut context = Context::from_waker(Waker::noop()); + let pending = matches!(Future::poll(edit.as_mut(), &mut context), Poll::Pending); + release.wait(); + assert!(worker.join().unwrap(), "first admission must retry"); + assert!( + pending, + "editing an in-flight item must wait, not report already consumed" + ); + assert!(edit.await); + assert_eq!( + queue.snapshot().await, + vec![(QueueLane::Followup, "revised".to_string())] + ); +} + +#[tokio::test] +async fn move_waits_for_in_flight_admission_before_reporting_consumed() { + use std::future::Future; + use std::task::{Context, Poll, Waker}; + + let queue = Arc::new(RunQueue::new()); + let entered = Arc::new(std::sync::Barrier::new(2)); + let release = Arc::new(std::sync::Barrier::new(2)); + queue + .push_guarded( + QueueLane::Followup, + "pending".to_string(), + Arc::new(PausedRetryAdmission { + entered: Arc::clone(&entered), + release: Arc::clone(&release), + }), + ) + .await; + let worker_queue = Arc::clone(&queue); + let worker = std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .build() + .unwrap() + .block_on(worker_queue.take_admitted(QueueLane::Followup, QueueMode::OneAtATime)) + .is_empty() + }); + entered.wait(); + let mut move_item = std::pin::pin!(queue.move_first_where( + |item| item == "pending", + QueueLane::Steer, + 0, + )); + let mut context = Context::from_waker(Waker::noop()); + let pending = matches!(Future::poll(move_item.as_mut(), &mut context), Poll::Pending); + release.wait(); + assert!(worker.join().unwrap(), "first admission must retry"); + assert!( + pending, + "moving an in-flight item must wait, not report already consumed" + ); + assert!(move_item.await); + assert_eq!( + queue.snapshot().await, + vec![(QueueLane::Steer, "pending".to_string())] + ); +} + #[tokio::test] async fn transient_admission_result_retains_item_for_the_next_boundary() { let queue = RunQueue::new(); From 94a306b0111602174b1cd397da0fbb47b8cfa99b Mon Sep 17 00:00:00 2001 From: Farnam Date: Tue, 27 Oct 2026 22:42:47 +0330 Subject: [PATCH 14/14] fix(queue): serialize edit and lane moves with in-flight durable admission --- crates/tinyagents-harness/src/run_queue/mod.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/crates/tinyagents-harness/src/run_queue/mod.rs b/crates/tinyagents-harness/src/run_queue/mod.rs index 5b0b8fbc5..992b34f54 100644 --- a/crates/tinyagents-harness/src/run_queue/mod.rs +++ b/crates/tinyagents-harness/src/run_queue/mod.rs @@ -58,6 +58,11 @@ pub use types::{ #[derive(Debug)] pub struct RunQueue { inner: Mutex>, + /// Serialize admission with host edits and lane moves. Admission releases + /// the lane mutex while consulting its lifecycle capability, so without + /// this separate gate a Retry item temporarily looks already consumed. + /// Producers still push without taking this gate. + admission_mutation_gate: tokio::sync::Mutex<()>, } /// The three FIFO lanes behind [`RunQueue`]'s lock. @@ -121,6 +126,7 @@ impl RunQueue { followups: Vec::new(), collects: Vec::new(), }), + admission_mutation_gate: tokio::sync::Mutex::new(()), } } @@ -172,6 +178,11 @@ impl RunQueue { lane: QueueLane, mode: QueueMode, ) -> Vec> { + // A temporarily removed item must be either fully admitted/rejected or + // requeued before edit_first_where/move_first_where decide it is gone. + // Never hold the lane mutex while acquiring a durable permit; queue + // producers are not serialized by this gate. + let _admission_guard = self.admission_mutation_gate.lock().await; if mode == QueueMode::OneAtATime { let mut permits = std::collections::HashMap::new(); let count = { @@ -477,6 +488,9 @@ impl RunQueue { pred: impl Fn(&T) -> bool, edit: impl FnOnce(&mut T), ) -> bool { + // If take_admitted is inspecting a removed guarded item, wait until + // Retry has requeued it or acceptance has definitively consumed it. + let _admission_guard = self.admission_mutation_gate.lock().await; let mut inner = self.lock(); let RunQueueInner { steers, @@ -510,6 +524,7 @@ impl RunQueue { lane: QueueLane, index: usize, ) -> bool { + let _admission_guard = self.admission_mutation_gate.lock().await; let mut inner = self.lock(); let item = { let RunQueueInner {