From 3fbcb4b0d32412a9df089faf513f88856c38e38a Mon Sep 17 00:00:00 2001 From: sanil-23 Date: Wed, 14 Oct 2026 21:34:27 +0530 Subject: [PATCH 1/2] Let recall and fetch leave a thread's turns out Add exclude_thread, a ThreadWindow, to RecallRequest and FetchRequest. A host reading inside a conversation logs the user's turn before it reads, so without an exclusion the message being answered comes back as a memory. ThreadWindow moves into tinymemory-api (tinymemory-tools re-exports it), with covers_meta and validate. The reference engine drops covered items; the CortexDB engine drops them from the decoded pack events, so fetch hits and recall citations leave the thread out (CortexDB narrows only by label, and its answer text is still written from the whole pack). Holistic recall's answer sections pass the pack's window to the engine. A new conformance check makes every engine honour it. The exclusion is on read requests only, never on MetaFilter, so it cannot narrow a forget or an erase. Co-Authored-By: Claude Opus 5.5 --- .../src/conformance/reference/mod.rs | 8 +- .../src/conformance/suite/checks.rs | 80 ++++++++++++++++++- crates/tinymemory-api/src/lib.rs | 3 +- crates/tinymemory-api/src/query/mod.rs | 79 +++++++++++++++++- crates/tinymemory-api/src/query/mod_tests.rs | 48 +++++++++++ .../src/cortex/engine/fetch.rs | 19 ++++- .../src/cortex/engine/fetch_tests.rs | 19 ++++- .../src/cortex/engine/recall.rs | 10 ++- crates/tinymemory-tools/src/recall/gather.rs | 3 + crates/tinymemory-tools/src/recall/types.rs | 27 +------ crates/tinymemory-tools/src/tools/read/mod.rs | 2 + docs/architecture/cortex-flows.md | 10 ++- docs/specs/agent-memory.md | 4 +- 13 files changed, 269 insertions(+), 43 deletions(-) diff --git a/crates/tinymemory-api/src/conformance/reference/mod.rs b/crates/tinymemory-api/src/conformance/reference/mod.rs index d9783ab3..aac00b87 100644 --- a/crates/tinymemory-api/src/conformance/reference/mod.rs +++ b/crates/tinymemory-api/src/conformance/reference/mod.rs @@ -159,7 +159,10 @@ impl MemoryEngine for ReferenceEngine { async fn recall(&self, req: RecallRequest) -> Result { req.validate()?; - let hits = self.ranked(&req.question, FetchMode::Hybrid, &req.filter)?; + let mut hits = self.ranked(&req.question, FetchMode::Hybrid, &req.filter)?; + if let Some(window) = &req.exclude_thread { + hits.retain(|hit| !window.covers(hit)); + } let citations: Vec = hits .into_iter() .take(req.limit) @@ -192,6 +195,9 @@ impl MemoryEngine for ReferenceEngine { self.descriptor.ensure_mode(req.mode)?; req.validate()?; let mut hits = self.ranked(&req.query, req.mode, &req.filter)?; + if let Some(window) = &req.exclude_thread { + hits.retain(|hit| !window.covers(hit)); + } if let Some(hint) = &req.refers_to { hint.rank(&mut hits); } diff --git a/crates/tinymemory-api/src/conformance/suite/checks.rs b/crates/tinymemory-api/src/conformance/suite/checks.rs index 3852a2ae..cdf73df3 100644 --- a/crates/tinymemory-api/src/conformance/suite/checks.rs +++ b/crates/tinymemory-api/src/conformance/suite/checks.rs @@ -4,7 +4,7 @@ use std::collections::BTreeSet; use crate::{ Error as ApiError, FetchMode, FetchRequest, ForgetTarget, ItemId, MemoryMeta, MetaFilter, - RecallRequest, + RecallRequest, Role, StoreItem, ThreadWindow, Turn, TurnRange, }; use super::{Ctx, ensure}; @@ -21,6 +21,7 @@ pub(super) async fn all(ctx: &Ctx<'_>) -> Result<()> { super::bulk::store_many(ctx).await?; super::lifecycle::store_with(ctx).await?; fetch_filters(ctx).await?; + excluded_thread(ctx).await?; unsupported_modes(ctx).await?; super::namespaces::namespaces(ctx).await?; empty_forget(ctx).await?; @@ -181,6 +182,83 @@ async fn fetch_filters(ctx: &Ctx<'_>) -> Result<()> { Ok(()) } +/// A read's `exclude_thread` leaves that thread's turns out of every fetch +/// mode and of recall's citations, and a window starting after a turn keeps +/// it. +async fn excluded_thread(ctx: &Ctx<'_>) -> Result<()> { + const CHECK: &str = "excluded_thread"; + let turn = |thread: &str| { + let mut meta = ctx.run.meta(); + meta.thread_id = Some(thread.to_string()); + meta.turns = Some(TurnRange { first: 0, last: 0 }); + StoreItem::Conversation { + turns: vec![Turn::new( + Role::User, + format!("{} the turn being answered in {thread}", ctx.run.marker), + )], + meta, + } + }; + let (live, other) = ( + format!("{}-live", ctx.run.marker), + format!("{}-other", ctx.run.marker), + ); + let live_id = ctx.call(CHECK, ctx.engine.store(turn(&live))).await?.id; + let other_id = ctx.call(CHECK, ctx.engine.store(turn(&other))).await?.id; + let modes = ctx.engine.descriptor().fetch_modes.clone(); + for mode in &modes { + let mut request = FetchRequest::new(ctx.run.marker.clone(), *mode, 50); + request.filter = ctx.run.filter(); + request.exclude_thread = Some(ThreadWindow::whole(&live)); + let page = ctx.call(CHECK, ctx.engine.fetch(request.clone())).await?; + let found: BTreeSet<&ItemId> = page.hits.iter().map(|hit| &hit.id).collect(); + ensure( + CHECK, + !found.contains(&live_id) && found.contains(&other_id), + || { + format!( + "{} fetch excluding {live} returned {found:?}: expected {other_id}, not {live_id}", + mode.as_str() + ) + }, + )?; + request.exclude_thread = Some(ThreadWindow { + thread_id: live.clone(), + from_turn: 1, + }); + let page = ctx.call(CHECK, ctx.engine.fetch(request)).await?; + ensure(CHECK, page.hits.iter().any(|hit| hit.id == live_id), || { + format!( + "{} fetch excluding turns of {live} from 1 on dropped its turn 0", + mode.as_str() + ) + })?; + } + let mut request = RecallRequest::new(ctx.run.marker.clone(), 50); + request.filter = ctx.run.filter(); + request.exclude_thread = Some(ThreadWindow::whole(&live)); + let answer = ctx.call(CHECK, ctx.engine.recall(request)).await?; + ensure( + CHECK, + answer + .citations + .iter() + .all(|citation| citation.id != live_id), + || format!("a recall excluding {live} cited its turn {live_id}"), + )?; + let blank = ThreadWindow::whole(" "); + let mut request = FetchRequest::new(ctx.run.marker.clone(), FetchMode::Hybrid, 5); + request.exclude_thread = Some(blank.clone()); + ensure(CHECK, request.validate().is_err(), || { + "a fetch excluding a blank thread id validated".to_string() + })?; + let mut request = RecallRequest::new(ctx.run.marker.clone(), 5); + request.exclude_thread = Some(blank); + ensure(CHECK, request.validate().is_err(), || { + "a recall excluding a blank thread id validated".to_string() + }) +} + async fn unsupported_modes(ctx: &Ctx<'_>) -> Result<()> { const CHECK: &str = "unsupported_modes"; for mode in FetchMode::ALL { diff --git a/crates/tinymemory-api/src/lib.rs b/crates/tinymemory-api/src/lib.rs index 737800c1..830d420e 100644 --- a/crates/tinymemory-api/src/lib.rs +++ b/crates/tinymemory-api/src/lib.rs @@ -85,7 +85,8 @@ pub use namespace::{Namespace, Reach, Segment, SegmentKind}; pub use provenance::{ProvenancePolicy, wrap_untrusted_for_agent}; pub use query::{ Citation, EraseReport, EraseRequest, ExportPage, Exported, FetchMode, FetchPage, FetchRequest, - ForgetReport, ForgetTarget, Hit, ListPage, ListRequest, RecallAnswer, RecallRequest, TimeHint, + ForgetReport, ForgetTarget, Hit, ListPage, ListRequest, RecallAnswer, RecallRequest, + ThreadWindow, TimeHint, }; pub use write::{WaitFor, WriteOptions}; diff --git a/crates/tinymemory-api/src/query/mod.rs b/crates/tinymemory-api/src/query/mod.rs index 6f176f4f..97a41f45 100644 --- a/crates/tinymemory-api/src/query/mod.rs +++ b/crates/tinymemory-api/src/query/mod.rs @@ -39,6 +39,11 @@ pub struct RecallRequest { /// ranking hint, never a filter (see [`TimeHint`]). #[serde(default, skip_serializing_if = "Option::is_none")] pub refers_to: Option, + /// Conversation turns never cited: the calling thread's, which its + /// prompt already holds (see [`ThreadWindow`]). Read-only, so it can + /// never narrow a forget or an erase. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub exclude_thread: Option, } impl RecallRequest { @@ -51,6 +56,7 @@ impl RecallRequest { limit, instructions: None, refers_to: None, + exclude_thread: None, } } @@ -58,15 +64,72 @@ impl RecallRequest { /// /// # Errors /// - /// [`Error::InvalidRequest`] for a blank question, a zero limit or an - /// invalid [`TimeHint`]. + /// [`Error::InvalidRequest`] for a blank question, a zero limit, an + /// invalid [`TimeHint`] or a blank excluded thread. pub fn validate(&self) -> Result<()> { non_blank("recall question", &self.question)?; positive("recall limit", self.limit)?; + self.exclude_thread + .as_ref() + .map_or(Ok(()), ThreadWindow::validate)?; self.refers_to.as_ref().map_or(Ok(()), TimeHint::validate) } } +/// A thread's turns a read leaves out: every turn of `thread_id` from +/// `from_turn` on. A caller reading inside a thread sets it to the turns its +/// prompt already holds, so the message being answered (logged before the +/// read) does not come back as a memory; `from_turn: 0` leaves the whole +/// thread out. Engines apply it to what they return; an engine whose store +/// cannot exclude server side drops the matching items from its results. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ThreadWindow { + /// The thread. + pub thread_id: String, + /// The first turn index still in the prompt; turns from here on are + /// left out. + #[serde(default)] + pub from_turn: u32, +} + +impl ThreadWindow { + /// The whole of `thread_id`. + #[must_use] + pub fn whole(thread_id: impl Into) -> Self { + Self { + thread_id: thread_id.into(), + from_turn: 0, + } + } + + /// Whether `hit` lies inside the window. + #[must_use] + pub fn covers(&self, hit: &Hit) -> bool { + self.covers_meta(&hit.meta) + } + + /// Whether an item carrying `meta` lies inside the window: it belongs + /// to the thread, and its turns (when it records them) reach + /// `from_turn`. + #[must_use] + pub fn covers_meta(&self, meta: &MemoryMeta) -> bool { + meta.thread_id.as_deref() == Some(self.thread_id.as_str()) + && meta + .turns + .as_ref() + .is_none_or(|turns| turns.last >= self.from_turn) + } + + /// Checks the window names a thread. + /// + /// # Errors + /// + /// [`Error::InvalidRequest`] for a blank thread id. + pub fn validate(&self) -> Result<()> { + non_blank("excluded thread id", &self.thread_id) + } +} + /// A synthesised answer. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct RecallAnswer { @@ -157,6 +220,11 @@ pub struct FetchRequest { /// scopes ignores it. Must be positive. #[serde(default, skip_serializing_if = "Option::is_none")] pub max_scopes: Option, + /// Conversation turns never returned: the calling thread's (see + /// [`ThreadWindow`]). Read-only, so it can never narrow a forget or an + /// erase. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub exclude_thread: Option, } /// Whether `n` is zero (serde's skip test). @@ -177,6 +245,7 @@ impl FetchRequest { beliefs: 0, refers_to: None, max_scopes: None, + exclude_thread: None, } } @@ -185,13 +254,17 @@ impl FetchRequest { /// # Errors /// /// [`Error::InvalidRequest`] for a blank query, a zero limit, a zero - /// [`FetchRequest::max_scopes`] or an invalid [`TimeHint`]. + /// [`FetchRequest::max_scopes`], an invalid [`TimeHint`] or a blank + /// excluded thread. pub fn validate(&self) -> Result<()> { non_blank("fetch query", &self.query)?; positive("fetch limit", self.limit)?; if let Some(scopes) = self.max_scopes { positive("fetch max_scopes", scopes)?; } + self.exclude_thread + .as_ref() + .map_or(Ok(()), ThreadWindow::validate)?; self.refers_to.as_ref().map_or(Ok(()), TimeHint::validate) } } diff --git a/crates/tinymemory-api/src/query/mod_tests.rs b/crates/tinymemory-api/src/query/mod_tests.rs index 79db2264..a711c788 100644 --- a/crates/tinymemory-api/src/query/mod_tests.rs +++ b/crates/tinymemory-api/src/query/mod_tests.rs @@ -83,3 +83,51 @@ fn a_fetch_reads_every_scope_unless_capped_and_never_zero() { let json = serde_json::to_value(FetchRequest::new("q", FetchMode::Hybrid, 3)).unwrap(); assert!(json.get("max_scopes").is_none()); } + +#[test] +fn a_thread_window_covers_its_thread_from_its_first_turn_on() { + let turn = |thread: Option<&str>, turns: Option<(u32, u32)>| MemoryMeta { + thread_id: thread.map(str::to_string), + turns: turns.map(|(first, last)| crate::meta::TurnRange { first, last }), + ..MemoryMeta::default() + }; + let whole = ThreadWindow::whole("t1"); + assert!(whole.covers_meta(&turn(Some("t1"), Some((0, 0))))); + assert!( + whole.covers_meta(&turn(Some("t1"), None)), + "an item without turns is the thread's" + ); + assert!(!whole.covers_meta(&turn(Some("t2"), Some((0, 0))))); + assert!(!whole.covers_meta(&turn(None, None))); + let in_prompt = ThreadWindow { + thread_id: "t1".into(), + from_turn: 4, + }; + assert!( + !in_prompt.covers_meta(&turn(Some("t1"), Some((0, 3)))), + "compacted turns stay recallable" + ); + assert!(in_prompt.covers_meta(&turn(Some("t1"), Some((2, 4))))); +} + +#[test] +fn reads_refuse_a_blank_excluded_thread() { + let mut recall = RecallRequest::new("q", 5); + recall.exclude_thread = Some(ThreadWindow::whole("")); + assert!(matches!(recall.validate(), Err(Error::InvalidRequest(_)))); + recall.exclude_thread = Some(ThreadWindow::whole("t1")); + assert!(recall.validate().is_ok()); + let mut fetch = FetchRequest::new("q", FetchMode::Hybrid, 5); + fetch.exclude_thread = Some(ThreadWindow::whole(" ")); + assert!(matches!(fetch.validate(), Err(Error::InvalidRequest(_)))); +} + +#[test] +fn a_read_without_an_excluded_thread_serializes_as_before() { + let json = serde_json::to_value(RecallRequest::new("q", 5)).unwrap(); + assert!(json.get("exclude_thread").is_none()); + let parsed: FetchRequest = + serde_json::from_value(serde_json::json!({"query": "q", "mode": "hybrid", "limit": 5})) + .unwrap(); + assert_eq!(parsed.exclude_thread, None); +} diff --git a/crates/tinymemory-integrations/src/cortex/engine/fetch.rs b/crates/tinymemory-integrations/src/cortex/engine/fetch.rs index 486e7254..377f5d63 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/fetch.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/fetch.rs @@ -32,7 +32,7 @@ use std::time::Instant; use futures::{StreamExt, TryStreamExt, stream}; use serde_json::{Value, json}; use tinymemory_api::{ - FetchPage, FetchRequest, Hit, ItemKind, MetaFilter, Namespace, Reach, StoreItem, + FetchPage, FetchRequest, Hit, ItemKind, MetaFilter, Namespace, Reach, StoreItem, ThreadWindow, }; use super::CortexEngine; @@ -116,8 +116,13 @@ pub(super) fn whole_items_budget(items: usize) -> usize { } /// The distinct items of `kind` a pack's events decode to, best rank first, -/// keeping only what `filter` matches. -pub(super) fn ranked(pack: &Value, kind: Option, filter: &MetaFilter) -> Vec { +/// keeping only what `filter` matches and `exclude` does not cover. +pub(super) fn ranked( + pack: &Value, + kind: Option, + filter: &MetaFilter, + exclude: Option<&ThreadWindow>, +) -> Vec { let mut seen = HashSet::new(); pack.pointer("/layers/events") .and_then(Value::as_array) @@ -126,6 +131,9 @@ pub(super) fn ranked(pack: &Value, kind: Option, filter: &MetaFilter) .filter_map(decode_event) .map(|decoded| decoded.envelope) .filter(|envelope| keeps(filter, kind.unwrap_or(envelope.kind), envelope)) + // CortexDB's recall filters can only narrow by label, so an excluded + // thread is dropped here, from what the pack returned. + .filter(|envelope| exclude.is_none_or(|window| !window.covers_meta(&envelope.meta))) .filter(|envelope| seen.insert(envelope.id.clone())) .collect() } @@ -201,7 +209,10 @@ impl CortexEngine { } else { Vec::new() }; - Ok::<_, Error>((ranked(&pack, Some(scope.kind), &req.filter), beliefs)) + Ok::<_, Error>(( + ranked(&pack, Some(scope.kind), &req.filter, req.exclude_thread.as_ref()), + beliefs, + )) }) .buffered(PACKS_AT_ONCE) .try_collect() diff --git a/crates/tinymemory-integrations/src/cortex/engine/fetch_tests.rs b/crates/tinymemory-integrations/src/cortex/engine/fetch_tests.rs index e331c425..dabb616f 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/fetch_tests.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/fetch_tests.rs @@ -25,7 +25,12 @@ fn a_pack_ranks_each_item_once_at_its_best_position_and_filters() { event("e2", &b), event("e3", &a), ] } }); - let all = ranked(&pack, Some(ItemKind::Document), &MetaFilter::default()); + let all = ranked( + &pack, + Some(ItemKind::Document), + &MetaFilter::default(), + None, + ); let ids: Vec<_> = all.iter().map(|e| e.id.clone()).collect(); assert_eq!(ids, vec![a.fingerprint(), b.fingerprint()]); @@ -33,10 +38,18 @@ fn a_pack_ranks_each_item_once_at_its_best_position_and_filters() { repo: Some("o/b".into()), ..MetaFilter::default() }; - let filtered = ranked(&pack, Some(ItemKind::Document), &only_b); + let filtered = ranked(&pack, Some(ItemKind::Document), &only_b, None); assert_eq!(filtered.len(), 1); assert_eq!(filtered[0].text, "beta"); - assert!(ranked(&pack, Some(ItemKind::Learning), &MetaFilter::default()).is_empty()); + assert!( + ranked( + &pack, + Some(ItemKind::Learning), + &MetaFilter::default(), + None + ) + .is_empty() + ); } #[test] diff --git a/crates/tinymemory-integrations/src/cortex/engine/recall.rs b/crates/tinymemory-integrations/src/cortex/engine/recall.rs index 85f9c35a..720e2c6c 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/recall.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/recall.rs @@ -22,11 +22,17 @@ //! `subtree` lane (experimental and off by default in 0.10.4). //! //! Citations come from the packs' `layers.events`, decoded back to items, -//! filtered by the full [`tinymemory_api::MetaFilter`] (reach included), and +//! filtered by the full [`tinymemory_api::MetaFilter`] (reach included) and +//! [`tinymemory_api::RecallRequest::exclude_thread`], and //! merged rank by rank across the packs (each pack's best, the most specific //! node's first, then each pack's second, …), one per item, at most `limit`. //! CortexDB scores none of them. A pack with no decodable events still //! returns the engine's answer, with no citations. +//! +//! **An excluded thread leaves the citations, not the answer text.** CortexDB +//! narrows a recall only by label, so `exclude_thread` is applied to the +//! decoded events; the server writes the answer from the whole pack, which +//! may still hold the excluded turns. use std::collections::HashSet; @@ -198,7 +204,7 @@ impl CortexEngine { .await?; let per_pack: Vec> = packs .iter() - .map(|(_, pack)| ranked(pack, None, &req.filter)) + .map(|(_, pack)| ranked(pack, None, &req.filter, req.exclude_thread.as_ref())) .collect(); let chosen = per_pack .iter() diff --git a/crates/tinymemory-tools/src/recall/gather.rs b/crates/tinymemory-tools/src/recall/gather.rs index a4149af5..f4ad29e8 100644 --- a/crates/tinymemory-tools/src/recall/gather.rs +++ b/crates/tinymemory-tools/src/recall/gather.rs @@ -228,6 +228,9 @@ async fn answer( limit: section.limit, instructions, refers_to: request.refers_to.clone(), + // The engine leaves the thread out itself, so the window's turns + // do not use up the section's citations. + exclude_thread: request.exclude_thread.clone(), }) .await?; if answer.citations.is_empty() || answer.answer.trim().is_empty() { diff --git a/crates/tinymemory-tools/src/recall/types.rs b/crates/tinymemory-tools/src/recall/types.rs index bda0c8cb..d850e710 100644 --- a/crates/tinymemory-tools/src/recall/types.rs +++ b/crates/tinymemory-tools/src/recall/types.rs @@ -142,30 +142,9 @@ impl ScopeSection { } } -/// Part of a conversation thread a pack leaves out because the host's prompt -/// already holds it. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct ThreadWindow { - /// The thread. - pub thread_id: String, - /// The first turn index still in the prompt; turns from here on are - /// left out. - #[serde(default)] - pub from_turn: u32, -} - -impl ThreadWindow { - /// Whether `hit` lies inside the window. - #[must_use] - pub fn covers(&self, hit: &Hit) -> bool { - hit.meta.thread_id.as_deref() == Some(self.thread_id.as_str()) - && hit - .meta - .turns - .as_ref() - .is_none_or(|turns| turns.last >= self.from_turn) - } -} +/// The turns a recall leaves out; defined beside the read requests that +/// also carry it ([`tinymemory_api::RecallRequest::exclude_thread`]). +pub use tinymemory_api::ThreadWindow; /// A recall across several scopes at once, rendered into one budgeted /// markdown block: the single read every lifecycle step is built from. diff --git a/crates/tinymemory-tools/src/tools/read/mod.rs b/crates/tinymemory-tools/src/tools/read/mod.rs index dedd4191..e3793a4d 100644 --- a/crates/tinymemory-tools/src/tools/read/mod.rs +++ b/crates/tinymemory-tools/src/tools/read/mod.rs @@ -42,6 +42,7 @@ pub(crate) async fn recall( limit: args.count("limit", DEFAULT_LIMIT, MAX_LIMIT)?, instructions: args.string("instructions")?, refers_to: time_hint(&args, "refers_to", scope)?, + exclude_thread: None, }; Ok(render::recall(&engine.recall(request).await?)) } @@ -81,6 +82,7 @@ pub(crate) async fn fetch( beliefs: 0, refers_to: time_hint(&args, "refers_to", scope)?, max_scopes: None, + exclude_thread: None, }; Ok(render::fetch(&engine.fetch(request).await?)) } diff --git a/docs/architecture/cortex-flows.md b/docs/architecture/cortex-flows.md index 141a379e..fa77d6ce 100644 --- a/docs/architecture/cortex-flows.md +++ b/docs/architecture/cortex-flows.md @@ -176,8 +176,10 @@ Only `Hybrid`. Other modes fail `Error::Unsupported` before any request. (Three raw events per wanted hit, because a conversation contributes several turns and the client-side filter drops some. 1000 events is the deepest a fetch page can go.) -4. Decode each pack's events, apply the **full filter** client-side, keep each - item once at its best rank. +4. Decode each pack's events, apply the **full filter** client-side, drop the + turns `exclude_thread` covers (CortexDB's recall filters only narrow by + label, so an exclusion cannot be sent), and keep each item once at its + best rank. 5. **Interleave** the scopes rank by rank: every scope's best, then every scope's second, and so on. 6. Take the page `[offset, end)`. A conversation hit carries the whole @@ -206,7 +208,9 @@ Recall builds a pack, asks the answer route **once** with `use_pack_id` 2. Each pack's events budget is `2 * limit`, with the derived layers sharing `limit` (see [the wire](cortex-wire.md#recall-recall)). 3. Decode and filter each pack's events with the full `MetaFilter` (reach - included). + included), and drop the turns `exclude_thread` covers. The exclusion + reaches the **citations only**: CortexDB writes the answer text from the + whole pack (step 4), which can still hold the excluded turns. 4. The answer comes from the pack holding the **most admitted events**, the most specific node on a tie. A missing `pack_id` is `Error::Engine`. `/v1/answer` takes one `use_pack_id` (its scope must match), so the diff --git a/docs/specs/agent-memory.md b/docs/specs/agent-memory.md index 6ade9fac..5725e844 100644 --- a/docs/specs/agent-memory.md +++ b/docs/specs/agent-memory.md @@ -152,7 +152,9 @@ skipped, engine }`. - **Exclusions:** - `exclude_ids` drops named items, such as the turn just logged. - `exclude_thread { thread_id, from_turn }` drops the turns still in the - prompt. + prompt. The same `ThreadWindow` rides on `RecallRequest` and + `FetchRequest` (`tinymemory-api`), so an answer section's recall leaves + the window out at the engine and a host's own reads can too. - **Dates.** A conversation bullet whose turn carries a time (`meta.observed_at`, set from `PreTurn::at` and `PostTurn::at`) is led by it, `[YYYY-MM-DD HH:MM]`. Hosts should pass the time the message was sent: From d693a6e974af38107958777fe9e9fc028d6bd1aa Mon Sep 17 00:00:00 2001 From: sanil-23 Date: Wed, 14 Oct 2026 22:37:23 +0530 Subject: [PATCH 2/2] Add an evidence mode to recall RecallRequest gains mode: RecallMode, Answer (the default, unchanged) or Evidence. In evidence mode the engine writes no answer: RecallAnswer's answer carries the recalled evidence as text for the caller's own model, and model is None. The CortexDB engine reads the same per-scope granular packs and never asks /v1/answer. It renders the evidence itself (engine/evidence.rs): dated facts, beliefs, then the admitted turns as [date] [role] text, merged rank by rank across the packs. Turns an exclude_thread window covers never appear, nor do facts supported by them or beliefs resting on those facts. Derived records carry no item metadata, so they are rendered only for a filter that narrows by reach and kinds alone. The reference engine returns its matching items as lines. Conformance checks that an evidence recall returns evidence and citations, reports no model, and never contains an excluded turn's text. Measured in openhuman-benchmarks #38: skipping the answer route removed recall timeouts and the answer/citation mismatch, and evidence recall was the best memory arm through OpenHuman. Co-Authored-By: Claude Opus 5.5 --- .../src/conformance/reference/mod.rs | 15 +- .../src/conformance/suite/checks.rs | 46 ++- crates/tinymemory-api/src/engine/mod.rs | 4 +- crates/tinymemory-api/src/lib.rs | 4 +- crates/tinymemory-api/src/query/mod.rs | 37 ++- .../src/cortex/engine/evidence.rs | 303 ++++++++++++++++++ .../src/cortex/engine/evidence_tests.rs | 161 ++++++++++ .../src/cortex/engine/mod.rs | 1 + .../src/cortex/engine/recall.rs | 13 +- crates/tinymemory-tools/src/recall/gather.rs | 3 +- crates/tinymemory-tools/src/tools/read/mod.rs | 3 +- docs/architecture/cortex-flows.md | 25 ++ 12 files changed, 599 insertions(+), 16 deletions(-) create mode 100644 crates/tinymemory-integrations/src/cortex/engine/evidence.rs create mode 100644 crates/tinymemory-integrations/src/cortex/engine/evidence_tests.rs diff --git a/crates/tinymemory-api/src/conformance/reference/mod.rs b/crates/tinymemory-api/src/conformance/reference/mod.rs index aac00b87..acd68849 100644 --- a/crates/tinymemory-api/src/conformance/reference/mod.rs +++ b/crates/tinymemory-api/src/conformance/reference/mod.rs @@ -20,8 +20,8 @@ use crate::{ Citation, ConsolidateReceipt, ConsolidateRequest, ConsolidateStatus, Consolidation, EngineDescriptor, EngineHealth, EraseReport, EraseRequest, Error, ExportPage, Exported, FetchMode, FetchPage, FetchRequest, ForgetReport, ForgetTarget, Hit, ItemId, ListPage, - ListRequest, MemoryEngine, MetaFilter, Reach, RecallAnswer, RecallRequest, Result, StoreItem, - StoreReceipt, + ListRequest, MemoryEngine, MetaFilter, Reach, RecallAnswer, RecallMode, RecallRequest, Result, + StoreItem, StoreReceipt, }; use async_trait::async_trait; @@ -174,6 +174,17 @@ impl MemoryEngine for ReferenceEngine { score: Some(hit.score), }) .collect(); + if req.mode == RecallMode::Evidence { + let lines: Vec = citations + .iter() + .map(|c| format!("- {}", c.snippet)) + .collect(); + return Ok(RecallAnswer { + answer: lines.join("\n"), + citations, + model: None, + }); + } let answer = if citations.is_empty() { "Nothing stored answers this question.".to_string() } else { diff --git a/crates/tinymemory-api/src/conformance/suite/checks.rs b/crates/tinymemory-api/src/conformance/suite/checks.rs index cdf73df3..5aad1007 100644 --- a/crates/tinymemory-api/src/conformance/suite/checks.rs +++ b/crates/tinymemory-api/src/conformance/suite/checks.rs @@ -4,7 +4,7 @@ use std::collections::BTreeSet; use crate::{ Error as ApiError, FetchMode, FetchRequest, ForgetTarget, ItemId, MemoryMeta, MetaFilter, - RecallRequest, Role, StoreItem, ThreadWindow, Turn, TurnRange, + RecallMode, RecallRequest, Role, StoreItem, ThreadWindow, Turn, TurnRange, }; use super::{Ctx, ensure}; @@ -237,7 +237,7 @@ async fn excluded_thread(ctx: &Ctx<'_>) -> Result<()> { let mut request = RecallRequest::new(ctx.run.marker.clone(), 50); request.filter = ctx.run.filter(); request.exclude_thread = Some(ThreadWindow::whole(&live)); - let answer = ctx.call(CHECK, ctx.engine.recall(request)).await?; + let answer = ctx.call(CHECK, ctx.engine.recall(request.clone())).await?; ensure( CHECK, answer @@ -246,6 +246,27 @@ async fn excluded_thread(ctx: &Ctx<'_>) -> Result<()> { .all(|citation| citation.id != live_id), || format!("a recall excluding {live} cited its turn {live_id}"), )?; + // In evidence mode the engine writes no answer of its own, so the text it + // returns is all evidence, and none of it may come from the window. + request.mode = RecallMode::Evidence; + let evidence = ctx.call(CHECK, ctx.engine.recall(request)).await?; + let live_text = format!("the turn being answered in {live}"); + ensure( + CHECK, + !evidence.answer.contains(&live_text) + && evidence + .citations + .iter() + .all(|citation| citation.id != live_id), + || format!("an evidence recall excluding {live} returned its turn {live_id}"), + )?; + ensure( + CHECK, + evidence + .answer + .contains(&format!("the turn being answered in {other}")), + || format!("an evidence recall left out {other}, which no window covers"), + )?; let blank = ThreadWindow::whole(" "); let mut request = FetchRequest::new(ctx.run.marker.clone(), FetchMode::Hybrid, 5); request.exclude_thread = Some(blank.clone()); @@ -382,6 +403,27 @@ async fn recall(ctx: &Ctx<'_>) -> Result<()> { ensure(CHECK, answer.citations.len() <= 5, || { format!("{} citations exceed the limit of 5", answer.citations.len()) })?; + let mut evidence = RecallRequest::new(ctx.run.marker.clone(), 5); + evidence.filter = ctx.run.filter(); + evidence.mode = RecallMode::Evidence; + let evidence = ctx.call(CHECK, ctx.engine.recall(evidence)).await?; + ensure( + CHECK, + !evidence.answer.trim().is_empty() && !evidence.citations.is_empty(), + || "an evidence recall over matching items returned no evidence".to_string(), + )?; + ensure(CHECK, evidence.model.is_none(), || { + format!( + "an evidence recall reported answering model {:?}", + evidence.model + ) + })?; + ensure(CHECK, evidence.citations.len() <= 5, || { + format!( + "{} evidence citations exceed the limit of 5", + evidence.citations.len() + ) + })?; let listed = ctx.list_all(CHECK, &ctx.run.filter()).await?; for citation in &answer.citations { let resolved = listed.iter().find(|hit| hit.id == citation.id); diff --git a/crates/tinymemory-api/src/engine/mod.rs b/crates/tinymemory-api/src/engine/mod.rs index 777d093b..ab46e528 100644 --- a/crates/tinymemory-api/src/engine/mod.rs +++ b/crates/tinymemory-api/src/engine/mod.rs @@ -33,7 +33,9 @@ pub trait MemoryEngine: Send + Sync { /// Whether the engine can serve right now. async fn health(&self) -> EngineHealth; - /// Answers a question from stored items. + /// Answers a question from stored items, or, in + /// [`crate::RecallMode::Evidence`], returns the evidence for the caller + /// to answer from without writing an answer. /// /// # Errors /// diff --git a/crates/tinymemory-api/src/lib.rs b/crates/tinymemory-api/src/lib.rs index 830d420e..da05816f 100644 --- a/crates/tinymemory-api/src/lib.rs +++ b/crates/tinymemory-api/src/lib.rs @@ -85,8 +85,8 @@ pub use namespace::{Namespace, Reach, Segment, SegmentKind}; pub use provenance::{ProvenancePolicy, wrap_untrusted_for_agent}; pub use query::{ Citation, EraseReport, EraseRequest, ExportPage, Exported, FetchMode, FetchPage, FetchRequest, - ForgetReport, ForgetTarget, Hit, ListPage, ListRequest, RecallAnswer, RecallRequest, - ThreadWindow, TimeHint, + ForgetReport, ForgetTarget, Hit, ListPage, ListRequest, RecallAnswer, RecallMode, + RecallRequest, ThreadWindow, TimeHint, }; pub use write::{WaitFor, WriteOptions}; diff --git a/crates/tinymemory-api/src/query/mod.rs b/crates/tinymemory-api/src/query/mod.rs index 97a41f45..fcc57855 100644 --- a/crates/tinymemory-api/src/query/mod.rs +++ b/crates/tinymemory-api/src/query/mod.rs @@ -1,7 +1,8 @@ //! Requests and responses for recall, fetch, list and forget. //! //! - **Recall** ([`RecallRequest`] → [`RecallAnswer`]) asks a question and gets -//! a synthesised answer with [`Citation`]s. +//! a synthesised answer with [`Citation`]s, or, in [`RecallMode::Evidence`], +//! the evidence for the caller's own model to answer from. //! - **Fetch** ([`FetchRequest`] → [`FetchPage`]) is raw retrieval in a //! [`FetchMode`], filtered by metadata. //! - **List** ([`ListRequest`] → [`ListPage`]) pages through stored items @@ -44,6 +45,33 @@ pub struct RecallRequest { /// never narrow a forget or an erase. #[serde(default, skip_serializing_if = "Option::is_none")] pub exclude_thread: Option, + /// Whether the engine writes the answer ([`RecallMode::Answer`], the + /// default) or returns the evidence for the caller to answer from. + #[serde(default, skip_serializing_if = "RecallMode::is_answer")] + pub mode: RecallMode, +} + +/// What [`crate::MemoryEngine::recall`] puts in [`RecallAnswer::answer`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RecallMode { + /// An answer the engine writes from what it recalled (an engine may run + /// a model for it). + #[default] + Answer, + /// The recalled evidence itself, as text, for a caller whose own model + /// writes the answer: the engine runs no answer step, and + /// [`RecallAnswer::model`] is `None`. The citations are the items the + /// evidence holds. + Evidence, +} + +impl RecallMode { + /// Whether this is [`RecallMode::Answer`] (serde's skip test). + #[must_use] + pub fn is_answer(&self) -> bool { + matches!(self, Self::Answer) + } } impl RecallRequest { @@ -57,6 +85,7 @@ impl RecallRequest { instructions: None, refers_to: None, exclude_thread: None, + mode: RecallMode::Answer, } } @@ -133,12 +162,14 @@ impl ThreadWindow { /// A synthesised answer. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct RecallAnswer { - /// The answer text. + /// The answer text; in [`RecallMode::Evidence`], the recalled evidence + /// as text (empty when nothing was recalled). pub answer: String, /// The items it drew on. #[serde(default)] pub citations: Vec, - /// The model that answered, when the engine reports it. + /// The model that answered, when the engine reports it; `None` in + /// [`RecallMode::Evidence`]. #[serde(default, skip_serializing_if = "Option::is_none")] pub model: Option, } diff --git a/crates/tinymemory-integrations/src/cortex/engine/evidence.rs b/crates/tinymemory-integrations/src/cortex/engine/evidence.rs new file mode 100644 index 00000000..eccc683f --- /dev/null +++ b/crates/tinymemory-integrations/src/cortex/engine/evidence.rs @@ -0,0 +1,303 @@ +//! Recall in [`tinymemory_api::RecallMode::Evidence`]: the evidence an answer +//! would be written from, rendered here, with no answer route. +//! +//! The packs are recall's own (`recall::pack`): one per kind scope the filter +//! reads, exact (`view: "granular"`), events first and then the derived +//! layers, most specific node first. Nothing is sent to `/v1/answer`, so no +//! model runs after retrieval and there is no answer whose grounding differs +//! from its citations. The text is rendered from what the packs decode to: +//! +//! - **Facts:** each pack's `facts` layer, `[valid_from date] statement`. +//! - **Beliefs:** each pack's `beliefs` layer, `subject predicate object`. +//! - **Conversation:** the admitted events, merged rank by rank across the +//! packs, `[YYYY-MM-DD HH:MM UTC] [role] text`, separated by `---`. +//! +//! **Exclusion.** [`RecallRequest::exclude_thread`] removes the window's turns +//! before rendering, so their text never appears. A fact whose supporting +//! events include an excluded turn the pack returned is left out too, and so +//! is a belief resting on such a fact. A fact derived from an excluded turn +//! the pack did not return cannot be told apart; enrichment usually derives +//! facts after the turn, so a turn logged just before the read has none yet. +//! +//! **Filters.** Derived records carry no item metadata, so facts and beliefs +//! are rendered only for a filter that narrows by reach and kinds alone. Any +//! other field (a workspace, a thread, a tag) leaves them out rather than show +//! a record from an item the filter does not admit. The `episodes` and +//! `understanding` layers (session and activity roll-ups) are not rendered. +//! +//! The citations are the conversation's items, at most `limit`; the evidence +//! may hold more events than that (each pack's events budget is `2 * limit`). + +use std::collections::HashSet; + +use futures::{StreamExt, TryStreamExt, stream}; +use serde_json::Value; +use tinymemory_api::{ + Citation, ItemId, ItemKind, MetaFilter, RecallAnswer, RecallRequest, Role, ThreadWindow, +}; + +use super::CortexEngine; +use super::fetch::interleave; +use super::items::{keeps, located_meta}; +use super::recall::PACKS_AT_ONCE; +use crate::cortex::envelope::{Envelope, decode_event}; +use crate::cortex::error::Result; + +/// What one pack contributes to the evidence. +#[derive(Debug, Default)] +struct Gathered { + /// Admitted events, best rank first, each item once. + events: Vec, + /// Rendered facts. + facts: Vec, + /// Rendered beliefs. + beliefs: Vec, +} + +/// The items of `pack`'s `layer`. +fn layer<'a>(pack: &'a Value, name: &str) -> impl Iterator { + pack.pointer(&format!("/layers/{name}")) + .and_then(Value::as_array) + .into_iter() + .flatten() +} + +/// The string ids an array field lists, or each entry's `id` when its +/// entries are objects (a belief's `supports`). +fn ids(record: &Value, field: &str) -> Vec { + record + .get(field) + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|entry| { + entry + .as_str() + .or_else(|| entry.get("id").and_then(Value::as_str)) + .map(str::to_owned) + }) + .collect() +} + +/// An entity's readable name: its `name`, else its id without CortexDB's +/// `ent_` prefix. `None` for an anonymous fact subject. +fn entity_name(entity: &Value) -> Option { + if let Some(name) = entity.get("name").and_then(Value::as_str) { + return Some(name.to_string()); + } + let id = entity.get("id").and_then(Value::as_str)?; + if id.starts_with("ent_fact_subject_") { + return None; + } + Some(id.strip_prefix("ent_").unwrap_or(id).to_string()) +} + +/// A claim's object: a literal's value, else the entity's name. +fn object_text(object: &Value) -> Option { + object + .get("value") + .and_then(Value::as_str) + .map(str::to_owned) + .or_else(|| entity_name(object)) +} + +/// `subject predicate object`; a `states` claim with no named subject is +/// its object alone. +fn claim_text(subject: &Value, predicate: &str, object: &Value) -> Option { + let object = object_text(object)?; + let predicate = predicate.replace('_', " "); + match entity_name(subject) { + Some(subject) => Some(format!("{subject} {predicate} {object}")), + None if predicate == "states" => Some(object), + None => Some(format!("{predicate} {object}")), + } +} + +/// A fact as one line: `[YYYY-MM-DD] statement` when it is dated. +fn fact_line(fact: &Value) -> Option { + let text = claim_text( + fact.get("subject")?, + fact.get("predicate").and_then(Value::as_str).unwrap_or(""), + fact.get("object")?, + )?; + let date = fact + .get("valid_from") + .and_then(Value::as_str) + .and_then(|at| at.get(..10)); + Some(match date { + Some(date) => format!("[{date}] {text}"), + None => text, + }) +} + +/// A belief's claim as one line. +fn belief_line(belief: &Value) -> Option { + let claim = belief.get("claim")?; + claim_text( + claim.get("subject")?, + claim.get("predicate").and_then(Value::as_str).unwrap_or(""), + claim.get("object")?, + ) +} + +/// Whether `filter` narrows by reach and kinds alone (see the module docs). +fn scope_only(filter: &MetaFilter) -> bool { + let mut rest = filter.clone(); + rest.reach = None; + rest.kinds.clear(); + rest.is_empty() +} + +/// What `pack` contributes, with `exclude`'s turns left out. +fn gather(pack: &Value, filter: &MetaFilter, exclude: Option<&ThreadWindow>) -> Gathered { + let mut gathered = Gathered::default(); + let mut excluded = HashSet::new(); + let mut seen = HashSet::new(); + for decoded in layer(pack, "events").filter_map(decode_event) { + let envelope = decoded.envelope; + if !keeps(filter, envelope.kind, &envelope) { + continue; + } + if exclude.is_some_and(|window| window.covers_meta(&envelope.meta)) { + excluded.insert(decoded.event_id); + continue; + } + if seen.insert(envelope.id.clone()) { + gathered.events.push(envelope); + } + } + if !scope_only(filter) { + return gathered; + } + let mut dropped = HashSet::new(); + for fact in layer(pack, "facts") { + if ids(fact, "supports").iter().any(|id| excluded.contains(id)) { + if let Some(id) = fact.get("id").and_then(Value::as_str) { + dropped.insert(id.to_string()); + } + continue; + } + gathered.facts.extend(fact_line(fact)); + } + for belief in layer(pack, "beliefs") { + if ids(belief, "supports") + .iter() + .any(|id| dropped.contains(id)) + { + continue; + } + gathered.beliefs.extend(belief_line(belief)); + } + gathered +} + +/// Who an event is from: a turn's role, else its item kind. +fn speaker(envelope: &Envelope) -> String { + match (&envelope.turn, envelope.kind) { + (Some(turn), _) => match turn.role { + Role::User => "user", + Role::Assistant => "assistant", + Role::System => "system", + Role::Tool => "tool", + } + .to_string(), + (None, ItemKind::Document) => match &envelope.title { + Some(title) => format!("document: {title}"), + None => "document".to_string(), + }, + (None, ItemKind::Learning) => "learning".to_string(), + (None, ItemKind::Conversation) => "conversation".to_string(), + } +} + +/// One event as an evidence entry: `[YYYY-MM-DD HH:MM UTC] [speaker] text`. +fn event_entry(envelope: &Envelope) -> String { + let at = envelope + .turn + .as_ref() + .and_then(|turn| turn.at) + .or(envelope.meta.observed_at); + let speaker = speaker(envelope); + match at { + Some(at) => format!( + "[{}] [{speaker}] {}", + at.format("%Y-%m-%d %H:%M UTC"), + envelope.text + ), + None => format!("[{speaker}] {}", envelope.text), + } +} + +/// The evidence text: facts, beliefs, then the conversation, each line once. +/// Empty when nothing was gathered. +fn render(facts: &[String], beliefs: &[String], events: &[Envelope]) -> String { + let unique = |lines: &[String]| -> Vec { + let mut seen = HashSet::new(); + lines + .iter() + .filter(|line| seen.insert(line.to_lowercase())) + .map(|line| format!("- {line}")) + .collect() + }; + let mut sections = Vec::new(); + let facts = unique(facts); + if !facts.is_empty() { + sections.push(format!("## Facts\n{}", facts.join("\n"))); + } + let beliefs = unique(beliefs); + if !beliefs.is_empty() { + sections.push(format!("## Beliefs\n{}", beliefs.join("\n"))); + } + if !events.is_empty() { + let entries: Vec = events.iter().map(event_entry).collect(); + sections.push(format!("## Conversation\n{}", entries.join("\n\n---\n\n"))); + } + sections.join("\n\n") +} + +impl CortexEngine { + /// See the module docs. `paths` are the kind scopes to read, most + /// specific node first. + pub(super) async fn recall_evidence( + &self, + req: &RecallRequest, + paths: &[String], + ) -> Result { + let packs: Vec = stream::iter(paths.to_vec()) + .map(|path| async move { self.pack(req, &path).await }) + .buffered(PACKS_AT_ONCE) + .try_collect() + .await?; + let mut facts = Vec::new(); + let mut beliefs = Vec::new(); + let mut per_pack = Vec::new(); + for pack in &packs { + let gathered = gather(pack, &req.filter, req.exclude_thread.as_ref()); + facts.extend(gathered.facts); + beliefs.extend(gathered.beliefs); + per_pack.push(gathered.events); + } + let events = interleave(per_pack); + let answer = render(&facts, &beliefs, &events); + let citations = events + .into_iter() + .take(req.limit) + .map(|envelope| Citation { + meta: located_meta(&envelope), + id: ItemId::new(envelope.id), + kind: envelope.kind, + snippet: envelope.text, + score: None, + }) + .collect(); + Ok(RecallAnswer { + answer, + citations, + model: None, + }) + } +} + +#[cfg(test)] +#[path = "evidence_tests.rs"] +mod tests; diff --git a/crates/tinymemory-integrations/src/cortex/engine/evidence_tests.rs b/crates/tinymemory-integrations/src/cortex/engine/evidence_tests.rs new file mode 100644 index 00000000..9bf07e74 --- /dev/null +++ b/crates/tinymemory-integrations/src/cortex/engine/evidence_tests.rs @@ -0,0 +1,161 @@ +//! Tests for rendering recall packs as evidence. + +use super::*; +use serde_json::json; +use tinymemory_api::{MemoryMeta, Reach, StoreItem, Turn, TurnRange}; + +fn turn(thread: &str, role: Role, text: &str) -> StoreItem { + let mut meta = MemoryMeta { + thread_id: Some(thread.to_string()), + turns: Some(TurnRange { first: 0, last: 0 }), + ..MemoryMeta::default() + }; + meta.observed_at = Some("2023-06-27T10:37:00Z".parse().unwrap()); + let mut said = Turn::new(role, text); + said.at = meta.observed_at; + StoreItem::Conversation { + turns: vec![said], + meta, + } +} + +fn event(id: &str, item: &StoreItem) -> Value { + let fingerprint = item.fingerprint(); + let envelope = &Envelope::for_item(item, &fingerprint).unwrap()[0]; + json!({ "id": id, "content": { "role": "user", "text": envelope.encode().unwrap() } }) +} + +fn fact(id: &str, supports: &[&str], value: &str) -> Value { + json!({ + "id": id, + "subject": { "type": "entity", "id": format!("ent_fact_subject_{id}") }, + "predicate": "states", + "object": { "type": "literal", "datatype": "string", "value": value }, + "supports": supports, + "valid_from": "2023-06-27T10:37:00Z", + }) +} + +fn belief(supports: &[&str], name: &str, predicate: &str, value: &str) -> Value { + let supports: Vec = supports + .iter() + .map(|id| json!({ "type": "fact", "id": id })) + .collect(); + json!({ + "claim": { + "subject": { "type": "entity", "id": format!("ent_{name}"), "name": name }, + "predicate": predicate, + "object": { "type": "literal", "value": value }, + }, + "supports": supports, + }) +} + +fn pack() -> Value { + json!({ "layers": { + "events": [ + event("e_live", &turn("live", Role::User, "what did grandma give me")), + event("e_gift", &turn("past", Role::User, "my grandma gave me a necklace")), + { "id": "e_other", "content": { "text": "somebody else's event" } }, + ], + "facts": [ + fact("f_live", &["e_live"], "the user asked about a gift"), + fact("f_gift", &["e_gift"], "grandma gave the user a necklace"), + ], + "beliefs": [ + belief(&["f_live"], "user", "asked_about", "a gift"), + belief(&["f_gift"], "Caroline", "owns", "a necklace"), + ], + } }) +} + +#[test] +fn an_excluded_turn_and_what_was_derived_from_it_are_left_out() { + let gathered = gather( + &pack(), + &MetaFilter::default(), + Some(&ThreadWindow::whole("live")), + ); + let texts: Vec<&str> = gathered.events.iter().map(|e| e.text.as_str()).collect(); + assert_eq!(texts, vec!["my grandma gave me a necklace"]); + assert_eq!( + gathered.facts, + vec!["[2023-06-27] grandma gave the user a necklace"] + ); + assert_eq!(gathered.beliefs, vec!["Caroline owns a necklace"]); +} + +#[test] +fn without_a_window_every_admitted_record_is_kept() { + let gathered = gather(&pack(), &MetaFilter::default(), None); + assert_eq!(gathered.events.len(), 2, "the foreign event never decodes"); + assert_eq!(gathered.facts.len(), 2); + assert_eq!( + gathered.beliefs, + vec!["user asked about a gift", "Caroline owns a necklace"] + ); +} + +#[test] +fn a_metadata_filter_keeps_derived_records_out() { + let scoped = MetaFilter { + reach: Some(Reach::subtree(tinymemory_api::Namespace::ROOT)), + kinds: vec![ItemKind::Conversation], + ..MetaFilter::default() + }; + assert!( + !gather(&pack(), &scoped, None).facts.is_empty(), + "reach and kinds alone keep them" + ); + let narrowed = MetaFilter { + thread_id: Some("past".into()), + ..MetaFilter::default() + }; + let gathered = gather(&pack(), &narrowed, None); + assert_eq!(gathered.events.len(), 1); + assert!(gathered.facts.is_empty() && gathered.beliefs.is_empty()); +} + +#[test] +fn the_evidence_lists_facts_beliefs_then_dated_turns() { + let gathered = gather( + &pack(), + &MetaFilter::default(), + Some(&ThreadWindow::whole("live")), + ); + let text = render(&gathered.facts, &gathered.beliefs, &gathered.events); + assert_eq!( + text, + "## Facts\n- [2023-06-27] grandma gave the user a necklace\n\n\ + ## Beliefs\n- Caroline owns a necklace\n\n\ + ## Conversation\n[2023-06-27 10:37 UTC] [user] my grandma gave me a necklace" + ); + assert!(!text.contains("what did grandma give me")); +} + +#[test] +fn repeated_lines_render_once_and_nothing_renders_empty() { + let facts = vec!["A".to_string(), "a".to_string()]; + assert_eq!(render(&facts, &[], &[]), "## Facts\n- A"); + assert_eq!(render(&[], &[], &[]), ""); +} + +#[test] +fn turns_are_separated_and_attributed_by_role() { + let events: Vec = [ + turn("t", Role::User, "hello"), + turn("t", Role::Assistant, "hi there"), + ] + .iter() + .map(|item| { + Envelope::for_item(item, &item.fingerprint()) + .unwrap() + .remove(0) + }) + .collect(); + assert_eq!( + render(&[], &[], &events), + "## Conversation\n[2023-06-27 10:37 UTC] [user] hello\n\n---\n\n\ + [2023-06-27 10:37 UTC] [assistant] hi there" + ); +} diff --git a/crates/tinymemory-integrations/src/cortex/engine/mod.rs b/crates/tinymemory-integrations/src/cortex/engine/mod.rs index b731370b..a3563d89 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/mod.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/mod.rs @@ -18,6 +18,7 @@ mod beliefs; mod consolidate; mod cursor; mod erase; +mod evidence; mod explore; mod fetch; mod forget; diff --git a/crates/tinymemory-integrations/src/cortex/engine/recall.rs b/crates/tinymemory-integrations/src/cortex/engine/recall.rs index 720e2c6c..08c631e6 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/recall.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/recall.rs @@ -1,5 +1,7 @@ //! Recall: one recall pack per scope, then the answer route once with one -//! of them. +//! of them. In [`tinymemory_api::RecallMode::Evidence`] the same packs are +//! rendered as evidence instead and the answer route is never asked (see +//! `evidence`). //! //! The scopes are the filter's (see `scopes`): the kind scopes of its reach, //! or, with no reach (an unscoped, administrative read), every kind scope the @@ -38,7 +40,7 @@ use std::collections::HashSet; use futures::{StreamExt, TryStreamExt, stream}; use serde_json::{Value, json}; -use tinymemory_api::{Citation, ItemId, Namespace, Reach, RecallAnswer, RecallRequest}; +use tinymemory_api::{Citation, ItemId, Namespace, Reach, RecallAnswer, RecallMode, RecallRequest}; use super::CortexEngine; use super::fetch::{interleave, ranked, recall_body, whole_items_budget}; @@ -49,7 +51,7 @@ use crate::cortex::envelope::Envelope; use crate::cortex::error::{Error, Result}; /// Recall packs built at once when a reach spans several scopes. -const PACKS_AT_ONCE: usize = 4; +pub(super) const PACKS_AT_ONCE: usize = 4; /// Rounds (packs, then one answer) at most per recall: the first, and one /// after each time CortexDB dropped the packs before the answer used one. @@ -141,6 +143,9 @@ impl CortexEngine { let mut ordered: Vec<&KindScope> = scopes.iter().collect(); ordered.sort_by_key(|scope| std::cmp::Reverse(scope.namespace.depth())); let paths: Vec = ordered.into_iter().map(|s| s.path.clone()).collect(); + if req.mode == RecallMode::Evidence { + return self.recall_evidence(&req, &paths).await; + } // A pack lives 60 s, and CortexDB drops every pack it holds once // anything is forgotten (measured on 0.10.4: a forget in another // scope turns the next `use_pack_id` into a 404). On a 404 every @@ -225,7 +230,7 @@ impl CortexEngine { /// A recall pack for `req` over exactly `scope`, sized for `req.limit` /// citations. It includes the derived layers the answer route reads, /// after the events the citations come from. - async fn pack(&self, req: &RecallRequest, scope: &str) -> Result { + pub(super) async fn pack(&self, req: &RecallRequest, scope: &str) -> Result { let mut body = recall_body(scope, &req.question, 0, &req.filter); body["include"] = pack_layers(); body["budgets"]["per_layer_limits"] = pack_budgets(req.limit); diff --git a/crates/tinymemory-tools/src/recall/gather.rs b/crates/tinymemory-tools/src/recall/gather.rs index f4ad29e8..74a250e4 100644 --- a/crates/tinymemory-tools/src/recall/gather.rs +++ b/crates/tinymemory-tools/src/recall/gather.rs @@ -35,7 +35,7 @@ use std::time::Instant; use futures::future::join; use tinymemory_api::{ BeliefsRequest, FetchMode, FetchRequest, Hit, ItemId, ItemKind, ListRequest, MemoryEngine, - MetaFilter, Namespace, Reach, RecallRequest, TimeHint, + MetaFilter, Namespace, Reach, RecallMode, RecallRequest, TimeHint, }; use super::render::{Body, Line, Section, shorten, single_line}; @@ -231,6 +231,7 @@ async fn answer( // The engine leaves the thread out itself, so the window's turns // do not use up the section's citations. exclude_thread: request.exclude_thread.clone(), + mode: RecallMode::Answer, }) .await?; if answer.citations.is_empty() || answer.answer.trim().is_empty() { diff --git a/crates/tinymemory-tools/src/tools/read/mod.rs b/crates/tinymemory-tools/src/tools/read/mod.rs index e3793a4d..497b512b 100644 --- a/crates/tinymemory-tools/src/tools/read/mod.rs +++ b/crates/tinymemory-tools/src/tools/read/mod.rs @@ -10,7 +10,7 @@ use serde_json::Value; use tinymemory_api::{ Error, ExploreRequest, FetchRequest, GetRequest, Hit, ItemId, ListRequest, MemoryEngine, - RecallRequest, Result, + RecallMode, RecallRequest, Result, }; use super::ToolScope; @@ -43,6 +43,7 @@ pub(crate) async fn recall( instructions: args.string("instructions")?, refers_to: time_hint(&args, "refers_to", scope)?, exclude_thread: None, + mode: RecallMode::Answer, }; Ok(render::recall(&engine.recall(request).await?)) } diff --git a/docs/architecture/cortex-flows.md b/docs/architecture/cortex-flows.md index fa77d6ce..8c8aba96 100644 --- a/docs/architecture/cortex-flows.md +++ b/docs/architecture/cortex-flows.md @@ -234,6 +234,31 @@ Recall builds a pack, asks the answer route **once** with `use_pack_id` envelope's text as the snippet. A pack with no decodable events still returns the answer, with no citations. +### Recall in evidence mode + +`RecallMode::Evidence` reads the same packs (step 1, with step 2's budgets) +and **never asks the answer route**: no model runs after retrieval, and no +answer is grounded on one pack while citing others. The engine renders the +evidence itself (`engine/evidence.rs`): + +1. Decode each pack's events, keep what the full `MetaFilter` admits, and + drop the turns `exclude_thread` covers, noting their event ids. +2. When the filter narrows by reach and kinds alone, render the `facts` + layer (`[valid_from date] statement`) and the `beliefs` layer + (`subject predicate object`), leaving out a fact whose `supports` name an + excluded event and a belief resting on such a fact. Derived records carry + no item metadata, so any other filter field leaves them out. `episodes` + and `understanding` are not rendered. +3. Merge the events rank by rank across the packs (most specific node + first) and render each as `[YYYY-MM-DD HH:MM UTC] [role] text`. +4. The answer is `## Facts`, `## Beliefs` and `## Conversation`, each line + once, empty when nothing was gathered; `model` is `None`. Citations are + the conversation's items, at most `limit`. + +Because the text is rendered from decoded events, an excluded turn never +appears in it, unlike the answer route, whose text CortexDB writes from the +whole pack. + ## Forget `forget` looks the items' events up, then removes them by `memory_ids`. The