diff --git a/crates/tinymemory-api/src/conformance/reference/mod.rs b/crates/tinymemory-api/src/conformance/reference/mod.rs index acd68849..5055c896 100644 --- a/crates/tinymemory-api/src/conformance/reference/mod.rs +++ b/crates/tinymemory-api/src/conformance/reference/mod.rs @@ -161,7 +161,7 @@ impl MemoryEngine for ReferenceEngine { req.validate()?; let mut hits = self.ranked(&req.question, FetchMode::Hybrid, &req.filter)?; if let Some(window) = &req.exclude_thread { - hits.retain(|hit| !window.covers(hit)); + hits.retain(|hit| !window.excludes(hit.kind, &hit.meta)); } let citations: Vec = hits .into_iter() @@ -207,7 +207,7 @@ impl MemoryEngine for ReferenceEngine { req.validate()?; let mut hits = self.ranked(&req.query, req.mode, &req.filter)?; if let Some(window) = &req.exclude_thread { - hits.retain(|hit| !window.covers(hit)); + hits.retain(|hit| !window.excludes(hit.kind, &hit.meta)); } if let Some(hint) = &req.refers_to { hint.rank(&mut hits); diff --git a/crates/tinymemory-api/src/conformance/suite/checks.rs b/crates/tinymemory-api/src/conformance/suite/checks.rs index 5aad1007..d15e4227 100644 --- a/crates/tinymemory-api/src/conformance/suite/checks.rs +++ b/crates/tinymemory-api/src/conformance/suite/checks.rs @@ -3,8 +3,8 @@ use std::collections::BTreeSet; use crate::{ - Error as ApiError, FetchMode, FetchRequest, ForgetTarget, ItemId, MemoryMeta, MetaFilter, - RecallMode, RecallRequest, Role, StoreItem, ThreadWindow, Turn, TurnRange, + Error as ApiError, FetchMode, FetchRequest, ForgetTarget, ItemId, LearningKind, MemoryMeta, + MetaFilter, RecallMode, RecallRequest, Role, StoreItem, ThreadWindow, Turn, TurnRange, }; use super::{Ctx, ensure}; @@ -21,7 +21,6 @@ pub(super) async fn all(ctx: &Ctx<'_>) -> Result<()> { super::bulk::store_many(ctx).await?; super::lifecycle::store_with(ctx).await?; fetch_filters(ctx).await?; - excluded_thread(ctx).await?; unsupported_modes(ctx).await?; super::namespaces::namespaces(ctx).await?; empty_forget(ctx).await?; @@ -29,6 +28,8 @@ pub(super) async fn all(ctx: &Ctx<'_>) -> Result<()> { forget_by_filter(ctx).await?; super::erase::erase(ctx).await?; recall(ctx).await?; + // After `recall`, so a recall fault is reported by the check named for it. + excluded_thread(ctx).await?; super::lifecycle::consolidate(ctx).await } @@ -183,8 +184,8 @@ async fn fetch_filters(ctx: &Ctx<'_>) -> Result<()> { } /// A read's `exclude_thread` leaves that thread's turns out of every fetch -/// mode and of recall's citations, and a window starting after a turn keeps -/// it. +/// mode and of recall's citations, a window starting after a turn keeps it, +/// and a learning stamped with the thread stays readable. async fn excluded_thread(ctx: &Ctx<'_>) -> Result<()> { const CHECK: &str = "excluded_thread"; let turn = |thread: &str| { @@ -205,6 +206,21 @@ async fn excluded_thread(ctx: &Ctx<'_>) -> Result<()> { ); let live_id = ctx.call(CHECK, ctx.engine.store(turn(&live))).await?.id; let other_id = ctx.call(CHECK, ctx.engine.store(turn(&other))).await?.id; + let mut learned_meta = ctx.run.meta(); + learned_meta.thread_id = Some(live.clone()); + let learned_text = format!("{} a learning made in {live}", ctx.run.marker); + let learned_id = ctx + .call( + CHECK, + ctx.engine.store(StoreItem::learning( + learned_text.clone(), + LearningKind::Fact, + 0.9, + learned_meta, + )), + ) + .await? + .id; let modes = ctx.engine.descriptor().fetch_modes.clone(); for mode in &modes { let mut request = FetchRequest::new(ctx.run.marker.clone(), *mode, 50); @@ -222,6 +238,12 @@ async fn excluded_thread(ctx: &Ctx<'_>) -> Result<()> { ) }, )?; + ensure(CHECK, found.contains(&learned_id), || { + format!( + "{} fetch excluding {live} dropped its learning {learned_id}: only conversation turns are excluded", + mode.as_str() + ) + })?; request.exclude_thread = Some(ThreadWindow { thread_id: live.clone(), from_turn: 1, @@ -246,6 +268,14 @@ async fn excluded_thread(ctx: &Ctx<'_>) -> Result<()> { .all(|citation| citation.id != live_id), || format!("a recall excluding {live} cited its turn {live_id}"), )?; + ensure( + CHECK, + answer + .citations + .iter() + .any(|citation| citation.id == learned_id), + || format!("a recall excluding {live} left out its learning {learned_id}"), + )?; // In evidence mode the engine writes no answer of its own, so the text it // returns is all evidence, and none of it may come from the window. request.mode = RecallMode::Evidence; diff --git a/crates/tinymemory-api/src/query/mod.rs b/crates/tinymemory-api/src/query/mod.rs index fcc57855..765d6c11 100644 --- a/crates/tinymemory-api/src/query/mod.rs +++ b/crates/tinymemory-api/src/query/mod.rs @@ -109,8 +109,9 @@ impl RecallRequest { /// `from_turn` on. A caller reading inside a thread sets it to the turns its /// prompt already holds, so the message being answered (logged before the /// read) does not come back as a memory; `from_turn: 0` leaves the whole -/// thread out. Engines apply it to what they return; an engine whose store -/// cannot exclude server side drops the matching items from its results. +/// thread out. Engines apply it to what they return ([`ThreadWindow::excludes`]: +/// conversation turns only); an engine whose store cannot exclude server side +/// drops the matching items from its results. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ThreadWindow { /// The thread. @@ -149,6 +150,16 @@ impl ThreadWindow { .is_none_or(|turns| turns.last >= self.from_turn) } + /// Whether a read's `exclude_thread` leaves out an item of `kind` + /// carrying `meta`: only **conversation** turns are left out. A learning + /// or a document stamped with the thread (a host records where it was + /// learned) stays readable, since the prompt holds the thread's turns, + /// not what was learned from them. + #[must_use] + pub fn excludes(&self, kind: ItemKind, meta: &MemoryMeta) -> bool { + kind == ItemKind::Conversation && self.covers_meta(meta) + } + /// Checks the window names a thread. /// /// # Errors diff --git a/crates/tinymemory-api/src/query/mod_tests.rs b/crates/tinymemory-api/src/query/mod_tests.rs index a711c788..3b9ecadf 100644 --- a/crates/tinymemory-api/src/query/mod_tests.rs +++ b/crates/tinymemory-api/src/query/mod_tests.rs @@ -131,3 +131,18 @@ fn a_read_without_an_excluded_thread_serializes_as_before() { .unwrap(); assert_eq!(parsed.exclude_thread, None); } + +#[test] +fn a_thread_window_excludes_conversation_turns_only() { + let meta = MemoryMeta { + thread_id: Some("t1".into()), + ..MemoryMeta::default() + }; + let window = ThreadWindow::whole("t1"); + assert!(window.excludes(ItemKind::Conversation, &meta)); + assert!( + !window.excludes(ItemKind::Learning, &meta), + "a learning stamped with the thread stays readable" + ); + assert!(!window.excludes(ItemKind::Document, &meta)); +} diff --git a/crates/tinymemory-integrations/src/cortex/engine/evidence.rs b/crates/tinymemory-integrations/src/cortex/engine/evidence.rs index eccc683f..5f840227 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/evidence.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/evidence.rs @@ -158,7 +158,7 @@ fn gather(pack: &Value, filter: &MetaFilter, exclude: Option<&ThreadWindow>) -> if !keeps(filter, envelope.kind, &envelope) { continue; } - if exclude.is_some_and(|window| window.covers_meta(&envelope.meta)) { + if exclude.is_some_and(|window| window.excludes(envelope.kind, &envelope.meta)) { excluded.insert(decoded.event_id); continue; } diff --git a/crates/tinymemory-integrations/src/cortex/engine/evidence_tests.rs b/crates/tinymemory-integrations/src/cortex/engine/evidence_tests.rs index 9bf07e74..58163698 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/evidence_tests.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/evidence_tests.rs @@ -159,3 +159,27 @@ fn turns_are_separated_and_attributed_by_role() { [2023-06-27 10:37 UTC] [assistant] hi there" ); } + +#[test] +fn a_learning_made_in_an_excluded_thread_stays_in_the_evidence() { + let learned = StoreItem::learning( + "the user prefers green tea", + tinymemory_api::LearningKind::Preference, + 0.9, + MemoryMeta { + thread_id: Some("live".into()), + ..MemoryMeta::default() + }, + ); + let pack = json!({ "layers": { "events": [ + event("e_live", &turn("live", Role::User, "what tea do I like")), + event("e_learned", &learned), + ] } }); + let gathered = gather( + &pack, + &MetaFilter::default(), + Some(&ThreadWindow::whole("live")), + ); + let texts: Vec<&str> = gathered.events.iter().map(|e| e.text.as_str()).collect(); + assert_eq!(texts, vec!["the user prefers green tea"]); +} diff --git a/crates/tinymemory-integrations/src/cortex/engine/fetch.rs b/crates/tinymemory-integrations/src/cortex/engine/fetch.rs index 377f5d63..22dd9b0a 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/fetch.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/fetch.rs @@ -133,7 +133,9 @@ pub(super) fn ranked( .filter(|envelope| keeps(filter, kind.unwrap_or(envelope.kind), envelope)) // CortexDB's recall filters can only narrow by label, so an excluded // thread is dropped here, from what the pack returned. - .filter(|envelope| exclude.is_none_or(|window| !window.covers_meta(&envelope.meta))) + .filter(|envelope| { + exclude.is_none_or(|window| !window.excludes(envelope.kind, &envelope.meta)) + }) .filter(|envelope| seen.insert(envelope.id.clone())) .collect() }