Exclude only conversation turns from a read's thread window

crates/tinymemory-api/src/conformance/reference/mod.rs

@@ -161,7 +161,7 @@ impl MemoryEngine for ReferenceEngine {
         req.validate()?;
         let mut hits = self.ranked(&req.question, FetchMode::Hybrid, &req.filter)?;
         if let Some(window) = &req.exclude_thread {
-            hits.retain(|hit| !window.covers(hit));
+            hits.retain(|hit| !window.excludes(hit.kind, &hit.meta));
         }
         let citations: Vec<Citation> = hits
             .into_iter()
@@ -207,7 +207,7 @@ impl MemoryEngine for ReferenceEngine {
         req.validate()?;
         let mut hits = self.ranked(&req.query, req.mode, &req.filter)?;
         if let Some(window) = &req.exclude_thread {
-            hits.retain(|hit| !window.covers(hit));
+            hits.retain(|hit| !window.excludes(hit.kind, &hit.meta));
         }
         if let Some(hint) = &req.refers_to {
             hint.rank(&mut hits);

crates/tinymemory-api/src/conformance/suite/checks.rs

@@ -3,8 +3,8 @@
 use std::collections::BTreeSet;
 
 use crate::{
-    Error as ApiError, FetchMode, FetchRequest, ForgetTarget, ItemId, MemoryMeta, MetaFilter,
-    RecallMode, RecallRequest, Role, StoreItem, ThreadWindow, Turn, TurnRange,
+    Error as ApiError, FetchMode, FetchRequest, ForgetTarget, ItemId, LearningKind, MemoryMeta,
+    MetaFilter, RecallMode, RecallRequest, Role, StoreItem, ThreadWindow, Turn, TurnRange,
 };
 
 use super::{Ctx, ensure};
@@ -21,14 +21,15 @@ pub(super) async fn all(ctx: &Ctx<'_>) -> Result<()> {
     super::bulk::store_many(ctx).await?;
     super::lifecycle::store_with(ctx).await?;
     fetch_filters(ctx).await?;
-    excluded_thread(ctx).await?;
     unsupported_modes(ctx).await?;
     super::namespaces::namespaces(ctx).await?;
     empty_forget(ctx).await?;
     forget_by_id(ctx).await?;
     forget_by_filter(ctx).await?;
     super::erase::erase(ctx).await?;
     recall(ctx).await?;
+    // After `recall`, so a recall fault is reported by the check named for it.
+    excluded_thread(ctx).await?;
     super::lifecycle::consolidate(ctx).await
 }
 
@@ -183,8 +184,8 @@ async fn fetch_filters(ctx: &Ctx<'_>) -> Result<()> {
 }
 
 /// A read's `exclude_thread` leaves that thread's turns out of every fetch
-/// mode and of recall's citations, and a window starting after a turn keeps
-/// it.
+/// mode and of recall's citations, a window starting after a turn keeps it,
+/// and a learning stamped with the thread stays readable.
 async fn excluded_thread(ctx: &Ctx<'_>) -> Result<()> {
     const CHECK: &str = "excluded_thread";
     let turn = |thread: &str| {
@@ -205,6 +206,21 @@ async fn excluded_thread(ctx: &Ctx<'_>) -> Result<()> {
     );
     let live_id = ctx.call(CHECK, ctx.engine.store(turn(&live))).await?.id;
     let other_id = ctx.call(CHECK, ctx.engine.store(turn(&other))).await?.id;
+    let mut learned_meta = ctx.run.meta();
+    learned_meta.thread_id = Some(live.clone());
+    let learned_text = format!("{} a learning made in {live}", ctx.run.marker);
+    let learned_id = ctx
+        .call(
+            CHECK,
+            ctx.engine.store(StoreItem::learning(
+                learned_text.clone(),
+                LearningKind::Fact,
+                0.9,
+                learned_meta,
+            )),
+        )
+        .await?
+        .id;
     let modes = ctx.engine.descriptor().fetch_modes.clone();
     for mode in &modes {
         let mut request = FetchRequest::new(ctx.run.marker.clone(), *mode, 50);
@@ -222,6 +238,12 @@ async fn excluded_thread(ctx: &Ctx<'_>) -> Result<()> {
                 )
             },
         )?;
+        ensure(CHECK, found.contains(&learned_id), || {
+            format!(
+                "{} fetch excluding {live} dropped its learning {learned_id}: only conversation turns are excluded",
+                mode.as_str()
+            )
+        })?;
         request.exclude_thread = Some(ThreadWindow {
             thread_id: live.clone(),
             from_turn: 1,
@@ -246,6 +268,14 @@ async fn excluded_thread(ctx: &Ctx<'_>) -> Result<()> {
             .all(|citation| citation.id != live_id),
         || format!("a recall excluding {live} cited its turn {live_id}"),
     )?;
+    ensure(
+        CHECK,
+        answer
+            .citations
+            .iter()
+            .any(|citation| citation.id == learned_id),
+        || format!("a recall excluding {live} left out its learning {learned_id}"),
+    )?;
     // In evidence mode the engine writes no answer of its own, so the text it
     // returns is all evidence, and none of it may come from the window.
     request.mode = RecallMode::Evidence;

crates/tinymemory-api/src/query/mod.rs

@@ -109,8 +109,9 @@ impl RecallRequest {
 /// `from_turn` on. A caller reading inside a thread sets it to the turns its
 /// prompt already holds, so the message being answered (logged before the
 /// read) does not come back as a memory; `from_turn: 0` leaves the whole
-/// thread out. Engines apply it to what they return; an engine whose store
-/// cannot exclude server side drops the matching items from its results.
+/// thread out. Engines apply it to what they return ([`ThreadWindow::excludes`]:
+/// conversation turns only); an engine whose store cannot exclude server side
+/// drops the matching items from its results.
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub struct ThreadWindow {
     /// The thread.
@@ -149,6 +150,16 @@ impl ThreadWindow {
                 .is_none_or(|turns| turns.last >= self.from_turn)
     }
 
+    /// Whether a read's `exclude_thread` leaves out an item of `kind`
+    /// carrying `meta`: only **conversation** turns are left out. A learning
+    /// or a document stamped with the thread (a host records where it was
+    /// learned) stays readable, since the prompt holds the thread's turns,
+    /// not what was learned from them.
+    #[must_use]
+    pub fn excludes(&self, kind: ItemKind, meta: &MemoryMeta) -> bool {
+        kind == ItemKind::Conversation && self.covers_meta(meta)
+    }
+
     /// Checks the window names a thread.
     ///
     /// # Errors

crates/tinymemory-api/src/query/mod_tests.rs

@@ -131,3 +131,18 @@ fn a_read_without_an_excluded_thread_serializes_as_before() {
             .unwrap();
     assert_eq!(parsed.exclude_thread, None);
 }
+
+#[test]
+fn a_thread_window_excludes_conversation_turns_only() {
+    let meta = MemoryMeta {
+        thread_id: Some("t1".into()),
+        ..MemoryMeta::default()
+    };
+    let window = ThreadWindow::whole("t1");
+    assert!(window.excludes(ItemKind::Conversation, &meta));
+    assert!(
+        !window.excludes(ItemKind::Learning, &meta),
+        "a learning stamped with the thread stays readable"
+    );
+    assert!(!window.excludes(ItemKind::Document, &meta));
+}

crates/tinymemory-integrations/src/cortex/engine/evidence.rs

@@ -158,7 +158,7 @@ fn gather(pack: &Value, filter: &MetaFilter, exclude: Option<&ThreadWindow>) ->
         if !keeps(filter, envelope.kind, &envelope) {
             continue;
         }
-        if exclude.is_some_and(|window| window.covers_meta(&envelope.meta)) {
+        if exclude.is_some_and(|window| window.excludes(envelope.kind, &envelope.meta)) {
             excluded.insert(decoded.event_id);
             continue;
         }

crates/tinymemory-integrations/src/cortex/engine/evidence_tests.rs

@@ -159,3 +159,27 @@ fn turns_are_separated_and_attributed_by_role() {
          [2023-06-27 10:37 UTC] [assistant] hi there"
     );
 }
+
+#[test]
+fn a_learning_made_in_an_excluded_thread_stays_in_the_evidence() {
+    let learned = StoreItem::learning(
+        "the user prefers green tea",
+        tinymemory_api::LearningKind::Preference,
+        0.9,
+        MemoryMeta {
+            thread_id: Some("live".into()),
+            ..MemoryMeta::default()
+        },
+    );
+    let pack = json!({ "layers": { "events": [
+        event("e_live", &turn("live", Role::User, "what tea do I like")),
+        event("e_learned", &learned),
+    ] } });
+    let gathered = gather(
+        &pack,
+        &MetaFilter::default(),
+        Some(&ThreadWindow::whole("live")),
+    );
+    let texts: Vec<&str> = gathered.events.iter().map(|e| e.text.as_str()).collect();
+    assert_eq!(texts, vec!["the user prefers green tea"]);
+}

crates/tinymemory-integrations/src/cortex/engine/fetch.rs

@@ -133,7 +133,9 @@ pub(super) fn ranked(
         .filter(|envelope| keeps(filter, kind.unwrap_or(envelope.kind), envelope))
         // CortexDB's recall filters can only narrow by label, so an excluded
         // thread is dropped here, from what the pack returned.
-        .filter(|envelope| exclude.is_none_or(|window| !window.covers_meta(&envelope.meta)))
+        .filter(|envelope| {
+            exclude.is_none_or(|window| !window.excludes(envelope.kind, &envelope.meta))
+        })
         .filter(|envelope| seen.insert(envelope.id.clone()))
         .collect()
 }