diff --git a/crates/tinyagents-harness/src/summarization/README.md b/crates/tinyagents-harness/src/summarization/README.md index 7fac40b08..6ee36fa3b 100644 --- a/crates/tinyagents-harness/src/summarization/README.md +++ b/crates/tinyagents-harness/src/summarization/README.md @@ -92,6 +92,18 @@ loop. ### Rendering - `render_message_for_summary(&Message) -> String`. +- `render_message_for_summary_with(&Message, ToolRendering) -> Option` + — the same, in a chosen mode; `None` when the mode omits the message. +- `ToolRendering` — `Full` (default: tool calls with ids and arguments, tool + results elided) or `Markers` (conversation only: each tool call is one + `[used : ]` line, arguments cut to 120 chars with an + ellipsis, no call ids; tool results omitted). + +### `ModelSummarizer` +- `with_tool_rendering(ToolRendering)` — the transcript's rendering mode; + `Full` unless set. +- `with_max_output_tokens(u32)` — sends `max_tokens` on the summary request. + Unset by default; the request sets no sampling parameters of its own. ## Files @@ -100,7 +112,8 @@ loop. | `mod.rs` | `estimate_tokens`, `Summarizer for ConcatSummarizer`, `SummarizationPolicy` impls; wires the submodules together. | | `types.rs` | All public struct/enum/trait definitions. | | `pairing.rs` | Tool-call-pairing-safe cut-point repair. | -| `render.rs` | `render_message_for_summary`. | +| `render.rs` | `render_message_for_summary`, `render_message_for_summary_with`. | +| `render_tests.rs` | Coverage for both rendering modes and marker truncation. | | `trim.rs` | `trim_messages`/`trim_messages_with`/`trim_messages_to_token_budget_with`. | | `compaction.rs` | `find_cut_point`, `summarize_with_split`, `OverflowClassifier`, `CompactionContext`/`CompactionDecision`. | | `response_overflow.rs` | `detect_response_overflow`, `ResponseOverflowDetection`: overflow read from a successful response (usage above the window, zero-output `length` stop, opt-in short `length` stop). | @@ -142,6 +155,11 @@ loop. agent loop, and a model continues it with its next tool call. The request declares no tools, so that call arrives as plain-text markup (DeepSeek V4: `<｜DSML｜invoke …>`). +- **`ToolRendering::Markers` drops tool output, never the conversation.** + User, assistant and system text render exactly as under `Full`; only tool + results are omitted and tool calls shortened to one marker line each, so a + small instruction-light summarization model sees what was said and done. + `Full` stays the default, byte-identical to `render_message_for_summary`. - **A tool call is never accepted as a summary.** A reply the tool-call grammars parse as a call is retried once, then returned as an error, so `FaultTolerantCachingSummarizer` falls back to its deterministic trim diff --git a/crates/tinyagents-harness/src/summarization/mod.rs b/crates/tinyagents-harness/src/summarization/mod.rs index 841d453b0..579026d5d 100644 --- a/crates/tinyagents-harness/src/summarization/mod.rs +++ b/crates/tinyagents-harness/src/summarization/mod.rs @@ -60,7 +60,7 @@ pub use pairing::{ advance_past_orphan_tools, find_safe_cutoff_point, is_tool_calling_assistant, retract_orphan_tool_calls, tool_pairing_is_intact, }; -pub use render::render_message_for_summary; +pub use render::{render_message_for_summary, render_message_for_summary_with}; pub use resilient::FaultTolerantCachingSummarizer; pub use response_overflow::{ResponseOverflowDetection, detect_response_overflow}; pub use split_turn::{SPLIT_TURN_HEADING, split_turn_start, summarize_split_turn}; diff --git a/crates/tinyagents-harness/src/summarization/model_summarizer.rs b/crates/tinyagents-harness/src/summarization/model_summarizer.rs index 2a67e0cc6..fa3fc0cb8 100644 --- a/crates/tinyagents-harness/src/summarization/model_summarizer.rs +++ b/crates/tinyagents-harness/src/summarization/model_summarizer.rs @@ -23,7 +23,7 @@ use tinyinference_llm::model::{ChatModel, ModelRequest}; use super::types::ModelSummarizer; use super::{ CompressionProvenance, SummarizationPolicy, Summarizer, SummaryKind, SummaryRecord, - SummaryRequest, estimate_tokens, render_message_for_summary, + SummaryRequest, ToolRendering, estimate_tokens, render_message_for_summary_with, }; use crate::error::{Result, TinyAgentsError}; use crate::token_estimation::estimate_slice_tokens; @@ -52,9 +52,28 @@ impl ModelSummarizer { model, model_id: model_id.into(), threshold_fraction: DEFAULT_SUMMARIZE_THRESHOLD_FRACTION, + tool_rendering: ToolRendering::Full, + max_output_tokens: None, } } + /// Choose how tool calls and results are rendered into the transcript the + /// model summarizes. The default, [`ToolRendering::Full`], sends them all. + #[must_use] + pub fn with_tool_rendering(mut self, rendering: ToolRendering) -> Self { + self.tool_rendering = rendering; + self + } + + /// Cap the summary's length: the request carries `max_tokens` set to + /// `max_output_tokens`. Unset by default, which leaves the provider's own + /// default in force. + #[must_use] + pub fn with_max_output_tokens(mut self, max_output_tokens: u32) -> Self { + self.max_output_tokens = Some(max_output_tokens); + self + } + /// Override the threshold fraction recorded in summary provenance. Use the /// same value passed to [`summarization_policy_with`]. #[must_use] @@ -99,7 +118,7 @@ impl ModelSummarizer { let transcript = messages .iter() - .map(render_message_for_summary) + .filter_map(|message| render_message_for_summary_with(message, self.tool_rendering)) .collect::>() .join("\n"); let (system_prompt, request_text) = match kind { @@ -120,10 +139,11 @@ impl ModelSummarizer { "[tinyagents::summarize] dispatching context-window summary" ); - let request = ModelRequest::new(vec![ + let mut request = ModelRequest::new(vec![ Message::system(system_prompt), Message::user(request_text), ]); + request.max_tokens = self.max_output_tokens; let (summary, usage) = self.summarize_once(request).await.map_err(|failure| { let (error, usage) = *failure; match usage { @@ -370,6 +390,13 @@ Produce exactly these sections (write \"None\" when a section is empty):\n\ ## Goal\n\ What the user is ultimately trying to accomplish.\n\ \n\ +## About the User\n\ +Every fact the user shared about themselves (name, places, people, preferences, routines), \ +stated as they gave it.\n\ +\n\ +## Commitments\n\ +What the assistant promised or scheduled (reminders, follow-ups, deadlines), with when.\n\ +\n\ ## Completed Actions\n\ Numbered list of what has already been done, with key results/outputs.\n\ \n\ diff --git a/crates/tinyagents-harness/src/summarization/model_summarizer_tests.rs b/crates/tinyagents-harness/src/summarization/model_summarizer_tests.rs index 8b86df381..d3cbafc1f 100644 --- a/crates/tinyagents-harness/src/summarization/model_summarizer_tests.rs +++ b/crates/tinyagents-harness/src/summarization/model_summarizer_tests.rs @@ -14,7 +14,7 @@ use tinyinference_llm::usage::Usage; use super::{ DEFAULT_SUMMARIZE_KEEP_LAST, DEFAULT_SUMMARIZE_THRESHOLD_FRACTION, FaultTolerantCachingSummarizer, ModelSummarizer, SummarizationPolicy, Summarizer, - SummaryRecord, SummaryRequest, summarization_policy, summarization_policy_with, + SummaryRecord, SummaryRequest, ToolRendering, summarization_policy, summarization_policy_with, }; use crate::error::{Result, TinyAgentsError}; use crate::testkit::ScriptedModel; @@ -360,6 +360,23 @@ async fn pending_work_is_handed_on_as_in_progress_not_stale() { assert!(prompt.contains("already answered"), "{prompt}"); } +/// A compaction must not cost what the user told the assistant about +/// themselves: the checkpoint keeps those facts, and the commitments the +/// assistant made, in sections of their own. +#[tokio::test] +async fn the_checkpoint_keeps_facts_about_the_user_and_commitments() { + let model = Arc::new(ScriptedModel::replies(vec!["## Goal\nx"])); + let summarizer = ModelSummarizer::new(model.clone(), "m"); + summarizer + .summarize(&[Message::user("I live in Pune"), Message::assistant("noted")]) + .await + .unwrap(); + + let prompt = model.requests()[0].messages[0].text(); + assert!(prompt.contains("## About the User"), "{prompt}"); + assert!(prompt.contains("## Commitments"), "{prompt}"); +} + #[tokio::test] async fn a_turn_prefix_request_uses_its_own_prompt() { let model = Arc::new(ScriptedModel::replies(vec![ @@ -398,3 +415,94 @@ async fn a_turn_prefix_request_uses_its_own_prompt() { .contains("beginning of a turn") ); } + +/// A short exchange around one tool call: the user's ask, a call with a long +/// argument, its result carrying a sentinel, and the assistant's reply. +fn cron_exchange() -> Vec { + vec![ + Message::user("remind me to water the plants"), + Message::Assistant(AssistantMessage { + id: None, + content: Vec::new(), + tool_calls: vec![ToolCall::new( + "cron-1", + "cron", + json!({"schedule": "0 9 * * *", "note": format!("water the plants {}", "z".repeat(300))}), + )], + usage: None, + origin: None, + }), + Message::tool("cron-1", "SENTINEL-TOOL-RESULT job 42 created"), + Message::assistant("Done, I will remind you at 9 every morning."), + ] +} + +#[tokio::test] +async fn the_default_rendering_is_unchanged() { + let expected = "user: remind me to water the plants\n\ +assistant: I should inspect the matching records. \ +{\"query\":\"open issues\"}\n\ +tool: {\"matches\":2}\n\ +assistant: There are 2."; + let mut messages = vec![Message::user("remind me to water the plants")]; + messages.extend(tool_call_messages(json!({"query": "open issues"}))); + messages.push(Message::assistant("There are 2.")); + + for rendering in [None, Some(ToolRendering::Full)] { + let model = Arc::new(ScriptedModel::replies(vec!["## Goal\nx"])); + let mut summarizer = ModelSummarizer::new(model.clone(), "m"); + if let Some(rendering) = rendering { + summarizer = summarizer.with_tool_rendering(rendering); + } + summarizer.summarize(&messages).await.unwrap(); + let request = &model.requests()[0]; + let sent = request.messages[1].text(); + assert!( + sent.contains(&format!("\n{expected}\n")), + "{sent}" + ); + assert_eq!(request.max_tokens, None); + assert_eq!(request.temperature, None); + } +} + +#[tokio::test] +async fn markers_send_the_conversation_without_tool_output() { + let model = Arc::new(ScriptedModel::replies(vec!["## Goal\nx"])); + let summarizer = + ModelSummarizer::new(model.clone(), "m").with_tool_rendering(ToolRendering::Markers); + summarizer.summarize(&cron_exchange()).await.unwrap(); + + let sent = model.requests()[0].messages[1].text(); + assert!( + sent.contains("user: remind me to water the plants"), + "{sent}" + ); + assert!( + sent.contains("assistant: Done, I will remind you at 9 every morning."), + "{sent}" + ); + assert!(sent.contains("[used cron: "), "{sent}"); + assert!(!sent.contains("SENTINEL-TOOL-RESULT"), "{sent}"); + assert!(!sent.contains(" &'static str { match message { @@ -147,3 +153,57 @@ pub fn render_message_for_summary(message: &Message) -> String { format!("{}: {}", role_label(message), parts.join(" ")) } + +/// Renders a single message for a summary transcript in the given +/// [`ToolRendering`] mode, or `None` when the mode leaves the message out. +/// +/// [`ToolRendering::Full`] is exactly [`render_message_for_summary`]. +/// [`ToolRendering::Markers`] renders system, user, assistant and custom +/// content the same way, but: +/// +/// - each assistant tool call becomes its own line, +/// `[used : ]`, without its call id and with the arguments' +/// JSON cut to 120 characters (an ellipsis marks the cut); +/// - an assistant turn that only called tools is just those lines; +/// - a tool result ([`Message::Tool`]) is omitted: `None`; +/// - so is an assistant turn with neither prose nor tool calls. +pub fn render_message_for_summary_with( + message: &Message, + rendering: ToolRendering, +) -> Option { + match (rendering, message) { + (ToolRendering::Full, _) => Some(render_message_for_summary(message)), + (ToolRendering::Markers, Message::Tool(_)) => None, + (ToolRendering::Markers, Message::Assistant(assistant)) => { + let parts = render_content(&assistant.content); + let prose = (!parts.is_empty()) + .then(|| format!("{}: {}", role_label(message), parts.join(" "))); + let markers = assistant.tool_calls.iter().map(|call| { + format!( + "[used {}: {}]", + call.name, + shorten(&call.arguments.to_string(), MARKER_ARGUMENT_CHARS) + ) + }); + let lines = prose.into_iter().chain(markers).collect::>(); + (!lines.is_empty()).then(|| lines.join("\n")) + } + (ToolRendering::Markers, _) => Some(render_message_for_summary(message)), + } +} + +/// Cuts `text` to at most `max_chars` characters, the last of them an +/// ellipsis when anything was dropped. Counts characters, so a cut never +/// splits one. +fn shorten(text: &str, max_chars: usize) -> String { + if text.chars().count() <= max_chars { + return text.to_string(); + } + let mut kept: String = text.chars().take(max_chars.saturating_sub(1)).collect(); + kept.push('…'); + kept +} + +#[cfg(test)] +#[path = "render_tests.rs"] +mod tests; diff --git a/crates/tinyagents-harness/src/summarization/render_tests.rs b/crates/tinyagents-harness/src/summarization/render_tests.rs new file mode 100644 index 000000000..038e87fa4 --- /dev/null +++ b/crates/tinyagents-harness/src/summarization/render_tests.rs @@ -0,0 +1,129 @@ +//! Tests for [`render_message_for_summary_with`]: the [`ToolRendering::Full`] +//! mode matches [`render_message_for_summary`], and [`ToolRendering::Markers`] +//! keeps the conversation while reducing tools to one-line markers. + +use super::*; +use crate::summarization::ToolRendering; +use serde_json::json; +use tinyinference_llm::message::AssistantMessage; +use tinyinference_llm::tool::ToolCall; + +fn assistant(text: Option<&str>, calls: Vec) -> Message { + Message::Assistant(AssistantMessage { + id: None, + content: text + .map(|t| ContentBlock::Text(t.into())) + .into_iter() + .collect(), + tool_calls: calls, + usage: None, + origin: None, + }) +} + +#[test] +fn full_rendering_matches_the_default_renderer_for_every_message_kind() { + let messages = vec![ + Message::system("be brief"), + Message::user("remind me at 9"), + assistant( + Some("on it"), + vec![ToolCall::new("c1", "cron", json!({"at": "9am"}))], + ), + Message::tool("c1", "scheduled"), + ]; + for message in &messages { + assert_eq!( + render_message_for_summary_with(message, ToolRendering::Full), + Some(render_message_for_summary(message)) + ); + } +} + +#[test] +fn markers_keep_prose_and_system_messages_as_rendered_today() { + for message in [ + Message::system("be brief"), + Message::user("remind me at 9"), + Message::assistant("done, see you then"), + ] { + assert_eq!( + render_message_for_summary_with(&message, ToolRendering::Markers), + Some(render_message_for_summary(&message)) + ); + } +} + +#[test] +fn markers_omit_tool_results() { + let result = Message::tool("c1", "SENTINEL-RESULT"); + assert_eq!( + render_message_for_summary_with(&result, ToolRendering::Markers), + None + ); +} + +#[test] +fn markers_render_each_call_on_its_own_line_without_its_id() { + let message = assistant( + Some("on it"), + vec![ + ToolCall::new("c1", "cron", json!({"at": "9am"})), + ToolCall::new("c2", "search", json!({"q": "news"})), + ], + ); + assert_eq!( + render_message_for_summary_with(&message, ToolRendering::Markers).as_deref(), + Some("assistant: on it\n[used cron: {\"at\":\"9am\"}]\n[used search: {\"q\":\"news\"}]") + ); +} + +#[test] +fn markers_still_render_an_assistant_turn_that_only_called_tools() { + let message = assistant( + None, + vec![ToolCall::new("c1", "cron", json!({"at": "9am"}))], + ); + assert_eq!( + render_message_for_summary_with(&message, ToolRendering::Markers).as_deref(), + Some("[used cron: {\"at\":\"9am\"}]") + ); +} + +#[test] +fn marker_arguments_up_to_the_cap_are_kept_whole() { + // `{"k":"…"}` is 8 characters around the value: 112 + 8 = exactly the cap. + let arguments = json!({"k": "a".repeat(112)}); + assert_eq!(arguments.to_string().chars().count(), MARKER_ARGUMENT_CHARS); + let message = assistant(None, vec![ToolCall::new("c1", "cron", arguments.clone())]); + assert_eq!( + render_message_for_summary_with(&message, ToolRendering::Markers), + Some(format!("[used cron: {arguments}]")) + ); +} + +#[test] +fn long_marker_arguments_are_cut_on_a_char_boundary_with_an_ellipsis() { + let arguments = json!({"note": "é".repeat(300)}); + let message = assistant(None, vec![ToolCall::new("c1", "cron", arguments.clone())]); + let line = render_message_for_summary_with(&message, ToolRendering::Markers) + .expect("an assistant turn is never omitted"); + + let shown = line + .strip_prefix("[used cron: ") + .and_then(|rest| rest.strip_suffix(']')) + .expect("marker shape"); + assert_eq!(shown.chars().count(), MARKER_ARGUMENT_CHARS); + assert!(shown.ends_with('…'), "{shown}"); + let kept = shown.trim_end_matches('…'); + assert!(arguments.to_string().starts_with(kept), "{shown}"); +} + +#[test] +fn markers_leave_out_an_assistant_turn_with_neither_prose_nor_calls() { + let empty = assistant(None, Vec::new()); + assert_eq!( + render_message_for_summary_with(&empty, ToolRendering::Markers), + None + ); +} diff --git a/crates/tinyagents-harness/src/summarization/types.rs b/crates/tinyagents-harness/src/summarization/types.rs index b7c119374..0fe2989f6 100644 --- a/crates/tinyagents-harness/src/summarization/types.rs +++ b/crates/tinyagents-harness/src/summarization/types.rs @@ -23,6 +23,28 @@ pub struct ModelSummarizer { pub(super) model_id: String, /// Threshold reported in summary provenance; the policy owns the trigger. pub(super) threshold_fraction: f64, + /// How tool calls and results are rendered into the transcript. + pub(super) tool_rendering: ToolRendering, + /// Output cap sent as the request's `max_tokens`; `None` leaves it unset. + pub(super) max_output_tokens: Option, +} + +/// How a summary transcript renders tool activity. +/// +/// See [`render_message_for_summary_with`](super::render_message_for_summary_with). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ToolRendering { + /// Every tool call with its id and arguments, and every tool result, in + /// the tagged form of + /// [`render_message_for_summary`](super::render_message_for_summary). + #[default] + Full, + /// The conversation only: each tool call becomes one + /// `[used : ]` line with its arguments cut short, and tool + /// results are left out. For small summarization models that should see + /// what was said and done, not the tools' output. + Markers, } impl std::fmt::Debug for ModelSummarizer { @@ -30,6 +52,8 @@ impl std::fmt::Debug for ModelSummarizer { f.debug_struct("ModelSummarizer") .field("model_id", &self.model_id) .field("threshold_fraction", &self.threshold_fraction) + .field("tool_rendering", &self.tool_rendering) + .field("max_output_tokens", &self.max_output_tokens) .finish_non_exhaustive() } }