From 05688fbb32f22470a17f7c95616ce5c473dd3e34 Mon Sep 17 00:00:00 2001 From: EdgeCaser Date: Wed, 8 Jul 2026 10:13:37 -0700 Subject: [PATCH] fix(dedupe): reject LLM output that introduces lines not in the original MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit semantic_dedupe_memory only guarded against too-short responses. But the observed failure mode makes the file LONGER: the model ignores "return ONLY the cleaned file" and prepends chat commentary ("No duplicates were found…", or a flag about session context), which --apply would then write verbatim as the opening lines of GLOBAL_MEMORY.md and load into every session. A dedupe may only remove lines, never introduce them. Fail closed if any non-blank output line was not present verbatim in the input. This is a deterministic code-level guard (per PITFALLS #1: enforce in code, not prompt), and makes the command safe to run unattended in a scheduled preview. Tests: subset-removal passes; too-short still rejected; prepended and appended commentary both rejected; unchanged-file passes. Also fixes the mock target in the new tests (function-local `from memsync.llm import call_llm` must be patched at memsync.llm, not memsync.sync). Co-Authored-By: Claude Opus 4.8 --- memsync/sync.py | 20 +++++++++++++++++++ tests/test_sync.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/memsync/sync.py b/memsync/sync.py index 9bd67fa..5cd0718 100644 --- a/memsync/sync.py +++ b/memsync/sync.py @@ -312,6 +312,26 @@ def semantic_dedupe_memory(content: str, config: Config) -> str: f"Semantic dedupe response too short ({len(cleaned)} chars vs " f"{len(content)} original) — rejecting to avoid data loss." ) + # Guard: a dedupe may only REMOVE lines, never introduce new ones. Models + # sometimes ignore "return ONLY the cleaned file" and prepend chat commentary + # ("No duplicates were found…") or flag something — which would then be + # written verbatim into GLOBAL_MEMORY.md and loaded into every session. The + # short-response guard above misses this because such pollution makes the + # file LONGER. Fail closed if the output contains any non-blank line that + # was not present verbatim in the input. + original_lines = {ln.strip() for ln in content.splitlines() if ln.strip()} + introduced = [ + ln.strip() + for ln in cleaned.splitlines() + if ln.strip() and ln.strip() not in original_lines + ] + if introduced: + preview = introduced[0][:80] + raise ValueError( + f"Semantic dedupe introduced {len(introduced)} line(s) absent from the " + f"original (e.g. {preview!r}) — rejecting to avoid writing model " + "commentary into memory." + ) return cleaned diff --git a/tests/test_sync.py b/tests/test_sync.py index 1c8a72f..47f7d91 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -494,3 +494,53 @@ def test_malformed_response_sets_malformed_flag(self): }): result = merge_candidates_into_memory("- fact\n", SAMPLE_MEMORY, config) assert result["malformed"] is True + + +class TestSemanticDedupeMemory: + @staticmethod + def _llm_result(text: str) -> dict: + return {"text": text, "input_tokens": 5, "output_tokens": 5, "truncated": False} + + def test_removes_duplicate_returns_subset(self): + from memsync.sync import semantic_dedupe_memory + config = Config() + # Model correctly drops the second near-duplicate, keeps everything else verbatim. + cleaned = SAMPLE_MEMORY.replace("- Always backup before writing\n", "") + with patch("memsync.llm.call_llm", return_value=self._llm_result(cleaned)): + result = semantic_dedupe_memory(SAMPLE_MEMORY, config) + assert "Never rewrite from scratch" in result + assert "Always backup before writing" not in result + + def test_rejects_too_short_response(self): + from memsync.sync import semantic_dedupe_memory + config = Config() + with patch("memsync.llm.call_llm", return_value=self._llm_result("# Global Memory\n")): + with pytest.raises(ValueError, match="too short"): + semantic_dedupe_memory(SAMPLE_MEMORY, config) + + def test_rejects_prepended_commentary(self): + # The real failure mode: model prepends chat commentary as file content. + from memsync.sync import semantic_dedupe_memory + config = Config() + polluted = ( + "No semantic duplicate bullets were found — the file is returned unchanged.\n\n" + + SAMPLE_MEMORY + ) + with patch("memsync.llm.call_llm", return_value=self._llm_result(polluted)): + with pytest.raises(ValueError, match="introduced"): + semantic_dedupe_memory(SAMPLE_MEMORY, config) + + def test_rejects_appended_commentary(self): + from memsync.sync import semantic_dedupe_memory + config = Config() + polluted = SAMPLE_MEMORY + "\nLet me know if you'd like anything else!\n" + with patch("memsync.llm.call_llm", return_value=self._llm_result(polluted)): + with pytest.raises(ValueError, match="introduced"): + semantic_dedupe_memory(SAMPLE_MEMORY, config) + + def test_unchanged_file_passes(self): + from memsync.sync import semantic_dedupe_memory + config = Config() + with patch("memsync.llm.call_llm", return_value=self._llm_result(SAMPLE_MEMORY)): + result = semantic_dedupe_memory(SAMPLE_MEMORY, config) + assert "Never rewrite from scratch" in result