From 9de88754af8f6b423e4aa9d4bca202e38b2becf2 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:05:37 +0100 Subject: [PATCH 01/31] mason: add read-only Git index reader and golden fixtures Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 2f67e5589a47fa7696cae759c83769233b198758) --- crates/aft/src/git_index_reader.rs | 1291 ++++++++++++++++++++++++++++ crates/aft/src/lib.rs | 1 + 2 files changed, 1292 insertions(+) create mode 100644 crates/aft/src/git_index_reader.rs diff --git a/crates/aft/src/git_index_reader.rs b/crates/aft/src/git_index_reader.rs new file mode 100644 index 0000000000..4cfc79c5bf --- /dev/null +++ b/crates/aft/src/git_index_reader.rs @@ -0,0 +1,1291 @@ +//! Read-only Git index snapshots and counted, deadline-bound Git metadata queries. +//! +//! Nothing in production routes through this module yet. A snapshot records only +//! the index bytes observed, not proof that worktree bytes are clean. Callers must +//! verify worktree content independently. + +use crate::file_identity::FileIdentity; +use sha1::Digest; +use std::fs::{self, File, Metadata}; +use std::io::{self, Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output, Stdio}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::mpsc; +use std::thread; +use std::time::{Duration, Instant, SystemTime}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ObjectFormat { + Sha1, + Sha256, +} + +impl ObjectFormat { + fn oid_len(self) -> usize { + match self { + Self::Sha1 => 20, + Self::Sha256 => 32, + } + } + + fn digest(self, bytes: &[u8]) -> Vec { + match self { + Self::Sha1 => sha1::Sha1::digest(bytes).to_vec(), + Self::Sha256 => sha2::Sha256::digest(bytes).to_vec(), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct IndexTime { + pub seconds: u32, + pub nanoseconds: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct IndexStat { + pub ctime: IndexTime, + pub mtime: IndexTime, + pub dev: u32, + pub ino: u32, + pub mode: u32, + pub uid: u32, + pub gid: u32, + pub size: u32, +} + +/// Full-width lstat fields supplied by the caller; times are Unix seconds. +/// On Windows ctime here means Git's creation time, not change-sensitive evidence. +#[derive(Clone, Debug)] +pub struct WorktreeStat { + pub ctime_seconds: i64, + pub ctime_nanoseconds: u32, + pub mtime_seconds: i64, + pub mtime_nanoseconds: u32, + pub dev: u64, + pub ino: u64, + pub mode: u32, + pub size: u64, +} + +impl IndexStat { + /// Compare the recorded tuple only, never infer clean content from it. + /// Git stores low 32 bits, including for signed seconds. Git for Windows + /// does not record device/inode, so those fields cannot participate there. + pub fn matches_stat(&self, stat: &WorktreeStat) -> bool { + let identity_matches = + cfg!(windows) || (self.dev == stat.dev as u32 && self.ino == stat.ino as u32); + identity_matches + && self.size == stat.size as u32 + && self.mode == stat.mode + && self.ctime.seconds == stat.ctime_seconds as u32 + && self.ctime.nanoseconds == stat.ctime_nanoseconds + && self.mtime.seconds == stat.mtime_seconds as u32 + && self.mtime.nanoseconds == stat.mtime_nanoseconds + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum EntryException { + Unmerged(u8), + AssumeValid, + SkipWorktree, + IntentToAdd, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct IndexEntry { + /// Git paths are bytes, not necessarily UTF-8, and use '/' on every platform. + pub path: Vec, + pub oid: Vec, + pub stat: IndexStat, + pub stage: u8, + pub assume_valid: bool, + pub skip_worktree: bool, + pub intent_to_add: bool, +} + +impl IndexEntry { + pub fn exceptions(&self) -> Vec { + let mut exceptions = Vec::new(); + if self.stage != 0 { + exceptions.push(EntryException::Unmerged(self.stage)); + } + if self.assume_valid { + exceptions.push(EntryException::AssumeValid); + } + if self.skip_worktree { + exceptions.push(EntryException::SkipWorktree); + } + if self.intent_to_add { + exceptions.push(EntryException::IntentToAdd); + } + exceptions + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct IndexSnapshot { + pub version: u32, + pub object_format: ObjectFormat, + pub entries: Vec, + /// The verified trailer also identifies the exact index bytes observed. + pub checksum: Vec, + pub modified: SystemTime, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum StrictReason { + IndexLocked, + Unreadable(io::ErrorKind), + Unstable, + ChecksumMismatch, + UnsupportedVersion(u32), + RequiredExtension([u8; 4]), + SparseDirectory, + Malformed(&'static str), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum IndexRead { + Snapshot(IndexSnapshot), + Strict(StrictReason), +} + +#[derive(Clone, Copy, Debug)] +pub struct ReadOptions { + pub attempts: usize, + /// Total budget for waiting on index.lock, shared by all attempts. + pub lock_wait: Duration, + pub retry_delay: Duration, +} + +impl Default for ReadOptions { + fn default() -> Self { + Self { + attempts: 3, + lock_wait: Duration::from_millis(100), + retry_delay: Duration::from_millis(5), + } + } +} + +/// Read the explicitly supplied index (including linked-worktree/custom indexes). +/// The caller supplies the repository's object format; no Git discovery or writes +/// are hidden in a snapshot read. Strict requires an independent worktree scan +/// rather than using the index to avoid content reads. +pub fn read_index(path: &Path, format: ObjectFormat, options: ReadOptions) -> IndexRead { + read_index_with_hook(path, format, options, || {}) +} + +fn read_index_with_hook( + path: &Path, + format: ObjectFormat, + options: ReadOptions, + mut after_read: impl FnMut(), +) -> IndexRead { + let mut lock_name = path.as_os_str().to_os_string(); + lock_name.push(".lock"); + let lock_path = PathBuf::from(lock_name); + let mut lock_remaining = options.lock_wait; + let mut reason = StrictReason::Unstable; + for attempt in 0..options.attempts { + if let Err(reason) = wait_for_unlock(&lock_path, &mut lock_remaining) { + return IndexRead::Strict(reason); + } + let result = read_once(path, format, &mut after_read); + // A lock appearing during the read invalidates even checksum-valid bytes. + let saw_lock = match wait_for_unlock(&lock_path, &mut lock_remaining) { + Ok(saw_lock) => saw_lock, + Err(reason) => return IndexRead::Strict(reason), + }; + match if saw_lock { + Err(StrictReason::Unstable) + } else { + result + } { + Ok(snapshot) => { + // If the writer replaced the pathname while we held the old open + // file, read_once rejects it rather than publishing the old index. + return IndexRead::Snapshot(snapshot); + } + Err(error) => { + let retry = matches!( + error, + StrictReason::Unstable + | StrictReason::ChecksumMismatch + | StrictReason::Unreadable(_) + ); + reason = error; + if !retry { + break; + } + } + } + if attempt + 1 < options.attempts { + thread::sleep(options.retry_delay); + } + } + IndexRead::Strict(reason) +} + +fn wait_for_unlock(path: &Path, remaining: &mut Duration) -> Result { + let start = Instant::now(); + let mut saw_lock = false; + loop { + match fs::symlink_metadata(path) { + Err(error) if error.kind() == io::ErrorKind::NotFound => { + *remaining = remaining.saturating_sub(start.elapsed()); + return Ok(saw_lock); + } + Err(error) => return Err(StrictReason::Unreadable(error.kind())), + Ok(_) => saw_lock = true, + } + if start.elapsed() >= *remaining { + *remaining = Duration::ZERO; + return Err(StrictReason::IndexLocked); + } + thread::sleep(Duration::from_millis(5).min(*remaining - start.elapsed().min(*remaining))); + } +} + +fn read_once( + path: &Path, + format: ObjectFormat, + after_read: &mut impl FnMut(), +) -> Result { + let io_reason = |error: io::Error| StrictReason::Unreadable(error.kind()); + let mut file = File::open(path).map_err(io_reason)?; + let before = file.metadata().map_err(io_reason)?; + let before_identity = FileIdentity::of_file(&file); + if !before.is_file() { + return Err(StrictReason::Malformed("index is not a regular file")); + } + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes).map_err(io_reason)?; + after_read(); + let after = file.metadata().map_err(io_reason)?; + let after_identity = FileIdentity::of_file(&file); + let (current, current_identity) = crate::file_identity::metadata(path).map_err(io_reason)?; + if !same_snapshot(&before, before_identity, &after, after_identity) + || !same_snapshot(&after, after_identity, ¤t, current_identity) + || before.len() != bytes.len() as u64 + { + return Err(StrictReason::Unstable); + } + parse_index(&bytes, format, before.modified().map_err(io_reason)?) +} + +fn same_snapshot( + before: &Metadata, + before_identity: FileIdentity, + after: &Metadata, + after_identity: FileIdentity, +) -> bool { + before.len() == after.len() + && before.modified().ok().is_some() + && before.modified().ok() == after.modified().ok() + && before_identity != FileIdentity::Unavailable + && before_identity == after_identity +} + +struct Cursor<'a> { + bytes: &'a [u8], + pos: usize, +} + +impl<'a> Cursor<'a> { + fn take(&mut self, len: usize) -> Result<&'a [u8], StrictReason> { + let end = self + .pos + .checked_add(len) + .ok_or(StrictReason::Malformed("length overflow"))?; + let bytes = self + .bytes + .get(self.pos..end) + .ok_or(StrictReason::Malformed("truncated index"))?; + self.pos = end; + Ok(bytes) + } + + fn u32(&mut self) -> Result { + Ok(u32::from_be_bytes(self.take(4)?.try_into().unwrap())) + } + + fn u16(&mut self) -> Result { + Ok(u16::from_be_bytes(self.take(2)?.try_into().unwrap())) + } + + fn nul_string(&mut self) -> Result<&'a [u8], StrictReason> { + let len = self.bytes[self.pos..] + .iter() + .position(|&b| b == 0) + .ok_or(StrictReason::Malformed("unterminated path"))?; + let string = self.take(len)?; + self.take(1)?; + Ok(string) + } + + fn strip_count(&mut self) -> Result { + let mut byte = self.take(1)?[0]; + let mut count = usize::from(byte & 0x7f); + while byte & 0x80 != 0 { + byte = self.take(1)?[0]; + count = count + .checked_add(1) + .and_then(|n| n.checked_mul(128)) + .and_then(|n| n.checked_add(usize::from(byte & 0x7f))) + .ok_or(StrictReason::Malformed("v4 prefix overflow"))?; + } + Ok(count) + } +} + +fn parse_index( + bytes: &[u8], + format: ObjectFormat, + modified: SystemTime, +) -> Result { + let trailer = format.oid_len(); + let body_len = bytes + .len() + .checked_sub(trailer) + .filter(|&n| n >= 12) + .ok_or(StrictReason::ChecksumMismatch)?; + let (body, checksum) = bytes.split_at(body_len); + if format.digest(body) != checksum { + return Err(StrictReason::ChecksumMismatch); + } + let mut cursor = Cursor { + bytes: body, + pos: 0, + }; + if cursor.take(4)? != b"DIRC" { + return Err(StrictReason::Malformed("invalid signature")); + } + let version = cursor.u32()?; + if !(2..=4).contains(&version) { + return Err(StrictReason::UnsupportedVersion(version)); + } + let count = cursor.u32()?; + let mut entries: Vec = Vec::new(); + for _ in 0..count { + let start = cursor.pos; + let stat = IndexStat { + ctime: IndexTime { + seconds: cursor.u32()?, + nanoseconds: cursor.u32()?, + }, + mtime: IndexTime { + seconds: cursor.u32()?, + nanoseconds: cursor.u32()?, + }, + dev: cursor.u32()?, + ino: cursor.u32()?, + mode: cursor.u32()?, + uid: cursor.u32()?, + gid: cursor.u32()?, + size: cursor.u32()?, + }; + if stat.ctime.nanoseconds >= 1_000_000_000 || stat.mtime.nanoseconds >= 1_000_000_000 { + return Err(StrictReason::Malformed("invalid stat nanoseconds")); + } + let oid = cursor.take(trailer)?.to_vec(); + let flags = cursor.u16()?; + let extended = if flags & 0x4000 != 0 { + if version == 2 { + return Err(StrictReason::Malformed("extended flags in v2")); + } + let extended = cursor.u16()?; + if extended & !0x6000 != 0 { + return Err(StrictReason::Malformed("unknown extended entry flags")); + } + extended + } else { + 0 + }; + let path = if version == 4 { + let strip = cursor.strip_count()?; + let previous = entries.last().map_or(&[][..], |entry| &entry.path); + let prefix_len = previous + .len() + .checked_sub(strip) + .ok_or(StrictReason::Malformed("v4 prefix exceeds previous path"))?; + let mut path = previous[..prefix_len].to_vec(); + path.extend_from_slice(cursor.nul_string()?); + path + } else { + let path = cursor.nul_string()?.to_vec(); + let padding = (8 - (cursor.pos - start) % 8) % 8; + if cursor.take(padding)?.iter().any(|&byte| byte != 0) { + return Err(StrictReason::Malformed("nonzero entry padding")); + } + path + }; + if usize::from(flags & 0x0fff) != path.len().min(0x0fff) { + return Err(StrictReason::Malformed("entry path length mismatch")); + } + entries.push(IndexEntry { + path, + oid, + stat, + stage: ((flags >> 12) & 3) as u8, + assume_valid: flags & 0x8000 != 0, + skip_worktree: extended & 0x4000 != 0, + intent_to_add: extended & 0x2000 != 0, + }); + } + while cursor.pos < body.len() { + let signature: [u8; 4] = cursor.take(4)?.try_into().unwrap(); + let len = cursor.u32()? as usize; + cursor.take(len)?; + if !signature[0].is_ascii_uppercase() { + return Err(StrictReason::RequiredExtension(signature)); + } + } + // Split indexes can contain empty names; inspect extensions before validating + // paths so their mandatory 'link' extension is reported as the strict reason. + for (i, entry) in entries.iter().enumerate() { + if entry.stat.mode & 0o170000 == 0o040000 { + return Err(StrictReason::SparseDirectory); + } + if entry.path.is_empty() + || entry + .path + .split(|&b| b == b'/') + .any(|p| p.is_empty() || p == b"." || p == b"..") + { + return Err(StrictReason::Malformed("invalid repository-relative path")); + } + if i > 0 { + let previous = &entries[i - 1]; + if (&previous.path, previous.stage) >= (&entry.path, entry.stage) { + return Err(StrictReason::Malformed("entries out of order")); + } + } + } + Ok(IndexSnapshot { + version, + object_format: format, + entries, + checksum: checksum.to_vec(), + modified, + }) +} + +/// Git metadata queries for one snapshot evaluation. Every spawned Git child, including a +/// failed or timed-out command, increments the counter. Output remains binary +/// because the NUL-delimited protocols must preserve arbitrary Git path bytes. +/// Callers must treat unsuccessful exit status as strict, not as empty evidence. +pub struct GitQueries { + root: PathBuf, + timeout: Duration, + spawned: AtomicU64, +} + +impl GitQueries { + pub fn new(root: PathBuf, timeout: Duration) -> Self { + Self { + root, + timeout, + spawned: AtomicU64::new(0), + } + } + + pub fn spawned(&self) -> u64 { + self.spawned.load(Ordering::Relaxed) + } + + pub fn head_tree(&self) -> io::Result { + self.run(&["rev-parse", "HEAD^{tree}"], Vec::new()) + } + + pub fn diff_trees(&self, old: &str, new: &str) -> io::Result { + // Only object IDs, never revision expressions or command-line options. + let valid = + |oid: &str| matches!(oid.len(), 40 | 64) && oid.bytes().all(|b| b.is_ascii_hexdigit()); + if !valid(old) || !valid(new) || old.len() != new.len() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "invalid tree OID", + )); + } + self.run( + &["diff-tree", "-r", "-z", "--no-renames", old, new, "--"], + Vec::new(), + ) + } + + pub fn check_attributes(&self, paths: &[&[u8]]) -> io::Result { + let mut input = Vec::new(); + for path in paths { + if path.contains(&0) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "NUL in Git path", + )); + } + input.extend_from_slice(path); + input.push(0); + } + self.run( + &[ + "check-attr", + "--stdin", + "-z", + "--", + "filter", + "text", + "eol", + "working-tree-encoding", + "ident", + ], + input, + ) + } + + /// Return config values with their origins for the caller to interpret and + /// retain. Callers should issue this query only once per snapshot evaluation. + pub fn config(&self) -> io::Result { + self.run(&["config", "--list", "--show-origin", "-z"], Vec::new()) + } + + fn run(&self, args: &[&str], input: Vec) -> io::Result { + let mut command = Command::new("git"); + command + .current_dir(&self.root) + .args(args) + .env("GIT_OPTIONAL_LOCKS", "0") + .env("LC_ALL", "C"); + #[cfg(test)] + crate::test_env::apply_hermetic_git_env(&mut command); + pump(&mut command, input, self.timeout, &self.spawned) + } +} + +enum PumpResult { + Input(io::Result<()>), + Stdout(io::Result>), + Stderr(io::Result>), +} + +fn pump( + command: &mut Command, + input: Vec, + timeout: Duration, + spawned: &AtomicU64, +) -> io::Result { + crate::format::isolate_in_process_group(command); + let deadline = Instant::now() + timeout; + let mut child = command + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + spawned.fetch_add(1, Ordering::Relaxed); + let mut stdin = child.stdin.take().expect("piped stdin"); + let mut stdout = child.stdout.take().expect("piped stdout"); + let mut stderr = child.stderr.take().expect("piped stderr"); + let (tx, rx) = mpsc::channel(); + let input_tx = tx.clone(); + thread::spawn(move || { + let result = stdin.write_all(&input); + drop(stdin); + let _ = input_tx.send(PumpResult::Input(result)); + }); + let stdout_tx = tx.clone(); + thread::spawn(move || { + let mut bytes = Vec::new(); + let result = stdout.read_to_end(&mut bytes).map(|_| bytes); + let _ = stdout_tx.send(PumpResult::Stdout(result)); + }); + thread::spawn(move || { + let mut bytes = Vec::new(); + let result = stderr.read_to_end(&mut bytes).map(|_| bytes); + let _ = tx.send(PumpResult::Stderr(result)); + }); + let mut status = None; + let mut input_done = false; + let mut output = None; + let mut error_output = None; + let result = (|| loop { + if Instant::now() >= deadline { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "Git metadata deadline exceeded", + )); + } + if status.is_none() { + status = child.try_wait()?; + } + if status.is_some() && input_done && output.is_some() && error_output.is_some() { + return Ok(Output { + status: status.unwrap(), + stdout: output.take().unwrap(), + stderr: error_output.take().unwrap(), + }); + } + // Completion includes EOF on both pipes and the stdin writer, not just + // process exit. A descendant retaining a pipe must not defeat the deadline. + match rx.recv_timeout( + deadline + .saturating_duration_since(Instant::now()) + .min(Duration::from_millis(5)), + ) { + Ok(PumpResult::Input(result)) => { + result?; + input_done = true; + } + Ok(PumpResult::Stdout(result)) => output = Some(result?), + Ok(PumpResult::Stderr(result)) => error_output = Some(result?), + Err(mpsc::RecvTimeoutError::Timeout) => {} + Err(mpsc::RecvTimeoutError::Disconnected) => { + if !input_done || output.is_none() || error_output.is_none() { + return Err(io::Error::other("Git pipe pump disconnected")); + } + thread::sleep(Duration::from_millis(1)); + } + } + })(); + if result.is_err() { + crate::format::kill_process_tree(&mut child); + let _ = child.wait(); + } + result +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + fn git(root: &Path, args: &[&str], input: &[u8]) -> Output { + let mut command = Command::new("git"); + command + .current_dir(root) + .args(args) + .env("GIT_OPTIONAL_LOCKS", "0"); + crate::test_env::apply_hermetic_git_env(&mut command); + let output = pump( + &mut command, + input.to_vec(), + Duration::from_secs(15), + &AtomicU64::new(0), + ) + .unwrap(); + assert!( + output.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + output + } + + fn fixture(format: ObjectFormat) -> TempDir { + let dir = tempfile::tempdir().unwrap(); + git( + dir.path(), + &[ + "init", + if format == ObjectFormat::Sha1 { + "--object-format=sha1" + } else { + "--object-format=sha256" + }, + ], + &[], + ); + git( + dir.path(), + &["config", "user.email", "index-reader@example.invalid"], + &[], + ); + git(dir.path(), &["config", "user.name", "Index Reader"], &[]); + git(dir.path(), &["config", "core.autocrlf", "false"], &[]); + git(dir.path(), &["config", "core.splitIndex", "false"], &[]); + for path in [ + "plain", + "skip", + "dir/sub/alpha", + "dir/sub/alphabet", + "with space", + ] { + let path = dir.path().join(path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, b"hello\n").unwrap(); + } + // A long first path followed by a short unrelated one exercises the + // multi-byte remove-count encoding in Git's real v4 prefix compressor. + fs::write(dir.path().join("a".repeat(140)), b"long\n").unwrap(); + git(dir.path(), &["add", "."], &[]); + git( + dir.path(), + &["-c", "commit.gpgsign=false", "commit", "-m", "fixture"], + &[], + ); + dir + } + + fn index(root: &Path) -> PathBuf { + root.join(".git/index") + } + + fn read(root: &Path, format: ObjectFormat) -> IndexRead { + read_index(&index(root), format, ReadOptions::default()) + } + + fn snapshot(root: &Path, format: ObjectFormat) -> IndexSnapshot { + match read(root, format) { + IndexRead::Snapshot(snapshot) => snapshot, + other => panic!("expected snapshot: {other:?}"), + } + } + + fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() + } + + fn assert_matches_git(root: &Path, snapshot: &IndexSnapshot) { + let mut actual = Vec::new(); + for entry in &snapshot.entries { + actual.extend_from_slice( + format!( + "{:06o} {} {}\t", + entry.stat.mode, + hex(&entry.oid), + entry.stage + ) + .as_bytes(), + ); + actual.extend_from_slice(&entry.path); + actual.push(0); + } + assert_eq!( + actual, + git(root, &["ls-files", "--stage", "-z"], &[]).stdout + ); + let plain = snapshot + .entries + .iter() + .find(|e| e.path == b"plain") + .unwrap(); + assert_eq!(plain.stat.size, 6); + assert_eq!(plain.stat.mode, 0o100644); + assert_eq!(snapshot.entries.len(), 6); + } + + #[test] + fn real_git_v2_golden() { + let dir = fixture(ObjectFormat::Sha1); + git( + dir.path(), + &["update-index", "--assume-unchanged", "plain"], + &[], + ); + git(dir.path(), &["update-index", "--index-version", "2"], &[]); + let snapshot = snapshot(dir.path(), ObjectFormat::Sha1); + assert_eq!(snapshot.version, 2); + assert_matches_git(dir.path(), &snapshot); + assert_eq!( + snapshot + .entries + .iter() + .find(|e| e.path == b"plain") + .unwrap() + .exceptions(), + vec![EntryException::AssumeValid] + ); + } + + #[test] + fn real_git_v3_golden_and_extended_flags() { + let dir = fixture(ObjectFormat::Sha1); + git( + dir.path(), + &["update-index", "--skip-worktree", "skip"], + &[], + ); + fs::write(dir.path().join("intent"), b"not staged\n").unwrap(); + git(dir.path(), &["add", "--intent-to-add", "intent"], &[]); + git(dir.path(), &["update-index", "--index-version", "3"], &[]); + let snapshot = snapshot(dir.path(), ObjectFormat::Sha1); + assert_eq!(snapshot.version, 3); + assert_eq!( + snapshot + .entries + .iter() + .find(|e| e.path == b"skip") + .unwrap() + .exceptions(), + vec![EntryException::SkipWorktree] + ); + assert_eq!( + snapshot + .entries + .iter() + .find(|e| e.path == b"intent") + .unwrap() + .exceptions(), + vec![EntryException::IntentToAdd] + ); + assert_eq!(snapshot.entries.len(), 7); + } + + #[test] + fn real_git_v4_golden_prefix_compression() { + let dir = fixture(ObjectFormat::Sha1); + git(dir.path(), &["update-index", "--index-version", "4"], &[]); + let snapshot = snapshot(dir.path(), ObjectFormat::Sha1); + assert_eq!(snapshot.version, 4); + assert_matches_git(dir.path(), &snapshot); + assert_eq!(snapshot.entries[0].path, vec![b'a'; 140]); + assert_eq!(snapshot.entries[1].path, b"dir/sub/alpha"); + assert_eq!(snapshot.entries[2].path, b"dir/sub/alphabet"); + } + + #[test] + fn real_git_sha256_golden_all_versions() { + let dir = fixture(ObjectFormat::Sha256); + for version in ["2", "3", "4"] { + if version == "3" { + git( + dir.path(), + &["update-index", "--skip-worktree", "skip"], + &[], + ); + } + git( + dir.path(), + &["update-index", "--index-version", version], + &[], + ); + let snapshot = snapshot(dir.path(), ObjectFormat::Sha256); + assert_eq!(snapshot.version.to_string(), version); + assert_eq!(snapshot.checksum.len(), 32); + assert!(snapshot.entries.iter().all(|e| e.oid.len() == 32)); + assert_matches_git(dir.path(), &snapshot); + } + assert_eq!( + read(dir.path(), ObjectFormat::Sha1), + IndexRead::Strict(StrictReason::ChecksumMismatch) + ); + } + + #[test] + fn real_git_unmerged_stages_are_exceptions() { + let dir = fixture(ObjectFormat::Sha1); + let oid = git(dir.path(), &["rev-parse", "HEAD:plain"], &[]).stdout; + let oid = String::from_utf8(oid).unwrap(); + let records: String = (1..=3) + .map(|stage| format!("100644 {} {stage}\tconflict\n", oid.trim())) + .collect(); + git( + dir.path(), + &["update-index", "--index-info"], + records.as_bytes(), + ); + let snapshot = snapshot(dir.path(), ObjectFormat::Sha1); + let stages: Vec<_> = snapshot + .entries + .iter() + .filter(|e| e.path == b"conflict") + .map(|e| e.exceptions()) + .collect(); + assert_eq!( + stages, + vec![ + vec![EntryException::Unmerged(1)], + vec![EntryException::Unmerged(2)], + vec![EntryException::Unmerged(3)] + ] + ); + } + + #[test] + fn real_git_split_index_is_strict() { + let dir = fixture(ObjectFormat::Sha1); + git(dir.path(), &["update-index", "--split-index"], &[]); + assert_eq!( + read(dir.path(), ObjectFormat::Sha1), + IndexRead::Strict(StrictReason::RequiredExtension(*b"link")) + ); + } + + #[test] + fn real_git_sparse_index_is_strict() { + let dir = fixture(ObjectFormat::Sha1); + git( + dir.path(), + &["sparse-checkout", "init", "--cone", "--sparse-index"], + &[], + ); + git( + dir.path(), + &["sparse-checkout", "set", "--sparse-index", "other"], + &[], + ); + assert_eq!( + read(dir.path(), ObjectFormat::Sha1), + IndexRead::Strict(StrictReason::RequiredExtension(*b"sdir")) + ); + } + + #[test] + fn held_index_lock_waits_then_strict_without_touching_it() { + let dir = fixture(ObjectFormat::Sha1); + let lock = dir.path().join(".git/index.lock"); + fs::write(&lock, b"writer owns this").unwrap(); + let options = ReadOptions { + lock_wait: Duration::from_millis(20), + ..ReadOptions::default() + }; + let start = Instant::now(); + assert_eq!( + read_index(&index(dir.path()), ObjectFormat::Sha1, options), + IndexRead::Strict(StrictReason::IndexLocked) + ); + assert!(start.elapsed() >= options.lock_wait); + assert_eq!(fs::read(lock).unwrap(), b"writer owns this"); + } + + #[test] + fn released_index_lock_allows_snapshot() { + let dir = fixture(ObjectFormat::Sha1); + let lock = dir.path().join(".git/index.lock"); + fs::write(&lock, b"writer").unwrap(); + let writer = thread::spawn(move || { + thread::sleep(Duration::from_millis(20)); + fs::remove_file(lock).unwrap(); + }); + let result = read_index( + &index(dir.path()), + ObjectFormat::Sha1, + ReadOptions { + lock_wait: Duration::from_secs(5), + ..ReadOptions::default() + }, + ); + writer.join().unwrap(); + assert!(matches!(result, IndexRead::Snapshot(_))); + } + + #[test] + fn real_index_torn_read_retries_then_recovers() { + let dir = fixture(ObjectFormat::Sha1); + let path = index(dir.path()); + let original = fs::read(&path).unwrap(); + let mut attempts = 0; + let result = + read_index_with_hook(&path, ObjectFormat::Sha1, ReadOptions::default(), || { + attempts += 1; + match attempts { + 1 => fs::write(&path, &original[..original.len() / 2]).unwrap(), + 2 => fs::write(&path, &original).unwrap(), + _ => {} + } + }); + assert_eq!(attempts, 3); + assert!(matches!(result, IndexRead::Snapshot(_))); + } + + #[test] + fn real_index_persistently_torn_read_is_strict() { + let dir = fixture(ObjectFormat::Sha1); + let path = index(dir.path()); + let original = fs::read(&path).unwrap(); + let mut attempts = 0; + let result = + read_index_with_hook(&path, ObjectFormat::Sha1, ReadOptions::default(), || { + attempts += 1; + fs::write( + &path, + if attempts % 2 == 0 { + &original[..] + } else { + &original[..original.len() / 2] + }, + ) + .unwrap(); + }); + assert_eq!(attempts, 3); + assert_eq!(result, IndexRead::Strict(StrictReason::Unstable)); + } + + #[test] + fn checksum_mismatch_retries_then_strict() { + let dir = fixture(ObjectFormat::Sha1); + let path = index(dir.path()); + let mut bytes = fs::read(&path).unwrap(); + bytes[20] ^= 1; + fs::write(&path, bytes).unwrap(); + let mut attempts = 0; + let result = + read_index_with_hook(&path, ObjectFormat::Sha1, ReadOptions::default(), || { + attempts += 1 + }); + assert_eq!(attempts, 3); + assert_eq!(result, IndexRead::Strict(StrictReason::ChecksumMismatch)); + } + + #[test] + fn replacing_index_with_same_size_and_mtime_is_not_coherent() { + let dir = fixture(ObjectFormat::Sha1); + let path = index(dir.path()); + let original = fs::read(&path).unwrap(); + let mtime = filetime::FileTime::from_last_modification_time(&fs::metadata(&path).unwrap()); + let result = read_index_with_hook( + &path, + ObjectFormat::Sha1, + ReadOptions { + attempts: 1, + ..ReadOptions::default() + }, + || { + let replacement = path.with_extension("replacement"); + fs::write(&replacement, &original).unwrap(); + filetime::set_file_mtime(&replacement, mtime).unwrap(); + // Windows cannot rename over an existing file. Moving the old name + // first keeps the original open handle alive on all three platforms. + fs::rename(&path, path.with_extension("old")).unwrap(); + fs::rename(replacement, &path).unwrap(); + }, + ); + assert_eq!(result, IndexRead::Strict(StrictReason::Unstable)); + } + + #[test] + fn same_size_in_place_mtime_change_is_not_coherent() { + let dir = fixture(ObjectFormat::Sha1); + let path = index(dir.path()); + let result = read_index_with_hook( + &path, + ObjectFormat::Sha1, + ReadOptions { + attempts: 1, + ..ReadOptions::default() + }, + || { + filetime::set_file_mtime(&path, filetime::FileTime::from_unix_time(1, 123)) + .unwrap(); + }, + ); + assert_eq!(result, IndexRead::Strict(StrictReason::Unstable)); + } + + fn append_extension(bytes: &[u8], signature: &[u8; 4], payload: &[u8]) -> Vec { + let mut body = bytes[..bytes.len() - 20].to_vec(); + body.extend_from_slice(signature); + body.extend_from_slice(&(payload.len() as u32).to_be_bytes()); + body.extend_from_slice(payload); + let digest = ObjectFormat::Sha1.digest(&body); + body.extend_from_slice(&digest); + body + } + + #[test] + fn optional_extensions_are_ignored_required_extensions_are_strict() { + let dir = fixture(ObjectFormat::Sha1); + let path = index(dir.path()); + let original = fs::read(&path).unwrap(); + let expected = snapshot(dir.path(), ObjectFormat::Sha1).entries; + for signature in [b"ABCD", b"Z123"] { + fs::write( + &path, + append_extension(&original, signature, b"optional payload"), + ) + .unwrap(); + assert_eq!(snapshot(dir.path(), ObjectFormat::Sha1).entries, expected); + } + for signature in [b"abcd", b"0123", b"link", b"sdir"] { + fs::write( + &path, + append_extension(&original, signature, b"required payload"), + ) + .unwrap(); + assert_eq!( + read(dir.path(), ObjectFormat::Sha1), + IndexRead::Strict(StrictReason::RequiredExtension(*signature)) + ); + } + } + + fn seal(mut body: Vec) -> Vec { + body.extend_from_slice(&ObjectFormat::Sha1.digest(&body)); + body + } + + #[test] + fn malformed_entries_extensions_and_versions_fail_closed() { + let dir = fixture(ObjectFormat::Sha1); + git(dir.path(), &["update-index", "--index-version", "4"], &[]); + let path = index(dir.path()); + let original = fs::read(&path).unwrap(); + let body = &original[..original.len() - 20]; + let mut invalid_prefix = body.to_vec(); + invalid_prefix[12 + 40 + 20 + 2] = 1; + let mut invalid_version = body.to_vec(); + invalid_version[4..8].copy_from_slice(&5u32.to_be_bytes()); + let mut invalid_count = body.to_vec(); + invalid_count[8..12].copy_from_slice(&u32::MAX.to_be_bytes()); + let mut invalid_nanoseconds = body.to_vec(); + invalid_nanoseconds[16..20].copy_from_slice(&1_000_000_000u32.to_be_bytes()); + let mut truncated_extension = body.to_vec(); + truncated_extension.extend_from_slice(b"ABCD\0\0\0\x10short"); + for malformed in [ + invalid_prefix, + invalid_version, + invalid_count, + invalid_nanoseconds, + truncated_extension, + body[..16].to_vec(), + ] { + fs::write(&path, seal(malformed)).unwrap(); + assert!(matches!( + read(dir.path(), ObjectFormat::Sha1), + IndexRead::Strict(_) + )); + } + } + + #[test] + fn stat_comparison_uses_low_32_bits_and_windows_unrecorded_identity() { + let stat = IndexStat { + ctime: IndexTime { + seconds: u32::MAX, + nanoseconds: 321, + }, + mtime: IndexTime { + seconds: 7, + nanoseconds: 654, + }, + dev: 3, + ino: 4, + mode: 0o100644, + uid: 10, + gid: 20, + size: 5, + }; + let mut observed = WorktreeStat { + ctime_seconds: -1, + ctime_nanoseconds: 321, + mtime_seconds: (1i64 << 32) + 7, + mtime_nanoseconds: 654, + dev: (1u64 << 32) + 3, + ino: (1u64 << 32) + 4, + mode: 0o100644, + size: (1u64 << 32) + 5, + }; + assert!(stat.matches_stat(&observed)); + observed.size += 1; + assert!(!stat.matches_stat(&observed)); + observed.size -= 1; + observed.dev = 0; + observed.ino = 0; + assert_eq!(stat.matches_stat(&observed), cfg!(windows)); + observed.mtime_nanoseconds += 1; + assert!(!stat.matches_stat(&observed)); + } + + #[cfg(unix)] + #[test] + fn real_git_preserves_non_utf8_paths() { + use std::ffi::OsStr; + use std::os::unix::ffi::OsStrExt; + let dir = fixture(ObjectFormat::Sha1); + let name = b"non-utf8-\xff"; + fs::write(dir.path().join(OsStr::from_bytes(name)), b"raw\n").unwrap(); + git(dir.path(), &["add", "."], &[]); + git(dir.path(), &["update-index", "--index-version", "4"], &[]); + assert!(snapshot(dir.path(), ObjectFormat::Sha1) + .entries + .iter() + .any(|e| e.path == name)); + } + + #[test] + fn counted_read_only_git_queries_and_large_bidirectional_pump() { + let dir = fixture(ObjectFormat::Sha1); + let path = index(dir.path()); + let original = fs::read(&path).unwrap(); + fs::write( + dir.path().join(".gitattributes"), + b"plain text eol=lf filter=demo\n", + ) + .unwrap(); + let queries = GitQueries::new(dir.path().to_path_buf(), Duration::from_secs(15)); + let first = queries.head_tree().unwrap(); + assert!(first.status.success()); + let first = String::from_utf8(first.stdout).unwrap(); + git(dir.path(), &["rm", "with space"], &[]); + git( + dir.path(), + &["-c", "commit.gpgsign=false", "commit", "-m", "second"], + &[], + ); + let second = queries.head_tree().unwrap(); + let second = String::from_utf8(second.stdout).unwrap(); + let after_commit = fs::read(&path).unwrap(); + let diff = queries.diff_trees(first.trim(), second.trim()).unwrap(); + assert!(diff.status.success()); + assert!(diff.stdout.ends_with(b"\0with space\0")); + assert!(queries.diff_trees("--bad", second.trim()).is_err()); + // Both the request and reply exceed pipe capacity, so serial write/read + // would deadlock. Repeating paths is valid check-attr input on every OS. + let paths = vec![b"plain".as_slice(); 30_000]; + let attrs = queries.check_attributes(&paths).unwrap(); + assert!( + attrs.status.success(), + "{}", + String::from_utf8_lossy(&attrs.stderr) + ); + assert_eq!(attrs.stdout.len(), b"plain\0filter\0demo\0plain\0text\0set\0plain\0eol\0lf\0plain\0working-tree-encoding\0unspecified\0plain\0ident\0unspecified\0".len() * paths.len()); + assert!(attrs + .stdout + .starts_with(b"plain\0filter\0demo\0plain\0text\0set\0")); + let config = queries.config().unwrap(); + assert!(config.status.success()); + assert!(config + .stdout + .windows(b"core.autocrlf\nfalse\0".len()) + .any(|w| w == b"core.autocrlf\nfalse\0")); + assert_eq!(queries.spawned(), 5); + assert_eq!(fs::read(&path).unwrap(), after_commit); + assert!(!dir.path().join(".git/index.lock").exists()); + assert_ne!(original, after_commit); + } + + #[test] + fn git_pump_deadline_and_spawn_accounting() { + // Re-exec this libtest binary with a private ignored sleeper, avoiding + // shell syntax and utilities unavailable on native Windows runners. + let mut command = Command::new(std::env::current_exe().unwrap()); + command.args([ + "--ignored", + "--exact", + "git_index_reader::tests::pump_sleep_child", + "--nocapture", + ]); + let spawned = AtomicU64::new(0); + let start = Instant::now(); + let error = pump( + &mut command, + vec![b'x'; 1024 * 1024], + Duration::from_millis(100), + &spawned, + ) + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::TimedOut); + assert_eq!(spawned.load(Ordering::Relaxed), 1); + assert!(start.elapsed() < Duration::from_secs(10)); + let mut missing = Command::new("aft-nonexistent-index-reader-test-command"); + assert!(pump(&mut missing, Vec::new(), Duration::from_secs(1), &spawned).is_err()); + assert_eq!(spawned.load(Ordering::Relaxed), 1); + } + + #[test] + #[ignore = "subprocess helper; invoked explicitly by git_pump_deadline_and_spawn_accounting"] + fn pump_sleep_child() { + thread::sleep(Duration::from_secs(60)); + } +} diff --git a/crates/aft/src/lib.rs b/crates/aft/src/lib.rs index c36fb03cbd..06f87df36d 100644 --- a/crates/aft/src/lib.rs +++ b/crates/aft/src/lib.rs @@ -113,6 +113,7 @@ pub mod gh_shim_ticket; pub(crate) mod gh_shim_worker_broker; #[cfg(unix)] pub(crate) mod gh_shim_worker_endpoints; +pub mod git_index_reader; pub(crate) mod git_probe; pub mod github_read; pub mod grep_executor; From c72523c0b66eb5428203b93c550804eb07fa89a8 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:21:58 +0100 Subject: [PATCH 02/31] mason: correct check-attr stdin argument syntax The large bidirectional-pump golden test exposed BrokenPipe because check-attr --stdin treats arguments after -- as forbidden pathnames, not attribute names. Keep the fixed conversion attribute names before any pathname separator. The initial module run passed the other 19 tests; the corrected query test passes on Linux. Windows tests compile with warnings denied. Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 32d148904afb9261a7bd1fbe6b4d9f18706b7c84) --- crates/aft/src/git_index_reader.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/aft/src/git_index_reader.rs b/crates/aft/src/git_index_reader.rs index 4cfc79c5bf..2356bea40e 100644 --- a/crates/aft/src/git_index_reader.rs +++ b/crates/aft/src/git_index_reader.rs @@ -534,7 +534,6 @@ impl GitQueries { "check-attr", "--stdin", "-z", - "--", "filter", "text", "eol", From 4b6ffcd514f939c85383b17f23a677356845e749 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:13:10 +0100 Subject: [PATCH 03/31] mason: audit watcher frontier coverage before cookie wiring Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 9f8edb63efe442423662ac9babc35bca3b343558) --- docs/design/views-4v-frontier-audit.md | 162 +++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 docs/design/views-4v-frontier-audit.md diff --git a/docs/design/views-4v-frontier-audit.md b/docs/design/views-4v-frontier-audit.md new file mode 100644 index 0000000000..3c171f6ff8 --- /dev/null +++ b/docs/design/views-4v-frontier-audit.md @@ -0,0 +1,162 @@ +# Watcher delivery-frontier audit + +## Decision and scope + +This audit concerns **verified bind**, not whether ordinary watching works. A +cookie acknowledgment must mean that every relevant write completed before +cookie creation has reached the views driver as reflected data or pending work. +An empty queue, a quiet interval, or a native flush returning is not that proof. +The cookie lives directly in the checkout root and is created exclusively as +`.aft-cookie--`; it must never become corpus membership. + +The audit starts from commit `a8e301bea1eb7d2cba7ecbaccfeecc4a461c6138`. +No backend is enabled for verified bind by this slice. In particular, FSEvents +is **off pending proof**, not demonstrated incorrect. Cookie/overflow +infrastructure can be built without relaxing that restriction. E0 is the start +of a fully registered watcher subscription; F is the later cookie delivery +frontier. A delivery sequence orders invalidations at the driver, and the +durable cut is the exact prefix of those changes persisted in a publication. +Connecting those points to ready publication belongs to later verified-bind work. +Ordinary strict reconciliation remains available with its existing, weaker +per-path stability and subsequent-event-invalidation contract. + +## Shared filters, batching and the driver queue + +`watcher_filter::WatcherFilterThread` receives normalized native events over an +MPSC channel. Ordinary paths are canonicalized, deduplicated, and batched before +corpus filtering. `send_dispatch` and `retry_backlog` feed a bounded FIFO driver +channel; held batches may merge, and overflow replaces paths with a rescan. +Consequently, observing a cookie at raw intake alone cannot acknowledge a +views frontier: earlier paths may still be in either batch, backlog, channel, +or `runtime_drain`'s sliced `pending_paths`. + +The additive cookie plumbing must bypass ignore/infra filters, unlink a root +cookie immediately at intake, flush earlier paths, and enqueue a distinct +control rather than a membership path. A mixed event must retain its non-cookie +paths. Only a create event may acknowledge a ticket; unlink/metadata events must +not. The shared `watcher_backend::flush::is_marker_path` admission predicate must +also reject cookies during a filesystem walk, not just during event delivery. +The command-report `.aft-flush` acknowledgment remains a separate protocol. + +The driver-side cookie arm must record all preceding queued paths as pending +before acknowledging the exact registered cookie. A rescan must enter the +driver as `WatcherState::Overflowed` and invalidate an in-flight installation; +a subsequent ordinary path must not reset that state to Healthy. A cookie held +in a backlog whose paths are discarded must be discarded too (timeout, not a +false successful frontier). This provides an ordered **delivery acknowledgment**, +not ready publication: the bind owner must still establish E0, reject any gap, +replay through the acknowledged cut, and revalidate before publication. + +The planned additive implementation includes regression tests named +`watcher_cookie_delayed_delivery_past_read_inotify`, +`watcher_cookie_delayed_delivery_past_read_fsevents`, and +`watcher_cookie_delayed_delivery_past_read_windows`, using a backend-delivery shim +which holds a write notification and its following cookie past a source read. +They must show no acknowledgment before release and a pending path before the +acknowledgment afterwards. They test the application queue, **not** a kernel's +completed-write coverage. Native macOS and Windows execution is delegated to CI. + +## Linux: inotify + +`inotify::ProjectWatcher::create_with_plan` creates one notify inotify instance, +collects directories, and synchronously installs non-recursive watches before +spawning its backend thread. Every per-directory descriptor belongs to that +instance's ordered queue; extra watches also use that instance. Successful +return is later than registration of the collected set, but is not yet a safe +E0: `collect_watch_directories` walks **before** the root watch is installed. +A directory created after its parent was enumerated but before the parent was +watched can be absent from the collected set without a queued create event. +The later bind must close that registration gap before recording E0, and must +not mistake a returned watcher handle for a complete subscription proof. + +For newly delivered directory creates/renames, `watch_new_tree` does install +each directory's watch before `exclusion_read_dir`. Already-populated files are +recovered by listing. `PendingNewDirectoryFiles` delays that synthetic file +batch; it must be flushed before a root cookie even when command-report tracking +is inactive. Otherwise the root cookie could overtake a recovered child write. +An independent metadata walk cannot assume the backend has already processed +every directory create: watch-before-list coordination with that walk remains +an enablement obligation of the bind implementation. + +Upstream notify's inotify queue-overflow flag is forwarded as `need_rescan`. +An incomplete/over-budget new-tree listing also emits a rescan. The filter maps +these to `RescanRequired`, and the context adapter must mark the installed +driver Overflowed before any subsequent cookie acknowledgment. Watch-limit and +other fatal backend errors terminate delivery; a disconnected/error stream +cannot certify a frontier. Retained watches are re-added without unwatching in +`reconcile_watch_directories`, but policy/boundary changes still require bind +revalidation. A single queue is necessary, not proof that every directory was +covered when a write occurred. + +## macOS: FSEvents + +`fsevents::FsEventsStream::start_paths` starts one recursive project stream with +FileEvents and waits for `FSEventStreamStart` on its run loop. There is no +per-directory watch-registration walk: a recursive stream covers newly created +descendants without adding directory handles. Excluded/cross-device descendants +are not thereby certified. A later bind can record E0 only after successful +startup and coverage/root/policy validation, not before the startup handshake. + +The callback translates in native array order, hands off one vector, and advances +its replay cursor only after handoff. The backend forwards each vector in order +to the filter. Stream replacement replays from the old delivered event ID rather +than SinceNow; a missing/wrapped cursor requires a rescan. MustScanSubDirs, +UserDropped and KernelDropped all emit `Flag::Rescan` in `translate_event`; +those controls must survive filters and mark the driver Overflowed. + +FSEvents is **off: completed-write-before-cookie coverage through the views +queue is unproved**. Auxiliary streams are independently ordered +(`fsevents.rs:163-164` at the audited commit). FlushSync covers only events +already in the pipeline (`fsevents.rs:214-222`), not all completed filesystem +writes. Preserving callback/event-ID order does not by itself close that gap. +No root cookie certifies an independently ordered auxiliary stream. No +quiet-window substitute is accepted. + +The later native test required to turn it on is +`fsevents_completed_writes_before_cookie_reach_views_frontier`: on a writable +same-device local root, establish E0, hold delivery beyond the stat/content read, +complete writes in multiple existing and newly created descendants, create the +cookie, and prove every pre-cookie write reflected or pending through the exact +views delivery cut before ready publication. Exercise coalesced callbacks, +stream handoff, and drops (drops must refuse verified bind); separately fence +auxiliary streams or exclude their state from the frontier claim. The shim test +above is necessary application-ordering evidence, not this native proof. + +## Windows: ReadDirectoryChangesW + +`windows::ProjectWatcher::start` blocks on `start_rx` until `open_watches` has +opened and submitted every initial read. `project_watch_targets` includes one +recursive root handle and separate non-recursive extra-path/parent handles. +Each `DirectoryWatch` has its own ordered notification buffer and pending read. +The recursive root handle already covers newly created directories; there is no +per-child registration/listing seam within that handle. E0 must follow all read +submissions and root/target validation. + +`run_completion_loop` rearms a read before translating its completed buffer, and +forwards records in buffer order. A completion port merges independent handles; +it does not give their writes a shared filesystem ordering. A root cookie +certifies neither an earlier undelivered extra-handle completion nor handle +replacement. Single-recursive-handle checkout-only coverage is a possible later +restriction, not a proof supplied by this slice. + +ERROR_NOTIFY_ENUM_DIR and successful zero-byte completions produce the typed +buffer-overflow rescan. These must mark the driver Overflowed; other fatal +completion errors end the stream and invalidate any bind. DirectoryWatcher +replacement drops its old watcher before starting the next; no verified bind +may span that registration gap. Native completed-write coverage (including +last-write/close semantics) and multi-handle fencing remain unproved. + +## Verified-bind on/off table + +These are enablement decisions, not changes to ordinary watcher selection. + +| Backend / root | Verified bind | Reason / remaining obligation | +| --- | --- | --- | +| Linux inotify, writable local same-device root | **Off pending coverage proof** | Initial collect-before-watch gap; coordinate new-directory registration before subscription E0 and during the bind walk through cookie frontier F. One instance plus ordered cookies alone is insufficient. | +| macOS FSEvents, writable local same-device root | **Off pending proof** | Completed-write-before-cookie coverage through views queue unproved; independent auxiliary streams; FlushSync only covers events already in pipeline. Native turn-on test named above belongs to later verified-bind work. | +| Windows native recursive root plus auxiliary handles | **Off pending proof** | Ordered buffers are per handle, not a global frontier; native completed-write coverage and handle lifecycle must be proved. | +| notify fallback; polling; any other unproved backend | **Off** | No proved ordered completed-write frontier. | +| Network filesystems (including NFS/SMB); FAT | **Off** | Backend/local timestamp and delivery assumptions do not apply. | +| Read-only roots | **Off** | Cannot create an exclusive cookie. | +| Cross-mount or cross-device subtrees | **Off for the subtree** | A root stream/cookie does not prove coverage on another mount/device. | +| Cookie timeout, watcher loss, root/policy change, unregistered directory | **Off for that bind** | Strict reconcile; never treat silence as continuity. | From 468639c0fde301a2b751f817935fd0d5a521d0b9 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:32:12 +0100 Subject: [PATCH 04/31] mason: deliver ordered watcher cookies and views overflow state Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit ffd27611beaf10a53c225095ad4a29a6673d19ab) --- crates/aft/src/context.rs | 30 ++ crates/aft/src/runtime_drain.rs | 16 + crates/aft/src/views/first_load.rs | 46 +++ crates/aft/src/watcher_backend/cookie.rs | 151 +++++++++ .../aft/src/watcher_backend/cookie_tests.rs | 299 ++++++++++++++++++ crates/aft/src/watcher_backend/flush.rs | 8 +- crates/aft/src/watcher_backend/fsevents.rs | 1 + crates/aft/src/watcher_backend/inotify.rs | 52 ++- crates/aft/src/watcher_backend/mod.rs | 3 + crates/aft/src/watcher_filter.rs | 170 +++++++++- docs/design/views-4v-frontier-audit.md | 7 +- 11 files changed, 770 insertions(+), 13 deletions(-) create mode 100644 crates/aft/src/watcher_backend/cookie.rs create mode 100644 crates/aft/src/watcher_backend/cookie_tests.rs diff --git a/crates/aft/src/context.rs b/crates/aft/src/context.rs index ac0b5e22ef..715f945994 100644 --- a/crates/aft/src/context.rs +++ b/crates/aft/src/context.rs @@ -1720,6 +1720,9 @@ pub(crate) struct WatcherCounters { pub(crate) bash_flush: Arc, pub(crate) bash_changes: std::sync::OnceLock>, overflow_ignore_evidence: AtomicU8, + /// Legacy rescan work coalesces repeated drops, but a newly installed views + /// snapshot must still learn that delivery was lost during that work. + checkout_watcher_gap_pending: AtomicBool, fsevents_stream_creations_total: AtomicU64, raw_events_total: AtomicU64, raw_events_since_last_rescan: AtomicU64, @@ -1891,6 +1894,8 @@ impl WatcherCounters { .is_ok() { self.overflows_during_rescan.fetch_add(1, Ordering::Relaxed); + self.checkout_watcher_gap_pending + .store(true, Ordering::Release); return true; } } @@ -10515,6 +10520,31 @@ impl AppContext { self.checkout_semantic.wake(); } + pub(crate) fn record_checkout_watcher_overflow(&self) { + self.reset_view_publication_retry(); + if let Some(driver) = self.checkout_driver.get() { + driver.record_watcher_overflow(); + } + self.checkout_semantic.wake(); + } + + pub(crate) fn record_checkout_watcher_cookie(&self, path: &Path) { + self.record_checkout_pending_watcher_gap(); + if let Some(driver) = self.checkout_driver.get() { + driver.record_watcher_cookie(path); + } + } + + pub(crate) fn record_checkout_pending_watcher_gap(&self) { + if self + .watcher_counters() + .checkout_watcher_gap_pending + .swap(false, Ordering::AcqRel) + { + self.record_checkout_watcher_overflow(); + } + } + pub(crate) fn view_publication_retry( &self, ) -> &parking_lot::Mutex { diff --git a/crates/aft/src/runtime_drain.rs b/crates/aft/src/runtime_drain.rs index 3431d1b973..8dbac6988e 100644 --- a/crates/aft/src/runtime_drain.rs +++ b/crates/aft/src/runtime_drain.rs @@ -3695,6 +3695,7 @@ pub fn drain_watcher_events_bounded(ctx: &AppContext, max_paths: usize) -> Drain { return outcome; } + ctx.record_checkout_pending_watcher_gap(); let mut state = match ctx.watcher_drain_slice().lock().take() { Some(state) if state.configure_generation == configure_generation => state, // Lifecycle-only generation change (transient unbind + equivalent @@ -3754,6 +3755,7 @@ pub fn drain_watcher_events_bounded(ctx: &AppContext, max_paths: usize) -> Drain } } Ok(WatcherDispatchEvent::RescanRequired(reason)) => { + ctx.record_checkout_watcher_overflow(); state.view_publication_reason.events |= crate::views::publication_reason::Reason::RESCAN; state.view_publication_full = true; @@ -3776,6 +3778,20 @@ pub fn drain_watcher_events_bounded(ctx: &AppContext, max_paths: usize) -> Drain state.semantic_refresh_paths.clear(); state.scheduler_changed_path_count = 0; } + Ok(WatcherDispatchEvent::CookieObserved(path)) => { + dispatch_events_received += 1; + // Earlier batches may still await sliced Tier-2 work. Record + // their intents now so the cookie cannot overtake them. + for pending in &state.pending_paths { + ctx.record_checkout_watcher_change(pending); + } + if let WatcherDrainPhase::Apply { paths, .. } = &state.phase { + for pending in paths { + ctx.record_checkout_watcher_change(pending); + } + } + ctx.record_checkout_watcher_cookie(&path); + } Ok(WatcherDispatchEvent::IgnoreRulesChanged { paths }) => { state.view_publication_reason.events |= crate::views::publication_reason::Reason::IGNORE; diff --git a/crates/aft/src/views/first_load.rs b/crates/aft/src/views/first_load.rs index fa9743e8a3..433d2ba3c3 100644 --- a/crates/aft/src/views/first_load.rs +++ b/crates/aft/src/views/first_load.rs @@ -5,6 +5,7 @@ use std::collections::BTreeMap; use std::sync::Arc; use crate::blob_store::v2::FamilyPlane; +pub use crate::watcher_backend::cookie::{CookieObservation, CookieTicket}; use super::contracts::{LoaderStage, PlaneError, ViewAccess}; use super::manifest_v2::Producers; @@ -1483,6 +1484,7 @@ pub struct CheckoutDriver { // without holding query locks during I/O or plane-reader retirement. publication_transition: std::sync::Mutex<()>, retry: std::sync::atomic::AtomicBool, + watcher_cookies: crate::watcher_backend::cookie::CookieBarrier, #[cfg(test)] work: std::sync::Mutex, } @@ -1553,6 +1555,7 @@ impl CheckoutDriver { adapters: Vec::new(), publish_fence: std::sync::RwLock::new(None), observed: Default::default(), + watcher_cookies: Default::default(), installed: std::sync::Mutex::new(Arc::new(InstalledBundle { seq: 0, composed, @@ -1615,6 +1618,8 @@ impl CheckoutDriver { delta.record_intent(path); // Membership is unchanged, so are the ignore rules it was walked under. delta.record_root_ignore_files(installed.snapshot.root_ignore_files().cloned()); + // A known path cannot repair an earlier stream-wide delivery gap. + delta.set_watcher(installed.snapshot.watcher()); installed.snapshot = delta .snapshot() .with_readers(installed.readers.clone()) @@ -1627,6 +1632,47 @@ impl CheckoutDriver { }); } + /// Invalidates stale builds and indexed membership until strict reconciliation. + pub fn record_watcher_overflow(&self) { + let mut installed = self + .installed + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + installed.revision += 1; + let mut delta = + super::snapshot::LiveDelta::new(Arc::clone(installed.snapshot.generation())); + for (path, entry) in installed.snapshot.live_entries() { + delta.apply(path.clone(), entry.clone()); + } + for path in installed.snapshot.pending_intent() { + delta.record_intent(path.clone()); + } + delta.record_root_ignore_files(installed.snapshot.root_ignore_files().cloned()); + delta.set_watcher(super::snapshot::WatcherState::Overflowed); + installed.snapshot = delta.snapshot(); + } + + /// Creates an exclusive root cookie. This does not enable verified bind; + /// the caller must establish subscription coverage and reject delivery gaps. + pub fn begin_watcher_cookie(&self) -> std::io::Result { + self.watcher_cookies.begin(self.owner.root()) + } + + /// Called only after preceding dispatch paths have entered the driver. + pub(crate) fn record_watcher_cookie(&self, path: &std::path::Path) { + let installed = self + .installed + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + self.watcher_cookies.observe( + path, + CookieObservation { + revision: installed.revision, + watcher: installed.snapshot.watcher(), + }, + ); + } + fn check_owner(&self, access: &ViewAccess) -> Result<(), PlaneError> { match access { ViewAccess::Owner(owner) diff --git a/crates/aft/src/watcher_backend/cookie.rs b/crates/aft/src/watcher_backend/cookie.rs new file mode 100644 index 0000000000..3bc0e3ddfc --- /dev/null +++ b/crates/aft/src/watcher_backend/cookie.rs @@ -0,0 +1,151 @@ +//! In-band views delivery acknowledgments, distinct from command-report flushes. +use crate::views::snapshot::WatcherState; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::sync::{atomic::AtomicU64, atomic::Ordering, mpsc, Arc, Mutex}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +const PREFIX: &str = ".aft-cookie-"; +static NEXT: AtomicU64 = AtomicU64::new(0); +type Pending = Arc>>>; + +/// A driver delivery cut, not a verified-bind or durable-publication certificate. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CookieObservation { + pub revision: u64, + pub watcher: WatcherState, +} + +/// An exclusive root cookie. Dropping a timed-out ticket cleans up its file. +pub struct CookieTicket { + path: PathBuf, + pending: Pending, + rx: mpsc::Receiver, +} + +impl CookieTicket { + pub fn path(&self) -> &Path { + &self.path + } + + /// On timeout or an untrusted watcher, use strict reconciliation rather + /// than relying on the cookie acknowledgment. + pub fn wait_timeout( + &self, + timeout: Duration, + ) -> Result { + self.rx.recv_timeout(timeout) + } +} + +impl Drop for CookieTicket { + fn drop(&mut self) { + self.pending + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&self.path); + let _ = std::fs::remove_file(&self.path); + } +} + +#[derive(Default)] +pub(crate) struct CookieBarrier { + pending: Pending, +} + +impl CookieBarrier { + pub(crate) fn begin(&self, root: &Path) -> std::io::Result { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + let path = root.join(format!( + "{PREFIX}{}-{nonce:x}{:016x}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let (tx, rx) = mpsc::channel(); + // Hold the registry lock across creation: even immediate delivery must + // find the waiter, and create_new cannot overwrite a checkout file. + let mut pending = self + .pending + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&path)?; + pending.insert(path.clone(), tx); + Ok(CookieTicket { + path, + pending: Arc::clone(&self.pending), + rx, + }) + } + + pub(crate) fn observe(&self, path: &Path, observation: CookieObservation) { + if let Some(tx) = self + .pending + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(path) + { + let _ = tx.send(observation); + } + } +} + +pub(crate) fn is_cookie_path(path: &Path) -> bool { + path.file_name() + .and_then(|name| name.to_str()) + .and_then(|name| name.strip_prefix(PREFIX)) + .and_then(|suffix| suffix.split_once('-')) + .is_some_and(|(pid, nonce)| { + !pid.is_empty() + && pid.bytes().all(|byte| byte.is_ascii_digit()) + && !nonce.is_empty() + && nonce + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-') + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn watcher_cookie_tickets_match_exact_names_and_cleanup_on_timeout() { + let root = tempfile::tempdir().unwrap(); + let barrier = CookieBarrier::default(); + let ticket = barrier.begin(root.path()).unwrap(); + let path = ticket.path().to_path_buf(); + assert!(is_cookie_path(&path)); + let observation = CookieObservation { + revision: 7, + watcher: WatcherState::Healthy, + }; + barrier.observe(&root.path().join(".aft-cookie-1-unregistered"), observation); + assert_eq!( + ticket.wait_timeout(Duration::ZERO), + Err(mpsc::RecvTimeoutError::Timeout) + ); + barrier.observe(&path, observation); + assert_eq!(ticket.wait_timeout(Duration::ZERO), Ok(observation)); + drop(ticket); + assert!(!path.exists()); + let ticket = barrier.begin(root.path()).unwrap(); + let path = ticket.path().to_path_buf(); + drop(ticket); + assert!(!path.exists()); + assert!(barrier.pending.lock().unwrap().is_empty()); + for name in [ + ".aft-cookie", + ".aft-cookie--nonce", + ".aft-cookie-pid-nonce", + ".aft-cookie-1-", + ] { + assert!(!is_cookie_path(Path::new(name))); + } + } +} diff --git a/crates/aft/src/watcher_backend/cookie_tests.rs b/crates/aft/src/watcher_backend/cookie_tests.rs new file mode 100644 index 0000000000..05c1a8b882 --- /dev/null +++ b/crates/aft/src/watcher_backend/cookie_tests.rs @@ -0,0 +1,299 @@ +//! Backend delivery shim: the native producer is held while the real filter and +//! driver drain remain live. This tests application ordering, not kernel coverage. +use crate::config::Config; +use crate::context::{default_language_provider_factory, AppContext}; +use crate::runtime_drain::{drain_watcher_events_bounded, drain_watcher_to_completion_for_test}; +use crate::views::contracts::ViewAccess; +use crate::views::first_load::{CheckoutDriver, ConfiguredMembershipWalker, FirstLoadDriver}; +use crate::views::manifest_v2::Producers; +use crate::views::registry::FamilyRegistry; +use crate::views::snapshot::WatcherState; +use crate::views::RelPath; +use crate::watcher_filter::{ + run_watcher_thread, watcher_dispatch_channel, WatcherDispatchEvent, WatcherFilterConfig, +}; +use notify::{event::CreateKind, event::DataChange, event::ModifyKind, Event, EventKind}; +use std::path::Path; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{mpsc, Arc, RwLock}; +use std::time::Duration; + +fn driver(root: &Path, storage: &Path) -> (Arc, ViewAccess) { + let registry = FamilyRegistry::open(storage, "family").unwrap(); + let owner = registry + .register_view_with_storage(registry.storage(), "cookie", root) + .unwrap(); + let access = ViewAccess::Owner(owner.clone()); + ( + Arc::new(CheckoutDriver::new( + owner, + Producers { + trigram: "unused".into(), + semantic: None, + callgraph: "unused".into(), + }, + None, + Arc::new(ConfiguredMembershipWalker), + vec![], + )), + access, + ) +} + +fn context(root: &Path, driver: &Arc) -> AppContext { + let ctx = AppContext::new(default_language_provider_factory(), Config::default()); + ctx.update_config(|config| config.project_root = Some(root.to_path_buf())); + ctx.set_canonical_cache_root(root.to_path_buf()); + ctx.mark_subc_bound(); + ctx.install_checkout_driver(Arc::clone(driver)); + ctx +} + +fn delayed_delivery_past_read() { + let temp = tempfile::tempdir().unwrap(); + let storage = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let changed = root.join("source.rs"); + std::fs::write(&changed, b"old bytes").unwrap(); + let (driver, _) = driver(&root, storage.path()); + let ctx = context(&root, &driver); + let (dispatch_tx, dispatch_rx) = watcher_dispatch_channel(); + *ctx.watcher_rx().lock() = Some(dispatch_rx.clone()); + let shutdown = Arc::new(AtomicBool::new(false)); + let (attached_tx, attached_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + let (script_tx, script_rx) = mpsc::channel::>(); + let config = WatcherFilterConfig::new(root.clone(), None); + let thread_shutdown = Arc::clone(&shutdown); + let filter = std::thread::spawn(move || { + run_watcher_thread( + config, + vec![], + Arc::new(RwLock::new(None)), + Arc::new(AtomicU64::new(0)), + dispatch_tx, + thread_shutdown, + move |_, _, raw_tx| { + // Holding at the backend handoff, not at the reader, reproduces + // a completed write whose invalidation arrives after the read. + let shim = std::thread::spawn(move || { + attached_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + for event in script_rx.recv().unwrap() { + raw_tx.send(Ok(event)).unwrap(); + } + // Keep the raw stream connected until the filter stops. + raw_tx + }); + Ok::<_, String>(shim) + }, + ); + }); + attached_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + let read = std::fs::read(&changed).unwrap(); + std::fs::write(&changed, b"new bytes").unwrap(); + assert_eq!(std::fs::read(&changed).unwrap(), b"new bytes"); + let ticket = driver.begin_watcher_cookie().unwrap(); + let cookie_path = ticket.path().to_path_buf(); + script_tx + .send(vec![ + Event::new(EventKind::Modify(ModifyKind::Data(DataChange::Content))) + .add_path(changed.clone()), + Event::new(EventKind::Create(CreateKind::File)).add_path(cookie_path.clone()), + ]) + .unwrap(); + drain_watcher_events_bounded(&ctx, 1); + assert_eq!(read, b"old bytes"); + assert_eq!( + ticket.wait_timeout(Duration::ZERO), + Err(mpsc::RecvTimeoutError::Timeout) + ); + assert!(driver + .installed_snapshot() + .pending_intent() + .next() + .is_none()); + release_tx.send(()).unwrap(); + + // Peek by consuming then restoring the complete prefix. This is a bounded + // handshake, not a timing assumption about how fast the worker is scheduled. + let first = dispatch_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + let second = dispatch_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + assert!( + matches!(&first, WatcherDispatchEvent::ContentOnlyPaths(paths) if paths == &vec![changed.clone()]) + ); + assert_eq!( + second, + WatcherDispatchEvent::CookieObserved(cookie_path.clone()) + ); + assert!( + !cookie_path.exists(), + "unlink must precede driver acknowledgment" + ); + assert_eq!( + ticket.wait_timeout(Duration::ZERO), + Err(mpsc::RecvTimeoutError::Timeout) + ); + let (tx, rx) = crossbeam_channel::unbounded(); + tx.send(first).unwrap(); + tx.send(second).unwrap(); + *ctx.watcher_rx().lock() = Some(rx); + drain_watcher_to_completion_for_test(&ctx, 1); + let observed = ticket.wait_timeout(Duration::ZERO).unwrap(); + assert!(observed.revision > 0); + assert!(driver + .installed_snapshot() + .pending_intent() + .any(|path| path == &RelPath::new("source.rs").unwrap())); + assert!(!driver + .installed_snapshot() + .membership() + .keys() + .any(|path| path.as_bytes().starts_with(b".aft-cookie-"))); + shutdown.store(true, Ordering::SeqCst); + filter.join().unwrap(); +} + +#[cfg(target_os = "linux")] +#[test] +fn watcher_cookie_delayed_delivery_past_read_inotify() { + delayed_delivery_past_read(); +} + +#[cfg(target_os = "macos")] +#[test] +fn watcher_cookie_delayed_delivery_past_read_fsevents() { + delayed_delivery_past_read(); +} + +#[cfg(windows)] +#[test] +fn watcher_cookie_delayed_delivery_past_read_windows() { + delayed_delivery_past_read(); +} + +#[test] +fn watcher_cookie_is_excluded_from_walk_membership_before_delivery() { + let temp = tempfile::tempdir().unwrap(); + let storage = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + std::fs::write(root.join("source.rs"), b"source").unwrap(); + let (driver, access) = driver(&root, storage.path()); + let ticket = driver.begin_watcher_cookie().unwrap(); + assert!(ticket.path().exists()); + let reconciled = driver.reconcile(&access).unwrap(); + assert_eq!(reconciled.entries.len(), 1); + assert!(reconciled + .entries + .contains_key(&RelPath::new("source.rs").unwrap())); + assert!( + ticket.path().exists(), + "only delivery or ticket cleanup unlinks the cookie" + ); + assert!(!super::flush::FlushControl::default().observe( + &Event::new(EventKind::Create(CreateKind::File)).add_path(ticket.path().to_path_buf()) + )); +} + +#[test] +fn watcher_cookie_rescan_dispatch_marks_driver_and_survives_later_paths() { + use crate::watcher_filter::RescanReason; + let temp = tempfile::tempdir().unwrap(); + let storage = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let (driver, access) = driver(&root, storage.path()); + let ctx = context(&root, &driver); + let (tx, rx) = crossbeam_channel::unbounded(); + *ctx.watcher_rx().lock() = Some(rx); + for reason in [ + RescanReason::Unknown, + RescanReason::UserDropped, + RescanReason::KernelDropped, + RescanReason::BufferOverflow, + ] { + let before = driver.revision(&access); + let ticket = driver.begin_watcher_cookie().unwrap(); + tx.send(WatcherDispatchEvent::RescanRequired(reason)) + .unwrap(); + tx.send(WatcherDispatchEvent::CookieObserved( + ticket.path().to_path_buf(), + )) + .unwrap(); + drain_watcher_to_completion_for_test(&ctx, 1); + let observed = ticket.wait_timeout(Duration::ZERO).unwrap(); + assert_eq!(observed.watcher, WatcherState::Overflowed); + assert!(observed.revision > before); + driver.record_absolute_change(&root.join("later.rs")); + assert_eq!( + driver.installed_snapshot().watcher(), + WatcherState::Overflowed + ); + } +} + +#[test] +fn watcher_cookie_fences_paths_already_parked_in_an_apply_slice() { + use crate::context::{WatcherDrainApplyPhase, WatcherDrainPhase, WatcherDrainSliceState}; + let temp = tempfile::tempdir().unwrap(); + let storage = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let (driver, _) = driver(&root, storage.path()); + let ctx = context(&root, &driver); + let mut state = WatcherDrainSliceState::new( + ctx.configure_generation(), + ctx.configure_content_generation(), + ); + state.phase = WatcherDrainPhase::Apply { + stage: WatcherDrainApplyPhase::PendingTier2, + paths: [root.join("parked.rs")].into(), + remaining: 1, + oversized_inline_batch: false, + }; + *ctx.watcher_drain_slice().lock() = Some(state); + let (tx, rx) = crossbeam_channel::unbounded(); + *ctx.watcher_rx().lock() = Some(rx); + let ticket = driver.begin_watcher_cookie().unwrap(); + tx.send(WatcherDispatchEvent::CookieObserved( + ticket.path().to_path_buf(), + )) + .unwrap(); + drain_watcher_to_completion_for_test(&ctx, 1); + assert!(ticket.wait_timeout(Duration::ZERO).unwrap().revision > 0); + assert!(driver + .installed_snapshot() + .pending_intent() + .any(|path| path == &RelPath::new("parked.rs").unwrap())); +} + +#[test] +fn watcher_cookie_coalesced_rescan_gap_still_invalidates_the_current_driver() { + use crate::watcher_filter::RescanReason; + let temp = tempfile::tempdir().unwrap(); + let storage = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let (driver, access) = driver(&root, storage.path()); + let ctx = context(&root, &driver); + let (_tx, rx) = crossbeam_channel::unbounded(); + *ctx.watcher_rx().lock() = Some(rx); + ctx.watcher_counters().start_rescan(); + assert!(ctx + .watcher_counters() + .note_overflow(RescanReason::BufferOverflow, vec![])); + let before = driver.revision(&access); + drain_watcher_events_bounded(&ctx, 1); + assert!(driver.revision(&access) > before); + assert_eq!( + driver.installed_snapshot().watcher(), + WatcherState::Overflowed + ); + assert_eq!( + ctx.watcher_counters() + .snapshot() + .rescans_buffer_overflow_total, + 0 + ); + assert_eq!( + ctx.watcher_counters().finish_rescan_walk(), + Some(RescanReason::BufferOverflow) + ); +} diff --git a/crates/aft/src/watcher_backend/flush.rs b/crates/aft/src/watcher_backend/flush.rs index b404e63748..ad2a822c53 100644 --- a/crates/aft/src/watcher_backend/flush.rs +++ b/crates/aft/src/watcher_backend/flush.rs @@ -221,7 +221,7 @@ impl FlushControl { if event.need_rescan() { return false; } - let marker_event = event.paths.iter().any(|path| is_marker_path(path)); + let marker_event = event.paths.iter().any(|path| is_flush_marker_path(path)); if !marker_event { return false; } @@ -243,6 +243,12 @@ impl FlushControl { } pub(crate) fn is_marker_path(path: &Path) -> bool { + // Corpus admission excludes both protocols, but command-report intake must + // not consume a views cookie before it reaches the driver's ordered queue. + super::cookie::is_cookie_path(path) || is_flush_marker_path(path) +} + +fn is_flush_marker_path(path: &Path) -> bool { let Some(name) = path.file_name().and_then(|name| name.to_str()) else { return false; }; diff --git a/crates/aft/src/watcher_backend/fsevents.rs b/crates/aft/src/watcher_backend/fsevents.rs index 8d29737957..e28a5a50f0 100644 --- a/crates/aft/src/watcher_backend/fsevents.rs +++ b/crates/aft/src/watcher_backend/fsevents.rs @@ -1878,6 +1878,7 @@ mod tests { Ok( WatcherDispatchEvent::Paths(_) | WatcherDispatchEvent::ContentOnlyPaths(_) + | WatcherDispatchEvent::CookieObserved(_) | WatcherDispatchEvent::IgnoreRulesChanged { .. } | WatcherDispatchEvent::RootDeleted, ) => {} diff --git a/crates/aft/src/watcher_backend/inotify.rs b/crates/aft/src/watcher_backend/inotify.rs index e133f89270..fc529f5d22 100644 --- a/crates/aft/src/watcher_backend/inotify.rs +++ b/crates/aft/src/watcher_backend/inotify.rs @@ -335,12 +335,11 @@ impl ProjectWatcher { match backend_rx.recv_timeout(BACKEND_POLL_INTERVAL) { Ok(Ok(event)) => { - if counters.bash_changes.get().is_some() - && event - .paths - .iter() - .any(|path| super::flush::is_marker_path(path)) - { + if event.paths.iter().any(|path| { + super::cookie::is_cookie_path(path) + || (counters.bash_changes.get().is_some() + && super::flush::is_marker_path(path)) + }) { // New-directory recovery waits briefly for duplicate kernel // events. Publish its files before the ordered marker too. if !new_directory_files.files.is_empty() @@ -813,6 +812,47 @@ fn desired_watch_directories( #[cfg(test)] pub(super) mod tests { + #[test] + fn watcher_cookie_inotify_recovery_precedes_root_cookie_without_command_tracking() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let staging = tempfile::tempdir().unwrap(); + let tree = staging.path().join("tree"); + std::fs::create_dir(&tree).unwrap(); + std::fs::write(tree.join("source.rs"), b"already populated").unwrap(); + let counters = crate::context::watcher_counters_for_root(&root); + assert!(counters.bash_changes.get().is_none()); + let (tx, rx) = mpsc::channel(); + let _watcher = ProjectWatcher::create( + root.clone(), + vec![], + tx, + Arc::new(RwLock::new(None)), + Arc::new(AtomicU64::new(0)), + ) + .unwrap(); + std::fs::rename(&tree, root.join("tree")).unwrap(); + let cookie = root.join(".aft-cookie-123-nonce"); + std::fs::write(&cookie, b"").unwrap(); + let deadline = Instant::now() + Duration::from_secs(5); + let mut recovered = false; + loop { + let event = rx + .recv_timeout(deadline.saturating_duration_since(Instant::now())) + .unwrap() + .unwrap(); + assert!(!event.need_rescan()); + recovered |= event.paths.contains(&root.join("tree/source.rs")); + if event.paths.contains(&cookie) { + assert!( + recovered, + "cookie overtook recovered new-directory membership" + ); + break; + } + } + } + use std::sync::{Condvar, Mutex, OnceLock, RwLock}; use std::time::{Duration, Instant}; diff --git a/crates/aft/src/watcher_backend/mod.rs b/crates/aft/src/watcher_backend/mod.rs index 5785c74302..1f901c54d6 100644 --- a/crates/aft/src/watcher_backend/mod.rs +++ b/crates/aft/src/watcher_backend/mod.rs @@ -3,6 +3,9 @@ use std::sync::atomic::AtomicU64; use std::sync::{mpsc, Arc}; use crate::watcher_filter::SharedGitignore; +pub(crate) mod cookie; +#[cfg(test)] +mod cookie_tests; pub(crate) mod flush; #[cfg(target_os = "macos")] diff --git a/crates/aft/src/watcher_filter.rs b/crates/aft/src/watcher_filter.rs index eb4933d394..b91224fe79 100644 --- a/crates/aft/src/watcher_filter.rs +++ b/crates/aft/src/watcher_filter.rs @@ -180,6 +180,8 @@ pub enum WatcherDispatchEvent { Paths(Vec), /// Every path had only explicit data modifications, never topology events. ContentOnlyPaths(Vec), + /// Ordered control; preceding changes must enter the driver before its ack. + CookieObserved(PathBuf), RescanRequired(RescanReason), IgnoreRulesChanged { paths: Vec, @@ -285,7 +287,9 @@ pub fn watcher_event_invalidates(kind: ¬ify::EventKind) -> bool { pub fn watcher_path_is_infra_skip(path: &Path) -> bool { // Operating-system metadata files (a Finder `.DS_Store` rewrite, say) are // never indexed by the project walks, so their events carry no change. - if crate::project_content::is_excluded_event(path) { + if crate::watcher_backend::cookie::is_cookie_path(path) + || crate::project_content::is_excluded_event(path) + { return true; } watcher_path_has_infra_component(path) @@ -2403,10 +2407,11 @@ impl WatcherFilterThread { } match raw_rx.recv_timeout(self.next_recv_timeout()) { - Ok(Ok(event)) => { + Ok(Ok(mut event)) => { #[cfg(feature = "test-timing-hooks")] hold_watcher_intake_for_test(&self.shutdown); self.config.counters.note_raw_event(); + let cookies = self.take_cookie_paths(&mut event); if self.config.counters.bash_changes.get().is_some() && self.config.counters.bash_flush.observe(&event) { @@ -2460,6 +2465,14 @@ impl WatcherFilterThread { return; } } + for cookie in cookies { + if !self.flush_pending() + || self.send_dispatch(WatcherDispatchEvent::CookieObserved(cookie)) + == Dispatch::Closed + { + return; + } + } } Ok(Err(error)) => { if let Some(journal) = self.config.counters.bash_changes.get() { @@ -2486,6 +2499,33 @@ impl WatcherFilterThread { } } + fn take_cookie_paths(&self, event: &mut notify::Event) -> Vec { + let acknowledge = + !event.need_rescan() && matches!(event.kind, notify::EventKind::Create(_)); + let mut cookies = Vec::new(); + event.paths.retain(|path| { + if !crate::watcher_backend::cookie::is_cookie_path(path) { + return true; + } + // Only root cookies are controls. Do not unlink a similarly named + // file in an auxiliary watch or a nested directory. + if path.parent() == Some(self.config.project_root.as_path()) { + let removed = std::fs::remove_file(path).or_else(|error| { + if error.kind() == std::io::ErrorKind::NotFound { + Ok(()) + } else { + Err(error) + } + }); + if acknowledge && removed.is_ok() { + cookies.push(path.clone()); + } + } + false + }); + cookies + } + fn project_root_was_deleted(&mut self, now: Instant) -> bool { // Check on a wall-clock cadence, not once per raw event. Ignored build // bursts can otherwise turn the root-deletion fallback into a stat loop. @@ -2705,6 +2745,7 @@ impl WatcherFilterThread { .strip_prefix(&self.config.project_root) .is_ok_and(watcher_path_is_high_churn_infra) || crate::os_metadata::is_os_metadata_path(&path) + || crate::watcher_backend::cookie::is_cookie_path(&path) { continue; } @@ -2981,12 +3022,21 @@ impl WatcherFilterThread { WatcherDispatchEvent::Paths(paths) => self.hold_paths(paths, false), WatcherDispatchEvent::ContentOnlyPaths(paths) => self.hold_paths(paths, true), WatcherDispatchEvent::RescanRequired(reason) => { - self.backlog.retain(|held| !is_path_batch(held)); + // Losing the paths before a held cookie also loses its proof. + self.backlog.retain(|held| { + !is_path_batch(held) && !matches!(held, WatcherDispatchEvent::CookieObserved(_)) + }); if !rescan_held { self.backlog .push_back(WatcherDispatchEvent::RescanRequired(reason)); } } + WatcherDispatchEvent::CookieObserved(path) => { + if !rescan_held { + self.backlog + .push_back(WatcherDispatchEvent::CookieObserved(path)); + } + } WatcherDispatchEvent::IgnoreRulesChanged { paths } => match self.backlog.back_mut() { Some(WatcherDispatchEvent::IgnoreRulesChanged { paths: held }) => { merge_paths(held, paths); @@ -3036,7 +3086,9 @@ impl WatcherFilterThread { // logged and counted like an operating-system event overflow. A // rescan already running is told to run again instead. let during_rescan = self.log_overflow(RescanReason::BufferOverflow); - self.backlog.retain(|held| !is_path_batch(held)); + self.backlog.retain(|held| { + !is_path_batch(held) && !matches!(held, WatcherDispatchEvent::CookieObserved(_)) + }); if !during_rescan { self.backlog.push_back(WatcherDispatchEvent::RescanRequired( RescanReason::BufferOverflow, @@ -3149,6 +3201,116 @@ pub(crate) fn take_watcher_overflow_logs_for_test() -> Vec { #[cfg(test)] mod tests { + #[test] + fn watcher_cookie_bypasses_ignore_and_flushes_mixed_paths_before_ack() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let source = root.join("source.rs"); + let cookie = root.join(".aft-cookie-123-nonce"); + std::fs::write(&source, b"source").unwrap(); + std::fs::write(&cookie, b"").unwrap(); + let mut builder = ignore::gitignore::GitignoreBuilder::new(&root); + builder.add_line(None, ".aft-cookie-*").unwrap(); + let matcher = Arc::new(RwLock::new(Some(Arc::new(builder.build().unwrap())))); + let (tx, rx) = watcher_dispatch_channel(); + let (raw_tx, raw_rx) = mpsc::channel(); + let shutdown = Arc::new(AtomicBool::new(false)); + let mut filter = WatcherFilterThread::new( + WatcherFilterConfig::new(root.clone(), None), + matcher, + Arc::new(AtomicU64::new(0)), + tx, + Arc::clone(&shutdown), + ); + let worker = thread::spawn(move || filter.run(raw_rx)); + raw_tx + .send(Ok(notify::Event::new(notify::EventKind::Create( + notify::event::CreateKind::File, + )) + .add_path(source.clone()) + .add_path(cookie.clone()))) + .unwrap(); + assert_eq!( + rx.recv_timeout(Duration::from_secs(5)).unwrap(), + WatcherDispatchEvent::Paths(vec![source]) + ); + assert_eq!( + rx.recv_timeout(Duration::from_secs(5)).unwrap(), + WatcherDispatchEvent::CookieObserved(cookie.clone()) + ); + assert!(!cookie.exists()); + shutdown.store(true, Ordering::SeqCst); + drop(raw_tx); + worker.join().unwrap(); + } + + #[test] + fn watcher_cookie_unlinks_on_overflow_but_only_create_can_ack() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let (filter, _rx, _) = filter_with_full_dispatch_channel(&root); + let cookie = root.join(".aft-cookie-123-nonce"); + for kind in [ + notify::EventKind::Remove(notify::event::RemoveKind::File), + notify::EventKind::Any, + ] { + std::fs::write(&cookie, b"").unwrap(); + let mut event = notify::Event::new(kind).add_path(cookie.clone()); + assert!(filter.take_cookie_paths(&mut event).is_empty()); + assert!(event.paths.is_empty()); + assert!(!cookie.exists()); + } + std::fs::write(&cookie, b"").unwrap(); + let mut event = + notify::Event::new(notify::EventKind::Create(notify::event::CreateKind::File)) + .add_path(cookie.clone()) + .set_flag(notify::event::Flag::Rescan); + assert!(filter.take_cookie_paths(&mut event).is_empty()); + assert!(event.need_rescan()); + assert!(!cookie.exists()); + let nested = root.join("nested/.aft-cookie-123-nonce"); + std::fs::create_dir_all(nested.parent().unwrap()).unwrap(); + std::fs::write(&nested, b"not a root control").unwrap(); + let mut event = + notify::Event::new(notify::EventKind::Create(notify::event::CreateKind::File)) + .add_path(nested.clone()); + assert!(filter.take_cookie_paths(&mut event).is_empty()); + assert!(nested.exists()); + } + + #[test] + fn watcher_cookie_backlog_preserves_barriers_and_drops_them_with_lost_paths() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let (mut filter, _rx, _) = filter_with_full_dispatch_channel(&root); + filter.send_dispatch(WatcherDispatchEvent::Paths(vec![root.join("a.rs")])); + filter.send_dispatch(WatcherDispatchEvent::CookieObserved( + root.join(".aft-cookie-1-nonce"), + )); + filter.send_dispatch(WatcherDispatchEvent::Paths(vec![root.join("b.rs")])); + assert_eq!( + filter.backlog.len(), + 3, + "paths must not merge across the cookie" + ); + filter.send_dispatch(WatcherDispatchEvent::RescanRequired( + RescanReason::BufferOverflow, + )); + assert_eq!(filter.backlog.len(), 1); + assert!(matches!( + filter.backlog.front(), + Some(WatcherDispatchEvent::RescanRequired(_)) + )); + filter.send_dispatch(WatcherDispatchEvent::CookieObserved( + root.join(".aft-cookie-1-next"), + )); + assert_eq!( + filter.backlog.len(), + 1, + "a held rescan cannot certify a cookie" + ); + } + #[test] fn issue645_nested_git_and_local_manifests_get_kernel_exclusions() { use super::*; diff --git a/docs/design/views-4v-frontier-audit.md b/docs/design/views-4v-frontier-audit.md index 3c171f6ff8..bf4fa4a194 100644 --- a/docs/design/views-4v-frontier-audit.md +++ b/docs/design/views-4v-frontier-audit.md @@ -43,11 +43,14 @@ before acknowledging the exact registered cookie. A rescan must enter the driver as `WatcherState::Overflowed` and invalidate an in-flight installation; a subsequent ordinary path must not reset that state to Healthy. A cookie held in a backlog whose paths are discarded must be discarded too (timeout, not a -false successful frontier). This provides an ordered **delivery acknowledgment**, +false successful frontier). Drops coalesced during a legacy rescan also set a +pending driver-gap signal, consumed at drain entry and before a cookie ack; +legacy rescan work coalescing must not hide a gap from a newly installed driver. +This provides an ordered **delivery acknowledgment**, not ready publication: the bind owner must still establish E0, reject any gap, replay through the acknowledged cut, and revalidate before publication. -The planned additive implementation includes regression tests named +The additive implementation includes regression tests named `watcher_cookie_delayed_delivery_past_read_inotify`, `watcher_cookie_delayed_delivery_past_read_fsevents`, and `watcher_cookie_delayed_delivery_past_read_windows`, using a backend-delivery shim From 044d36aa5a692cb5f4bc190ee93dc885863a2585 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 19:46:20 +0100 Subject: [PATCH 05/31] mason: expose plane-independent verified bind snapshot hook Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/views/first_load.rs | 41 ++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/crates/aft/src/views/first_load.rs b/crates/aft/src/views/first_load.rs index 433d2ba3c3..7bc22df3ff 100644 --- a/crates/aft/src/views/first_load.rs +++ b/crates/aft/src/views/first_load.rs @@ -565,6 +565,14 @@ impl SiblingLoader { &self, access: &ViewAccess, base: Arc, + ) -> Result<(Snapshot, u64), PlaneError> { + Self::reconciled_snapshot(self.driver.as_ref(), access, base) + } + + fn reconciled_snapshot( + driver: &dyn FirstLoadDriver, + access: &ViewAccess, + base: Arc, ) -> Result<(Snapshot, u64), PlaneError> { for _ in 0..3 { // Read before the walk, so an ignore edit racing the walk shows up @@ -576,8 +584,8 @@ impl SiblingLoader { } _ => None, }; - let checkout = self.driver.reconcile(access)?; - if checkout.cut.is_none() && self.driver.revision(access) != checkout.revision { + let checkout = driver.reconcile(access)?; + if checkout.cut.is_none() && driver.revision(access) != checkout.revision { continue; } let mut delta = super::snapshot::LiveDelta::new(Arc::clone(&base)); @@ -1395,6 +1403,23 @@ mod stat_tests { .unwrap(); } + #[test] + fn verified_bind_hook_without_a_frontier_strictly_verifies_membership() { + let root = tempfile::tempdir().unwrap(); + let storage = tempfile::tempdir().unwrap(); + std::fs::write(root.path().join("file.rs"), b"fn fresh() {}\n").unwrap(); + let (driver, access) = driver(root.path(), storage.path()); + let base = driver.installed_snapshot().generation().clone(); + let (snapshot, _) = driver.verified_bind(&access, base).unwrap(); + let path = RelPath::new(b"file.rs".to_vec()).unwrap(); + assert_eq!( + snapshot.disk_state(&path), + super::super::snapshot::DiskState::of_bytes(b"fn fresh() {}\n") + ); + assert_eq!(driver.take_reconcile_work().reads, 1); + assert!(snapshot.install_cut().is_some()); + } + #[test] fn stat_reconcile_detects_unreported_same_size_backdated_edit() { let root = tempfile::tempdir().unwrap(); @@ -1506,6 +1531,18 @@ pub(crate) struct ReconcileWork { } impl CheckoutDriver { + /// Verifies this checkout against `base` and returns a reconciliation cut. + /// Unsupported freshness proofs use strict reconciliation instead. This + /// does not publish or install the cut; those steps still validate its epoch + /// and installed-bundle identity. The hook is independent of plane selection. + pub fn verified_bind( + &self, + access: &ViewAccess, + base: Arc, + ) -> Result<(Snapshot, u64), PlaneError> { + SiblingLoader::reconciled_snapshot(self, access, base) + } + pub fn take_revalidated_job(&self) -> bool { self.retry.swap(false, std::sync::atomic::Ordering::AcqRel) } From 215e04a59a5ec5b82df95e3dccbf7e6a217d5f04 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 19:54:49 +0100 Subject: [PATCH 06/31] mason: reject unstable view reads and validate after publication waits Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/views/base_refs.rs | 9 +- crates/aft/src/views/base_refs_tests.rs | 56 ++++ crates/aft/src/views/first_load.rs | 246 +++++++++++++++--- crates/aft/src/views/readiness.rs | 11 + crates/aft/src/views/semantic_runtime.rs | 4 +- .../aft/src/views/semantic_runtime_tests.rs | 29 +++ crates/aft/src/views/snapshot.rs | 2 + crates/aft/src/views/v3_plane_tests.rs | 1 + 8 files changed, 323 insertions(+), 35 deletions(-) diff --git a/crates/aft/src/views/base_refs.rs b/crates/aft/src/views/base_refs.rs index d3dc104fe2..4b2d5f6c6c 100644 --- a/crates/aft/src/views/base_refs.rs +++ b/crates/aft/src/views/base_refs.rs @@ -680,7 +680,7 @@ pub fn publish_worker( request: WorkerPublication<'_>, observer: Option<&dyn DurabilityObserver>, ) -> Result { - publish_worker_with(request, observer, &mut |_| Ok(())) + publish_worker_with(request, observer, &mut |_| Ok(()), &mut || Ok(())) } /// [`publish_worker`] with a step that runs once the final generation name is @@ -688,10 +688,14 @@ pub fn publish_worker( /// its manifest is durable) and before the pointer CAS. Plane finishers /// rename private derived files to that name here. An error aborts the job /// like a failed verify. +/// `final_validate` runs after marker and registry verification, immediately +/// before the pointer CAS, so admission and artifact work cannot stale a +/// checkout's freshness proof. pub fn publish_worker_with( request: WorkerPublication<'_>, observer: Option<&dyn DurabilityObserver>, before_cas: &mut dyn FnMut(&str) -> Result<()>, + final_validate: &mut dyn FnMut() -> Result<()>, ) -> Result { let WorkerPublication { storage, @@ -846,6 +850,9 @@ pub fn publish_worker_with( if let Err(error) = verified { return Err(abort(error, marker)); } + if let Err(error) = final_validate() { + return Err(abort(error, marker)); + } // (4) The CAS. From here on the reference may be durable. let outcome = worker_store.commit_v2(prepared, observer); diff --git a/crates/aft/src/views/base_refs_tests.rs b/crates/aft/src/views/base_refs_tests.rs index 55a562e3c2..531cf93845 100644 --- a/crates/aft/src/views/base_refs_tests.rs +++ b/crates/aft/src/views/base_refs_tests.rs @@ -402,6 +402,62 @@ fn a_failed_verify_cleans_up_its_intent_before_any_pointer_moves() { assert_eq!(marker_files(&base_dir), 0); } +#[test] +fn final_checkout_validation_runs_after_marker_validation_and_can_refuse_cas() { + let _serial = v3::serial_for_test(); + for remove_base_before_validation in [false, true] { + let storage = tempfile::tempdir().unwrap(); + let (identity, generation) = + publish_test_base(storage.path(), &[("src/changed.rs", b"old")]); + let header = identity.header(&generation); + let root = storage.path().join("worker-root"); + std::fs::create_dir(&root).unwrap(); + let finished = std::cell::Cell::new(false); + let validated = std::cell::Cell::new(false); + let result = publish_worker_with( + WorkerPublication { + storage: storage.path(), + family: FAMILY, + worker_scope: "worker-a", + root: &root, + base: &header, + delta: worker_delta(&identity, &generation, b"new"), + previous: None, + }, + None, + &mut |_| { + finished.set(true); + if remove_base_before_validation { + with_base_barrier(storage.path(), FAMILY, |tx| { + tx.execute( + "UPDATE members SET state='evicting' WHERE scope=?1", + [&identity.scope], + )?; + Ok(()) + })?; + } + Ok(()) + }, + &mut || { + assert!(finished.get()); + validated.set(true); + Err(BaseRefError::Unavailable( + "HEAD changed after admission".into(), + )) + }, + ); + assert!(result.is_err()); + assert_eq!(validated.get(), !remove_base_before_validation); + assert!(refs(storage.path()).is_empty()); + let worker = v3::view_dir(storage.path(), FAMILY, "worker-a").unwrap(); + assert!(ViewStore::existing_dir(worker) + .unwrap() + .current_generation_read_only() + .unwrap() + .is_none()); + } +} + #[test] fn a_landed_cas_is_published_and_never_erased_as_abort_cleanup() { let _serial = v3::serial_for_test(); diff --git a/crates/aft/src/views/first_load.rs b/crates/aft/src/views/first_load.rs index 7bc22df3ff..fdaf881924 100644 --- a/crates/aft/src/views/first_load.rs +++ b/crates/aft/src/views/first_load.rs @@ -602,7 +602,11 @@ impl SiblingLoader { } } for (path, entry) in checkout.entries { - delta.apply(path, entry); + if entry.reconciliation == super::readiness::ReconciledDelta::Pending { + delta.record_intent(path); + } else { + delta.apply(path, entry); + } } for (path, patch) in checkout.readiness { delta.set_readiness(path, patch); @@ -1332,6 +1336,7 @@ struct ObservedStat { #[cfg(unix)] device: u64, recorded_ns: u128, + delivery_seq: u64, } /// Publish stat evidence together with the extracted content it validates. @@ -1353,6 +1358,7 @@ impl ObservedStat { metadata.dev() }, recorded_ns, + delivery_seq: 0, } } @@ -1420,6 +1426,67 @@ mod stat_tests { assert!(snapshot.install_cut().is_some()); } + #[test] + fn unstable_strict_reads_retry_three_times_and_name_pending_without_bytes() { + let root = tempfile::tempdir().unwrap(); + let storage = tempfile::tempdir().unwrap(); + std::fs::write(root.path().join("file.rs"), b"initial").unwrap(); + let (driver, access) = driver(root.path(), storage.path()); + *driver.read_hook.lock().unwrap() = Some(Box::new(|file, attempt| { + std::fs::write(file, vec![b'x'; 20 + attempt]).unwrap(); + })); + let base = driver.installed_snapshot().generation().clone(); + let (snapshot, _) = driver.verified_bind(&access, base).unwrap(); + let path = RelPath::new(b"file.rs".to_vec()).unwrap(); + assert_eq!(snapshot.pending_intent().collect::>(), vec![&path]); + assert_eq!( + snapshot.disk_state(&path), + super::super::snapshot::DiskState::Absent + ); + assert_eq!(driver.take_reconcile_work().reads, 3); + assert!(driver.observed.lock().unwrap().stats.is_empty()); + } + + #[test] + fn event_during_exception_read_invalidates_even_unchanged_stats() { + let root = tempfile::tempdir().unwrap(); + let storage = tempfile::tempdir().unwrap(); + std::fs::write(root.path().join("file.rs"), b"unchanged").unwrap(); + let (driver, access) = driver(root.path(), storage.path()); + let driver = Arc::new(driver); + let weak = Arc::downgrade(&driver); + let path = RelPath::new(b"file.rs".to_vec()).unwrap(); + let changed = path.clone(); + *driver.read_hook.lock().unwrap() = Some(Box::new(move |_, attempt| { + if attempt == 0 { + weak.upgrade().unwrap().record_change(changed.clone()); + } + })); + driver.reconcile(&access).unwrap(); + assert_eq!(driver.take_reconcile_work().reads, 2); + assert_eq!(driver.observed.lock().unwrap().stats[&path].delivery_seq, 1); + } + + #[test] + fn overflow_updates_the_canonical_epoch_and_preserves_bundle_readers() { + let root = tempfile::tempdir().unwrap(); + let storage = tempfile::tempdir().unwrap(); + let (driver, access) = driver(root.path(), storage.path()); + let before = driver.installed_snapshot(); + driver.record_watcher_overflow(); + let after = driver.installed_snapshot(); + assert_eq!( + after.watcher(), + super::super::snapshot::WatcherState::Overflowed + ); + assert!(after.install_cut().unwrap().epoch > before.install_cut().unwrap().epoch); + assert!(Arc::ptr_eq( + after.readers().unwrap(), + before.readers().unwrap() + )); + assert!(driver.validate_cut(&access, &before, 0).is_err()); + } + #[test] fn stat_reconcile_detects_unreported_same_size_backdated_edit() { let root = tempfile::tempdir().unwrap(); @@ -1510,10 +1577,28 @@ pub struct CheckoutDriver { publication_transition: std::sync::Mutex<()>, retry: std::sync::atomic::AtomicBool, watcher_cookies: crate::watcher_backend::cookie::CookieBarrier, + delivery: std::sync::Mutex, + #[cfg(test)] + read_hook: std::sync::Mutex>, #[cfg(test)] work: std::sync::Mutex, } +#[derive(Default)] +struct DeliveryState { + seq: u64, + paths: BTreeMap, +} + +#[cfg(test)] +type ReadHook = Box; + +enum StableRead { + Absent, + Bytes(Vec, ObservedStat), + Pending, +} + struct PreparedReaders { seq: u64, readers: Arc, @@ -1531,6 +1616,66 @@ pub(crate) struct ReconcileWork { } impl CheckoutDriver { + fn stable_read( + &self, + path: &RelPath, + absolute: &std::path::Path, + recorded_ns: u128, + first_seq: u64, + ) -> Result { + for attempt in 0..3 { + let seq = if attempt == 0 { + first_seq + } else { + self.delivery + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .seq + }; + let before = self.source_stat(absolute, recorded_ns)?; + let bytes = super::assembly::read_working_tree_file(absolute) + .map_err(|error| SiblingLoader::error(error.to_string()))?; + #[cfg(test)] + { + if let Some(bytes) = &bytes { + let mut work = self.work.lock().unwrap(); + work.reads += 1; + work.content_hash_bytes += bytes.len(); + } + if let Some(hook) = &*self.read_hook.lock().unwrap() { + hook(absolute, attempt); + } + } + let after = self.source_stat(absolute, recorded_ns)?; + let delivery = self + .delivery + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let unchanged = delivery.paths.get(path).copied().unwrap_or(0) <= seq; + drop(delivery); + match (before, after, bytes) { + (None, None, None) if unchanged => return Ok(StableRead::Absent), + (Some(mut before), Some(after), Some(bytes)) if unchanged => { + let stable = before.stamp == after.stamp; + #[cfg(unix)] + let stable = stable && before.device == after.device; + if stable { + before.delivery_seq = seq; + return Ok(StableRead::Bytes(bytes, before)); + } + } + _ => {} + } + // A changing file must not contribute bytes to a ready generation. + // Back off between bounded attempts; tests inject changes directly. + #[cfg(not(test))] + if attempt < 2 { + std::thread::sleep(std::time::Duration::from_millis(1 << attempt)); + } + } + Ok(StableRead::Pending) + } + /// Verifies this checkout against `base` and returns a reconciliation cut. /// Unsupported freshness proofs use strict reconciliation instead. This /// does not publish or install the cut; those steps still validate its epoch @@ -1593,6 +1738,9 @@ impl CheckoutDriver { publish_fence: std::sync::RwLock::new(None), observed: Default::default(), watcher_cookies: Default::default(), + delivery: Default::default(), + #[cfg(test)] + read_hook: Default::default(), installed: std::sync::Mutex::new(Arc::new(InstalledBundle { seq: 0, composed, @@ -1642,6 +1790,15 @@ impl CheckoutDriver { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let installed = Arc::make_mut(&mut installed); + { + let mut delivery = self + .delivery + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + delivery.seq += 1; + let seq = delivery.seq; + delivery.paths.insert(path.clone(), seq); + } installed.revision += 1; // Keep prior membership until reconciliation, but mark the edited path // as unreflected so queries cannot report stale data as installed. @@ -1671,22 +1828,28 @@ impl CheckoutDriver { /// Invalidates stale builds and indexed membership until strict reconciliation. pub fn record_watcher_overflow(&self) { + let mut delta = self + .live + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); let mut installed = self .installed .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); + let installed = Arc::make_mut(&mut installed); installed.revision += 1; - let mut delta = - super::snapshot::LiveDelta::new(Arc::clone(installed.snapshot.generation())); - for (path, entry) in installed.snapshot.live_entries() { - delta.apply(path.clone(), entry.clone()); - } - for path in installed.snapshot.pending_intent() { - delta.record_intent(path.clone()); - } - delta.record_root_ignore_files(installed.snapshot.root_ignore_files().cloned()); + delta.bump_epoch(); delta.set_watcher(super::snapshot::WatcherState::Overflowed); - installed.snapshot = delta.snapshot(); + installed.snapshot = delta + .snapshot() + .with_readers(installed.readers.clone()) + .with_install_cut(InstallCut { + seq: installed.seq, + resident: installed.composed.clone(), + epoch: delta.epoch(), + version: delta.version(), + intent_version: delta.snapshot().intent_version(), + }); } /// Creates an exclusive root cookie. This does not enable verified bind; @@ -1829,6 +1992,11 @@ impl FirstLoadDriver for CheckoutDriver { .unwrap_or_default() .as_secs(), ) * 1_000_000_000; + let delivery_seq = self + .delivery + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .seq; let before = self.source_stat(&absolute, recorded_ns)?; if !pending.contains(&path) { if let (Some(old), Some(cached), Some(current)) = ( @@ -1836,7 +2004,15 @@ impl FirstLoadDriver for CheckoutDriver { previous.stats.get(&path), before.as_ref(), ) { - if cached.reusable(current) { + let delivered = self + .delivery + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .paths + .get(&path) + .copied() + .unwrap_or(0); + if cached.reusable(current) && delivered <= cached.delivery_seq { let mut entry = old.clone(); entry.seq = revision; entries.insert(path.clone(), entry); @@ -1845,27 +2021,20 @@ impl FirstLoadDriver for CheckoutDriver { } } } - let Some(bytes) = super::assembly::read_working_tree_file(&absolute) - .map_err(|error| SiblingLoader::error(error.to_string()))? - else { - continue; - }; - let after = self.source_stat(&absolute, recorded_ns)?; - if let (Some(before), Some(after)) = (before, after) { - let stable = before.stamp == after.stamp; - #[cfg(unix)] - let stable = stable && before.device == after.device; - if stable { - stats.insert(path.clone(), before); - } - } - #[cfg(test)] - { - let mut work = self.work.lock().unwrap(); - work.reads += 1; - work.content_hash_bytes += bytes.len(); - } + let (bytes, evidence) = + match self.stable_read(&path, &absolute, recorded_ns, delivery_seq)? { + StableRead::Absent => continue, + StableRead::Bytes(bytes, evidence) => (bytes, evidence), + StableRead::Pending => { + let mut entry = LiveEntry::new(installed.disk_state(&path), revision); + entry.reconciliation = super::readiness::ReconciledDelta::Pending; + entries.insert(path, entry); + continue; + } + }; + stats.insert(path.clone(), evidence); let mut entry = LiveEntry::new(super::snapshot::DiskState::of_bytes(&bytes), revision); + entry.reconciliation = super::readiness::ReconciledDelta::Reconciled; for plane in &self.planes { if plane.applies_to(&path) { #[cfg(test)] @@ -2408,6 +2577,10 @@ impl CheckoutDriver { } Ok(()) }, + &mut || { + self.validate_cut(access, snapshot, 0) + .map_err(|error| super::base_refs::BaseRefError::Unavailable(error.reason)) + }, ); drop(live); match outcome { @@ -2495,7 +2668,14 @@ impl QueryState for CheckoutDriver { gaps.extend( snapshot .live_entries() - .filter(|(path, _)| applicable(path)) + .filter(|(path, entry)| { + applicable(path) + && (entry.reconciliation != super::readiness::ReconciledDelta::Reconciled + || !matches!( + snapshot.plane_readiness(path, plane), + Some(super::readiness::PlaneState::Ready { .. }) + )) + }) .map(|(path, _)| path.clone()), ); for (path, _entry) in snapshot.generation().entries() { diff --git a/crates/aft/src/views/readiness.rs b/crates/aft/src/views/readiness.rs index 682aeeab10..479f565e30 100644 --- a/crates/aft/src/views/readiness.rs +++ b/crates/aft/src/views/readiness.rs @@ -18,6 +18,17 @@ use super::manifest_v2::{EntryV2, ManifestV2, Producers}; use super::snapshot::{ComposedGeneration, DiskState}; use super::{RelPath, Result, ViewError}; +/// Whether a live delta's bytes were stably observed, independently of whether +/// its plane artifacts have finished materializing. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub enum ReconciledDelta { + #[default] + Unreconciled, + Reconciled, + /// Repeated unstable reads supplied no bytes safe to insert. + Pending, +} + /// The state of one plane for one manifest entry. #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(tag = "state", rename_all = "snake_case")] diff --git a/crates/aft/src/views/semantic_runtime.rs b/crates/aft/src/views/semantic_runtime.rs index 4083c2542d..c97b2a4f4a 100644 --- a/crates/aft/src/views/semantic_runtime.rs +++ b/crates/aft/src/views/semantic_runtime.rs @@ -730,7 +730,9 @@ pub struct CheckoutSemanticMemory { fn needs_reload(snapshot: &Snapshot) -> bool { snapshot.generation().name() == "empty" || snapshot.pending_intent().next().is_some() - || snapshot.live_entries().next().is_some() + || snapshot + .live_entries() + .any(|(_, entry)| entry.reconciliation != super::readiness::ReconciledDelta::Reconciled) } /// Whether reconciling `snapshot` needs a cold-build slot. A reconcile walks diff --git a/crates/aft/src/views/semantic_runtime_tests.rs b/crates/aft/src/views/semantic_runtime_tests.rs index 808ce60b98..4c3634fba0 100644 --- a/crates/aft/src/views/semantic_runtime_tests.rs +++ b/crates/aft/src/views/semantic_runtime_tests.rs @@ -12,6 +12,35 @@ use std::sync::atomic::AtomicUsize; /// row, and the marker row saying the run's vector matches that chunk row. const ROWS_PER_ONE_CHUNK_FILE: u64 = 3; +#[test] +fn reconciled_delta_does_not_request_another_source_walk() { + let manifest = + super::super::manifest_v2::ManifestV2::new(super::super::manifest_v2::ManifestHeader { + producers: Producers { + trigram: "unused".into(), + semantic: None, + callgraph: "unused".into(), + }, + head_tree: None, + ignore_fingerprint: None, + segment: None, + }); + let mut delta = LiveDelta::new(Arc::new(ComposedGeneration::new( + "published", + manifest, + None, + ))); + let path = super::super::RelPath::new(b"file.rs".to_vec()).unwrap(); + let mut entry = super::super::snapshot::LiveEntry::new(DiskState::of_bytes(b"new"), 1); + delta.apply(path.clone(), entry.clone()); + assert!(super::needs_reload(&delta.snapshot())); + entry.reconciliation = super::super::readiness::ReconciledDelta::Reconciled; + delta.apply(path.clone(), entry); + assert!(!super::needs_reload(&delta.snapshot())); + delta.record_intent(path); + assert!(super::needs_reload(&delta.snapshot())); +} + /// The test prints public setup, query answers, a v3 file inventory and exact /// reader-floor bytes. Query expectations come from an independent cold index, /// not the view. diff --git a/crates/aft/src/views/snapshot.rs b/crates/aft/src/views/snapshot.rs index a987f1c866..e89db24033 100644 --- a/crates/aft/src/views/snapshot.rs +++ b/crates/aft/src/views/snapshot.rs @@ -535,6 +535,7 @@ pub struct LiveEntry { /// The watcher or intent sequence that produced this state. pub seq: u64, pub attachments: BTreeMap, + pub reconciliation: super::readiness::ReconciledDelta, } impl fmt::Debug for LiveEntry { @@ -553,6 +554,7 @@ impl LiveEntry { disk, seq, attachments: BTreeMap::new(), + reconciliation: Default::default(), } } } diff --git a/crates/aft/src/views/v3_plane_tests.rs b/crates/aft/src/views/v3_plane_tests.rs index 9f110e28ba..a3df1dbde7 100644 --- a/crates/aft/src/views/v3_plane_tests.rs +++ b/crates/aft/src/views/v3_plane_tests.rs @@ -150,6 +150,7 @@ impl Fixture { }, None, before_cas, + &mut || Ok(()), ) .unwrap() { From b1df8f9b295a7528d1b7f671fbe3805e6220025e Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:13:29 +0100 Subject: [PATCH 07/31] mason: arm inotify directories before initial enumeration Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/watcher_backend/inotify.rs | 122 ++++++++++++++++++---- 1 file changed, 103 insertions(+), 19 deletions(-) diff --git a/crates/aft/src/watcher_backend/inotify.rs b/crates/aft/src/watcher_backend/inotify.rs index fc529f5d22..bbcdc5b2f9 100644 --- a/crates/aft/src/watcher_backend/inotify.rs +++ b/crates/aft/src/watcher_backend/inotify.rs @@ -114,30 +114,18 @@ impl ProjectWatcher { let (backend_tx, backend_rx) = mpsc::channel(); let mut watcher = notify::recommended_watcher(backend_tx)?; - let walked_directories = exclusion_state - .boundary() - .map_or_else(BTreeMap::new, |boundary| { - collect_watch_directories(boundary, &matcher, &exclusion_paths) - }); counters.set_backend_exclusions( observed_generation.unwrap_or(0), exclusion_paths.clone(), watcher_exclusion_paths(&plan.dropped), ); - let mut installed = BTreeSet::new(); - for directory in desired_watch_directories( - walked_directories, - &BTreeSet::new(), - &matcher, - &exclusion_paths, - &root, - ) { - if watch_directory(&mut watcher, &directory, &mut watch, &counters)? { - installed.insert(directory.clone()); - } else if directory == root { - return Err(notify::Error::path_not_found().add_path(root)); - } - } + let boundary = exclusion_state + .boundary() + .ok_or_else(|| notify::Error::path_not_found().add_path(root.clone()))?; + let installed = + register_initial_tree(boundary, &matcher, &exclusion_paths, &mut |directory| { + watch_directory(&mut watcher, directory, &mut watch, &counters) + })?; let mut watched_directories = installed; let git_directory = root.join(".git"); let mut git_watched = watched_directories.contains(&git_directory); @@ -734,6 +722,42 @@ impl DirectoryIdentity { } } +/// Arm each parent before listing it. Children created during registration +/// are then either enumerated or reported on the already armed parent stream. +fn register_initial_tree( + boundary: &crate::watcher_filter::WatcherWalkBoundary, + matcher: &SharedGitignore, + exclusions: &[PathBuf], + watch: &mut dyn FnMut(&Path) -> notify::Result, +) -> notify::Result> { + let root = boundary.root(); + let mut stack = vec![root.to_path_buf()]; + let mut installed = BTreeSet::new(); + while let Some(directory) = stack.pop() { + if exclusions + .iter() + .any(|excluded| directory.starts_with(excluded)) + || watcher_directory_is_ignored_by_matcher(matcher, &directory, root) + || !boundary.allows_directory(&directory) + { + continue; + } + if !watch(&directory)? { + return Err(notify::Error::path_not_found().add_path(directory)); + } + installed.insert(directory.clone()); + for entry in + crate::watcher_filter::exclusion_read_dir(&directory).map_err(notify::Error::io)? + { + let entry = entry.map_err(notify::Error::io)?; + if entry.file_type().map_err(notify::Error::io)?.is_dir() { + stack.push(entry.path()); + } + } + } + Ok(installed) +} + fn collect_watch_directories( boundary: &crate::watcher_filter::WatcherWalkBoundary, matcher: &SharedGitignore, @@ -812,6 +836,66 @@ fn desired_watch_directories( #[cfg(test)] pub(super) mod tests { + #[test] + fn initial_subscription_arms_parent_before_listing_new_descendants() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let boundary = crate::watcher_filter::WatcherWalkBoundary::for_root(&root).unwrap(); + let mut order = Vec::new(); + let installed = register_initial_tree( + &boundary, + &Arc::new(RwLock::new(None)), + &[], + &mut |directory| { + order.push(directory.to_path_buf()); + if directory == root { + std::fs::create_dir(root.join("during")).unwrap(); + } + if directory == root.join("during") { + std::fs::create_dir(directory.join("nested")).unwrap(); + } + Ok(true) + }, + ) + .unwrap(); + assert_eq!( + order, + vec![ + root.clone(), + root.join("during"), + root.join("during/nested") + ] + ); + assert_eq!(installed.len(), 3); + } + + #[test] + fn initial_subscription_refuses_a_missing_watch_or_incomplete_listing() { + for remove_after_watch in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + std::fs::create_dir(root.join("child")).unwrap(); + let boundary = crate::watcher_filter::WatcherWalkBoundary::for_root(&root).unwrap(); + let result = register_initial_tree( + &boundary, + &Arc::new(RwLock::new(None)), + &[], + &mut |directory| { + if directory == root { + return Ok(true); + } + if remove_after_watch { + std::fs::remove_dir(directory).unwrap(); + Ok(true) + } else { + Ok(false) + } + }, + ); + assert!(result.is_err()); + } + } + #[test] fn watcher_cookie_inotify_recovery_precedes_root_cookie_without_command_tracking() { let temp = tempfile::tempdir().unwrap(); From de24132e50525fe7663d66bc2c0b67ede8b61804 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:13:31 +0100 Subject: [PATCH 08/31] mason: pair v3 stat evidence with generations and mint advisory view tokens Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/blob_store/v3.rs | 56 ++ crates/aft/src/commands/mod.rs | 1 + crates/aft/src/commands/view_token.rs | 62 ++ crates/aft/src/main.rs | 1 + crates/aft/src/persisted_format.rs | 2 +- crates/aft/src/views/first_load.rs | 398 +++++++++- crates/aft/src/views/mod.rs | 1 + crates/aft/src/views/registry.rs | 380 ++++++++++ crates/aft/src/views/semantic.rs | 16 + crates/aft/src/views/snapshot.rs | 4 + crates/aft/src/views/verification.rs | 689 ++++++++++++++++++ docs/design/views-v3-format.md | 27 + .../src/__tests__/view-token.test.ts | 27 + packages/aft-bridge/src/index.ts | 2 + packages/aft-bridge/src/view-token.ts | 31 + 15 files changed, 1670 insertions(+), 27 deletions(-) create mode 100644 crates/aft/src/commands/view_token.rs create mode 100644 crates/aft/src/views/verification.rs create mode 100644 packages/aft-bridge/src/__tests__/view-token.test.ts create mode 100644 packages/aft-bridge/src/view-token.ts diff --git a/crates/aft/src/blob_store/v3.rs b/crates/aft/src/blob_store/v3.rs index 06da17f730..ef6d118a33 100644 --- a/crates/aft/src/blob_store/v3.rs +++ b/crates/aft/src/blob_store/v3.rs @@ -52,6 +52,7 @@ const META: &str = "CREATE TABLE format_meta (singleton INTEGER PRIMARY KEY CHEC /// [`RegistryWriter::upgrade_to_base_refs`], only by a writer that raised the /// registry reader floor to 2 first. pub const REGISTRY_BASE_REFS_VERSION: u32 = 2; +pub const REGISTRY_VERIFICATION_VERSION: u32 = 3; macro_rules! registry_v1_schema { () => { " @@ -97,6 +98,21 @@ CREATE TABLE sweep_state (singleton INTEGER PRIMARY KEY CHECK(singleton=1), const REGISTRY_SCHEMA: &str = registry_v1_schema!(); const BASE_REFS_SCHEMA: &str = base_refs_schema!(); const REGISTRY_SCHEMA_V2: &str = concat!(registry_v1_schema!(), base_refs_schema!()); +macro_rules! view_tokens_schema { + () => { + "CREATE TABLE view_tokens (nonce TEXT NOT NULL PRIMARY KEY, root BLOB NOT NULL, + expected_tree TEXT NOT NULL, minted_ms INTEGER NOT NULL, expires_ms INTEGER NOT NULL, + bound_root BLOB, bound_gitdir BLOB, consumed_ms INTEGER, + CHECK((bound_root IS NULL AND bound_gitdir IS NULL AND consumed_ms IS NULL) OR + (bound_root IS NOT NULL AND bound_gitdir IS NOT NULL AND consumed_ms IS NOT NULL)));" + }; +} +const VIEW_TOKENS_SCHEMA: &str = view_tokens_schema!(); +const REGISTRY_SCHEMA_V3: &str = concat!( + registry_v1_schema!(), + base_refs_schema!(), + view_tokens_schema!() +); const PLANE_SCHEMA: &str = " CREATE TABLE store_meta (singleton INTEGER PRIMARY KEY CHECK(singleton=1), plane TEXT NOT NULL, gc_epoch INTEGER NOT NULL); CREATE TABLE blob_payloads (full_key BLOB NOT NULL PRIMARY KEY CHECK(length(full_key)=32), @@ -487,6 +503,7 @@ fn validate(c: &Connection, path: &Path, kind: Kind) -> Result<()> { // A registry of the base-reference protocol carries its own exact // text; any other version/schema pair falls through to the drift check. (Kind::Registry, 2, 2) if layout == 0 => Some(REGISTRY_SCHEMA_V2), + (Kind::Registry, 3, 3) if layout == 0 => Some(REGISTRY_SCHEMA_V3), _ => kind.schema_for(layout), }; let Some(compiled_schema) = compiled_schema else { @@ -508,6 +525,11 @@ fn validate(c: &Connection, path: &Path, kind: Kind) -> Result<()> { u64::from(REGISTRY_BASE_REFS_VERSION), REGISTRY_BASE_REFS_VERSION, ) + || (version, schema) + == ( + u64::from(REGISTRY_VERIFICATION_VERSION), + REGISTRY_VERIFICATION_VERSION, + ) } Kind::Plane(_) => version == u64::from(FORMAT_VERSION) && schema == SCHEMA_VERSION, }; @@ -906,6 +928,8 @@ impl<'a> RegistryWriter<'a> { )?; if (version, schema) == (REGISTRY_BASE_REFS_VERSION, REGISTRY_BASE_REFS_VERSION) && text == format!("{META}{REGISTRY_SCHEMA_V2}") + || (version, schema) == (REGISTRY_VERIFICATION_VERSION, REGISTRY_VERIFICATION_VERSION) + && text == format!("{META}{REGISTRY_SCHEMA_V3}") { return Ok(()); } @@ -931,6 +955,19 @@ impl<'a> RegistryWriter<'a> { Ok(()) }) } + /// Adds only the registered v3 token protocol; plane stores and legacy + /// registries retain their own versions and schema. + pub fn upgrade_to_verification(&mut self) -> Result<()> { + self.upgrade_to_base_refs()?; + self.with_barrier(|tx| { + let version: u32 = tx.query_row("SELECT format_version FROM format_meta WHERE singleton=1", [], |r| r.get(0))?; + if version == REGISTRY_VERIFICATION_VERSION { return Ok(()); } + tx.execute_batch(VIEW_TOKENS_SCHEMA)?; + tx.execute("UPDATE format_meta SET format_version=?1,schema_version=?1,schema_text=?2 WHERE singleton=1", + params![REGISTRY_VERIFICATION_VERSION, format!("{META}{REGISTRY_SCHEMA_V3}")])?; + Ok(()) + }) + } fn connection(&self) -> Result> { self.database.connection() } @@ -1437,6 +1474,25 @@ pub fn open_base_refs_registry<'a>( ) .map_err(|e| Error::Io(std::io::Error::other(e)))? } +/// Raises the verification floor before any token or stat-evidence bytes exist. +pub fn open_verification_registry<'a>( + storage: &Path, + family: &str, + lock: &'a ReservationLock, +) -> Result> { + crate::reader_floor::publish_v3_with_floor( + storage, + REGISTRY_VERIFICATION_VERSION, + FORMAT_VERSION, + || { + let mut registry = open_base_refs_registry(storage, family, lock, None)?; + registry.upgrade_to_verification()?; + Ok(registry) + }, + ) + .map_err(|e| Error::Io(std::io::Error::other(e)))? +} + fn publish_under_lock( storage: &Path, family: &str, diff --git a/crates/aft/src/commands/mod.rs b/crates/aft/src/commands/mod.rs index 76df2b11d6..d2cb062195 100644 --- a/crates/aft/src/commands/mod.rs +++ b/crates/aft/src/commands/mod.rs @@ -79,6 +79,7 @@ pub mod undo; pub mod untrust_filter_project; pub(crate) mod url_output; mod url_resolution; +pub mod view_token; pub mod write; pub mod writes_census; pub mod zoom; diff --git a/crates/aft/src/commands/view_token.rs b/crates/aft/src/commands/view_token.rs new file mode 100644 index 0000000000..29416fb372 --- /dev/null +++ b/crates/aft/src/commands/view_token.rs @@ -0,0 +1,62 @@ +//! Internal provisioning request, deliberately not an agent tool. Tokens are +//! advisory and are never a substitute for a driver's checkout verification. + +use crate::{ + context::AppContext, + protocol::{RawRequest, Response}, +}; + +pub fn handle_view_token(request: &RawRequest, context: &AppContext) -> Response { + if !context.config().views.enabled { + return Response::error(&request.id, "view tokens require views to be enabled"); + } + let Some(root) = request.params.get("root").and_then(|v| v.as_str()) else { + return Response::error(&request.id, "root is required"); + }; + let Some(tree) = request.params.get("expected_tree").and_then(|v| v.as_str()) else { + return Response::error(&request.id, "expected_tree is required"); + }; + let Some(project) = context.canonical_cache_root_opt() else { + return Response::error(&request.id, "view token requires a configured repository"); + }; + let storage = context.storage_paths().artifacts; + let family = match crate::search_index::artifact_cache_key_read_only(&project, &storage) { + Ok(family) => family, + Err(error) => return Response::error(&request.id, &error.to_string()), + }; + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64; + match crate::views::registry::mint_view_token( + &storage, + &family, + std::path::Path::new(root), + tree, + now, + ) { + Ok(token) => Response::success( + &request.id, + serde_json::json!({ "token": token.token, "expires_ms": token.expires_ms, "advisory": true }), + ), + Err(error) => Response::error(&request.id, &error.to_string()), + } +} + +#[cfg(test)] +mod tests { + #[test] + fn view_token_request_with_views_off_creates_no_v3_files() { + let storage = tempfile::tempdir().unwrap(); + let context = crate::context::AppContext::new( + Box::new(crate::parser::TreeSitterProvider::new()), + crate::config::Config { + storage_dir: Some(storage.path().to_path_buf()), + ..Default::default() + }, + ); + let request = serde_json::from_value(serde_json::json!({ "id": "token", "command": "view_token", "root": "/future", "expected_tree": "a".repeat(40) })).unwrap(); + assert!(!super::handle_view_token(&request, &context).success); + assert!(!storage.path().join("blobs/v3").exists()); + } +} diff --git a/crates/aft/src/main.rs b/crates/aft/src/main.rs index 800a191914..81e8108c83 100644 --- a/crates/aft/src/main.rs +++ b/crates/aft/src/main.rs @@ -1518,6 +1518,7 @@ fn dispatch_command(req: RawRequest, ctx: &AppContext) -> Response { "remove_import" => aft::commands::remove_import::handle_remove_import(&req, ctx), "organize_imports" => aft::commands::organize_imports::handle_organize_imports(&req, ctx), "configure" => aft::commands::configure::handle_configure(&req, ctx), + "view_token" => aft::commands::view_token::handle_view_token(&req, ctx), "glob" => aft::commands::glob::handle_glob(&req, ctx), "grep" => aft::commands::grep::handle_grep(&req, ctx), "semantic_search" => { diff --git a/crates/aft/src/persisted_format.rs b/crates/aft/src/persisted_format.rs index 3444fc5aef..abe48e4be0 100644 --- a/crates/aft/src/persisted_format.rs +++ b/crates/aft/src/persisted_format.rs @@ -111,7 +111,7 @@ impl PersistedStore { PersistedStore::ArtifactOwner => crate::artifact_owner::SCHEMA_VERSION, PersistedStore::BackupMeta => crate::backup::SCHEMA_VERSION, PersistedStore::BashTask => crate::bash_background::persistence::SCHEMA_VERSION, - PersistedStore::ViewsV3Registry => crate::blob_store::v3::REGISTRY_BASE_REFS_VERSION, + PersistedStore::ViewsV3Registry => crate::blob_store::v3::REGISTRY_VERIFICATION_VERSION, PersistedStore::ViewsV3Store => 1, } } diff --git a/crates/aft/src/views/first_load.rs b/crates/aft/src/views/first_load.rs index fdaf881924..78eab56433 100644 --- a/crates/aft/src/views/first_load.rs +++ b/crates/aft/src/views/first_load.rs @@ -1238,6 +1238,29 @@ pub trait CompositePlane: Send + Sync { fn plane(&self) -> FamilyPlane; fn applies_to(&self, path: &RelPath) -> bool; + /// Derive a payload key without extraction. Planes with deferred extraction + /// may leave this absent and keep their own cheap attachment format. + fn key( + &self, + _path: &RelPath, + _content: crate::blob_store::v2::ContentHash, + ) -> Option { + None + } + fn cached_attachment( + &self, + _content: crate::blob_store::v2::ContentHash, + key: crate::blob_store::v2::FamilyKey, + ) -> super::snapshot::PlaneAttachment { + Arc::new(CachedAttachment(key)) + } + fn attachment_key( + &self, + _attachment: &super::snapshot::PlaneAttachment, + ) -> Option { + None + } + /// Called during strict reconciliation with the very bytes that were hashed. /// May compute extraction data, but must not write artifacts or read source. fn attachment( @@ -1337,6 +1360,7 @@ struct ObservedStat { device: u64, recorded_ns: u128, delivery_seq: u64, + native: Option, } /// Publish stat evidence together with the extracted content it validates. @@ -1346,6 +1370,8 @@ struct ObservedStat { struct ObservedCheckout { entries: BTreeMap, stats: BTreeMap, + evidence: BTreeMap, + guard: Option>, } impl ObservedStat { @@ -1359,10 +1385,14 @@ impl ObservedStat { }, recorded_ns, delivery_seq: 0, + native: None, } } fn reusable(&self, current: &Self) -> bool { + if let (Some(old), Some(current)) = (&self.native, ¤t.native) { + return old.reusable(current, self.recorded_ns); + } #[cfg(unix)] { self.stamp == current.stamp && self.device == current.device @@ -1578,6 +1608,8 @@ pub struct CheckoutDriver { retry: std::sync::atomic::AtomicBool, watcher_cookies: crate::watcher_backend::cookie::CookieBarrier, delivery: std::sync::Mutex, + v3_reconcile: std::sync::atomic::AtomicBool, + advisory_token: std::sync::Mutex>, #[cfg(test)] read_hook: std::sync::Mutex>, #[cfg(test)] @@ -1599,6 +1631,15 @@ enum StableRead { Pending, } +#[derive(Clone)] +struct CachedAttachment(crate::blob_store::v2::FamilyKey); + +fn v3_contains(route: &super::snapshot::V3Route, key: &crate::blob_store::v2::FamilyKey) -> bool { + crate::blob_store::v3::PlaneReader::open_existing(&route.storage, &route.family, key.plane()) + .and_then(|reader| reader.contains(key)) + .unwrap_or(false) +} + struct PreparedReaders { seq: u64, readers: Arc, @@ -1640,7 +1681,7 @@ impl CheckoutDriver { if let Some(bytes) = &bytes { let mut work = self.work.lock().unwrap(); work.reads += 1; - work.content_hash_bytes += bytes.len(); + let _ = bytes; } if let Some(hook) = &*self.read_hook.lock().unwrap() { hook(absolute, attempt); @@ -1656,7 +1697,7 @@ impl CheckoutDriver { match (before, after, bytes) { (None, None, None) if unchanged => return Ok(StableRead::Absent), (Some(mut before), Some(after), Some(bytes)) if unchanged => { - let stable = before.stamp == after.stamp; + let stable = before.stamp == after.stamp && before.native == after.native; #[cfg(unix)] let stable = stable && before.device == after.device; if stable { @@ -1685,9 +1726,41 @@ impl CheckoutDriver { access: &ViewAccess, base: Arc, ) -> Result<(Snapshot, u64), PlaneError> { + if let Some(token) = self + .advisory_token + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take() + { + let route = self.owner.v3_route(); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64; + // Token rejection is advisory; every outcome follows the same + // verification path. Tree-base warming is not enabled yet. + let _ = super::registry::consume_view_token( + &route.storage, + &route.family, + &token, + self.owner.root(), + now, + ); + } + self.v3_reconcile.store( + base.v3_route().is_some(), + std::sync::atomic::Ordering::Release, + ); SiblingLoader::reconciled_snapshot(self, access, base) } + pub fn set_view_token(&self, token: String) { + *self + .advisory_token + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(token); + } + pub fn take_revalidated_job(&self) -> bool { self.retry.swap(false, std::sync::atomic::Ordering::AcqRel) } @@ -1727,6 +1800,7 @@ impl CheckoutDriver { epoch: delta.epoch(), version: delta.version(), intent_version: delta.snapshot().intent_version(), + verification: None, }); Self { owner, @@ -1739,6 +1813,8 @@ impl CheckoutDriver { observed: Default::default(), watcher_cookies: Default::default(), delivery: Default::default(), + v3_reconcile: Default::default(), + advisory_token: Default::default(), #[cfg(test)] read_hook: Default::default(), installed: std::sync::Mutex::new(Arc::new(InstalledBundle { @@ -1823,6 +1899,10 @@ impl CheckoutDriver { epoch: delta.epoch(), version: delta.version(), intent_version: delta.snapshot().intent_version(), + verification: installed + .snapshot + .install_cut() + .and_then(|cut| cut.verification.clone()), }); } @@ -1849,6 +1929,10 @@ impl CheckoutDriver { epoch: delta.epoch(), version: delta.version(), intent_version: delta.snapshot().intent_version(), + verification: installed + .snapshot + .install_cut() + .and_then(|cut| cut.verification.clone()), }); } @@ -1970,8 +2054,42 @@ impl FirstLoadDriver for CheckoutDriver { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .clone(); + let v3 = self.v3_reconcile.load(std::sync::atomic::Ordering::Acquire) + || installed.generation().v3_route().is_some(); + let guard = if v3 { + Some(Arc::new( + super::verification::BindGuard::capture(self.owner.root(), None) + .map_err(|e| SiblingLoader::error(e.to_string()))?, + )) + } else { + None + }; + let route = self.owner.v3_route(); + let durable = if v3 && previous.entries.is_empty() { + super::base_refs::open_foreign_v3( + &route.storage, + &route.family, + &route.scope, + &self.producers, + ) + .ok() + .flatten() + } else { + None + }; + let persisted = durable + .as_ref() + .and_then(|generation| { + super::verification::read_evidence( + &route.member_dir(), + generation.name(), + guard.as_ref().unwrap(), + ) + }) + .unwrap_or_default(); let mut entries = BTreeMap::new(); let mut stats = BTreeMap::new(); + let mut evidence = BTreeMap::new(); #[cfg(test)] { self.work.lock().unwrap().walks += 1; @@ -1986,12 +2104,7 @@ impl FirstLoadDriver for CheckoutDriver { .map_err(|error| SiblingLoader::error(error.to_string()))?; // Round down to the second so coarse filesystem timestamps in the // observation's second are also treated as racily clean. - let recorded_ns = u128::from( - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs(), - ) * 1_000_000_000; + let recorded_ns = super::verification::now_ns(); let delivery_seq = self .delivery .lock() @@ -2012,11 +2125,79 @@ impl FirstLoadDriver for CheckoutDriver { .get(&path) .copied() .unwrap_or(0); - if cached.reusable(current) && delivered <= cached.delivery_seq { + let policy_matches = !v3 + || guard.as_ref().is_some_and(|guard| { + guard.policy.reusable + && previous.guard.as_ref().is_some_and(|old| { + old.policy == guard.policy && old.root == guard.root + }) + }); + let keys_present = old.attachments.values().all(|a| { + a.downcast_ref::() + .is_none_or(|cached| v3_contains(&route, &cached.0)) + }); + if cached.reusable(current) + && delivered <= cached.delivery_seq + && policy_matches + && keys_present + { let mut entry = old.clone(); entry.seq = revision; entries.insert(path.clone(), entry); - stats.insert(path, cached.clone()); + stats.insert(path.clone(), cached.clone()); + if let Some(record) = previous.evidence.get(&path) { + evidence.insert(path.clone(), record.clone()); + } + continue; + } + } + } + if !pending.contains(&path) && guard.as_ref().is_some_and(|g| g.policy.reusable) { + if let (Some(record), Some(current), Some(generation)) = + (persisted.get(&path), before.as_ref(), durable.as_ref()) + { + if record.stat.reusable( + current.native.as_ref().unwrap_or(&record.stat), + record.observed_ns, + ) && current.native.is_some() + && record.content().is_some_and(|content| { + generation.disk_state(&path) + == (super::snapshot::DiskState::Present { + content, + size: record.size, + }) + }) + && self.planes.iter().filter(|p| p.applies_to(&path)).all(|p| { + record + .key(p.plane()) + .is_some_and(|key| v3_contains(&route, &key)) + }) + && self + .delivery + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .paths + .get(&path) + .copied() + .unwrap_or(0) + == 0 + { + let mut entry = LiveEntry::new(generation.disk_state(&path), revision); + entry.reconciliation = super::readiness::ReconciledDelta::Reconciled; + for plane in &self.planes { + if plane.applies_to(&path) { + let key = record.key(plane.plane()).unwrap(); + entry.attachments.insert( + plane.plane(), + plane.cached_attachment(record.content().unwrap(), key), + ); + } + } + let mut current = current.clone(); + current.recorded_ns = record.observed_ns; + stats.insert(path.clone(), current); + entries.insert(path.clone(), entry); + evidence.insert(path, record.clone()); continue; } } @@ -2033,19 +2214,59 @@ impl FirstLoadDriver for CheckoutDriver { } }; stats.insert(path.clone(), evidence); - let mut entry = LiveEntry::new(super::snapshot::DiskState::of_bytes(&bytes), revision); + let content = crate::blob_store::v2::ContentHash::of(&bytes); + #[cfg(test)] + { + self.work.lock().unwrap().content_hash_bytes += bytes.len(); + } + let mut entry = LiveEntry::new( + super::snapshot::DiskState::Present { + content, + size: bytes.len() as u64, + }, + revision, + ); entry.reconciliation = super::readiness::ReconciledDelta::Reconciled; + let mut keys = BTreeMap::new(); for plane in &self.planes { if plane.applies_to(&path) { + let key = plane.key(&path, content); + if v3 && key.is_some_and(|key| v3_contains(&route, &key)) { + let key = key.unwrap(); + keys.insert(format!("{:?}", plane.plane()), key.to_hex()); + entry + .attachments + .insert(plane.plane(), plane.cached_attachment(content, key)); + continue; + } #[cfg(test)] { self.work.lock().unwrap().attachments += 1; } - entry - .attachments - .insert(plane.plane(), plane.attachment(&path, &bytes)?); + let attachment = plane.attachment(&path, &bytes)?; + if let Some(key) = key.or_else(|| plane.attachment_key(&attachment)) { + keys.insert(format!("{:?}", plane.plane()), key.to_hex()); + } + entry.attachments.insert(plane.plane(), attachment); } } + if let (Some(guard), Some(stat)) = + (&guard, stats.get(&path).and_then(|s| s.native.clone())) + { + evidence.insert( + path.clone(), + super::verification::Evidence { + version: 1, + stat, + observed_ns: recorded_ns, + content: content.to_hex(), + size: bytes.len() as u64, + git_oid: None, + policy: guard.policy.digest.clone(), + keys, + }, + ); + } entries.insert(path, entry); } *self @@ -2054,11 +2275,17 @@ impl FirstLoadDriver for CheckoutDriver { .unwrap_or_else(std::sync::PoisonError::into_inner) = ObservedCheckout { entries: entries.clone(), stats, + evidence, + guard: guard.clone(), }; + let mut cut = installed.install_cut().cloned(); + if let Some(cut) = &mut cut { + cut.verification = guard; + } Ok(ReconciledCheckout { revision, entries, - cut: installed.install_cut().cloned(), + cut, readiness: installed.readiness_patches().clone(), publication_base: Some(publication_base), }) @@ -2155,6 +2382,7 @@ impl FirstLoadDriver for CheckoutDriver { epoch: successor.epoch(), version: successor.version(), intent_version: successor.snapshot().intent_version(), + verification: cut.verification.clone(), }); if snapshot.reconciled_epoch().is_some() { if let Some(base) = snapshot.publication_base() { @@ -2575,9 +2803,30 @@ impl CheckoutDriver { super::base_refs::BaseRefError::Io(std::io::Error::other(error.reason)) })?; } + if let Some(guard) = snapshot + .install_cut() + .and_then(|cut| cut.verification.as_ref()) + { + let evidence = self + .observed + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .evidence + .clone(); + super::verification::write_evidence(&route, published, guard, &evidence) + .map_err(super::base_refs::BaseRefError::Io)?; + } Ok(()) }, &mut || { + if let Some(guard) = snapshot + .install_cut() + .and_then(|cut| cut.verification.as_ref()) + { + guard + .validate(self.owner.root()) + .map_err(super::base_refs::BaseRefError::Io)?; + } self.validate_cut(access, snapshot, 0) .map_err(|error| super::base_refs::BaseRefError::Unavailable(error.reason)) }, @@ -3822,6 +4071,30 @@ impl CompositePlane for CallgraphBridge { use super::contracts::PlaneAdapter; self.adapter.applies_to(path) } + fn key( + &self, + path: &RelPath, + content: crate::blob_store::v2::ContentHash, + ) -> Option { + let language = if super::assembly::is_resolution_input(path.as_bytes()) { + "config".to_owned() + } else { + let path = std::str::from_utf8(path.as_bytes()).ok()?; + format!( + "{:?}", + crate::parser::detect_language(std::path::Path::new(path))? + ) + .to_lowercase() + }; + Some(crate::blob_store::v2::FamilyKey::from( + &crate::blob_store::CallgraphKey::from_source_digest( + *content.as_bytes(), + language, + crate::blob_store::CALLGRAPH_PRODUCER_VERSION, + ) + .full_key(), + )) + } fn attachment( &self, path: &RelPath, @@ -3995,6 +4268,25 @@ impl CompositePlane for CallgraphBridge { { continue; } + if let Some(cached) = entry + .attachments + .get(&FamilyPlane::Callgraph) + .and_then(|a| a.downcast_ref::()) + { + live.protect(&[cached.0]) + .map_err(|e| error(e.to_string()))?; + if !v3_contains(route, &cached.0) { + return Err(error( + "cached callgraph row disappeared before protection".into(), + )); + } + if let Some(super::manifest_v2::EntryV2::Regular { planes, .. }) = + delta.get_mut(path) + { + planes.callgraph = Some(super::readiness::PlaneState::ready(&cached.0)); + } + continue; + } let attachment = entry .attachments .get(&FamilyPlane::Callgraph) @@ -4657,6 +4949,19 @@ impl CompositePlane for TrigramBridge { fn applies_to(&self, path: &RelPath) -> bool { !path.is_synthetic() } + fn key( + &self, + _: &RelPath, + content: crate::blob_store::v2::ContentHash, + ) -> Option { + Some( + crate::blob_store::v2::TrigramKey { + content, + policy: self.policy, + } + .family_key(), + ) + } fn attachment( &self, _: &RelPath, @@ -4698,7 +5003,7 @@ impl CompositePlane for TrigramBridge { } fn materialize_v3( &self, - owner: &super::registry::ViewRegistration, + _owner: &super::registry::ViewRegistration, route: &super::snapshot::V3Route, _: &str, _: &Arc, @@ -4706,15 +5011,53 @@ impl CompositePlane for TrigramBridge { delta: &mut super::manifest_v2::ManifestV2, live: &mut crate::pins::LivePin, ) -> Result<(), PlaneError> { - super::trigram::materialize_v3_observed( - &route.storage, - &route.family, - owner.root(), - delta, - observed, - live, - self.policy, - ) + let mut keys = Vec::new(); + let mut payloads = Vec::new(); + for (path, entry) in delta.entries() { + let super::manifest_v2::EntryV2::Regular { content, .. } = entry else { + continue; + }; + let key = self.key(path, *content).unwrap(); + live.protect(&[key]) + .map_err(|e| SiblingLoader::error(e.to_string()))?; + if !v3_contains(route, &key) { + let attachment = observed + .get(path) + .filter(|e| e.disk == super::snapshot::DiskState::in_generation(delta, path)) + .and_then(|e| e.attachments.get(&FamilyPlane::Trigram)) + .and_then(|a| a.downcast_ref::()) + .filter(|a| a.content == *content && a.policy == self.policy.fingerprint()) + .ok_or_else(|| { + SiblingLoader::error( + "trigram row missing; reconcile source before retrying", + ) + })?; + payloads.push((key, attachment.payload.encode())); + } + keys.push((path.clone(), key)); + } + if !payloads.is_empty() { + let request = payloads + .iter() + .map(|(_, p)| p.len() as u64) + .sum::() + .saturating_mul(2) + .saturating_add(1 << 20); + crate::blob_store::v3::publish(&route.storage, &route.family, request, None, |p| { + for (key, payload) in &payloads { + p.put(key, payload, FamilyPlane::Trigram.payload_schema())?; + } + Ok(()) + }) + .map_err(|e| SiblingLoader::error(e.to_string()))?; + } + for (path, key) in keys { + if let Some(super::manifest_v2::EntryV2::Regular { planes, .. }) = delta.get_mut(&path) + { + planes.trigram = Some(super::readiness::PlaneState::ready(&key)); + } + } + Ok(()) } } @@ -4730,7 +5073,9 @@ impl CheckoutDriver { } match std::fs::symlink_metadata(path) { Ok(metadata) if metadata.is_file() => { - Ok(Some(ObservedStat::new(&metadata, recorded_ns))) + let mut stat = ObservedStat::new(&metadata, recorded_ns); + stat.native = super::verification::NativeStat::read(path).ok(); + Ok(Some(stat)) } Ok(_) => Ok(None), Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), @@ -4814,6 +5159,7 @@ impl super::intent::WriteIntentListener for CheckoutDriver { epoch: live.epoch(), version: live.version(), intent_version: live.snapshot().intent_version(), + verification: None, }); } let snapshot = self diff --git a/crates/aft/src/views/mod.rs b/crates/aft/src/views/mod.rs index 542f4b6594..67e7433e8e 100644 --- a/crates/aft/src/views/mod.rs +++ b/crates/aft/src/views/mod.rs @@ -64,6 +64,7 @@ pub mod semantic; pub mod semantic_arena; pub mod semantic_runtime; pub mod trigram; +mod verification; #[cfg(test)] mod dispatch_parity_probe; diff --git a/crates/aft/src/views/registry.rs b/crates/aft/src/views/registry.rs index d0aee2b097..5ed7326166 100644 --- a/crates/aft/src/views/registry.rs +++ b/crates/aft/src/views/registry.rs @@ -1410,6 +1410,191 @@ fn now_ms() -> u64 { crate::pins::now_ms() } +pub const VIEW_TOKEN_TTL_MS: u64 = 600_000; + +#[derive(Clone, Debug, serde::Serialize)] +pub struct ViewToken { + pub token: String, + pub expires_ms: u64, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum TokenUse { + Accepted { expected_tree: String }, + Forged, + Expired, + WrongRoot, + WrongTree, + IdentityMismatch, + Replayed, +} + +/// Canonicalize an existing parent without requiring the future checkout to +/// exist. A later symlink substitution changes the canonical consumption root. +pub fn canonical_token_root(path: &Path) -> std::io::Result { + if !path.is_absolute() + || path + .components() + .any(|c| matches!(c, std::path::Component::ParentDir)) + { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "token root must be absolute without parent components", + )); + } + let mut parent = path; + let mut suffix = Vec::new(); + while !parent.try_exists()? { + suffix.push( + parent + .file_name() + .ok_or_else(|| std::io::Error::other("no existing root ancestor"))? + .to_os_string(), + ); + parent = parent + .parent() + .ok_or_else(|| std::io::Error::other("no existing root ancestor"))?; + } + let mut result = std::fs::canonicalize(parent)?; + for component in suffix.into_iter().rev() { + result.push(component); + } + Ok(result) +} + +fn token_store_error(error: crate::blob_store::v3::Error) -> RegistryError { + RegistryError::Io(std::io::Error::other(error)) +} + +/// Minting is advisory and creates only the registered v3 token row. No base, +/// source bytes, view pointer or legacy row is written. +pub fn mint_view_token( + storage: &Path, + family: &str, + root: &Path, + tree: &str, + now_ms: u64, +) -> RegistryResult { + if !matches!(tree.len(), 40 | 64) + || !tree + .bytes() + .all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()) + { + return Err(RegistryError::InvalidScope( + "expected tree must be a lowercase Git object ID".into(), + )); + } + let root = canonical_token_root(root)?; + let root_bytes = encode_root(&root).ok_or_else(|| { + RegistryError::InvalidScope("token root cannot be represented exactly".into()) + })?; + let mut nonce = [0u8; 32]; + getrandom::fill(&mut nonce) + .map_err(|error| RegistryError::Io(std::io::Error::other(error.to_string())))?; + let token = crate::blob_store::v2::to_hex(&nonce); + let expires_ms = now_ms + .checked_add(VIEW_TOKEN_TTL_MS) + .ok_or_else(|| RegistryError::InvalidScope("token clock overflow".into()))?; + let lock = + crate::blob_store::v3::ReservationLock::acquire(storage).map_err(token_store_error)?; + let mut registry = crate::blob_store::v3::open_verification_registry(storage, family, &lock) + .map_err(token_store_error)?; + registry.with_barrier(|tx| { + tx.execute("DELETE FROM view_tokens WHERE expires_ms<=?1", [now_ms])?; + tx.execute("INSERT INTO view_tokens(nonce,root,expected_tree,minted_ms,expires_ms) VALUES(?1,?2,?3,?4,?5)", params![token, root_bytes, tree, now_ms, expires_ms])?; + Ok(()) + }).map_err(token_store_error)?; + Ok(ViewToken { token, expires_ms }) +} + +/// A token never supplies a freshness proof. Even an accepted token only +/// suggests which tree to pre-warm; callers must still reconcile the checkout. +pub fn consume_view_token( + storage: &Path, + family: &str, + token: &str, + root: &Path, + now_ms: u64, +) -> RegistryResult { + let root = canonical_token_root(root)?; + let directory = match super::verification::gitdir(&root) { + Ok(directory) => directory, + Err(_) => return Ok(TokenUse::IdentityMismatch), + }; + let identities = match ( + super::verification::NativeStat::read(&root), + super::verification::NativeStat::read(&directory), + ) { + (Ok(root), Ok(gitdir)) => (root.identity, (directory, gitdir.identity)), + _ => return Ok(TokenUse::IdentityMismatch), + }; + let queries = + crate::git_index_reader::GitQueries::new(root.clone(), std::time::Duration::from_secs(2)); + let tree = queries.head_tree()?; + if !tree.status.success() { + return Ok(TokenUse::WrongTree); + } + let tree = String::from_utf8_lossy(&tree.stdout).trim().to_owned(); + let bound_root = serde_json::to_vec(&identities.0) + .map_err(|e| RegistryError::Io(std::io::Error::other(e)))?; + let bound_gitdir = serde_json::to_vec(&identities.1) + .map_err(|e| RegistryError::Io(std::io::Error::other(e)))?; + consume_token_identity( + storage, + family, + token, + &root, + &tree, + &bound_root, + &bound_gitdir, + now_ms, + ) +} + +#[allow(clippy::too_many_arguments)] +fn consume_token_identity( + storage: &Path, + family: &str, + token: &str, + root: &Path, + tree: &str, + root_identity: &[u8], + gitdir_identity: &[u8], + now_ms: u64, +) -> RegistryResult { + // A forged token does not create a family or advance its reader floor. + if !crate::blob_store::v3::family_dir(storage, family) + .map_err(token_store_error)? + .join("members.sqlite") + .try_exists()? + { + return Ok(TokenUse::Forged); + } + let lock = + crate::blob_store::v3::ReservationLock::acquire(storage).map_err(token_store_error)?; + let mut registry = + crate::blob_store::v3::RegistryWriter::open_for_barrier(storage, family, &lock) + .map_err(token_store_error)?; + if registry.registry_version().map_err(token_store_error)? + < crate::blob_store::v3::REGISTRY_VERIFICATION_VERSION + { + return Ok(TokenUse::Forged); + } + registry.with_barrier(|tx| { + type Row = (Vec, String, u64, u64, Option>, Option>, Option); + let row: Option = tx.query_row("SELECT root,expected_tree,minted_ms,expires_ms,bound_root,bound_gitdir,consumed_ms FROM view_tokens WHERE nonce=?1", [token], |r| Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?,r.get(6)?))).optional()?; + let Some((expected_root, expected_tree, minted, expires, bound_root, bound_gitdir, consumed)) = row else { return Ok(TokenUse::Forged); }; + if now_ms < minted || now_ms >= expires { return Ok(TokenUse::Expired); } + if Some(expected_root) != encode_root(root) { return Ok(TokenUse::WrongRoot); } + if expected_tree != tree { return Ok(TokenUse::WrongTree); } + if consumed.is_some() { + return Ok(if bound_root.as_deref() == Some(root_identity) && bound_gitdir.as_deref() == Some(gitdir_identity) { TokenUse::Replayed } else { TokenUse::IdentityMismatch }); + } + let changed = tx.execute("UPDATE view_tokens SET bound_root=?1,bound_gitdir=?2,consumed_ms=?3 WHERE nonce=?4 AND consumed_ms IS NULL", params![root_identity, gitdir_identity, now_ms, token])?; + Ok(if changed == 1 { TokenUse::Accepted { expected_tree } } else { TokenUse::Replayed }) + }).map_err(token_store_error) +} + #[cfg(test)] mod tests { use super::*; @@ -1688,3 +1873,198 @@ mod tests { ); } } + +#[cfg(test)] +mod view_token_tests { + use super::*; + const TREE: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + + #[test] + fn view_token_mints_before_root_exists_and_binds_both_identities_once() { + let _serial = crate::blob_store::v3::serial_for_test(); + let storage = tempfile::tempdir().unwrap(); + let parent = tempfile::tempdir().unwrap(); + let root = std::fs::canonicalize(parent.path()).unwrap().join("future"); + let minted = mint_view_token(storage.path(), "family", &root, TREE, 10).unwrap(); + assert!(!root.exists()); + let ledger = + crate::blob_store::v3::LedgerReader::open_existing(storage.path(), "family").unwrap(); + assert!(ledger.read(|c| Ok(c.query_row("SELECT bound_root IS NULL AND bound_gitdir IS NULL AND consumed_ms IS NULL FROM view_tokens", [], |r| r.get::<_, bool>(0))?)).unwrap()); + assert_eq!( + consume_token_identity( + storage.path(), + "family", + &minted.token, + &root, + TREE, + b"root-id", + b"gitdir-id", + 11 + ) + .unwrap(), + TokenUse::Accepted { + expected_tree: TREE.into() + } + ); + assert_eq!( + consume_token_identity( + storage.path(), + "family", + &minted.token, + &root, + TREE, + b"root-id", + b"gitdir-id", + 12 + ) + .unwrap(), + TokenUse::Replayed + ); + for (root_id, gitdir_id) in [ + (b"replacement".as_slice(), b"gitdir-id".as_slice()), + (b"root-id".as_slice(), b"replacement".as_slice()), + ] { + assert_eq!( + consume_token_identity( + storage.path(), + "family", + &minted.token, + &root, + TREE, + root_id, + gitdir_id, + 12 + ) + .unwrap(), + TokenUse::IdentityMismatch + ); + } + let floor = crate::reader_floor::read(storage.path()).unwrap().unwrap(); + assert_eq!(floor.stores["views_v3_registry"], 3); + assert_eq!(floor.stores["views_v3_store"], 1); + assert!(!storage.path().join("blobs/family").exists()); + assert!(!storage.path().join("blobs/v2").exists()); + } + + #[test] + fn view_token_concurrent_first_consumption_has_exactly_one_winner() { + let _serial = crate::blob_store::v3::serial_for_test(); + let storage = tempfile::tempdir().unwrap(); + let parent = tempfile::tempdir().unwrap(); + let root = std::fs::canonicalize(parent.path()).unwrap().join("future"); + let token = mint_view_token(storage.path(), "family", &root, TREE, 1).unwrap(); + let start = std::sync::Arc::new(std::sync::Barrier::new(2)); + let outcomes = std::thread::scope(|scope| { + let jobs: Vec<_> = (0..2) + .map(|_| { + let start = &start; + let token = &token.token; + let root = &root; + let storage = storage.path(); + scope.spawn(move || { + start.wait(); + consume_token_identity( + storage, "family", token, root, TREE, b"root", b"gitdir", 2, + ) + .unwrap() + }) + }) + .collect(); + jobs.into_iter() + .map(|j| j.join().unwrap()) + .collect::>() + }); + assert_eq!( + outcomes + .iter() + .filter(|r| matches!(r, TokenUse::Accepted { .. })) + .count(), + 1 + ); + assert_eq!( + outcomes + .iter() + .filter(|r| **r == TokenUse::Replayed) + .count(), + 1 + ); + } + + #[test] + fn view_token_forged_expired_wrong_root_and_wrong_tree_never_consume() { + let _serial = crate::blob_store::v3::serial_for_test(); + let storage = tempfile::tempdir().unwrap(); + let parent = tempfile::tempdir().unwrap(); + let root = std::fs::canonicalize(parent.path()).unwrap().join("future"); + assert_eq!( + consume_token_identity( + storage.path(), + "family", + "forged", + &root, + TREE, + b"r", + b"g", + 1 + ) + .unwrap(), + TokenUse::Forged + ); + assert!(!storage.path().join("blobs/v3").exists()); + let token = mint_view_token(storage.path(), "family", &root, TREE, 10).unwrap(); + for (nonce, path, tree, now, expected) in [ + ("forged", root.clone(), TREE, 11, TokenUse::Forged), + ( + &token.token, + root.join("wrong"), + TREE, + 11, + TokenUse::WrongRoot, + ), + ( + &token.token, + root.clone(), + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + 11, + TokenUse::WrongTree, + ), + ( + &token.token, + root.clone(), + TREE, + token.expires_ms, + TokenUse::Expired, + ), + (&token.token, root.clone(), TREE, 9, TokenUse::Expired), + ] { + assert_eq!( + consume_token_identity( + storage.path(), + "family", + nonce, + &path, + tree, + b"r", + b"g", + now + ) + .unwrap(), + expected + ); + } + assert!(matches!( + consume_token_identity( + storage.path(), + "family", + &token.token, + &root, + TREE, + b"r", + b"g", + 11 + ) + .unwrap(), + TokenUse::Accepted { .. } + )); + } +} diff --git a/crates/aft/src/views/semantic.rs b/crates/aft/src/views/semantic.rs index d5c76fdd69..ae753850d9 100644 --- a/crates/aft/src/views/semantic.rs +++ b/crates/aft/src/views/semantic.rs @@ -2613,6 +2613,22 @@ impl super::first_load::CompositePlane for SemanticPlane { fn applies_to(&self, rel_path: &RelPath) -> bool { applies_to(rel_path) } + fn attachment_key(&self, attachment: &super::snapshot::PlaneAttachment) -> Option { + attachment + .downcast_ref::() + .map(|a| a.key) + } + fn cached_attachment( + &self, + content: ContentHash, + key: FamilyKey, + ) -> super::snapshot::PlaneAttachment { + Arc::new(SemanticAttachment { + content, + producer: self.producer.id(), + key, + }) + } fn attachment( &self, diff --git a/crates/aft/src/views/snapshot.rs b/crates/aft/src/views/snapshot.rs index e89db24033..6a3069034e 100644 --- a/crates/aft/src/views/snapshot.rs +++ b/crates/aft/src/views/snapshot.rs @@ -1314,6 +1314,7 @@ mod mutation_tests { epoch: live.epoch(), version: live.version(), intent_version: 0, + verification: None, }; let changed = RelPath::new(b"changed.rs".to_vec()).unwrap(); let transient = RelPath::new(b"transient.rs".to_vec()).unwrap(); @@ -1376,6 +1377,7 @@ mod mutation_tests { epoch: live.epoch(), version: live.version(), intent_version: 0, + verification: None, }; let mut walked = LiveDelta::new(live.base().clone()); walked.apply(file.clone(), LiveEntry::new(DiskState::of_bytes(b"B"), 2)); @@ -1410,6 +1412,7 @@ mod mutation_tests { epoch: live.epoch(), version: live.version(), intent_version: 0, + verification: None, }; let mut walked = LiveDelta::new(live.base().clone()); walked.apply(file.clone(), LiveEntry::new(DiskState::of_bytes(b"B"), 2)); @@ -1684,6 +1687,7 @@ pub struct InstallCut { pub epoch: u64, pub version: u64, pub intent_version: u64, + pub(crate) verification: Option>, } /// A lease on resident adapters. Query snapshots retain this lease, so removing diff --git a/crates/aft/src/views/verification.rs b/crates/aft/src/views/verification.rs new file mode 100644 index 0000000000..eb0c74874b --- /dev/null +++ b/crates/aft/src/views/verification.rs @@ -0,0 +1,689 @@ +//! Fail-closed metadata and policy evidence for checkout reconciliation. +//! Git index tuples alone never authorize reuse of working-tree bytes. + +use std::collections::BTreeMap; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use serde::{Deserialize, Serialize}; + +use super::RelPath; +use crate::blob_store::v2::{ContentHash, FamilyKey, FamilyPlane}; + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub(crate) enum FileIdentity { + Unix { + dev: u64, + ino: u64, + birth: Option, + }, + Windows { + volume: u64, + id: [u8; 16], + creation: i64, + }, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub(crate) struct NativeStat { + pub identity: FileIdentity, + pub size: u64, + pub modified_ns: i128, + pub changed_ns: i128, + pub mode: u32, +} + +impl NativeStat { + pub fn read(path: &Path) -> io::Result { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let m = fs::symlink_metadata(path)?; + Ok(Self { + identity: FileIdentity::Unix { + dev: m.dev(), + ino: m.ino(), + birth: m + .created() + .ok() + .and_then(|t| t.duration_since(UNIX_EPOCH).ok()) + .map(|d| d.as_nanos()), + }, + size: m.len(), + modified_ns: i128::from(m.mtime()) * 1_000_000_000 + i128::from(m.mtime_nsec()), + changed_ns: i128::from(m.ctime()) * 1_000_000_000 + i128::from(m.ctime_nsec()), + mode: m.mode(), + }) + } + #[cfg(windows)] + { + windows_stat(path) + } + #[cfg(not(any(unix, windows)))] + { + let _ = path; + Err(io::Error::new( + io::ErrorKind::Unsupported, + "stable file identity unavailable", + )) + } + } + pub fn reusable(&self, current: &Self, observed_ns: u128) -> bool { + self == current && self.modified_ns >= 0 && (self.modified_ns as u128) < observed_ns + } +} + +#[cfg(windows)] +fn windows_stat(path: &Path) -> io::Result { + use std::os::windows::{fs::OpenOptionsExt, io::AsRawHandle}; + use windows_sys::Win32::Storage::FileSystem::{ + FileBasicInfo, FileIdInfo, GetFileInformationByHandleEx, GetVolumeInformationByHandleW, + FILE_BASIC_INFO, FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, FILE_ID_INFO, + FILE_READ_ATTRIBUTES, + }; + let file = fs::OpenOptions::new() + .read(true) + .access_mode(FILE_READ_ATTRIBUTES) + .share_mode(7) + .custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT) + .open(path)?; + let handle = file.as_raw_handle(); + let mut id: FILE_ID_INFO = unsafe { std::mem::zeroed() }; + let mut basic: FILE_BASIC_INFO = unsafe { std::mem::zeroed() }; + let mut filesystem = [0u16; 32]; + // All fields come from one metadata-only handle. A 64-bit file index is + // insufficient on ReFS, and creation time is not a substitute for ChangeTime. + let ok = unsafe { + GetFileInformationByHandleEx( + handle, + FileIdInfo, + &mut id as *mut _ as _, + std::mem::size_of::() as u32, + ) != 0 + && GetFileInformationByHandleEx( + handle, + FileBasicInfo, + &mut basic as *mut _ as _, + std::mem::size_of::() as u32, + ) != 0 + && GetVolumeInformationByHandleW( + handle, + std::ptr::null_mut(), + 0, + std::ptr::null_mut(), + std::ptr::null_mut(), + std::ptr::null_mut(), + filesystem.as_mut_ptr(), + filesystem.len() as u32, + ) != 0 + }; + if !ok { + return Err(io::Error::last_os_error()); + } + let end = filesystem + .iter() + .position(|c| *c == 0) + .unwrap_or(filesystem.len()); + let name = String::from_utf16_lossy(&filesystem[..end]); + if !matches!(name.as_str(), "NTFS" | "ReFS") { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + "stat reuse requires NTFS or ReFS", + )); + } + let unix_ns = |ticks: i64| (i128::from(ticks) - 116_444_736_000_000_000i128) * 100; + Ok(NativeStat { + identity: FileIdentity::Windows { + volume: id.VolumeSerialNumber, + id: id.FileId.Identifier, + creation: basic.CreationTime, + }, + size: file.metadata()?.len(), + modified_ns: unix_ns(basic.LastWriteTime), + changed_ns: unix_ns(basic.ChangeTime), + mode: basic.FileAttributes, + }) +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub(crate) struct Evidence { + pub version: u32, + pub stat: NativeStat, + pub observed_ns: u128, + pub content: String, + pub size: u64, + pub git_oid: Option, + pub policy: String, + pub keys: BTreeMap, +} + +impl Evidence { + pub fn content(&self) -> Option { + ContentHash::from_hex(&self.content) + } + pub fn key(&self, plane: FamilyPlane) -> Option { + let name = format!("{plane:?}"); + Some(FamilyKey::new( + plane, + crate::blob_store::v2::parse_hex32(self.keys.get(&name)?)?, + )) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct Policy { + pub digest: String, + /// Missing or unscopable conversion sources forbid stat-based byte reuse. + pub reusable: bool, +} + +fn metadata_source(path: &Path, hash: &mut blake3::Hasher) -> io::Result<()> { + let name = path.as_os_str().as_encoded_bytes(); + hash.update(&(name.len() as u64).to_le_bytes()); + hash.update(name); + match fs::read(path) { + Ok(bytes) => { + hash.update(&[1]); + hash.update(&(bytes.len() as u64).to_le_bytes()); + hash.update(&bytes); + } + Err(e) if e.kind() == io::ErrorKind::NotFound => { + hash.update(&[0]); + } + Err(e) => return Err(e), + } + Ok(()) +} + +impl Policy { + pub fn capture(root: &Path, metadata: &[PathBuf]) -> io::Result { + let mut hash = blake3::Hasher::new(); + hash.update(b"aft/views-v3/conversion-policy/1"); + let queries = + crate::git_index_reader::GitQueries::new(root.to_path_buf(), Duration::from_secs(2)); + let output = queries.config()?; + if !output.status.success() { + return Ok(Self { + digest: "non-git".into(), + reusable: false, + }); + } + hash.update(&output.stdout); + let mut reusable = true; + // Config --show-origin records include targets and worktree config. + // Hash their bytes as well as the effective values, including empty files. + for item in output.stdout.split(|b| *b == 0) { + if let Some(path) = item.strip_prefix(b"file:") { + let path = path_from_bytes(path)?; + metadata_source( + &if path.is_absolute() { + path + } else { + root.join(path) + }, + &mut hash, + )?; + } + let lower = String::from_utf8_lossy(item).to_lowercase(); + if lower.starts_with("filter.") + || lower.starts_with("core.autocrlf\n") && lower != "core.autocrlf\nfalse" + || lower == "core.eol\ncrlf" + { + reusable = false; + } + if let Some(path) = item.strip_prefix(b"core.attributesfile\n") { + metadata_source(&path_from_bytes(path)?, &mut hash)?; + reusable = false; + } + } + for path in metadata { + metadata_source(path, &mut hash)?; + if path.exists() + && path + .file_name() + .is_some_and(|name| name == ".gitattributes" || name == "attributes") + { + // Until path-scoped attribute provenance is available, every + // attribute source is conservatively an unscopable conversion. + reusable = false; + } + } + if let Some(home) = std::env::var_os("HOME").or_else(|| std::env::var_os("USERPROFILE")) { + for relative in [".gitattributes", ".config/git/attributes"] { + let path = PathBuf::from(&home).join(relative); + metadata_source(&path, &mut hash)?; + reusable &= !path.exists(); + } + } + let system = Path::new("/etc/gitattributes"); + metadata_source(system, &mut hash)?; + reusable &= !system.exists(); + Ok(Self { + digest: hash.finalize().to_hex().to_string(), + reusable, + }) + } +} + +pub(crate) fn path_from_bytes(bytes: &[u8]) -> io::Result { + #[cfg(unix)] + { + use std::os::unix::ffi::OsStringExt; + Ok(std::ffi::OsString::from_vec(bytes.to_vec()).into()) + } + #[cfg(not(unix))] + { + std::str::from_utf8(bytes) + .map(PathBuf::from) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) + } +} + +pub(crate) fn now_ns() -> u128 { + u128::from( + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(), + ) * 1_000_000_000 +} + +/// Enumerate policy inputs with metadata only. Source files are never opened. +pub(crate) fn policy_inputs(root: &Path) -> io::Result> { + fn walk(root: &Path, out: &mut Vec) -> io::Result<()> { + for name in [".gitattributes", ".gitignore", ".ignore"] { + out.push(root.join(name)); + } + for entry in fs::read_dir(root)? { + let entry = entry?; + let kind = entry.file_type()?; + let name = entry.file_name(); + if name == ".git" { + continue; + } + if kind.is_dir() { + walk(&entry.path(), out)?; + } + } + Ok(()) + } + let mut paths = Vec::new(); + walk(root, &mut paths)?; + let marker = root.join(".git"); + let gitdir = if marker.is_dir() { + Some(marker) + } else { + fs::read(&marker) + .ok() + .and_then(|bytes| { + bytes + .strip_prefix(b"gitdir: ") + .map(|p| p.trim_ascii().to_vec()) + }) + .and_then(|p| path_from_bytes(&p).ok()) + .map(|p| if p.is_absolute() { p } else { root.join(p) }) + }; + if let Some(gitdir) = gitdir { + paths.push(gitdir.join("info/attributes")); + paths.push(gitdir.join("info/exclude")); + } + paths.sort(); + Ok(paths) +} + +pub(crate) fn gitdir(root: &Path) -> io::Result { + let marker = root.join(".git"); + if marker.is_dir() { + return fs::canonicalize(marker); + } + let bytes = fs::read(marker)?; + let path = bytes + .strip_prefix(b"gitdir: ") + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "invalid gitdir marker"))?; + let path = path_from_bytes(path.trim_ascii())?; + fs::canonicalize(if path.is_absolute() { + path + } else { + root.join(path) + }) +} + +/// Captured even for strict loads: admission waits cannot transfer an old +/// root, Git state or policy observation to a new pointer publication. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct BindGuard { + pub root: FileIdentity, + pub gitdir: Option<(PathBuf, FileIdentity)>, + pub head_tree: Option>, + pub index: Option<(NativeStat, [u8; 32])>, + pub policy: Policy, + pub inputs: Vec, +} + +impl BindGuard { + pub fn capture(root: &Path, known_inputs: Option<&[PathBuf]>) -> io::Result { + let inputs = match known_inputs { + Some(inputs) => inputs.to_vec(), + None => policy_inputs(root)?, + }; + let policy = Policy::capture(root, &inputs)?; + let root_identity = NativeStat::read(root)?.identity; + let directory = match gitdir(root) { + Ok(path) => Some((path.clone(), NativeStat::read(&path)?.identity)), + Err(e) if e.kind() == io::ErrorKind::NotFound => None, + Err(e) => return Err(e), + }; + let mut head_tree = None; + let mut index = None; + if let Some((directory, _)) = &directory { + let queries = crate::git_index_reader::GitQueries::new( + root.to_path_buf(), + Duration::from_secs(2), + ); + let head = queries.head_tree()?; + if head.status.success() { + head_tree = Some(head.stdout); + } + let path = directory.join("index"); + match NativeStat::read(&path) { + Ok(before) => { + let bytes = fs::read(&path)?; + let after = NativeStat::read(&path)?; + if before != after { + return Err(io::Error::other("Git index changed during verification")); + } + index = Some((before, *blake3::hash(&bytes).as_bytes())); + } + Err(e) if e.kind() == io::ErrorKind::NotFound => {} + Err(e) => return Err(e), + } + } + Ok(Self { + root: root_identity, + gitdir: directory, + head_tree, + index, + policy, + inputs, + }) + } + pub fn validate(&self, root: &Path) -> io::Result<()> { + let current = Self::capture(root, None)?; + if ¤t != self { + return Err(io::Error::other( + "checkout Git, root identity or conversion policy changed", + )); + } + Ok(()) + } +} + +const EVIDENCE_APP: i64 = 0x41465445; +const EVIDENCE_SCHEMA: &str = "CREATE TABLE evidence_meta(singleton INTEGER PRIMARY KEY CHECK(singleton=1), format_version INTEGER NOT NULL, schema_version INTEGER NOT NULL, generation TEXT NOT NULL, root BLOB NOT NULL, policy TEXT NOT NULL); +CREATE TABLE stat_evidence(path BLOB PRIMARY KEY, record BLOB NOT NULL);"; + +pub(crate) fn evidence_path(dir: &Path, generation: &str) -> PathBuf { + dir.join(format!("stat-evidence-{generation}.sqlite")) +} + +/// Writes the evidence before the CAS. Its generation and content digests are +/// checked on read, so an unpublished or obsolete sidecar supplies no proof. +pub(crate) fn write_evidence( + route: &super::snapshot::V3Route, + generation: &str, + guard: &BindGuard, + records: &BTreeMap, +) -> io::Result<()> { + let lock = crate::blob_store::v3::ReservationLock::acquire(&route.storage) + .map_err(io::Error::other)?; + let registry = + crate::blob_store::v3::open_verification_registry(&route.storage, &route.family, &lock) + .map_err(io::Error::other)?; + drop(registry); + drop(lock); + let root = serde_json::to_vec(&guard.root)?; + let dir = route.member_dir(); + let destination = evidence_path(&dir, generation); + let staging = dir.join(format!( + "stat-evidence-{}.staging", + super::base_refs::fresh_nonce() + )); + let result = (|| -> Result<(), Box> { + let mut db = rusqlite::Connection::open(&staging)?; + db.pragma_update(None, "application_id", EVIDENCE_APP)?; + db.pragma_update(None, "user_version", 1)?; + db.pragma_update(None, "journal_mode", "DELETE")?; + db.pragma_update(None, "synchronous", "FULL")?; + db.execute_batch(EVIDENCE_SCHEMA)?; + let tx = db.transaction()?; + tx.execute( + "INSERT INTO evidence_meta VALUES(1,1,1,?1,?2,?3)", + rusqlite::params![generation, root, guard.policy.digest], + )?; + for (path, record) in records { + tx.execute( + "INSERT INTO stat_evidence VALUES(?1,?2)", + rusqlite::params![path.as_bytes(), serde_json::to_vec(record)?], + )?; + } + tx.commit()?; + drop(db); + fs::File::open(&staging)?.sync_all()?; + if destination.exists() { + fs::remove_file(&staging)?; + } else { + fs::rename(&staging, &destination)?; + } + super::sync_directory(&dir)?; + Ok(()) + })(); + if let Err(error) = result { + let _ = fs::remove_file(&staging); + return Err(io::Error::other(error.to_string())); + } + Ok(()) +} + +pub(crate) fn read_evidence( + dir: &Path, + generation: &str, + guard: &BindGuard, +) -> Option> { + let db = rusqlite::Connection::open_with_flags( + evidence_path(dir, generation), + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .ok()?; + if db + .pragma_query_value::(None, "application_id", |r| r.get(0)) + .ok()? + != EVIDENCE_APP + || db + .pragma_query_value::(None, "user_version", |r| r.get(0)) + .ok()? + != 1 + { + return None; + } + let (format, schema, observed_generation, root, policy): (u32, u32, String, Vec, String) = db.query_row( + "SELECT format_version,schema_version,generation,root,policy FROM evidence_meta WHERE singleton=1", [], + |r| Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?)), + ).ok()?; + if (format, schema) != (1, 1) + || observed_generation != generation + || root != serde_json::to_vec(&guard.root).ok()? + || policy != guard.policy.digest + { + return None; + } + let mut statement = db + .prepare("SELECT path,record FROM stat_evidence ORDER BY path") + .ok()?; + let rows = statement + .query_map([], |r| { + Ok((r.get::<_, Vec>(0)?, r.get::<_, Vec>(1)?)) + }) + .ok()?; + let mut result = BTreeMap::new(); + for row in rows { + let (path, bytes) = row.ok()?; + let record: Evidence = serde_json::from_slice(&bytes).ok()?; + if record.version != 1 || record.policy != guard.policy.digest { + return None; + } + result.insert(RelPath::new(path).ok()?, record); + } + Some(result) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn git(root: &Path, args: &[&str]) { + let output = std::process::Command::new("git") + .current_dir(root) + .args(args) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + } + + #[test] + fn stat_evidence_is_generation_root_policy_and_version_paired() { + let _serial = crate::blob_store::v3::serial_for_test(); + let storage = tempfile::tempdir().unwrap(); + let root = tempfile::tempdir().unwrap(); + let file = root.path().join("file.rs"); + fs::write(&file, b"stable").unwrap(); + let guard = BindGuard::capture(root.path(), None).unwrap(); + let route = super::super::snapshot::V3Route { + storage: storage.path().to_owned(), + family: "evidence".into(), + scope: "worker".into(), + }; + fs::create_dir_all(route.member_dir()).unwrap(); + let path = RelPath::new(b"file.rs".to_vec()).unwrap(); + let content = ContentHash::of(b"stable"); + let record = Evidence { + version: 1, + stat: NativeStat::read(&file).unwrap(), + observed_ns: now_ns(), + content: content.to_hex(), + size: 6, + git_oid: None, + policy: guard.policy.digest.clone(), + keys: BTreeMap::from([("Trigram".into(), "ab".repeat(32))]), + }; + write_evidence( + &route, + "g-one", + &guard, + &BTreeMap::from([(path.clone(), record.clone())]), + ) + .unwrap(); + let restored = read_evidence(&route.member_dir(), "g-one", &guard).unwrap(); + assert_eq!(restored[&path].content(), Some(content)); + assert_eq!( + restored[&path].key(FamilyPlane::Trigram).unwrap().to_hex(), + "ab".repeat(32) + ); + assert!(restored[&path].git_oid.is_none()); + assert_eq!( + crate::reader_floor::read(storage.path()) + .unwrap() + .unwrap() + .stores["views_v3_registry"], + 3 + ); + let mut wrong = guard.clone(); + wrong.policy.digest = "changed".into(); + assert!(read_evidence(&route.member_dir(), "g-one", &wrong).is_none()); + wrong = guard.clone(); + wrong.root = FileIdentity::Windows { + volume: 1, + id: [2; 16], + creation: 3, + }; + assert!(read_evidence(&route.member_dir(), "g-one", &wrong).is_none()); + fs::copy( + evidence_path(&route.member_dir(), "g-one"), + evidence_path(&route.member_dir(), "g-two"), + ) + .unwrap(); + assert!(read_evidence(&route.member_dir(), "g-two", &guard).is_none()); + let db = rusqlite::Connection::open(evidence_path(&route.member_dir(), "g-one")).unwrap(); + db.pragma_update(None, "user_version", 2).unwrap(); + assert!(read_evidence(&route.member_dir(), "g-one", &guard).is_none()); + } + + #[test] + fn stat_evidence_discards_identity_change_and_racy_observations() { + let root = tempfile::tempdir().unwrap(); + let file = root.path().join("file.rs"); + fs::write(&file, b"one").unwrap(); + let stat = NativeStat::read(&file).unwrap(); + let observed = u128::try_from(stat.modified_ns).unwrap(); + assert!(!stat.reusable(&stat, observed)); + assert!(stat.reusable(&stat, observed + 1)); + let mut changed = stat.clone(); + changed.changed_ns += 1; + assert!(!stat.reusable(&changed, observed + 1)); + changed = stat.clone(); + changed.identity = FileIdentity::Windows { + volume: 42, + id: [3; 16], + creation: 0, + }; + assert!(!stat.reusable(&changed, observed + 1)); + } + + #[test] + fn final_revalidation_refuses_git_index_head_root_and_policy_changes() { + let root = tempfile::tempdir().unwrap(); + git(root.path(), &["init", "-q"]); + git(root.path(), &["config", "user.name", "test"]); + git( + root.path(), + &["config", "user.email", "test@example.invalid"], + ); + fs::write(root.path().join("file.rs"), b"initial").unwrap(); + git(root.path(), &["add", "."]); + git(root.path(), &["commit", "-qm", "first"]); + let guard = BindGuard::capture(root.path(), None).unwrap(); + guard.validate(root.path()).unwrap(); + fs::write(root.path().join("file.rs"), b"staged").unwrap(); + git(root.path(), &["add", "."]); + assert!(guard.validate(root.path()).is_err()); + let staged = BindGuard::capture(root.path(), None).unwrap(); + git(root.path(), &["commit", "-qm", "second"]); + assert!(staged.validate(root.path()).is_err()); + let committed = BindGuard::capture(root.path(), None).unwrap(); + git(root.path(), &["config", "core.autocrlf", "true"]); + assert!(committed.validate(root.path()).is_err()); + assert!( + !BindGuard::capture(root.path(), None) + .unwrap() + .policy + .reusable + ); + git(root.path(), &["config", "core.autocrlf", "false"]); + let before_attribute = BindGuard::capture(root.path(), None).unwrap(); + fs::write(root.path().join(".gitattributes"), b"*.rs text\n").unwrap(); + assert!(before_attribute.validate(root.path()).is_err()); + assert!( + !BindGuard::capture(root.path(), None) + .unwrap() + .policy + .reusable + ); + let replacement = tempfile::tempdir().unwrap(); + assert!(committed.validate(replacement.path()).is_err()); + } +} diff --git a/docs/design/views-v3-format.md b/docs/design/views-v3-format.md index f17d4436aa..244ebb82ae 100644 --- a/docs/design/views-v3-format.md +++ b/docs/design/views-v3-format.md @@ -658,6 +658,33 @@ contract is [views-slice4-opening.md](views-slice4-opening.md). | 6a | `F/symbol.sqlite` | `views_v3_store` 1/1, new-store layout 2; store floor 1 | The v3 symbol plane uses the same immutable `payload`/`payload_digest` contract and reservation protocol. No in-place legacy conversion. Implemented; see *Import ledger and symbol plane*. | | 6a | `F/imports.sqlite` (sidecar, not a registry table) | Import ledger 1/1, `application_id` `0x41465449`; registry stays 1/1 or 2/2 | Record completion/rejection per legacy key, artifact, compatible producer and destination plane, with owner-token claim/recovery semantics. Import into v3 only, stream inputs above 1 GiB and retain legacy bytes. Implemented; see *Import ledger and symbol plane*. | +#### Stat evidence and advisory tokens + +Registry 3/3 is an exact-predecessor upgrade of 2/2 under the v3 reservation +barrier, after raising only `views_v3_registry` to 3 and `views_v3_store` to 1. +It adds `view_tokens(nonce TEXT PRIMARY KEY, root BLOB NOT NULL, +expected_tree TEXT NOT NULL, minted_ms INTEGER NOT NULL, expires_ms INTEGER NOT NULL, +bound_root BLOB, bound_gitdir BLOB, consumed_ms INTEGER)`. Root is the canonical +future checkout path. The two bound identity blobs and consume time are all +NULL at mint and all set in one first-consumption transaction. A consumed row +is retained until expiry so replay and identity mismatch can be distinguished; +neither can authorize verification or publication. The lifetime is ten minutes. +Identity blobs are versioned JSON, not a Git stat-cache tuple. Older registry +readers supporting at most 2 refuse registry 3; v1/v2 store formats are unchanged. + +Stat-evidence 1/1 uses `application_id=0x41465445`, `PRAGMA user_version=1`, and +`evidence_meta(singleton INTEGER PRIMARY KEY CHECK(singleton=1), +format_version INTEGER NOT NULL, schema_version INTEGER NOT NULL, +generation TEXT NOT NULL, root BLOB NOT NULL, policy TEXT NOT NULL)` plus +`stat_evidence(path BLOB PRIMARY KEY, record BLOB NOT NULL)`. Records are +versioned JSON containing root/file identity, size, mtime, ctime or Windows +ChangeTime, observation time, content digest, an optional proven Git OID, +the policy digest paired with that observation, and per-plane keys. The file +is staged and synced before the generation pointer CAS and is reusable only +when its generation, root, policy, file identity, timestamps and racy checks +all agree. Missing, corrupt or future evidence means reading source again. +Evidence does not upgrade membership-keyed bases to tree-keyed identities. + The registry-version increments above are serial handoffs, not permission for an earlier slice to write a later owner's rows. Owners implement and test their assigned schema and encoding before publishing any such bytes. Registration diff --git a/packages/aft-bridge/src/__tests__/view-token.test.ts b/packages/aft-bridge/src/__tests__/view-token.test.ts new file mode 100644 index 0000000000..561609bd30 --- /dev/null +++ b/packages/aft-bridge/src/__tests__/view-token.test.ts @@ -0,0 +1,27 @@ +import { expect, test } from "bun:test"; +import type { AftProjectTransport } from "../transport.js"; +import { mintViewToken } from "../view-token.js"; + +test("view token uses a management request, not an agent tool", async () => { + const calls: unknown[] = []; + const transport = { + send: async (...args: unknown[]) => { + calls.push(args); + return { success: true, data: { token: "opaque", expires_ms: 100, advisory: true } }; + }, + } as unknown as AftProjectTransport; + expect(await mintViewToken(transport, "/future/root", "abc")).toEqual({ + token: "opaque", expires_ms: 100, advisory: true, + }); + expect(calls).toEqual([["view_token", { root: "/future/root", expected_tree: "abc" }]]); +}); + +test("view token refuses a non-advisory or failed response", async () => { + for (const response of [ + { success: false, error: "views disabled", data: {} }, + { success: true, data: { token: "opaque", expires_ms: 100, advisory: false } }, + ]) { + const transport = { send: async () => response } as unknown as AftProjectTransport; + await expect(mintViewToken(transport, "/future/root", "abc")).rejects.toThrow(); + } +}); diff --git a/packages/aft-bridge/src/index.ts b/packages/aft-bridge/src/index.ts index f5cbadd1c5..41c1ab3721 100644 --- a/packages/aft-bridge/src/index.ts +++ b/packages/aft-bridge/src/index.ts @@ -100,6 +100,8 @@ export { readBinaryIdentity, writeStampedFileDigest, } from "./binary-identity.js"; +export { mintViewToken, type ViewToken } from "./view-token.js"; + export type { BashCompletedPayload, BashLongRunningPayload, diff --git a/packages/aft-bridge/src/view-token.ts b/packages/aft-bridge/src/view-token.ts new file mode 100644 index 0000000000..b2c4fbe345 --- /dev/null +++ b/packages/aft-bridge/src/view-token.ts @@ -0,0 +1,31 @@ +import type { AftProjectTransport } from "./transport.js"; + +export interface ViewToken { + token: string; + expires_ms: number; + advisory: true; +} + +/** Mint a provisioning hint. The daemon still verifies every checkout bind. */ +export async function mintViewToken( + transport: AftProjectTransport, + root: string, + expectedTree: string, +): Promise { + const response = await transport.send("view_token", { + root, + expected_tree: expectedTree, + }); + if (!response.success) { + throw new Error(response.error ?? "View token mint failed"); + } + const value = response.data; + if ( + typeof value.token !== "string" || + typeof value.expires_ms !== "number" || + value.advisory !== true + ) { + throw new Error("Invalid advisory view token response"); + } + return value as unknown as ViewToken; +} From a47b6b1456d945c2b89ba793d88a8ef5570a1ed1 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:33:56 +0100 Subject: [PATCH 09/31] mason: bootstrap fresh verified-bind drivers through v3-only registration Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/blob_store/v2.rs | 14 + crates/aft/src/views/first_load.rs | 489 +++++++++++++++++- crates/aft/src/views/registry.rs | 171 ++++-- crates/aft/src/views/semantic_runtime.rs | 6 +- crates/aft/src/views/snapshot.rs | 13 + crates/aft/src/views/verification.rs | 18 +- crates/aft/src/watcher_backend/mod.rs | 1 + .../aft/src/watcher_backend/subscription.rs | 85 +++ crates/aft/src/watcher_filter.rs | 1 + 9 files changed, 728 insertions(+), 70 deletions(-) create mode 100644 crates/aft/src/watcher_backend/subscription.rs diff --git a/crates/aft/src/blob_store/v2.rs b/crates/aft/src/blob_store/v2.rs index ab8d71f328..9529b867bd 100644 --- a/crates/aft/src/blob_store/v2.rs +++ b/crates/aft/src/blob_store/v2.rs @@ -363,6 +363,7 @@ pub type StoreResult = Result; pub struct StoreWriteAccess { storage: PathBuf, family: String, + legacy_allowed: bool, } impl StoreWriteAccess { @@ -371,12 +372,20 @@ impl StoreWriteAccess { Self { storage: storage.to_path_buf(), family: family.to_owned(), + legacy_allowed: true, } } pub fn storage(&self) -> &Path { &self.storage } + pub(crate) fn v3_only(storage: &Path, family: &str) -> Self { + Self { + storage: storage.to_path_buf(), + family: family.to_owned(), + legacy_allowed: false, + } + } pub fn family(&self) -> &str { &self.family @@ -442,6 +451,11 @@ pub struct FamilyStoreReader { impl FamilyStore { /// Opens (creating when absent) `/blobs/v2//.sqlite`. pub fn open(access: &StoreWriteAccess, plane: FamilyPlane) -> StoreResult { + if !access.legacy_allowed { + return Err(StoreError::InvalidFamily( + "v3-only registration cannot open a v2 writer".into(), + )); + } let path = plane_path(&access.storage, &access.family, plane)?; if let Some(parent) = path.parent() { crate::private_storage::open_dir(&access.storage, parent)?; diff --git a/crates/aft/src/views/first_load.rs b/crates/aft/src/views/first_load.rs index 78eab56433..a6dd686002 100644 --- a/crates/aft/src/views/first_load.rs +++ b/crates/aft/src/views/first_load.rs @@ -1237,6 +1237,9 @@ mod tests { pub trait CompositePlane: Send + Sync { fn plane(&self) -> FamilyPlane; fn applies_to(&self, path: &RelPath) -> bool; + fn bootstrap_policy(&self) -> Option { + None + } /// Derive a payload key without extraction. Planes with deferred extraction /// may leave this absent and keep their own cheap attachment format. @@ -1333,6 +1336,13 @@ pub trait CompositePlane: Send + Sync { /// adapters must return walker errors rather than silently truncating membership. pub trait MembershipWalker: Send + Sync { fn files(&self, root: &std::path::Path) -> Result, PlaneError>; + fn changed_files( + &self, + _root: &std::path::Path, + _candidates: &std::collections::BTreeSet, + ) -> Result>, PlaneError> { + Ok(None) + } } #[derive(Clone)] @@ -1372,6 +1382,7 @@ struct ObservedCheckout { stats: BTreeMap, evidence: BTreeMap, guard: Option>, + subscription: Option, } impl ObservedStat { @@ -1440,13 +1451,13 @@ mod stat_tests { } #[test] - fn verified_bind_hook_without_a_frontier_strictly_verifies_membership() { + fn strict_reconcile_without_a_frontier_verifies_membership() { let root = tempfile::tempdir().unwrap(); let storage = tempfile::tempdir().unwrap(); std::fs::write(root.path().join("file.rs"), b"fn fresh() {}\n").unwrap(); let (driver, access) = driver(root.path(), storage.path()); let base = driver.installed_snapshot().generation().clone(); - let (snapshot, _) = driver.verified_bind(&access, base).unwrap(); + let (snapshot, _) = SiblingLoader::reconciled_snapshot(&driver, &access, base).unwrap(); let path = RelPath::new(b"file.rs".to_vec()).unwrap(); assert_eq!( snapshot.disk_state(&path), @@ -1466,7 +1477,7 @@ mod stat_tests { std::fs::write(file, vec![b'x'; 20 + attempt]).unwrap(); })); let base = driver.installed_snapshot().generation().clone(); - let (snapshot, _) = driver.verified_bind(&access, base).unwrap(); + let (snapshot, _) = SiblingLoader::reconciled_snapshot(&driver, &access, base).unwrap(); let path = RelPath::new(b"file.rs".to_vec()).unwrap(); assert_eq!(snapshot.pending_intent().collect::>(), vec![&path]); assert_eq!( @@ -1610,6 +1621,7 @@ pub struct CheckoutDriver { delivery: std::sync::Mutex, v3_reconcile: std::sync::atomic::AtomicBool, advisory_token: std::sync::Mutex>, + base_claim: std::sync::Mutex>, #[cfg(test)] read_hook: std::sync::Mutex>, #[cfg(test)] @@ -1718,14 +1730,21 @@ impl CheckoutDriver { } /// Verifies this checkout against `base` and returns a reconciliation cut. - /// Unsupported freshness proofs use strict reconciliation instead. This - /// does not publish or install the cut; those steps still validate its epoch - /// and installed-bundle identity. The hook is independent of plane selection. + /// Unsupported freshness proofs use strict reconciliation instead. + /// A fresh checkout bootstraps a lexical-ready v3 base and publishes a v3 + /// worker over it; it never falls back to legacy storage. Installation is + /// left to the caller and still validates the cut's epoch and bundle identity. pub fn verified_bind( &self, access: &ViewAccess, base: Arc, ) -> Result<(Snapshot, u64), PlaneError> { + self.check_owner(access)?; + if !self.owner.registry().is_v3_only() { + return Err(SiblingLoader::error( + "verified_bind requires a v3-only registration", + )); + } if let Some(token) = self .advisory_token .lock() @@ -1747,11 +1766,189 @@ impl CheckoutDriver { now, ); } - self.v3_reconcile.store( - base.v3_route().is_some(), - std::sync::atomic::Ordering::Release, + let bootstrap = base.base_header().is_none(); + self.v3_reconcile + .store(true, std::sync::atomic::Ordering::Release); + let base = if bootstrap { + self.bootstrap_v3_base(access, base)? + } else { + base + }; + let (snapshot, revision) = SiblingLoader::reconciled_snapshot(self, access, base)?; + if !bootstrap { + return Ok((snapshot, revision)); + } + let own = self.build_own_generation(access, &snapshot, DERIVED_CALLGRAPH_SEEDING)?; + if self + .planes + .iter() + .all(|plane| plane.plane() == FamilyPlane::Trigram) + { + if let Some(claim) = self + .base_claim + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take() + { + claim + .release() + .map_err(|e| SiblingLoader::error(e.to_string()))?; + } + } + let mut delta = LiveDelta::from_snapshot(&snapshot); + let draft = super::snapshot::derive_successor( + &delta.cut(), + own, + &super::snapshot::carry_disk_state_only, + ); + delta + .replay_and_swap(draft, &super::snapshot::carry_disk_state_only) + .map_err(|error| SiblingLoader::error(format!("{error:?}")))?; + Ok(( + delta + .snapshot() + .with_install_cut(snapshot.install_cut().unwrap().clone()) + .with_reconciled_epoch( + snapshot + .reconciled_epoch() + .unwrap_or(snapshot.install_cut().unwrap().epoch), + ), + revision, + )) + } + + fn bootstrap_v3_base( + &self, + access: &ViewAccess, + seed: Arc, + ) -> Result, PlaneError> { + let policy = self + .planes + .iter() + .find_map(|plane| plane.bootstrap_policy()) + .ok_or_else(|| { + SiblingLoader::error("v3 bootstrap requires a registered lexical plane") + })?; + if self.producers.trigram != policy.fingerprint_hex() { + return Err(SiblingLoader::error( + "v3 bootstrap lexical producer mismatch", + )); + } + let (checkout, _) = SiblingLoader::reconciled_snapshot(self, access, seed)?; + if checkout.pending_intent().next().is_some() { + return Err(SiblingLoader::error( + "v3 bootstrap has pending unstable paths", + )); + } + let route = self.owner.v3_route(); + let ignore_fingerprint = checkout + .install_cut() + .and_then(|cut| cut.verification.as_ref()) + .map(|g| g.policy.digest.clone()); + let header = super::manifest_v2::ManifestHeader { + producers: self.producers.clone(), + head_tree: None, + ignore_fingerprint: ignore_fingerprint.clone(), + segment: None, + }; + let mut manifest = super::manifest_v2::ManifestV2::new_v3(header, None, Default::default()); + for (path, state) in checkout.membership() { + if let super::snapshot::DiskState::Present { content, size } = state { + manifest + .insert( + path, + super::manifest_v2::EntryV2::regular(content, size, Default::default()), + ) + .map_err(|e| SiblingLoader::error(e.to_string()))?; + } + } + let expected = super::base_refs::BaseIdentity::of(&route.family, &manifest) + .map_err(|e| SiblingLoader::error(e.to_string()))?; + let resolved = + match super::base_refs::resolve_base(&route.storage, &route.family, &expected.scope) { + Ok(base) => base, + Err(_) => { + let paths: Vec<_> = manifest.entries().map(|(path, _)| path.clone()).collect(); + match super::base_builder::build_base( + super::base_builder::BaseBuildInput { + storage: &route.storage, + family: &route.family, + root: self.owner.root(), + paths: &paths, + policy, + semantic_producer: self.producers.semantic.clone(), + callgraph_producer: self.producers.callgraph.clone(), + ignore_fingerprint, + history: None, + budget: Default::default(), + }, + None, + ) + .map_err(|e| SiblingLoader::error(e.to_string()))? + { + super::base_builder::BaseBuild::Published { + identity, claim, .. + } => { + *self + .base_claim + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(claim); + super::base_refs::resolve_base( + &route.storage, + &route.family, + &identity.scope, + ) + .map_err(|e| SiblingLoader::error(e.to_string()))? + } + super::base_builder::BaseBuild::Busy => super::base_refs::resolve_base( + &route.storage, + &route.family, + &expected.scope, + ) + .map_err(|_| { + SiblingLoader::error( + "v3 base build busy; retain direct-scan coverage and retry", + ) + })?, + super::base_builder::BaseBuild::Strict(reason) => { + return Err(SiblingLoader::error(format!( + "v3 base admission refused: {reason}" + ))) + } + } + } + }; + let header = resolved.header(); + let generation = resolved.into_generation(&route.storage, &route.family); + let manifest = super::manifest_v2::ManifestV2::new_v3( + generation.header().clone(), + Some(header), + Default::default(), ); - SiblingLoader::reconciled_snapshot(self, access, base) + let seed_route = generation.v3_route().unwrap().clone(); + let composed = + ComposedGeneration::compose(generation.name(), manifest, generation.clone(), None) + .map_err(|e| SiblingLoader::error(e.to_string()))?; + Ok(Arc::new(composed.with_v3_route(seed_route))) + } + + #[allow(clippy::too_many_arguments)] + pub fn new_v3( + storage: &std::path::Path, + family: &str, + scope: &str, + root: &std::path::Path, + producers: Producers, + walker: Arc, + planes: Vec>, + ) -> Result { + let owner = super::registry::ViewRegistration::register_v3(storage, family, scope, root) + .map_err(|e| SiblingLoader::error(e.to_string()))?; + let driver = Self::new(owner, producers, None, walker, planes); + driver + .v3_reconcile + .store(true, std::sync::atomic::Ordering::Release); + Ok(driver) } pub fn set_view_token(&self, token: String) { @@ -1761,6 +1958,10 @@ impl CheckoutDriver { .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(token); } + pub fn access(&self) -> ViewAccess { + ViewAccess::Owner(self.owner.clone()) + } + pub fn take_revalidated_job(&self) -> bool { self.retry.swap(false, std::sync::atomic::Ordering::AcqRel) } @@ -1815,6 +2016,7 @@ impl CheckoutDriver { delivery: Default::default(), v3_reconcile: Default::default(), advisory_token: Default::default(), + base_claim: Default::default(), #[cfg(test)] read_hook: Default::default(), installed: std::sync::Mutex::new(Arc::new(InstalledBundle { @@ -1908,6 +2110,7 @@ impl CheckoutDriver { /// Invalidates stale builds and indexed membership until strict reconciliation. pub fn record_watcher_overflow(&self) { + crate::watcher_backend::subscription::gap(self.owner.root()); let mut delta = self .live .lock() @@ -2087,16 +2290,86 @@ impl FirstLoadDriver for CheckoutDriver { ) }) .unwrap_or_default(); - let mut entries = BTreeMap::new(); - let mut stats = BTreeMap::new(); - let mut evidence = BTreeMap::new(); + let subscription = crate::watcher_backend::subscription::current(self.owner.root()); + let mut changed = pending.clone(); + changed.extend( + installed + .live_entries() + .filter(|(_, entry)| { + entry.reconciliation != super::readiness::ReconciledDelta::Reconciled + }) + .map(|(path, _)| path.clone()), + ); + let mut candidates = std::collections::BTreeSet::new(); + let mut policy_or_directory = false; + for path in &changed { + let relative = super::segment_store::rel_path_to_os(path) + .map_err(|e| SiblingLoader::error(e.to_string()))?; + let absolute = self.owner.root().join(relative); + let name = absolute.file_name().and_then(|n| n.to_str()).unwrap_or(""); + policy_or_directory |= matches!( + name, + ".gitattributes" | ".gitignore" | ".ignore" | ".aftignore" + ) || path.as_bytes().starts_with(b".git/") + || super::assembly::is_resolution_input(path.as_bytes()) + || std::fs::symlink_metadata(&absolute).is_ok_and(|m| m.is_dir()); + candidates.insert(absolute); + } + let continuous = v3 + && subscription.is_some() + && subscription == previous.subscription + && installed.watcher() == super::snapshot::WatcherState::Healthy + && guard + .as_ref() + .zip(previous.guard.as_ref()) + .is_some_and(|(now, old)| now.root == old.root && now.policy == old.policy) + && !policy_or_directory; + let changed_files = if continuous { + self.walker.changed_files(self.owner.root(), &candidates)? + } else { + None + }; + let incremental = changed_files.is_some(); + let files = match changed_files { + Some(files) => files, + None => self.walker.files(self.owner.root())?, + }; + let mut entries = if incremental { + previous.entries.clone() + } else { + BTreeMap::new() + }; + let mut stats = if incremental { + previous.stats.clone() + } else { + BTreeMap::new() + }; + let mut evidence = if incremental { + previous.evidence.clone() + } else { + BTreeMap::new() + }; + if incremental { + for path in &changed { + entries.remove(path); + stats.remove(path); + evidence.remove(path); + let mut prefix = path.as_bytes().to_vec(); + prefix.push(b'/'); + entries.retain(|p, _| !p.as_bytes().starts_with(&prefix)); + stats.retain(|p, _| !p.as_bytes().starts_with(&prefix)); + evidence.retain(|p, _| !p.as_bytes().starts_with(&prefix)); + } + } #[cfg(test)] { - self.work.lock().unwrap().walks += 1; + if !incremental { + self.work.lock().unwrap().walks += 1; + } } - // Full configured membership, even for copied and non-git roots. Stat - // reuse avoids source I/O, not the discovery of unreported changes. - for absolute in self.walker.files(self.owner.root())? { + // Restart and delivery gaps require full membership. A continuous + // subscription checks only reported files under the same walker policy. + for absolute in files { let relative = absolute .strip_prefix(self.owner.root()) .map_err(|error| SiblingLoader::error(error.to_string()))?; @@ -2168,9 +2441,11 @@ impl FirstLoadDriver for CheckoutDriver { }) }) && self.planes.iter().filter(|p| p.applies_to(&path)).all(|p| { - record - .key(p.plane()) - .is_some_and(|key| v3_contains(&route, &key)) + record.key(p.plane()).is_some_and(|key| { + matches!(generation.plane_readiness(&path, p.plane()), Some(super::readiness::PlaneState::Ready { key: ready }) if *ready == key.to_hex()) + && record.content().is_some_and(|content| p.key(&path, content).is_none_or(|expected| expected == key)) + && v3_contains(&route, &key) + }) }) && self .delivery @@ -2277,6 +2552,7 @@ impl FirstLoadDriver for CheckoutDriver { stats, evidence, guard: guard.clone(), + subscription, }; let mut cut = installed.install_cut().cloned(); if let Some(cut) = &mut cut { @@ -2464,6 +2740,11 @@ impl FirstLoadDriver for CheckoutDriver { if let Some(header) = snapshot.generation().base_header() { return self.build_v3_worker_generation(access, snapshot, header.clone()); } + if self.owner.registry().is_v3_only() { + return Err(SiblingLoader::error( + "v3-only driver requires verified_bind bootstrap before publication", + )); + } super::io::WORK .own_generation_folds .fetch_add(1, std::sync::atomic::Ordering::Relaxed); @@ -4090,7 +4371,7 @@ impl CompositePlane for CallgraphBridge { &crate::blob_store::CallgraphKey::from_source_digest( *content.as_bytes(), language, - crate::blob_store::CALLGRAPH_PRODUCER_VERSION, + super::callgraph::PRODUCER, ) .full_key(), )) @@ -4856,6 +5137,20 @@ thread_local! { pub(crate) static MEMBERSHIP_WALKS: std::cell::Cell = const { std::cell::Cell::new(0) }; } impl MembershipWalker for ConfiguredMembershipWalker { + fn changed_files( + &self, + root: &std::path::Path, + candidates: &std::collections::BTreeSet, + ) -> Result>, PlaneError> { + let listing = crate::search_index::project_walk_lists(root, candidates); + if candidates + .iter() + .any(|path| listing.gap_for(path).is_some()) + { + return Ok(None); + } + Ok(Some(listing.listed.into_iter().collect())) + } fn files(&self, root: &std::path::Path) -> Result, PlaneError> { #[cfg(test)] MEMBERSHIP_WALKS.with(|walks| walks.set(walks.get() + 1)); @@ -4943,6 +5238,9 @@ impl TrigramBridge { } } impl CompositePlane for TrigramBridge { + fn bootstrap_policy(&self) -> Option { + Some(self.policy) + } fn plane(&self) -> FamilyPlane { FamilyPlane::Trigram } @@ -5423,3 +5721,152 @@ mod checkout_kind_tests { } } } + +#[cfg(test)] +mod v3_bootstrap_tests { + use super::*; + + fn inventory(root: &std::path::Path) -> BTreeMap> { + fn visit( + base: &std::path::Path, + path: &std::path::Path, + files: &mut BTreeMap>, + ) { + if !path.exists() { + return; + } + for entry in std::fs::read_dir(path).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + visit(base, &path, files); + } else { + files.insert( + path.strip_prefix(base).unwrap().to_owned(), + std::fs::read(path).unwrap(), + ); + } + } + } + let mut files = BTreeMap::new(); + visit(root, root, &mut files); + files + } + + #[test] + fn verified_bind_fresh_trigram_driver_bootstraps_and_publishes_only_v3() { + let _serial = crate::blob_store::v3::serial_for_test(); + let root = tempfile::tempdir().unwrap(); + let storage = tempfile::tempdir().unwrap(); + let source = b"fn fresh_checkout() { let marker = 42; }\n"; + std::fs::write(root.path().join("file.rs"), source).unwrap(); + std::fs::File::open(root.path().join("file.rs")) + .unwrap() + .set_times( + std::fs::FileTimes::new() + .set_modified(std::time::UNIX_EPOCH + std::time::Duration::from_secs(60)), + ) + .unwrap(); + let legacy_v1 = storage.path().join("blobs/family"); + let legacy_v2 = storage.path().join("blobs/v2/family"); + for dir in [&legacy_v1, &legacy_v2] { + std::fs::create_dir_all(dir).unwrap(); + std::fs::write(dir.join("untouched"), b"legacy sentinel").unwrap(); + } + let before = (inventory(&legacy_v1), inventory(&legacy_v2)); + let policy = crate::blob_store::v2::TrigramPolicy { + max_file_size: 1 << 20, + }; + let producers = Producers { + trigram: policy.fingerprint_hex(), + semantic: None, + callgraph: "unregistered".into(), + }; + let plane = Arc::new(TrigramBridge::new(storage.path().to_owned(), policy)); + let driver = CheckoutDriver::new_v3( + storage.path(), + "family", + "worker", + root.path(), + producers.clone(), + Arc::new(ConfiguredMembershipWalker), + vec![plane], + ) + .unwrap(); + let access = driver.access(); + assert!(driver.owner.open_store(FamilyPlane::Trigram).is_err()); + assert_eq!((inventory(&legacy_v1), inventory(&legacy_v2)), before); + let empty = driver.installed_snapshot().generation().clone(); + let (snapshot, revision) = driver.verified_bind(&access, empty).unwrap(); + assert!(snapshot.generation().base_header().is_some()); + assert_eq!(snapshot.generation().v3_route().unwrap().scope, "worker"); + assert!(snapshot.generation().header().head_tree.is_none()); + let path = RelPath::new(b"file.rs".to_vec()).unwrap(); + assert_eq!( + snapshot.disk_state(&path), + super::super::snapshot::DiskState::of_bytes(source) + ); + assert!(matches!( + snapshot.plane_readiness(&path, FamilyPlane::Trigram), + Some(super::super::readiness::PlaneState::Ready { .. }) + )); + driver.install(&access, &snapshot, revision).unwrap(); + assert!(!driver.installed_snapshot().needs_reconcile()); + let route = driver.owner.v3_route(); + let store = super::super::ViewStore::existing_dir(route.member_dir()).unwrap(); + let generation = store.current_generation_read_only().unwrap().unwrap(); + let manifest = store.load_manifest_v2(&generation).unwrap(); + assert_eq!( + manifest.len(), + 0, + "unchanged bytes inherit the lexical-ready base" + ); + assert!( + super::super::verification::evidence_path(&route.member_dir(), &generation).exists() + ); + assert_eq!((inventory(&legacy_v1), inventory(&legacy_v2)), before); + let ledger = + crate::blob_store::v3::LedgerReader::open_existing(storage.path(), "family").unwrap(); + assert_eq!( + ledger + .read(|c| Ok(c.query_row( + "SELECT COUNT(*) FROM view_sessions WHERE scope='worker'", + [], + |r| r.get::<_, u32>(0) + )?)) + .unwrap(), + 1 + ); + let second = CheckoutDriver::new_v3( + storage.path(), + "family", + "worker", + root.path(), + producers, + Arc::new(ConfiguredMembershipWalker), + vec![Arc::new(TrigramBridge::new( + storage.path().to_owned(), + policy, + ))], + ) + .unwrap(); + let base = super::super::base_refs::open_foreign_v3( + storage.path(), + "family", + "worker", + &second.producers, + ) + .unwrap() + .unwrap(); + second.verified_bind(&second.access(), base).unwrap(); + let work = second.take_reconcile_work(); + assert_eq!( + work.walks, 1, + "membership-keyed bases still take strict startup verification" + ); + assert_eq!( + work.reads, 0, + "generation-paired evidence avoids rereading stable source" + ); + assert_eq!((inventory(&legacy_v1), inventory(&legacy_v2)), before); + } +} diff --git a/crates/aft/src/views/registry.rs b/crates/aft/src/views/registry.rs index 5ed7326166..beef416018 100644 --- a/crates/aft/src/views/registry.rs +++ b/crates/aft/src/views/registry.rs @@ -236,7 +236,7 @@ struct RegistryInner { durable_storage: Mutex>, family: String, path: PathBuf, - connection: Mutex, + connection: Option>, /// The disk limits this process applies to the family; see /// `views::eviction`. Shared by every handle on this registry file. budget: Mutex, @@ -299,6 +299,30 @@ impl RegistryLocation { } impl FamilyRegistry { + /// Identity and barrier access for a v3-only owner. No legacy connection is + /// opened, including during session cleanup. + fn open_v3(storage: &Path, family: &str) -> RegistryResult { + let path = crate::blob_store::v3::family_dir(storage, family) + .map_err(token_store_error)? + .join("members.sqlite"); + Ok(Self { + inner: Arc::new(RegistryInner { + storage: storage.to_path_buf(), + durable_storage: Mutex::new(None), + family: family.to_owned(), + path, + connection: None, + budget: Mutex::new(Default::default()), + size_caches: Mutex::default(), + ended_sessions: Mutex::new(Vec::new()), + }), + }) + } + + pub fn is_v3_only(&self) -> bool { + self.inner.connection.is_none() + } + /// Opens the registry, creating it when absent. pub fn open(storage: &Path, family: &str) -> RegistryResult { let dir = family_dir(storage, family)?; @@ -371,7 +395,7 @@ impl FamilyRegistry { durable_storage: Mutex::new(None), family: family.to_owned(), path, - connection: Mutex::new(connection), + connection: Some(Mutex::new(connection)), budget: Mutex::new(super::eviction::DiskBudget::default()), size_caches: Mutex::default(), ended_sessions: Mutex::new(Vec::new()), @@ -408,13 +432,18 @@ impl FamilyRegistry { durable_storage: Mutex::new(None), family: family.to_owned(), path: path.clone(), - connection: Mutex::new(connection), + connection: Some(Mutex::new(connection)), budget: Mutex::new(super::eviction::DiskBudget::default()), size_caches: Mutex::default(), ended_sessions: Mutex::new(Vec::new()), }); { - let mut connection = lock(&inner.connection); + let mut connection = lock( + inner + .connection + .as_ref() + .expect("legacy registry connection"), + ); if create { crate::blob_store::retry_while_busy( Duration::from_millis(crate::blob_store::BUSY_TIMEOUT_MS), @@ -446,21 +475,23 @@ impl FamilyRegistry { /// The registry connection, after writing any session ends queued while /// it was busy, so that nothing read through it sees a session that has /// already ended in this process. - fn connection(&self) -> MutexGuard<'_, TrackedConnection> { + fn connection(&self) -> RegistryResult> { + let mutex = self.inner.connection.as_ref().ok_or_else(|| { + RegistryError::Incompatible( + "legacy registry operation on a v3-only registration".into(), + ) + })?; #[cfg(test)] CONNECTION_BLOCKED.with(|hook| { - if matches!( - self.inner.connection.try_lock(), - Err(std::sync::TryLockError::WouldBlock) - ) { + if matches!(mutex.try_lock(), Err(std::sync::TryLockError::WouldBlock)) { if let Some(blocked) = hook.borrow_mut().take() { blocked.send(()).unwrap(); } } }); - let connection = lock(&self.inner.connection); + let connection = lock(mutex); flush_ended_sessions(&self.inner, &connection); - connection + Ok(connection) } pub fn storage(&self) -> &Path { @@ -534,6 +565,11 @@ impl FamilyRegistry { scope: &str, root: &Path, ) -> RegistryResult { + if self.is_v3_only() { + return Err(RegistryError::Incompatible( + "use ViewRegistration::register_v3 for v3-only owners".into(), + )); + } self.bind_durable_storage(durable_storage)?; if matches!(root.try_exists(), Ok(true)) { let paths = crate::root_cache::StoragePaths::new( @@ -554,7 +590,7 @@ impl FamilyRegistry { owner.start_time, SESSION_SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed) ); - let mut connection = self.connection(); + let mut connection = self.connection()?; let tx = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; tx.execute_batch(RUNTIME_SCHEMA)?; let previous: Option = tx @@ -633,7 +669,7 @@ impl FamilyRegistry { /// Every recorded view session, live or not. pub fn sessions(&self) -> RegistryResult> { - let connection = self.connection(); + let connection = self.connection()?; read_sessions(&connection, None) } @@ -647,7 +683,7 @@ impl FamilyRegistry { .filter(|session| !crate::pins::owner_is_live(&session.owner)) .map(|session| session.session_id) .collect::>(); - let connection = self.connection(); + let connection = self.connection()?; for session_id in &dead { connection.execute( "DELETE FROM view_sessions WHERE session_id = ?1", @@ -668,7 +704,7 @@ impl FamilyRegistry { owner.start_time, READER_SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed) ); - self.connection().execute( + self.connection()?.execute( "INSERT INTO readers (reader_id, label, owner_pid, owner_start, registered_at_ms) VALUES (?1, ?2, ?3, ?4, ?5)", params![ @@ -687,11 +723,11 @@ impl FamilyRegistry { } pub fn members(&self) -> RegistryResult> { - read_members(&self.connection()) + read_members(&*self.connection()?) } pub(crate) fn members_bounded(&self, limit: usize) -> RegistryResult> { - let connection = self.connection(); + let connection = self.connection()?; let members = read_members_limited(&connection, limit.saturating_add(1))?; if members.len() > limit { return Err(RegistryError::Incompatible(format!( @@ -702,13 +738,13 @@ impl FamilyRegistry { } pub fn member(&self, scope: &str) -> RegistryResult> { - Ok(read_members(&self.connection())? + Ok(read_members(&*self.connection()?)? .into_iter() .find(|member| member.scope == scope)) } pub fn readers(&self) -> RegistryResult> { - let connection = self.connection(); + let connection = self.connection()?; let mut statement = connection.prepare( "SELECT reader_id, label, owner_pid, owner_start, registered_at_ms FROM readers ORDER BY reader_id", @@ -728,7 +764,7 @@ impl FamilyRegistry { } pub fn gc_epoch(&self) -> RegistryResult { - let epoch: i64 = self.connection().query_row( + let epoch: i64 = self.connection()?.query_row( "SELECT gc_epoch FROM registry_meta WHERE singleton = 1", [], |row| row.get(0), @@ -738,7 +774,7 @@ impl FamilyRegistry { /// Records a successful publication of `scope`. pub fn note_publish(&self, scope: &str) -> RegistryResult<()> { - self.connection().execute( + self.connection()?.execute( "UPDATE members SET last_publish_ms = ?2 WHERE scope = ?1", params![scope, now_ms() as i64], )?; @@ -759,7 +795,7 @@ impl FamilyRegistry { /// A live owner keeps the lease; a dead owner's lease is taken over. pub(crate) fn acquire_sweep(&self) -> RegistryResult> { let owner = current_owner(); - let mut connection = self.connection(); + let mut connection = self.connection()?; let tx = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; let holder: Option<(i64, i64)> = tx .query_row( @@ -812,7 +848,18 @@ impl FamilyRegistry { &self, body: impl FnOnce(&rusqlite::Connection) -> RegistryResult, ) -> RegistryResult { - body(&self.connection()) + if self.is_v3_only() { + return crate::blob_store::v3::LedgerReader::open_existing( + self.storage(), + self.family(), + ) + .map_err(token_store_error)? + .read(|c| { + body(c).map_err(|e| crate::blob_store::v3::Error::Io(std::io::Error::other(e))) + }) + .map_err(token_store_error); + } + body(&*self.connection()?) } /// Runs `body` inside one IMMEDIATE transaction on the registry. This is @@ -822,7 +869,14 @@ impl FamilyRegistry { &self, body: impl FnOnce(&rusqlite::Transaction<'_>) -> RegistryResult, ) -> RegistryResult { - let mut connection = self.connection(); + if self.is_v3_only() { + return super::base_refs::with_base_barrier(self.storage(), self.family(), |tx| { + flush_ended_sessions(&self.inner, tx); + body(tx).map_err(|e| super::base_refs::BaseRefError::Io(std::io::Error::other(e))) + }) + .map_err(|e| RegistryError::Io(std::io::Error::other(e))); + } + let mut connection = self.connection()?; let tx = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; let value = body(&tx)?; tx.commit()?; @@ -851,7 +905,7 @@ impl FamilyRegistry { .filter(|reader| !crate::pins::owner_is_live(&reader.owner)) .map(|reader| reader.reader_id) .collect::>(); - let connection = self.connection(); + let connection = self.connection()?; for reader_id in &dead { connection.execute( "DELETE FROM readers WHERE reader_id = ?1", @@ -900,7 +954,7 @@ impl SweepLease { /// A forced admission pass ignores it. Failed or partial passes do not /// postpone a retry. pub(crate) fn due(&self, force: bool) -> RegistryResult { - let connection = self.registry.connection(); + let connection = self.registry.connection()?; connection.execute_batch("CREATE TABLE IF NOT EXISTS sweep_schedule (singleton INTEGER PRIMARY KEY CHECK(singleton = 1), completed_at_ms INTEGER NOT NULL)")?; let last: Option = connection .query_row( @@ -917,7 +971,7 @@ impl SweepLease { } pub(crate) fn completed(&self) -> RegistryResult<()> { - self.registry.connection().execute("INSERT INTO sweep_schedule VALUES (1, ?1) ON CONFLICT(singleton) DO UPDATE SET completed_at_ms = excluded.completed_at_ms", params![now_ms() as i64])?; + self.registry.connection()?.execute("INSERT INTO sweep_schedule VALUES (1, ?1) ON CONFLICT(singleton) DO UPDATE SET completed_at_ms = excluded.completed_at_ms", params![now_ms() as i64])?; Ok(()) } @@ -934,10 +988,12 @@ impl SweepLease { return; } self.released = true; - let _ = self.registry.connection().execute( + if let Ok(connection) = self.registry.connection() { + let _ = connection.execute( "DELETE FROM sweep_lease WHERE singleton = 1 AND owner_pid = ?1 AND owner_start = ?2", params![i64::from(self.owner.pid), self.owner.start_time as i64], ); + } } } @@ -993,7 +1049,16 @@ impl Drop for ViewSession { ended_at_ms: now_ms(), }; lock(&self.registry.inner.ended_sessions).push(ended); - match self.registry.inner.connection.try_lock() { + let Some(connection) = &self.registry.inner.connection else { + let registry = self.registry.clone(); + let _ = std::thread::Builder::new() + .name("aft-v3-view-session-end".into()) + .spawn(move || { + let _ = registry.with_barrier(|_| Ok(())); + }); + return; + }; + match connection.try_lock() { Ok(connection) => flush_ended_sessions(&self.registry.inner, &connection), Err(std::sync::TryLockError::Poisoned(poisoned)) => { flush_ended_sessions(&self.registry.inner, &poisoned.into_inner()) @@ -1039,6 +1104,36 @@ pub struct SessionRecord { } impl ViewRegistration { + /// Registers only in blobs/v3. It grants no legacy store-write capability. + pub fn register_v3( + storage: &Path, + family: &str, + scope: &str, + root: &Path, + ) -> RegistryResult { + let registry = FamilyRegistry::open_v3(storage, family)?; + let root = std::fs::canonicalize(root)?; + let owner = current_owner(); + let session_id = super::base_refs::fresh_nonce(); + let view_dir = registry.with_barrier(|tx| { + let dir = super::base_refs::register_view_in(tx, storage, family, scope, &root) + .map_err(|e| RegistryError::Io(std::io::Error::other(e)))?; + tx.execute("INSERT INTO view_sessions(session_id,scope,owner_pid,owner_start,opened_at_ms) VALUES(?1,?2,?3,?4,?5)", + params![session_id, scope, owner.pid, owner.start_time as i64, now_ms()])?; + Ok(dir) + })?; + Ok(Self { + registry: registry.clone(), + scope: scope.to_owned(), + root, + view_dir, + session: Arc::new(ViewSession { + registry, + session_id, + scope: scope.to_owned(), + }), + }) + } pub fn registry(&self) -> &FamilyRegistry { &self.registry } @@ -1065,6 +1160,9 @@ impl ViewRegistration { } pub fn write_access(&self) -> StoreWriteAccess { + if self.registry.is_v3_only() { + return StoreWriteAccess::v3_only(self.registry.storage(), self.family()); + } StoreWriteAccess::for_registered_view(self.registry.storage(), self.registry.family()) } @@ -1084,6 +1182,9 @@ impl ViewRegistration { /// The view's own pointer and manifests in `views/v2/`. pub fn view_store(&self) -> RegistryResult { + if self.registry.is_v3_only() { + return Ok(super::ViewStore::open_dir(self.view_dir.clone())?); + } let view = super::ViewStore::open_dir(self.view_dir.clone())? .with_sweep_registry(self.registry.clone()); view.recover_abandoned_derived(Some(self.registry())); @@ -1211,10 +1312,12 @@ impl ReaderRegistration { impl Drop for ReaderRegistration { fn drop(&mut self) { - let _ = self.registry.connection().execute( - "DELETE FROM readers WHERE reader_id = ?1", - params![self.reader_id], - ); + if let Ok(connection) = self.registry.connection() { + let _ = connection.execute( + "DELETE FROM readers WHERE reader_id = ?1", + params![self.reader_id], + ); + } } } @@ -1723,7 +1826,7 @@ mod tests { super::super::generation::schedule_generation_sweep(view.clone()); entered_rx.recv_timeout(Duration::from_secs(10)).unwrap(); let held = { - let mut connection = registry.connection(); + let mut connection = registry.connection().unwrap(); connection.busy_timeout(Duration::ZERO).unwrap(); let blocked = connection .transaction_with_behavior(TransactionBehavior::Immediate) diff --git a/crates/aft/src/views/semantic_runtime.rs b/crates/aft/src/views/semantic_runtime.rs index c97b2a4f4a..e27e9ba87d 100644 --- a/crates/aft/src/views/semantic_runtime.rs +++ b/crates/aft/src/views/semantic_runtime.rs @@ -728,11 +728,7 @@ pub struct CheckoutSemanticMemory { /// before its vectors can be brought up to date. The initial snapshot (before /// any generation was published) always needs one. fn needs_reload(snapshot: &Snapshot) -> bool { - snapshot.generation().name() == "empty" - || snapshot.pending_intent().next().is_some() - || snapshot - .live_entries() - .any(|(_, entry)| entry.reconciliation != super::readiness::ReconciledDelta::Reconciled) + snapshot.needs_reconcile() } /// Whether reconciling `snapshot` needs a cold-build slot. A reconcile walks diff --git a/crates/aft/src/views/snapshot.rs b/crates/aft/src/views/snapshot.rs index 6a3069034e..a932bf713c 100644 --- a/crates/aft/src/views/snapshot.rs +++ b/crates/aft/src/views/snapshot.rs @@ -821,6 +821,12 @@ impl LiveDelta { reconciled_epoch: None, publication_base: None, readers: None, + needs_reconcile: self.base.name() == "empty" + || !self.intent.is_empty() + || self.watcher != WatcherState::Healthy + || self.entries.values().any(|entry| { + entry.reconciliation != super::readiness::ReconciledDelta::Reconciled + }), }), } } @@ -1665,6 +1671,7 @@ pub fn carry_disk_state_only(path: &RelPath, generation: &ComposedGeneration) -> #[derive(Clone, Debug)] struct SnapshotInner { + needs_reconcile: bool, reconciled_epoch: Option, publication_base: Option>, generation: Arc, @@ -1991,6 +1998,12 @@ pub struct Snapshot { } impl Snapshot { + /// Checks the cached reconciliation flag. Work is summarized when the + /// immutable snapshot is created, not by scanning the delta on each query. + pub fn needs_reconcile(&self) -> bool { + self.inner.needs_reconcile + } + pub(crate) fn with_publication_base(mut self, base: Option) -> Self { Arc::make_mut(&mut self.inner).publication_base = Some(base); self diff --git a/crates/aft/src/views/verification.rs b/crates/aft/src/views/verification.rs index eb0c74874b..4ece51f167 100644 --- a/crates/aft/src/views/verification.rs +++ b/crates/aft/src/views/verification.rs @@ -203,18 +203,14 @@ impl Policy { hash.update(b"aft/views-v3/conversion-policy/1"); let queries = crate::git_index_reader::GitQueries::new(root.to_path_buf(), Duration::from_secs(2)); - let output = queries.config()?; - if !output.status.success() { - return Ok(Self { - digest: "non-git".into(), - reusable: false, - }); - } - hash.update(&output.stdout); - let mut reusable = true; + let output = queries.config(); + let mut reusable = output.as_ref().is_ok_and(|output| output.status.success()); + hash.update(&[u8::from(reusable)]); + let config = output.map(|output| output.stdout).unwrap_or_default(); + hash.update(&config); // Config --show-origin records include targets and worktree config. // Hash their bytes as well as the effective values, including empty files. - for item in output.stdout.split(|b| *b == 0) { + for item in config.split(|b| *b == 0) { if let Some(path) = item.strip_prefix(b"file:") { let path = path_from_bytes(path)?; metadata_source( @@ -360,6 +356,7 @@ pub(crate) struct BindGuard { pub index: Option<(NativeStat, [u8; 32])>, pub policy: Policy, pub inputs: Vec, + pub subscription: Option, } impl BindGuard { @@ -407,6 +404,7 @@ impl BindGuard { index, policy, inputs, + subscription: crate::watcher_backend::subscription::current(root), }) } pub fn validate(&self, root: &Path) -> io::Result<()> { diff --git a/crates/aft/src/watcher_backend/mod.rs b/crates/aft/src/watcher_backend/mod.rs index 1f901c54d6..795ccedf00 100644 --- a/crates/aft/src/watcher_backend/mod.rs +++ b/crates/aft/src/watcher_backend/mod.rs @@ -7,6 +7,7 @@ pub(crate) mod cookie; #[cfg(test)] mod cookie_tests; pub(crate) mod flush; +pub(crate) mod subscription; #[cfg(target_os = "macos")] mod fsevents; diff --git a/crates/aft/src/watcher_backend/subscription.rs b/crates/aft/src/watcher_backend/subscription.rs new file mode 100644 index 0000000000..e1a9cb9399 --- /dev/null +++ b/crates/aft/src/watcher_backend/subscription.rs @@ -0,0 +1,85 @@ +//! Subscription generation and observed gaps for steady-state refresh. This +//! does not prove that all writes before a cookie reached the driver, so it +//! cannot justify a bind based on Git/stat observations. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, OnceLock, Weak}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct Continuity { + pub generation: u64, + pub gap: u64, +} +struct State { + generation: u64, + gap: AtomicU64, +} +pub(crate) struct Subscription { + _state: Arc, +} +static NEXT: AtomicU64 = AtomicU64::new(1); +fn roots() -> &'static Mutex>> { + static ROOTS: OnceLock>>> = OnceLock::new(); + ROOTS.get_or_init(Default::default) +} + +pub(crate) fn start(root: &Path) -> Subscription { + let state = Arc::new(State { + generation: NEXT.fetch_add(1, Ordering::Relaxed), + gap: AtomicU64::new(0), + }); + roots() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert(root.to_path_buf(), Arc::downgrade(&state)); + Subscription { _state: state } +} + +pub(crate) fn current(root: &Path) -> Option { + let mut roots = roots() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + match roots.get(root).and_then(Weak::upgrade) { + Some(state) => Some(Continuity { + generation: state.generation, + gap: state.gap.load(Ordering::Acquire), + }), + None => { + roots.remove(root); + None + } + } +} + +pub(crate) fn gap(root: &Path) { + if let Some(state) = roots() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(root) + .and_then(Weak::upgrade) + { + state.gap.fetch_add(1, Ordering::AcqRel); + } +} + +#[cfg(test)] +mod tests { + #[test] + fn subscription_restart_stop_and_gap_change_the_continuity_token() { + let root = tempfile::tempdir().unwrap(); + assert!(super::current(root.path()).is_none()); + let first = super::start(root.path()); + let before = super::current(root.path()).unwrap(); + super::gap(root.path()); + assert_ne!(super::current(root.path()).unwrap(), before); + let second = super::start(root.path()); + let after = super::current(root.path()).unwrap(); + assert_ne!(before.generation, after.generation); + drop(first); + assert_eq!(super::current(root.path()), Some(after)); + drop(second); + assert!(super::current(root.path()).is_none()); + } +} diff --git a/crates/aft/src/watcher_filter.rs b/crates/aft/src/watcher_filter.rs index b91224fe79..0276a8e610 100644 --- a/crates/aft/src/watcher_filter.rs +++ b/crates/aft/src/watcher_filter.rs @@ -2293,6 +2293,7 @@ pub fn run_watcher_thread( return; } crate::slog_info!("watcher started: {}", root_path.display()); + let _subscription = crate::watcher_backend::subscription::start(&root_path); let mut filter = WatcherFilterThread::new( config, matcher, From 2bbc1b62849c049fd9ba0aa5e710d533ac3f510f Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:35:12 +0100 Subject: [PATCH 10/31] mason: align advisory token wire types and current registry floor Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/commands/view_token.rs | 20 +++++++++++++------ crates/aft/src/reader_floor.rs | 2 +- .../src/__tests__/view-token.test.ts | 6 +++--- packages/aft-bridge/src/view-token.ts | 8 ++++++-- 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/crates/aft/src/commands/view_token.rs b/crates/aft/src/commands/view_token.rs index 29416fb372..fb14ca48a2 100644 --- a/crates/aft/src/commands/view_token.rs +++ b/crates/aft/src/commands/view_token.rs @@ -8,21 +8,29 @@ use crate::{ pub fn handle_view_token(request: &RawRequest, context: &AppContext) -> Response { if !context.config().views.enabled { - return Response::error(&request.id, "view tokens require views to be enabled"); + return Response::error( + &request.id, + "VIEW_TOKEN", + "view tokens require views to be enabled", + ); } let Some(root) = request.params.get("root").and_then(|v| v.as_str()) else { - return Response::error(&request.id, "root is required"); + return Response::error(&request.id, "VIEW_TOKEN", "root is required"); }; let Some(tree) = request.params.get("expected_tree").and_then(|v| v.as_str()) else { - return Response::error(&request.id, "expected_tree is required"); + return Response::error(&request.id, "VIEW_TOKEN", "expected_tree is required"); }; let Some(project) = context.canonical_cache_root_opt() else { - return Response::error(&request.id, "view token requires a configured repository"); + return Response::error( + &request.id, + "VIEW_TOKEN", + "view token requires a configured repository", + ); }; let storage = context.storage_paths().artifacts; let family = match crate::search_index::artifact_cache_key_read_only(&project, &storage) { Ok(family) => family, - Err(error) => return Response::error(&request.id, &error.to_string()), + Err(error) => return Response::error(&request.id, "VIEW_TOKEN", &error.to_string()), }; let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -39,7 +47,7 @@ pub fn handle_view_token(request: &RawRequest, context: &AppContext) -> Response &request.id, serde_json::json!({ "token": token.token, "expires_ms": token.expires_ms, "advisory": true }), ), - Err(error) => Response::error(&request.id, &error.to_string()), + Err(error) => Response::error(&request.id, "VIEW_TOKEN", &error.to_string()), } } diff --git a/crates/aft/src/reader_floor.rs b/crates/aft/src/reader_floor.rs index 67c786e59d..04206c2b14 100644 --- a/crates/aft/src/reader_floor.rs +++ b/crates/aft/src/reader_floor.rs @@ -787,7 +787,7 @@ pub(crate) mod tests { let formats = persisted_format::formats_json(); // The registry reads the base-reference protocol (2/2); plane stores // keep format 1. Neither appears in the startup baseline floor. - for (name, version) in [("views_v3_registry", 2), ("views_v3_store", 1)] { + for (name, version) in [("views_v3_registry", 3), ("views_v3_store", 1)] { assert_eq!(formats["stores"][name], version, "{name}"); let store = PersistedStore::from_name(name).expect("registered store"); assert!(PersistedStore::ALL.contains(&store)); diff --git a/packages/aft-bridge/src/__tests__/view-token.test.ts b/packages/aft-bridge/src/__tests__/view-token.test.ts index 561609bd30..10e22dc51b 100644 --- a/packages/aft-bridge/src/__tests__/view-token.test.ts +++ b/packages/aft-bridge/src/__tests__/view-token.test.ts @@ -7,7 +7,7 @@ test("view token uses a management request, not an agent tool", async () => { const transport = { send: async (...args: unknown[]) => { calls.push(args); - return { success: true, data: { token: "opaque", expires_ms: 100, advisory: true } }; + return { success: true, token: "opaque", expires_ms: 100, advisory: true }; }, } as unknown as AftProjectTransport; expect(await mintViewToken(transport, "/future/root", "abc")).toEqual({ @@ -18,8 +18,8 @@ test("view token uses a management request, not an agent tool", async () => { test("view token refuses a non-advisory or failed response", async () => { for (const response of [ - { success: false, error: "views disabled", data: {} }, - { success: true, data: { token: "opaque", expires_ms: 100, advisory: false } }, + { success: false, code: "VIEW_TOKEN", message: "views disabled" }, + { success: true, token: "opaque", expires_ms: 100, advisory: false }, ]) { const transport = { send: async () => response } as unknown as AftProjectTransport; await expect(mintViewToken(transport, "/future/root", "abc")).rejects.toThrow(); diff --git a/packages/aft-bridge/src/view-token.ts b/packages/aft-bridge/src/view-token.ts index b2c4fbe345..9f693acb12 100644 --- a/packages/aft-bridge/src/view-token.ts +++ b/packages/aft-bridge/src/view-token.ts @@ -17,9 +17,13 @@ export async function mintViewToken( expected_tree: expectedTree, }); if (!response.success) { - throw new Error(response.error ?? "View token mint failed"); + throw new Error(typeof response.message === "string" ? response.message : "View token mint failed"); } - const value = response.data; + const data = response; + if (typeof data !== "object" || data === null) { + throw new Error("Invalid advisory view token response"); + } + const value = data as Record; if ( typeof value.token !== "string" || typeof value.expires_ms !== "number" || From 39aff972fdb0f1bececd7c08368c2b06f4d354bf Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:38:19 +0100 Subject: [PATCH 11/31] mason: use the public trigram attachment type in v3 materialization Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/reader_floor.rs | 2 +- crates/aft/src/views/first_load.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/aft/src/reader_floor.rs b/crates/aft/src/reader_floor.rs index 04206c2b14..7b67b387ba 100644 --- a/crates/aft/src/reader_floor.rs +++ b/crates/aft/src/reader_floor.rs @@ -785,7 +785,7 @@ pub(crate) mod tests { #[test] fn v3_registration_is_advertised_but_not_prepared() { let formats = persisted_format::formats_json(); - // The registry reads the base-reference protocol (2/2); plane stores + // The registry reads verification format/schema version 3; plane stores // keep format 1. Neither appears in the startup baseline floor. for (name, version) in [("views_v3_registry", 3), ("views_v3_store", 1)] { assert_eq!(formats["stores"][name], version, "{name}"); diff --git a/crates/aft/src/views/first_load.rs b/crates/aft/src/views/first_load.rs index a6dd686002..2ca0f92d42 100644 --- a/crates/aft/src/views/first_load.rs +++ b/crates/aft/src/views/first_load.rs @@ -5323,7 +5323,7 @@ impl CompositePlane for TrigramBridge { .get(path) .filter(|e| e.disk == super::snapshot::DiskState::in_generation(delta, path)) .and_then(|e| e.attachments.get(&FamilyPlane::Trigram)) - .and_then(|a| a.downcast_ref::()) + .and_then(|a| a.downcast_ref::()) .filter(|a| a.content == *content && a.policy == self.policy.fingerprint()) .ok_or_else(|| { SiblingLoader::error( From e59e1318867545d9d5f552348a074140342a72ba Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:39:51 +0100 Subject: [PATCH 12/31] mason: keep stable-read stats distinct from persisted evidence rows Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/views/first_load.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/aft/src/views/first_load.rs b/crates/aft/src/views/first_load.rs index 2ca0f92d42..6ec588c991 100644 --- a/crates/aft/src/views/first_load.rs +++ b/crates/aft/src/views/first_load.rs @@ -2477,7 +2477,7 @@ impl FirstLoadDriver for CheckoutDriver { } } } - let (bytes, evidence) = + let (bytes, stat_evidence) = match self.stable_read(&path, &absolute, recorded_ns, delivery_seq)? { StableRead::Absent => continue, StableRead::Bytes(bytes, evidence) => (bytes, evidence), @@ -2488,7 +2488,7 @@ impl FirstLoadDriver for CheckoutDriver { continue; } }; - stats.insert(path.clone(), evidence); + stats.insert(path.clone(), stat_evidence); let content = crate::blob_store::v2::ContentHash::of(&bytes); #[cfg(test)] { From 15049cfae1de76915eeb9b52deb5ec031fa98dcb Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:09:38 +0100 Subject: [PATCH 13/31] mason: recover overflow only after a strict walk of its exact epoch Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/views/first_load.rs | 11 ++- .../first_load/overflow_recovery_tests.rs | 90 +++++++++++++++++++ 2 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 crates/aft/src/views/first_load/overflow_recovery_tests.rs diff --git a/crates/aft/src/views/first_load.rs b/crates/aft/src/views/first_load.rs index 6ec588c991..a9e0ee7ca4 100644 --- a/crates/aft/src/views/first_load.rs +++ b/crates/aft/src/views/first_load.rs @@ -14,6 +14,10 @@ use super::snapshot::{ }; use super::RelPath; +#[cfg(test)] +#[path = "first_load/overflow_recovery_tests.rs"] +mod overflow_recovery_tests; + /// A complete strict content walk under the checkout's current ignore rules. /// The revision must change for every delivered edit, write intent, watcher gap, /// or membership change; a partial walk must return an error, not this value. @@ -2195,7 +2199,12 @@ impl CheckoutDriver { } match live.watcher() { super::snapshot::WatcherState::Healthy => Ok(()), - super::snapshot::WatcherState::Reconciling if strict => Ok(()), + super::snapshot::WatcherState::Reconciling + | super::snapshot::WatcherState::Overflowed + if strict => + { + Ok(()) + } _ => Err(SiblingLoader::error(format!( "{CHECKOUT_CHANGED_BEFORE_INSTALL}: watcher is not healthy or strictly reconciled" ))), diff --git a/crates/aft/src/views/first_load/overflow_recovery_tests.rs b/crates/aft/src/views/first_load/overflow_recovery_tests.rs new file mode 100644 index 0000000000..57f60e3dbb --- /dev/null +++ b/crates/aft/src/views/first_load/overflow_recovery_tests.rs @@ -0,0 +1,90 @@ +use super::*; + +fn fixture() -> ( + tempfile::TempDir, + tempfile::TempDir, + Arc, + ViewAccess, + crate::watcher_backend::subscription::Subscription, +) { + let root = tempfile::tempdir().unwrap(); + let storage = tempfile::tempdir().unwrap(); + std::fs::write(root.path().join("file.rs"), b"fn content() {}\n").unwrap(); + let subscription = + crate::watcher_backend::subscription::start(&std::fs::canonicalize(root.path()).unwrap()); + let policy = crate::blob_store::v2::TrigramPolicy { + max_file_size: 1 << 20, + }; + let driver = Arc::new( + CheckoutDriver::new_v3( + storage.path(), + "family", + "worker", + root.path(), + Producers { + trigram: policy.fingerprint_hex(), + semantic: None, + callgraph: "unregistered".into(), + }, + Arc::new(ConfiguredMembershipWalker), + vec![Arc::new(TrigramBridge::new( + storage.path().to_owned(), + policy, + ))], + ) + .unwrap(), + ); + let access = driver.access(); + let (snapshot, revision) = driver + .verified_bind(&access, driver.installed_snapshot().generation().clone()) + .unwrap(); + driver.install(&access, &snapshot, revision).unwrap(); + driver.take_reconcile_work(); + (root, storage, driver, access, subscription) +} + +#[test] +fn completed_strict_reconcile_recovers_the_overflow_epoch_it_covered() { + let _serial = crate::blob_store::v3::serial_for_test(); + let (_root, _storage, driver, access, _subscription) = fixture(); + driver.record_watcher_overflow(); + let epoch = driver.live.lock().unwrap().epoch(); + let (snapshot, revision) = driver + .verified_bind(&access, driver.installed_snapshot().generation().clone()) + .unwrap(); + assert_eq!(driver.take_reconcile_work().walks, 1); + assert_eq!(snapshot.reconciled_epoch(), Some(epoch)); + assert_eq!( + driver.live.lock().unwrap().watcher(), + super::super::snapshot::WatcherState::Overflowed + ); + driver.install(&access, &snapshot, revision).unwrap(); + assert_eq!( + driver.live.lock().unwrap().watcher(), + super::super::snapshot::WatcherState::Healthy + ); +} + +#[test] +fn overflow_during_strict_reconcile_cannot_be_cleared_by_its_older_cut() { + let _serial = crate::blob_store::v3::serial_for_test(); + let (_root, _storage, driver, access, _subscription) = fixture(); + driver.record_watcher_overflow(); + let covered = driver.live.lock().unwrap().epoch(); + driver.observed.lock().unwrap().stats.clear(); + let weak = Arc::downgrade(&driver); + *driver.read_hook.lock().unwrap() = Some(Box::new(move |_, _| { + weak.upgrade().unwrap().record_watcher_overflow() + })); + let (snapshot, revision) = driver + .verified_bind(&access, driver.installed_snapshot().generation().clone()) + .unwrap(); + assert_eq!(driver.take_reconcile_work().walks, 1); + assert_eq!(snapshot.reconciled_epoch(), Some(covered)); + assert!(driver.live.lock().unwrap().epoch() > covered); + assert!(driver.install(&access, &snapshot, revision).is_err()); + assert_eq!( + driver.live.lock().unwrap().watcher(), + super::super::snapshot::WatcherState::Overflowed + ); +} From 8aa9c7c2211f7f13be8c6e48975ab79603dd9a23 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:23:24 +0100 Subject: [PATCH 14/31] mason: close provenance gaps and guard steady refresh by subscription continuity Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/git_index_reader.rs | 40 ++- crates/aft/src/views/first_load.rs | 256 +++++++++++++++++- crates/aft/src/views/io.rs | 3 + crates/aft/src/views/registry.rs | 63 ++++- crates/aft/src/views/verification.rs | 374 +++++++++++++++++++++++++-- docs/design/views-v3-format.md | 9 + 6 files changed, 705 insertions(+), 40 deletions(-) diff --git a/crates/aft/src/git_index_reader.rs b/crates/aft/src/git_index_reader.rs index 2356bea40e..4de1a275fd 100644 --- a/crates/aft/src/git_index_reader.rs +++ b/crates/aft/src/git_index_reader.rs @@ -1,8 +1,7 @@ //! Read-only Git index snapshots and counted, deadline-bound Git metadata queries. //! -//! Nothing in production routes through this module yet. A snapshot records only -//! the index bytes observed, not proof that worktree bytes are clean. Callers must -//! verify worktree content independently. +//! A snapshot records the index bytes observed, not proof that working-tree bytes +//! are clean. Callers must verify working-tree content independently. use crate::file_identity::FileIdentity; use sha1::Digest; @@ -549,6 +548,21 @@ impl GitQueries { pub fn config(&self) -> io::Result { self.run(&["config", "--list", "--show-origin", "-z"], Vec::new()) } + /// Ask Git for its compiled/default attribute path rather than guessing an + /// installation prefix. Older Git versions may refuse this variable. + pub fn attribute_source(&self, global: bool) -> io::Result { + self.run( + &[ + "var", + if global { + "GIT_ATTR_GLOBAL" + } else { + "GIT_ATTR_SYSTEM" + }, + ], + Vec::new(), + ) + } fn run(&self, args: &[&str], input: Vec) -> io::Result { let mut command = Command::new("git"); @@ -583,6 +597,9 @@ fn pump( .stderr(Stdio::piped()) .spawn()?; spawned.fetch_add(1, Ordering::Relaxed); + crate::views::io::WORK + .git_subprocesses + .fetch_add(1, Ordering::Relaxed); let mut stdin = child.stdin.take().expect("piped stdin"); let mut stdout = child.stdout.take().expect("piped stdout"); let mut stderr = child.stderr.take().expect("piped stderr"); @@ -1288,3 +1305,20 @@ mod tests { thread::sleep(Duration::from_secs(60)); } } +#[test] +fn attribute_source_queries_count_both_spawns_even_on_unsupported_git() { + let root = tempfile::tempdir().unwrap(); + let queries = GitQueries::new(root.path().to_owned(), Duration::from_secs(2)); + let before = crate::views::io::WORK + .git_subprocesses + .load(Ordering::Relaxed); + queries.attribute_source(false).unwrap(); + queries.attribute_source(true).unwrap(); + assert_eq!(queries.spawned(), 2); + assert!( + crate::views::io::WORK + .git_subprocesses + .load(Ordering::Relaxed) + >= before + 2 + ); +} diff --git a/crates/aft/src/views/first_load.rs b/crates/aft/src/views/first_load.rs index a9e0ee7ca4..0e24c00831 100644 --- a/crates/aft/src/views/first_load.rs +++ b/crates/aft/src/views/first_load.rs @@ -978,7 +978,7 @@ mod tests { drop(generation); } - fn inventory(root: &std::path::Path) -> BTreeMap> { + fn legacy_inventory(storage: &std::path::Path) -> BTreeMap> { fn visit( root: &std::path::Path, dir: &std::path::Path, @@ -1364,9 +1364,9 @@ pub struct InstalledBundle { own_installed: bool, } -/// In-memory evidence that a previous source read still describes this file. -/// No stamps survive a restart. Platforms without a kernel change time and -/// file identity keep reading bytes instead of trusting mtime alone. +/// Evidence that a previous source read still describes this file. V3 stores +/// stable observations with their published generation. Missing change-time +/// or file-identity support means reading bytes instead of trusting mtime alone. #[derive(Clone)] struct ObservedStat { stamp: crate::watcher::FileStamp, @@ -1971,6 +1971,7 @@ impl CheckoutDriver { } pub fn installed_bundle(&self) -> Arc { + self.check_subscription(); self.installed .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) @@ -2332,6 +2333,7 @@ impl FirstLoadDriver for CheckoutDriver { .as_ref() .zip(previous.guard.as_ref()) .is_some_and(|(now, old)| now.root == old.root && now.policy == old.policy) + && guard.as_ref().is_some_and(|guard| guard.policy.reusable) && !policy_or_directory; let changed_files = if continuous { self.walker.changed_files(self.owner.root(), &candidates)? @@ -3172,6 +3174,7 @@ impl QueryState for CheckoutDriver { access: &ViewAccess, plane: FamilyPlane, ) -> (Snapshot, Vec) { + self.check_subscription(); let installed = self .installed .lock() @@ -5369,6 +5372,28 @@ impl CompositePlane for TrigramBridge { } impl CheckoutDriver { + fn check_subscription(&self) { + let stale = { + let installed = self + .installed + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + installed.snapshot.watcher() == super::snapshot::WatcherState::Healthy + && installed + .snapshot + .install_cut() + .and_then(|cut| cut.verification.as_ref()) + .and_then(|guard| guard.subscription) + .is_some_and(|expected| { + crate::watcher_backend::subscription::current(self.owner.root()) + != Some(expected) + }) + }; + if stale { + self.record_watcher_overflow(); + } + } + fn source_stat( &self, path: &std::path::Path, @@ -5381,7 +5406,7 @@ impl CheckoutDriver { match std::fs::symlink_metadata(path) { Ok(metadata) if metadata.is_file() => { let mut stat = ObservedStat::new(&metadata, recorded_ns); - stat.native = super::verification::NativeStat::read(path).ok(); + stat.native = super::verification::NativeStat::strict_read(path).ok(); Ok(Some(stat)) } Ok(_) => Ok(None), @@ -5406,6 +5431,7 @@ impl CheckoutDriver { /// The snapshot last installed for this checkout, without computing the /// per-plane gaps `installed_state` reports. pub fn installed_snapshot(&self) -> Snapshot { + self.check_subscription(); self.installed .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) @@ -5735,12 +5761,152 @@ mod checkout_kind_tests { mod v3_bootstrap_tests { use super::*; + fn trigram_driver(root: &std::path::Path, storage: &std::path::Path) -> CheckoutDriver { + let policy = crate::blob_store::v2::TrigramPolicy { + max_file_size: 1 << 20, + }; + let plane = Arc::new(TrigramBridge::new(storage.to_owned(), policy)); + CheckoutDriver::new_v3( + storage, + "family", + "worker", + root, + Producers { + trigram: policy.fingerprint_hex(), + semantic: None, + callgraph: "unregistered".into(), + }, + Arc::new(ConfiguredMembershipWalker), + vec![plane.clone()], + ) + .unwrap() + .with_adapters(vec![plane.adapter.clone()]) + } + + #[test] + fn continuous_refresh_reads_only_reported_paths_and_gaps_or_policy_edits_walk() { + let _serial = crate::blob_store::v3::serial_for_test(); + let root = tempfile::tempdir().unwrap(); + let storage = tempfile::tempdir().unwrap(); + let _attributes = super::super::verification::known_attribute_sources_for_test(root.path()); + for name in ["file.rs", "other.rs"] { + std::fs::write(root.path().join(name), b"fn before() {}\n").unwrap(); + } + let subscription = crate::watcher_backend::subscription::start( + &std::fs::canonicalize(root.path()).unwrap(), + ); + let driver = trigram_driver(root.path(), storage.path()); + let access = driver.access(); + let (first, revision) = driver + .verified_bind(&access, driver.installed_snapshot().generation().clone()) + .unwrap(); + driver.install(&access, &first, revision).unwrap(); + driver.take_reconcile_work(); + std::fs::write(root.path().join("file.rs"), b"fn edited() {}\n").unwrap(); + driver.record_change(RelPath::new(b"file.rs".to_vec()).unwrap()); + let (changed, revision) = driver + .verified_bind(&access, first.generation().clone()) + .unwrap(); + let work = driver.take_reconcile_work(); + assert_eq!(work.walks, 0); + assert_eq!(work.reads, 1); + assert!(work.stats <= 3); + driver.install(&access, &changed, revision).unwrap(); + assert!(!driver.installed_snapshot().needs_reconcile()); + driver.record_watcher_overflow(); + let (reconciled, revision) = driver + .verified_bind(&access, changed.generation().clone()) + .unwrap(); + assert_eq!(driver.take_reconcile_work().walks, 1); + driver.install(&access, &reconciled, revision).unwrap(); + std::fs::write(root.path().join(".gitignore"), b"other.rs\n").unwrap(); + driver.record_change(RelPath::new(b".gitignore".to_vec()).unwrap()); + let (ignored, revision) = driver + .verified_bind(&access, reconciled.generation().clone()) + .unwrap(); + assert_eq!(driver.take_reconcile_work().walks, 1); + assert_eq!( + ignored.disk_state(&RelPath::new(b"other.rs".to_vec()).unwrap()), + super::super::snapshot::DiskState::Absent + ); + driver.install(&access, &ignored, revision).unwrap(); + drop(subscription); + assert!(driver.installed_snapshot().needs_reconcile()); + assert_eq!( + driver.installed_snapshot().watcher(), + super::super::snapshot::WatcherState::Overflowed + ); + } + + #[test] + fn v3_lazy_attachments_probe_the_actual_producer_key_before_extraction() { + let _serial = crate::blob_store::v3::serial_for_test(); + let root = tempfile::tempdir().unwrap(); + let storage = tempfile::tempdir().unwrap(); + let bytes = b"pub fn cached() -> u32 { 42 }\n"; + std::fs::write(root.path().join("file.rs"), bytes).unwrap(); + let path = RelPath::new(b"file.rs".to_vec()).unwrap(); + let content = crate::blob_store::v2::ContentHash::of(bytes); + let policy = crate::blob_store::v2::TrigramPolicy { + max_file_size: 1 << 20, + }; + let graph = CallgraphBridge::default(); + let attachment = graph.attachment(&path, bytes).unwrap(); + let extracted = attachment + .downcast_ref::() + .unwrap(); + assert_eq!(graph.key(&path, content), Some(extracted.key)); + let trigram_key = crate::blob_store::v2::TrigramKey { content, policy }.family_key(); + let trigram = super::super::segment_store::TrigramPayload::extract(bytes, &policy).encode(); + let graph_payload = extracted.blob.to_bytes().unwrap(); + crate::blob_store::v3::publish(storage.path(), "family", 1 << 20, None, |p| { + p.put( + &trigram_key, + &trigram, + FamilyPlane::Trigram.payload_schema(), + )?; + p.put( + &extracted.key, + &graph_payload, + FamilyPlane::Callgraph.payload_schema(), + )?; + Ok(()) + }) + .unwrap(); + for (family, expected_extractions) in [("family", 0), ("miss", 2)] { + let driver = CheckoutDriver::new_v3( + storage.path(), + family, + "worker", + root.path(), + Producers { + trigram: policy.fingerprint_hex(), + semantic: None, + callgraph: super::super::callgraph::PRODUCER.into(), + }, + Arc::new(ConfiguredMembershipWalker), + vec![ + Arc::new(TrigramBridge::new(storage.path().to_owned(), policy)), + Arc::new(CallgraphBridge::default()), + ], + ) + .unwrap(); + driver.reconcile(&driver.access()).unwrap(); + let work = driver.take_reconcile_work(); + assert_eq!(work.reads, 1); + assert_eq!(work.attachments, expected_extractions); + } + } + fn inventory(root: &std::path::Path) -> BTreeMap> { fn visit( base: &std::path::Path, path: &std::path::Path, files: &mut BTreeMap>, ) { + if path == base.join("v3") { + return; + } if !path.exists() { return; } @@ -5757,7 +5923,8 @@ mod v3_bootstrap_tests { } } let mut files = BTreeMap::new(); - visit(root, root, &mut files); + let root = storage.join("blobs"); + visit(&root, &root, &mut files); files } @@ -5766,6 +5933,7 @@ mod v3_bootstrap_tests { let _serial = crate::blob_store::v3::serial_for_test(); let root = tempfile::tempdir().unwrap(); let storage = tempfile::tempdir().unwrap(); + let _attributes = super::super::verification::known_attribute_sources_for_test(root.path()); let source = b"fn fresh_checkout() { let marker = 42; }\n"; std::fs::write(root.path().join("file.rs"), source).unwrap(); std::fs::File::open(root.path().join("file.rs")) @@ -5781,7 +5949,7 @@ mod v3_bootstrap_tests { std::fs::create_dir_all(dir).unwrap(); std::fs::write(dir.join("untouched"), b"legacy sentinel").unwrap(); } - let before = (inventory(&legacy_v1), inventory(&legacy_v2)); + let before = legacy_inventory(storage.path()); let policy = crate::blob_store::v2::TrigramPolicy { max_file_size: 1 << 20, }; @@ -5798,12 +5966,13 @@ mod v3_bootstrap_tests { root.path(), producers.clone(), Arc::new(ConfiguredMembershipWalker), - vec![plane], + vec![plane.clone()], ) - .unwrap(); + .unwrap() + .with_adapters(vec![plane.adapter.clone()]); let access = driver.access(); assert!(driver.owner.open_store(FamilyPlane::Trigram).is_err()); - assert_eq!((inventory(&legacy_v1), inventory(&legacy_v2)), before); + assert_eq!(legacy_inventory(storage.path()), before); let empty = driver.installed_snapshot().generation().clone(); let (snapshot, revision) = driver.verified_bind(&access, empty).unwrap(); assert!(snapshot.generation().base_header().is_some()); @@ -5820,6 +5989,10 @@ mod v3_bootstrap_tests { )); driver.install(&access, &snapshot, revision).unwrap(); assert!(!driver.installed_snapshot().needs_reconcile()); + assert!(driver + .installed_state(&access, FamilyPlane::Trigram) + .1 + .is_empty()); let route = driver.owner.v3_route(); let store = super::super::ViewStore::existing_dir(route.member_dir()).unwrap(); let generation = store.current_generation_read_only().unwrap().unwrap(); @@ -5832,7 +6005,7 @@ mod v3_bootstrap_tests { assert!( super::super::verification::evidence_path(&route.member_dir(), &generation).exists() ); - assert_eq!((inventory(&legacy_v1), inventory(&legacy_v2)), before); + assert_eq!(legacy_inventory(storage.path()), before); let ledger = crate::blob_store::v3::LedgerReader::open_existing(storage.path(), "family").unwrap(); assert_eq!( @@ -5876,6 +6049,65 @@ mod v3_bootstrap_tests { work.reads, 0, "generation-paired evidence avoids rereading stable source" ); - assert_eq!((inventory(&legacy_v1), inventory(&legacy_v2)), before); + let other_content = crate::blob_store::v2::ContentHash::of(b"other bytes"); + let other_key = crate::blob_store::v2::TrigramKey { + content: other_content, + policy, + } + .family_key(); + let other_payload = + super::super::segment_store::TrigramPayload::extract(b"other bytes", &policy).encode(); + crate::blob_store::v3::publish(storage.path(), "family", 1 << 20, None, |p| { + p.put( + &other_key, + &other_payload, + FamilyPlane::Trigram.payload_schema(), + )?; + Ok(()) + }) + .unwrap(); + let evidence = rusqlite::Connection::open(super::super::verification::evidence_path( + &route.member_dir(), + &generation, + )) + .unwrap(); + let bytes: Vec = evidence + .query_row( + "SELECT record FROM stat_evidence WHERE path=?1", + [path.as_bytes()], + |r| r.get(0), + ) + .unwrap(); + let mut record: super::super::verification::Evidence = + serde_json::from_slice(&bytes).unwrap(); + record.keys.insert("Trigram".into(), other_key.to_hex()); + evidence + .execute( + "UPDATE stat_evidence SET record=?1 WHERE path=?2", + rusqlite::params![serde_json::to_vec(&record).unwrap(), path.as_bytes()], + ) + .unwrap(); + drop(evidence); + let third = trigram_driver(root.path(), storage.path()); + let base = super::super::base_refs::open_foreign_v3( + storage.path(), + "family", + "worker", + &third.producers, + ) + .unwrap() + .unwrap(); + third.set_view_token("forged-advisory-token".into()); + let (checked, _) = third.verified_bind(&third.access(), base).unwrap(); + assert_eq!( + third.take_reconcile_work().reads, + 1, + "an existing but unrelated key is not stat provenance, even with a token" + ); + assert_eq!( + checked.disk_state(&path), + super::super::snapshot::DiskState::of_bytes(source) + ); + assert_eq!(legacy_inventory(storage.path()), before); } } diff --git a/crates/aft/src/views/io.rs b/crates/aft/src/views/io.rs index 0846fb354c..d63b65c250 100644 --- a/crates/aft/src/views/io.rs +++ b/crates/aft/src/views/io.rs @@ -13,6 +13,7 @@ pub(crate) struct WorkCounters { pub content_reads: AtomicU64, pub content_read_bytes: AtomicU64, pub full_walks: AtomicU64, + pub git_subprocesses: AtomicU64, pub applicability_walks: AtomicU64, pub logical_aggregates: AtomicU64, pub own_generation_folds: AtomicU64, @@ -35,6 +36,7 @@ pub(crate) static WORK: WorkCounters = WorkCounters { content_reads: AtomicU64::new(0), content_read_bytes: AtomicU64::new(0), full_walks: AtomicU64::new(0), + git_subprocesses: AtomicU64::new(0), applicability_walks: AtomicU64::new(0), logical_aggregates: AtomicU64::new(0), own_generation_folds: AtomicU64::new(0), @@ -74,6 +76,7 @@ pub(crate) fn work_snapshot() -> serde_json::Value { "content_reads": WORK.content_reads.load(Ordering::Relaxed), "content_read_bytes": WORK.content_read_bytes.load(Ordering::Relaxed), "full_walks": WORK.full_walks.load(Ordering::Relaxed), + "git_subprocesses": WORK.git_subprocesses.load(Ordering::Relaxed), "applicability_walks": WORK.applicability_walks.load(Ordering::Relaxed), "logical_aggregates": WORK.logical_aggregates.load(Ordering::Relaxed), "own_generation_folds": WORK.own_generation_folds.load(Ordering::Relaxed), diff --git a/crates/aft/src/views/registry.rs b/crates/aft/src/views/registry.rs index beef416018..2ab2581a60 100644 --- a/crates/aft/src/views/registry.rs +++ b/crates/aft/src/views/registry.rs @@ -1625,8 +1625,8 @@ pub fn consume_view_token( Err(_) => return Ok(TokenUse::IdentityMismatch), }; let identities = match ( - super::verification::NativeStat::read(&root), - super::verification::NativeStat::read(&directory), + super::verification::NativeStat::strict_read(&root), + super::verification::NativeStat::strict_read(&directory), ) { (Ok(root), Ok(gitdir)) => (root.identity, (directory, gitdir.identity)), _ => return Ok(TokenUse::IdentityMismatch), @@ -1982,6 +1982,65 @@ mod view_token_tests { use super::*; const TREE: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + #[test] + fn view_token_upgrade_preserves_existing_registry_two_members_and_sessions() { + let _serial = crate::blob_store::v3::serial_for_test(); + let storage = tempfile::tempdir().unwrap(); + let root = tempfile::tempdir().unwrap(); + super::super::base_refs::with_base_barrier(storage.path(), "family", |tx| { + super::super::base_refs::register_view_in( + tx, + storage.path(), + "family", + "before", + root.path(), + )?; + tx.execute( + "INSERT INTO view_sessions VALUES('before-session','before',123,456,789)", + [], + )?; + Ok(()) + }) + .unwrap(); + let reader = + crate::blob_store::v3::LedgerReader::open_existing(storage.path(), "family").unwrap(); + assert_eq!( + reader + .read(|c| Ok(c + .query_row("SELECT format_version FROM format_meta", [], |r| r + .get::<_, u32>(0))?)) + .unwrap(), + 2 + ); + mint_view_token( + storage.path(), + "family", + &root.path().join("future"), + TREE, + 1, + ) + .unwrap(); + assert_eq!( + reader + .read(|c| Ok(c + .query_row("SELECT format_version FROM format_meta", [], |r| r + .get::<_, u32>(0))?)) + .unwrap(), + 3 + ); + assert_eq!( + reader + .read(|c| Ok(c.query_row( + "SELECT COUNT(*) FROM members WHERE scope='before'", + [], + |r| r.get::<_, u32>(0) + )?)) + .unwrap(), + 1 + ); + assert_eq!(reader.read(|c| Ok(c.query_row("SELECT owner_pid,owner_start,opened_at_ms FROM view_sessions WHERE session_id='before-session'", [], |r| Ok((r.get::<_, u32>(0)?,r.get::<_, u32>(1)?,r.get::<_, u32>(2)?)))?)).unwrap(), (123,456,789)); + } + #[test] fn view_token_mints_before_root_exists_and_binds_both_identities_once() { let _serial = crate::blob_store::v3::serial_for_test(); diff --git a/crates/aft/src/views/verification.rs b/crates/aft/src/views/verification.rs index 4ece51f167..5521990ffb 100644 --- a/crates/aft/src/views/verification.rs +++ b/crates/aft/src/views/verification.rs @@ -10,6 +10,28 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use serde::{Deserialize, Serialize}; use super::RelPath; + +#[cfg(test)] +thread_local! { + static ATTRIBUTE_SOURCES: std::cell::RefCell> = const { std::cell::RefCell::new(None) }; +} +#[cfg(test)] +pub(crate) struct AttributeSourcesGuard(Option<[PathBuf; 2]>); +#[cfg(test)] +impl Drop for AttributeSourcesGuard { + fn drop(&mut self) { + ATTRIBUTE_SOURCES.with(|paths| *paths.borrow_mut() = self.0.take()); + } +} +#[cfg(test)] +pub(crate) fn known_attribute_sources_for_test(root: &Path) -> AttributeSourcesGuard { + let sources = [ + root.join("absent-system-attributes"), + root.join("absent-global-attributes"), + ]; + assert!(sources.iter().all(|p| !p.exists())); + AttributeSourcesGuard(ATTRIBUTE_SOURCES.with(|paths| paths.replace(Some(sources)))) +} use crate::blob_store::v2::{ContentHash, FamilyKey, FamilyPlane}; #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] @@ -24,6 +46,11 @@ pub(crate) enum FileIdentity { id: [u8; 16], creation: i64, }, + StrictOnly { + volume: u64, + index: u64, + birth: Option, + }, } #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] @@ -33,6 +60,7 @@ pub(crate) struct NativeStat { pub modified_ns: i128, pub changed_ns: i128, pub mode: u32, + pub reuse_supported: bool, } impl NativeStat { @@ -55,6 +83,7 @@ impl NativeStat { modified_ns: i128::from(m.mtime()) * 1_000_000_000 + i128::from(m.mtime_nsec()), changed_ns: i128::from(m.ctime()) * 1_000_000_000 + i128::from(m.ctime_nsec()), mode: m.mode(), + reuse_supported: local_precise_filesystem(path), }) } #[cfg(windows)] @@ -71,7 +100,69 @@ impl NativeStat { } } pub fn reusable(&self, current: &Self, observed_ns: u128) -> bool { - self == current && self.modified_ns >= 0 && (self.modified_ns as u128) < observed_ns + self.reuse_supported + && self == current + && self.modified_ns >= 0 + && (self.modified_ns as u128) < observed_ns + } + + pub fn strict_read(path: &Path) -> io::Result { + match Self::read(path) { + Ok(stat) => Ok(stat), + Err(error) => { + let (metadata, identity) = crate::file_identity::symlink_metadata(path)?; + let crate::file_identity::FileIdentity::Known { volume, index } = identity else { + return Err(error); + }; + Ok(Self { + identity: FileIdentity::StrictOnly { + volume, + index, + birth: metadata + .created() + .ok() + .and_then(|t| t.duration_since(UNIX_EPOCH).ok()) + .map(|d| d.as_nanos()), + }, + size: metadata.len(), + modified_ns: metadata + .modified() + .ok() + .and_then(|t| t.duration_since(UNIX_EPOCH).ok()) + .map_or(-1, |d| d.as_nanos() as i128), + changed_ns: -1, + mode: 0, + reuse_supported: false, + }) + } + } + } +} + +#[cfg(unix)] +fn local_precise_filesystem(path: &Path) -> bool { + use std::os::unix::ffi::OsStrExt; + let Ok(path) = std::ffi::CString::new(path.as_os_str().as_bytes()) else { + return false; + }; + let mut stat: libc::statfs = unsafe { std::mem::zeroed() }; + if unsafe { libc::statfs(path.as_ptr(), &mut stat) } != 0 { + return false; + } + #[cfg(target_os = "linux")] + { + matches!( + stat.f_type as u64, + 0xef53 | 0x9123683e | 0x58465342 | 0x794c7630 | 0x01021994 | 0x2fc12fc1 + ) + } + #[cfg(target_os = "macos")] + { + unsafe { std::ffi::CStr::from_ptr(stat.f_fstypename.as_ptr()) }.to_bytes() == b"apfs" + } + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + false } } @@ -144,6 +235,7 @@ fn windows_stat(path: &Path) -> io::Result { modified_ns: unix_ns(basic.LastWriteTime), changed_ns: unix_ns(basic.ChangeTime), mode: basic.FileAttributes, + reuse_supported: true, }) } @@ -199,15 +291,71 @@ fn metadata_source(path: &Path, hash: &mut blake3::Hasher) -> io::Result<()> { impl Policy { pub fn capture(root: &Path, metadata: &[PathBuf]) -> io::Result { - let mut hash = blake3::Hasher::new(); - hash.update(b"aft/views-v3/conversion-policy/1"); let queries = crate::git_index_reader::GitQueries::new(root.to_path_buf(), Duration::from_secs(2)); - let output = queries.config(); + #[cfg(test)] + if let Some(paths) = ATTRIBUTE_SOURCES.with(|paths| paths.borrow().clone()) { + #[cfg(unix)] + use std::os::unix::process::ExitStatusExt; + #[cfg(windows)] + use std::os::windows::process::ExitStatusExt; + return Self::capture_sources( + root, + metadata, + queries.config(), + paths.map(|path| { + Ok(std::process::Output { + status: std::process::ExitStatus::from_raw(0), + stdout: path.as_os_str().as_encoded_bytes().to_vec(), + stderr: Vec::new(), + }) + }), + ); + } + Self::capture_sources( + root, + metadata, + queries.config(), + [ + queries.attribute_source(false), + queries.attribute_source(true), + ], + ) + } + + fn capture_sources( + root: &Path, + metadata: &[PathBuf], + output: io::Result, + sources: [io::Result; 2], + ) -> io::Result { + let mut hash = blake3::Hasher::new(); + hash.update(b"aft/views-v3/conversion-policy/1"); let mut reusable = output.as_ref().is_ok_and(|output| output.status.success()); hash.update(&[u8::from(reusable)]); let config = output.map(|output| output.stdout).unwrap_or_default(); hash.update(&config); + for output in sources { + match output { + Ok(output) if output.status.success() => { + let bytes = output.stdout.strip_suffix(b"\n").unwrap_or(&output.stdout); + let bytes = bytes.strip_suffix(b"\r").unwrap_or(bytes); + if bytes.is_empty() || bytes.contains(&b'\n') { + reusable = false; + continue; + } + let path = path_from_bytes(bytes)?; + let path = if path.is_absolute() { + path + } else { + root.join(path) + }; + metadata_source(&path, &mut hash)?; + reusable &= !path.exists(); + } + _ => reusable = false, + } + } // Config --show-origin records include targets and worktree config. // Hash their bytes as well as the effective values, including empty files. for item in config.split(|b| *b == 0) { @@ -229,9 +377,28 @@ impl Policy { { reusable = false; } - if let Some(path) = item.strip_prefix(b"core.attributesfile\n") { - metadata_source(&path_from_bytes(path)?, &mut hash)?; - reusable = false; + for (key, conversion) in [ + (b"core.attributesfile\n".as_slice(), true), + (b"core.excludesfile\n".as_slice(), false), + ] { + if let Some(path) = item.strip_prefix(key) { + let path = path_from_bytes(path)?; + let path = + if let Some(suffix) = path.to_str().and_then(|p| p.strip_prefix("~/")) { + std::env::var_os("HOME") + .or_else(|| std::env::var_os("USERPROFILE")) + .map(|home| PathBuf::from(home).join(suffix)) + .ok_or_else(|| { + io::Error::other("cannot resolve Git policy home directory") + })? + } else if path.is_absolute() { + path + } else { + root.join(path) + }; + metadata_source(&path, &mut hash)?; + reusable &= !conversion; + } } } for path in metadata { @@ -246,16 +413,6 @@ impl Policy { reusable = false; } } - if let Some(home) = std::env::var_os("HOME").or_else(|| std::env::var_os("USERPROFILE")) { - for relative in [".gitattributes", ".config/git/attributes"] { - let path = PathBuf::from(&home).join(relative); - metadata_source(&path, &mut hash)?; - reusable &= !path.exists(); - } - } - let system = Path::new("/etc/gitattributes"); - metadata_source(system, &mut hash)?; - reusable &= !system.exists(); Ok(Self { digest: hash.finalize().to_hex().to_string(), reusable, @@ -289,7 +446,7 @@ pub(crate) fn now_ns() -> u128 { /// Enumerate policy inputs with metadata only. Source files are never opened. pub(crate) fn policy_inputs(root: &Path) -> io::Result> { fn walk(root: &Path, out: &mut Vec) -> io::Result<()> { - for name in [".gitattributes", ".gitignore", ".ignore"] { + for name in [".gitattributes", ".gitignore", ".ignore", ".aftignore"] { out.push(root.join(name)); } for entry in fs::read_dir(root)? { @@ -324,6 +481,22 @@ pub(crate) fn policy_inputs(root: &Path) -> io::Result> { if let Some(gitdir) = gitdir { paths.push(gitdir.join("info/attributes")); paths.push(gitdir.join("info/exclude")); + let common_marker = gitdir.join("commondir"); + match fs::read(&common_marker) { + Ok(bytes) => { + let common = path_from_bytes(bytes.trim_ascii())?; + let common = if common.is_absolute() { + common + } else { + gitdir.join(common) + }; + paths.push(common.join("info/attributes")); + paths.push(common.join("info/exclude")); + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + paths.push(common_marker); } paths.sort(); Ok(paths) @@ -365,10 +538,10 @@ impl BindGuard { Some(inputs) => inputs.to_vec(), None => policy_inputs(root)?, }; - let policy = Policy::capture(root, &inputs)?; - let root_identity = NativeStat::read(root)?.identity; + let mut policy = Policy::capture(root, &inputs)?; + let root_identity = NativeStat::strict_read(root)?.identity; let directory = match gitdir(root) { - Ok(path) => Some((path.clone(), NativeStat::read(&path)?.identity)), + Ok(path) => Some((path.clone(), NativeStat::strict_read(&path)?.identity)), Err(e) if e.kind() == io::ErrorKind::NotFound => None, Err(e) => return Err(e), }; @@ -384,14 +557,40 @@ impl BindGuard { head_tree = Some(head.stdout); } let path = directory.join("index"); - match NativeStat::read(&path) { + match NativeStat::strict_read(&path) { Ok(before) => { let bytes = fs::read(&path)?; - let after = NativeStat::read(&path)?; + let after = NativeStat::strict_read(&path)?; if before != after { return Err(io::Error::other("Git index changed during verification")); } index = Some((before, *blake3::hash(&bytes).as_bytes())); + let format = if head_tree + .as_ref() + .is_some_and(|tree| tree.trim_ascii().len() == 64) + { + crate::git_index_reader::ObjectFormat::Sha256 + } else { + crate::git_index_reader::ObjectFormat::Sha1 + }; + match crate::git_index_reader::read_index(&path, format, Default::default()) { + crate::git_index_reader::IndexRead::Snapshot(snapshot) => { + let mut digest = blake3::Hasher::new(); + digest.update(policy.digest.as_bytes()); + for entry in snapshot.entries.iter().filter(|entry| { + entry.path.rsplit(|b| *b == b'/').next() == Some(b".gitattributes") + }) { + policy.reusable = false; + digest.update(&(entry.path.len() as u64).to_le_bytes()); + digest.update(&entry.path); + digest.update(&entry.oid); + } + policy.digest = digest.finalize().to_hex().to_string(); + } + crate::git_index_reader::IndexRead::Strict(_) => { + policy.reusable = false; + } + } } Err(e) if e.kind() == io::ErrorKind::NotFound => {} Err(e) => return Err(e), @@ -540,6 +739,75 @@ pub(crate) fn read_evidence( mod tests { use super::*; + fn success(bytes: &[u8]) -> io::Result { + #[cfg(unix)] + use std::os::unix::process::ExitStatusExt; + #[cfg(windows)] + use std::os::windows::process::ExitStatusExt; + Ok(std::process::Output { + status: std::process::ExitStatus::from_raw(0), + stdout: bytes.to_vec(), + stderr: Vec::new(), + }) + } + + #[test] + fn conversion_or_unknown_attribute_sources_disable_stat_provenance() { + let root = tempfile::tempdir().unwrap(); + let sources = || { + [ + success( + root.path() + .join("system-attributes") + .as_os_str() + .as_encoded_bytes(), + ), + success( + root.path() + .join("global-attributes") + .as_os_str() + .as_encoded_bytes(), + ), + ] + }; + assert!( + Policy::capture_sources(root.path(), &[], success(b""), sources()) + .unwrap() + .reusable + ); + for config in [ + b"core.autocrlf\ntrue\0".as_slice(), + b"core.eol\ncrlf\0", + b"filter.custom.clean\ncustom-driver\0", + ] { + assert!( + !Policy::capture_sources(root.path(), &[], success(config), sources()) + .unwrap() + .reusable + ); + } + assert!( + !Policy::capture_sources( + root.path(), + &[], + success(b""), + [ + Err(io::Error::other("unsupported Git source query")), + success(b"unused") + ] + ) + .unwrap() + .reusable + ); + let attribute = root.path().join(".gitattributes"); + fs::write(&attribute, b"*.rs text\n").unwrap(); + assert!( + !Policy::capture_sources(root.path(), &[attribute], success(b""), sources()) + .unwrap() + .reusable + ); + } + fn git(root: &Path, args: &[&str]) { let output = std::process::Command::new("git") .current_dir(root) @@ -553,6 +821,60 @@ mod tests { ); } + #[test] + fn failed_config_disables_reuse_but_still_hashes_known_policy_sources() { + let root = tempfile::tempdir().unwrap(); + let source = root.path().join(".gitattributes"); + fs::write(&source, b"*.rs text\n").unwrap(); + let capture = || { + Policy::capture_sources( + root.path(), + std::slice::from_ref(&source), + Err(io::Error::other("config unavailable")), + [ + Err(io::Error::other("unknown system attributes")), + Err(io::Error::other("unknown global attributes")), + ], + ) + .unwrap() + }; + let before = capture(); + assert!(!before.reusable); + fs::write(&source, b"*.rs filter=custom\n").unwrap(); + let after = capture(); + assert!(!after.reusable); + assert_ne!(before.digest, after.digest); + } + + #[test] + fn final_policy_digest_covers_git_stat_settings_and_include_targets() { + let root = tempfile::tempdir().unwrap(); + git(root.path(), &["init", "-q"]); + for (key, value) in [ + ("core.autocrlf", "true"), + ("core.eol", "crlf"), + ("core.symlinks", "false"), + ("core.ignorecase", "true"), + ("core.checkStat", "minimal"), + ("core.trustctime", "false"), + ("core.fileMode", "false"), + ] { + let before = BindGuard::capture(root.path(), None).unwrap(); + git(root.path(), &["config", key, value]); + assert!(before.validate(root.path()).is_err(), "{key}"); + } + let external = tempfile::tempdir().unwrap(); + let include = external.path().join("included.config"); + fs::write(&include, b"[core]\n eol = lf\n").unwrap(); + git( + root.path(), + &["config", "include.path", include.to_str().unwrap()], + ); + let before = BindGuard::capture(root.path(), None).unwrap(); + fs::write(&include, b"[core]\n eol = crlf\n").unwrap(); + assert!(before.validate(root.path()).is_err()); + } + #[test] fn stat_evidence_is_generation_root_policy_and_version_paired() { let _serial = crate::blob_store::v3::serial_for_test(); @@ -619,6 +941,9 @@ mod tests { let db = rusqlite::Connection::open(evidence_path(&route.member_dir(), "g-one")).unwrap(); db.pragma_update(None, "user_version", 2).unwrap(); assert!(read_evidence(&route.member_dir(), "g-one", &guard).is_none()); + db.pragma_update(None, "user_version", 1).unwrap(); + db.pragma_update(None, "application_id", 0x1234).unwrap(); + assert!(read_evidence(&route.member_dir(), "g-one", &guard).is_none()); } #[test] @@ -631,6 +956,9 @@ mod tests { assert!(!stat.reusable(&stat, observed)); assert!(stat.reusable(&stat, observed + 1)); let mut changed = stat.clone(); + changed.reuse_supported = false; + assert!(!changed.reusable(&changed, observed + 1)); + changed = stat.clone(); changed.changed_ns += 1; assert!(!stat.reusable(&changed, observed + 1)); changed = stat.clone(); diff --git a/docs/design/views-v3-format.md b/docs/design/views-v3-format.md index 244ebb82ae..734b0e2cfc 100644 --- a/docs/design/views-v3-format.md +++ b/docs/design/views-v3-format.md @@ -685,6 +685,15 @@ when its generation, root, policy, file identity, timestamps and racy checks all agree. Missing, corrupt or future evidence means reading source again. Evidence does not upgrade membership-keyed bases to tree-keyed identities. +Policy capture uses counted `git config --list --show-origin -z` **and** +`git var GIT_ATTR_SYSTEM` / `git var GIT_ATTR_GLOBAL`, rather than guessing +an installation prefix for attribute sources. This supersedes the earlier +config-only query wording in the views design. Every successful spawn, +including an unsupported-variable exit, contributes to `git_subprocesses` +in the process work counters. Either source query failing disables stat reuse +and takes strict reconciliation. Measurements of these additional queries +belong to the later performance gate. + The registry-version increments above are serial handoffs, not permission for an earlier slice to write a later owner's rows. Owners implement and test their assigned schema and encoding before publishing any such bytes. Registration From 314f530cf7bc8dc82cd3982a54b9deb8457c3ca2 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:25:03 +0100 Subject: [PATCH 15/31] mason: cover view-token management dispatch without enabling views Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/main.rs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/crates/aft/src/main.rs b/crates/aft/src/main.rs index 81e8108c83..721cf8b09b 100644 --- a/crates/aft/src/main.rs +++ b/crates/aft/src/main.rs @@ -3113,6 +3113,32 @@ mod deferred_semantic_search_tests { } } + #[test] + fn view_token_management_dispatch_reaches_the_default_off_guard() { + let storage = tempfile::tempdir().unwrap(); + let context = AppContext::new( + Box::new(TreeSitterProvider::new()), + Config { + storage_dir: Some(storage.path().to_owned()), + ..Default::default() + }, + ); + let response = super::dispatch_command( + request( + "token", + "view_token", + serde_json::json!({ + "root": storage.path().join("future"), "expected_tree": "a".repeat(40), + }), + ), + &context, + ); + let wire = serde_json::to_value(response).unwrap(); + assert_eq!(wire["success"], false); + assert_eq!(wire["code"], "VIEW_TOKEN"); + assert!(!storage.path().join("blobs").exists()); + } + fn slow_search_request(id: &str) -> RawRequest { request( id, From b13e57955e23c7e12de48baf78032aca75bb60fb Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:26:30 +0100 Subject: [PATCH 16/31] mason: preserve deterministic registration-walk test interception Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/watcher_backend/inotify.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/aft/src/watcher_backend/inotify.rs b/crates/aft/src/watcher_backend/inotify.rs index bbcdc5b2f9..45f19cd054 100644 --- a/crates/aft/src/watcher_backend/inotify.rs +++ b/crates/aft/src/watcher_backend/inotify.rs @@ -746,6 +746,8 @@ fn register_initial_tree( return Err(notify::Error::path_not_found().add_path(directory)); } installed.insert(directory.clone()); + #[cfg(test)] + tests::block_watch_walk(&directory); for entry in crate::watcher_filter::exclusion_read_dir(&directory).map_err(notify::Error::io)? { From 31835083162cc2966c6dbe77f8b1668562fc9b71 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:29:52 +0100 Subject: [PATCH 17/31] mason: return only validated advisory-token fields from bridge minting Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- packages/aft-bridge/src/view-token.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/aft-bridge/src/view-token.ts b/packages/aft-bridge/src/view-token.ts index 9f693acb12..ac040a42a4 100644 --- a/packages/aft-bridge/src/view-token.ts +++ b/packages/aft-bridge/src/view-token.ts @@ -31,5 +31,5 @@ export async function mintViewToken( ) { throw new Error("Invalid advisory view token response"); } - return value as unknown as ViewToken; + return { token: value.token, expires_ms: value.expires_ms, advisory: true }; } From 05247908bb4cc9ec12e4e89f32a4ca0649bac461 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:46:33 +0100 Subject: [PATCH 18/31] mason: keep legacy and v3 inventory test helpers in their own modules Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/views/first_load.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/aft/src/views/first_load.rs b/crates/aft/src/views/first_load.rs index 0e24c00831..5f554bf340 100644 --- a/crates/aft/src/views/first_load.rs +++ b/crates/aft/src/views/first_load.rs @@ -978,7 +978,7 @@ mod tests { drop(generation); } - fn legacy_inventory(storage: &std::path::Path) -> BTreeMap> { + fn inventory(root: &std::path::Path) -> BTreeMap> { fn visit( root: &std::path::Path, dir: &std::path::Path, @@ -5898,7 +5898,7 @@ mod v3_bootstrap_tests { } } - fn inventory(root: &std::path::Path) -> BTreeMap> { + fn legacy_inventory(storage: &std::path::Path) -> BTreeMap> { fn visit( base: &std::path::Path, path: &std::path::Path, From 870f8f08039f93db9a1580ed2ad60254278b9568 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:46:34 +0100 Subject: [PATCH 19/31] mason: reclaim stat evidence through the protected generation sweep Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/views/generation.rs | 69 +++++++- .../views/generation/stat_evidence_tests.rs | 155 ++++++++++++++++++ crates/aft/src/views/verification.rs | 7 +- docs/design/views-v3-format.md | 11 ++ 4 files changed, 234 insertions(+), 8 deletions(-) create mode 100644 crates/aft/src/views/generation/stat_evidence_tests.rs diff --git a/crates/aft/src/views/generation.rs b/crates/aft/src/views/generation.rs index 0520263bf2..c955735362 100644 --- a/crates/aft/src/views/generation.rs +++ b/crates/aft/src/views/generation.rs @@ -19,6 +19,10 @@ use serde::{ use super::{Result, ViewError, ViewStore}; +#[cfg(test)] +#[path = "generation/stat_evidence_tests.rs"] +mod stat_evidence_tests; + // Protection scans must finish within one common allowance before any unlink. pub(super) const GENERATION_DIRECTORY_LIMIT: usize = 65_536; const MAX_CLAIM_FILES: usize = 16; @@ -94,6 +98,48 @@ fn unlink_generation_file(path: &Path) -> Option { Some(bytes) } +fn stat_evidence_namespace(view_dir: &Path) -> bool { + fn named(path: &Path, name: &str) -> bool { + path.file_name() + .and_then(|n| n.to_str()) + .is_some_and(|actual| { + if cfg!(windows) { + actual.eq_ignore_ascii_case(name) + } else { + actual == name + } + }) + } + let Some(views) = view_dir.parent() else { + return false; + }; + let Some(family) = views.parent() else { + return false; + }; + let Some(v3) = family.parent() else { + return false; + }; + let Some(blobs) = v3.parent() else { + return false; + }; + named(views, "views") && named(v3, "v3") && named(blobs, "blobs") +} + +fn stat_evidence_generation(name: &str) -> Option<&str> { + name.strip_prefix("stat-evidence-") + .and_then(|name| { + [".sqlite", ".sqlite-wal", ".sqlite-shm", ".sqlite-journal"] + .iter() + .find_map(|suffix| name.strip_suffix(suffix)) + }) + .or_else(|| { + name.strip_prefix(".stat-evidence-") + .and_then(|name| name.split_once(".sqlite.tmp.")) + .filter(|(_, nonce)| !nonce.is_empty()) + .map(|(generation, _)| generation) + }) +} + // A view has at most one cleanup worker. Publications arriving during a sweep // request another lap rather than starting competing pointer transactions. static GENERATION_SWEEPS: OnceLock>> = OnceLock::new(); @@ -780,6 +826,7 @@ impl ViewStore { fn has_superseded_artifacts(&self) -> Result { let current = self.pointer_census()?.generation; + let evidence = stat_evidence_namespace(self.view_dir()); let (entries, reached_end) = bounded_directory_entries(self.view_dir(), 0, GENERATION_DIRECTORY_LIMIT)?; if !reached_end { @@ -820,7 +867,8 @@ impl ViewStore { name.split_once(".json.tmp.") .map(|(generation, _)| generation) }) - }); + }) + .or_else(|| evidence.then(|| stat_evidence_generation(name)).flatten()); if generation.is_some_and(|generation| current.as_deref() != Some(generation)) { return Ok(true); } @@ -855,6 +903,7 @@ impl ViewStore { let mut generations = std::collections::BTreeSet::new(); let mut derived_owners = std::collections::BTreeSet::new(); let mut temporaries: HashMap> = HashMap::new(); + let evidence = stat_evidence_namespace(self.view_dir()); // A directory over the limit is examined one window per pass, so the // leak it holds can still shrink. Each pass moves the window on. let windows = OVERSIZED_WINDOWS.get_or_init(Default::default); @@ -912,9 +961,12 @@ impl ViewStore { .or_else(|| { name.strip_prefix(".manifest-") .and_then(|s| s.split_once(".json.tmp.").map(|(generation, _)| generation)) - }); + }) + .or_else(|| evidence.then(|| stat_evidence_generation(name)).flatten()); if let Some(generation) = generation { - if name.starts_with(".manifest-") { + if name.starts_with(".manifest-") + || (evidence && name.starts_with(".stat-evidence-")) + { temporaries .entry(generation.to_owned()) .or_default() @@ -1069,13 +1121,20 @@ impl ViewStore { // claim bound stops this pass partway, the manifest may be left // without its sidecar (blob GC then decodes the JSON), but the // sidecar is never left behind its manifest. - for path in [ + let mut artifacts = vec![ self.view_dir().join(format!("derived-{generation}.sqlite")), self.view_dir().join(format!("derived-{generation}.ref")), self.trigram_path(generation)?, sidecar.clone(), self.manifest_path(generation)?, - ] { + ]; + if stat_evidence_namespace(self.view_dir()) { + artifacts.insert( + 3, + super::verification::evidence_path(self.view_dir(), generation), + ); + } + for path in artifacts { let sqlite = path.extension().is_some_and(|ext| ext == "sqlite"); // A kept clone-baseline manifest keeps its sidecar too. if keep_derived diff --git a/crates/aft/src/views/generation/stat_evidence_tests.rs b/crates/aft/src/views/generation/stat_evidence_tests.rs new file mode 100644 index 0000000000..3bc62b70e3 --- /dev/null +++ b/crates/aft/src/views/generation/stat_evidence_tests.rs @@ -0,0 +1,155 @@ +use super::*; +use crate::views::manifest_v2::{GenerationName, ManifestHeader, ManifestV2, Producers, PublishV2}; + +fn fixture() -> (tempfile::TempDir, ViewStore) { + let storage = tempfile::tempdir().unwrap(); + let directory = crate::blob_store::v3::view_dir(storage.path(), "family", "worker").unwrap(); + crate::reader_floor::publish_v3_with_floor(storage.path(), 3, 1, || { + std::fs::create_dir_all(&directory).unwrap(); + }) + .unwrap(); + let view = ViewStore::open_dir(directory).unwrap(); + (storage, view) +} + +fn manifest(tag: &str) -> ManifestV2 { + ManifestV2::new_v3( + ManifestHeader { + producers: Producers { + trigram: "trigram".into(), + semantic: None, + callgraph: "graph".into(), + }, + head_tree: None, + ignore_fingerprint: Some(tag.into()), + segment: None, + }, + None, + Default::default(), + ) +} + +fn publish(view: &ViewStore, tag: &str) -> String { + let manifest = manifest(tag); + let generation = GenerationName::for_manifest(&manifest).unwrap(); + let previous = view.current_generation_read_only().unwrap(); + assert_eq!( + view.publish_v2(&generation, previous.as_deref(), &manifest, None) + .unwrap(), + PublishV2::Published + ); + wait_for_generation_sweep_for_test(view.view_dir()); + generation.to_string() +} + +fn stage(view: &ViewStore, generation: &str, nonce: &str) -> PathBuf { + view.view_dir() + .join(format!(".stat-evidence-{generation}.sqlite.tmp.{nonce}")) +} + +#[test] +fn stat_evidence_retires_with_its_generation_but_survives_current_and_read_pins() { + let (_storage, view) = fixture(); + let old = publish(&view, "old"); + let old_evidence = crate::views::verification::evidence_path(view.view_dir(), &old); + fs::write(&old_evidence, vec![7; 32_768]).unwrap(); + let old_stage = stage(&view, &old, "abandoned"); + fs::write(&old_stage, b"private evidence").unwrap(); + let marker = crate::root_cache::ReadMarker::create(view.view_dir(), &old).unwrap(); + let current = publish(&view, "current"); + let current_evidence = crate::views::verification::evidence_path(view.view_dir(), ¤t); + fs::write(¤t_evidence, b"current evidence").unwrap(); + let current_stage = stage(&view, ¤t, "still-current"); + fs::write(¤t_stage, b"current staging").unwrap(); + view.sweep_generations().unwrap(); + assert!(old_evidence.exists()); + assert!(old_stage.exists()); + view.sweep_generations().unwrap(); + assert!(old_evidence.exists()); + assert!(current_evidence.exists()); + assert!(current_stage.exists()); + drop(marker); + let swept = view.sweep_generations_with_bytes().unwrap(); + assert!(!old_evidence.exists()); + assert!(!old_stage.exists()); + assert!(swept.removed_generations > 0); + assert!(current_evidence.exists()); + assert!(current_stage.exists()); +} + +#[test] +fn stat_evidence_staging_never_ages_out_under_a_prepared_generation_pin() { + let (_storage, view) = fixture(); + publish(&view, "current"); + let manifest = manifest("unpublished"); + let generation = GenerationName::for_manifest(&manifest).unwrap(); + let previous = view.current_generation_read_only().unwrap(); + let prepared = view + .prepare_v2(&generation, previous.as_deref(), &manifest, None) + .unwrap(); + let staging = stage(&view, &generation.to_string(), "crashed-writer"); + fs::write(&staging, b"partly written database").unwrap(); + fs::File::open(&staging) + .unwrap() + .set_times( + fs::FileTimes::new().set_modified(std::time::UNIX_EPOCH + Duration::from_secs(60)), + ) + .unwrap(); + view.sweep_generations().unwrap(); + assert!(staging.exists()); + assert!(view + .manifest_path(&generation.to_string()) + .unwrap() + .exists()); + drop(prepared); + view.sweep_generations().unwrap(); + assert!(!staging.exists()); + assert!(!view + .manifest_path(&generation.to_string()) + .unwrap() + .exists()); +} + +#[test] +fn stat_evidence_orphans_and_sqlite_temporaries_use_existing_removed_byte_accounting() { + let (_storage, view) = fixture(); + publish(&view, "current"); + let database = crate::views::verification::evidence_path(view.view_dir(), "orphan"); + let journal = database.with_file_name("stat-evidence-orphan.sqlite-journal"); + let temporary = stage(&view, "orphan", "dead-writer"); + let temporary_journal = temporary.with_file_name(format!( + "{}-journal", + temporary.file_name().unwrap().to_str().unwrap() + )); + for (path, size) in [ + (&database, 65_536), + (&journal, 4_096), + (&temporary, 8_192), + (&temporary_journal, 512), + ] { + fs::write(path, vec![3; size]).unwrap(); + } + assert!(view.has_superseded_artifacts().unwrap()); + let swept = view.sweep_generations_with_bytes().unwrap(); + assert_eq!(swept.removed_bytes, 65_536 + 4_096 + 8_192 + 512); + assert_eq!(swept.removed_generations, 1); + for path in [&database, &journal, &temporary, &temporary_journal] { + assert!(!path.exists()); + } +} + +#[test] +fn stat_evidence_discovery_is_confined_to_v3_member_directories() { + let storage = tempfile::tempdir().unwrap(); + let registry = crate::views::registry::FamilyRegistry::open(storage.path(), "family").unwrap(); + let owner = registry + .register_view_with_storage(registry.storage(), "legacy", storage.path()) + .unwrap(); + let legacy = owner.view_store().unwrap(); + let evidence = crate::views::verification::evidence_path(legacy.view_dir(), "orphan"); + fs::write(&evidence, b"not a legacy-owned artifact").unwrap(); + assert!(!stat_evidence_namespace(legacy.view_dir())); + assert!(!legacy.has_superseded_artifacts().unwrap()); + legacy.sweep_generations().unwrap(); + assert_eq!(fs::read(evidence).unwrap(), b"not a legacy-owned artifact"); +} diff --git a/crates/aft/src/views/verification.rs b/crates/aft/src/views/verification.rs index 5521990ffb..ad5fdbbda6 100644 --- a/crates/aft/src/views/verification.rs +++ b/crates/aft/src/views/verification.rs @@ -625,8 +625,9 @@ pub(crate) fn evidence_path(dir: &Path, generation: &str) -> PathBuf { dir.join(format!("stat-evidence-{generation}.sqlite")) } -/// Writes the evidence before the CAS. Its generation and content digests are -/// checked on read, so an unpublished or obsolete sidecar supplies no proof. +/// Writes evidence while the caller holds the prepared-generation pin, before +/// the CAS. Generation and content digests are checked on read, so an +/// unpublished or obsolete sidecar supplies no proof. pub(crate) fn write_evidence( route: &super::snapshot::V3Route, generation: &str, @@ -644,7 +645,7 @@ pub(crate) fn write_evidence( let dir = route.member_dir(); let destination = evidence_path(&dir, generation); let staging = dir.join(format!( - "stat-evidence-{}.staging", + ".stat-evidence-{generation}.sqlite.tmp.{}", super::base_refs::fresh_nonce() )); let result = (|| -> Result<(), Box> { diff --git a/docs/design/views-v3-format.md b/docs/design/views-v3-format.md index 734b0e2cfc..0cf7312021 100644 --- a/docs/design/views-v3-format.md +++ b/docs/design/views-v3-format.md @@ -685,6 +685,17 @@ when its generation, root, policy, file identity, timestamps and racy checks all agree. Missing, corrupt or future evidence means reading source again. Evidence does not upgrade membership-keyed bases to tree-keyed identities. +Evidence staging is `M/.stat-evidence-.sqlite.tmp.` (and +SQLite's `-journal`, `-wal`, `-shm` suffixes if present), so a crashed write has +an attributable generation. The prepared-generation pin is acquired before +the manifest or evidence is exposed and remains held through publication. +The existing generation sweep discovers the permanent database and these +temporaries and retires them in the same pin-checked claim as the generation; +its existing moved-file/unlink accounting includes their bytes. Current and +pinned generations retain evidence and staging regardless of age. Abandoned +staging is removed only when its generation retires, never by a second sweep +or by a timer that can bypass a live pin. + Policy capture uses counted `git config --list --show-origin -z` **and** `git var GIT_ATTR_SYSTEM` / `git var GIT_ATTR_GLOBAL`, rather than guessing an installation prefix for attribute sources. This supersedes the earlier From bd9e418881ef2ea1a6854035d1dc2d0b98b1539e Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:54:02 +0100 Subject: [PATCH 20/31] mason: refuse initial coverage when the root disappears before registration Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/watcher_backend/inotify.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/crates/aft/src/watcher_backend/inotify.rs b/crates/aft/src/watcher_backend/inotify.rs index 45f19cd054..8625a0e2c7 100644 --- a/crates/aft/src/watcher_backend/inotify.rs +++ b/crates/aft/src/watcher_backend/inotify.rs @@ -757,6 +757,9 @@ fn register_initial_tree( } } } + if !installed.contains(root) { + return Err(notify::Error::path_not_found().add_path(root.to_path_buf())); + } Ok(installed) } @@ -898,6 +901,19 @@ pub(super) mod tests { } } + #[test] + fn initial_subscription_cannot_certify_a_root_lost_before_registration() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let boundary = crate::watcher_filter::WatcherWalkBoundary::for_root(&root).unwrap(); + std::fs::remove_dir(&root).unwrap(); + let result = + register_initial_tree(&boundary, &Arc::new(RwLock::new(None)), &[], &mut |_| { + Ok(true) + }); + assert!(result.is_err()); + } + #[test] fn watcher_cookie_inotify_recovery_precedes_root_cookie_without_command_tracking() { let temp = tempfile::tempdir().unwrap(); From 71b1e97a37d186de78667bfbd118652fa497981b Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:56:48 +0100 Subject: [PATCH 21/31] mason: publish sweep fixtures through the prepared generation CAS Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/views/generation/stat_evidence_tests.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/aft/src/views/generation/stat_evidence_tests.rs b/crates/aft/src/views/generation/stat_evidence_tests.rs index 3bc62b70e3..82efee0d6a 100644 --- a/crates/aft/src/views/generation/stat_evidence_tests.rs +++ b/crates/aft/src/views/generation/stat_evidence_tests.rs @@ -33,9 +33,11 @@ fn publish(view: &ViewStore, tag: &str) -> String { let manifest = manifest(tag); let generation = GenerationName::for_manifest(&manifest).unwrap(); let previous = view.current_generation_read_only().unwrap(); + let prepared = view + .prepare_v2(&generation, previous.as_deref(), &manifest, None) + .unwrap(); assert_eq!( - view.publish_v2(&generation, previous.as_deref(), &manifest, None) - .unwrap(), + view.commit_v2(prepared, None).unwrap(), PublishV2::Published ); wait_for_generation_sweep_for_test(view.view_dir()); From 77a9b01fad01415d77935a7edbc0b6fd659bd627 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:03:51 +0100 Subject: [PATCH 22/31] mason: open writable handles for portable timestamp fixtures Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/views/first_load.rs | 4 +++- crates/aft/src/views/generation/stat_evidence_tests.rs | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/aft/src/views/first_load.rs b/crates/aft/src/views/first_load.rs index 5f554bf340..cfb610c967 100644 --- a/crates/aft/src/views/first_load.rs +++ b/crates/aft/src/views/first_load.rs @@ -5936,7 +5936,9 @@ mod v3_bootstrap_tests { let _attributes = super::super::verification::known_attribute_sources_for_test(root.path()); let source = b"fn fresh_checkout() { let marker = 42; }\n"; std::fs::write(root.path().join("file.rs"), source).unwrap(); - std::fs::File::open(root.path().join("file.rs")) + std::fs::OpenOptions::new() + .write(true) + .open(root.path().join("file.rs")) .unwrap() .set_times( std::fs::FileTimes::new() diff --git a/crates/aft/src/views/generation/stat_evidence_tests.rs b/crates/aft/src/views/generation/stat_evidence_tests.rs index 82efee0d6a..9ec8bbbde5 100644 --- a/crates/aft/src/views/generation/stat_evidence_tests.rs +++ b/crates/aft/src/views/generation/stat_evidence_tests.rs @@ -91,7 +91,9 @@ fn stat_evidence_staging_never_ages_out_under_a_prepared_generation_pin() { .unwrap(); let staging = stage(&view, &generation.to_string(), "crashed-writer"); fs::write(&staging, b"partly written database").unwrap(); - fs::File::open(&staging) + fs::OpenOptions::new() + .write(true) + .open(&staging) .unwrap() .set_times( fs::FileTimes::new().set_modified(std::time::UNIX_EPOCH + Duration::from_secs(60)), From 61714c0dde4fa3f503923e26313b1d460a6a2610 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:16:51 +0100 Subject: [PATCH 23/31] mason: preserve canonical watcher gaps and require live steady subscriptions Keep the sparse-v3 fixture's sparse/restart/no-v2 assertions unchanged, but establish a real reconciled and installed driver cut before publication. Merely naming its generation installed-worker no longer satisfies the final CAS identity check. Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/views/first_load.rs | 52 ++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/crates/aft/src/views/first_load.rs b/crates/aft/src/views/first_load.rs index cfb610c967..02c255402c 100644 --- a/crates/aft/src/views/first_load.rs +++ b/crates/aft/src/views/first_load.rs @@ -2095,8 +2095,8 @@ impl CheckoutDriver { delta.record_intent(path); // Membership is unchanged, so are the ignore rules it was walked under. delta.record_root_ignore_files(installed.snapshot.root_ignore_files().cloned()); - // A known path cannot repair an earlier stream-wide delivery gap. - delta.set_watcher(installed.snapshot.watcher()); + // One known-path event cannot prove that delivery remained continuous + // for the whole watcher stream. installed.snapshot = delta .snapshot() .with_readers(installed.readers.clone()) @@ -4216,6 +4216,17 @@ mod composite_tests { ), ); let snapshot = delta.snapshot(); + // The install check needs an identity recorded by the driver. Reconcile + // and install here; the name `installed-worker` alone is not that proof. + let (reconciled, revision) = SiblingLoader::reconciled_snapshot( + driver.as_ref(), + &access, + snapshot.generation().clone(), + ) + .unwrap(); + driver.install(&access, &reconciled, revision).unwrap(); + let snapshot = + snapshot.with_install_cut(driver.installed_snapshot().install_cut().unwrap().clone()); let published = driver .build_own_generation(&access, &snapshot, false) .unwrap(); @@ -5383,10 +5394,10 @@ impl CheckoutDriver { .snapshot .install_cut() .and_then(|cut| cut.verification.as_ref()) - .and_then(|guard| guard.subscription) - .is_some_and(|expected| { - crate::watcher_backend::subscription::current(self.owner.root()) - != Some(expected) + .is_some_and(|guard| { + guard.subscription.is_none() + || crate::watcher_backend::subscription::current(self.owner.root()) + != guard.subscription }) }; if stale { @@ -5838,6 +5849,32 @@ mod v3_bootstrap_tests { ); } + #[test] + fn no_subscription_keeps_v3_queries_on_the_strict_path_until_registration() { + let _serial = crate::blob_store::v3::serial_for_test(); + let root = tempfile::tempdir().unwrap(); + let storage = tempfile::tempdir().unwrap(); + let _attributes = super::super::verification::known_attribute_sources_for_test(root.path()); + std::fs::write(root.path().join("file.rs"), b"fn observed() {}\n").unwrap(); + let driver = trigram_driver(root.path(), storage.path()); + let access = driver.access(); + let (snapshot, revision) = driver + .verified_bind(&access, driver.installed_snapshot().generation().clone()) + .unwrap(); + driver.install(&access, &snapshot, revision).unwrap(); + assert!(driver.installed_snapshot().needs_reconcile()); + driver.take_reconcile_work(); + let _subscription = crate::watcher_backend::subscription::start( + &std::fs::canonicalize(root.path()).unwrap(), + ); + let (snapshot, revision) = driver + .verified_bind(&access, snapshot.generation().clone()) + .unwrap(); + assert_eq!(driver.take_reconcile_work().walks, 1); + driver.install(&access, &snapshot, revision).unwrap(); + assert!(!driver.installed_snapshot().needs_reconcile()); + } + #[test] fn v3_lazy_attachments_probe_the_actual_producer_key_before_extraction() { let _serial = crate::blob_store::v3::serial_for_test(); @@ -5934,6 +5971,9 @@ mod v3_bootstrap_tests { let root = tempfile::tempdir().unwrap(); let storage = tempfile::tempdir().unwrap(); let _attributes = super::super::verification::known_attribute_sources_for_test(root.path()); + let _subscription = crate::watcher_backend::subscription::start( + &std::fs::canonicalize(root.path()).unwrap(), + ); let source = b"fn fresh_checkout() { let marker = 42; }\n"; std::fs::write(root.path().join("file.rs"), source).unwrap(); std::fs::OpenOptions::new() From 7fd7fd61153a3263ad79d3ea154695b23aad7ae2 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:16:53 +0100 Subject: [PATCH 24/31] mason: skip source admission probes for already reconciled deltas Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/views/semantic_runtime.rs | 5 +++++ crates/aft/src/views/semantic_runtime_tests.rs | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/crates/aft/src/views/semantic_runtime.rs b/crates/aft/src/views/semantic_runtime.rs index e27e9ba87d..ab5e17883f 100644 --- a/crates/aft/src/views/semantic_runtime.rs +++ b/crates/aft/src/views/semantic_runtime.rs @@ -748,6 +748,11 @@ fn needs_reload(snapshot: &Snapshot) -> bool { /// regular file counts as large. A changed path that no longer exists counts /// as zero bytes, because the reconcile has nothing of it to read. fn needs_cold_reconcile(snapshot: &Snapshot, root: &Path) -> bool { + // Reconciled bytes may still have pending planes. Their fill admission + // does not require probing source files for another reconciliation. + if !needs_reload(snapshot) { + return false; + } if snapshot.generation().name() == "empty" { return true; } diff --git a/crates/aft/src/views/semantic_runtime_tests.rs b/crates/aft/src/views/semantic_runtime_tests.rs index 4c3634fba0..45135e8cac 100644 --- a/crates/aft/src/views/semantic_runtime_tests.rs +++ b/crates/aft/src/views/semantic_runtime_tests.rs @@ -37,8 +37,12 @@ fn reconciled_delta_does_not_request_another_source_walk() { entry.reconciliation = super::super::readiness::ReconciledDelta::Reconciled; delta.apply(path.clone(), entry); assert!(!super::needs_reload(&delta.snapshot())); + let root = tempfile::tempdir().unwrap(); + let missing = root.path().join("source-probe-must-not-run"); + assert!(!super::needs_cold_reconcile(&delta.snapshot(), &missing)); delta.record_intent(path); assert!(super::needs_reload(&delta.snapshot())); + assert!(super::needs_cold_reconcile(&delta.snapshot(), &missing)); } /// The test prints public setup, query answers, a v3 file inventory and exact From 02b6788b55cac5e2c9634d6142a3c37e45a186e1 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:23:28 +0100 Subject: [PATCH 25/31] mason: keep unsupported roots out of watcher-trusted steady refresh Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/views/first_load.rs | 16 ++++++++++++++-- crates/aft/src/views/verification.rs | 9 ++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/crates/aft/src/views/first_load.rs b/crates/aft/src/views/first_load.rs index 02c255402c..47ed534c83 100644 --- a/crates/aft/src/views/first_load.rs +++ b/crates/aft/src/views/first_load.rs @@ -2333,7 +2333,9 @@ impl FirstLoadDriver for CheckoutDriver { .as_ref() .zip(previous.guard.as_ref()) .is_some_and(|(now, old)| now.root == old.root && now.policy == old.policy) - && guard.as_ref().is_some_and(|guard| guard.policy.reusable) + && guard + .as_ref() + .is_some_and(|guard| guard.policy.reusable && guard.steady_supported) && !policy_or_directory; let changed_files = if continuous { self.walker.changed_files(self.owner.root(), &candidates)? @@ -5395,7 +5397,8 @@ impl CheckoutDriver { .install_cut() .and_then(|cut| cut.verification.as_ref()) .is_some_and(|guard| { - guard.subscription.is_none() + !guard.steady_supported + || guard.subscription.is_none() || crate::watcher_backend::subscription::current(self.owner.root()) != guard.subscription }) @@ -5873,6 +5876,15 @@ mod v3_bootstrap_tests { assert_eq!(driver.take_reconcile_work().walks, 1); driver.install(&access, &snapshot, revision).unwrap(); assert!(!driver.installed_snapshot().needs_reconcile()); + let installed = driver.installed_snapshot(); + let mut cut = installed.install_cut().unwrap().clone(); + let mut guard = cut.verification.as_ref().unwrap().as_ref().clone(); + guard.steady_supported = false; + cut.verification = Some(Arc::new(guard)); + driver + .install(&access, &installed.with_install_cut(cut), revision) + .unwrap(); + assert!(driver.installed_snapshot().needs_reconcile()); } #[test] diff --git a/crates/aft/src/views/verification.rs b/crates/aft/src/views/verification.rs index ad5fdbbda6..918ccde52a 100644 --- a/crates/aft/src/views/verification.rs +++ b/crates/aft/src/views/verification.rs @@ -530,6 +530,9 @@ pub(crate) struct BindGuard { pub policy: Policy, pub inputs: Vec, pub subscription: Option, + /// Stable local identities and a native backend allow event-based refresh, + /// but do not prove that all writes before a cookie reached the driver. + pub steady_supported: bool, } impl BindGuard { @@ -539,7 +542,10 @@ impl BindGuard { None => policy_inputs(root)?, }; let mut policy = Policy::capture(root, &inputs)?; - let root_identity = NativeStat::strict_read(root)?.identity; + let root_stat = NativeStat::strict_read(root)?; + let steady_supported = root_stat.reuse_supported + && cfg!(any(target_os = "linux", target_os = "macos", windows)); + let root_identity = root_stat.identity; let directory = match gitdir(root) { Ok(path) => Some((path.clone(), NativeStat::strict_read(&path)?.identity)), Err(e) if e.kind() == io::ErrorKind::NotFound => None, @@ -604,6 +610,7 @@ impl BindGuard { policy, inputs, subscription: crate::watcher_backend::subscription::current(root), + steady_supported, }) } pub fn validate(&self, root: &Path) -> io::Result<()> { From 02781f13b0619ccffc0c216f7726ec0ca7edecf9 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:53:00 +0100 Subject: [PATCH 26/31] mason: fence v3 final validation and CAS with driver ownership Keep artifact construction and admission outside the ownership fence. Preserve the old publication helper as an unfenced delegate, serialize the driver's installed-bundle transition with validate+CAS, and release a completed lexical-only bootstrap claim even when publication is refused. Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/views/base_refs.rs | 52 ++++- crates/aft/src/views/first_load.rs | 37 +++- .../first_load/publication_fence_tests.rs | 191 ++++++++++++++++++ 3 files changed, 273 insertions(+), 7 deletions(-) create mode 100644 crates/aft/src/views/first_load/publication_fence_tests.rs diff --git a/crates/aft/src/views/base_refs.rs b/crates/aft/src/views/base_refs.rs index 4b2d5f6c6c..27f7029358 100644 --- a/crates/aft/src/views/base_refs.rs +++ b/crates/aft/src/views/base_refs.rs @@ -696,6 +696,25 @@ pub fn publish_worker_with( observer: Option<&dyn DurabilityObserver>, before_cas: &mut dyn FnMut(&str) -> Result<()>, final_validate: &mut dyn FnMut() -> Result<()>, +) -> Result { + publish_worker_guarded( + request, + observer, + before_cas, + final_validate, + &mut |commit| commit(), + ) +} + +/// Publishes with ownership serialization around final validation and the CAS. +/// `publish_cas` may decline without calling its operation, but may call it only +/// once. Artifact construction and admission happen before this short fence. +pub fn publish_worker_guarded( + request: WorkerPublication<'_>, + observer: Option<&dyn DurabilityObserver>, + before_cas: &mut dyn FnMut(&str) -> Result<()>, + final_validate: &mut dyn FnMut() -> Result<()>, + publish_cas: &mut dyn FnMut(&mut dyn FnMut() -> Result) -> Result, ) -> Result { let WorkerPublication { storage, @@ -850,16 +869,39 @@ pub fn publish_worker_with( if let Err(error) = verified { return Err(abort(error, marker)); } - if let Err(error) = final_validate() { - return Err(abort(error, marker)); - } // (4) The CAS. From here on the reference may be durable. - let outcome = worker_store.commit_v2(prepared, observer); + let mut prepared = Some(prepared); + let mut invoked = false; + let mut cas_started = false; + let outcome = publish_cas(&mut || { + if invoked { + return Err(BaseRefError::Unavailable( + "publication fence invoked CAS twice".into(), + )); + } + invoked = true; + final_validate()?; + cas_started = true; + worker_store + .commit_v2( + prepared.take().expect("prepared CAS consumed once"), + observer, + ) + .map_err(BaseRefError::from) + }); + if !cas_started { + return Err(abort( + outcome.err().unwrap_or_else(|| { + BaseRefError::Unavailable("publication fence did not invoke CAS".into()) + }), + marker, + )); + } #[cfg(test)] let outcome = match AFTER_WORKER_CAS.with(|hook| hook.borrow_mut().as_mut().and_then(|hook| hook())) { - Some(error) => Err(super::ViewError::Io(error)), + Some(error) => Err(error.into()), None => outcome, }; let landed = match outcome { diff --git a/crates/aft/src/views/first_load.rs b/crates/aft/src/views/first_load.rs index 47ed534c83..ea20fe8fab 100644 --- a/crates/aft/src/views/first_load.rs +++ b/crates/aft/src/views/first_load.rs @@ -18,6 +18,10 @@ use super::RelPath; #[path = "first_load/overflow_recovery_tests.rs"] mod overflow_recovery_tests; +#[cfg(test)] +#[path = "first_load/publication_fence_tests.rs"] +mod publication_fence_tests; + /// A complete strict content walk under the checkout's current ignore rules. /// The revision must change for every delivered edit, write intent, watcher gap, /// or membership change; a partial walk must return an error, not this value. @@ -1782,7 +1786,7 @@ impl CheckoutDriver { if !bootstrap { return Ok((snapshot, revision)); } - let own = self.build_own_generation(access, &snapshot, DERIVED_CALLGRAPH_SEEDING)?; + let own = self.build_own_generation(access, &snapshot, DERIVED_CALLGRAPH_SEEDING); if self .planes .iter() @@ -1799,6 +1803,7 @@ impl CheckoutDriver { .map_err(|e| SiblingLoader::error(e.to_string()))?; } } + let own = own?; let mut delta = LiveDelta::from_snapshot(&snapshot); let draft = super::snapshot::derive_successor( &delta.cut(), @@ -3078,7 +3083,12 @@ impl CheckoutDriver { .flatten(), }; let planes = &self.planes; - let outcome = super::base_refs::publish_worker_with( + let fence = self + .publish_fence + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + let outcome = super::base_refs::publish_worker_guarded( super::base_refs::WorkerPublication { storage: &storage, family: &family, @@ -3124,6 +3134,29 @@ impl CheckoutDriver { self.validate_cut(access, snapshot, 0) .map_err(|error| super::base_refs::BaseRefError::Unavailable(error.reason)) }, + &mut |commit| { + let mut actual = None; + let mut publish = || { + let _transition = self + .publication_transition + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + actual = + Some(commit().map_err(|error| SiblingLoader::error(error.to_string()))?); + Ok(()) + }; + match &fence { + Some(fence) => fence.publish(&mut publish), + None => publish(), + } + .map_err(|error| super::base_refs::BaseRefError::Unavailable(error.reason))?; + drop(publish); + actual.ok_or_else(|| { + super::base_refs::BaseRefError::Unavailable( + "publication fence did not invoke CAS".into(), + ) + }) + }, ); drop(live); match outcome { diff --git a/crates/aft/src/views/first_load/publication_fence_tests.rs b/crates/aft/src/views/first_load/publication_fence_tests.rs new file mode 100644 index 0000000000..afa3f5d7ba --- /dev/null +++ b/crates/aft/src/views/first_load/publication_fence_tests.rs @@ -0,0 +1,191 @@ +use super::*; +use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Mutex; + +struct LeaseFence { + live: Mutex, + calls: AtomicUsize, + policy_change: Mutex>, +} +impl LeaseFence { + fn live() -> Arc { + Arc::new(Self { + live: Mutex::new(true), + calls: AtomicUsize::new(0), + policy_change: Mutex::new(None), + }) + } + fn revoke(&self) { + *self.live.lock().unwrap() = false; + } +} +impl PublishFence for LeaseFence { + fn publish( + &self, + commit: &mut dyn FnMut() -> Result<(), PlaneError>, + ) -> Result<(), PlaneError> { + let live = self.live.lock().unwrap(); + self.calls.fetch_add(1, Ordering::SeqCst); + if !*live { + return Err(SiblingLoader::error("driver ownership retired")); + } + if let Some(path) = self.policy_change.lock().unwrap().take() { + std::fs::write(path, b"*.rs text\n").unwrap(); + } + commit() + } +} + +struct Fixture { + root: tempfile::TempDir, + storage: tempfile::TempDir, + driver: CheckoutDriver, + access: ViewAccess, +} +impl Fixture { + fn new() -> Self { + let root = tempfile::tempdir().unwrap(); + let storage = tempfile::tempdir().unwrap(); + std::fs::write(root.path().join("file.rs"), b"fn before() {}\n").unwrap(); + let policy = crate::blob_store::v2::TrigramPolicy { + max_file_size: 1 << 20, + }; + let driver = CheckoutDriver::new_v3( + storage.path(), + "family", + "worker", + root.path(), + Producers { + trigram: policy.fingerprint_hex(), + semantic: None, + callgraph: "unregistered".into(), + }, + Arc::new(ConfiguredMembershipWalker), + vec![Arc::new(TrigramBridge::new( + storage.path().to_owned(), + policy, + ))], + ) + .unwrap(); + let access = driver.access(); + Self { + root, + storage, + driver, + access, + } + } + fn bootstrap(&self) { + let (snapshot, revision) = self + .driver + .verified_bind( + &self.access, + self.driver.installed_snapshot().generation().clone(), + ) + .unwrap(); + self.driver + .install(&self.access, &snapshot, revision) + .unwrap(); + } + fn pointer(&self) -> Option { + super::super::ViewStore::existing_dir(self.driver.owner.v3_route().member_dir()) + .map(|store| store.current_generation_read_only().unwrap()) + .flatten() + } + fn validated_change(&self) -> Snapshot { + std::fs::write( + self.root.path().join("file.rs"), + b"fn after() { let changed = true; }\n", + ) + .unwrap(); + self.driver + .record_change(RelPath::new(b"file.rs".to_vec()).unwrap()); + let (snapshot, revision) = self + .driver + .verified_bind( + &self.access, + self.driver.installed_snapshot().generation().clone(), + ) + .unwrap(); + self.driver + .validate_cut(&self.access, &snapshot, revision) + .unwrap(); + snapshot + } +} + +#[test] +fn v3_cas_fence_rejects_retirement_after_validation_and_allows_a_live_driver() { + let _serial = crate::blob_store::v3::serial_for_test(); + for retired in [true, false] { + let fixture = Fixture::new(); + fixture.bootstrap(); + let previous = fixture.pointer(); + let fence = LeaseFence::live(); + fixture.driver.set_publish_fence(fence.clone()); + let snapshot = fixture.validated_change(); + if retired { + fence.revoke(); + } + let result = fixture + .driver + .build_own_generation(&fixture.access, &snapshot, false); + assert_eq!(fence.calls.load(Ordering::SeqCst), 1); + if retired { + assert!(result.is_err()); + assert_eq!(fixture.pointer(), previous); + } else { + let generation = result.unwrap(); + assert_ne!(fixture.pointer(), previous); + assert_eq!(fixture.pointer().as_deref(), Some(generation.name())); + } + } +} + +#[test] +fn v3_final_metadata_validation_runs_inside_the_ownership_fence() { + let _serial = crate::blob_store::v3::serial_for_test(); + let fixture = Fixture::new(); + fixture.bootstrap(); + let previous = fixture.pointer(); + let fence = LeaseFence::live(); + fixture.driver.set_publish_fence(fence.clone()); + let snapshot = fixture.validated_change(); + *fence.policy_change.lock().unwrap() = Some(fixture.root.path().join(".gitattributes")); + assert!(fixture + .driver + .build_own_generation(&fixture.access, &snapshot, false) + .is_err()); + assert_eq!(fence.calls.load(Ordering::SeqCst), 1); + assert_eq!(fixture.pointer(), previous); +} + +#[test] +fn refused_fresh_v3_publication_releases_the_completed_lexical_claim() { + let _serial = crate::blob_store::v3::serial_for_test(); + let fixture = Fixture::new(); + let fence = LeaseFence::live(); + fence.revoke(); + fixture.driver.set_publish_fence(fence); + assert!(fixture + .driver + .verified_bind( + &fixture.access, + fixture.driver.installed_snapshot().generation().clone() + ) + .is_err()); + assert!(fixture.pointer().is_none()); + let registry = + crate::blob_store::v3::LedgerReader::open_existing(fixture.storage.path(), "family") + .unwrap(); + assert_eq!( + registry + .read(|c| Ok( + c.query_row("SELECT COUNT(*) FROM base_builds", [], |row| row + .get::<_, u64>(0))? + )) + .unwrap(), + 0 + ); +} From 796f8b98d9f92c9923908f86b2efc8163785a632 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:16:11 +0100 Subject: [PATCH 27/31] mason: retain partial shared watchers with typed untrusted coverage Preserve views-off startup when a descendant cannot be listed or watched. Initial coverage now names gaps on the retained native handle; v3 trust refuses incomplete or opaque handles. This intentionally replaces the temporary whole-watcher-refusal test with partial-service and strict-v3 controls, per the compatibility ruling. Configure's generic attachment API is unchanged. Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/views/first_load.rs | 3 + .../first_load/overflow_recovery_tests.rs | 6 +- crates/aft/src/views/verification.rs | 10 +- crates/aft/src/watcher_backend/inotify.rs | 279 ++++++++++++++++-- crates/aft/src/watcher_backend/mod.rs | 54 ++++ .../aft/src/watcher_backend/subscription.rs | 86 +++++- crates/aft/src/watcher_filter.rs | 5 +- 7 files changed, 404 insertions(+), 39 deletions(-) diff --git a/crates/aft/src/views/first_load.rs b/crates/aft/src/views/first_load.rs index ea20fe8fab..6bae55a45c 100644 --- a/crates/aft/src/views/first_load.rs +++ b/crates/aft/src/views/first_load.rs @@ -5841,6 +5841,7 @@ mod v3_bootstrap_tests { } let subscription = crate::watcher_backend::subscription::start( &std::fs::canonicalize(root.path()).unwrap(), + crate::watcher_backend::InitialCoverage::Complete, ); let driver = trigram_driver(root.path(), storage.path()); let access = driver.access(); @@ -5902,6 +5903,7 @@ mod v3_bootstrap_tests { driver.take_reconcile_work(); let _subscription = crate::watcher_backend::subscription::start( &std::fs::canonicalize(root.path()).unwrap(), + crate::watcher_backend::InitialCoverage::Complete, ); let (snapshot, revision) = driver .verified_bind(&access, snapshot.generation().clone()) @@ -6018,6 +6020,7 @@ mod v3_bootstrap_tests { let _attributes = super::super::verification::known_attribute_sources_for_test(root.path()); let _subscription = crate::watcher_backend::subscription::start( &std::fs::canonicalize(root.path()).unwrap(), + crate::watcher_backend::InitialCoverage::Complete, ); let source = b"fn fresh_checkout() { let marker = 42; }\n"; std::fs::write(root.path().join("file.rs"), source).unwrap(); diff --git a/crates/aft/src/views/first_load/overflow_recovery_tests.rs b/crates/aft/src/views/first_load/overflow_recovery_tests.rs index 57f60e3dbb..5aa2f2d39a 100644 --- a/crates/aft/src/views/first_load/overflow_recovery_tests.rs +++ b/crates/aft/src/views/first_load/overflow_recovery_tests.rs @@ -10,8 +10,10 @@ fn fixture() -> ( let root = tempfile::tempdir().unwrap(); let storage = tempfile::tempdir().unwrap(); std::fs::write(root.path().join("file.rs"), b"fn content() {}\n").unwrap(); - let subscription = - crate::watcher_backend::subscription::start(&std::fs::canonicalize(root.path()).unwrap()); + let subscription = crate::watcher_backend::subscription::start( + &std::fs::canonicalize(root.path()).unwrap(), + crate::watcher_backend::InitialCoverage::Complete, + ); let policy = crate::blob_store::v2::TrigramPolicy { max_file_size: 1 << 20, }; diff --git a/crates/aft/src/views/verification.rs b/crates/aft/src/views/verification.rs index 918ccde52a..0f53132c47 100644 --- a/crates/aft/src/views/verification.rs +++ b/crates/aft/src/views/verification.rs @@ -530,6 +530,7 @@ pub(crate) struct BindGuard { pub policy: Policy, pub inputs: Vec, pub subscription: Option, + pub initial_coverage: Option, /// Stable local identities and a native backend allow event-based refresh, /// but do not prove that all writes before a cookie reached the driver. pub steady_supported: bool, @@ -543,8 +544,12 @@ impl BindGuard { }; let mut policy = Policy::capture(root, &inputs)?; let root_stat = NativeStat::strict_read(root)?; + let registration = crate::watcher_backend::subscription::status(root); let steady_supported = root_stat.reuse_supported - && cfg!(any(target_os = "linux", target_os = "macos", windows)); + && cfg!(any(target_os = "linux", target_os = "macos", windows)) + && registration + .as_ref() + .is_some_and(|status| status.coverage.is_complete()); let root_identity = root_stat.identity; let directory = match gitdir(root) { Ok(path) => Some((path.clone(), NativeStat::strict_read(&path)?.identity)), @@ -609,7 +614,8 @@ impl BindGuard { index, policy, inputs, - subscription: crate::watcher_backend::subscription::current(root), + subscription: registration.as_ref().map(|status| status.continuity), + initial_coverage: registration.map(|status| status.coverage), steady_supported, }) } diff --git a/crates/aft/src/watcher_backend/inotify.rs b/crates/aft/src/watcher_backend/inotify.rs index 8625a0e2c7..f7e61dd69e 100644 --- a/crates/aft/src/watcher_backend/inotify.rs +++ b/crates/aft/src/watcher_backend/inotify.rs @@ -30,9 +30,13 @@ pub(crate) struct ProjectWatcher { shutdown: Arc, join: Option>, watched_directory_count: Arc, + initial_coverage: super::InitialCoverage, } impl ProjectWatcher { + pub(crate) fn initial_coverage(&self) -> super::InitialCoverage { + self.initial_coverage.clone() + } pub(crate) fn create( root: PathBuf, extra_watch_paths: Vec, @@ -122,18 +126,56 @@ impl ProjectWatcher { let boundary = exclusion_state .boundary() .ok_or_else(|| notify::Error::path_not_found().add_path(root.clone()))?; - let installed = - register_initial_tree(boundary, &matcher, &exclusion_paths, &mut |directory| { - watch_directory(&mut watcher, directory, &mut watch, &counters) - })?; + let InitialRegistration { + watched: installed, + mut gaps, + } = register_initial_tree(boundary, &matcher, &exclusion_paths, &mut |directory| { + watch_directory(&mut watcher, directory, &mut watch, &counters) + })?; let mut watched_directories = installed; let git_directory = root.join(".git"); let mut git_watched = watched_directories.contains(&git_directory); for path in extra_watch_paths { - if path.exists() { - let _ = watch_directory(&mut watcher, &path, &mut watch, &counters)?; + let exists = match path.try_exists() { + Ok(exists) => exists, + Err(error) => { + gaps.push(super::CoverageGap { + path: path.clone(), + kind: super::CoverageGapKind::MetadataFailed(error.kind()), + }); + if path == git_directory { + git_watched = false; + } + continue; + } + }; + if exists { + let watched = match watch_directory(&mut watcher, &path, &mut watch, &counters) { + Ok(watched) => watched, + Err(error) if fatal_watch_error(&error) => return Err(error), + Err(error) => { + let kind = match &error.kind { + ErrorKind::Io(error) => error.kind(), + _ => std::io::ErrorKind::Other, + }; + gaps.push(super::CoverageGap { + path: path.clone(), + kind: super::CoverageGapKind::WatchFailed(kind), + }); + if path == git_directory { + git_watched = false; + } + continue; + } + }; + if !watched { + gaps.push(super::CoverageGap { + path: path.clone(), + kind: super::CoverageGapKind::WatchLost, + }); + } if path == git_directory { - git_watched = true; + git_watched = watched; } } } @@ -452,6 +494,11 @@ impl ProjectWatcher { shutdown, join: Some(join), watched_directory_count, + initial_coverage: if gaps.is_empty() { + super::InitialCoverage::Complete + } else { + super::InitialCoverage::Incomplete(gaps) + }, }) } @@ -494,6 +541,11 @@ impl Drop for ProjectWatcher { // A removed directory cannot contain live files to invalidate. A move out // produces a rename event for its old path; a move into the root is scanned // on Rename(To), including any files already present in its subtree. +fn fatal_watch_error(error: ¬ify::Error) -> bool { + matches!(error.kind, ErrorKind::MaxFilesWatch) + || matches!(&error.kind, ErrorKind::Io(io) if matches!(io.raw_os_error(), Some(libc::ENOSPC | libc::EMFILE | libc::ENFILE))) +} + fn watch_directory( watcher: &mut notify::RecommendedWatcher, path: &Path, @@ -508,8 +560,7 @@ where Err(error) => { // Limits and resource exhaustion are failures of the watcher, even // if the directory happens to disappear before the recheck. - let fatal = matches!(error.kind, ErrorKind::MaxFilesWatch) - || matches!(&error.kind, ErrorKind::Io(io) if matches!(io.raw_os_error(), Some(libc::ENOSPC | libc::EMFILE | libc::ENFILE))); + let fatal = fatal_watch_error(&error); if !fatal && (matches!(error.kind, ErrorKind::PathNotFound) || matches!(&error.kind, ErrorKind::Io(io) if io.kind() == std::io::ErrorKind::NotFound) @@ -722,45 +773,119 @@ impl DirectoryIdentity { } } -/// Arm each parent before listing it. Children created during registration -/// are then either enumerated or reported on the already armed parent stream. +struct InitialRegistration { + watched: BTreeSet, + gaps: Vec, +} + +/// Arm each parent before listing it. Keep covered directories running when a +/// descendant fails, but retain every gap so indexing cannot trust full coverage. fn register_initial_tree( boundary: &crate::watcher_filter::WatcherWalkBoundary, matcher: &SharedGitignore, exclusions: &[PathBuf], watch: &mut dyn FnMut(&Path) -> notify::Result, -) -> notify::Result> { +) -> notify::Result { let root = boundary.root(); let mut stack = vec![root.to_path_buf()]; let mut installed = BTreeSet::new(); + let mut gaps = Vec::new(); while let Some(directory) = stack.pop() { if exclusions .iter() .any(|excluded| directory.starts_with(excluded)) || watcher_directory_is_ignored_by_matcher(matcher, &directory, root) - || !boundary.allows_directory(&directory) { continue; } - if !watch(&directory)? { - return Err(notify::Error::path_not_found().add_path(directory)); + match std::fs::symlink_metadata(&directory) { + Ok(metadata) if metadata.is_dir() => {} + Ok(_) => { + gaps.push(super::CoverageGap { + path: directory, + kind: super::CoverageGapKind::WatchLost, + }); + continue; + } + Err(error) if directory == root => { + return Err(notify::Error::io(error).add_path(directory)) + } + Err(error) => { + gaps.push(super::CoverageGap { + path: directory, + kind: super::CoverageGapKind::MetadataFailed(error.kind()), + }); + continue; + } + } + if !boundary.allows_directory(&directory) { + continue; + } + match watch(&directory) { + Ok(true) => {} + Ok(false) => { + gaps.push(super::CoverageGap { + path: directory, + kind: super::CoverageGapKind::WatchLost, + }); + continue; + } + Err(error) if directory == root || fatal_watch_error(&error) => return Err(error), + Err(error) => { + let kind = match &error.kind { + ErrorKind::Io(error) => error.kind(), + _ => std::io::ErrorKind::Other, + }; + gaps.push(super::CoverageGap { + path: directory, + kind: super::CoverageGapKind::WatchFailed(kind), + }); + continue; + } } installed.insert(directory.clone()); #[cfg(test)] tests::block_watch_walk(&directory); - for entry in - crate::watcher_filter::exclusion_read_dir(&directory).map_err(notify::Error::io)? - { - let entry = entry.map_err(notify::Error::io)?; - if entry.file_type().map_err(notify::Error::io)?.is_dir() { - stack.push(entry.path()); + #[cfg(test)] + let entries = if tests::initial_listing_denied(&directory) { + Err(std::io::Error::from(std::io::ErrorKind::PermissionDenied)) + } else { + crate::watcher_filter::exclusion_read_dir(&directory) + }; + #[cfg(not(test))] + let entries = crate::watcher_filter::exclusion_read_dir(&directory); + match entries { + Ok(entries) => { + for entry in entries { + match entry { + Ok(entry) => match entry.file_type() { + Ok(kind) if kind.is_dir() => stack.push(entry.path()), + Ok(_) => {} + Err(error) => gaps.push(super::CoverageGap { + path: entry.path(), + kind: super::CoverageGapKind::MetadataFailed(error.kind()), + }), + }, + Err(error) => gaps.push(super::CoverageGap { + path: directory.clone(), + kind: super::CoverageGapKind::ListingFailed(error.kind()), + }), + } + } } + Err(error) => gaps.push(super::CoverageGap { + path: directory, + kind: super::CoverageGapKind::ListingFailed(error.kind()), + }), } } if !installed.contains(root) { return Err(notify::Error::path_not_found().add_path(root.to_path_buf())); } - Ok(installed) + Ok(InitialRegistration { + watched: installed, + gaps, + }) } fn collect_watch_directories( @@ -841,6 +966,22 @@ fn desired_watch_directories( #[cfg(test)] pub(super) mod tests { + thread_local! { + static INITIAL_LISTING_DENIED: std::cell::RefCell> = const { std::cell::RefCell::new(None) }; + } + pub(super) fn initial_listing_denied(path: &Path) -> bool { + INITIAL_LISTING_DENIED.with(|denied| denied.borrow().as_deref() == Some(path)) + } + struct InitialListingDenial(Option); + impl Drop for InitialListingDenial { + fn drop(&mut self) { + INITIAL_LISTING_DENIED.with(|denied| *denied.borrow_mut() = self.0.take()); + } + } + fn deny_initial_listing(path: PathBuf) -> InitialListingDenial { + InitialListingDenial(INITIAL_LISTING_DENIED.with(|denied| denied.replace(Some(path)))) + } + #[test] fn initial_subscription_arms_parent_before_listing_new_descendants() { let temp = tempfile::tempdir().unwrap(); @@ -871,11 +1012,12 @@ pub(super) mod tests { root.join("during/nested") ] ); - assert_eq!(installed.len(), 3); + assert_eq!(installed.watched.len(), 3); + assert!(installed.gaps.is_empty()); } #[test] - fn initial_subscription_refuses_a_missing_watch_or_incomplete_listing() { + fn initial_subscription_reports_partial_watches_and_incomplete_listings() { for remove_after_watch in [false, true] { let temp = tempfile::tempdir().unwrap(); let root = temp.path().canonicalize().unwrap(); @@ -897,7 +1039,12 @@ pub(super) mod tests { } }, ); - assert!(result.is_err()); + let registration = result.unwrap(); + assert!(registration.watched.contains(&root)); + assert!(registration + .gaps + .iter() + .any(|gap| gap.path == root.join("child"))); } } @@ -914,6 +1061,88 @@ pub(super) mod tests { assert!(result.is_err()); } + #[test] + fn partial_initial_coverage_keeps_shared_watching_but_never_grants_v3_trust() { + let _serial = crate::blob_store::v3::serial_for_test(); + let temp = tempfile::tempdir().unwrap(); + let storage = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + std::fs::create_dir_all(root.join("readable")).unwrap(); + std::fs::create_dir_all(root.join("unreadable/nested")).unwrap(); + std::fs::write(root.join("unreadable/nested/file.rs"), b"fn hidden() {}\n").unwrap(); + let _denied = deny_initial_listing(root.join("unreadable")); + let (tx, rx) = mpsc::channel(); + let watcher = ProjectWatcher::create( + root.clone(), + vec![], + tx, + Arc::new(RwLock::new(None)), + Arc::new(AtomicU64::new(0)), + ) + .unwrap(); + let coverage = super::super::initial_coverage_of(&watcher); + let super::super::InitialCoverage::Incomplete(gaps) = &coverage else { + panic!("partial registration must name its gap") + }; + assert!(gaps.iter().any(|gap| gap.path == root.join("unreadable") + && gap.kind + == super::super::CoverageGapKind::ListingFailed( + std::io::ErrorKind::PermissionDenied + ))); + let changed = root.join("readable/still-watched.rs"); + std::fs::write(&changed, b"fn visible() {}\n").unwrap(); + let deadline = Instant::now() + Duration::from_secs(5); + loop { + let event = rx + .recv_timeout(deadline.saturating_duration_since(Instant::now())) + .unwrap() + .unwrap(); + if event.paths.contains(&changed) { + break; + } + } + let _subscription = super::super::subscription::start(&root, coverage.clone()); + assert_eq!( + super::super::subscription::status(&root).unwrap().coverage, + coverage + ); + assert!(super::super::subscription::current(&root).is_none()); + let policy = crate::blob_store::v2::TrigramPolicy { + max_file_size: 1 << 20, + }; + let driver = crate::views::first_load::CheckoutDriver::new_v3( + storage.path(), + "family", + "worker", + &root, + crate::views::manifest_v2::Producers { + trigram: policy.fingerprint_hex(), + semantic: None, + callgraph: "unregistered".into(), + }, + Arc::new(crate::views::first_load::ConfiguredMembershipWalker), + vec![Arc::new(crate::views::first_load::TrigramBridge::new( + storage.path().to_owned(), + policy, + ))], + ) + .unwrap(); + let access = driver.access(); + let (snapshot, revision) = driver + .verified_bind(&access, driver.installed_snapshot().generation().clone()) + .unwrap(); + assert!(snapshot + .generation() + .published() + .unwrap() + .verified_frontier + .is_none()); + assert!(driver.take_reconcile_work().walks > 0); + use crate::views::first_load::FirstLoadDriver; + driver.install(&access, &snapshot, revision).unwrap(); + assert!(driver.installed_snapshot().needs_reconcile()); + } + #[test] fn watcher_cookie_inotify_recovery_precedes_root_cookie_without_command_tracking() { let temp = tempfile::tempdir().unwrap(); diff --git a/crates/aft/src/watcher_backend/mod.rs b/crates/aft/src/watcher_backend/mod.rs index 795ccedf00..b372e42284 100644 --- a/crates/aft/src/watcher_backend/mod.rs +++ b/crates/aft/src/watcher_backend/mod.rs @@ -9,6 +9,60 @@ mod cookie_tests; pub(crate) mod flush; pub(crate) mod subscription; +// Inotify reports per-directory gaps. Recursive-handle backends currently +// report only successful or unproved registration at construction. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum CoverageGapKind { + ListingFailed(std::io::ErrorKind), + MetadataFailed(std::io::ErrorKind), + WatchFailed(std::io::ErrorKind), + WatchLost, +} + +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CoverageGap { + pub path: PathBuf, + pub kind: CoverageGapKind, +} + +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum InitialCoverage { + Complete, + Incomplete(Vec), + Unproved, +} + +impl InitialCoverage { + pub(crate) fn is_complete(&self) -> bool { + matches!(self, Self::Complete) + } +} + +// Raw-event test shims and other opaque handles cannot grant kernel coverage. +// Inspect the actual retained native handle without constraining the generic +// filter API used by those event injectors. +pub(crate) fn initial_coverage_of(handle: &dyn std::any::Any) -> InitialCoverage { + handle + .downcast_ref::() + .map_or(InitialCoverage::Unproved, |watcher| { + watcher.initial_coverage() + }) +} + +#[cfg(not(target_os = "linux"))] +impl ProjectWatcher { + pub(crate) fn initial_coverage(&self) -> InitialCoverage { + if cfg!(any(target_os = "macos", windows)) { + InitialCoverage::Complete + } else { + InitialCoverage::Unproved + } + } +} + #[cfg(target_os = "macos")] mod fsevents; #[cfg(target_os = "linux")] diff --git a/crates/aft/src/watcher_backend/subscription.rs b/crates/aft/src/watcher_backend/subscription.rs index e1a9cb9399..64e88a30ca 100644 --- a/crates/aft/src/watcher_backend/subscription.rs +++ b/crates/aft/src/watcher_backend/subscription.rs @@ -7,6 +7,8 @@ use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex, OnceLock, Weak}; +use super::InitialCoverage; + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) struct Continuity { pub generation: u64, @@ -15,9 +17,30 @@ pub(crate) struct Continuity { struct State { generation: u64, gap: AtomicU64, + coverage: InitialCoverage, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct SubscriptionStatus { + pub continuity: Continuity, + pub coverage: InitialCoverage, } pub(crate) struct Subscription { _state: Arc, + root: PathBuf, +} +impl Drop for Subscription { + fn drop(&mut self) { + let mut roots = roots() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if roots + .get(&self.root) + .is_some_and(|state| state.ptr_eq(&Arc::downgrade(&self._state))) + { + roots.remove(&self.root); + } + } } static NEXT: AtomicU64 = AtomicU64::new(1); fn roots() -> &'static Mutex>> { @@ -25,27 +48,28 @@ fn roots() -> &'static Mutex>> { ROOTS.get_or_init(Default::default) } -pub(crate) fn start(root: &Path) -> Subscription { +pub(crate) fn start(root: &Path, coverage: InitialCoverage) -> Subscription { let state = Arc::new(State { generation: NEXT.fetch_add(1, Ordering::Relaxed), gap: AtomicU64::new(0), + coverage, }); roots() .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .insert(root.to_path_buf(), Arc::downgrade(&state)); - Subscription { _state: state } + Subscription { + _state: state, + root: root.to_path_buf(), + } } -pub(crate) fn current(root: &Path) -> Option { +fn retained(root: &Path) -> Option> { let mut roots = roots() .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); match roots.get(root).and_then(Weak::upgrade) { - Some(state) => Some(Continuity { - generation: state.generation, - gap: state.gap.load(Ordering::Acquire), - }), + Some(state) => Some(state), None => { roots.remove(root); None @@ -53,6 +77,25 @@ pub(crate) fn current(root: &Path) -> Option { } } +pub(crate) fn current(root: &Path) -> Option { + retained(root) + .filter(|state| state.coverage.is_complete()) + .map(|state| Continuity { + generation: state.generation, + gap: state.gap.load(Ordering::Acquire), + }) +} + +pub(crate) fn status(root: &Path) -> Option { + retained(root).map(|state| SubscriptionStatus { + continuity: Continuity { + generation: state.generation, + gap: state.gap.load(Ordering::Acquire), + }, + coverage: state.coverage.clone(), + }) +} + pub(crate) fn gap(root: &Path) { if let Some(state) = roots() .lock() @@ -66,15 +109,40 @@ pub(crate) fn gap(root: &Path) { #[cfg(test)] mod tests { + #[test] + fn incomplete_and_unproved_handles_never_grant_a_continuity_token() { + let root = tempfile::tempdir().unwrap(); + let coverage = + super::InitialCoverage::Incomplete(vec![crate::watcher_backend::CoverageGap { + path: root.path().join("unreadable"), + kind: crate::watcher_backend::CoverageGapKind::ListingFailed( + std::io::ErrorKind::PermissionDenied, + ), + }]); + let partial = super::start(root.path(), coverage.clone()); + assert_eq!(super::status(root.path()).unwrap().coverage, coverage); + assert!(super::current(root.path()).is_none()); + drop(partial); + let _unproved = super::start( + root.path(), + crate::watcher_backend::initial_coverage_of(&()), + ); + assert_eq!( + super::status(root.path()).unwrap().coverage, + super::InitialCoverage::Unproved + ); + assert!(super::current(root.path()).is_none()); + } + #[test] fn subscription_restart_stop_and_gap_change_the_continuity_token() { let root = tempfile::tempdir().unwrap(); assert!(super::current(root.path()).is_none()); - let first = super::start(root.path()); + let first = super::start(root.path(), super::InitialCoverage::Complete); let before = super::current(root.path()).unwrap(); super::gap(root.path()); assert_ne!(super::current(root.path()).unwrap(), before); - let second = super::start(root.path()); + let second = super::start(root.path(), super::InitialCoverage::Complete); let after = super::current(root.path()).unwrap(); assert_ne!(before.generation, after.generation); drop(first); diff --git a/crates/aft/src/watcher_filter.rs b/crates/aft/src/watcher_filter.rs index 0276a8e610..c76df5b841 100644 --- a/crates/aft/src/watcher_filter.rs +++ b/crates/aft/src/watcher_filter.rs @@ -2293,7 +2293,10 @@ pub fn run_watcher_thread( return; } crate::slog_info!("watcher started: {}", root_path.display()); - let _subscription = crate::watcher_backend::subscription::start(&root_path); + let _subscription = crate::watcher_backend::subscription::start( + &root_path, + crate::watcher_backend::initial_coverage_of(&_watcher), + ); let mut filter = WatcherFilterThread::new( config, matcher, From 9d6ee581bc862e904a652ecfcc14a7c58d03197f Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:16:47 +0100 Subject: [PATCH 28/31] mason: refuse unowned fresh planes and complete source metadata guards Fresh bootstrap is lexical-only until the G/H base completion coordinators exist; typed preflight refuses unsupported planes before reads or claims. Existing bases remain plane-independent. Reuse the source stamp already taken before the read instead of adding a duplicate stat, and preserve resolution-input metadata on cached callgraph attachments. Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/views/base_refs.rs | 2 +- crates/aft/src/views/first_load.rs | 327 +++++++++++++++++++++++++++-- 2 files changed, 312 insertions(+), 17 deletions(-) diff --git a/crates/aft/src/views/base_refs.rs b/crates/aft/src/views/base_refs.rs index 27f7029358..eb700352f9 100644 --- a/crates/aft/src/views/base_refs.rs +++ b/crates/aft/src/views/base_refs.rs @@ -708,7 +708,7 @@ pub fn publish_worker_with( /// Publishes with ownership serialization around final validation and the CAS. /// `publish_cas` may decline without calling its operation, but may call it only -/// once. Artifact construction and admission happen before this short fence. +/// once. Artifact construction and admission happen before this fence. pub fn publish_worker_guarded( request: WorkerPublication<'_>, observer: Option<&dyn DurabilityObserver>, diff --git a/crates/aft/src/views/first_load.rs b/crates/aft/src/views/first_load.rs index 6bae55a45c..b33b572a4e 100644 --- a/crates/aft/src/views/first_load.rs +++ b/crates/aft/src/views/first_load.rs @@ -1676,13 +1676,72 @@ pub(crate) struct ReconcileWork { pub attachments: usize, } +/// Why this driver cannot own and complete a fresh base. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum FreshBootstrapRefusal { + NonLexicalPlane(FamilyPlane), + MissingLexicalPlane, + LexicalProducerMismatch, +} + +impl std::fmt::Display for FreshBootstrapRefusal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::NonLexicalPlane(plane) => write!( + f, + "fresh_bootstrap_nonlexical_plane: {} requires its base completion coordinator", + plane.as_str() + ), + Self::MissingLexicalPlane => f.write_str("fresh_bootstrap_missing_lexical_plane"), + Self::LexicalProducerMismatch => { + f.write_str("fresh_bootstrap_lexical_producer_mismatch") + } + } + } +} +impl std::error::Error for FreshBootstrapRefusal {} + impl CheckoutDriver { + /// Pure eligibility check for callers choosing between new and existing + /// runtime paths. The bind hook reports the same refusal through PlaneError. + pub fn fresh_bootstrap_policy( + &self, + ) -> Result { + if let Some(plane) = self + .planes + .iter() + .map(|plane| plane.plane()) + .find(|plane| *plane != FamilyPlane::Trigram) + { + return Err(FreshBootstrapRefusal::NonLexicalPlane(plane)); + } + if self.producers.semantic.is_some() { + return Err(FreshBootstrapRefusal::NonLexicalPlane( + FamilyPlane::Semantic, + )); + } + if self.producers.callgraph != super::semantic_runtime::UNREGISTERED_PRODUCER { + return Err(FreshBootstrapRefusal::NonLexicalPlane( + FamilyPlane::Callgraph, + )); + } + let policy = self + .planes + .iter() + .find_map(|plane| plane.bootstrap_policy()) + .ok_or(FreshBootstrapRefusal::MissingLexicalPlane)?; + if self.producers.trigram != policy.fingerprint_hex() { + return Err(FreshBootstrapRefusal::LexicalProducerMismatch); + } + Ok(policy) + } fn stable_read( &self, path: &RelPath, absolute: &std::path::Path, recorded_ns: u128, first_seq: u64, + first_stat: Option, ) -> Result { for attempt in 0..3 { let seq = if attempt == 0 { @@ -1693,7 +1752,11 @@ impl CheckoutDriver { .unwrap_or_else(std::sync::PoisonError::into_inner) .seq }; - let before = self.source_stat(absolute, recorded_ns)?; + let before = if attempt == 0 { + first_stat.clone() + } else { + self.source_stat(absolute, recorded_ns)? + }; let bytes = super::assembly::read_working_tree_file(absolute) .map_err(|error| SiblingLoader::error(error.to_string()))?; #[cfg(test)] @@ -1742,6 +1805,8 @@ impl CheckoutDriver { /// A fresh checkout bootstraps a lexical-ready v3 base and publishes a v3 /// worker over it; it never falls back to legacy storage. Installation is /// left to the caller and still validates the cut's epoch and bundle identity. + /// Fresh bases are lexical-only until nonlexical completion can be scheduled; + /// already-built bases can be verified for any registered plane. pub fn verified_bind( &self, access: &ViewAccess, @@ -1753,6 +1818,15 @@ impl CheckoutDriver { "verified_bind requires a v3-only registration", )); } + let bootstrap = base.base_header().is_none(); + if bootstrap { + self.fresh_bootstrap_policy() + .map_err(|reason| SiblingLoader::error(reason.to_string()))?; + } else { + base.ensure_producers(&self.producers).map_err(|error| { + SiblingLoader::stage_error(LoaderStage::ProducerValidation, error.to_string()) + })?; + } if let Some(token) = self .advisory_token .lock() @@ -1774,7 +1848,6 @@ impl CheckoutDriver { now, ); } - let bootstrap = base.base_header().is_none(); self.v3_reconcile .store(true, std::sync::atomic::Ordering::Release); let base = if bootstrap { @@ -1832,17 +1905,8 @@ impl CheckoutDriver { seed: Arc, ) -> Result, PlaneError> { let policy = self - .planes - .iter() - .find_map(|plane| plane.bootstrap_policy()) - .ok_or_else(|| { - SiblingLoader::error("v3 bootstrap requires a registered lexical plane") - })?; - if self.producers.trigram != policy.fingerprint_hex() { - return Err(SiblingLoader::error( - "v3 bootstrap lexical producer mismatch", - )); - } + .fresh_bootstrap_policy() + .map_err(|reason| SiblingLoader::error(reason.to_string()))?; let (checkout, _) = SiblingLoader::reconciled_snapshot(self, access, seed)?; if checkout.pending_intent().next().is_some() { return Err(SiblingLoader::error( @@ -2496,7 +2560,7 @@ impl FirstLoadDriver for CheckoutDriver { } } let (bytes, stat_evidence) = - match self.stable_read(&path, &absolute, recorded_ns, delivery_seq)? { + match self.stable_read(&path, &absolute, recorded_ns, delivery_seq, before)? { StableRead::Absent => continue, StableRead::Bytes(bytes, evidence) => (bytes, evidence), StableRead::Pending => { @@ -4619,10 +4683,14 @@ impl CompositePlane for CallgraphBridge { "cached callgraph row disappeared before protection".into(), )); } - if let Some(super::manifest_v2::EntryV2::Regular { planes, .. }) = - delta.get_mut(path) + if let Some(super::manifest_v2::EntryV2::Regular { + planes, + resolution_input, + .. + }) = delta.get_mut(path) { planes.callgraph = Some(super::readiness::PlaneState::ready(&cached.0)); + *resolution_input = super::assembly::is_resolution_input(path.as_bytes()); } continue; } @@ -5367,6 +5435,9 @@ impl CompositePlane for TrigramBridge { delta: &mut super::manifest_v2::ManifestV2, live: &mut crate::pins::LivePin, ) -> Result<(), PlaneError> { + if delta.header().producers.trigram != self.policy.fingerprint_hex() { + return Err(SiblingLoader::error("trigram producer mismatch")); + } let mut keys = Vec::new(); let mut payloads = Vec::new(); for (path, entry) in delta.entries() { @@ -5808,6 +5879,112 @@ mod checkout_kind_tests { mod v3_bootstrap_tests { use super::*; + struct NamedPlane(FamilyPlane); + impl CompositePlane for NamedPlane { + fn plane(&self) -> FamilyPlane { + self.0 + } + fn applies_to(&self, _: &RelPath) -> bool { + false + } + fn attachment( + &self, + _: &RelPath, + _: &[u8], + ) -> Result { + panic!("unsupported bootstrap must not extract") + } + fn materialize( + &self, + _: &super::super::registry::ViewRegistration, + _: &str, + _: &Snapshot, + _: &BTreeMap, + _: &mut super::super::manifest_v2::ManifestV2, + _: &mut crate::pins::LivePin, + _: bool, + ) -> Result<(), PlaneError> { + panic!("unsupported bootstrap must not materialize") + } + fn finish_generation( + &self, + _: &super::super::registry::ViewRegistration, + _: &str, + _: &str, + ) -> Result<(), PlaneError> { + Ok(()) + } + } + + #[test] + fn fresh_nonlexical_bootstrap_has_a_typed_refusal_and_takes_no_claim() { + let _serial = crate::blob_store::v3::serial_for_test(); + for plane in [FamilyPlane::Semantic, FamilyPlane::Callgraph] { + let root = tempfile::tempdir().unwrap(); + let storage = tempfile::tempdir().unwrap(); + std::fs::write(root.path().join("file.rs"), b"fn untouched() {}\n").unwrap(); + let policy = crate::blob_store::v2::TrigramPolicy { + max_file_size: 1 << 20, + }; + let driver = CheckoutDriver::new_v3( + storage.path(), + "family", + "worker", + root.path(), + Producers { + trigram: policy.fingerprint_hex(), + semantic: (plane == FamilyPlane::Semantic).then(|| "model".into()), + callgraph: if plane == FamilyPlane::Callgraph { + super::super::callgraph::PRODUCER.into() + } else { + "unregistered".into() + }, + }, + Arc::new(ConfiguredMembershipWalker), + vec![ + Arc::new(TrigramBridge::new(storage.path().to_owned(), policy)), + Arc::new(NamedPlane(plane)), + ], + ) + .unwrap(); + assert_eq!( + driver.fresh_bootstrap_policy(), + Err(FreshBootstrapRefusal::NonLexicalPlane(plane)) + ); + let error = driver + .verified_bind( + &driver.access(), + driver.installed_snapshot().generation().clone(), + ) + .unwrap_err(); + assert!(error.reason.contains("fresh_bootstrap_nonlexical_plane")); + let work = driver.take_reconcile_work(); + assert_eq!((work.walks, work.reads), (0, 0)); + let registry = + crate::blob_store::v3::LedgerReader::open_existing(storage.path(), "family") + .unwrap(); + assert_eq!( + registry + .read( + |c| Ok(c.query_row("SELECT COUNT(*) FROM base_builds", [], |r| r + .get::<_, u64>(0))?) + ) + .unwrap(), + 0 + ); + assert_eq!( + registry + .read(|c| Ok(c.query_row( + "SELECT COUNT(*) FROM members WHERE kind='base'", + [], + |r| r.get::<_, u64>(0) + )?)) + .unwrap(), + 0 + ); + } + } + fn trigram_driver(root: &std::path::Path, storage: &std::path::Path) -> CheckoutDriver { let policy = crate::blob_store::v2::TrigramPolicy { max_file_size: 1 << 20, @@ -5979,9 +6156,127 @@ mod v3_bootstrap_tests { let work = driver.take_reconcile_work(); assert_eq!(work.reads, 1); assert_eq!(work.attachments, expected_extractions); + let mut header = driver.installed_snapshot().generation().header().clone(); + header.producers.trigram = "wrong-producer".into(); + let mut delta = + super::super::manifest_v2::ManifestV2::new_v3(header, None, Default::default()); + let bridge = TrigramBridge::new(storage.path().to_owned(), policy); + let mut pin = + crate::pins::LivePin::create_v3(storage.path(), family, "worker").unwrap(); + assert!(bridge + .materialize_v3( + &driver.owner, + &driver.owner.v3_route(), + "unused", + driver.installed_snapshot().generation(), + &BTreeMap::new(), + &mut delta, + &mut pin + ) + .is_err()); } } + #[test] + fn cached_callgraph_configuration_keeps_resolution_input_metadata() { + let _serial = crate::blob_store::v3::serial_for_test(); + let root = tempfile::tempdir().unwrap(); + let storage = tempfile::tempdir().unwrap(); + let owner = super::super::registry::ViewRegistration::register_v3( + storage.path(), + "family", + "worker", + root.path(), + ) + .unwrap(); + let bridge = CallgraphBridge::default(); + let path = RelPath::new(b"package.json".to_vec()).unwrap(); + let bytes = b"{\"name\":\"cached-configuration\"}"; + let content = crate::blob_store::v2::ContentHash::of(bytes); + let attachment = bridge.attachment(&path, bytes).unwrap(); + let attachment = attachment + .downcast_ref::() + .unwrap(); + let key = attachment.key; + crate::blob_store::v3::publish(storage.path(), "family", 1 << 20, None, |p| { + p.put( + &key, + &attachment.blob.to_bytes().unwrap(), + FamilyPlane::Callgraph.payload_schema(), + )?; + Ok(()) + }) + .unwrap(); + let header = super::super::manifest_v2::ManifestHeader { + producers: Producers { + trigram: "unused".into(), + semantic: None, + callgraph: super::super::callgraph::PRODUCER.into(), + }, + head_tree: None, + ignore_fingerprint: None, + segment: None, + }; + let mut base_manifest = + super::super::manifest_v2::ManifestV2::new_v3(header.clone(), None, Default::default()); + base_manifest + .insert( + RelPath::new(b"base.rs".to_vec()).unwrap(), + super::super::manifest_v2::EntryV2::regular( + crate::blob_store::v2::ContentHash::of(b"base"), + 4, + super::super::manifest_v2::EntryPlanes { + callgraph: Some(super::super::readiness::PlaneState::pending( + "base graph pending", + )), + ..Default::default() + }, + ), + ) + .unwrap(); + let identity = super::super::base_refs::BaseIdentity::of("family", &base_manifest).unwrap(); + let base = Arc::new(ComposedGeneration::new("base", base_manifest, None)); + let mut delta = super::super::manifest_v2::ManifestV2::new_v3( + header, + Some(identity.header("base")), + Default::default(), + ); + delta + .insert( + path.clone(), + super::super::manifest_v2::EntryV2::regular( + content, + bytes.len() as u64, + Default::default(), + ), + ) + .unwrap(); + let mut entry = LiveEntry::new(super::super::snapshot::DiskState::of_bytes(bytes), 0); + entry.attachments.insert( + FamilyPlane::Callgraph, + bridge.cached_attachment(content, key), + ); + let mut pin = crate::pins::LivePin::create_v3(storage.path(), "family", "worker").unwrap(); + bridge + .materialize_v3( + &owner, + &owner.v3_route(), + "private", + &base, + &BTreeMap::from([(path.clone(), entry)]), + &mut delta, + &mut pin, + ) + .unwrap(); + assert!(matches!( + delta.get(&path), + Some(super::super::manifest_v2::EntryV2::Regular { + resolution_input: true, + .. + }) + )); + } + fn legacy_inventory(storage: &std::path::Path) -> BTreeMap> { fn visit( base: &std::path::Path, From bae4932951b82bd746568aebc72158df141dfb45 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:16:49 +0100 Subject: [PATCH 29/31] mason: keep newer-registry refusal tests above the supported format Registry format 3 is now supported. Probe the compiled maximum plus one so the existing newer-format refusal assertion remains a newer-format test rather than schema drift within a supported version. Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/blob_store/v3/tests.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/aft/src/blob_store/v3/tests.rs b/crates/aft/src/blob_store/v3/tests.rs index fc905f832a..d568cee51e 100644 --- a/crates/aft/src/blob_store/v3/tests.rs +++ b/crates/aft/src/blob_store/v3/tests.rs @@ -859,15 +859,15 @@ fn newer_format_keeps_artifact_refusal_and_malformed_file_reports_found_zero() { let root = tempdir().unwrap(); seed(root.path(), A); let path = db(root.path(), A); + let future = REGISTRY_VERIFICATION_VERSION + 1; Connection::open(&path) .unwrap() - // Registry 2 is the base-reference protocol this build reads; the - // first newer version is 3. - .execute("UPDATE format_meta SET format_version=3", []) + // Keep testing a future registry, not drift within a supported version. + .execute("UPDATE format_meta SET format_version=?1", [future]) .unwrap(); match LedgerReader::open_existing(root.path(), A).err().unwrap() { Error::Incompatible(error) => { - assert_eq!(error.found, 3); + assert_eq!(error.found, u64::from(future)); assert_eq!(error.source, persisted_format::RefusalSource::Artifact); } error => panic!("{error}"), From b7a708969f29c7d710a28e7c7da76cfeee73447e Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:26:20 +0100 Subject: [PATCH 30/31] mason: make the source-probe control exercise a nonregular path Missing paths intentionally cost zero bytes, so the earlier missing-file fixture could not distinguish a skipped probe. Keep the same assertions but use a directory at the live file path: the probe must classify it as large after an intent, and must not run for an already reconciled delta. Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/views/semantic_runtime_tests.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/aft/src/views/semantic_runtime_tests.rs b/crates/aft/src/views/semantic_runtime_tests.rs index 45135e8cac..74c54de9af 100644 --- a/crates/aft/src/views/semantic_runtime_tests.rs +++ b/crates/aft/src/views/semantic_runtime_tests.rs @@ -38,11 +38,11 @@ fn reconciled_delta_does_not_request_another_source_walk() { delta.apply(path.clone(), entry); assert!(!super::needs_reload(&delta.snapshot())); let root = tempfile::tempdir().unwrap(); - let missing = root.path().join("source-probe-must-not-run"); - assert!(!super::needs_cold_reconcile(&delta.snapshot(), &missing)); + std::fs::create_dir(root.path().join("file.rs")).unwrap(); + assert!(!super::needs_cold_reconcile(&delta.snapshot(), root.path())); delta.record_intent(path); assert!(super::needs_reload(&delta.snapshot())); - assert!(super::needs_cold_reconcile(&delta.snapshot(), &missing)); + assert!(super::needs_cold_reconcile(&delta.snapshot(), root.path())); } /// The test prints public setup, query answers, a v3 file inventory and exact From 1b15e302cd607d0b9e2498239df1d1f84189ecd2 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:43:03 +0100 Subject: [PATCH 31/31] search-quality descriptor for views-d-verified-bind Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- .../land-views-d-verified-bind.json | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 benchmarks/aft-search/slice-descriptors/land-views-d-verified-bind.json diff --git a/benchmarks/aft-search/slice-descriptors/land-views-d-verified-bind.json b/benchmarks/aft-search/slice-descriptors/land-views-d-verified-bind.json new file mode 100644 index 0000000000..f09d77027f --- /dev/null +++ b/benchmarks/aft-search/slice-descriptors/land-views-d-verified-bind.json @@ -0,0 +1,13 @@ +{ + "slice_class": "engine_unwired", + "targeted_mechanism": "none", + "kind": "harness", + "fixtures": [ + "crates/aft/tests/integration/tool_call_parity_test.rs", + "crates/aft/src/lib.rs" + ], + "slices": [ + "views-d-verified-bind" + ], + "note": "The only ranking fence hit is crates/aft/src/lib.rs, which exports the git index reader module brought in from 4v-1. Views stay default-off and unrouted in this step; no search, routing or ranking code changes, so ranked rows must remain byte-identical." +}