From 7d2bc52efa5ff3e0beee60fca0e913b60ea6f873 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:59:41 +0100 Subject: [PATCH 1/9] mason: register tree-keyed base identity and tree proof format Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit f3c935edb548a56d523b24ce4263582e584cd53a) --- docs/design/views-v3-format.md | 112 ++++++++++++++++++++++++++++++++- 1 file changed, 111 insertions(+), 1 deletion(-) diff --git a/docs/design/views-v3-format.md b/docs/design/views-v3-format.md index 0cf731202..36072d87f 100644 --- a/docs/design/views-v3-format.md +++ b/docs/design/views-v3-format.md @@ -565,7 +565,9 @@ CREATE TABLE sweep_state (singleton INTEGER PRIMARY KEY CHECK(singleton=1), overrides is not a base. The identity digest (tag `aft/views-v3/base-identity/1`) covers family, all producers, the ignore fingerprint, the platform family, `head_tree:none` and the membership - digest. The base scope is `base-m1-`. + digest. The base scope is `base-m1-`. A second, tree-keyed + base kind (`base-t1-`) is defined in *Tree-keyed bases and blob receipts* + below; a scope is never relabelled from one kind to the other. - **Authentication.** Every consumer resolves a base by pin-then-verify and recomputes both digests from the pinned manifest; a `BaseHeader` is checked against that result, never trusted. `bound_generation` is @@ -653,6 +655,7 @@ contract is [views-slice4-opening.md](views-slice4-opening.md). | 4 | Plane-row sweep over `F/{semantic,callgraph,trigram}.sqlite`; progress at `retention/cursors/v3-plane-rows/.json` | Store 1/1; cursor 1; registry 2/2 for sweep coordination | Implement in `gc/family.rs` and `storage_retention.rs`. Mark the composed depth-one closure, including overrides, then delete only unmarked eligible rows under the epoch/lease protocol. Bound the producing iterator and deadline; resume rather than treating a partial scan as complete. Include later registered planes when they exist. | | 4 | `DiskBudget` wiring in `crates/aft/src/views/eviction.rs`, using `F/members.sqlite` reservations/accounting and the existing namespace inventory | Registry 2/2; no separate budget file or new config format | Preserve allocated-byte charging, the 2 GiB family target, 4 GiB hard/namespace limits and max(4 GiB, 5%) device floor. Refuse with the named storage gap and no legacy fallback. | | 4v | `M/stat-evidence-.sqlite` | Stat-evidence 1/1, registry floor 3 | Generation-paired path/root/file identity, size/times, stable content digest, proven Git OID if any, observation-time conversion-policy digest and per-plane keys. No stat-cache or current-policy-only proof. | +| 4v | `M/tree-proof.sqlite` in a `base-t1-` member | Tree proof 1/1, registry floor 3 | Full-index tree descriptor proven from the coherent index and counted `rev-parse HEAD^{tree}`, plus one byte-bound Git-blob receipt per base member. Never a header string or a stat-cache tuple. See *Tree-keyed bases and blob receipts*. | | 4v | `F/members.sqlite`: `view_tokens` rows | `views_v3_registry` 3/3 | Single-use advisory token bound to identity at first consumption, with expiry and consume state. Stored in daemon storage, never the checkout. Stat evidence and tokens do not replace freshness validation. | | 5 | `M/donor--.seal.json` beside `M/donor--.sqlite` | Donor seal 1, registry floor 4 | Atomically publish fingerprint, size and `journal_mode=DELETE` only after the detached donor is checkpointed and synced and has no WAL/SHM. A published manifest names only a completed seal. Never copy/clone a live SQLite file set. | | 6a | `F/symbol.sqlite` | `views_v3_store` 1/1, new-store layout 2; store floor 1 | The v3 symbol plane uses the same immutable `payload`/`payload_digest` contract and reservation protocol. No in-place legacy conversion. Implemented; see *Import ledger and symbol plane*. | @@ -716,6 +719,113 @@ Slice 4's binder must honour the `evicting` contract above: when it finds its member in `evicting`, it never uses that view directory, and either treats the member as absent and publishes a fresh one or waits for the eviction to finish. +#### Tree-keyed bases and blob receipts + +Source: `git_index_reader.rs` (tree computation and blob object IDs), +`views/base_refs.rs` (identity), `views/tree_base.rs` (builder), +`views/verification.rs` (proof file and bind-time checks). + +**Identity.** A base manifest whose header carries `head_tree` is a +tree-keyed base; one without it keeps the membership identity above, +unchanged. A tree-keyed manifest has only regular entries, each with a +`git_oid`, and no `base` header, tombstones or overrides. Its tree-membership +digest is BLAKE3 over the length-prefixed tag +`aft/views-v3/base-tree-membership/1`, the entry count, then per path (in path +order) the path bytes, kind byte 1, mode, content hash, size, +resolution-input flag and the `git_oid` text. Every OID and the tree are +lowercase hex of one length (40 for SHA-1, 64 for SHA-256). The identity digest +(tag `aft/views-v3/base-identity-tree/1`) covers family, all producers, the +ignore fingerprint, the platform family, the object format name (`sha1` or +`sha256`), `head_tree:` and the tree-membership digest. The scope is +`base-t1-`. The membership digest still refuses any manifest +with `head_tree` or a `git_oid`, and membership resolution still requires the +`base-m1-` prefix. + +The identity only names the tree a base claims. Header fields never prove it. +Trust needs the proof file below and a live check against Git at bind time. +Without both, a tree-keyed base is treated like a membership base: every path +takes the full check. + +**Older readers.** A tree-keyed base is published only after the registry floor +is raised to 3, so readers that support at most registry 2 refuse the domain. +A registry-3 reader without this section refuses the scope: membership +resolution rejects the `base-t1-` prefix, and `membership_digest` rejects the +manifest. Such a reader never authenticates a tree-keyed base as +membership-keyed. Its sweep records the base's edges as `base_corrupt` (strict +rebuild for those workers) and can then reclaim the base as unreferenced. That +costs a rebuild in a mixed-version fleet; it does not change any answer. + +**Builder.** The claim owner reads a coherent index snapshot. It computes the +full index tree in Git's object format and compares it with a counted +`git rev-parse HEAD^{tree}`. The computation refuses unmerged stages, +assume-valid, skip-worktree and intent-to-add entries, and duplicate paths. It +also refuses file/directory conflicts, empty, `.` or `..` components, any +component equal to `.git` (ASCII case-insensitive), and modes other than +100644, 100755, 120000 and 160000. Split and sparse indexes were already +refused by the reader. Every member is a walker-admitted path with a stage-0 +regular index entry. It is read with the same native stat before and after +the read. The bytes are hashed as BLAKE3 content and as Git's +`blob \0` object. The blob OID must equal the index entry's. On +unix the executable bit must agree with the index mode. A failure of any check +for any member means no tree-keyed base: the bind builds a membership base as +before. The conversion policy must be reusable (see *Stat evidence*). The +receipt pairs the policy digest captured for this build with that read, never +with an older index tuple. + +**Proof file `M/tree-proof.sqlite`.** One per tree-keyed base member. The +base's identity and membership never change across readiness successors, so +one proof serves every generation of the scope. It is written once by the +claim owner, after the member is registered and the registry floor is 3, and +before the first publication. After that it is never rewritten. +`application_id=0x41465454`, `PRAGMA user_version=1`, `journal_mode=DELETE`: + +```sql +CREATE TABLE proof_meta(singleton INTEGER PRIMARY KEY CHECK(singleton=1), + format_version INTEGER NOT NULL, schema_version INTEGER NOT NULL, + base_scope TEXT NOT NULL, identity_digest TEXT NOT NULL, membership_digest TEXT NOT NULL, + object_format TEXT NOT NULL, head_tree TEXT NOT NULL, root BLOB NOT NULL, + policy TEXT NOT NULL, receipts_digest TEXT NOT NULL); +CREATE TABLE blob_receipts(path BLOB PRIMARY KEY, record BLOB NOT NULL); +``` + +Format/schema are 1/1. `root` is the versioned JSON root identity used by stat +evidence. A receipt record is versioned JSON (`version: 1`): mode, size, +content digest, Git blob OID, the native stat observed around the read, +observation time (rounded down to the second), policy digest and delivery +sequence. `receipts_digest` is BLAKE3 over the tag `aft/views-v3/tree-receipts/1`, +the receipt count, then each path and its record bytes, length-prefixed and in +path order. + +Staging is `M/.tree-proof.sqlite.tmp.`. It is synced, renamed into place +and followed by a directory sync. A failed writer removes its own staging. A +crash can leave one staging file, which stays until the member is reclaimed. +The generation sweep does not match either name. Both are member files, so +the existing per-member accounting counts them, and whole-base reclaim removes +them. A reader reads the proof only while it holds the authenticated base's +exact-generation read marker. + +**Bind-time check.** A worker whose durable generation names a `base-t1-` base +resolves that base (pin-then-verify) and reads the proof. The bind refuses the +proof unless all of these hold: + +- the file's application ID, version and format/schema are as above; +- `receipts_digest` matches the stored receipts; +- scope, identity and membership digests equal the authenticated base; +- head tree and object format equal the base manifest; +- the root identity and policy digest equal the current bind guard, and the + current policy is reusable; +- every base member has exactly one receipt, and each receipt's mode, size, + content and OID equal its manifest entry. + +Git must then agree now. The guard's counted `rev-parse HEAD^{tree}` must name +the proof's tree. A coherent index read now must compute that same tree. Each +receipt's path must have a stage-0 index entry with no exception flag, the +same mode and the receipt's OID. A receipt that passes is only a candidate. +The stat-evidence reuse rules still decide each path: equal native stat and +file identity, mtime before the observation second, no watcher delivery for +the path, matching disk state, and ready, present plane keys. Any failed check +means that path, or with a refused proof every path, takes the full check. + ## Import ledger and symbol plane Sources: `crates/aft/src/migration/v3_import.rs` (decoders, staging, From 9883ddd97801ac60c636add8a52646ad51a5bd39 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:01:03 +0100 Subject: [PATCH 2/9] mason: compute full index trees and Git blob object IDs Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 775d4d79aa7b2c4bd5a82d7a2e111abe475d2990) --- crates/aft/src/git_index_reader.rs | 292 +++++++++++++++++++++++++++++ 1 file changed, 292 insertions(+) diff --git a/crates/aft/src/git_index_reader.rs b/crates/aft/src/git_index_reader.rs index 4de1a275f..4d9615071 100644 --- a/crates/aft/src/git_index_reader.rs +++ b/crates/aft/src/git_index_reader.rs @@ -21,6 +21,40 @@ pub enum ObjectFormat { } impl ObjectFormat { + /// The format whose object IDs are `hex_len` hex digits long. + pub fn from_hex_len(hex_len: usize) -> Option { + match hex_len { + 40 => Some(Self::Sha1), + 64 => Some(Self::Sha256), + _ => None, + } + } + + /// Git's name for the format (`extensions.objectFormat`). + pub fn name(self) -> &'static str { + match self { + Self::Sha1 => "sha1", + Self::Sha256 => "sha256", + } + } + + /// The object ID Git gives `bytes` stored as a blob: the digest of + /// `blob \0`. These are the exact bytes read, so the caller + /// must already know that no conversion applies between them and the blob. + pub fn blob_oid(self, bytes: &[u8]) -> Vec { + self.object_oid(b"blob", bytes) + } + + fn object_oid(self, kind: &[u8], body: &[u8]) -> Vec { + let mut object = Vec::with_capacity(body.len() + 32); + object.extend_from_slice(kind); + object.push(b' '); + object.extend_from_slice(body.len().to_string().as_bytes()); + object.push(0); + object.extend_from_slice(body); + self.digest(&object) + } + fn oid_len(self) -> usize { match self { Self::Sha1 => 20, @@ -473,6 +507,110 @@ fn parse_index( }) } +/// Why a coherent index snapshot cannot prove a full tree. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum TreeRefusal { + /// An unmerged, assume-valid, skip-worktree or intent-to-add entry. + Exception(Vec, EntryException), + /// A mode other than regular, executable, symlink or gitlink. + Mode(Vec, u32), + /// An empty, `.`, `..` or `.git` component. + Path(Vec), + /// The same path twice, or a path that is both a file and a directory. + Conflict(Vec), +} + +enum TreeNode<'a> { + Leaf { mode: u32, oid: &'a [u8] }, + Directory(std::collections::BTreeMap<&'a [u8], TreeNode<'a>>), +} + +/// Computes the root tree object ID that `git write-tree` would write for +/// this index, from every entry. Any entry that a plain tree cannot +/// represent exactly refuses the whole computation rather than being skipped, +/// so an equal result proves the full index, not a subset of it. +pub fn index_tree(snapshot: &IndexSnapshot) -> Result, TreeRefusal> { + let mut root = std::collections::BTreeMap::new(); + for entry in &snapshot.entries { + if let Some(exception) = entry.exceptions().into_iter().next() { + return Err(TreeRefusal::Exception(entry.path.clone(), exception)); + } + if !matches!(entry.stat.mode, 0o100644 | 0o100755 | 0o120000 | 0o160000) { + return Err(TreeRefusal::Mode(entry.path.clone(), entry.stat.mode)); + } + let components: Vec<&[u8]> = entry.path.split(|&b| b == b'/').collect(); + if components.iter().any(|component| { + component.is_empty() + || *component == b"." + || *component == b".." + || component.eq_ignore_ascii_case(b".git") + }) { + return Err(TreeRefusal::Path(entry.path.clone())); + } + let conflict = || TreeRefusal::Conflict(entry.path.clone()); + let (leaf, parents) = components.split_last().expect("split yields one item"); + let mut directory = &mut root; + for parent in parents { + let node = directory + .entry(*parent) + .or_insert_with(|| TreeNode::Directory(Default::default())); + directory = match node { + TreeNode::Directory(children) => children, + TreeNode::Leaf { .. } => return Err(conflict()), + }; + } + if directory + .insert( + leaf, + TreeNode::Leaf { + mode: entry.stat.mode, + oid: &entry.oid, + }, + ) + .is_some() + { + return Err(conflict()); + } + } + Ok(write_tree(snapshot.object_format, &root)) +} + +fn write_tree( + format: ObjectFormat, + children: &std::collections::BTreeMap<&[u8], TreeNode<'_>>, +) -> Vec { + // Git orders tree entries by name, comparing a directory as if its name + // ended in '/'. + let mut rows: Vec<(Vec, String, &[u8], Vec)> = children + .iter() + .map(|(name, node)| match node { + TreeNode::Leaf { mode, oid } => { + (name.to_vec(), format!("{mode:o}"), *name, oid.to_vec()) + } + TreeNode::Directory(grandchildren) => { + let mut key = name.to_vec(); + key.push(b'/'); + ( + key, + "40000".to_owned(), + *name, + write_tree(format, grandchildren), + ) + } + }) + .collect(); + rows.sort_by(|a, b| a.0.cmp(&b.0)); + let mut body = Vec::new(); + for (_, mode, name, oid) in rows { + body.extend_from_slice(mode.as_bytes()); + body.push(b' '); + body.extend_from_slice(name); + body.push(0); + body.extend_from_slice(&oid); + } + format.object_oid(b"tree", &body) +} + /// Git metadata queries for one snapshot evaluation. Every spawned Git child, including a /// failed or timed-out command, increments the counter. Output remains binary /// because the NUL-delimited protocols must preserve arbitrary Git path bytes. @@ -1219,6 +1357,160 @@ mod tests { .any(|e| e.path == name)); } + fn trimmed_hex(output: Output) -> String { + String::from_utf8(output.stdout).unwrap().trim().to_owned() + } + + #[test] + fn index_tree_and_blob_oids_match_git_for_both_object_formats() { + for format in [ObjectFormat::Sha1, ObjectFormat::Sha256] { + let dir = fixture(format); + // Names whose Git tree order differs from plain byte order once + // the directory `a` is compared as `a/`. + for path in ["order/a-b", "order/a/x", "order/a.b", "order/a0"] { + let path = dir.path().join(path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + let contents = path_bytes(&path); + fs::write(path, contents).unwrap(); + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let tool = dir.path().join("tool.sh"); + fs::write(&tool, b"#!/bin/sh\n").unwrap(); + fs::set_permissions(&tool, fs::Permissions::from_mode(0o755)).unwrap(); + std::os::unix::fs::symlink("plain", dir.path().join("link")).unwrap(); + } + git(dir.path(), &["add", "."], &[]); + let commit = trimmed_hex(git(dir.path(), &["rev-parse", "HEAD"], &[])); + git( + dir.path(), + &[ + "update-index", + "--add", + "--cacheinfo", + &format!("160000,{commit},module"), + ], + &[], + ); + let snapshot = snapshot(dir.path(), format); + assert_eq!( + hex(&index_tree(&snapshot).unwrap()), + trimmed_hex(git(dir.path(), &["write-tree"], &[])), + "{format:?}" + ); + let bytes = b"blob bytes\0with a NUL\n"; + assert_eq!( + hex(&format.blob_oid(bytes)), + trimmed_hex(git(dir.path(), &["hash-object", "--stdin"], bytes)) + ); + assert_eq!( + ObjectFormat::from_hex_len(hex(&format.blob_oid(b"")).len()), + Some(format) + ); + } + } + + fn path_bytes(path: &Path) -> Vec { + path.file_name() + .unwrap() + .to_string_lossy() + .as_bytes() + .to_vec() + } + + #[test] + fn index_tree_refuses_real_git_exception_entries() { + let dir = fixture(ObjectFormat::Sha1); + let tree = |dir: &TempDir| index_tree(&snapshot(dir.path(), ObjectFormat::Sha1)); + assert!(tree(&dir).is_ok()); + fs::write(dir.path().join("new"), b"new\n").unwrap(); + git(dir.path(), &["add", "-N", "new"], &[]); + assert_eq!( + tree(&dir), + Err(TreeRefusal::Exception( + b"new".to_vec(), + EntryException::IntentToAdd + )) + ); + git(dir.path(), &["update-index", "--force-remove", "new"], &[]); + git(dir.path(), &["update-index", "--skip-worktree", "skip"], &[]); + assert_eq!( + tree(&dir), + Err(TreeRefusal::Exception( + b"skip".to_vec(), + EntryException::SkipWorktree + )) + ); + git(dir.path(), &["update-index", "--no-skip-worktree", "skip"], &[]); + git(dir.path(), &["update-index", "--assume-unchanged", "plain"], &[]); + assert_eq!( + tree(&dir), + Err(TreeRefusal::Exception( + b"plain".to_vec(), + EntryException::AssumeValid + )) + ); + } + + #[test] + fn index_tree_refuses_modes_paths_and_conflicts_a_tree_cannot_hold() { + let entry = |path: &[u8], mode: u32| IndexEntry { + path: path.to_vec(), + oid: vec![7; 20], + stat: IndexStat { + ctime: IndexTime { + seconds: 0, + nanoseconds: 0, + }, + mtime: IndexTime { + seconds: 0, + nanoseconds: 0, + }, + dev: 0, + ino: 0, + mode, + uid: 0, + gid: 0, + size: 0, + }, + stage: 0, + assume_valid: false, + skip_worktree: false, + intent_to_add: false, + }; + let tree = |entries: Vec| { + index_tree(&IndexSnapshot { + version: 2, + object_format: ObjectFormat::Sha1, + entries, + checksum: Vec::new(), + modified: SystemTime::UNIX_EPOCH, + }) + }; + assert!(tree(vec![entry(b"a", 0o100644), entry(b"b/c", 0o120000)]).is_ok()); + assert_eq!( + tree(vec![entry(b"a", 0o100664)]), + Err(TreeRefusal::Mode(b"a".to_vec(), 0o100664)) + ); + assert_eq!( + tree(vec![entry(b"x/.GIT/y", 0o100644)]), + Err(TreeRefusal::Path(b"x/.GIT/y".to_vec())) + ); + assert_eq!( + tree(vec![entry(b"a", 0o100644), entry(b"a/b", 0o100644)]), + Err(TreeRefusal::Conflict(b"a/b".to_vec())) + ); + assert_eq!( + tree(vec![entry(b"a/b", 0o100644), entry(b"a", 0o100644)]), + Err(TreeRefusal::Conflict(b"a".to_vec())) + ); + assert_eq!( + tree(vec![entry(b"a", 0o100644), entry(b"a", 0o100755)]), + Err(TreeRefusal::Conflict(b"a".to_vec())) + ); + } + #[test] fn counted_read_only_git_queries_and_large_bidirectional_pump() { let dir = fixture(ObjectFormat::Sha1); From 46dfb79978c5202645cf71ec5c77566def3a2c84 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:09:21 +0100 Subject: [PATCH 3/9] mason: tree-keyed base identity, proof file and receipt fast path (wip, untested) Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 3362271a7a4527309ca2a8b8aa316364a46ca631) --- crates/aft/src/git_index_reader.rs | 18 +- crates/aft/src/views/base_refs.rs | 148 +++++++++- crates/aft/src/views/base_refs_tests.rs | 135 +++++++++ crates/aft/src/views/first_load.rs | 31 +- crates/aft/src/views/mod.rs | 1 + crates/aft/src/views/tree_base.rs | 348 ++++++++++++++++++++++ crates/aft/src/views/tree_base_tests.rs | 373 ++++++++++++++++++++++++ crates/aft/src/views/verification.rs | 320 ++++++++++++++++++++ 8 files changed, 1362 insertions(+), 12 deletions(-) create mode 100644 crates/aft/src/views/tree_base.rs create mode 100644 crates/aft/src/views/tree_base_tests.rs diff --git a/crates/aft/src/git_index_reader.rs b/crates/aft/src/git_index_reader.rs index 4d9615071..f7a17cbed 100644 --- a/crates/aft/src/git_index_reader.rs +++ b/crates/aft/src/git_index_reader.rs @@ -1434,7 +1434,11 @@ mod tests { )) ); git(dir.path(), &["update-index", "--force-remove", "new"], &[]); - git(dir.path(), &["update-index", "--skip-worktree", "skip"], &[]); + git( + dir.path(), + &["update-index", "--skip-worktree", "skip"], + &[], + ); assert_eq!( tree(&dir), Err(TreeRefusal::Exception( @@ -1442,8 +1446,16 @@ mod tests { EntryException::SkipWorktree )) ); - git(dir.path(), &["update-index", "--no-skip-worktree", "skip"], &[]); - git(dir.path(), &["update-index", "--assume-unchanged", "plain"], &[]); + git( + dir.path(), + &["update-index", "--no-skip-worktree", "skip"], + &[], + ); + git( + dir.path(), + &["update-index", "--assume-unchanged", "plain"], + &[], + ); assert_eq!( tree(&dir), Err(TreeRefusal::Exception( diff --git a/crates/aft/src/views/base_refs.rs b/crates/aft/src/views/base_refs.rs index eb700352f..244eaf3da 100644 --- a/crates/aft/src/views/base_refs.rs +++ b/crates/aft/src/views/base_refs.rs @@ -46,9 +46,17 @@ use super::ViewStore; pub const MEMBERSHIP_DIGEST_TAG: &[u8] = b"aft/views-v3/base-membership/1"; /// Domain tag of the effective-identity digest, version 1. pub const IDENTITY_DIGEST_TAG: &[u8] = b"aft/views-v3/base-identity/1"; -/// Membership-keyed base scopes. Slice 4v's tree-keyed scopes will use a -/// different prefix; a scope is never relabelled from one to the other. +/// Membership-keyed base scopes. Tree-keyed scopes use +/// [`TREE_BASE_SCOPE_PREFIX`]; a scope is never relabelled from one to the other. pub const BASE_SCOPE_PREFIX: &str = "base-m1-"; +/// Domain tag of a tree-keyed base's membership digest, version 1. It covers +/// each entry's `git_oid`, which the membership digest refuses. +pub const TREE_MEMBERSHIP_DIGEST_TAG: &[u8] = b"aft/views-v3/base-tree-membership/1"; +/// Domain tag of a tree-keyed base's effective-identity digest, version 1. +pub const TREE_IDENTITY_DIGEST_TAG: &[u8] = b"aft/views-v3/base-identity-tree/1"; +/// Tree-keyed base scopes. Readers that know only membership-keyed bases +/// refuse this prefix, so they never authenticate such a base. +pub const TREE_BASE_SCOPE_PREFIX: &str = "base-t1-"; /// Pin-then-verify attempts before a base is reported unavailable. pub const RESOLVE_ATTEMPTS: usize = 3; /// `base_refs.state` values. @@ -148,8 +156,9 @@ impl TaggedHasher { /// entry kind, mode, content hash, size and resolution-input flag, in path /// order. Readiness is excluded, so readiness successors keep the digest. /// -/// Slice 4 bases are membership-keyed: a manifest that claims a HEAD tree or -/// any `git_oid`, or carries composition sections, is not a valid base. +/// Membership-keyed bases carry no tree claim: a manifest that claims a HEAD +/// tree or any `git_oid`, or carries composition sections, is not a valid +/// membership-keyed base. Tree claims use [`tree_membership_digest`]. pub fn membership_digest(manifest: &ManifestV2) -> Result<[u8; 32]> { if manifest.base_header().is_some() { return Err(BaseRefError::Corrupt( @@ -229,6 +238,115 @@ pub fn base_scope_for(identity: &[u8; 32]) -> String { format!("{BASE_SCOPE_PREFIX}{}", to_hex(identity)) } +/// The two kinds of base. The kind follows from the manifest (a `head_tree` +/// makes it tree-keyed) and is visible in the scope prefix. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BaseKind { + Membership, + Tree, +} + +/// The kind of base `scope` names, or `None` when it names no base. +pub fn base_kind(scope: &str) -> Option { + if scope.starts_with(BASE_SCOPE_PREFIX) { + Some(BaseKind::Membership) + } else if scope.starts_with(TREE_BASE_SCOPE_PREFIX) { + Some(BaseKind::Tree) + } else { + None + } +} + +/// The object format of a lowercase-hex object ID, or `None` when `text` is +/// not one. Uppercase is refused so that each OID has a single spelling. +fn object_format_of(text: &str) -> Option { + text.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + .then(|| crate::git_index_reader::ObjectFormat::from_hex_len(text.len())) + .flatten() +} + +/// The membership digest of a tree-keyed base: like [`membership_digest`], +/// plus each entry's `git_oid`. Returns the object format shared by the +/// manifest's tree and every OID. These strings are checked for shape only; +/// whether the bytes really are that tree is proven at bind time, never +/// taken from the manifest. +pub fn tree_membership_digest( + manifest: &ManifestV2, +) -> Result<([u8; 32], crate::git_index_reader::ObjectFormat)> { + if manifest.base_header().is_some() { + return Err(BaseRefError::Corrupt( + "a base manifest names another base".into(), + )); + } + if !manifest.tombstones().is_empty() || !manifest.readiness_overrides().is_empty() { + return Err(BaseRefError::Corrupt( + "a base manifest carries worker composition sections".into(), + )); + } + let format = manifest + .header() + .head_tree + .as_deref() + .and_then(object_format_of) + .ok_or_else(|| { + BaseRefError::Corrupt("a tree-keyed base needs a hex HEAD tree object ID".into()) + })?; + let mut hasher = TaggedHasher::new(TREE_MEMBERSHIP_DIGEST_TAG); + hasher.u64(manifest.len() as u64); + for (path, entry) in manifest.entries() { + let EntryV2::Regular { + mode, + content, + size, + git_oid: Some(git_oid), + resolution_input, + .. + } = entry + else { + return Err(BaseRefError::Corrupt(format!( + "tree-keyed base entry {path:?} is not a regular file with a git_oid" + ))); + }; + if object_format_of(git_oid) != Some(format) { + return Err(BaseRefError::Corrupt(format!( + "tree-keyed base entry {path:?} has a malformed git_oid" + ))); + } + hasher.field(path.as_bytes()); + hasher.byte(1); + hasher.u64(u64::from(*mode)); + hasher.field(content.as_bytes()); + hasher.u64(*size); + hasher.byte(u8::from(*resolution_input)); + hasher.field(git_oid.as_bytes()); + } + Ok((hasher.finish(), format)) +} + +/// The effective identity of a tree-keyed base: the inputs of +/// [`identity_digest`] with the object format and HEAD tree in place of +/// `head_tree:none`, over the tree-membership digest. +pub fn tree_identity_digest( + family: &str, + header: &ManifestHeader, + format: crate::git_index_reader::ObjectFormat, + membership: &[u8; 32], +) -> [u8; 32] { + let mut hasher = TaggedHasher::new(TREE_IDENTITY_DIGEST_TAG); + hasher.field(family.as_bytes()); + hasher.field(header.producers.trigram.as_bytes()); + hasher.optional(header.producers.semantic.as_deref().map(str::as_bytes)); + hasher.field(header.producers.callgraph.as_bytes()); + hasher.optional(header.ignore_fingerprint.as_deref().map(str::as_bytes)); + hasher.field(std::env::consts::FAMILY.as_bytes()); + hasher.field(format.name().as_bytes()); + let tree = format!("head_tree:{}", header.head_tree.as_deref().unwrap_or("")); + hasher.field(tree.as_bytes()); + hasher.field(membership); + hasher.finish() +} + /// The authenticated identity of one base manifest, without any protection. /// Builders use it before a base is published; readers use /// [`AuthenticatedBase`]. @@ -240,7 +358,18 @@ pub struct BaseIdentity { } impl BaseIdentity { + /// A manifest with a `head_tree` gets a tree-keyed identity; any other + /// manifest keeps the membership-keyed identity, computed exactly as before. pub fn of(family: &str, manifest: &ManifestV2) -> Result { + if manifest.header().head_tree.is_some() { + let (membership, format) = tree_membership_digest(manifest)?; + let identity = tree_identity_digest(family, manifest.header(), format, &membership); + return Ok(Self { + scope: format!("{TREE_BASE_SCOPE_PREFIX}{}", to_hex(&identity)), + membership, + identity, + }); + } let membership = membership_digest(manifest)?; let identity = identity_digest(family, manifest.header(), &membership); Ok(Self { @@ -250,6 +379,11 @@ impl BaseIdentity { }) } + /// Which kind of base this identity names. + pub fn kind(&self) -> BaseKind { + base_kind(&self.scope).expect("identities only derive base scopes") + } + pub fn header(&self, bound_generation: &str) -> BaseHeader { BaseHeader { scope: self.scope.clone(), @@ -363,9 +497,9 @@ fn expected_dir(storage: &Path, family: &str, scope: &str, stored: &str) -> Resu /// [`RESOLVE_ATTEMPTS`] times when the pointer or row moves; a missing base is /// reported unavailable, never replaced by another generation. pub fn resolve_base(storage: &Path, family: &str, scope: &str) -> Result { - if !scope.starts_with(BASE_SCOPE_PREFIX) { + if base_kind(scope).is_none() { return Err(BaseRefError::Corrupt(format!( - "{scope} is not a membership-keyed base scope" + "{scope} is not a base scope" ))); } for _ in 0..RESOLVE_ATTEMPTS { @@ -540,7 +674,7 @@ pub(crate) fn register_view_in( scope: &str, root: &Path, ) -> Result { - if scope.starts_with(BASE_SCOPE_PREFIX) { + if base_kind(scope).is_some() { return Err(BaseRefError::Corrupt(format!( "{scope} is reserved for bases" ))); diff --git a/crates/aft/src/views/base_refs_tests.rs b/crates/aft/src/views/base_refs_tests.rs index 531cf9384..6111f91b0 100644 --- a/crates/aft/src/views/base_refs_tests.rs +++ b/crates/aft/src/views/base_refs_tests.rs @@ -227,6 +227,141 @@ fn a_membership_keyed_base_never_claims_tree_identity() { )); } +/// A tree claim over `files`: every entry carries its real Git blob OID and +/// the header names `tree` (a well-formed but unproven string). +pub(crate) fn tree_manifest(files: &[(&str, &[u8])], tree: &str) -> ManifestV2 { + let mut manifest = base_manifest(files); + manifest.header_mut().head_tree = Some(tree.into()); + for (name, bytes) in files { + if let Some(EntryV2::Regular { git_oid, .. }) = manifest.get_mut(&path(name)) { + *git_oid = Some(to_hex( + &crate::git_index_reader::ObjectFormat::Sha1.blob_oid(bytes), + )); + } + } + manifest +} + +#[test] +fn tree_claims_get_their_own_identity_kind_and_never_a_membership_scope() { + let files: &[(&str, &[u8])] = &[("a.rs", b"one"), ("b.rs", b"two")]; + let tree = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; + let manifest = tree_manifest(files, tree); + let identity = BaseIdentity::of(FAMILY, &manifest).unwrap(); + assert_eq!(identity.kind(), BaseKind::Tree); + assert!(identity.scope.starts_with(TREE_BASE_SCOPE_PREFIX)); + let membership = BaseIdentity::of(FAMILY, &base_manifest(files)).unwrap(); + assert_eq!(membership.kind(), BaseKind::Membership); + assert_ne!(identity.scope, membership.scope); + assert_ne!(identity.membership, membership.membership); + + // The tree string and every OID are part of the identity. + let other_tree = tree_manifest(files, &"1".repeat(40)); + assert_ne!(BaseIdentity::of(FAMILY, &other_tree).unwrap(), identity); + let mut other_oid = manifest.clone(); + if let Some(EntryV2::Regular { git_oid, .. }) = other_oid.get_mut(&path("a.rs")) { + *git_oid = Some("2".repeat(40)); + } + assert_ne!(BaseIdentity::of(FAMILY, &other_oid).unwrap(), identity); + // Readiness successors keep the identity. + let mut ready = manifest.clone(); + if let Some(EntryV2::Regular { planes, .. }) = ready.get_mut(&path("a.rs")) { + planes.semantic = Some(PlaneState::ready(&FamilyKey::new( + FamilyPlane::Semantic, + [9; 32], + ))); + } + assert_eq!(BaseIdentity::of(FAMILY, &ready).unwrap(), identity); + + // Malformed claims are refused, never reinterpreted. + for forged in ["caller-supplied", &tree.to_uppercase(), &tree[..39]] { + let mut bad = manifest.clone(); + bad.header_mut().head_tree = Some(forged.to_owned()); + assert!(matches!( + BaseIdentity::of(FAMILY, &bad), + Err(BaseRefError::Corrupt(_)) + )); + } + let mut missing_oid = manifest.clone(); + if let Some(EntryV2::Regular { git_oid, .. }) = missing_oid.get_mut(&path("b.rs")) { + *git_oid = None; + } + assert!(matches!( + BaseIdentity::of(FAMILY, &missing_oid), + Err(BaseRefError::Corrupt(_)) + )); + let mut sha256_oid = manifest.clone(); + if let Some(EntryV2::Regular { git_oid, .. }) = sha256_oid.get_mut(&path("b.rs")) { + *git_oid = Some("3".repeat(64)); + } + assert!(matches!( + BaseIdentity::of(FAMILY, &sha256_oid), + Err(BaseRefError::Corrupt(_)) + )); + // Dropping the tree while keeping the OIDs does not relabel the claim as + // membership-keyed. + let mut relabelled = manifest.clone(); + relabelled.header_mut().head_tree = None; + assert!(matches!( + BaseIdentity::of(FAMILY, &relabelled), + Err(BaseRefError::Corrupt(_)) + )); +} + +/// A reader that knows only membership-keyed bases (the rules this module had +/// before tree-keyed bases): it accepts a scope only with the membership +/// prefix, and only a manifest whose membership digest derives that scope. +fn membership_only_reader_accepts(manifest: &ManifestV2, scope: &str) -> bool { + scope.starts_with(BASE_SCOPE_PREFIX) + && membership_digest(manifest).is_ok_and(|membership| { + base_scope_for(&identity_digest(FAMILY, manifest.header(), &membership)) == scope + }) +} + +#[test] +fn an_older_membership_only_reader_refuses_a_published_tree_keyed_base() { + let _serial = v3::serial_for_test(); + let storage = tempfile::tempdir().unwrap(); + let files: &[(&str, &[u8])] = &[("a.rs", b"alpha")]; + put_rows(storage.path(), &[b"alpha"]); + let manifest = tree_manifest(files, "4b825dc642cb6eb9a060e54bf8d69288fbee4904"); + let identity = BaseIdentity::of(FAMILY, &manifest).unwrap(); + let BasePublication::Published { generation, .. } = + publish_base(storage.path(), FAMILY, &manifest, None).unwrap() + else { + panic!("first tree base publication lost") + }; + // This reader resolves it with the tree identity. + let resolved = resolve_base(storage.path(), FAMILY, &identity.scope).unwrap(); + assert_eq!(resolved.identity().kind(), BaseKind::Tree); + assert_eq!(resolved.generation(), generation); + // The durable manifest, as an older reader would load it, is refused + // under both of its rules: the scope prefix and the membership digest. + let stored = ViewStore::existing_dir(resolved.view_dir().to_path_buf()) + .unwrap() + .load_manifest_v2(&generation) + .unwrap(); + assert!(!membership_only_reader_accepts(&stored, &identity.scope)); + assert!(matches!( + membership_digest(&stored), + Err(BaseRefError::Corrupt(_)) + )); + // No membership-keyed scope can name the tree-keyed manifest either. + let membership = BaseIdentity::of(FAMILY, &base_manifest(files)).unwrap(); + assert!(!membership_only_reader_accepts(&stored, &membership.scope)); + // A worker cannot register under a tree-keyed base scope. + assert!(matches!( + with_base_barrier(storage.path(), FAMILY, |tx| register_view_in( + tx, + storage.path(), + FAMILY, + &identity.scope, + Path::new("/checkout") + )), + Err(BaseRefError::Corrupt(_)) + )); +} + #[test] fn resolution_authenticates_the_pinned_base_and_rejects_forged_headers() { let _serial = v3::serial_for_test(); diff --git a/crates/aft/src/views/first_load.rs b/crates/aft/src/views/first_load.rs index b33b572a4..6480f0e39 100644 --- a/crates/aft/src/views/first_load.rs +++ b/crates/aft/src/views/first_load.rs @@ -1674,6 +1674,8 @@ pub(crate) struct ReconcileWork { pub reads: usize, pub content_hash_bytes: usize, pub attachments: usize, + /// Paths whose read was skipped on a tree-keyed base's blob receipt. + pub tree_receipts: usize, } /// Why this driver cannot own and complete a fresh base. @@ -1937,11 +1939,24 @@ impl CheckoutDriver { } let expected = super::base_refs::BaseIdentity::of(&route.family, &manifest) .map_err(|e| SiblingLoader::error(e.to_string()))?; + // A Git-proven tree-keyed base is preferred; without that proof the + // membership-keyed base below is resolved or built exactly as before. + let paths: Vec<_> = manifest.entries().map(|(path, _)| path.clone()).collect(); + let tree_base = super::tree_base::bootstrap( + super::tree_base::TreeBaseInput { + route: &route, + root: self.owner.root(), + paths: &paths, + policy, + producers: &self.producers, + ignore_fingerprint: ignore_fingerprint.clone(), + }, + &self.base_claim, + ); let resolved = - match super::base_refs::resolve_base(&route.storage, &route.family, &expected.scope) { + match super::tree_base::resolve_preferring(tree_base, &route, &expected.scope) { Ok(base) => base, Err(_) => { - let paths: Vec<_> = manifest.entries().map(|(path, _)| path.clone()).collect(); match super::base_builder::build_base( super::base_builder::BaseBuildInput { storage: &route.storage, @@ -2369,6 +2384,14 @@ impl FirstLoadDriver for CheckoutDriver { ) }) .unwrap_or_default(); + // A proven tree-keyed base adds its blob receipts as candidates; + // membership-keyed bases add none. + let (persisted, _receipt_paths) = super::tree_base::with_receipts( + persisted, + durable.as_deref(), + guard.as_deref(), + &route, + ); let subscription = crate::watcher_backend::subscription::current(self.owner.root()); let mut changed = pending.clone(); changed.extend( @@ -2552,6 +2575,10 @@ impl FirstLoadDriver for CheckoutDriver { } let mut current = current.clone(); current.recorded_ns = record.observed_ns; + #[cfg(test)] + if _receipt_paths.contains(&path) { + self.work.lock().unwrap().tree_receipts += 1; + } stats.insert(path.clone(), current); entries.insert(path.clone(), entry); evidence.insert(path, record.clone()); diff --git a/crates/aft/src/views/mod.rs b/crates/aft/src/views/mod.rs index 67e7433e8..a9f727880 100644 --- a/crates/aft/src/views/mod.rs +++ b/crates/aft/src/views/mod.rs @@ -63,6 +63,7 @@ pub mod query_wait; pub mod semantic; pub mod semantic_arena; pub mod semantic_runtime; +mod tree_base; pub mod trigram; mod verification; diff --git a/crates/aft/src/views/tree_base.rs b/crates/aft/src/views/tree_base.rs new file mode 100644 index 000000000..bb4a917f9 --- /dev/null +++ b/crates/aft/src/views/tree_base.rs @@ -0,0 +1,348 @@ +//! Tree-keyed bases and their blob receipts +//! (docs/design/views-v3-format.md, "Tree-keyed bases and blob receipts"). +//! +//! A fresh checkout gets a tree-keyed base only when Git itself proves the +//! claim: the coherent index computes HEAD's tree, and every member's bytes, +//! read once under a stable native stat, hash to that member's index blob. +//! Each such read leaves a receipt in the base's proof file. A later bind of +//! the same checkout can then skip rereading a member whose receipt still +//! holds. Anything short of that proof keeps today's membership-keyed base and +//! its full check. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; +use std::sync::Mutex; + +use crate::blob_store::v2::{to_hex, FamilyPlane, TrigramPolicy}; +use crate::blob_store::v3; +use crate::git_index_reader::{index_tree, read_index, IndexRead, ObjectFormat}; + +use super::base_builder::{claim_base, initial_reservation, BuildClaim, ClaimOutcome}; +use super::base_refs::{ + resolve_base, with_base_barrier, AuthenticatedBase, BaseIdentity, BaseKind, +}; +use super::manifest_v2::{EntryPlanes, EntryV2, ManifestHeader, ManifestV2, Producers}; +use super::readiness::PlaneState; +use super::snapshot::{ComposedGeneration, V3Route}; +use super::verification::{BindGuard, BlobReceipt, Evidence, NativeStat, TreeProof}; +use super::RelPath; + +/// What a fresh checkout needs to build a tree-keyed base. +pub(crate) struct TreeBaseInput<'a> { + pub route: &'a V3Route, + pub root: &'a Path, + /// Walker-admitted paths of the checkout; only tracked regular files + /// among them become members. + pub paths: &'a [RelPath], + pub policy: TrigramPolicy, + pub producers: &'a Producers, + pub ignore_fingerprint: Option, +} + +/// A base manifest whose every member was read and matched to the index, +/// with the bytes read and their receipts. +struct ProvenMembers { + manifest: ManifestV2, + bytes: BTreeMap>, + receipts: BTreeMap, + format: ObjectFormat, +} + +/// Reads and proves the members, or `None` when any check fails. +fn prove(input: &TreeBaseInput<'_>, guard: &BindGuard) -> Option { + if !guard.policy.reusable { + return None; + } + let head_tree = std::str::from_utf8(guard.head_tree.as_deref()?.trim_ascii()).ok()?; + let format = ObjectFormat::from_hex_len(head_tree.len())?; + let index_path = guard.gitdir.as_ref()?.0.join("index"); + let IndexRead::Snapshot(snapshot) = read_index(&index_path, format, Default::default()) else { + return None; + }; + if NativeStat::strict_read(&index_path).ok()? != guard.index.as_ref()?.0 + || to_hex(&index_tree(&snapshot).ok()?) != head_tree + { + return None; + } + let index: BTreeMap<&[u8], _> = snapshot + .entries + .iter() + .map(|entry| (entry.path.as_slice(), entry)) + .collect(); + let header = ManifestHeader { + producers: input.producers.clone(), + head_tree: Some(head_tree.to_owned()), + ignore_fingerprint: input.ignore_fingerprint.clone(), + segment: None, + }; + let mut manifest = ManifestV2::new_v3(header, None, Default::default()); + let mut bytes = BTreeMap::new(); + let mut receipts = BTreeMap::new(); + for path in input.paths { + // Untracked files and tracked symlinks or gitlinks stay out of the + // base; the worker's own delta carries whatever the walker admits. + let Some(entry) = index.get(path.as_bytes()) else { + continue; + }; + if !matches!(entry.stat.mode, 0o100644 | 0o100755) { + continue; + } + let absolute = input + .root + .join(super::segment_store::rel_path_to_os(path).ok()?); + let observed_ns = super::verification::now_ns(); + let before = NativeStat::read(&absolute).ok()?; + if !std::fs::symlink_metadata(&absolute).ok()?.is_file() { + return None; + } + let source = std::fs::read(&absolute).ok()?; + let after = NativeStat::read(&absolute).ok()?; + if before != after || before.size != source.len() as u64 { + return None; + } + let oid = format.blob_oid(&source); + if oid != entry.oid { + return None; + } + #[cfg(unix)] + if (before.mode & 0o111 != 0) != (entry.stat.mode == 0o100755) { + return None; + } + let content = crate::blob_store::v2::ContentHash::of(&source); + let planes = EntryPlanes { + trigram: Some(PlaneState::pending("lexical plane not built")), + semantic: input + .producers + .semantic + .as_ref() + .filter(|_| super::semantic::applies_to(path)) + .map(|_| PlaneState::pending("awaiting the base owner")), + callgraph: Some(PlaneState::pending("awaiting the base owner")), + }; + manifest + .insert( + path.clone(), + EntryV2::Regular { + mode: entry.stat.mode, + content, + size: source.len() as u64, + git_oid: Some(to_hex(&oid)), + resolution_input: false, + planes, + }, + ) + .ok()?; + receipts.insert( + path.clone(), + BlobReceipt { + version: 1, + mode: entry.stat.mode, + size: source.len() as u64, + content: content.to_hex(), + git_oid: to_hex(&oid), + stat: before, + observed_ns, + policy: guard.policy.digest.clone(), + delivery_seq: 0, + }, + ); + bytes.insert(path.clone(), source); + } + (!manifest.is_empty()).then_some(ProvenMembers { + manifest, + bytes, + receipts, + format, + }) +} + +/// Resolves or builds the tree-keyed base of a fresh checkout. `None` means no +/// tree-keyed base is available now (no proof, a busy or capped claim, a +/// refused admission or a failed build); the caller then takes its +/// membership-keyed path unchanged. A claim this call keeps is left in +/// `claim_slot` for the caller to retire, as for membership-keyed builds. +pub(crate) fn bootstrap( + input: TreeBaseInput<'_>, + claim_slot: &Mutex>, +) -> Option { + let guard = BindGuard::capture(input.root, None).ok()?; + let mut proven = prove(&input, &guard)?; + let route = input.route; + let identity = BaseIdentity::of(&route.family, &proven.manifest).ok()?; + if identity.kind() != BaseKind::Tree { + return None; + } + if let Ok(base) = resolve_base(&route.storage, &route.family, &identity.scope) { + return Some(base); + } + let ClaimOutcome::Claimed(claim) = + claim_base(&route.storage, &route.family, &identity.scope, None).ok()? + else { + return None; + }; + match build(route, &identity, &guard, &mut proven, input.policy, &claim) { + Ok(()) => { + let base = resolve_base(&route.storage, &route.family, &identity.scope).ok(); + *claim_slot + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(claim); + base + } + Err(_) => { + let _ = claim.release(); + None + } + } +} + +/// The base a fresh checkout binds: `tree` when [`bootstrap`] produced one, +/// otherwise the membership-keyed base `scope`, resolved as before. +pub(crate) fn resolve_preferring( + tree: Option, + route: &V3Route, + scope: &str, +) -> super::base_refs::Result { + match tree { + Some(base) => Ok(base), + None => resolve_base(&route.storage, &route.family, scope), + } +} + +/// The claimed build: admission, member registration, the proof file, the +/// lexical plane and the first publication, in that order. +fn build( + route: &V3Route, + identity: &BaseIdentity, + guard: &BindGuard, + proven: &mut ProvenMembers, + policy: TrigramPolicy, + claim: &BuildClaim, +) -> Result<(), String> { + let storage = route.storage.as_path(); + let family = route.family.as_str(); + let error = |e: &dyn std::fmt::Display| e.to_string(); + let source_bytes = proven.bytes.values().map(|b| b.len() as u64).sum(); + let request = initial_reservation(proven.bytes.len() as u64, source_bytes, None); + let reservation = { + let lock = v3::ReservationLock::acquire(storage).map_err(|e| error(&e))?; + let mut registry = + v3::open_base_refs_registry(storage, family, &lock, None).map_err(|e| error(&e))?; + super::eviction::DiskBudget::default() + .admit_v3(&mut registry, request) + .map_err(|e| error(&e))? + .reservation + }; + let built = (|| -> Result<(), String> { + let (dir, _) = with_base_barrier(storage, family, |tx| { + super::base_builder::register_base_in(tx, storage, family, &identity.scope) + }) + .map_err(|e| error(&e))?; + super::verification::write_tree_proof( + storage, + family, + &dir, + &TreeProof { + base_scope: identity.scope.clone(), + identity_digest: to_hex(&identity.identity), + membership_digest: to_hex(&identity.membership), + object_format: proven.format.name().to_owned(), + head_tree: proven + .manifest + .header() + .head_tree + .clone() + .unwrap_or_default(), + root: guard.root.clone(), + policy: guard.policy.digest.clone(), + receipts: proven.receipts.clone(), + }, + ) + .map_err(|e| error(&e))?; + let mut live = crate::pins::LivePin::create_v3(storage, family, &identity.scope) + .map_err(|e| error(&e))?; + super::trigram::materialize_v3( + storage, + family, + &mut proven.manifest, + &proven.bytes, + &mut live, + policy, + ) + .map_err(|e| e.reason)?; + if !claim.verify_held().map_err(|e| error(&e))? { + return Err("the build claim was taken over".into()); + } + match super::base_builder::publish_base(storage, family, &proven.manifest, None) + .map_err(|e| error(&e))? + { + super::base_builder::BasePublication::Published { .. } => Ok(()), + super::base_builder::BasePublication::Lost { .. } => { + Err("another publisher moved the base pointer".into()) + } + } + })(); + let released = (|| -> Result<(), String> { + let lock = v3::ReservationLock::acquire(storage).map_err(|e| error(&e))?; + let mut registry = + v3::RegistryWriter::open_for_barrier(storage, family, &lock).map_err(|e| error(&e))?; + registry.release(reservation).map_err(|e| error(&e)) + })(); + built?; + released?; + claim.mark_serving().map_err(|e| error(&e)) +} + +/// Adds the receipts of `durable`'s tree-keyed base to the checkout's own +/// generation evidence, as candidates for the same stat-reuse checks. A +/// receipt replaces an evidence record only when it records the same stat +/// (the same observation) or the path has no record. Returns the merged +/// candidates and the paths whose candidate is a receipt. A membership-keyed +/// base, or a tree-keyed base whose proof does not hold now, adds nothing. +pub(crate) fn with_receipts( + mut evidence: BTreeMap, + durable: Option<&ComposedGeneration>, + guard: Option<&BindGuard>, + route: &V3Route, +) -> (BTreeMap, BTreeSet) { + let mut from_receipts = BTreeSet::new(); + let (Some(durable), Some(guard)) = (durable, guard) else { + return (evidence, from_receipts); + }; + let Some(header) = durable.base_header() else { + return (evidence, from_receipts); + }; + if super::base_refs::base_kind(&header.scope) != Some(BaseKind::Tree) { + return (evidence, from_receipts); + } + let Some(base) = resolve_base(&route.storage, &route.family, &header.scope) + .ok() + .filter(|base| base.check_header(header).is_ok()) + else { + return (evidence, from_receipts); + }; + let Some(receipts) = super::verification::tree_receipt_evidence(&base, guard) else { + return (evidence, from_receipts); + }; + for (path, mut record) in receipts { + if evidence + .get(&path) + .is_some_and(|existing| existing.stat != record.stat) + { + continue; + } + // Plane keys come from the checkout's own readiness; the reuse check + // still requires each to be ready, expected and present. + for plane in FamilyPlane::ALL { + if let Some(PlaneState::Ready { key }) = durable.plane_readiness(&path, plane) { + record.keys.insert(format!("{plane:?}"), key.clone()); + } + } + evidence.insert(path.clone(), record); + from_receipts.insert(path); + } + (evidence, from_receipts) +} + +#[cfg(test)] +#[path = "tree_base_tests.rs"] +mod tests; diff --git a/crates/aft/src/views/tree_base_tests.rs b/crates/aft/src/views/tree_base_tests.rs new file mode 100644 index 000000000..c11f345ab --- /dev/null +++ b/crates/aft/src/views/tree_base_tests.rs @@ -0,0 +1,373 @@ +//! Tree-keyed bases through the checkout driver: which binds may skip a +//! member's read on its blob receipt, and which must take the full check. + +use std::path::Path; +use std::process::Command; +use std::sync::Arc; + +use crate::blob_store::v2::{FamilyPlane, TrigramPolicy}; +use crate::views::base_refs::{base_kind, open_foreign_v3, BaseKind}; +use crate::views::first_load::{ + CheckoutDriver, ConfiguredMembershipWalker, FirstLoadDriver, ReconcileWork, TrigramBridge, +}; +use crate::views::manifest_v2::Producers; +use crate::views::snapshot::{DiskState, Snapshot}; +use crate::views::verification::{ + evidence_path, known_attribute_sources_for_test, read_tree_proof, write_tree_proof, + BlobReceipt, FileIdentity, NativeStat, TreeProof, TREE_PROOF_APP, TREE_PROOF_FILE, +}; +use crate::views::RelPath; + +const FAMILY: &str = "family"; +const WORKER: &str = "worker"; +const FILES: [(&str, &[u8]); 2] = [ + ("file.rs", b"fn tree_member() { let marker = 41; }\n"), + ("lib.rs", b"pub fn other_member() -> u8 { 7 }\n"), +]; + +fn git(root: &Path, args: &[&str]) { + let mut command = Command::new("git"); + command.current_dir(root).args(args); + crate::test_env::apply_hermetic_git_env(&mut command); + let output = command.output().unwrap(); + assert!( + output.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); +} + +/// Writes `bytes` with an mtime well before any observation second, so a +/// stable read of it is reusable evidence. +fn write_old(path: &Path, bytes: &[u8]) { + std::fs::write(path, bytes).unwrap(); + std::fs::OpenOptions::new() + .write(true) + .open(path) + .unwrap() + .set_times( + std::fs::FileTimes::new() + .set_modified(std::time::UNIX_EPOCH + std::time::Duration::from_secs(60)), + ) + .unwrap(); +} + +/// A checkout of `FILES`; committed (clean) when `commit` is set, otherwise +/// a plain directory without Git. +fn checkout(commit: bool) -> tempfile::TempDir { + let root = tempfile::tempdir().unwrap(); + for (name, bytes) in FILES { + write_old(&root.path().join(name), bytes); + } + if commit { + git(root.path(), &["init", "-q", "--object-format=sha1"]); + git( + root.path(), + &["config", "user.email", "tree@example.invalid"], + ); + git(root.path(), &["config", "user.name", "Tree Base"]); + git(root.path(), &["config", "core.autocrlf", "false"]); + git(root.path(), &["add", "."]); + git( + root.path(), + &[ + "-c", + "commit.gpgsign=false", + "commit", + "-q", + "-m", + "fixture", + ], + ); + } + root +} + +fn policy() -> TrigramPolicy { + TrigramPolicy { + max_file_size: 1 << 20, + } +} + +fn producers() -> Producers { + Producers { + trigram: policy().fingerprint_hex(), + semantic: None, + callgraph: "unregistered".into(), + } +} + +fn driver(root: &Path, storage: &Path) -> CheckoutDriver { + let plane = Arc::new(TrigramBridge::new(storage.to_owned(), policy())); + CheckoutDriver::new_v3( + storage, + FAMILY, + WORKER, + root, + producers(), + Arc::new(ConfiguredMembershipWalker), + vec![plane.clone()], + ) + .unwrap() + .with_adapters(vec![plane.adapter.clone()]) +} + +/// The first bind of a fresh checkout: bootstraps a base and publishes the +/// worker over it. Returns the base scope. +fn bootstrap(root: &Path, storage: &Path) -> String { + let first = driver(root, storage); + let access = first.access(); + let empty = first.installed_snapshot().generation().clone(); + let (snapshot, revision) = first.verified_bind(&access, empty).unwrap(); + first.install(&access, &snapshot, revision).unwrap(); + snapshot.generation().base_header().unwrap().scope.clone() +} + +/// A later bind of the same checkout by a fresh driver (as after a restart). +fn rebind(root: &Path, storage: &Path) -> (Snapshot, ReconcileWork) { + let next = driver(root, storage); + let published = open_foreign_v3(storage, FAMILY, WORKER, &producers()) + .unwrap() + .unwrap(); + let (snapshot, _) = next.verified_bind(&next.access(), published).unwrap(); + (snapshot, next.take_reconcile_work()) +} + +fn member_dir(storage: &Path, scope: &str) -> std::path::PathBuf { + crate::blob_store::v3::view_dir(storage, FAMILY, scope).unwrap() +} + +/// Removes the worker generation's own stat evidence, so only a base's +/// receipts could avoid a read. +fn remove_worker_evidence(storage: &Path) { + let dir = member_dir(storage, WORKER); + let generation = crate::views::ViewStore::existing_dir(dir.clone()) + .unwrap() + .current_generation_read_only() + .unwrap() + .unwrap(); + std::fs::remove_file(evidence_path(&dir, &generation)).unwrap(); +} + +fn rel(name: &str) -> RelPath { + RelPath::new(name.as_bytes().to_vec()).unwrap() +} + +fn assert_current_bytes(snapshot: &Snapshot, root: &Path) { + for (name, _) in FILES { + assert_eq!( + snapshot.disk_state(&rel(name)), + DiskState::of_bytes(&std::fs::read(root.join(name)).unwrap()), + "{name}" + ); + assert!(matches!( + snapshot.plane_readiness(&rel(name), FamilyPlane::Trigram), + Some(crate::views::readiness::PlaneState::Ready { .. }) + )); + } +} + +#[test] +fn a_proven_tree_base_takes_the_receipt_fast_path() { + let _serial = crate::blob_store::v3::serial_for_test(); + let root = checkout(true); + let storage = tempfile::tempdir().unwrap(); + let _attributes = known_attribute_sources_for_test(root.path()); + let scope = bootstrap(root.path(), storage.path()); + assert_eq!(base_kind(&scope), Some(BaseKind::Tree)); + let proof = read_tree_proof(&member_dir(storage.path(), &scope)).unwrap(); + assert_eq!(proof.receipts.len(), FILES.len()); + for (name, bytes) in FILES { + let receipt = &proof.receipts[&rel(name)]; + assert_eq!( + receipt.git_oid, + crate::blob_store::v2::to_hex( + &crate::git_index_reader::ObjectFormat::Sha1.blob_oid(bytes) + ) + ); + } + + let (snapshot, work) = rebind(root.path(), storage.path()); + assert_eq!(work.tree_receipts, FILES.len(), "{work:?}"); + assert_eq!(work.reads, 0, "{work:?}"); + assert_current_bytes(&snapshot, root.path()); + + // The receipts alone suffice: the worker's own evidence is not needed. + remove_worker_evidence(storage.path()); + let (snapshot, work) = rebind(root.path(), storage.path()); + assert_eq!(work.tree_receipts, FILES.len(), "{work:?}"); + assert_eq!(work.reads, 0, "{work:?}"); + assert_current_bytes(&snapshot, root.path()); +} + +#[test] +fn a_membership_base_never_takes_the_receipt_fast_path() { + let _serial = crate::blob_store::v3::serial_for_test(); + for commit in [false, true] { + let root = checkout(commit); + let storage = tempfile::tempdir().unwrap(); + let _attributes = known_attribute_sources_for_test(root.path()); + if commit { + // A dirty tracked file: the index no longer proves the bytes. + write_old(&root.path().join("file.rs"), b"fn edited() {}\n"); + } + let scope = bootstrap(root.path(), storage.path()); + assert_eq!(base_kind(&scope), Some(BaseKind::Membership), "{commit}"); + assert!(!member_dir(storage.path(), &scope) + .join(TREE_PROOF_FILE) + .exists()); + + let (_, work) = rebind(root.path(), storage.path()); + assert_eq!(work.tree_receipts, 0, "{work:?}"); + remove_worker_evidence(storage.path()); + let (snapshot, work) = rebind(root.path(), storage.path()); + assert_eq!(work.tree_receipts, 0, "{work:?}"); + assert_eq!(work.reads, FILES.len(), "full check: {work:?}"); + assert_current_bytes(&snapshot, root.path()); + } +} + +#[test] +fn a_tree_base_without_a_holding_proof_falls_back_to_the_full_check() { + let _serial = crate::blob_store::v3::serial_for_test(); + let root = checkout(true); + let storage = tempfile::tempdir().unwrap(); + let _attributes = known_attribute_sources_for_test(root.path()); + let scope = bootstrap(root.path(), storage.path()); + assert_eq!(base_kind(&scope), Some(BaseKind::Tree)); + remove_worker_evidence(storage.path()); + let proof_path = member_dir(storage.path(), &scope).join(TREE_PROOF_FILE); + let original = std::fs::read(&proof_path).unwrap(); + let full_check = |label: &str| { + let (snapshot, work) = rebind(root.path(), storage.path()); + assert_eq!(work.tree_receipts, 0, "{label}: {work:?}"); + assert_eq!(work.reads, FILES.len(), "{label}: {work:?}"); + assert_current_bytes(&snapshot, root.path()); + }; + + // No proof file at all. + std::fs::remove_file(&proof_path).unwrap(); + full_check("missing proof"); + + // A receipt edited after the proof was written. + std::fs::write(&proof_path, &original).unwrap(); + { + let db = rusqlite::Connection::open(&proof_path).unwrap(); + let record: Vec = db + .query_row( + "SELECT record FROM blob_receipts WHERE path=?1", + [b"lib.rs".as_slice()], + |r| r.get(0), + ) + .unwrap(); + let mut receipt: BlobReceipt = serde_json::from_slice(&record).unwrap(); + receipt.observed_ns += 1; + db.execute( + "UPDATE blob_receipts SET record=?1 WHERE path=?2", + rusqlite::params![serde_json::to_vec(&receipt).unwrap(), b"lib.rs".as_slice()], + ) + .unwrap(); + } + full_check("receipt digest mismatch"); + + // A proof from a reader of another format. + std::fs::write(&proof_path, &original).unwrap(); + rusqlite::Connection::open(&proof_path) + .unwrap() + .pragma_update(None, "user_version", 2) + .unwrap(); + full_check("newer proof version"); + + // An intact proof, but Git no longer proves the full index. + std::fs::write(&proof_path, &original).unwrap(); + git( + root.path(), + &["update-index", "--assume-unchanged", "lib.rs"], + ); + full_check("assume-unchanged entry"); + git( + root.path(), + &["update-index", "--no-assume-unchanged", "lib.rs"], + ); + + // Restoring everything restores the fast path. + let (_, work) = rebind(root.path(), storage.path()); + assert_eq!(work.tree_receipts, FILES.len(), "{work:?}"); + assert_eq!(work.reads, 0, "{work:?}"); +} + +#[test] +fn a_file_whose_bytes_no_longer_match_its_receipt_takes_the_full_check() { + let _serial = crate::blob_store::v3::serial_for_test(); + let root = checkout(true); + let storage = tempfile::tempdir().unwrap(); + let _attributes = known_attribute_sources_for_test(root.path()); + let scope = bootstrap(root.path(), storage.path()); + assert_eq!(base_kind(&scope), Some(BaseKind::Tree)); + remove_worker_evidence(storage.path()); + // Same length and the same mtime as the receipt recorded: only the change + // time and the bytes differ, and the index still names the old blob. + let edited = FILES[0].1.to_vec(); + let edited: Vec = edited + .into_iter() + .map(|b| if b == b'1' { b'2' } else { b }) + .collect(); + assert_ne!(edited, FILES[0].1); + write_old(&root.path().join(FILES[0].0), &edited); + + let (snapshot, work) = rebind(root.path(), storage.path()); + assert_eq!(work.reads, 1, "{work:?}"); + assert_eq!(work.tree_receipts, 1, "{work:?}"); + assert_eq!( + snapshot.disk_state(&rel(FILES[0].0)), + DiskState::of_bytes(&edited) + ); + assert_current_bytes(&snapshot, root.path()); +} + +#[test] +fn a_tree_proof_round_trips_and_refuses_another_application() { + let _serial = crate::blob_store::v3::serial_for_test(); + let storage = tempfile::tempdir().unwrap(); + let dir = storage.path().join("member"); + std::fs::create_dir_all(&dir).unwrap(); + let file = dir.join("source"); + std::fs::write(&file, b"bytes").unwrap(); + let receipt = BlobReceipt { + version: 1, + mode: 0o100644, + size: 5, + content: crate::blob_store::v2::ContentHash::of(b"bytes").to_hex(), + git_oid: "1".repeat(40), + stat: NativeStat::read(&file).unwrap(), + observed_ns: 7, + policy: "policy".into(), + delivery_seq: 0, + }; + let proof = TreeProof { + base_scope: "base-t1-00".into(), + identity_digest: "aa".into(), + membership_digest: "bb".into(), + object_format: "sha1".into(), + head_tree: "2".repeat(40), + root: FileIdentity::StrictOnly { + volume: 1, + index: 2, + birth: None, + }, + policy: "policy".into(), + receipts: [(rel("a.rs"), receipt)].into_iter().collect(), + }; + write_tree_proof(storage.path(), FAMILY, &dir, &proof).unwrap(); + assert_eq!(read_tree_proof(&dir), Some(proof)); + assert!(std::fs::read_dir(&dir).unwrap().all(|entry| !entry + .unwrap() + .file_name() + .to_string_lossy() + .contains(".tmp."))); + let db = rusqlite::Connection::open(dir.join(TREE_PROOF_FILE)).unwrap(); + db.pragma_update(None, "application_id", TREE_PROOF_APP + 1) + .unwrap(); + drop(db); + assert_eq!(read_tree_proof(&dir), None); +} diff --git a/crates/aft/src/views/verification.rs b/crates/aft/src/views/verification.rs index 0f53132c4..f347740fb 100644 --- a/crates/aft/src/views/verification.rs +++ b/crates/aft/src/views/verification.rs @@ -749,6 +749,326 @@ pub(crate) fn read_evidence( Some(result) } +/// SQLite application ID of a tree-keyed base's proof file ("AFTT"). +pub(crate) const TREE_PROOF_APP: i64 = 0x41465454; +/// The proof file's name inside a tree-keyed base's member directory. +pub(crate) const TREE_PROOF_FILE: &str = "tree-proof.sqlite"; +const TREE_PROOF_SCHEMA: &str = "CREATE TABLE proof_meta(singleton INTEGER PRIMARY KEY CHECK(singleton=1), format_version INTEGER NOT NULL, schema_version INTEGER NOT NULL, base_scope TEXT NOT NULL, identity_digest TEXT NOT NULL, membership_digest TEXT NOT NULL, object_format TEXT NOT NULL, head_tree TEXT NOT NULL, root BLOB NOT NULL, policy TEXT NOT NULL, receipts_digest TEXT NOT NULL); +CREATE TABLE blob_receipts(path BLOB PRIMARY KEY, record BLOB NOT NULL);"; +const TREE_RECEIPTS_TAG: &[u8] = b"aft/views-v3/tree-receipts/1"; + +/// What one stable read of a tree-keyed base member observed: the bytes' +/// content digest and Git blob OID, the native stat taken around the read, +/// and the conversion policy in force at that read. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub(crate) struct BlobReceipt { + pub version: u32, + pub mode: u32, + pub size: u64, + pub content: String, + pub git_oid: String, + pub stat: NativeStat, + pub observed_ns: u128, + pub policy: String, + pub delivery_seq: u64, +} + +/// The durable proof of one tree-keyed base. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct TreeProof { + pub base_scope: String, + pub identity_digest: String, + pub membership_digest: String, + pub object_format: String, + pub head_tree: String, + pub root: FileIdentity, + pub policy: String, + pub receipts: BTreeMap, +} + +fn receipts_digest<'a>(records: impl ExactSizeIterator) -> String { + let mut hash = blake3::Hasher::new(); + let field = |hash: &mut blake3::Hasher, bytes: &[u8]| { + hash.update(&(bytes.len() as u64).to_le_bytes()); + hash.update(bytes); + }; + field(&mut hash, TREE_RECEIPTS_TAG); + hash.update(&(records.len() as u64).to_le_bytes()); + for (path, record) in records { + field(&mut hash, path); + field(&mut hash, record); + } + hash.finalize().to_hex().to_string() +} + +/// Writes `proof` into `dir` (the base's member directory). Only the build +/// claim owner calls this, before the base's first publication; afterwards +/// the file is never rewritten. The registry floor is raised to the +/// verification version first, so readers without that support refuse. +pub(crate) fn write_tree_proof( + storage: &Path, + family: &str, + dir: &Path, + proof: &TreeProof, +) -> io::Result<()> { + let lock = + crate::blob_store::v3::ReservationLock::acquire(storage).map_err(io::Error::other)?; + drop( + crate::blob_store::v3::open_verification_registry(storage, family, &lock) + .map_err(io::Error::other)?, + ); + drop(lock); + let mut encoded = BTreeMap::new(); + for (path, receipt) in &proof.receipts { + encoded.insert(path.as_bytes().to_vec(), serde_json::to_vec(receipt)?); + } + let digest = receipts_digest( + encoded + .iter() + .map(|(path, record)| (path.as_slice(), record.as_slice())), + ); + let root = serde_json::to_vec(&proof.root)?; + let destination = dir.join(TREE_PROOF_FILE); + let staging = dir.join(format!( + ".{TREE_PROOF_FILE}.tmp.{}", + super::base_refs::fresh_nonce() + )); + let result = (|| -> Result<(), Box> { + let mut db = rusqlite::Connection::open(&staging)?; + db.pragma_update(None, "application_id", TREE_PROOF_APP)?; + db.pragma_update(None, "user_version", 1)?; + db.pragma_update(None, "journal_mode", "DELETE")?; + db.pragma_update(None, "synchronous", "FULL")?; + db.execute_batch(TREE_PROOF_SCHEMA)?; + let tx = db.transaction()?; + tx.execute( + "INSERT INTO proof_meta VALUES(1,1,1,?1,?2,?3,?4,?5,?6,?7,?8)", + rusqlite::params![ + proof.base_scope, + proof.identity_digest, + proof.membership_digest, + proof.object_format, + proof.head_tree, + root, + proof.policy, + digest + ], + )?; + for (path, record) in &encoded { + tx.execute( + "INSERT INTO blob_receipts VALUES(?1,?2)", + rusqlite::params![path, record], + )?; + } + tx.commit()?; + drop(db); + fs::File::open(&staging)?.sync_all()?; + fs::rename(&staging, &destination)?; + super::sync_directory(dir)?; + Ok(()) + })(); + if let Err(error) = result { + let _ = fs::remove_file(&staging); + return Err(io::Error::other(error.to_string())); + } + Ok(()) +} + +/// Reads the proof in `dir`. `None` for a missing, foreign, newer, corrupt or +/// internally inconsistent file: the caller then takes the full check. +pub(crate) fn read_tree_proof(dir: &Path) -> Option { + let db = rusqlite::Connection::open_with_flags( + dir.join(TREE_PROOF_FILE), + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .ok()?; + if db + .pragma_query_value::(None, "application_id", |r| r.get(0)) + .ok()? + != TREE_PROOF_APP + || db + .pragma_query_value::(None, "user_version", |r| r.get(0)) + .ok()? + != 1 + { + return None; + } + type Meta = ( + u32, + u32, + String, + String, + String, + String, + String, + Vec, + String, + String, + ); + let meta: Meta = db + .query_row( + "SELECT format_version,schema_version,base_scope,identity_digest,membership_digest,object_format,head_tree,root,policy,receipts_digest FROM proof_meta WHERE singleton=1", + [], + |r| { + Ok(( + r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?, + r.get(5)?, r.get(6)?, r.get(7)?, r.get(8)?, r.get(9)?, + )) + }, + ) + .ok()?; + let ( + format, + schema, + base_scope, + identity_digest, + membership_digest, + object_format, + head_tree, + root, + policy, + digest, + ) = meta; + if (format, schema) != (1, 1) { + return None; + } + let mut statement = db + .prepare("SELECT path,record FROM blob_receipts ORDER BY path") + .ok()?; + let rows = statement + .query_map([], |r| { + Ok((r.get::<_, Vec>(0)?, r.get::<_, Vec>(1)?)) + }) + .ok()? + .collect::>>() + .ok()?; + if receipts_digest( + rows.iter() + .map(|(path, record)| (path.as_slice(), record.as_slice())), + ) != digest + { + return None; + } + let mut receipts = BTreeMap::new(); + for (path, record) in rows { + let receipt: BlobReceipt = serde_json::from_slice(&record).ok()?; + if receipt.version != 1 || receipt.policy != policy { + return None; + } + receipts.insert(RelPath::new(path).ok()?, receipt); + } + Some(TreeProof { + base_scope, + identity_digest, + membership_digest, + object_format, + head_tree, + root: serde_json::from_slice(&root).ok()?, + policy, + receipts, + }) +} + +/// Stat-evidence candidates from a tree-keyed base's blob receipts, or `None` +/// when the base is membership-keyed or its proof does not hold now (every +/// path then takes the full check). +/// +/// The proof must match the authenticated base, the current root and the +/// current reusable policy. Git must still agree: `rev-parse HEAD^{tree}` +/// (taken by the guard) names the base's tree, and a coherent index read of +/// the same index file the guard observed computes that tree. A receipt is +/// offered only for a path whose stage-0 index entry has no exception flag +/// and the receipt's mode and OID. Callers still apply the stat-evidence +/// reuse rules to every candidate before skipping a read. +pub(crate) fn tree_receipt_evidence( + base: &super::base_refs::AuthenticatedBase, + guard: &BindGuard, +) -> Option> { + use crate::blob_store::v2::to_hex; + use crate::git_index_reader::{index_tree, read_index, IndexRead, ObjectFormat}; + if base.identity().kind() != super::base_refs::BaseKind::Tree || !guard.policy.reusable { + return None; + } + let manifest = base.manifest(); + let head_tree = manifest.header().head_tree.as_deref()?; + let format = ObjectFormat::from_hex_len(head_tree.len())?; + let proof = read_tree_proof(base.view_dir())?; + if proof.base_scope != base.scope() + || proof.identity_digest != to_hex(&base.identity().identity) + || proof.membership_digest != to_hex(&base.identity().membership) + || proof.object_format != format.name() + || proof.head_tree != head_tree + || proof.root != guard.root + || proof.policy != guard.policy.digest + || proof.receipts.len() != manifest.len() + { + return None; + } + if guard.head_tree.as_deref()?.trim_ascii() != head_tree.as_bytes() { + return None; + } + let index_path = guard.gitdir.as_ref()?.0.join("index"); + let IndexRead::Snapshot(snapshot) = read_index(&index_path, format, Default::default()) else { + return None; + }; + // The snapshot must come from the index file the guard recorded, which + // final validation re-checks before publication. + if NativeStat::strict_read(&index_path).ok()? != guard.index.as_ref()?.0 + || to_hex(&index_tree(&snapshot).ok()?) != head_tree + { + return None; + } + let index: BTreeMap<&[u8], &crate::git_index_reader::IndexEntry> = snapshot + .entries + .iter() + .map(|entry| (entry.path.as_slice(), entry)) + .collect(); + let mut candidates = BTreeMap::new(); + for (path, entry) in manifest.entries() { + let super::manifest_v2::EntryV2::Regular { + mode, + content, + size, + git_oid: Some(git_oid), + .. + } = entry + else { + return None; + }; + let receipt = proof.receipts.get(path)?; + if receipt.mode != *mode + || receipt.size != *size + || receipt.content != content.to_hex() + || &receipt.git_oid != git_oid + { + return None; + } + let Some(current) = index.get(path.as_bytes()) else { + continue; + }; + if !current.exceptions().is_empty() + || current.stat.mode != *mode + || to_hex(¤t.oid) != *git_oid + { + continue; + } + candidates.insert( + path.clone(), + Evidence { + version: 1, + stat: receipt.stat.clone(), + observed_ns: receipt.observed_ns, + content: receipt.content.clone(), + size: receipt.size, + git_oid: Some(receipt.git_oid.clone()), + policy: receipt.policy.clone(), + keys: BTreeMap::new(), + }, + ); + } + Some(candidates) +} + #[cfg(test)] mod tests { use super::*; From 857d12274cb3e9dcbfd34050d155d7eebc6495f4 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:18:14 +0100 Subject: [PATCH 4/9] mason: name tree proof refusals and keep published proofs unchanged Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit ed0172f1483cc180e4fe0a0fe789b4d25e556ff8) --- crates/aft/src/views/tree_base.rs | 72 +++++++++++++++++-------- crates/aft/src/views/tree_base_tests.rs | 44 +++++++++++++++ 2 files changed, 94 insertions(+), 22 deletions(-) diff --git a/crates/aft/src/views/tree_base.rs b/crates/aft/src/views/tree_base.rs index bb4a917f9..5be32b17a 100644 --- a/crates/aft/src/views/tree_base.rs +++ b/crates/aft/src/views/tree_base.rs @@ -48,21 +48,31 @@ struct ProvenMembers { format: ObjectFormat, } -/// Reads and proves the members, or `None` when any check fails. -fn prove(input: &TreeBaseInput<'_>, guard: &BindGuard) -> Option { +/// Why a fresh checkout gets no tree-keyed base. +pub(crate) type NoTreeProof = &'static str; + +/// Reads and proves the members, or names the first check that failed. +fn prove(input: &TreeBaseInput<'_>, guard: &BindGuard) -> Result { if !guard.policy.reusable { - return None; + return Err("conversion_policy_not_reusable"); } - let head_tree = std::str::from_utf8(guard.head_tree.as_deref()?.trim_ascii()).ok()?; - let format = ObjectFormat::from_hex_len(head_tree.len())?; - let index_path = guard.gitdir.as_ref()?.0.join("index"); + let head_tree = guard + .head_tree + .as_deref() + .and_then(|tree| std::str::from_utf8(tree.trim_ascii()).ok()) + .ok_or("no_head_tree")?; + let format = ObjectFormat::from_hex_len(head_tree.len()).ok_or("unknown_object_format")?; + let index_path = guard.gitdir.as_ref().ok_or("no_gitdir")?.0.join("index"); let IndexRead::Snapshot(snapshot) = read_index(&index_path, format, Default::default()) else { - return None; + return Err("index_not_coherent"); }; - if NativeStat::strict_read(&index_path).ok()? != guard.index.as_ref()?.0 - || to_hex(&index_tree(&snapshot).ok()?) != head_tree + if NativeStat::strict_read(&index_path).ok().as_ref() + != guard.index.as_ref().map(|(stat, _)| stat) { - return None; + return Err("index_changed_since_guard"); + } + if to_hex(&index_tree(&snapshot).map_err(|_| "index_tree_refused")?) != head_tree { + return Err("index_tree_differs_from_head"); } let index: BTreeMap<&[u8], _> = snapshot .entries @@ -89,24 +99,28 @@ fn prove(input: &TreeBaseInput<'_>, guard: &BindGuard) -> Option } let absolute = input .root - .join(super::segment_store::rel_path_to_os(path).ok()?); + .join(super::segment_store::rel_path_to_os(path).map_err(|_| "unrepresentable_path")?); let observed_ns = super::verification::now_ns(); - let before = NativeStat::read(&absolute).ok()?; - if !std::fs::symlink_metadata(&absolute).ok()?.is_file() { - return None; + let unreadable = |_| "member_unreadable"; + let before = NativeStat::read(&absolute).map_err(unreadable)?; + if !std::fs::symlink_metadata(&absolute) + .map_err(unreadable)? + .is_file() + { + return Err("member_not_a_regular_file"); } - let source = std::fs::read(&absolute).ok()?; - let after = NativeStat::read(&absolute).ok()?; + let source = std::fs::read(&absolute).map_err(unreadable)?; + let after = NativeStat::read(&absolute).map_err(unreadable)?; if before != after || before.size != source.len() as u64 { - return None; + return Err("member_changed_during_read"); } let oid = format.blob_oid(&source); if oid != entry.oid { - return None; + return Err("member_differs_from_index_blob"); } #[cfg(unix)] if (before.mode & 0o111 != 0) != (entry.stat.mode == 0o100755) { - return None; + return Err("member_mode_differs_from_index"); } let content = crate::blob_store::v2::ContentHash::of(&source); let planes = EntryPlanes { @@ -131,7 +145,7 @@ fn prove(input: &TreeBaseInput<'_>, guard: &BindGuard) -> Option planes, }, ) - .ok()?; + .map_err(|_| "invalid_manifest_entry")?; receipts.insert( path.clone(), BlobReceipt { @@ -148,7 +162,10 @@ fn prove(input: &TreeBaseInput<'_>, guard: &BindGuard) -> Option ); bytes.insert(path.clone(), source); } - (!manifest.is_empty()).then_some(ProvenMembers { + if manifest.is_empty() { + return Err("no_tracked_members"); + } + Ok(ProvenMembers { manifest, bytes, receipts, @@ -166,7 +183,7 @@ pub(crate) fn bootstrap( claim_slot: &Mutex>, ) -> Option { let guard = BindGuard::capture(input.root, None).ok()?; - let mut proven = prove(&input, &guard)?; + let mut proven = prove(&input, &guard).ok()?; let route = input.route; let identity = BaseIdentity::of(&route.family, &proven.manifest).ok()?; if identity.kind() != BaseKind::Tree { @@ -237,6 +254,17 @@ fn build( super::base_builder::register_base_in(tx, storage, family, &identity.scope) }) .map_err(|e| error(&e))?; + // A base that is already published keeps its proof unchanged: a + // proof is written only before a scope's first publication. + if super::ViewStore::existing_dir(dir.clone()) + .map(|store| store.current_generation_read_only()) + .transpose() + .map_err(|e| error(&e))? + .flatten() + .is_some() + { + return Ok(()); + } super::verification::write_tree_proof( storage, family, diff --git a/crates/aft/src/views/tree_base_tests.rs b/crates/aft/src/views/tree_base_tests.rs index c11f345ab..127f42333 100644 --- a/crates/aft/src/views/tree_base_tests.rs +++ b/crates/aft/src/views/tree_base_tests.rs @@ -325,6 +325,50 @@ fn a_file_whose_bytes_no_longer_match_its_receipt_takes_the_full_check() { assert_current_bytes(&snapshot, root.path()); } +#[test] +fn the_tree_proof_names_the_first_check_a_checkout_fails() { + let _serial = crate::blob_store::v3::serial_for_test(); + let storage = tempfile::tempdir().unwrap(); + let route = crate::views::snapshot::V3Route { + storage: storage.path().to_owned(), + family: FAMILY.into(), + scope: WORKER.into(), + }; + let paths: Vec = FILES.iter().map(|(name, _)| rel(name)).collect(); + let producers = producers(); + let outcome = |root: &Path| { + let input = super::TreeBaseInput { + route: &route, + root, + paths: &paths, + policy: policy(), + producers: &producers, + ignore_fingerprint: None, + }; + let guard = crate::views::verification::BindGuard::capture(root, None).unwrap(); + super::prove(&input, &guard).map(|proven| proven.receipts.len()) + }; + let plain = checkout(false); + let _plain_attributes = known_attribute_sources_for_test(plain.path()); + assert_eq!(outcome(plain.path()), Err("no_head_tree")); + let root = checkout(true); + let _attributes = known_attribute_sources_for_test(root.path()); + assert_eq!(outcome(root.path()), Ok(FILES.len())); + git( + root.path(), + &["update-index", "--assume-unchanged", "lib.rs"], + ); + assert_eq!(outcome(root.path()), Err("index_tree_refused")); + git( + root.path(), + &["update-index", "--no-assume-unchanged", "lib.rs"], + ); + write_old(&root.path().join("lib.rs"), b"pub fn edited() {}\n"); + assert_eq!(outcome(root.path()), Err("member_differs_from_index_blob")); + git(root.path(), &["add", "lib.rs"]); + assert_eq!(outcome(root.path()), Err("index_tree_differs_from_head")); +} + #[test] fn a_tree_proof_round_trips_and_refuses_another_application() { let _serial = crate::blob_store::v3::serial_for_test(); From 799e1a0649843f94351222f48acbc31336bca8f5 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:26:00 +0100 Subject: [PATCH 5/9] mason: report why a tree-keyed bootstrap is unavailable Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 1fd271284572f785def2ecd7ba687ded8706018d) --- crates/aft/src/views/tree_base.rs | 31 +++++++++++++++---------- crates/aft/src/views/tree_base_tests.rs | 19 +++++++++++++++ 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/crates/aft/src/views/tree_base.rs b/crates/aft/src/views/tree_base.rs index 5be32b17a..43e4df15d 100644 --- a/crates/aft/src/views/tree_base.rs +++ b/crates/aft/src/views/tree_base.rs @@ -182,32 +182,39 @@ pub(crate) fn bootstrap( input: TreeBaseInput<'_>, claim_slot: &Mutex>, ) -> Option { - let guard = BindGuard::capture(input.root, None).ok()?; - let mut proven = prove(&input, &guard).ok()?; + try_bootstrap(input, claim_slot).ok() +} + +/// [`bootstrap`] with the reason a tree-keyed base is unavailable. +fn try_bootstrap( + input: TreeBaseInput<'_>, + claim_slot: &Mutex>, +) -> Result { + let guard = BindGuard::capture(input.root, None).map_err(|e| e.to_string())?; + let mut proven = prove(&input, &guard)?; let route = input.route; - let identity = BaseIdentity::of(&route.family, &proven.manifest).ok()?; - if identity.kind() != BaseKind::Tree { - return None; - } + let identity = BaseIdentity::of(&route.family, &proven.manifest).map_err(|e| e.to_string())?; if let Ok(base) = resolve_base(&route.storage, &route.family, &identity.scope) { - return Some(base); + return Ok(base); } let ClaimOutcome::Claimed(claim) = - claim_base(&route.storage, &route.family, &identity.scope, None).ok()? + claim_base(&route.storage, &route.family, &identity.scope, None) + .map_err(|e| e.to_string())? else { - return None; + return Err("tree_base_claim_unavailable".into()); }; match build(route, &identity, &guard, &mut proven, input.policy, &claim) { Ok(()) => { - let base = resolve_base(&route.storage, &route.family, &identity.scope).ok(); + let base = resolve_base(&route.storage, &route.family, &identity.scope) + .map_err(|e| e.to_string()); *claim_slot .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(claim); base } - Err(_) => { + Err(error) => { let _ = claim.release(); - None + Err(error) } } } diff --git a/crates/aft/src/views/tree_base_tests.rs b/crates/aft/src/views/tree_base_tests.rs index 127f42333..512cdaaab 100644 --- a/crates/aft/src/views/tree_base_tests.rs +++ b/crates/aft/src/views/tree_base_tests.rs @@ -354,6 +354,25 @@ fn the_tree_proof_names_the_first_check_a_checkout_fails() { let root = checkout(true); let _attributes = known_attribute_sources_for_test(root.path()); assert_eq!(outcome(root.path()), Ok(FILES.len())); + let slot = std::sync::Mutex::new(None); + let built = super::try_bootstrap( + super::TreeBaseInput { + route: &route, + root: root.path(), + paths: &paths, + policy: policy(), + producers: &producers, + ignore_fingerprint: None, + }, + &slot, + ); + assert_eq!( + built.as_ref().err(), + None, + "a clean checkout builds a tree-keyed base" + ); + assert_eq!(built.unwrap().identity().kind(), BaseKind::Tree); + slot.lock().unwrap().take().unwrap().release().unwrap(); git( root.path(), &["update-index", "--assume-unchanged", "lib.rs"], From 7fd67ef4b2ec7d11bcac6fad520e196f01ffcbcb Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:29:25 +0100 Subject: [PATCH 6/9] mason: create the tree base member directory before writing its proof Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 21536e8dceaf1fd798d813853f2216b81cbc33b1) --- crates/aft/src/views/tree_base.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/aft/src/views/tree_base.rs b/crates/aft/src/views/tree_base.rs index 43e4df15d..06005f026 100644 --- a/crates/aft/src/views/tree_base.rs +++ b/crates/aft/src/views/tree_base.rs @@ -262,12 +262,11 @@ fn build( }) .map_err(|e| error(&e))?; // A base that is already published keeps its proof unchanged: a - // proof is written only before a scope's first publication. - if super::ViewStore::existing_dir(dir.clone()) - .map(|store| store.current_generation_read_only()) - .transpose() + // proof is written only before a scope's first publication. Opening + // the store also creates the member directory the proof lives in. + if super::ViewStore::open_dir(dir.clone()) + .and_then(|store| store.current_generation()) .map_err(|e| error(&e))? - .flatten() .is_some() { return Ok(()); From 7a5d796253743fa8159a67f559ace19e81cc3489 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:32:04 +0100 Subject: [PATCH 7/9] mason: clarify which resolver the tree-base test exercises Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 9ee68f4a022572214d60cbc3df9dffb7632244d3) --- crates/aft/src/views/base_refs_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/aft/src/views/base_refs_tests.rs b/crates/aft/src/views/base_refs_tests.rs index 6111f91b0..d2aae788b 100644 --- a/crates/aft/src/views/base_refs_tests.rs +++ b/crates/aft/src/views/base_refs_tests.rs @@ -331,7 +331,7 @@ fn an_older_membership_only_reader_refuses_a_published_tree_keyed_base() { else { panic!("first tree base publication lost") }; - // This reader resolves it with the tree identity. + // The current resolver authenticates the scope with its tree identity. let resolved = resolve_base(storage.path(), FAMILY, &identity.scope).unwrap(); assert_eq!(resolved.identity().kind(), BaseKind::Tree); assert_eq!(resolved.generation(), generation); From 0d9c4bc08a0409d7ab7a2d475af6024182660353 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:38:44 +0100 Subject: [PATCH 8/9] mason: check only the unchanged member's readiness after an edit Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit eb65b7e2034f962d24ccc2e173ef7ab670d1994b) --- crates/aft/src/views/tree_base_tests.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/aft/src/views/tree_base_tests.rs b/crates/aft/src/views/tree_base_tests.rs index 512cdaaab..6e8898f97 100644 --- a/crates/aft/src/views/tree_base_tests.rs +++ b/crates/aft/src/views/tree_base_tests.rs @@ -322,7 +322,16 @@ fn a_file_whose_bytes_no_longer_match_its_receipt_takes_the_full_check() { snapshot.disk_state(&rel(FILES[0].0)), DiskState::of_bytes(&edited) ); - assert_current_bytes(&snapshot, root.path()); + // The unchanged member keeps its ready base entry; the edited one waits + // for its own lexical entry like any other changed file. + assert_eq!( + snapshot.disk_state(&rel(FILES[1].0)), + DiskState::of_bytes(FILES[1].1) + ); + assert!(matches!( + snapshot.plane_readiness(&rel(FILES[1].0), FamilyPlane::Trigram), + Some(crate::views::readiness::PlaneState::Ready { .. }) + )); } #[test] From 12c67f89cb7addbcf0fb2105401602ac4774c6fc Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:59:32 +0100 Subject: [PATCH 9/9] Search-quality descriptor for the tree-proven views bases Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- .../slice-descriptors/land-views-tree-proven-bases.json | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 benchmarks/aft-search/slice-descriptors/land-views-tree-proven-bases.json diff --git a/benchmarks/aft-search/slice-descriptors/land-views-tree-proven-bases.json b/benchmarks/aft-search/slice-descriptors/land-views-tree-proven-bases.json new file mode 100644 index 000000000..ebd574711 --- /dev/null +++ b/benchmarks/aft-search/slice-descriptors/land-views-tree-proven-bases.json @@ -0,0 +1,8 @@ +{ + "targeted_mechanism": "none", + "kind": "harness", + "slice_class": "non_ranking", + "fixtures": [ + "harness-goldens" + ] +}