From 691c90f1ab85982fd9885de0c5c2ea0086409ae6 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:54:56 +0100 Subject: [PATCH 1/8] mason: register the v3 import ledger and symbol plane formats; add the v3-only symbol plane Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit b25f98bc736c63bd411c933b779bfc3c458ce3b0) --- crates/aft/src/blob_store/v2.rs | 30 +++++++ crates/aft/src/blob_store/v3.rs | 21 ++++- crates/aft/src/views/manifest_v2.rs | 9 ++ docs/design/views-v3-format.md | 130 ++++++++++++++++++++++++++-- 4 files changed, 180 insertions(+), 10 deletions(-) diff --git a/crates/aft/src/blob_store/v2.rs b/crates/aft/src/blob_store/v2.rs index 372f72505..ab8d71f32 100644 --- a/crates/aft/src/blob_store/v2.rs +++ b/crates/aft/src/blob_store/v2.rs @@ -44,6 +44,9 @@ pub const STORE_SCHEMA_VERSION: i64 = 1; /// Payload encoding version of trigram blobs. Bumping it must also change the /// trigram policy fingerprint, because that fingerprint is part of every key. pub const TRIGRAM_PAYLOAD_SCHEMA: u32 = 1; +/// Layout of a v3 symbol-plane payload (`views::symbol_store`). The symbol +/// plane exists only in v3; no v2 store is ever opened for it. +pub const SYMBOL_PAYLOAD_SCHEMA: u32 = 1; const STORE_SCHEMA: &str = r#" CREATE TABLE IF NOT EXISTS store_meta ( @@ -84,16 +87,28 @@ pub enum FamilyPlane { Trigram, Semantic, Callgraph, + /// Parsed symbols of one file, keyed by content, language and the + /// extraction schema. A v3-only plane: it has no manifest readiness and + /// no v2 store, so it is deliberately absent from [`Self::ALL`], which + /// the manifest, v2 GC, eviction and per-entry code iterate. The v3 + /// plane-row sweep covers it through [`Self::V3_STORES`]. + Symbol, } impl FamilyPlane { + /// The planes a manifest entry, a v2 family store and the v2 GC know. + /// Never add [`Self::Symbol`] here: `EntryPlanes::get_mut` has no field + /// for it and panics. pub const ALL: [Self; 3] = [Self::Trigram, Self::Semantic, Self::Callgraph]; + /// Every plane that can have a store under `blobs/v3//`. + pub const V3_STORES: [Self; 4] = [Self::Trigram, Self::Semantic, Self::Callgraph, Self::Symbol]; pub const fn as_str(self) -> &'static str { match self { Self::Trigram => "trigram", Self::Semantic => "semantic", Self::Callgraph => "callgraph", + Self::Symbol => "symbol", } } @@ -102,6 +117,7 @@ impl FamilyPlane { Self::Trigram => TRIGRAM_PAYLOAD_SCHEMA, Self::Semantic => crate::blob_store::SEMANTIC_PAYLOAD_SCHEMA, Self::Callgraph => crate::blob_store::CALLGRAPH_PAYLOAD_SCHEMA, + Self::Symbol => SYMBOL_PAYLOAD_SCHEMA, } } } @@ -1892,6 +1908,20 @@ mod tests { .family_key() } + #[test] + fn symbol_is_not_a_manifest_plane() { + // `EntryPlanes::get_mut` panics for Symbol because a manifest entry + // has no symbol state. That panic is unreachable only while every + // list the manifest, v2 store, GC, eviction and per-entry code + // iterates (`FamilyPlane::ALL`) excludes the v3-only symbol plane. + assert!(!FamilyPlane::ALL.contains(&FamilyPlane::Symbol)); + assert!(FamilyPlane::V3_STORES.contains(&FamilyPlane::Symbol)); + for plane in FamilyPlane::ALL { + assert!(FamilyPlane::V3_STORES.contains(&plane)); + } + assert_eq!(FamilyPlane::Symbol.as_str(), "symbol"); + } + #[test] fn a_conflicting_payload_for_one_key_is_rejected_and_the_stored_payload_is_kept() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/aft/src/blob_store/v3.rs b/crates/aft/src/blob_store/v3.rs index 36913daef..06da17f73 100644 --- a/crates/aft/src/blob_store/v3.rs +++ b/crates/aft/src/blob_store/v3.rs @@ -24,6 +24,10 @@ pub const LAYOUT_TAG: u32 = 2; pub const REGISTRY_APPLICATION_ID: i64 = 0x41465452; pub const PLANE_APPLICATION_ID: i64 = 0x41465450; pub const NAMESPACE_BUDGET: u64 = 4 * 1024 * 1024 * 1024; +/// Sweep epochs a symbol-plane row survives without being touched. Symbol +/// rows are a reusable cache that no manifest names, so the plane-row sweep +/// ages them out by epoch instead of by marking. +pub const SYMBOL_RETENTION_SWEEPS: i64 = 8; const TIMEOUT_MS: i64 = 5_000; mod connections; mod copy; @@ -1315,7 +1319,8 @@ impl RegistryWriter<'_> { complete: true, ..PlaneRowSweep::default() }; - for plane in FamilyPlane::ALL { + // Every v3 plane store, including the v3-only symbol plane. + for plane in FamilyPlane::V3_STORES { let path = self.dir.join(format!("{}.sqlite", plane.as_str())); if !path.try_exists()? { cursor.after.remove(plane.as_str()); @@ -1349,6 +1354,15 @@ impl RegistryWriter<'_> { .and_then(|hex| super::v2::parse_hex32(hex)) .map(|key| key.to_vec()) .unwrap_or_default(); + // No manifest names a symbol row, so marking never protects one. + // A symbol row is instead kept until it has gone untouched for + // SYMBOL_RETENTION_SWEEPS sweep epochs; every other plane is + // collected as soon as it is unmarked and older than this epoch. + let collect_below = if plane == FamilyPlane::Symbol { + epoch.saturating_sub(SYMBOL_RETENTION_SWEEPS) + } else { + epoch + }; loop { if report.examined >= limit || deadline.is_some_and(|d| Instant::now() >= d) { report.complete = false; @@ -1357,7 +1371,7 @@ impl RegistryWriter<'_> { let tx = c.transaction_with_behavior(TransactionBehavior::Immediate)?; let keys: Vec> = tx .prepare(select)? - .query_map(params![epoch, after], |r| r.get(0))? + .query_map(params![collect_below, after], |r| r.get(0))? .collect::>()?; if keys.is_empty() { tx.commit()?; @@ -1369,7 +1383,7 @@ impl RegistryWriter<'_> { let protected = <[u8; 32]>::try_from(key.as_slice()) .map_or(true, |key| marked.contains(&key)); if !protected { - report.deleted += tx.execute(delete, params![key, epoch])? as u64; + report.deleted += tx.execute(delete, params![key, collect_below])? as u64; } } tx.commit()?; @@ -1662,6 +1676,7 @@ pub fn v3_format_report(path: &Path) -> Result { Some("semantic") => Kind::Plane(FamilyPlane::Semantic), Some("callgraph") => Kind::Plane(FamilyPlane::Callgraph), Some("trigram") => Kind::Plane(FamilyPlane::Trigram), + Some("symbol") => Kind::Plane(FamilyPlane::Symbol), _ => return Err(Error::InvalidIdentity), }; let database = connections::shared(path, kind, false, true)?; diff --git a/crates/aft/src/views/manifest_v2.rs b/crates/aft/src/views/manifest_v2.rs index 48031a684..52351b652 100644 --- a/crates/aft/src/views/manifest_v2.rs +++ b/crates/aft/src/views/manifest_v2.rs @@ -55,6 +55,8 @@ impl Producers { FamilyPlane::Trigram => Some(&self.trigram), FamilyPlane::Semantic => self.semantic.as_deref(), FamilyPlane::Callgraph => Some(&self.callgraph), + // Symbols are not a manifest plane: no producer is recorded here. + FamilyPlane::Symbol => None, } } } @@ -118,6 +120,7 @@ impl EntryPlanes { FamilyPlane::Trigram => self.trigram.as_ref(), FamilyPlane::Semantic => self.semantic.as_ref(), FamilyPlane::Callgraph => self.callgraph.as_ref(), + FamilyPlane::Symbol => None, } } @@ -126,6 +129,12 @@ impl EntryPlanes { FamilyPlane::Trigram => &mut self.trigram, FamilyPlane::Semantic => &mut self.semantic, FamilyPlane::Callgraph => &mut self.callgraph, + // Unreachable while FamilyPlane::ALL excludes Symbol (pinned by + // `blob_store::v2` test `symbol_is_not_a_manifest_plane`). A write + // here could not be persisted, so it fails loudly instead. + FamilyPlane::Symbol => { + panic!("symbol readiness is not a manifest plane; EntryPlanes has no symbol state") + } } } } diff --git a/docs/design/views-v3-format.md b/docs/design/views-v3-format.md index d6dc89e32..aef2ebf5f 100644 --- a/docs/design/views-v3-format.md +++ b/docs/design/views-v3-format.md @@ -609,7 +609,10 @@ contract is [views-slice4-opening.md](views-slice4-opening.md). - The implemented registry remains `views_v3_registry` format/schema **1/1**. Reserve **2/2** for slice 4's base-reference protocol, **3/3** for 4v's token and stat-evidence protocol, **4/4** for slice 5's donor-seal protocol, and - **5/5** for 6a's symbol/import protocol. These versions gate the whole member + 5/5, earlier reserved for 6a, is **not used** by 6a: its import ledger is a + sidecar file and its symbol plane is an ordinary plane store (see *Import + ledger and symbol plane* below), so neither changes the registry schema. + These versions gate the whole member protocol, including its sidecars; a version can advance without adding a registry table. An owning slice installs its complete schema through an explicit versioned upgrade, never a @@ -633,10 +636,11 @@ contract is [views-slice4-opening.md](views-slice4-opening.md). registry-floor version that makes unsupported readers refuse. - Every artifact publication uses the combined v3 floor boundary. Slice 4 requests registry floor 2 before its artifacts; 4v requests 3 before stat - evidence or tokens; 5 requests 4 before donor/seal publication; 6a requests 5 - before **either** symbol-plane or import-ledger publication. Every publisher - also requests store floor 1. These are future written versions, not changes - to today's startup baseline. + evidence or tokens; 5 requests 4 before donor/seal publication. 6a writes + its symbol plane and import ledger only inside `blob_store::v3::publish`, + which requests registry floor 1 and store floor 1 before creating either + file. Every publisher also requests store floor 1. These are future written + versions, not changes to today's startup baseline. ### Assigned paths and owners @@ -651,8 +655,8 @@ contract is [views-slice4-opening.md](views-slice4-opening.md). | 4v | `M/stat-evidence-.sqlite` | Stat-evidence 1/1, registry floor 3 | Generation-paired path/root/file identity, size/times, stable content digest, proven Git OID if any, observation-time conversion-policy digest and per-plane keys. No stat-cache or current-policy-only proof. | | 4v | `F/members.sqlite`: `view_tokens` rows | `views_v3_registry` 3/3 | Single-use advisory token bound to identity at first consumption, with expiry and consume state. Stored in daemon storage, never the checkout. Stat evidence and tokens do not replace freshness validation. | | 5 | `M/donor--.seal.json` beside `M/donor--.sqlite` | Donor seal 1, registry floor 4 | Atomically publish fingerprint, size and `journal_mode=DELETE` only after the detached donor is checkpointed and synced and has no WAL/SHM. A published manifest names only a completed seal. Never copy/clone a live SQLite file set. | -| 6a | `F/symbol.sqlite` | `views_v3_store` 1/1, new-store layout 2; registry floor 5 | The v3 symbol plane uses the same immutable `payload`/`payload_digest` contract and reservation protocol. No in-place legacy conversion. | -| 6a | `F/members.sqlite`: `imports` rows | `views_v3_registry` 5/5 | Record completion/rejection per artifact, compatible producer and destination plane, with owner-token claim/recovery semantics. Import into v3 only, stream inputs above 1 GiB and retain legacy bytes. | +| 6a | `F/symbol.sqlite` | `views_v3_store` 1/1, new-store layout 2; store floor 1 | The v3 symbol plane uses the same immutable `payload`/`payload_digest` contract and reservation protocol. No in-place legacy conversion. Implemented; see *Import ledger and symbol plane*. | +| 6a | `F/imports.sqlite` (sidecar, not a registry table) | Import ledger 1/1, `application_id` `0x41465449`; registry stays 1/1 or 2/2 | Record completion/rejection per legacy key, artifact, compatible producer and destination plane, with owner-token claim/recovery semantics. Import into v3 only, stream inputs above 1 GiB and retain legacy bytes. Implemented; see *Import ledger and symbol plane*. | The registry-version increments above are serial handoffs, not permission for an earlier slice to write a later owner's rows. Owners implement and test their @@ -664,3 +668,115 @@ and member retention remain unchanged. Slice 4's binder must honour the `evicting` contract above: when it finds its member in `evicting`, it never uses that view directory, and either treats the member as absent and publishes a fresh one or waits for the eviction to finish. + +## Import ledger and symbol plane (6a) + +Sources: `crates/aft/src/migration/v3_import.rs` (ledger, decoders, staging, +registration), `crates/aft/src/migration/v3_symbols.rs` (symbol importer) and +`crates/aft/src/views/symbol_store.rs` (symbol plane). Imports write only +under `blobs/v3//` and their own staging directory; they never write +a v1 or v2 row, manifest, pointer or schema, and never move, change or delete +a legacy file. + +### Symbol plane `F/symbol.sqlite` + +- An ordinary v3 plane store (`views_v3_store` 1/1, layout 2, `store_meta.plane + = 'symbol'`), opened and written through the same `blob_store::v3` + connections, `Publication` and `PlaneReader` as the other planes: + `FamilyPlane::Symbol`. It is a v3-only plane: it is not in + `FamilyPlane::ALL` (which manifests, v2 stores, v2 GC and eviction + iterate) but is in `FamilyPlane::V3_STORES`, which the v3 plane-row sweep + and `v3_format_report` use. No manifest entry records symbol readiness. +- Key: BLAKE3 over the tag `aft/views-v3/symbol/v1`, then the content hash + (32 bytes), the language id and the producer id, each preceded by its + length as a big-endian u64. The producer id is + `aft-symbols/extraction-` (`symbol_cache_disk::SCHEMA_VERSION`, + bumped on any grammar, query or extractor change), so a parser or schema + change produces a different key and misses. A source hash alone is never + a key. +- Payload (payload schema 1): `u8` layout 1, then three length-prefixed + (`u32` LE) fields: producer id, language id, and the symbols as the JSON + array `symbols.bin` stores. A reader checks the layout, producer and + language against its key's inputs before decoding. +- **GC.** Manifests never name symbol rows, so the sweep does not use marking + for them: it deletes a symbol row only when its `ref_epoch` is more than + `SYMBOL_RETENTION_SWEEPS` (8) sweep epochs old. A lookup that uses a row + touches it inside a publication, which restamps it. +- **Residency.** Decoded symbol lists are shared as `Arc>` + through a process-wide cache in `views/symbol_store.rs`, bounded at 64 MiB + of estimated bytes; eviction from it never touches stored rows. +- **Floor and accounting.** Written only inside `v3::publish` (registry floor + 1, store floor 1, a reservation per batch); its bytes are part of the + family's namespace inventory like any plane store. + +### Import ledger `F/imports.sqlite` + +A sidecar SQLite file, not a registry table: `members.sqlite` and its +version are unchanged. It is created, and every row transition is written, +inside a `v3::publish` call (64 KiB reservation), so the file exists only +after the v3 floor is raised and each write holds the reservation lock. +Readers (status, prune eligibility, retention) copy the database and its WAL +to a private temporary directory and read the copy, so a read never writes +under the storage root. + +```sql +CREATE TABLE format_meta (singleton INTEGER PRIMARY KEY CHECK(singleton=1), + format_version INTEGER NOT NULL, schema_version INTEGER NOT NULL, schema_text TEXT NOT NULL); +CREATE TABLE imports (legacy_key TEXT NOT NULL, artifact TEXT NOT NULL, + producer TEXT NOT NULL, plane TEXT NOT NULL, + state TEXT NOT NULL CHECK(state IN ('claimed','staged','validated','registered','done','rejected')), + attempt INTEGER NOT NULL CHECK(attempt>=1), owner_pid INTEGER, owner_start INTEGER, owner_token TEXT, + source_hash BLOB, source_len INTEGER, staged_hash BLOB, + imported INTEGER NOT NULL DEFAULT 0, skipped INTEGER NOT NULL DEFAULT 0, + reason TEXT, updated_ms INTEGER NOT NULL, completed_ms INTEGER, + PRIMARY KEY(legacy_key, artifact, producer, plane)); +``` + +Format/schema 1/1, `application_id` `0x41465449` ("AFTI"), WAL, +`synchronous=FULL`. A newer format is refused by name. `artifact` is +`trigram` (`index//cache.bin`), `semantic` (`semantic//semantic.bin`) +or `symbols` (`symbols//symbols.bin`); `producer` is the trigram policy +fingerprint, the semantic producer id or the symbol producer id; `plane` is +the destination plane. A row completes only its own (artifact, producer, +plane): a semantic-only completion never suppresses a trigram or symbol +import, and a producer change starts a new row. + +- **States.** `claimed → staged → validated → registered → done`, or + `rejected{reason}` from any non-terminal state. Each transition is one + `BEGIN IMMEDIATE` compare-and-set on the row's state and owner token. A + live owner makes other callers back off; a dead or released owner's row is + taken over with `attempt + 1` and resumes from its durable state, + repeating at most one idempotent step. +- **staged:** the canonical conversion is streamed to + `/migration/v3///--.bundle` (temp, fsync, rename, directory sync), with the + legacy source's BLAKE3 and length recorded. Old binaries never look under + `migration/v3`. A bundle is a magic `AFTIMP03`, the artifact byte, entries + (`u8` 1, path, content hash, size, key, payload), then `u8` 0, the entry + count and a BLAKE3 of every earlier byte. +- **validated:** the bundle is re-read and every key re-derived, and the + legacy source is re-hashed and must still be the converted bytes. +- **registered:** every payload is in the destination plane, written in + batches of at most 32 MiB, each one `v3::publish` with a reservation of + twice its bytes plus 1 MiB. A refused reservation reports the plane's + named storage gap `storage_gap:` (`v3_namespace_budget`, + `device_free_space_floor`, `reservation_ledger_unreadable`); the row keeps + its state with the owner released and the reason recorded, so a later + run resumes. Nothing falls back to a legacy reader. +- **done:** the staging bundle is removed. + +Inputs have no size limit. The decoders stream: the trigram `cache.bin` +is checksummed in one sequential pass, then inverted in file-id ranges +whose postings fit a working-set budget (64 MiB by default), one pass over +the posting blob per range; `semantic.bin` is decoded one file's rows at a +time. Memory is bounded by the per-file tables (path, size and hash per +file) plus the budget, never by the input size. The `semantic.bin` decoder +is read-only and refuses trailing bytes (appended segments or a torn tail) +rather than repairing or folding them; a path whose rows are not +contiguous is skipped and left for the plane to embed. + +Imported trigram and semantic rows are not named by any manifest until a +base or worker build reuses them (`contains`, then `put_or_touch`). A v3 +plane-row sweep that runs between the import and that build may collect +them; the build then recomputes those files. Imported symbol rows follow the +symbol plane's epoch rule above. From 07143134a6a77a4dd589ebd45180a2ed66eef715 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:20:58 +0100 Subject: [PATCH 2/8] mason: stream legacy postings and vectors into v3 plane rows through staged bundles Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit d18ea48005e640c55aa3333a8671a76a6f3e7c9f) --- crates/aft/src/migration/mod.rs | 1 + crates/aft/src/migration/per_checkout.rs | 246 ++-- crates/aft/src/migration/v3_import.rs | 1416 +++++++++++++++++++ crates/aft/src/migration/v3_import_tests.rs | 441 ++++++ 4 files changed, 1983 insertions(+), 121 deletions(-) create mode 100644 crates/aft/src/migration/v3_import.rs create mode 100644 crates/aft/src/migration/v3_import_tests.rs diff --git a/crates/aft/src/migration/mod.rs b/crates/aft/src/migration/mod.rs index badf2ceeb..d9f9f9acd 100644 --- a/crates/aft/src/migration/mod.rs +++ b/crates/aft/src/migration/mod.rs @@ -15,6 +15,7 @@ pub mod per_checkout; pub mod prune_legacy; +pub mod v3_import; use std::collections::BTreeMap; use std::fmt; diff --git a/crates/aft/src/migration/per_checkout.rs b/crates/aft/src/migration/per_checkout.rs index c6cd767d7..5ab898240 100644 --- a/crates/aft/src/migration/per_checkout.rs +++ b/crates/aft/src/migration/per_checkout.rs @@ -171,7 +171,7 @@ impl ImportState { } } - fn parse(value: &str) -> Option { + pub(super) fn parse(value: &str) -> Option { [ Self::Claimed, Self::Staged, @@ -200,7 +200,7 @@ pub struct ClaimOwner { } impl ClaimOwner { - fn new() -> Self { + pub(super) fn new() -> Self { static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); let me = this_process(); let seq = SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed); @@ -237,10 +237,10 @@ fn active_claims() -> &'static Mutex> { } /// Marks a claim token live for as long as the import that owns it runs. -struct ActiveClaim(String); +pub(super) struct ActiveClaim(String); impl ActiveClaim { - fn register(owner: &ClaimOwner) -> Self { + pub(super) fn register(owner: &ClaimOwner) -> Self { lock(active_claims()).insert(owner.token.clone()); Self(owner.token.clone()) } @@ -258,7 +258,7 @@ fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { .unwrap_or_else(std::sync::PoisonError::into_inner) } -fn now_ms() -> u64 { +pub(super) fn now_ms() -> u64 { SystemTime::now() .duration_since(UNIX_EPOCH) .map(|elapsed| elapsed.as_millis().min(u128::from(u64::MAX)) as u64) @@ -781,18 +781,18 @@ fn status_of(rows: Vec) -> ImportStatus { /// One converted file: the manifest facts and the payload stored under `key`. #[derive(Clone, Debug, Eq, PartialEq)] -struct BundleEntry { - rel_path: RelPath, - content: ContentHash, - size: u64, - key: [u8; 32], - payload: Vec, +pub(super) struct BundleEntry { + pub(super) rel_path: RelPath, + pub(super) content: ContentHash, + pub(super) size: u64, + pub(super) key: [u8; 32], + pub(super) payload: Vec, } /// The staged, canonical conversion of one legacy artifact. #[derive(Clone, Debug, Default, Eq, PartialEq)] -struct Bundle { - entries: Vec, +pub(super) struct Bundle { + pub(super) entries: Vec, } impl Bundle { @@ -868,7 +868,7 @@ fn staged_path(ledger: &ImportLedger, artifact: Artifact, staged_hash: &[u8; 32] /// Writes `bytes` to `path` through temp + fsync + rename, then syncs the /// directory, so `staged` is only recorded for bytes that reached the disk. -fn write_durable(path: &Path, bytes: &[u8]) -> io::Result<()> { +pub(super) fn write_durable(path: &Path, bytes: &[u8]) -> io::Result<()> { let parent = path.parent().unwrap_or_else(|| Path::new(".")); crate::private_storage::create_dir_all(parent)?; let temporary = path.with_extension(format!("tmp.{}.{}", std::process::id(), now_ms())); @@ -950,7 +950,7 @@ impl<'a> LeReader<'a> { /// failed checksum, a path that is not relative, or a malformed table. /// Files without a recorded content hash are left out; the first load's /// strict reconciliation adds them from the checkout. -fn convert_trigram(bytes: &[u8], policy: &TrigramPolicy) -> Result { +pub(super) fn convert_trigram(bytes: &[u8], policy: &TrigramPolicy) -> Result { let mut reader = LeReader::new(bytes); if reader.u32()? != LEGACY_CACHE_MAGIC { return Err("not a trigram cache".to_owned()); @@ -1251,21 +1251,12 @@ fn capped_legacy_summary(current: &ChunkShape, legacy: &ChunkShape, budget: usiz /// A legacy small-cap summary may instead be the exact uncapped input: keep /// that run under its old key so the owner can replace only the summary. /// Every other file is left for the plane to embed. -fn convert_semantic( +pub(super) fn convert_semantic( bytes: &[u8], root: &Path, producer: &SemanticProducer, ) -> Result<(Bundle, usize), String> { - let summary_budget = serde_json::from_str::( - &producer.model_fingerprint, - ) - .ok() - .filter(|identity| { - identity.embed_text_caps == producer.caps - && identity.embed_text_caps.total_chars - < crate::semantic_index::EmbedTextCaps::default().total_chars - }) - .map(|identity| identity.embed_text_caps.total_chars); + let summary_budget = small_summary_budget(producer); let parsed = match super::parse_legacy_snapshot(bytes, &producer.model_fingerprint) { Ok(parsed) => parsed, Err(error) @@ -1284,108 +1275,121 @@ fn convert_semantic( let mut entries = Vec::new(); let mut skipped = 0usize; for file in parsed.files { - let Some(rel_path) = super::legacy_path_to_rel_path(&file.path, &root) else { - skipped += 1; - continue; - }; - let Ok(relative) = segment_store::rel_path_to_os(&rel_path) else { - skipped += 1; - continue; - }; - let absolute = root.join(&relative); - let fits = fs::symlink_metadata(&absolute) - .is_ok_and(|meta| meta.is_file() && meta.len() <= MAX_SEMANTIC_SOURCE_BYTES); - let Some(source) = fits.then(|| fs::read(&absolute).ok()).flatten() else { - skipped += 1; - continue; - }; - if file.content_hash == [0; 32] || blake3::hash(&source).as_bytes() != &file.content_hash { - skipped += 1; - continue; + match convert_semantic_file(&root, producer, summary_budget, file) { + Some(entry) => entries.push(entry), + None => skipped += 1, } - let chunks = match crate::semantic_index::chunk_view_file( - &root, - &relative, - &source, - producer.caps, - ) { + } + entries.sort_by(|left, right| left.rel_path.cmp(&right.rel_path)); + Ok((Bundle { entries }, skipped)) +} + +/// The small text cap a producer's fingerprint records, when it is below the +/// default: a legacy file summary embedded under that cap may then be the +/// exact uncapped input of the current one (see [`capped_legacy_summary`]). +pub(super) fn small_summary_budget(producer: &SemanticProducer) -> Option { + serde_json::from_str::( + &producer.model_fingerprint, + ) + .ok() + .filter(|identity| { + identity.embed_text_caps == producer.caps + && identity.embed_text_caps.total_chars + < crate::semantic_index::EmbedTextCaps::default().total_chars + }) + .map(|identity| identity.embed_text_caps.total_chars) +} + +/// Converts one legacy file's rows under `producer`, or `None` when the file +/// cannot be imported and is left for the plane to embed: its path is not +/// under `root`, its bytes changed since the snapshot, or this release chunks +/// them into other rows than the snapshot holds. `root` is canonical. +pub(super) fn convert_semantic_file( + root: &Path, + producer: &SemanticProducer, + summary_budget: Option, + file: super::LegacyFile, +) -> Option { + let rel_path = super::legacy_path_to_rel_path(&file.path, root)?; + let relative = segment_store::rel_path_to_os(&rel_path).ok()?; + let absolute = root.join(&relative); + let fits = fs::symlink_metadata(&absolute) + .is_ok_and(|meta| meta.is_file() && meta.len() <= MAX_SEMANTIC_SOURCE_BYTES); + let source = fits.then(|| fs::read(&absolute).ok()).flatten()?; + if file.content_hash == [0; 32] || blake3::hash(&source).as_bytes() != &file.content_hash { + return None; + } + let chunks = + match crate::semantic_index::chunk_view_file(root, &relative, &source, producer.caps) { Ok(Some(chunks)) => chunks, - _ => { - skipped += 1; - continue; - } + _ => return None, }; - let mut stored: BTreeMap>> = BTreeMap::new(); - for entry in file.entries { - stored - .entry(ChunkShape { - name: entry.name, - qualified_name: entry.qualified_name.unwrap_or_default(), - kind: entry.kind, - start_line: entry.start_line, - end_line: entry.end_line, - exported: entry.exported, - snippet: entry.snippet, - embed_text: entry.embed_text, - }) - .or_default() - .push(entry.vector); - } - let chunk_count = chunks.len(); - let mut rows = Vec::with_capacity(chunk_count); - let mut summary_repair_pending = false; - for chunk in chunks { - let shape = ChunkShape { - name: chunk.name, - qualified_name: chunk.qualified_name.unwrap_or_default(), - kind: stored_kind(&chunk.kind), - start_line: chunk.start_line, - end_line: chunk.end_line, - exported: chunk.exported, - snippet: chunk.snippet, - embed_text: chunk.embed_text, - }; - if let Some(vector) = stored.get_mut(&shape).and_then(Vec::pop) { - rows.push((shape, vector)); - } else if let Some(legacy) = summary_budget - .and_then(|budget| { - stored - .keys() - .find(|legacy| capped_legacy_summary(&shape, legacy, budget)) - }) - .cloned() - { - let Some(vector) = stored.get_mut(&legacy).and_then(Vec::pop) else { - break; - }; - rows.push((legacy, vector)); - summary_repair_pending = true; - } else { + let mut stored: BTreeMap>> = BTreeMap::new(); + for entry in file.entries { + stored + .entry(ChunkShape { + name: entry.name, + qualified_name: entry.qualified_name.unwrap_or_default(), + kind: entry.kind, + start_line: entry.start_line, + end_line: entry.end_line, + exported: entry.exported, + snippet: entry.snippet, + embed_text: entry.embed_text, + }) + .or_default() + .push(entry.vector); + } + let chunk_count = chunks.len(); + let mut rows = Vec::with_capacity(chunk_count); + let mut summary_repair_pending = false; + for chunk in chunks { + let shape = ChunkShape { + name: chunk.name, + qualified_name: chunk.qualified_name.unwrap_or_default(), + kind: stored_kind(&chunk.kind), + start_line: chunk.start_line, + end_line: chunk.end_line, + exported: chunk.exported, + snippet: chunk.snippet, + embed_text: chunk.embed_text, + }; + if let Some(vector) = stored.get_mut(&shape).and_then(Vec::pop) { + rows.push((shape, vector)); + } else if let Some(legacy) = summary_budget + .and_then(|budget| { + stored + .keys() + .find(|legacy| capped_legacy_summary(&shape, legacy, budget)) + }) + .cloned() + { + let Some(vector) = stored.get_mut(&legacy).and_then(Vec::pop) else { break; - } - } - let all_matched = rows.len() == chunk_count && stored.values().all(Vec::is_empty); - if !all_matched { - skipped += 1; - continue; - } - let key = if summary_repair_pending { - producer.legacy_key(&source, &rel_path) + }; + rows.push((legacy, vector)); + summary_repair_pending = true; } else { - producer.key(&source, &rel_path) - }; - entries.push(BundleEntry { - content: ContentHash::from_bytes(file.content_hash), - size: source.len() as u64, - key: *key.as_bytes(), - payload: semantic_payload(producer, &rows), - rel_path, - }); + break; + } } - entries.sort_by(|left, right| left.rel_path.cmp(&right.rel_path)); - Ok((Bundle { entries }, skipped)) + let all_matched = rows.len() == chunk_count && stored.values().all(Vec::is_empty); + if !all_matched { + return None; + } + let key = if summary_repair_pending { + producer.legacy_key(&source, &rel_path) + } else { + producer.key(&source, &rel_path) + }; + Some(BundleEntry { + content: ContentHash::from_bytes(file.content_hash), + size: source.len() as u64, + key: *key.as_bytes(), + payload: semantic_payload(producer, &rows), + rel_path, + }) } - // --------------------------------------------------------------------------- // The import diff --git a/crates/aft/src/migration/v3_import.rs b/crates/aft/src/migration/v3_import.rs new file mode 100644 index 000000000..bbfa72656 --- /dev/null +++ b/crates/aft/src/migration/v3_import.rs @@ -0,0 +1,1416 @@ +//! Import of legacy (pre-view) index artifacts into the v3 family stores. +//! +//! The legacy artifacts of one key are `index//cache.bin` (trigram +//! postings), `semantic//semantic.bin` (vectors) and +//! `symbols//symbols.bin` (parsed symbols). This importer converts what +//! can be proven equal to a cold build into rows of the family's v3 planes +//! (`blobs/v3//{trigram,semantic,symbol}.sqlite`), so the base or +//! worker that later needs those files finds the rows by key instead of +//! re-tokenizing, re-embedding or re-parsing them. +//! +//! What it never does: +//! +//! - write a v1 or v2 row, manifest, pointer or schema (it opens no v1/v2 +//! store at all); +//! - move, change or delete a legacy file: only `aft cache prune-legacy` +//! removes legacy sets, and only after the retention window; +//! - fall back to a legacy reader when v3 admission refuses a reservation: +//! the plane reports a named storage gap and serves what v3 already holds; +//! - call an embedding model or the tree-sitter parser for an imported row +//! of the trigram or symbol plane. The semantic plane re-chunks the +//! checkout's bytes to prove the stored rows match, as the v2 import does, +//! but never calls a model. +//! +//! Inputs of any size are streamed: memory is bounded by the per-file tables +//! of an artifact plus a working-set budget, never by the input size. +//! +//! Progress is recorded in the import ledger `blobs/v3//imports.sqlite` +//! (format in docs/design/views-v3-format.md, *Import ledger and symbol +//! plane*), one row per legacy key, artifact, producer and destination plane: +//! +//! ```text +//! (absent) → claimed → staged → validated → registered → done +//! ↘ rejected{reason} (from any non-terminal state) +//! ``` +//! +//! Every transition is a compare-and-set on the row's state and owner token, +//! so a crash at any boundary leaves a row the next attempt resumes by +//! repeating at most one idempotent step. + +use std::collections::{BTreeMap, HashSet}; +use std::fmt; +use std::fs; +use std::io::{self, BufReader, BufWriter, Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +use super::per_checkout::{convert_semantic_file, now_ms, small_summary_budget, BundleEntry}; +use crate::blob_store::v2::{ + to_hex, ContentHash, FamilyKey, FamilyPlane, TrigramKey, TrigramPolicy, +}; +use crate::blob_store::v3; +use crate::views::segment_store::{TrigramFlag, TrigramPayload, TrigramRecord}; +use crate::views::semantic::SemanticProducer; +use crate::views::RelPath; + +/// Format and schema of the v3 import ledger this build writes and reads. +pub const LEDGER_FORMAT_VERSION: i64 = 1; +/// `PRAGMA application_id` of the ledger ("AFTI"). +pub const LEDGER_APPLICATION_ID: i64 = 0x4146_5449; +/// Working-set budget of the trigram inversion when the caller sets none. +pub const DEFAULT_WORKING_SET: usize = 64 << 20; +/// Payload bytes registered by one v3 publication. +const REGISTER_BATCH_BYTES: u64 = 32 << 20; +const STAGED_MAGIC: &[u8; 8] = b"AFTIMP03"; +/// Size of one sequential read of a legacy source. +const READ_CHUNK: usize = 64 * 1024; +/// Longest path field accepted from a legacy artifact or a staged bundle. +const MAX_PATH_BYTES: usize = 64 * 1024; +/// Longest single non-path field (a snippet, an embedded text, a vector or a +/// staged payload) the decoders allocate. +const MAX_FIELD_BYTES: usize = 256 << 20; +/// Upper bound on files listed by one legacy artifact. +const MAX_LEGACY_FILES: usize = 2_000_000; + +// Layout constants of the legacy trigram `cache.bin`, as written by the +// search index of the previous release (`search_index::write_cache_file_from_sources`). +const LEGACY_CACHE_MAGIC: u32 = 0x3144_4958; +const LEGACY_INDEX_MAGIC: &[u8; 8] = b"AFTIDX01"; +const LEGACY_LOOKUP_MAGIC: &[u8; 8] = b"AFTLKP01"; +const LEGACY_POSTING_BYTES: usize = 6; +const LEGACY_LOOKUP_ENTRY_BYTES: usize = 16; + +/// Refusals of a v3 reservation that are a lack of storage, reported as the +/// plane's named storage gap rather than as an import failure. +const STORAGE_GAP_REFUSALS: [&str; 3] = [ + "v3_namespace_budget", + "device_free_space_floor", + "reservation_ledger_unreadable", +]; + +// --------------------------------------------------------------------------- +// Errors + +#[derive(Debug)] +pub enum V3ImportError { + Io(io::Error), + Sqlite(rusqlite::Error), + Store(v3::Error), + /// The ledger was written by a newer AFT. + NewerLedger(i64), + /// The legacy key or family cannot name a directory. + InvalidKey(String), + /// Another owner took the row over between two steps of this attempt. + LostClaim(String), + Corrupt(String), +} + +impl fmt::Display for V3ImportError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Io(error) => write!(f, "v3 legacy import I/O error: {error}"), + Self::Sqlite(error) => write!(f, "v3 legacy import ledger error: {error}"), + Self::Store(error) => write!(f, "v3 legacy import store error: {error}"), + Self::NewerLedger(version) => write!( + f, + "v3 import ledger format {version} is newer than this build reads ({LEDGER_FORMAT_VERSION})" + ), + Self::InvalidKey(key) => write!(f, "invalid legacy key or family `{key}`"), + Self::LostClaim(row) => write!(f, "v3 import claim of {row} was taken over"), + Self::Corrupt(reason) => write!(f, "v3 import state is corrupt: {reason}"), + } + } +} + +impl std::error::Error for V3ImportError {} + +impl From for V3ImportError { + fn from(error: io::Error) -> Self { + Self::Io(error) + } +} + +impl From for V3ImportError { + fn from(error: rusqlite::Error) -> Self { + Self::Sqlite(error) + } +} + +impl From for V3ImportError { + fn from(error: v3::Error) -> Self { + Self::Store(error) + } +} + +pub type V3Result = Result; + +// --------------------------------------------------------------------------- +// Artifacts and paths + +/// One legacy artifact the v3 importer reads. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub enum LegacyArtifact { + Trigram, + Semantic, + Symbols, +} + +impl LegacyArtifact { + pub const ALL: [Self; 3] = [Self::Trigram, Self::Semantic, Self::Symbols]; + + pub const fn as_str(self) -> &'static str { + match self { + Self::Trigram => "trigram", + Self::Semantic => "semantic", + Self::Symbols => "symbols", + } + } + + pub(super) fn parse(value: &str) -> Option { + Self::ALL + .into_iter() + .find(|artifact| artifact.as_str() == value) + } + + /// The v3 plane this artifact's rows go to. + pub const fn plane(self) -> FamilyPlane { + match self { + Self::Trigram => FamilyPlane::Trigram, + Self::Semantic => FamilyPlane::Semantic, + Self::Symbols => FamilyPlane::Symbol, + } + } + + /// The legacy file this artifact is read from. + pub fn source_path(self, storage: &Path, legacy_key: &str) -> PathBuf { + match self { + Self::Trigram => super::per_checkout::legacy_trigram_path(storage, legacy_key), + Self::Semantic => super::per_checkout::legacy_semantic_path(storage, legacy_key), + Self::Symbols => crate::symbol_cache_disk::cache_path(storage, legacy_key), + } + } +} + +/// `/migration/v3//`: staging bundles. It lies +/// outside `blobs/v3` and outside every directory older binaries walk. +pub fn staging_dir(storage: &Path, family: &str, legacy_key: &str) -> PathBuf { + storage + .join("migration") + .join("v3") + .join(family) + .join(legacy_key) +} + +fn hash_prefix(hash: &[u8; 32]) -> String { + to_hex(hash)[..32].to_owned() +} + +// --------------------------------------------------------------------------- +// Bounded reads of a legacy source + +/// What a decode read and held, for the bounded-memory contract. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct ReadStats { + /// Bytes read from the legacy source, summed over every pass. + pub bytes_read: u64, + /// Largest single read from the legacy source. + pub largest_read: usize, + /// Passes over the trigram posting blob after the checksum pass. + pub posting_passes: u32, + /// Largest number of bytes of decoded postings held at once. + pub peak_working_set: u64, +} + +/// The file identity a decode started from. A legacy writer replaces its +/// file by rename, so an unchanged identity on the open handle means the +/// bytes did not change under the decode. +#[derive(Clone, Debug, Eq, PartialEq)] +struct SourceIdentity { + len: u64, + modified: Option, +} + +impl SourceIdentity { + fn of(metadata: &fs::Metadata) -> Self { + Self { + len: metadata.len(), + modified: metadata.modified().ok(), + } + } +} + +/// A legacy source opened once, read only through bounded reads. It is opened +/// read-only and never written, truncated or repaired. +struct Source { + file: fs::File, + len: u64, + identity: SourceIdentity, + stats: ReadStats, +} + +impl Source { + fn open(path: &Path) -> Result { + let file = fs::File::open(path).map_err(|error| match error.kind() { + io::ErrorKind::NotFound => "no legacy artifact".to_owned(), + _ => format!("legacy artifact unreadable: {error}"), + })?; + let metadata = file + .metadata() + .map_err(|error| format!("legacy artifact unreadable: {error}"))?; + if !metadata.is_file() { + return Err("legacy artifact is not a file".to_owned()); + } + Ok(Self { + len: metadata.len(), + identity: SourceIdentity::of(&metadata), + file, + stats: ReadStats::default(), + }) + } + + fn read_at(&mut self, offset: u64, buf: &mut [u8]) -> Result<(), String> { + offset + .checked_add(buf.len() as u64) + .filter(|end| *end <= self.len) + .ok_or("unexpected end of legacy artifact")?; + self.file + .seek(SeekFrom::Start(offset)) + .and_then(|_| self.file.read_exact(buf)) + .map_err(|error| format!("legacy artifact read failed: {error}"))?; + self.stats.bytes_read += buf.len() as u64; + self.stats.largest_read = self.stats.largest_read.max(buf.len()); + Ok(()) + } + + /// Whether the open file still has the identity it had when opened. + fn unchanged(&self) -> bool { + self.file + .metadata() + .is_ok_and(|metadata| SourceIdentity::of(&metadata) == self.identity) + } + + /// One sequential pass over the whole file: its BLAKE3 and the CRC32 of + /// each `[start, end)` range in `crc_ranges`. + fn checksum_pass(&mut self, crc_ranges: &[(u64, u64)]) -> Result<([u8; 32], Vec), String> { + let mut hasher = blake3::Hasher::new(); + let mut crcs = vec![crc32fast::Hasher::new(); crc_ranges.len()]; + let mut buffer = vec![0_u8; READ_CHUNK]; + let mut offset = 0_u64; + while offset < self.len { + let len = (self.len - offset).min(READ_CHUNK as u64) as usize; + self.read_at(offset, &mut buffer[..len])?; + let chunk = &buffer[..len]; + hasher.update(chunk); + for ((start, end), crc) in crc_ranges.iter().zip(crcs.iter_mut()) { + let from = (*start).max(offset); + let to = (*end).min(offset + len as u64); + if from < to { + crc.update(&chunk[(from - offset) as usize..(to - offset) as usize]); + } + } + offset += len as u64; + } + Ok(( + *hasher.finalize().as_bytes(), + crcs.into_iter().map(crc32fast::Hasher::finalize).collect(), + )) + } +} + +/// A little-endian cursor over `[offset, end)` of a source, refilled one +/// bounded window at a time. +struct SeqReader<'a> { + source: &'a mut Source, + window: Vec, + window_start: u64, + offset: u64, + end: u64, +} + +impl<'a> SeqReader<'a> { + fn new(source: &'a mut Source, offset: u64, end: u64) -> Self { + Self { + source, + window: Vec::new(), + window_start: offset, + offset, + end, + } + } + + fn position(&self) -> u64 { + self.offset + } + + fn is_exhausted(&self) -> bool { + self.offset == self.end + } + + fn bytes(&mut self, out: &mut [u8]) -> Result<(), String> { + if self.offset + out.len() as u64 > self.end { + return Err("unexpected end of data".to_owned()); + } + let mut written = 0; + while written < out.len() { + let window_end = self.window_start + self.window.len() as u64; + if self.offset < self.window_start || self.offset >= window_end { + let len = (self.end - self.offset).min(READ_CHUNK as u64) as usize; + self.window.resize(len, 0); + self.source.read_at(self.offset, &mut self.window)?; + self.window_start = self.offset; + continue; + } + let at = (self.offset - self.window_start) as usize; + let take = (self.window.len() - at).min(out.len() - written); + out[written..written + take].copy_from_slice(&self.window[at..at + take]); + written += take; + self.offset += take as u64; + } + Ok(()) + } + + fn skip(&mut self, len: u64) -> Result<(), String> { + self.offset = self + .offset + .checked_add(len) + .filter(|end| *end <= self.end) + .ok_or("unexpected end of data")?; + Ok(()) + } + + fn vec(&mut self, len: usize, cap: usize) -> Result, String> { + if len > cap || len as u64 > self.end - self.offset { + return Err(format!("field of {len} bytes exceeds its bound")); + } + let mut out = vec![0_u8; len]; + self.bytes(&mut out)?; + Ok(out) + } + + fn u8(&mut self) -> Result { + let mut out = [0_u8; 1]; + self.bytes(&mut out)?; + Ok(out[0]) + } + + fn u32(&mut self) -> Result { + let mut out = [0_u8; 4]; + self.bytes(&mut out)?; + Ok(u32::from_le_bytes(out)) + } + + fn u64(&mut self) -> Result { + let mut out = [0_u8; 8]; + self.bytes(&mut out)?; + Ok(u64::from_le_bytes(out)) + } + + fn array32(&mut self) -> Result<[u8; 32], String> { + let mut out = [0_u8; 32]; + self.bytes(&mut out)?; + Ok(out) + } + + /// A `u32`-length-prefixed field. + fn field(&mut self, cap: usize) -> Result, String> { + let len = self.u32()? as usize; + self.vec(len, cap) + } + + fn string(&mut self, cap: usize) -> Result { + String::from_utf8(self.field(cap)?).map_err(|_| "legacy string is not UTF-8".to_owned()) + } +} + +// --------------------------------------------------------------------------- +// Streaming conversion: trigram + +struct TrigramFile { + rel_path: RelPath, + size: u64, + content: [u8; 32], + unindexed: bool, +} + +/// Where the sections of a checksummed legacy `cache.bin` are. +struct TrigramLayout { + files: Vec, + blob_start: u64, + blob_len: u64, + lookup_entries_start: u64, + entry_count: u64, +} + +/// Verifies both checksums of a legacy `cache.bin` in one sequential pass and +/// reads its file table. Returns the layout and the file's BLAKE3. +fn trigram_layout( + source: &mut Source, + policy: &TrigramPolicy, +) -> Result<(TrigramLayout, [u8; 32]), String> { + let source_len = source.len; + let mut header = SeqReader::new(source, 0, 16.min(source_len)); + if header.u32()? != LEGACY_CACHE_MAGIC { + return Err("not a trigram cache".to_owned()); + } + let version = header.u32()?; + if version != crate::search_index::INDEX_FORMAT_VERSION { + return Err(format!( + "trigram cache version {version} differs from this build's {}", + crate::search_index::INDEX_FORMAT_VERSION + )); + } + let postings_len_total = header.u64()?; + let postings_end = 16_u64 + .checked_add(postings_len_total) + .filter(|end| *end <= source.len && postings_len_total >= 4) + .ok_or("trigram cache postings section overruns the file")?; + let crc_at = postings_end - 4; + let lookup_crc_at = source + .len + .checked_sub(4) + .filter(|at| *at >= postings_end) + .ok_or("trigram cache lookup section is truncated")?; + let (source_hash, crcs) = + source.checksum_pass(&[(16, crc_at), (postings_end, lookup_crc_at)])?; + let mut stored = [0_u8; 4]; + source.read_at(crc_at, &mut stored)?; + if crcs[0] != u32::from_le_bytes(stored) { + return Err("trigram cache postings checksum mismatch".to_owned()); + } + source.read_at(lookup_crc_at, &mut stored)?; + if crcs[1] != u32::from_le_bytes(stored) { + return Err("trigram cache lookup checksum mismatch".to_owned()); + } + + let mut reader = SeqReader::new(source, 16, crc_at); + let mut magic = [0_u8; 8]; + reader.bytes(&mut magic)?; + if &magic != LEGACY_INDEX_MAGIC || reader.u32()? != version { + return Err("trigram cache postings header mismatch".to_owned()); + } + let head_len = reader.u32()?; + let root_len = reader.u32()?; + let ignore_len = reader.u32()?; + let max_file_size = reader.u64()?; + if max_file_size != policy.max_file_size { + return Err(format!( + "trigram cache max_file_size {max_file_size} differs from the family policy {}", + policy.max_file_size + )); + } + let file_count = reader.u32()? as usize; + if file_count > MAX_LEGACY_FILES { + return Err(format!("trigram cache lists {file_count} files")); + } + reader.skip(u64::from(head_len))?; + reader.skip(u64::from(root_len))?; + reader.skip(u64::from(ignore_len))?; + let mut files = Vec::with_capacity(file_count.min((crc_at / 57) as usize)); + let mut seen = HashSet::new(); + for _ in 0..file_count { + let unindexed = match reader.u8()? { + 0 => false, + 1 => true, + other => return Err(format!("trigram cache file flag {other}")), + }; + let path_len = reader.u32()? as usize; + let size = reader.u64()?; + let _secs = reader.u64()?; + let _nanos = reader.u32()?; + let content = reader.array32()?; + let path = reader.vec(path_len, MAX_PATH_BYTES)?; + let path = std::str::from_utf8(&path).map_err(|_| "trigram cache path is not UTF-8")?; + let path = Path::new(path); + if path.is_absolute() + || path + .components() + .any(|component| !matches!(component, std::path::Component::Normal(_))) + { + return Err(format!( + "trigram cache path {} is not relative", + path.display() + )); + } + let rel_path = RelPath::from_os_path(path).map_err(|error| error.to_string())?; + // A digest per path keeps the duplicate check smaller than the table. + if !seen.insert(*blake3::hash(rel_path.as_bytes()).as_bytes()) { + return Err("trigram cache lists a path twice".to_owned()); + } + files.push(TrigramFile { + rel_path, + size, + content, + unindexed, + }); + } + drop(seen); + let blob_len = reader.u64()?; + let blob_start = reader.position(); + reader.skip(blob_len)?; + if blob_len % LEGACY_POSTING_BYTES as u64 != 0 { + return Err("trigram cache posting blob has a partial record".to_owned()); + } + + let mut lookup = SeqReader::new(source, postings_end, lookup_crc_at); + lookup.bytes(&mut magic)?; + if &magic != LEGACY_LOOKUP_MAGIC || lookup.u32()? != version { + return Err("trigram cache lookup header mismatch".to_owned()); + } + let entry_count = u64::from(lookup.u32()?); + let lookup_entries_start = lookup.position(); + if entry_count + .checked_mul(LEGACY_LOOKUP_ENTRY_BYTES as u64) + .and_then(|len| lookup_entries_start.checked_add(len)) + != Some(lookup_crc_at) + { + return Err("trigram cache lookup table does not fill its section".to_owned()); + } + Ok(( + TrigramLayout { + files, + blob_start, + blob_len, + lookup_entries_start, + entry_count, + }, + source_hash, + )) +} + +/// Calls `visit(trigram, postings)` for every posting list, in lookup order, +/// with each list delivered in bounded chunks of whole postings. Checks that +/// the lookup is strictly sorted and every list lies inside the blob. +fn for_each_posting_chunk( + source: &mut Source, + layout: &TrigramLayout, + visit: &mut dyn FnMut(u32, &[u8]) -> Result<(), String>, +) -> Result<(), String> { + const ENTRIES_PER_READ: usize = READ_CHUNK / LEGACY_LOOKUP_ENTRY_BYTES; + const POSTINGS_PER_READ: usize = READ_CHUNK / LEGACY_POSTING_BYTES; + let mut entries = vec![0_u8; ENTRIES_PER_READ * LEGACY_LOOKUP_ENTRY_BYTES]; + let mut postings = vec![0_u8; POSTINGS_PER_READ * LEGACY_POSTING_BYTES]; + let mut previous = None; + let mut index = 0_u64; + while index < layout.entry_count { + let batch = (layout.entry_count - index).min(ENTRIES_PER_READ as u64) as usize; + let bytes = &mut entries[..batch * LEGACY_LOOKUP_ENTRY_BYTES]; + source.read_at( + layout.lookup_entries_start + index * LEGACY_LOOKUP_ENTRY_BYTES as u64, + bytes, + )?; + for entry in bytes.chunks_exact(LEGACY_LOOKUP_ENTRY_BYTES) { + let trigram = u32::from_le_bytes(entry[..4].try_into().expect("4")); + let offset = u64::from_le_bytes(entry[4..12].try_into().expect("8")); + let count = u64::from(u32::from_le_bytes(entry[12..16].try_into().expect("4"))); + if previous.is_some_and(|previous| previous >= trigram) { + return Err("trigram cache lookup is not strictly sorted".to_owned()); + } + previous = Some(trigram); + count + .checked_mul(LEGACY_POSTING_BYTES as u64) + .and_then(|len| offset.checked_add(len)) + .filter(|end| *end <= layout.blob_len) + .ok_or("trigram cache posting list overruns the blob")?; + let mut done = 0_u64; + while done < count { + let take = (count - done).min(POSTINGS_PER_READ as u64) as usize; + let chunk = &mut postings[..take * LEGACY_POSTING_BYTES]; + source.read_at( + layout.blob_start + offset + done * LEGACY_POSTING_BYTES as u64, + chunk, + )?; + visit(trigram, chunk)?; + done += take as u64; + } + } + index += batch as u64; + } + Ok(()) +} + +/// Converts a legacy `cache.bin` into canonical per-file trigram entries, +/// handing each to `emit`, with memory bounded by the file table plus +/// `working_set` bytes of decoded postings (or one file's postings, if a +/// single file needs more). +/// +/// The legacy postings are the search index's own posting fold of each file, +/// so inverting them gives exactly the payload `TrigramPayload::extract` +/// computes from the same bytes; no source file is read. Anything that cannot +/// be proven equal is a rejection: another index or tokenizer version, another +/// size limit, a failed checksum, a path that is not relative, or a malformed +/// table. Files without a recorded content hash are left out. +/// +/// The inversion runs in file-id ranges: one counting pass over the posting +/// blob sizes each file's postings, then one pass per range collects the +/// postings of that range's files only. +fn convert_trigram_streaming( + source: &mut Source, + policy: &TrigramPolicy, + working_set: usize, + emit: &mut dyn FnMut(BundleEntry) -> Result<(), String>, +) -> Result<[u8; 32], String> { + let (layout, source_hash) = trigram_layout(source, policy)?; + let file_count = layout.files.len(); + let mut counts = vec![0_u32; file_count]; + let mut last: Vec> = vec![None; file_count]; + for_each_posting_chunk(source, &layout, &mut |trigram, chunk| { + for posting in chunk.chunks_exact(LEGACY_POSTING_BYTES) { + let file_id = u32::from_le_bytes(posting[..4].try_into().expect("4")) as usize; + let slot = last + .get_mut(file_id) + .ok_or("trigram cache posting names an unknown file")?; + if *slot == Some(trigram) { + return Err("trigram cache lists one file twice under a trigram".to_owned()); + } + *slot = Some(trigram); + counts[file_id] += 1; + } + Ok(()) + })?; + drop(last); + source.stats.posting_passes += 1; + + let record_bytes = std::mem::size_of::() as u64; + let budget = working_set.max(1) as u64; + let mut start = 0; + while start < file_count { + let mut end = start; + let mut held = 0_u64; + while end < file_count { + let next = u64::from(counts[end]) * record_bytes; + if end > start && held + next > budget { + break; + } + held += next; + end += 1; + } + source.stats.peak_working_set = source.stats.peak_working_set.max(held); + let mut records: Vec> = counts[start..end] + .iter() + .map(|count| Vec::with_capacity(*count as usize)) + .collect(); + for_each_posting_chunk(source, &layout, &mut |trigram, chunk| { + for posting in chunk.chunks_exact(LEGACY_POSTING_BYTES) { + let file_id = u32::from_le_bytes(posting[..4].try_into().expect("4")) as usize; + if (start..end).contains(&file_id) { + records[file_id - start].push(TrigramRecord { + trigram, + next_mask: posting[4], + loc_mask: posting[5], + }); + } + } + Ok(()) + })?; + source.stats.posting_passes += 1; + for (file, records) in layout.files[start..end].iter().zip(records) { + if file.content == [0; 32] { + continue; + } + let flag = if !file.unindexed { + TrigramFlag::Indexed + } else if file.size > policy.max_file_size { + TrigramFlag::UnindexedOversize + } else { + TrigramFlag::UnindexedBinary + }; + if flag != TrigramFlag::Indexed && !records.is_empty() { + return Err("trigram cache holds postings for an unindexed file".to_owned()); + } + let content = ContentHash::from_bytes(file.content); + emit(BundleEntry { + rel_path: file.rel_path.clone(), + content, + size: file.size, + key: *TrigramKey { + content, + policy: *policy, + } + .family_key() + .as_bytes(), + payload: TrigramPayload { flag, records }.encode(), + })?; + } + start = end; + } + Ok(source_hash) +} + +// --------------------------------------------------------------------------- +// Streaming conversion: semantic + +struct SemanticHeader { + version: u8, + dimension: usize, + entry_count: u64, +} + +fn semantic_header( + reader: &mut SeqReader<'_>, + producer: &SemanticProducer, +) -> Result { + let version = reader.u8()?; + if version != super::LEGACY_SEMANTIC_V6 && version != super::LEGACY_SEMANTIC_V7 { + return Err(format!( + "semantic.bin version {version} has no established producer stamps" + )); + } + let dimension = reader.u32()? as usize; + if dimension == 0 || dimension > super::MAX_LEGACY_DIMENSION { + return Err(format!("invalid semantic dimension {dimension}")); + } + let entry_count = reader.u32()? as usize; + if entry_count > super::MAX_LEGACY_ROWS { + return Err(format!("too many semantic entries {entry_count}")); + } + let fingerprint = reader.string(MAX_FIELD_BYTES)?; + if !super::fingerprint_is_compatible(&fingerprint, &producer.model_fingerprint) { + return Err("model fingerprint or chunker version is incompatible".to_owned()); + } + Ok(SemanticHeader { + version, + dimension, + entry_count: entry_count as u64, + }) +} + +/// Reads the metadata table: each path's recorded content hash. +fn semantic_metadata(reader: &mut SeqReader<'_>) -> Result, [u8; 32]>, String> { + let count = reader.u32()? as usize; + if count > super::MAX_LEGACY_ROWS { + return Err(format!("too many semantic metadata rows {count}")); + } + let mut metadata = BTreeMap::new(); + for _ in 0..count { + let path = reader.field(MAX_PATH_BYTES)?; + let _seconds = reader.u64()?; + let nanos = reader.u32()?; + if nanos >= 1_000_000_000 { + return Err(format!("invalid semantic mtime nanos {nanos}")); + } + let _size = reader.u64()?; + let content_hash = reader.array32()?; + if metadata.insert(path, content_hash).is_some() { + return Err("duplicate semantic metadata path".to_owned()); + } + } + Ok(metadata) +} + +fn semantic_entry( + reader: &mut SeqReader<'_>, + header: &SemanticHeader, +) -> Result<(Vec, super::LegacyEntry), String> { + let path = reader.field(MAX_PATH_BYTES)?; + let name = reader.string(MAX_FIELD_BYTES)?; + let qualified_name = if header.version == super::LEGACY_SEMANTIC_V7 { + let qualified = reader.string(MAX_FIELD_BYTES)?; + (!qualified.is_empty()).then_some(qualified) + } else { + None + }; + let kind = reader.u8()?; + let start_line = reader.u32()?; + let end_line = reader.u32()?; + let exported = reader.u8()? != 0; + let snippet = reader.string(MAX_FIELD_BYTES)?; + let embed_text = reader.string(MAX_FIELD_BYTES)?; + let vector_len = header + .dimension + .checked_mul(std::mem::size_of::()) + .ok_or("semantic vector length overflow")?; + let vector = reader.vec(vector_len, MAX_FIELD_BYTES)?; + Ok(( + path, + super::LegacyEntry { + name, + qualified_name, + kind, + start_line, + end_line, + exported, + snippet, + embed_text, + vector, + }, + )) +} + +fn path_digest(path: &[u8]) -> [u8; 32] { + *blake3::hash(path).as_bytes() +} + +/// Converts a legacy `semantic.bin` into view payloads under `producer`, one +/// file's rows at a time, handing each imported file to `emit`. Returns the +/// file's BLAKE3 and the number of files left for the plane to embed. +/// +/// The decoder is read-only and strict: a version without producer stamps, +/// another model or chunker, or trailing bytes after the snapshot (appended +/// segments or a torn tail) reject the whole artifact. Nothing is repaired or +/// folded. Each file is then checked as the v2 import does +/// (`convert_semantic_file`): its current bytes must still hash to the +/// snapshot's hash and chunk into exactly the snapshot's rows. A path whose +/// rows are not contiguous in the snapshot is left for the plane. +fn convert_semantic_streaming( + source: &mut Source, + root: &Path, + producer: &SemanticProducer, + emit: &mut dyn FnMut(BundleEntry) -> Result<(), String>, +) -> Result<([u8; 32], u64), String> { + let (source_hash, _) = source.checksum_pass(&[])?; + let len = source.len; + // Pass 1: validate the whole snapshot and find paths whose rows are split. + let mut seen = HashSet::new(); + let mut split = HashSet::new(); + { + let mut reader = SeqReader::new(source, 0, len); + let header = semantic_header(&mut reader, producer)?; + semantic_metadata(&mut reader)?; + let mut current: Option<[u8; 32]> = None; + for _ in 0..header.entry_count { + let (path, _) = semantic_entry(&mut reader, &header)?; + let digest = path_digest(&path); + if current != Some(digest) { + if !seen.insert(digest) { + split.insert(digest); + } + current = Some(digest); + } + } + if !reader.is_exhausted() { + return Err( + "trailing bytes after semantic snapshot (segments or a torn tail are not repaired)" + .to_owned(), + ); + } + } + // Pass 2: convert one contiguous group of rows at a time. + let root = fs::canonicalize(root).map_err(|error| error.to_string())?; + let summary_budget = small_summary_budget(producer); + let mut reader = SeqReader::new(source, 0, len); + let header = semantic_header(&mut reader, producer)?; + let metadata = semantic_metadata(&mut reader)?; + let mut skipped = 0_u64; + let mut convert = |path: Vec, entries: Vec| -> Result<(), String> { + let digest = path_digest(&path); + let Some(content_hash) = metadata.get(&path).copied() else { + // Rows without metadata cannot prove they belong to this root. + skipped += 1; + return Ok(()); + }; + if split.contains(&digest) { + skipped += 1; + return Ok(()); + } + let file = super::LegacyFile { + path, + content_hash, + entries, + }; + match convert_semantic_file(&root, producer, summary_budget, file) { + Some(entry) => emit(entry), + None => { + skipped += 1; + Ok(()) + } + } + }; + let mut group: Option<(Vec, Vec)> = None; + for _ in 0..header.entry_count { + let (path, entry) = semantic_entry(&mut reader, &header)?; + match &mut group { + Some((current, rows)) if *current == path => rows.push(entry), + _ => { + if let Some((path, rows)) = group.take() { + convert(path, rows)?; + } + group = Some((path, vec![entry])); + } + } + } + if let Some((path, rows)) = group.take() { + convert(path, rows)?; + } + // A file the snapshot lists without rows is imported when this release + // also chunks it into no rows, as the v2 import does. + for path in metadata.keys() { + if !seen.contains(&path_digest(path)) { + convert(path.clone(), Vec::new())?; + } + } + Ok((source_hash, skipped)) +} + +// --------------------------------------------------------------------------- +// Staged bundles + +/// Streams converted entries to a staging file. The final name is derived +/// from the hash of the whole file, so a resumed attempt finds exactly the +/// bytes its ledger row recorded. +struct StagedWriter { + file: BufWriter, + temporary: PathBuf, + body: blake3::Hasher, + whole: blake3::Hasher, + count: u64, +} + +impl StagedWriter { + fn create(dir: &Path, artifact: LegacyArtifact) -> io::Result { + static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + crate::private_storage::create_dir_all(dir)?; + let temporary = dir.join(format!( + "{}.tmp.{}.{}.{}", + artifact.as_str(), + std::process::id(), + now_ms(), + SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + )); + let file = crate::private_storage::options() + .create_new(true) + .write(true) + .open(&temporary)?; + let mut writer = Self { + file: BufWriter::new(file), + temporary, + body: blake3::Hasher::new(), + whole: blake3::Hasher::new(), + count: 0, + }; + writer.write(STAGED_MAGIC)?; + writer.write(&[artifact as u8])?; + Ok(writer) + } + + fn write(&mut self, bytes: &[u8]) -> io::Result<()> { + self.body.update(bytes); + self.whole.update(bytes); + self.file.write_all(bytes) + } + + fn entry(&mut self, entry: &BundleEntry) -> io::Result<()> { + let path = entry.rel_path.as_bytes(); + let payload_len = u32::try_from(entry.payload.len()) + .ok() + .filter(|len| *len as usize <= MAX_FIELD_BYTES) + .ok_or_else(|| io::Error::other("staged payload too large"))?; + self.write(&[1])?; + self.write(&(path.len() as u32).to_le_bytes())?; + self.write(path)?; + self.write(entry.content.as_bytes())?; + self.write(&entry.size.to_le_bytes())?; + self.write(&entry.key)?; + self.write(&payload_len.to_le_bytes())?; + self.write(&entry.payload)?; + self.count += 1; + Ok(()) + } + + /// Writes the trailer, makes the file durable under its final name and + /// returns that name and the file's hash. + fn finish(mut self, dir: &Path, artifact: LegacyArtifact) -> io::Result<(PathBuf, [u8; 32])> { + let digest = *self.body.finalize().as_bytes(); + self.write(&[0])?; + let count = self.count.to_le_bytes(); + self.write(&count)?; + self.write(&digest)?; + let hash = *self.whole.finalize().as_bytes(); + let file = self.file.into_inner().map_err(|error| error.into_error())?; + file.sync_all()?; + drop(file); + let path = staged_path(dir, artifact, &hash); + crate::fs_lock::rename_over(&self.temporary, &path)?; + crate::fs_lock::sync_parent(&path); + Ok((path, hash)) + } + + fn abandon(self) { + let temporary = self.temporary.clone(); + drop(self); + let _ = fs::remove_file(temporary); + } +} + +pub(super) fn staged_path(dir: &Path, artifact: LegacyArtifact, hash: &[u8; 32]) -> PathBuf { + dir.join(format!( + "{}-{}-{}.bundle", + artifact.as_str(), + artifact.plane().as_str(), + hash_prefix(hash) + )) +} + +/// Streams the entries of a staged bundle to `visit`, then checks its +/// trailer, entry count and whole-file hash against `expected`. `visit` may +/// see entries of a bundle that later fails these checks, so a caller that +/// must not act on such entries runs a checking pass first. +fn read_staged( + path: &Path, + artifact: LegacyArtifact, + expected: &[u8; 32], + visit: &mut dyn FnMut(BundleEntry) -> Result<(), String>, +) -> Result { + struct Hashing { + inner: R, + body: blake3::Hasher, + whole: blake3::Hasher, + in_body: bool, + } + impl Hashing { + fn take(&mut self, out: &mut [u8]) -> Result<(), String> { + self.inner + .read_exact(out) + .map_err(|error| format!("staged bundle unreadable: {error}"))?; + if self.in_body { + self.body.update(out); + } + self.whole.update(out); + Ok(()) + } + fn u32(&mut self) -> Result { + let mut out = [0_u8; 4]; + self.take(&mut out)?; + Ok(u32::from_le_bytes(out)) + } + fn u64(&mut self) -> Result { + let mut out = [0_u8; 8]; + self.take(&mut out)?; + Ok(u64::from_le_bytes(out)) + } + fn array32(&mut self) -> Result<[u8; 32], String> { + let mut out = [0_u8; 32]; + self.take(&mut out)?; + Ok(out) + } + fn vec(&mut self, len: usize, cap: usize) -> Result, String> { + if len > cap { + return Err("staged bundle field exceeds its bound".to_owned()); + } + let mut out = vec![0_u8; len]; + self.take(&mut out)?; + Ok(out) + } + } + let file = + fs::File::open(path).map_err(|error| format!("staged bundle unreadable: {error}"))?; + let mut reader = Hashing { + inner: BufReader::with_capacity(READ_CHUNK, file), + body: blake3::Hasher::new(), + whole: blake3::Hasher::new(), + in_body: true, + }; + let mut magic = [0_u8; 9]; + reader.take(&mut magic)?; + if &magic[..8] != STAGED_MAGIC || magic[8] != artifact as u8 { + return Err("staged bundle has another format or artifact".to_owned()); + } + let mut count = 0_u64; + loop { + let mut marker = [0_u8; 1]; + // The end marker is the first trailer byte: it is not part of the + // body digest the writer recorded. + reader.in_body = false; + reader.take(&mut marker)?; + match marker[0] { + 0 => break, + 1 => { + reader.body.update(&marker); + reader.in_body = true; + } + other => return Err(format!("staged bundle marker {other}")), + } + let path_len = reader.u32()? as usize; + let rel_path = RelPath::new(reader.vec(path_len, MAX_PATH_BYTES)?) + .map_err(|error| error.to_string())?; + let content = ContentHash::from_bytes(reader.array32()?); + let size = reader.u64()?; + let key = reader.array32()?; + let payload_len = reader.u32()? as usize; + let payload = reader.vec(payload_len, MAX_FIELD_BYTES)?; + count += 1; + visit(BundleEntry { + rel_path, + content, + size, + key, + payload, + })?; + } + let body_digest = *reader.body.finalize().as_bytes(); + let recorded_count = reader.u64()?; + let recorded_digest = reader.array32()?; + if recorded_count != count || recorded_digest != body_digest { + return Err("staged bundle trailer does not match its entries".to_owned()); + } + let mut extra = [0_u8; 1]; + if reader + .inner + .read(&mut extra) + .map_err(|error| error.to_string())? + != 0 + { + return Err("staged bundle has trailing bytes".to_owned()); + } + if reader.whole.finalize().as_bytes() != expected { + return Err("staged bundle does not match its recorded hash".to_owned()); + } + Ok(count) +} + +/// The result of converting one legacy artifact to a staged bundle. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Staged { + pub path: PathBuf, + pub hash: [u8; 32], + pub source_hash: [u8; 32], + pub source_len: u64, + pub entries: u64, + /// Files the artifact lists that were left for the plane to build. + pub skipped: u64, + pub stats: ReadStats, +} + +/// What a staging attempt needs besides the artifact. +pub(super) struct StageInput<'a> { + pub storage: &'a Path, + pub legacy_key: &'a str, + pub root: &'a Path, + pub dir: &'a Path, + pub trigram: &'a TrigramPolicy, + pub semantic: Option<&'a SemanticProducer>, + pub working_set: usize, +} + +/// Converts `artifact` into a durable staged bundle. `Ok(Err(reason))` is a +/// rejection of the artifact: absent, incompatible, malformed, or changed +/// while it was read. `Err` is a failure to write the staging file. +pub(super) fn stage( + artifact: LegacyArtifact, + input: &StageInput<'_>, +) -> V3Result> { + let mut source = match Source::open(&artifact.source_path(input.storage, input.legacy_key)) { + Ok(source) => source, + Err(reason) => return Ok(Err(reason)), + }; + let mut writer = StagedWriter::create(input.dir, artifact)?; + let mut write_error = None; + let mut emit = |entry: BundleEntry| -> Result<(), String> { + writer.entry(&entry).map_err(|error| { + let message = error.to_string(); + write_error = Some(error); + message + }) + }; + let converted = match artifact { + LegacyArtifact::Trigram => { + convert_trigram_streaming(&mut source, input.trigram, input.working_set, &mut emit) + .map(|hash| (hash, 0)) + } + LegacyArtifact::Semantic => match input.semantic { + Some(producer) => { + convert_semantic_streaming(&mut source, input.root, producer, &mut emit) + } + None => Err("the request carries no semantic producer".to_owned()), + }, + LegacyArtifact::Symbols => Err("SYMBOLS_PENDING".to_owned()), + }; + if let Some(error) = write_error { + writer.abandon(); + return Err(error.into()); + } + let (source_hash, skipped) = match converted { + Ok(converted) => converted, + Err(reason) => { + writer.abandon(); + return Ok(Err(reason)); + } + }; + if !source.unchanged() { + writer.abandon(); + return Ok(Err( + "the legacy artifact changed during the import".to_owned() + )); + } + let entries = writer.count; + let (path, hash) = writer.finish(input.dir, artifact)?; + Ok(Ok(Staged { + path, + hash, + source_hash, + source_len: source.len, + entries, + skipped, + stats: source.stats, + })) +} + +/// Re-reads a staged bundle and re-derives what can be derived from each +/// entry: a trigram entry's key from its content and the policy, and that +/// every payload decodes for its plane and producer. +pub(super) fn check_staged( + artifact: LegacyArtifact, + path: &Path, + hash: &[u8; 32], + input: &StageInput<'_>, +) -> Result { + read_staged(path, artifact, hash, &mut |entry| match artifact { + LegacyArtifact::Trigram => { + TrigramPayload::decode(&entry.payload).map_err(|error| error.to_string())?; + let key = TrigramKey { + content: entry.content, + policy: *input.trigram, + } + .family_key(); + (key.as_bytes() == &entry.key) + .then_some(()) + .ok_or_else(|| "staged trigram key differs from its content".to_owned()) + } + LegacyArtifact::Semantic => { + let producer = input + .semantic + .ok_or("the request carries no semantic producer")?; + let relative = crate::views::segment_store::rel_path_to_os(&entry.rel_path) + .map_err(|error| error.to_string())?; + crate::semantic_index::SemanticVectors::decode_view_payload( + &entry.payload, + &relative, + &crate::semantic_index::ViewPayloadProducer { + chunker_version: &producer.chunker_version, + template_version: &producer.template_version, + model_fingerprint: &producer.model_fingerprint, + }, + ) + .map(|_| ()) + } + LegacyArtifact::Symbols => Err("SYMBOLS_PENDING".to_owned()), + }) +} + +/// Re-hashes a legacy source in one bounded sequential pass. +pub(super) fn source_identity( + artifact: LegacyArtifact, + storage: &Path, + legacy_key: &str, +) -> Result<([u8; 32], u64), String> { + let mut source = Source::open(&artifact.source_path(storage, legacy_key))?; + let (hash, _) = source.checksum_pass(&[])?; + Ok((hash, source.len)) +} + +// --------------------------------------------------------------------------- +// Registration into v3 + +/// What putting a staged bundle into its plane did. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct Registered { + pub inserted: u64, + /// Rows already present with an equal payload; only touched. + pub reused: u64, + /// Rows already present with another payload for the same key. The stored + /// row is kept: semantic keys address the input and producer, not the + /// model's output floats, and a stored trigram row came from the same bytes. + pub kept_existing: u64, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum RegisterOutcome { + Registered(Registered), + /// A reservation was refused for lack of storage. The plane serves what + /// v3 already holds; nothing falls back to the legacy artifact. + StorageGap(String), +} + +fn storage_gap(error: &v3::Error) -> Option { + match error { + v3::Error::Refused(reason) if STORAGE_GAP_REFUSALS.contains(reason) => { + Some(format!("storage_gap:{reason}")) + } + _ => None, + } +} + +/// Puts every entry of a checked staged bundle into the destination plane, +/// in batches of at most [`REGISTER_BATCH_BYTES`] payload bytes, each one +/// v3 publication under its own reservation. Every put is idempotent, so a +/// resumed attempt repeats the batches a crash interrupted. +pub(super) fn register_staged( + storage: &Path, + family: &str, + artifact: LegacyArtifact, + path: &Path, + hash: &[u8; 32], + after_batch: &mut dyn FnMut(u64), +) -> V3Result> { + let plane = artifact.plane(); + let schema = plane.payload_schema(); + let mut totals = Registered::default(); + let mut batch: Vec<(FamilyKey, Vec)> = Vec::new(); + let mut batch_bytes = 0_u64; + let mut batches = 0_u64; + let mut failure: Option = None; + let mut gap: Option = None; + let mut flush = |batch: &mut Vec<(FamilyKey, Vec)>, + batch_bytes: &mut u64, + totals: &mut Registered| + -> Result<(), String> { + if batch.is_empty() { + return Ok(()); + } + let request = batch_bytes.saturating_mul(2).saturating_add(1 << 20); + let result = v3::publish(storage, family, request, None, |publication| { + let mut done = Registered::default(); + for (key, payload) in batch.iter() { + match publication.put_or_touch(key, payload, schema) { + Ok(true) => done.inserted += 1, + Ok(false) => done.reused += 1, + Err(v3::Error::Refused("payload_mismatch")) => { + publication.touch(&[*key])?; + done.kept_existing += 1; + } + Err(error) => return Err(error), + } + } + Ok(done) + }); + match result { + Ok(done) => { + totals.inserted += done.inserted; + totals.reused += done.reused; + totals.kept_existing += done.kept_existing; + batches += 1; + after_batch(batches); + batch.clear(); + *batch_bytes = 0; + Ok(()) + } + Err(error) => { + let message = error.to_string(); + match storage_gap(&error) { + Some(name) => gap = Some(name), + None => failure = Some(error.into()), + } + Err(message) + } + } + }; + let read = read_staged(path, artifact, hash, &mut |entry| { + batch_bytes += entry.payload.len() as u64; + batch.push((FamilyKey::new(plane, entry.key), entry.payload)); + if batch_bytes >= REGISTER_BATCH_BYTES { + flush(&mut batch, &mut batch_bytes, &mut totals)?; + } + Ok(()) + }); + let read = read.and_then(|_| flush(&mut batch, &mut batch_bytes, &mut totals)); + if let Some(name) = gap { + return Ok(Ok(RegisterOutcome::StorageGap(name))); + } + if let Some(error) = failure { + return Err(error); + } + Ok(read.map(|_| RegisterOutcome::Registered(totals))) +} + +#[cfg(test)] +#[path = "v3_import_tests.rs"] +pub(crate) mod tests; diff --git a/crates/aft/src/migration/v3_import_tests.rs b/crates/aft/src/migration/v3_import_tests.rs new file mode 100644 index 000000000..4eccc5407 --- /dev/null +++ b/crates/aft/src/migration/v3_import_tests.rs @@ -0,0 +1,441 @@ +use std::path::{Path, PathBuf}; + +use super::*; +use crate::semantic_index::{EmbedTextCaps, SemanticIndex, SemanticIndexFingerprint}; + +pub(crate) const POLICY: TrigramPolicy = TrigramPolicy { + max_file_size: 1 << 20, +}; +pub(crate) const FAMILY: &str = "family"; +pub(crate) const KEY: &str = "family"; + +pub(crate) fn write_tree(root: &Path, files: &[(&str, &[u8])]) { + for (path, bytes) in files { + let path = root.join(path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, bytes).unwrap(); + } +} + +/// A checkout with a few source files, prose and a binary file. +pub(crate) fn checkout(dir: &Path) -> PathBuf { + let root = dir.join("checkout"); + fs::create_dir_all(&root).unwrap(); + write_tree( + &root, + &[ + ( + "src/alpha.rs", + b"pub fn alpha_total(values: &[u32]) -> u32 {\n values.iter().sum()\n}\n", + ), + ( + "src/beta.rs", + b"pub struct BetaCache {\n entries: Vec,\n}\n", + ), + ( + "notes/readme.md", + b"# Notes\n\nSome prose about the alpha total.\n", + ), + ("assets/blob.bin", &[0, 159, 146, 150, 0, 1, 2, 3, 0, 0]), + ], + ); + fs::canonicalize(&root).unwrap() +} + +/// Adds `count` generated source files, so an artifact is many times larger +/// than a small working set. +pub(crate) fn add_generated(root: &Path, count: usize) { + for index in 0..count { + let body = (0..20) + .map(|line| { + format!("pub fn generated_{index}_{line}(value: u64) -> u64 {{ value ^ {line} }}\n") + }) + .collect::(); + write_tree(root, &[(&format!("gen/file_{index}.rs"), body.as_bytes())]); + } +} + +/// Writes the previous release's trigram cache for `root` where the importer +/// reads it, and returns its bytes. +pub(crate) fn legacy_trigram(storage: &Path, root: &Path) -> Vec { + let dir = storage.join("index").join(KEY); + let mut index = crate::search_index::SearchIndex::build_with_limit(root, POLICY.max_file_size); + assert!(index.write_to_disk(&dir, None), "legacy cache written"); + fs::read(dir.join("cache.bin")).unwrap() +} + +pub(crate) fn deterministic(text: &str) -> Vec { + blake3::hash(text.as_bytes()).as_bytes()[..16] + .iter() + .map(|byte| (f32::from(*byte) - 127.5) / 127.5) + .collect() +} + +pub(crate) fn fingerprint() -> SemanticIndexFingerprint { + SemanticIndexFingerprint { + backend: "test".to_owned(), + model: "deterministic".to_owned(), + base_url: "none".to_owned(), + dimension: 16, + chunking_version: 2, + embed_text_caps: EmbedTextCaps::default(), + ..SemanticIndexFingerprint::default() + } +} + +pub(crate) fn producer() -> SemanticProducer { + let fingerprint = fingerprint(); + SemanticProducer::current(fingerprint.as_string(), fingerprint.embed_text_caps) +} + +/// Writes the previous release's `semantic.bin` for `root` where the +/// importer reads it, and returns its bytes. +pub(crate) fn legacy_semantic(storage: &Path, root: &Path) -> Vec { + let files = ["src/alpha.rs", "src/beta.rs", "notes/readme.md"] + .iter() + .map(|path| root.join(path)) + .collect::>(); + let mut index = SemanticIndex::build( + root, + &files, + &mut |texts: Vec| Ok(texts.iter().map(|text| deterministic(text)).collect()), + 64, + ) + .unwrap(); + index.set_fingerprint(fingerprint()); + let bytes = index.to_bytes(); + let path = super::super::per_checkout::legacy_semantic_path(storage, KEY); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(&path, &bytes).unwrap(); + bytes +} + +fn sorted(mut entries: Vec) -> Vec { + entries.sort_by(|left, right| left.rel_path.cmp(&right.rel_path)); + entries +} + +fn stream_trigram( + path: &Path, + working_set: usize, +) -> Result<(Vec, ReadStats), String> { + let mut source = Source::open(path)?; + let mut entries = Vec::new(); + convert_trigram_streaming(&mut source, &POLICY, working_set, &mut |entry| { + entries.push(entry); + Ok(()) + })?; + Ok((entries, source.stats)) +} + +pub(super) fn input<'a>( + storage: &'a Path, + root: &'a Path, + dir: &'a Path, + semantic: Option<&'a SemanticProducer>, +) -> StageInput<'a> { + StageInput { + storage, + legacy_key: KEY, + root, + dir, + trigram: &POLICY, + semantic, + working_set: DEFAULT_WORKING_SET, + } +} + +/// Every file below `dir`, with its bytes, for byte-identity checks. +pub(crate) fn snapshot_tree(dir: &Path) -> BTreeMap> { + let mut files = BTreeMap::new(); + let mut stack = vec![dir.to_path_buf()]; + while let Some(dir) = stack.pop() { + let Ok(entries) = fs::read_dir(&dir) else { + continue; + }; + for entry in entries { + let path = entry.unwrap().path(); + if path.is_dir() { + stack.push(path); + } else { + files.insert(path.clone(), fs::read(&path).unwrap()); + } + } + } + files +} + +/// The streaming inversion equals the whole-file v2 conversion even when its +/// working set is far smaller than the artifact, and it never holds more +/// than the budget (or one file's postings) of decoded postings at once. +#[test] +fn streaming_trigram_conversion_is_bounded_and_equals_the_whole_file_conversion() { + let dir = tempfile::tempdir().unwrap(); + let root = checkout(dir.path()); + add_generated(&root, 150); + let storage = dir.path().join("storage"); + let bytes = legacy_trigram(&storage, &root); + let whole = super::super::per_checkout::convert_trigram(&bytes, &POLICY).unwrap(); + let path = storage.join("index").join(KEY).join("cache.bin"); + + const BUDGET: usize = 2 * 1024; + let (entries, stats) = stream_trigram(&path, BUDGET).unwrap(); + assert_eq!(sorted(entries), whole.entries); + let largest_file = whole + .entries + .iter() + .map(|entry| { + TrigramPayload::decode(&entry.payload) + .unwrap() + .records + .len() as u64 + * std::mem::size_of::() as u64 + }) + .max() + .unwrap(); + assert!( + bytes.len() > 20 * BUDGET, + "the artifact ({} bytes) must dwarf the working set", + bytes.len() + ); + assert!( + stats.peak_working_set <= (BUDGET as u64).max(largest_file), + "held {} bytes of postings with a {BUDGET}-byte budget", + stats.peak_working_set + ); + assert!(stats.posting_passes > 3, "{stats:?}"); + assert!(stats.largest_read <= READ_CHUNK, "{stats:?}"); + + // With the default budget the same artifact needs one counting pass and + // one collecting pass. + let (entries, stats) = stream_trigram(&path, DEFAULT_WORKING_SET).unwrap(); + assert_eq!(sorted(entries), whole.entries); + assert_eq!(stats.posting_passes, 2); +} + +/// Anything not provably equal to a cold build is rejected, never relabelled. +#[test] +fn streaming_trigram_conversion_rejects_incompatible_and_corrupt_caches() { + let dir = tempfile::tempdir().unwrap(); + let root = checkout(dir.path()); + let storage = dir.path().join("storage"); + let bytes = legacy_trigram(&storage, &root); + let path = dir.path().join("variant.bin"); + let run = |bytes: &[u8], policy: &TrigramPolicy| { + fs::write(&path, bytes).unwrap(); + let mut source = Source::open(&path).unwrap(); + convert_trigram_streaming(&mut source, policy, DEFAULT_WORKING_SET, &mut |_| Ok(())) + .map(|_| ()) + }; + assert!(run( + &bytes, + &TrigramPolicy { + max_file_size: 4096 + } + ) + .unwrap_err() + .contains("max_file_size")); + let mut version = bytes.clone(); + version[4] ^= 0xff; + assert!(run(&version, &POLICY).unwrap_err().contains("version")); + let mut flipped = bytes.clone(); + let middle = 40 + (bytes.len() - 40) / 3; + flipped[middle] ^= 0x01; + assert!(run(&flipped, &POLICY).unwrap_err().contains("checksum")); + assert!(run(&bytes[..bytes.len() - 7], &POLICY).is_err()); + assert!(run(&bytes, &POLICY).is_ok()); +} + +#[test] +fn streaming_semantic_conversion_equals_the_v2_conversion() { + let dir = tempfile::tempdir().unwrap(); + let root = checkout(dir.path()); + let storage = dir.path().join("storage"); + let snapshot = legacy_semantic(&storage, &root); + fs::write(root.join("src/beta.rs"), b"pub struct BetaCache;\n").unwrap(); + let producer = producer(); + let (v2, v2_skipped) = + super::super::per_checkout::convert_semantic(&snapshot, &root, &producer).unwrap(); + let mut source = Source::open(&super::super::per_checkout::legacy_semantic_path( + &storage, KEY, + )) + .unwrap(); + let mut entries = Vec::new(); + let (hash, skipped) = convert_semantic_streaming(&mut source, &root, &producer, &mut |entry| { + entries.push(entry); + Ok(()) + }) + .unwrap(); + assert!(!v2.entries.is_empty()); + assert_eq!(sorted(entries), v2.entries); + assert_eq!(skipped, v2_skipped as u64); + assert_eq!(hash, *blake3::hash(&snapshot).as_bytes()); + assert!(source.stats.largest_read <= READ_CHUNK); +} + +/// The semantic decoder never repairs: appended bytes (a segment or a torn +/// tail) reject the artifact, whose bytes stay exactly as they were. +#[test] +fn semantic_trailing_bytes_are_refused_without_repair() { + let dir = tempfile::tempdir().unwrap(); + let root = checkout(dir.path()); + let storage = dir.path().join("storage"); + let mut snapshot = legacy_semantic(&storage, &root); + snapshot.extend_from_slice(b"torn tail"); + let path = super::super::per_checkout::legacy_semantic_path(&storage, KEY); + fs::write(&path, &snapshot).unwrap(); + let staging = dir.path().join("staging"); + let producer = producer(); + let rejected = stage( + LegacyArtifact::Semantic, + &input(&storage, &root, &staging, Some(&producer)), + ) + .unwrap() + .unwrap_err(); + assert!(rejected.contains("trailing bytes"), "{rejected}"); + assert_eq!(fs::read(&path).unwrap(), snapshot); + // Nothing is left in staging after a rejection. + assert!(snapshot_tree(&staging).is_empty()); +} + +/// Imported postings are the rows a cold v3 build of the same bytes would +/// store, under the same keys, and producing them reads no checkout file: +/// the import retokenizes nothing. It writes only under `blobs/v3` (and its +/// staging directory) and leaves the legacy cache byte-identical. +#[test] +fn imported_trigram_rows_equal_a_cold_build_and_read_no_checkout_file() { + let _serial = crate::blob_store::v3::serial_for_test(); + let dir = tempfile::tempdir().unwrap(); + let root = checkout(dir.path()); + let storage = dir.path().join("storage"); + let bytes = legacy_trigram(&storage, &root); + let sources = snapshot_tree(&root) + .into_iter() + .map(|(path, bytes)| (path.strip_prefix(&root).unwrap().to_path_buf(), bytes)) + .collect::>(); + // The checkout is gone: every row must come from the legacy postings. + fs::remove_dir_all(&root).unwrap(); + let before = snapshot_tree(&storage); + let staging = staging_dir(&storage, FAMILY, KEY); + let input = input(&storage, &root, &staging, None); + let staged = stage(LegacyArtifact::Trigram, &input).unwrap().unwrap(); + assert_eq!(staged.source_hash, *blake3::hash(&bytes).as_bytes()); + check_staged(LegacyArtifact::Trigram, &staged.path, &staged.hash, &input).unwrap(); + let mut entries = Vec::new(); + read_staged( + &staged.path, + LegacyArtifact::Trigram, + &staged.hash, + &mut |entry| { + entries.push(entry); + Ok(()) + }, + ) + .unwrap(); + assert!(entries.len() >= 3, "{} entries", entries.len()); + let outcome = register_staged( + &storage, + FAMILY, + LegacyArtifact::Trigram, + &staged.path, + &staged.hash, + &mut |_| {}, + ) + .unwrap() + .unwrap(); + assert_eq!( + outcome, + RegisterOutcome::Registered(Registered { + inserted: entries.len() as u64, + ..Registered::default() + }) + ); + let reader = v3::PlaneReader::open_existing(&storage, FAMILY, FamilyPlane::Trigram).unwrap(); + for entry in &entries { + let relative = crate::views::segment_store::rel_path_to_os(&entry.rel_path).unwrap(); + let source = &sources[&relative]; + let key = TrigramKey { + content: ContentHash::of(source), + policy: POLICY, + } + .family_key(); + assert_eq!(key.as_bytes(), &entry.key, "key of {}", relative.display()); + assert_eq!( + reader.get(&key).unwrap(), + Some(TrigramPayload::extract(source, &POLICY).encode()), + "row of {}", + relative.display() + ); + } + // Only v3 rows and staging were added; legacy bytes are untouched and no + // v1/v2 store, view or ledger was created. + let after = snapshot_tree(&storage); + for (path, bytes) in &before { + assert_eq!(after.get(path), Some(bytes), "{} changed", path.display()); + } + for path in after.keys().filter(|path| !before.contains_key(*path)) { + let relative = path.strip_prefix(&storage).unwrap(); + // The floor file is the v3 publication boundary's reader floor. + assert!( + relative.starts_with("blobs/v3") + || relative.starts_with("migration/v3") + || relative == Path::new("reader-floor.json"), + "import wrote {}", + relative.display() + ); + } + assert!(after + .keys() + .all(|path| !path.starts_with(storage.join("blobs/v2")))); + + // Registering again is idempotent: every row is reused. + let again = register_staged( + &storage, + FAMILY, + LegacyArtifact::Trigram, + &staged.path, + &staged.hash, + &mut |_| {}, + ) + .unwrap() + .unwrap(); + assert_eq!( + again, + RegisterOutcome::Registered(Registered { + reused: entries.len() as u64, + ..Registered::default() + }) + ); +} + +/// A reservation refused for lack of space is reported as the plane's named +/// storage gap: no row is written and nothing reads the legacy artifact in +/// its place. +#[test] +fn a_refused_reservation_is_a_named_storage_gap() { + let _serial = crate::blob_store::v3::serial_for_test(); + let dir = tempfile::tempdir().unwrap(); + let root = checkout(dir.path()); + let storage = dir.path().join("storage"); + legacy_trigram(&storage, &root); + let staging = staging_dir(&storage, FAMILY, KEY); + let input = input(&storage, &root, &staging, None); + let staged = stage(LegacyArtifact::Trigram, &input).unwrap().unwrap(); + v3::set_free_space_for_test(1 << 20, 1 << 40); + let outcome = register_staged( + &storage, + FAMILY, + LegacyArtifact::Trigram, + &staged.path, + &staged.hash, + &mut |_| {}, + ); + v3::set_free_space_for_test(u64::MAX, u64::MAX / 2); + assert_eq!( + outcome.unwrap().unwrap(), + RegisterOutcome::StorageGap("storage_gap:device_free_space_floor".to_owned()) + ); + assert!(!v3::plane_path(&storage, FAMILY, FamilyPlane::Trigram) + .unwrap() + .exists()); +} From 58b284d26e6ccfc582f533fa01f1f22e6c426981 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:21:00 +0100 Subject: [PATCH 3/8] mason: v3 import ledger per artifact, producer and plane, with crash-resume and storage-gap release Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit e4424f60f21a63dcab0860b53e7f7697210e1e97) --- crates/aft/src/migration/mod.rs | 1 + crates/aft/src/migration/v3_ledger.rs | 942 ++++++++++++++++++++ crates/aft/src/migration/v3_ledger_tests.rs | 541 +++++++++++ 3 files changed, 1484 insertions(+) create mode 100644 crates/aft/src/migration/v3_ledger.rs create mode 100644 crates/aft/src/migration/v3_ledger_tests.rs diff --git a/crates/aft/src/migration/mod.rs b/crates/aft/src/migration/mod.rs index d9f9f9acd..eaf91d6ea 100644 --- a/crates/aft/src/migration/mod.rs +++ b/crates/aft/src/migration/mod.rs @@ -16,6 +16,7 @@ pub mod per_checkout; pub mod prune_legacy; pub mod v3_import; +pub mod v3_ledger; use std::collections::BTreeMap; use std::fmt; diff --git a/crates/aft/src/migration/v3_ledger.rs b/crates/aft/src/migration/v3_ledger.rs new file mode 100644 index 000000000..45d08540f --- /dev/null +++ b/crates/aft/src/migration/v3_ledger.rs @@ -0,0 +1,942 @@ +//! The v3 import ledger (`blobs/v3//imports.sqlite`) and the driver +//! that runs one legacy key's import through it. +//! +//! One row per legacy key, artifact, producer and destination plane, so an +//! earlier completion of one artifact or producer never stands for another: +//! a semantic-only completion leaves the trigram and symbol rows to run, and a +//! changed producer starts a new row. Claims follow the v2 import ledger's +//! owner-token protocol (`per_checkout::ClaimOwner`): a live owner makes other +//! callers back off, a dead or released owner's row is taken over with +//! `attempt + 1` and resumes from its durable state. +//! +//! The file is created, and every row transition written, inside a +//! `v3::publish` call, so it exists only after the v3 reader floor is raised +//! and each write holds the storage root's reservation lock. Readers that +//! must not write under the storage root copy it first (see [`read_rows`]). + +use std::collections::{BTreeMap, HashMap}; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, MutexGuard, OnceLock, Weak}; + +use rusqlite::{params, OptionalExtension, TransactionBehavior}; + +use super::per_checkout::{now_ms, validate_legacy_key, ActiveClaim, ClaimOwner, ImportState}; +use super::v3_import::{ + check_staged, register_staged, source_identity, stage, staged_path, staging_dir, + LegacyArtifact, ReadStats, RegisterOutcome, Registered, StageInput, V3ImportError, V3Result, + DEFAULT_WORKING_SET, LEDGER_APPLICATION_ID, LEDGER_FORMAT_VERSION, +}; +use crate::blob_store::v2::{FamilyPlane, TrigramPolicy}; +use crate::blob_store::v3; +use crate::db::lifecycle::{SqliteStore, TrackedConnection}; +use crate::views::semantic::SemanticProducer; + +const LEDGER_FILE: &str = "imports.sqlite"; +const LEDGER_SCHEMA_VERSION: i64 = 1; +/// Each ledger write is one v3 publication; this covers its WAL frames and +/// the registry transaction, independent of the import's size. +const LEDGER_RESERVATION: u64 = 64 * 1024; +const STORE: SqliteStore = SqliteStore::Unmapped("v3_import_ledger"); + +const SCHEMA: &str = "CREATE TABLE format_meta (singleton INTEGER PRIMARY KEY CHECK(singleton=1), + format_version INTEGER NOT NULL, schema_version INTEGER NOT NULL, schema_text TEXT NOT NULL); +CREATE TABLE imports (legacy_key TEXT NOT NULL, artifact TEXT NOT NULL, + producer TEXT NOT NULL, plane TEXT NOT NULL, + state TEXT NOT NULL CHECK(state IN ('claimed','staged','validated','registered','done','rejected')), + attempt INTEGER NOT NULL CHECK(attempt>=1), owner_pid INTEGER, owner_start INTEGER, owner_token TEXT, + source_hash BLOB, source_len INTEGER, staged_hash BLOB, + imported INTEGER NOT NULL DEFAULT 0, skipped INTEGER NOT NULL DEFAULT 0, + reason TEXT, updated_ms INTEGER NOT NULL, completed_ms INTEGER, + PRIMARY KEY(legacy_key, artifact, producer, plane)); +"; + +/// Identifies one ledger row. +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct RowKey { + pub legacy_key: String, + pub artifact: LegacyArtifact, + /// The trigram policy fingerprint, semantic producer id or symbol + /// producer id the row's payloads were converted for. + pub producer: String, + pub plane: FamilyPlane, +} + +impl RowKey { + fn describe(&self) -> String { + format!( + "{}/{}/{}", + self.legacy_key, + self.artifact.as_str(), + self.plane.as_str() + ) + } +} + +/// One row of the v3 import ledger. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct V3ImportRow { + pub key: RowKey, + pub state: ImportState, + pub attempt: u64, + pub owner: Option, + pub source_hash: Option<[u8; 32]>, + pub source_len: Option, + pub staged_hash: Option<[u8; 32]>, + /// Rows put into (or found in) the plane. + pub imported: u64, + /// Files of the artifact left for the plane to build. + pub skipped: u64, + /// Why the row was rejected, or the storage gap that stopped it. + pub reason: Option, + pub updated_ms: u64, + pub completed_ms: Option, +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +type SharedConnection = Arc>; + +/// Every handle of a process shares one connection per ledger: a second +/// descriptor on the same database would drop the process's POSIX locks on +/// it when closed. +fn open_connections() -> &'static Mutex>>> { + static OPEN: OnceLock>>>> = + OnceLock::new(); + OPEN.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn check_format(connection: &rusqlite::Connection) -> V3Result<()> { + let application_id: i64 = + connection.query_row("PRAGMA application_id", [], |row| row.get(0))?; + if application_id != LEDGER_APPLICATION_ID { + return Err(V3ImportError::Corrupt(format!( + "import ledger application_id {application_id:#x}" + ))); + } + let meta: Option<(i64, i64, String)> = connection + .query_row( + "SELECT format_version, schema_version, schema_text FROM format_meta WHERE singleton=1", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .optional()?; + match meta { + Some((format, _, _)) if format > LEDGER_FORMAT_VERSION => { + Err(V3ImportError::NewerLedger(format)) + } + Some((_, schema, text)) if schema == LEDGER_SCHEMA_VERSION && text == SCHEMA => Ok(()), + Some(_) => Err(V3ImportError::Corrupt( + "import ledger schema differs from this build's".to_owned(), + )), + None => Err(V3ImportError::Corrupt( + "import ledger has no format row".to_owned(), + )), + } +} + +/// The ledger of one v3 family. +#[derive(Clone, Debug)] +pub struct V3Ledger { + storage: PathBuf, + family: String, + path: PathBuf, +} + +enum Claim { + Owned(V3ImportRow), + Terminal(V3ImportRow), + Busy(ClaimOwner), +} + +#[derive(Default)] +struct Advance { + source_hash: Option<[u8; 32]>, + source_len: Option, + staged_hash: Option<[u8; 32]>, + imported: Option, + skipped: Option, + reason: Option, +} + +const ROW_COLUMNS: &str = "legacy_key, artifact, producer, plane, state, attempt, owner_pid, + owner_start, owner_token, source_hash, source_len, staged_hash, imported, skipped, reason, + updated_ms, completed_ms"; + +fn decode_row(row: &rusqlite::Row<'_>) -> rusqlite::Result> { + let artifact: String = row.get(1)?; + let plane: String = row.get(3)?; + let state: String = row.get(4)?; + let hash = |index: usize| -> rusqlite::Result> { + let bytes: Option> = row.get(index)?; + Ok(bytes.and_then(|bytes| bytes.try_into().ok())) + }; + let owner = match ( + row.get::<_, Option>(6)?, + row.get::<_, Option>(7)?, + row.get::<_, Option>(8)?, + ) { + (Some(pid), Some(start), Some(token)) => Some(ClaimOwner { + pid: pid as u32, + start_time: start as u64, + token, + }), + _ => None, + }; + let (Some(artifact), Some(plane), Some(state)) = ( + LegacyArtifact::parse(&artifact), + FamilyPlane::V3_STORES + .into_iter() + .find(|candidate| candidate.as_str() == plane), + ImportState::parse(&state), + ) else { + return Ok(Err(V3ImportError::Corrupt(format!( + "unknown import row {artifact}/{plane}/{state}" + )))); + }; + Ok(Ok(V3ImportRow { + key: RowKey { + legacy_key: row.get(0)?, + artifact, + producer: row.get(2)?, + plane, + }, + state, + attempt: row.get::<_, i64>(5)? as u64, + owner, + source_hash: hash(9)?, + source_len: row.get::<_, Option>(10)?.map(|len| len as u64), + staged_hash: hash(11)?, + imported: row.get::<_, i64>(12)? as u64, + skipped: row.get::<_, i64>(13)? as u64, + reason: row.get(14)?, + updated_ms: row.get::<_, i64>(15)? as u64, + completed_ms: row.get::<_, Option>(16)?.map(|ms| ms as u64), + })) +} + +fn read_row(connection: &rusqlite::Connection, key: &RowKey) -> V3Result> { + connection + .query_row( + &format!( + "SELECT {ROW_COLUMNS} FROM imports + WHERE legacy_key=?1 AND artifact=?2 AND producer=?3 AND plane=?4" + ), + params![ + key.legacy_key, + key.artifact.as_str(), + key.producer, + key.plane.as_str() + ], + decode_row, + ) + .optional()? + .transpose() +} + +fn read_all(connection: &rusqlite::Connection) -> V3Result> { + let mut statement = connection.prepare(&format!( + "SELECT {ROW_COLUMNS} FROM imports ORDER BY legacy_key, artifact, producer, plane" + ))?; + let rows = statement + .query_map([], decode_row)? + .collect::, _>>()?; + rows.into_iter().collect() +} + +impl V3Ledger { + pub fn new(storage: &Path, family: &str) -> V3Result { + let dir = v3::family_dir(storage, family) + .map_err(|_| V3ImportError::InvalidKey(family.to_owned()))?; + Ok(Self { + storage: storage.to_path_buf(), + family: family.to_owned(), + path: dir.join(LEDGER_FILE), + }) + } + + pub fn path(&self) -> &Path { + &self.path + } + + /// The process's connection to the ledger. Without `create`, a missing + /// ledger is `None`; with it, the caller must be inside a v3 publication. + fn connection(&self, create: bool) -> V3Result> { + let mut open = lock(open_connections()); + open.retain(|_, connection| connection.strong_count() > 0); + if let Some(shared) = open.get(&self.path).and_then(Weak::upgrade) { + return Ok(Some(shared)); + } + if !create && !self.path.is_file() { + return Ok(None); + } + let exists = self.path.is_file(); + let mut connection = TrackedConnection::open(&self.path, STORE)?; + connection.busy_timeout(std::time::Duration::from_millis( + crate::blob_store::BUSY_TIMEOUT_MS, + ))?; + if !exists { + // A fresh file: select WAL, then create the schema and identity + // in one transaction, so a crash leaves either nothing usable to + // misread or the whole schema. + connection.pragma_update(None, "journal_mode", "WAL")?; + let tx = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let empty: i64 = + tx.query_row("SELECT count(*) FROM sqlite_schema", [], |row| row.get(0))?; + if empty == 0 { + tx.execute_batch(SCHEMA)?; + tx.pragma_update(None, "application_id", LEDGER_APPLICATION_ID)?; + tx.execute( + "INSERT INTO format_meta (singleton, format_version, schema_version, schema_text) + VALUES (1, ?1, ?2, ?3)", + params![LEDGER_FORMAT_VERSION, LEDGER_SCHEMA_VERSION, SCHEMA], + )?; + } + tx.commit()?; + } + connection.pragma_update(None, "synchronous", "FULL")?; + check_format(&connection)?; + let shared = Arc::new(Mutex::new(connection)); + open.insert(self.path.clone(), Arc::downgrade(&shared)); + Ok(Some(shared)) + } + + /// Runs `body` in one IMMEDIATE ledger transaction inside a v3 + /// publication: the floor is raised before the ledger can be created and + /// the write holds the reservation lock. + fn write( + &self, + body: impl FnOnce(&rusqlite::Transaction<'_>) -> V3Result, + ) -> V3Result { + let mut outcome = None; + v3::publish( + &self.storage, + &self.family, + LEDGER_RESERVATION, + None, + |_publication| { + outcome = Some((|| { + let shared = self.connection(true)?.expect("created ledger"); + let mut connection = lock(&shared); + let tx = + connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let result = body(&tx)?; + tx.commit()?; + Ok(result) + })()); + Ok(()) + }, + )?; + outcome.expect("publication ran its body") + } + + /// Every row, read through this process's connection when it has one, + /// otherwise from a private copy (see [`read_rows`]). + pub fn rows(&self) -> V3Result> { + Ok(read_rows(&self.storage, &self.family)?.unwrap_or_default()) + } + + pub fn row(&self, key: &RowKey) -> V3Result> { + Ok(self.rows()?.into_iter().find(|row| &row.key == key)) + } + + fn claim(&self, key: &RowKey, owner: &ClaimOwner) -> V3Result { + self.write(|tx| { + let now = now_ms() as i64; + Ok(match read_row(tx, key)? { + None => { + tx.execute( + "INSERT INTO imports (legacy_key, artifact, producer, plane, state, attempt, + owner_pid, owner_start, owner_token, updated_ms) + VALUES (?1, ?2, ?3, ?4, 'claimed', 1, ?5, ?6, ?7, ?8)", + params![ + key.legacy_key, + key.artifact.as_str(), + key.producer, + key.plane.as_str(), + owner.pid, + owner.start_time as i64, + owner.token, + now + ], + )?; + Claim::Owned(read_row(tx, key)?.expect("inserted row")) + } + Some(row) if row.state.is_terminal() => Claim::Terminal(row), + Some(V3ImportRow { + owner: Some(holder), + .. + }) if holder.is_live() => Claim::Busy(holder), + Some(_) => { + // The previous owner is gone or released the row: resume + // where it stopped. Its state is durable, so resuming + // repeats at most the one step that was in flight. + tx.execute( + "UPDATE imports SET attempt = attempt + 1, owner_pid = ?5, + owner_start = ?6, owner_token = ?7, updated_ms = ?8 + WHERE legacy_key=?1 AND artifact=?2 AND producer=?3 AND plane=?4", + params![ + key.legacy_key, + key.artifact.as_str(), + key.producer, + key.plane.as_str(), + owner.pid, + owner.start_time as i64, + owner.token, + now + ], + )?; + Claim::Owned(read_row(tx, key)?.expect("claimed row")) + } + }) + }) + } + + /// Moves `key` from `from` to `to` if `owner` still holds it there. A + /// terminal state releases the owner and stamps completion. + fn advance( + &self, + key: &RowKey, + owner: &ClaimOwner, + from: ImportState, + to: ImportState, + update: Advance, + ) -> V3Result { + self.write(|tx| { + let now = now_ms() as i64; + let changed = tx.execute( + "UPDATE imports SET state = ?7, updated_ms = ?8, completed_ms = ?9, + source_hash = COALESCE(?10, source_hash), + source_len = COALESCE(?11, source_len), + staged_hash = COALESCE(?12, staged_hash), + imported = COALESCE(?13, imported), + skipped = COALESCE(?14, skipped), + reason = COALESCE(?15, reason), + owner_pid = CASE WHEN ?16 THEN NULL ELSE owner_pid END, + owner_start = CASE WHEN ?16 THEN NULL ELSE owner_start END, + owner_token = CASE WHEN ?16 THEN NULL ELSE owner_token END + WHERE legacy_key=?1 AND artifact=?2 AND producer=?3 AND plane=?4 + AND state = ?5 AND owner_token = ?6", + params![ + key.legacy_key, + key.artifact.as_str(), + key.producer, + key.plane.as_str(), + from.as_str(), + owner.token, + to.as_str(), + now, + to.is_terminal().then_some(now), + update.source_hash.as_ref().map(|hash| hash.to_vec()), + update.source_len.map(|len| len as i64), + update.staged_hash.as_ref().map(|hash| hash.to_vec()), + update.imported.map(|count| count as i64), + update.skipped.map(|count| count as i64), + update.reason, + to.is_terminal(), + ], + )?; + if changed != 1 { + return Err(V3ImportError::LostClaim(key.describe())); + } + Ok(read_row(tx, key)?.expect("advanced row")) + }) + } + + /// Releases `owner`'s claim without changing the row's state, recording + /// why it stopped. The next run takes the row over and resumes it. + fn release(&self, key: &RowKey, owner: &ClaimOwner, reason: &str) -> V3Result { + self.write(|tx| { + let changed = tx.execute( + "UPDATE imports SET owner_pid = NULL, owner_start = NULL, owner_token = NULL, + reason = ?6, updated_ms = ?7 + WHERE legacy_key=?1 AND artifact=?2 AND producer=?3 AND plane=?4 + AND owner_token = ?5", + params![ + key.legacy_key, + key.artifact.as_str(), + key.producer, + key.plane.as_str(), + owner.token, + reason, + now_ms() as i64 + ], + )?; + if changed != 1 { + return Err(V3ImportError::LostClaim(key.describe())); + } + Ok(read_row(tx, key)?.expect("released row")) + }) + } +} + +/// Reads every row of a family's ledger without writing a byte under the +/// storage root. A connection this process already holds is used as is. +/// Otherwise the database and its WAL are copied to a private temporary +/// directory and read there: an ordinary open would recover and checkpoint +/// a WAL a killed importer left behind. A copy taken while another process +/// writes can only be older or unreadable, and rows only move towards +/// completion, so neither makes an import look more complete than it is. +pub fn read_rows(storage: &Path, family: &str) -> V3Result>> { + let ledger = V3Ledger::new(storage, family)?; + if let Some(shared) = lock(open_connections()) + .get(&ledger.path) + .and_then(Weak::upgrade) + { + return read_all(&lock(&shared)).map(Some); + } + if !ledger.path.is_file() { + return Ok(None); + } + static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let copy_dir = std::env::temp_dir().join(format!( + "aft-v3-ledger-read-{}-{}", + std::process::id(), + SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + )); + let result = (|| { + crate::private_storage::create_dir_all(©_dir)?; + let copy = copy_dir.join(LEDGER_FILE); + crate::private_storage::copy(&ledger.path, ©)?; + let wal = ledger.path.with_file_name(format!("{LEDGER_FILE}-wal")); + match crate::private_storage::copy(&wal, copy.with_file_name(format!("{LEDGER_FILE}-wal"))) + { + Ok(_) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + let connection = TrackedConnection::open(©, STORE)?; + check_format(&connection)?; + let rows = read_all(&connection)?; + drop(connection); + Ok(Some(rows)) + })(); + let _ = fs::remove_dir_all(©_dir); + result +} + +// --------------------------------------------------------------------------- +// Legacy-key status for retention and pruning + +/// How far a legacy key's v3 import got, read without writing. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct LegacyImportStatus { + pub rows: Vec, + /// Some row has a claim whose owner is running: that process may be + /// reading the legacy artifact right now. + pub live_claim: bool, + /// Some row is not terminal yet. + pub incomplete: bool, + /// When the last row became terminal, once every row is. + pub completed_ms: Option, +} + +/// The v3 import status of `legacy_key`. The previous release named a legacy +/// set with the same key as its family, so the family ledger of that name is +/// the one that can hold the key's rows. `None` when no v3 import was ever +/// recorded for it. +pub fn legacy_key_status(storage: &Path, legacy_key: &str) -> V3Result> { + if validate_legacy_key(legacy_key).is_err() + || crate::blob_store::v2::validate_family(legacy_key).is_err() + { + return Ok(None); + } + let Some(rows) = read_rows(storage, legacy_key)? else { + return Ok(None); + }; + let rows = rows + .into_iter() + .filter(|row| row.key.legacy_key == legacy_key) + .collect::>(); + if rows.is_empty() { + return Ok(None); + } + let live_claim = rows + .iter() + .any(|row| row.owner.as_ref().is_some_and(ClaimOwner::is_live)); + let incomplete = rows.iter().any(|row| !row.state.is_terminal()); + let completed_ms = (!incomplete) + .then(|| rows.iter().filter_map(|row| row.completed_ms).max()) + .flatten(); + Ok(Some(LegacyImportStatus { + rows, + live_claim, + incomplete, + completed_ms, + })) +} + +// --------------------------------------------------------------------------- +// The driver + +/// What one caller asks the v3 import to produce. The trigram policy is not +/// optional: every request imports the search postings, so no request can +/// complete only the semantic plane and leave the trigram plane unasked. +#[derive(Clone, Debug)] +pub struct V3ImportRequest { + /// The storage root holding both the legacy artifacts and `blobs/v3`. + pub storage: PathBuf, + pub family: String, + /// The key of the legacy set (`index/`, `semantic/`, …). + pub legacy_key: String, + /// The checkout the semantic import verifies its files against. + pub root: PathBuf, + pub trigram: TrigramPolicy, + /// The semantic producer, when semantic indexing is configured. + pub semantic: Option, + pub symbols: bool, + /// Working-set budget of the streaming trigram inversion, in bytes. + pub working_set: usize, +} + +impl V3ImportRequest { + /// The production request: every plane the family has. + pub fn full_plane( + storage: PathBuf, + family: String, + legacy_key: String, + root: PathBuf, + trigram: TrigramPolicy, + semantic: Option, + ) -> Self { + Self { + storage, + family, + legacy_key, + root, + trigram, + semantic, + symbols: true, + working_set: DEFAULT_WORKING_SET, + } + } + + /// The ledger rows this request drives, one per artifact it imports. + pub fn rows(&self) -> Vec { + let mut rows = vec![self.row(LegacyArtifact::Trigram, self.trigram.fingerprint_hex())]; + if let Some(producer) = &self.semantic { + rows.push(self.row(LegacyArtifact::Semantic, producer.id())); + } + // SYMBOLS_ROW + rows + } + + fn row(&self, artifact: LegacyArtifact, producer: String) -> RowKey { + RowKey { + legacy_key: self.legacy_key.clone(), + artifact, + producer, + plane: artifact.plane(), + } + } + + fn stage_input<'a>(&'a self, dir: &'a Path) -> StageInput<'a> { + StageInput { + storage: &self.storage, + legacy_key: &self.legacy_key, + root: &self.root, + dir, + trigram: &self.trigram, + semantic: self.semantic.as_ref(), + working_set: self.working_set, + } + } +} + +/// A durable boundary of one artifact's import, reported after it commits. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum V3Step { + Claimed, + Staged, + Validated, + /// The `n`th registration batch is committed in the plane. + BatchRegistered(u64), + Registered, + Done, + Rejected, + /// The claim was released after a storage gap. + Released, +} + +/// Observes durable boundaries. Tests park or kill the process here. +pub trait V3ImportObserver: Sync { + fn reached(&self, artifact: LegacyArtifact, step: V3Step); +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ArtifactOutcome { + /// The legacy set has no such artifact; no row was written. + Absent, + /// The row was already terminal; nothing was done. + AlreadyComplete(ImportState), + /// This call finished the import, possibly resuming an earlier attempt. + Done { + registered: Registered, + skipped: u64, + }, + Rejected(String), + /// Another live owner holds the row. + Busy { + owner_pid: u32, + }, + /// A reservation was refused; the plane reports this named gap and + /// serves what v3 already holds. A later run resumes the row. + StorageGap(String), +} + +#[derive(Clone, Debug, Default)] +pub struct V3ImportReport { + pub outcomes: BTreeMap, + /// Durable boundaries committed by this call, in order. + pub steps: Vec<(LegacyArtifact, V3Step)>, + /// Read statistics of the artifacts this call staged. + pub stats: BTreeMap, +} + +struct Run<'a> { + request: &'a V3ImportRequest, + ledger: V3Ledger, + owner: ClaimOwner, + dir: PathBuf, + observer: Option<&'a dyn V3ImportObserver>, + report: V3ImportReport, +} + +impl Run<'_> { + fn observe(&mut self, artifact: LegacyArtifact, step: V3Step) { + self.report.steps.push((artifact, step)); + if let Some(observer) = self.observer { + observer.reached(artifact, step); + } + } + + fn staged_file(&self, row: &V3ImportRow) -> Option { + row.staged_hash + .map(|hash| staged_path(&self.dir, row.key.artifact, &hash)) + } + + fn reject(&mut self, row: &V3ImportRow, reason: String) -> V3Result { + crate::slog_info!( + "v3 legacy import {} rejected: {}", + row.key.describe(), + reason + ); + self.ledger.advance( + &row.key, + &self.owner, + row.state, + ImportState::Rejected, + Advance { + reason: Some(reason.clone()), + ..Advance::default() + }, + )?; + if let Some(path) = self.staged_file(row) { + let _ = fs::remove_file(path); + } + self.observe(row.key.artifact, V3Step::Rejected); + Ok(ArtifactOutcome::Rejected(reason)) + } + + fn drive(&mut self, mut row: V3ImportRow) -> V3Result { + let artifact = row.key.artifact; + let mut registered = Registered::default(); + loop { + match row.state { + ImportState::Claimed => { + let staged = stage(artifact, &self.request.stage_input(&self.dir))?; + let staged = match staged { + Ok(staged) => staged, + Err(reason) => return self.reject(&row, reason), + }; + self.report.stats.insert(artifact, staged.stats); + row = self.ledger.advance( + &row.key, + &self.owner, + ImportState::Claimed, + ImportState::Staged, + Advance { + source_hash: Some(staged.source_hash), + source_len: Some(staged.source_len), + staged_hash: Some(staged.hash), + skipped: Some(staged.skipped), + ..Advance::default() + }, + )?; + self.observe(artifact, V3Step::Staged); + } + ImportState::Staged => { + let Some(path) = self.staged_file(&row) else { + return self.reject(&row, "the row records no staged bundle".to_owned()); + }; + let hash = row.staged_hash.expect("staged file implies hash"); + if let Err(reason) = + check_staged(artifact, &path, &hash, &self.request.stage_input(&self.dir)) + { + return self.reject(&row, reason); + } + // An older daemon may have replaced its artifact since it + // was converted; the bundle then describes bytes that no + // longer exist and the plane is built from the checkout. + let current = + source_identity(artifact, &self.request.storage, &self.request.legacy_key); + if current.ok() != row.source_hash.zip(row.source_len) { + return self.reject( + &row, + "the legacy artifact changed during the import".to_owned(), + ); + } + row = self.ledger.advance( + &row.key, + &self.owner, + ImportState::Staged, + ImportState::Validated, + Advance::default(), + )?; + self.observe(artifact, V3Step::Validated); + } + ImportState::Validated => { + let Some(path) = self.staged_file(&row) else { + return self.reject(&row, "the row records no staged bundle".to_owned()); + }; + let hash = row.staged_hash.expect("staged file implies hash"); + let mut batches = Vec::new(); + let outcome = register_staged( + &self.request.storage, + &self.request.family, + artifact, + &path, + &hash, + &mut |batch| { + batches.push(batch); + if let Some(observer) = self.observer { + observer.reached(artifact, V3Step::BatchRegistered(batch)); + } + }, + )?; + for batch in batches { + self.report + .steps + .push((artifact, V3Step::BatchRegistered(batch))); + } + match outcome { + Err(reason) => return self.reject(&row, reason), + Ok(RegisterOutcome::StorageGap(gap)) => { + self.ledger.release(&row.key, &self.owner, &gap)?; + self.observe(artifact, V3Step::Released); + return Ok(ArtifactOutcome::StorageGap(gap)); + } + Ok(RegisterOutcome::Registered(done)) => { + registered = done.clone(); + row = self.ledger.advance( + &row.key, + &self.owner, + ImportState::Validated, + ImportState::Registered, + Advance { + imported: Some( + done.inserted + done.reused + done.kept_existing, + ), + ..Advance::default() + }, + )?; + self.observe(artifact, V3Step::Registered); + } + } + } + ImportState::Registered => { + if let Some(path) = self.staged_file(&row) { + match fs::remove_file(path) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + } + row = self.ledger.advance( + &row.key, + &self.owner, + ImportState::Registered, + ImportState::Done, + Advance::default(), + )?; + self.observe(artifact, V3Step::Done); + } + ImportState::Done => { + return Ok(ArtifactOutcome::Done { + registered, + skipped: row.skipped, + }) + } + ImportState::Rejected => { + return Ok(ArtifactOutcome::Rejected(row.reason.unwrap_or_default())) + } + } + } + } +} + +/// Runs, resumes or observes the v3 import of `request`'s legacy key. +/// +/// Call it from a background thread, never on a bind path: a first import +/// reads every legacy artifact. Artifacts the legacy set lacks write no row, +/// so a key without legacy data never creates a ledger. +pub fn run_v3_import( + request: &V3ImportRequest, + observer: Option<&dyn V3ImportObserver>, +) -> V3Result { + validate_legacy_key(&request.legacy_key) + .map_err(|_| V3ImportError::InvalidKey(request.legacy_key.clone()))?; + let owner = ClaimOwner::new(); + let _active = ActiveClaim::register(&owner); + let mut run = Run { + request, + ledger: V3Ledger::new(&request.storage, &request.family)?, + owner, + dir: staging_dir(&request.storage, &request.family, &request.legacy_key), + observer, + report: V3ImportReport::default(), + }; + for key in request.rows() { + let artifact = key.artifact; + let existing = run.ledger.row(&key)?; + if existing.is_none() + && !artifact + .source_path(&request.storage, &request.legacy_key) + .is_file() + { + run.report + .outcomes + .insert(artifact, ArtifactOutcome::Absent); + continue; + } + if let Some(row) = existing.filter(|row| row.state.is_terminal()) { + run.report + .outcomes + .insert(artifact, ArtifactOutcome::AlreadyComplete(row.state)); + continue; + } + let outcome = match run.ledger.claim(&key, &run.owner)? { + Claim::Terminal(row) => ArtifactOutcome::AlreadyComplete(row.state), + Claim::Busy(holder) => ArtifactOutcome::Busy { + owner_pid: holder.pid, + }, + Claim::Owned(row) => { + if row.attempt == 1 && row.state == ImportState::Claimed { + run.observe(artifact, V3Step::Claimed); + } + run.drive(row)? + } + }; + run.report.outcomes.insert(artifact, outcome); + } + Ok(run.report) +} + +#[cfg(test)] +#[path = "v3_ledger_tests.rs"] +mod tests; diff --git a/crates/aft/src/migration/v3_ledger_tests.rs b/crates/aft/src/migration/v3_ledger_tests.rs new file mode 100644 index 000000000..af90ad932 --- /dev/null +++ b/crates/aft/src/migration/v3_ledger_tests.rs @@ -0,0 +1,541 @@ +use std::path::Path; +use std::time::{Duration, Instant}; + +use super::*; +use crate::blob_store::v2::{ContentHash, FamilyKey, TrigramKey}; +use crate::migration::v3_import::tests::{ + checkout, deterministic, legacy_semantic, legacy_trigram, producer, snapshot_tree, FAMILY, KEY, + POLICY, +}; +use crate::views::segment_store::TrigramPayload; + +fn request(storage: &Path, root: &Path, semantic: Option) -> V3ImportRequest { + let mut request = V3ImportRequest::full_plane( + storage.to_path_buf(), + FAMILY.to_owned(), + KEY.to_owned(), + root.to_path_buf(), + POLICY, + semantic, + ); + // The symbol artifact has its own tests; these cover the two others. + request.symbols = false; + request +} + +fn outcome(report: &V3ImportReport, artifact: LegacyArtifact) -> ArtifactOutcome { + report.outcomes[&artifact].clone() +} + +fn row_of(storage: &Path, artifact: LegacyArtifact) -> V3ImportRow { + read_rows(storage, FAMILY) + .unwrap() + .unwrap() + .into_iter() + .find(|row| row.key.artifact == artifact) + .unwrap() +} + +fn trigram_keys( + root_sources: &BTreeMap>, + policy: TrigramPolicy, +) -> Vec { + root_sources + .values() + .map(|source| { + TrigramKey { + content: ContentHash::of(source), + policy, + } + .family_key() + }) + .collect() +} + +fn present(storage: &Path, plane: FamilyPlane, keys: &[FamilyKey]) -> usize { + if !v3::plane_path(storage, FAMILY, plane).unwrap().exists() { + return 0; + } + let reader = v3::PlaneReader::open_existing(storage, FAMILY, plane).unwrap(); + keys.iter() + .filter(|key| reader.contains(key).unwrap()) + .count() +} + +/// A full import puts every compatible row into v3, calling no model and +/// reading no source for the trigram plane, and records one done row per +/// artifact; a second run does nothing. Legacy bytes stay identical. +#[test] +fn a_full_import_lands_every_artifact_and_a_second_run_is_already_complete() { + let _serial = v3::serial_for_test(); + let dir = tempfile::tempdir().unwrap(); + let root = checkout(dir.path()); + let storage = dir.path().join("storage"); + legacy_trigram(&storage, &root); + legacy_semantic(&storage, &root); + let legacy = snapshot_tree(&storage); + let producer = producer(); + let report = run_v3_import(&request(&storage, &root, Some(producer.clone())), None).unwrap(); + for artifact in [LegacyArtifact::Trigram, LegacyArtifact::Semantic] { + assert!( + matches!(outcome(&report, artifact), ArtifactOutcome::Done { ref registered, .. } if registered.inserted > 0), + "{artifact:?}: {:?}", + outcome(&report, artifact) + ); + let row = row_of(&storage, artifact); + assert_eq!( + (row.state, row.attempt, row.owner), + (ImportState::Done, 1, None) + ); + } + // Every imported semantic payload is the one a cold fill stores under + // the same key; the import itself had no model to call. + let reader = v3::PlaneReader::open_existing(&storage, FAMILY, FamilyPlane::Semantic).unwrap(); + let payload_producer = crate::semantic_index::ViewPayloadProducer { + chunker_version: &producer.chunker_version, + template_version: &producer.template_version, + model_fingerprint: &producer.model_fingerprint, + }; + let mut model_calls = 0; + let mut checked = 0; + for path in ["src/alpha.rs", "src/beta.rs", "notes/readme.md"] { + let rel_path = crate::views::RelPath::new(path.as_bytes().to_vec()).unwrap(); + if !crate::views::semantic::applies_to(&rel_path) { + continue; + } + let source = fs::read(root.join(path)).unwrap(); + let chunks = + crate::semantic_index::chunk_view_file(&root, Path::new(path), &source, producer.caps) + .unwrap() + .unwrap(); + let mut embed = |texts: Vec| { + model_calls += 1; + Ok(texts.iter().map(|text| deterministic(text)).collect()) + }; + let runs = crate::semantic_index::embed_view_files(vec![chunks], &mut embed, 64).unwrap(); + assert_eq!( + reader.get(&producer.key(&source, &rel_path)).unwrap(), + Some(runs[0].encode_view_payload(&payload_producer)), + "semantic row of {path}" + ); + checked += 1; + } + assert!( + checked >= 2 && model_calls >= checked, + "{checked} {model_calls}" + ); + let again = run_v3_import(&request(&storage, &root, Some(producer)), None).unwrap(); + assert_eq!( + outcome(&again, LegacyArtifact::Trigram), + ArtifactOutcome::AlreadyComplete(ImportState::Done) + ); + assert!(again.steps.is_empty()); + let after = snapshot_tree(&storage); + for (path, bytes) in &legacy { + assert_eq!(after.get(path), Some(bytes), "{} changed", path.display()); + } + assert!(snapshot_tree(&staging_dir(&storage, FAMILY, KEY)).is_empty()); + assert!(!storage.join("migration/v2").exists()); + assert!(!storage.join("blobs/v2").exists()); +} + +/// Completion is per artifact, producer and plane: a run that completed only +/// the semantic plane (the legacy set had no postings yet) leaves the trigram +/// import to a later run instead of standing for it. +#[test] +fn a_semantic_only_completion_does_not_suppress_a_later_trigram_import() { + let _serial = v3::serial_for_test(); + let dir = tempfile::tempdir().unwrap(); + let root = checkout(dir.path()); + let storage = dir.path().join("storage"); + legacy_semantic(&storage, &root); + let request = request(&storage, &root, Some(producer())); + let first = run_v3_import(&request, None).unwrap(); + assert_eq!( + outcome(&first, LegacyArtifact::Trigram), + ArtifactOutcome::Absent + ); + assert!(matches!( + outcome(&first, LegacyArtifact::Semantic), + ArtifactOutcome::Done { .. } + )); + let status = legacy_key_status(&storage, KEY).unwrap().unwrap(); + assert!(!status.incomplete && status.completed_ms.is_some()); + + legacy_trigram(&storage, &root); + let second = run_v3_import(&request, None).unwrap(); + assert!( + matches!(outcome(&second, LegacyArtifact::Trigram), ArtifactOutcome::Done { ref registered, .. } if registered.inserted > 0), + "{:?}", + second.outcomes + ); + assert_eq!( + outcome(&second, LegacyArtifact::Semantic), + ArtifactOutcome::AlreadyComplete(ImportState::Done) + ); + let sources = snapshot_tree(&root); + assert!( + present( + &storage, + FamilyPlane::Trigram, + &trigram_keys(&sources, POLICY) + ) >= 3 + ); +} + +/// A changed producer is a new row that has to prove compatibility again: +/// a cache built under another size limit or another model is rejected, +/// nothing is stored under the new producer's keys, and the earlier +/// producer's completed row stays as it was. +#[test] +fn a_changed_producer_starts_a_new_row_and_provably_misses() { + let _serial = v3::serial_for_test(); + let dir = tempfile::tempdir().unwrap(); + let root = checkout(dir.path()); + let storage = dir.path().join("storage"); + legacy_trigram(&storage, &root); + legacy_semantic(&storage, &root); + run_v3_import(&request(&storage, &root, Some(producer())), None).unwrap(); + let sources = snapshot_tree(&root); + + let other_policy = TrigramPolicy { + max_file_size: 4096, + }; + let mut other_fingerprint = crate::migration::v3_import::tests::fingerprint(); + other_fingerprint.model = "another".to_owned(); + let other_producer = SemanticProducer::current( + other_fingerprint.as_string(), + other_fingerprint.embed_text_caps, + ); + let mut changed = request(&storage, &root, Some(other_producer.clone())); + changed.trigram = other_policy; + let report = run_v3_import(&changed, None).unwrap(); + for artifact in [LegacyArtifact::Trigram, LegacyArtifact::Semantic] { + assert!( + matches!(outcome(&report, artifact), ArtifactOutcome::Rejected(_)), + "{artifact:?}: {:?}", + outcome(&report, artifact) + ); + } + // Counted misses: no row exists under any key of the changed producers. + assert_eq!( + present( + &storage, + FamilyPlane::Trigram, + &trigram_keys(&sources, other_policy) + ), + 0 + ); + let semantic_keys = sources + .iter() + .map(|(path, source)| { + let relative = path.strip_prefix(&root).unwrap(); + other_producer.key( + source, + &crate::views::RelPath::from_os_path(relative).unwrap(), + ) + }) + .collect::>(); + assert_eq!(present(&storage, FamilyPlane::Semantic, &semantic_keys), 0); + // The original producer's rows are still the ones the first run stored. + assert!( + present( + &storage, + FamilyPlane::Trigram, + &trigram_keys(&sources, POLICY) + ) >= 3 + ); + let rows = read_rows(&storage, FAMILY).unwrap().unwrap(); + assert_eq!(rows.len(), 4, "{rows:#?}"); + assert_eq!( + rows.iter() + .filter(|row| row.state == ImportState::Done) + .count(), + 2 + ); +} + +/// A refused reservation is the plane's named storage gap. The row keeps its +/// state with its claim released and the gap recorded, nothing reads the +/// legacy artifact in place of v3, and a later run with room resumes it. +#[test] +fn a_storage_gap_releases_the_row_and_a_later_run_resumes_it() { + let _serial = v3::serial_for_test(); + let dir = tempfile::tempdir().unwrap(); + let root = checkout(dir.path()); + let storage = dir.path().join("storage"); + legacy_trigram(&storage, &root); + // Room for a 64 KiB ledger write above the 4 GiB floor, not for a batch. + v3::set_free_space_for_test((4 << 30) + (512 << 10), 1 << 30); + let report = run_v3_import(&request(&storage, &root, None), None); + v3::set_free_space_for_test(u64::MAX, u64::MAX / 2); + let report = report.unwrap(); + assert_eq!( + outcome(&report, LegacyArtifact::Trigram), + ArtifactOutcome::StorageGap("storage_gap:device_free_space_floor".to_owned()) + ); + let row = row_of(&storage, LegacyArtifact::Trigram); + assert_eq!(row.state, ImportState::Validated); + assert_eq!(row.owner, None); + assert_eq!( + row.reason.as_deref(), + Some("storage_gap:device_free_space_floor") + ); + let sources = snapshot_tree(&root); + assert_eq!( + present( + &storage, + FamilyPlane::Trigram, + &trigram_keys(&sources, POLICY) + ), + 0 + ); + + let resumed = run_v3_import(&request(&storage, &root, None), None).unwrap(); + assert!(matches!( + outcome(&resumed, LegacyArtifact::Trigram), + ArtifactOutcome::Done { .. } + )); + let row = row_of(&storage, LegacyArtifact::Trigram); + assert_eq!((row.state, row.attempt), (ImportState::Done, 2)); + assert!( + present( + &storage, + FamilyPlane::Trigram, + &trigram_keys(&sources, POLICY) + ) >= 3 + ); +} + +/// A live claim is single flight; once its owner is gone the row is taken +/// over and finished. +#[test] +fn a_live_claim_is_single_flight_and_an_abandoned_one_is_taken_over() { + let _serial = v3::serial_for_test(); + let dir = tempfile::tempdir().unwrap(); + let root = checkout(dir.path()); + let storage = dir.path().join("storage"); + legacy_trigram(&storage, &root); + let request = request(&storage, &root, None); + let key = request.rows().remove(0); + let ledger = V3Ledger::new(&storage, FAMILY).unwrap(); + let other = ClaimOwner::new(); + let active = ActiveClaim::register(&other); + assert!(matches!( + ledger.claim(&key, &other).unwrap(), + Claim::Owned(_) + )); + let busy = run_v3_import(&request, None).unwrap(); + assert!(matches!( + outcome(&busy, LegacyArtifact::Trigram), + ArtifactOutcome::Busy { .. } + )); + let status = legacy_key_status(&storage, KEY).unwrap().unwrap(); + assert!(status.live_claim && status.incomplete); + drop(active); + let done = run_v3_import(&request, None).unwrap(); + assert!(matches!( + outcome(&done, LegacyArtifact::Trigram), + ArtifactOutcome::Done { .. } + )); + assert_eq!(row_of(&storage, LegacyArtifact::Trigram).attempt, 2); +} + +#[test] +fn a_ledger_from_a_newer_build_is_refused_by_name() { + let _serial = v3::serial_for_test(); + let dir = tempfile::tempdir().unwrap(); + let root = checkout(dir.path()); + let storage = dir.path().join("storage"); + legacy_trigram(&storage, &root); + run_v3_import(&request(&storage, &root, None), None).unwrap(); + let path = V3Ledger::new(&storage, FAMILY) + .unwrap() + .path() + .to_path_buf(); + // Close this process's shared connection before editing the file. + lock(open_connections()).clear(); + let connection = rusqlite::Connection::open(&path).unwrap(); + connection + .execute("UPDATE format_meta SET format_version = 2", []) + .unwrap(); + drop(connection); + assert!(matches!( + read_rows(&storage, FAMILY), + Err(V3ImportError::NewerLedger(2)) + )); +} + +// --------------------------------------------------------------------------- +// Crash transitions: a child process is killed with SIGKILL right after each +// durable boundary of the trigram import; the next run must finish the import +// from that state with nothing lost, duplicated or left in staging. + +const KILL_SUBJECT: &str = "AFT_V3_IMPORT_KILL_SUBJECT"; +const KILL_AT: &str = "AFT_V3_IMPORT_KILL_AT"; + +fn step_name(step: V3Step) -> String { + match step { + V3Step::BatchRegistered(batch) => format!("batch-{batch}"), + other => format!("{other:?}").to_lowercase(), + } +} + +struct Park { + at: String, + ready: PathBuf, +} + +impl V3ImportObserver for Park { + fn reached(&self, artifact: LegacyArtifact, step: V3Step) { + if artifact != LegacyArtifact::Trigram || step_name(step) != self.at { + return; + } + fs::write(&self.ready, step_name(step)).unwrap(); + let deadline = Instant::now() + Duration::from_secs(30); + while Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + panic!("kill-test parent did not send SIGKILL within 30 seconds"); + } +} + +#[cfg(unix)] +#[test] +fn v3_import_kill_child() { + let Ok(subject) = std::env::var(KILL_SUBJECT) else { + return; + }; + let at = std::env::var(KILL_AT).unwrap(); + let subject = PathBuf::from(subject); + v3::set_free_space_for_test(u64::MAX, u64::MAX / 2); + let park = Park { + at, + ready: subject.join("ready"), + }; + let root = subject.join("checkout"); + run_v3_import(&request(&subject.join("storage"), &root, None), Some(&park)).unwrap(); + panic!("the import finished without reaching the kill point"); +} + +#[cfg(unix)] +fn kill_child_at(subject: &Path, at: &str) { + use std::os::unix::process::ExitStatusExt; + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "migration::v3_ledger::tests::v3_import_kill_child", + "--nocapture", + ]) + .env(KILL_SUBJECT, subject) + .env(KILL_AT, at) + .spawn() + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(60); + while !subject.join("ready").exists() { + assert!( + child.try_wait().unwrap().is_none(), + "kill-test child exited before reaching {at}" + ); + if Instant::now() >= deadline { + child.kill().unwrap(); + panic!("kill-test child did not reach {at} within 60 seconds"); + } + std::thread::sleep(Duration::from_millis(10)); + } + child.kill().unwrap(); + assert_eq!(child.wait().unwrap().signal(), Some(9)); +} + +#[cfg(unix)] +#[test] +fn every_ledger_crash_transition_resumes_to_the_same_rows() { + let _serial = v3::serial_for_test(); + let reference = { + let dir = tempfile::tempdir().unwrap(); + let root = checkout(dir.path()); + let storage = dir.path().join("storage"); + legacy_trigram(&storage, &root); + run_v3_import(&request(&storage, &root, None), None).unwrap(); + let reader = + v3::PlaneReader::open_existing(&storage, FAMILY, FamilyPlane::Trigram).unwrap(); + let keys = trigram_keys(&snapshot_tree(&root), POLICY) + .into_iter() + .filter(|key| reader.contains(key).unwrap()) + .collect::>(); + assert!(keys.len() >= 3); + (keys, row_of(&storage, LegacyArtifact::Trigram).imported) + }; + for at in [ + "claimed", + "staged", + "validated", + "batch-1", + "registered", + "done", + ] { + let dir = tempfile::tempdir().unwrap(); + let subject = dir.path(); + let root = checkout(subject); + let storage = subject.join("storage"); + legacy_trigram(&storage, &root); + let legacy = snapshot_tree(&storage); + kill_child_at(subject, at); + let killed = row_of(&storage, LegacyArtifact::Trigram); + assert_eq!( + killed.owner.is_some(), + at != "done", + "{at}: the killed child held the row until it was done" + ); + + let report = run_v3_import(&request(&storage, &root, None), None).unwrap(); + let expected = if at == "done" { + ArtifactOutcome::AlreadyComplete(ImportState::Done) + } else { + match outcome(&report, LegacyArtifact::Trigram) { + done @ ArtifactOutcome::Done { .. } => done, + other => panic!("{at}: {other:?}"), + } + }; + assert_eq!(outcome(&report, LegacyArtifact::Trigram), expected, "{at}"); + let row = row_of(&storage, LegacyArtifact::Trigram); + assert_eq!(row.state, ImportState::Done, "{at}"); + assert_eq!(row.attempt, if at == "done" { 1 } else { 2 }, "{at}"); + assert_eq!(row.imported, reference.1, "{at}"); + assert_eq!( + present(&storage, FamilyPlane::Trigram, &reference.0), + reference.0.len(), + "{at}: every row is in v3" + ); + let reader = + v3::PlaneReader::open_existing(&storage, FAMILY, FamilyPlane::Trigram).unwrap(); + for (_, source) in snapshot_tree(&root) { + let key = TrigramKey { + content: ContentHash::of(&source), + policy: POLICY, + } + .family_key(); + if !reference.0.contains(&key) { + continue; + } + assert_eq!( + reader.get(&key).unwrap(), + Some(TrigramPayload::extract(&source, &POLICY).encode()), + "{at}" + ); + } + assert!( + snapshot_tree(&staging_dir(&storage, FAMILY, KEY)).is_empty(), + "{at}: staging left behind" + ); + let after = snapshot_tree(&storage); + for (path, bytes) in &legacy { + assert_eq!( + after.get(path), + Some(bytes), + "{at}: {} changed", + path.display() + ); + } + } +} From af803873131938517d6a016fe4e5188e9aec1213 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:30:14 +0100 Subject: [PATCH 4/8] mason: v3 symbol plane store and parse-free legacy symbol importer Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 59f5b5c4ac7f0039d05fafd8a88da29e652931fc) --- crates/aft/src/migration/mod.rs | 1 + crates/aft/src/migration/v3_import.rs | 43 ++- crates/aft/src/migration/v3_ledger.rs | 7 +- crates/aft/src/migration/v3_symbols.rs | 122 ++++++++ crates/aft/src/migration/v3_symbols_tests.rs | 204 ++++++++++++++ crates/aft/src/parser.rs | 9 + crates/aft/src/symbol_cache_disk.rs | 52 +++- crates/aft/src/views/mod.rs | 1 + crates/aft/src/views/symbol_store.rs | 280 +++++++++++++++++++ crates/aft/src/views/symbol_store_tests.rs | 186 ++++++++++++ 10 files changed, 888 insertions(+), 17 deletions(-) create mode 100644 crates/aft/src/migration/v3_symbols.rs create mode 100644 crates/aft/src/migration/v3_symbols_tests.rs create mode 100644 crates/aft/src/views/symbol_store.rs create mode 100644 crates/aft/src/views/symbol_store_tests.rs diff --git a/crates/aft/src/migration/mod.rs b/crates/aft/src/migration/mod.rs index eaf91d6ea..0e40b00d6 100644 --- a/crates/aft/src/migration/mod.rs +++ b/crates/aft/src/migration/mod.rs @@ -17,6 +17,7 @@ pub mod per_checkout; pub mod prune_legacy; pub mod v3_import; pub mod v3_ledger; +pub mod v3_symbols; use std::collections::BTreeMap; use std::fmt; diff --git a/crates/aft/src/migration/v3_import.rs b/crates/aft/src/migration/v3_import.rs index bbfa72656..43a06d831 100644 --- a/crates/aft/src/migration/v3_import.rs +++ b/crates/aft/src/migration/v3_import.rs @@ -241,7 +241,7 @@ impl SourceIdentity { /// A legacy source opened once, read only through bounded reads. It is opened /// read-only and never written, truncated or repaired. -struct Source { +pub(super) struct Source { file: fs::File, len: u64, identity: SourceIdentity, @@ -282,6 +282,14 @@ impl Source { Ok(()) } + /// A sequential reader over the whole source, through bounded reads. + pub(super) fn sequential(&mut self) -> SourceRead<'_> { + SourceRead { + source: self, + offset: 0, + } + } + /// Whether the open file still has the identity it had when opened. fn unchanged(&self) -> bool { self.file @@ -291,7 +299,10 @@ impl Source { /// One sequential pass over the whole file: its BLAKE3 and the CRC32 of /// each `[start, end)` range in `crc_ranges`. - fn checksum_pass(&mut self, crc_ranges: &[(u64, u64)]) -> Result<([u8; 32], Vec), String> { + pub(super) fn checksum_pass( + &mut self, + crc_ranges: &[(u64, u64)], + ) -> Result<([u8; 32], Vec), String> { let mut hasher = blake3::Hasher::new(); let mut crcs = vec![crc32fast::Hasher::new(); crc_ranges.len()]; let mut buffer = vec![0_u8; READ_CHUNK]; @@ -317,6 +328,28 @@ impl Source { } } +/// `Read` over a [`Source`] from its start; every read is at most +/// [`READ_CHUNK`] bytes. +pub(super) struct SourceRead<'a> { + source: &'a mut Source, + offset: u64, +} + +impl Read for SourceRead<'_> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let remaining = self.source.len - self.offset; + let len = (buf.len() as u64).min(remaining).min(READ_CHUNK as u64) as usize; + if len == 0 { + return Ok(0); + } + self.source + .read_at(self.offset, &mut buf[..len]) + .map_err(io::Error::other)?; + self.offset += len as u64; + Ok(len) + } +} + /// A little-endian cursor over `[offset, end)` of a source, refilled one /// bounded window at a time. struct SeqReader<'a> { @@ -1212,7 +1245,9 @@ pub(super) fn stage( } None => Err("the request carries no semantic producer".to_owned()), }, - LegacyArtifact::Symbols => Err("SYMBOLS_PENDING".to_owned()), + LegacyArtifact::Symbols => { + super::v3_symbols::convert_symbols_streaming(&mut source, input.root, &mut emit) + } }; if let Some(error) = write_error { writer.abandon(); @@ -1282,7 +1317,7 @@ pub(super) fn check_staged( ) .map(|_| ()) } - LegacyArtifact::Symbols => Err("SYMBOLS_PENDING".to_owned()), + LegacyArtifact::Symbols => super::v3_symbols::check_staged_entry(&entry), }) } diff --git a/crates/aft/src/migration/v3_ledger.rs b/crates/aft/src/migration/v3_ledger.rs index 45d08540f..70c4eb95b 100644 --- a/crates/aft/src/migration/v3_ledger.rs +++ b/crates/aft/src/migration/v3_ledger.rs @@ -622,7 +622,12 @@ impl V3ImportRequest { if let Some(producer) = &self.semantic { rows.push(self.row(LegacyArtifact::Semantic, producer.id())); } - // SYMBOLS_ROW + if self.symbols { + rows.push(self.row( + LegacyArtifact::Symbols, + crate::views::symbol_store::SymbolProducer::current().id(), + )); + } rows } diff --git a/crates/aft/src/migration/v3_symbols.rs b/crates/aft/src/migration/v3_symbols.rs new file mode 100644 index 000000000..2e48af9bf --- /dev/null +++ b/crates/aft/src/migration/v3_symbols.rs @@ -0,0 +1,122 @@ +//! Import of a legacy symbol cache (`symbols//symbols.bin`) into the v3 +//! symbol plane, without parsing. +//! +//! The legacy cache stores, per checkout-relative path, the content hash and +//! size of the bytes it parsed and their symbols. This importer adapts the +//! entries the disk cache reader validates (`DecodedSymbolCache`): the path +//! must stay under the current root and the file there must still be the +//! recorded bytes (same size, same hash). Each such entry becomes a row keyed +//! by content, language and the extraction schema (`views::symbol_store`), +//! so any checkout of the family with the same bytes reuses it. A cache of +//! another format or extraction schema yields nothing: a parser or schema +//! change misses rather than reusing stale symbols. +//! +//! The import reads the legacy cache once for its hash and once to convert +//! it, through bounded reads, and reads each listed source file to verify its +//! hash. It never calls the parser. + +use std::fs; +use std::io::BufReader; +use std::path::Path; + +use super::per_checkout::BundleEntry; +use super::v3_import::Source; +use crate::blob_store::v2::ContentHash; +use crate::symbols::Symbol; +use crate::views::symbol_store::{self, SymbolKey, SymbolProducer}; +use crate::views::RelPath; + +/// Source files larger than this are not read to verify a cached entry; the +/// symbol plane fills them on demand instead. +const MAX_SYMBOL_SOURCE_BYTES: u64 = 16 << 20; +const READ_CHUNK: usize = 64 * 1024; + +/// Converts a legacy `symbols.bin` into symbol-plane entries, handing each +/// to `emit`. Returns the file's BLAKE3 and the number of entries left for +/// the plane to fill (path outside the root, unknown language, source +/// changed or unreadable, undecodable symbols). +pub(super) fn convert_symbols_streaming( + source: &mut Source, + root: &Path, + emit: &mut dyn FnMut(BundleEntry) -> Result<(), String>, +) -> Result<([u8; 32], u64), String> { + let (source_hash, _) = source.checksum_pass(&[])?; + let root = fs::canonicalize(root).map_err(|error| error.to_string())?; + let producer = SymbolProducer::current(); + let mut canonical_parents = crate::search_index::ParentCanonicalizationMemo::default(); + let mut skipped = 0_u64; + let reader = BufReader::with_capacity(READ_CHUNK, source.sequential()); + crate::symbol_cache_disk::for_each_raw_entry(reader, &mut |raw| { + let Some(language) = symbol_store::language_of(&raw.relative_path) else { + skipped += 1; + return Ok(()); + }; + // The validated relative entry: it maps under the current root, and + // the file there still holds the bytes the symbols were parsed from. + let Some(path) = crate::search_index::cached_path_under_root_with_memo( + &root, + &raw.relative_path, + &mut canonical_parents, + ) else { + skipped += 1; + return Ok(()); + }; + let bytes = fs::symlink_metadata(&path) + .ok() + .filter(|meta| { + meta.is_file() && meta.len() == raw.size && meta.len() <= MAX_SYMBOL_SOURCE_BYTES + }) + .and_then(|_| fs::read(&path).ok()); + let Some(bytes) = bytes.filter(|bytes| blake3::hash(bytes) == raw.content_hash) else { + skipped += 1; + return Ok(()); + }; + let Ok(symbols) = serde_json::from_slice::>(&raw.symbols_json) else { + skipped += 1; + return Ok(()); + }; + let Ok(rel_path) = RelPath::from_os_path(&raw.relative_path) else { + skipped += 1; + return Ok(()); + }; + let content = ContentHash::of(&bytes); + let key = SymbolKey { + content, + language: language.clone(), + producer, + } + .family_key(); + emit(BundleEntry { + rel_path, + content, + size: raw.size, + key: *key.as_bytes(), + payload: symbol_store::encode_payload(&producer, &language, &symbols), + }) + })?; + Ok((source_hash, skipped)) +} + +/// Re-derives a staged symbol entry's key from its content and path and +/// checks that its payload decodes for that language and this producer. +pub(super) fn check_staged_entry(entry: &BundleEntry) -> Result<(), String> { + let relative = crate::views::segment_store::rel_path_to_os(&entry.rel_path) + .map_err(|error| error.to_string())?; + let language = + symbol_store::language_of(&relative).ok_or("staged symbol entry has no language")?; + let producer = SymbolProducer::current(); + symbol_store::decode_payload(&entry.payload, &producer, &language)?; + let key = SymbolKey { + content: entry.content, + language, + producer, + } + .family_key(); + (key.as_bytes() == &entry.key) + .then_some(()) + .ok_or_else(|| "staged symbol key differs from its content".to_owned()) +} + +#[cfg(test)] +#[path = "v3_symbols_tests.rs"] +mod tests; diff --git a/crates/aft/src/migration/v3_symbols_tests.rs b/crates/aft/src/migration/v3_symbols_tests.rs new file mode 100644 index 000000000..20f532b92 --- /dev/null +++ b/crates/aft/src/migration/v3_symbols_tests.rs @@ -0,0 +1,204 @@ +use std::path::Path; + +use super::*; +use crate::blob_store::v2::FamilyPlane; +use crate::blob_store::v3; +use crate::migration::per_checkout::ImportState; +use crate::migration::v3_import::tests::{checkout, snapshot_tree, FAMILY, KEY, POLICY}; +use crate::migration::v3_import::LegacyArtifact; +use crate::migration::v3_ledger::{read_rows, run_v3_import, ArtifactOutcome, V3ImportRequest}; +use crate::parser::work_counters::cached_parser_parses; + +/// Parses every supported file under `root` (this is the cold work an import +/// must avoid) and writes the previous release's `symbols.bin` for it. +/// Returns each relative path's bytes and symbols. +fn legacy_symbols(storage: &Path, root: &Path) -> Vec<(std::path::PathBuf, Vec, Vec)> { + let mut parser = crate::parser::FileParser::new(); + let mut cache = crate::parser::SymbolCache::new(); + cache.set_project_root(root.to_path_buf()); + let mut files = Vec::new(); + for (path, bytes) in snapshot_tree(root) { + if crate::parser::detect_language(&path).is_none() { + continue; + } + let symbols = parser.extract_symbols(&path).unwrap(); + let metadata = fs::metadata(&path).unwrap(); + cache.insert( + path.clone(), + metadata.modified().unwrap(), + metadata.len(), + blake3::hash(&bytes), + symbols.clone(), + ); + files.push(( + path.strip_prefix(root).unwrap().to_path_buf(), + bytes, + symbols, + )); + } + crate::symbol_cache_disk::write_to_disk(&cache, storage, KEY).unwrap(); + assert!(crate::symbol_cache_disk::cache_path(storage, KEY).is_file()); + files +} + +fn request(storage: &Path, root: &Path) -> V3ImportRequest { + V3ImportRequest::full_plane( + storage.to_path_buf(), + FAMILY.to_owned(), + KEY.to_owned(), + root.to_path_buf(), + POLICY, + None, + ) +} + +/// A compatible symbol cache is imported and then served by key with zero +/// tree-sitter parses; the counter is shown to count real parses. +#[test] +fn a_compatible_symbol_import_needs_no_parse_and_serves_lookups() { + let _serial = v3::serial_for_test(); + let dir = tempfile::tempdir().unwrap(); + let root = checkout(dir.path()); + let storage = dir.path().join("storage"); + let files = legacy_symbols(&storage, &root); + assert!(files.len() >= 3, "{} parsed files", files.len()); + let legacy = snapshot_tree(&storage); + + let parses = cached_parser_parses(); + let report = run_v3_import(&request(&storage, &root), None).unwrap(); + let ArtifactOutcome::Done { registered, .. } = &report.outcomes[&LegacyArtifact::Symbols] + else { + panic!("{:?}", report.outcomes); + }; + assert_eq!(registered.inserted as usize, files.len()); + for (relative, bytes, symbols) in &files { + let stored = symbol_store::lookup(&storage, FAMILY, ContentHash::of(bytes), relative) + .unwrap() + .unwrap_or_else(|| panic!("no row for {}", relative.display())); + assert_eq!(&*stored, symbols, "{}", relative.display()); + } + assert_eq!( + cached_parser_parses() - parses, + 0, + "import and lookups must not parse" + ); + // Control: the counter does count a real parse on this thread. + crate::parser::FileParser::new() + .extract_symbols(&root.join(&files[0].0)) + .unwrap(); + assert!(cached_parser_parses() > parses); + + let rows = read_rows(&storage, FAMILY).unwrap().unwrap(); + let row = rows + .iter() + .find(|row| row.key.artifact == LegacyArtifact::Symbols) + .unwrap(); + assert_eq!(row.state, ImportState::Done); + assert_eq!(row.key.plane, FamilyPlane::Symbol); + assert_eq!(row.key.producer, SymbolProducer::current().id()); + let after = snapshot_tree(&storage); + for (path, bytes) in &legacy { + assert_eq!(after.get(path), Some(bytes), "{} changed", path.display()); + } +} + +/// A cache written under another extraction schema is rejected whole, and +/// a key under another producer never matches a stored row: a parser or +/// schema change misses. +#[test] +fn a_parser_or_schema_change_misses() { + let _serial = v3::serial_for_test(); + let dir = tempfile::tempdir().unwrap(); + let root = checkout(dir.path()); + let storage = dir.path().join("storage"); + let files = legacy_symbols(&storage, &root); + let path = crate::symbol_cache_disk::cache_path(&storage, KEY); + let mut bytes = fs::read(&path).unwrap(); + let schema = crate::symbol_cache_disk::SCHEMA_VERSION; + bytes[12..16].copy_from_slice(&(schema - 1).to_le_bytes()); + fs::write(&path, &bytes).unwrap(); + let report = run_v3_import(&request(&storage, &root), None).unwrap(); + let ArtifactOutcome::Rejected(reason) = &report.outcomes[&LegacyArtifact::Symbols] else { + panic!("{:?}", report.outcomes); + }; + assert!(reason.contains("schema"), "{reason}"); + assert_eq!( + fs::read(&path).unwrap(), + bytes, + "the legacy cache is left as it was" + ); + for (relative, source, _) in &files { + assert!( + symbol_store::lookup(&storage, FAMILY, ContentHash::of(source), relative) + .unwrap() + .is_none() + ); + } + + // A compatible cache imports, but a lookup key of another producer still + // misses every row. + let other = tempfile::tempdir().unwrap(); + let storage = other.path().join("storage"); + let root = checkout(other.path()); + let files = legacy_symbols(&storage, &root); + run_v3_import(&request(&storage, &root), None).unwrap(); + let reader = v3::PlaneReader::open_existing(&storage, FAMILY, FamilyPlane::Symbol).unwrap(); + for (relative, source, _) in &files { + let language = symbol_store::language_of(relative).unwrap(); + let current = SymbolKey { + content: ContentHash::of(source), + language: language.clone(), + producer: SymbolProducer::current(), + }; + let changed = SymbolKey { + producer: SymbolProducer { + extraction_schema: schema + 1, + }, + ..current.clone() + }; + assert!(reader.contains(¤t.family_key()).unwrap()); + assert!(!reader.contains(&changed.family_key()).unwrap()); + } +} + +/// An entry whose file changed since it was parsed cannot vouch for the new +/// bytes and is left for the plane; the others are imported. +#[test] +fn a_changed_source_is_left_for_the_plane() { + let _serial = v3::serial_for_test(); + let dir = tempfile::tempdir().unwrap(); + let root = checkout(dir.path()); + let storage = dir.path().join("storage"); + let files = legacy_symbols(&storage, &root); + let changed = b"pub fn alpha_total() -> u32 { 7 }\n"; + fs::write(root.join("src/alpha.rs"), changed).unwrap(); + let report = run_v3_import(&request(&storage, &root), None).unwrap(); + let ArtifactOutcome::Done { + registered, + skipped, + } = &report.outcomes[&LegacyArtifact::Symbols] + else { + panic!("{:?}", report.outcomes); + }; + assert_eq!(*skipped, 1); + assert_eq!(registered.inserted as usize, files.len() - 1); + for (relative, source, _) in &files { + let found = symbol_store::lookup(&storage, FAMILY, ContentHash::of(source), relative) + .unwrap() + .is_some(); + assert_eq!( + found, + relative != Path::new("src/alpha.rs"), + "{}", + relative.display() + ); + } + assert!(symbol_store::lookup( + &storage, + FAMILY, + ContentHash::of(changed), + Path::new("src/alpha.rs") + ) + .unwrap() + .is_none()); +} diff --git a/crates/aft/src/parser.rs b/crates/aft/src/parser.rs index 4c773a6f8..fc810a11a 100644 --- a/crates/aft/src/parser.rs +++ b/crates/aft/src/parser.rs @@ -1009,6 +1009,13 @@ pub(crate) mod work_counters { pub(crate) static SOURCE_READS: Cell = const { Cell::new(0) }; pub(crate) static OWNED_SYMBOL_COPIES: Cell = const { Cell::new(0) }; pub(crate) static RUST_MODULE_SCANS: Cell = const { Cell::new(0) }; + /// Tree-sitter parses through this thread's retained parsers, by any + /// caller (FileParser, semantic chunking, symbol extraction). + pub(crate) static CACHED_PARSER_PARSES: Cell = const { Cell::new(0) }; + } + + pub(crate) fn cached_parser_parses() -> usize { + CACHED_PARSER_PARSES.with(Cell::get) } pub(crate) fn tree_parses() -> usize { @@ -1037,6 +1044,8 @@ pub(crate) fn parse_source_with_cached_parser( source: &str, lang: LangId, ) -> Result { + #[cfg(test)] + work_counters::CACHED_PARSER_PARSES.with(|count| count.set(count.get() + 1)); REUSABLE_PARSERS.with(|parsers| { let mut parsers = parsers.borrow_mut(); if let std::collections::hash_map::Entry::Vacant(entry) = parsers.entry(lang) { diff --git a/crates/aft/src/symbol_cache_disk.rs b/crates/aft/src/symbol_cache_disk.rs index 3c610c3a8..3a4391728 100644 --- a/crates/aft/src/symbol_cache_disk.rs +++ b/crates/aft/src/symbol_cache_disk.rs @@ -311,8 +311,41 @@ pub(crate) fn check_disk_format( } fn read_cache_file(path: &Path) -> Result { - let mut reader = BufReader::new(File::open(path).map_err(|error| error.to_string())?); + let reader = BufReader::new(File::open(path).map_err(|error| error.to_string())?); + let mut entries = Vec::new(); + for_each_raw_entry(reader, &mut |raw| { + let symbols: Vec = serde_json::from_slice(&raw.symbols_json) + .map_err(|error| format!("failed to decode cached symbols: {error}"))?; + entries.push(DiskSymbolEntry { + relative_path: raw.relative_path, + mtime: raw.mtime, + size: raw.size, + content_hash: raw.content_hash, + symbols, + }); + Ok(()) + })?; + Ok(DiskSymbolCache { entries }) +} +/// One entry of `symbols.bin` with its symbols still in the stored JSON. +pub(crate) struct RawSymbolEntry { + pub(crate) relative_path: PathBuf, + pub(crate) mtime: SystemTime, + pub(crate) size: u64, + pub(crate) content_hash: blake3::Hash, + pub(crate) symbols_json: Vec, +} + +/// Streams the entries of a `symbols.bin` from `reader` to `visit`, one at a +/// time, with the header, format, schema, bound and relative-path checks of +/// the disk cache reader. A file of another format or extraction schema is an +/// error, so a parser or schema change never yields entries. Oversized +/// entries are skipped, as the cache reader skips them. +pub(crate) fn for_each_raw_entry( + mut reader: R, + visit: &mut dyn FnMut(RawSymbolEntry) -> Result<(), String>, +) -> Result<(), String> { let mut magic = [0u8; 8]; reader .read_exact(&mut magic) @@ -345,7 +378,6 @@ fn read_cache_file(path: &Path) -> Result { } let _project_root = PathBuf::from(read_string_with_len(&mut reader, root_len)?); - let mut entries = Vec::with_capacity(entry_count); for _ in 0..entry_count { let path_len = read_u32(&mut reader)? as usize; @@ -390,23 +422,19 @@ fn read_cache_file(path: &Path) -> Result { continue; } - let mut symbol_bytes = vec![0u8; symbol_bytes_len]; + let mut symbols_json = vec![0u8; symbol_bytes_len]; reader - .read_exact(&mut symbol_bytes) + .read_exact(&mut symbols_json) .map_err(|error| format!("failed to read symbol payload: {error}"))?; - let symbols: Vec = serde_json::from_slice(&symbol_bytes) - .map_err(|error| format!("failed to decode cached symbols: {error}"))?; - - entries.push(DiskSymbolEntry { + visit(RawSymbolEntry { relative_path, mtime: system_time_from_parts(mtime_secs, mtime_nanos)?, size, content_hash, - symbols, - }); + symbols_json, + })?; } - - Ok(DiskSymbolCache { entries }) + Ok(()) } fn write_cache_file( diff --git a/crates/aft/src/views/mod.rs b/crates/aft/src/views/mod.rs index 4e5e4bfcc..542f4b659 100644 --- a/crates/aft/src/views/mod.rs +++ b/crates/aft/src/views/mod.rs @@ -49,6 +49,7 @@ pub mod readiness; pub mod registry; pub mod segment_store; pub mod snapshot; +pub mod symbol_store; // Plane and runtime modules of the per-checkout views. Each starts as a // documented empty module that its plane fills in. diff --git a/crates/aft/src/views/symbol_store.rs b/crates/aft/src/views/symbol_store.rs new file mode 100644 index 000000000..886158604 --- /dev/null +++ b/crates/aft/src/views/symbol_store.rs @@ -0,0 +1,280 @@ +//! The v3 symbol plane: parsed symbols of one file, shared by every checkout +//! of a family and keyed by what decides them, never by path. +//! +//! Storage is the ordinary v3 plane store `blobs/v3//symbol.sqlite` +//! (`FamilyPlane::Symbol`, format in docs/design/views-v3-format.md, *Import +//! ledger and symbol plane*). This module owns the plane's key, payload, +//! residency and the policy parts of its GC and retention: +//! +//! - **Key.** Content hash, language and producer (the extraction schema, +//! `symbol_cache_disk::SCHEMA_VERSION`). A parser or schema change changes +//! the producer, so its lookups miss instead of returning stale symbols. +//! - **Writes** go through `blob_store::v3::publish` only: the v3 reader floor +//! is raised first and every batch holds its own reservation. +//! - **GC.** No manifest names a symbol row, so the v3 plane-row sweep keeps +//! a row until it has gone [`SYMBOL_RETENTION_SWEEPS`] sweep epochs without +//! a put or [`touch`]. Callers that rely on a row touch it. +//! - **Retention.** Rows count in the family's v3 namespace inventory and +//! budget like every plane; nothing else deletes them. +//! - **Residency.** Decoded symbol lists are shared as `Arc>` +//! through one process-wide cache bounded by [`RESIDENT_BYTES`]; evicting +//! from it never touches stored rows. + +use std::collections::{HashMap, VecDeque}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, OnceLock}; + +use crate::blob_store::v2::{ContentHash, FamilyKey, FamilyPlane}; +use crate::blob_store::v3; +use crate::symbols::Symbol; + +pub use crate::blob_store::v3::SYMBOL_RETENTION_SWEEPS; + +/// Layout byte of a symbol payload. +const PAYLOAD_LAYOUT: u8 = 1; +const KEY_TAG: &[u8] = b"aft/views-v3/symbol/v1"; +/// Estimated bytes of decoded symbol lists kept resident per process. +pub const RESIDENT_BYTES: u64 = 64 << 20; +/// Payload bytes written by one v3 publication. +const PUT_BATCH_BYTES: u64 = 32 << 20; + +/// What produced a symbol list: the extraction schema, which is bumped on +/// any grammar, query or extractor change. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +pub struct SymbolProducer { + pub extraction_schema: u32, +} + +impl SymbolProducer { + /// This build's producer. + pub fn current() -> Self { + Self { + extraction_schema: crate::symbol_cache_disk::SCHEMA_VERSION, + } + } + + pub fn id(&self) -> String { + format!("aft-symbols/extraction-{}", self.extraction_schema) + } +} + +/// The language id stored in symbol keys and payloads: the parser's +/// language name. +pub fn language_id(language: crate::parser::LangId) -> String { + format!("{language:?}") +} + +/// The language a path's symbols are extracted with, if any. +pub fn language_of(path: &Path) -> Option { + crate::parser::detect_language(path).map(language_id) +} + +/// Everything that decides a symbol list. +#[derive(Clone, Debug, Eq, PartialEq, Hash)] +pub struct SymbolKey { + pub content: ContentHash, + pub language: String, + pub producer: SymbolProducer, +} + +impl SymbolKey { + pub fn family_key(&self) -> FamilyKey { + let mut hasher = blake3::Hasher::new(); + hasher.update(KEY_TAG); + for field in [ + &self.content.as_bytes()[..], + self.language.as_bytes(), + self.producer.id().as_bytes(), + ] { + hasher.update(&(field.len() as u64).to_be_bytes()); + hasher.update(field); + } + FamilyKey::new(FamilyPlane::Symbol, *hasher.finalize().as_bytes()) + } +} + +fn push_field(output: &mut Vec, bytes: &[u8]) { + output.extend_from_slice(&(bytes.len() as u32).to_le_bytes()); + output.extend_from_slice(bytes); +} + +/// The canonical payload of `symbols`: the layout byte, then the producer +/// id, the language id and the symbols as JSON, each length-prefixed. +pub fn encode_payload(producer: &SymbolProducer, language: &str, symbols: &[Symbol]) -> Vec { + let json = serde_json::to_vec(symbols).expect("symbols serialize"); + let mut payload = vec![PAYLOAD_LAYOUT]; + push_field(&mut payload, producer.id().as_bytes()); + push_field(&mut payload, language.as_bytes()); + push_field(&mut payload, &json); + payload +} + +/// Decodes a payload stored for `producer` and `language`; a payload of any +/// other layout, producer or language is refused, never reinterpreted. +pub fn decode_payload( + payload: &[u8], + producer: &SymbolProducer, + language: &str, +) -> Result, String> { + let mut rest = payload + .strip_prefix(&[PAYLOAD_LAYOUT]) + .ok_or("symbol payload has another layout")?; + let mut field = || -> Result<&[u8], String> { + if rest.len() < 4 { + return Err("symbol payload is truncated".to_owned()); + } + let len = u32::from_le_bytes(rest[..4].try_into().expect("4")) as usize; + let value = rest.get(4..4 + len).ok_or("symbol payload is truncated")?; + rest = &rest[4 + len..]; + Ok(value) + }; + if field()? != producer.id().as_bytes() { + return Err("symbol payload has another producer".to_owned()); + } + if field()? != language.as_bytes() { + return Err("symbol payload has another language".to_owned()); + } + let json = field()?; + if !rest.is_empty() { + return Err("symbol payload has trailing bytes".to_owned()); + } + serde_json::from_slice(json).map_err(|error| format!("symbol payload JSON: {error}")) +} + +/// Stores `(key, payload)` rows, in batches of at most 32 MiB of payload, +/// each one v3 publication under its own reservation. Returns how many rows +/// were inserted (the rest were equal rows, only touched). +pub fn put(storage: &Path, family: &str, rows: &[(FamilyKey, Vec)]) -> Result { + let schema = FamilyPlane::Symbol.payload_schema(); + let mut inserted = 0; + let mut start = 0; + while start < rows.len() { + let mut end = start; + let mut bytes = 0_u64; + while end < rows.len() && (end == start || bytes < PUT_BATCH_BYTES) { + bytes += rows[end].1.len() as u64; + end += 1; + } + let request = bytes.saturating_mul(2).saturating_add(1 << 20); + inserted += v3::publish(storage, family, request, None, |publication| { + let mut inserted = 0; + for (key, payload) in &rows[start..end] { + if publication.put_or_touch(key, payload, schema)? { + inserted += 1; + } + } + Ok(inserted) + })?; + start = end; + } + Ok(inserted) +} + +/// Restamps existing rows with the current sweep epoch, so the plane-row +/// sweep keeps them another [`SYMBOL_RETENTION_SWEEPS`] epochs. Returns the +/// keys that have no row. +pub fn touch( + storage: &Path, + family: &str, + keys: &[FamilyKey], +) -> Result, v3::Error> { + if keys.is_empty() { + return Ok(Vec::new()); + } + let request = (keys.len() as u64).saturating_mul(64 * 1024).max(1 << 20); + v3::publish(storage, family, request, None, |publication| { + publication.touch(keys) + }) +} + +/// Resident lists are keyed by the plane store they were read from as well +/// as the key, so a lookup in one storage root or family never answers from +/// rows another one stored. +type ResidentKey = (PathBuf, [u8; 32]); + +struct Residency { + entries: HashMap>, u64)>, + order: VecDeque, + bytes: u64, +} + +fn residency() -> &'static Mutex { + static RESIDENT: OnceLock> = OnceLock::new(); + RESIDENT.get_or_init(|| { + Mutex::new(Residency { + entries: HashMap::new(), + order: VecDeque::new(), + bytes: 0, + }) + }) +} + +fn resident(key: &ResidentKey) -> Option>> { + residency() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .entries + .get(key) + .map(|(symbols, _)| Arc::clone(symbols)) +} + +fn make_resident(key: ResidentKey, symbols: Arc>, bytes: u64) { + let mut cache = residency() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if cache.entries.contains_key(&key) || bytes > RESIDENT_BYTES { + return; + } + while cache.bytes + bytes > RESIDENT_BYTES { + let Some(oldest) = cache.order.pop_front() else { + break; + }; + if let Some((_, size)) = cache.entries.remove(&oldest) { + cache.bytes -= size; + } + } + cache.entries.insert(key.clone(), (symbols, bytes)); + cache.order.push_back(key); + cache.bytes += bytes; +} + +/// The stored symbols of the file at `path` whose bytes hash to `content`, +/// under this build's producer, or `None` when the plane has no such row. +/// Never parses: a miss is the caller's to fill. +pub fn lookup( + storage: &Path, + family: &str, + content: ContentHash, + path: &Path, +) -> Result>>, String> { + let Some(language) = language_of(path) else { + return Ok(None); + }; + let producer = SymbolProducer::current(); + let key = SymbolKey { + content, + language: language.clone(), + producer, + } + .family_key(); + let plane = v3::plane_path(storage, family, FamilyPlane::Symbol).map_err(|e| e.to_string())?; + let resident_key = (plane.clone(), *key.as_bytes()); + if let Some(symbols) = resident(&resident_key) { + return Ok(Some(symbols)); + } + if !plane.exists() { + return Ok(None); + } + let reader = v3::PlaneReader::open_existing(storage, family, FamilyPlane::Symbol) + .map_err(|error| error.to_string())?; + let Some(payload) = reader.get(&key).map_err(|error| error.to_string())? else { + return Ok(None); + }; + let symbols = Arc::new(decode_payload(&payload, &producer, &language)?); + make_resident(resident_key, Arc::clone(&symbols), payload.len() as u64); + Ok(Some(symbols)) +} + +#[cfg(test)] +#[path = "symbol_store_tests.rs"] +mod tests; diff --git a/crates/aft/src/views/symbol_store_tests.rs b/crates/aft/src/views/symbol_store_tests.rs new file mode 100644 index 000000000..ae8beef02 --- /dev/null +++ b/crates/aft/src/views/symbol_store_tests.rs @@ -0,0 +1,186 @@ +use std::collections::HashSet; + +use super::*; +use crate::symbols::{Range, SymbolKind}; + +const FAMILY: &str = "family"; + +fn symbol(name: &str) -> Symbol { + Symbol { + name: name.to_owned(), + kind: SymbolKind::Function, + range: Range { + start_line: 1, + start_col: 0, + end_line: 3, + end_col: 1, + }, + signature: Some(format!("fn {name}()")), + scope_chain: Vec::new(), + exported: true, + parent: None, + } +} + +fn key(content: &[u8]) -> SymbolKey { + SymbolKey { + content: ContentHash::of(content), + language: "Rust".to_owned(), + producer: SymbolProducer::current(), + } +} + +#[test] +fn a_payload_decodes_only_for_its_producer_and_language() { + let producer = SymbolProducer::current(); + let symbols = vec![symbol("alpha"), symbol("beta")]; + let payload = encode_payload(&producer, "Rust", &symbols); + assert_eq!( + decode_payload(&payload, &producer, "Rust").unwrap(), + symbols + ); + let other = SymbolProducer { + extraction_schema: producer.extraction_schema + 1, + }; + assert!(decode_payload(&payload, &other, "Rust") + .unwrap_err() + .contains("producer")); + assert!(decode_payload(&payload, &producer, "Go") + .unwrap_err() + .contains("language")); + assert!(decode_payload(&payload[..payload.len() - 1], &producer, "Rust").is_err()); + // Every input of the key separates keys. + let base = key(b"fn alpha() {}"); + assert_ne!(base.family_key(), key(b"fn beta() {}").family_key()); + assert_ne!( + base.family_key(), + SymbolKey { + language: "Go".to_owned(), + ..base.clone() + } + .family_key() + ); + assert_ne!( + base.family_key(), + SymbolKey { + producer: other, + ..base.clone() + } + .family_key() + ); + assert_eq!(base.family_key().plane(), FamilyPlane::Symbol); +} + +fn sweep(storage: &Path, epoch: i64) -> u64 { + let lock = v3::ReservationLock::acquire(storage).unwrap(); + let mut registry = v3::RegistryWriter::open_existing(storage, FAMILY, &lock).unwrap(); + let mut cursor = v3::PlaneRowCursor::default(); + let swept = registry + .sweep_plane_rows(epoch, &HashSet::new(), &mut cursor, 1_000, None) + .unwrap(); + assert!(swept.complete); + swept.deleted +} + +fn has(storage: &Path, plane: FamilyPlane, key: &FamilyKey) -> bool { + v3::PlaneReader::open_existing(storage, FAMILY, plane) + .unwrap() + .contains(key) + .unwrap() +} + +/// No manifest marks a symbol row, so the plane-row sweep ages symbol rows +/// out by epoch: an untouched row survives `SYMBOL_RETENTION_SWEEPS` sweeps, +/// a touch restarts its count, and an unmarked row of a manifest plane is +/// still collected at the first sweep. +#[test] +fn the_sweep_keeps_symbol_rows_for_their_retention_epochs() { + let _serial = v3::serial_for_test(); + let storage = tempfile::tempdir().unwrap(); + let storage = storage.path(); + let producer = SymbolProducer::current(); + let kept = key(b"fn kept() {}").family_key(); + let touched = key(b"fn touched() {}").family_key(); + let rows = [kept, touched] + .iter() + .map(|key| (*key, encode_payload(&producer, "Rust", &[symbol("row")]))) + .collect::>(); + assert_eq!(put(storage, FAMILY, &rows).unwrap(), 2); + assert_eq!( + put(storage, FAMILY, &rows).unwrap(), + 0, + "a second put reuses" + ); + let trigram = FamilyKey::new(FamilyPlane::Trigram, [3; 32]); + v3::publish(storage, FAMILY, 1 << 20, None, |publication| { + publication.put(&trigram, b"unmarked", 1) + }) + .unwrap(); + + sweep(storage, 1); + assert!(!has(storage, FamilyPlane::Trigram, &trigram)); + for epoch in 2..=SYMBOL_RETENTION_SWEEPS { + if epoch == 4 { + assert!(touch(storage, FAMILY, &[touched]).unwrap().is_empty()); + } + assert_eq!(sweep(storage, epoch), 0, "epoch {epoch}"); + } + assert!(has(storage, FamilyPlane::Symbol, &kept)); + // The untouched row was stamped at epoch 0 and is now one sweep too old. + assert_eq!(sweep(storage, SYMBOL_RETENTION_SWEEPS + 1), 1); + assert!(!has(storage, FamilyPlane::Symbol, &kept)); + assert!(has(storage, FamilyPlane::Symbol, &touched)); + // The touch at epoch 3's stamp keeps the other row until epoch 3 + 8 + 1. + for epoch in SYMBOL_RETENTION_SWEEPS + 2..=3 + SYMBOL_RETENTION_SWEEPS { + assert_eq!(sweep(storage, epoch), 0, "epoch {epoch}"); + } + assert_eq!(sweep(storage, 4 + SYMBOL_RETENTION_SWEEPS), 1); + assert_eq!( + v3::v3_format_report(&v3::plane_path(storage, FAMILY, FamilyPlane::Symbol).unwrap()) + .unwrap() + .layout_tag, + v3::LAYOUT_TAG + ); +} + +#[test] +fn a_lookup_reads_a_stored_row_and_misses_without_parsing() { + let _serial = v3::serial_for_test(); + let storage = tempfile::tempdir().unwrap(); + let storage = storage.path(); + let source = b"pub fn stored() {}\n"; + let path = Path::new("src/stored.rs"); + assert!(lookup(storage, FAMILY, ContentHash::of(source), path) + .unwrap() + .is_none()); + let producer = SymbolProducer::current(); + let language = language_of(path).unwrap(); + let key = SymbolKey { + content: ContentHash::of(source), + language: language.clone(), + producer, + }; + let symbols = vec![symbol("stored")]; + put( + storage, + FAMILY, + &[( + key.family_key(), + encode_payload(&producer, &language, &symbols), + )], + ) + .unwrap(); + let found = lookup(storage, FAMILY, ContentHash::of(source), path) + .unwrap() + .unwrap(); + assert_eq!(*found, symbols); + // The same content under another language has another key. + assert!(lookup( + storage, + FAMILY, + ContentHash::of(source), + Path::new("src/stored.go") + ) + .unwrap() + .is_none()); +} From 05465e553e48c1931d5ff90685609943df9c7122 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:55:53 +0100 Subject: [PATCH 5/8] mason: retained legacy inventory, live import sources kept by retention, prune-legacy consults v3 ledgers and base builds Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 2a0960762595a330762eb6773ebf6cccbe324774) --- crates/aft/src/cli/prune_legacy.rs | 5 +- crates/aft/src/migration/prune_legacy.rs | 174 +++++++++++- .../aft/src/migration/prune_legacy_tests.rs | 250 ++++++++++++++++++ crates/aft/src/migration/v3_ledger.rs | 62 ++++- crates/aft/src/storage_retention.rs | 227 ++++++++++++++++ 5 files changed, 698 insertions(+), 20 deletions(-) diff --git a/crates/aft/src/cli/prune_legacy.rs b/crates/aft/src/cli/prune_legacy.rs index 3a6eb9cef..677be0c9c 100644 --- a/crates/aft/src/cli/prune_legacy.rs +++ b/crates/aft/src/cli/prune_legacy.rs @@ -20,7 +20,10 @@ fn usage() -> String { Lists every legacy (pre-view) index set under the AFT storage root with its bytes, and with --yes deletes the sets whose import into per-checkout views -completed at least {days} days ago. Without --yes it is a dry run. +or into the v3 stores completed at least {days} days ago. A set that a running +import or a v3 base build may still read is kept. Older v1 views and blob +stores and v2 family stores are listed as retained and never deleted. Without +--yes it is a dry run. {OPERATOR_CONTRACT} diff --git a/crates/aft/src/migration/prune_legacy.rs b/crates/aft/src/migration/prune_legacy.rs index d288ab414..b97ad4a36 100644 --- a/crates/aft/src/migration/prune_legacy.rs +++ b/crates/aft/src/migration/prune_legacy.rs @@ -1,12 +1,15 @@ //! `aft cache prune-legacy`: the only path that deletes a legacy (pre-view) -//! artifact set. +//! artifact set (`index`, `semantic`, `callgraph`, `artifact-owners` and +//! `symbols` directories of one key). //! //! Nothing deletes legacy sets automatically: not startup, not a sweep, not //! maintenance. The operator runs this command, which is a dry run unless //! `--yes` is given. It lists each legacy set as a whole with its bytes and -//! prunes a set only when the set's import into per-checkout views completed -//! and the retention window (14 days) has passed since, so an offline -//! rollback to the previous binary keeps its caches in between. +//! prunes a set only when the set's import into per-checkout views (v2) or +//! into the v3 stores completed and the retention window (14 days) has +//! passed since, so an offline rollback to the previous binary keeps its +//! caches in between. A set that a running v3 import or a v3 base build may +//! still read is kept whatever its ledgers say. //! //! The process census is a guard against a mistake, not process exclusion: //! @@ -49,14 +52,17 @@ pub enum LegacyKind { Semantic, Callgraph, ArtifactOwners, + /// `symbols/`: the parsed-symbol cache, which the v3 importer reads. + Symbols, } impl LegacyKind { - pub const ALL: [Self; 4] = [ + pub const ALL: [Self; 5] = [ Self::Index, Self::Semantic, Self::Callgraph, Self::ArtifactOwners, + Self::Symbols, ]; /// The legacy root under the storage directory. @@ -66,6 +72,7 @@ impl LegacyKind { Self::Semantic => "semantic", Self::Callgraph => "callgraph", Self::ArtifactOwners => "artifact-owners", + Self::Symbols => "symbols", } } @@ -100,6 +107,10 @@ pub enum Eligibility { LedgerUnreadable(String), /// The set could not be inventoried completely; it is never moved. InventoryIncomplete(String), + /// Something may still read the set as an import source, whatever its + /// ledgers say: a running v3 import holds a claim on it, or a v3 base of + /// the family of the same name is being built. + PinnedSource(String), } impl Eligibility { @@ -120,6 +131,9 @@ impl Eligibility { ), Self::LedgerUnreadable(reason) => format!("kept: import ledger unreadable: {reason}"), Self::InventoryIncomplete(reason) => format!("kept: inventory incomplete: {reason}"), + Self::PinnedSource(reason) => { + format!("kept: still pinned as an import source: {reason}") + } } } } @@ -156,7 +170,7 @@ pub struct Inventory { /// Size of a directory tree without following symlinks. The walk stops at /// `limit` entries; `complete` is false when it did. -fn tree_size(path: &Path, limit: usize) -> io::Result<(u64, u64, bool)> { +pub(crate) fn tree_size(path: &Path, limit: usize) -> io::Result<(u64, u64, bool)> { let mut bytes = 0_u64; let mut entries = 0_u64; let mut stack = vec![path.to_path_buf()]; @@ -220,14 +234,103 @@ fn civil_from_days(days: i64) -> (i64, u32, u32) { (year, month, day) } -/// Decides eligibility from the key's import ledger, read without writing. +/// Whether something may still read a legacy set as an import source. +enum SourcePin { + Free, + Pinned(String), + Uncertain(String), +} + +/// Consults the v3 import ledger and the v3 base-build claims (the +/// base-reference protocol of `views::base_refs`) of the family named like +/// the legacy key, reading both without writing. A finished import does not +/// free a set that a live import or a live base build may still read. +fn source_pin(storage: &Path, key: &str) -> SourcePin { + match super::v3_ledger::legacy_key_status(storage, key) { + Ok(Some(status)) if status.live_claim => { + return SourcePin::Pinned("a running v3 import holds a claim on it".to_owned()) + } + Ok(_) => {} + Err(error) => return SourcePin::Uncertain(error.to_string()), + } + if crate::blob_store::v2::validate_family(key).is_err() { + return SourcePin::Free; + } + let Ok(registry) = + crate::blob_store::v3::family_dir(storage, key).map(|family| family.join("members.sqlite")) + else { + return SourcePin::Free; + }; + if !registry.is_file() { + return SourcePin::Free; + } + // Read a private copy: even a read-only open of the live registry can + // leave WAL sidecars beside it, and this command's dry run writes nothing. + let building = super::v3_ledger::read_sqlite_copy(®istry, |connection| { + if !crate::views::base_refs::has_base_refs(connection)? { + return Ok(None); + } + let mut statement = connection.prepare( + "SELECT base_scope, owner_pid, owner_start FROM base_builds WHERE state='building'", + )?; + let claims = statement + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + crate::pins::PinOwner { + pid: row.get(1)?, + start_time: row.get(2)?, + }, + )) + })? + .collect::>>()?; + Ok(claims + .into_iter() + .find(|(_, owner)| crate::pins::owner_is_live(owner)) + .map(|(scope, _)| scope)) + }); + match building { + Ok(Some(scope)) => SourcePin::Pinned(format!("v3 base {scope} is being built")), + Ok(None) => SourcePin::Free, + Err(error) => SourcePin::Uncertain(format!("v3 registry unreadable: {error}")), + } +} + +/// Decides eligibility from the key's import ledgers (the v2 per-checkout +/// ledger and the v3 ledger), read without writing. Every ledger that has +/// rows for the key must be complete; the retention window runs from the +/// latest completion; and nothing may still pin the set as a source. fn eligibility(storage: &Path, key: &str, now: SystemTime) -> Eligibility { - let status = match ImportLedger::read_status(storage, key) { - Ok(Some(status)) => status, - Ok(None) => return Eligibility::NotImported, + match source_pin(storage, key) { + SourcePin::Free => {} + SourcePin::Pinned(reason) => return Eligibility::PinnedSource(reason), + SourcePin::Uncertain(reason) => return Eligibility::LedgerUnreadable(reason), + } + let v2 = match ImportLedger::read_status(storage, key) { + Ok(status) => status, Err(error) => return Eligibility::LedgerUnreadable(error.to_string()), }; - let (true, Some(completed_ms)) = (status.complete, status.completed_ms) else { + let v3 = match super::v3_ledger::legacy_key_status(storage, key) { + Ok(status) => status, + Err(error) => return Eligibility::LedgerUnreadable(error.to_string()), + }; + if v2.is_none() && v3.is_none() { + return Eligibility::NotImported; + } + let mut completions = Vec::new(); + if let Some(status) = v2 { + match (status.complete, status.completed_ms) { + (true, Some(completed_ms)) => completions.push(completed_ms), + _ => return Eligibility::ImportIncomplete, + } + } + if let Some(status) = v3 { + match (status.incomplete, status.completed_ms) { + (false, Some(completed_ms)) => completions.push(completed_ms), + _ => return Eligibility::ImportIncomplete, + } + } + let Some(completed_ms) = completions.into_iter().max() else { return Eligibility::ImportIncomplete; }; let retention_ms = LEGACY_RETENTION.as_millis() as u64; @@ -761,6 +864,7 @@ pub fn prune_legacy( writeln!(out)?; let inventory = inventory(storage, options.now)?; write_inventory(out, storage, &inventory)?; + write_retained(out, storage)?; if let Some(reasons) = census_refusal(census) { writeln!(out, "Refusing: the storage root may be in use.")?; @@ -936,6 +1040,54 @@ fn write_inventory(out: &mut dyn Write, storage: &Path, inventory: &Inventory) - Ok(()) } +/// Lists the legacy data this command never removes (v1 views and blob +/// families, v2 family stores), so the operator sees what stays for rollback +/// and which families v3 already holds. Symbol caches are part of the legacy +/// sets above and are pruned with them. +fn write_retained(out: &mut dyn Write, storage: &Path) -> io::Result<()> { + let items = match crate::storage_retention::retained_legacy_inventory(storage) { + Ok(items) => items + .into_iter() + .filter(|item| item.kind != crate::storage_retention::RetainedKind::SymbolCache) + .collect::>(), + Err(error) => { + writeln!(out, "Retained legacy data could not be listed: {error}")?; + writeln!(out)?; + return Ok(()); + } + }; + if items.is_empty() { + return Ok(()); + } + writeln!(out, "Retained legacy data (never removed by this command):")?; + for item in &items { + let mut notes = Vec::new(); + if item.copied_into_v3 { + notes.push("v3 family present".to_owned()); + } + if let Some(reason) = &item.pinned { + notes.push(format!("pinned: {reason}")); + } + writeln!( + out, + " {} {} {}{}", + item.kind.describe(), + item.path + .strip_prefix(storage) + .unwrap_or(&item.path) + .display(), + human_bytes(item.bytes), + if notes.is_empty() { + String::new() + } else { + format!(" ({})", notes.join("; ")) + } + )?; + } + writeln!(out)?; + Ok(()) +} + /// The retention window as a whole number of days, for help text. pub fn retention_days() -> u64 { LEGACY_RETENTION.as_secs() / Duration::from_secs(86_400).as_secs() diff --git a/crates/aft/src/migration/prune_legacy_tests.rs b/crates/aft/src/migration/prune_legacy_tests.rs index 8a941aaf1..fba7aab28 100644 --- a/crates/aft/src/migration/prune_legacy_tests.rs +++ b/crates/aft/src/migration/prune_legacy_tests.rs @@ -116,3 +116,253 @@ fn a_tree_walk_stops_at_its_bound() { assert!(complete); assert_eq!(entries, 10); } + +// --------------------------------------------------------------------------- +// v3 import sources + +use std::collections::BTreeMap; + +use crate::blob_store::v3; +use crate::migration::v3_import::tests::{checkout, legacy_trigram, snapshot_tree, KEY}; +use crate::migration::v3_ledger::{hold_claim_for_test, run_v3_import, V3ImportRequest}; + +struct Census(Vec); + +impl ProcessCensus for Census { + fn take(&self) -> Result, String> { + Ok(self.0.clone()) + } +} + +const FIFTEEN_DAYS: Duration = Duration::from_secs(15 * 24 * 60 * 60); + +/// A storage root whose legacy set `KEY` was imported into v3 (and never +/// through v2), plus retained v1 and v2 data and a v3 family. +fn v3_imported_root(dir: &Path) -> PathBuf { + let root = checkout(dir); + let storage = dir.join("storage"); + legacy_trigram(&storage, &root); + let symbols = crate::symbol_cache_disk::cache_path(&storage, KEY); + fs::create_dir_all(symbols.parent().unwrap()).unwrap(); + fs::write(&symbols, b"not a symbol cache").unwrap(); + let mut request = V3ImportRequest::full_plane( + storage.clone(), + KEY.to_owned(), + KEY.to_owned(), + root, + crate::migration::v3_import::tests::POLICY, + None, + ); + request.symbols = true; + run_v3_import(&request, None).unwrap(); + for retained in [ + "views/0123456789abcdef/manifest.json", + "blobs/0123456789abcdef/semantic.sqlite", + "blobs/v2/family/semantic.sqlite", + ] { + let path = storage.join(retained); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, b"retained").unwrap(); + } + storage +} + +/// Every file's bytes except SQLite shared-memory files, which a reader +/// updates by design (read marks) without changing any stored data. +fn data_files(storage: &Path) -> BTreeMap> { + snapshot_tree(storage) + .into_iter() + .filter(|(path, _)| !path.to_string_lossy().ends_with("-shm")) + .collect() +} + +/// Fails naming every file whose presence or bytes differ. +fn assert_same( + before: &BTreeMap>, + after: &BTreeMap>, + what: &str, +) { + let changed = before + .keys() + .chain(after.keys()) + .filter(|path| before.get(*path) != after.get(*path)) + .map(|path| { + format!( + "{} ({:?} -> {:?} bytes)", + path.display(), + before.get(path).map(Vec::len), + after.get(path).map(Vec::len) + ) + }) + .collect::>(); + assert!(changed.is_empty(), "{what}: {changed:#?}"); +} + +fn eligibility_of(storage: &Path, now: SystemTime) -> Eligibility { + inventory(storage, now) + .unwrap() + .sets + .into_iter() + .find(|set| set.key == KEY) + .unwrap() + .eligibility +} + +fn run(storage: &Path, yes: bool, census: &Census) -> (PruneExit, String) { + let mut out = Vec::new(); + let exit = prune_legacy( + storage, + &PruneOptions { + yes, + now: SystemTime::now() + FIFTEEN_DAYS, + }, + census, + None, + &mut out, + ) + .unwrap(); + (exit, String::from_utf8(out).unwrap()) +} + +/// A v3 import's completion makes the set (including its symbol cache) +/// eligible after the retention window, but never while a running import +/// holds a claim on it, whatever its other rows say. +#[test] +fn a_set_a_live_v3_import_reads_is_never_prune_eligible() { + let _serial = v3::serial_for_test(); + let dir = tempfile::tempdir().unwrap(); + let storage = v3_imported_root(dir.path()); + let now = SystemTime::now(); + assert!(matches!( + eligibility_of(&storage, now), + Eligibility::Retained { .. } + )); + assert!(matches!( + eligibility_of(&storage, now + FIFTEEN_DAYS), + Eligibility::Eligible { .. } + )); + let set = inventory(&storage, now).unwrap().sets.remove(0); + assert!(set.dirs.iter().any(|dir| dir.kind == LegacyKind::Symbols)); + + let held = hold_claim_for_test(&storage, KEY, KEY); + let pinned = eligibility_of(&storage, now + FIFTEEN_DAYS); + assert!( + matches!(pinned, Eligibility::PinnedSource(ref reason) if reason.contains("v3 import")), + "{pinned:?}" + ); + let before = data_files(&storage); + let (exit, out) = run(&storage, true, &Census(Vec::new())); + assert_eq!(exit, PruneExit::Clean, "{out}"); + assert!( + out.contains("kept: still pinned as an import source"), + "{out}" + ); + assert_same(&before, &data_files(&storage), "a pinned set lost bytes"); + drop(held); +} + +/// The slice-4 base protocol is consulted: a live base build in the family +/// of the same name pins the legacy set; a dead builder does not. +#[test] +fn a_live_v3_base_build_pins_the_family_legacy_set() { + let _serial = v3::serial_for_test(); + let dir = tempfile::tempdir().unwrap(); + let storage = v3_imported_root(dir.path()); + let lock = v3::ReservationLock::acquire(&storage).unwrap(); + let mut registry = v3::open_base_refs_registry(&storage, KEY, &lock, None).unwrap(); + let me = crate::views::registry::current_owner(); + registry + .with_barrier(|tx| { + tx.execute( + "INSERT INTO base_builds (base_scope, owner_pid, owner_start, owner_token, attempt, state, heartbeat_at_ms) + VALUES ('base-under-test', ?1, ?2, 'token', 1, 'building', 0)", + rusqlite::params![me.pid, me.start_time as i64], + )?; + Ok(()) + }) + .unwrap(); + drop(registry); + drop(lock); + let later = SystemTime::now() + FIFTEEN_DAYS; + let pinned = eligibility_of(&storage, later); + assert!( + matches!(pinned, Eligibility::PinnedSource(ref reason) if reason.contains("base-under-test")), + "{pinned:?}" + ); + let lock = v3::ReservationLock::acquire(&storage).unwrap(); + let mut registry = v3::open_base_refs_registry(&storage, KEY, &lock, None).unwrap(); + registry + .with_barrier(|tx| { + // A pid no process holds: the builder is gone. + tx.execute("UPDATE base_builds SET owner_pid = 2147483646", [])?; + Ok(()) + }) + .unwrap(); + drop(registry); + drop(lock); + assert!(matches!( + eligibility_of(&storage, later), + Eligibility::Eligible { .. } + )); +} + +/// The safeguards stay: every run prints the stop-all warning, a dry run +/// changes nothing, a live AFT process refuses the run, and `--yes` renames +/// the eligible set aside before deleting it. Retained v1/v2 data is listed +/// and never touched. +#[cfg(unix)] +#[test] +fn prune_keeps_its_safeguards_and_never_touches_retained_legacy_data() { + let _serial = v3::serial_for_test(); + let dir = tempfile::tempdir().unwrap(); + let storage = v3_imported_root(dir.path()); + let retained = data_files(&storage) + .into_iter() + .filter(|(path, _)| { + let relative = path.strip_prefix(&storage).unwrap(); + relative.starts_with("views") + || relative.starts_with("blobs") + || relative.starts_with("migration") + }) + .collect::>(); + let before = data_files(&storage); + + let (exit, out) = run(&storage, false, &Census(Vec::new())); + assert_eq!(exit, PruneExit::Clean, "{out}"); + assert!(out.starts_with(OPERATOR_CONTRACT), "{out}"); + assert!(out.contains("Dry run: 1 legacy set(s)"), "{out}"); + assert!(out.contains("Retained legacy data"), "{out}"); + assert!(out.contains("v2 family store blobs/v2/family"), "{out}"); + assert!(out.contains("v1 view views/0123456789abcdef"), "{out}"); + assert!(out.contains("v3 family present"), "{out}"); + assert_same(&before, &data_files(&storage), "a dry run changed bytes"); + + let holder = CensusFinding::Aft { + pid: 4242, + daemon: true, + command: "aft --subc /tmp/c.json".to_owned(), + }; + let (exit, out) = run(&storage, true, &Census(vec![holder])); + assert_eq!(exit, PruneExit::Refused, "{out}"); + assert_same( + &before, + &data_files(&storage), + "a refused run changed bytes", + ); + + let (exit, out) = run(&storage, true, &Census(Vec::new())); + assert_eq!(exit, PruneExit::Clean, "{out}"); + assert!(out.contains(&format!("Moved {KEY} aside")), "{out}"); + for kind in ["index", "symbols"] { + assert!(!storage.join(kind).join(KEY).exists(), "{kind}: {out}"); + } + let after = data_files(&storage); + for (path, bytes) in &retained { + assert_eq!(after.get(path), Some(bytes), "{} changed", path.display()); + } + assert!(fs::read_dir(&storage).unwrap().all(|entry| !entry + .unwrap() + .file_name() + .to_string_lossy() + .starts_with(PRUNE_DIR_PREFIX))); +} diff --git a/crates/aft/src/migration/v3_ledger.rs b/crates/aft/src/migration/v3_ledger.rs index 70c4eb95b..21b177320 100644 --- a/crates/aft/src/migration/v3_ledger.rs +++ b/crates/aft/src/migration/v3_ledger.rs @@ -493,28 +493,49 @@ pub fn read_rows(storage: &Path, family: &str) -> V3Result( + path: &Path, + body: impl FnOnce(&rusqlite::Connection) -> V3Result, +) -> V3Result { static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let name = path + .file_name() + .ok_or_else(|| V3ImportError::Corrupt(format!("{} names no file", path.display())))?; let copy_dir = std::env::temp_dir().join(format!( - "aft-v3-ledger-read-{}-{}", + "aft-v3-sqlite-read-{}-{}", std::process::id(), SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed) )); let result = (|| { crate::private_storage::create_dir_all(©_dir)?; - let copy = copy_dir.join(LEDGER_FILE); - crate::private_storage::copy(&ledger.path, ©)?; - let wal = ledger.path.with_file_name(format!("{LEDGER_FILE}-wal")); - match crate::private_storage::copy(&wal, copy.with_file_name(format!("{LEDGER_FILE}-wal"))) + let copy = copy_dir.join(name); + crate::private_storage::copy(path, ©)?; + let mut wal_name = name.to_os_string(); + wal_name.push("-wal"); + match crate::private_storage::copy(path.with_file_name(&wal_name), copy_dir.join(&wal_name)) { Ok(_) => {} Err(error) if error.kind() == io::ErrorKind::NotFound => {} Err(error) => return Err(error.into()), } let connection = TrackedConnection::open(©, STORE)?; - check_format(&connection)?; - let rows = read_all(&connection)?; + let result = body(&connection); drop(connection); - Ok(Some(rows)) + result })(); let _ = fs::remove_dir_all(©_dir); result @@ -945,3 +966,28 @@ pub fn run_v3_import( #[cfg(test)] #[path = "v3_ledger_tests.rs"] mod tests; + +/// Keeps a test's claim live until it is dropped. +#[cfg(test)] +pub(crate) struct HeldClaim(#[allow(dead_code)] ActiveClaim); + +/// Test seam: claims a row of `legacy_key` in `family`'s ledger for an owner +/// this process keeps live until the returned guard drops, as a running +/// import would. +#[cfg(test)] +pub(crate) fn hold_claim_for_test(storage: &Path, family: &str, legacy_key: &str) -> HeldClaim { + let owner = ClaimOwner::new(); + let active = ActiveClaim::register(&owner); + let key = RowKey { + legacy_key: legacy_key.to_owned(), + artifact: LegacyArtifact::Trigram, + producer: "held-for-test".to_owned(), + plane: FamilyPlane::Trigram, + }; + let claimed = V3Ledger::new(storage, family) + .unwrap() + .claim(&key, &owner) + .unwrap(); + assert!(matches!(claimed, Claim::Owned(_))); + HeldClaim(active) +} diff --git a/crates/aft/src/storage_retention.rs b/crates/aft/src/storage_retention.rs index 0876d923f..30c7cb367 100644 --- a/crates/aft/src/storage_retention.rs +++ b/crates/aft/src/storage_retention.rs @@ -2279,6 +2279,144 @@ fn protected(cache: &Path) -> io::Result { Ok(false) } +/// Whether a due legacy cache must stay because a v3 import is reading it. +/// +/// The v3 importer reads `index/`, `semantic/` and `symbols/` +/// while it holds a claim in the family's import ledger +/// (`migration::v3_ledger`). Retiring such a set would pull the source out +/// from under that import, so a live claim protects it, and a ledger that +/// cannot be read is uncertainty and protects it too. A finished, abandoned +/// or absent import leaves the binding-age rules alone. +fn legacy_import_source_pinned(storage: &StoragePaths, cache: &Path, key: &str) -> bool { + let domain = cache + .parent() + .and_then(Path::file_name) + .and_then(|name| name.to_str()); + if !matches!(domain, Some("index" | "semantic" | "symbols")) { + return false; + } + match crate::migration::v3_ledger::legacy_key_status(&storage.artifacts, key) { + Ok(Some(status)) => status.live_claim, + Ok(None) => false, + Err(_) => true, + } +} + +/// One kind of legacy (pre-v3) data that stays on disk for an offline +/// rollback to the previous binary. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)] +pub enum RetainedKind { + /// `symbols/`: a parsed-symbol cache, a source of the v3 symbol import. + SymbolCache, + /// `views/`: a v1 view, whose generations name v1 blob rows. + V1View, + /// `blobs/`: v1 blob stores, the copy-on-first-use donors of v3. + V1BlobFamily, + /// `blobs/v2/`: v2 family stores, also copy-on-first-use donors. + V2FamilyStore, +} + +impl RetainedKind { + pub const fn describe(self) -> &'static str { + match self { + Self::SymbolCache => "symbol cache", + Self::V1View => "v1 view", + Self::V1BlobFamily => "v1 blob family", + Self::V2FamilyStore => "v2 family store", + } + } +} + +/// One retained legacy item. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RetainedItem { + pub kind: RetainedKind, + pub path: PathBuf, + pub bytes: u64, + /// The v3 family of the same name exists, so rows of this item may have + /// been copied into v3 (a copy never removes or changes its source). + pub copied_into_v3: bool, + /// Why the item is protected right now (a live import claim, live pins + /// or readers), or `None`. A protected item is never retired. + pub pinned: Option, +} + +/// Upper bound on items listed per retained kind. +const RETAINED_ITEMS_PER_KIND: usize = 10_000; + +/// Lists the legacy data this build keeps but never writes: symbol caches, +/// v1 views and blob families, and v2 family stores, with whether v3 holds +/// a family of the same name and whether something protects them now. It +/// reads directory listings, sizes, ledgers and pin files only; it writes +/// nothing under the storage root. +pub fn retained_legacy_inventory(storage: &Path) -> io::Result> { + let paths = StoragePaths::same(storage); + let v3_family = |name: &str| storage.join("blobs/v3").join(name).is_dir(); + let mut items = Vec::new(); + for (kind, parent) in [ + (RetainedKind::SymbolCache, storage.join("symbols")), + (RetainedKind::V1View, storage.join("views")), + (RetainedKind::V1BlobFamily, storage.join("blobs")), + (RetainedKind::V2FamilyStore, storage.join("blobs/v2")), + ] { + let entries = match fs::read_dir(&parent) { + Ok(entries) => entries, + Err(error) if error.kind() == io::ErrorKind::NotFound => continue, + Err(error) => return Err(error), + }; + let mut names = Vec::new(); + for entry in entries.take(RETAINED_ITEMS_PER_KIND) { + let entry = entry?; + if !entry.file_type()?.is_dir() { + continue; + } + let Some(name) = entry.file_name().to_str().map(str::to_owned) else { + continue; + }; + let listed = match kind { + RetainedKind::SymbolCache => { + crate::migration::per_checkout::validate_legacy_key(&name).is_ok() + } + RetainedKind::V1View | RetainedKind::V1BlobFamily => key_valid(&name), + RetainedKind::V2FamilyStore => { + crate::blob_store::v2::validate_family(&name).is_ok() + } + }; + if listed { + names.push(name); + } + } + names.sort(); + for name in names { + let path = parent.join(&name); + let bytes = crate::migration::prune_legacy::tree_size(&path, TREE_ENTRIES) + .map(|(bytes, _, _)| bytes) + .unwrap_or(0); + let pinned = match kind { + RetainedKind::SymbolCache => legacy_import_source_pinned(&paths, &path, &name) + .then(|| "a v3 import is reading it".to_owned()), + RetainedKind::V1View => match protected(&path) { + Ok(true) => Some("live pins or readers".to_owned()), + Ok(false) => None, + Err(error) => Some(format!("pins unreadable: {error}")), + }, + // Their rows are protected one by one: a copier pins each + // source row before reading it, and the v1/v2 sweeps honour + // those pins. Nothing protects the store as a whole. + RetainedKind::V1BlobFamily | RetainedKind::V2FamilyStore => None, + }; + items.push(RetainedItem { + kind, + copied_into_v3: v3_family(&name), + path, + bytes, + pinned, + }); + } + } + Ok(items) +} + fn owner_has_residency_marker( storage: &StoragePaths, owner: &crate::artifact_owner::ArtifactOwnerManifest, @@ -3478,6 +3616,7 @@ impl PreparedDue { || crate::root_cache::live_scope_keys_for_storage(&storage.durable).contains(key) || owners.protect(storage, key, current.contains_key(key))? || protected(&cache)? + || legacy_import_source_pinned(storage, &cache, key) { report.skipped_protected += 1; return Ok(()); @@ -3948,6 +4087,7 @@ fn candidate_inner( report, )? || crate::root_cache::live_scope_keys_for_storage(&storage.durable).contains(key) || protected(&cache)? + || legacy_import_source_pinned(storage, &cache, key) { report.skipped_protected += 1; return Ok(()); @@ -9217,6 +9357,93 @@ mod storage_retention_tests { assert!(report.skipped_protected > 0); } + #[test] + fn retention_due_legacy_cache_keeps_a_live_v3_import_source() { + let _serial = crate::blob_store::v3::serial_for_test(); + let (temp, root, scope) = fixture(); + fs::remove_dir(&root).unwrap(); + let path = cache(temp.path(), "index", &scope); + // The v3 importer of this key's family holds a live claim: it may be + // reading `index/` right now. + let held = crate::migration::v3_ledger::hold_claim_for_test(temp.path(), &scope, &scope); + let report = run_pass(temp.path(), &|| false); + assert_eq!(report.removed_roots, 0, "{report:?}"); + assert!(path.join("payload.bin").exists()); + assert!(report.skipped_protected > 0, "{report:?}"); + // Once the import is gone the binding-age rule applies again. + drop(held); + let report = run_pass(temp.path(), &|| false); + assert_eq!(report.removed_roots, 1, "{report:?}"); + assert!(!path.exists()); + } + + #[test] + fn retained_legacy_inventory_lists_every_kind_without_writing() { + let temp = tempfile::tempdir().unwrap(); + let storage = temp.path(); + for (path, bytes) in [ + ("symbols/0123456789abcdef/symbols.bin", &b"symbols"[..]), + ("views/0123456789abcdef/manifest.json", b"view"), + ("views/v2/scope/manifest.json", b"v2 view"), + ("blobs/fedcba9876543210/semantic.sqlite", b"v1 blobs"), + ("blobs/v2/family/semantic.sqlite", b"v2 blobs"), + ("blobs/v3/family/members.sqlite", b"v3 registry"), + ] { + let path = storage.join(path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, bytes).unwrap(); + } + let pins = storage.join("views/0123456789abcdef/pins"); + fs::create_dir_all(&pins).unwrap(); + let _pin = + crate::pins::QueryPin::acquire(&storage.join("views/0123456789abcdef"), "g").unwrap(); + let snapshot = crate::migration::v3_import::tests::snapshot_tree; + let before = snapshot(storage); + let items = retained_legacy_inventory(storage).unwrap(); + let summary = items + .iter() + .map(|item| { + ( + item.kind, + item.path.strip_prefix(storage).unwrap().to_path_buf(), + item.copied_into_v3, + item.pinned.is_some(), + ) + }) + .collect::>(); + assert_eq!( + summary, + vec![ + ( + RetainedKind::SymbolCache, + PathBuf::from("symbols/0123456789abcdef"), + false, + false + ), + ( + RetainedKind::V1View, + PathBuf::from("views/0123456789abcdef"), + false, + true + ), + ( + RetainedKind::V1BlobFamily, + PathBuf::from("blobs/fedcba9876543210"), + false, + false + ), + ( + RetainedKind::V2FamilyStore, + PathBuf::from("blobs/v2/family"), + true, + false + ), + ] + ); + assert!(items.iter().all(|item| item.bytes > 0)); + assert_eq!(snapshot(storage), before); + } + #[test] fn retention_due_root_keeps_writer_lock() { let (temp, _, _, path) = due_cache_fixture(); From 5293521f9e19cf0745e3da5f7337fcf0ad428e14 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:59:57 +0100 Subject: [PATCH 6/8] mason: gate unix-only kill-test helpers, and clarify the import ledger and symbol plane format text Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit d559a9a292ec464d4f983b35ebf8086ee15cbd62) --- .../aft/src/migration/prune_legacy_tests.rs | 4 +- crates/aft/src/migration/v3_ledger_tests.rs | 7 +++ docs/design/views-v3-format.md | 44 ++++++++++--------- 3 files changed, 33 insertions(+), 22 deletions(-) diff --git a/crates/aft/src/migration/prune_legacy_tests.rs b/crates/aft/src/migration/prune_legacy_tests.rs index fba7aab28..f722304c7 100644 --- a/crates/aft/src/migration/prune_legacy_tests.rs +++ b/crates/aft/src/migration/prune_legacy_tests.rs @@ -261,8 +261,8 @@ fn a_set_a_live_v3_import_reads_is_never_prune_eligible() { drop(held); } -/// The slice-4 base protocol is consulted: a live base build in the family -/// of the same name pins the legacy set; a dead builder does not. +/// The v3 registry's base-build claims are consulted: a live base build in +/// the family of the same name pins the legacy set; a dead builder does not. #[test] fn a_live_v3_base_build_pins_the_family_legacy_set() { let _serial = v3::serial_for_test(); diff --git a/crates/aft/src/migration/v3_ledger_tests.rs b/crates/aft/src/migration/v3_ledger_tests.rs index af90ad932..75afc354b 100644 --- a/crates/aft/src/migration/v3_ledger_tests.rs +++ b/crates/aft/src/migration/v3_ledger_tests.rs @@ -1,4 +1,5 @@ use std::path::Path; +#[cfg(unix)] use std::time::{Duration, Instant}; use super::*; @@ -7,6 +8,7 @@ use crate::migration::v3_import::tests::{ checkout, deterministic, legacy_semantic, legacy_trigram, producer, snapshot_tree, FAMILY, KEY, POLICY, }; +#[cfg(unix)] use crate::views::segment_store::TrigramPayload; fn request(storage: &Path, root: &Path, semantic: Option) -> V3ImportRequest { @@ -371,9 +373,12 @@ fn a_ledger_from_a_newer_build_is_refused_by_name() { // durable boundary of the trigram import; the next run must finish the import // from that state with nothing lost, duplicated or left in staging. +#[cfg(unix)] const KILL_SUBJECT: &str = "AFT_V3_IMPORT_KILL_SUBJECT"; +#[cfg(unix)] const KILL_AT: &str = "AFT_V3_IMPORT_KILL_AT"; +#[cfg(unix)] fn step_name(step: V3Step) -> String { match step { V3Step::BatchRegistered(batch) => format!("batch-{batch}"), @@ -381,11 +386,13 @@ fn step_name(step: V3Step) -> String { } } +#[cfg(unix)] struct Park { at: String, ready: PathBuf, } +#[cfg(unix)] impl V3ImportObserver for Park { fn reached(&self, artifact: LegacyArtifact, step: V3Step) { if artifact != LegacyArtifact::Trigram || step_name(step) != self.at { diff --git a/docs/design/views-v3-format.md b/docs/design/views-v3-format.md index aef2ebf5f..f17d4436a 100644 --- a/docs/design/views-v3-format.md +++ b/docs/design/views-v3-format.md @@ -608,11 +608,11 @@ contract is [views-slice4-opening.md](views-slice4-opening.md). - The implemented registry remains `views_v3_registry` format/schema **1/1**. Reserve **2/2** for slice 4's base-reference protocol, **3/3** for 4v's token - and stat-evidence protocol, **4/4** for slice 5's donor-seal protocol, and - 5/5, earlier reserved for 6a, is **not used** by 6a: its import ledger is a - sidecar file and its symbol plane is an ordinary plane store (see *Import - ledger and symbol plane* below), so neither changes the registry schema. - These versions gate the whole member + and stat-evidence protocol and **4/4** for slice 5's donor-seal protocol. + Version 5/5, earlier reserved for slice 6a's symbol and import protocol, is + **not used**: the import ledger is a sidecar file and the symbol plane is an + ordinary plane store (see *Import ledger and symbol plane* below), so neither + changes the registry schema. These versions gate the whole member protocol, including its sidecars; a version can advance without adding a registry table. An owning slice installs its complete schema through an explicit versioned upgrade, never a @@ -636,11 +636,11 @@ contract is [views-slice4-opening.md](views-slice4-opening.md). registry-floor version that makes unsupported readers refuse. - Every artifact publication uses the combined v3 floor boundary. Slice 4 requests registry floor 2 before its artifacts; 4v requests 3 before stat - evidence or tokens; 5 requests 4 before donor/seal publication. 6a writes - its symbol plane and import ledger only inside `blob_store::v3::publish`, - which requests registry floor 1 and store floor 1 before creating either - file. Every publisher also requests store floor 1. These are future written - versions, not changes to today's startup baseline. + evidence or tokens; 5 requests 4 before donor/seal publication. Slice 6a + writes its symbol plane and import ledger only inside + `blob_store::v3::publish`, which requests registry floor 1 and store floor 1 + before creating either file. Every publisher also requests store floor 1. + These are future written versions, not changes to today's startup baseline. ### Assigned paths and owners @@ -669,10 +669,11 @@ Slice 4's binder must honour the `evicting` contract above: when it finds its member in `evicting`, it never uses that view directory, and either treats the member as absent and publishes a fresh one or waits for the eviction to finish. -## Import ledger and symbol plane (6a) +## Import ledger and symbol plane -Sources: `crates/aft/src/migration/v3_import.rs` (ledger, decoders, staging, -registration), `crates/aft/src/migration/v3_symbols.rs` (symbol importer) and +Sources: `crates/aft/src/migration/v3_import.rs` (decoders, staging, +registration), `crates/aft/src/migration/v3_ledger.rs` (ledger and driver), +`crates/aft/src/migration/v3_symbols.rs` (symbol importer) and `crates/aft/src/views/symbol_store.rs` (symbol plane). Imports write only under `blobs/v3//` and their own staging directory; they never write a v1 or v2 row, manifest, pointer or schema, and never move, change or delete @@ -688,7 +689,8 @@ a legacy file. iterate) but is in `FamilyPlane::V3_STORES`, which the v3 plane-row sweep and `v3_format_report` use. No manifest entry records symbol readiness. - Key: BLAKE3 over the tag `aft/views-v3/symbol/v1`, then the content hash - (32 bytes), the language id and the producer id, each preceded by its + (32 bytes), the language id (the parser's `LangId` name, for example + `Rust`) and the producer id, each preceded by its length as a big-endian u64. The producer id is `aft-symbols/extraction-` (`symbol_cache_disk::SCHEMA_VERSION`, bumped on any grammar, query or extractor change), so a parser or schema @@ -715,9 +717,10 @@ A sidecar SQLite file, not a registry table: `members.sqlite` and its version are unchanged. It is created, and every row transition is written, inside a `v3::publish` call (64 KiB reservation), so the file exists only after the v3 floor is raised and each write holds the reservation lock. -Readers (status, prune eligibility, retention) copy the database and its WAL -to a private temporary directory and read the copy, so a read never writes -under the storage root. +Readers (status, prune eligibility, retention) use this process's ledger +connection when it has one; otherwise they copy the database and its WAL to a +private temporary directory and read the copy, so a read never writes under +the storage root. `aft cache prune-legacy` reads `members.sqlite` the same way. ```sql CREATE TABLE format_meta (singleton INTEGER PRIMARY KEY CHECK(singleton=1), @@ -766,9 +769,10 @@ import, and a producer change starts a new row. - **done:** the staging bundle is removed. Inputs have no size limit. The decoders stream: the trigram `cache.bin` -is checksummed in one sequential pass, then inverted in file-id ranges -whose postings fit a working-set budget (64 MiB by default), one pass over -the posting blob per range; `semantic.bin` is decoded one file's rows at a +is checksummed in one sequential pass, its postings are counted per file in +one pass over the posting blob, then inverted in file-id ranges whose +postings fit a working-set budget (64 MiB by default), one more pass per +range; `semantic.bin` is decoded one file's rows at a time. Memory is bounded by the per-file tables (path, size and hash per file) plus the budget, never by the input size. The `semantic.bin` decoder is read-only and refuses trailing bytes (appended segments or a torn tail) From c80fcfe1dc6995036c9566dee09c1a9f206159fa Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 19:12:29 +0100 Subject: [PATCH 7/8] mason: hold the ledger open lock across ledger copies so no descriptor closes under a live connection; assert exact imported path sets Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> (cherry picked from commit 8ca96e51a2e3d76fde3e3d69cd5e3fefd5a234d6) --- crates/aft/src/migration/v3_import_tests.rs | 25 ++++++- crates/aft/src/migration/v3_ledger.rs | 53 +++++++++++--- crates/aft/src/migration/v3_ledger_tests.rs | 75 +++++++++++++++++++- crates/aft/src/migration/v3_symbols_tests.rs | 12 +++- 4 files changed, 153 insertions(+), 12 deletions(-) diff --git a/crates/aft/src/migration/v3_import_tests.rs b/crates/aft/src/migration/v3_import_tests.rs index 4eccc5407..3f96c8219 100644 --- a/crates/aft/src/migration/v3_import_tests.rs +++ b/crates/aft/src/migration/v3_import_tests.rs @@ -332,7 +332,30 @@ fn imported_trigram_rows_equal_a_cold_build_and_read_no_checkout_file() { }, ) .unwrap(); - assert!(entries.len() >= 3, "{} entries", entries.len()); + // Exactly the files a cold build indexes are imported: every checkout + // file whose trigram extraction is indexed (the binary file is not), and + // no other path. + let imported = entries + .iter() + .map(|entry| crate::views::segment_store::rel_path_to_os(&entry.rel_path).unwrap()) + .collect::>(); + let eligible = sources + .iter() + .filter(|(_, bytes)| { + TrigramPayload::extract(bytes, &POLICY).flag + == crate::views::segment_store::TrigramFlag::Indexed + }) + .map(|(path, _)| path.clone()) + .collect::>(); + assert_eq!(imported, eligible); + assert_eq!( + eligible, + ["notes/readme.md", "src/alpha.rs", "src/beta.rs"] + .into_iter() + .map(PathBuf::from) + .collect() + ); + assert_eq!(entries.len(), imported.len(), "a path was imported twice"); let outcome = register_staged( &storage, FAMILY, diff --git a/crates/aft/src/migration/v3_ledger.rs b/crates/aft/src/migration/v3_ledger.rs index 21b177320..f358cedc5 100644 --- a/crates/aft/src/migration/v3_ledger.rs +++ b/crates/aft/src/migration/v3_ledger.rs @@ -266,6 +266,10 @@ impl V3Ledger { /// The process's connection to the ledger. Without `create`, a missing /// ledger is `None`; with it, the caller must be inside a v3 publication. + /// + /// The `open_connections` lock is held from before the open until the + /// connection is registered, the same lock [`read_rows`] holds across a + /// copy of the ledger's files, so an open never overlaps such a copy. fn connection(&self, create: bool) -> V3Result> { let mut open = lock(open_connections()); open.retain(|_, connection| connection.strong_count() > 0); @@ -482,21 +486,48 @@ impl V3Ledger { /// a WAL a killed importer left behind. A copy taken while another process /// writes can only be older or unreadable, and rows only move towards /// completion, so neither makes an import look more complete than it is. +/// +/// The copy opens and closes descriptors on the ledger's files, and closing +/// any descriptor on a SQLite file drops every POSIX lock this process holds +/// on it. So the lookup and the whole copy run under the `open_connections` +/// lock, which every ledger open in this process takes before it opens and +/// keeps until it has registered its connection: an open and a copy can +/// never overlap, and a copy only runs while this process has no connection +/// to the ledger. pub fn read_rows(storage: &Path, family: &str) -> V3Result>> { let ledger = V3Ledger::new(storage, family)?; - if let Some(shared) = lock(open_connections()) - .get(&ledger.path) - .and_then(Weak::upgrade) - { + let mut open = lock(open_connections()); + open.retain(|_, connection| connection.strong_count() > 0); + if let Some(shared) = open.get(&ledger.path).and_then(Weak::upgrade) { + drop(open); return read_all(&lock(&shared)).map(Some); } if !ledger.path.is_file() { return Ok(None); } - read_sqlite_copy(&ledger.path, |connection| { + let rows = read_sqlite_copy(&ledger.path, |connection| { check_format(connection)?; read_all(connection).map(Some) - }) + }); + drop(open); + rows +} + +#[cfg(test)] +thread_local! { + /// Test hook: called on the copying thread at named points of + /// [`read_sqlite_copy`], so a test can hold a copy part-way through. + static COPY_HOOK: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; +} + +#[cfg(test)] +fn copy_hook(stage: &'static str) { + COPY_HOOK.with(|hook| { + if let Some(hook) = hook.borrow_mut().as_mut() { + hook(stage); + } + }); } /// Runs `body` on a private copy of the SQLite database at `path` (and its @@ -505,8 +536,10 @@ pub fn read_rows(storage: &Path, family: &str) -> V3Result( path: &Path, body: impl FnOnce(&rusqlite::Connection) -> V3Result, @@ -524,6 +557,8 @@ pub(crate) fn read_sqlite_copy( crate::private_storage::create_dir_all(©_dir)?; let copy = copy_dir.join(name); crate::private_storage::copy(path, ©)?; + #[cfg(test)] + copy_hook("database-copied"); let mut wal_name = name.to_os_string(); wal_name.push("-wal"); match crate::private_storage::copy(path.with_file_name(&wal_name), copy_dir.join(&wal_name)) @@ -538,6 +573,8 @@ pub(crate) fn read_sqlite_copy( result })(); let _ = fs::remove_dir_all(©_dir); + #[cfg(test)] + copy_hook("finished"); result } diff --git a/crates/aft/src/migration/v3_ledger_tests.rs b/crates/aft/src/migration/v3_ledger_tests.rs index 75afc354b..3269a0ed3 100644 --- a/crates/aft/src/migration/v3_ledger_tests.rs +++ b/crates/aft/src/migration/v3_ledger_tests.rs @@ -343,6 +343,76 @@ fn a_live_claim_is_single_flight_and_an_abandoned_one_is_taken_over() { assert_eq!(row_of(&storage, LegacyArtifact::Trigram).attempt, 2); } +/// A ledger read that has to copy the ledger's files holds descriptors on +/// them; closing those would drop the POSIX locks of any connection this +/// process had open. The copy is held part-way through: it then holds the +/// lock every ledger open takes, and an open attempted meanwhile completes +/// only after the copy has finished. +#[test] +fn a_ledger_open_waits_for_an_in_flight_copy_of_its_files() { + use std::sync::mpsc; + let _serial = v3::serial_for_test(); + let dir = tempfile::tempdir().unwrap(); + let root = checkout(dir.path()); + let storage = dir.path().join("storage"); + legacy_trigram(&storage, &root); + run_v3_import(&request(&storage, &root, None), None).unwrap(); + let ledger = V3Ledger::new(&storage, FAMILY).unwrap(); + assert!(ledger.path.is_file()); + // The import's connection is closed (its last handle dropped), so the + // next read has to copy the files. + assert!(lock(open_connections()) + .get(&ledger.path) + .and_then(Weak::upgrade) + .is_none()); + + let log = Arc::new(Mutex::new(Vec::new())); + let (paused_tx, paused_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel::<()>(); + let reader = { + let storage = storage.clone(); + let log = Arc::clone(&log); + std::thread::spawn(move || { + COPY_HOOK.with(|hook| { + *hook.borrow_mut() = Some(Box::new(move |stage| match stage { + "database-copied" => { + paused_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + } + "finished" => lock(&log).push("copy-finished"), + _ => {} + })); + }); + read_rows(&storage, FAMILY).unwrap().unwrap().len() + }) + }; + paused_rx.recv().unwrap(); + // The copy is in flight and holds the lock every ledger open takes first. + assert!(matches!( + open_connections().try_lock(), + Err(std::sync::TryLockError::WouldBlock) + )); + let (attempting_tx, attempting_rx) = mpsc::channel(); + let opener = { + let log = Arc::clone(&log); + std::thread::spawn(move || { + attempting_tx.send(()).unwrap(); + let connection = ledger.connection(false).unwrap().unwrap(); + lock(&log).push("opened"); + drop(connection); + }) + }; + attempting_rx.recv().unwrap(); + assert!( + lock(&log).is_empty(), + "nothing finished while the copy is held" + ); + release_tx.send(()).unwrap(); + assert_eq!(reader.join().unwrap(), 1, "the copy read the trigram row"); + opener.join().unwrap(); + assert_eq!(*lock(&log), vec!["copy-finished", "opened"]); +} + #[test] fn a_ledger_from_a_newer_build_is_refused_by_name() { let _serial = v3::serial_for_test(); @@ -355,8 +425,9 @@ fn a_ledger_from_a_newer_build_is_refused_by_name() { .unwrap() .path() .to_path_buf(); - // Close this process's shared connection before editing the file. - lock(open_connections()).clear(); + // Forget this process's ledger handle (its connection is already closed) + // so the next read opens the edited file. + lock(open_connections()).remove(&path); let connection = rusqlite::Connection::open(&path).unwrap(); connection .execute("UPDATE format_meta SET format_version = 2", []) diff --git a/crates/aft/src/migration/v3_symbols_tests.rs b/crates/aft/src/migration/v3_symbols_tests.rs index 20f532b92..5151a1a86 100644 --- a/crates/aft/src/migration/v3_symbols_tests.rs +++ b/crates/aft/src/migration/v3_symbols_tests.rs @@ -61,7 +61,17 @@ fn a_compatible_symbol_import_needs_no_parse_and_serves_lookups() { let root = checkout(dir.path()); let storage = dir.path().join("storage"); let files = legacy_symbols(&storage, &root); - assert!(files.len() >= 3, "{} parsed files", files.len()); + // Every checkout file the parser has a language for, and only those. + assert_eq!( + files + .iter() + .map(|(relative, _, _)| relative.clone()) + .collect::>(), + ["notes/readme.md", "src/alpha.rs", "src/beta.rs"] + .into_iter() + .map(std::path::PathBuf::from) + .collect() + ); let legacy = snapshot_tree(&storage); let parses = cached_parser_parses(); From 9cb2469527cb27c795c71920007c408bed221ec6 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 19:30:03 +0100 Subject: [PATCH 8/8] search-quality descriptor for views-e-legacy-imports Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- .../slice-descriptors/land-views-e-legacy-imports.json | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 benchmarks/aft-search/slice-descriptors/land-views-e-legacy-imports.json diff --git a/benchmarks/aft-search/slice-descriptors/land-views-e-legacy-imports.json b/benchmarks/aft-search/slice-descriptors/land-views-e-legacy-imports.json new file mode 100644 index 000000000..ebd574711 --- /dev/null +++ b/benchmarks/aft-search/slice-descriptors/land-views-e-legacy-imports.json @@ -0,0 +1,8 @@ +{ + "targeted_mechanism": "none", + "kind": "harness", + "slice_class": "non_ranking", + "fixtures": [ + "harness-goldens" + ] +}