Views: import existing index data into v3 (postings, vectors, symbols)

benchmarks/aft-search/slice-descriptors/land-views-e-legacy-imports.json

@@ -0,0 +1,8 @@
+{
+  "targeted_mechanism": "none",
+  "kind": "harness",
+  "slice_class": "non_ranking",
+  "fixtures": [
+    "harness-goldens"
+  ]
+}

crates/aft/src/blob_store/v2.rs

@@ -44,6 +44,9 @@ pub const STORE_SCHEMA_VERSION: i64 = 1;
 /// Payload encoding version of trigram blobs. Bumping it must also change the
 /// trigram policy fingerprint, because that fingerprint is part of every key.
 pub const TRIGRAM_PAYLOAD_SCHEMA: u32 = 1;
+/// Layout of a v3 symbol-plane payload (`views::symbol_store`). The symbol
+/// plane exists only in v3; no v2 store is ever opened for it.
+pub const SYMBOL_PAYLOAD_SCHEMA: u32 = 1;
 
 const STORE_SCHEMA: &str = r#"
 CREATE TABLE IF NOT EXISTS store_meta (
@@ -84,16 +87,28 @@ pub enum FamilyPlane {
     Trigram,
     Semantic,
     Callgraph,
+    /// Parsed symbols of one file, keyed by content, language and the
+    /// extraction schema. A v3-only plane: it has no manifest readiness and
+    /// no v2 store, so it is deliberately absent from [`Self::ALL`], which
+    /// the manifest, v2 GC, eviction and per-entry code iterate. The v3
+    /// plane-row sweep covers it through [`Self::V3_STORES`].
+    Symbol,
 }
 
 impl FamilyPlane {
+    /// The planes a manifest entry, a v2 family store and the v2 GC know.
+    /// Never add [`Self::Symbol`] here: `EntryPlanes::get_mut` has no field
+    /// for it and panics.
     pub const ALL: [Self; 3] = [Self::Trigram, Self::Semantic, Self::Callgraph];
+    /// Every plane that can have a store under `blobs/v3/<family>/`.
+    pub const V3_STORES: [Self; 4] = [Self::Trigram, Self::Semantic, Self::Callgraph, Self::Symbol];
 
     pub const fn as_str(self) -> &'static str {
         match self {
             Self::Trigram => "trigram",
             Self::Semantic => "semantic",
             Self::Callgraph => "callgraph",
+            Self::Symbol => "symbol",
         }
     }
 
@@ -102,6 +117,7 @@ impl FamilyPlane {
             Self::Trigram => TRIGRAM_PAYLOAD_SCHEMA,
             Self::Semantic => crate::blob_store::SEMANTIC_PAYLOAD_SCHEMA,
             Self::Callgraph => crate::blob_store::CALLGRAPH_PAYLOAD_SCHEMA,
+            Self::Symbol => SYMBOL_PAYLOAD_SCHEMA,
         }
     }
 }
@@ -1892,6 +1908,20 @@ mod tests {
         .family_key()
     }
 
+    #[test]
+    fn symbol_is_not_a_manifest_plane() {
+        // `EntryPlanes::get_mut` panics for Symbol because a manifest entry
+        // has no symbol state. That panic is unreachable only while every
+        // list the manifest, v2 store, GC, eviction and per-entry code
+        // iterates (`FamilyPlane::ALL`) excludes the v3-only symbol plane.
+        assert!(!FamilyPlane::ALL.contains(&FamilyPlane::Symbol));
+        assert!(FamilyPlane::V3_STORES.contains(&FamilyPlane::Symbol));
+        for plane in FamilyPlane::ALL {
+            assert!(FamilyPlane::V3_STORES.contains(&plane));
+        }
+        assert_eq!(FamilyPlane::Symbol.as_str(), "symbol");
+    }
+
     #[test]
     fn a_conflicting_payload_for_one_key_is_rejected_and_the_stored_payload_is_kept() {
         let dir = tempfile::tempdir().unwrap();

crates/aft/src/blob_store/v3.rs

@@ -24,6 +24,10 @@ pub const LAYOUT_TAG: u32 = 2;
 pub const REGISTRY_APPLICATION_ID: i64 = 0x41465452;
 pub const PLANE_APPLICATION_ID: i64 = 0x41465450;
 pub const NAMESPACE_BUDGET: u64 = 4 * 1024 * 1024 * 1024;
+/// Sweep epochs a symbol-plane row survives without being touched. Symbol
+/// rows are a reusable cache that no manifest names, so the plane-row sweep
+/// ages them out by epoch instead of by marking.
+pub const SYMBOL_RETENTION_SWEEPS: i64 = 8;
 const TIMEOUT_MS: i64 = 5_000;
 mod connections;
 mod copy;
@@ -1315,7 +1319,8 @@ impl RegistryWriter<'_> {
             complete: true,
             ..PlaneRowSweep::default()
         };
-        for plane in FamilyPlane::ALL {
+        // Every v3 plane store, including the v3-only symbol plane.
+        for plane in FamilyPlane::V3_STORES {
             let path = self.dir.join(format!("{}.sqlite", plane.as_str()));
             if !path.try_exists()? {
                 cursor.after.remove(plane.as_str());
@@ -1349,6 +1354,15 @@ impl RegistryWriter<'_> {
                 .and_then(|hex| super::v2::parse_hex32(hex))
                 .map(|key| key.to_vec())
                 .unwrap_or_default();
+            // No manifest names a symbol row, so marking never protects one.
+            // A symbol row is instead kept until it has gone untouched for
+            // SYMBOL_RETENTION_SWEEPS sweep epochs; every other plane is
+            // collected as soon as it is unmarked and older than this epoch.
+            let collect_below = if plane == FamilyPlane::Symbol {
+                epoch.saturating_sub(SYMBOL_RETENTION_SWEEPS)
+            } else {
+                epoch
+            };
             loop {
                 if report.examined >= limit || deadline.is_some_and(|d| Instant::now() >= d) {
                     report.complete = false;
@@ -1357,7 +1371,7 @@ impl RegistryWriter<'_> {
                 let tx = c.transaction_with_behavior(TransactionBehavior::Immediate)?;
                 let keys: Vec<Vec<u8>> = tx
                     .prepare(select)?
-                    .query_map(params![epoch, after], |r| r.get(0))?
+                    .query_map(params![collect_below, after], |r| r.get(0))?
                     .collect::<rusqlite::Result<_>>()?;
                 if keys.is_empty() {
                     tx.commit()?;
@@ -1369,7 +1383,7 @@ impl RegistryWriter<'_> {
                     let protected = <[u8; 32]>::try_from(key.as_slice())
                         .map_or(true, |key| marked.contains(&key));
                     if !protected {
-                        report.deleted += tx.execute(delete, params![key, epoch])? as u64;
+                        report.deleted += tx.execute(delete, params![key, collect_below])? as u64;
                     }
                 }
                 tx.commit()?;
@@ -1662,6 +1676,7 @@ pub fn v3_format_report(path: &Path) -> Result<V3FormatReport> {
         Some("semantic") => Kind::Plane(FamilyPlane::Semantic),
         Some("callgraph") => Kind::Plane(FamilyPlane::Callgraph),
         Some("trigram") => Kind::Plane(FamilyPlane::Trigram),
+        Some("symbol") => Kind::Plane(FamilyPlane::Symbol),
         _ => return Err(Error::InvalidIdentity),
     };
     let database = connections::shared(path, kind, false, true)?;

crates/aft/src/cli/prune_legacy.rs

@@ -20,7 +20,10 @@ fn usage() -> String {
 
 Lists every legacy (pre-view) index set under the AFT storage root with its
 bytes, and with --yes deletes the sets whose import into per-checkout views
-completed at least {days} days ago. Without --yes it is a dry run.
+or into the v3 stores completed at least {days} days ago. A set that a running
+import or a v3 base build may still read is kept. Older v1 views and blob
+stores and v2 family stores are listed as retained and never deleted. Without
+--yes it is a dry run.
 
 {OPERATOR_CONTRACT}
 

crates/aft/src/migration/mod.rs

@@ -15,6 +15,9 @@
 
 pub mod per_checkout;
 pub mod prune_legacy;
+pub mod v3_import;
+pub mod v3_ledger;
+pub mod v3_symbols;
 
 use std::collections::BTreeMap;
 use std::fmt;