Snapshot directories are btrfs subvolumes, and a diff is staged from its parent's

.claude/CLAUDE.md

@@ -2547,6 +2547,17 @@ fcvm runs two microVM backends behind a pluggable `Hypervisor` trait (`src/hyper
   owner, so it would get a disk it cannot write or remove). `rm -rf` removes subvolumes, so teardown is unchanged, but
   btrfs reclaims a removed subvolume's space on its cleaner thread after the rmdir returns, where an unlinked reflink
   freed its space at once; `df` right after many clone teardowns can read high for a while.
+- Each snapshot directory is a btrfs subvolume too, and a diff snapshot is staged as a subvolume snapshot of its
+  parent's directory, keeping only the parent's `memory.bin`, which the diff is then written into
+  (`stage_from_parent_subvolume`). A memory file written by in-place diffs fragments: one written by eight of them
+  held 11,824,233 extents, a reflink of it took 133 s and removing that copy 52 s, while the subvolume snapshot and
+  its delete each finished within a second (2026-10-27). The parent's directory was synced when its own snapshot
+  finished, so the flush a subvolume snapshot does first finds little. Snapshot directories are deleted with
+  `btrfs subvolume delete` (`remove_snapshot_dir`), which detaches them at once, where `rm -rf` would drop the memory
+  file's references one by one first. A parent that is not a subvolume (made before this, or on a store that cannot
+  make one) is reflinked as before. An unprivileged user may delete a subvolume only on a filesystem mounted with
+  `user_subvol_rm_allowed`; fcvm's own loopback store is mounted with it, and on a btrfs store mounted without it a
+  rootless delete falls back to removing the files one by one.
 - Initrd: `/mnt/fcvm-btrfs/initrd/fc-agent-{sha}.initrd` (injects fc-agent at boot)
 
 **Per-VM Disk Sizing (`--rootfs-size`):**

src/setup/storage.rs

@@ -233,9 +233,12 @@ fn ensure_storage_with(
 
     // Mount the loopback image
     info!("Mounting btrfs filesystem...");
+    // user_subvol_rm_allowed lets an unprivileged owner delete the subvolumes fcvm makes for
+    // snapshot directories; without it a rootless delete removes them file by file, one
+    // reference per extent of each memory file.
     let status = Command::new("mount")
         .arg("-o")
-        .arg("loop")
+        .arg("loop,user_subvol_rm_allowed")
         .arg(&loopback_image)
         .arg(&mount_point)
         .status()

src/storage/disk.rs

@@ -39,7 +39,7 @@ fn is_snapshot_disk_subvolume(dir: &Path) -> bool {
 /// would get a disk it cannot open for writing and a directory it cannot
 /// remove at teardown; it reflinks the disk instead, which makes a copy it
 /// owns. Root can write and remove anything.
-fn can_own_subvolume_snapshot_of(base_dir: &Path) -> bool {
+pub fn can_own_subvolume_snapshot_of(base_dir: &Path) -> bool {
     use std::os::unix::fs::MetadataExt;
     let euid = nix::unistd::geteuid().as_raw();
     euid == 0
@@ -76,13 +76,170 @@ pub async fn create_dir_or_subvolume(path: &Path) -> Result<()> {
     info!(
         path = %path.display(),
         %reason,
-        "not a btrfs subvolume; created as a directory (clones of it reflink the disk)"
+        "not a btrfs subvolume; created as a directory (copies of it reflink its files)"
     );
     fs::create_dir_all(path)
         .await
         .with_context(|| format!("creating {}", path.display()))
 }
 
+/// Remove a snapshot directory. A snapshot directory that is a btrfs subvolume (and its
+/// `disk/` subvolume) is deleted as a subvolume, which detaches it at once and leaves the
+/// cleaner thread to drop its extent references in the background: unlinking a memory file
+/// written by in-place diffs drops one reference per extent first, 52 to 93 s for one with
+/// 11.8 million extents. Anything else, or a subvolume this process may not delete (an
+/// unprivileged caller without user_subvol_rm_allowed), is removed file by file.
+pub async fn remove_snapshot_dir(dir: &Path) -> std::io::Result<()> {
+    if is_btrfs_subvolume(dir) {
+        let disk = dir.join(crate::storage::snapshot::SNAPSHOT_DISK_SUBDIR);
+        let disk_gone = !is_btrfs_subvolume(&disk) || delete_subvolume(&disk).await;
+        if disk_gone && delete_subvolume(dir).await && !dir.exists() {
+            return Ok(());
+        }
+    }
+    match fs::remove_dir_all(dir).await {
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
+        other => other,
+    }
+}
+
+/// Remove a snapshot create's leftover staging directory, and fail if it is still there: a
+/// subvolume snapshot made onto an existing directory lands inside it, and a subvolume
+/// cannot be created over one, so a leftover would be silently reused.
+pub async fn clear_staging(staging: &Path) -> Result<()> {
+    clear_staging_with(staging, remove_snapshot_dir(staging).await)
+}
+
+/// `clear_staging` after its removal returned `removed`.
+fn clear_staging_with(staging: &Path, removed: std::io::Result<()>) -> Result<()> {
+    if staging.exists() {
+        let reason = match removed {
+            Err(error) => error.to_string(),
+            Ok(()) => "it is still there after its removal".to_string(),
+        };
+        bail!(
+            "could not remove the leftover staging directory {}: {reason}",
+            staging.display()
+        );
+    }
+    Ok(())
+}
+
+/// Wait for btrfs to free the space of subvolumes deleted on the filesystem holding `path`
+/// (`btrfs subvolume sync`, bounded): a deleted subvolume's space comes back on the cleaner
+/// thread after the delete returns, so a free-space check right after one can read low.
+pub async fn wait_for_deleted_subvolumes(path: &Path) {
+    let sync = tokio::process::Command::new("btrfs")
+        .args(["subvolume", "sync"])
+        .arg(path)
+        .kill_on_drop(true)
+        .output();
+    match tokio::time::timeout(std::time::Duration::from_secs(120), sync).await {
+        Ok(Ok(output)) if output.status.success() => {}
+        Ok(Ok(output)) => debug!(path = %path.display(),
+            stderr = %String::from_utf8_lossy(&output.stderr).trim(), "btrfs subvolume sync refused"),
+        Ok(Err(error)) => debug!(path = %path.display(), %error, "btrfs tool unavailable"),
+        Err(_) => warn!(path = %path.display(), "btrfs subvolume sync did not finish in 120 s"),
+    }
+}
+
+/// `btrfs subvolume delete`, without waiting for the cleaner. Whether it succeeded.
+async fn delete_subvolume(path: &Path) -> bool {
+    let output = tokio::process::Command::new("btrfs")
+        .args(["subvolume", "delete"])
+        .arg(path)
+        .output()
+        .await;
+    match output {
+        Ok(output) if output.status.success() => true,
+        Ok(output) => {
+            debug!(path = %path.display(), stderr = %String::from_utf8_lossy(&output.stderr).trim(),
+                "btrfs subvolume delete refused; removing file by file");
+            false
+        }
+        Err(error) => {
+            debug!(path = %path.display(), %error, "btrfs tool unavailable; removing file by file");
+            false
+        }
+    }
+}
+
+/// Stage a diff snapshot's directory as a subvolume snapshot of its parent's, keeping only
+/// the parent's memory file, which the diff is then written into. A subvolume snapshot costs
+/// one transaction commit whatever the memory file's extent count, where a reflink of the
+/// file costs one reference per extent: 133 s for a memory file of 11.8 million extents
+/// written by eight in-place diffs. Returns false, leaving no `staging`, when the parent's
+/// directory is not a subvolume this process may snapshot or the snapshot fails; the caller
+/// then reflinks the memory file. A failed staging that cannot be removed is an error, since
+/// the reflink path would otherwise create its directory over the parent's leftover files.
+///
+/// The parent's other files (its config, vmstate, recorded working set, locks) are removed
+/// from the staging directory, and so is its `disk/`, which a subvolume snapshot shows as an
+/// empty placeholder directory when it is a subvolume of its own.
+pub async fn stage_from_parent_subvolume(parent: &Path, staging: &Path) -> Result<bool> {
+    if !is_btrfs_subvolume(parent) || !can_own_subvolume_snapshot_of(parent) {
+        return Ok(false);
+    }
+    if staging.exists() {
+        bail!(
+            "{} exists; a subvolume snapshot would land inside it",
+            staging.display()
+        );
+    }
+    let started = std::time::Instant::now();
+    let output = tokio::process::Command::new("btrfs")
+        .args(["subvolume", "snapshot"])
+        .arg(parent)
+        .arg(staging)
+        .output()
+        .await;
+    let refused = match output {
+        Ok(output) if output.status.success() => None,
+        Ok(output) => Some(String::from_utf8_lossy(&output.stderr).trim().to_string()),
+        Err(error) => Some(format!("btrfs tool unavailable: {error}")),
+    };
+    if let Some(reason) = refused {
+        info!(parent = %parent.display(), %reason,
+            "could not snapshot the parent's subvolume; reflinking its memory file instead");
+        clear_staging(staging).await?;
+        return Ok(false);
+    }
+    let snapshot_ms = started.elapsed().as_millis() as u64;
+    if let Err(error) = keep_only_memory_file(staging).await {
+        warn!(staging = %staging.display(), %error,
+            "could not clear the parent's other files from the staged snapshot; reflinking instead");
+        clear_staging(staging).await?;
+        return Ok(false);
+    }
+    debug!(parent = %parent.display(), staging = %staging.display(), snapshot_ms,
+        "staged the snapshot as a subvolume snapshot of its parent");
+    Ok(true)
+}
+
+/// Remove everything in `dir` but the memory file: files, directories, and the empty
+/// placeholder a nested subvolume leaves in a subvolume snapshot.
+async fn keep_only_memory_file(dir: &Path) -> Result<()> {
+    let mut entries = fs::read_dir(dir)
+        .await
+        .with_context(|| format!("reading {}", dir.display()))?;
+    while let Some(entry) = entries.next_entry().await? {
+        if entry.file_name() == crate::storage::snapshot::SNAPSHOT_MEMORY_FILE {
+            continue;
+        }
+        let path = entry.path();
+        if entry.file_type().await?.is_dir() {
+            fs::remove_dir_all(&path)
+                .await
+                .with_context(|| format!("removing {}", path.display()))?;
+        } else {
+            fs::remove_file(&path)
+                .await
+                .with_context(|| format!("removing {}", path.display()))?;
+        }
+    }
+    Ok(())
+}
+
 /// Configuration for a VM disk
 #[derive(Debug, Clone)]
 pub struct DiskConfig {
@@ -544,3 +701,81 @@ pub fn parse_size(s: &str) -> Result<u64> {
     num.checked_mul(multiplier)
         .with_context(|| format!("size overflow: {} * {}", num, multiplier))
 }
+
+#[cfg(test)]
+mod snapshot_dir_tests {
+    use super::*;
+
+    /// A leftover staging directory that cannot be removed fails the create: a subvolume
+    /// snapshot made onto it would land inside it. The removal's failure is given to the
+    /// check, since root removes files from a read-only directory.
+    #[tokio::test]
+    async fn an_unremovable_leftover_staging_directory_fails() {
+        let dir = tempfile::tempdir().unwrap();
+        let staging = dir.path().join("snap.creating");
+        std::fs::create_dir_all(staging.join("disk")).unwrap();
+        let refused = std::io::Error::from_raw_os_error(libc::EPERM);
+        let error = clear_staging_with(&staging, Err(refused))
+            .expect_err("an unremovable leftover must fail the create");
+        assert!(
+            error.to_string().contains("leftover staging directory"),
+            "{error}"
+        );
+        clear_staging_with(&staging, Ok(())).expect_err("a leftover still present must fail");
+        clear_staging(&staging).await.unwrap();
+        assert!(!staging.exists());
+        clear_staging(&staging).await.unwrap();
+    }
+
+    /// A staged snapshot keeps the parent's memory file and nothing else of its directory.
+    #[tokio::test]
+    async fn staging_keeps_only_the_memory_file() {
+        let dir = tempfile::tempdir().unwrap();
+        let staged = dir.path();
+        std::fs::write(staged.join("memory.bin"), b"memory").unwrap();
+        std::fs::write(staged.join("config.json"), b"{}").unwrap();
+        std::fs::write(staged.join("memory.bin.working-set"), b"set").unwrap();
+        std::fs::create_dir(staged.join("disk")).unwrap();
+        std::fs::write(staged.join("disk").join("disk.raw"), b"disk").unwrap();
+        std::fs::create_dir(staged.join("empty-placeholder")).unwrap();
+        keep_only_memory_file(staged).await.unwrap();
+        let mut left: Vec<String> = std::fs::read_dir(staged)
+            .unwrap()
+            .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned())
+            .collect();
+        left.sort();
+        assert_eq!(left, ["memory.bin"]);
+        assert_eq!(std::fs::read(staged.join("memory.bin")).unwrap(), b"memory");
+    }
+
+    /// A parent directory that is not a subvolume stages nothing, and the caller reflinks.
+    #[tokio::test]
+    async fn a_plain_parent_directory_is_not_staged() {
+        let dir = tempfile::tempdir().unwrap();
+        let parent = dir.path().join("parent");
+        std::fs::create_dir(&parent).unwrap();
+        std::fs::write(parent.join("memory.bin"), b"memory").unwrap();
+        let staging = dir.path().join("parent.creating");
+        assert!(!stage_from_parent_subvolume(&parent, &staging)
+            .await
+            .unwrap());
+        assert!(
+            !staging.exists(),
+            "a refused staging left {}",
+            staging.display()
+        );
+    }
+
+    /// A plain snapshot directory is removed file by file, and a missing one is not an error.
+    #[tokio::test]
+    async fn a_plain_snapshot_directory_is_removed() {
+        let dir = tempfile::tempdir().unwrap();
+        let snapshot = dir.path().join("snap");
+        std::fs::create_dir_all(snapshot.join("disk")).unwrap();
+        std::fs::write(snapshot.join("memory.bin"), b"memory").unwrap();
+        std::fs::write(snapshot.join("disk").join("disk.raw"), b"disk").unwrap();
+        remove_snapshot_dir(&snapshot).await.unwrap();
+        assert!(!snapshot.exists());
+        remove_snapshot_dir(&snapshot).await.unwrap();
+    }
+}

src/storage/snapshot.rs

@@ -87,6 +87,9 @@ impl std::fmt::Display for SnapshotKind {
 /// kernel flushes nothing but the disks (reflinks, so already clean) to take it.
 pub const SNAPSHOT_DISK_SUBDIR: &str = "disk";
 
+/// The memory file's name in a snapshot's directory.
+pub const SNAPSHOT_MEMORY_FILE: &str = "memory.bin";
+
 /// Where a snapshot directory keeps its disks.
 pub fn snapshot_disk_dir(snapshot_dir: &Path) -> PathBuf {
     snapshot_dir.join(SNAPSHOT_DISK_SUBDIR)
@@ -482,7 +485,7 @@ impl SnapshotManager {
                 .with_context(|| format!("locking snapshot '{}' for deletion", name))?;
 
         if snapshot_dir.exists() {
-            fs::remove_dir_all(&snapshot_dir)
+            crate::storage::disk::remove_snapshot_dir(&snapshot_dir)
                 .await
                 .context("removing snapshot directory")?;
 
@@ -527,7 +530,7 @@ impl SnapshotManager {
             return Ok(false);
         }
 
-        fs::remove_dir_all(&snapshot_dir)
+        crate::storage::disk::remove_snapshot_dir(&snapshot_dir)
             .await
             .context("removing invalid snapshot generation")?;
         info!(snapshot = name, "snapshot generation deleted");

src/uffd/release.rs

@@ -54,8 +54,7 @@ use std::path::Path;
 use std::time::Duration;
 use tracing::{info, warn};
 
-/// The memory file's name in a snapshot's directory.
-const MEMORY_FILE: &str = "memory.bin";
+use crate::storage::snapshot::SNAPSHOT_MEMORY_FILE as MEMORY_FILE;
 
 /// The endings of the directories a snapshot create works in beside the installed
 /// snapshots (`commands::common::snapshot_sibling`).

tests/test_diff_snapshot.rs

@@ -404,6 +404,37 @@ async fn test_user_snapshot_from_clone_uses_parent() -> Result<()> {
         anyhow::bail!("User snapshot from clone failed: {}", stderr);
     }
     println!("  ✓ User snapshot created: {}", snapshot2_name);
+    // On a btrfs store each snapshot directory is a subvolume, and a diff is staged as a
+    // subvolume snapshot of its parent's, which costs one commit where a reflink of the
+    // parent's memory file costs one reference per extent.
+    let snapshot1_dir = snapshot_dir().join(&snapshot1_name);
+    let on_btrfs = nix::sys::statfs::statfs(&snapshot1_dir)
+        .map(|fs| fs.filesystem_type() == nix::sys::statfs::BTRFS_SUPER_MAGIC)
+        .unwrap_or(false);
+    if on_btrfs {
+        assert!(
+            fcvm::storage::disk::is_btrfs_subvolume(&snapshot1_dir),
+            "the first snapshot's directory {} is not a btrfs subvolume",
+            snapshot1_dir.display()
+        );
+        assert!(
+            stderr.contains("staged as a subvolume snapshot of the parent"),
+            "the diff was not staged from the parent's subvolume:\n{stderr}"
+        );
+        let snapshot2_dir = snapshot_dir().join(&snapshot2_name);
+        assert!(
+            fcvm::storage::disk::is_btrfs_subvolume(&snapshot2_dir),
+            "the diff snapshot's directory is not a btrfs subvolume"
+        );
+        // The staging snapshot holds an empty placeholder where the parent's disk subvolume
+        // was; the diff's own disks must be a subvolume again, or its clones reflink them.
+        assert!(
+            fcvm::storage::disk::is_btrfs_subvolume(
+                &snapshot2_dir.join(fcvm::storage::snapshot::SNAPSHOT_DISK_SUBDIR)
+            ),
+            "the diff snapshot's disk directory is not a btrfs subvolume"
+        );
+    }
     // What the merge did, for the log of this run.
     if let Some(line) = stderr
         .lines()
@@ -477,6 +508,23 @@ async fn test_user_snapshot_from_clone_uses_parent() -> Result<()> {
         stderr
     );
 
+    // Deleting a snapshot that is a subvolume holding a subvolume removes both.
+    let deleted = tokio::process::Command::new(&fcvm_path)
+        .args(["snapshots", "delete", "--force", &snapshot2_name])
+        .output()
+        .await
+        .context("running snapshots delete")?;
+    assert!(
+        deleted.status.success(),
+        "snapshots delete failed: {}",
+        String::from_utf8_lossy(&deleted.stderr)
+    );
+    assert!(
+        !snapshot2_dir.exists(),
+        "snapshots delete left {}",
+        snapshot2_dir.display()
+    );
+
     println!("\n✅ USER SNAPSHOT FROM CLONE TEST PASSED!");
     Ok(())
 }