warm-root: list the prefetch from a launch that restores the snapshot

fleet/ami/check.sh

@@ -233,6 +233,18 @@ grep -qx 'DefaultDependencies=no' fleet/ami/host/fcvm-fleet-prefetch.service &&
   no "fcvm-fleet-prefetch.service must start once local file systems are up (DefaultDependencies=no)"
 sed -n '/^prep)/,/;;/p' fleet/ami/host/build.sh | grep -qE '^  systemctl enable .*fcvm-fleet-prefetch\.service' ||
   no "host/build.sh prep does not enable fcvm-fleet-prefetch.service"
+# warm-root.sh lists the prefetch from a launch that restores the snapshot (what a woken
+# host's slots run), not from the first launch, which also boots and exports the image.
+wr=fleet/ami/host/warm-root.sh
+first=$(grep -n '^launch ||' "$wr" | head -1 | cut -d: -f1)
+traced=$(grep -n '^launch trace ||' "$wr" | head -1 | cut -d: -f1)
+# fcvm logs the hit at info and its default filter is warn: the launches set RUST_LOG.
+# A hit is logged before the load, and a failed load falls back to a cold boot: the traced
+# launch's own log must hold the hit and no miss.
+[ -n "$first" ] && [ -n "$traced" ] && [ "$first" -lt "$traced" ] &&
+  grep -qF "grep -q 'Pre-start snapshot hit' \"\$LOG.trace\" && ! grep -q 'Snapshot miss' \"\$LOG.trace\"" "$wr" &&
+  grep -qE 'RUST_LOG=[^ ]*info[^ ]* +"\$\{trace\[@\]\}"|RUST_LOG=info' "$wr" ||
+  no "warm-root.sh must list the prefetch from a second, traced launch that hits the first one's snapshot"
 # prefetch.sh reads every listed file in 1 MiB pieces (a file of 2 MiB and one byte is three
 # pieces) and skips a listed file the image lacks.
 pf=$(mktemp -d)

fleet/ami/host/warm-root.sh

@@ -39,23 +39,41 @@ P=$(curl -fsS -H "X-aws-ec2-metadata-token: $T" "http://169.254.169.254/latest/m
 [ -n "$P" ] || { echo "warm-root: this instance has no IPv6 prefix (build-ami.sh launch host gives it one)"; exit 1; }
 # The agent's slot address: the prefix with the GPU index plus two in its last group.
 A=$(python3 -c 'import ipaddress,sys; print(ipaddress.ip_network(sys.argv[1], strict=False).network_address + 2)' "$P")
-t0=$(date +%s%N)
-setsid strace -f -qq --seccomp-bpf -e trace=openat,execve -o /tmp/warm-root.strace \
-  fcvm podman run --name warm-root --kernel-profile gpu --gpu "$G" --publish 127.0.0.1:20099:8080 \
-  --network routed --ipv6-prefix "$P" --ipv6-address "$A" --publish 127.0.0.1:20098:443 \
-  --map "$TLS_DIR":/run/fleet-tls:ro --env TLS_CERT_FILE=/run/fleet-tls/tls.json --env TLS_PORT=443 "$IMG" \
-  > "$LOG" 2>&1 < /dev/null &
-WP=$!
-ok=0
-for _ in $(seq 1 1800); do
-  if curl -fsS -m 2 http://127.0.0.1:20099/health 2>/dev/null | jq -e '.ready == true and .frame > 0' >/dev/null 2>&1; then ok=1; break; fi
-  kill -0 $WP 2>/dev/null || break
-  sleep 1
-done
-echo "warm-root: ok=$ok first_frame_ms=$(( ($(date +%s%N) - t0) / 1000000 )) gpu=$G"
-pkill -TERM -x fcvm; wait $WP
+# One launch, as the agent launches a slot; with "trace", under strace into /tmp/warm-root.strace.
+# At info, so the log says whether the launch restored a snapshot (fcvm defaults to warn).
+launch() {
+  local trace=()
+  [ "${1:-}" = trace ] && trace=(strace -f -qq --seccomp-bpf -e "trace=openat,execve" -o /tmp/warm-root.strace)
+  local t0; t0=$(date +%s%N)
+  RUST_LOG=info setsid "${trace[@]}" \
+    fcvm podman run --name warm-root --kernel-profile gpu --gpu "$G" --publish 127.0.0.1:20099:8080 \
+    --network routed --ipv6-prefix "$P" --ipv6-address "$A" --publish 127.0.0.1:20098:443 \
+    --map "$TLS_DIR":/run/fleet-tls:ro --env TLS_CERT_FILE=/run/fleet-tls/tls.json --env TLS_PORT=443 "$IMG" \
+    > "$LOG.${1:-launch}" 2>&1 < /dev/null &
+  local wp=$! ok=0
+  for _ in $(seq 1 1800); do
+    if curl -fsS -m 2 http://127.0.0.1:20099/health 2>/dev/null | jq -e '.ready == true and .frame > 0' >/dev/null 2>&1; then ok=1; break; fi
+    kill -0 $wp 2>/dev/null || break
+    sleep 1
+  done
+  echo "warm-root: ${1:-launch} ok=$ok first_frame_ms=$(( ($(date +%s%N) - t0) / 1000000 )) gpu=$G"
+  pkill -TERM -x fcvm; wait $wp
+  cat "$LOG.${1:-launch}" >> "$LOG"
+  [ $ok = 1 ] || { tail -20 "$LOG.${1:-launch}"; return 1; }
+}
+# The first launch boots the VM and takes the pre-start snapshot (and exports the image for
+# root's podman store). The list comes from a second launch, which restores that snapshot
+# as a woken host's slots do: listed from the first, it named 6163 files (334 MiB, mostly
+# image layers the export read) that the prefetch then read for 20 s at every boot, beside
+# cloud-init and the store (2026-10-27).
+: > "$LOG"; rm -f "$LOG.launch" "$LOG.trace"
+launch || { rm -rf "$TLS_DIR"; exit 1; }
+launch trace || { rm -rf "$TLS_DIR"; exit 1; }
 rm -rf "$TLS_DIR"
-[ $ok = 1 ] || { tail -20 "$LOG"; exit 1; }
+# The traced launch must have restored the snapshot. fcvm logs the hit before it loads the
+# snapshot, and a load that fails is logged as a miss and cold-boots instead.
+{ grep -q 'Pre-start snapshot hit' "$LOG.trace" && ! grep -q 'Snapshot miss' "$LOG.trace"; } ||
+  { echo "warm-root: the traced launch did not restore the first one's snapshot"; exit 1; }
 grep -oE '(openat\([^"]*|execve\()"[^"]+"' /tmp/warm-root.strace | sed -E 's/.*"([^"]+)"$/\1/' |
   grep '^/' | grep -vE '^/(proc|sys|dev|run|tmp|mnt/fcvm-btrfs)/' | sort -u |
   while read -r f; do [ -f "$f" ] && echo "$f"; done > "$LIST"