diff --git a/fleet/ami/check.sh b/fleet/ami/check.sh index 16aee022c..d716ced23 100644 --- a/fleet/ami/check.sh +++ b/fleet/ami/check.sh @@ -233,6 +233,18 @@ grep -qx 'DefaultDependencies=no' fleet/ami/host/fcvm-fleet-prefetch.service && no "fcvm-fleet-prefetch.service must start once local file systems are up (DefaultDependencies=no)" sed -n '/^prep)/,/;;/p' fleet/ami/host/build.sh | grep -qE '^ systemctl enable .*fcvm-fleet-prefetch\.service' || no "host/build.sh prep does not enable fcvm-fleet-prefetch.service" +# warm-root.sh lists the prefetch from a launch that restores the snapshot (what a woken +# host's slots run), not from the first launch, which also boots and exports the image. +wr=fleet/ami/host/warm-root.sh +first=$(grep -n '^launch ||' "$wr" | head -1 | cut -d: -f1) +traced=$(grep -n '^launch trace ||' "$wr" | head -1 | cut -d: -f1) +# fcvm logs the hit at info and its default filter is warn: the launches set RUST_LOG. +# A hit is logged before the load, and a failed load falls back to a cold boot: the traced +# launch's own log must hold the hit and no miss. +[ -n "$first" ] && [ -n "$traced" ] && [ "$first" -lt "$traced" ] && + grep -qF "grep -q 'Pre-start snapshot hit' \"\$LOG.trace\" && ! grep -q 'Snapshot miss' \"\$LOG.trace\"" "$wr" && + grep -qE 'RUST_LOG=[^ ]*info[^ ]* +"\$\{trace\[@\]\}"|RUST_LOG=info' "$wr" || + no "warm-root.sh must list the prefetch from a second, traced launch that hits the first one's snapshot" # prefetch.sh reads every listed file in 1 MiB pieces (a file of 2 MiB and one byte is three # pieces) and skips a listed file the image lacks. pf=$(mktemp -d) diff --git a/fleet/ami/host/warm-root.sh b/fleet/ami/host/warm-root.sh index 90c76765d..9fc594246 100755 --- a/fleet/ami/host/warm-root.sh +++ b/fleet/ami/host/warm-root.sh @@ -39,23 +39,41 @@ P=$(curl -fsS -H "X-aws-ec2-metadata-token: $T" "http://169.254.169.254/latest/m [ -n "$P" ] || { echo "warm-root: this instance has no IPv6 prefix (build-ami.sh launch host gives it one)"; exit 1; } # The agent's slot address: the prefix with the GPU index plus two in its last group. A=$(python3 -c 'import ipaddress,sys; print(ipaddress.ip_network(sys.argv[1], strict=False).network_address + 2)' "$P") -t0=$(date +%s%N) -setsid strace -f -qq --seccomp-bpf -e trace=openat,execve -o /tmp/warm-root.strace \ - fcvm podman run --name warm-root --kernel-profile gpu --gpu "$G" --publish 127.0.0.1:20099:8080 \ - --network routed --ipv6-prefix "$P" --ipv6-address "$A" --publish 127.0.0.1:20098:443 \ - --map "$TLS_DIR":/run/fleet-tls:ro --env TLS_CERT_FILE=/run/fleet-tls/tls.json --env TLS_PORT=443 "$IMG" \ - > "$LOG" 2>&1 < /dev/null & -WP=$! -ok=0 -for _ in $(seq 1 1800); do - if curl -fsS -m 2 http://127.0.0.1:20099/health 2>/dev/null | jq -e '.ready == true and .frame > 0' >/dev/null 2>&1; then ok=1; break; fi - kill -0 $WP 2>/dev/null || break - sleep 1 -done -echo "warm-root: ok=$ok first_frame_ms=$(( ($(date +%s%N) - t0) / 1000000 )) gpu=$G" -pkill -TERM -x fcvm; wait $WP +# One launch, as the agent launches a slot; with "trace", under strace into /tmp/warm-root.strace. +# At info, so the log says whether the launch restored a snapshot (fcvm defaults to warn). +launch() { + local trace=() + [ "${1:-}" = trace ] && trace=(strace -f -qq --seccomp-bpf -e "trace=openat,execve" -o /tmp/warm-root.strace) + local t0; t0=$(date +%s%N) + RUST_LOG=info setsid "${trace[@]}" \ + fcvm podman run --name warm-root --kernel-profile gpu --gpu "$G" --publish 127.0.0.1:20099:8080 \ + --network routed --ipv6-prefix "$P" --ipv6-address "$A" --publish 127.0.0.1:20098:443 \ + --map "$TLS_DIR":/run/fleet-tls:ro --env TLS_CERT_FILE=/run/fleet-tls/tls.json --env TLS_PORT=443 "$IMG" \ + > "$LOG.${1:-launch}" 2>&1 < /dev/null & + local wp=$! ok=0 + for _ in $(seq 1 1800); do + if curl -fsS -m 2 http://127.0.0.1:20099/health 2>/dev/null | jq -e '.ready == true and .frame > 0' >/dev/null 2>&1; then ok=1; break; fi + kill -0 $wp 2>/dev/null || break + sleep 1 + done + echo "warm-root: ${1:-launch} ok=$ok first_frame_ms=$(( ($(date +%s%N) - t0) / 1000000 )) gpu=$G" + pkill -TERM -x fcvm; wait $wp + cat "$LOG.${1:-launch}" >> "$LOG" + [ $ok = 1 ] || { tail -20 "$LOG.${1:-launch}"; return 1; } +} +# The first launch boots the VM and takes the pre-start snapshot (and exports the image for +# root's podman store). The list comes from a second launch, which restores that snapshot +# as a woken host's slots do: listed from the first, it named 6163 files (334 MiB, mostly +# image layers the export read) that the prefetch then read for 20 s at every boot, beside +# cloud-init and the store (2026-10-27). +: > "$LOG"; rm -f "$LOG.launch" "$LOG.trace" +launch || { rm -rf "$TLS_DIR"; exit 1; } +launch trace || { rm -rf "$TLS_DIR"; exit 1; } rm -rf "$TLS_DIR" -[ $ok = 1 ] || { tail -20 "$LOG"; exit 1; } +# The traced launch must have restored the snapshot. fcvm logs the hit before it loads the +# snapshot, and a load that fails is logged as a miss and cold-boots instead. +{ grep -q 'Pre-start snapshot hit' "$LOG.trace" && ! grep -q 'Snapshot miss' "$LOG.trace"; } || + { echo "warm-root: the traced launch did not restore the first one's snapshot"; exit 1; } grep -oE '(openat\([^"]*|execve\()"[^"]+"' /tmp/warm-root.strace | sed -E 's/.*"([^"]+)"$/\1/' | grep '^/' | grep -vE '^/(proc|sys|dev|run|tmp|mnt/fcvm-btrfs)/' | sort -u | while read -r f; do [ -f "$f" ] && echo "$f"; done > "$LIST"