fix(backup): refuse incomplete restore points after interruption

.changeset/interrupted-restore-point.md

@@ -0,0 +1,5 @@
+---
+"ftw": patch
+---
+
+Record complete restore points before activation. Refuse incomplete or older unverified points before reverting, and retain moved data if rollback fails. Full backups from older releases still import normally.

go/internal/backup/archive.go

@@ -34,6 +34,8 @@ const (
 
 var ensureBackupDiskSpace = state.EnsureDiskSpace
 
+var moveRestoreEntry = os.Rename
+
 // ComponentInventory records the independently versioned runtime that created
 // the backup. Driver artifacts themselves are also captured under data/.
 type ComponentInventory struct {
@@ -96,8 +98,9 @@ type Info struct {
 }
 
 type RestoreResult struct {
-	DataDir   string `json:"data_dir"`
-	SafetyDir string `json:"safety_dir,omitempty"`
+	DataDir        string `json:"data_dir"`
+	SafetyDir      string `json:"safety_dir,omitempty"`
+	SafetyComplete bool   `json:"safety_complete"`
 }
 
 // Inspect performs a full verification and returns stable archive metadata.
@@ -867,6 +870,9 @@ func RestoreContents(archivePath, dataDir string, now time.Time) (RestoreResult,
 	if !info.IsDir() {
 		return RestoreResult{}, fmt.Errorf("backup: restore target is not a directory: %s", dataDir)
 	}
+	if err := checkRestorePointsBeforeRestore(dataDir); err != nil {
+		return RestoreResult{}, err
+	}
 	manifest, err := verifyInWorkspace(archivePath, dataDir)
 	if err != nil {
 		return RestoreResult{}, err
@@ -900,27 +906,33 @@ func RestoreContents(archivePath, dataDir string, now time.Time) (RestoreResult,
 		_ = os.Remove(safetyDir)
 		return RestoreResult{}, err
 	}
+	if err := writeRestorePoint(safetyDir, current, false); err != nil {
+		return RestoreResult{}, fmt.Errorf("backup: record previous file list: %w", err)
+	}
 	movedCurrent := make([]string, 0, len(current))
 	for _, name := range current {
-		if err := os.Rename(filepath.Join(dataDir, name), filepath.Join(safetyDir, name)); err != nil {
-			rollbackErr := restoreMoveSet(safetyDir, dataDir, movedCurrent)
-			_ = os.Remove(safetyDir)
+		if err := moveRestoreEntry(filepath.Join(dataDir, name), filepath.Join(safetyDir, name)); err != nil {
+			rollbackErr := rollbackRestorePoint(safetyDir, dataDir, movedCurrent)
 			return RestoreResult{}, errors.Join(fmt.Errorf("backup: preserve current %s: %w", name, err), rollbackErr)
 		}
 		movedCurrent = append(movedCurrent, name)
 	}
+	if err := completeRestorePoint(dataDir, safetyDir, current); err != nil {
+		rollbackErr := rollbackRestorePoint(safetyDir, dataDir, movedCurrent)
+		return RestoreResult{}, errors.Join(fmt.Errorf("backup: confirm previous files: %w", err), rollbackErr)
+	}
 
 	restored, err := topLevelEntries(staging)
 	if err != nil {
-		rollbackErr := restoreMoveSet(safetyDir, dataDir, movedCurrent)
+		rollbackErr := rollbackRestorePoint(safetyDir, dataDir, movedCurrent)
 		return RestoreResult{}, errors.Join(err, rollbackErr)
 	}
 	activated := make([]string, 0, len(restored))
 	for _, name := range restored {
-		if err := os.Rename(filepath.Join(staging, name), filepath.Join(dataDir, name)); err != nil {
+		if err := moveRestoreEntry(filepath.Join(staging, name), filepath.Join(dataDir, name)); err != nil {
 			rollbackErr := errors.Join(
 				restoreMoveSet(dataDir, staging, activated),
-				restoreMoveSet(safetyDir, dataDir, movedCurrent),
+				rollbackRestorePoint(safetyDir, dataDir, movedCurrent),
 			)
 			return RestoreResult{}, errors.Join(fmt.Errorf("backup: activate restored %s: %w", name, err), rollbackErr)
 		}
@@ -929,12 +941,12 @@ func RestoreContents(archivePath, dataDir string, now time.Time) (RestoreResult,
 	if err := syncDir(dataDir); err != nil {
 		rollbackErr := errors.Join(
 			restoreMoveSet(dataDir, staging, activated),
-			restoreMoveSet(safetyDir, dataDir, movedCurrent),
+			rollbackRestorePoint(safetyDir, dataDir, movedCurrent),
 			syncDir(dataDir),
 		)
 		return RestoreResult{}, errors.Join(fmt.Errorf("backup: sync activated restore: %w", err), rollbackErr)
 	}
-	return RestoreResult{DataDir: dataDir, SafetyDir: safetyDir}, nil
+	return RestoreResult{DataDir: dataDir, SafetyDir: safetyDir, SafetyComplete: true}, nil
 }
 
 // RevertContents puts a RestoreContents safety directory back into service.
@@ -953,12 +965,16 @@ func RevertContents(dataDir, safetyDir string) (RestoreResult, error) {
 	if filepath.Dir(safetyDir) != dataDir || !strings.HasPrefix(filepath.Base(safetyDir), ".ftw-pre-restore-") {
 		return RestoreResult{}, errors.New("backup: safety directory is not a direct FTW restore point")
 	}
-	if info, statErr := os.Stat(safetyDir); statErr != nil || !info.IsDir() {
+	if info, statErr := os.Lstat(safetyDir); statErr != nil || !info.IsDir() {
 		if statErr != nil {
 			return RestoreResult{}, statErr
 		}
 		return RestoreResult{}, errors.New("backup: safety path is not a directory")
 	}
+	if err := verifyRestorePoint(safetyDir); err != nil {
+		return RestoreResult{}, err
+	}
+	currentComplete := currentRestoreStateComplete(dataDir)
 	rejected, err := os.MkdirTemp(dataDir, ".ftw-rejected-restore-")
 	if err != nil {
 		return RestoreResult{}, err
@@ -973,30 +989,49 @@ func RevertContents(dataDir, safetyDir string) (RestoreResult, error) {
 	if err != nil {
 		return RestoreResult{}, err
 	}
+	if err := writeRestorePoint(rejected, current, false); err != nil {
+		return RestoreResult{}, fmt.Errorf("backup: record rejected file list: %w", err)
+	}
+	// Retain moved data even if an error prevents rollback. A failed revert
+	// must never delete readings that are absent from the source archive.
+	keepRejected = true
 	movedCurrent := make([]string, 0, len(current))
 	for _, name := range current {
-		if err := os.Rename(filepath.Join(dataDir, name), filepath.Join(rejected, name)); err != nil {
-			rollbackErr := restoreMoveSet(rejected, dataDir, movedCurrent)
+		if err := moveRestoreEntry(filepath.Join(dataDir, name), filepath.Join(rejected, name)); err != nil {
+			rollbackErr := rollbackRestorePoint(rejected, dataDir, movedCurrent)
 			return RestoreResult{}, errors.Join(fmt.Errorf("backup: stage rejected %s: %w", name, err), rollbackErr)
 		}
 		movedCurrent = append(movedCurrent, name)
 	}
+	if err := errors.Join(syncDir(dataDir), syncDir(rejected)); err != nil {
+		rollbackErr := rollbackRestorePoint(rejected, dataDir, movedCurrent)
+		return RestoreResult{}, errors.Join(fmt.Errorf("backup: confirm rejected files: %w", err), rollbackErr)
+	}
+	if currentComplete {
+		if err := writeRestorePoint(rejected, current, true); err != nil {
+			rollbackErr := rollbackRestorePoint(rejected, dataDir, movedCurrent)
+			return RestoreResult{}, errors.Join(fmt.Errorf("backup: confirm rejected file list: %w", err), rollbackErr)
+		}
+	}
 	previous, err := topLevelEntries(safetyDir)
 	if err != nil {
-		rollbackErr := restoreMoveSet(rejected, dataDir, movedCurrent)
+		rollbackErr := rollbackRestorePoint(rejected, dataDir, movedCurrent)
 		return RestoreResult{}, errors.Join(err, rollbackErr)
 	}
 	restored := make([]string, 0, len(previous))
 	for _, name := range previous {
-		if err := os.Rename(filepath.Join(safetyDir, name), filepath.Join(dataDir, name)); err != nil {
-			rollbackErr := errors.Join(
-				restoreMoveSet(dataDir, safetyDir, restored),
-				restoreMoveSet(rejected, dataDir, movedCurrent),
-			)
+		if err := moveRestoreEntry(filepath.Join(safetyDir, name), filepath.Join(dataDir, name)); err != nil {
+			rollbackErr := restoreMoveSet(dataDir, safetyDir, restored)
+			if rollbackErr == nil {
+				rollbackErr = rollbackRestorePoint(rejected, dataDir, movedCurrent)
+			}
 			return RestoreResult{}, errors.Join(fmt.Errorf("backup: reactivate previous %s: %w", name, err), rollbackErr)
 		}
 		restored = append(restored, name)
 	}
+	if err := os.Remove(filepath.Join(safetyDir, restorePointFile)); err != nil {
+		return RestoreResult{DataDir: dataDir, SafetyDir: rejected}, fmt.Errorf("backup: previous data is active but rejected restore remains at %s: %w", rejected, err)
+	}
 	if err := os.Remove(safetyDir); err != nil {
 		keepRejected = true
 		return RestoreResult{DataDir: dataDir, SafetyDir: rejected}, fmt.Errorf("backup: previous data is active but rejected restore remains at %s: %w", rejected, err)
@@ -1009,7 +1044,7 @@ func RevertContents(dataDir, safetyDir string) (RestoreResult, error) {
 	if err := syncDir(dataDir); err != nil {
 		return RestoreResult{}, err
 	}
-	return RestoreResult{DataDir: dataDir, SafetyDir: safetyDir}, nil
+	return RestoreResult{DataDir: dataDir, SafetyDir: safetyDir, SafetyComplete: currentComplete}, nil
 }
 
 func topLevelEntries(dir string, exclude ...string) ([]string, error) {
@@ -1035,15 +1070,15 @@ func topLevelEntries(dir string, exclude ...string) ([]string, error) {
 }
 
 func isRestoreInternalName(name string) bool {
-	return strings.HasPrefix(name, ".ftw-restore-stage-") ||
+	return name == restorePointFile || strings.HasPrefix(name, ".ftw-restore-stage-") ||
 		strings.HasPrefix(name, ".ftw-pre-restore-") ||
 		strings.HasPrefix(name, ".ftw-rejected-restore-")
 }
 
 func restoreMoveSet(fromDir, toDir string, names []string) error {
 	var result error
 	for i := len(names) - 1; i >= 0; i-- {
-		if err := os.Rename(filepath.Join(fromDir, names[i]), filepath.Join(toDir, names[i])); err != nil {
+		if err := moveRestoreEntry(filepath.Join(fromDir, names[i]), filepath.Join(toDir, names[i])); err != nil {
 			result = errors.Join(result, fmt.Errorf("backup: roll back move for %s: %w", names[i], err))
 		}
 	}

go/internal/backup/archive_test.go

@@ -440,6 +440,15 @@ func TestRestoreContentsAndRevertPreserveBothStates(t *testing.T) {
 	if err != nil || string(got) != "generation: backup\n" {
 		t.Fatalf("rejected restore not retained = %q err=%v", got, err)
 	}
+	// The rejected state is itself a complete point, so another revert can
+	// switch back without accepting a stale list from the previous state.
+	if _, err := RevertContents(dataDir, reverted.SafetyDir); err != nil {
+		t.Fatal(err)
+	}
+	got, err = os.ReadFile(filepath.Join(dataDir, "config.yaml"))
+	if err != nil || string(got) != "generation: backup\n" {
+		t.Fatalf("second revert config = %q err=%v", got, err)
+	}
 }
 
 func writeTestFile(t *testing.T, filename, body string) {

go/internal/backup/restore_crash_test.go

@@ -0,0 +1,158 @@
+//go:build !windows
+
+package backup
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/srcfl/ftw/go/internal/state"
+)
+
+func TestRestoreCrashChild(t *testing.T) {
+	phase := os.Getenv("FTW_RESTORE_CRASH_PHASE")
+	if phase == "" {
+		return
+	}
+	data := os.Getenv("FTW_RESTORE_CRASH_DATA")
+	marker := os.Getenv("FTW_RESTORE_CRASH_MARKER")
+	realMove := moveRestoreEntry
+	moved := 0
+	moveRestoreEntry = func(from, to string) error {
+		if err := realMove(from, to); err != nil {
+			return err
+		}
+		preserving := filepath.Dir(from) == data && strings.HasPrefix(filepath.Base(filepath.Dir(to)), ".ftw-pre-restore-")
+		activating := filepath.Dir(to) == data && strings.HasPrefix(filepath.Base(filepath.Dir(from)), ".ftw-restore-stage-")
+		if (phase == "preserving" && preserving) || (phase == "activating" && activating) {
+			moved++
+			if moved == 64 {
+				if err := os.WriteFile(marker, []byte(filepath.Dir(to)), 0o600); err != nil {
+					return err
+				}
+				// Hold a real, partly completed filesystem transaction for the
+				// parent's SIGKILL without depending on scheduler speed.
+				for {
+					time.Sleep(time.Minute)
+				}
+			}
+		}
+		return nil
+	}
+	if _, err := RestoreContents(os.Getenv("FTW_RESTORE_CRASH_ARCHIVE"), data, time.Now()); err != nil {
+		t.Fatal(err)
+	}
+	t.Fatal("restore completed without reaching the crash window")
+}
+
+func TestRestoreProcessCrashPreservesDataAndRefusesPartialPoints(t *testing.T) {
+	for _, phase := range []string{"preserving", "activating"} {
+		t.Run(phase, func(t *testing.T) {
+			root := t.TempDir()
+			source := filepath.Join(root, "source")
+			if err := os.Mkdir(source, 0o700); err != nil {
+				t.Fatal(err)
+			}
+			st, err := state.Open(filepath.Join(source, "state.db"))
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer st.Close()
+			if err := st.SaveConfig("goal", "synthetic saved goal"); err != nil {
+				t.Fatal(err)
+			}
+			for i := range 128 {
+				writeTestFile(t, filepath.Join(source, fmt.Sprintf("%04d-new", i)), "new data")
+			}
+			archive, err := Create(context.Background(), CreateOptions{State: st, StatePath: filepath.Join(source, "state.db"), DataDir: source, OutputDir: filepath.Join(root, "backups")})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if err := st.Close(); err != nil {
+				t.Fatal(err)
+			}
+			data := filepath.Join(root, "target")
+			for i := range 128 {
+				writeTestFile(t, filepath.Join(data, fmt.Sprintf("%04d-old", i)), "original data")
+			}
+			marker := filepath.Join(root, "crash-window")
+			child := exec.Command(os.Args[0], "-test.run=^TestRestoreCrashChild$", "-test.count=1")
+			child.Env = append(os.Environ(), "FTW_RESTORE_CRASH_PHASE="+phase, "FTW_RESTORE_CRASH_DATA="+data, "FTW_RESTORE_CRASH_ARCHIVE="+archive.Path, "FTW_RESTORE_CRASH_MARKER="+marker)
+			log, err := os.Create(filepath.Join(root, "child.log"))
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer log.Close()
+			child.Stdout, child.Stderr = log, log
+			if err := child.Start(); err != nil {
+				t.Fatal(err)
+			}
+			t.Cleanup(func() { _ = child.Process.Kill() })
+			deadline := time.Now().Add(30 * time.Second)
+			for {
+				if _, err := os.Stat(marker); err == nil {
+					break
+				}
+				if time.Now().After(deadline) {
+					body, _ := os.ReadFile(filepath.Join(root, "child.log"))
+					t.Fatalf("crash window not reached: %s", body)
+				}
+				time.Sleep(time.Millisecond)
+			}
+			if err := child.Process.Kill(); err != nil {
+				t.Fatal(err)
+			}
+			if err := child.Wait(); err == nil {
+				t.Fatal("child survived SIGKILL")
+			} else if status, ok := child.ProcessState.Sys().(syscall.WaitStatus); !ok || !status.Signaled() || status.Signal() != syscall.SIGKILL {
+				t.Fatalf("child did not terminate through SIGKILL: %v", child.ProcessState)
+			}
+			points, err := filepath.Glob(filepath.Join(data, ".ftw-pre-restore-*"))
+			if err != nil || len(points) != 1 {
+				t.Fatalf("missing retained point: %v %v", points, err)
+			}
+			point := points[0]
+			if phase == "preserving" {
+				if _, err := RevertContents(data, point); err == nil {
+					t.Fatal("partial preservation was accepted")
+				}
+				if _, err := RestoreContents(archive.Path, data, time.Now()); err == nil {
+					t.Fatal("partial active root was certified by a retry")
+				}
+			} else {
+				result, err := RevertContents(data, point)
+				if err != nil || result.SafetyComplete {
+					t.Fatalf("complete source did not recover with partial target retained: %+v %v", result, err)
+				}
+				if _, err := RevertContents(data, point); err == nil {
+					t.Fatal("partial target was accepted for reactivation")
+				}
+			}
+			for i := range 128 {
+				name := fmt.Sprintf("%04d-old", i)
+				atRoot, rootErr := os.ReadFile(filepath.Join(data, name))
+				atPoint, pointErr := os.ReadFile(filepath.Join(point, name))
+				if phase == "activating" && (rootErr != nil || string(atRoot) != "original data") {
+					t.Fatalf("original %s not active after recovery", name)
+				}
+				if (rootErr != nil || string(atRoot) != "original data") && (pointErr != nil || string(atPoint) != "original data") {
+					t.Fatalf("original %s lost after %s crash", name, phase)
+				}
+			}
+			fresh := filepath.Join(root, "fresh-recovery")
+			if err := os.Mkdir(fresh, 0o700); err != nil {
+				t.Fatal(err)
+			}
+			if _, err := RestoreContents(archive.Path, fresh, time.Now()); err != nil {
+				t.Fatal("full backup did not recover on a separate copy:", err)
+			}
+		})
+	}
+}