feat: Retriever can produce a .tar archive: BED-10076

README.md

@@ -114,9 +114,10 @@ PostgreSQL and Neo4j, uncompressed, gzip, and zstd fragments, bounded keyset
 scans, resumable dump checkpoints, checksum validation before load, optional
 deterministic property scrubbing, optional write-only Parquet sidecars for
 analytical readers, and a read-throughput benchmark mode. It can also package
-dumps as single HPKE/ML-KEM encrypted TAR archives. See
+dumps as single unencrypted TAR or HPKE/ML-KEM encrypted TAR archives. Archive
+file creation refuses existing output paths. See
 [cmd/retriever/README.md](cmd/retriever/README.md) for dump, Parquet sidecar,
-encrypted archive, load, scrubbed dump, metrics verification, and benchmark
+archive, load, scrubbed dump, metrics verification, and benchmark
 examples.
 The same import/export functionality is available to library consumers from
 `github.com/specterops/dawgs/retriever`; callers provide an already-open

cmd/retriever/README.md

@@ -163,7 +163,28 @@ The listener shuts down when the Retriever command exits, and failure to bind
 the requested address fails the command before database work starts. Treat all
 captured profiles as sensitive operational artifacts.
 
-## Encrypted Archives
+## Archives
+
+Pass `-archive-out` without `-recipient` to write an unencrypted TAR file
+alongside the collection directory:
+
+```bash
+retriever dump \
+  -connection "$CONNECTION_STRING" \
+  -out ./dumpdir \
+  -archive-out ./dump.tar
+```
+
+The TAR contains the manifest and its JSONL fragments. `-out` remains the
+collection directory; `-archive-out` adds a second artifact. Parquet sidecars,
+when requested, stay in the directory and are not included in the TAR.
+Existing archive files are refused, and partial archive files are removed if
+packaging fails.
+To load an unencrypted TAR, extract it into a collection directory and pass
+that directory to `retriever load -in`. The `retriever unpack` and
+`retriever load -archive` commands currently accept encrypted archives only.
+
+### Encrypted Archives
 
 Generate a recipient key pair before creating encrypted archives:
 
@@ -177,8 +198,8 @@ The private key is written as an unencrypted JSON key envelope with restrictive
 file permissions. Store it separately from archives and treat it as sensitive.
 Key generation refuses to replace existing key files.
 
-Pass `-archive-out` and `-recipient` to create a single encrypted TAR archive
-after a dump succeeds:
+Pass `-archive-out` with `-recipient` to write an encrypted TAR archive
+alongside the collection directory:
 
 ```bash
 retriever dump \
@@ -192,10 +213,10 @@ retriever dump \
 ```
 
 The archive layer uses an uncompressed TAR stream encrypted with HPKE using
-ML-KEM-1024, HKDF-SHA512, and AES-256-GCM. Fragment compression inside the dump
-directory remains controlled by `-compression`; the archive itself is not
-compressed. If archive creation fails, the command fails and removes partial
-archive output while leaving the completed dump directory for inspection.
+ML-KEM-1024, HKDF-SHA512, and AES-256-GCM. Fragment compression remains
+controlled by `-compression`; the archive itself is not compressed. If archive
+creation fails, the command removes partial archive output and leaves the
+completed collection directory for inspection.
 
 Unpack an encrypted archive with the private key:
 
@@ -247,7 +268,7 @@ retriever load \
   -in ./dumpdir
 ```
 
-Encrypted archives produced by `dump -archive-out` can be loaded directly:
+Encrypted archives produced by `dump -archive-out -recipient` can be loaded directly:
 
 ```bash
 retriever load \

cmd/retriever/main.go

@@ -92,8 +92,8 @@ func (s commandRuntime) runDump(ctx context.Context, args []string) error {
 	flags.BoolVar(&cfg.Force, "force", false, "Replace an existing non-empty output directory.")
 	flags.BoolVar(&cfg.Resume, "resume", false, "Resume an interrupted dump from its validated checkpoint.")
 	flags.BoolVar(&cfg.Parquet, "parquet", false, "Also write Parquet sidecars for each JSONL fragment.")
-	flags.StringVar(&archiveOut, "archive-out", "", "Optional encrypted archive output path.")
-	flags.StringVar(&recipientPath, "recipient", "", "Recipient public key for -archive-out.")
+	flags.StringVar(&archiveOut, "archive-out", "", "Optional TAR archive output path.")
+	flags.StringVar(&recipientPath, "recipient", "", "Recipient public key to encrypt -archive-out.")
 	flags.StringVar(&scrubValue, "scrub", string(cfg.Scrub), "Scrub mode: none or full.")
 	flags.StringVar(&cfg.Salt, "salt", "", "Scrub salt. Overrides RETRIEVER_SCRUB_SALT and is never written.")
 	flags.StringVar(&scrubConfig, "config", "", "Optional retriever TOML config for scrub classifier settings.")
@@ -134,16 +134,13 @@ func (s commandRuntime) runDump(ctx context.Context, args []string) error {
 
 	var archiveRecipient hpke.PublicKey
 	if strings.TrimSpace(archiveOut) != "" {
-		if strings.TrimSpace(recipientPath) == "" {
-			return fmt.Errorf("-archive-out requires -recipient")
+		if strings.TrimSpace(recipientPath) != "" {
+			var err error
+			archiveRecipient, err = retriever.LoadArchivePublicKey(recipientPath)
+			if err != nil {
+				return err
+			}
 		}
-
-		var err error
-		archiveRecipient, err = retriever.LoadArchivePublicKey(recipientPath)
-		if err != nil {
-			return err
-		}
-
 		if err := retriever.PreflightArchiveOutputPath(archiveOut); err != nil {
 			return err
 		}
@@ -175,10 +172,13 @@ func (s commandRuntime) runDump(ctx context.Context, args []string) error {
 
 	var archiveLine string
 	if strings.TrimSpace(archiveOut) != "" {
-		if err := retriever.WriteEncryptedCollectionArchiveFile(cfg.OutputDir, archiveOut, archiveRecipient); err != nil {
+		if err := retriever.WriteCollectionArchiveFile(retriever.ArchiveFileOptions{
+			CollectionDir: cfg.OutputDir,
+			OutputPath:    archiveOut,
+			Recipient:     archiveRecipient,
+		}); err != nil {
 			return err
 		}
-
 		archiveLine = fmt.Sprintf("archive: %s\n", archiveOut)
 	}
 

cmd/retriever/main_test.go

@@ -91,3 +91,35 @@ func TestDumpArchiveOutputPreflightBeforeDatabase(t *testing.T) {
 		t.Fatalf("expected archive preflight error before database open, got %v", err)
 	}
 }
+
+func TestDumpUnencryptedArchiveValidationBeforeDatabase(t *testing.T) {
+	runtime := commandRuntime{stdout: &bytes.Buffer{}, stderr: &bytes.Buffer{}}
+	dir := t.TempDir()
+	archivePath := filepath.Join(dir, "dump.tar")
+	dumpDir := filepath.Join(dir, "dump")
+
+	for name, testCase := range map[string]struct {
+		args []string
+		want string
+	}{
+		"missing output":            {[]string{"dump", "-archive-out", archivePath}, "output directory is required"},
+		"recipient without archive": {[]string{"dump", "-out", dumpDir, "-recipient", "public.key"}, "-recipient requires -archive-out"},
+	} {
+		t.Run(name, func(t *testing.T) {
+			err := runtime.run(context.Background(), testCase.args)
+			if err == nil || !strings.Contains(err.Error(), testCase.want) {
+				t.Fatalf("unexpected validation error: %v", err)
+			}
+		})
+	}
+
+	if err := os.WriteFile(archivePath, []byte("exists"), 0o600); err != nil {
+		t.Fatalf("write existing archive: %v", err)
+	}
+	err := runtime.run(context.Background(), []string{
+		"dump", "-out", dumpDir, "-archive-out", archivePath,
+	})
+	if err == nil || !strings.Contains(err.Error(), "already exists") {
+		t.Fatalf("expected archive preflight error before database open, got %v", err)
+	}
+}

cmd/retriever/packaging_plan.md

@@ -15,7 +15,7 @@ Assumption: "QC resistant" means quantum-computer-resistant cryptography.
 Use a raw TAR packaging layer plus an encrypted streaming envelope:
 
 ```text
-retriever dump -> dump directory -> raw tar stream -> HPKE/ML-KEM encrypted archive
+retriever dump -> dump directory -> raw tar stream -> archive file
 ```
 
 Default crypto suite:
@@ -152,7 +152,7 @@ detected.
 4. Add TAR unpacker with path traversal protections.
 5. Add encrypted streaming writer using an HPKE sender context.
 6. Add encrypted streaming reader using an HPKE recipient context.
-7. Wire `dump -archive-out -recipient`.
+7. Wire `dump -archive-out [-recipient]`.
 8. Add `retriever unpack`.
 9. Optionally wire `load -archive -identity`.
 10. Add `slog` progress for archive walking, TAR streaming, encryption,

cmd/retriever/retriever_integration_test.go

@@ -19,10 +19,14 @@
 package main
 
 import (
+	"archive/tar"
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"os"
+	"path/filepath"
 	"reflect"
 	"strings"
 	"testing"
@@ -188,6 +192,49 @@ func TestDumpLoadRoundTrip(t *testing.T) {
 		t.Fatalf("unexpected dump metrics counts: %+v", dumpResult.Manifest.Metrics.Graphs[0])
 	}
 
+	archiveBase := t.TempDir()
+	archiveDumpDir := filepath.Join(archiveBase, "dumpdir")
+	archivePath := filepath.Join(archiveBase, "dump.tar")
+	var archiveOutput bytes.Buffer
+	runtime := commandRuntime{stdout: &archiveOutput, stderr: io.Discard}
+	if err := runtime.run(ctx, []string{
+		"dump", "-connection", connection, "-out", archiveDumpDir,
+		"-archive-out", archivePath,
+		"-graph", graphName, "-compression", "gzip", "-shard-size", "3",
+	}); err != nil {
+		t.Fatalf("dump unencrypted archive: %v", err)
+	}
+	if !strings.Contains(archiveOutput.String(), "archive: "+archivePath) {
+		t.Fatalf("dump output omits archive path: %s", archiveOutput.String())
+	}
+	if entries, err := os.ReadDir(archiveBase); err != nil || len(entries) != 2 || entries[0].Name() != "dump.tar" || entries[1].Name() != "dumpdir" {
+		t.Fatalf("archive output directory contains %v, error = %v", entries, err)
+	}
+	archiveFile, err := os.Open(archivePath)
+	if err != nil {
+		t.Fatalf("open unencrypted archive: %v", err)
+	}
+	defer archiveFile.Close()
+	archiveReader := tar.NewReader(archiveFile)
+	var archivedPaths []string
+	for {
+		header, err := archiveReader.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatalf("read unencrypted archive: %v", err)
+		}
+		archivedPaths = append(archivedPaths, header.Name)
+	}
+	expectedPaths, err := retriever.ArchivePathsFromManifest(dumpResult.Manifest)
+	if err != nil {
+		t.Fatalf("expected archive paths: %v", err)
+	}
+	if !reflect.DeepEqual(archivedPaths, expectedPaths) {
+		t.Fatalf("archive paths = %v, want %v", archivedPaths, expectedPaths)
+	}
+
 	if _, err := retriever.Load(ctx, db, driverName, retriever.LoadOptions{
 		InputDir:      dumpDir,
 		BatchSize:     batchSize,

retriever/archive_envelope.go

@@ -104,71 +104,78 @@ func WriteEncryptedCollectionArchiveWithOptions(writer io.Writer, dumpDir string
 	return nil
 }
 
-func WriteEncryptedCollectionArchiveFile(dumpDir, archivePath string, recipient hpke.PublicKey) error {
-	return WriteEncryptedCollectionArchiveFileWithOptions(dumpDir, archivePath, recipient, ArchiveOptions{})
-}
-
-func writeEncryptedCollectionArchive(dumpDir, archivePath string, recipient hpke.PublicKey) error {
-	return WriteEncryptedCollectionArchiveFile(dumpDir, archivePath, recipient)
-}
-
-func WriteEncryptedCollectionArchiveFileWithOptions(dumpDir, archivePath string, recipient hpke.PublicKey, options ArchiveOptions) error {
-	archivePath = strings.TrimSpace(archivePath)
-	if err := preflightArchiveOutputPath(archivePath); err != nil {
+// ArchiveFileOptions packages an existing collection at OutputPath.
+// A nil Recipient writes a plain TAR; a non-nil Recipient encrypts it.
+type ArchiveFileOptions struct {
+	CollectionDir string
+	OutputPath    string
+	Recipient     hpke.PublicKey
+	Progress      ProgressFunc
+}
+
+// WriteCollectionArchiveFile writes one archive without replacing an existing file.
+func WriteCollectionArchiveFile(options ArchiveFileOptions) error {
+	collectionDir := strings.TrimSpace(options.CollectionDir)
+	outputPath := strings.TrimSpace(options.OutputPath)
+	if collectionDir == "" {
+		return fmt.Errorf("collection directory is required")
+	}
+	if err := preflightArchiveOutputPath(outputPath); err != nil {
 		return err
 	}
 
-	tempFile, err := os.CreateTemp(filepath.Dir(archivePath), filepath.Base(archivePath)+".*.tmp")
+	tempFile, err := os.CreateTemp(filepath.Dir(outputPath), filepath.Base(outputPath)+".*.tmp")
 	if err != nil {
 		return fmt.Errorf("create archive temp file: %w", err)
 	}
 	tempPath := tempFile.Name()
-	cleanupTemp := true
 	defer func() {
-		if cleanupTemp {
-			_ = os.Remove(tempPath)
-		}
+		_ = os.Remove(tempPath)
 	}()
 
-	slog.Info("retriever archive encryption started",
-		slog.String("input_dir", dumpDir),
-		slog.String("archive", archivePath),
-	)
-	options.Progress.emit(ProgressEvent{
-		Operation:   OperationArchive,
-		Message:     "retriever archive encryption started",
-		InputDir:    dumpDir,
-		ArchivePath: archivePath,
-	})
+	if options.Recipient != nil {
+		slog.Info("retriever archive encryption started",
+			slog.String("input_dir", collectionDir),
+			slog.String("archive", outputPath),
+		)
+		options.Progress.emit(ProgressEvent{
+			Operation:   OperationArchive,
+			Message:     "retriever archive encryption started",
+			InputDir:    collectionDir,
+			ArchivePath: outputPath,
+		})
+	}
 
-	if err := WriteEncryptedCollectionArchiveWithOptions(tempFile, dumpDir, recipient, options, archivePath, tempPath); err != nil {
+	archiveOptions := ArchiveOptions{Progress: options.Progress}
+	if options.Recipient == nil {
+		err = WriteCollectionTarWithOptions(tempFile, collectionDir, archiveOptions, outputPath, tempPath)
+	} else {
+		err = WriteEncryptedCollectionArchiveWithOptions(tempFile, collectionDir, options.Recipient, archiveOptions, outputPath, tempPath)
+	}
+	if err != nil {
 		_ = tempFile.Close()
 		return err
 	}
-
 	if err := tempFile.Close(); err != nil {
 		return fmt.Errorf("close archive temp file: %w", err)
 	}
-
-	if err := requirePathDoesNotExist(archivePath); err != nil {
-		return err
+	// Linking publishes the completed file only if the destination is absent.
+	// Unlike a check followed by Rename, this cannot replace a concurrent output.
+	if err := os.Link(tempPath, outputPath); err != nil {
+		return fmt.Errorf("publish archive: %w", err)
 	}
 
-	if err := os.Rename(tempPath, archivePath); err != nil {
-		return fmt.Errorf("rename archive: %w", err)
+	if options.Recipient != nil {
+		slog.Info("retriever archive encryption completed",
+			slog.String("archive", outputPath),
+		)
+		options.Progress.emit(ProgressEvent{
+			Operation:   OperationArchive,
+			Message:     "retriever archive encryption completed",
+			InputDir:    collectionDir,
+			ArchivePath: outputPath,
+		})
 	}
-
-	cleanupTemp = false
-
-	slog.Info("retriever archive encryption completed",
-		slog.String("archive", archivePath),
-	)
-	options.Progress.emit(ProgressEvent{
-		Operation:   OperationArchive,
-		Message:     "retriever archive encryption completed",
-		InputDir:    dumpDir,
-		ArchivePath: archivePath,
-	})
 	return nil
 }