diff --git a/Makefile b/Makefile index f6f5baf..64072fd 100644 --- a/Makefile +++ b/Makefile @@ -78,8 +78,10 @@ tidy-mod: preload-images: test/fixtures/preload-images.sh +# -p 1: the test binaries share one dockerd, one publisher dockerd and one app store, +# so the packages must not run concurrently test-e2e: $(exe) preload-images - @$(GO) test -v ./... + @$(GO) test -v -p 1 ./... test-smoke: $(exe) preload-images @$(GO) test -v -run TestSmoke test/integration/smoke_test.go diff --git a/cmd/composectl/cmd/prune.go b/cmd/composectl/cmd/prune.go index 766adf3..8fbc597 100644 --- a/cmd/composectl/cmd/prune.go +++ b/cmd/composectl/cmd/prune.go @@ -3,6 +3,7 @@ package composectl import ( "encoding/json" "fmt" + "github.com/foundriesio/composeapp/pkg/compose" v1 "github.com/foundriesio/composeapp/pkg/compose/v1" "github.com/spf13/cobra" ) @@ -34,8 +35,13 @@ func init() { func pruneApps(cmd *cobra.Command, opts *pruneOptions) { cs, err := v1.NewAppStore(config.StoreRoot, config.Platform) DieNotNil(err) + unlock, err := compose.LockInstall(config.StoreRoot) + DieNotNil(err) + defer unlock() prunedBlobs, err := cs.Prune(cmd.Context()) DieNotNil(err) + // Leases left by interrupted installs hold blobs and snapshots in dockerd's containerd. + DieNotNil(compose.PruneInstallLeases(cmd.Context(), config), "failed to prune install leases") if opts.Format == "json" { if b, err := json.MarshalIndent(prunedBlobs, "", " "); err == nil { fmt.Println(string(b)) diff --git a/docs/containerd-install-plan-2026-10-08.md b/docs/containerd-install-plan-2026-10-08.md new file mode 100644 index 0000000..044cde9 --- /dev/null +++ b/docs/containerd-install-plan-2026-10-08.md @@ -0,0 +1,429 @@ +# Installing app images into dockerd's containerd + +*Design note, 2026-10-08, revised 2026-10-09 after the phase 0 checks and the second +Astra review, and 2026-10-12 with the device result for the minimum Docker version. +Implemented on this branch; the code review record is in the pull request.* + +## Decision + +On a dockerd that uses the containerd image store, `composectl install` streams each +service image from composectl's store into dockerd's containerd under a short install +lease, unpacks it, and creates the image records. Deleting the lease then releases +the compressed layers, and containerd's GC removes them unless something else, such +as an image pulled with `docker pull`, still references them. The only persistent +compressed copy of an app image is then the one in composectl's store. On a +graphdriver dockerd, and on a Docker older than the qualified minimum, install keeps +using `docker load`. + +Pull, the store layout, status, remove, uninstall, the update states and the +aktualizr-lite interface stay as they are. + +| | Compressed copies kept | Unpacked | +|---|---|---| +| Today, containerd image store | 2 (composectl store and containerd content store) | 1 | +| This plan | 1 (composectl store) | 1 | + +On the 380 MB test image this goes from about 1853 MB to about 1473 MB. The extra +compressed copy during install is temporary. + +## Why this route + +- **The store copy keeps the bytes for a repair local.** In the power-cut + experiments (Exp P), a cut up to about 35 s after a pull left snapshots committed + but empty, and a re-pull did not repair them. With the store copy, reinstalling + needs no network. The bytes alone do not repair a broken snapshot: see "Crash + recovery" for what does. +- **aktualizr-lite behaves the same as today.** After a store prune it calls + `/images/prune` with `dangling=false`, which removes images no container uses. + `check --install` then reports the image missing, and aktualizr-lite reinstalls it + from the store. There is no durable lease that the prune would have to work around. +- **Install skips layers that are already unpacked.** Unpack stats each chain ID + before it applies the layer, so base layers shared with the running version are + not streamed again. Today `docker load` sends every blob. The stat checks only that + the snapshot exists, not that its files are intact; "Crash recovery" says why this + is safe for snapshots composectl creates. +- **The registry path does not change.** composectl's resolver, the token proxy, + resume and the split between fetch and install all stay as they are. + +### Options not taken now + +- **Link-in** (hardlink the store blob into the content root): phase 2. Build it only + if device measurements show that the write volume or the install peak matters. Try + the link and fall back to streaming on any error, because a hardlink fails across + mounts. Read the blob back after registration, because GC can remove the linked + file before commit. +- **Pull into containerd and discard** (the 2026-10-08 brief): phase 3. Build it only + if the persistent compressed copy is shown to be the limit. Its retention model + (durable leases on snapshots, three predicates for status, a reconcile step) is + much larger. The Astra review records its blockers. +- **`docker pull` through the Docker API**: rejected. It loses the token proxy auth + path and the split between fetch and install. +- **`docker load` of an OCI layout followed by stripping layer labels**: this was the + fallback if new `go.mod` modules were refused. It is not needed now. Keeping the + pinned digest through this path is untested. + +## Install sequence (containerd image store) + +composectl takes an exclusive lock on `/.install.lock` (`syscall.Flock`) +for every operation that writes to containerd or deletes from it: install, +uninstall, `rm` and `prune`. A second composectl waits for the lock. Concurrent +installs are not supported, so nothing more is needed. + +Under the lock, for each app: + +1. Delete every lease whose ID starts with `composectl/install/`, synchronously. + Under the lock no other install is running, so any such lease was left by an + interrupted install. This also releases the snapshots that install created + (see "Crash recovery"). Then create the lease `composectl/install/@` + with no expiry. +2. For each image, in manifest order, find the first layer whose chain ID has no + snapshot. Stream that layer and every layer after it from the store with + `content.WriteBlob` under the lease. Also stream the config and the manifest, and + the index when the image is pinned by one. +3. Write the manifest without `gc.ref.content.l.N` labels, so that the layers are + held only by the lease. Keep the labels for the config, and on an index keep + `gc.ref.content.m.N` pointing at the platform manifest. `WriteBlob` does not + change the labels of a manifest that already exists, so a manifest written + earlier by `docker pull` keeps its layers (see "Accepted consequences"). +4. Run `Image.Unpack` on the snapshotter that `/info` reports. If it fails because + a layer blob is missing (a base snapshot was removed between step 2 and step 4, + for example by `docker image prune`), stream every layer of that image and run + `Unpack` once more. +5. Call `syncfs` on the snapshotter root, once per app. Required: see check 6. +6. Create or update the image records: the pinned digest ref, and the `GetTagRef` + tag that `ps` and status read today. Use `Create`, and `Update` of the target + when the record already exists, so that a retry is idempotent. +7. Report each installed image through the existing progress callback with + `ImageLoadStateImageLoaded`, so that `LoadedImages` and `update cancel` keep + working. +8. Delete the lease synchronously. containerd's GC then removes the compressed + layers that nothing else references. + +The lease has no expiry. containerd measures `gc.expire` by wall clock, and boards +without a real-time clock can jump forward by months when NTP syncs, which could +expire the lease in the middle of an install. Orphan leases are removed in step 1 +of the next install and by `composectl prune`. + +## Crash recovery + +The rule: a snapshot that a composectl image record references is durable, because +step 5 runs `syncfs` before step 6 creates the records. A snapshot created after +the last `syncfs` can only be referenced through the install lease. + +So after an interrupted install: + +- Snapshots created by the interrupted install are held only by its lease. Step 1 + of the next install deletes that lease synchronously, and GC removes them before + step 2 looks for existing snapshots. A broken snapshot is not reused. +- Records the interrupted install already created point at synced snapshots. The + retry updates them (step 6). +- If the crash came during lease deletion, the lease may be gone with GC not + finished. The records protect everything that must stay. The layer records are + removed before the files are unlinked, so a cut in between leaves orphan files + that containerd removes at its next content cleanup (see the 2026-10-13 results). + +Out of scope, and documented as unsupported on devices: + +- Snapshots created by something other than composectl, such as `docker pull` or + `docker build`, that a crash left broken. composectl cannot tell them from intact + ones, and step 2 would reuse them. The manual repair is to remove the images that + use them and run `docker image prune`. +- A snapshotter whose GC fails to remove a broken snapshot. Exp P saw this with + erofs. Qualifying erofs is a deployment gate (see "Still unproven"). + +## Discovery and host support + +Docker's `/info` is read with a raw GET through the transport of the existing +Docker client. Docker v25.0.3 has no `Containerd` field, and this avoids bumping the +Docker module. composectl uses these fields: + +- `Containerd.Address` and `Containerd.Namespaces.Containers`; +- `Driver`, the snapshotter name, when `DriverStatus` has the entry + `driver-type` = `io.containerd.snapshotter.v1`. This replaces today's `overlayfs` + hardcode and `DriverStatus[0][1]` in `adjustOptionsIfContainerd`; +- `ServerVersion`. + +composectl uses the containerd install only when all of these hold; otherwise it +uses `docker load`, unchanged: + +- the daemon uses the containerd image store (the `driver-type` entry is present); +- the Docker host is a local unix socket, because composectl connects to the + containerd socket and reads the snapshotter root on the same host; +- the server version is at least the qualified minimum. Docker 25 leaves images + whose layers are missing out of `docker image ls` + (`daemon/containerd/image_list.go`), and `ps` and `check --install` read that + list. Docker 29.7.2 lists them (checked, see the results). The minimum is set to + the oldest version on which the listing is checked: 29.5.2, checked on the board + (see "Results on the device, 2026-10-12"). The version string may carry a `v` + prefix (`v29.5.2` on the Arduino build of Docker); the comparison strips it. + +This covers stock Docker and the Foundries-patched dockerd; the containerd route +uses only stock APIs and does not depend on the patch. + +The containerd socket is normally writable only by root. If composectl cannot +connect to it on a daemon that qualifies, it fails with an error that says so. It +does not fall back to `docker load`, because that would quietly bring back the +second compressed copy. + +## Storage precheck + +`checkApps` (`cmd/composectl/cmd/check.go`) checks the total size against +`StoreRoot` only. The new precheck checks each filesystem against its own needs: + +| Term | Bytes | Path | +|---|---|---| +| store | missing blobs, aligned to the block size | `StoreRoot` | +| bundle | the extracted app bundle | `ComposeRoot` | +| install peak | compressed bytes of the layers that have to be streamed | containerd content root | +| runtime | unpacked size of the missing snapshots | snapshotter root | + +The content root and the snapshotter root come from the introspection API +(`Exports["root"]` of each plugin). Terms that share an `st_dev` are added together +before they are compared with the free space. The runtime term counts each missing +snapshot once, by chain ID, even when its compressed blob is already in the store: +the same blob above a different parent is a different snapshot. The unpacked size +comes from the layers metadata, with the existing fallback when an app has none +(`pkg/compose/v1/app.go`), which is an estimate. The current watermark and +reserved-storage options apply to each filesystem. The check counts bytes only: +inodes and concurrent writers are not covered. + +`check --format json` keeps its current fields and adds one entry per filesystem: + +```json +"storage": [ + {"path": "/var/sota/reset-apps", "required": 412000000, "free": 5100000000, "ok": true}, + {"path": "/var/lib/containerd", "required": 1090000000, "free": 5100000000, "ok": true} +] +``` + +aktualizr-lite should check this list instead of using the Docker root alone. That +change is made in the aktualizr-lite repository. + +On a host that uses `docker load` the current single term stays. + +The calculation lives in `pkg/compose`, used by both `cmd/composectl/cmd/check.go` +and `pkg/update/init.go`. `init.go` today adds `blobStoreSize` to +`runtimeSizeTotal` (and adds it twice to `totalSize`); both should use +`blobRuntimeSize`. This fixes the reported numbers only: `init.go` does not reject +an update on size, and that stays as it is. + +The erofs snapshotter may need temporary space beyond the unpacked size while it +converts a layer. That term is added when erofs is qualified. + +## Code changes + +| File | Change | +|---|---| +| `pkg/compose/docker_client.go` | `/info` discovery (address, namespace, snapshotter) | +| `pkg/compose/ctrd_install.go` (new) | the install sequence above, crash recovery and the install lock | +| `pkg/compose/install.go` | split `Install` (88 lines, over the 60-line limit) and pick containerd or `docker load` per daemon | +| `pkg/compose/image_loader.go` | `adjustOptionsIfContainerd` uses the discovered snapshotter | +| `pkg/compose/storage.go` (new) | the per-filesystem sizing shared by `check` and `update init` | +| `cmd/composectl/cmd/check.go` | uses it; the `storage` JSON field | +| `pkg/update/init.go` | uses it, and the runtime-size fix | +| `pkg/compose/remove.go`, `uninstall.go`, `cmd/composectl/cmd/prune.go` | take the install lock; `prune` deletes orphan install leases | +| `go.mod`, `go.sum` | the root package of the containerd v1.7.15 client is imported. This adds 17 indirect modules and makes `golang.org/x/sys` direct (list below). No Go bump: v1.7.15 declares Go 1.21. The v2 client needs Go 1.23 (v2.0, v2.1) or 1.24.3 (v2.2). | +| `test/compose/docker-compose.yml`, `test/integration/` | a stock dockerd 29 service with the containerd image store, and the tests below | + +New files: the containerd install is a second implementation next to `docker load`; +putting it into `install.go` would mix the two paths. The sizing is shared by two +callers that today compute it separately, and differently. + +Unchanged: fetch, status (`ps` and `check --install` read the Docker image list, +which lists these images on qualified versions), the store, the update state +machine, and `MakeAkliteHappy`. + +Not emitted: images created directly in containerd produce no Docker `load` or +`pull` event. composectl does not read Docker events; whether aktualizr-lite does is +to be checked in its repository. + +Modules added to `go.mod` as indirect, plus `golang.org/x/sys` moving from indirect to +direct: +`AdamKorcz/go-118-fuzz-build`, `containerd/cgroups`, `containerd/continuity`, +`containerd/fifo`, `containerd/ttrpc`, `containerd/typeurl/v2`, +`docker/go-events`, `golang/groupcache`, `google/go-cmp`, `google/uuid`, +`moby/sys/mountinfo`, `moby/sys/signal`, `opencontainers/runtime-spec`, +`opencontainers/selinux`, `go.opencensus.io`, `golang.org/x/text`, +`google.golang.org/genproto`. AGENTS.md asks for approval of new modules; this list +is that request. + +## Phase 0: rig checks before code + +1. The containerd v1.7.15 client against containerd 2.4.1 under dockerd 29: + `WriteBlob` under a lease, `Unpack`, `Create` image, and synchronous lease delete. +2. dockerd lists, runs and `compose up`s the image offline, both by the pinned digest + and by the short tag. +3. The compressed layers are gone from the content root after install, and the + snapshots stay. +4. `/images/prune` with aktualizr-lite's filter removes the unused image, and + reinstalling from the store works offline. +5. An update that shares base layers streams only the layers that differ. +6. Exp P repeated with `syncfs` before the records are created: no committed but + broken snapshot after a cut. +7. On the target eMMC: bytes written and install time, compared with `docker load`. + +If check 1 fails, the fallback is a newer 1.7.x client before considering v2. + +### Results, 2026-10-09 (dev container) + +Run in the test environment of this repository on the containerd image store with +dockerd 29.7.2 and 29.7.2+fio, both on containerd 2.4.1 with the overlayfs +snapshotter. A temporary probe test published and pulled two app versions with the +test fixtures and ran the install sequence above against dockerd's containerd. Both +daemons gave the same results. + +| Check | Result | +|---|---| +| 1. v1.7.15 client against containerd 2.4.1 | Pass. `WriteBlob` under the lease, `Unpack`, image create and synchronous lease delete all work, also for an image pinned by an index. Discovery through a raw `/info` GET and the content and snapshotter roots from introspection work as described. | +| 2. Offline run | Pass. `docker image inspect` and `docker run --pull never --network none` work by the pinned digest and by the short tag; `docker compose up --pull never` of the installed bundle starts every service. | +| 3. Layers gone, snapshots kept | Pass. After install no layer blob is in the content root; the manifest, config and every snapshot in the chain are. | +| 4. Prune and offline reinstall | Pass. `/images/prune` with `dangling=false` removes the records, the snapshots and the config; reinstalling from the store streams every layer again and the app runs offline. | +| 5. Shared base layers | Pass. v2 replaces one image with an image built on top of v1's base: install streamed 1 of 3 layers. `Unpack` does not read the content of a layer whose snapshot exists. | +| 6. Power cut with `syncfs` | On the device, see below. | +| 7. eMMC writes and time | On the device, see below. In the dev container, one comparison for 8.6 MB of layers: 0.49 s for the install, 0.40 s for `docker load`. | +| Status on the discarded layers (added after the review) | Pass on 29.7.2, stock and fio. `docker image ls` lists both images with tag, digest, ID and size; `composectl check --install` reports no missing image; `composectl run` and `ps` report the app running; `stop`, `uninstall` and `rm` work. | + +### Results on the device, 2026-10-09 (checks 6 and 7) + +Board: Arduino UNO Q `unoq-dev`, kernel 7.0.14, ext4 on eMMC, stock dockerd 29.5.2 on +the containerd image store with containerd 2.3.3 and the overlayfs snapshotter. App: +node-red 4.1.11 (v1, 17 layers, 208 MB compressed), updated to 4.1.12 (v2, 13 new +layers, 163 MB). A probe binary ran the install sequence above against a composectl +store copied to the board, or `docker load` from the same store (the unpatched path). + +**Check 7, 3 rounds, arms alternated:** + +| | containerd install | `docker load` | +|---|---|---| +| v1 cold: time | 160–161 s | 153–178 s | +| v1 cold: written to eMMC | 993 MB | 992–993 MB | +| v1 → v2 update: time | 126–128 s | 123–128 s | +| v1 → v2 update: written to eMMC | 797–801 MB | 846–849 MB | +| Disk used with v1 and v2 installed | 1244 MiB | 1602 MiB | + +Cold installs write the same and take about the same time. The update writes 45–52 MB +less with the containerd install in all three rounds, about 6 %. Three rounds per +arm are not enough to call the times equal; per-stage times were not recorded. The +containerd install keeps 358 MiB less on disk, which is the compressed copy of both +versions that `docker load` leaves in containerd. The update streamed only the 13 +layers v1 did not have. + +**Check 6: crash right after install returns.** `echo b > /proc/sysrq-trigger` with +no sync, so the page cache is lost as in a power cut; the eMMC stays powered, so this +is milder than a real cut. 3 trials per arm, compared with a content hash of the +image taken after a clean install. + +| Arm | Result | +|---|---| +| no `syncfs` | 3 of 3 broken: the record and all 17 snapshots are committed, but `docker create` fails. In trials 2 and 3 the error is `failed to stat parent: stat .../snapshots/1139/fs: no such file or directory`; trial 1 ran a probe that did not record the error text | +| `syncfs` before the records | 3 of 3 intact. The content hash matches the clean install | + +`syncfs` (step 5) is required, not optional. Without it, a crash leaves an image that +looks installed and cannot run, and a reinstall would not repair it, because step 2 +skips layers whose snapshot exists. Removing the image records let GC delete the +broken snapshots in all three trials. + +The crash came after the records were created and the lease deleted. The content +hash covers the path, type, size and contents of every regular file, not +permissions, owners, link targets or xattrs. Cuts at other points are listed under +"Still unproven". + +One change to the plan: importing the root package of the client adds 17 indirect +modules to `go.mod`, not three, and makes `golang.org/x/sys` direct; see "Code +changes" for the list. + +### Results on the device, 2026-10-12 (minimum Docker version) + +Same board, Docker v29.5.2, with `composectl` built from the implementation branch +(arm64), the node-red apps above copied to the board as a store. + +The first run found a gate bug: `docker info` on this board reports the server +version as `v29.5.2`, the version comparison read that as `0.0.0`, and install +silently used `docker load` (the content store kept the compressed layers and the +records were the `docker load` ones). The comparison now strips a leading `v`. + +The rerun with the fix took the containerd path: + +| Step | Result | +|---|---| +| install v1 | 3m12s. Snapshots 0 → 17; content blobs 4 → 7 (index, manifest, config, no layers); records for the tag and the pinned digest, both targeting the OCI index; no lease left | +| `docker image ls --digests` | lists the image with tag, digest, ID and size | +| `check --install`, `ps` | no missing image | +| `run` (install + `docker compose up`) | the container starts from the installed image, no pull | +| install v2 (update) | 2m30s; only the 13 new layers streamed; both images listed | +| `uninstall --prune`, `rm` | board back to 0 images, 0 snapshots, 4 content blobs, no composectl leases | + +The listing is therefore checked on 29.5.2 and 29.7.2; the minimum stays 29.5.2. + +### Results on the device, 2026-10-13 (cuts during install, real composectl) + +Same board and apps, `composectl` built from the reviewed tree. Every cut is a +sysrq-b with nothing synced, fired on the board itself: during unpack by a loop +that watches the snapshot directory count; between the first image record and the +lease delete by a probe that polls containerd's image service every 300 µs for the +pinned-digest record; right after `install` returns by the shell. The board came +back on its own each time (77–246 s) and the checks ran with no manual step. + +| Cut | Trials | State after the reboot | Recovery | +|---|---|---|---| +| During unpack, at snapshot 3, 9 and 15 of 17 | 3 | Lease present, the 17 streamed layer blobs present. Snapshots torn in two trials: the metadata had 9 and 15 committed snapshots, the disk had 3 and 7 directories. `check --install` reports the image missing | The next `install` deleted the lease, GC dropped the torn snapshots, the image was unpacked again (17 snapshots, blobs back to 7), the container ran, `check` clean. 3 of 3 | +| After the first image record, before the lease delete | 3 | Both records, 17 snapshots, 24 blobs and the lease present; no bundle yet, so `check --install` reports the image present and `docker-compose.yml` missing | The next `install` (or `run`) deleted the stale lease, GC dropped the layers (blobs 7), the bundle was written, the container ran, `check` clean. 3 of 3 | +| Right after `install` returned | 5 | Records, snapshots and the lease deletion durable in all five; the image ran in all five. In two, the content metadata had 7 records but 24 files on disk: the unlinks of the 17 layer files were lost. The bundle was missing in all five | The next `install` rewrote the bundle. The orphan layer files survived that install's GC and were removed by the uninstall | + +`docker image prune -a` with a stopped container of the app keeps the image (Docker +drops the tag record only); without a container it removes the image, `check +--install` reports it missing, and `install` brings it back from the store and the +container runs. + +Two findings, neither introduced by this change: + +1. The compose bundle is not durable when `install` returns: `InstallCompose` + untars it without a sync, on both routes. A cut inside the writeback window + loses it, `check --install` reports it, the next install rewrites it. Follow-up: + `syncfs` on the compose root after the untar. +2. A cut between the GC's metadata commit and its unlinks leaves the compressed + layers as files unknown to containerd (208 MB here). containerd removes such + files only in `cleanupContent`, which walks the blob directory after a GC that + removed a content record, so they stay until the next image removal on the + device (the uninstall of the previous version at the next update, or `docker + image prune`). Bounded to one install's compressed layers and self-healing. + Follow-up option: let `composectl prune` force a content GC. + +## Still unproven, ranked by risk + +1. Real power cuts on the eMMC, and the fleet's snapshotter. The crash trials used a + sysrq reboot and overlayfs only. If the fleet uses erofs (with its differ and + fs-verity), the crash checks are repeated there before deployment; overlayfs + results do not qualify erofs. +2. The storage precheck on separate mounts and on a nearly full filesystem (an app + whose image config is unreadable is covered by the integration test). + +1 is a deployment gate; 2 and the two follow-ups above are follow-ups. + +## Integration test + +Against the dockerd service on the containerd image store, stock and fio: + +- install from the store with no network; +- an update that shares layers with the installed version; +- prune followed by an offline reinstall; +- a lease left by a killed install is removed on the next install, and the app runs; +- a second composectl waits for the install lock; +- a base snapshot removed by `docker image prune` between the scan and `Unpack` + (the retry in step 4); +- an image pinned by an index; +- `ps`, `check --install`, `update cancel` and uninstall after a containerd install; +- the graphdriver path is unchanged (the existing tests). + +## Accepted consequences (user, 2026-10-08) + +- The persistent compressed copy in the store is acceptable for the first phases. +- `docker save` and `docker push` of app images on the device no longer work on the + containerd image store: the layers are not in containerd. +- The fleet is new. Images loaded the old way, or pulled with `docker pull`, whose + manifests carry layer labels, keep their compressed layers in containerd. +- composectl is the only writer of app images on devices. `docker pull` and + `docker build` there are not supported for app images (see "Crash recovery"). +- Concurrent composectl installs are serialized, not run in parallel. +- `/var/sota/reset-apps` and containerd's root are on the same mount on devices. + composectl does not rely on this; the precheck merges terms by `st_dev`. diff --git a/go.mod b/go.mod index 967fcc8..5e25e65 100644 --- a/go.mod +++ b/go.mod @@ -20,28 +20,39 @@ require ( go.etcd.io/bbolt v1.3.7 golang.org/x/net v0.26.0 golang.org/x/sync v0.7.0 + golang.org/x/sys v0.29.0 gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect + github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/Microsoft/hcsshim v0.11.4 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/containerd/cgroups v1.1.0 // indirect + github.com/containerd/continuity v0.4.2 // indirect + github.com/containerd/fifo v1.1.0 // indirect github.com/containerd/log v0.1.0 // indirect + github.com/containerd/ttrpc v1.2.3 // indirect + github.com/containerd/typeurl/v2 v2.1.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/docker-credential-helpers v0.7.0 // indirect github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.3.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/uuid v1.3.1 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -50,9 +61,13 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/locker v1.0.1 // indirect + github.com/moby/sys/mountinfo v0.6.2 // indirect github.com/moby/sys/sequential v0.5.0 // indirect + github.com/moby/sys/signal v0.7.0 // indirect github.com/moby/sys/user v0.1.0 // indirect github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/runtime-spec v1.1.0 // indirect + github.com/opencontainers/selinux v1.11.0 // indirect github.com/prometheus/client_golang v1.17.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.44.0 // indirect @@ -64,6 +79,7 @@ require ( github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect + go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect go.opentelemetry.io/otel v1.21.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 // indirect @@ -72,8 +88,9 @@ require ( go.opentelemetry.io/otel/trace v1.21.0 // indirect golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect golang.org/x/mod v0.17.0 // indirect - golang.org/x/sys v0.29.0 // indirect + golang.org/x/text v0.16.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/grpc v1.59.0 // indirect google.golang.org/protobuf v1.33.0 // indirect diff --git a/go.sum b/go.sum index e2a914a..7be12f7 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,11 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 h1:59MxjQVfjXsBpLy+dbd2/ELV5ofnUkUZBvWSC85sheA= +github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0/go.mod h1:OahwfttHWG6eJ0clwcfBAHoDI6X/LV/15hx/wlMZSrU= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8= @@ -14,8 +18,11 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/compose-spec/compose-go v1.20.2 h1:u/yfZHn4EaHGdidrZycWpxXgFffjYULlTbRfJ51ykjQ= github.com/compose-spec/compose-go v1.20.2/go.mod h1:+MdqXV4RA7wdFsahh/Kb8U0pAJqkg7mr4PM9tFKU8RM= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= @@ -24,8 +31,14 @@ github.com/containerd/containerd v1.7.15 h1:afEHXdil9iAm03BmhjzKyXnnEBtjaLJefdU7 github.com/containerd/containerd v1.7.15/go.mod h1:ISzRRTMF8EXNpJlTzyr2XMhN+j9K302C21/+cr3kUnY= github.com/containerd/continuity v0.4.2 h1:v3y/4Yz5jwnvqPKJJ+7Wf93fyWoCB3F5EclWG023MDM= github.com/containerd/continuity v0.4.2/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= +github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY= +github.com/containerd/fifo v1.1.0/go.mod h1:bmC4NWMbXlt2EZ0Hc7Fx7QzTFxgPID13eH0Qu+MAb2o= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/ttrpc v1.2.3 h1:4jlhbXIGvijRtNC8F/5CpuJZ7yKOBFGFOOXg1bkISz0= +github.com/containerd/ttrpc v1.2.3/go.mod h1:ieWsXucbb8Mj9PH0rXCw1i8IunRbbAiDkpXkbfflWBM= +github.com/containerd/typeurl/v2 v2.1.1 h1:3Q4Pt7i8nYwy2KmQWIw2+1hTvwTE/6w9FqcttATPO/4= +github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3HZj1hsSQlywkQ0= github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= @@ -45,12 +58,18 @@ github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryef github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -65,17 +84,37 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= @@ -112,6 +151,8 @@ github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vyg github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/moby/sys/signal v0.7.0 h1:25RW3d5TnQEoKvRbEKUGay6DCQ46IxAVTT9CUMgmsSI= +github.com/moby/sys/signal v0.7.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= @@ -129,6 +170,10 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg= +github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= +github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -142,6 +187,7 @@ github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1 github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -166,9 +212,14 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -204,20 +255,31 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -225,6 +287,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -232,6 +295,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -242,6 +307,10 @@ golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -251,13 +320,35 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= +google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q= google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -270,3 +361,5 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/compose/ctrd_install.go b/pkg/compose/ctrd_install.go new file mode 100644 index 0000000..d909799 --- /dev/null +++ b/pkg/compose/ctrd_install.go @@ -0,0 +1,479 @@ +package compose + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/containerd/containerd" + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/leases" + "github.com/containerd/containerd/platforms" + dockerclient "github.com/docker/docker/client" + "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/identity" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "golang.org/x/sys/unix" +) + +// This file installs app images straight into the containerd behind dockerd, following +// docs/containerd-install-plan-2026-10-08.md. The compressed layers are streamed under a short +// lease and dropped by containerd's GC once the lease is deleted, so the only persistent compressed +// copy of an app image is the one in composectl's store. + +const ( + // MinDockerVersionForCtrdInstall is the oldest dockerd on which `docker image ls` lists an image + // whose compressed layers are not in containerd; `ps` and `check --install` read that list. + MinDockerVersionForCtrdInstall = "29.5.2" + + installLeasePrefix = "composectl/install/" + installLockFileName = ".install.lock" + contentPluginType = "io.containerd.content.v1" + gcRefConfigLabel = "containerd.io/gc.ref.content.config" + gcRefManifestLabel = "containerd.io/gc.ref.content.m.0" + // recordCleanupTimeout bounds the removal of partial image records after a failed write. + recordCleanupTimeout = 30 * time.Second +) + +type ( + // ctrdClient is a connection to the containerd behind dockerd. + ctrdClient struct { + *containerd.Client + snapshotter string + contentRoot string + snapshotterRoot string + } + + // ctrdImage is one app image as the containerd install needs it. + ctrdImage struct { + ref string // the pinned digest reference the app uses + tagRef string // the short tag reference that `ps` and status read + root ocispec.Descriptor // what ref points at: the platform manifest or an index + manifest ocispec.Descriptor // the platform manifest selected when the app tree was loaded + config ocispec.Descriptor + layers []ocispec.Descriptor + chainIDs []digest.Digest // chain ID of the snapshot of each layer + } +) + +// LockInstall takes the lock that serializes every operation that writes to or deletes from +// dockerd's containerd: install, uninstall, rm and prune. It blocks until a concurrent composectl +// releases it. +func LockInstall(storeRoot string) (unlock func(), err error) { + f, err := os.OpenFile(filepath.Join(storeRoot, installLockFileName), os.O_RDWR|os.O_CREATE, 0o644) + if err != nil { + return nil, fmt.Errorf("failed to open the install lock: %w", err) + } + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil { + _ = f.Close() + return nil, fmt.Errorf("failed to take the install lock: %w", err) + } + return func() { + _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + _ = f.Close() + }, nil +} + +// PruneInstallLeases deletes the leases that interrupted installs left in dockerd's containerd, +// and with them the blobs and snapshots that only they hold. Call it under the install lock. +// It does nothing on a daemon whose images are loaded with `docker load`. +func PruneInstallLeases(ctx context.Context, cfg *Config) error { + c, err := ctrdIfInstallTarget(ctx, cfg) + if err != nil || c == nil { + return err + } + defer c.Close() + if err := deleteInstallLeases(ctx, c.LeasesService()); err != nil { + return fmt.Errorf("failed to prune install leases: %w", err) + } + return nil +} + +// connectCtrdIfInstallTarget returns a client to dockerd's containerd when app images are installed +// into it directly, or nil when they are loaded with `docker load`: a graphdriver daemon, a daemon +// older than MinDockerVersionForCtrdInstall or one that is not on a local unix socket. +func connectCtrdIfInstallTarget(ctx context.Context, cli *dockerclient.Client) (*ctrdClient, error) { + info, err := GetDockerInfo(ctx, cli) + if err != nil { + return nil, fmt.Errorf("failed to discover the docker image store: %w", err) + } + if !isCtrdInstallTarget(info, cli.DaemonHost()) { + return nil, nil + } + return newCtrdClient(ctx, info) +} + +// ctrdIfInstallTarget is connectCtrdIfInstallTarget for the operations that worked without dockerd +// before (the storage check, prune): a dockerd that cannot be reached is treated like a `docker load` one. +func ctrdIfInstallTarget(ctx context.Context, cfg *Config) (*ctrdClient, error) { + cli, err := GetDockerClient(cfg.DockerHost) + if err != nil { + return nil, nil + } + defer cli.Close() + info, err := GetDockerInfo(ctx, cli) + if err != nil || !isCtrdInstallTarget(info, cli.DaemonHost()) { + return nil, nil + } + c, err := newCtrdClient(ctx, info) + if err != nil { + return nil, fmt.Errorf("failed to connect to dockerd's containerd: %w", err) + } + return c, nil +} + +func isCtrdInstallTarget(info *DockerInfo, dockerHost string) bool { + return info.Snapshotter() != "" && + info.Containerd != nil && len(info.Containerd.Address) > 0 && + isLocalDockerHost(dockerHost) && + versionAtLeast(info.ServerVersion, MinDockerVersionForCtrdInstall) +} + +func newCtrdClient(ctx context.Context, info *DockerInfo) (*ctrdClient, error) { + c, err := containerd.New(info.Containerd.Address, + containerd.WithDefaultNamespace(info.Containerd.Namespaces.Containers)) + if err != nil { + return nil, fmt.Errorf("dockerd %s uses the containerd image store, but composectl cannot connect"+ + " to its containerd at %s (the socket is usually writable by root only): %w", + info.ServerVersion, info.Containerd.Address, err) + } + cc := &ctrdClient{Client: c, snapshotter: info.Snapshotter()} + if cc.contentRoot, err = pluginRoot(ctx, c, contentPluginType, "content"); err == nil { + cc.snapshotterRoot, err = pluginRoot(ctx, c, snapshotterPluginType, cc.snapshotter) + } + if err != nil { + _ = c.Close() + return nil, err + } + return cc, nil +} + +func pluginRoot(ctx context.Context, c *containerd.Client, pluginType, id string) (string, error) { + resp, err := c.IntrospectionService().Plugins(ctx, []string{"type==" + pluginType + ",id==" + id}) + if err != nil { + return "", fmt.Errorf("failed to introspect containerd plugin %s.%s: %w", pluginType, id, err) + } + if len(resp.Plugins) != 1 || resp.Plugins[0].Exports["root"] == "" { + return "", fmt.Errorf("containerd plugin %s.%s does not export its root", pluginType, id) + } + return resp.Plugins[0].Exports["root"], nil +} + +// installImagesCtrd installs the images of one app into dockerd's containerd. Under a lease it streams +// the layers that have no snapshot yet, unpacks every image and syncs the snapshotter's filesystem, +// and only then creates the image records: a record never points at a snapshot that a power cut +// can still lose. Deleting the lease lets containerd's GC drop the compressed layers. +func installImagesCtrd(ctx context.Context, c *ctrdClient, app App, cfg *Config, report ProgressCallbackFn) error { + blobsRoot := cfg.GetBlobsRoot() + imgs, err := ctrdImagesOf(ctx, app, NewStoreBlobProvider(blobsRoot)) + if err != nil { + return fmt.Errorf("failed to read the app images from the store: %w", err) + } + ls := c.LeasesService() + if err := deleteInstallLeases(ctx, ls); err != nil { + return fmt.Errorf("failed to remove leases of interrupted installs: %w", err) + } + lease, err := ls.Create(ctx, leases.WithID(installLeaseID(app.Name(), app.Ref().Digest))) + if err != nil { + return fmt.Errorf("failed to create the install lease: %w", err) + } + lctx := leases.WithLease(ctx, lease.ID) + platform := platforms.OnlyStrict(cfg.Platform) + for _, img := range imgs { + if err := c.installImage(lctx, img, blobsRoot, platform, report); err != nil { + return fmt.Errorf("failed to install image %s: %w", img.ref, err) + } + } + if err := syncFs(c.snapshotterRoot); err != nil { + return fmt.Errorf("failed to sync the snapshots of app %s: %w", app.Name(), err) + } + for _, img := range imgs { + if err := putImageRecords(ctx, c.ImageService(), img); err != nil { + return fmt.Errorf("failed to record image %s: %w", img.ref, err) + } + reportImageState(report, ImageLoadStateImageLoaded, img.ref) + } + if err := ls.Delete(ctx, lease, leases.SynchronousDelete); err != nil { + return fmt.Errorf("failed to delete the install lease: %w", err) + } + return nil +} + +func installLeaseID(appName string, appDigest digest.Digest) string { + return installLeasePrefix + appName + "@" + appDigest.String() +} + +func installLeases(all []leases.Lease) []leases.Lease { + var found []leases.Lease + for _, l := range all { + if strings.HasPrefix(l.ID, installLeasePrefix) { + found = append(found, l) + } + } + return found +} + +// deleteInstallLeases removes the leases that interrupted installs left behind. Under the install +// lock no other install is running, so every install lease is such a leftover. +func deleteInstallLeases(ctx context.Context, ls leases.Manager) error { + all, err := ls.List(ctx) + if err != nil { + return fmt.Errorf("failed to list containerd leases: %w", err) + } + for _, l := range installLeases(all) { + if err := ls.Delete(ctx, l, leases.SynchronousDelete); err != nil && !errdefs.IsNotFound(err) { + return fmt.Errorf("failed to delete the install lease %s: %w", l.ID, err) + } + } + return nil +} + +// installImage streams what image needs and unpacks it. If a base snapshot went away between the +// scan and the unpack (for example `docker image prune` removed the image that held it), the unpack +// fails on the missing layer blob; then every layer is streamed and the unpack is run once more. +func (c *ctrdClient) installImage(ctx context.Context, img *ctrdImage, blobsRoot string, + platform platforms.MatchComparer, report ProgressCallbackFn) error { + first, err := c.firstMissingSnapshot(ctx, img.chainIDs) + if err != nil { + return fmt.Errorf("failed to scan the snapshots: %w", err) + } + reportImageState(report, ImageLoadStateImageLoading, img.ref) + if err := c.streamImage(ctx, img, blobsRoot, first); err != nil { + return fmt.Errorf("failed to stream the image to containerd: %w", err) + } + err = c.unpack(ctx, img, platform) + if err == nil { + return nil + } + if !errdefs.IsNotFound(err) { + return fmt.Errorf("failed to unpack: %w", err) + } + if err := c.streamLayers(ctx, img.layers, blobsRoot); err != nil { + return fmt.Errorf("failed to stream every layer to containerd after a snapshot went away: %w", err) + } + if err := c.unpack(ctx, img, platform); err != nil { + return fmt.Errorf("failed to unpack after streaming every layer: %w", err) + } + return nil +} + +// firstMissingSnapshot returns the index of the first layer whose snapshot does not exist; every +// layer after it has to be streamed and unpacked too, because a snapshot needs its parent. +func (c *ctrdClient) firstMissingSnapshot(ctx context.Context, chainIDs []digest.Digest) (int, error) { + sn := c.SnapshotService(c.snapshotter) + for i, chainID := range chainIDs { + _, err := sn.Stat(ctx, chainID.String()) + if errdefs.IsNotFound(err) { + return i, nil + } + if err != nil { + return 0, fmt.Errorf("failed to stat snapshot %s: %w", chainID, err) + } + } + return len(chainIDs), nil +} + +// streamImage writes the layers from first on, the config, the manifest and, when the image is +// pinned by one, the index. The manifest gets no layer labels: only the lease holds the layers. +// WriteBlob leaves the labels of a blob that already exists as they are. +func (c *ctrdClient) streamImage(ctx context.Context, img *ctrdImage, blobsRoot string, first int) error { + if err := c.streamLayers(ctx, img.layers[first:], blobsRoot); err != nil { + return err + } + if err := c.writeBlob(ctx, blobsRoot, img.config, nil); err != nil { + return err + } + manifestLabels := map[string]string{gcRefConfigLabel: img.config.Digest.String()} + if err := c.writeBlob(ctx, blobsRoot, img.manifest, manifestLabels); err != nil { + return err + } + if img.root.Digest == img.manifest.Digest { + return nil + } + err := c.writeBlob(ctx, blobsRoot, img.root, map[string]string{gcRefManifestLabel: img.manifest.Digest.String()}) + if errors.Is(err, os.ErrNotExist) { + // The app was pulled by skopeo: the store has the image's platform manifest but not the index + // the app pins it by. The image records point at the manifest then, as `docker load` does. + img.root = img.manifest + return nil + } + return err +} + +func (c *ctrdClient) streamLayers(ctx context.Context, layers []ocispec.Descriptor, blobsRoot string) error { + for _, layer := range layers { + if err := c.writeBlob(ctx, blobsRoot, layer, nil); err != nil { + return err + } + } + return nil +} + +func (c *ctrdClient) writeBlob(ctx context.Context, blobsRoot string, desc ocispec.Descriptor, labels map[string]string) error { + f, err := os.Open(filepath.Join(blobsRoot, desc.Digest.Encoded())) + if err != nil { + return fmt.Errorf("failed to open blob: %w", err) + } + defer f.Close() + var opts []content.Opt + if labels != nil { + opts = append(opts, content.WithLabels(labels)) + } + if err := content.WriteBlob(ctx, c.ContentStore(), "composectl-"+desc.Digest.String(), f, desc, opts...); err != nil { + return fmt.Errorf("failed to write blob %s to containerd: %w", desc.Digest, err) + } + return nil +} + +// unpack creates the snapshots of the image's layers. The target is the platform manifest the app +// tree selected, not the index: on arm64 the default platform matcher would also accept arm/v7. +func (c *ctrdClient) unpack(ctx context.Context, img *ctrdImage, platform platforms.MatchComparer) error { + ci := containerd.NewImageWithPlatform(c.Client, images.Image{Name: img.ref, Target: img.manifest}, platform) + return ci.Unpack(ctx, c.snapshotter) +} + +// putImageRecords creates the pinned digest reference and the short tag reference, or points the +// existing records at the image, so that a retried install is idempotent. If the second record +// fails, the records this call created are deleted again (best effort), so that an image is +// either fully recorded and reported as loaded, or absent and still held only by the lease. +func putImageRecords(ctx context.Context, is images.Store, img *ctrdImage) error { + var created []string + for _, name := range []string{img.ref, img.tagRef} { + rec := images.Image{Name: name, Target: img.root} + _, err := is.Create(ctx, rec) + if err == nil { + created = append(created, name) + continue + } + if errdefs.IsAlreadyExists(err) { + _, err = is.Update(ctx, rec, "target") + } + if err != nil { + deleteImageRecords(ctx, is, created) + return fmt.Errorf("failed to create image record %s: %w", name, err) + } + } + return nil +} + +// deleteImageRecords removes the records a failed putImageRecords created. The failure may be the +// cancellation of ctx itself, so the deletes run on a context that is not cancelled, with a bound. +func deleteImageRecords(ctx context.Context, is images.Store, names []string) { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), recordCleanupTimeout) + defer cancel() + for _, name := range names { + _ = is.Delete(cleanupCtx, name) + } +} + +// syncFs flushes the filesystem that holds path. A power cut after the image records are created +// must not lose the snapshots they point at. +func syncFs(path string) error { + fd, err := unix.Open(path, unix.O_RDONLY|unix.O_DIRECTORY, 0) + if err != nil { + return fmt.Errorf("failed to open %s: %w", path, err) + } + defer unix.Close(fd) + if err := unix.Syncfs(fd); err != nil { + return fmt.Errorf("failed to sync the filesystem of %s: %w", path, err) + } + return nil +} + +func reportImageState(report ProgressCallbackFn, state ImageLoadState, imageID string) { + if report != nil { + report(&LoadImageProgress{State: state, ImageID: imageID}) + } +} + +// ctrdImagesOf describes each image of the app as the containerd install and the storage precheck +// need it. The app tree already holds the platform manifest of each image. +func ctrdImagesOf(ctx context.Context, app App, provider BlobProvider) ([]*ctrdImage, error) { + var imgs []*ctrdImage + for _, root := range app.GetComposeRoot().Children { + img, err := ctrdImageOf(ctx, root, provider) + if err != nil { + return nil, fmt.Errorf("failed to read image %s: %w", root.Ref(), err) + } + imgs = append(imgs, img) + } + return imgs, nil +} + +func ctrdImageOf(ctx context.Context, root *TreeNode, provider BlobProvider) (*ctrdImage, error) { + ref, err := ParseImageRef(root.Ref()) + if err != nil { + return nil, fmt.Errorf("failed to parse the image reference: %w", err) + } + mNode, err := platformManifestNode(root) + if err != nil { + return nil, fmt.Errorf("failed to find the platform manifest: %w", err) + } + manifest, err := ReadImageManifest(ctx, provider, mNode.Ref(), mNode.Descriptor.Size) + if err != nil { + return nil, fmt.Errorf("failed to read image manifest: %w", err) + } + cfgNode := childOfType(mNode, BlobTypeImageConfig) + if cfgNode == nil { + return nil, fmt.Errorf("image config is not found") + } + b, err := ReadBlob(ctx, provider, cfgNode.Ref(), cfgNode.Descriptor.Digest, cfgNode.Descriptor.Size) + if err != nil { + return nil, fmt.Errorf("failed to read image config: %w", err) + } + var imageConfig ocispec.Image + if err := json.Unmarshal(b, &imageConfig); err != nil { + return nil, fmt.Errorf("failed to unmarshal image config: %w", err) + } + if len(imageConfig.RootFS.DiffIDs) != len(manifest.Layers) { + return nil, fmt.Errorf("image config has %d diff IDs for %d layers", + len(imageConfig.RootFS.DiffIDs), len(manifest.Layers)) + } + return &ctrdImage{ + ref: root.Ref(), + tagRef: ref.GetTagRef(), + root: plainDescriptor(root.Descriptor), + manifest: plainDescriptor(mNode.Descriptor), + config: plainDescriptor(&manifest.Config), + layers: manifest.Layers, + chainIDs: identity.ChainIDs(imageConfig.RootFS.DiffIDs), + }, nil +} + +// platformManifestNode walks from the image root down to its platform manifest. +func platformManifestNode(root *TreeNode) (*TreeNode, error) { + node := root + for node.Type != BlobTypeImageManifest { + if node.Type != BlobTypeImageIndex && node.Type != BlobTypeSkopeoImageIndex { + return nil, fmt.Errorf("invalid image type: %s", node.Type) + } + if len(node.Children) != 1 { + return nil, fmt.Errorf("the image index has %d manifests for the platform, expected one", len(node.Children)) + } + node = node.Children[0] + } + return node, nil +} + +func childOfType(node *TreeNode, blobType BlobType) *TreeNode { + for _, child := range node.Children { + if child.Type == blobType { + return child + } + } + return nil +} + +// plainDescriptor drops what the app tree adds to a descriptor (composectl's references in URLs, +// the index's platform), keeping what containerd records and content need. +func plainDescriptor(desc *ocispec.Descriptor) ocispec.Descriptor { + return ocispec.Descriptor{MediaType: desc.MediaType, Digest: desc.Digest, Size: desc.Size} +} diff --git a/pkg/compose/ctrd_install_rig_test.go b/pkg/compose/ctrd_install_rig_test.go new file mode 100644 index 0000000..6559aba --- /dev/null +++ b/pkg/compose/ctrd_install_rig_test.go @@ -0,0 +1,108 @@ +package compose_test + +// Drives the containerd install directly, with its synchronous progress callback, against the +// test dockerd on the containerd image store. Skips when that dockerd or the composectl binary +// (built by `make`) is not there, as under `make test-unit`. + +import ( + "context" + "fmt" + "os" + "os/exec" + "testing" + "time" + + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/platforms" + "github.com/foundriesio/composeapp/pkg/compose" + f "github.com/foundriesio/composeapp/test/fixtures" +) + +const derivedAppDef = ` +services: + derived: + image: %s + command: sh -c "while true; do sleep 60; done" +` + +func ctrdRig(t *testing.T) *compose.CtrdClient { + t.Helper() + if _, err := os.Stat(os.Getenv("COMPOSECTL_EXE")); err != nil { + t.Skip("composectl binary is not built; run `make` first") + } + cli, err := compose.GetDockerClient("") + f.Check(t, err) + defer cli.Close() + c, err := compose.ConnectCtrdIfInstallTarget(context.Background(), cli) + f.Check(t, err) + if c == nil { + t.Skip("dockerd does not use the containerd image store") + } + t.Cleanup(func() { c.Close() }) + return c +} + +// TestCtrdInstallRetriesAfterBaseSnapshotRemoval: the base snapshot found by the scan is removed +// before the unpack, as `docker image prune` could do, and the install streams every layer and +// unpacks again. The removal runs in the install's progress callback, which installImagesCtrd +// calls synchronously after the scan and before the layers are streamed. +func TestCtrdInstallRetriesAfterBaseSnapshotRemoval(t *testing.T) { + c := ctrdRig(t) + ctx := context.Background() + cfg := f.NewTestConfig(t) + store := compose.NewStoreBlobProvider(cfg.GetBlobsRoot()) + const base = "ghcr.io/foundriesio/busybox:1.36" + + baseApp := f.NewApp(t, fmt.Sprintf(derivedAppDef, base)) + baseApp.Publish(t) + baseApp.Pull(t) + defer baseApp.Remove(t) + baseApp.Install(t) + defer baseApp.Uninstall(t) + baseTree, err := cfg.AppLoader.LoadAppTree(ctx, store, platforms.OnlyStrict(cfg.Platform), baseApp.PublishedUri) + f.Check(t, err) + baseChains, err := compose.ChainIDsOf(ctx, baseTree, store) + f.Check(t, err) + baseChain := baseChains[0] + baseRef, err := compose.ParseImageRef(baseTree.GetComposeRoot().Children[0].Ref()) + f.Check(t, err) + baseRefs := []string{baseTree.GetComposeRoot().Children[0].Ref(), baseRef.GetTagRef()} + + derivedApp := f.NewApp(t, fmt.Sprintf(derivedAppDef, f.PublishDerivedImage(t, base, 4))) + derivedApp.Publish(t) + derivedApp.Pull(t) + defer derivedApp.Remove(t) + derivedTree, err := cfg.AppLoader.LoadAppTree(ctx, store, platforms.OnlyStrict(cfg.Platform), derivedApp.PublishedUri) + f.Check(t, err) + + sn := c.SnapshotService(compose.CtrdSnapshotter(c)) + var removedAt time.Time + removeBase := func(p *compose.LoadImageProgress) { + if p.State != compose.ImageLoadStateImageLoading || !removedAt.IsZero() { + return + } + for _, ref := range baseRefs { + f.Check(t, c.ImageService().Delete(ctx, ref, images.SynchronousDelete())) + } + removedAt = time.Now() + if _, err := sn.Stat(ctx, baseChain[len(baseChain)-1]); !errdefs.IsNotFound(err) { + t.Fatalf("the base snapshot is still there after its images were deleted: %v", err) + } + } + f.Check(t, compose.InstallImagesCtrd(ctx, c, derivedTree, cfg, removeBase)) + defer derivedApp.Uninstall(t) + if removedAt.IsZero() { + t.Fatal("the progress callback did not run") + } + for _, chainID := range baseChain { + info, err := sn.Stat(ctx, chainID) + f.Check(t, err) + if !info.Created.After(removedAt) { + t.Errorf("base snapshot %s was not recreated by the retry: created %v, removed at %v", chainID, info.Created, removedAt) + } + } + derivedRef := derivedTree.GetComposeRoot().Children[0].Ref() + out, err := exec.Command("docker", "run", "--rm", "--pull", "never", "--network", "none", derivedRef, "echo", "ok").CombinedOutput() + f.Checkf(t, err, "docker run %s: %s", derivedRef, out) +} diff --git a/pkg/compose/ctrd_install_test.go b/pkg/compose/ctrd_install_test.go new file mode 100644 index 0000000..4448741 --- /dev/null +++ b/pkg/compose/ctrd_install_test.go @@ -0,0 +1,123 @@ +package compose + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/leases" + "github.com/opencontainers/go-digest" +) + +func TestInstallLeaseID(t *testing.T) { + d := digest.FromString("app") + id := installLeaseID("myapp", d) + if id != "composectl/install/myapp@"+d.String() { + t.Fatalf("unexpected lease ID: %s", id) + } + if !strings.HasPrefix(id, installLeasePrefix) { + t.Fatalf("lease ID %s does not start with %s", id, installLeasePrefix) + } +} + +func TestInstallLeasesFilter(t *testing.T) { + all := []leases.Lease{ + {ID: "composectl/install/app1@sha256:aaaa"}, + {ID: "moby-pull-123"}, + {ID: "composectl/install/app2@sha256:bbbb"}, + {ID: "composectl/other"}, + } + got := installLeases(all) + if len(got) != 2 || got[0].ID != all[0].ID || got[1].ID != all[2].ID { + t.Fatalf("unexpected install leases: %+v", got) + } + if got := installLeases(nil); len(got) != 0 { + t.Fatalf("expected no leases, got %+v", got) + } +} + +func TestIsCtrdInstallTarget(t *testing.T) { + ctrdStore := func(version string) *DockerInfo { + return &DockerInfo{ + ServerVersion: version, + Driver: "overlayfs", + DriverStatus: [][2]string{{"driver-type", snapshotterPluginType}}, + Containerd: &ContainerdInfo{Address: "/run/docker/containerd/containerd.sock"}, + } + } + const local = "unix:///var/run/docker.sock" + if !isCtrdInstallTarget(ctrdStore("29.5.2"), local) { + t.Fatal("the containerd image store on the minimum version over a unix socket must qualify") + } + if isCtrdInstallTarget(ctrdStore("29.5.1"), local) { + t.Fatal("a dockerd older than the minimum must not qualify") + } + if isCtrdInstallTarget(ctrdStore("29.7.2"), "tcp://10.0.0.1:2375") { + t.Fatal("a remote dockerd must not qualify") + } + graphdriver := ctrdStore("29.7.2") + graphdriver.Driver = "overlay2" + graphdriver.DriverStatus = [][2]string{{"Backing Filesystem", "extfs"}} + if isCtrdInstallTarget(graphdriver, local) { + t.Fatal("a graphdriver dockerd must not qualify") + } + noAddress := ctrdStore("29.7.2") + noAddress.Containerd = nil + if isCtrdInstallTarget(noAddress, local) { + t.Fatal("a dockerd that reports no containerd address must not qualify") + } +} + +// fakeImageStore is an images.Store whose Create cancels the install context after the first +// record, as a cancellation between the two record writes does, and records the deletes. +type fakeImageStore struct { + cancel context.CancelFunc + created []string + deleted []string + // deletedWithLiveCtx is false when a delete came with a context that was already done. + deletedWithLiveCtx bool +} + +func (s *fakeImageStore) Get(context.Context, string) (images.Image, error) { + return images.Image{}, errdefs.ErrNotFound +} +func (s *fakeImageStore) List(context.Context, ...string) ([]images.Image, error) { return nil, nil } +func (s *fakeImageStore) Create(ctx context.Context, img images.Image) (images.Image, error) { + if err := ctx.Err(); err != nil { + return images.Image{}, err + } + s.created = append(s.created, img.Name) + s.cancel() + return img, nil +} +func (s *fakeImageStore) Update(ctx context.Context, img images.Image, _ ...string) (images.Image, error) { + return images.Image{}, ctx.Err() +} +func (s *fakeImageStore) Delete(ctx context.Context, name string, _ ...images.DeleteOpt) error { + s.deleted = append(s.deleted, name) + s.deletedWithLiveCtx = ctx.Err() == nil + return nil +} + +func TestPutImageRecordsRollsBackOnCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + store := &fakeImageStore{cancel: cancel} + img := &ctrdImage{ref: "registry/app@sha256:aaaa", tagRef: "registry/app:aaaa"} + err := putImageRecords(ctx, store, img) + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected the cancellation to be returned, got %v", err) + } + if len(store.created) != 1 || store.created[0] != img.ref { + t.Fatalf("expected the digest record only to be created, got %v", store.created) + } + if len(store.deleted) != 1 || store.deleted[0] != img.ref { + t.Fatalf("expected the digest record to be deleted again, got %v", store.deleted) + } + if !store.deletedWithLiveCtx { + t.Fatal("the rollback ran on the cancelled context") + } +} diff --git a/pkg/compose/docker_client.go b/pkg/compose/docker_client.go index 2acd74f..4eea676 100644 --- a/pkg/compose/docker_client.go +++ b/pkg/compose/docker_client.go @@ -1,9 +1,34 @@ package compose import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "strings" + dockerclient "github.com/docker/docker/client" ) +type ( + // DockerInfo is the part of dockerd's `/info` that composectl reads. + // It is decoded from a raw GET because the Docker v25 client's Info type has no Containerd field. + DockerInfo struct { + ServerVersion string + Driver string + DriverStatus [][2]string + Containerd *ContainerdInfo + } + ContainerdInfo struct { + Address string + Namespaces struct{ Containers string } + } +) + +const snapshotterPluginType = "io.containerd.snapshotter.v1" + func GetDockerClient(dockerHost string) (*dockerclient.Client, error) { opts := []dockerclient.Opt{ dockerclient.FromEnv, @@ -13,3 +38,92 @@ func GetDockerClient(dockerHost string) (*dockerclient.Client, error) { } return dockerclient.NewClientWithOpts(opts...) } + +// GetDockerInfo reads dockerd's `/info`. On a local socket it is a raw GET through the client's +// transport, which is the only way to get the Containerd field with the Docker v25 client. On a +// TCP host the raw request would not reach the daemon (the client's address and TLS live in its +// request path), so the typed Info is used there; it has no Containerd field, and that host is +// never an install target anyway (see isCtrdInstallTarget). +func GetDockerInfo(ctx context.Context, cli *dockerclient.Client) (*DockerInfo, error) { + if !isLocalDockerHost(cli.DaemonHost()) { + info, err := cli.Info(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get docker info: %w", err) + } + return &DockerInfo{ServerVersion: info.ServerVersion, Driver: info.Driver, DriverStatus: info.DriverStatus}, nil + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://docker/info", nil) + if err != nil { + return nil, fmt.Errorf("failed to build the docker info request: %w", err) + } + resp, err := cli.HTTPClient().Do(req) + if err != nil { + return nil, fmt.Errorf("failed to get docker info: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to get docker info: %s", resp.Status) + } + info, err := parseDockerInfo(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to decode docker info: %w", err) + } + return info, nil +} + +// isLocalDockerHost reports whether the docker host is a socket on this machine. +func isLocalDockerHost(dockerHost string) bool { + return strings.HasPrefix(dockerHost, "unix://") || strings.HasPrefix(dockerHost, "npipe://") +} + +func parseDockerInfo(r io.Reader) (*DockerInfo, error) { + var info DockerInfo + if err := json.NewDecoder(r).Decode(&info); err != nil { + return nil, err + } + return &info, nil +} + +// Snapshotter returns the name of the containerd snapshotter that dockerd uses, +// or "" when dockerd uses a graphdriver instead of the containerd image store. +func (i *DockerInfo) Snapshotter() string { + for _, kv := range i.DriverStatus { + if kv[0] == "driver-type" && kv[1] == snapshotterPluginType { + return i.Driver + } + } + return "" +} + +// versionAtLeast reports whether a dotted version such as "29.7.2", "v29.5.2" or "20.10.24+dfsg1" +// is not older than min. Only the leading numeric components are compared. +func versionAtLeast(version, min string) bool { + have := versionNumbers(version) + want := versionNumbers(min) + for len(have) < len(want) { + have = append(have, 0) + } + for i, w := range want { + if have[i] != w { + return have[i] > w + } + } + return true +} + +func versionNumbers(version string) []int { + version = strings.TrimLeft(version, "vV") // some builds report "v29.5.2" + end := strings.IndexFunc(version, func(r rune) bool { return r != '.' && (r < '0' || r > '9') }) + if end >= 0 { + version = version[:end] + } + var numbers []int + for _, part := range strings.Split(version, ".") { + n, err := strconv.Atoi(part) + if err != nil { + break + } + numbers = append(numbers, n) + } + return numbers +} diff --git a/pkg/compose/docker_client_test.go b/pkg/compose/docker_client_test.go new file mode 100644 index 0000000..fe945f4 --- /dev/null +++ b/pkg/compose/docker_client_test.go @@ -0,0 +1,149 @@ +package compose + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + dockerclient "github.com/docker/docker/client" +) + +// Trimmed `/info` responses of dockerd 29.7.2 on the containerd image store and on the overlay2 graphdriver. +const ( + infoContainerdStore = `{ + "ID": "6ef1", + "Driver": "overlayfs", + "DriverStatus": [["driver-type", "io.containerd.snapshotter.v1"]], + "ServerVersion": "29.7.2", + "Containerd": { + "Address": "/var/run/docker/containerd/containerd.sock", + "Namespaces": {"Containers": "moby", "Plugins": "plugins.moby"} + } +}` + infoGraphdriver = `{ + "ID": "6ef1", + "Driver": "overlay2", + "DriverStatus": [["Backing Filesystem", "extfs"], ["Supports d_type", "true"]], + "ServerVersion": "29.7.2", + "Containerd": { + "Address": "/var/run/docker/containerd/containerd.sock", + "Namespaces": {"Containers": "moby", "Plugins": "plugins.moby"} + } +}` + infoDocker25 = `{"Driver": "overlay2", "DriverStatus": [], "ServerVersion": "25.0.3"}` +) + +func TestParseDockerInfoContainerdStore(t *testing.T) { + info, err := parseDockerInfo(strings.NewReader(infoContainerdStore)) + if err != nil { + t.Fatal(err) + } + if info.Snapshotter() != "overlayfs" { + t.Fatalf("snapshotter: got %q, want overlayfs", info.Snapshotter()) + } + if info.Containerd == nil || info.Containerd.Address != "/var/run/docker/containerd/containerd.sock" { + t.Fatalf("containerd address: got %+v", info.Containerd) + } + if info.Containerd.Namespaces.Containers != "moby" { + t.Fatalf("containerd namespace: got %q, want moby", info.Containerd.Namespaces.Containers) + } + if info.ServerVersion != "29.7.2" { + t.Fatalf("server version: got %q", info.ServerVersion) + } +} + +func TestParseDockerInfoGraphdriver(t *testing.T) { + info, err := parseDockerInfo(strings.NewReader(infoGraphdriver)) + if err != nil { + t.Fatal(err) + } + if info.Snapshotter() != "" { + t.Fatalf("snapshotter: got %q, want none", info.Snapshotter()) + } + info, err = parseDockerInfo(strings.NewReader(infoDocker25)) + if err != nil { + t.Fatal(err) + } + if info.Snapshotter() != "" || info.Containerd != nil { + t.Fatalf("docker 25 info: got snapshotter %q, containerd %+v", info.Snapshotter(), info.Containerd) + } +} + +func TestVersionAtLeast(t *testing.T) { + for _, tc := range []struct { + version string + min string + want bool + }{ + {"29.5.2", "29.5.2", true}, + {"29.7.2", "29.5.2", true}, + {"29.5.1", "29.5.2", false}, + {"29.4.9", "29.5.2", false}, + {"30.0.0", "29.5.2", true}, + {"28.9.9", "29.5.2", false}, + {"25.0.3", "29.5.2", false}, + {"29.7.2+fio", "29.5.2", true}, + {"20.10.24+dfsg1", "29.5.2", false}, + {"29.5.2-rc1", "29.5.2", true}, + {"v29.5.2", "29.5.2", true}, + {"v29.5.2", "29.7.2", false}, + {"V29.7.2", "29.5.2", true}, + {"v", "29.5.2", false}, + {"29.5", "29.5.2", false}, + {"29.6", "29.5.2", true}, + {"", "29.5.2", false}, + {"garbage", "29.5.2", false}, + } { + if got := versionAtLeast(tc.version, tc.min); got != tc.want { + t.Errorf("versionAtLeast(%q, %q) = %v, want %v", tc.version, tc.min, got, tc.want) + } + } +} + +// On a TCP host the raw GET through the transport cannot reach the daemon, so the typed Info is +// used; it has no Containerd field, and such a host never qualifies for the containerd install. +func TestGetDockerInfoOverTCP(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.HasSuffix(r.URL.Path, "/info") { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, infoContainerdStore) + })) + defer srv.Close() + cli, err := dockerclient.NewClientWithOpts(dockerclient.WithHost("tcp://" + srv.Listener.Addr().String())) + if err != nil { + t.Fatal(err) + } + defer cli.Close() + info, err := GetDockerInfo(context.Background(), cli) + if err != nil { + t.Fatal(err) + } + if info.ServerVersion != "29.7.2" || info.Snapshotter() != "overlayfs" { + t.Fatalf("unexpected info over tcp: %+v", info) + } + if info.Containerd != nil { + t.Fatalf("the typed info must not carry a containerd address: %+v", info.Containerd) + } + if isCtrdInstallTarget(info, cli.DaemonHost()) { + t.Fatal("a tcp host must not be an install target") + } +} + +func TestIsLocalDockerHost(t *testing.T) { + for host, want := range map[string]bool{ + "unix:///var/run/docker.sock": true, + "npipe:////./pipe/docker": true, + "tcp://10.0.0.1:2376": false, + "": false, + } { + if got := isLocalDockerHost(host); got != want { + t.Errorf("isLocalDockerHost(%q) = %v, want %v", host, got, want) + } + } +} diff --git a/pkg/compose/export_test.go b/pkg/compose/export_test.go new file mode 100644 index 0000000..82039ba --- /dev/null +++ b/pkg/compose/export_test.go @@ -0,0 +1,33 @@ +package compose + +import "context" + +// Exported for the tests in package compose_test that drive the containerd install directly. + +type CtrdClient = ctrdClient + +var ( + ConnectCtrdIfInstallTarget = connectCtrdIfInstallTarget + InstallImagesCtrd = installImagesCtrd +) + +func CtrdSnapshotter(c *CtrdClient) string { + return c.snapshotter +} + +// ChainIDsOf returns the chain IDs of each app image, in the app's image order. +func ChainIDsOf(ctx context.Context, app App, provider BlobProvider) ([][]string, error) { + imgs, err := ctrdImagesOf(ctx, app, provider) + if err != nil { + return nil, err + } + var out [][]string + for _, img := range imgs { + var ids []string + for _, id := range img.chainIDs { + ids = append(ids, id.String()) + } + out = append(out, ids) + } + return out, nil +} diff --git a/pkg/compose/image_loader.go b/pkg/compose/image_loader.go index c27738e..dcb97d9 100644 --- a/pkg/compose/image_loader.go +++ b/pkg/compose/image_loader.go @@ -348,25 +348,15 @@ func getImageID(imageURIs []imageURI2RefCounter, imageIndex int, printWarning bo } func adjustOptionsIfContainerd(ctx context.Context, client *client.Client, options *LoadImageOptions) bool { - var isCtrd = false - // Check if the docker daemon is using ctrd storage, which requires special handling when loading images. - if dockerInfo, err := client.Info(ctx); err == nil { - if dockerInfo.Driver == "overlayfs" && - len(dockerInfo.DriverStatus) > 0 && - dockerInfo.DriverStatus[0][1] == "io.containerd.snapshotter.v1" { - isCtrd = true - } - } - - if isCtrd { - if options.ReadBlobsFromStore { - // The ctrd storage does not support loading image layers from an external directory, - // thus, the image load will fail if `options.ReadBlobsFromStore` is set. - options.ReadBlobsFromStore = false - } - } - - return isCtrd + // Check if the docker daemon uses the containerd image store, which requires special handling when loading images. + info, err := GetDockerInfo(ctx, client) + if err != nil || info.Snapshotter() == "" { + return false + } + // The containerd image store does not support loading image layers from an external directory, + // thus, the image load will fail if `options.ReadBlobsFromStore` is set. + options.ReadBlobsFromStore = false + return true } func reportProgressIfContainerd(body io.ReadCloser, imageURIs []imageURI2RefCounter, reporter progress.Reporter[LoadImageProgress]) error { diff --git a/pkg/compose/install.go b/pkg/compose/install.go index 2959bfb..f8a6073 100644 --- a/pkg/compose/install.go +++ b/pkg/compose/install.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/containerd/containerd/platforms" + "github.com/docker/docker/client" "github.com/docker/docker/pkg/archive" "github.com/foundriesio/composeapp/internal/progress" "os" @@ -60,85 +61,119 @@ func Install(ctx context.Context, o(&opts) } + unlock, err := LockInstall(cfg.StoreRoot) + if err != nil { + return err + } + defer unlock() + cs, err := cfg.AppStoreFactory() if err != nil { return fmt.Errorf("failed to create app store instance: %w", err) } app, err := cfg.AppLoader.LoadAppTree(ctx, cs, platforms.OnlyStrict(cfg.Platform), appURI) if err != nil { - return fmt.Errorf("failed to load app %s: %w", app, err) + return fmt.Errorf("failed to load app %s: %w", appURI, err) } - if opts.ProgressReporter != nil { - // TODO: Implement progress reporting for app compose installation - opts.ProgressReporter.Update(InstallProgress{ - AppInstallState: AppInstallStateComposeInstalling, - AppID: app.Ref().String(), - }) + // TODO: Implement progress reporting for app compose installation + reportInstallState(&opts, app, AppInstallStateComposeInstalling) + + if err := installImages(ctx, cfg, app, &opts); err != nil { + return fmt.Errorf("failed to load images for app %s: %w", app.Ref().String(), err) + } + + if err := InstallCompose(ctx, app, cs, cfg.ComposeRoot); err != nil { + return err + } + // TODO: Implement progress reporting for app compose installation checking + reportInstallState(&opts, app, AppInstallStateComposeChecking) + return checkComposeInstallation(ctx, app, cs, cfg) +} + +func reportInstallState(opts *InstallOptions, app App, state AppInstallState) { + if opts.ProgressReporter == nil { + return + } + opts.ProgressReporter.Update(InstallProgress{ + AppInstallState: state, + AppID: app.Ref().String(), + }) +} + +// imageProgressCallback forwards the image load progress to the install progress reporter and +// records the loaded images; it is nil when neither is wanted. +func imageProgressCallback(opts *InstallOptions, app App) ProgressCallbackFn { + if opts.ProgressReporter == nil && opts.LoadedImages == nil { + return nil + } + return func(ilp *LoadImageProgress) { + if opts.LoadedImages != nil && ilp.State == ImageLoadStateImageLoaded { + opts.LoadedImages[ilp.ImageID] = struct{}{} + } + if opts.ProgressReporter != nil { + opts.ProgressReporter.Update(InstallProgress{ + AppInstallState: AppInstallStateImagesLoading, + AppID: app.Ref().String(), + ImageLoadState: ilp.State, + ImageID: ilp.ImageID, + ID: ilp.ID, + Current: ilp.Current, + Total: ilp.Total, + }) + } } +} +// installImages puts the app images into dockerd: straight into its containerd on a daemon that +// uses the containerd image store, with `docker load` otherwise. +func installImages(ctx context.Context, cfg *Config, app App, opts *InstallOptions) error { cli, err := GetDockerClient(cfg.DockerHost) if err != nil { return err } - - var loadImageOptions []LoadImageOption - var withProgressOpt LoadImageOption - if opts.ProgressReporter != nil || opts.LoadedImages != nil { - withProgressOpt = WithProgressReporting(func(ilp *LoadImageProgress) { - if opts.LoadedImages != nil { - if ilp.State == ImageLoadStateImageLoaded { - opts.LoadedImages[ilp.ImageID] = struct{}{} - } - } - if opts.ProgressReporter != nil { - opts.ProgressReporter.Update(InstallProgress{ - AppInstallState: AppInstallStateImagesLoading, - AppID: app.Ref().String(), - ImageLoadState: ilp.State, - ImageID: ilp.ImageID, - ID: ilp.ID, - Current: ilp.Current, - Total: ilp.Total, - }) - } - }) - loadImageOptions = append(loadImageOptions, withProgressOpt) - } - - loadImageOptionsRequiringPatch := append(loadImageOptions, WithRefWithDigest(), WithBlobReadingFromStore()) - // Try to load app images with reading blobs directly from the store and specifying image digests (URI with hashes) - err = LoadImages(ctx, cli, app, cfg.GetBlobsRoot(), loadImageOptionsRequiringPatch...) + defer cli.Close() + report := imageProgressCallback(opts, app) + ctrd, err := connectCtrdIfInstallTarget(ctx, cli) if err != nil { - // Retry loading images without reading blobs directly from the store and specifying the digest, - // in case if the docker daemon is not patch with the Foundries patches - err = LoadImages(ctx, cli, app, cfg.GetBlobsRoot(), loadImageOptions...) + return err + } + if ctrd != nil { + defer ctrd.Close() + return installImagesCtrd(ctx, ctrd, app, cfg, report) + } + return loadImagesWithDocker(ctx, cli, app, cfg.GetBlobsRoot(), report) +} + +// loadImagesWithDocker tries first to load the images with reading blobs directly from the store +// and with digest references, which needs the Foundries patches to dockerd, then the plain way. +func loadImagesWithDocker(ctx context.Context, cli *client.Client, app App, blobsRoot string, report ProgressCallbackFn) error { + var loadOpts []LoadImageOption + if report != nil { + loadOpts = append(loadOpts, WithProgressReporting(report)) } + patchedOpts := append([]LoadImageOption{WithRefWithDigest(), WithBlobReadingFromStore()}, loadOpts...) + err := LoadImages(ctx, cli, app, blobsRoot, patchedOpts...) if err != nil { - return fmt.Errorf("failed to load images for app %s: %w", app.Ref().String(), err) + err = LoadImages(ctx, cli, app, blobsRoot, loadOpts...) } + return err +} - if err := InstallCompose(ctx, app, cs, cfg.ComposeRoot); err != nil { +func checkComposeInstallation(ctx context.Context, app App, cs AppStore, cfg *Config) error { + checkErrMap, err := app.CheckComposeInstallation(ctx, cs, cfg.GetAppComposeDir(app.Name())) + if err != nil { return err } - if opts.ProgressReporter != nil { - // TODO: Implement progress reporting for app compose installation checking - opts.ProgressReporter.Update(InstallProgress{ - AppInstallState: AppInstallStateComposeChecking, - AppID: app.Ref().String(), - }) + if len(checkErrMap) == 0 { + return nil } - if checkErrMap, err := app.CheckComposeInstallation(ctx, cs, cfg.GetAppComposeDir(app.Name())); err != nil { - return err - } else if len(checkErrMap) > 0 { - // TODO: remove prints and return error map - fmt.Println("the following app bundle files are not correctly installed") - for filePath, fileErr := range checkErrMap { - fmt.Printf("\t%s\t%s\n", filePath, fileErr) - } - return fmt.Errorf("app bundle is not installed completely") + // TODO: remove prints and return error map + fmt.Println("the following app bundle files are not correctly installed") + for filePath, fileErr := range checkErrMap { + fmt.Printf("\t%s\t%s\n", filePath, fileErr) } - return err + return fmt.Errorf("app bundle is not installed completely") } func InstallCompose(ctx context.Context, app App, provider BlobProvider, composeRoot string) error { diff --git a/pkg/compose/remove.go b/pkg/compose/remove.go index 4ecf500..3ab8721 100644 --- a/pkg/compose/remove.go +++ b/pkg/compose/remove.go @@ -32,6 +32,11 @@ func RemoveApps(ctx context.Context, cfg *Config, appRefs []string, options ...R for _, o := range options { o(opts) } + unlock, err := LockInstall(cfg.StoreRoot) + if err != nil { + return err + } + defer unlock() if opts.CheckStatus { appsStatus, err := CheckAppsStatus(ctx, cfg, appRefs) if err != nil { diff --git a/pkg/compose/uninstall.go b/pkg/compose/uninstall.go index d550a06..b020fa3 100644 --- a/pkg/compose/uninstall.go +++ b/pkg/compose/uninstall.go @@ -48,8 +48,12 @@ func UninstallApps(ctx context.Context, cfg *Config, appRefs []string, options . if len(appRefs) == 0 && (!opts.Prune || opts.PruneType != PruneTypeAllUnusedImages) { return nil } + unlock, err := LockInstall(cfg.StoreRoot) + if err != nil { + return err + } + defer unlock() - var err error var status *AppsStatus if len(appRefs) > 0 { status, err = CheckAppsStatus(ctx, cfg, appRefs) diff --git a/pkg/update/cancel.go b/pkg/update/cancel.go index 03b8038..b9f7f24 100644 --- a/pkg/update/cancel.go +++ b/pkg/update/cancel.go @@ -12,6 +12,12 @@ import ( ) func (u *runnerImpl) cancel(ctx context.Context) (err error) { + // Removing the loaded images and the fetched blobs must not race with an install. + unlock, err := compose.LockInstall(u.config.StoreRoot) + if err != nil { + return fmt.Errorf("failed to lock the store for the cancel: %w", err) + } + defer unlock() if len(u.LoadedImages) > 0 { cli, err := compose.GetDockerClient(u.config.DockerHost) diff --git a/pkg/update/complete.go b/pkg/update/complete.go index 0c5c73c..4f71830 100644 --- a/pkg/update/complete.go +++ b/pkg/update/complete.go @@ -108,11 +108,17 @@ func (u *runnerImpl) complete(ctx context.Context, options ...CompleteOpt) error } } - // Prune blobs in the app store + // Prune blobs in the app store. UninstallApps and RemoveApps above take the install lock + // themselves, so it is taken only now, for the prune. store, err := u.config.AppStoreFactory() if err != nil { return err } + unlock, err := compose.LockInstall(u.config.StoreRoot) + if err != nil { + return fmt.Errorf("failed to lock the store for the prune: %w", err) + } + defer unlock() _, err = store.Prune(ctx) return err } diff --git a/test/fixtures/composectl_cmds.go b/test/fixtures/composectl_cmds.go index 5385323..c2fa347 100644 --- a/test/fixtures/composectl_cmds.go +++ b/test/fixtures/composectl_cmds.go @@ -483,3 +483,23 @@ func randomString(length int) string { } return string(b) } + +// PublishDerivedImage builds an image on top of base with one layer of sizeMB random bytes and +// pushes it to the test registry through the publisher dockerd, which the publish step reads the +// image layers from. It returns the image reference. +func PublishDerivedImage(t *testing.T, base string, sizeMB int) string { + t.Helper() + name := "derived-" + randomString(8) + ref := "registry:5000/factory/" + name + ":v1" + pub := func(args ...string) { + out, err := publisherDockerCommand(args...).CombinedOutput() + Checkf(t, err, "docker %v: %s", args, out) + } + pub("pull", base) + pub("run", "--name", name, base, "sh", "-c", fmt.Sprintf("head -c %d /dev/urandom > /derived", sizeMB<<20)) + pub("commit", name, ref) + pub("rm", name) + pub("push", ref) + pub("image", "rm", ref) + return ref +} diff --git a/test/integration/ctrd_install_test.go b/test/integration/ctrd_install_test.go new file mode 100644 index 0000000..6f706b1 --- /dev/null +++ b/test/integration/ctrd_install_test.go @@ -0,0 +1,447 @@ +package e2e_tests + +// Tests of the install of app images into dockerd's containerd, see +// docs/containerd-install-plan-2026-10-08.md. They need the dockerd on the containerd image store +// (CONTAINERD_SNAPSHOTTER=true) and skip on the graphdriver one. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "github.com/containerd/containerd" + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/leases" + "github.com/containerd/containerd/platforms" + "github.com/containerd/containerd/snapshots" + "github.com/docker/docker/api/types/filters" + "github.com/foundriesio/composeapp/pkg/compose" + v1 "github.com/foundriesio/composeapp/pkg/compose/v1" + "github.com/foundriesio/composeapp/pkg/update" + f "github.com/foundriesio/composeapp/test/fixtures" + "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/identity" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +const ( + ctrdTestApp = ` +services: + base: + image: ghcr.io/foundriesio/busybox:1.36 + command: sh -c "while true; do sleep 60; done" + multi: + image: registry:5000/factory/runner-image:v0.1 + command: sh -c "while true; do sleep 60; done" +` + ctrdDerivedApp = ` +services: + derived: + image: %s + command: sh -c "while true; do sleep 60; done" +` + ctrdKilledApp = ` +services: + a-base: + image: %s + command: sh -c "while true; do sleep 60; done" + b-derived: + image: %s + command: sh -c "while true; do sleep 60; done" +` +) + +type ( + // ctrdRig is the containerd behind the test dockerd. + ctrdRig struct { + t *testing.T + c *containerd.Client + snapshotter string + contentRoot string + blobsRoot string + } + + ctrdTestImage struct { + refs []string // the pinned digest reference and the short tag reference + manifest ocispec.Manifest + chainIDs []digest.Digest + } +) + +// newCtrdRig connects to dockerd's containerd, or skips the test on a graphdriver dockerd. +func newCtrdRig(t *testing.T) *ctrdRig { + t.Helper() + cli, err := compose.GetDockerClient("") + f.Check(t, err) + defer cli.Close() + info, err := compose.GetDockerInfo(context.Background(), cli) + f.Check(t, err) + if info.Snapshotter() == "" { + t.Skip("dockerd uses a graphdriver, not the containerd image store") + } + c, err := containerd.New(info.Containerd.Address, containerd.WithDefaultNamespace(info.Containerd.Namespaces.Containers)) + f.Check(t, err) + t.Cleanup(func() { c.Close() }) + r := &ctrdRig{t: t, c: c, snapshotter: info.Snapshotter(), blobsRoot: compose.GetBlobsRootFor(f.AppStoreRoot)} + resp, err := c.IntrospectionService().Plugins(context.Background(), []string{"type==io.containerd.content.v1,id==content"}) + f.Check(t, err) + if len(resp.Plugins) != 1 { + t.Fatalf("content plugin not found: %+v", resp.Plugins) + } + r.contentRoot = resp.Plugins[0].Exports["root"] + return r +} + +func (r *ctrdRig) readJSON(d digest.Digest, v any) { + b, err := os.ReadFile(filepath.Join(r.blobsRoot, d.Encoded())) + f.Check(r.t, err) + f.Check(r.t, json.Unmarshal(b, v)) +} + +// images describes the images of the app from the composectl store. +func (r *ctrdRig) images(appURI string) []ctrdTestImage { + cfg := f.NewTestConfig(r.t) + app, err := v1.NewAppLoader().LoadAppTree(context.Background(), compose.NewStoreBlobProvider(r.blobsRoot), + platforms.OnlyStrict(cfg.Platform), appURI) + f.Check(r.t, err) + var out []ctrdTestImage + for _, root := range app.GetComposeRoot().Children { + ref, err := compose.ParseImageRef(root.Ref()) + f.Check(r.t, err) + img := ctrdTestImage{refs: []string{root.Ref(), ref.GetTagRef()}} + mNode := root + if images.IsIndexType(root.Descriptor.MediaType) { + mNode = root.Children[0] + } + r.readJSON(mNode.Descriptor.Digest, &img.manifest) + var imageConfig ocispec.Image + r.readJSON(img.manifest.Config.Digest, &imageConfig) + img.chainIDs = identity.ChainIDs(imageConfig.RootFS.DiffIDs) + out = append(out, img) + } + return out +} + +func (r *ctrdRig) blobInContentRoot(d digest.Digest) bool { + _, err := os.Stat(filepath.Join(r.contentRoot, "blobs", d.Algorithm().String(), d.Encoded())) + if err != nil && !errors.Is(err, os.ErrNotExist) { + f.Check(r.t, err) + } + return err == nil +} + +func (r *ctrdRig) snapshot(chainID digest.Digest) (snapshots.Info, bool) { + info, err := r.c.SnapshotService(r.snapshotter).Stat(context.Background(), chainID.String()) + if err != nil && !errdefs.IsNotFound(err) { + f.Check(r.t, err) + } + return info, err == nil +} + +func (r *ctrdRig) installLeases() []leases.Lease { + all, err := r.c.LeasesService().List(context.Background()) + f.Check(r.t, err) + var found []leases.Lease + for _, l := range all { + if strings.HasPrefix(l.ID, "composectl/install/") { + found = append(found, l) + } + } + return found +} + +func dockerImageExists(ref string) bool { + return exec.Command("docker", "image", "inspect", ref).Run() == nil +} + +// checkInstalled verifies the outcome of the install: dockerd runs every image by both references +// offline, no compressed layer stays in containerd, and every snapshot of the image is there. +func (r *ctrdRig) checkInstalled(app *f.App) { + t := r.t + for _, img := range r.images(app.PublishedUri) { + for _, ref := range img.refs { + c := exec.Command("docker", "run", "--rm", "--pull", "never", "--network", "none", ref, "echo", "ok") + out, err := c.CombinedOutput() + f.Checkf(t, err, "docker run %s: %s", ref, out) + } + for _, layer := range img.manifest.Layers { + if r.blobInContentRoot(layer.Digest) { + t.Errorf("layer %s is still in the containerd content store", layer.Digest) + } + } + if !r.blobInContentRoot(img.manifest.Config.Digest) { + t.Errorf("config %s is not in the containerd content store", img.manifest.Config.Digest) + } + for _, chainID := range img.chainIDs { + if _, ok := r.snapshot(chainID); !ok { + t.Errorf("snapshot %s is missing", chainID) + } + } + } + if leftovers := r.installLeases(); len(leftovers) > 0 { + t.Errorf("install leases left after the install: %+v", leftovers) + } +} + +func (r *ctrdRig) checkNotInstalled(app *f.App) { + for _, img := range r.images(app.PublishedUri) { + for _, ref := range img.refs { + if dockerImageExists(ref) { + r.t.Errorf("image %s is still in dockerd", ref) + } + } + } +} + +// pruneImages removes the images no container uses, as aktualizr-lite does after a store prune, +// and waits for containerd's GC to remove the snapshots of the app images. +func (r *ctrdRig) pruneImages(topChainIDs []digest.Digest) { + cli, err := compose.GetDockerClient("") + f.Check(r.t, err) + defer cli.Close() + _, err = cli.ImagesPrune(context.Background(), filters.NewArgs(filters.Arg("dangling", "false"))) + f.Check(r.t, err) + for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(200 * time.Millisecond) { + if !r.anySnapshot(topChainIDs) { + return + } + } + r.t.Fatal("snapshots of the app images are still there after the prune") +} + +func (r *ctrdRig) anySnapshot(chainIDs []digest.Digest) bool { + for _, chainID := range chainIDs { + if _, ok := r.snapshot(chainID); ok { + return true + } + } + return false +} + +// topChainIDs returns the chain ID of the top snapshot of each app image. +func (r *ctrdRig) topChainIDs(app *f.App) []digest.Digest { + var tops []digest.Digest + for _, img := range r.images(app.PublishedUri) { + tops = append(tops, img.chainIDs[len(img.chainIDs)-1]) + } + return tops +} + +// TestCtrdInstall: an app with an image pinned by a manifest and one pinned by an index is installed +// from the store, runs, is seen by `check --install` and `ps`, and is removed by uninstall. +func TestCtrdInstall(t *testing.T) { + r := newCtrdRig(t) + app := f.NewApp(t, ctrdTestApp) + app.Publish(t) + app.Pull(t) + defer app.Remove(t) + + app.Install(t) + r.checkInstalled(app) + app.CheckInstalled(t) + + app.Run(t) + app.CheckRunning(t) + app.Stop(t) + app.Uninstall(t) + r.checkNotInstalled(app) +} + +// TestCtrdInstallSharedLayers: an update whose image is built on top of the installed version's image +// keeps the base snapshots and adds the new one. +func TestCtrdInstallSharedLayers(t *testing.T) { + r := newCtrdRig(t) + const base = "ghcr.io/foundriesio/busybox:1.36" + appV1 := f.NewApp(t, fmt.Sprintf(ctrdDerivedApp, base), "ctrdshared") + appV1.Publish(t) + appV1.Pull(t) + appV1.Install(t) + r.checkInstalled(appV1) + + baseChain := r.images(appV1.PublishedUri)[0].chainIDs + var baseCreated []time.Time + for _, chainID := range baseChain { + info, ok := r.snapshot(chainID) + if !ok { + t.Fatalf("base snapshot %s is missing", chainID) + } + baseCreated = append(baseCreated, info.Created) + } + + derived := f.PublishDerivedImage(t, base, 4) + appV2 := f.NewApp(t, fmt.Sprintf(ctrdDerivedApp, derived), "ctrdshared") + appV2.Publish(t) + appV2.Pull(t) + defer appV2.Remove(t) + app2Chain := r.images(appV2.PublishedUri)[0].chainIDs + if len(app2Chain) != len(baseChain)+1 || app2Chain[len(baseChain)-1] != baseChain[len(baseChain)-1] { + t.Fatalf("the derived image does not share the base chain: %v vs %v", app2Chain, baseChain) + } + + appV2.Install(t) + defer r.pruneImages(baseChain[len(baseChain)-1:]) // v1's images stay in dockerd until v2 is uninstalled + defer appV2.Uninstall(t) + appV1.Remove(t) // `run` by name needs one version of the app in the store + r.checkInstalled(appV2) + for i, chainID := range baseChain { + info, ok := r.snapshot(chainID) + if !ok || !info.Created.Equal(baseCreated[i]) { + t.Errorf("base snapshot %s was recreated by the update: %v -> %v", chainID, baseCreated[i], info.Created) + } + } + appV2.Run(t) + appV2.CheckRunning(t) + appV2.Stop(t) +} + +// TestCtrdInstallAfterPrune: `/images/prune` with aktualizr-lite's filter removes the installed app +// images, `check --install` reports them missing, and the app is reinstalled from the store. +func TestCtrdInstallAfterPrune(t *testing.T) { + r := newCtrdRig(t) + app := f.NewApp(t, ctrdTestApp) + app.Publish(t) + app.Pull(t) + defer app.Remove(t) + app.Install(t) + r.checkInstalled(app) + + r.pruneImages(r.topChainIDs(app)) + if res := app.GetInstallCheckRes(t); len(res.MissingImages) == 0 { + t.Fatal("check --install reports no missing image after the prune") + } + + app.Install(t) + defer app.Uninstall(t) + r.checkInstalled(app) + app.CheckInstalled(t) + app.Run(t) + app.CheckRunning(t) + app.Stop(t) +} + +// TestCtrdInstallRecoversFromKilledInstall: an install is killed after it created snapshots; its +// lease stays, the next install removes the lease and with it the snapshots it alone held, and +// recreates them, so that nothing a power cut may have broken is reused. +func TestCtrdInstallRecoversFromKilledInstall(t *testing.T) { + r := newCtrdRig(t) + const base = "ghcr.io/foundriesio/busybox:1.36" + // The small base image is installed first; the big derived layer keeps the install busy after that. + app := f.NewApp(t, fmt.Sprintf(ctrdKilledApp, base, f.PublishDerivedImage(t, base, 48))) + app.Publish(t) + app.Pull(t) + defer app.Remove(t) + r.pruneImages(r.topChainIDs(app)) + firstChain := r.images(app.PublishedUri)[0].chainIDs + + install := exec.Command(os.Getenv("COMPOSECTL_EXE"), "install", app.PublishedUri) + f.Check(t, install.Start()) + defer func() { _ = install.Process.Kill(); _ = install.Wait() }() + var killedAt time.Time + for deadline := time.Now().Add(2 * time.Minute); killedAt.IsZero(); time.Sleep(5 * time.Millisecond) { + if time.Now().After(deadline) { + t.Fatal("the install created no lease and snapshot in time") + } + if _, ok := r.snapshot(firstChain[0]); ok && len(r.installLeases()) > 0 { + f.Check(t, install.Process.Kill()) + killedAt = time.Now() + } + } + _ = install.Wait() + if leftovers := r.installLeases(); len(leftovers) != 1 { + t.Fatalf("expected the lease of the killed install to stay, got %+v", leftovers) + } + + app.Install(t) + defer app.Uninstall(t) + r.checkInstalled(app) + for _, img := range r.images(app.PublishedUri) { + for _, chainID := range img.chainIDs { + info, ok := r.snapshot(chainID) + if !ok || !info.Created.After(killedAt) { + t.Errorf("snapshot %s was not recreated after the killed install: created %v, killed at %v", chainID, info.Created, killedAt) + } + } + } + app.Run(t) + app.CheckRunning(t) + app.Stop(t) +} + +// TestCtrdInstallWaitsForLock: a second composectl waits for the install lock instead of installing +// concurrently. +func TestCtrdInstallWaitsForLock(t *testing.T) { + r := newCtrdRig(t) + app := f.NewApp(t, ctrdTestApp) + app.Publish(t) + app.Pull(t) + defer app.Remove(t) + + lock, err := os.OpenFile(filepath.Join(f.AppStoreRoot, ".install.lock"), os.O_RDWR|os.O_CREATE, 0o644) + f.Check(t, err) + defer lock.Close() + f.Check(t, syscall.Flock(int(lock.Fd()), syscall.LOCK_EX)) + + install := exec.Command(os.Getenv("COMPOSECTL_EXE"), "install", app.PublishedUri) + install.Stdout = os.Stdout + install.Stderr = os.Stderr + f.Check(t, install.Start()) + done := make(chan error, 1) + go func() { done <- install.Wait() }() + select { + case err := <-done: + t.Fatalf("install did not wait for the lock: %v", err) + case <-time.After(3 * time.Second): + } + if _, ok := r.snapshot(r.images(app.PublishedUri)[0].chainIDs[0]); ok { + t.Fatal("install created a snapshot while the lock was held") + } + + f.Check(t, syscall.Flock(int(lock.Fd()), syscall.LOCK_UN)) + select { + case err := <-done: + f.Checkf(t, err, "install failed after the lock was released: %v", err) + case <-time.After(5 * time.Minute): + t.Fatal("install did not finish after the lock was released") + } + defer app.Uninstall(t) + r.checkInstalled(app) +} + +// TestCtrdUpdateCancel: `update cancel` after an install into containerd removes the loaded images. +func TestCtrdUpdateCancel(t *testing.T) { + r := newCtrdRig(t) + app := f.NewApp(t, ctrdTestApp) + app.Publish(t) + + ctx := context.Background() + ur, err := update.NewUpdate(f.NewTestConfig(t), "target-ctrd-cancel") + f.Check(t, err) + f.Check(t, ur.Init(ctx, []string{app.PublishedUri})) + f.Check(t, ur.Fetch(ctx)) + f.Check(t, ur.Install(ctx)) + r.checkInstalled(app) + imgs := r.images(app.PublishedUri) // the cancel removes the app blobs from the store + if len(ur.Status().LoadedImages) != 2 { + t.Fatalf("expected 2 loaded images, got %+v", ur.Status().LoadedImages) + } + f.Check(t, ur.Cancel(ctx)) + if ur.Status().State != update.StateCanceled { + t.Fatalf("update not canceled: %s", ur.Status().State) + } + for _, img := range imgs { + if dockerImageExists(img.refs[1]) { + t.Errorf("image %s is still in dockerd after the cancel", img.refs[1]) + } + } + f.Check(t, os.RemoveAll(f.NewTestConfig(t).GetAppComposeDir(app.Name))) + f.Check(t, compose.RemoveApps(ctx, f.NewTestConfig(t), []string{app.PublishedUri}, compose.WithCheckStatus(false))) +}