From d708a829d42f2cc352e13c6ad70d7486f840d247 Mon Sep 17 00:00:00 2001 From: Mike Sul Date: Wed, 21 Oct 2026 14:55:22 +0200 Subject: [PATCH 1/5] docs(design): plan and record the install into containerd On a dockerd with the containerd image store, every app image is kept compressed twice: in composectl's store and in containerd's content store after docker load. The note decides to stream the images from the store into containerd under an install lease and drop the compressed layers after unpack, keeping the store copy as the source for offline repair and reinstall after aktualizr-lite's image prune. It records the install lock, the crash recovery rule (syncfs before the image records), the host gate, the per-filesystem storage precheck, the deferred and rejected options, and the evidence: the rig checks, the write and crash measurements on the board, the Docker 29.5.2 listing check that sets the minimum version, and the power-cut trials that demonstrate the recovery rule, with the two pre-existing limits they surfaced. Assisted-by: Claude Code:claude-fable-5-1 Signed-off-by: Mike Sul --- docs/containerd-install-plan-2026-10-08.md | 429 +++++++++++++++++++++ 1 file changed, 429 insertions(+) create mode 100644 docs/containerd-install-plan-2026-10-08.md diff --git a/docs/containerd-install-plan-2026-10-08.md b/docs/containerd-install-plan-2026-10-08.md new file mode 100644 index 0000000..044cde9 --- /dev/null +++ b/docs/containerd-install-plan-2026-10-08.md @@ -0,0 +1,429 @@ +# Installing app images into dockerd's containerd + +*Design note, 2026-10-08, revised 2026-10-09 after the phase 0 checks and the second +Astra review, and 2026-10-12 with the device result for the minimum Docker version. +Implemented on this branch; the code review record is in the pull request.* + +## Decision + +On a dockerd that uses the containerd image store, `composectl install` streams each +service image from composectl's store into dockerd's containerd under a short install +lease, unpacks it, and creates the image records. Deleting the lease then releases +the compressed layers, and containerd's GC removes them unless something else, such +as an image pulled with `docker pull`, still references them. The only persistent +compressed copy of an app image is then the one in composectl's store. On a +graphdriver dockerd, and on a Docker older than the qualified minimum, install keeps +using `docker load`. + +Pull, the store layout, status, remove, uninstall, the update states and the +aktualizr-lite interface stay as they are. + +| | Compressed copies kept | Unpacked | +|---|---|---| +| Today, containerd image store | 2 (composectl store and containerd content store) | 1 | +| This plan | 1 (composectl store) | 1 | + +On the 380 MB test image this goes from about 1853 MB to about 1473 MB. The extra +compressed copy during install is temporary. + +## Why this route + +- **The store copy keeps the bytes for a repair local.** In the power-cut + experiments (Exp P), a cut up to about 35 s after a pull left snapshots committed + but empty, and a re-pull did not repair them. With the store copy, reinstalling + needs no network. The bytes alone do not repair a broken snapshot: see "Crash + recovery" for what does. +- **aktualizr-lite behaves the same as today.** After a store prune it calls + `/images/prune` with `dangling=false`, which removes images no container uses. + `check --install` then reports the image missing, and aktualizr-lite reinstalls it + from the store. There is no durable lease that the prune would have to work around. +- **Install skips layers that are already unpacked.** Unpack stats each chain ID + before it applies the layer, so base layers shared with the running version are + not streamed again. Today `docker load` sends every blob. The stat checks only that + the snapshot exists, not that its files are intact; "Crash recovery" says why this + is safe for snapshots composectl creates. +- **The registry path does not change.** composectl's resolver, the token proxy, + resume and the split between fetch and install all stay as they are. + +### Options not taken now + +- **Link-in** (hardlink the store blob into the content root): phase 2. Build it only + if device measurements show that the write volume or the install peak matters. Try + the link and fall back to streaming on any error, because a hardlink fails across + mounts. Read the blob back after registration, because GC can remove the linked + file before commit. +- **Pull into containerd and discard** (the 2026-10-08 brief): phase 3. Build it only + if the persistent compressed copy is shown to be the limit. Its retention model + (durable leases on snapshots, three predicates for status, a reconcile step) is + much larger. The Astra review records its blockers. +- **`docker pull` through the Docker API**: rejected. It loses the token proxy auth + path and the split between fetch and install. +- **`docker load` of an OCI layout followed by stripping layer labels**: this was the + fallback if new `go.mod` modules were refused. It is not needed now. Keeping the + pinned digest through this path is untested. + +## Install sequence (containerd image store) + +composectl takes an exclusive lock on `/.install.lock` (`syscall.Flock`) +for every operation that writes to containerd or deletes from it: install, +uninstall, `rm` and `prune`. A second composectl waits for the lock. Concurrent +installs are not supported, so nothing more is needed. + +Under the lock, for each app: + +1. Delete every lease whose ID starts with `composectl/install/`, synchronously. + Under the lock no other install is running, so any such lease was left by an + interrupted install. This also releases the snapshots that install created + (see "Crash recovery"). Then create the lease `composectl/install/@` + with no expiry. +2. For each image, in manifest order, find the first layer whose chain ID has no + snapshot. Stream that layer and every layer after it from the store with + `content.WriteBlob` under the lease. Also stream the config and the manifest, and + the index when the image is pinned by one. +3. Write the manifest without `gc.ref.content.l.N` labels, so that the layers are + held only by the lease. Keep the labels for the config, and on an index keep + `gc.ref.content.m.N` pointing at the platform manifest. `WriteBlob` does not + change the labels of a manifest that already exists, so a manifest written + earlier by `docker pull` keeps its layers (see "Accepted consequences"). +4. Run `Image.Unpack` on the snapshotter that `/info` reports. If it fails because + a layer blob is missing (a base snapshot was removed between step 2 and step 4, + for example by `docker image prune`), stream every layer of that image and run + `Unpack` once more. +5. Call `syncfs` on the snapshotter root, once per app. Required: see check 6. +6. Create or update the image records: the pinned digest ref, and the `GetTagRef` + tag that `ps` and status read today. Use `Create`, and `Update` of the target + when the record already exists, so that a retry is idempotent. +7. Report each installed image through the existing progress callback with + `ImageLoadStateImageLoaded`, so that `LoadedImages` and `update cancel` keep + working. +8. Delete the lease synchronously. containerd's GC then removes the compressed + layers that nothing else references. + +The lease has no expiry. containerd measures `gc.expire` by wall clock, and boards +without a real-time clock can jump forward by months when NTP syncs, which could +expire the lease in the middle of an install. Orphan leases are removed in step 1 +of the next install and by `composectl prune`. + +## Crash recovery + +The rule: a snapshot that a composectl image record references is durable, because +step 5 runs `syncfs` before step 6 creates the records. A snapshot created after +the last `syncfs` can only be referenced through the install lease. + +So after an interrupted install: + +- Snapshots created by the interrupted install are held only by its lease. Step 1 + of the next install deletes that lease synchronously, and GC removes them before + step 2 looks for existing snapshots. A broken snapshot is not reused. +- Records the interrupted install already created point at synced snapshots. The + retry updates them (step 6). +- If the crash came during lease deletion, the lease may be gone with GC not + finished. The records protect everything that must stay. The layer records are + removed before the files are unlinked, so a cut in between leaves orphan files + that containerd removes at its next content cleanup (see the 2026-10-13 results). + +Out of scope, and documented as unsupported on devices: + +- Snapshots created by something other than composectl, such as `docker pull` or + `docker build`, that a crash left broken. composectl cannot tell them from intact + ones, and step 2 would reuse them. The manual repair is to remove the images that + use them and run `docker image prune`. +- A snapshotter whose GC fails to remove a broken snapshot. Exp P saw this with + erofs. Qualifying erofs is a deployment gate (see "Still unproven"). + +## Discovery and host support + +Docker's `/info` is read with a raw GET through the transport of the existing +Docker client. Docker v25.0.3 has no `Containerd` field, and this avoids bumping the +Docker module. composectl uses these fields: + +- `Containerd.Address` and `Containerd.Namespaces.Containers`; +- `Driver`, the snapshotter name, when `DriverStatus` has the entry + `driver-type` = `io.containerd.snapshotter.v1`. This replaces today's `overlayfs` + hardcode and `DriverStatus[0][1]` in `adjustOptionsIfContainerd`; +- `ServerVersion`. + +composectl uses the containerd install only when all of these hold; otherwise it +uses `docker load`, unchanged: + +- the daemon uses the containerd image store (the `driver-type` entry is present); +- the Docker host is a local unix socket, because composectl connects to the + containerd socket and reads the snapshotter root on the same host; +- the server version is at least the qualified minimum. Docker 25 leaves images + whose layers are missing out of `docker image ls` + (`daemon/containerd/image_list.go`), and `ps` and `check --install` read that + list. Docker 29.7.2 lists them (checked, see the results). The minimum is set to + the oldest version on which the listing is checked: 29.5.2, checked on the board + (see "Results on the device, 2026-10-12"). The version string may carry a `v` + prefix (`v29.5.2` on the Arduino build of Docker); the comparison strips it. + +This covers stock Docker and the Foundries-patched dockerd; the containerd route +uses only stock APIs and does not depend on the patch. + +The containerd socket is normally writable only by root. If composectl cannot +connect to it on a daemon that qualifies, it fails with an error that says so. It +does not fall back to `docker load`, because that would quietly bring back the +second compressed copy. + +## Storage precheck + +`checkApps` (`cmd/composectl/cmd/check.go`) checks the total size against +`StoreRoot` only. The new precheck checks each filesystem against its own needs: + +| Term | Bytes | Path | +|---|---|---| +| store | missing blobs, aligned to the block size | `StoreRoot` | +| bundle | the extracted app bundle | `ComposeRoot` | +| install peak | compressed bytes of the layers that have to be streamed | containerd content root | +| runtime | unpacked size of the missing snapshots | snapshotter root | + +The content root and the snapshotter root come from the introspection API +(`Exports["root"]` of each plugin). Terms that share an `st_dev` are added together +before they are compared with the free space. The runtime term counts each missing +snapshot once, by chain ID, even when its compressed blob is already in the store: +the same blob above a different parent is a different snapshot. The unpacked size +comes from the layers metadata, with the existing fallback when an app has none +(`pkg/compose/v1/app.go`), which is an estimate. The current watermark and +reserved-storage options apply to each filesystem. The check counts bytes only: +inodes and concurrent writers are not covered. + +`check --format json` keeps its current fields and adds one entry per filesystem: + +```json +"storage": [ + {"path": "/var/sota/reset-apps", "required": 412000000, "free": 5100000000, "ok": true}, + {"path": "/var/lib/containerd", "required": 1090000000, "free": 5100000000, "ok": true} +] +``` + +aktualizr-lite should check this list instead of using the Docker root alone. That +change is made in the aktualizr-lite repository. + +On a host that uses `docker load` the current single term stays. + +The calculation lives in `pkg/compose`, used by both `cmd/composectl/cmd/check.go` +and `pkg/update/init.go`. `init.go` today adds `blobStoreSize` to +`runtimeSizeTotal` (and adds it twice to `totalSize`); both should use +`blobRuntimeSize`. This fixes the reported numbers only: `init.go` does not reject +an update on size, and that stays as it is. + +The erofs snapshotter may need temporary space beyond the unpacked size while it +converts a layer. That term is added when erofs is qualified. + +## Code changes + +| File | Change | +|---|---| +| `pkg/compose/docker_client.go` | `/info` discovery (address, namespace, snapshotter) | +| `pkg/compose/ctrd_install.go` (new) | the install sequence above, crash recovery and the install lock | +| `pkg/compose/install.go` | split `Install` (88 lines, over the 60-line limit) and pick containerd or `docker load` per daemon | +| `pkg/compose/image_loader.go` | `adjustOptionsIfContainerd` uses the discovered snapshotter | +| `pkg/compose/storage.go` (new) | the per-filesystem sizing shared by `check` and `update init` | +| `cmd/composectl/cmd/check.go` | uses it; the `storage` JSON field | +| `pkg/update/init.go` | uses it, and the runtime-size fix | +| `pkg/compose/remove.go`, `uninstall.go`, `cmd/composectl/cmd/prune.go` | take the install lock; `prune` deletes orphan install leases | +| `go.mod`, `go.sum` | the root package of the containerd v1.7.15 client is imported. This adds 17 indirect modules and makes `golang.org/x/sys` direct (list below). No Go bump: v1.7.15 declares Go 1.21. The v2 client needs Go 1.23 (v2.0, v2.1) or 1.24.3 (v2.2). | +| `test/compose/docker-compose.yml`, `test/integration/` | a stock dockerd 29 service with the containerd image store, and the tests below | + +New files: the containerd install is a second implementation next to `docker load`; +putting it into `install.go` would mix the two paths. The sizing is shared by two +callers that today compute it separately, and differently. + +Unchanged: fetch, status (`ps` and `check --install` read the Docker image list, +which lists these images on qualified versions), the store, the update state +machine, and `MakeAkliteHappy`. + +Not emitted: images created directly in containerd produce no Docker `load` or +`pull` event. composectl does not read Docker events; whether aktualizr-lite does is +to be checked in its repository. + +Modules added to `go.mod` as indirect, plus `golang.org/x/sys` moving from indirect to +direct: +`AdamKorcz/go-118-fuzz-build`, `containerd/cgroups`, `containerd/continuity`, +`containerd/fifo`, `containerd/ttrpc`, `containerd/typeurl/v2`, +`docker/go-events`, `golang/groupcache`, `google/go-cmp`, `google/uuid`, +`moby/sys/mountinfo`, `moby/sys/signal`, `opencontainers/runtime-spec`, +`opencontainers/selinux`, `go.opencensus.io`, `golang.org/x/text`, +`google.golang.org/genproto`. AGENTS.md asks for approval of new modules; this list +is that request. + +## Phase 0: rig checks before code + +1. The containerd v1.7.15 client against containerd 2.4.1 under dockerd 29: + `WriteBlob` under a lease, `Unpack`, `Create` image, and synchronous lease delete. +2. dockerd lists, runs and `compose up`s the image offline, both by the pinned digest + and by the short tag. +3. The compressed layers are gone from the content root after install, and the + snapshots stay. +4. `/images/prune` with aktualizr-lite's filter removes the unused image, and + reinstalling from the store works offline. +5. An update that shares base layers streams only the layers that differ. +6. Exp P repeated with `syncfs` before the records are created: no committed but + broken snapshot after a cut. +7. On the target eMMC: bytes written and install time, compared with `docker load`. + +If check 1 fails, the fallback is a newer 1.7.x client before considering v2. + +### Results, 2026-10-09 (dev container) + +Run in the test environment of this repository on the containerd image store with +dockerd 29.7.2 and 29.7.2+fio, both on containerd 2.4.1 with the overlayfs +snapshotter. A temporary probe test published and pulled two app versions with the +test fixtures and ran the install sequence above against dockerd's containerd. Both +daemons gave the same results. + +| Check | Result | +|---|---| +| 1. v1.7.15 client against containerd 2.4.1 | Pass. `WriteBlob` under the lease, `Unpack`, image create and synchronous lease delete all work, also for an image pinned by an index. Discovery through a raw `/info` GET and the content and snapshotter roots from introspection work as described. | +| 2. Offline run | Pass. `docker image inspect` and `docker run --pull never --network none` work by the pinned digest and by the short tag; `docker compose up --pull never` of the installed bundle starts every service. | +| 3. Layers gone, snapshots kept | Pass. After install no layer blob is in the content root; the manifest, config and every snapshot in the chain are. | +| 4. Prune and offline reinstall | Pass. `/images/prune` with `dangling=false` removes the records, the snapshots and the config; reinstalling from the store streams every layer again and the app runs offline. | +| 5. Shared base layers | Pass. v2 replaces one image with an image built on top of v1's base: install streamed 1 of 3 layers. `Unpack` does not read the content of a layer whose snapshot exists. | +| 6. Power cut with `syncfs` | On the device, see below. | +| 7. eMMC writes and time | On the device, see below. In the dev container, one comparison for 8.6 MB of layers: 0.49 s for the install, 0.40 s for `docker load`. | +| Status on the discarded layers (added after the review) | Pass on 29.7.2, stock and fio. `docker image ls` lists both images with tag, digest, ID and size; `composectl check --install` reports no missing image; `composectl run` and `ps` report the app running; `stop`, `uninstall` and `rm` work. | + +### Results on the device, 2026-10-09 (checks 6 and 7) + +Board: Arduino UNO Q `unoq-dev`, kernel 7.0.14, ext4 on eMMC, stock dockerd 29.5.2 on +the containerd image store with containerd 2.3.3 and the overlayfs snapshotter. App: +node-red 4.1.11 (v1, 17 layers, 208 MB compressed), updated to 4.1.12 (v2, 13 new +layers, 163 MB). A probe binary ran the install sequence above against a composectl +store copied to the board, or `docker load` from the same store (the unpatched path). + +**Check 7, 3 rounds, arms alternated:** + +| | containerd install | `docker load` | +|---|---|---| +| v1 cold: time | 160–161 s | 153–178 s | +| v1 cold: written to eMMC | 993 MB | 992–993 MB | +| v1 → v2 update: time | 126–128 s | 123–128 s | +| v1 → v2 update: written to eMMC | 797–801 MB | 846–849 MB | +| Disk used with v1 and v2 installed | 1244 MiB | 1602 MiB | + +Cold installs write the same and take about the same time. The update writes 45–52 MB +less with the containerd install in all three rounds, about 6 %. Three rounds per +arm are not enough to call the times equal; per-stage times were not recorded. The +containerd install keeps 358 MiB less on disk, which is the compressed copy of both +versions that `docker load` leaves in containerd. The update streamed only the 13 +layers v1 did not have. + +**Check 6: crash right after install returns.** `echo b > /proc/sysrq-trigger` with +no sync, so the page cache is lost as in a power cut; the eMMC stays powered, so this +is milder than a real cut. 3 trials per arm, compared with a content hash of the +image taken after a clean install. + +| Arm | Result | +|---|---| +| no `syncfs` | 3 of 3 broken: the record and all 17 snapshots are committed, but `docker create` fails. In trials 2 and 3 the error is `failed to stat parent: stat .../snapshots/1139/fs: no such file or directory`; trial 1 ran a probe that did not record the error text | +| `syncfs` before the records | 3 of 3 intact. The content hash matches the clean install | + +`syncfs` (step 5) is required, not optional. Without it, a crash leaves an image that +looks installed and cannot run, and a reinstall would not repair it, because step 2 +skips layers whose snapshot exists. Removing the image records let GC delete the +broken snapshots in all three trials. + +The crash came after the records were created and the lease deleted. The content +hash covers the path, type, size and contents of every regular file, not +permissions, owners, link targets or xattrs. Cuts at other points are listed under +"Still unproven". + +One change to the plan: importing the root package of the client adds 17 indirect +modules to `go.mod`, not three, and makes `golang.org/x/sys` direct; see "Code +changes" for the list. + +### Results on the device, 2026-10-12 (minimum Docker version) + +Same board, Docker v29.5.2, with `composectl` built from the implementation branch +(arm64), the node-red apps above copied to the board as a store. + +The first run found a gate bug: `docker info` on this board reports the server +version as `v29.5.2`, the version comparison read that as `0.0.0`, and install +silently used `docker load` (the content store kept the compressed layers and the +records were the `docker load` ones). The comparison now strips a leading `v`. + +The rerun with the fix took the containerd path: + +| Step | Result | +|---|---| +| install v1 | 3m12s. Snapshots 0 → 17; content blobs 4 → 7 (index, manifest, config, no layers); records for the tag and the pinned digest, both targeting the OCI index; no lease left | +| `docker image ls --digests` | lists the image with tag, digest, ID and size | +| `check --install`, `ps` | no missing image | +| `run` (install + `docker compose up`) | the container starts from the installed image, no pull | +| install v2 (update) | 2m30s; only the 13 new layers streamed; both images listed | +| `uninstall --prune`, `rm` | board back to 0 images, 0 snapshots, 4 content blobs, no composectl leases | + +The listing is therefore checked on 29.5.2 and 29.7.2; the minimum stays 29.5.2. + +### Results on the device, 2026-10-13 (cuts during install, real composectl) + +Same board and apps, `composectl` built from the reviewed tree. Every cut is a +sysrq-b with nothing synced, fired on the board itself: during unpack by a loop +that watches the snapshot directory count; between the first image record and the +lease delete by a probe that polls containerd's image service every 300 µs for the +pinned-digest record; right after `install` returns by the shell. The board came +back on its own each time (77–246 s) and the checks ran with no manual step. + +| Cut | Trials | State after the reboot | Recovery | +|---|---|---|---| +| During unpack, at snapshot 3, 9 and 15 of 17 | 3 | Lease present, the 17 streamed layer blobs present. Snapshots torn in two trials: the metadata had 9 and 15 committed snapshots, the disk had 3 and 7 directories. `check --install` reports the image missing | The next `install` deleted the lease, GC dropped the torn snapshots, the image was unpacked again (17 snapshots, blobs back to 7), the container ran, `check` clean. 3 of 3 | +| After the first image record, before the lease delete | 3 | Both records, 17 snapshots, 24 blobs and the lease present; no bundle yet, so `check --install` reports the image present and `docker-compose.yml` missing | The next `install` (or `run`) deleted the stale lease, GC dropped the layers (blobs 7), the bundle was written, the container ran, `check` clean. 3 of 3 | +| Right after `install` returned | 5 | Records, snapshots and the lease deletion durable in all five; the image ran in all five. In two, the content metadata had 7 records but 24 files on disk: the unlinks of the 17 layer files were lost. The bundle was missing in all five | The next `install` rewrote the bundle. The orphan layer files survived that install's GC and were removed by the uninstall | + +`docker image prune -a` with a stopped container of the app keeps the image (Docker +drops the tag record only); without a container it removes the image, `check +--install` reports it missing, and `install` brings it back from the store and the +container runs. + +Two findings, neither introduced by this change: + +1. The compose bundle is not durable when `install` returns: `InstallCompose` + untars it without a sync, on both routes. A cut inside the writeback window + loses it, `check --install` reports it, the next install rewrites it. Follow-up: + `syncfs` on the compose root after the untar. +2. A cut between the GC's metadata commit and its unlinks leaves the compressed + layers as files unknown to containerd (208 MB here). containerd removes such + files only in `cleanupContent`, which walks the blob directory after a GC that + removed a content record, so they stay until the next image removal on the + device (the uninstall of the previous version at the next update, or `docker + image prune`). Bounded to one install's compressed layers and self-healing. + Follow-up option: let `composectl prune` force a content GC. + +## Still unproven, ranked by risk + +1. Real power cuts on the eMMC, and the fleet's snapshotter. The crash trials used a + sysrq reboot and overlayfs only. If the fleet uses erofs (with its differ and + fs-verity), the crash checks are repeated there before deployment; overlayfs + results do not qualify erofs. +2. The storage precheck on separate mounts and on a nearly full filesystem (an app + whose image config is unreadable is covered by the integration test). + +1 is a deployment gate; 2 and the two follow-ups above are follow-ups. + +## Integration test + +Against the dockerd service on the containerd image store, stock and fio: + +- install from the store with no network; +- an update that shares layers with the installed version; +- prune followed by an offline reinstall; +- a lease left by a killed install is removed on the next install, and the app runs; +- a second composectl waits for the install lock; +- a base snapshot removed by `docker image prune` between the scan and `Unpack` + (the retry in step 4); +- an image pinned by an index; +- `ps`, `check --install`, `update cancel` and uninstall after a containerd install; +- the graphdriver path is unchanged (the existing tests). + +## Accepted consequences (user, 2026-10-08) + +- The persistent compressed copy in the store is acceptable for the first phases. +- `docker save` and `docker push` of app images on the device no longer work on the + containerd image store: the layers are not in containerd. +- The fleet is new. Images loaded the old way, or pulled with `docker pull`, whose + manifests carry layer labels, keep their compressed layers in containerd. +- composectl is the only writer of app images on devices. `docker pull` and + `docker build` there are not supported for app images (see "Crash recovery"). +- Concurrent composectl installs are serialized, not run in parallel. +- `/var/sota/reset-apps` and containerd's root are on the same mount on devices. + composectl does not rely on this; the precheck merges terms by `st_dev`. From 938c693c1ff023b889e8295fd6cdeae324107bc5 Mon Sep 17 00:00:00 2001 From: Mike Sul Date: Mon, 12 Oct 2026 14:17:09 +0200 Subject: [PATCH 2/5] feat(compose): discover containerd behind dockerd from /info The install into dockerd's containerd needs the containerd socket, the namespace and the snapshotter that dockerd uses. The Docker v25 client bundled with composectl has no Containerd field in its Info type, and bumping the Docker module is out of scope, so on a local socket `/info` is read with a raw GET through the client's own transport and decoded into the few fields composectl needs. On a tcp host that raw request would not reach the daemon (the client's address and TLS live in its request path), so the typed Info is used there; it has no Containerd field, and such a host is never an install target anyway. The snapshotter now comes from the `driver-type` entry of DriverStatus instead of the `overlayfs` hardcode and `DriverStatus[0][1]`, so a daemon on another snapshotter (erofs, for example) is detected too. The server version is compared numerically with a leading v stripped: the Arduino build of Docker reports `v29.5.2`. Signed-off-by: Mike Sul Assisted-by: Claude Code:claude-fable-5-1 (cherry picked from commit ae1118100fb3c5fc7f971e4ca5b79704fdf98405) --- pkg/compose/docker_client.go | 114 +++++++++++++++++++++++ pkg/compose/docker_client_test.go | 146 ++++++++++++++++++++++++++++++ 2 files changed, 260 insertions(+) create mode 100644 pkg/compose/docker_client_test.go diff --git a/pkg/compose/docker_client.go b/pkg/compose/docker_client.go index 2acd74f..4eea676 100644 --- a/pkg/compose/docker_client.go +++ b/pkg/compose/docker_client.go @@ -1,9 +1,34 @@ package compose import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "strings" + dockerclient "github.com/docker/docker/client" ) +type ( + // DockerInfo is the part of dockerd's `/info` that composectl reads. + // It is decoded from a raw GET because the Docker v25 client's Info type has no Containerd field. + DockerInfo struct { + ServerVersion string + Driver string + DriverStatus [][2]string + Containerd *ContainerdInfo + } + ContainerdInfo struct { + Address string + Namespaces struct{ Containers string } + } +) + +const snapshotterPluginType = "io.containerd.snapshotter.v1" + func GetDockerClient(dockerHost string) (*dockerclient.Client, error) { opts := []dockerclient.Opt{ dockerclient.FromEnv, @@ -13,3 +38,92 @@ func GetDockerClient(dockerHost string) (*dockerclient.Client, error) { } return dockerclient.NewClientWithOpts(opts...) } + +// GetDockerInfo reads dockerd's `/info`. On a local socket it is a raw GET through the client's +// transport, which is the only way to get the Containerd field with the Docker v25 client. On a +// TCP host the raw request would not reach the daemon (the client's address and TLS live in its +// request path), so the typed Info is used there; it has no Containerd field, and that host is +// never an install target anyway (see isCtrdInstallTarget). +func GetDockerInfo(ctx context.Context, cli *dockerclient.Client) (*DockerInfo, error) { + if !isLocalDockerHost(cli.DaemonHost()) { + info, err := cli.Info(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get docker info: %w", err) + } + return &DockerInfo{ServerVersion: info.ServerVersion, Driver: info.Driver, DriverStatus: info.DriverStatus}, nil + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://docker/info", nil) + if err != nil { + return nil, fmt.Errorf("failed to build the docker info request: %w", err) + } + resp, err := cli.HTTPClient().Do(req) + if err != nil { + return nil, fmt.Errorf("failed to get docker info: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to get docker info: %s", resp.Status) + } + info, err := parseDockerInfo(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to decode docker info: %w", err) + } + return info, nil +} + +// isLocalDockerHost reports whether the docker host is a socket on this machine. +func isLocalDockerHost(dockerHost string) bool { + return strings.HasPrefix(dockerHost, "unix://") || strings.HasPrefix(dockerHost, "npipe://") +} + +func parseDockerInfo(r io.Reader) (*DockerInfo, error) { + var info DockerInfo + if err := json.NewDecoder(r).Decode(&info); err != nil { + return nil, err + } + return &info, nil +} + +// Snapshotter returns the name of the containerd snapshotter that dockerd uses, +// or "" when dockerd uses a graphdriver instead of the containerd image store. +func (i *DockerInfo) Snapshotter() string { + for _, kv := range i.DriverStatus { + if kv[0] == "driver-type" && kv[1] == snapshotterPluginType { + return i.Driver + } + } + return "" +} + +// versionAtLeast reports whether a dotted version such as "29.7.2", "v29.5.2" or "20.10.24+dfsg1" +// is not older than min. Only the leading numeric components are compared. +func versionAtLeast(version, min string) bool { + have := versionNumbers(version) + want := versionNumbers(min) + for len(have) < len(want) { + have = append(have, 0) + } + for i, w := range want { + if have[i] != w { + return have[i] > w + } + } + return true +} + +func versionNumbers(version string) []int { + version = strings.TrimLeft(version, "vV") // some builds report "v29.5.2" + end := strings.IndexFunc(version, func(r rune) bool { return r != '.' && (r < '0' || r > '9') }) + if end >= 0 { + version = version[:end] + } + var numbers []int + for _, part := range strings.Split(version, ".") { + n, err := strconv.Atoi(part) + if err != nil { + break + } + numbers = append(numbers, n) + } + return numbers +} diff --git a/pkg/compose/docker_client_test.go b/pkg/compose/docker_client_test.go new file mode 100644 index 0000000..378f734 --- /dev/null +++ b/pkg/compose/docker_client_test.go @@ -0,0 +1,146 @@ +package compose + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + dockerclient "github.com/docker/docker/client" +) + +// Trimmed `/info` responses of dockerd 29.7.2 on the containerd image store and on the overlay2 graphdriver. +const ( + infoContainerdStore = `{ + "ID": "6ef1", + "Driver": "overlayfs", + "DriverStatus": [["driver-type", "io.containerd.snapshotter.v1"]], + "ServerVersion": "29.7.2", + "Containerd": { + "Address": "/var/run/docker/containerd/containerd.sock", + "Namespaces": {"Containers": "moby", "Plugins": "plugins.moby"} + } +}` + infoGraphdriver = `{ + "ID": "6ef1", + "Driver": "overlay2", + "DriverStatus": [["Backing Filesystem", "extfs"], ["Supports d_type", "true"]], + "ServerVersion": "29.7.2", + "Containerd": { + "Address": "/var/run/docker/containerd/containerd.sock", + "Namespaces": {"Containers": "moby", "Plugins": "plugins.moby"} + } +}` + infoDocker25 = `{"Driver": "overlay2", "DriverStatus": [], "ServerVersion": "25.0.3"}` +) + +func TestParseDockerInfoContainerdStore(t *testing.T) { + info, err := parseDockerInfo(strings.NewReader(infoContainerdStore)) + if err != nil { + t.Fatal(err) + } + if info.Snapshotter() != "overlayfs" { + t.Fatalf("snapshotter: got %q, want overlayfs", info.Snapshotter()) + } + if info.Containerd == nil || info.Containerd.Address != "/var/run/docker/containerd/containerd.sock" { + t.Fatalf("containerd address: got %+v", info.Containerd) + } + if info.Containerd.Namespaces.Containers != "moby" { + t.Fatalf("containerd namespace: got %q, want moby", info.Containerd.Namespaces.Containers) + } + if info.ServerVersion != "29.7.2" { + t.Fatalf("server version: got %q", info.ServerVersion) + } +} + +func TestParseDockerInfoGraphdriver(t *testing.T) { + info, err := parseDockerInfo(strings.NewReader(infoGraphdriver)) + if err != nil { + t.Fatal(err) + } + if info.Snapshotter() != "" { + t.Fatalf("snapshotter: got %q, want none", info.Snapshotter()) + } + info, err = parseDockerInfo(strings.NewReader(infoDocker25)) + if err != nil { + t.Fatal(err) + } + if info.Snapshotter() != "" || info.Containerd != nil { + t.Fatalf("docker 25 info: got snapshotter %q, containerd %+v", info.Snapshotter(), info.Containerd) + } +} + +func TestVersionAtLeast(t *testing.T) { + for _, tc := range []struct { + version string + min string + want bool + }{ + {"29.5.2", "29.5.2", true}, + {"29.7.2", "29.5.2", true}, + {"29.5.1", "29.5.2", false}, + {"29.4.9", "29.5.2", false}, + {"30.0.0", "29.5.2", true}, + {"28.9.9", "29.5.2", false}, + {"25.0.3", "29.5.2", false}, + {"29.7.2+fio", "29.5.2", true}, + {"20.10.24+dfsg1", "29.5.2", false}, + {"29.5.2-rc1", "29.5.2", true}, + {"v29.5.2", "29.5.2", true}, + {"v29.5.2", "29.7.2", false}, + {"V29.7.2", "29.5.2", true}, + {"v", "29.5.2", false}, + {"29.5", "29.5.2", false}, + {"29.6", "29.5.2", true}, + {"", "29.5.2", false}, + {"garbage", "29.5.2", false}, + } { + if got := versionAtLeast(tc.version, tc.min); got != tc.want { + t.Errorf("versionAtLeast(%q, %q) = %v, want %v", tc.version, tc.min, got, tc.want) + } + } +} + +// On a TCP host the raw GET through the transport cannot reach the daemon, so the typed Info is +// used; it has no Containerd field, and such a host never qualifies for the containerd install. +func TestGetDockerInfoOverTCP(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.HasSuffix(r.URL.Path, "/info") { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, infoContainerdStore) + })) + defer srv.Close() + cli, err := dockerclient.NewClientWithOpts(dockerclient.WithHost("tcp://" + srv.Listener.Addr().String())) + if err != nil { + t.Fatal(err) + } + defer cli.Close() + info, err := GetDockerInfo(context.Background(), cli) + if err != nil { + t.Fatal(err) + } + if info.ServerVersion != "29.7.2" || info.Snapshotter() != "overlayfs" { + t.Fatalf("unexpected info over tcp: %+v", info) + } + if info.Containerd != nil { + t.Fatalf("the typed info must not carry a containerd address: %+v", info.Containerd) + } +} + +func TestIsLocalDockerHost(t *testing.T) { + for host, want := range map[string]bool{ + "unix:///var/run/docker.sock": true, + "npipe:////./pipe/docker": true, + "tcp://10.0.0.1:2376": false, + "": false, + } { + if got := isLocalDockerHost(host); got != want { + t.Errorf("isLocalDockerHost(%q) = %v, want %v", host, got, want) + } + } +} From 3cf4d2ea95f3ac6ad8c9fe9c0e9817e296d59e56 Mon Sep 17 00:00:00 2001 From: Mike Sul Date: Mon, 12 Oct 2026 14:17:10 +0200 Subject: [PATCH 3/5] feat(compose): install app images into dockerd's containerd On a dockerd that uses the containerd image store, `docker load` keeps a second compressed copy of every app image next to the one in composectl's store. Streaming the images straight into dockerd's containerd under a short lease, unpacking them and then deleting the lease lets containerd's GC drop the compressed layers, so the store copy is the only persistent one (about 380 MB less per 380 MB image). Layers whose snapshot already exists are not streamed, which makes an update that shares base layers with the installed version cheaper than `docker load`, which sends every blob. The manifest is written without layer labels so that only the lease holds the layers; the snapshotter's filesystem is synced before the image records are created, because a power cut after the records otherwise leaves an image that looks installed but cannot run. Leases left by an interrupted install are deleted synchronously by the next install, which also drops the snapshots that only they hold, so a broken snapshot is never reused. If the second image record cannot be written, the first is deleted again on a context that survives the caller's cancellation, so an image is either fully recorded and reported loaded, or absent. Installs, uninstalls, removals and prunes are serialized by a flock on `/.install.lock`. The route is used only on a daemon with the containerd image store, on a local unix socket and at least Docker 29.5.2, the oldest version on which `docker image ls` lists images whose layers are not in containerd. If containerd cannot be reached on such a daemon the install fails instead of falling back to `docker load`, which would quietly bring the second copy back. Other daemons keep `docker load`. Importing the root package of the containerd v1.7.15 client adds 17 indirect modules and makes golang.org/x/sys direct; `go mod tidy` was run with the go directive kept at 1.22.12. Design note: docs/containerd-install-plan-2026-10-08.md. Signed-off-by: Mike Sul Assisted-by: Claude Code:claude-fable-5-1 (cherry picked from commit 10f6ec8a15d82a1eccd5eedbe349a2a4b0df307e) --- go.mod | 19 +- go.sum | 93 ++++++ pkg/compose/ctrd_install.go | 479 ++++++++++++++++++++++++++++++ pkg/compose/ctrd_install_test.go | 123 ++++++++ pkg/compose/docker_client_test.go | 3 + pkg/compose/image_loader.go | 28 +- pkg/compose/install.go | 147 +++++---- 7 files changed, 816 insertions(+), 76 deletions(-) create mode 100644 pkg/compose/ctrd_install.go create mode 100644 pkg/compose/ctrd_install_test.go diff --git a/go.mod b/go.mod index 967fcc8..5e25e65 100644 --- a/go.mod +++ b/go.mod @@ -20,28 +20,39 @@ require ( go.etcd.io/bbolt v1.3.7 golang.org/x/net v0.26.0 golang.org/x/sync v0.7.0 + golang.org/x/sys v0.29.0 gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect + github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/Microsoft/hcsshim v0.11.4 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/containerd/cgroups v1.1.0 // indirect + github.com/containerd/continuity v0.4.2 // indirect + github.com/containerd/fifo v1.1.0 // indirect github.com/containerd/log v0.1.0 // indirect + github.com/containerd/ttrpc v1.2.3 // indirect + github.com/containerd/typeurl/v2 v2.1.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/docker-credential-helpers v0.7.0 // indirect github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.3.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/uuid v1.3.1 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -50,9 +61,13 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/locker v1.0.1 // indirect + github.com/moby/sys/mountinfo v0.6.2 // indirect github.com/moby/sys/sequential v0.5.0 // indirect + github.com/moby/sys/signal v0.7.0 // indirect github.com/moby/sys/user v0.1.0 // indirect github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/runtime-spec v1.1.0 // indirect + github.com/opencontainers/selinux v1.11.0 // indirect github.com/prometheus/client_golang v1.17.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.44.0 // indirect @@ -64,6 +79,7 @@ require ( github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect + go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect go.opentelemetry.io/otel v1.21.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 // indirect @@ -72,8 +88,9 @@ require ( go.opentelemetry.io/otel/trace v1.21.0 // indirect golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect golang.org/x/mod v0.17.0 // indirect - golang.org/x/sys v0.29.0 // indirect + golang.org/x/text v0.16.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/grpc v1.59.0 // indirect google.golang.org/protobuf v1.33.0 // indirect diff --git a/go.sum b/go.sum index e2a914a..7be12f7 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,11 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 h1:59MxjQVfjXsBpLy+dbd2/ELV5ofnUkUZBvWSC85sheA= +github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0/go.mod h1:OahwfttHWG6eJ0clwcfBAHoDI6X/LV/15hx/wlMZSrU= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8= @@ -14,8 +18,11 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/compose-spec/compose-go v1.20.2 h1:u/yfZHn4EaHGdidrZycWpxXgFffjYULlTbRfJ51ykjQ= github.com/compose-spec/compose-go v1.20.2/go.mod h1:+MdqXV4RA7wdFsahh/Kb8U0pAJqkg7mr4PM9tFKU8RM= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= @@ -24,8 +31,14 @@ github.com/containerd/containerd v1.7.15 h1:afEHXdil9iAm03BmhjzKyXnnEBtjaLJefdU7 github.com/containerd/containerd v1.7.15/go.mod h1:ISzRRTMF8EXNpJlTzyr2XMhN+j9K302C21/+cr3kUnY= github.com/containerd/continuity v0.4.2 h1:v3y/4Yz5jwnvqPKJJ+7Wf93fyWoCB3F5EclWG023MDM= github.com/containerd/continuity v0.4.2/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= +github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY= +github.com/containerd/fifo v1.1.0/go.mod h1:bmC4NWMbXlt2EZ0Hc7Fx7QzTFxgPID13eH0Qu+MAb2o= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/ttrpc v1.2.3 h1:4jlhbXIGvijRtNC8F/5CpuJZ7yKOBFGFOOXg1bkISz0= +github.com/containerd/ttrpc v1.2.3/go.mod h1:ieWsXucbb8Mj9PH0rXCw1i8IunRbbAiDkpXkbfflWBM= +github.com/containerd/typeurl/v2 v2.1.1 h1:3Q4Pt7i8nYwy2KmQWIw2+1hTvwTE/6w9FqcttATPO/4= +github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3HZj1hsSQlywkQ0= github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= @@ -45,12 +58,18 @@ github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryef github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -65,17 +84,37 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= @@ -112,6 +151,8 @@ github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vyg github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/moby/sys/signal v0.7.0 h1:25RW3d5TnQEoKvRbEKUGay6DCQ46IxAVTT9CUMgmsSI= +github.com/moby/sys/signal v0.7.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= @@ -129,6 +170,10 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg= +github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= +github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -142,6 +187,7 @@ github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1 github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -166,9 +212,14 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -204,20 +255,31 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -225,6 +287,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -232,6 +295,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -242,6 +307,10 @@ golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -251,13 +320,35 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= +google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q= google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -270,3 +361,5 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/compose/ctrd_install.go b/pkg/compose/ctrd_install.go new file mode 100644 index 0000000..d909799 --- /dev/null +++ b/pkg/compose/ctrd_install.go @@ -0,0 +1,479 @@ +package compose + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/containerd/containerd" + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/leases" + "github.com/containerd/containerd/platforms" + dockerclient "github.com/docker/docker/client" + "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/identity" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "golang.org/x/sys/unix" +) + +// This file installs app images straight into the containerd behind dockerd, following +// docs/containerd-install-plan-2026-10-08.md. The compressed layers are streamed under a short +// lease and dropped by containerd's GC once the lease is deleted, so the only persistent compressed +// copy of an app image is the one in composectl's store. + +const ( + // MinDockerVersionForCtrdInstall is the oldest dockerd on which `docker image ls` lists an image + // whose compressed layers are not in containerd; `ps` and `check --install` read that list. + MinDockerVersionForCtrdInstall = "29.5.2" + + installLeasePrefix = "composectl/install/" + installLockFileName = ".install.lock" + contentPluginType = "io.containerd.content.v1" + gcRefConfigLabel = "containerd.io/gc.ref.content.config" + gcRefManifestLabel = "containerd.io/gc.ref.content.m.0" + // recordCleanupTimeout bounds the removal of partial image records after a failed write. + recordCleanupTimeout = 30 * time.Second +) + +type ( + // ctrdClient is a connection to the containerd behind dockerd. + ctrdClient struct { + *containerd.Client + snapshotter string + contentRoot string + snapshotterRoot string + } + + // ctrdImage is one app image as the containerd install needs it. + ctrdImage struct { + ref string // the pinned digest reference the app uses + tagRef string // the short tag reference that `ps` and status read + root ocispec.Descriptor // what ref points at: the platform manifest or an index + manifest ocispec.Descriptor // the platform manifest selected when the app tree was loaded + config ocispec.Descriptor + layers []ocispec.Descriptor + chainIDs []digest.Digest // chain ID of the snapshot of each layer + } +) + +// LockInstall takes the lock that serializes every operation that writes to or deletes from +// dockerd's containerd: install, uninstall, rm and prune. It blocks until a concurrent composectl +// releases it. +func LockInstall(storeRoot string) (unlock func(), err error) { + f, err := os.OpenFile(filepath.Join(storeRoot, installLockFileName), os.O_RDWR|os.O_CREATE, 0o644) + if err != nil { + return nil, fmt.Errorf("failed to open the install lock: %w", err) + } + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil { + _ = f.Close() + return nil, fmt.Errorf("failed to take the install lock: %w", err) + } + return func() { + _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + _ = f.Close() + }, nil +} + +// PruneInstallLeases deletes the leases that interrupted installs left in dockerd's containerd, +// and with them the blobs and snapshots that only they hold. Call it under the install lock. +// It does nothing on a daemon whose images are loaded with `docker load`. +func PruneInstallLeases(ctx context.Context, cfg *Config) error { + c, err := ctrdIfInstallTarget(ctx, cfg) + if err != nil || c == nil { + return err + } + defer c.Close() + if err := deleteInstallLeases(ctx, c.LeasesService()); err != nil { + return fmt.Errorf("failed to prune install leases: %w", err) + } + return nil +} + +// connectCtrdIfInstallTarget returns a client to dockerd's containerd when app images are installed +// into it directly, or nil when they are loaded with `docker load`: a graphdriver daemon, a daemon +// older than MinDockerVersionForCtrdInstall or one that is not on a local unix socket. +func connectCtrdIfInstallTarget(ctx context.Context, cli *dockerclient.Client) (*ctrdClient, error) { + info, err := GetDockerInfo(ctx, cli) + if err != nil { + return nil, fmt.Errorf("failed to discover the docker image store: %w", err) + } + if !isCtrdInstallTarget(info, cli.DaemonHost()) { + return nil, nil + } + return newCtrdClient(ctx, info) +} + +// ctrdIfInstallTarget is connectCtrdIfInstallTarget for the operations that worked without dockerd +// before (the storage check, prune): a dockerd that cannot be reached is treated like a `docker load` one. +func ctrdIfInstallTarget(ctx context.Context, cfg *Config) (*ctrdClient, error) { + cli, err := GetDockerClient(cfg.DockerHost) + if err != nil { + return nil, nil + } + defer cli.Close() + info, err := GetDockerInfo(ctx, cli) + if err != nil || !isCtrdInstallTarget(info, cli.DaemonHost()) { + return nil, nil + } + c, err := newCtrdClient(ctx, info) + if err != nil { + return nil, fmt.Errorf("failed to connect to dockerd's containerd: %w", err) + } + return c, nil +} + +func isCtrdInstallTarget(info *DockerInfo, dockerHost string) bool { + return info.Snapshotter() != "" && + info.Containerd != nil && len(info.Containerd.Address) > 0 && + isLocalDockerHost(dockerHost) && + versionAtLeast(info.ServerVersion, MinDockerVersionForCtrdInstall) +} + +func newCtrdClient(ctx context.Context, info *DockerInfo) (*ctrdClient, error) { + c, err := containerd.New(info.Containerd.Address, + containerd.WithDefaultNamespace(info.Containerd.Namespaces.Containers)) + if err != nil { + return nil, fmt.Errorf("dockerd %s uses the containerd image store, but composectl cannot connect"+ + " to its containerd at %s (the socket is usually writable by root only): %w", + info.ServerVersion, info.Containerd.Address, err) + } + cc := &ctrdClient{Client: c, snapshotter: info.Snapshotter()} + if cc.contentRoot, err = pluginRoot(ctx, c, contentPluginType, "content"); err == nil { + cc.snapshotterRoot, err = pluginRoot(ctx, c, snapshotterPluginType, cc.snapshotter) + } + if err != nil { + _ = c.Close() + return nil, err + } + return cc, nil +} + +func pluginRoot(ctx context.Context, c *containerd.Client, pluginType, id string) (string, error) { + resp, err := c.IntrospectionService().Plugins(ctx, []string{"type==" + pluginType + ",id==" + id}) + if err != nil { + return "", fmt.Errorf("failed to introspect containerd plugin %s.%s: %w", pluginType, id, err) + } + if len(resp.Plugins) != 1 || resp.Plugins[0].Exports["root"] == "" { + return "", fmt.Errorf("containerd plugin %s.%s does not export its root", pluginType, id) + } + return resp.Plugins[0].Exports["root"], nil +} + +// installImagesCtrd installs the images of one app into dockerd's containerd. Under a lease it streams +// the layers that have no snapshot yet, unpacks every image and syncs the snapshotter's filesystem, +// and only then creates the image records: a record never points at a snapshot that a power cut +// can still lose. Deleting the lease lets containerd's GC drop the compressed layers. +func installImagesCtrd(ctx context.Context, c *ctrdClient, app App, cfg *Config, report ProgressCallbackFn) error { + blobsRoot := cfg.GetBlobsRoot() + imgs, err := ctrdImagesOf(ctx, app, NewStoreBlobProvider(blobsRoot)) + if err != nil { + return fmt.Errorf("failed to read the app images from the store: %w", err) + } + ls := c.LeasesService() + if err := deleteInstallLeases(ctx, ls); err != nil { + return fmt.Errorf("failed to remove leases of interrupted installs: %w", err) + } + lease, err := ls.Create(ctx, leases.WithID(installLeaseID(app.Name(), app.Ref().Digest))) + if err != nil { + return fmt.Errorf("failed to create the install lease: %w", err) + } + lctx := leases.WithLease(ctx, lease.ID) + platform := platforms.OnlyStrict(cfg.Platform) + for _, img := range imgs { + if err := c.installImage(lctx, img, blobsRoot, platform, report); err != nil { + return fmt.Errorf("failed to install image %s: %w", img.ref, err) + } + } + if err := syncFs(c.snapshotterRoot); err != nil { + return fmt.Errorf("failed to sync the snapshots of app %s: %w", app.Name(), err) + } + for _, img := range imgs { + if err := putImageRecords(ctx, c.ImageService(), img); err != nil { + return fmt.Errorf("failed to record image %s: %w", img.ref, err) + } + reportImageState(report, ImageLoadStateImageLoaded, img.ref) + } + if err := ls.Delete(ctx, lease, leases.SynchronousDelete); err != nil { + return fmt.Errorf("failed to delete the install lease: %w", err) + } + return nil +} + +func installLeaseID(appName string, appDigest digest.Digest) string { + return installLeasePrefix + appName + "@" + appDigest.String() +} + +func installLeases(all []leases.Lease) []leases.Lease { + var found []leases.Lease + for _, l := range all { + if strings.HasPrefix(l.ID, installLeasePrefix) { + found = append(found, l) + } + } + return found +} + +// deleteInstallLeases removes the leases that interrupted installs left behind. Under the install +// lock no other install is running, so every install lease is such a leftover. +func deleteInstallLeases(ctx context.Context, ls leases.Manager) error { + all, err := ls.List(ctx) + if err != nil { + return fmt.Errorf("failed to list containerd leases: %w", err) + } + for _, l := range installLeases(all) { + if err := ls.Delete(ctx, l, leases.SynchronousDelete); err != nil && !errdefs.IsNotFound(err) { + return fmt.Errorf("failed to delete the install lease %s: %w", l.ID, err) + } + } + return nil +} + +// installImage streams what image needs and unpacks it. If a base snapshot went away between the +// scan and the unpack (for example `docker image prune` removed the image that held it), the unpack +// fails on the missing layer blob; then every layer is streamed and the unpack is run once more. +func (c *ctrdClient) installImage(ctx context.Context, img *ctrdImage, blobsRoot string, + platform platforms.MatchComparer, report ProgressCallbackFn) error { + first, err := c.firstMissingSnapshot(ctx, img.chainIDs) + if err != nil { + return fmt.Errorf("failed to scan the snapshots: %w", err) + } + reportImageState(report, ImageLoadStateImageLoading, img.ref) + if err := c.streamImage(ctx, img, blobsRoot, first); err != nil { + return fmt.Errorf("failed to stream the image to containerd: %w", err) + } + err = c.unpack(ctx, img, platform) + if err == nil { + return nil + } + if !errdefs.IsNotFound(err) { + return fmt.Errorf("failed to unpack: %w", err) + } + if err := c.streamLayers(ctx, img.layers, blobsRoot); err != nil { + return fmt.Errorf("failed to stream every layer to containerd after a snapshot went away: %w", err) + } + if err := c.unpack(ctx, img, platform); err != nil { + return fmt.Errorf("failed to unpack after streaming every layer: %w", err) + } + return nil +} + +// firstMissingSnapshot returns the index of the first layer whose snapshot does not exist; every +// layer after it has to be streamed and unpacked too, because a snapshot needs its parent. +func (c *ctrdClient) firstMissingSnapshot(ctx context.Context, chainIDs []digest.Digest) (int, error) { + sn := c.SnapshotService(c.snapshotter) + for i, chainID := range chainIDs { + _, err := sn.Stat(ctx, chainID.String()) + if errdefs.IsNotFound(err) { + return i, nil + } + if err != nil { + return 0, fmt.Errorf("failed to stat snapshot %s: %w", chainID, err) + } + } + return len(chainIDs), nil +} + +// streamImage writes the layers from first on, the config, the manifest and, when the image is +// pinned by one, the index. The manifest gets no layer labels: only the lease holds the layers. +// WriteBlob leaves the labels of a blob that already exists as they are. +func (c *ctrdClient) streamImage(ctx context.Context, img *ctrdImage, blobsRoot string, first int) error { + if err := c.streamLayers(ctx, img.layers[first:], blobsRoot); err != nil { + return err + } + if err := c.writeBlob(ctx, blobsRoot, img.config, nil); err != nil { + return err + } + manifestLabels := map[string]string{gcRefConfigLabel: img.config.Digest.String()} + if err := c.writeBlob(ctx, blobsRoot, img.manifest, manifestLabels); err != nil { + return err + } + if img.root.Digest == img.manifest.Digest { + return nil + } + err := c.writeBlob(ctx, blobsRoot, img.root, map[string]string{gcRefManifestLabel: img.manifest.Digest.String()}) + if errors.Is(err, os.ErrNotExist) { + // The app was pulled by skopeo: the store has the image's platform manifest but not the index + // the app pins it by. The image records point at the manifest then, as `docker load` does. + img.root = img.manifest + return nil + } + return err +} + +func (c *ctrdClient) streamLayers(ctx context.Context, layers []ocispec.Descriptor, blobsRoot string) error { + for _, layer := range layers { + if err := c.writeBlob(ctx, blobsRoot, layer, nil); err != nil { + return err + } + } + return nil +} + +func (c *ctrdClient) writeBlob(ctx context.Context, blobsRoot string, desc ocispec.Descriptor, labels map[string]string) error { + f, err := os.Open(filepath.Join(blobsRoot, desc.Digest.Encoded())) + if err != nil { + return fmt.Errorf("failed to open blob: %w", err) + } + defer f.Close() + var opts []content.Opt + if labels != nil { + opts = append(opts, content.WithLabels(labels)) + } + if err := content.WriteBlob(ctx, c.ContentStore(), "composectl-"+desc.Digest.String(), f, desc, opts...); err != nil { + return fmt.Errorf("failed to write blob %s to containerd: %w", desc.Digest, err) + } + return nil +} + +// unpack creates the snapshots of the image's layers. The target is the platform manifest the app +// tree selected, not the index: on arm64 the default platform matcher would also accept arm/v7. +func (c *ctrdClient) unpack(ctx context.Context, img *ctrdImage, platform platforms.MatchComparer) error { + ci := containerd.NewImageWithPlatform(c.Client, images.Image{Name: img.ref, Target: img.manifest}, platform) + return ci.Unpack(ctx, c.snapshotter) +} + +// putImageRecords creates the pinned digest reference and the short tag reference, or points the +// existing records at the image, so that a retried install is idempotent. If the second record +// fails, the records this call created are deleted again (best effort), so that an image is +// either fully recorded and reported as loaded, or absent and still held only by the lease. +func putImageRecords(ctx context.Context, is images.Store, img *ctrdImage) error { + var created []string + for _, name := range []string{img.ref, img.tagRef} { + rec := images.Image{Name: name, Target: img.root} + _, err := is.Create(ctx, rec) + if err == nil { + created = append(created, name) + continue + } + if errdefs.IsAlreadyExists(err) { + _, err = is.Update(ctx, rec, "target") + } + if err != nil { + deleteImageRecords(ctx, is, created) + return fmt.Errorf("failed to create image record %s: %w", name, err) + } + } + return nil +} + +// deleteImageRecords removes the records a failed putImageRecords created. The failure may be the +// cancellation of ctx itself, so the deletes run on a context that is not cancelled, with a bound. +func deleteImageRecords(ctx context.Context, is images.Store, names []string) { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), recordCleanupTimeout) + defer cancel() + for _, name := range names { + _ = is.Delete(cleanupCtx, name) + } +} + +// syncFs flushes the filesystem that holds path. A power cut after the image records are created +// must not lose the snapshots they point at. +func syncFs(path string) error { + fd, err := unix.Open(path, unix.O_RDONLY|unix.O_DIRECTORY, 0) + if err != nil { + return fmt.Errorf("failed to open %s: %w", path, err) + } + defer unix.Close(fd) + if err := unix.Syncfs(fd); err != nil { + return fmt.Errorf("failed to sync the filesystem of %s: %w", path, err) + } + return nil +} + +func reportImageState(report ProgressCallbackFn, state ImageLoadState, imageID string) { + if report != nil { + report(&LoadImageProgress{State: state, ImageID: imageID}) + } +} + +// ctrdImagesOf describes each image of the app as the containerd install and the storage precheck +// need it. The app tree already holds the platform manifest of each image. +func ctrdImagesOf(ctx context.Context, app App, provider BlobProvider) ([]*ctrdImage, error) { + var imgs []*ctrdImage + for _, root := range app.GetComposeRoot().Children { + img, err := ctrdImageOf(ctx, root, provider) + if err != nil { + return nil, fmt.Errorf("failed to read image %s: %w", root.Ref(), err) + } + imgs = append(imgs, img) + } + return imgs, nil +} + +func ctrdImageOf(ctx context.Context, root *TreeNode, provider BlobProvider) (*ctrdImage, error) { + ref, err := ParseImageRef(root.Ref()) + if err != nil { + return nil, fmt.Errorf("failed to parse the image reference: %w", err) + } + mNode, err := platformManifestNode(root) + if err != nil { + return nil, fmt.Errorf("failed to find the platform manifest: %w", err) + } + manifest, err := ReadImageManifest(ctx, provider, mNode.Ref(), mNode.Descriptor.Size) + if err != nil { + return nil, fmt.Errorf("failed to read image manifest: %w", err) + } + cfgNode := childOfType(mNode, BlobTypeImageConfig) + if cfgNode == nil { + return nil, fmt.Errorf("image config is not found") + } + b, err := ReadBlob(ctx, provider, cfgNode.Ref(), cfgNode.Descriptor.Digest, cfgNode.Descriptor.Size) + if err != nil { + return nil, fmt.Errorf("failed to read image config: %w", err) + } + var imageConfig ocispec.Image + if err := json.Unmarshal(b, &imageConfig); err != nil { + return nil, fmt.Errorf("failed to unmarshal image config: %w", err) + } + if len(imageConfig.RootFS.DiffIDs) != len(manifest.Layers) { + return nil, fmt.Errorf("image config has %d diff IDs for %d layers", + len(imageConfig.RootFS.DiffIDs), len(manifest.Layers)) + } + return &ctrdImage{ + ref: root.Ref(), + tagRef: ref.GetTagRef(), + root: plainDescriptor(root.Descriptor), + manifest: plainDescriptor(mNode.Descriptor), + config: plainDescriptor(&manifest.Config), + layers: manifest.Layers, + chainIDs: identity.ChainIDs(imageConfig.RootFS.DiffIDs), + }, nil +} + +// platformManifestNode walks from the image root down to its platform manifest. +func platformManifestNode(root *TreeNode) (*TreeNode, error) { + node := root + for node.Type != BlobTypeImageManifest { + if node.Type != BlobTypeImageIndex && node.Type != BlobTypeSkopeoImageIndex { + return nil, fmt.Errorf("invalid image type: %s", node.Type) + } + if len(node.Children) != 1 { + return nil, fmt.Errorf("the image index has %d manifests for the platform, expected one", len(node.Children)) + } + node = node.Children[0] + } + return node, nil +} + +func childOfType(node *TreeNode, blobType BlobType) *TreeNode { + for _, child := range node.Children { + if child.Type == blobType { + return child + } + } + return nil +} + +// plainDescriptor drops what the app tree adds to a descriptor (composectl's references in URLs, +// the index's platform), keeping what containerd records and content need. +func plainDescriptor(desc *ocispec.Descriptor) ocispec.Descriptor { + return ocispec.Descriptor{MediaType: desc.MediaType, Digest: desc.Digest, Size: desc.Size} +} diff --git a/pkg/compose/ctrd_install_test.go b/pkg/compose/ctrd_install_test.go new file mode 100644 index 0000000..4448741 --- /dev/null +++ b/pkg/compose/ctrd_install_test.go @@ -0,0 +1,123 @@ +package compose + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/leases" + "github.com/opencontainers/go-digest" +) + +func TestInstallLeaseID(t *testing.T) { + d := digest.FromString("app") + id := installLeaseID("myapp", d) + if id != "composectl/install/myapp@"+d.String() { + t.Fatalf("unexpected lease ID: %s", id) + } + if !strings.HasPrefix(id, installLeasePrefix) { + t.Fatalf("lease ID %s does not start with %s", id, installLeasePrefix) + } +} + +func TestInstallLeasesFilter(t *testing.T) { + all := []leases.Lease{ + {ID: "composectl/install/app1@sha256:aaaa"}, + {ID: "moby-pull-123"}, + {ID: "composectl/install/app2@sha256:bbbb"}, + {ID: "composectl/other"}, + } + got := installLeases(all) + if len(got) != 2 || got[0].ID != all[0].ID || got[1].ID != all[2].ID { + t.Fatalf("unexpected install leases: %+v", got) + } + if got := installLeases(nil); len(got) != 0 { + t.Fatalf("expected no leases, got %+v", got) + } +} + +func TestIsCtrdInstallTarget(t *testing.T) { + ctrdStore := func(version string) *DockerInfo { + return &DockerInfo{ + ServerVersion: version, + Driver: "overlayfs", + DriverStatus: [][2]string{{"driver-type", snapshotterPluginType}}, + Containerd: &ContainerdInfo{Address: "/run/docker/containerd/containerd.sock"}, + } + } + const local = "unix:///var/run/docker.sock" + if !isCtrdInstallTarget(ctrdStore("29.5.2"), local) { + t.Fatal("the containerd image store on the minimum version over a unix socket must qualify") + } + if isCtrdInstallTarget(ctrdStore("29.5.1"), local) { + t.Fatal("a dockerd older than the minimum must not qualify") + } + if isCtrdInstallTarget(ctrdStore("29.7.2"), "tcp://10.0.0.1:2375") { + t.Fatal("a remote dockerd must not qualify") + } + graphdriver := ctrdStore("29.7.2") + graphdriver.Driver = "overlay2" + graphdriver.DriverStatus = [][2]string{{"Backing Filesystem", "extfs"}} + if isCtrdInstallTarget(graphdriver, local) { + t.Fatal("a graphdriver dockerd must not qualify") + } + noAddress := ctrdStore("29.7.2") + noAddress.Containerd = nil + if isCtrdInstallTarget(noAddress, local) { + t.Fatal("a dockerd that reports no containerd address must not qualify") + } +} + +// fakeImageStore is an images.Store whose Create cancels the install context after the first +// record, as a cancellation between the two record writes does, and records the deletes. +type fakeImageStore struct { + cancel context.CancelFunc + created []string + deleted []string + // deletedWithLiveCtx is false when a delete came with a context that was already done. + deletedWithLiveCtx bool +} + +func (s *fakeImageStore) Get(context.Context, string) (images.Image, error) { + return images.Image{}, errdefs.ErrNotFound +} +func (s *fakeImageStore) List(context.Context, ...string) ([]images.Image, error) { return nil, nil } +func (s *fakeImageStore) Create(ctx context.Context, img images.Image) (images.Image, error) { + if err := ctx.Err(); err != nil { + return images.Image{}, err + } + s.created = append(s.created, img.Name) + s.cancel() + return img, nil +} +func (s *fakeImageStore) Update(ctx context.Context, img images.Image, _ ...string) (images.Image, error) { + return images.Image{}, ctx.Err() +} +func (s *fakeImageStore) Delete(ctx context.Context, name string, _ ...images.DeleteOpt) error { + s.deleted = append(s.deleted, name) + s.deletedWithLiveCtx = ctx.Err() == nil + return nil +} + +func TestPutImageRecordsRollsBackOnCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + store := &fakeImageStore{cancel: cancel} + img := &ctrdImage{ref: "registry/app@sha256:aaaa", tagRef: "registry/app:aaaa"} + err := putImageRecords(ctx, store, img) + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected the cancellation to be returned, got %v", err) + } + if len(store.created) != 1 || store.created[0] != img.ref { + t.Fatalf("expected the digest record only to be created, got %v", store.created) + } + if len(store.deleted) != 1 || store.deleted[0] != img.ref { + t.Fatalf("expected the digest record to be deleted again, got %v", store.deleted) + } + if !store.deletedWithLiveCtx { + t.Fatal("the rollback ran on the cancelled context") + } +} diff --git a/pkg/compose/docker_client_test.go b/pkg/compose/docker_client_test.go index 378f734..fe945f4 100644 --- a/pkg/compose/docker_client_test.go +++ b/pkg/compose/docker_client_test.go @@ -130,6 +130,9 @@ func TestGetDockerInfoOverTCP(t *testing.T) { if info.Containerd != nil { t.Fatalf("the typed info must not carry a containerd address: %+v", info.Containerd) } + if isCtrdInstallTarget(info, cli.DaemonHost()) { + t.Fatal("a tcp host must not be an install target") + } } func TestIsLocalDockerHost(t *testing.T) { diff --git a/pkg/compose/image_loader.go b/pkg/compose/image_loader.go index c27738e..dcb97d9 100644 --- a/pkg/compose/image_loader.go +++ b/pkg/compose/image_loader.go @@ -348,25 +348,15 @@ func getImageID(imageURIs []imageURI2RefCounter, imageIndex int, printWarning bo } func adjustOptionsIfContainerd(ctx context.Context, client *client.Client, options *LoadImageOptions) bool { - var isCtrd = false - // Check if the docker daemon is using ctrd storage, which requires special handling when loading images. - if dockerInfo, err := client.Info(ctx); err == nil { - if dockerInfo.Driver == "overlayfs" && - len(dockerInfo.DriverStatus) > 0 && - dockerInfo.DriverStatus[0][1] == "io.containerd.snapshotter.v1" { - isCtrd = true - } - } - - if isCtrd { - if options.ReadBlobsFromStore { - // The ctrd storage does not support loading image layers from an external directory, - // thus, the image load will fail if `options.ReadBlobsFromStore` is set. - options.ReadBlobsFromStore = false - } - } - - return isCtrd + // Check if the docker daemon uses the containerd image store, which requires special handling when loading images. + info, err := GetDockerInfo(ctx, client) + if err != nil || info.Snapshotter() == "" { + return false + } + // The containerd image store does not support loading image layers from an external directory, + // thus, the image load will fail if `options.ReadBlobsFromStore` is set. + options.ReadBlobsFromStore = false + return true } func reportProgressIfContainerd(body io.ReadCloser, imageURIs []imageURI2RefCounter, reporter progress.Reporter[LoadImageProgress]) error { diff --git a/pkg/compose/install.go b/pkg/compose/install.go index 2959bfb..f8a6073 100644 --- a/pkg/compose/install.go +++ b/pkg/compose/install.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/containerd/containerd/platforms" + "github.com/docker/docker/client" "github.com/docker/docker/pkg/archive" "github.com/foundriesio/composeapp/internal/progress" "os" @@ -60,85 +61,119 @@ func Install(ctx context.Context, o(&opts) } + unlock, err := LockInstall(cfg.StoreRoot) + if err != nil { + return err + } + defer unlock() + cs, err := cfg.AppStoreFactory() if err != nil { return fmt.Errorf("failed to create app store instance: %w", err) } app, err := cfg.AppLoader.LoadAppTree(ctx, cs, platforms.OnlyStrict(cfg.Platform), appURI) if err != nil { - return fmt.Errorf("failed to load app %s: %w", app, err) + return fmt.Errorf("failed to load app %s: %w", appURI, err) } - if opts.ProgressReporter != nil { - // TODO: Implement progress reporting for app compose installation - opts.ProgressReporter.Update(InstallProgress{ - AppInstallState: AppInstallStateComposeInstalling, - AppID: app.Ref().String(), - }) + // TODO: Implement progress reporting for app compose installation + reportInstallState(&opts, app, AppInstallStateComposeInstalling) + + if err := installImages(ctx, cfg, app, &opts); err != nil { + return fmt.Errorf("failed to load images for app %s: %w", app.Ref().String(), err) + } + + if err := InstallCompose(ctx, app, cs, cfg.ComposeRoot); err != nil { + return err + } + // TODO: Implement progress reporting for app compose installation checking + reportInstallState(&opts, app, AppInstallStateComposeChecking) + return checkComposeInstallation(ctx, app, cs, cfg) +} + +func reportInstallState(opts *InstallOptions, app App, state AppInstallState) { + if opts.ProgressReporter == nil { + return + } + opts.ProgressReporter.Update(InstallProgress{ + AppInstallState: state, + AppID: app.Ref().String(), + }) +} + +// imageProgressCallback forwards the image load progress to the install progress reporter and +// records the loaded images; it is nil when neither is wanted. +func imageProgressCallback(opts *InstallOptions, app App) ProgressCallbackFn { + if opts.ProgressReporter == nil && opts.LoadedImages == nil { + return nil + } + return func(ilp *LoadImageProgress) { + if opts.LoadedImages != nil && ilp.State == ImageLoadStateImageLoaded { + opts.LoadedImages[ilp.ImageID] = struct{}{} + } + if opts.ProgressReporter != nil { + opts.ProgressReporter.Update(InstallProgress{ + AppInstallState: AppInstallStateImagesLoading, + AppID: app.Ref().String(), + ImageLoadState: ilp.State, + ImageID: ilp.ImageID, + ID: ilp.ID, + Current: ilp.Current, + Total: ilp.Total, + }) + } } +} +// installImages puts the app images into dockerd: straight into its containerd on a daemon that +// uses the containerd image store, with `docker load` otherwise. +func installImages(ctx context.Context, cfg *Config, app App, opts *InstallOptions) error { cli, err := GetDockerClient(cfg.DockerHost) if err != nil { return err } - - var loadImageOptions []LoadImageOption - var withProgressOpt LoadImageOption - if opts.ProgressReporter != nil || opts.LoadedImages != nil { - withProgressOpt = WithProgressReporting(func(ilp *LoadImageProgress) { - if opts.LoadedImages != nil { - if ilp.State == ImageLoadStateImageLoaded { - opts.LoadedImages[ilp.ImageID] = struct{}{} - } - } - if opts.ProgressReporter != nil { - opts.ProgressReporter.Update(InstallProgress{ - AppInstallState: AppInstallStateImagesLoading, - AppID: app.Ref().String(), - ImageLoadState: ilp.State, - ImageID: ilp.ImageID, - ID: ilp.ID, - Current: ilp.Current, - Total: ilp.Total, - }) - } - }) - loadImageOptions = append(loadImageOptions, withProgressOpt) - } - - loadImageOptionsRequiringPatch := append(loadImageOptions, WithRefWithDigest(), WithBlobReadingFromStore()) - // Try to load app images with reading blobs directly from the store and specifying image digests (URI with hashes) - err = LoadImages(ctx, cli, app, cfg.GetBlobsRoot(), loadImageOptionsRequiringPatch...) + defer cli.Close() + report := imageProgressCallback(opts, app) + ctrd, err := connectCtrdIfInstallTarget(ctx, cli) if err != nil { - // Retry loading images without reading blobs directly from the store and specifying the digest, - // in case if the docker daemon is not patch with the Foundries patches - err = LoadImages(ctx, cli, app, cfg.GetBlobsRoot(), loadImageOptions...) + return err + } + if ctrd != nil { + defer ctrd.Close() + return installImagesCtrd(ctx, ctrd, app, cfg, report) + } + return loadImagesWithDocker(ctx, cli, app, cfg.GetBlobsRoot(), report) +} + +// loadImagesWithDocker tries first to load the images with reading blobs directly from the store +// and with digest references, which needs the Foundries patches to dockerd, then the plain way. +func loadImagesWithDocker(ctx context.Context, cli *client.Client, app App, blobsRoot string, report ProgressCallbackFn) error { + var loadOpts []LoadImageOption + if report != nil { + loadOpts = append(loadOpts, WithProgressReporting(report)) } + patchedOpts := append([]LoadImageOption{WithRefWithDigest(), WithBlobReadingFromStore()}, loadOpts...) + err := LoadImages(ctx, cli, app, blobsRoot, patchedOpts...) if err != nil { - return fmt.Errorf("failed to load images for app %s: %w", app.Ref().String(), err) + err = LoadImages(ctx, cli, app, blobsRoot, loadOpts...) } + return err +} - if err := InstallCompose(ctx, app, cs, cfg.ComposeRoot); err != nil { +func checkComposeInstallation(ctx context.Context, app App, cs AppStore, cfg *Config) error { + checkErrMap, err := app.CheckComposeInstallation(ctx, cs, cfg.GetAppComposeDir(app.Name())) + if err != nil { return err } - if opts.ProgressReporter != nil { - // TODO: Implement progress reporting for app compose installation checking - opts.ProgressReporter.Update(InstallProgress{ - AppInstallState: AppInstallStateComposeChecking, - AppID: app.Ref().String(), - }) + if len(checkErrMap) == 0 { + return nil } - if checkErrMap, err := app.CheckComposeInstallation(ctx, cs, cfg.GetAppComposeDir(app.Name())); err != nil { - return err - } else if len(checkErrMap) > 0 { - // TODO: remove prints and return error map - fmt.Println("the following app bundle files are not correctly installed") - for filePath, fileErr := range checkErrMap { - fmt.Printf("\t%s\t%s\n", filePath, fileErr) - } - return fmt.Errorf("app bundle is not installed completely") + // TODO: remove prints and return error map + fmt.Println("the following app bundle files are not correctly installed") + for filePath, fileErr := range checkErrMap { + fmt.Printf("\t%s\t%s\n", filePath, fileErr) } - return err + return fmt.Errorf("app bundle is not installed completely") } func InstallCompose(ctx context.Context, app App, provider BlobProvider, composeRoot string) error { From 5cc3aa645209ddbf8dd8c2d56514c3c6dfa6b370 Mon Sep 17 00:00:00 2001 From: Mike Sul Date: Mon, 12 Oct 2026 14:17:11 +0200 Subject: [PATCH 4/5] feat(cli): take the install lock in rm, uninstall and prune Every operation that writes to or deletes from dockerd's containerd has to be serialized with an install, otherwise an uninstall or a prune can delete what a concurrent install is counting on. rm, uninstall and prune now take the same flock as install, and so do `update cancel` around its image and blob removal and `update complete` around its final store prune (its uninstall and remove calls lock on their own). prune also deletes the leases that an interrupted install left behind, which hold compressed layers and snapshots that nothing else references. Signed-off-by: Mike Sul Assisted-by: Claude Code:claude-fable-5-1 (cherry picked from commit 1c93c48d034a993eadd3e45ffd0872b03a07915d) --- cmd/composectl/cmd/prune.go | 6 ++++++ pkg/compose/remove.go | 5 +++++ pkg/compose/uninstall.go | 6 +++++- pkg/update/cancel.go | 6 ++++++ pkg/update/complete.go | 8 +++++++- 5 files changed, 29 insertions(+), 2 deletions(-) diff --git a/cmd/composectl/cmd/prune.go b/cmd/composectl/cmd/prune.go index 766adf3..8fbc597 100644 --- a/cmd/composectl/cmd/prune.go +++ b/cmd/composectl/cmd/prune.go @@ -3,6 +3,7 @@ package composectl import ( "encoding/json" "fmt" + "github.com/foundriesio/composeapp/pkg/compose" v1 "github.com/foundriesio/composeapp/pkg/compose/v1" "github.com/spf13/cobra" ) @@ -34,8 +35,13 @@ func init() { func pruneApps(cmd *cobra.Command, opts *pruneOptions) { cs, err := v1.NewAppStore(config.StoreRoot, config.Platform) DieNotNil(err) + unlock, err := compose.LockInstall(config.StoreRoot) + DieNotNil(err) + defer unlock() prunedBlobs, err := cs.Prune(cmd.Context()) DieNotNil(err) + // Leases left by interrupted installs hold blobs and snapshots in dockerd's containerd. + DieNotNil(compose.PruneInstallLeases(cmd.Context(), config), "failed to prune install leases") if opts.Format == "json" { if b, err := json.MarshalIndent(prunedBlobs, "", " "); err == nil { fmt.Println(string(b)) diff --git a/pkg/compose/remove.go b/pkg/compose/remove.go index 4ecf500..3ab8721 100644 --- a/pkg/compose/remove.go +++ b/pkg/compose/remove.go @@ -32,6 +32,11 @@ func RemoveApps(ctx context.Context, cfg *Config, appRefs []string, options ...R for _, o := range options { o(opts) } + unlock, err := LockInstall(cfg.StoreRoot) + if err != nil { + return err + } + defer unlock() if opts.CheckStatus { appsStatus, err := CheckAppsStatus(ctx, cfg, appRefs) if err != nil { diff --git a/pkg/compose/uninstall.go b/pkg/compose/uninstall.go index d550a06..b020fa3 100644 --- a/pkg/compose/uninstall.go +++ b/pkg/compose/uninstall.go @@ -48,8 +48,12 @@ func UninstallApps(ctx context.Context, cfg *Config, appRefs []string, options . if len(appRefs) == 0 && (!opts.Prune || opts.PruneType != PruneTypeAllUnusedImages) { return nil } + unlock, err := LockInstall(cfg.StoreRoot) + if err != nil { + return err + } + defer unlock() - var err error var status *AppsStatus if len(appRefs) > 0 { status, err = CheckAppsStatus(ctx, cfg, appRefs) diff --git a/pkg/update/cancel.go b/pkg/update/cancel.go index 03b8038..b9f7f24 100644 --- a/pkg/update/cancel.go +++ b/pkg/update/cancel.go @@ -12,6 +12,12 @@ import ( ) func (u *runnerImpl) cancel(ctx context.Context) (err error) { + // Removing the loaded images and the fetched blobs must not race with an install. + unlock, err := compose.LockInstall(u.config.StoreRoot) + if err != nil { + return fmt.Errorf("failed to lock the store for the cancel: %w", err) + } + defer unlock() if len(u.LoadedImages) > 0 { cli, err := compose.GetDockerClient(u.config.DockerHost) diff --git a/pkg/update/complete.go b/pkg/update/complete.go index 0c5c73c..4f71830 100644 --- a/pkg/update/complete.go +++ b/pkg/update/complete.go @@ -108,11 +108,17 @@ func (u *runnerImpl) complete(ctx context.Context, options ...CompleteOpt) error } } - // Prune blobs in the app store + // Prune blobs in the app store. UninstallApps and RemoveApps above take the install lock + // themselves, so it is taken only now, for the prune. store, err := u.config.AppStoreFactory() if err != nil { return err } + unlock, err := compose.LockInstall(u.config.StoreRoot) + if err != nil { + return fmt.Errorf("failed to lock the store for the prune: %w", err) + } + defer unlock() _, err = store.Prune(ctx) return err } From 954e58a0794b362944479aa9394ece6908703ddb Mon Sep 17 00:00:00 2001 From: Mike Sul Date: Wed, 21 Oct 2026 14:58:24 +0200 Subject: [PATCH 5/5] test(integration): cover the containerd install path The containerd install route only runs against the dockerd with the containerd image store, so the existing tests do not reach it. The new tests skip on a graphdriver dockerd and cover: an install from the store of an image pinned by a manifest and one pinned by an index, status, run, check --install and uninstall afterwards; an update that shares base layers with the installed version; a reinstall after `/images/prune` removed the images; an install killed after its lease and first snapshot exist, then recovered by the next install; a second composectl waiting for the install lock; and `update cancel` after the install. The retry after a base snapshot disappears between the scan and the unpack is tested in pkg/compose through the synchronous progress callback, so that the removal happens at a known point with no data race. That test shares the daemons and the store with the integration tests, so the e2e target runs the packages one at a time. Signed-off-by: Mike Sul Assisted-by: Claude Code:claude-fable-5-1 (cherry picked from commit b051ed42abbed539e0c0c840e10a6bda71e833a8) --- Makefile | 4 +- pkg/compose/ctrd_install_rig_test.go | 108 +++++++ pkg/compose/export_test.go | 33 ++ test/fixtures/composectl_cmds.go | 20 ++ test/integration/ctrd_install_test.go | 447 ++++++++++++++++++++++++++ 5 files changed, 611 insertions(+), 1 deletion(-) create mode 100644 pkg/compose/ctrd_install_rig_test.go create mode 100644 pkg/compose/export_test.go create mode 100644 test/integration/ctrd_install_test.go diff --git a/Makefile b/Makefile index f6f5baf..64072fd 100644 --- a/Makefile +++ b/Makefile @@ -78,8 +78,10 @@ tidy-mod: preload-images: test/fixtures/preload-images.sh +# -p 1: the test binaries share one dockerd, one publisher dockerd and one app store, +# so the packages must not run concurrently test-e2e: $(exe) preload-images - @$(GO) test -v ./... + @$(GO) test -v -p 1 ./... test-smoke: $(exe) preload-images @$(GO) test -v -run TestSmoke test/integration/smoke_test.go diff --git a/pkg/compose/ctrd_install_rig_test.go b/pkg/compose/ctrd_install_rig_test.go new file mode 100644 index 0000000..6559aba --- /dev/null +++ b/pkg/compose/ctrd_install_rig_test.go @@ -0,0 +1,108 @@ +package compose_test + +// Drives the containerd install directly, with its synchronous progress callback, against the +// test dockerd on the containerd image store. Skips when that dockerd or the composectl binary +// (built by `make`) is not there, as under `make test-unit`. + +import ( + "context" + "fmt" + "os" + "os/exec" + "testing" + "time" + + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/platforms" + "github.com/foundriesio/composeapp/pkg/compose" + f "github.com/foundriesio/composeapp/test/fixtures" +) + +const derivedAppDef = ` +services: + derived: + image: %s + command: sh -c "while true; do sleep 60; done" +` + +func ctrdRig(t *testing.T) *compose.CtrdClient { + t.Helper() + if _, err := os.Stat(os.Getenv("COMPOSECTL_EXE")); err != nil { + t.Skip("composectl binary is not built; run `make` first") + } + cli, err := compose.GetDockerClient("") + f.Check(t, err) + defer cli.Close() + c, err := compose.ConnectCtrdIfInstallTarget(context.Background(), cli) + f.Check(t, err) + if c == nil { + t.Skip("dockerd does not use the containerd image store") + } + t.Cleanup(func() { c.Close() }) + return c +} + +// TestCtrdInstallRetriesAfterBaseSnapshotRemoval: the base snapshot found by the scan is removed +// before the unpack, as `docker image prune` could do, and the install streams every layer and +// unpacks again. The removal runs in the install's progress callback, which installImagesCtrd +// calls synchronously after the scan and before the layers are streamed. +func TestCtrdInstallRetriesAfterBaseSnapshotRemoval(t *testing.T) { + c := ctrdRig(t) + ctx := context.Background() + cfg := f.NewTestConfig(t) + store := compose.NewStoreBlobProvider(cfg.GetBlobsRoot()) + const base = "ghcr.io/foundriesio/busybox:1.36" + + baseApp := f.NewApp(t, fmt.Sprintf(derivedAppDef, base)) + baseApp.Publish(t) + baseApp.Pull(t) + defer baseApp.Remove(t) + baseApp.Install(t) + defer baseApp.Uninstall(t) + baseTree, err := cfg.AppLoader.LoadAppTree(ctx, store, platforms.OnlyStrict(cfg.Platform), baseApp.PublishedUri) + f.Check(t, err) + baseChains, err := compose.ChainIDsOf(ctx, baseTree, store) + f.Check(t, err) + baseChain := baseChains[0] + baseRef, err := compose.ParseImageRef(baseTree.GetComposeRoot().Children[0].Ref()) + f.Check(t, err) + baseRefs := []string{baseTree.GetComposeRoot().Children[0].Ref(), baseRef.GetTagRef()} + + derivedApp := f.NewApp(t, fmt.Sprintf(derivedAppDef, f.PublishDerivedImage(t, base, 4))) + derivedApp.Publish(t) + derivedApp.Pull(t) + defer derivedApp.Remove(t) + derivedTree, err := cfg.AppLoader.LoadAppTree(ctx, store, platforms.OnlyStrict(cfg.Platform), derivedApp.PublishedUri) + f.Check(t, err) + + sn := c.SnapshotService(compose.CtrdSnapshotter(c)) + var removedAt time.Time + removeBase := func(p *compose.LoadImageProgress) { + if p.State != compose.ImageLoadStateImageLoading || !removedAt.IsZero() { + return + } + for _, ref := range baseRefs { + f.Check(t, c.ImageService().Delete(ctx, ref, images.SynchronousDelete())) + } + removedAt = time.Now() + if _, err := sn.Stat(ctx, baseChain[len(baseChain)-1]); !errdefs.IsNotFound(err) { + t.Fatalf("the base snapshot is still there after its images were deleted: %v", err) + } + } + f.Check(t, compose.InstallImagesCtrd(ctx, c, derivedTree, cfg, removeBase)) + defer derivedApp.Uninstall(t) + if removedAt.IsZero() { + t.Fatal("the progress callback did not run") + } + for _, chainID := range baseChain { + info, err := sn.Stat(ctx, chainID) + f.Check(t, err) + if !info.Created.After(removedAt) { + t.Errorf("base snapshot %s was not recreated by the retry: created %v, removed at %v", chainID, info.Created, removedAt) + } + } + derivedRef := derivedTree.GetComposeRoot().Children[0].Ref() + out, err := exec.Command("docker", "run", "--rm", "--pull", "never", "--network", "none", derivedRef, "echo", "ok").CombinedOutput() + f.Checkf(t, err, "docker run %s: %s", derivedRef, out) +} diff --git a/pkg/compose/export_test.go b/pkg/compose/export_test.go new file mode 100644 index 0000000..82039ba --- /dev/null +++ b/pkg/compose/export_test.go @@ -0,0 +1,33 @@ +package compose + +import "context" + +// Exported for the tests in package compose_test that drive the containerd install directly. + +type CtrdClient = ctrdClient + +var ( + ConnectCtrdIfInstallTarget = connectCtrdIfInstallTarget + InstallImagesCtrd = installImagesCtrd +) + +func CtrdSnapshotter(c *CtrdClient) string { + return c.snapshotter +} + +// ChainIDsOf returns the chain IDs of each app image, in the app's image order. +func ChainIDsOf(ctx context.Context, app App, provider BlobProvider) ([][]string, error) { + imgs, err := ctrdImagesOf(ctx, app, provider) + if err != nil { + return nil, err + } + var out [][]string + for _, img := range imgs { + var ids []string + for _, id := range img.chainIDs { + ids = append(ids, id.String()) + } + out = append(out, ids) + } + return out, nil +} diff --git a/test/fixtures/composectl_cmds.go b/test/fixtures/composectl_cmds.go index 5385323..c2fa347 100644 --- a/test/fixtures/composectl_cmds.go +++ b/test/fixtures/composectl_cmds.go @@ -483,3 +483,23 @@ func randomString(length int) string { } return string(b) } + +// PublishDerivedImage builds an image on top of base with one layer of sizeMB random bytes and +// pushes it to the test registry through the publisher dockerd, which the publish step reads the +// image layers from. It returns the image reference. +func PublishDerivedImage(t *testing.T, base string, sizeMB int) string { + t.Helper() + name := "derived-" + randomString(8) + ref := "registry:5000/factory/" + name + ":v1" + pub := func(args ...string) { + out, err := publisherDockerCommand(args...).CombinedOutput() + Checkf(t, err, "docker %v: %s", args, out) + } + pub("pull", base) + pub("run", "--name", name, base, "sh", "-c", fmt.Sprintf("head -c %d /dev/urandom > /derived", sizeMB<<20)) + pub("commit", name, ref) + pub("rm", name) + pub("push", ref) + pub("image", "rm", ref) + return ref +} diff --git a/test/integration/ctrd_install_test.go b/test/integration/ctrd_install_test.go new file mode 100644 index 0000000..6f706b1 --- /dev/null +++ b/test/integration/ctrd_install_test.go @@ -0,0 +1,447 @@ +package e2e_tests + +// Tests of the install of app images into dockerd's containerd, see +// docs/containerd-install-plan-2026-10-08.md. They need the dockerd on the containerd image store +// (CONTAINERD_SNAPSHOTTER=true) and skip on the graphdriver one. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "github.com/containerd/containerd" + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/leases" + "github.com/containerd/containerd/platforms" + "github.com/containerd/containerd/snapshots" + "github.com/docker/docker/api/types/filters" + "github.com/foundriesio/composeapp/pkg/compose" + v1 "github.com/foundriesio/composeapp/pkg/compose/v1" + "github.com/foundriesio/composeapp/pkg/update" + f "github.com/foundriesio/composeapp/test/fixtures" + "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/identity" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +const ( + ctrdTestApp = ` +services: + base: + image: ghcr.io/foundriesio/busybox:1.36 + command: sh -c "while true; do sleep 60; done" + multi: + image: registry:5000/factory/runner-image:v0.1 + command: sh -c "while true; do sleep 60; done" +` + ctrdDerivedApp = ` +services: + derived: + image: %s + command: sh -c "while true; do sleep 60; done" +` + ctrdKilledApp = ` +services: + a-base: + image: %s + command: sh -c "while true; do sleep 60; done" + b-derived: + image: %s + command: sh -c "while true; do sleep 60; done" +` +) + +type ( + // ctrdRig is the containerd behind the test dockerd. + ctrdRig struct { + t *testing.T + c *containerd.Client + snapshotter string + contentRoot string + blobsRoot string + } + + ctrdTestImage struct { + refs []string // the pinned digest reference and the short tag reference + manifest ocispec.Manifest + chainIDs []digest.Digest + } +) + +// newCtrdRig connects to dockerd's containerd, or skips the test on a graphdriver dockerd. +func newCtrdRig(t *testing.T) *ctrdRig { + t.Helper() + cli, err := compose.GetDockerClient("") + f.Check(t, err) + defer cli.Close() + info, err := compose.GetDockerInfo(context.Background(), cli) + f.Check(t, err) + if info.Snapshotter() == "" { + t.Skip("dockerd uses a graphdriver, not the containerd image store") + } + c, err := containerd.New(info.Containerd.Address, containerd.WithDefaultNamespace(info.Containerd.Namespaces.Containers)) + f.Check(t, err) + t.Cleanup(func() { c.Close() }) + r := &ctrdRig{t: t, c: c, snapshotter: info.Snapshotter(), blobsRoot: compose.GetBlobsRootFor(f.AppStoreRoot)} + resp, err := c.IntrospectionService().Plugins(context.Background(), []string{"type==io.containerd.content.v1,id==content"}) + f.Check(t, err) + if len(resp.Plugins) != 1 { + t.Fatalf("content plugin not found: %+v", resp.Plugins) + } + r.contentRoot = resp.Plugins[0].Exports["root"] + return r +} + +func (r *ctrdRig) readJSON(d digest.Digest, v any) { + b, err := os.ReadFile(filepath.Join(r.blobsRoot, d.Encoded())) + f.Check(r.t, err) + f.Check(r.t, json.Unmarshal(b, v)) +} + +// images describes the images of the app from the composectl store. +func (r *ctrdRig) images(appURI string) []ctrdTestImage { + cfg := f.NewTestConfig(r.t) + app, err := v1.NewAppLoader().LoadAppTree(context.Background(), compose.NewStoreBlobProvider(r.blobsRoot), + platforms.OnlyStrict(cfg.Platform), appURI) + f.Check(r.t, err) + var out []ctrdTestImage + for _, root := range app.GetComposeRoot().Children { + ref, err := compose.ParseImageRef(root.Ref()) + f.Check(r.t, err) + img := ctrdTestImage{refs: []string{root.Ref(), ref.GetTagRef()}} + mNode := root + if images.IsIndexType(root.Descriptor.MediaType) { + mNode = root.Children[0] + } + r.readJSON(mNode.Descriptor.Digest, &img.manifest) + var imageConfig ocispec.Image + r.readJSON(img.manifest.Config.Digest, &imageConfig) + img.chainIDs = identity.ChainIDs(imageConfig.RootFS.DiffIDs) + out = append(out, img) + } + return out +} + +func (r *ctrdRig) blobInContentRoot(d digest.Digest) bool { + _, err := os.Stat(filepath.Join(r.contentRoot, "blobs", d.Algorithm().String(), d.Encoded())) + if err != nil && !errors.Is(err, os.ErrNotExist) { + f.Check(r.t, err) + } + return err == nil +} + +func (r *ctrdRig) snapshot(chainID digest.Digest) (snapshots.Info, bool) { + info, err := r.c.SnapshotService(r.snapshotter).Stat(context.Background(), chainID.String()) + if err != nil && !errdefs.IsNotFound(err) { + f.Check(r.t, err) + } + return info, err == nil +} + +func (r *ctrdRig) installLeases() []leases.Lease { + all, err := r.c.LeasesService().List(context.Background()) + f.Check(r.t, err) + var found []leases.Lease + for _, l := range all { + if strings.HasPrefix(l.ID, "composectl/install/") { + found = append(found, l) + } + } + return found +} + +func dockerImageExists(ref string) bool { + return exec.Command("docker", "image", "inspect", ref).Run() == nil +} + +// checkInstalled verifies the outcome of the install: dockerd runs every image by both references +// offline, no compressed layer stays in containerd, and every snapshot of the image is there. +func (r *ctrdRig) checkInstalled(app *f.App) { + t := r.t + for _, img := range r.images(app.PublishedUri) { + for _, ref := range img.refs { + c := exec.Command("docker", "run", "--rm", "--pull", "never", "--network", "none", ref, "echo", "ok") + out, err := c.CombinedOutput() + f.Checkf(t, err, "docker run %s: %s", ref, out) + } + for _, layer := range img.manifest.Layers { + if r.blobInContentRoot(layer.Digest) { + t.Errorf("layer %s is still in the containerd content store", layer.Digest) + } + } + if !r.blobInContentRoot(img.manifest.Config.Digest) { + t.Errorf("config %s is not in the containerd content store", img.manifest.Config.Digest) + } + for _, chainID := range img.chainIDs { + if _, ok := r.snapshot(chainID); !ok { + t.Errorf("snapshot %s is missing", chainID) + } + } + } + if leftovers := r.installLeases(); len(leftovers) > 0 { + t.Errorf("install leases left after the install: %+v", leftovers) + } +} + +func (r *ctrdRig) checkNotInstalled(app *f.App) { + for _, img := range r.images(app.PublishedUri) { + for _, ref := range img.refs { + if dockerImageExists(ref) { + r.t.Errorf("image %s is still in dockerd", ref) + } + } + } +} + +// pruneImages removes the images no container uses, as aktualizr-lite does after a store prune, +// and waits for containerd's GC to remove the snapshots of the app images. +func (r *ctrdRig) pruneImages(topChainIDs []digest.Digest) { + cli, err := compose.GetDockerClient("") + f.Check(r.t, err) + defer cli.Close() + _, err = cli.ImagesPrune(context.Background(), filters.NewArgs(filters.Arg("dangling", "false"))) + f.Check(r.t, err) + for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(200 * time.Millisecond) { + if !r.anySnapshot(topChainIDs) { + return + } + } + r.t.Fatal("snapshots of the app images are still there after the prune") +} + +func (r *ctrdRig) anySnapshot(chainIDs []digest.Digest) bool { + for _, chainID := range chainIDs { + if _, ok := r.snapshot(chainID); ok { + return true + } + } + return false +} + +// topChainIDs returns the chain ID of the top snapshot of each app image. +func (r *ctrdRig) topChainIDs(app *f.App) []digest.Digest { + var tops []digest.Digest + for _, img := range r.images(app.PublishedUri) { + tops = append(tops, img.chainIDs[len(img.chainIDs)-1]) + } + return tops +} + +// TestCtrdInstall: an app with an image pinned by a manifest and one pinned by an index is installed +// from the store, runs, is seen by `check --install` and `ps`, and is removed by uninstall. +func TestCtrdInstall(t *testing.T) { + r := newCtrdRig(t) + app := f.NewApp(t, ctrdTestApp) + app.Publish(t) + app.Pull(t) + defer app.Remove(t) + + app.Install(t) + r.checkInstalled(app) + app.CheckInstalled(t) + + app.Run(t) + app.CheckRunning(t) + app.Stop(t) + app.Uninstall(t) + r.checkNotInstalled(app) +} + +// TestCtrdInstallSharedLayers: an update whose image is built on top of the installed version's image +// keeps the base snapshots and adds the new one. +func TestCtrdInstallSharedLayers(t *testing.T) { + r := newCtrdRig(t) + const base = "ghcr.io/foundriesio/busybox:1.36" + appV1 := f.NewApp(t, fmt.Sprintf(ctrdDerivedApp, base), "ctrdshared") + appV1.Publish(t) + appV1.Pull(t) + appV1.Install(t) + r.checkInstalled(appV1) + + baseChain := r.images(appV1.PublishedUri)[0].chainIDs + var baseCreated []time.Time + for _, chainID := range baseChain { + info, ok := r.snapshot(chainID) + if !ok { + t.Fatalf("base snapshot %s is missing", chainID) + } + baseCreated = append(baseCreated, info.Created) + } + + derived := f.PublishDerivedImage(t, base, 4) + appV2 := f.NewApp(t, fmt.Sprintf(ctrdDerivedApp, derived), "ctrdshared") + appV2.Publish(t) + appV2.Pull(t) + defer appV2.Remove(t) + app2Chain := r.images(appV2.PublishedUri)[0].chainIDs + if len(app2Chain) != len(baseChain)+1 || app2Chain[len(baseChain)-1] != baseChain[len(baseChain)-1] { + t.Fatalf("the derived image does not share the base chain: %v vs %v", app2Chain, baseChain) + } + + appV2.Install(t) + defer r.pruneImages(baseChain[len(baseChain)-1:]) // v1's images stay in dockerd until v2 is uninstalled + defer appV2.Uninstall(t) + appV1.Remove(t) // `run` by name needs one version of the app in the store + r.checkInstalled(appV2) + for i, chainID := range baseChain { + info, ok := r.snapshot(chainID) + if !ok || !info.Created.Equal(baseCreated[i]) { + t.Errorf("base snapshot %s was recreated by the update: %v -> %v", chainID, baseCreated[i], info.Created) + } + } + appV2.Run(t) + appV2.CheckRunning(t) + appV2.Stop(t) +} + +// TestCtrdInstallAfterPrune: `/images/prune` with aktualizr-lite's filter removes the installed app +// images, `check --install` reports them missing, and the app is reinstalled from the store. +func TestCtrdInstallAfterPrune(t *testing.T) { + r := newCtrdRig(t) + app := f.NewApp(t, ctrdTestApp) + app.Publish(t) + app.Pull(t) + defer app.Remove(t) + app.Install(t) + r.checkInstalled(app) + + r.pruneImages(r.topChainIDs(app)) + if res := app.GetInstallCheckRes(t); len(res.MissingImages) == 0 { + t.Fatal("check --install reports no missing image after the prune") + } + + app.Install(t) + defer app.Uninstall(t) + r.checkInstalled(app) + app.CheckInstalled(t) + app.Run(t) + app.CheckRunning(t) + app.Stop(t) +} + +// TestCtrdInstallRecoversFromKilledInstall: an install is killed after it created snapshots; its +// lease stays, the next install removes the lease and with it the snapshots it alone held, and +// recreates them, so that nothing a power cut may have broken is reused. +func TestCtrdInstallRecoversFromKilledInstall(t *testing.T) { + r := newCtrdRig(t) + const base = "ghcr.io/foundriesio/busybox:1.36" + // The small base image is installed first; the big derived layer keeps the install busy after that. + app := f.NewApp(t, fmt.Sprintf(ctrdKilledApp, base, f.PublishDerivedImage(t, base, 48))) + app.Publish(t) + app.Pull(t) + defer app.Remove(t) + r.pruneImages(r.topChainIDs(app)) + firstChain := r.images(app.PublishedUri)[0].chainIDs + + install := exec.Command(os.Getenv("COMPOSECTL_EXE"), "install", app.PublishedUri) + f.Check(t, install.Start()) + defer func() { _ = install.Process.Kill(); _ = install.Wait() }() + var killedAt time.Time + for deadline := time.Now().Add(2 * time.Minute); killedAt.IsZero(); time.Sleep(5 * time.Millisecond) { + if time.Now().After(deadline) { + t.Fatal("the install created no lease and snapshot in time") + } + if _, ok := r.snapshot(firstChain[0]); ok && len(r.installLeases()) > 0 { + f.Check(t, install.Process.Kill()) + killedAt = time.Now() + } + } + _ = install.Wait() + if leftovers := r.installLeases(); len(leftovers) != 1 { + t.Fatalf("expected the lease of the killed install to stay, got %+v", leftovers) + } + + app.Install(t) + defer app.Uninstall(t) + r.checkInstalled(app) + for _, img := range r.images(app.PublishedUri) { + for _, chainID := range img.chainIDs { + info, ok := r.snapshot(chainID) + if !ok || !info.Created.After(killedAt) { + t.Errorf("snapshot %s was not recreated after the killed install: created %v, killed at %v", chainID, info.Created, killedAt) + } + } + } + app.Run(t) + app.CheckRunning(t) + app.Stop(t) +} + +// TestCtrdInstallWaitsForLock: a second composectl waits for the install lock instead of installing +// concurrently. +func TestCtrdInstallWaitsForLock(t *testing.T) { + r := newCtrdRig(t) + app := f.NewApp(t, ctrdTestApp) + app.Publish(t) + app.Pull(t) + defer app.Remove(t) + + lock, err := os.OpenFile(filepath.Join(f.AppStoreRoot, ".install.lock"), os.O_RDWR|os.O_CREATE, 0o644) + f.Check(t, err) + defer lock.Close() + f.Check(t, syscall.Flock(int(lock.Fd()), syscall.LOCK_EX)) + + install := exec.Command(os.Getenv("COMPOSECTL_EXE"), "install", app.PublishedUri) + install.Stdout = os.Stdout + install.Stderr = os.Stderr + f.Check(t, install.Start()) + done := make(chan error, 1) + go func() { done <- install.Wait() }() + select { + case err := <-done: + t.Fatalf("install did not wait for the lock: %v", err) + case <-time.After(3 * time.Second): + } + if _, ok := r.snapshot(r.images(app.PublishedUri)[0].chainIDs[0]); ok { + t.Fatal("install created a snapshot while the lock was held") + } + + f.Check(t, syscall.Flock(int(lock.Fd()), syscall.LOCK_UN)) + select { + case err := <-done: + f.Checkf(t, err, "install failed after the lock was released: %v", err) + case <-time.After(5 * time.Minute): + t.Fatal("install did not finish after the lock was released") + } + defer app.Uninstall(t) + r.checkInstalled(app) +} + +// TestCtrdUpdateCancel: `update cancel` after an install into containerd removes the loaded images. +func TestCtrdUpdateCancel(t *testing.T) { + r := newCtrdRig(t) + app := f.NewApp(t, ctrdTestApp) + app.Publish(t) + + ctx := context.Background() + ur, err := update.NewUpdate(f.NewTestConfig(t), "target-ctrd-cancel") + f.Check(t, err) + f.Check(t, ur.Init(ctx, []string{app.PublishedUri})) + f.Check(t, ur.Fetch(ctx)) + f.Check(t, ur.Install(ctx)) + r.checkInstalled(app) + imgs := r.images(app.PublishedUri) // the cancel removes the app blobs from the store + if len(ur.Status().LoadedImages) != 2 { + t.Fatalf("expected 2 loaded images, got %+v", ur.Status().LoadedImages) + } + f.Check(t, ur.Cancel(ctx)) + if ur.Status().State != update.StateCanceled { + t.Fatalf("update not canceled: %s", ur.Status().State) + } + for _, img := range imgs { + if dockerImageExists(img.refs[1]) { + t.Errorf("image %s is still in dockerd after the cancel", img.refs[1]) + } + } + f.Check(t, os.RemoveAll(f.NewTestConfig(t).GetAppComposeDir(app.Name))) + f.Check(t, compose.RemoveApps(ctx, f.NewTestConfig(t), []string{app.PublishedUri}, compose.WithCheckStatus(false))) +}