diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f75c36..e71c916 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,7 @@ jobs: go-version: "1.26.x" - run: make test - run: make profiles-check + - run: python3 scripts/test-install.py # Every platform a release ships must keep compiling. - name: Cross-compile if: runner.arch == 'X64' @@ -66,7 +67,7 @@ jobs: with: src: python # warmctl.io/install.sh runs under any POSIX sh: dash, busybox, macOS. - - run: shellcheck -s sh scripts/install.sh + - run: shellcheck -s sh scripts/install.sh scripts/fetch-dmtcp.sh vulncheck: runs-on: ubuntu-24.04 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6c85486..b8892d0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,6 +11,10 @@ on: - .goreleaser.yaml - .github/workflows/release.yml - scripts/release-notes.sh + - scripts/fetch-dmtcp.sh + - scripts/install.sh + - scripts/test-install.py + - internal/dmtcp/** - Makefile permissions: @@ -63,6 +67,11 @@ jobs: name: cuinterpose-linux-amd64 path: ${{ runner.temp }} - run: mkdir -p bin && tar -xf "$RUNNER_TEMP/cuinterpose-linux-amd64.tar" -C bin && ls -l bin/cuinterpose + - name: Fetch the pinned DMTCP release packages + run: | + version=$(cat internal/dmtcp/VERSION) + scripts/fetch-dmtcp.sh amd64 "dmtcp/linux-amd64/$version" + scripts/fetch-dmtcp.sh arm64 "dmtcp/linux-arm64/$version" - uses: anchore/sbom-action/download-syft@66cbf4bc1f1c0d2edc94016e65bc221b6bb0ad6c # v0.24.3 with: syft-version: ${{ env.SYFT_VERSION }} @@ -76,12 +85,12 @@ jobs: TAG: ${{ startsWith(github.ref, 'refs/tags/') && github.ref_name || 'v2.0.0-beta.5' }} run: scripts/release-notes.sh "$TAG" > "$RUNNER_TEMP/notes.md" - - name: Publish + - name: Build signed release archives if: startsWith(github.ref, 'refs/tags/') uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7.3.0 with: version: ${{ env.GORELEASER_VERSION }} - args: release --clean --release-notes ${{ runner.temp }}/notes.md + args: release --clean --skip=publish --release-notes ${{ runner.temp }}/notes.md env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Provenance @@ -104,11 +113,83 @@ jobs: tar -xzf dist/warmctl-*-linux-amd64.tar.gz -C "$RUNNER_TEMP" warmctl cuinterpose/libcuinterpose.so "$RUNNER_TEMP/warmctl" version test -x "$RUNNER_TEMP/warmctl" - - if: "!startsWith(github.ref, 'refs/tags/')" - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: snapshot + name: release-packages path: | dist/*.tar.gz dist/*checksums.txt + dist/*.sigstore.json + dist/*.sbom.json retention-days: 7 + + verify-native: + needs: release + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-24.04 + arch: amd64 + - runner: ubuntu-24.04-arm + arch: arm64 + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: release-packages + path: packages + - name: Check the actual native archive and plugin + run: | + mkdir unpacked + tar -xzf packages/*-linux-${{ matrix.arch }}.tar.gz -C unpacked + unpacked/warmctl version + version=$(cat internal/dmtcp/VERSION) + root="$PWD/unpacked/dmtcp/linux-${{ matrix.arch }}/$version" + "$root/bin/dmtcp_launch" --version + "$root/bin/dmtcp_coordinator" --daemon --exit-on-last --port 0 --port-file "$RUNNER_TEMP/dmtcp.port" + port=$(cat "$RUNNER_TEMP/dmtcp.port") + trap '"$root/bin/dmtcp_command" --coord-port "$port" --quit || true' EXIT + DMTCP_PLUGIN="$root/lib/libdmtcp_fdkeep.so" "$root/bin/dmtcp_launch" --join-coordinator --coord-port "$port" /bin/true + - name: Checkpoint and restore with the packaged runtime on Ubuntu 24.04 + env: + GH_TOKEN: ${{ github.token }} + run: | + version=$(cat internal/dmtcp/VERSION) + mkdir "$RUNNER_TEMP/dmtcp-source" + gh release download "$version" --repo TensorRails/dmtcp --pattern "dmtcp-$version-source.tar.gz" --dir "$RUNNER_TEMP/dmtcp-source" + (cd "$RUNNER_TEMP/dmtcp-source" && sha256sum --ignore-missing -c "$GITHUB_WORKSPACE/internal/dmtcp/SHA256SUMS") + tar -xzf "$RUNNER_TEMP/dmtcp-source/dmtcp-$version-source.tar.gz" --strip-components=1 -C "$RUNNER_TEMP/dmtcp-source" + root="$PWD/unpacked/dmtcp/linux-${{ matrix.arch }}/$version" + # Use the generated version header from the installed package. + CPATH="$root/include" python3 "$RUNNER_TEMP/dmtcp-source/test/restore-memory-check.py" --root "$root" \ + --plugin "$root/lib/libdmtcp_fdkeep.so" --output "$RUNNER_TEMP/dmtcp-installed-check" + - name: Install, reinstall, preserve old snapshots and refuse replacement + run: INSTALL_TEST_ARCHIVE=$(echo "$PWD"/packages/*-linux-${{ matrix.arch }}.tar.gz) python3 scripts/test-install.py + + publish: + if: startsWith(github.ref, 'refs/tags/') + needs: verify-native + runs-on: ubuntu-24.04 + permissions: + contents: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: release-packages + path: packages + - name: Publish verified archives + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.ref_name }} + run: | + scripts/release-notes.sh "$TAG" > "$RUNNER_TEMP/notes.md" + flags=() + case "$TAG" in *-*) flags+=(--prerelease) ;; esac + gh release create "$TAG" packages/* --verify-tag --title "$TAG" --notes-file "$RUNNER_TEMP/notes.md" "${flags[@]}" diff --git a/.github/workflows/runtime-image.yml b/.github/workflows/runtime-image.yml new file mode 100644 index 0000000..66c9514 --- /dev/null +++ b/.github/workflows/runtime-image.yml @@ -0,0 +1,112 @@ +name: runtime image + +# Publish only an already released CLI. GPU cold-start qualification follows +# separately; this workflow deliberately does not move the latest image tag. +on: + workflow_dispatch: + inputs: + release: + description: Published warmctl release tag + required: true + image_tag: + description: Optional image tag (defaults to the release tag) + required: false + base: + description: Optional pinned base image (defaults to the Dockerfile base) + required: false + +permissions: + contents: read + attestations: read + +concurrency: + group: runtime-image-${{ inputs.image_tag || inputs.release }} + cancel-in-progress: false + +jobs: + publish: + runs-on: ubuntu-24.04 + timeout-minutes: 60 + env: + RELEASE: ${{ inputs.release }} + IMAGE_TAG: ${{ inputs.image_tag || inputs.release }} + BASE_IMAGE: ${{ inputs.base }} + GH_TOKEN: ${{ github.token }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.release }} + persist-credentials: false + - name: Validate release and image tag + run: | + [[ "$RELEASE" =~ ^v[0-9][0-9A-Za-z._-]*$ ]] + [[ "$IMAGE_TAG" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]*$ ]] + gh release view "$RELEASE" --repo TensorRails/warmctl + if [ -n "$BASE_IMAGE" ]; then [[ "$BASE_IMAGE" == *@sha256:* ]]; fi + - name: Make room for the vLLM image + run: | + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /opt/hostedtoolcache + df -h / + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + - name: Fetch and verify the released cuInterpose bundle + run: | + mkdir -p "$RUNNER_TEMP/release" bin + gh release download "$RELEASE" --repo TensorRails/warmctl --dir "$RUNNER_TEMP/release" \ + --pattern "warmctl-$RELEASE-linux-amd64.tar.gz" --pattern "warmctl-$RELEASE-checksums.txt" + (cd "$RUNNER_TEMP/release" && sha256sum --ignore-missing -c "warmctl-$RELEASE-checksums.txt") + gh attestation verify "$RUNNER_TEMP/release/warmctl-$RELEASE-linux-amd64.tar.gz" --repo TensorRails/warmctl + tar -xzf "$RUNNER_TEMP/release/warmctl-$RELEASE-linux-amd64.tar.gz" -C bin cuinterpose + - name: Build the released runtime + run: | + args=() + if [ -n "$BASE_IMAGE" ]; then args+=(--build-arg "BASE=$BASE_IMAGE"); fi + docker build -t "localhost:5000/warmctl/runtime:$IMAGE_TAG" \ + -f deploy/kubernetes/Dockerfile --build-context cuinterpose=bin/cuinterpose \ + --build-arg "VERSION=$RELEASE" "${args[@]}" . + - name: Verify runtime programs and native plugin + run: | + docker run --rm --entrypoint /bin/bash "localhost:5000/warmctl/runtime:$IMAGE_TAG" -lc ' + set -e + warmctl version + cat /opt/warmctl/RUNTIME /opt/warmctl/dmtcp/BUILDINFO + python3 -c "import importlib.metadata as m; print(\"vllm\", m.version(\"vllm\"))" + root=/opt/warmctl/dmtcp + "$root/bin/dmtcp_coordinator" --daemon --exit-on-last --port 0 --port-file /tmp/dmtcp.port + port=$(cat /tmp/dmtcp.port) + DMTCP_PLUGIN="$root/lib/libdmtcp_fdkeep.so" "$root/bin/dmtcp_launch" --join-coordinator --coord-port "$port" /bin/true + ' + - name: Install the pinned registry client + run: | + curl -fsSL --retry 3 https://github.com/regclient/regclient/releases/download/v0.11.7/regctl-linux-amd64 -o "$RUNNER_TEMP/regctl" + echo "3737fbbdd791e04e7a831640b681f3397a376498e80e2ad9b1b2d27296f239bc $RUNNER_TEMP/regctl" | sha256sum -c - + chmod 0755 "$RUNNER_TEMP/regctl" + - name: Publish with bounded registry chunks + env: + WARMCTL_REGISTRY_AUTH: ${{ secrets.WARMCTL_REGISTRY_AUTH }} + run: | + python3 - <<'PY' + import json, os + from pathlib import Path + config = json.loads(os.environ['WARMCTL_REGISTRY_AUTH']) + assert set(config['auths']) == {'registry.warmctl.io'} + path = Path.home() / '.docker/config.json' + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(config)) + path.chmod(0o600) + PY + docker run -d --name release-registry -p 127.0.0.1:5000:5000 registry:2 + docker push "localhost:5000/warmctl/runtime:$IMAGE_TAG" + "$RUNNER_TEMP/regctl" registry set localhost:5000 --tls disabled + "$RUNNER_TEMP/regctl" registry set registry.warmctl.io --blob-chunk 67108864 --blob-max 67108864 + "$RUNNER_TEMP/regctl" image copy "localhost:5000/warmctl/runtime:$IMAGE_TAG" "registry.warmctl.io/warmctl/runtime:$IMAGE_TAG" + digest=$("$RUNNER_TEMP/regctl" image digest "registry.warmctl.io/warmctl/runtime:$IMAGE_TAG") + printf 'registry.warmctl.io/warmctl/runtime:%s@%s\n' "$IMAGE_TAG" "$digest" | tee runtime-image.txt + cat runtime-image.txt >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: runtime-image + path: runtime-image.txt + if-no-files-found: error + - name: Remove credentials + if: always() + run: rm -f "$HOME/.docker/config.json" diff --git a/.gitignore b/.gitignore index cab98b8..ad5f8a3 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ __pycache__/ .venv/ *.test .cloudflare/ +/dmtcp/ diff --git a/.goreleaser.yaml b/.goreleaser.yaml index f8f25a4..6f2e4e5 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -12,7 +12,17 @@ builds: goos: [linux] goarch: [amd64] flags: [-trimpath] - ldflags: ['-s -w -X main.version={{ if .IsSnapshot }}{{ .Tag }}-dev+{{ .ShortCommit }}{{ else }}{{ .Tag }}{{ end }}'] + ldflags: + - '-s -w -X main.version={{ if .IsSnapshot }}{{ .Tag }}-dev+{{ .ShortCommit }}{{ else }}{{ .Tag }}{{ end }}' + mod_timestamp: "{{ .CommitTimestamp }}" + - id: linux-arm64 + main: ./cmd/warmctl + binary: warmctl + env: [CGO_ENABLED=0] + targets: [linux_arm64] + flags: [-trimpath] + ldflags: + - '-s -w -X main.version={{ if .IsSnapshot }}{{ .Tag }}-dev+{{ .ShortCommit }}{{ else }}{{ .Tag }}{{ end }}' mod_timestamp: "{{ .CommitTimestamp }}" # linux/arm64 has no cuInterpose build yet; macOS runs the registry side # (catalog, images, mirror, push, pull, profile), not build or up. @@ -20,7 +30,7 @@ builds: main: ./cmd/warmctl binary: warmctl env: [CGO_ENABLED=0] - targets: [linux_arm64, darwin_amd64, darwin_arm64] + targets: [darwin_amd64, darwin_arm64] flags: [-trimpath] ldflags: ['-s -w -X main.version={{ if .IsSnapshot }}{{ .Tag }}-dev+{{ .ShortCommit }}{{ else }}{{ .Tag }}{{ end }}'] mod_timestamp: "{{ .CommitTimestamp }}" @@ -37,6 +47,16 @@ archives: - src: bin/cuinterpose/* dst: cuinterpose strip_parent: true + - dmtcp/linux-amd64/**/* + - id: linux-arm64 + ids: [linux-arm64] + formats: [tar.gz] + name_template: "warmctl-{{ .Tag }}-{{ .Os }}-{{ .Arch }}" + files: + - LICENSE + - NOTICE + - README.md + - dmtcp/linux-arm64/**/* - id: plain ids: [plain] formats: [tar.gz] diff --git a/CHANGELOG.md b/CHANGELOG.md index 085dcb4..f2fc65f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,37 @@ # Changelog +## v2.0.0-beta.17 — 2026-11-01 + +Large local DMTCP snapshots restore faster with huge pages and parallel +CPU-memory reads. On one H200, Qwen3.8-Flash-Next-NVFP4 reached health in +20.21–23.43 s across four runs (median 20.83 s, first streaming token +20.92 s), versus 77.36/82.14 s with the ordinary reader on the same host +and snapshot. These were local restores with hot file cache and sufficient +RAM, excluding provisioning and snapshot transfer; this is not a <=20 s SLA. + +- `up` supplies restart-only huge-page advice and up to 16 parallel readers, + bounded by Go's container-aware CPU budget. Explicit + `DMTCP_RESTART_HUGEPAGES` and `DMTCP_RESTART_READERS` values override the + defaults. Private mmap remains disabled by default because it shifted + work into CUDA and worsened full recovery time. +- Linux archives now include the separately released TensorRails DMTCP + `v5.0.0-warmctl.1` runtime and fdkeep plugin for amd64 and arm64. The + runtime Dockerfile downloads the same packages with source-pinned SHA-256 + checksums, with no DMTCP compilation during the warmctl build. Packages + require Debian 12-compatible glibc/libstdc++; macOS remains client-only. +- The CLI finds its versioned runtime beside its own executable, including + custom install prefixes. The installer preserves runtime bytes and mtimes, + retains older versions, and refuses to replace a different build at the + same version. Bake new snapshots after upgrading; old snapshots still + need their original libraries and paths. +- Release archives and the installer are tested natively on both Linux + architectures before publication. ARM64 is CPU-tested; GPU serving and + the parallel reader remain x86-64-only qualifications. +- All 20 GPU experiments matched weight fingerprints and passed streaming + and tool-call tests; 640 short requests succeeded. The numerical canary + remains unqualified: the donor already varied before checkpoint. Its + tolerance is unchanged. See docs/dmtcp.md for the phase breakdown. + ## v2.0.0-beta.16 — 2026-11-01 warmctl now runs where CRIU cannot: in containers without privileges, such diff --git a/Makefile b/Makefile index cf89074..1cccd20 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: catalog-push demo build linux test pytest lint profiles-check check criu-test criu-image e2e-test cuinterpose pod-image release clean +.PHONY: catalog-push demo build linux test pytest lint profiles-check check criu-test criu-image e2e-test cuinterpose pod-image dmtcp release clean GO ?= go PY ?= python3 @@ -96,9 +96,16 @@ pod-image: docker build -t $(POD_IMAGE) -f deploy/kubernetes/Dockerfile --build-context cuinterpose=bin/cuinterpose \ --build-arg VERSION=$(VERSION) . +# Fetch the independently released native runtimes for Linux CLI archives. +dmtcp: + @for arch in amd64 arm64; do \ + dest="dmtcp/linux-$$arch/$$(cat internal/dmtcp/VERSION)"; \ + test -f "$$dest/BUILDINFO" || scripts/fetch-dmtcp.sh "$$arch" "$$dest" || exit $$?; \ + done + # The release archives as .github/workflows/release.yml builds them, as a # local snapshot in dist/ (nothing signed or published). Needs goreleaser v2. -release: +release: dmtcp test -f bin/cuinterpose/libcuinterpose.so || $(MAKE) cuinterpose goreleaser release --snapshot --clean --skip=sign,sbom diff --git a/NOTICE b/NOTICE index f526d6e..b03bbba 100644 --- a/NOTICE +++ b/NOTICE @@ -3,8 +3,16 @@ Copyright 2026 The warmctl Authors This product is licensed under the Apache License, Version 2.0 (LICENSE). -Release archives also contain cuInterpose (cuinterpose/), Copyright (c) 2026 +Linux amd64 release archives also contain cuInterpose (cuinterpose/), Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES, licensed under the Apache License, Version 2.0 (cuinterpose/LICENSE). It is built unmodified from https://github.com/ai-dynamo/snapshot at commit 50bf0db3575df64c968e3f3255d5fa8938950b7a (v0.2.0-rc.2), agent/cmd/cuinterpose. + +Linux release archives also contain TensorRails' DMTCP fork, based on DMTCP +5.0, under the GNU Lesser General Public License version 3 or later. See +COPYING, COPYING.LESSER and AUTHORS under dmtcp/linux-ARCH/VERSION/share/doc/dmtcp. +Corresponding source is published with each independently versioned release: +https://github.com/TensorRails/dmtcp/releases +The included fdkeep plugin is Copyright 2026 The warmctl Authors, under the +Apache License 2.0; its license is in the same runtime's share/doc/dmtcp/fdkeep. diff --git a/README.md b/README.md index ca15b16..a30cfb8 100644 --- a/README.md +++ b/README.md @@ -157,11 +157,15 @@ driver branch it was built on (580.x, 610.x). Proven on H100 SXM, H200, B200, B300, L40S and RTX PRO 6000. `warmctl hardware` lists the GPU models warmctl knows and marks this host's; another model is a row in a `hardware.yaml` next to your profiles (`--profile-dir`). -Keep `warmctl` and `cuinterpose/` from the linux-amd64 archive side by side, -or build both with `make build cuinterpose`. Releases also ship linux-arm64 -(no cuInterpose yet, not proven on a GPU) and macOS builds, which run the -registry side: `catalog`, `images`, `mirror`, `push`, `pull`. Every release -is signed and has build provenance ([verify](RELEASING.md#verify-a-release)). +Keep `warmctl`, `cuinterpose/` and the versioned `dmtcp/` directory from the +linux-amd64 archive side by side; the installer does this automatically. +Linux arm64 archives include DMTCP but no cuInterpose yet and are CPU-tested, +not GPU-qualified. The native DMTCP packages require Debian 12-compatible +glibc (>= 2.36) and libstdc++. macOS builds run the registry side: `catalog`, +`images`, `mirror`, `push`, `pull`. Source builds use `make build cuinterpose`; +see [DMTCP runtime installation and upgrades](docs/dmtcp.md). +Every release is signed and has build provenance +([verify](RELEASING.md#verify-the-cli-release)). ## Develop diff --git a/RELEASING.md b/RELEASING.md index 4f4b987..08dca54 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -12,13 +12,14 @@ git push origin main vX.Y.Z ``` -3. `.github/workflows/release.yml` builds cuInterpose and publishes the GitHub - release: +3. `.github/workflows/release.yml` builds cuInterpose, fetches the pinned + DMTCP packages, builds and signs archives, tests the Linux archives and + installer on native amd64 and arm64 runners, then publishes the release: | Asset | What | |---|---| - | `warmctl-vX.Y.Z-linux-amd64.tar.gz` | `warmctl`, `cuinterpose/`, LICENSE, NOTICE, README | - | `warmctl-vX.Y.Z-linux-arm64.tar.gz` | `warmctl` without cuInterpose; not proven on a GPU yet | + | `warmctl-vX.Y.Z-linux-amd64.tar.gz` | `warmctl`, `cuinterpose/`, versioned `dmtcp/`, LICENSE, NOTICE, README | + | `warmctl-vX.Y.Z-linux-arm64.tar.gz` | `warmctl` and versioned `dmtcp/`, without cuInterpose; CPU-tested, not GPU-qualified | | `warmctl-vX.Y.Z-darwin-{amd64,arm64}.tar.gz` | `warmctl` for the registry side: `catalog`, `images`, `mirror`, `push`, `pull`, `profile` | | `warmctl-vX.Y.Z-checksums.txt` | SHA-256 of every archive | | `warmctl-vX.Y.Z-checksums.txt.sigstore.json` | cosign signature of the checksums, keyless from the workflow | @@ -42,9 +43,50 @@ regctl image copy registry.warmctl.io/warmctl/runtime:vX.Y.Z registry.warmctl.io/warmctl/runtime:latest ``` + Alternatively, dispatch `.github/workflows/runtime-image.yml` with the + published release tag. It uses the same Dockerfile, verifies the released + cuInterpose archive and provenance, runs CPU runtime checks, then copies + the image to registry.warmctl.io in bounded chunks. Configure the repository + secret `WARMCTL_REGISTRY_AUTH` as a Docker auth JSON containing only that + registry. An optional custom base must be pinned by digest. The workflow + publishes the requested tag and records the digest; promote `latest` only + after the GPU bake and cold-start checks succeed. + 5. If `catalog/catalog.yaml` changed: `make catalog-push`. -## Verify a release +## DMTCP dependency + +DMTCP has independent `v5.0.0-warmctl.N` tags in TensorRails/dmtcp, with +native Linux amd64 and arm64 packages containing both the runtime and fdkeep. +The plugin source is maintained in that fork's `plugin/fdkeep`, not duplicated +here. DMTCP and its shared libraries are native Linux code, not portable Go +executables. Packages are built on Debian 12 (glibc >= 2.36 and compatible +libstdc++); macOS archives contain only the warmctl client. + +For an update, first publish and verify the DMTCP release. Update +`internal/dmtcp/VERSION` and copy its published archive checksums into +`internal/dmtcp/SHA256SUMS`. `scripts/fetch-dmtcp.sh` verifies those pinned +checksums before extracting. Both GoReleaser and the runtime Dockerfile +consume these assets; neither compiles the fork. `make dmtcp` fetches them +for local release builds. + +Linux CLI archives keep the runtime at +`dmtcp/linux-ARCH/VERSION/` beside `warmctl`. The CLI resolves its executable +symlink to find that directory, including custom installation prefixes. The +installer preserves its bytes and mtimes, refuses to overwrite a different +build at the same version, and retains older versions for existing snapshots. +Source-built CLIs without a bundle fall back to `/opt/warmctl/dmtcp`, as used +by the runtime image. Explicit `--dmtcp-dir` and `--dmtcp-plugin` still override +the defaults. + +Bake fresh snapshots after a runtime upgrade. Donor and receiver need the +same mapped libraries at the same paths; keep old runtime images and versioned +directories for old snapshots. An updated CLI archive does not publish a new +vLLM runtime image: build and publish that separately using step 4. ARM64 +packages pass CPU checkpoint/restore tests; the parallel reader and H200 GPU +qualification are x86-64 only. + +## Verify the CLI release ```bash V=vX.Y.Z diff --git a/deploy/kubernetes/Dockerfile b/deploy/kubernetes/Dockerfile index 9e04351..b3aa364 100644 --- a/deploy/kubernetes/Dockerfile +++ b/deploy/kubernetes/Dockerfile @@ -26,17 +26,15 @@ ARG VERSION=dev RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \ go build -trimpath -ldflags "-X main.version=${VERSION}" -o /out/warmctl ./cmd/warmctl -# DMTCP for the dmtcp backend (docs/dmtcp.md): TensorRails' fork at a pinned -# commit and warmctl's fdkeep plugin against it, built on the base so they -# link the libc the engine runs with. A restore maps DMTCP's own text from -# these files, so donor and receiver need this very build. +# The same released DMTCP+fdkeep bytes carried by the CLI archive. The package +# requires Debian 12-compatible glibc/libstdc++; execute it during the build +# so an incompatible BASE fails here. No fork compilation during image build. FROM ${BASE} AS dmtcp -ARG DMTCP_COMMIT=fc4c8a3878606dad0b487cd716ddaa5b1ea0195d -RUN apt-get update && apt-get install -y --no-install-recommends build-essential ca-certificates git \ - && git clone https://github.com/TensorRails/dmtcp /src/dmtcp && cd /src/dmtcp && git checkout ${DMTCP_COMMIT} \ - && ./configure --prefix=/opt/warmctl/dmtcp && make -j"$(nproc)" && make install -COPY internal/dmtcp/fdkeep/fdkeep.c /src/fdkeep.c -RUN gcc -Wall -O2 -shared -fPIC -I/opt/warmctl/dmtcp/include -o /opt/warmctl/dmtcp/lib/libdmtcp_fdkeep.so /src/fdkeep.c +COPY scripts/fetch-dmtcp.sh /src/scripts/fetch-dmtcp.sh +COPY internal/dmtcp/VERSION internal/dmtcp/SHA256SUMS /src/internal/dmtcp/ +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl libstdc++6 \ + && sh /src/scripts/fetch-dmtcp.sh amd64 /opt/warmctl/dmtcp \ + && /opt/warmctl/dmtcp/bin/dmtcp_launch --version # cuInterpose's build (make cuinterpose), passed as a named build context; # empty without one. A restore runs the snapshot's own copy, so only a bake @@ -44,6 +42,10 @@ RUN gcc -Wall -O2 -shared -fPIC -I/opt/warmctl/dmtcp/include -o /opt/warmctl/dmt FROM scratch AS cuinterpose FROM ${BASE} +# The native DMTCP package also works with minimal Debian/Ubuntu test bases. +# --no-upgrade keeps an existing runtime's C++ library unchanged. +RUN apt-get update && apt-get install -y --no-install-recommends --no-upgrade libstdc++6 \ + && rm -rf /var/lib/apt/lists/* ARG BASE ARG BASE_REF=${BASE} ARG VERSION=dev diff --git a/docs/dmtcp.md b/docs/dmtcp.md index e56bb85..ace436a 100644 --- a/docs/dmtcp.md +++ b/docs/dmtcp.md @@ -26,6 +26,71 @@ rsync -a --exclude manifest.json /snap/q8/ /nv/snap/q8/ && cp /snap/q8/manifest. warmctl up /nv/snap/q8 --mode in-place # or up NAME: the index entry this host can restore ``` +## Large local images and the versioned fork + +From warmctl beta.17, the receiver supplies `DMTCP_RESTART_HUGEPAGES=1` and +`DMTCP_RESTART_READERS` for DMTCP restores. Readers are bounded by Go's +container-aware CPU budget: 1–8, or 16 when at least 16 CPUs are available. +They operate per process and only on raw regular-file payloads of at least +1 GiB on x86-64. Explicit values in the receiver environment override these +defaults. Compressed streams and older DMTCP builds keep ordinary reading. +`DMTCP_RESTART_MMAP` defaults to 0: CUDA restoration can force private copies +and erase its memory-pass benefit. Profiling is opt-in: + +```bash +DMTCP_RESTART_PROFILE=1 warmctl up SNAPSHOT --mode in-place +# Compare with the ordinary reader: +DMTCP_RESTART_HUGEPAGES=0 DMTCP_RESTART_READERS=1 warmctl up SNAPSHOT --mode in-place +``` + +**One H200, Qwen3.8-Flash-Next-NVFP4, local snapshot with hot file cache:** +four optimized runs reached health in **20.21, 23.43, 20.27, 21.39 s**; +median **20.83 s**, first streaming token **20.92 s**. The same host and +snapshot took 77.36/82.14 s with the ordinary reader. One full pair: + +| Phase | Ordinary reader | Huge pages + 16 readers | +| --- | ---: | ---: | +| Setup and filesystem | 3.09 s | 3.33 s | +| DMTCP | 54.09 s | 4.04 s | +| CUDA | 13.52 s | 6.73 s | +| Wake and reload weights | 5.62 s | 5.12 s | +| KV, health, canary | 0.99 s | 0.99 s | +| Total including health polling | 77.36 s | 20.27 s | + +The slow path incurred 14.15 million minor faults while loading a 47.68 GiB +anonymous region already in page cache. Huge pages and parallel reads +reduced its load to 1.27–1.33 s. The cgroup had 20.4 CPUs and 233.76 GiB RAM; +unused experimental-image cache was evicted before the fast series, which +peaked at 213.08 GiB without direct reclaim. These numbers exclude provisioning +and snapshot transfer and are not a <=20 s SLA. All 20 experiment restores +matched weight fingerprints and passed streaming/tool checks; all 640 short +requests succeeded. Numerical equivalence remains unqualified: the donor +already varied before checkpoint. The canary tolerance remains 0.001. +The [full experiment record](https://github.com/TensorRails/dmtcp/blob/v5.0.0-warmctl.1/doc/h200-restart-performance.md) +includes all modes, cache conditions and the unsuccessful mmap experiment. + +The fork is released independently as +[`v5.0.0-warmctl.1`](https://github.com/TensorRails/dmtcp/releases/tag/v5.0.0-warmctl.1), +with source, Debian 12-compatible Linux amd64 and arm64 packages, checksums +and provenance. Both packages include fdkeep. Linux warmctl archives bundle +them at `dmtcp/linux-ARCH/VERSION/` beside the CLI; the runtime Dockerfile +fetches the same assets into `/opt/warmctl/dmtcp`. Downloads are verified +against SHA-256 values pinned in warmctl's source. No fork compilation is +needed during warmctl packaging. macOS archives remain client-only. + +**Bake a new snapshot after upgrading the runtime.** Donor and receiver need +byte-identical mapped libraries at the same paths. The installer preserves +versioned DMTCP directories and their mtimes, and keeps older versions for +existing snapshots. Use an old snapshot's original runtime image or explicit +`--dmtcp-dir` / `--dmtcp-plugin` paths. Publishing a CLI release does not +replace an existing runtime image. See the fork's +[release procedure](https://github.com/TensorRails/dmtcp/blob/v5.0.0-warmctl.1/RELEASING.md). + +The packages require glibc >= 2.36 and compatible libstdc++. ARM64 passes +native CPU checkpoint/restore tests, including fdkeep; GPU serving has not +been qualified on ARM64. Parallel pread is x86-64-only; ARM64 uses the +ordinary reader with optional huge pages. + ## Choosing the backend | | Decided by | @@ -100,14 +165,15 @@ A pod restores a snapshot another pod took when it runs the same image, GPU mode - **The engine's image.** An in-place bake records the runtime image's base (`/opt/warmctl/RUNTIME`) as the engine's image, which a receiver compares with its own. - **DMTCP.** Library text is mapped from the receiver's files (see the fork below), so DMTCP and every library - must be byte-identical: the same image. `--dmtcp-dir` defaults to the image's `/opt/warmctl/dmtcp/bin`, not - PATH. A shell in a RunPod pod lacks the image's PATH, and any other DMTCP would not match. + must be byte-identical. `--dmtcp-dir` defaults to the bundled versioned runtime beside the CLI, + or `/opt/warmctl/dmtcp/bin` in the runtime image. It never selects an arbitrary build from PATH. ## Restoring from network storage A RunPod Pod gets its snapshot from a Global Volume (S3 behind geesefs, FUSE on the host) or a network -volume (MooseFS over FUSE in EU-NL-1, VAST NFS in CA-MTL-1). The restore's readers, `dmtcp_restart` and -the weights reload, read one stream each, and FUSE keeps no page cache the prefetch could fill for them. +volume (MooseFS over FUSE in EU-NL-1, VAST NFS in CA-MTL-1). These mounts can limit throughput +and page-cache reuse. The parallel DMTCP reader improves local raw-image restoration; network staging +still handles remote throughput and independent compressed frames. So `up` from a FUSE directory, or from any snapshot stored encoded, stages a local copy (`restore.Stage`): @@ -144,8 +210,7 @@ records each file's encoding, frame ends and byte-plane split; sizes and SHA-256 `--compress`. The source keeps its files, so one bake gives both: a snapshot `push` publishes, and a smaller copy for a network volume. -Compression stays out of the restore itself. The fork can pipe its images through zstd (`DMTCP_ZSTD=1`, -commit `fbbaf3f`, after the one warmctl pins), but `mtcp_restart` then reads one decoded stream through a +Compression stays out of the restore itself. The fork can pipe its images through zstd (`DMTCP_ZSTD=1`), but `mtcp_restart` then reads one decoded stream through a pipe at 0.67 GB/s. A restore from a local copy went from 10.3 to 15.4 s that way, so the images stay raw on disk. @@ -154,8 +219,8 @@ are [#60](https://github.com/TensorRails/warmctl/issues/60). ## DMTCP: the fork and the plugin -Branch `warmctl` of [TensorRails/dmtcp](https://github.com/TensorRails/dmtcp/tree/warmctl) is DMTCP 5.0 -with these patches: +The released [TensorRails/dmtcp](https://github.com/TensorRails/dmtcp) fork is DMTCP 5.0 +with the restart changes above and these earlier patches: | Patch | Why | |---|---| @@ -171,7 +236,8 @@ with these patches: | Accept an already unlinked shm file after restart | Processes that share an area of a deleted `/dev/shm` file (CPython's named semaphores, unlinked at once) recreate one file at restart and each unlinks it again; every one but the first found it gone, and every vLLM restart printed a warning. | | Change a socket's connection only with the fd table locked | `setsockopt`, `connect`, `bind` and `listen` changed the connection they had looked up while another thread could delete it: `add()` drops the entry of an fd closed past the close wrapper once the number comes back. A worker restarted under cuInterpose, whose Rust threads make and close some sockets with raw syscalls, segfaulted in `SocketConnection::addSetsockopt` once in about a dozen restores. | -The plugin `internal/dmtcp/fdkeep/fdkeep.c` repairs what DMTCP loses: +The plugin [fdkeep](https://github.com/TensorRails/dmtcp/tree/v5.0.0-warmctl.1/plugin/fdkeep), +now maintained and released with the fork, repairs what DMTCP loses: - the CUDA driver's eventfds and pipes, recreated as real pipes; - thread names, which `cuda-checkpoint` uses to find the restore thread; @@ -182,8 +248,8 @@ The plugin `internal/dmtcp/fdkeep/fdkeep.c` repairs what DMTCP loses: ## The runtime image -`deploy/kubernetes/Dockerfile` builds the fork at a pinned commit and fdkeep against it, on the base, into -`/opt/warmctl/dmtcp`. +`deploy/kubernetes/Dockerfile` downloads the pinned DMTCP release with fdkeep into +`/opt/warmctl/dmtcp`, checks its SHA-256, and executes it to reject incompatible base images. - **Base.** The default base (vLLM's cu134 nightly, CUDA 13.4) runs on RunPod's 595.91 driver (CUDA 13.2) through CUDA's minor-version compatibility: verified on 4×RTX PRO 6000 in EU-RO-1. The experimental image @@ -279,6 +345,5 @@ runs at that pod's upload speed (0.22 GiB/s): geesefs uploads in the background ```bash ./configure --prefix=/opt/warmctl/dmtcp && make -j && make install -gcc -shared -fPIC -O2 -I/opt/warmctl/dmtcp/include -o /opt/warmctl/dmtcp/lib/libdmtcp_fdkeep.so \ - internal/dmtcp/fdkeep/fdkeep.c +make -C plugin/fdkeep install PREFIX=/opt/warmctl/dmtcp ``` diff --git a/internal/cli/commands.go b/internal/cli/commands.go index c22e872..70b54c6 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -61,8 +61,9 @@ func (c *common) register(fs *flag.FlagSet) { fs.BoolVar(&c.noVerify, "no-verify", false, "canary only warns") fs.IntVar(&c.threads, "threads", 0, "pool copy threads per rank (default 4)") fs.BoolVar(&c.cpuTest, "test-cpu", false, "end-to-end test on a CPU host: no GPU devices, no io_uring check") - fs.StringVar(&c.dmtcp.Dir, "dmtcp-dir", dmtcp.DefaultDir, "directory of DMTCP's programs, for the dmtcp backend") - fs.StringVar(&c.dmtcp.Plugin, "dmtcp-plugin", dmtcp.DefaultPlugin, + dmtcpDefaults := dmtcp.DefaultTools() + fs.StringVar(&c.dmtcp.Dir, "dmtcp-dir", dmtcpDefaults.Dir, "directory of DMTCP's programs, for the dmtcp backend") + fs.StringVar(&c.dmtcp.Plugin, "dmtcp-plugin", dmtcpDefaults.Plugin, "warmctl's fdkeep DMTCP plugin, for the dmtcp backend") } diff --git a/internal/cli/oci.go b/internal/cli/oci.go index b5c3e58..ec670a0 100644 --- a/internal/cli/oci.go +++ b/internal/cli/oci.go @@ -155,7 +155,7 @@ func runPull(ctx context.Context, args []string, stdout, stderr io.Writer) error mode := fs.String("mode", restore.ModeAuto, "where the snapshot will restore, as up's --mode: of an index's "+ "entries, one this host can restore that way (auto, host or in-place)") sock := fs.String("docker", docker.DefaultSocket, "Docker socket") - tools := dmtcp.Tools{Dir: dmtcp.DefaultDir, Plugin: dmtcp.DefaultPlugin} + tools := dmtcp.DefaultTools() fs.StringVar(&tools.Dir, "dmtcp-dir", tools.Dir, "directory of DMTCP's programs, for the dmtcp backend") pos, err := positionals(fs, args) if err != nil { diff --git a/internal/dmtcp/SHA256SUMS b/internal/dmtcp/SHA256SUMS new file mode 100644 index 0000000..20faca5 --- /dev/null +++ b/internal/dmtcp/SHA256SUMS @@ -0,0 +1,3 @@ +496b54b3405ed69dce8d0c3562d644b698150c21a921b9abed3562814724d9eb dmtcp-v5.0.0-warmctl.1-linux-amd64-debian12.tar.gz +90cf76a7733fd2998a29df4e299d9337fdbd0a8c919ccd2db3219c1e263ffd21 dmtcp-v5.0.0-warmctl.1-linux-arm64-debian12.tar.gz +f639b50aa5ecb5b3115f36f67d8ed7ad2894010dcdff43ed4e5a6276f4eeb0b1 dmtcp-v5.0.0-warmctl.1-source.tar.gz diff --git a/internal/dmtcp/VERSION b/internal/dmtcp/VERSION new file mode 100644 index 0000000..9b13a7f --- /dev/null +++ b/internal/dmtcp/VERSION @@ -0,0 +1 @@ +v5.0.0-warmctl.1 diff --git a/internal/dmtcp/dmtcp.go b/internal/dmtcp/dmtcp.go index 4d926cf..068daa9 100644 --- a/internal/dmtcp/dmtcp.go +++ b/internal/dmtcp/dmtcp.go @@ -15,6 +15,7 @@ package dmtcp import ( "context" + _ "embed" "errors" "fmt" "io" @@ -22,6 +23,7 @@ import ( "os/exec" "path/filepath" "regexp" + "runtime" "sort" "strconv" "strings" @@ -37,6 +39,31 @@ const ( DefaultPlugin = "/opt/warmctl/dmtcp/lib/libdmtcp_fdkeep.so" ) +//go:embed VERSION +var bundledVersion string + +// DefaultTools uses the release's own runtime, never another build from PATH. +// Resolve the executable symlink before looking beside it (custom prefixes). +func DefaultTools() Tools { + self, _ := os.Executable() + return defaultTools(self, runtime.GOOS+"-"+runtime.GOARCH, strings.TrimSpace(bundledVersion)) +} + +func defaultTools(self, platform, version string) Tools { + legacy := Tools{Dir: DefaultDir, Plugin: DefaultPlugin} + if self == "" || version == "" { + return legacy + } + if resolved, err := filepath.EvalSymlinks(self); err == nil { + self = resolved + } + root := filepath.Join(filepath.Dir(self), "dmtcp", platform, version) + if info, err := os.Stat(root); err != nil || !info.IsDir() { + return legacy + } + return Tools{Dir: filepath.Join(root, "bin"), Plugin: filepath.Join(root, "lib", "libdmtcp_fdkeep.so")} +} + // Env is what an engine needs to run under DMTCP: // - fork with live OpenBLAS/OpenMP thread pools hangs the child (torch's // import runs ldconfig in a subprocess), so the pools get one thread; @@ -237,6 +264,7 @@ func (t Tools) RestartArgs(images []string) []string { func (t Tools) Restart(images []string, log io.Writer) (*exec.Cmd, error) { args := t.RestartArgs(images) cmd := exec.Command(t.path("dmtcp_restart"), args...) + cmd.Env = restartEnv(os.Environ(), runtime.GOMAXPROCS(0)) cmd.Stdout, cmd.Stderr = log, log if err := cmd.Start(); err != nil { return nil, err @@ -244,6 +272,39 @@ func (t Tools) Restart(images []string, log io.Writer) (*exec.Cmd, error) { return cmd, nil } +// restartEnv enables the fork's large-image reader only for restart. Go's +// GOMAXPROCS default respects container CPU quota; use a supported reader +// count no larger than that budget. Explicit environment settings, including +// an empty value, win so a caller can compare or disable either optimization. +// Older DMTCP builds ignore these variables. Private mmap remains opt-in: +// CUDA host-memory registration can make it much slower than copying. +func restartEnv(env []string, cpus int) []string { + readers := min(8, max(1, cpus)) + if cpus >= 16 { + readers = 16 + } + defaults := []string{ + "DMTCP_RESTART_HUGEPAGES=1", + "DMTCP_RESTART_READERS=" + strconv.Itoa(readers), + "DMTCP_RESTART_MMAP=0", + } + result := append([]string(nil), env...) + for _, setting := range defaults { + key, _, _ := strings.Cut(setting, "=") + found := false + for _, entry := range env { + if strings.HasPrefix(entry, key+"=") { + found = true + break + } + } + if !found { + result = append(result, setting) + } + } + return result +} + // Peer is a process of the computation. type Peer struct { Name string `json:"name"` diff --git a/internal/dmtcp/fdkeep/fdkeep.c b/internal/dmtcp/fdkeep/fdkeep.c deleted file mode 100644 index 47a2b96..0000000 --- a/internal/dmtcp/fdkeep/fdkeep.c +++ /dev/null @@ -1,513 +0,0 @@ -// DMTCP plugin: keep eventfds and pipes that DMTCP did not track. -// The CUDA driver creates the eventfds and pipes its helper threads poll during -// cuInit; DMTCP drops them on restart, the helper threads see POLLNVAL and exit, -// and cuda-checkpoint can no longer find the restore thread. Before a checkpoint -// we record every eventfd and pipe end; on restart we recreate the ones that are -// missing at the same fd numbers, with the same flags, counter and pipe bytes. -// Listening Unix sockets bound to a path are kept the same way: cuInterpose -// makes its control socket with raw syscalls, which DMTCP never sees. -// -// Some calls bypass libc with raw syscalls, where DMTCP's wrappers must not see them (its pipe() returns a -// socketpair, its listen() takes a lock the checkpoint thread holds): x86-64 only. -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "dmtcp.h" - -#if !defined(__x86_64__) -#error "fdkeep's raw syscalls are x86-64's" -#endif - -#define MAXFD 512 -#define MAXWATCH 4096 - -enum kind { NONE, EVENTFD, PIPE, LISTEN }; - -struct rec { - enum kind kind; - int flags; // file status flags from fdinfo (octal) - int cloexec; - unsigned long long count; // eventfd counter - unsigned long inode; // pipe inode, pairs read and write ends - int len; // pipe bytes pending (read end only) - char *data; - char *path; // a listening Unix socket's path - int type; // its socket type - mode_t mode; // and the path's permissions -}; - -static struct rec recs[MAXFD]; - -static void logf_(const char *fmt, ...) -{ - char buf[512]; - va_list ap; - va_start(ap, fmt); - int n = vsnprintf(buf, sizeof buf, fmt, ap); - va_end(ap); - int fd = open("/tmp/fdkeep.log", O_WRONLY | O_CREAT | O_APPEND | O_CLOEXEC, 0644); - if (fd >= 0) { - if (write(fd, buf, n) < 0) {} - close(fd); - } -} - -static int fdinfo(int fd, int *flags, unsigned long long *count) -{ - char path[64], line[256]; - snprintf(path, sizeof path, "/proc/self/fdinfo/%d", fd); - FILE *f = fopen(path, "re"); - if (!f) return -1; - while (fgets(line, sizeof line, f)) { - if (sscanf(line, "flags: %o", flags) == 1) continue; - sscanf(line, "eventfd-count: %llx", count); - } - fclose(f); - return 0; -} - -static int nwatches, lostwatches; -static void record_epoll(int epfd); - -// A listening Unix socket's path, type and permissions, kept in case DMTCP does not restore it. -static void record_unix(int fd, struct rec *r) -{ - int domain = 0; - socklen_t len = sizeof domain; - struct sockaddr_un a; - socklen_t alen = sizeof a; - struct stat st; - if (getsockopt(fd, SOL_SOCKET, SO_DOMAIN, &domain, &len) != 0 || domain != AF_UNIX) return; - len = sizeof r->type; - if (getsockopt(fd, SOL_SOCKET, SO_TYPE, &r->type, &len) != 0) return; - memset(&a, 0, sizeof a); - // An abstract name starts with a NUL and has no file to bind again. - if (getsockname(fd, (struct sockaddr *)&a, &alen) != 0 || a.sun_path[0] == 0) return; - a.sun_path[sizeof a.sun_path - 1] = 0; - if (lstat(a.sun_path, &st) != 0 || !S_ISSOCK(st.st_mode)) return; - r->path = strdup(a.sun_path); - r->mode = st.st_mode & 07777; -} - -static void record(void) -{ - for (int fd = 0; fd < MAXFD; fd++) { - free(recs[fd].data); - free(recs[fd].path); - } - memset(recs, 0, sizeof recs); - nwatches = lostwatches = 0; - int past = 0; // fds of a kind we keep, numbered past MAXFD: not kept - DIR *d = opendir("/proc/self/fd"); - if (!d) return; - int self = dirfd(d); - struct dirent *e; - while ((e = readdir(d))) { - int fd = atoi(e->d_name); - if (e->d_name[0] == '.' || fd == self) continue; - char path[64], target[128]; - snprintf(path, sizeof path, "/proc/self/fd/%d", fd); - ssize_t n = readlink(path, target, sizeof target - 1); - if (n <= 0) continue; - target[n] = 0; - if (fd >= MAXFD) { - // DMTCP's own descriptors (its coordinator's socket, a pipe: 821 and 828) are its to restore. - if (dmtcp_is_protected_fd(fd)) continue; - if (strncmp(target, "anon_inode:[event", 17) == 0 || strncmp(target, "pipe:", 5) == 0 || - strncmp(target, "socket:", 7) == 0) - past++; - continue; - } - struct rec *r = &recs[fd]; - if (strcmp(target, "anon_inode:[eventfd]") == 0) { - r->kind = EVENTFD; - } else if (sscanf(target, "pipe:[%lu]", &r->inode) == 1) { - r->kind = PIPE; - } else if (strcmp(target, "anon_inode:[eventpoll]") == 0) { - record_epoll(fd); - continue; - } else if (strncmp(target, "socket:", 7) == 0) { - // DMTCP sometimes misses listen() and restores the socket bound but not listening. - int on = 0; - socklen_t len = sizeof on; - if (getsockopt(fd, SOL_SOCKET, SO_ACCEPTCONN, &on, &len) != 0 || !on) continue; - r->kind = LISTEN; - record_unix(fd, r); - fdinfo(fd, &r->flags, &r->count); - r->cloexec = (fcntl(fd, F_GETFD) & FD_CLOEXEC) != 0; - continue; - } else { - continue; - } - fdinfo(fd, &r->flags, &r->count); - r->cloexec = (fcntl(fd, F_GETFD) & FD_CLOEXEC) != 0; - if (r->kind == PIPE && (r->flags & O_ACCMODE) == O_RDONLY) { - int avail = 0; - if (ioctl(fd, FIONREAD, &avail) == 0 && avail > 0) { - // Take the bytes out to save them, then put them back for the live process. - r->data = malloc(avail); - int fl = fcntl(fd, F_GETFL); - fcntl(fd, F_SETFL, fl | O_NONBLOCK); - r->len = read(fd, r->data, avail); - fcntl(fd, F_SETFL, fl); - if (r->len < 0) r->len = 0; - for (int w = 0; w < MAXFD && r->len > 0; w++) { - if (recs[w].kind == PIPE && recs[w].inode == r->inode && (recs[w].flags & O_ACCMODE) == O_WRONLY) { - if (write(w, r->data, r->len) < 0) {} - break; - } - } - } - } - } - closedir(d); - int ne = 0, np = 0, nl = 0; - for (int fd = 0; fd < MAXFD; fd++) { - if (recs[fd].kind == EVENTFD) ne++; - if (recs[fd].kind == PIPE) np++; - if (recs[fd].kind == LISTEN) nl++; - } - logf_("pid %d precheckpoint: %d eventfds, %d pipe ends, %d listening sockets, %d epoll watches\n", getpid(), ne, - np, nl, nwatches); - if (past > 0 || lostwatches > 0) - logf_("pid %d precheckpoint: NOT KEPT: %d eventfds, pipes, epolls or sockets past fd %d, %d epoll watches past %d\n", - getpid(), past, MAXFD, lostwatches, MAXWATCH); -} - -static long raw_pipe2(int *p, int flags) -{ - long ret; - __asm__ volatile("syscall" : "=a"(ret) : "0"((long)SYS_pipe2), "D"(p), "S"((long)flags) : "rcx", "r11", "memory"); - return ret; -} - -// Epoll watches. A forked child shares its parent's epoll fds and sockets, and DMTCP refills a shared -// epoll from the watch list of whichever process restores it: when that is the child, the watches the -// parent added after the fork (uvloop's listening socket in vLLM's API server) are gone, and the -// server never accepts. Record the kernel's own list, keeping only watches whose fd in this process -// is still the watched file, and add back whatever is missing after restart. -struct watch { - int epfd, tfd; - unsigned int events; - unsigned long long data; -}; -static struct watch watches[MAXWATCH]; - -static void record_epoll(int epfd) -{ - char path[64], line[256]; - snprintf(path, sizeof path, "/proc/self/fdinfo/%d", epfd); - FILE *f = fopen(path, "re"); - if (!f) return; - while (fgets(line, sizeof line, f)) { - int tfd; - unsigned int events; - unsigned long long data; - unsigned long ino; - if (sscanf(line, "tfd: %d events: %x data: %llx pos:%*d ino:%lx", &tfd, &events, &data, &ino) != 4) continue; - struct stat st; - if (fstat(tfd, &st) != 0 || st.st_ino != ino) continue; // registered through another process's fd table - if (nwatches == MAXWATCH) { - lostwatches++; - continue; - } - watches[nwatches++] = (struct watch){epfd, tfd, events, data}; - } - fclose(f); -} - -static long raw_epoll_ctl(int epfd, int op, int fd, struct epoll_event *ev) -{ - long ret; - register long r10 __asm__("r10") = (long)ev; - __asm__ volatile("syscall" : "=a"(ret) : "0"((long)SYS_epoll_ctl), "D"((long)epfd), "S"((long)op), "d"((long)fd), "r"(r10) : "rcx", "r11", "memory"); - return ret; -} - -static void rewatch(void) -{ - int added = 0, failed = 0; - for (int i = 0; i < nwatches; i++) { - struct epoll_event ev = {.events = watches[i].events, .data.u64 = watches[i].data}; - long r = raw_epoll_ctl(watches[i].epfd, EPOLL_CTL_ADD, watches[i].tfd, &ev); - if (r == 0) added++; - else if (r != -EEXIST) failed++; - } - logf_("pid %d restart: %d of %d epoll watches were missing and added back, %d failed\n", getpid(), added, nwatches, failed); -} - -// DMTCP's listen() wrapper takes a lock the checkpoint thread may hold while threads resume. -static long raw_listen(int fd, int backlog) -{ - long ret; - __asm__ volatile("syscall" : "=a"(ret) : "0"((long)SYS_listen), "D"((long)fd), "S"((long)backlog) : "rcx", "r11", "memory"); - return ret; -} - -static long raw3(long nr, long a, long b, long c) -{ - long ret; - __asm__ volatile("syscall" : "=a"(ret) : "0"(nr), "D"(a), "S"(b), "d"(c) : "rcx", "r11", "memory"); - return ret; -} - -// Move a fresh fd above every slot we may fill, so placing one never clobbers another. -static int high(int fd) -{ - int h = fcntl(fd, F_DUPFD_CLOEXEC, MAXFD + 100); - close(fd); - return h; -} - -static void place(int src, int fd, const struct rec *r) -{ - dup2(src, fd); - fcntl(fd, F_SETFL, r->flags & (O_NONBLOCK | O_APPEND)); - fcntl(fd, F_SETFD, r->cloexec ? FD_CLOEXEC : 0); -} - -// Is fd open and of the recorded kind? DMTCP restores the pipes it tracked as -// socketpairs, and cuda-checkpoint wakes the restore thread by opening -// /proc/PID/fd/N of its pipe, which fails with ENXIO on a socket. -static int same_kind(int fd, enum kind kind) -{ - char path[64], target[128]; - snprintf(path, sizeof path, "/proc/self/fd/%d", fd); - ssize_t n = readlink(path, target, sizeof target - 1); - if (n <= 0) return 0; - target[n] = 0; - if (kind == EVENTFD) return strcmp(target, "anon_inode:[eventfd]") == 0; - return strncmp(target, "pipe:", 5) == 0; -} - -// Bind a Unix listener DMTCP did not restore again at its path, as its process made it (cuInterpose: -// nonblocking, the path 0600 before listen). The path left by the process checkpointed is removed only -// when it is a socket of ours that nothing listens on. -static int relisten(int fd, const struct rec *r) -{ - struct sockaddr_un a = {.sun_family = AF_UNIX}; - struct stat st; - strncpy(a.sun_path, r->path, sizeof a.sun_path - 1); - if (lstat(r->path, &st) == 0) { - long probe = raw3(SYS_socket, AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0); - long c = probe < 0 ? probe : raw3(SYS_connect, probe, (long)&a, sizeof a); - if (probe >= 0) close((int)probe); - if (!S_ISSOCK(st.st_mode) || st.st_uid != geteuid() || c != -ECONNREFUSED) return -1; - if (unlink(r->path) != 0) return -1; - } - long s = raw3(SYS_socket, AF_UNIX, r->type | SOCK_CLOEXEC, 0); - if (s < 0) return -1; - int h = high((int)s); - if (h < 0 || raw3(SYS_bind, h, (long)&a, sizeof a) != 0 || chmod(r->path, r->mode) != 0 || - raw_listen(h, SOMAXCONN) != 0) { - if (h >= 0) close(h); - return -1; - } - place(h, fd, r); - close(h); - return 0; -} - -static void recreate(void) -{ - int made = 0, kept = 0; - // A pipe is redone whole when either end is missing or not a pipe. - for (int fd = 0; fd < MAXFD; fd++) { - struct rec *r = &recs[fd]; - if (r->kind != PIPE || r->inode == 0 || same_kind(fd, PIPE)) continue; - for (int g = 0; g < MAXFD; g++) { - if (recs[g].kind == PIPE && recs[g].inode == r->inode && fcntl(g, F_GETFD) != -1) { - close(g); - } - } - } - for (int fd = 0; fd < MAXFD; fd++) { - struct rec *r = &recs[fd]; - if (r->kind == NONE) continue; - if (r->kind == LISTEN) { - int on = 0; - socklen_t len = sizeof on; - if (getsockopt(fd, SOL_SOCKET, SO_ACCEPTCONN, &on, &len) == 0) { - if (!on) { - logf_(" fd %d was listening, restored without listen: listen -> %ld\n", fd, raw_listen(fd, 4096)); - made++; - } - } else if (r->path && fcntl(fd, F_GETFD) == -1) { - int ok = relisten(fd, r); - logf_(" fd %d listened on %s, not restored: %s\n", fd, r->path, ok == 0 ? "bound again" : "FAILED"); - if (ok == 0) made++; - } - continue; - } - if (fcntl(fd, F_GETFD) != -1) { // DMTCP restored it, or the slot is taken - if (r->kind == EVENTFD || r->inode != 0) kept++; - continue; - } - if (r->kind == EVENTFD) { - int e = high(eventfd(r->count, 0)); - place(e, fd, r); - close(e); - made++; - continue; - } - // Pipe: create once per inode, then place both ends wherever they lived. - if (r->inode == 0) continue; - unsigned long ino = r->inode; - int p[2]; - // DMTCP's pipe() and syscall() wrappers return a socketpair; we need a real pipe. - if (raw_pipe2(p, 0) != 0) continue; - p[0] = high(p[0]); - p[1] = high(p[1]); - for (int g = fd; g < MAXFD; g++) { - struct rec *q = &recs[g]; - if (q->kind != PIPE || q->inode != ino || fcntl(g, F_GETFD) != -1) continue; - int rd = (q->flags & O_ACCMODE) == O_RDONLY; - if (rd && q->len > 0 && write(p[1], q->data, q->len) < 0) {} - place(rd ? p[0] : p[1], g, q); - q->inode = 0; // done - made++; - } - close(p[0]); - close(p[1]); - } - logf_("pid %d restart: recreated %d fds, %d already open\n", getpid(), made, kept); - for (int fd = 0; fd < MAXFD; fd++) { - if (recs[fd].kind == PIPE && !same_kind(fd, PIPE)) logf_(" fd %d is not a pipe after fixup\n", fd); - } -} - -// Thread names. DMTCP restarts every thread as "DMTCP:" cut to 15 bytes, -// and cuda-checkpoint finds the driver's restore thread by its name -// (cuda). Save the names before a checkpoint and put them back after. -#define MAXTHREADS 256 -static char names[MAXTHREADS][16]; -static int nnames; - -static int readcomm(const char *tid, char *out) -{ - char path[64]; - snprintf(path, sizeof path, "/proc/self/task/%s/comm", tid); - int fd = open(path, O_RDONLY | O_CLOEXEC); - if (fd < 0) return -1; - int n = read(fd, out, 15); - close(fd); - if (n <= 0) return -1; - out[n] = 0; - out[strcspn(out, "\n")] = 0; - return 0; -} - -static void savenames(void) -{ - nnames = 0; - int past = 0; - DIR *d = opendir("/proc/self/task"); - if (!d) return; - struct dirent *e; - while ((e = readdir(d))) { - if (e->d_name[0] == '.') continue; - if (nnames == MAXTHREADS) { - past++; - continue; - } - if (readcomm(e->d_name, names[nnames]) == 0) nnames++; - } - closedir(d); - if (past > 0) logf_("pid %d precheckpoint: NOT KEPT: the names of %d threads past %d\n", getpid(), past, MAXTHREADS); -} - -static void restorenames(void) -{ - char used[MAXTHREADS] = {0}; - int fixed = 0; - DIR *d = opendir("/proc/self/task"); - if (!d) return; - struct dirent *e; - while ((e = readdir(d))) { - char cur[16]; - if (e->d_name[0] == '.' || readcomm(e->d_name, cur) != 0) continue; - if (strncmp(cur, "DMTCP:", 6) != 0) continue; - const char *stem = cur + 6; - for (int i = 0; i < nnames; i++) { - if (used[i] || strncmp(names[i], stem, strlen(stem)) != 0) continue; - used[i] = 1; - char path[64]; - snprintf(path, sizeof path, "/proc/self/task/%.20s/comm", e->d_name); // a TID: digits - int fd = open(path, O_WRONLY | O_CLOEXEC); - if (fd >= 0) { - if (write(fd, names[i], strlen(names[i])) > 0) fixed++; - close(fd); - } - break; - } - } - closedir(d); - logf_("pid %d restart: renamed %d of %d threads\n", getpid(), fixed, nnames); -} - -// RESTART runs plugins in reverse order, so ours runs before DMTCP's file and -// socket plugins, which then dup their socketpairs over our pipes. THREAD_RESUME -// comes after them and before any thread returns to application code: the first -// thread there does the fixup, the rest wait for it. -static volatile int restarted, fixing, fixed; - -static void hook(DmtcpEvent_t event, DmtcpEventData_t *data) -{ - (void)data; - switch (event) { - case DMTCP_EVENT_INIT: - case DMTCP_EVENT_ATFORK_CHILD: - // Yama ptrace_scope=1 lets only ancestors attach; with FDKEEP_PTRACER_ANY set, gdb can attach to a - // process stuck in a checkpoint from any shell. - if (getenv("FDKEEP_PTRACER_ANY")) prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY, 0, 0, 0); - break; - case DMTCP_EVENT_PRECHECKPOINT: - record(); - savenames(); - break; - case DMTCP_EVENT_RESTART: - fixing = fixed = 0; - __atomic_store_n(&restarted, 1, __ATOMIC_SEQ_CST); - break; - case DMTCP_EVENT_THREAD_RESUME: - if (!__atomic_load_n(&restarted, __ATOMIC_SEQ_CST)) break; - if (__atomic_exchange_n(&fixing, 1, __ATOMIC_SEQ_CST) == 0) { - recreate(); - rewatch(); - restorenames(); - __atomic_store_n(&restarted, 0, __ATOMIC_SEQ_CST); - __atomic_store_n(&fixed, 1, __ATOMIC_SEQ_CST); - } else { - while (!__atomic_load_n(&fixed, __ATOMIC_SEQ_CST)) usleep(100); - } - break; - default: - break; - } -} - -DmtcpPluginDescriptor_t fdkeep_plugin = { - DMTCP_PLUGIN_API_VERSION, - DMTCP_PACKAGE_VERSION, - "fdkeep", - "warmctl", - "", - "Keep what DMTCP loses: CUDA's eventfds and pipes, thread names, listening sockets, epoll watches", - hook -}; - -DMTCP_DECL_PLUGIN(fdkeep_plugin); diff --git a/internal/dmtcp/restart_env_test.go b/internal/dmtcp/restart_env_test.go new file mode 100644 index 0000000..4e35503 --- /dev/null +++ b/internal/dmtcp/restart_env_test.go @@ -0,0 +1,86 @@ +package dmtcp + +import ( + "io" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +func TestRestartEnv(t *testing.T) { + for _, tc := range []struct{ cpus, readers int }{ + {0, 1}, {1, 1}, {2, 2}, {4, 4}, {8, 8}, {9, 8}, {15, 8}, {16, 16}, {24, 16}, + } { + t.Run(strconv.Itoa(tc.cpus), func(t *testing.T) { + got := strings.Join(restartEnv([]string{"UNCHANGED=yes"}, tc.cpus), "\n") + want := "UNCHANGED=yes\nDMTCP_RESTART_HUGEPAGES=1\nDMTCP_RESTART_READERS=" + + strconv.Itoa(tc.readers) + "\nDMTCP_RESTART_MMAP=0" + if got != want { + t.Fatalf("got %q, want %q", got, want) + } + }) + } + explicit := []string{"DMTCP_RESTART_HUGEPAGES=0", "DMTCP_RESTART_READERS=1", "DMTCP_RESTART_MMAP="} + if got := strings.Join(restartEnv(explicit, 32), "\n"); got != strings.Join(explicit, "\n") { + t.Fatalf("overrode explicit settings: %q", got) + } + prefixed := restartEnv([]string{"DMTCP_RESTART_READERS_EXTRA=1"}, 16) + if !strings.Contains(strings.Join(prefixed, "\n"), "DMTCP_RESTART_READERS=16") { + t.Fatal("mistook a variable prefix for an override") + } +} + +func TestRestartPassesReaderEnvironment(t *testing.T) { + dir := t.TempDir() + output := filepath.Join(dir, "environment") + script := "#!/bin/sh\nprintf '%s\\n' \"$DMTCP_RESTART_HUGEPAGES\" \"$DMTCP_RESTART_READERS\" \"$DMTCP_RESTART_MMAP\" > \"$RESTART_ENV_OUTPUT\"\n" + if err := os.WriteFile(filepath.Join(dir, "dmtcp_restart"), []byte(script), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("RESTART_ENV_OUTPUT", output) + t.Setenv("DMTCP_RESTART_HUGEPAGES", "0") + t.Setenv("DMTCP_RESTART_READERS", "4") + t.Setenv("DMTCP_RESTART_MMAP", "0") + cmd, err := (Tools{Dir: dir}).Restart([]string{"unused.dmtcp"}, io.Discard) + if err != nil { + t.Fatal(err) + } + if err := cmd.Wait(); err != nil { + t.Fatal(err) + } + got, err := os.ReadFile(output) + if err != nil || string(got) != "0\n4\n0\n" { + t.Fatalf("restart environment = %q, %v", got, err) + } +} + +func TestDefaultToolsBundle(t *testing.T) { + prefix := t.TempDir() + self := filepath.Join(prefix, "warmctl") + if err := os.WriteFile(self, nil, 0o755); err != nil { + t.Fatal(err) + } + link := filepath.Join(t.TempDir(), "warmctl") + if err := os.Symlink(self, link); err != nil { + t.Fatal(err) + } + prefix, err := filepath.EvalSymlinks(prefix) + if err != nil { + t.Fatal(err) + } + root := filepath.Join(prefix, "dmtcp", "linux-amd64", "v5.0.0-warmctl.1") + if err := os.MkdirAll(root, 0o755); err != nil { + t.Fatal(err) + } + for _, path := range []string{self, link} { + tools := defaultTools(path, "linux-amd64", "v5.0.0-warmctl.1") + if tools.Dir != filepath.Join(root, "bin") || tools.Plugin != filepath.Join(root, "lib", "libdmtcp_fdkeep.so") { + t.Fatalf("wrong bundle for %s: %+v", path, tools) + } + } + if tools := defaultTools(self, "linux-amd64", "missing"); tools.Dir != DefaultDir || tools.Plugin != DefaultPlugin { + t.Fatalf("source build lost runtime image defaults: %+v", tools) + } +} diff --git a/scripts/fetch-dmtcp.sh b/scripts/fetch-dmtcp.sh new file mode 100755 index 0000000..1048e83 --- /dev/null +++ b/scripts/fetch-dmtcp.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# Fetch a released native runtime with the checksum pinned in this source tree. +# Usage: scripts/fetch-dmtcp.sh amd64|arm64 NEW_DESTINATION +set -eu +repo=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +arch=${1:?usage: fetch-dmtcp.sh amd64|arm64 NEW_DESTINATION} +dest=${2:?usage: fetch-dmtcp.sh amd64|arm64 NEW_DESTINATION} +case "$arch" in amd64|arm64) ;; *) echo 'unsupported DMTCP architecture' >&2; exit 1 ;; esac +[ ! -e "$dest" ] || { echo "refusing to overwrite $dest" >&2; exit 1; } +version=$(cat "$repo/internal/dmtcp/VERSION") +asset=dmtcp-$version-linux-$arch-debian12.tar.gz +expected=$(awk -v name="$asset" '$2 == name { print $1 }' "$repo/internal/dmtcp/SHA256SUMS") +[ ${#expected} -eq 64 ] || { echo "missing pinned checksum for $asset" >&2; exit 1; } +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +trap 'exit 1' HUP INT TERM +curl --proto '=https' --tlsv1.2 -fsSL --retry 3 \ + "https://github.com/TensorRails/dmtcp/releases/download/$version/$asset" -o "$tmp/$asset" +if command -v sha256sum >/dev/null 2>&1; then + printf '%s %s\n' "$expected" "$tmp/$asset" | sha256sum -c - +else + printf '%s %s\n' "$expected" "$tmp/$asset" | shasum -a 256 -c - +fi +mkdir "$tmp/runtime" +tar -xzf "$tmp/$asset" --strip-components=1 -C "$tmp/runtime" +test -x "$tmp/runtime/bin/dmtcp_launch" +test -f "$tmp/runtime/lib/libdmtcp_fdkeep.so" +grep -Fqx "release=$version" "$tmp/runtime/BUILDINFO" +grep -Fqx "arch=$arch" "$tmp/runtime/BUILDINFO" +mkdir -p "$(dirname "$dest")" +mv "$tmp/runtime" "$dest" diff --git a/scripts/install.sh b/scripts/install.sh index a9c10b0..0ed6e9a 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -3,8 +3,9 @@ # # curl -fsSL https://warmctl.io/install.sh | sh # -# It puts the release in /opt/warmctl (warmctl and, on linux-amd64, -# cuinterpose/ next to it) and links /usr/local/bin/warmctl to it. Run it +# It puts the release in /opt/warmctl (warmctl, versioned Linux DMTCP +# runtimes and, on linux-amd64, cuinterpose/) and links /usr/local/bin/warmctl +# to it. Run it # again to upgrade. Settings, from the environment: # # WARMCTL_VERSION release tag, e.g. v2.0.0-beta.5 (default: the newest, @@ -174,6 +175,35 @@ main() { # Some archives carry cuinterpose/ as 0700; it is read by everyone. chmod -R a+rX "$tmp/x" + # DMTCP libraries are mapped by baked engines. Keep each released runtime + # immutable at its versioned path, including mtimes, and retain older ones. + # Never overwrite a build still referenced by an existing snapshot. + if [ -d "$tmp/x/dmtcp" ]; then + for platform_dir in "$tmp/x/dmtcp/"*; do + [ -d "$platform_dir" ] || continue + component_platform=${platform_dir##*/} + for component in "$platform_dir/"*; do + [ -d "$component" ] || continue + component_version=${component##*/} + target=$prefix/dmtcp/$component_platform/$component_version + as_root mkdir -p "$(dirname "$target")" + if [ -e "$target" ]; then + as_root diff -qr "$component" "$target" >/dev/null || + die "existing DMTCP $component_version differs; preserving it for existing snapshots" + else + stage=$(as_root mktemp -d "$(dirname "$target")/.install.XXXXXXXX") + as_root cp -pR "$component/." "$stage/" + # Linux packages only. -n also protects against another installer. + as_root mv -T -n "$stage" "$target" + if [ -d "$stage" ]; then + as_root rm -rf "$stage" + as_root diff -qr "$component" "$target" >/dev/null || die "concurrent DMTCP install differs" + fi + fi + done + done + fi + # Every file goes in under a temporary name and is renamed over the old # one, so a running warmctl keeps its binary and a failed install leaves # the old release whole. diff --git a/scripts/test-install.py b/scripts/test-install.py new file mode 100644 index 0000000..a4ee2e9 --- /dev/null +++ b/scripts/test-install.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Exercise the real installer with local downloads; no network/root writes. + +INSTALL_TEST_ARCHIVE may point to the native release archive. Otherwise a +small fixture covers immutable component paths and mtime preservation. +""" +import hashlib +import os +from pathlib import Path +import platform +import shutil +import subprocess +import sys +import tarfile +import tempfile +import unittest + + +@unittest.skipUnless(sys.platform == 'linux', 'Linux runtime installer') +class InstallTest(unittest.TestCase): + def test_versioned_runtime(self): + with tempfile.TemporaryDirectory(prefix='warmctl-install-test-') as tmp: + root = Path(tmp) + downloads, commands, prefix = [root / n for n in ('downloads', 'commands', 'prefix')] + downloads.mkdir() + commands.mkdir() + arch = {'x86_64': 'amd64', 'aarch64': 'arm64'}[platform.machine()] + version = 'v0.0.0-installer-test' + name = f'warmctl-{version}' + archive = downloads / f'{name}-linux-{arch}.tar.gz' + supplied = os.environ.get('INSTALL_TEST_ARCHIVE') + if supplied: + shutil.copyfile(supplied, archive) + else: + fixture = root / 'fixture' + fixture.mkdir() + binary = fixture / 'warmctl' + binary.write_text('#!/bin/sh\necho warmctl-test\n') + binary.chmod(0o755) + component = fixture / 'dmtcp' / f'linux-{arch}' / 'v5.0.0-warmctl.1' + component.mkdir(parents=True) + (component / 'BUILDINFO').write_text('release=test\n') + os.utime(component / 'BUILDINFO', ns=(1700000000000000000,) * 2) + with tarfile.open(archive, 'w:gz') as tar: + for child in fixture.iterdir(): + tar.add(child, arcname=child.name) + with tarfile.open(archive) as tar: + markers = [m for m in tar.getmembers() if m.name.endswith('/BUILDINFO')] + self.assertEqual(len(markers), 1) + archive_mtime = markers[0].mtime + digest = hashlib.sha256(archive.read_bytes()).hexdigest() + (downloads / f'{name}-checksums.txt').write_text(f'{digest} {archive.name}\n') + # Use actual filesystem tools and SHA-256; only HTTPS transport is + # mocked. No cosign in this PATH, exercising its optional absence. + for command in ('uname', 'id', 'mktemp', 'rm', 'mkdir', 'tar', 'chmod', + 'cp', 'mv', 'ln', 'dirname', 'diff', 'sha256sum', 'cut', + 'awk', 'sed', 'head', 'gzip'): + os.symlink(shutil.which(command), commands / command) + curl = commands / 'curl' + curl.write_text(f'''#!{sys.executable} +import os,sys,shutil +from pathlib import Path +from urllib.parse import urlparse +args=sys.argv[1:] +url=next(a for a in args if a.startswith('https://')) +source=Path(os.environ['INSTALL_TEST_DOWNLOADS'])/Path(urlparse(url).path).name +if not source.is_file(): sys.exit(22) +if any(a.startswith('-') and not a.startswith('--') and 'I' in a for a in args): sys.exit(0) +out=args[args.index('-o')+1] +if out=='-': sys.stdout.buffer.write(source.read_bytes()) +else: shutil.copyfile(source,out) +''') + curl.chmod(0o755) + env = dict(os.environ, PATH=str(commands), WARMCTL_VERSION=version, + WARMCTL_PREFIX=str(prefix), WARMCTL_BIN_DIR='', + INSTALL_TEST_DOWNLOADS=str(downloads)) + installer = Path(__file__).resolve().with_name('install.sh') + + def install(): + return subprocess.run(['/bin/sh', str(installer)], env=env, + text=True, capture_output=True, timeout=60) + + first = install() + self.assertEqual(first.returncode, 0, first.stderr) + components = list((prefix / 'dmtcp' / f'linux-{arch}').glob('*/BUILDINFO')) + self.assertEqual(len(components), 1) + marker = components[0] + self.assertAlmostEqual(marker.stat().st_mtime, archive_mtime, places=5) + before = (marker.read_bytes(), marker.stat().st_mtime_ns, marker.stat().st_ino) + old = marker.parent.parent / 'v0-old-snapshot' + old.mkdir() + (old / 'library').write_text('old snapshot library') + again = install() + self.assertEqual(again.returncode, 0, again.stderr) + self.assertEqual(before, (marker.read_bytes(), marker.stat().st_mtime_ns, marker.stat().st_ino)) + self.assertEqual((old / 'library').read_text(), 'old snapshot library') + marker.write_text('different installed build') + refused = install() + self.assertNotEqual(refused.returncode, 0) + self.assertIn('preserving it for existing snapshots', refused.stderr) + self.assertEqual(marker.read_text(), 'different installed build') + self.assertTrue((prefix / 'warmctl').is_file()) + + +if __name__ == '__main__': + unittest.main()