Bundle the DMTCP fork release and accelerate large-memory restores

.github/workflows/ci.yml

@@ -28,6 +28,7 @@ jobs:
           go-version: "1.26.x"
       - run: make test
       - run: make profiles-check
+      - run: python3 scripts/test-install.py
       # Every platform a release ships must keep compiling.
       - name: Cross-compile
         if: runner.arch == 'X64'
@@ -66,7 +67,7 @@ jobs:
         with:
           src: python
       # warmctl.io/install.sh runs under any POSIX sh: dash, busybox, macOS.
-      - run: shellcheck -s sh scripts/install.sh
+      - run: shellcheck -s sh scripts/install.sh scripts/fetch-dmtcp.sh
 
   vulncheck:
     runs-on: ubuntu-24.04

.github/workflows/release.yml

@@ -11,6 +11,10 @@ on:
       - .goreleaser.yaml
       - .github/workflows/release.yml
       - scripts/release-notes.sh
+      - scripts/fetch-dmtcp.sh
+      - scripts/install.sh
+      - scripts/test-install.py
+      - internal/dmtcp/**
       - Makefile
 
 permissions:
@@ -63,6 +67,11 @@ jobs:
           name: cuinterpose-linux-amd64
           path: ${{ runner.temp }}
       - run: mkdir -p bin && tar -xf "$RUNNER_TEMP/cuinterpose-linux-amd64.tar" -C bin && ls -l bin/cuinterpose
+      - name: Fetch the pinned DMTCP release packages
+        run: |
+          version=$(cat internal/dmtcp/VERSION)
+          scripts/fetch-dmtcp.sh amd64 "dmtcp/linux-amd64/$version"
+          scripts/fetch-dmtcp.sh arm64 "dmtcp/linux-arm64/$version"
       - uses: anchore/sbom-action/download-syft@66cbf4bc1f1c0d2edc94016e65bc221b6bb0ad6c # v0.24.3
         with:
           syft-version: ${{ env.SYFT_VERSION }}
@@ -76,12 +85,12 @@ jobs:
           TAG: ${{ startsWith(github.ref, 'refs/tags/') && github.ref_name || 'v2.0.0-beta.5' }}
         run: scripts/release-notes.sh "$TAG" > "$RUNNER_TEMP/notes.md"
 
-      - name: Publish
+      - name: Build signed release archives
         if: startsWith(github.ref, 'refs/tags/')
         uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7.3.0
         with:
           version: ${{ env.GORELEASER_VERSION }}
-          args: release --clean --release-notes ${{ runner.temp }}/notes.md
+          args: release --clean --skip=publish --release-notes ${{ runner.temp }}/notes.md
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Provenance
@@ -104,11 +113,83 @@ jobs:
           tar -xzf dist/warmctl-*-linux-amd64.tar.gz -C "$RUNNER_TEMP" warmctl cuinterpose/libcuinterpose.so
           "$RUNNER_TEMP/warmctl" version
           test -x "$RUNNER_TEMP/warmctl"
-      - if: "!startsWith(github.ref, 'refs/tags/')"
-        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
         with:
-          name: snapshot
+          name: release-packages
           path: |
             dist/*.tar.gz
             dist/*checksums.txt
+            dist/*.sigstore.json
+            dist/*.sbom.json
           retention-days: 7
+
+  verify-native:
+    needs: release
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - runner: ubuntu-24.04
+            arch: amd64
+          - runner: ubuntu-24.04-arm
+            arch: arm64
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        with:
+          name: release-packages
+          path: packages
+      - name: Check the actual native archive and plugin
+        run: |
+          mkdir unpacked
+          tar -xzf packages/*-linux-${{ matrix.arch }}.tar.gz -C unpacked
+          unpacked/warmctl version
+          version=$(cat internal/dmtcp/VERSION)
+          root="$PWD/unpacked/dmtcp/linux-${{ matrix.arch }}/$version"
+          "$root/bin/dmtcp_launch" --version
+          "$root/bin/dmtcp_coordinator" --daemon --exit-on-last --port 0 --port-file "$RUNNER_TEMP/dmtcp.port"
+          port=$(cat "$RUNNER_TEMP/dmtcp.port")
+          trap '"$root/bin/dmtcp_command" --coord-port "$port" --quit || true' EXIT
+          DMTCP_PLUGIN="$root/lib/libdmtcp_fdkeep.so" "$root/bin/dmtcp_launch" --join-coordinator --coord-port "$port" /bin/true
+      - name: Checkpoint and restore with the packaged runtime on Ubuntu 24.04
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          version=$(cat internal/dmtcp/VERSION)
+          mkdir "$RUNNER_TEMP/dmtcp-source"
+          gh release download "$version" --repo TensorRails/dmtcp --pattern "dmtcp-$version-source.tar.gz" --dir "$RUNNER_TEMP/dmtcp-source"
+          (cd "$RUNNER_TEMP/dmtcp-source" && sha256sum --ignore-missing -c "$GITHUB_WORKSPACE/internal/dmtcp/SHA256SUMS")
+          tar -xzf "$RUNNER_TEMP/dmtcp-source/dmtcp-$version-source.tar.gz" --strip-components=1 -C "$RUNNER_TEMP/dmtcp-source"
+          root="$PWD/unpacked/dmtcp/linux-${{ matrix.arch }}/$version"
+          # Use the generated version header from the installed package.
+          CPATH="$root/include" python3 "$RUNNER_TEMP/dmtcp-source/test/restore-memory-check.py" --root "$root" \
+            --plugin "$root/lib/libdmtcp_fdkeep.so" --output "$RUNNER_TEMP/dmtcp-installed-check"
+      - name: Install, reinstall, preserve old snapshots and refuse replacement
+        run: INSTALL_TEST_ARCHIVE=$(echo "$PWD"/packages/*-linux-${{ matrix.arch }}.tar.gz) python3 scripts/test-install.py
+
+  publish:
+    if: startsWith(github.ref, 'refs/tags/')
+    needs: verify-native
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        with:
+          name: release-packages
+          path: packages
+      - name: Publish verified archives
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ github.ref_name }}
+        run: |
+          scripts/release-notes.sh "$TAG" > "$RUNNER_TEMP/notes.md"
+          flags=()
+          case "$TAG" in *-*) flags+=(--prerelease) ;; esac
+          gh release create "$TAG" packages/* --verify-tag --title "$TAG" --notes-file "$RUNNER_TEMP/notes.md" "${flags[@]}"

.github/workflows/runtime-image.yml

@@ -0,0 +1,112 @@
+name: runtime image
+
+# Publish only an already released CLI. GPU cold-start qualification follows
+# separately; this workflow deliberately does not move the latest image tag.
+on:
+  workflow_dispatch:
+    inputs:
+      release:
+        description: Published warmctl release tag
+        required: true
+      image_tag:
+        description: Optional image tag (defaults to the release tag)
+        required: false
+      base:
+        description: Optional pinned base image (defaults to the Dockerfile base)
+        required: false
+
+permissions:
+  contents: read
+  attestations: read
+
+concurrency:
+  group: runtime-image-${{ inputs.image_tag || inputs.release }}
+  cancel-in-progress: false
+
+jobs:
+  publish:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    env:
+      RELEASE: ${{ inputs.release }}
+      IMAGE_TAG: ${{ inputs.image_tag || inputs.release }}
+      BASE_IMAGE: ${{ inputs.base }}
+      GH_TOKEN: ${{ github.token }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ inputs.release }}
+          persist-credentials: false
+      - name: Validate release and image tag
+        run: |
+          [[ "$RELEASE" =~ ^v[0-9][0-9A-Za-z._-]*$ ]]
+          [[ "$IMAGE_TAG" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]*$ ]]
+          gh release view "$RELEASE" --repo TensorRails/warmctl
+          if [ -n "$BASE_IMAGE" ]; then [[ "$BASE_IMAGE" == *@sha256:* ]]; fi
+      - name: Make room for the vLLM image
+        run: |
+          sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /opt/hostedtoolcache
+          df -h /
+      - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
+      - name: Fetch and verify the released cuInterpose bundle
+        run: |
+          mkdir -p "$RUNNER_TEMP/release" bin
+          gh release download "$RELEASE" --repo TensorRails/warmctl --dir "$RUNNER_TEMP/release" \
+            --pattern "warmctl-$RELEASE-linux-amd64.tar.gz" --pattern "warmctl-$RELEASE-checksums.txt"
+          (cd "$RUNNER_TEMP/release" && sha256sum --ignore-missing -c "warmctl-$RELEASE-checksums.txt")
+          gh attestation verify "$RUNNER_TEMP/release/warmctl-$RELEASE-linux-amd64.tar.gz" --repo TensorRails/warmctl
+          tar -xzf "$RUNNER_TEMP/release/warmctl-$RELEASE-linux-amd64.tar.gz" -C bin cuinterpose
+      - name: Build the released runtime
+        run: |
+          args=()
+          if [ -n "$BASE_IMAGE" ]; then args+=(--build-arg "BASE=$BASE_IMAGE"); fi
+          docker build -t "localhost:5000/warmctl/runtime:$IMAGE_TAG" \
+            -f deploy/kubernetes/Dockerfile --build-context cuinterpose=bin/cuinterpose \
+            --build-arg "VERSION=$RELEASE" "${args[@]}" .
+      - name: Verify runtime programs and native plugin
+        run: |
+          docker run --rm --entrypoint /bin/bash "localhost:5000/warmctl/runtime:$IMAGE_TAG" -lc '
+            set -e
+            warmctl version
+            cat /opt/warmctl/RUNTIME /opt/warmctl/dmtcp/BUILDINFO
+            python3 -c "import importlib.metadata as m; print(\"vllm\", m.version(\"vllm\"))"
+            root=/opt/warmctl/dmtcp
+            "$root/bin/dmtcp_coordinator" --daemon --exit-on-last --port 0 --port-file /tmp/dmtcp.port
+            port=$(cat /tmp/dmtcp.port)
+            DMTCP_PLUGIN="$root/lib/libdmtcp_fdkeep.so" "$root/bin/dmtcp_launch" --join-coordinator --coord-port "$port" /bin/true
+          '
+      - name: Install the pinned registry client
+        run: |
+          curl -fsSL --retry 3 https://github.com/regclient/regclient/releases/download/v0.11.7/regctl-linux-amd64 -o "$RUNNER_TEMP/regctl"
+          echo "3737fbbdd791e04e7a831640b681f3397a376498e80e2ad9b1b2d27296f239bc  $RUNNER_TEMP/regctl" | sha256sum -c -
+          chmod 0755 "$RUNNER_TEMP/regctl"
+      - name: Publish with bounded registry chunks
+        env:
+          WARMCTL_REGISTRY_AUTH: ${{ secrets.WARMCTL_REGISTRY_AUTH }}
+        run: |
+          python3 - <<'PY'
+          import json, os
+          from pathlib import Path
+          config = json.loads(os.environ['WARMCTL_REGISTRY_AUTH'])
+          assert set(config['auths']) == {'registry.warmctl.io'}
+          path = Path.home() / '.docker/config.json'
+          path.parent.mkdir(parents=True, exist_ok=True)
+          path.write_text(json.dumps(config))
+          path.chmod(0o600)
+          PY
+          docker run -d --name release-registry -p 127.0.0.1:5000:5000 registry:2
+          docker push "localhost:5000/warmctl/runtime:$IMAGE_TAG"
+          "$RUNNER_TEMP/regctl" registry set localhost:5000 --tls disabled
+          "$RUNNER_TEMP/regctl" registry set registry.warmctl.io --blob-chunk 67108864 --blob-max 67108864
+          "$RUNNER_TEMP/regctl" image copy "localhost:5000/warmctl/runtime:$IMAGE_TAG" "registry.warmctl.io/warmctl/runtime:$IMAGE_TAG"
+          digest=$("$RUNNER_TEMP/regctl" image digest "registry.warmctl.io/warmctl/runtime:$IMAGE_TAG")
+          printf 'registry.warmctl.io/warmctl/runtime:%s@%s\n' "$IMAGE_TAG" "$digest" | tee runtime-image.txt
+          cat runtime-image.txt >> "$GITHUB_STEP_SUMMARY"
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: runtime-image
+          path: runtime-image.txt
+          if-no-files-found: error
+      - name: Remove credentials
+        if: always()
+        run: rm -f "$HOME/.docker/config.json"

.gitignore

@@ -7,3 +7,4 @@ __pycache__/
 .venv/
 *.test
 .cloudflare/
+/dmtcp/

.goreleaser.yaml

@@ -12,15 +12,25 @@ builds:
     goos: [linux]
     goarch: [amd64]
     flags: [-trimpath]
-    ldflags: ['-s -w -X main.version={{ if .IsSnapshot }}{{ .Tag }}-dev+{{ .ShortCommit }}{{ else }}{{ .Tag }}{{ end }}']
+    ldflags:
+      - '-s -w -X main.version={{ if .IsSnapshot }}{{ .Tag }}-dev+{{ .ShortCommit }}{{ else }}{{ .Tag }}{{ end }}'
+    mod_timestamp: "{{ .CommitTimestamp }}"
+  - id: linux-arm64
+    main: ./cmd/warmctl
+    binary: warmctl
+    env: [CGO_ENABLED=0]
+    targets: [linux_arm64]
+    flags: [-trimpath]
+    ldflags:
+      - '-s -w -X main.version={{ if .IsSnapshot }}{{ .Tag }}-dev+{{ .ShortCommit }}{{ else }}{{ .Tag }}{{ end }}'
     mod_timestamp: "{{ .CommitTimestamp }}"
   # linux/arm64 has no cuInterpose build yet; macOS runs the registry side
   # (catalog, images, mirror, push, pull, profile), not build or up.
   - id: plain
     main: ./cmd/warmctl
     binary: warmctl
     env: [CGO_ENABLED=0]
-    targets: [linux_arm64, darwin_amd64, darwin_arm64]
+    targets: [darwin_amd64, darwin_arm64]
     flags: [-trimpath]
     ldflags: ['-s -w -X main.version={{ if .IsSnapshot }}{{ .Tag }}-dev+{{ .ShortCommit }}{{ else }}{{ .Tag }}{{ end }}']
     mod_timestamp: "{{ .CommitTimestamp }}"
@@ -37,6 +47,16 @@ archives:
       - src: bin/cuinterpose/*
         dst: cuinterpose
         strip_parent: true
+      - dmtcp/linux-amd64/**/*
+  - id: linux-arm64
+    ids: [linux-arm64]
+    formats: [tar.gz]
+    name_template: "warmctl-{{ .Tag }}-{{ .Os }}-{{ .Arch }}"
+    files:
+      - LICENSE
+      - NOTICE
+      - README.md
+      - dmtcp/linux-arm64/**/*
   - id: plain
     ids: [plain]
     formats: [tar.gz]

CHANGELOG.md

@@ -1,5 +1,37 @@
 # Changelog
 
+## v2.0.0-beta.17 — 2026-11-01
+
+Large local DMTCP snapshots restore faster with huge pages and parallel
+CPU-memory reads. On one H200, Qwen3.8-Flash-Next-NVFP4 reached health in
+20.21–23.43 s across four runs (median 20.83 s, first streaming token
+20.92 s), versus 77.36/82.14 s with the ordinary reader on the same host
+and snapshot. These were local restores with hot file cache and sufficient
+RAM, excluding provisioning and snapshot transfer; this is not a <=20 s SLA.
+
+- `up` supplies restart-only huge-page advice and up to 16 parallel readers,
+  bounded by Go's container-aware CPU budget. Explicit
+  `DMTCP_RESTART_HUGEPAGES` and `DMTCP_RESTART_READERS` values override the
+  defaults. Private mmap remains disabled by default because it shifted
+  work into CUDA and worsened full recovery time.
+- Linux archives now include the separately released TensorRails DMTCP
+  `v5.0.0-warmctl.1` runtime and fdkeep plugin for amd64 and arm64. The
+  runtime Dockerfile downloads the same packages with source-pinned SHA-256
+  checksums, with no DMTCP compilation during the warmctl build. Packages
+  require Debian 12-compatible glibc/libstdc++; macOS remains client-only.
+- The CLI finds its versioned runtime beside its own executable, including
+  custom install prefixes. The installer preserves runtime bytes and mtimes,
+  retains older versions, and refuses to replace a different build at the
+  same version. Bake new snapshots after upgrading; old snapshots still
+  need their original libraries and paths.
+- Release archives and the installer are tested natively on both Linux
+  architectures before publication. ARM64 is CPU-tested; GPU serving and
+  the parallel reader remain x86-64-only qualifications.
+- All 20 GPU experiments matched weight fingerprints and passed streaming
+  and tool-call tests; 640 short requests succeeded. The numerical canary
+  remains unqualified: the donor already varied before checkpoint. Its
+  tolerance is unchanged. See docs/dmtcp.md for the phase breakdown.
+
 ## v2.0.0-beta.16 — 2026-11-01
 
 warmctl now runs where CRIU cannot: in containers without privileges, such

Makefile

@@ -1,4 +1,4 @@
-.PHONY: catalog-push demo build linux test pytest lint profiles-check check criu-test criu-image e2e-test cuinterpose pod-image release clean
+.PHONY: catalog-push demo build linux test pytest lint profiles-check check criu-test criu-image e2e-test cuinterpose pod-image dmtcp release clean
 
 GO ?= go
 PY ?= python3
@@ -96,9 +96,16 @@ pod-image:
 	docker build -t $(POD_IMAGE) -f deploy/kubernetes/Dockerfile --build-context cuinterpose=bin/cuinterpose \
 		--build-arg VERSION=$(VERSION) .
 
+# Fetch the independently released native runtimes for Linux CLI archives.
+dmtcp:
+	@for arch in amd64 arm64; do \
+		dest="dmtcp/linux-$$arch/$$(cat internal/dmtcp/VERSION)"; \
+		test -f "$$dest/BUILDINFO" || scripts/fetch-dmtcp.sh "$$arch" "$$dest" || exit $$?; \
+	done
+
 # The release archives as .github/workflows/release.yml builds them, as a
 # local snapshot in dist/ (nothing signed or published). Needs goreleaser v2.
-release:
+release: dmtcp
 	test -f bin/cuinterpose/libcuinterpose.so || $(MAKE) cuinterpose
 	goreleaser release --snapshot --clean --skip=sign,sbom
 

NOTICE

@@ -3,8 +3,16 @@ Copyright 2026 The warmctl Authors
 
 This product is licensed under the Apache License, Version 2.0 (LICENSE).
 
-Release archives also contain cuInterpose (cuinterpose/), Copyright (c) 2026
+Linux amd64 release archives also contain cuInterpose (cuinterpose/), Copyright (c) 2026
 NVIDIA CORPORATION & AFFILIATES, licensed under the Apache License, Version
 2.0 (cuinterpose/LICENSE). It is built unmodified from
 https://github.com/ai-dynamo/snapshot at commit
 50bf0db3575df64c968e3f3255d5fa8938950b7a (v0.2.0-rc.2), agent/cmd/cuinterpose.
+
+Linux release archives also contain TensorRails' DMTCP fork, based on DMTCP
+5.0, under the GNU Lesser General Public License version 3 or later. See
+COPYING, COPYING.LESSER and AUTHORS under dmtcp/linux-ARCH/VERSION/share/doc/dmtcp.
+Corresponding source is published with each independently versioned release:
+https://github.com/TensorRails/dmtcp/releases
+The included fdkeep plugin is Copyright 2026 The warmctl Authors, under the
+Apache License 2.0; its license is in the same runtime's share/doc/dmtcp/fdkeep.