Release the warmctl fork with parallel large-memory restart

.github/workflows/make-check.yml

@@ -6,6 +6,10 @@ on:
   pull_request:
     branches: [ main ]
 
+concurrency:
+  group: build-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   build:
 
@@ -20,8 +24,8 @@ jobs:
     - name: configure
       run: ./configure
     - name: make
-      run: make
+      run: make -j2
     - name: make tests
-      run: make tests
+      run: make tests -j2
     - name: make check
       run: make check AUTOTEST="--retry-once --jobs 8"

.github/workflows/warmctl-release.yml

@@ -0,0 +1,128 @@
+name: warmctl release
+
+on:
+  pull_request:
+    branches: [main, warmctl]
+  push:
+    tags: ['v*-warmctl.*']
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: warmctl-release-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  linux:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - runner: ubuntu-24.04
+            arch: amd64
+          - runner: ubuntu-24.04-arm
+            arch: arm64
+    runs-on: ${{ matrix.runner }}
+    container:
+      image: debian:bookworm-slim@sha256:88200866dfff7ea7f5cbcb6ec7c8a701889efe6fe859fe64d6990e4b07ea4171
+      # Restored process trees are adopted by PID 1; reap them in this test container.
+      options: --init
+    timeout-minutes: 30
+    steps:
+      - name: Install native build dependencies
+        run: apt-get update && apt-get install -y --no-install-recommends build-essential python3 git ca-certificates gzip zstd
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Build
+        run: |
+          ./configure --prefix=/opt/warmctl/dmtcp --docdir=/opt/warmctl/dmtcp/share/doc/dmtcp
+          make -j2
+          make tests -j2
+      - name: Check memory readers and checkpoint bytes
+        run: python3 test/restore-memory-check.py --root . --output "$RUNNER_TEMP/memory-check"
+      - name: Existing process, thread and memory regressions
+        run: DMTCP_RESTART_HUGEPAGES=1 DMTCP_RESTART_READERS=16 python3 test/autotest.py --retry-once --jobs 2 file1 file3 mmap1 mmap-noreserve zeropages zeropages-pread pthread1 forkexec sysv-shm1 modify-env
+      - name: Package installed runtime and source
+        env:
+          TAG: ${{ startsWith(github.ref, 'refs/tags/') && github.ref_name || 'snapshot' }}
+          ARCH: ${{ matrix.arch }}
+        run: |
+          make install DESTDIR="$RUNNER_TEMP/stage"
+          make -C plugin/fdkeep install DESTDIR="$RUNNER_TEMP/stage"
+          install -m 0644 COPYING.LESSER "$RUNNER_TEMP/stage/opt/warmctl/dmtcp/share/doc/dmtcp/"
+          mkdir -p dist
+          printf 'release=%s\ncommit=%s\nupstream_version=%s\nos=debian-12\narch=%s\nprefix=/opt/warmctl/dmtcp\n' \
+            "$TAG" "$GITHUB_SHA" "$(cat VERSION)" "$ARCH" > "$RUNNER_TEMP/stage/opt/warmctl/dmtcp/BUILDINFO"
+          ldd --version | head -1 >> "$RUNNER_TEMP/stage/opt/warmctl/dmtcp/BUILDINFO"
+          tar -czf "dist/dmtcp-$TAG-linux-$ARCH-debian12.tar.gz" -C "$RUNNER_TEMP/stage/opt/warmctl" dmtcp
+          if [ "$ARCH" = amd64 ]; then
+            git -c safe.directory="$GITHUB_WORKSPACE" archive --format=tar.gz --prefix="dmtcp-$TAG/" HEAD > "dist/dmtcp-$TAG-source.tar.gz"
+          fi
+          (cd dist && sha256sum *.tar.gz > "dmtcp-$TAG-linux-$ARCH-checksums.txt")
+      - name: Verify packaged runtime
+        run: |
+          mkdir -p "$RUNNER_TEMP/relocated"
+          tar -xzf dist/*-linux-${{ matrix.arch }}-debian12.tar.gz -C "$RUNNER_TEMP/relocated"
+          "$RUNNER_TEMP/relocated/dmtcp/bin/dmtcp_launch" --version
+          python3 test/restore-memory-check.py --root "$RUNNER_TEMP/relocated/dmtcp" \
+            --plugin "$RUNNER_TEMP/relocated/dmtcp/lib/libdmtcp_fdkeep.so" --output "$RUNNER_TEMP/package-check"
+      - name: Archive test logs
+        if: always()
+        run: |
+          mkdir -p dist
+          python3 - <<'PY'
+          import os
+          from pathlib import Path
+          import tarfile
+          root = Path(os.environ['RUNNER_TEMP'])
+          with tarfile.open('dist/test-logs-${{ matrix.arch }}.tgz', 'w:gz') as archive:
+              for name in ('memory-check', 'package-check'):
+                  for log in (root / name).rglob('*.log'):
+                      archive.add(log, arcname=str(log.relative_to(root)))
+          PY
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        if: always()
+        with:
+          name: dmtcp-linux-${{ matrix.arch }}
+          path: dist/
+          if-no-files-found: error
+          retention-days: 14
+
+  publish:
+    if: startsWith(github.ref, 'refs/tags/')
+    needs: linux
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: write
+      id-token: write
+      attestations: write
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        with:
+          pattern: dmtcp-linux-*
+          merge-multiple: true
+          path: artifacts
+      - name: Combined checksums
+        env:
+          TAG: ${{ github.ref_name }}
+        run: |
+          cd artifacts
+          rm *-linux-*-checksums.txt
+          sha256sum *.tar.gz > "dmtcp-$TAG-checksums.txt"
+      - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
+        with:
+          subject-path: artifacts/*.tar.gz
+      - name: Publish fork prerelease
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ github.ref_name }}
+        run: |
+          mapfile -t assets < <(find artifacts -type f \( -name '*.tar.gz' -o -name '*checksums.txt' \))
+          gh release create "$TAG" "${assets[@]}" --verify-tag --prerelease --title "DMTCP $TAG" --notes-file doc/warmctl-release-notes.md

RELEASING.md

@@ -0,0 +1,46 @@
+# TensorRails DMTCP releases
+
+The fork has its own tags, `v5.0.0-warmctl.N`, independently of warmctl's
+CLI versions. `VERSION` remains the upstream checkpoint ABI version (5.0);
+the tag and full commit in the installed `BUILDINFO` identify the fork build.
+Do not change the upstream checkpoint signature just to number a fork release.
+
+Merge a reviewed PR into `main`, with `Fixes #N` for its performance issue.
+The PR workflow builds natively in pinned Debian 12 containers on x86-64 and ARM64, runs the memory-reader and
+process regressions, packages an installed runtime, and tests that package.
+After the checks pass, tag the merge commit and push the tag:
+
+```sh
+git tag -a v5.0.0-warmctl.1 MERGE_COMMIT -m v5.0.0-warmctl.1
+git push origin v5.0.0-warmctl.1
+```
+
+The workflow reruns the checks and publishes a prerelease with source,
+Debian 12 x86-64 and ARM64 runtimes, SHA-256 checksums and GitHub build provenance.
+Release notes come from `doc/warmctl-release-notes.md`. Before each later
+release, update those notes in the PR. Keep the historical `warmctl` branch
+fast-forwarded to the released commit for consumers still using that branch.
+
+The binary archive contains `dmtcp/`, to extract under `/opt/warmctl`.
+It includes the separately licensed warmctl fdkeep plugin; cuInterpose is a
+separate GPU dependency. fdkeep is built from plugin/fdkeep and tested on both
+release architectures. The archive requires Debian 12-compatible glibc
+(2.36 or newer) and libstdc++ on the matching architecture; it is not a
+portable Linux binary. DMTCP runs on Linux, not macOS or Windows. Parallel
+pread helpers currently support x86-64 only; ARM64 uses the ordinary reader
+with optional huge pages and is CPU-tested, not GPU-qualified by this release.
+The package tests also bake and restore from a relocated installation path.
+warmctl bundles these released packages and uses the same assets in its runtime
+Dockerfile, verifying source-pinned SHA-256 checksums. For an older runtime
+base, build the tagged source and fdkeep inside that base instead.
+
+```sh
+sha256sum --ignore-missing -c dmtcp-v5.0.0-warmctl.1-checksums.txt
+gh attestation verify dmtcp-v5.0.0-warmctl.1-linux-amd64-debian12.tar.gz \
+  --repo TensorRails/dmtcp
+```
+
+Donor and receiver must use byte-identical mapped DMTCP libraries and runtime
+files. A new release requires a fresh bake and the same resulting runtime
+image on receivers. Never overwrite `libdmtcp.so` underneath a live engine or
+an old snapshot and assume compatibility. Preserve that snapshot's old image.

doc/h200-restart-performance.md

@@ -0,0 +1,75 @@
+# H200 local restart measurements, 2026-11-01
+
+Model: `nvidia/Qwen3.8-Flash-Next-NVFP4`, one H200. A local 135.43 GiB
+snapshot, with file cache hot, on a Xeon Platinum 8558 host; cgroup CPU quota
+20.4 CPUs and RAM limit 233.76 GiB. GPU weights occupy 76.50 GiB. This
+measures restart through HTTP readiness and the first streaming token, not
+pod provisioning, runtime installation or snapshot transfer.
+
+All modes used the same snapshot and H200. Earlier observations of 52.8 s
+were on another host with a subsequently lost snapshot and are not an A/B
+baseline for these measurements.
+
+| Mode | Runs to health (seconds) | Median |
+| --- | --- | --- |
+| Ordinary reader | 77.36, 82.14 | 79.75 |
+| Huge pages, one reader | 35.95, 37.46 | 36.71 |
+| Private mmap, unaligned image (fallback) | 79.24, 75.98 | 77.61 |
+| Private mmap, aligned copy | 77.02, 52.83 | 64.93 |
+| Ordinary reader, aligned copy | 83.12 | 83.12 |
+| Huge pages, four readers | 34.27, 26.58 | 30.42 |
+| Huge pages, eight readers | 38.73, 27.03 | 32.88 |
+| Huge pages, eight readers after cache preparation | 23.45, 23.77 | 23.61 |
+| Huge pages, sixteen readers after cache preparation | 20.21, 23.43, 20.27, 21.39 | **20.83** |
+| One-reader huge-page control after cache preparation | 44.97 | 44.97 |
+
+Cache preparation evicted only unused experimental aligned-image and CPU
+benchmark pages with `POSIX_FADV_DONTNEED`. It did not evict the primary
+snapshot, model or host-wide page cache. `memory.current` fell from 163.61
+to 135.07 GiB. The four fast runs peaked at 213.08 GiB, with no additional
+direct reclaim scans. This preparation and the small sample sizes matter:
+these are experiments, not a p95 or a <=20 s guarantee. The final binary,
+including helper parent-death protection, produced the last two fast runs.
+Median streaming first token from start across the four: **20.92 s**.
+
+One complete before/after pair:
+
+| Phase | Ordinary | Huge pages + 16 readers |
+| --- | ---: | ---: |
+| Setup and filesystem | 3.09 s | 3.33 s |
+| DMTCP | 54.09 s | 4.04 s |
+| CUDA | 13.52 s | 6.73 s |
+| Wake and reload weights | 5.62 s | 5.12 s |
+| KV, health, canary | 0.99 s | 0.99 s |
+| External health polling | 0.05 s | 0.07 s |
+| Total | 77.36 s | 20.27 s |
+
+The slow worker's memory pass took 53.61 s, with 14,154,801 minor faults,
+zero major faults and zero disk `read_bytes`. A single 47.6846 GiB anonymous
+payload took 47.05 s. Its size matches the model's CPU PLE/Engram table,
+allowing for allocation rounding; direct tensor-pointer attribution was
+not performed. Other processes finished in 0.08–3.80 s and waited at the
+barrier; adding their barrier waits would double-count elapsed time.
+
+THP reduced that worker's minor faults about 49-fold. Sixteen readers loaded
+the dominant region in 1.27–1.33 s and the whole worker in 3.56–3.73 s.
+Private mmap reached 2.68–2.84 s for DMTCP but 39.81–60.66 s for CUDA.
+Observed anonymous RSS growth is consistent with COW during CUDA host-memory
+registration, though the driver's internal function was not traced.
+
+## Correctness and limits
+
+All 20 GPU runs matched weight fingerprints, returned streaming READY and
+a valid forced echo tool call. All 640 short smoke requests succeeded.
+Numerical equivalence is not proven: identical donor requests *before*
+checkpoint, seed 42 and temperature 0, already differed in token sequence
+and by up to 1.15547 logprob on the five-token common prefix. Both reported
+zero cached tokens. The canary tolerance stayed 0.001; variability before
+checkpoint does not prove absence of restoration bugs.
+
+CPU tests verified every word of dense/sparse mappings, dirty-page writes,
+second checkpoint/restore, source-image digests, uneven parallel chunks,
+parent cancellation and private dirty bytes after file EOF. CI repeats
+these checks against both the build and the packaged runtime. The fork
+switches remain opt-in; CUDA mmap remains experimental and disabled in
+the recommended configuration.

doc/restart-memory-profiling.md

@@ -0,0 +1,77 @@
+# Restart memory profiling experiments
+
+These opt-in switches diagnose large CPU process images. Configure DMTCP
+normally. The default reader and checkpoint format stay unchanged.
+
+| Environment variable | Value | Effect |
+| --- | --- | --- |
+| `DMTCP_RESTART_PROFILE` | `1` | Log memory restoration, large/slow regions, page faults, the restart barrier and plugin phase. |
+| `DMTCP_RESTART_HUGEPAGES` | `1` | Advise huge pages for eligible large private anonymous mappings before loading their contents. |
+| `DMTCP_RESTART_MMAP` | `1` | Map eligible raw anonymous payloads of at least 64 MiB directly from the image using private mappings. |
+| `DMTCP_RESTART_READERS` | `2`–`8`, `16` | On x86-64, copy raw regular-file payloads of at least 1 GiB with parallel `pread` helpers. Default is one reader. |
+
+Use one optimization at a time for A/B measurements. Example:
+
+```sh
+DMTCP_RESTART_PROFILE=1 DMTCP_RESTART_HUGEPAGES=1 warmctl up SNAPSHOT --mode in-place
+DMTCP_RESTART_PROFILE=1 DMTCP_RESTART_MMAP=1 warmctl up SNAPSHOT --mode in-place
+```
+
+The profiling fields in `RestoreInfo` are transient. Modified `libdmtcp.so`
+must match the library present at checkpoint creation: rebuild the runtime and
+bake a new snapshot before testing its post-restart instrumentation. The MTCP
+memory reader uses raw syscalls and stores its counters in relocated state.
+
+`phase=memory` reports microseconds spent in the memory pass and payload reads,
+anonymous mmap setup time, bytes read or file-mapped in KiB, and minor/major
+faults. `pid` is the real process PID; `virtual_pid` is the checkpoint PID.
+`phase=area` reports large or slow payloads, mmap eligibility, offset alignment
+and mmap errors. An eligible payload with nonzero `offset_mod` falls back to
+reading. `restart_barrier`,
+`restart_directory`, `restart_plugins`, and `restart_maps_and_resume` are
+sequential post-memory phases, identified by DMTCP's virtual PID. Barrier wait
+times across processes overlap and must not be added to the critical path.
+
+Huge-page advice is best effort and leaves zero runs unpopulated. Actual THP
+availability, NUMA placement, pressure and pinning determine its effectiveness.
+
+The mmap experiment replaces only a region created by the restore itself,
+including bounded children of an anonymous parent region. It leaves stacks,
+small payloads, original file-backed regions and compressed/pipe inputs on the
+ordinary reader. Private mappings protect the source image against writes.
+Keep the source image immutable for the entire restored process lifetime.
+Do not truncate or rewrite it while mapped.
+
+Older raw checkpoints can have unaligned payloads after a partial file-backed
+mapping. The writer now saves the complete final page of regular files,
+bounded by the mapping. This also fixes loss of private dirty bytes between
+EOF and the end of that page. Existing readers honor the rounded
+`mmapFileSize`; no format version changes. `test/restore-partial-file.c`
+reproduces the old dirty-tail loss and verifies the corrected writer.
+This writer correction affects newly created snapshots, not existing ones.
+
+Mmap shifts faults to first access. Measure first useful work, CUDA restoration,
+and steady-state inference, not just the memory-pass duration. CUDA host-memory
+registration may force private copies and remove some or all of the benefit.
+Neither experiment is qualified for GPU production solely by CPU tests.
+
+Parallel readers can be combined with huge-page advice. Their temporary stacks
+are unmapped and every helper is reaped before moving to another memory area or
+resuming the application. Helpers share the address space but do not use libc,
+TLS or pthreads. Failed helper creation falls back to reading its range in the
+parent; failed reads fall back to the ordinary reader after all helpers exit.
+Helpers arm a parent-death signal and recheck their parent, so cancellation
+does not leave them holding the restore's address space.
+`parallel_kib` identifies actual use. `child_minflt` and `child_majflt` are the
+helpers' faults, separate from the parent's `minflt` and `majflt`.
+
+`test/restore-memory-bench.c` checkpoints dense or sparse anonymous memory,
+verifies every word after restart, and optionally writes to every page and
+checkpoints/restores again. The investigation runner compares both modes on the
+same images, explicitly warms their file cache, and checks source SHA-256 for
+mmap runs. Its wall time includes full data verification and process exit.
+
+During instrumentation, `dmtcp_get_restart_env()` was found to bound its scan
+by `sizeof(char *)`, which examined only the first environment entry. The fix
+uses the actual read length and requires an exact name followed by `=`. The
+benchmark verifies a later variable whose longer prefixed name occurs first.