From 1b20da1ac46264f4014df102955f938ca0fa0996 Mon Sep 17 00:00:00 2001 From: Simon Davies Date: Thu, 22 Oct 2026 14:36:47 +0100 Subject: [PATCH] Validate device readiness on KVM and MSHV runners Exercise live readiness on hosted KVM and the Azure Linux MSHV AMD and Intel pools as part of PR validation. Keep host device permissions unchanged and require an actual passing live test. Refs #22. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 38aaadb6-fb9d-4ecd-809f-e996b2f8c463 Signed-off-by: Simon Davies --- .github/workflows/pr-validation.yml | 66 +++++++++++++++++++++++++-- device-plugin/probe_kvm_linux_test.go | 18 ++++++++ docs/architecture.md | 2 + 3 files changed, 81 insertions(+), 5 deletions(-) create mode 100644 device-plugin/probe_kvm_linux_test.go diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 0bf0763..ff3cea6 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -41,10 +41,6 @@ jobs: - name: Run linters run: just lint-strict - - name: Test device readiness and recovery - working-directory: device-plugin - run: go test -race -count=1 -timeout=90s ./... - build: name: Build runs-on: ubuntu-latest @@ -64,6 +60,66 @@ jobs: - name: Build device plugin run: just plugin-build + readiness: + name: Device Readiness (${{ matrix.hypervisor }}, ${{ matrix.cpu_vendor }}) + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - hypervisor: kvm + cpu_vendor: hosted + live_test: TestKVMLiveVMCreation + live_env: TEST_KVM_LIVE + - hypervisor: mshv + cpu_vendor: amd + live_test: TestMSHVLivePartitionCreation + live_env: TEST_MSHV_LIVE + - hypervisor: mshv + cpu_vendor: intel + live_test: TestMSHVLivePartitionCreation + live_env: TEST_MSHV_LIVE + runs-on: ${{ matrix.hypervisor == 'kvm' && 'ubuntu-24.04' || fromJSON(format('["self-hosted", "Linux", "X64", "1ES.Pool=hld-azlinux3-mshv-{0}", "JobId=device-readiness-{0}-{1}-{2}-{3}"]', matrix.cpu_vendor, github.run_id, github.run_number, github.run_attempt)) }} + env: + HYPERVISOR: ${{ matrix.hypervisor }} + LIVE_TEST: ${{ matrix.live_test }} + LIVE_ENV: ${{ matrix.live_env }} + defaults: + run: + shell: bash + working-directory: device-plugin + steps: + - name: Checkout + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + + - name: Setup Go + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 + with: + go-version-file: device-plugin/go.mod + cache-dependency-path: device-plugin/go.sum + + - name: Check hypervisor host + run: | + uname -a + lscpu + git rev-parse HEAD + stat -c '%n %F %a %U:%G %t:%T' "/dev/$HYPERVISOR" + sudo -n test -c "/dev/$HYPERVISOR" + + - name: Test device readiness and recovery + run: go test -race -count=1 -timeout=90s ./... + + - name: Test live hypervisor readiness + run: | + go test -race -c -o "$RUNNER_TEMP/device-readiness.test" . + # Only the live probe needs root; leave host device permissions unchanged. + sudo -n env "$LIVE_ENV=1" "$RUNNER_TEMP/device-readiness.test" \ + -test.v -test.count=1 -test.timeout=60s -test.run "^${LIVE_TEST}$" \ + | tee "$RUNNER_TEMP/live-readiness.log" + grep -F -- "--- PASS: $LIVE_TEST (" "$RUNNER_TEMP/live-readiness.log" + integration: name: Integration Tests runs-on: ubuntu-24.04 @@ -91,7 +147,7 @@ jobs: - name: Install kind run: | go install sigs.k8s.io/kind@latest - echo "$(go env GOPATH)/bin" >> $GITHUB_PATH + echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH" - name: Install kubectl uses: azure/setup-kubectl@v4 diff --git a/device-plugin/probe_kvm_linux_test.go b/device-plugin/probe_kvm_linux_test.go new file mode 100644 index 0000000..e503639 --- /dev/null +++ b/device-plugin/probe_kvm_linux_test.go @@ -0,0 +1,18 @@ +// Copyright 2026 The Hyperlight Authors. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "testing" +) + +func TestKVMLiveVMCreation(t *testing.T) { + if os.Getenv("TEST_KVM_LIVE") != "1" { + t.Skip("set TEST_KVM_LIVE=1 on a KVM host") + } + t.Setenv("TEST_PROBE_SUCCESS", "0") + t.Setenv("TEST_PROBE_HANG", "0") + requireNoError(t, (&deviceProbe{}).check(t.Context(), "kvm", "/dev/kvm")) +} diff --git a/docs/architecture.md b/docs/architecture.md index e7586bd..e87219f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -104,6 +104,8 @@ For KVM, a helper opens the character device read/write, verifies API version 12 From `device-plugin/`, the opt-in `TEST_MSHV_LIVE=1 go test -count=1 -run '^TestMSHVLivePartitionCreation$' ./...` check requires an MSHV host and access to `/dev/mshv`. Ordinary unit tests exercise failure handling and descriptor cleanup without a real hypervisor. +PR Validation runs the readiness and recovery race tests on hosted Ubuntu KVM and the organization-wide Azure Linux 3 MSHV AMD and Intel runner pools. Each matrix entry also runs its live creation test through the production readiness helper: `TestKVMLiveVMCreation` with `TEST_KVM_LIVE=1`, or `TestMSHVLivePartitionCreation` with `TEST_MSHV_LIVE=1`. Only the live test binary runs as root, without changing host device ownership or permissions. Missing devices, failed partition/VM creation, and probe timeouts fail the job. The workflow logs the tested commit, kernel, CPU details and device metadata. This validates host readiness, not MSHV Kubernetes/CDI injection or guest execution; the existing KVM KIND integration job remains separate. + The bootstrap allocation gives the trusted plugin explicit hypervisor device access through kubelet and the container runtime. Host permissions and security policy still apply; denial fails readiness without changing those policies. The probe establishes usability for the plugin, not admission or device access for every application security context; fresh guest execution remains a separate acceptance check. The liveness probe performs a gRPC health request to the running server. Readiness additionally requires a successful, deadline-bounded kubelet registration, an active `ListAndWatch` stream and a successful readiness check. Socket existence alone satisfies neither probe. Registration failures stop the attempted server before retrying; kubelet socket cleanup causes the server to be recreated and registered again.