make the local venv match CI

.github/workflows/validate.yaml

@@ -13,35 +13,37 @@ jobs:
       # Read-only rendering only: satisfies the state-encryption config and
       # its >=16-char validation. No state is created or read in CI.
       TF_VAR_state_passphrase: ci-dummy-passphrase-render-only
+      # The same path tofu.env sets locally, so scripts/install-collections.sh
+      # and every ansible command after it agree on one location. github is
+      # available in a job env: block; runner is not - naming runner.temp in
+      # an expression here makes the whole workflow file invalid, and no
+      # pull_request run is created at all.
+      ANSIBLE_COLLECTIONS_PATH: ${{ github.workspace }}/.venv/share/ansible/collections
     steps:
       - uses: actions/checkout@v4
 
       - uses: opentofu/setup-opentofu@v1
 
+      # Exact tag: setup-uv stopped publishing moving major aliases after
+      # v8, so @v10 does not resolve.
+      - uses: astral-sh/setup-uv@v10.0.1
+
       - name: Install validation dependencies
         run: |
-          # pipx, not pip --user: it keeps ansible's dependencies in their
-          # own venvs. ansible-lint pulls ansible-core into its venv itself.
-          pipx install ansible-core
-          pipx install ansible-lint
-          # The roles' non-builtin FQCNs resolve locally because requirements.txt
-          # pins the batteries-included `ansible` package; here they have to be
-          # fetched - into one path every later step agrees on. Each tool has
-          # its own default view of where collections live: the runner ships
-          # some preinstalled, and ansible-lint runs from its own pipx venv and
-          # reads neither those nor ansible-core's. Without the explicit -p and
-          # --force, galaxy finds the runner's copies, reports "nothing to do",
-          # and ansible-lint still cannot resolve community.general.
-          #
-          # Exported through GITHUB_ENV rather than the job's env: block
-          # because that block cannot see the runner context. Singular _PATH -
-          # ansible-compat hard errors on the plural spelling - and outside the
-          # workspace, so ansible-lint never walks it as project files.
-          collections="$RUNNER_TEMP/ansible-collections"
-          echo "ANSIBLE_COLLECTIONS_PATH=$collections" >>"$GITHUB_ENV"
-          ansible-galaxy collection install -r ansible/requirements.yml \
-            -p "$collections" --force
-          ansible-galaxy collection list -p "$collections"
+          # The same environment README.md tells an operator to build, from the
+          # same pinned file: requirements.txt is a uv pip freeze, so CI and a
+          # workstation run identical versions on identical Python. That
+          # matters because requirements.txt pins ansible-core, which ships no
+          # collections - both sides get theirs from ansible/requirements.yml
+          # via the script below, and an unpinned community.* module therefore
+          # fails in both places rather than only here.
+          uv venv --python 3.14 --managed-python
+          uv pip install -r requirements.txt
+          # PATH, not activate: the venv has to outlive this step, and each
+          # step is its own shell. This also puts it ahead of the runner
+          # image's own preinstalled ansible.
+          echo "$PWD/.venv/bin" >>"$GITHUB_PATH"
+          ./scripts/install-collections.sh
 
       - name: copy examples
         run: |

CLAUDE.md

@@ -38,9 +38,19 @@ it is the only action that puts one in destroy range.
 ## Working on this
 
 Always `source tofu.env` first — it exports `TF_VAR_state_passphrase` from
-sops and redirects logs to `logs/tofu.log`. Without it every command fails on
-the state encryption key. It refuses to run if sops fails or you are not in
-the repo root.
+sops, redirects logs to `logs/tofu.log`, activates `.venv`, and exports
+`ANSIBLE_CONFIG` and `ANSIBLE_COLLECTIONS_PATH`. Without it every command
+fails on the state encryption key, and every ansible command looks for
+collections in the wrong place. It refuses to run if sops fails or you are not
+in the repo root.
+
+The venv is the whole Python environment and CI builds the same one from the
+same `requirements.txt` (a `uv pip freeze`; README.md has the setup steps).
+It pins `ansible-core`, not the batteries-included `ansible` package, so
+collections come only from `ansible/requirements.yml` via
+`./scripts/install-collections.sh`, which writes into
+`.venv/share/ansible/collections` — recreating the venv therefore requires
+re-running it.
 
 `terraform.tfvars` is gitignored and must exist locally; `ssh_public_keys` is
 the only required variable. See `terraform.tfvars.example`.
@@ -85,10 +95,11 @@ which serves interactive shells only). Underscore names,
 each a directory under `ansible/roles/`, typos fail at plan time. The dynamic
 inventory (`ansible/inventory/tofu.py`) reads `tofu output -json vms`, so it
 needs an applied state; versions are pinned in each role's
-`defaults/main.yaml`, and collections in `ansible/requirements.yml` (CI
-installs from it — `ansible-core` alone ships none, so a `community.*` task
-without a pin there passes locally and fails CI). The `base` role is in no
-spec and cannot be opted out of: `site.yaml` applies it to every host via
+`defaults/main.yaml`, and collections in `ansible/requirements.yml` (the only
+source of them in either environment — `ansible-core` ships none, so a
+`community.*` task without a pin there fails locally and in CI alike). The
+`base` role is in no spec and cannot be opted out of: `site.yaml` applies it
+to every host via
 `roles:`, ahead of the include loop. It carries what the cloud-init snippet
 used to — the apt snapshot pin, the `APT::Periodic` zeros, the apt-daily
 timers, key-only SSH, the timezone, and `spec.packages` — which means those

README.md

@@ -69,14 +69,14 @@ overall testing workflow.
 ├── modules/vm-pve/        # the contract: what a VM is
 ├── ansible/               # post-boot software
 │   ├── ansible.cfg
-│   ├── requirements.yml           # pinned collections (CI installs these)
+│   ├── requirements.yml           # pinned collections (the only source)
 │   ├── site.yaml                  # base, then each VM's declared roles
 │   ├── inventory/tofu.py          # dynamic inventory from tofu output
 │   └── roles/<role>/              # base (every VM), then nats_server, bun,
 │                                  # claude, docker, metafactory_arc
 ├── inventory/             # one YAML file per VM                       [EDIT]
 ├── scripts/               # build-image.sh, check-ansible.sh,
-│                          # vm-fingerprint.sh
+│                          # install-collections.sh, vm-fingerprint.sh
 ├── otel-lgtm/             # grafana/otel-lgtm observability stack      [EDIT]
 │                          # (docker compose; see its README.md)
 └── windmill/              # Windmill workflow engine (docker compose;  [EDIT]
@@ -196,6 +196,7 @@ Note: Edit all the copied files with your specific data.
 - `uv venv --python 3.14 --managed-python`
 - `uv pip install -r requirements.txt`
 - `source tofu.env` # this also sources .venv/bin/activate
+- `./scripts/install-collections.sh` # after tofu.env, which names the path
 - `tofu init` # the first init installs the providers
 - `tofu plan` # verify the output looks like what you expect
 
@@ -306,9 +307,11 @@ declarative grammar as the rest of the repo:
   loop. It owns the apt snapshot pin, the `APT::Periodic` zeros, the masked
   apt-daily timers, key-only SSH, the timezone, and the spec's `packages:` —
   the baseline that used to be cloud-init's, now re-appliable to a running VM.
-- **Collections are pinned** in `ansible/requirements.yml`, and CI installs
-  from it. `ansible-core` ships none, so a `community.*` task without an entry
-  there passes locally and fails in CI.
+- **Collections are pinned** in `ansible/requirements.yml`, and both a
+  workstation and CI install from it with `./scripts/install-collections.sh`.
+  `requirements.txt` pins `ansible-core`, which ships none, so a `community.*`
+  task without an entry there fails in both places — and `check-ansible.sh`
+  verifies the installed versions against the pins before it lints anything.
 - **Role names use underscores** (`nats_server`, not `nats-server`): they
   double as Ansible group names, which must be valid identifiers.
 - **Every download is verified, every version pinned** in the role's

ansible/requirements.yml

@@ -2,12 +2,11 @@
 # Collections used beyond ansible.builtin, pinned like every other version in
 # this repo.
 #
-# CI installs exactly these (.github/workflows/validate.yaml). Locally they
-# arrive incidentally, because requirements.txt pins the batteries-included
-# `ansible` package rather than `ansible-core` - so the versions below are the
-# ones that package currently ships. If the two ever diverge, local checks and
-# CI stop agreeing; the fix is to move requirements.txt to ansible-core and
-# install from this file locally too.
+# The only source of collections in either environment: requirements.txt pins
+# ansible-core, which ships none, and scripts/install-collections.sh installs
+# exactly what is below - locally into .venv/, in CI into the same path under
+# the workspace. So a community.* module missing from this file fails
+# everywhere, instead of passing on a workstation and failing in CI.
 collections:
   # timezone. The only declarative option for it - ansible.builtin has none.
   - name: community.general

requirements.txt

@@ -1,4 +1,3 @@
-ansible==14.3.1
 ansible-compat==26.8.0
 ansible-core==2.21.3
 ansible-lint==26.8.0
@@ -17,7 +16,7 @@ markupsafe==3.0.3
 mypy-extensions==1.1.0
 packaging==26.3
 pathspec==1.1.1
-platformdirs==4.11.4
+platformdirs==4.11.5
 pycparser==3.0
 pytokens==0.4.1
 pyyaml==6.0.3

scripts/check-ansible.sh

@@ -19,11 +19,44 @@ if [ -z "${TF_VAR_state_passphrase:-}" ]; then
   echo "TF_VAR_state_passphrase is not set; run 'source tofu.env' first so the inventory can read the tofu state" >&2
   exit 1
 fi
+if [ -z "${ANSIBLE_COLLECTIONS_PATH:-}" ]; then
+  echo "ANSIBLE_COLLECTIONS_PATH is not set; run 'source tofu.env' first" >&2
+  exit 1
+fi
 export ANSIBLE_CONFIG="$PWD/ansible/ansible.cfg"
 
 status=0
 
-# 1. Inventory: valid JSON with the hostvars site.yaml and the roles rely on,
+# 1. Collections: every pin in ansible/requirements.yml installed, at exactly
+#    the pinned version. Nothing else supplies them - requirements.txt pins
+#    ansible-core, which ships none - and without this check a missing or
+#    stale collection surfaces only as an opaque ansible-lint
+#    "couldn't resolve module/action" three checks later.
+if python3 -c '
+import json, sys
+from pathlib import Path
+import yaml
+
+root = Path(sys.argv[1]) / "ansible_collections"
+pins = yaml.safe_load(Path("ansible/requirements.yml").read_text())["collections"]
+for pin in pins:
+    fqcn = pin["name"]
+    manifest = root / Path(fqcn.replace(".", "/")) / "MANIFEST.json"
+    if not manifest.is_file():
+        sys.exit(f"{fqcn} is not installed - run ./scripts/install-collections.sh")
+    want = pin["version"]
+    found = json.loads(manifest.read_text())["collection_info"]["version"]
+    if found != want:
+        sys.exit(f"{fqcn} is {found}, pinned at {want} - run ./scripts/install-collections.sh")
+print(f"ok    collections ({len(pins)} pinned)")
+' "$ANSIBLE_COLLECTIONS_PATH"; then
+  :
+else
+  echo "FAIL  collections" >&2
+  status=1
+fi
+
+# 2. Inventory: valid JSON with the hostvars site.yaml and the roles rely on,
 #    and every declared role backed by a real role directory (belt and braces
 #    with the tofu plan-time validation).
 if ansible-inventory --list | python3 -c '
@@ -50,15 +83,15 @@ else
   status=1
 fi
 
-# 2. Playbook syntax.
+# 3. Playbook syntax.
 if ansible-playbook ansible/site.yaml --syntax-check >/dev/null; then
   echo "ok    site.yaml syntax"
 else
   echo "FAIL  site.yaml syntax" >&2
   status=1
 fi
 
-# 3. Lint, when available.
+# 4. Lint, when available.
 if command -v ansible-lint >/dev/null; then
   if (cd ansible && ansible-lint) >/dev/null 2>&1; then
     echo "ok    ansible-lint"

scripts/install-collections.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# Install the pinned ansible collections named in ansible/requirements.yml.
+#
+# One script for both environments: the local venv and CI run this same
+# command, so the two cannot drift in how the collections arrive. Where they
+# land is the caller's choice, named by ANSIBLE_COLLECTIONS_PATH - tofu.env
+# sets it locally, the workflow's job env: block sets it in CI - because that
+# is also the variable every later ansible command reads.
+
+set -euo pipefail
+
+cd "$(dirname "$0")/.."
+
+if ! command -v ansible-galaxy >/dev/null; then
+  echo "missing ansible-galaxy; create the venv first (see README)" >&2
+  exit 1
+fi
+
+# Singular _PATH, not the plural spelling: ansible-compat hard errors on
+# ANSIBLE_COLLECTIONS_PATHS, and ansible-lint runs through ansible-compat.
+if [ -z "${ANSIBLE_COLLECTIONS_PATH:-}" ]; then
+  echo "ANSIBLE_COLLECTIONS_PATH is not set; run 'source tofu.env' first" >&2
+  exit 1
+fi
+# A colon-separated list is valid for ansible itself but not for -p, which
+# takes exactly one directory. Fail rather than silently install into the head.
+case "$ANSIBLE_COLLECTIONS_PATH" in
+  *:*)
+    echo "ANSIBLE_COLLECTIONS_PATH must name a single directory: $ANSIBLE_COLLECTIONS_PATH" >&2
+    exit 1
+    ;;
+esac
+
+# --force is not cosmetic. Without it galaxy short-circuits to "Nothing to do"
+# whenever a copy of a requested collection exists anywhere in its own search
+# path - a runner image ships some preinstalled - and installs nothing into
+# the path everything else here reads.
+ansible-galaxy collection install -r ansible/requirements.yml \
+  -p "$ANSIBLE_COLLECTIONS_PATH" --force
+
+# Logged, not just installed: what landed is the thing worth seeing in a CI
+# run's output when a lint failure turns out to be a missing collection.
+ansible-galaxy collection list -p "$ANSIBLE_COLLECTIONS_PATH"

tofu.env

@@ -48,6 +48,18 @@ export TF_LOG_PATH="$PWD/logs/tofu.log"
 # without -i or cwd tricks (ANSIBLE_CONFIG outranks cwd-based discovery).
 export ANSIBLE_CONFIG="$PWD/ansible/ansible.cfg"
 
+# The only place collections are read from - scripts/install-collections.sh
+# writes here, every ansible command reads here, and CI names the same path in
+# its job env: block. Singular _PATH: ansible-compat hard errors on the plural
+# ANSIBLE_COLLECTIONS_PATHS, and ansible-lint runs through ansible-compat.
+#
+# Under .venv/ deliberately: recreating the venv takes the collections with it,
+# so a rebuilt environment can never keep serving a stale set. Setting this
+# also *replaces* ansible's defaults (~/.ansible/collections and
+# /usr/share/ansible/collections), which is the point - what is not pinned in
+# ansible/requirements.yml must not resolve here either.
+export ANSIBLE_COLLECTIONS_PATH="$PWD/.venv/share/ansible/collections"
+
 # Activate Python venv for ansible
 if [ -f "$PWD/.venv/bin/activate" ]; then
   source "$PWD/.venv/bin/activate"