From 8e486976f39de1ddb1668482de5f0d207755688a Mon Sep 17 00:00:00 2001 From: Mike Riddle Date: Mon, 2 Nov 2026 14:38:48 -0500 Subject: [PATCH 1/4] Make the pupmod acceptance jobs survive Launchpad PPA outages The acceptance jobs in both pupmod PR Tests templates installed vagrant and vagrant-libvirt from ppa:evgeni/vagrant alone, so a Launchpad outage failed every acceptance job and, through the required 'All checks' status, blocked merging. Ubuntu dropped both packages after 23.10, and the PPA carries vagrant 2.3.7 (the last MPL-licensed release) with a matching vagrant-libvirt. The PPA stays the primary source, wrapped so its outages don't matter: * The PPA's two .debs are kept in the Actions cache, keyed on the Ubuntu codename, architecture and the pinned package versions. On a cache hit the step installs them without contacting Launchpad; their dependencies come from the Ubuntu archive. * add-apt-repository, apt-get update and the download are retried three times, 30s and then 60s apart. * If the PPA is still down, the step installs Vagrant pinned to 2.4.9 from HashiCorp's apt repository and vagrant-libvirt 0.12.2 with `vagrant plugin install`, and emits a warning saying so. The PPA only publishes for noble, and ubuntu-latest is moving to 26.04 (resolute), where the old step fails on the PPA's missing Release file. On a series the PPA doesn't publish for, the step goes straight to the HashiCorp path with a notice. The acceptance block is repo-owned during syncs (preserve_blocks), so a template change to it never reached the fleet. merge_gha_workflows gains a `template_steps` option: inside a preserved job, the steps with those names are replaced by the template's, in template order, where the first of them was. A job with none of them is left alone, so podman-based and acceptance-free modules are unchanged. The new 20261102-vagrant-ppa-fallback config uses it to sync the change. Fixes #122 Co-Authored-By: Claude Opus 5.5 --- .../20261102-vagrant-ppa-fallback.yaml | 66 ++++++++++++ data/sync/configs/latest.yaml | 2 +- dist/puppetsync/plans/init.pp | 1 + .../puppetsync/tasks/merge_gha_workflows.json | 4 + dist/puppetsync/tasks/merge_gha_workflows.rb | 70 ++++++++++-- .../_github/workflows/pr_tests.simp-simp.yml | 101 +++++++++++++++++- .../pupmod/_github/workflows/pr_tests.yml | 101 +++++++++++++++++- spec/plans/merge_workflows_stage_spec.rb | 6 ++ spec/tasks/merge_gha_workflows_spec.rb | 95 +++++++++++++++- 9 files changed, 431 insertions(+), 15 deletions(-) create mode 100644 data/sync/configs/20261102-vagrant-ppa-fallback.yaml diff --git a/data/sync/configs/20261102-vagrant-ppa-fallback.yaml b/data/sync/configs/20261102-vagrant-ppa-fallback.yaml new file mode 100644 index 0000000..6080683 --- /dev/null +++ b/data/sync/configs/20261102-vagrant-ppa-fallback.yaml @@ -0,0 +1,66 @@ +--- +# Make the pupmod acceptance jobs survive Launchpad PPA outages +# (simp/puppetsync#122). +# +# The acceptance job's "Setup libvirt for Vagrant" step installed vagrant +# and vagrant-libvirt from ppa:evgeni/vagrant alone, so a Launchpad outage +# failed every acceptance job and, through 'All checks', blocked merges. +# The step now restores the PPA's .debs from the Actions cache, retries the +# PPA with backoff, and falls back to HashiCorp's apt repository and +# `vagrant plugin install vagrant-libvirt`. +# +# pr_tests.yml (merge_github_workflows): each repo's `jobs.acceptance` block +# is kept as-is, except for the Vagrant setup steps named in +# template_steps, which are replaced with the template's. Modules whose +# acceptance job uses podman, and modules with no acceptance job, are left +# unchanged. +puppetsync::plan_config: + puppetsync: + permitted_project_types: + - pupmod + - pupmod_skeleton + plans: + sync: + github_api_delay_seconds: 1 + merge_github_workflows: + files: + - pr_tests.yml + preserve_blocks: + - jobs.acceptance + template_steps: + - Choose the Vagrant packages + - Restore the cached Vagrant packages + - Setup libvirt for Vagrant + - Cache the Vagrant packages + stages: + - install_gems + - checkout_git_feature_branch_in_each_repo + - merge_github_workflows + - git_commit_changes + - ensure_github_fork + - ensure_git_remote + - git_push_to_remote + - ensure_github_pr + + approve_github_prs: + stages: + - approve_github_pr_for_each_repo + + merge_github_prs: + stages: + - merge_github_pr_for_each_repo + + git: + feature_branch: puppetsync/20261102-vagrant-ppa-fallback + commit_message: | + [puppetsync] Survive Launchpad PPA outages in acceptance + + The acceptance job installed Vagrant and vagrant-libvirt from + ppa:evgeni/vagrant alone, so a Launchpad outage failed every + acceptance run. The setup now installs the PPA's packages from the + Actions cache when it can, retries the PPA with backoff, and falls + back to HashiCorp's Vagrant and the vagrant-libvirt plugin. + + github: + pr_user: silug # This should be the account that *submitted* the PRs + approval_message: ':+1: :ghost:' diff --git a/data/sync/configs/latest.yaml b/data/sync/configs/latest.yaml index fd33f65..719f402 120000 --- a/data/sync/configs/latest.yaml +++ b/data/sync/configs/latest.yaml @@ -1 +1 @@ -20261014-all-checks-job.yaml \ No newline at end of file +20261102-vagrant-ppa-fallback.yaml \ No newline at end of file diff --git a/dist/puppetsync/plans/init.pp b/dist/puppetsync/plans/init.pp index 4b90ce4..2bd59a7 100644 --- a/dist/puppetsync/plans/init.pp +++ b/dist/puppetsync/plans/init.pp @@ -348,6 +348,7 @@ 'workflows' => $workflows, 'preserve_keys' => $opts.dig('merge_github_workflows', 'preserve_keys'), 'preserve_blocks' => $opts.dig('merge_github_workflows', 'preserve_blocks'), + 'template_steps' => $opts.dig('merge_github_workflows', 'template_steps'), }) } } diff --git a/dist/puppetsync/tasks/merge_gha_workflows.json b/dist/puppetsync/tasks/merge_gha_workflows.json index 15b558a..af69cf7 100644 --- a/dist/puppetsync/tasks/merge_gha_workflows.json +++ b/dist/puppetsync/tasks/merge_gha_workflows.json @@ -13,6 +13,10 @@ "preserve_blocks": { "description": "Dot-paths of mapping blocks the EXISTING file is authoritative for (e.g. ['jobs.acceptance']): its block replaces the template's, is appended when the template lacks it, and the template's block is removed when the existing file has none", "type": "Optional[Array[String[1]]]" + }, + "template_steps": { + "description": "Names of steps the template stays authoritative for inside preserved_blocks: a preserved job's steps with these names are replaced by the template's steps of the same names, in template order, where the first of them was. A preserved job with none of them is left alone", + "type": "Optional[Array[String[1]]]" } } } diff --git a/dist/puppetsync/tasks/merge_gha_workflows.rb b/dist/puppetsync/tasks/merge_gha_workflows.rb index 3d82bf2..76bb469 100644 --- a/dist/puppetsync/tasks/merge_gha_workflows.rb +++ b/dist/puppetsync/tasks/merge_gha_workflows.rb @@ -32,6 +32,13 @@ # other job's `needs:`, since GitHub rejects a workflow whose `needs:` # names a job that doesn't exist (the template's aggregate check job # needs every job, acceptance included). +# +# `template_steps` hands named steps back to the template inside those +# preserved blocks: a preserved job's steps with these names are replaced by +# the template's steps of the same names, in template order, at the first +# one's position. A preserved job with none of them keeps its steps, so a +# baseline fix to shared steps (e.g. the Vagrant setup) reaches every repo +# that has them without touching each repo's own matrix and suites. require 'fileutils' require 'json' @@ -102,11 +109,55 @@ def node_at_path(root, path) [key_node, node] end +# The steps of a job mapping as [name, start_line, end_line, column] (end +# exclusive, trailing blank lines left out, column of the step's first key). +# An item ends where the next one starts; the last ends with the sequence. +def job_steps(job, lines) + steps = job.is_a?(Psych::Nodes::Mapping) && node_at_path(job, 'steps') + return [] unless steps && steps[1].is_a?(Psych::Nodes::Sequence) + + items = steps[1].children + items.each_with_index.map do |item, i| + name = item.is_a?(Psych::Nodes::Mapping) && node_at_path(item, 'name') + stop = i + 1 < items.size ? items[i + 1].start_line : item.end_line + stop -= 1 while stop > item.start_line + 1 && lines[stop - 1].strip.empty? + [name && name[1].respond_to?(:value) ? name[1].value : nil, item.start_line, stop, item.start_column] + end +end + +# Within a preserved job block, make the template authoritative for the +# steps named in `names`: the block's steps with those names are replaced +# by the template's, in template order, where the first of them was. A +# block with none of them is returned unchanged, so a job set up some other +# way (podman instead of Vagrant, say) doesn't gain them. +def graft_template_steps(block, block_start, existing_lines, existing_job, template_lines, template_job, names) + existing = job_steps(existing_job, existing_lines).select { |name, _start, _stop, _col| names.include?(name) } + return block if existing.empty? + + template_steps = job_steps(template_job, template_lines).select { |name, _start, _stop, _col| names.include?(name) } + shift = template_steps.empty? ? 0 : existing.first[3] - template_steps.first[3] + replacement = template_steps.flat_map { |_name, start, stop, _col| template_lines[start...stop] }.map do |line| + next line if line.strip.empty? + + shift >= 0 ? (' ' * shift) + line : line.sub(/\A {0,#{-shift}}/, '') + end + + drop = existing.flat_map { |_name, start, stop, _col| ((start - block_start)...(stop - block_start)).to_a } + first = existing.first[1] - block_start + block.each_with_index.flat_map do |line, i| + next replacement if i == first + next [] if drop.include?(i) + + [line] + end +end + # Make the existing file authoritative for whole blocks: splice its version # over the template's, append it when the template lacks it, or remove the # template's when the existing file has none (Psych spans: -# key.start_line ... value.end_line, exclusive) -def preserve_existing_blocks(template, existing_text, paths) +# key.start_line ... value.end_line, exclusive). Steps named in +# `template_steps` stay the template's inside a preserved job block. +def preserve_existing_blocks(template, existing_text, paths, template_steps = []) preserved = [] removed = [] existing_lines = existing_text.split("\n", -1) @@ -127,6 +178,10 @@ def preserve_existing_blocks(template, existing_text, paths) removed << path elsif template_pair block = existing_lines[existing_pair[0].start_line...existing_pair[1].end_line] + unless template_steps.empty? + block = graft_template_steps(block, existing_pair[0].start_line, existing_lines, existing_pair[1], + template_lines, template_pair[1], template_steps) + end next if template_lines[template_pair[0].start_line...template_pair[1].end_line] == block template_lines[template_pair[0].start_line...template_pair[1].end_line] = block @@ -184,8 +239,8 @@ def prune_needs(template, job_names) end end -def merge_workflow(template, existing_text, keys, blocks) - template, preserved_blocks, removed_blocks = preserve_existing_blocks(template, existing_text, blocks) +def merge_workflow(template, existing_text, keys, blocks, steps = []) + template, preserved_blocks, removed_blocks = preserve_existing_blocks(template, existing_text, blocks, steps) removed_jobs = removed_blocks.filter_map { |path| path.delete_prefix('jobs.') if path.match?(/\Ajobs\.[^.]+\z/) } template = prune_needs(template, removed_jobs) existing = existing_values(existing_text, keys) @@ -217,7 +272,7 @@ def merge_workflow(template, existing_text, keys, blocks) [lines.join("\n"), updated, preserved_blocks] end -def merge_gha_workflows(workflows, keys, blocks) +def merge_gha_workflows(workflows, keys, blocks, steps) results = {} workflows.each do |wf| path = wf.fetch('path') @@ -232,7 +287,7 @@ def merge_gha_workflows(workflows, keys, blocks) end existing_text = File.read(path) - merged, updated, preserved_blocks = merge_workflow(template, existing_text, keys, blocks) + merged, updated, preserved_blocks = merge_workflow(template, existing_text, keys, blocks, steps) if merged == existing_text results[path] = { 'changed' => false } @@ -252,5 +307,6 @@ def merge_gha_workflows(workflows, keys, blocks) raise('No workflows given') unless workflows.is_a?(Array) keys = params.fetch('preserve_keys', nil) || DEFAULT_PRESERVE_KEYS blocks = params.fetch('preserve_blocks', nil) || [] +steps = params.fetch('template_steps', nil) || [] -puts JSON.generate(merge_gha_workflows(workflows, keys, blocks)) +puts JSON.generate(merge_gha_workflows(workflows, keys, blocks, steps)) diff --git a/modules/profile/files/pupmod/_github/workflows/pr_tests.simp-simp.yml b/modules/profile/files/pupmod/_github/workflows/pr_tests.simp-simp.yml index 3102366..088e02c 100644 --- a/modules/profile/files/pupmod/_github/workflows/pr_tests.simp-simp.yml +++ b/modules/profile/files/pupmod/_github/workflows/pr_tests.simp-simp.yml @@ -199,12 +199,107 @@ jobs: - name: bundle install run: | bundle install + - name: Choose the Vagrant packages + id: vagrant-debs + run: | + # ppa:evgeni/vagrant has vagrant 2.3.7 (the last MPL-licensed + # release) and a matching vagrant-libvirt for the Ubuntu series in + # ppa_series; Ubuntu dropped both after 23.10. HashiCorp's apt + # repository and rubygems.org are the fallback when the PPA is down + # and the cache is cold, and the only source on other series. + ppa_series='noble' + ppa_vagrant='2.3.7+git20230731.5fc64cde+dfsg-2~ppa+1' + ppa_vagrant_libvirt='0.12.2-1~ppa+1' + . /etc/os-release + { + if [[ " ${ppa_series} " == *" ${VERSION_CODENAME} "* ]]; then + echo 'ppa=true' + else + echo 'ppa=false' + fi + echo "ppa-vagrant=${ppa_vagrant}" + echo "ppa-vagrant-libvirt=${ppa_vagrant_libvirt}" + echo 'hashicorp-vagrant=2.4.9-1' + echo 'vagrant-libvirt-plugin=0.12.2' + echo "codename=${VERSION_CODENAME}" + echo "dir=${RUNNER_TEMP}/vagrant-debs" + echo "key=vagrant-debs-${VERSION_CODENAME}-$(dpkg --print-architecture)-${ppa_vagrant}-${ppa_vagrant_libvirt}" + } >> "$GITHUB_OUTPUT" + - name: Restore the cached Vagrant packages + id: vagrant-cache + uses: actions/cache/restore@v6 + with: + path: ${{ steps.vagrant-debs.outputs.dir }} + key: ${{ steps.vagrant-debs.outputs.key }} - name: Setup libvirt for Vagrant + id: vagrant-setup + env: + CACHE_HIT: ${{ steps.vagrant-cache.outputs.cache-hit }} + CODENAME: ${{ steps.vagrant-debs.outputs.codename }} + DEBS: ${{ steps.vagrant-debs.outputs.dir }} + PPA: ${{ steps.vagrant-debs.outputs.ppa }} + PPA_VAGRANT: ${{ steps.vagrant-debs.outputs.ppa-vagrant }} + PPA_VAGRANT_LIBVIRT: ${{ steps.vagrant-debs.outputs.ppa-vagrant-libvirt }} + HASHICORP_VAGRANT: ${{ steps.vagrant-debs.outputs.hashicorp-vagrant }} + VAGRANT_LIBVIRT_PLUGIN: ${{ steps.vagrant-debs.outputs.vagrant-libvirt-plugin }} run: | - sudo add-apt-repository ppa:evgeni/vagrant - sudo apt-get update - sudo apt-get install -y --no-install-recommends vagrant vagrant-libvirt libvirt-daemon-system libvirt-daemon qemu-system-x86 qemu-utils dnsmasq + # Up to three attempts, 30s and then 60s apart + retry() { + for delay in 30 60 0; do + "$@" && return 0 + if [ "$delay" -eq 0 ]; then + return 1 + fi + echo "::notice::Retrying in ${delay}s: $*" + sleep "$delay" + done + } + + from_ppa() { + retry sudo add-apt-repository -y -n ppa:evgeni/vagrant && + retry sudo apt-get update --error-on=any && + mkdir -p "$DEBS" && + (cd "$DEBS" && retry apt-get download "vagrant=${PPA_VAGRANT}" "vagrant-libvirt=${PPA_VAGRANT_LIBVIRT}") + } + + packages=(libvirt-daemon-system libvirt-daemon qemu-system-x86 qemu-utils dnsmasq) + if [ "$CACHE_HIT" = 'true' ]; then + source=cache + retry sudo apt-get update --error-on=any + elif [ "$PPA" = 'true' ] && from_ppa; then + source=ppa + else + source=hashicorp + fallback="BSL-licensed Vagrant ${HASHICORP_VAGRANT} from HashiCorp and vagrant-libvirt ${VAGRANT_LIBVIRT_PLUGIN} from rubygems.org" + if [ "$PPA" = 'true' ]; then + echo "::warning::ppa:evgeni/vagrant is unavailable, so this job uses ${fallback}" + sudo add-apt-repository -y -n -r ppa:evgeni/vagrant || true + rm -rf "$DEBS" + else + echo "::notice::ppa:evgeni/vagrant has no packages for Ubuntu ${CODENAME}, so this job uses ${fallback}" + fi + retry curl -fsSL -o "${RUNNER_TEMP}/hashicorp.asc" https://apt.releases.hashicorp.com/gpg + sudo gpg --batch --yes --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg "${RUNNER_TEMP}/hashicorp.asc" + echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com ${CODENAME} main" | + sudo tee /etc/apt/sources.list.d/hashicorp.list + retry sudo apt-get update --error-on=any + packages+=("vagrant=${HASHICORP_VAGRANT}" libvirt-dev build-essential pkg-config) + fi + echo "source=${source}" >> "$GITHUB_OUTPUT" + + if [ "$source" = 'hashicorp' ]; then + retry sudo apt-get install -y --no-install-recommends "${packages[@]}" + retry vagrant plugin install vagrant-libvirt --plugin-version "$VAGRANT_LIBVIRT_PLUGIN" + else + retry sudo apt-get install -y --no-install-recommends "$DEBS"/*.deb "${packages[@]}" + fi sudo chmod 666 /var/run/libvirt/libvirt-sock + - name: Cache the Vagrant packages + if: steps.vagrant-setup.outputs.source == 'ppa' + uses: actions/cache/save@v6 + with: + path: ${{ steps.vagrant-debs.outputs.dir }} + key: ${{ steps.vagrant-debs.outputs.key }} - name: beaker env: BEAKER_HYPERVISOR: 'vagrant_libvirt' diff --git a/modules/profile/files/pupmod/_github/workflows/pr_tests.yml b/modules/profile/files/pupmod/_github/workflows/pr_tests.yml index 678173b..0ff52f6 100644 --- a/modules/profile/files/pupmod/_github/workflows/pr_tests.yml +++ b/modules/profile/files/pupmod/_github/workflows/pr_tests.yml @@ -179,12 +179,107 @@ jobs: - name: bundle install run: | bundle install + - name: Choose the Vagrant packages + id: vagrant-debs + run: | + # ppa:evgeni/vagrant has vagrant 2.3.7 (the last MPL-licensed + # release) and a matching vagrant-libvirt for the Ubuntu series in + # ppa_series; Ubuntu dropped both after 23.10. HashiCorp's apt + # repository and rubygems.org are the fallback when the PPA is down + # and the cache is cold, and the only source on other series. + ppa_series='noble' + ppa_vagrant='2.3.7+git20230731.5fc64cde+dfsg-2~ppa+1' + ppa_vagrant_libvirt='0.12.2-1~ppa+1' + . /etc/os-release + { + if [[ " ${ppa_series} " == *" ${VERSION_CODENAME} "* ]]; then + echo 'ppa=true' + else + echo 'ppa=false' + fi + echo "ppa-vagrant=${ppa_vagrant}" + echo "ppa-vagrant-libvirt=${ppa_vagrant_libvirt}" + echo 'hashicorp-vagrant=2.4.9-1' + echo 'vagrant-libvirt-plugin=0.12.2' + echo "codename=${VERSION_CODENAME}" + echo "dir=${RUNNER_TEMP}/vagrant-debs" + echo "key=vagrant-debs-${VERSION_CODENAME}-$(dpkg --print-architecture)-${ppa_vagrant}-${ppa_vagrant_libvirt}" + } >> "$GITHUB_OUTPUT" + - name: Restore the cached Vagrant packages + id: vagrant-cache + uses: actions/cache/restore@v6 + with: + path: ${{ steps.vagrant-debs.outputs.dir }} + key: ${{ steps.vagrant-debs.outputs.key }} - name: Setup libvirt for Vagrant + id: vagrant-setup + env: + CACHE_HIT: ${{ steps.vagrant-cache.outputs.cache-hit }} + CODENAME: ${{ steps.vagrant-debs.outputs.codename }} + DEBS: ${{ steps.vagrant-debs.outputs.dir }} + PPA: ${{ steps.vagrant-debs.outputs.ppa }} + PPA_VAGRANT: ${{ steps.vagrant-debs.outputs.ppa-vagrant }} + PPA_VAGRANT_LIBVIRT: ${{ steps.vagrant-debs.outputs.ppa-vagrant-libvirt }} + HASHICORP_VAGRANT: ${{ steps.vagrant-debs.outputs.hashicorp-vagrant }} + VAGRANT_LIBVIRT_PLUGIN: ${{ steps.vagrant-debs.outputs.vagrant-libvirt-plugin }} run: | - sudo add-apt-repository ppa:evgeni/vagrant - sudo apt-get update - sudo apt-get install -y --no-install-recommends vagrant vagrant-libvirt libvirt-daemon-system libvirt-daemon qemu-system-x86 qemu-utils dnsmasq + # Up to three attempts, 30s and then 60s apart + retry() { + for delay in 30 60 0; do + "$@" && return 0 + if [ "$delay" -eq 0 ]; then + return 1 + fi + echo "::notice::Retrying in ${delay}s: $*" + sleep "$delay" + done + } + + from_ppa() { + retry sudo add-apt-repository -y -n ppa:evgeni/vagrant && + retry sudo apt-get update --error-on=any && + mkdir -p "$DEBS" && + (cd "$DEBS" && retry apt-get download "vagrant=${PPA_VAGRANT}" "vagrant-libvirt=${PPA_VAGRANT_LIBVIRT}") + } + + packages=(libvirt-daemon-system libvirt-daemon qemu-system-x86 qemu-utils dnsmasq) + if [ "$CACHE_HIT" = 'true' ]; then + source=cache + retry sudo apt-get update --error-on=any + elif [ "$PPA" = 'true' ] && from_ppa; then + source=ppa + else + source=hashicorp + fallback="BSL-licensed Vagrant ${HASHICORP_VAGRANT} from HashiCorp and vagrant-libvirt ${VAGRANT_LIBVIRT_PLUGIN} from rubygems.org" + if [ "$PPA" = 'true' ]; then + echo "::warning::ppa:evgeni/vagrant is unavailable, so this job uses ${fallback}" + sudo add-apt-repository -y -n -r ppa:evgeni/vagrant || true + rm -rf "$DEBS" + else + echo "::notice::ppa:evgeni/vagrant has no packages for Ubuntu ${CODENAME}, so this job uses ${fallback}" + fi + retry curl -fsSL -o "${RUNNER_TEMP}/hashicorp.asc" https://apt.releases.hashicorp.com/gpg + sudo gpg --batch --yes --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg "${RUNNER_TEMP}/hashicorp.asc" + echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com ${CODENAME} main" | + sudo tee /etc/apt/sources.list.d/hashicorp.list + retry sudo apt-get update --error-on=any + packages+=("vagrant=${HASHICORP_VAGRANT}" libvirt-dev build-essential pkg-config) + fi + echo "source=${source}" >> "$GITHUB_OUTPUT" + + if [ "$source" = 'hashicorp' ]; then + retry sudo apt-get install -y --no-install-recommends "${packages[@]}" + retry vagrant plugin install vagrant-libvirt --plugin-version "$VAGRANT_LIBVIRT_PLUGIN" + else + retry sudo apt-get install -y --no-install-recommends "$DEBS"/*.deb "${packages[@]}" + fi sudo chmod 666 /var/run/libvirt/libvirt-sock + - name: Cache the Vagrant packages + if: steps.vagrant-setup.outputs.source == 'ppa' + uses: actions/cache/save@v6 + with: + path: ${{ steps.vagrant-debs.outputs.dir }} + key: ${{ steps.vagrant-debs.outputs.key }} - name: beaker env: BEAKER_HYPERVISOR: 'vagrant_libvirt' diff --git a/spec/plans/merge_workflows_stage_spec.rb b/spec/plans/merge_workflows_stage_spec.rb index 2cd039a..1521ae8 100644 --- a/spec/plans/merge_workflows_stage_spec.rb +++ b/spec/plans/merge_workflows_stage_spec.rb @@ -93,6 +93,12 @@ def run_merge_stage(stage_opts = {}) expect(params['preserve_blocks']).to eq(['jobs.acceptance']) end + it 'passes the session-configured template_steps through to the task' do + params = run_merge_stage('merge_github_workflows' => { 'template_steps' => ['Setup libvirt for Vagrant'] }) + + expect(params['template_steps']).to eq(['Setup libvirt for Vagrant']) + end + it 'classifies a repo with a gemspec as a rubygem, excluding it from the merge' do # A gem can carry a pupmod-shaped metadata.json (compliance_engine does); # the gemspec must win or the gem's custom workflows get flattened diff --git a/spec/tasks/merge_gha_workflows_spec.rb b/spec/tasks/merge_gha_workflows_spec.rb index 869f11f..f52ad1e 100644 --- a/spec/tasks/merge_gha_workflows_spec.rb +++ b/spec/tasks/merge_gha_workflows_spec.rb @@ -13,9 +13,10 @@ end end - def run_merge(workflows, preserve_blocks: nil) + def run_merge(workflows, preserve_blocks: nil, template_steps: nil) params = { 'workflows' => workflows } params['preserve_blocks'] = preserve_blocks if preserve_blocks + params['template_steps'] = template_steps if template_steps run_task('merge_gha_workflows.rb', params) end @@ -443,6 +444,98 @@ def run_merge(workflows, preserve_blocks: nil) end end + context 'with template_steps' do + let(:vagrant_steps) do + ['Choose the Vagrant packages', 'Restore the cached Vagrant packages', + 'Setup libvirt for Vagrant', 'Cache the Vagrant packages'] + end + + # The deployed fleet's acceptance job before the Vagrant package cache: + # its own step names, matrix and suites, and the one-step PPA install + let(:fleet_acceptance) do + <<~YAML.gsub(/^(?=.)/, ' ') + acceptance: + runs-on: + - ubuntu-latest + strategy: + matrix: + node: + - oraclelinux8 + steps: + - name: checkout repo + uses: actions/checkout@v7 + - name: Setup libvirt for Vagrant + run: | + sudo add-apt-repository ppa:evgeni/vagrant + sudo apt-get update + + - name: beaker + run: | + bundle exec rake beaker:suites[custom,${{ matrix.node }}] + YAML + end + + def merge_acceptance(existing) + File.write(@wf, real_template.sub(/^ acceptance:.*\z/m, existing)) + run_merge([{ 'path' => @wf, 'template' => real_template }], + preserve_blocks: ['jobs.acceptance'], template_steps: vagrant_steps) + end + + def acceptance_steps + Psych.safe_load(File.read(@wf))['jobs']['acceptance']['steps'] + end + + it "replaces the named steps with the template's, keeping the rest of the job" do + stdout, stderr, status = merge_acceptance(fleet_acceptance) + + expect(status).to be_success, stderr + expect(JSON.parse(stdout)['files'][@wf]['changed']).to be true + expect(acceptance_steps.map { |s| s['name'] }).to eq(['checkout repo', *vagrant_steps, 'beaker']) + template_steps = Psych.safe_load(real_template)['jobs']['acceptance']['steps'] + expect(acceptance_steps[1..4]).to eq(template_steps.select { |s| vagrant_steps.include?(s['name']) }) + merged = File.read(@wf) + expect(merged).to include(" node:\n - oraclelinux8\n") + expect(merged).to include('beaker:suites[custom,${{ matrix.node }}]') + expect(merged).not_to include('sudo add-apt-repository ppa:evgeni/vagrant') + # The blank line after the replaced step is kept, after the last new one + expect(merged).to include("outputs.key }}\n\n - name: beaker\n") + end + + it 'is idempotent' do + merge_acceptance(fleet_acceptance) + first_pass = File.read(@wf) + + stdout, stderr, status = run_merge([{ 'path' => @wf, 'template' => real_template }], + preserve_blocks: ['jobs.acceptance'], template_steps: vagrant_steps) + + expect(status).to be_success, stderr + expect(JSON.parse(stdout)['changed']).to be false + expect(File.read(@wf)).to eq(first_pass) + end + + it "leaves a job that doesn't have the named steps alone" do + podman = fleet_acceptance.sub(/- name: Setup libvirt.*?\n\n/m, <<~YAML.gsub(/^/, ' ').lstrip) + - name: Setup podman + run: | + systemctl start --user podman.socket + YAML + stdout, stderr, status = merge_acceptance(podman) + + expect(status).to be_success, stderr + expect(JSON.parse(stdout)['files'][@wf]['changed']).to be false + expect(acceptance_steps.map { |s| s['name'] }).to eq(['checkout repo', 'Setup podman', 'beaker']) + end + + it 'reindents the template steps to match the job' do + # Steps indented two spaces less than the template's + stdout, stderr, status = merge_acceptance(fleet_acceptance.gsub(/^( )(.*)/) { " #{Regexp.last_match(2)}" }) + + expect(status).to be_success, stderr + expect(acceptance_steps.map { |s| s['name'] }).to eq(['checkout repo', *vagrant_steps, 'beaker']) + expect(File.read(@wf)).to include("\n - name: Choose the Vagrant packages\n id: vagrant-debs\n") + end + end + it 'fails when workflows param is missing' do _stdout, _stderr, status = run_task('merge_gha_workflows.rb', {}) expect(status).not_to be_success From da7ace8b54173c4afecbc324b122385e4b8e1c0e Mon Sep 17 00:00:00 2001 From: Mike Riddle Date: Mon, 2 Nov 2026 15:57:06 -0500 Subject: [PATCH 2/4] Address review: use a shared setup action and warm its cache from the default branch - Install Vagrant with simp/github-action-setup-vagrant-libvirt, which restores the PPA's packages from the Actions cache, then tries the PPA, then HashiCorp. It caches every package the PPA provides, unpinned, with the key derived from their file names; refreshes only the PPA's index with --error-on=any; and removes the PPA's sources file directly. - Add vagrant_package_cache.yml, which saves the cache from the default branch, where pull requests can restore it. - merge_gha_workflows template_steps: replace named steps in place, keep comments that belong to the next step, and fail when the template lacks a named step instead of deleting it. - Sync to a repolist of the 36 Vagrant repos only. Co-Authored-By: Claude Opus 5.5 --- .../20261102-vagrant-ppa-fallback.yaml | 36 ++++-- data/sync/repolists/latest.yaml | 2 +- .../repolists/pupmods_vagrant_acceptance.yaml | 113 ++++++++++++++++++ .../puppetsync/tasks/merge_gha_workflows.json | 2 +- dist/puppetsync/tasks/merge_gha_workflows.rb | 72 ++++++++--- .../_github/workflows/pr_tests.simp-simp.yml | 101 +--------------- .../pupmod/_github/workflows/pr_tests.yml | 101 +--------------- .../workflows/vagrant_package_cache.yml | 37 ++++++ spec/tasks/merge_gha_workflows_spec.rb | 98 +++++++++++---- 9 files changed, 308 insertions(+), 254 deletions(-) create mode 100644 data/sync/repolists/pupmods_vagrant_acceptance.yaml create mode 100644 modules/profile/files/pupmod/_github/workflows/vagrant_package_cache.yml diff --git a/data/sync/configs/20261102-vagrant-ppa-fallback.yaml b/data/sync/configs/20261102-vagrant-ppa-fallback.yaml index 6080683..78d3699 100644 --- a/data/sync/configs/20261102-vagrant-ppa-fallback.yaml +++ b/data/sync/configs/20261102-vagrant-ppa-fallback.yaml @@ -5,15 +5,22 @@ # The acceptance job's "Setup libvirt for Vagrant" step installed vagrant # and vagrant-libvirt from ppa:evgeni/vagrant alone, so a Launchpad outage # failed every acceptance job and, through 'All checks', blocked merges. -# The step now restores the PPA's .debs from the Actions cache, retries the -# PPA with backoff, and falls back to HashiCorp's apt repository and -# `vagrant plugin install vagrant-libvirt`. +# The step now uses simp/github-action-setup-vagrant-libvirt, which +# installs from the Actions cache, then the PPA (with retries), then +# HashiCorp's apt repository. +# +# A pull request can only restore caches saved from the default branch (or +# its own ref or base branch), so each repo also gets +# vagrant_package_cache.yml, which saves the PPA's packages from the +# default branch on push and twice a week. +# +# Run it with repolist=pupmods_vagrant_acceptance: ensure_files adds +# vagrant_package_cache.yml to every repo in the session, and modules whose +# acceptance job uses podman, or that have none, don't need it. # # pr_tests.yml (merge_github_workflows): each repo's `jobs.acceptance` block -# is kept as-is, except for the Vagrant setup steps named in -# template_steps, which are replaced with the template's. Modules whose -# acceptance job uses podman, and modules with no acceptance job, are left -# unchanged. +# is kept as-is, except for the step named in template_steps, which is +# replaced with the template's. puppetsync::plan_config: puppetsync: permitted_project_types: @@ -25,13 +32,13 @@ puppetsync::plan_config: merge_github_workflows: files: - pr_tests.yml + - vagrant_package_cache.yml + ensure_files: + - vagrant_package_cache.yml preserve_blocks: - jobs.acceptance template_steps: - - Choose the Vagrant packages - - Restore the cached Vagrant packages - Setup libvirt for Vagrant - - Cache the Vagrant packages stages: - install_gems - checkout_git_feature_branch_in_each_repo @@ -57,9 +64,12 @@ puppetsync::plan_config: The acceptance job installed Vagrant and vagrant-libvirt from ppa:evgeni/vagrant alone, so a Launchpad outage failed every - acceptance run. The setup now installs the PPA's packages from the - Actions cache when it can, retries the PPA with backoff, and falls - back to HashiCorp's Vagrant and the vagrant-libvirt plugin. + acceptance run. The setup step now uses + simp/github-action-setup-vagrant-libvirt, which installs the PPA's + packages from the Actions cache when it can, retries the PPA, and + falls back to HashiCorp's Vagrant and the vagrant-libvirt plugin. + The new 'Vagrant package cache' workflow keeps the packages cached + from the default branch, where pull requests can restore them. github: pr_user: silug # This should be the account that *submitted* the PRs diff --git a/data/sync/repolists/latest.yaml b/data/sync/repolists/latest.yaml index b36c9b8..d87854e 120000 --- a/data/sync/repolists/latest.yaml +++ b/data/sync/repolists/latest.yaml @@ -1 +1 @@ -github-org.yaml \ No newline at end of file +pupmods_vagrant_acceptance.yaml \ No newline at end of file diff --git a/data/sync/repolists/pupmods_vagrant_acceptance.yaml b/data/sync/repolists/pupmods_vagrant_acceptance.yaml new file mode 100644 index 0000000..486570d --- /dev/null +++ b/data/sync/repolists/pupmods_vagrant_acceptance.yaml @@ -0,0 +1,113 @@ +--- +# pupmod repos whose acceptance job runs Beaker on Vagrant with libvirt +# ("Setup libvirt for Vagrant" in pr_tests.yml), as of 2026-11-02. +# The rest of pupmods.yaml run acceptance on podman or have none. +puppetsync::repos_config: + + https://github.com/simp/pupmod-simp-acpid: + branch: master + + https://github.com/simp/pupmod-simp-aide: + branch: master + + https://github.com/simp/pupmod-simp-auditd: + branch: master + + https://github.com/simp/pupmod-simp-autofs: + branch: master + + https://github.com/simp/pupmod-simp-ds389: + branch: master + + https://github.com/simp/pupmod-simp-fips: + branch: master + + https://github.com/simp/pupmod-simp-gdm: + branch: master + + https://github.com/simp/pupmod-simp-ima: + branch: master + + https://github.com/simp/pupmod-simp-issue: + branch: master + + https://github.com/simp/pupmod-simp-libreswan: + branch: master + + https://github.com/simp/pupmod-simp-logrotate: + branch: master + + https://github.com/simp/pupmod-simp-mozilla: + branch: master + + https://github.com/simp/pupmod-simp-nfs: + branch: master + + https://github.com/simp/pupmod-simp-oath: + branch: master + + https://github.com/simp/pupmod-simp-oddjob: + branch: master + + https://github.com/simp/pupmod-simp-pam: + branch: master + + https://github.com/simp/pupmod-simp-pki: + branch: master + + https://github.com/simp/pupmod-simp-pupmod: + branch: master + + https://github.com/simp/pupmod-simp-rsync: + branch: master + + https://github.com/simp/pupmod-simp-rsyslog: + branch: master + + https://github.com/simp/pupmod-simp-selinux: + branch: master + + https://github.com/simp/pupmod-simp-simp: + branch: master + + https://github.com/simp/pupmod-simp-simp_apache: + branch: master + + https://github.com/simp/pupmod-simp-simp_ds389: + branch: master + + https://github.com/simp/pupmod-simp-simp_gitlab: + branch: master + + https://github.com/simp/pupmod-simp-simp_nfs: + branch: master + + https://github.com/simp/pupmod-simp-simp_rsyslog: + branch: master + + https://github.com/simp/pupmod-simp-simp_snmpd: + branch: master + + https://github.com/simp/pupmod-simp-ssh: + branch: master + + https://github.com/simp/pupmod-simp-sssd: + branch: master + + https://github.com/simp/pupmod-simp-stunnel: + branch: master + + https://github.com/simp/pupmod-simp-sudo: + branch: master + + https://github.com/simp/pupmod-simp-svckill: + branch: master + + https://github.com/simp/pupmod-simp-swap: + branch: master + + https://github.com/simp/pupmod-simp-tlog: + branch: master + + https://github.com/simp/pupmod-simp-vsftpd: + branch: master diff --git a/dist/puppetsync/tasks/merge_gha_workflows.json b/dist/puppetsync/tasks/merge_gha_workflows.json index af69cf7..98bdc20 100644 --- a/dist/puppetsync/tasks/merge_gha_workflows.json +++ b/dist/puppetsync/tasks/merge_gha_workflows.json @@ -15,7 +15,7 @@ "type": "Optional[Array[String[1]]]" }, "template_steps": { - "description": "Names of steps the template stays authoritative for inside preserved_blocks: a preserved job's steps with these names are replaced by the template's steps of the same names, in template order, where the first of them was. A preserved job with none of them is left alone", + "description": "Names of steps the template stays authoritative for inside preserved_blocks: each of a preserved job's steps with one of these names is replaced in place by the template's step of that name, and named steps the job lacks go in next to their template neighbours. A preserved job with none of them is left alone. Fails if the template's job lacks a name", "type": "Optional[Array[String[1]]]" } } diff --git a/dist/puppetsync/tasks/merge_gha_workflows.rb b/dist/puppetsync/tasks/merge_gha_workflows.rb index 76bb469..60d7892 100644 --- a/dist/puppetsync/tasks/merge_gha_workflows.rb +++ b/dist/puppetsync/tasks/merge_gha_workflows.rb @@ -34,11 +34,13 @@ # needs every job, acceptance included). # # `template_steps` hands named steps back to the template inside those -# preserved blocks: a preserved job's steps with these names are replaced by -# the template's steps of the same names, in template order, at the first -# one's position. A preserved job with none of them keeps its steps, so a -# baseline fix to shared steps (e.g. the Vagrant setup) reaches every repo -# that has them without touching each repo's own matrix and suites. +# preserved blocks: each of a preserved job's steps with one of these names +# is replaced in place by the template's step of that name, and named steps +# the job lacks go in next to their template neighbours. A preserved job +# with none of them keeps its steps, so a baseline fix to shared steps +# (e.g. the Vagrant setup) reaches every repo that has them without +# touching each repo's own matrix, suites and steps. A name the template +# lacks is an error rather than a deletion. require 'fileutils' require 'json' @@ -110,8 +112,9 @@ def node_at_path(root, path) end # The steps of a job mapping as [name, start_line, end_line, column] (end -# exclusive, trailing blank lines left out, column of the step's first key). -# An item ends where the next one starts; the last ends with the sequence. +# exclusive, column of the step's first key). An item ends where the next +# one starts, or the last with the sequence, less any trailing blank lines +# and comments outdented from its keys: those belong to whatever follows. def job_steps(job, lines) steps = job.is_a?(Psych::Nodes::Mapping) && node_at_path(job, 'steps') return [] unless steps && steps[1].is_a?(Psych::Nodes::Sequence) @@ -120,32 +123,65 @@ def job_steps(job, lines) items.each_with_index.map do |item, i| name = item.is_a?(Psych::Nodes::Mapping) && node_at_path(item, 'name') stop = i + 1 < items.size ? items[i + 1].start_line : item.end_line - stop -= 1 while stop > item.start_line + 1 && lines[stop - 1].strip.empty? + while stop > item.start_line + 1 + line = lines[stop - 1] + break unless line.strip.empty? || (line.lstrip.start_with?('#') && line[/\A */].size <= item.start_column) + + stop -= 1 + end [name && name[1].respond_to?(:value) ? name[1].value : nil, item.start_line, stop, item.start_column] end end # Within a preserved job block, make the template authoritative for the -# steps named in `names`: the block's steps with those names are replaced -# by the template's, in template order, where the first of them was. A -# block with none of them is returned unchanged, so a job set up some other -# way (podman instead of Vagrant, say) doesn't gain them. +# steps named in `names`. Each of the block's steps with one of those names +# is replaced in place by the template's step of that name; a named step +# the block lacks goes in next to the template step before it (or after +# it), so the block's own steps keep their positions. A block with none of +# the named steps is returned unchanged, so a job set up some other way +# (podman instead of Vagrant, say) doesn't gain them. def graft_template_steps(block, block_start, existing_lines, existing_job, template_lines, template_job, names) existing = job_steps(existing_job, existing_lines).select { |name, _start, _stop, _col| names.include?(name) } return block if existing.empty? template_steps = job_steps(template_job, template_lines).select { |name, _start, _stop, _col| names.include?(name) } - shift = template_steps.empty? ? 0 : existing.first[3] - template_steps.first[3] - replacement = template_steps.flat_map { |_name, start, stop, _col| template_lines[start...stop] }.map do |line| - next line if line.strip.empty? + missing = names - template_steps.map(&:first) + # Replacing with nothing would silently delete the repo's steps + raise "template_steps not in the template's job: #{missing.join(', ')}" unless missing.empty? + + shift = existing.first[3] - template_steps.first[3] + text = template_steps.to_h do |name, start, stop, _col| + lines = template_lines[start...stop].map do |line| + next line if line.strip.empty? + + shift >= 0 ? (' ' * shift) + line : line.sub(/\A {0,#{-shift}}/, '') + end + [name, lines] + end - shift >= 0 ? (' ' * shift) + line : line.sub(/\A {0,#{-shift}}/, '') + # Each existing named step's span is replaced by a chunk of template + # steps: its own, plus any the block lacks that follow it in the template + # (or, before the first one the block has, precede it) + have = existing.map(&:first) + chunks = Hash.new { |h, k| h[k] = [] } + anchor = nil + pending = [] + template_steps.each do |name, *| + if have.include?(name) + chunks[name].concat(pending.flat_map { |n| text[n] }, text[name]) + pending = [] + anchor = name + elsif anchor + chunks[anchor].concat(text[name]) + else + pending << name + end end + starts = existing.to_h { |name, start, _stop, _col| [start - block_start, name] } drop = existing.flat_map { |_name, start, stop, _col| ((start - block_start)...(stop - block_start)).to_a } - first = existing.first[1] - block_start block.each_with_index.flat_map do |line, i| - next replacement if i == first + next chunks[starts[i]] if starts.key?(i) next [] if drop.include?(i) [line] diff --git a/modules/profile/files/pupmod/_github/workflows/pr_tests.simp-simp.yml b/modules/profile/files/pupmod/_github/workflows/pr_tests.simp-simp.yml index 088e02c..d204251 100644 --- a/modules/profile/files/pupmod/_github/workflows/pr_tests.simp-simp.yml +++ b/modules/profile/files/pupmod/_github/workflows/pr_tests.simp-simp.yml @@ -199,107 +199,8 @@ jobs: - name: bundle install run: | bundle install - - name: Choose the Vagrant packages - id: vagrant-debs - run: | - # ppa:evgeni/vagrant has vagrant 2.3.7 (the last MPL-licensed - # release) and a matching vagrant-libvirt for the Ubuntu series in - # ppa_series; Ubuntu dropped both after 23.10. HashiCorp's apt - # repository and rubygems.org are the fallback when the PPA is down - # and the cache is cold, and the only source on other series. - ppa_series='noble' - ppa_vagrant='2.3.7+git20230731.5fc64cde+dfsg-2~ppa+1' - ppa_vagrant_libvirt='0.12.2-1~ppa+1' - . /etc/os-release - { - if [[ " ${ppa_series} " == *" ${VERSION_CODENAME} "* ]]; then - echo 'ppa=true' - else - echo 'ppa=false' - fi - echo "ppa-vagrant=${ppa_vagrant}" - echo "ppa-vagrant-libvirt=${ppa_vagrant_libvirt}" - echo 'hashicorp-vagrant=2.4.9-1' - echo 'vagrant-libvirt-plugin=0.12.2' - echo "codename=${VERSION_CODENAME}" - echo "dir=${RUNNER_TEMP}/vagrant-debs" - echo "key=vagrant-debs-${VERSION_CODENAME}-$(dpkg --print-architecture)-${ppa_vagrant}-${ppa_vagrant_libvirt}" - } >> "$GITHUB_OUTPUT" - - name: Restore the cached Vagrant packages - id: vagrant-cache - uses: actions/cache/restore@v6 - with: - path: ${{ steps.vagrant-debs.outputs.dir }} - key: ${{ steps.vagrant-debs.outputs.key }} - name: Setup libvirt for Vagrant - id: vagrant-setup - env: - CACHE_HIT: ${{ steps.vagrant-cache.outputs.cache-hit }} - CODENAME: ${{ steps.vagrant-debs.outputs.codename }} - DEBS: ${{ steps.vagrant-debs.outputs.dir }} - PPA: ${{ steps.vagrant-debs.outputs.ppa }} - PPA_VAGRANT: ${{ steps.vagrant-debs.outputs.ppa-vagrant }} - PPA_VAGRANT_LIBVIRT: ${{ steps.vagrant-debs.outputs.ppa-vagrant-libvirt }} - HASHICORP_VAGRANT: ${{ steps.vagrant-debs.outputs.hashicorp-vagrant }} - VAGRANT_LIBVIRT_PLUGIN: ${{ steps.vagrant-debs.outputs.vagrant-libvirt-plugin }} - run: | - # Up to three attempts, 30s and then 60s apart - retry() { - for delay in 30 60 0; do - "$@" && return 0 - if [ "$delay" -eq 0 ]; then - return 1 - fi - echo "::notice::Retrying in ${delay}s: $*" - sleep "$delay" - done - } - - from_ppa() { - retry sudo add-apt-repository -y -n ppa:evgeni/vagrant && - retry sudo apt-get update --error-on=any && - mkdir -p "$DEBS" && - (cd "$DEBS" && retry apt-get download "vagrant=${PPA_VAGRANT}" "vagrant-libvirt=${PPA_VAGRANT_LIBVIRT}") - } - - packages=(libvirt-daemon-system libvirt-daemon qemu-system-x86 qemu-utils dnsmasq) - if [ "$CACHE_HIT" = 'true' ]; then - source=cache - retry sudo apt-get update --error-on=any - elif [ "$PPA" = 'true' ] && from_ppa; then - source=ppa - else - source=hashicorp - fallback="BSL-licensed Vagrant ${HASHICORP_VAGRANT} from HashiCorp and vagrant-libvirt ${VAGRANT_LIBVIRT_PLUGIN} from rubygems.org" - if [ "$PPA" = 'true' ]; then - echo "::warning::ppa:evgeni/vagrant is unavailable, so this job uses ${fallback}" - sudo add-apt-repository -y -n -r ppa:evgeni/vagrant || true - rm -rf "$DEBS" - else - echo "::notice::ppa:evgeni/vagrant has no packages for Ubuntu ${CODENAME}, so this job uses ${fallback}" - fi - retry curl -fsSL -o "${RUNNER_TEMP}/hashicorp.asc" https://apt.releases.hashicorp.com/gpg - sudo gpg --batch --yes --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg "${RUNNER_TEMP}/hashicorp.asc" - echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com ${CODENAME} main" | - sudo tee /etc/apt/sources.list.d/hashicorp.list - retry sudo apt-get update --error-on=any - packages+=("vagrant=${HASHICORP_VAGRANT}" libvirt-dev build-essential pkg-config) - fi - echo "source=${source}" >> "$GITHUB_OUTPUT" - - if [ "$source" = 'hashicorp' ]; then - retry sudo apt-get install -y --no-install-recommends "${packages[@]}" - retry vagrant plugin install vagrant-libvirt --plugin-version "$VAGRANT_LIBVIRT_PLUGIN" - else - retry sudo apt-get install -y --no-install-recommends "$DEBS"/*.deb "${packages[@]}" - fi - sudo chmod 666 /var/run/libvirt/libvirt-sock - - name: Cache the Vagrant packages - if: steps.vagrant-setup.outputs.source == 'ppa' - uses: actions/cache/save@v6 - with: - path: ${{ steps.vagrant-debs.outputs.dir }} - key: ${{ steps.vagrant-debs.outputs.key }} + uses: simp/github-action-setup-vagrant-libvirt@v1.0.0 - name: beaker env: BEAKER_HYPERVISOR: 'vagrant_libvirt' diff --git a/modules/profile/files/pupmod/_github/workflows/pr_tests.yml b/modules/profile/files/pupmod/_github/workflows/pr_tests.yml index 0ff52f6..1faa08a 100644 --- a/modules/profile/files/pupmod/_github/workflows/pr_tests.yml +++ b/modules/profile/files/pupmod/_github/workflows/pr_tests.yml @@ -179,107 +179,8 @@ jobs: - name: bundle install run: | bundle install - - name: Choose the Vagrant packages - id: vagrant-debs - run: | - # ppa:evgeni/vagrant has vagrant 2.3.7 (the last MPL-licensed - # release) and a matching vagrant-libvirt for the Ubuntu series in - # ppa_series; Ubuntu dropped both after 23.10. HashiCorp's apt - # repository and rubygems.org are the fallback when the PPA is down - # and the cache is cold, and the only source on other series. - ppa_series='noble' - ppa_vagrant='2.3.7+git20230731.5fc64cde+dfsg-2~ppa+1' - ppa_vagrant_libvirt='0.12.2-1~ppa+1' - . /etc/os-release - { - if [[ " ${ppa_series} " == *" ${VERSION_CODENAME} "* ]]; then - echo 'ppa=true' - else - echo 'ppa=false' - fi - echo "ppa-vagrant=${ppa_vagrant}" - echo "ppa-vagrant-libvirt=${ppa_vagrant_libvirt}" - echo 'hashicorp-vagrant=2.4.9-1' - echo 'vagrant-libvirt-plugin=0.12.2' - echo "codename=${VERSION_CODENAME}" - echo "dir=${RUNNER_TEMP}/vagrant-debs" - echo "key=vagrant-debs-${VERSION_CODENAME}-$(dpkg --print-architecture)-${ppa_vagrant}-${ppa_vagrant_libvirt}" - } >> "$GITHUB_OUTPUT" - - name: Restore the cached Vagrant packages - id: vagrant-cache - uses: actions/cache/restore@v6 - with: - path: ${{ steps.vagrant-debs.outputs.dir }} - key: ${{ steps.vagrant-debs.outputs.key }} - name: Setup libvirt for Vagrant - id: vagrant-setup - env: - CACHE_HIT: ${{ steps.vagrant-cache.outputs.cache-hit }} - CODENAME: ${{ steps.vagrant-debs.outputs.codename }} - DEBS: ${{ steps.vagrant-debs.outputs.dir }} - PPA: ${{ steps.vagrant-debs.outputs.ppa }} - PPA_VAGRANT: ${{ steps.vagrant-debs.outputs.ppa-vagrant }} - PPA_VAGRANT_LIBVIRT: ${{ steps.vagrant-debs.outputs.ppa-vagrant-libvirt }} - HASHICORP_VAGRANT: ${{ steps.vagrant-debs.outputs.hashicorp-vagrant }} - VAGRANT_LIBVIRT_PLUGIN: ${{ steps.vagrant-debs.outputs.vagrant-libvirt-plugin }} - run: | - # Up to three attempts, 30s and then 60s apart - retry() { - for delay in 30 60 0; do - "$@" && return 0 - if [ "$delay" -eq 0 ]; then - return 1 - fi - echo "::notice::Retrying in ${delay}s: $*" - sleep "$delay" - done - } - - from_ppa() { - retry sudo add-apt-repository -y -n ppa:evgeni/vagrant && - retry sudo apt-get update --error-on=any && - mkdir -p "$DEBS" && - (cd "$DEBS" && retry apt-get download "vagrant=${PPA_VAGRANT}" "vagrant-libvirt=${PPA_VAGRANT_LIBVIRT}") - } - - packages=(libvirt-daemon-system libvirt-daemon qemu-system-x86 qemu-utils dnsmasq) - if [ "$CACHE_HIT" = 'true' ]; then - source=cache - retry sudo apt-get update --error-on=any - elif [ "$PPA" = 'true' ] && from_ppa; then - source=ppa - else - source=hashicorp - fallback="BSL-licensed Vagrant ${HASHICORP_VAGRANT} from HashiCorp and vagrant-libvirt ${VAGRANT_LIBVIRT_PLUGIN} from rubygems.org" - if [ "$PPA" = 'true' ]; then - echo "::warning::ppa:evgeni/vagrant is unavailable, so this job uses ${fallback}" - sudo add-apt-repository -y -n -r ppa:evgeni/vagrant || true - rm -rf "$DEBS" - else - echo "::notice::ppa:evgeni/vagrant has no packages for Ubuntu ${CODENAME}, so this job uses ${fallback}" - fi - retry curl -fsSL -o "${RUNNER_TEMP}/hashicorp.asc" https://apt.releases.hashicorp.com/gpg - sudo gpg --batch --yes --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg "${RUNNER_TEMP}/hashicorp.asc" - echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com ${CODENAME} main" | - sudo tee /etc/apt/sources.list.d/hashicorp.list - retry sudo apt-get update --error-on=any - packages+=("vagrant=${HASHICORP_VAGRANT}" libvirt-dev build-essential pkg-config) - fi - echo "source=${source}" >> "$GITHUB_OUTPUT" - - if [ "$source" = 'hashicorp' ]; then - retry sudo apt-get install -y --no-install-recommends "${packages[@]}" - retry vagrant plugin install vagrant-libvirt --plugin-version "$VAGRANT_LIBVIRT_PLUGIN" - else - retry sudo apt-get install -y --no-install-recommends "$DEBS"/*.deb "${packages[@]}" - fi - sudo chmod 666 /var/run/libvirt/libvirt-sock - - name: Cache the Vagrant packages - if: steps.vagrant-setup.outputs.source == 'ppa' - uses: actions/cache/save@v6 - with: - path: ${{ steps.vagrant-debs.outputs.dir }} - key: ${{ steps.vagrant-debs.outputs.key }} + uses: simp/github-action-setup-vagrant-libvirt@v1.0.0 - name: beaker env: BEAKER_HYPERVISOR: 'vagrant_libvirt' diff --git a/modules/profile/files/pupmod/_github/workflows/vagrant_package_cache.yml b/modules/profile/files/pupmod/_github/workflows/vagrant_package_cache.yml new file mode 100644 index 0000000..dd17035 --- /dev/null +++ b/modules/profile/files/pupmod/_github/workflows/vagrant_package_cache.yml @@ -0,0 +1,37 @@ +# Keep ppa:evgeni/vagrant's packages in the Actions cache for acceptance +# ------------------------------------------------------------------------------ +# +# NOTICE: **This file is maintained with puppetsync** +# +# This file is updated automatically as part of a puppet module baseline. +# +# The next baseline sync will overwrite any local changes to this file! +# +# ============================================================================== +# +# The acceptance job's "Setup libvirt for Vagrant" step installs from this +# cache when it can, so a Launchpad outage doesn't fail it. A pull request +# can only restore caches saved from its own ref, its base branch or the +# default branch, so the cache is saved here, from the default branch. The +# runner must be on an Ubuntu series the PPA publishes for, and the +# scheduled runs keep the entry from being evicted after seven days unused. +# See simp/puppetsync#122. +# +--- +name: 'Vagrant package cache' + +'on': + push: + branches: [master, main] + schedule: + - cron: '17 6 * * 1,4' + workflow_dispatch: {} + +jobs: + cache: + name: 'Cache the Vagrant packages' + runs-on: ubuntu-24.04 + steps: + - uses: simp/github-action-setup-vagrant-libvirt@v1.0.0 + with: + cache-only: 'true' diff --git a/spec/tasks/merge_gha_workflows_spec.rb b/spec/tasks/merge_gha_workflows_spec.rb index f52ad1e..89b1136 100644 --- a/spec/tasks/merge_gha_workflows_spec.rb +++ b/spec/tasks/merge_gha_workflows_spec.rb @@ -445,13 +445,8 @@ def run_merge(workflows, preserve_blocks: nil, template_steps: nil) end context 'with template_steps' do - let(:vagrant_steps) do - ['Choose the Vagrant packages', 'Restore the cached Vagrant packages', - 'Setup libvirt for Vagrant', 'Cache the Vagrant packages'] - end - - # The deployed fleet's acceptance job before the Vagrant package cache: - # its own step names, matrix and suites, and the one-step PPA install + # The deployed fleet's acceptance job before the setup action: its own + # step names, matrix and suites, and the one-step PPA install let(:fleet_acceptance) do <<~YAML.gsub(/^(?=.)/, ' ') acceptance: @@ -467,38 +462,42 @@ def run_merge(workflows, preserve_blocks: nil, template_steps: nil) - name: Setup libvirt for Vagrant run: | sudo add-apt-repository ppa:evgeni/vagrant + # the next line is part of the script sudo apt-get update + # Run the custom suite; see #123 - name: beaker run: | bundle exec rake beaker:suites[custom,${{ matrix.node }}] YAML end - def merge_acceptance(existing) - File.write(@wf, real_template.sub(/^ acceptance:.*\z/m, existing)) - run_merge([{ 'path' => @wf, 'template' => real_template }], - preserve_blocks: ['jobs.acceptance'], template_steps: vagrant_steps) + def merge_acceptance(existing, template: real_template, names: ['Setup libvirt for Vagrant']) + File.write(@wf, template.sub(/^ acceptance:.*\z/m, existing)) + run_merge([{ 'path' => @wf, 'template' => template }], + preserve_blocks: ['jobs.acceptance'], template_steps: names) end def acceptance_steps Psych.safe_load(File.read(@wf))['jobs']['acceptance']['steps'] end - it "replaces the named steps with the template's, keeping the rest of the job" do + it "replaces the named step with the template's, keeping the rest of the job" do stdout, stderr, status = merge_acceptance(fleet_acceptance) expect(status).to be_success, stderr expect(JSON.parse(stdout)['files'][@wf]['changed']).to be true - expect(acceptance_steps.map { |s| s['name'] }).to eq(['checkout repo', *vagrant_steps, 'beaker']) - template_steps = Psych.safe_load(real_template)['jobs']['acceptance']['steps'] - expect(acceptance_steps[1..4]).to eq(template_steps.select { |s| vagrant_steps.include?(s['name']) }) + expect(acceptance_steps.map { |s| s['name'] }).to eq(['checkout repo', 'Setup libvirt for Vagrant', 'beaker']) + template_step = Psych.safe_load(real_template)['jobs']['acceptance']['steps'] + .find { |s| s['name'] == 'Setup libvirt for Vagrant' } + expect(acceptance_steps[1]).to eq(template_step) merged = File.read(@wf) expect(merged).to include(" node:\n - oraclelinux8\n") expect(merged).to include('beaker:suites[custom,${{ matrix.node }}]') - expect(merged).not_to include('sudo add-apt-repository ppa:evgeni/vagrant') - # The blank line after the replaced step is kept, after the last new one - expect(merged).to include("outputs.key }}\n\n - name: beaker\n") + expect(merged).not_to include('ppa:evgeni/vagrant') + expect(merged).not_to include('the next line is part of the script') + # The blank line and the next step's comment stay with the next step + expect(merged).to include("@v1.0.0\n\n # Run the custom suite; see #123\n - name: beaker\n") end it 'is idempotent' do @@ -506,7 +505,8 @@ def acceptance_steps first_pass = File.read(@wf) stdout, stderr, status = run_merge([{ 'path' => @wf, 'template' => real_template }], - preserve_blocks: ['jobs.acceptance'], template_steps: vagrant_steps) + preserve_blocks: ['jobs.acceptance'], + template_steps: ['Setup libvirt for Vagrant']) expect(status).to be_success, stderr expect(JSON.parse(stdout)['changed']).to be false @@ -518,6 +518,7 @@ def acceptance_steps - name: Setup podman run: | systemctl start --user podman.socket + YAML stdout, stderr, status = merge_acceptance(podman) @@ -531,8 +532,63 @@ def acceptance_steps stdout, stderr, status = merge_acceptance(fleet_acceptance.gsub(/^( )(.*)/) { " #{Regexp.last_match(2)}" }) expect(status).to be_success, stderr - expect(acceptance_steps.map { |s| s['name'] }).to eq(['checkout repo', *vagrant_steps, 'beaker']) - expect(File.read(@wf)).to include("\n - name: Choose the Vagrant packages\n id: vagrant-debs\n") + expect(acceptance_steps.map { |s| s['name'] }).to eq(['checkout repo', 'Setup libvirt for Vagrant', 'beaker']) + expect(File.read(@wf)).to include("\n - name: Setup libvirt for Vagrant\n uses: simp/") + end + + it 'fails rather than delete steps the template lacks' do + # A typo, or a per-repo template without the step + stdout, stderr, status = merge_acceptance(fleet_acceptance, + names: ['Setup libvirt for Vagrant', 'Setup libvirt for Vagrnt']) + + expect(status).not_to be_success, stdout + expect(stderr).to include("template_steps not in the template's job: Setup libvirt for Vagrnt") + end + + context 'with several named steps' do + let(:names) { %w[one two three] } + let(:template) do + <<~YAML + jobs: + acceptance: + steps: + - name: checkout + uses: actions/checkout@v7 + - name: one + run: echo 1 + - name: two + run: echo 2 + - name: three + run: echo 3 + - name: beaker + run: rake beaker + YAML + end + + def existing_with(*steps) + " acceptance:\n steps:\n" + steps.map { |name, run| " - name: #{name}\n run: #{run}\n" }.join + end + + it "keeps the repo's own steps where they are" do + existing = existing_with(%w[checkout old], %w[one old], ['custom', 'repo step'], %w[two old], + %w[three old], %w[beaker old]) + stdout, stderr, status = merge_acceptance(existing, template: template, names: names) + + expect(status).to be_success, stderr + expect(JSON.parse(stdout)['files'][@wf]['changed']).to be true + expect(acceptance_steps.map { |s| [s['name'], s['run']] }).to eq( + [%w[checkout old], ['one', 'echo 1'], ['custom', 'repo step'], ['two', 'echo 2'], + ['three', 'echo 3'], %w[beaker old]], + ) + end + + it 'adds the steps the job lacks next to their template neighbours' do + existing = existing_with(%w[checkout old], ['custom', 'repo step'], %w[two old], %w[beaker old]) + stdout, stderr, status = merge_acceptance(existing, template: template, names: names) + + expect(status).to be_success, stderr + expect(acceptance_steps.map { |s| s['name'] }).to eq(%w[checkout custom one two three beaker]) + end end end From 38365139d372fb089ab51655c0d3a6f578bfecac Mon Sep 17 00:00:00 2001 From: Mike Riddle Date: Mon, 2 Nov 2026 16:52:21 -0500 Subject: [PATCH 3/4] Use the setup-vagrant-libvirt action from simp/gha-workflows The action moved from its own repo into simp/gha-workflows, so there is one repo to manage. Pin it at v1.1.0, the release that adds it. Co-Authored-By: Claude Opus 5.5 --- data/sync/configs/20261102-vagrant-ppa-fallback.yaml | 10 +++++----- .../pupmod/_github/workflows/pr_tests.simp-simp.yml | 2 +- .../files/pupmod/_github/workflows/pr_tests.yml | 2 +- .../pupmod/_github/workflows/vagrant_package_cache.yml | 2 +- spec/tasks/merge_gha_workflows_spec.rb | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/data/sync/configs/20261102-vagrant-ppa-fallback.yaml b/data/sync/configs/20261102-vagrant-ppa-fallback.yaml index 78d3699..82e5d9a 100644 --- a/data/sync/configs/20261102-vagrant-ppa-fallback.yaml +++ b/data/sync/configs/20261102-vagrant-ppa-fallback.yaml @@ -5,8 +5,8 @@ # The acceptance job's "Setup libvirt for Vagrant" step installed vagrant # and vagrant-libvirt from ppa:evgeni/vagrant alone, so a Launchpad outage # failed every acceptance job and, through 'All checks', blocked merges. -# The step now uses simp/github-action-setup-vagrant-libvirt, which -# installs from the Actions cache, then the PPA (with retries), then +# The step now uses the setup-vagrant-libvirt action in simp/gha-workflows, +# which installs from the Actions cache, then the PPA (with retries), then # HashiCorp's apt repository. # # A pull request can only restore caches saved from the default branch (or @@ -64,9 +64,9 @@ puppetsync::plan_config: The acceptance job installed Vagrant and vagrant-libvirt from ppa:evgeni/vagrant alone, so a Launchpad outage failed every - acceptance run. The setup step now uses - simp/github-action-setup-vagrant-libvirt, which installs the PPA's - packages from the Actions cache when it can, retries the PPA, and + acceptance run. The setup step now uses the setup-vagrant-libvirt + action from simp/gha-workflows, which installs the PPA's packages + from the Actions cache when it can, retries the PPA, and falls back to HashiCorp's Vagrant and the vagrant-libvirt plugin. The new 'Vagrant package cache' workflow keeps the packages cached from the default branch, where pull requests can restore them. diff --git a/modules/profile/files/pupmod/_github/workflows/pr_tests.simp-simp.yml b/modules/profile/files/pupmod/_github/workflows/pr_tests.simp-simp.yml index d204251..6580834 100644 --- a/modules/profile/files/pupmod/_github/workflows/pr_tests.simp-simp.yml +++ b/modules/profile/files/pupmod/_github/workflows/pr_tests.simp-simp.yml @@ -200,7 +200,7 @@ jobs: run: | bundle install - name: Setup libvirt for Vagrant - uses: simp/github-action-setup-vagrant-libvirt@v1.0.0 + uses: simp/gha-workflows/.github/actions/setup-vagrant-libvirt@v1.1.0 - name: beaker env: BEAKER_HYPERVISOR: 'vagrant_libvirt' diff --git a/modules/profile/files/pupmod/_github/workflows/pr_tests.yml b/modules/profile/files/pupmod/_github/workflows/pr_tests.yml index 1faa08a..51700d9 100644 --- a/modules/profile/files/pupmod/_github/workflows/pr_tests.yml +++ b/modules/profile/files/pupmod/_github/workflows/pr_tests.yml @@ -180,7 +180,7 @@ jobs: run: | bundle install - name: Setup libvirt for Vagrant - uses: simp/github-action-setup-vagrant-libvirt@v1.0.0 + uses: simp/gha-workflows/.github/actions/setup-vagrant-libvirt@v1.1.0 - name: beaker env: BEAKER_HYPERVISOR: 'vagrant_libvirt' diff --git a/modules/profile/files/pupmod/_github/workflows/vagrant_package_cache.yml b/modules/profile/files/pupmod/_github/workflows/vagrant_package_cache.yml index dd17035..d1c6896 100644 --- a/modules/profile/files/pupmod/_github/workflows/vagrant_package_cache.yml +++ b/modules/profile/files/pupmod/_github/workflows/vagrant_package_cache.yml @@ -32,6 +32,6 @@ jobs: name: 'Cache the Vagrant packages' runs-on: ubuntu-24.04 steps: - - uses: simp/github-action-setup-vagrant-libvirt@v1.0.0 + - uses: simp/gha-workflows/.github/actions/setup-vagrant-libvirt@v1.1.0 with: cache-only: 'true' diff --git a/spec/tasks/merge_gha_workflows_spec.rb b/spec/tasks/merge_gha_workflows_spec.rb index 89b1136..fa6ebf5 100644 --- a/spec/tasks/merge_gha_workflows_spec.rb +++ b/spec/tasks/merge_gha_workflows_spec.rb @@ -497,7 +497,7 @@ def acceptance_steps expect(merged).not_to include('ppa:evgeni/vagrant') expect(merged).not_to include('the next line is part of the script') # The blank line and the next step's comment stay with the next step - expect(merged).to include("@v1.0.0\n\n # Run the custom suite; see #123\n - name: beaker\n") + expect(merged).to include("setup-vagrant-libvirt@v1.1.0\n\n # Run the custom suite; see #123\n - name: beaker\n") end it 'is idempotent' do From bd824a85c2a29a0aa298b08c138df9705e479a07 Mon Sep 17 00:00:00 2001 From: Mike Riddle Date: Tue, 3 Nov 2026 11:29:28 -0500 Subject: [PATCH 4/4] Address review: add rngd and simp_authselect, quiet the cache workflow - pupmods_vagrant_acceptance: add pupmod-simp-rngd (main) and pupmod-simp-simp_authselect, which have the Vagrant step but aren't in pupmods.yaml. - vagrant_package_cache.yml runs on push only when the file itself changes, so it seeds the cache when it lands without a run (that fails during a Launchpad outage) on every merge. Its header says how it is delivered, since it isn't in the pupmod present_action_files. - merge_gha_workflows: template_steps fails when a name matches more than one step in a job instead of replacing both, keeps commented-out keys at a step's key column with the step, and the header documents that preserve_keys still applies to grafted steps. Co-Authored-By: Claude Opus 5.5 --- .../repolists/pupmods_vagrant_acceptance.yaml | 11 ++++- dist/puppetsync/tasks/merge_gha_workflows.rb | 19 ++++++++- .../workflows/vagrant_package_cache.yml | 10 ++++- spec/tasks/merge_gha_workflows_spec.rb | 41 +++++++++++++++++++ 4 files changed, 76 insertions(+), 5 deletions(-) diff --git a/data/sync/repolists/pupmods_vagrant_acceptance.yaml b/data/sync/repolists/pupmods_vagrant_acceptance.yaml index 486570d..2282f72 100644 --- a/data/sync/repolists/pupmods_vagrant_acceptance.yaml +++ b/data/sync/repolists/pupmods_vagrant_acceptance.yaml @@ -1,7 +1,8 @@ --- # pupmod repos whose acceptance job runs Beaker on Vagrant with libvirt -# ("Setup libvirt for Vagrant" in pr_tests.yml), as of 2026-11-02. -# The rest of pupmods.yaml run acceptance on podman or have none. +# ("Setup libvirt for Vagrant" in pr_tests.yml), as of 2026-11-03. +# The rest of pupmods.yaml run acceptance on podman or have none; +# pupmod-simp-rngd and pupmod-simp-simp_authselect aren't in pupmods.yaml. puppetsync::repos_config: https://github.com/simp/pupmod-simp-acpid: @@ -58,6 +59,9 @@ puppetsync::repos_config: https://github.com/simp/pupmod-simp-pupmod: branch: master + https://github.com/simp/pupmod-simp-rngd: + branch: main + https://github.com/simp/pupmod-simp-rsync: branch: master @@ -73,6 +77,9 @@ puppetsync::repos_config: https://github.com/simp/pupmod-simp-simp_apache: branch: master + https://github.com/simp/pupmod-simp-simp_authselect: + branch: master + https://github.com/simp/pupmod-simp-simp_ds389: branch: master diff --git a/dist/puppetsync/tasks/merge_gha_workflows.rb b/dist/puppetsync/tasks/merge_gha_workflows.rb index 60d7892..80142d1 100644 --- a/dist/puppetsync/tasks/merge_gha_workflows.rb +++ b/dist/puppetsync/tasks/merge_gha_workflows.rb @@ -40,7 +40,15 @@ # with none of them keeps its steps, so a baseline fix to shared steps # (e.g. the Vagrant setup) reaches every repo that has them without # touching each repo's own matrix, suites and steps. A name the template -# lacks is an error rather than a deletion. +# lacks is an error rather than a deletion, and so is a name that more than +# one step in a job has, since replacing both would drop whatever made them +# differ (an `if:`, say). +# +# `preserve_keys` still applies after the graft, so a grafted step keeps +# the repo's value for those keys: a `uses:` tag Renovate has bumped past +# the template's survives the next sync. A template change that needs a new +# tag together with new inputs therefore reaches each repo as the inputs +# now and the tag through that repo's Renovate PR. require 'fileutils' require 'json' @@ -115,6 +123,7 @@ def node_at_path(root, path) # exclusive, column of the step's first key). An item ends where the next # one starts, or the last with the sequence, less any trailing blank lines # and comments outdented from its keys: those belong to whatever follows. +# Comments at the keys' column (commented-out keys) stay with the step. def job_steps(job, lines) steps = job.is_a?(Psych::Nodes::Mapping) && node_at_path(job, 'steps') return [] unless steps && steps[1].is_a?(Psych::Nodes::Sequence) @@ -125,7 +134,7 @@ def job_steps(job, lines) stop = i + 1 < items.size ? items[i + 1].start_line : item.end_line while stop > item.start_line + 1 line = lines[stop - 1] - break unless line.strip.empty? || (line.lstrip.start_with?('#') && line[/\A */].size <= item.start_column) + break unless line.strip.empty? || (line.lstrip.start_with?('#') && line[/\A */].size < item.start_column) stop -= 1 end @@ -149,6 +158,12 @@ def graft_template_steps(block, block_start, existing_lines, existing_job, templ # Replacing with nothing would silently delete the repo's steps raise "template_steps not in the template's job: #{missing.join(', ')}" unless missing.empty? + # Two steps with one name can't both become the template's step + [existing, template_steps].each do |steps| + dups = steps.map(&:first).tally.select { |_name, count| count > 1 }.keys + raise "template_steps name more than one step in a job: #{dups.join(', ')}" unless dups.empty? + end + shift = existing.first[3] - template_steps.first[3] text = template_steps.to_h do |name, start, stop, _col| lines = template_lines[start...stop].map do |line| diff --git a/modules/profile/files/pupmod/_github/workflows/vagrant_package_cache.yml b/modules/profile/files/pupmod/_github/workflows/vagrant_package_cache.yml index d1c6896..bbb4615 100644 --- a/modules/profile/files/pupmod/_github/workflows/vagrant_package_cache.yml +++ b/modules/profile/files/pupmod/_github/workflows/vagrant_package_cache.yml @@ -15,7 +15,14 @@ # default branch, so the cache is saved here, from the default branch. The # runner must be on an Ubuntu series the PPA publishes for, and the # scheduled runs keep the entry from being evicted after seven days unused. -# See simp/puppetsync#122. +# On push it runs only when this file changes, which seeds the cache when +# the workflow lands without adding a run (that fails while Launchpad is +# down) to every merge. +# +# Only modules whose acceptance job uses Vagrant have this file, so it isn't +# in the pupmod baseline's present_action_files: a sync session adds it with +# merge_github_workflows' ensure_files, and later syncs refresh it where it +# exists. See simp/puppetsync#122. # --- name: 'Vagrant package cache' @@ -23,6 +30,7 @@ name: 'Vagrant package cache' 'on': push: branches: [master, main] + paths: ['.github/workflows/vagrant_package_cache.yml'] schedule: - cron: '17 6 * * 1,4' workflow_dispatch: {} diff --git a/spec/tasks/merge_gha_workflows_spec.rb b/spec/tasks/merge_gha_workflows_spec.rb index fa6ebf5..4ef83cd 100644 --- a/spec/tasks/merge_gha_workflows_spec.rb +++ b/spec/tasks/merge_gha_workflows_spec.rb @@ -545,6 +545,47 @@ def acceptance_steps expect(stderr).to include("template_steps not in the template's job: Setup libvirt for Vagrnt") end + it 'fails rather than replace two steps that share a name' do + # Both would become the template's step, losing what told them apart + twice = fleet_acceptance.sub(" - name: beaker\n", <<~YAML.gsub(/^/, ' ') + " - name: beaker\n") + - name: Setup libvirt for Vagrant + if: matrix.node == 'oraclelinux8' + run: sudo apt-get install -y qemu-system-x86 + YAML + stdout, stderr, status = merge_acceptance(twice) + + expect(status).not_to be_success, stdout + expect(stderr).to include('template_steps name more than one step in a job: Setup libvirt for Vagrant') + end + + it "replaces a step's trailing commented-out keys with it" do + commented = fleet_acceptance.sub(" sudo apt-get update\n", <<~YAML.gsub(/^/, ' ')) + sudo apt-get update + # with: + # retries: 3 + YAML + stdout, stderr, status = merge_acceptance(commented) + + expect(status).to be_success, stderr + merged = File.read(@wf) + expect(merged).not_to include('retries: 3') + expect(merged).to include("setup-vagrant-libvirt@v1.1.0\n\n # Run the custom suite; see #123\n - name: beaker\n") + end + + it "keeps the repo's tag on a grafted step (preserve_keys)" do + # Renovate bumped the repo past the template's tag; the next sync must not + # roll it back + merge_acceptance(fleet_acceptance) + File.write(@wf, File.read(@wf).sub('setup-vagrant-libvirt@v1.1.0', 'setup-vagrant-libvirt@v1.2.0')) + + _stdout, stderr, status = run_merge([{ 'path' => @wf, 'template' => real_template }], + preserve_blocks: ['jobs.acceptance'], + template_steps: ['Setup libvirt for Vagrant']) + + expect(status).to be_success, stderr + expect(File.read(@wf)).to include('setup-vagrant-libvirt@v1.2.0') + end + context 'with several named steps' do let(:names) { %w[one two three] } let(:template) do