build-python.yml

name: Build Source Portable Python

on:
  workflow_dispatch:
    inputs:
      force_rebuild:
        description: "Force build even if latest release already has same versions."
        required: false
        default: "false"
        type: choice
        options:
          - "false"
          - "true"
      major_filter:
        description: "Optional major to rebuild (example: 3.11)."
        required: false
        default: ""
        type: string
      target_os_filter:
        description: "Optional target OS filter."
        required: false
        default: "any"
        type: choice
        options:
          - any
          - linux
          - windows
          - macos
      target_arch_filter:
        description: "Optional target architecture filter."
        required: false
        default: "any"
        type: choice
        options:
          - any
          - x86_64
          - arm64
      release_tag:
        description: "Optional release tag (example: 2026.04.16). If set, create GitHub release."
        required: false
        type: string

permissions:
  contents: read

jobs:
  check-upstream:
    runs-on: ubuntu-latest
    outputs:
      should_build: ${{ steps.plan.outputs.should_build }}
      current_versions_json: ${{ steps.plan.outputs.current_versions_json }}
      changed_versions_json: ${{ steps.plan.outputs.changed_versions_json }}
      release_plan_json: ${{ steps.plan.outputs.release_plan_json }}
      release_tag: ${{ steps.plan.outputs.release_tag }}
      matrix: ${{ steps.make-matrix.outputs.matrix }}
      matrix_has_entries: ${{ steps.make-matrix.outputs.matrix_has_entries }}
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - uses: actions/setup-python@v6
        with:
          python-version: "3.12"

      - id: resolve
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          python scripts/resolve_latest_patch.py --majors-file config/majors.json --details > resolved-details.json
          cat resolved-details.json

      - id: release-exists
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -euo pipefail

          RELEASE_TAG="python-$(jq -r '[to_entries[] | .value.version] | join("-")' resolved-details.json)"
          echo "release_tag=${RELEASE_TAG}" >> "$GITHUB_OUTPUT"

          if git ls-remote --exit-code --tags origin "refs/tags/${RELEASE_TAG}" > /dev/null 2>&1; then
            TAG_EXISTS="true"
          else
            TAG_EXISTS="false"
          fi

          RELEASE_STATUS="$(curl -sS -o /dev/null -w "%{http_code}" \
            -H "Authorization: Bearer ${GITHUB_TOKEN}" \
            -H "Accept: application/vnd.github+json" \
            "${{ github.api_url }}/repos/${{ github.repository }}/releases/tags/${RELEASE_TAG}")"
          if [ "${RELEASE_STATUS}" = "200" ]; then
            RELEASE_EXISTS="true"
          else
            RELEASE_EXISTS="false"
          fi

          RELEASE_AVAILABLE="false"
          if [ "${TAG_EXISTS}" = "true" ] && [ "${RELEASE_EXISTS}" = "true" ]; then
            RELEASE_AVAILABLE="true"
          fi

          echo "tag_exists=${TAG_EXISTS}" >> "$GITHUB_OUTPUT"
          echo "release_exists=${RELEASE_EXISTS}" >> "$GITHUB_OUTPUT"
          echo "release_available=${RELEASE_AVAILABLE}" >> "$GITHUB_OUTPUT"
          echo "Expected release tag: ${RELEASE_TAG}"
          echo "Tag exists: ${TAG_EXISTS}"
          echo "Release exists: ${RELEASE_EXISTS}"

      - id: plan
        env:
          FORCE_REBUILD: ${{ github.event.inputs.force_rebuild || 'false' }}
          RELEASE_TAG_INPUT: ${{ github.event.inputs.release_tag || '' }}
          MAJOR_FILTER_INPUT: ${{ github.event.inputs.major_filter || '' }}
          TARGET_OS_FILTER_INPUT: ${{ github.event.inputs.target_os_filter || 'any' }}
          TARGET_ARCH_FILTER_INPUT: ${{ github.event.inputs.target_arch_filter || 'any' }}
          RELEASE_AVAILABLE: ${{ steps.release-exists.outputs.release_available }}
          EXPECTED_RELEASE_TAG: ${{ steps.release-exists.outputs.release_tag }}
        run: |
          set -euo pipefail

          PLAN_ARGS=()
          FORCE_REBUILD_EFFECTIVE="${FORCE_REBUILD}"
          RELEASE_TAG_EFFECTIVE="${RELEASE_TAG_INPUT}"
          if [ -z "${RELEASE_TAG_EFFECTIVE}" ]; then
            RELEASE_TAG_EFFECTIVE="${EXPECTED_RELEASE_TAG}"
          fi

          if [ -n "${MAJOR_FILTER_INPUT}" ] || [ "${TARGET_OS_FILTER_INPUT}" != "any" ] || [ "${TARGET_ARCH_FILTER_INPUT}" != "any" ]; then
            FORCE_REBUILD_EFFECTIVE="true"
          fi

          if [ "${FORCE_REBUILD_EFFECTIVE}" = "true" ]; then
            PLAN_ARGS+=(--force)
          fi
          if [ -n "${RELEASE_TAG_EFFECTIVE}" ]; then
            PLAN_ARGS+=(--release-tag "${RELEASE_TAG_EFFECTIVE}")
          fi

          python scripts/plan_release.py \
            --resolved-file resolved-details.json \
            --state-file release-state/latest.json \
            "${PLAN_ARGS[@]}" > release-plan.json

          cat release-plan.json
          echo "should_build=$(jq -r '.should_build' release-plan.json)" >> "$GITHUB_OUTPUT"
          echo "current_versions_json=$(jq -c '.current_versions' release-plan.json)" >> "$GITHUB_OUTPUT"
          echo "changed_versions_json=$(jq -c '.changed_versions' release-plan.json)" >> "$GITHUB_OUTPUT"
          echo "release_plan_json=$(jq -c . release-plan.json)" >> "$GITHUB_OUTPUT"
          echo "release_tag=$(jq -r '.release_tag' release-plan.json)" >> "$GITHUB_OUTPUT"

      - id: make-matrix
        env:
          MAJOR_FILTER_INPUT: ${{ github.event.inputs.major_filter || '' }}
          TARGET_OS_FILTER_INPUT: ${{ github.event.inputs.target_os_filter || 'any' }}
          TARGET_ARCH_FILTER_INPUT: ${{ github.event.inputs.target_arch_filter || 'any' }}
        shell: python
        run: |
          import json
          import os
          from pathlib import Path

          plan = json.loads(Path("release-plan.json").read_text(encoding="utf-8"))
          resolved = json.loads(Path("resolved-details.json").read_text(encoding="utf-8"))
          versions = plan["changed_versions"]
          major_filter = os.environ.get("MAJOR_FILTER_INPUT", "").strip()
          target_os_filter = os.environ.get("TARGET_OS_FILTER_INPUT", "any").strip().lower() or "any"
          target_arch_filter = os.environ.get("TARGET_ARCH_FILTER_INPUT", "any").strip().lower() or "any"

          valid_os_filters = {"any", "linux", "windows", "macos"}
          valid_arch_filters = {"any", "x86_64", "arm64"}
          if target_os_filter not in valid_os_filters:
              raise SystemExit(f"Invalid target_os_filter: {target_os_filter}")
          if target_arch_filter not in valid_arch_filters:
              raise SystemExit(f"Invalid target_arch_filter: {target_arch_filter}")

          targets = [
              {
                  "runner": "ubuntu-24.04",
                  "target_os": "linux",
                  "target_arch": "x86_64",
                  "archive_ext": ".tar.gz",
              },
              {
                  "runner": "windows-2022",
                  "target_os": "windows",
                  "target_arch": "x86_64",
                  "archive_ext": ".zip",
              },
              {
                  "runner": "macos-15-intel",
                  "target_os": "macos",
                  "target_arch": "x86_64",
                  "archive_ext": ".tar.gz",
              },
              {
                  "runner": "macos-15",
                  "target_os": "macos",
                  "target_arch": "arm64",
                  "archive_ext": ".tar.gz",
              },
          ]

          include = []
          for major, full_version in versions.items():
              if major_filter and major != major_filter:
                  continue

              detail = resolved.get(major, {})
              cpython_tag = detail.get("tag", f"v{full_version}")
              cpython_tag_commit_sha = detail.get("tag_commit_sha", "")
              for target in targets:
                  if target_os_filter != "any" and target["target_os"] != target_os_filter:
                      continue
                  if target_arch_filter != "any" and target["target_arch"] != target_arch_filter:
                      continue

                  artifact_name = (
                      f"python-{full_version}-{target['target_os']}-{target['target_arch']}"
                  )
                  include.append(
                      {
                          "major": major,
                          "python_version": full_version,
                          "cpython_tag": cpython_tag,
                          "cpython_tag_commit_sha": cpython_tag_commit_sha,
                          "artifact_name": artifact_name,
                          "archive_name": f"{artifact_name}{target['archive_ext']}",
                          **target,
                      }
                  )

          if versions and not include:
              raise SystemExit(
                  "Selected filters produced an empty build matrix. "
                  "Check major_filter/target_os_filter/target_arch_filter values."
              )

          matrix = {"include": include}
          line = f"matrix={json.dumps(matrix)}"
          with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as fh:
              fh.write(line + "\n")
              fh.write(f"matrix_has_entries={'true' if include else 'false'}\n")

  validate-scripts:
    needs: check-upstream
    if: needs.check-upstream.outputs.should_build == 'true'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6

      - uses: actions/setup-python@v6
        with:
          python-version: "3.12"

      - name: Verify Python scripts compile
        run: python -m compileall scripts

      - name: Re-resolve configured versions
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          python scripts/resolve_latest_patch.py --majors-file config/majors.json --details > resolved-details.json
          cat resolved-details.json

      - name: Smoke test validation CLIs
        run: |
          python scripts/check_stdlib.py --help
          python scripts/validate_distribution.py --help
          python scripts/plan_release.py --help
          python scripts/update_release_state.py --help

  build:
    needs: [check-upstream, validate-scripts]
    if: needs.check-upstream.outputs.should_build == 'true' && needs.check-upstream.outputs.matrix_has_entries == 'true'
    runs-on: ${{ matrix.runner }}
    permissions:
      contents: read
      id-token: write
      attestations: write
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.check-upstream.outputs.matrix) }}
    steps:
      - uses: actions/checkout@v6

      - uses: actions/setup-python@v6
        with:
          python-version: "3.12"

      - name: Install build dependencies (macOS)
        if: matrix.target_os == 'macos'
        env:
          HOMEBREW_NO_AUTO_UPDATE: "1"
          HOMEBREW_NO_INSTALL_UPGRADE: "1"
        run: |
          if [[ "${{ matrix.python_version }}" == 3.11.* ]]; then
            formulas=(openssl@3 sqlite xz zlib tcl-tk@8)
          else
            formulas=(openssl@3 sqlite xz zlib tcl-tk)
          fi

          for formula in "${formulas[@]}"; do
            if brew list --versions "$formula" >/dev/null; then
              echo "$formula already installed"
              continue
            fi
            brew install "$formula"
          done

      - name: Build archive
        shell: bash
        run: |
          set -euo pipefail
          if [[ "${{ matrix.target_os }}" == "linux" ]]; then
            docker run --rm \
              -v "$PWD:/work" \
              quay.io/pypa/manylinux2014_x86_64 \
              bash -c "
                yum install -y \
                  libffi-devel \
                  pkgconfig \
                  sqlite-devel \
                  bzip2-devel \
                  xz-devel \
                  libzstd-devel \
                  readline-devel \
                  ncurses-devel \
                  libuuid-devel \
                  tk-devel \
                  patchelf && \
                cd /work && /opt/python/cp312-cp312/bin/python scripts/build_portable.py \
                  '${{ matrix.major }}' \
                  --python-version '${{ matrix.python_version }}' \
                  --cpython-tag '${{ matrix.cpython_tag }}' \
                  --cpython-tag-commit-sha '${{ matrix.cpython_tag_commit_sha }}' \
                  --target-os '${{ matrix.target_os }}' \
                  --target-arch '${{ matrix.target_arch }}' \
                  --output-dir dist"
          else
            python scripts/build_portable.py \
              "${{ matrix.major }}" \
              --python-version "${{ matrix.python_version }}" \
              --cpython-tag "${{ matrix.cpython_tag }}" \
              --cpython-tag-commit-sha "${{ matrix.cpython_tag_commit_sha }}" \
              --target-os "${{ matrix.target_os }}" \
              --target-arch "${{ matrix.target_arch }}" \
              --output-dir dist
          fi

      - name: Attest archive
        uses: actions/attest@v4
        with:
          subject-path: dist/${{ matrix.archive_name }}

      - id: upload-artifact
        name: Upload artifact
        uses: actions/upload-artifact@v7
        continue-on-error: true
        with:
          name: ${{ matrix.artifact_name }}
          path: |
            dist/${{ matrix.archive_name }}
            dist/${{ matrix.archive_name }}.sha256
          if-no-files-found: error
          overwrite: true

      - name: Retry artifact upload
        if: steps.upload-artifact.outcome == 'failure'
        uses: actions/upload-artifact@v7
        with:
          name: ${{ matrix.artifact_name }}
          path: |
            dist/${{ matrix.archive_name }}
            dist/${{ matrix.archive_name }}.sha256
          if-no-files-found: error
          overwrite: true

  validate-artifacts:
    needs: [check-upstream, build]
    if: needs.check-upstream.outputs.should_build == 'true' && needs.check-upstream.outputs.matrix_has_entries == 'true'
    runs-on: ${{ matrix.runner }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.check-upstream.outputs.matrix) }}
    steps:
      - uses: actions/checkout@v6

      - uses: actions/setup-python@v6
        with:
          python-version: "3.12"

      - name: Download built archive
        uses: actions/download-artifact@v8
        with:
          name: ${{ matrix.artifact_name }}
          path: dist-archive

      - name: Diagnose packaged Python startup (Linux)
        if: matrix.target_os == 'linux'
        continue-on-error: true
        shell: bash
        run: |
          TMPDIR_D=$(mktemp -d)
          tar xzf "dist-archive/${{ matrix.archive_name }}" -C "$TMPDIR_D"
          PYTHON_BIN="$TMPDIR_D/python/bin/python3"
          echo "=== ldd python3 ===" | tee -a "$GITHUB_STEP_SUMMARY"
          ldd "$PYTHON_BIN" 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" || true
          echo "=== python3 startup test ===" | tee -a "$GITHUB_STEP_SUMMARY"
          "$PYTHON_BIN" -c "import sys; print(sys.version)" 2>&1 \
            | tee -a "$GITHUB_STEP_SUMMARY" || true
          rm -rf "$TMPDIR_D"

      - name: Validate packaged distribution
        shell: bash
        run: |
          set -euo pipefail
          python scripts/validate_distribution.py \
            "dist-archive/${{ matrix.archive_name }}" \
            --full-timeout-seconds 20 \
            2>&1 | tee -a "$GITHUB_STEP_SUMMARY"

  release:
    if: needs.check-upstream.outputs.should_build == 'true' && needs.check-upstream.outputs.matrix_has_entries == 'true'
    needs: [check-upstream, validate-artifacts]
    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
          ref: ${{ github.ref_name }}

      - uses: actions/setup-python@v6
        with:
          python-version: "3.12"

      - name: Restore release plan
        run: |
          printf '%s' '${{ needs.check-upstream.outputs.release_plan_json }}' > release-plan.json
          cat release-plan.json

      - name: Download all artifacts
        uses: actions/download-artifact@v8
        with:
          path: release-assets
          merge-multiple: true

      - name: Carry forward unchanged release assets
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          GH_REPO: ${{ github.repository }}
        run: |
          set -euo pipefail
          PREVIOUS_TAG="$(jq -r '.release_tag // empty' release-state/latest.json)"
          if [ -z "${PREVIOUS_TAG}" ]; then
            echo "No previous release tag; publishing newly built assets only."
            exit 0
          fi
          if [ "${PREVIOUS_TAG}" = "${{ needs.check-upstream.outputs.release_tag }}" ]; then
            echo "Release tag unchanged; skipping asset carry-forward."
            exit 0
          fi

          mkdir -p previous-assets
          if ! gh release download "${PREVIOUS_TAG}" --dir previous-assets --pattern 'python-*'; then
            echo "Could not download previous release ${PREVIOUS_TAG}; publishing newly built assets only."
            exit 0
          fi

          python scripts/carry_forward_release_assets.py \
            --plan-file release-plan.json \
            --source-dir previous-assets \
            --dest-dir release-assets

      - name: Generate checksums
        run: |
          cd release-assets
          find . -maxdepth 1 -type f ! -name '*.sha256' | sort | xargs sha256sum > SHA256SUMS.txt

      - name: Update release state files
        env:
          GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        run: |
          python scripts/update_release_state.py \
            --plan-file release-plan.json \
            --output-dir release-state \
            --build-source-commit "${{ github.sha }}" \
            --workflow-run-id "${{ github.run_id }}" \
            --workflow-run-url "${GITHUB_RUN_URL}" \
            --repository "${{ github.repository }}" \
            --event-name "${{ github.event_name }}"

      - id: commit-state
        name: Commit release state
        run: |
          set -euo pipefail

          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git add release-state

          if git diff --cached --quiet; then
            echo "release_commit_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          git commit -m "Record release state for ${{ needs.check-upstream.outputs.release_tag }}"
          git push origin HEAD:${{ github.ref_name }}
          echo "release_commit_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"

      - id: push-tag
        name: Create and push release tag
        env:
          RELEASE_TAG: ${{ needs.check-upstream.outputs.release_tag }}
          RELEASE_COMMIT_SHA: ${{ steps.commit-state.outputs.release_commit_sha }}
        run: |
          set -euo pipefail

          git fetch --tags --force --prune --prune-tags
          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"

          if git ls-remote --exit-code --tags origin "refs/tags/${RELEASE_TAG}" >/dev/null 2>&1; then
            echo "Tag already exists on origin: ${RELEASE_TAG}"
            echo "tag_name=${RELEASE_TAG}" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          if git rev-parse -q --verify "refs/tags/${RELEASE_TAG}" >/dev/null; then
            git tag -d "${RELEASE_TAG}"
          fi

          git tag -a "${RELEASE_TAG}" "${RELEASE_COMMIT_SHA}" -m "Release ${RELEASE_TAG}"
          git push origin "refs/tags/${RELEASE_TAG}"
          echo "tag_name=${RELEASE_TAG}" >> "$GITHUB_OUTPUT"

      - name: Create release
        uses: softprops/action-gh-release@v3
        with:
          tag_name: ${{ steps.push-tag.outputs.tag_name }}
          name: "Portable Python ${{ needs.check-upstream.outputs.release_tag }}"
          fail_on_unmatched_files: true
          generate_release_notes: true
          files: release-assets/*