From f5509f2f45d4687ea468d42cb2a083d8c140abc9 Mon Sep 17 00:00:00 2001 From: Scott Dixon Date: Sat, 24 Oct 2026 14:50:19 -0700 Subject: [PATCH 1/3] Simplify releases to verified stable versions and automatic recovery #verification --- .github/workflows/cetlvast.yml | 84 +------ .github/workflows/release-docs.yml | 98 ++++++++ CONTRIBUTING.md | 5 +- RELEASING.md | 39 +-- scripts/release.py | 367 +++++++++++++++------------ scripts/release_tools.py | 102 +++++--- scripts/tests/test_release.py | 388 ++++++++++++++++++++--------- 7 files changed, 649 insertions(+), 434 deletions(-) create mode 100644 .github/workflows/release-docs.yml diff --git a/.github/workflows/cetlvast.yml b/.github/workflows/cetlvast.yml index e29efed..419f247 100644 --- a/.github/workflows/cetlvast.yml +++ b/.github/workflows/cetlvast.yml @@ -1,11 +1,11 @@ name: "CETLVaSt" -run-name: CETLVaSt / ${{ inputs.docs_ref || github.ref_name }} +run-name: CETLVaSt / ${{ github.ref_name }} permissions: contents: read concurrency: - group: cetlvast-${{ github.ref }}-${{ inputs.docs_ref }} + group: cetlvast-${{ github.ref }} cancel-in-progress: false on: @@ -14,14 +14,7 @@ on: branches: - main - "issue/*" - release: - types: [published] workflow_dispatch: - inputs: - docs_ref: - description: "Release tag to verify in full and deploy from main (stable releases only)." - required: false - default: "" jobs: version: @@ -30,22 +23,14 @@ jobs: commit: ${{ steps.version.outputs.commit }} tag: ${{ steps.version.outputs.tag }} version: ${{ steps.version.outputs.version }} - prerelease: ${{ steps.version.outputs.prerelease }} steps: - uses: actions/checkout@v7 with: - ref: ${{ inputs.docs_ref || github.event.release.tag_name || github.sha }} + ref: ${{ github.sha }} fetch-depth: 0 - name: Validate library and release versions id: version - env: - DOCS_REF: ${{ inputs.docs_ref }} - run: | - if [ -n "$DOCS_REF" ]; then - python3 scripts/release_tools.py validate --tag "$DOCS_REF" - else - python3 scripts/release_tools.py validate - fi + run: python3 scripts/release_tools.py validate release-tests: runs-on: ubuntu-latest @@ -63,7 +48,6 @@ jobs: cache-warmup: if: > startsWith(github.ref, 'refs/tags/v') || - github.event_name == 'release' || github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.message, '#verification') || contains(github.event.head_commit.message, '#docs') || @@ -91,7 +75,6 @@ jobs: verification-amd64: if: > startsWith(github.ref, 'refs/tags/v') || - github.event_name == 'release' || github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.message, '#verification') || contains(github.ref, '/main') || @@ -135,7 +118,6 @@ jobs: verification-arm64: if: > startsWith(github.ref, 'refs/tags/v') || - github.event_name == 'release' || github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.message, '#verification') || contains(github.ref, '/main') || @@ -180,7 +162,6 @@ jobs: verification-legacy-arm64: if: > startsWith(github.ref, 'refs/tags/v') || - github.event_name == 'release' || github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.message, '#verification') || contains(github.ref, '/main') || @@ -226,7 +207,6 @@ jobs: docs: if: > startsWith(github.ref, 'refs/tags/v') || - github.event_name == 'release' || github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.message, '#verification') || contains(github.event.head_commit.message, '#docs') || @@ -262,15 +242,6 @@ jobs: run: >- python3 scripts/release_tools.py verify-docs --version "$CETL_RELEASE_VERSION" --path cetlvast/build/suites/docs/html/index.html - - name: Setup Pages - if: ${{ github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' }} - uses: actions/configure-pages@v6 - - name: Upload docs - if: ${{ github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' }} - uses: actions/upload-pages-artifact@v5 - with: - path: "cetlvast/build/suites/docs/html/" - - name: Save built release documentation uses: actions/upload-artifact@v7 with: @@ -284,7 +255,6 @@ jobs: if: >- always() && ( startsWith(github.ref, 'refs/tags/v') || - github.event_name == 'release' || github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.message, '#verification') || contains(github.ref, '/main') || @@ -320,16 +290,10 @@ jobs: path: release-docs - name: Package the verified source, headers, and documentation env: - RELEASE_TAG: ${{ needs.version.outputs.tag }} RELEASE_COMMIT: ${{ needs.version.outputs.commit }} LIBRARY_VERSION: ${{ needs.version.outputs.version }} run: | - # PR/branch runs rehearse packaging using a local tag; it is never pushed. - if [ -z "$RELEASE_TAG" ]; then - RELEASE_TAG="v$LIBRARY_VERSION-rc.$GITHUB_RUN_ID" - git tag "$RELEASE_TAG" "$RELEASE_COMMIT" - fi - python3 scripts/release_tools.py package --tag "$RELEASE_TAG" --commit "$RELEASE_COMMIT" \ + python3 scripts/release_tools.py package --tag "v$LIBRARY_VERSION" --commit "$RELEASE_COMMIT" \ --docs release-docs --output release-bundle python3 - <<'PYTHON' import json @@ -363,41 +327,3 @@ jobs: if-no-files-found: error overwrite: true retention-days: 30 - - deploy-docs: - if: ${{ github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && needs.version.outputs.tag != '' && needs.version.outputs.prerelease == 'false' }} - needs: [version, release-ready, release-assets] - concurrency: - group: cetl-stable-pages - cancel-in-progress: false - # Grant GITHUB_TOKEN the permissions required to make a Pages deployment - permissions: - contents: read - pages: write # to deploy to Pages - id-token: write # to verify the deployment originates from an appropriate source - - # Deploy to the github-pages environment - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - - # Specify runner + deployment step - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - with: - ref: ${{ needs.version.outputs.commit }} - - name: Prevent unpublished releases or older retries from replacing stable docs - env: - GH_TOKEN: ${{ github.token }} - RELEASE_TAG: ${{ needs.version.outputs.tag }} - PYTHONPATH: scripts - run: | - python3 - <<'PYTHON' - import os - from release import GitHub, require_latest_stable - require_latest_stable(GitHub(os.environ['GITHUB_REPOSITORY']), os.environ['RELEASE_TAG']) - PYTHON - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v5 diff --git a/.github/workflows/release-docs.yml b/.github/workflows/release-docs.yml new file mode 100644 index 0000000..45cc194 --- /dev/null +++ b/.github/workflows/release-docs.yml @@ -0,0 +1,98 @@ +name: Release documentation +run-name: CETL docs / ${{ inputs.tag }} + +on: + workflow_dispatch: + inputs: + tag: + description: Published release tag + required: true + type: string + +permissions: + contents: read + actions: read + pages: write + id-token: write + +concurrency: + group: cetl-stable-pages + cancel-in-progress: false + +jobs: + deploy-docs: + if: ${{ github.ref == 'refs/heads/main' }} + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + - uses: actions/setup-python@v7 + with: + python-version: '3.14' + - name: Validate release identity + id: version + env: + RELEASE_TAG: ${{ inputs.tag }} + PYTHONPATH: scripts + run: | + python3 - <<'PYTHON' + import os + from pathlib import Path + from release_tools import Git, parse_version + + tag = os.environ['RELEASE_TAG'] + if not tag.startswith('v'): + raise SystemExit('Expected a vX.Y.Z tag') + parse_version(tag[1:]) + git = Git(Path.cwd()) + commit = git.commit('refs/tags/' + tag) + git.run('merge-base', '--is-ancestor', commit, 'HEAD') + metadata = {**git.validate(commit, tag), 'commit': commit} + with open(os.environ['GITHUB_OUTPUT'], 'a') as output: + for key, value in metadata.items(): + output.write(f'{key}={value}\n') + PYTHON + - name: Download and verify published documentation + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ steps.version.outputs.tag }} + RELEASE_COMMIT: ${{ steps.version.outputs.commit }} + PYTHONPATH: scripts + run: | + python3 - <<'PYTHON' + import os + from pathlib import Path + from release import GitHub, require_latest_stable, wait_for_release_run + from release_tools import extract_documentation + + github = GitHub(os.environ['GITHUB_REPOSITORY']) + tag, commit = os.environ['RELEASE_TAG'], os.environ['RELEASE_COMMIT'] + require_latest_stable(github, tag) + wait_for_release_run(github, commit, 300, 15) + bundle = Path('release-bundle') + bundle.mkdir() + github.cli('release', 'download', tag, '--dir', str(bundle)) + extract_documentation(bundle, commit, tag, 'site') + PYTHON + - uses: actions/configure-pages@v6 + - uses: actions/upload-pages-artifact@v5 + with: + path: site/cetl-${{ steps.version.outputs.version }}/html + - name: Confirm release is still latest + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ steps.version.outputs.tag }} + PYTHONPATH: scripts + run: | + python3 - <<'PYTHON' + import os + from release import GitHub, require_latest_stable + require_latest_stable(GitHub(os.environ['GITHUB_REPOSITORY']), os.environ['RELEASE_TAG']) + PYTHON + - uses: actions/deploy-pages@v5 + id: deployment diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 06810de..3acaba6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -178,8 +178,9 @@ Note that if the job you requested is dependent on other jobs that are not trigg Version/header validation and release-tool tests run independently of hashtags. The `#verification` trigger includes documentation and release archive checks. -Version tags and release dispatches always run full verification. A release -documentation dispatch also verifies the complete release; it is not a docs-only shortcut. +Version tags and verification dispatches run full verification. Documentation +deployment verifies the published bundle and its successful main CI run, then +deploys the packaged documentation. See [Releasing CETL](RELEASING.md) for the maintainer Python CLI and recovery instructions. The CMake `release` target is a legacy build/test alias and does not publish a release. diff --git a/RELEASING.md b/RELEASING.md index 8977ff6..df76f60 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -1,36 +1,17 @@ # Releasing CETL -Merge the reviewed changes and version bump into `main`. Choose patch versions for -compatible fixes, minor versions for compatible additions, and major versions for -incompatible changes. Documentation/test-only changes do not require a version bump. +Merge the reviewed changes and version bump in `include/cetl/cetl.hpp` into `main`. +Choose patch versions for compatible fixes, minor versions for compatible additions, +and major versions for incompatible changes. ```sh -python3 scripts/release.py --dry-run +python3 scripts/release.py ``` -Run the printed **Release this checked commit** command to publish exactly the -version and commit that passed preflight. The script fetches remote `main`, reads -the header version, checks the tools/access/workflow/Pages setup, waits for full CI, -verifies the archives, publishes the release, and verifies the deployed documentation. -Failures report the corrective action. No manual test, checksum, or site-version -check is required. +The script fetches `main`, reads the header version, verifies CI and the release +bundle, then creates the tag, publishes the release, and deploys documentation. +Failed verification leaves the version available for a corrected commit. -For a prerelease, supply its version explicitly: - -```sh -python3 scripts/release.py --version 2.0.0-rc.1 --dry-run -``` - -Use `--ref COMMIT` to select a particular merged commit and `--notes-file PATH` for -reviewed release notes; otherwise GitHub generates the notes. See `--help` for -forks, alternate remotes, and wait limits. - -## Recovery - -Run the printed **Resume command** after fixing the reported problem. It retains -matching tags/assets, finishes missing uploads, and retries documentation deployment. -Published releases are verified from their actual assets, so expired CI artifacts -do not require manual recovery. Conflicting tags or asset contents are refused. - -For a transient CI failure, run the full-run retry command printed in the error, -then resume. A source fix needs a new reviewed commit and a new version/tag. +If interrupted, run the command again. Matching tags, drafts, and uploaded assets +are reused. The script also prints a retry command pinned to the selected commit. +Conflicting tags or asset contents are refused. diff --git a/scripts/release.py b/scripts/release.py index 77433c6..cc98d5c 100644 --- a/scripts/release.py +++ b/scripts/release.py @@ -2,7 +2,7 @@ """Publish verified CETL releases using Git and authenticated GitHub CLI access. Preflight selects a merged commit and validates the release configuration. -Publication waits for tagged CI, verifies assets, and deploys stable documentation. +Publication uses verified main CI, checks assets, and deploys documentation. Matching attempts can resume after interruption; completed remote operations are preserved. Python and the standard library implement the orchestration; ``release_tools`` supplies shared validation and packaging contracts. @@ -24,11 +24,15 @@ from urllib.error import HTTPError from urllib.request import Request, urlopen -from release_tools import (Git, ReleaseError, WORKFLOW, check_bundle, check_documentation, check_run, command, +from release_tools import (DOCS_WORKFLOW, Git, ReleaseError, WORKFLOW, check_bundle, check_documentation, check_run, command, parse_version, sha256) _PYTHON_MINIMUM = (3, 14) + +class _AssetsPending(ReleaseError): + """Expected uploads have not appeared in GitHub's asset listing yet.""" + def check_local_tools(python_version=None): """Require the supported Python version and Git/GitHub CLI operations. @@ -56,28 +60,21 @@ def check_local_tools(python_version=None): f"gh {' '.join(operation)} is unavailable.") from ex -def release_command(args, root, version, commit, resume=False): - """Format a shell command pinned to a previously checked release revision. +def release_command(args, root, commit): + """Format a retry command pinned to a checked source commit. Args: - args (argparse.Namespace): Parsed options containing repo, remote, - timeout, poll, and the optional notes_file Path. - root (str | Path): Checkout directory for the printed command. - version (str): Validated release version without the v prefix. - commit (str): Full commit object ID, retained even if main advances. - resume (bool): Include the option for continuing an existing attempt. + args (argparse.Namespace): Repository, remote, timeout, poll, and notes options. + root (str | Path): Checkout directory. + commit (str): Full source commit object ID. Returns: - str: Command using this interpreter/script, platform-appropriate quoting, - and an absolute notes path when notes_file is supplied. + str: Quoted command using this interpreter and script. """ parts = [sys.executable, str(Path(__file__).resolve()), "--root", str(root), "--repo", args.repo, - "--remote", args.remote, "--version", version, "--ref", commit, - "--timeout", str(args.timeout), "--poll", str(args.poll)] + "--remote", args.remote, "--ref", commit, "--timeout", str(args.timeout), "--poll", str(args.poll)] if args.notes_file: parts.extend(["--notes-file", str(args.notes_file.resolve())]) - if resume: - parts.append("--resume") return subprocess.list2cmdline(parts) if os.name == "nt" else shlex.join(parts) @@ -119,7 +116,7 @@ def cli(self, *args): except ReleaseError as ex: raise ReleaseError(f"gh {' '.join(args[:2])} failed. Run gh auth status --hostname github.com; " "release operations need Contents read/write and Actions read/write access. " - "Then retry the printed resume command.") from ex + "Then retry the same release command.") from ex def api(self, path, method="GET", **fields): """Issue a repository-relative REST request and decode its JSON response. @@ -142,6 +139,8 @@ def api(self, path, method="GET", **fields): """ args = ["gh", "api", "--hostname", "github.com", "--method", method, f"repos/{self.repository}/{path}".rstrip("/")] + if method == "GET": + args.extend(["-H", "Cache-Control: no-cache"]) for key, value in fields.items(): args.extend(["-F" if isinstance(value, bool) else "-f", f"{key}={str(value).lower() if isinstance(value, bool) else value}"]) @@ -179,6 +178,8 @@ def preflight(self, stable): raise ReleaseError(f"Enable CETLVaSt: gh workflow enable {WORKFLOW} --repo {self.repository}") if not stable: return + if self._api_object(f"actions/workflows/{DOCS_WORKFLOW}").get("state") != "active": + raise ReleaseError(f"Enable release documentation: gh workflow enable {DOCS_WORKFLOW} --repo {self.repository}") try: site = self._api_object("pages") environment = self._api_object("environments/github-pages") @@ -236,7 +237,7 @@ def verify_pages(self, version, timeout, poll): last_error = f"Could not read documentation ({type(ex).__name__}); check network/TLS access." time.sleep(poll) raise ReleaseError(f"Release is published, but documentation verification failed at {url}. {last_error} " - "Fix access or allow Pages/CDN propagation, then retry the printed resume command.") + "Fix access or allow Pages/CDN propagation, then retry the same release command.") def pages(self, path, key=None): """Collect a REST collection across pages of up to 100 records. @@ -286,6 +287,20 @@ def release(self, tag): """ return next((release for release in self.pages("releases") if release["tag_name"] == tag), None) + def release_by_id(self, release_id): + """Read a known release directly, bypassing release-list visibility delays. + + Args: + release_id (int): ID returned by release creation or discovery. + + Returns: + dict: GitHub release metadata. + + Raises: + ReleaseError: The request fails or does not return an object. + """ + return self._api_object(f"releases/{release_id}") + def published_stable_versions(self): """Collect numeric versions from published, stable, vX.Y.Z releases. @@ -304,10 +319,11 @@ def published_stable_versions(self): versions.add(tuple(map(int, match.groups()))) return versions - def runs(self, **filters): - """List CETLVaSt workflow runs matching GitHub's REST filter fields. + def runs(self, workflow=WORKFLOW, **filters): + """List workflow runs matching GitHub's REST filter fields. Args: + workflow (str): Workflow filename; defaults to CETLVaSt. **filters: Run-list parameters such as head_sha, event, and branch; values are URL-encoded before requesting every result page. @@ -317,7 +333,7 @@ def runs(self, **filters): Raises: ReleaseError: The workflow run listing cannot be completed. """ - return self.pages(f"actions/workflows/{WORKFLOW}/runs?{urlencode(filters)}", "workflow_runs") + return self.pages(f"actions/workflows/{workflow}/runs?{urlencode(filters)}", "workflow_runs") def jobs(self, run): """List jobs belonging to exactly the attempt identified by a run record. @@ -349,13 +365,12 @@ def assets(self, release): """ return self.pages(f"releases/{release['id']}/assets") - def create_draft(self, tag, commit, prerelease, marker, run, notes=None): + def create_draft(self, tag, commit, marker, run, notes=None): """Create a draft with source identity and successful verification evidence. Args: tag (str): Existing v-prefixed remote tag; also the release title. commit (str): Full source commit object ID. - prerelease (bool): Classification already validated against the tag. marker (str): Script-management marker used to identify matching retries. run (dict): Verified Actions run containing html_url. notes (str | None): Reviewed body text, or None to generate GitHub notes. @@ -363,14 +378,17 @@ def create_draft(self, tag, commit, prerelease, marker, run, notes=None): Raises: ReleaseError: Note generation or release creation fails. + Returns: + dict: Creation response, including the release ID. + Writes a remote draft. Callers must check for an existing release first and re-read remote state after an interrupted creation request. """ if notes is None: notes = self._api_object("releases/generate-notes", "POST", tag_name=tag, target_commitish=commit)["body"] body = f"{notes}\n\nSource commit: `{commit}`\n\nVerification: {run['html_url']}\n\n{marker}\n" - self.api("releases", "POST", tag_name=tag, target_commitish=commit, name=tag, - draft=True, prerelease=prerelease, body=body) + return self._api_object("releases", "POST", tag_name=tag, target_commitish=commit, name=tag, + draft=True, prerelease=False, body=body) def asset_matches(self, asset, path): """Compare a remote asset's uploaded bytes with a local file. @@ -398,56 +416,53 @@ def asset_matches(self, asset, path): return sha256(Path(directory) / path.name) == sha256(path) -def wait_for_release_run(github, commit, tag, timeout, poll): - """Wait for the newest tagged push/dispatch run and validate its full gate. +def wait_for_release_run(github, commit, timeout, poll): + """Wait for full verification of a main commit before creating its release tag. Args: - github (GitHub): Repository client used to list runs and attempt jobs. - commit (str): Full release commit object ID. - tag (str): Exact v-prefixed release tag. + github (GitHub): Repository client. + commit (str): Exact merged source commit. timeout (int | float): Polling budget in seconds. - poll (int | float): Seconds between polls while a run is absent/incomplete. + poll (int | float): Seconds between status reads. Returns: - dict: Completed successful run accepted by ``check_run``. + dict: Successful main-branch CETLVaSt run, including the artifact run ID. Raises: - ReleaseError: API access fails, a completed run fails validation, or the - budget expires. A failed gate includes a full-run retry command. + ReleaseError: CI fails, cannot be read, or does not finish within the budget. """ deadline = time.monotonic() + timeout while time.monotonic() < deadline: - candidates = [run for run in github.runs(head_sha=commit) - if run["head_branch"] == tag and run["event"] in ("push", "workflow_dispatch")] + candidates = [run for run in github.runs(head_sha=commit, branch="main", event="push") + if run["head_branch"] == "main" and run["event"] == "push"] if candidates: run = max(candidates, key=lambda item: item["id"]) if run["status"] == "completed": try: - check_run(run, github.jobs(run), commit, tag) + check_run(run, github.jobs(run), commit) except ReleaseError as ex: raise ReleaseError(f"{ex} See {run['html_url']}. For a transient failure run " - f"gh run rerun {run['id']} --repo {github.repository} " - "(the entire run), then use the printed resume command.") from ex + f"gh run rerun {run['id']} --repo {github.repository}, " + "then rerun the release command.") from ex print(f"Verification: {run['html_url']}") return run time.sleep(poll) - raise ReleaseError("Timed out waiting for release CI; leave the tag/draft intact and retry with --resume.") + raise ReleaseError("Timed out waiting for main CI; rerun the release command when verification finishes.") -def check_draft(release, tag, prerelease, marker): +def check_draft(release, tag, marker): """Require a release to belong to the same script-managed attempt. Args: release (dict): Draft or published release with tag_name, prerelease, and body fields. Publication state is checked by the caller. tag (str): Expected exact tag name. - prerelease (bool): Expected release classification. marker (str): Management marker that must appear in the release body. Raises: ReleaseError: The tag, classification, or management marker disagrees. """ - if (release.get("tag_name") != tag or release.get("prerelease") != prerelease + if (release.get("tag_name") != tag or release.get("prerelease") is not False or marker not in (release.get("body") or "")): raise ReleaseError("Existing release is not a matching script-managed release; refusing to change it.") @@ -456,7 +471,7 @@ def version_core(version): """Extract an orderable numeric core from a supported version string. Args: - version (str): Stable or prerelease version without a v prefix. + version (str): X.Y.Z version without a v prefix. Returns: tuple[int, int, int]: Major, minor, and patch as integers. @@ -472,7 +487,7 @@ def require_new_version(github, version): Args: github (GitHub): Repository client used to list published stable versions. - version (str): Candidate stable or prerelease version without v. + version (str): Candidate X.Y.Z version without v. Raises: ReleaseError: A published stable core is greater than or equal to the @@ -482,21 +497,23 @@ def require_new_version(github, version): raise ReleaseError("A new release must advance the numeric version beyond published stable releases.") -def require_latest_stable(github, tag): +def require_latest_stable(github, tag, release=None): """Require a tag to identify the newest published stable release. Args: github (GitHub): Repository client providing release and version records. tag (str): Validated v-prefixed version being considered for deployment. + release (dict | None): Known publication response; otherwise read the + published release by its tag. Raises: ReleaseError: The release is absent, draft, prerelease, older than the newest stable core, or remote state cannot be read. """ - release = github.release(tag) + release = release if release is not None else github._api_object(f"releases/tags/{tag}") versions = github.published_stable_versions() - if (not release or release["draft"] or release["prerelease"] or not versions - or version_core(tag[1:]) != max(versions)): + if (release.get("tag_name") != tag or release["draft"] or release["prerelease"] + or any(prior > version_core(tag[1:]) for prior in versions)): raise ReleaseError("Only the latest published stable release may replace the documentation site.") @@ -530,12 +547,43 @@ def sync_assets(github, release, tag, directory, names, upload): raise ReleaseError(f"Existing asset differs: {asset['name']}; no assets were overwritten.") missing = set(names) - {asset["name"] for asset in existing} if missing and not upload: - raise ReleaseError("Release is missing required assets.") + raise _AssetsPending("Release is missing required assets.") for name in sorted(missing): # No --clobber: concurrent attempts must fail safely rather than replace bytes. github.cli("release", "upload", tag, str(directory / name)) +def verify_assets(github, release, tag, directory, names, timeout, poll): + """Wait up to 60 seconds for uploaded assets to appear, then verify their bytes. + + Args: + github (GitHub): Repository client. + release (dict): Release with a known ID. + tag (str): Expected vX.Y.Z tag. + directory (Path): Verified local bundle directory. + names (Collection[str]): Expected asset filenames. + timeout (int | float): Upper bound for the visibility wait in seconds. + poll (int | float): Poll interval, capped at two seconds. + + Raises: + ReleaseError: Assets conflict, remain missing, or cannot be read. + """ + try: + sync_assets(github, release, tag, directory, names, upload=False) + return + except _AssetsPending: + pass + deadline = time.monotonic() + min(timeout, 60) + while time.monotonic() < deadline: + time.sleep(min(poll, 2)) + try: + sync_assets(github, release, tag, directory, names, upload=False) + return + except _AssetsPending: + pass + raise ReleaseError("Uploaded assets are not visible yet; rerun the release command.") + + def verify_published_assets(github, release, commit, tag): """Verify a published release directly, independent of CI artifact retention. @@ -562,10 +610,10 @@ def verify_published_assets(github, release, commit, tag): def deploy_docs(github, tag, timeout, poll): - """Complete a main-branch documentation deployment for a published stable tag. + """Deploy verified release documentation through the main-branch Pages environment. Args: - github (GitHub): Client used to find/dispatch CETLVaSt and inspect jobs. + github (GitHub): Client used to find/dispatch documentation runs and inspect jobs. tag (str): Published stable tag, checked by the caller before deployment. timeout (int | float): Polling budget in seconds. poll (int | float): Seconds between deployment-status polls. @@ -575,13 +623,13 @@ def deploy_docs(github, tag, timeout, poll): its deployment job is unsuccessful, or the budget expires. Reuses a successful or in-flight matching dispatch. Otherwise starts a run - on main with the release tag as docs_ref. Prints environment-approval links + on main with the release tag. Prints environment-approval links when required; site-content verification belongs to ``finish_documentation``. """ - title = f"CETLVaSt / {tag}" - runs = [run for run in github.runs(event="workflow_dispatch", branch="main") + title = f"CETL docs / {tag}" + runs = [run for run in github.runs(workflow=DOCS_WORKFLOW, event="workflow_dispatch", branch="main") if run["display_title"] == title] - # A completed successful deployment makes --resume a no-op. + # A completed successful deployment makes repeating the command a no-op. latest = max(runs, key=lambda run: run["id"]) if runs else None if latest is not None: if latest["status"] == "completed" and latest["conclusion"] == "success": @@ -593,11 +641,11 @@ def deploy_docs(github, tag, timeout, poll): previous = latest["id"] - 1 # Resume an in-flight deployment instead of queueing another full build. else: previous = max((run["id"] for run in runs), default=0) - github.cli("workflow", "run", WORKFLOW, "--ref", "main", "-f", f"docs_ref={tag}") + github.cli("workflow", "run", DOCS_WORKFLOW, "--ref", "main", "-f", f"tag={tag}") deadline = time.monotonic() + timeout approval_reported = set() while time.monotonic() < deadline: - runs = [run for run in github.runs(event="workflow_dispatch", branch="main") + runs = [run for run in github.runs(workflow=DOCS_WORKFLOW, event="workflow_dispatch", branch="main") if run["display_title"] == title and run["id"] > previous] if runs: run = max(runs, key=lambda item: item["id"]) @@ -609,180 +657,171 @@ def deploy_docs(github, tag, timeout, poll): if run["conclusion"] != "success" or not any( job["name"] == "deploy-docs" and job["conclusion"] == "success" for job in jobs ): - raise ReleaseError(f"Release is published, but documentation failed: {run['html_url']}; use --resume.") + raise ReleaseError(f"Release is published, but documentation failed: {run['html_url']}; rerun the release command.") print(f"Documentation: {run['html_url']}") return time.sleep(poll) - raise ReleaseError("Release is published, but documentation timed out; use --resume to recover.") + raise ReleaseError("Release is published, but documentation timed out; rerun the release command to recover.") def cut_release(args, git=None, github=None): - """Preflight, publish, or resume a CETL release and its stable documentation. + """Release the header version from verified main source, continuing matching attempts. Args: - args (argparse.Namespace): Options produced by ``main``: root (Path), - repo and remote (str), version and ref (str or None), notes_file - (Path or None), dry_run and resume (bool), and positive timeout and - poll durations in seconds. Omitted version/ref select the freshly - fetched main commit and its header version; resume requires both. - git (Git | None): Checkout adapter; None constructs Git(args.root). - github (GitHub | None): Repository adapter; None constructs GitHub(args.repo). - Tests may inject adapters implementing the same operations. + args (argparse.Namespace): root, repo, remote, optional ref and notes_file, + dry_run, and positive timeout/poll durations in seconds. + git (Git | None): Checkout adapter; None uses args.root. + github (GitHub | None): Repository adapter; None uses args.repo. Raises: - ReleaseError: Preflight, version/ref policy, CI, asset, publication, or - deployment checks fail. Matching remote state is retained for retry. - OSError: Notes, temporary bundle files, or subprocesses are inaccessible. - ValueError: Remote or bundle JSON cannot be decoded. + ReleaseError: Preflight, verification, identity, asset, or deployment checks fail. + OSError: Local files or subprocesses cannot be accessed. + ValueError: A service response or manifest is invalid JSON. KeyError: A service record lacks a required field. - Fetches remote main and prints commands pinned to the selected revision. - Dry runs stop before remote writes. Publication may push a tag, dispatch CI, - create/upload/publish a release, and deploy Pages. Resuming a published - release verifies its actual assets before finishing documentation. + Checks CI and the bundle before pushing a new tag. Existing matching tags, + drafts, uploads, and published releases are reused. Retry commands pin the + selected commit. Dry runs perform preflight without remote writes. """ check_local_tools() - if args.resume and (not args.version or not args.ref): - raise ReleaseError("Use the printed Resume command: --resume requires explicit --version and --ref " - "so an advancing main branch cannot change the release being resumed.") - if args.version: - parse_version(args.version) if not re.fullmatch(r"[\w.-]+/[\w.-]+", args.repo) or args.remote.startswith("-"): raise ReleaseError("Expected --repo OWNER/NAME and a named Git remote.") git = git or Git(args.root) github = github or GitHub(args.repo) if git.run("status", "--porcelain"): - raise ReleaseError("The checkout must be clean, including untracked files. " - "Run git status --short and commit or stash the reported changes.") + raise ReleaseError("The checkout must be clean. Commit or stash the changes reported by git status --short.") if git.remote_repository(args.remote).lower() != args.repo.lower(): raise ReleaseError("Git remote does not match --repo.") repository = github._api_object("") if repository.get("archived") or not repository.get("permissions", {}).get("push"): - raise ReleaseError("GitHub credentials require repository write access. " - "Run gh auth login --hostname github.com with a maintainer account.") + raise ReleaseError("GitHub credentials require repository write access. Run gh auth login --hostname github.com.") if repository["default_branch"] != "main": - raise ReleaseError("The supported release branch is main; review the policy before changing it.") + raise ReleaseError("The release branch must be main.") git.run("fetch", "--no-tags", args.remote, "refs/heads/main:refs/remotes/" + args.remote + "/main") main_commit = git.commit(f"refs/remotes/{args.remote}/main") commit = git.commit(args.ref) if args.ref else main_commit + version = git.validate(commit)["version"] + tag = "v" + version + remote_commit = git.remote_tag(args.remote, tag) + # A rerun of the default command continues the already selected release snapshot. + if remote_commit and not args.ref: + commit = git.commit(remote_commit) try: git.run("merge-base", "--is-ancestor", commit, f"refs/remotes/{args.remote}/main") except ReleaseError as ex: - raise ReleaseError("Release commit must already be merged into the remote main branch.") from ex + raise ReleaseError("Release commit must already be merged into remote main.") from ex for target in {main_commit, commit}: git.require_release_files(target) - version = args.version or git.validate(commit)["version"] - tag = "v" + version - metadata = git.validate(commit, tag) - print(f"Repository: {args.repo}\nTag: {tag}\nCommit: {commit}\nPrerelease: {metadata['prerelease']}") - print("Resume command: " + release_command(args, git.root, version, commit, resume=True)) - remote_commit = git.remote_tag(args.remote, tag) + git.validate(commit, tag) + print(f"Repository: {args.repo}\nTag: {tag}\nCommit: {commit}") + print("Retry command: " + release_command(args, git.root, commit)) if remote_commit and remote_commit != commit: raise ReleaseError(f"Remote tag {tag} points to another commit; refusing to move it.") - local_tags = git.run("tag", "--list", tag) - if local_tags and git.commit(f"refs/tags/{tag}") != commit: + if git.run("tag", "--list", tag) and git.commit(f"refs/tags/{tag}") != commit: raise ReleaseError(f"Local tag {tag} points to another commit.") - release = github.release(tag) - notes = args.notes_file.read_text(encoding="utf-8") if args.notes_file and not release else None - runs = github.runs(head_sha=commit) # Check Actions read access before pushing a tag. marker = f"" + release = github.release(tag) if release: - if not release["draft"] and marker not in (release.get("body") or "") and not args.resume: - raise ReleaseError(f"Version {version} is already published. Merge a reviewed version bump into main " - "before cutting a new release.") - check_draft(release, tag, metadata["prerelease"], marker) + release = github.release_by_id(release["id"]) + check_draft(release, tag, marker) if not remote_commit: - raise ReleaseError("A release exists without its remote tag; manual investigation is required.") + raise ReleaseError("A release exists without its remote tag; refusing to change it.") if not release or release["draft"]: require_new_version(github, version) - deploy_stable = not metadata["prerelease"] and not any( - prior > version_core(version) for prior in github.published_stable_versions()) + deploy_stable = not any(prior > version_core(version) for prior in github.published_stable_versions()) github.preflight(deploy_stable) - if (release or remote_commit) and not args.resume and not args.dry_run: - raise ReleaseError("Release/tag already exists; inspect it and use --resume for a matching attempt.") if not remote_commit: git.run("push", "--dry-run", args.remote, f"{commit}:refs/tags/{tag}") if args.dry_run: - print("Dry run: push immutable tag; await full CETLVaSt; download and verify bundle; create/resume draft; " - "upload matching assets; publish; deploy stable documentation. No remote writes performed.") - print("Release this checked commit: " + release_command( - args, git.root, version, commit, resume=bool(release or remote_commit or args.resume))) + print("Dry run: verify main CI and bundle; push tag; upload and publish release; deploy documentation. " + "No remote writes performed.") + print("Release this checked commit: " + release_command(args, git.root, commit)) return if release and not release["draft"]: verify_published_assets(github, release, commit, tag) - print(f"Release: https://github.com/{args.repo}/releases/tag/{tag}") - finish_documentation(github, tag, version, metadata["prerelease"], args.timeout, args.poll) + finish_documentation(github, tag, version, args.timeout, args.poll, release) return - if not remote_commit: - git.run("push", args.remote, f"{commit}:refs/tags/{tag}") - elif not any(run["head_branch"] == tag and run["event"] in ("push", "workflow_dispatch") - for run in runs): - github.cli("workflow", "run", WORKFLOW, "--ref", tag) - run = wait_for_release_run(github, commit, tag, args.timeout, args.poll) + notes = args.notes_file.read_text(encoding="utf-8") if args.notes_file and not release else None + run = wait_for_release_run(github, commit, args.timeout, args.poll) with tempfile.TemporaryDirectory(prefix="cetl-release-") as temporary: directory = Path(temporary) github.cli("run", "download", str(run["id"]), "--name", "release-bundle", "--dir", str(directory)) names = check_bundle(directory, commit, tag) - if git.remote_tag(args.remote, tag) != commit: - raise ReleaseError("Remote tag changed while waiting for CI.") - # Refresh after the wait: a concurrent attempt may have created or published the draft. + current_tag = git.remote_tag(args.remote, tag) + if current_tag not in (None, commit): + raise ReleaseError("Remote tag changed while waiting for verification.") + if current_tag is None: + git.run("push", args.remote, f"{commit}:refs/tags/{tag}") release = github.release(tag) - if not release: - github.create_draft(tag, commit, metadata["prerelease"], marker, run, notes) - release = github.release(tag) + if release is not None: + release = github.release_by_id(release["id"]) if release is None: - raise ReleaseError("Release is missing after draft creation; retry the printed resume command.") - check_draft(release, tag, metadata["prerelease"], marker) + try: + release = github.create_draft(tag, commit, marker, run, notes) + except ReleaseError: + # Creation can succeed before a lost response, or race another maintainer. + for attempt in range(5): + release = github.release(tag) + if release is not None: + break + time.sleep(min(args.poll, 2)) + if release is None: + raise + if not isinstance(release, dict): + raise ReleaseError("GitHub did not return release metadata; rerun the release command.") + check_draft(release, tag, marker) sync_assets(github, release, tag, directory, names, upload=release["draft"]) - release = github.release(tag) - if release is None: - raise ReleaseError("Release disappeared after asset verification; retry the printed resume command.") - check_draft(release, tag, metadata["prerelease"], marker) - sync_assets(github, release, tag, directory, names, upload=False) + verify_assets(github, release, tag, directory, names, args.timeout, args.poll) + release = github.release_by_id(release["id"]) + check_draft(release, tag, marker) if release["draft"]: require_new_version(github, version) if git.remote_tag(args.remote, tag) != commit: raise ReleaseError("Remote tag changed before publication.") - # Never edit a published release, including on a retry. - github.api(f"releases/{release['id']}", "PATCH", draft=False, make_latest="false" if metadata["prerelease"] else "true") - release = github.release(tag) - if release is None: - raise ReleaseError("Release is missing after publication; retry the printed resume command.") - check_draft(release, tag, metadata["prerelease"], marker) - if release["draft"]: - raise ReleaseError("GitHub still reports a draft; retry the printed resume command.") - sync_assets(github, release, tag, directory, names, upload=False) + publish_error = None + release_id = release["id"] + try: + release = github._api_object(f"releases/{release_id}", "PATCH", draft=False, make_latest="true") + except ReleaseError as ex: + publish_error = ex + release = github.release_by_id(release_id) + check_draft(release, tag, marker) + deadline = time.monotonic() + min(args.timeout, 60) if release["draft"] else 0 + while release["draft"] and time.monotonic() < deadline: + time.sleep(min(args.poll, 2)) + release = github.release_by_id(release["id"]) + check_draft(release, tag, marker) + if release["draft"]: + if publish_error is not None: + raise publish_error + raise ReleaseError("GitHub has not confirmed publication yet; rerun the release command.") + verify_assets(github, release, tag, directory, names, args.timeout, args.poll) print(f"Release: https://github.com/{args.repo}/releases/tag/{tag}") - finish_documentation(github, tag, version, metadata["prerelease"], args.timeout, args.poll) + finish_documentation(github, tag, version, args.timeout, args.poll, release) -def finish_documentation(github, tag, version, prerelease, timeout, poll): - """Deploy and verify documentation when a release is the latest stable one. +def finish_documentation(github, tag, version, timeout, poll, release=None): + """Deploy and verify documentation for the latest published release. Args: github (GitHub): Repository and Pages client. - tag (str): Published, validated release tag. - version (str): Corresponding version without v. - prerelease (bool): Classification validated against the tag. - timeout (int | float): Separate polling budget for deployment and HTML checks. - poll (int | float): Seconds between status/content checks. + tag (str): Published vX.Y.Z tag. + version (str): Matching header version. + timeout (int | float): Budget for each deployment/content check. + poll (int | float): Seconds between checks. + release (dict | None): Known publication response, avoiding list visibility delays. Raises: - ReleaseError: The latest-stable check, deployment, or public HTML - verification fails, or GitHub state cannot be read. + ReleaseError: Publication identity, deployment, or public HTML checks fail. - Prints a skip reason for prereleases and releases superseded by a newer - stable version. Eligible releases may dispatch a deployment through main. + Resuming an older release retains documentation belonging to a newer version. """ - if prerelease: - print("Documentation: built and attached; prereleases do not replace the stable Pages site.") + if any(prior > version_core(version) for prior in github.published_stable_versions()): + print("Documentation: a newer release is published; its site is retained.") else: - if any(prior > version_core(version) for prior in github.published_stable_versions()): - print("Documentation: a newer stable release is published; its site is retained.") - else: - require_latest_stable(github, tag) - deploy_docs(github, tag, timeout, poll) - github.verify_pages(version, timeout, poll) + require_latest_stable(github, tag, release) + deploy_docs(github, tag, timeout, poll) + github.verify_pages(version, timeout, poll) def main(): @@ -796,14 +835,12 @@ def main(): status 1, or keyboard interruption exits with status 130. """ parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--version", help="Override header version, e.g. X.Y.Z-rc.1 (without v)") parser.add_argument("--ref", help="Commit/ref merged into remote main; defaults to freshly fetched main") parser.add_argument("--repo", default="OpenCyphal/CETL") parser.add_argument("--remote", default="origin") parser.add_argument("--root", type=Path, default=Path.cwd()) parser.add_argument("--notes-file", type=Path, help="Reviewed release notes; otherwise generate notes from GitHub") parser.add_argument("--dry-run", action="store_true", help="Perform preflight reads, without remote writes") - parser.add_argument("--resume", action="store_true", help="Resume only an identical tag/script-managed release") parser.add_argument("--timeout", type=int, default=7200, help="Seconds to wait for each CI/deployment phase") parser.add_argument("--poll", type=int, default=15, help="Seconds between GitHub status reads") args = parser.parse_args() @@ -812,7 +849,7 @@ def main(): try: cut_release(args) except KeyboardInterrupt: - parser.exit(130, "Release interrupted. Use the printed Resume command to continue.\n") + parser.exit(130, "Release interrupted. Use the same release command to continue.\n") except (ReleaseError, OSError, ValueError, KeyError) as ex: parser.exit(1, f"Release stopped: {ex}\nFix the reported problem and retry; existing tags/assets are preserved.\n") diff --git a/scripts/release_tools.py b/scripts/release_tools.py index 7d5cb04..cc7f7f9 100644 --- a/scripts/release_tools.py +++ b/scripts/release_tools.py @@ -32,30 +32,29 @@ class ReleaseError(RuntimeError): """ -VERSION = re.compile(r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(?:-(alpha|beta|rc)\.([1-9][0-9]*))?") +VERSION = re.compile(r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)") HEADER = "include/cetl/cetl.hpp" WORKFLOW = "cetlvast.yml" +DOCS_WORKFLOW = "release-docs.yml" RELEASE_FILES = ("scripts/release.py", "scripts/release_tools.py", "scripts/tests/test_release.py", - f".github/workflows/{WORKFLOW}") + f".github/workflows/{WORKFLOW}", f".github/workflows/{DOCS_WORKFLOW}") def parse_version(value): - """Parse a supported stable or prerelease version without a tag prefix. + """Parse a release version without a tag prefix. Args: - value (str): X.Y.Z or X.Y.Z-{alpha,beta,rc}.N, with decimal components, - no leading zeros, and a positive prerelease sequence number. + value (str): X.Y.Z with decimal components and no leading zeros. Returns: - re.Match: Five groups: major, minor, patch, stage, and sequence number. - The final two groups are None for stable versions. + re.Match: Three groups: major, minor, and patch. Raises: ReleaseError: The entire string does not match the version grammar. """ match = VERSION.fullmatch(value) if not match: - raise ReleaseError("Expected X.Y.Z or X.Y.Z-{alpha,beta,rc}.N (no leading zeros; N >= 1).") + raise ReleaseError("Expected X.Y.Z with no leading zeros.") return match @@ -81,34 +80,28 @@ def header_version(text): return ".".join(parts) -def validate(text, tag=None, prerelease=None): - """Check agreement between header macros, a tag, and release classification. +def validate(text, tag=None): + """Check agreement between header macros and a release tag. Args: text (str): CETL header source to inspect. tag (str | None): A v-prefixed version; None uses the header's version. - prerelease (bool | None): Expected GitHub prerelease flag, or None to - derive the classification from the version suffix. Returns: - dict: ``version`` without v, ``tag`` (empty when omitted), and the - boolean ``prerelease`` classification. + dict: ``version`` without v and ``tag`` (empty when omitted). Raises: - ReleaseError: Header/version syntax is invalid, the tag's numeric core - differs from the header, or the supplied classification disagrees. + ReleaseError: Header/version syntax is invalid or the tag differs from + the header version. """ core = header_version(text) version = tag[1:] if tag and tag.startswith("v") else core if tag and not tag.startswith("v"): raise ReleaseError("Release tags must start with v.") - match = parse_version(version) - if ".".join(match.groups()[:3]) != core: + parse_version(version) + if version != core: raise ReleaseError(f"Tag {tag} does not match {HEADER} ({core}).") - is_prerelease = match.group(4) is not None - if prerelease is not None and prerelease != is_prerelease: - raise ReleaseError("GitHub's prerelease flag does not match the tag suffix.") - return {"version": version, "tag": tag or "", "prerelease": is_prerelease} + return {"version": version, "tag": tag or ""} @overload @@ -213,21 +206,20 @@ def commit(self, ref): # --end-of-options prevents a ref supplied at the CLI from becoming a Git option. return self.run("rev-parse", "--verify", "--end-of-options", ref + "^{commit}") - def validate(self, commit, tag=None, prerelease=None): + def validate(self, commit, tag=None): """Validate a committed header using ``validate``'s metadata contract. Args: commit (str): Commit or revision containing the header. tag (str | None): Expected v-prefixed release version. - prerelease (bool | None): Expected GitHub prerelease classification. Returns: - dict: Validated ``version``, ``tag``, and ``prerelease`` fields. + dict: Validated ``version`` and ``tag`` fields. Raises: ReleaseError: Git cannot read the header or its metadata disagrees. """ - return validate(self.run("show", f"{commit}:{HEADER}"), tag, prerelease) + return validate(self.run("show", f"{commit}:{HEADER}"), tag) def require_release_files(self, commit): """Require every path in RELEASE_FILES to exist in a committed tree. @@ -324,7 +316,7 @@ def required_jobs(): return names -def check_run(run: Mapping[str, Any], jobs: Iterable[Mapping[str, Any]], commit: str, tag: str) -> None: +def check_run(run: Mapping[str, Any], jobs: Iterable[Mapping[str, Any]], commit: str) -> None: """Require successful CETLVaSt verification of one exact release revision. Args: @@ -333,16 +325,15 @@ def check_run(run: Mapping[str, Any], jobs: Iterable[Mapping[str, Any]], commit: jobs (Iterable[dict]): Jobs from this run's current attempt, each with name and conclusion fields. The caller must isolate the attempt. commit (str): Expected full commit object ID. - tag (str): Expected v-prefixed tag used as the run's head_branch. Raises: ReleaseError: The run identifies another workflow/revision/event, is unsuccessful or incomplete, or lacks a successful required job. """ - if (run.get("head_sha") != commit or run.get("head_branch") != tag - or run.get("event") not in ("push", "workflow_dispatch") + if (run.get("head_sha") != commit or run.get("head_branch") != "main" + or run.get("event") != "push" or run.get("path") != f".github/workflows/{WORKFLOW}"): - raise ReleaseError("CI must be CETLVaSt on the exact release tag and commit (push or manual dispatch).") + raise ReleaseError("CI must be CETLVaSt for the exact main push commit.") if run.get("status") != "completed" or run.get("conclusion") != "success": raise ReleaseError("Release CI has not completed successfully.") successful = {job["name"] for job in jobs if job.get("conclusion") == "success"} @@ -388,7 +379,7 @@ def check_documentation(html, version): Args: html (str): Decoded HTML containing CETL's projectnumber span. - version (str): Expected version text, including any prerelease suffix. + version (str): Expected X.Y.Z version text. Raises: ReleaseError: The displayed version is absent or differs after stripping @@ -429,7 +420,7 @@ def package(git, commit, tag, docs, output): """Write reproducible release archives, a manifest, and their checksums. Args: - git (Git): Checkout containing the selected commit and local tag. + git (Git): Checkout containing the selected, CI-verified commit. commit (str): Full commit object ID to archive. tag (str): Matching v-prefixed release version. docs (str | Path): Built HTML directory containing index.html. @@ -445,8 +436,6 @@ def package(git, commit, tag, docs, output): bytes from identical inputs. Failures may leave partial local output. """ metadata = git.validate(commit, tag) - if git.commit(f"refs/tags/{tag}") != commit: - raise ReleaseError("Packaging commit differs from the local release tag.") docs, output = Path(docs), Path(output) if not (docs / "index.html").is_file(): raise ReleaseError("Built documentation must contain index.html.") @@ -505,7 +494,7 @@ def check_bundle(directory, commit, tag): if {path.name for path in directory.iterdir()} != expected | {"SHA256SUMS"}: raise ReleaseError("Release bundle contains missing or unexpected files.") manifest = json.loads((directory / "release.json").read_text()) - if manifest != {"version": version, "tag": tag, "prerelease": "-" in version, "commit": commit}: + if manifest != {"version": version, "tag": tag, "commit": commit}: raise ReleaseError("Release bundle metadata does not match the requested release.") sums = (directory / "SHA256SUMS").read_text() if sums != "".join(f"{sha256(directory / name)} {name}\n" for name in sorted(expected)): @@ -513,6 +502,36 @@ def check_bundle(directory, commit, tag): return sorted(expected | {"SHA256SUMS"}) + +def extract_documentation(directory, commit, tag, output): + """Verify a published bundle and extract its versioned documentation. + + Args: + directory (str | Path): Downloaded release assets. + commit (str): Expected full source commit ID. + tag (str): Expected vX.Y.Z tag. + output (str | Path): Empty or nonexistent extraction directory. + + Returns: + Path: HTML directory ready for Pages upload. + + Raises: + ReleaseError: Bundle identity, checksums, displayed version, or output state is invalid. + OSError: Release files cannot be read or extracted. + tarfile.TarError: The archive is invalid or its paths fail the data filter. + """ + check_bundle(directory, commit, tag) + output = Path(output) + if output.exists() and any(output.iterdir()): + raise ReleaseError("Documentation output directory must be empty.") + output.mkdir(parents=True, exist_ok=True) + with tarfile.open(Path(directory) / f"cetl-{tag[1:]}-docs.tar.gz") as archive: + archive.extractall(output, filter="data") + html = output / f"cetl-{tag[1:]}" / "html" + check_documentation((html / "index.html").read_text(), tag[1:]) + return html + + def main(): """Run the validation, packaging, or built-documentation CLI operation. @@ -528,8 +547,7 @@ def main(): subparsers = parser.add_subparsers(dest="operation", required=True) check = subparsers.add_parser("validate", help="Validate checkout and GitHub event metadata.") check.add_argument("--tag") - check.add_argument("--prerelease", choices=("true", "false")) - build = subparsers.add_parser("package", help="Package the validated tagged commit and built documentation.") + build = subparsers.add_parser("package", help="Package the verified commit and built documentation.") build.add_argument("--tag", required=True) build.add_argument("--commit", required=True) build.add_argument("--docs", required=True) @@ -548,15 +566,17 @@ def main(): if args.operation == "package": package(git, git.commit(args.commit), args.tag, args.docs, args.output) return - tag, prerelease = args.tag, None if args.prerelease is None else args.prerelease == "true" + tag = args.tag event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text()) if os.getenv("GITHUB_EVENT_PATH") else {} if os.getenv("GITHUB_EVENT_NAME") == "release": - tag, prerelease = event["release"]["tag_name"], event["release"]["prerelease"] + tag = event["release"]["tag_name"] + if event["release"]["prerelease"]: + raise ReleaseError("GitHub release must be a full release.") elif os.getenv("GITHUB_REF", "").startswith("refs/tags/"): tag = os.environ["GITHUB_REF"][len("refs/tags/"):] if tag and git.commit(f"refs/tags/{tag}") != commit: raise ReleaseError("Checkout does not match the requested tag's commit.") - result = {**git.validate(commit, tag, prerelease), "commit": commit} + result = {**git.validate(commit, tag), "commit": commit} print(json.dumps(result, indent=2)) if os.getenv("GITHUB_OUTPUT"): with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as stream: diff --git a/scripts/tests/test_release.py b/scripts/tests/test_release.py index eeddba0..e7001ff 100644 --- a/scripts/tests/test_release.py +++ b/scripts/tests/test_release.py @@ -21,6 +21,7 @@ import sys import tarfile import tempfile +import textwrap from typing import Any import unittest from unittest.mock import patch @@ -30,7 +31,7 @@ from release import GitHub, check_local_tools, cut_release, require_latest_stable # noqa: E402 from release_tools import (Git, HEADER, RELEASE_FILES, ReleaseError, check_bundle, check_documentation, check_run, command, # noqa: E402 - header_version, package, parse_version, repository_from_url, + extract_documentation, header_version, package, parse_version, repository_from_url, required_jobs, sha256, validate) HEADER_TEXT = "#define CETL_VERSION_MAJOR 2\n#define CETL_VERSION_MINOR 0\n#define CETL_VERSION_PATCH 0\n" @@ -47,16 +48,14 @@ def _workflow_group(pattern: str, workflow: str, flags: int = 0) -> str: class VersionTests(unittest.TestCase): """Check the accepted version grammar, header definitions, and remote identities.""" - def test_stable_and_prerelease(self): - """Accept matching stable versions and each supported prerelease stage.""" + def test_header_and_release_tag_agree(self): + """Derive the release version from the header and validate its exact tag.""" self.assertEqual(header_version(HEADER_TEXT), "2.0.0") - self.assertFalse(validate(HEADER_TEXT, "v2.0.0", False)["prerelease"]) - for stage in ("alpha", "beta", "rc"): - self.assertTrue(validate(HEADER_TEXT, f"v2.0.0-{stage}.1", True)["prerelease"]) + self.assertEqual(validate(HEADER_TEXT, "v2.0.0"), {"version": "2.0.0", "tag": "v2.0.0"}) def test_invalid_versions(self): """Reject ambiguous components, unsupported suffixes, and malformed version text.""" - for version in ("02.0.0", "2.00.0", "2.0", "v2.0.0", "2.0.0-rc.0", "2.0.0-rc.01", + for version in ("02.0.0", "2.00.0", "2.0", "v2.0.0", "2.0.0-rc.1", "2.0.0-alpha.1", "2.0.0-beta.1", "2.0.0-dev.1", "2.0.0+build", "2.0.0\n", "-n", "2.0.0-rc.1/branch"): with self.subTest(version=version), self.assertRaises(ReleaseError): parse_version(version) @@ -72,11 +71,11 @@ def test_bad_header_definitions(self): with self.subTest(header=header), self.assertRaises(ReleaseError): header_version(header) - def test_misaligned_tags_and_flags(self): - """Reject tag/header disagreements and inconsistent prerelease flags.""" - for tag, flag in (("v2.0.1", None), ("2.0.0", None), ("v2.0.0", True), ("v2.0.0-rc.1", False)): - with self.subTest(tag=tag, flag=flag), self.assertRaises(ReleaseError): - validate(HEADER_TEXT, tag, flag) + def test_misaligned_or_malformed_tags(self): + """Reject tags that differ from the header version or have invalid syntax.""" + for tag in ("v2.0.1", "2.0.0", "v2.0.0-rc.1"): + with self.subTest(tag=tag), self.assertRaises(ReleaseError): + validate(HEADER_TEXT, tag) def test_github_remote_formats(self): """Accept supported GitHub remotes and reject credentials or another host.""" @@ -89,17 +88,16 @@ def test_github_remote_formats(self): repository_from_url(url) -def successful_run(commit: str, tag: str) -> dict[str, Any]: - """Build a successful tagged-push Actions record for gate tests. +def successful_run(commit: str) -> dict[str, Any]: + """Build a successful main-push Actions record for gate tests. Args: commit (str): Source object ID exposed as head_sha. - tag (str): Exact release tag exposed as head_branch. Returns: dict: Mutable run fixture identifying CETLVaSt, attempt 1, and run 123. """ - return {"id": 123, "run_attempt": 1, "head_sha": commit, "head_branch": tag, + return {"id": 123, "run_attempt": 1, "head_sha": commit, "head_branch": "main", "event": "push", "path": ".github/workflows/cetlvast.yml", "status": "completed", "conclusion": "success", "html_url": "https://example.test/runs/123"} @@ -109,12 +107,12 @@ class GateTests(unittest.TestCase): def setUp(self): """Create an accepted run and one successful record for every required job.""" - self.workflow_run = successful_run("a" * 40, "v2.0.0") + self.workflow_run = successful_run("a" * 40) self.jobs = [{"name": name, "conclusion": "success"} for name in sorted(required_jobs())] def check(self): """Validate the mutable run/job fixtures against the fixed release identity.""" - check_run(self.workflow_run, self.jobs, "a" * 40, "v2.0.0") + check_run(self.workflow_run, self.jobs, "a" * 40) def test_success(self): """Accept a successful run containing every required release job.""" @@ -126,15 +124,16 @@ def test_missing_or_unsuccessful_job(self): with self.subTest(result=result): jobs = self.jobs[:-1] if result is None else [*self.jobs[:-1], {**self.jobs[-1], "conclusion": result}] with self.assertRaises(ReleaseError): - check_run(self.workflow_run, jobs, "a" * 40, "v2.0.0") + check_run(self.workflow_run, jobs, "a" * 40) def test_wrong_commit_event_workflow_tag_or_state(self): """Reject a run whose identity or completion state differs from the release.""" - for key, value in (("head_sha", "b" * 40), ("event", "pull_request"), ("head_branch", "main"), + for key, value in (("head_sha", "b" * 40), ("event", "pull_request"), ("event", "workflow_dispatch"), + ("head_branch", "feature"), ("path", ".github/workflows/other.yml"), ("status", "in_progress"), ("conclusion", "cancelled")): with self.subTest(key=key), self.assertRaises(ReleaseError): - check_run({**self.workflow_run, key: value}, self.jobs, "a" * 40, "v2.0.0") + check_run({**self.workflow_run, key: value}, self.jobs, "a" * 40) def test_matrix_and_gate_contract_matches_workflow(self): """Keep required job names and aggregate dependencies aligned with CETLVaSt.""" @@ -235,7 +234,7 @@ def test_wrong_published_site_version_fails_with_resume_instruction(self): patch("release.urlopen", return_value=response), \ patch("release.time.monotonic", side_effect=[0, 0, 2]), patch("release.time.sleep"), \ contextlib.redirect_stdout(io.StringIO()): - with self.assertRaisesRegex(ReleaseError, "Documentation displays 1.0.0, expected 2.0.0.*printed resume command"): + with self.assertRaisesRegex(ReleaseError, "Documentation displays 1.0.0, expected 2.0.0.*same release command"): github.verify_pages("2.0.0", 1, 1) def test_site_network_failure_is_distinguished_from_wrong_version(self): @@ -261,6 +260,22 @@ def test_cli_pins_host_and_api_encodes_booleans(self): self.assertIn("body=literal $() and `quotes`\nnext line", args) self.assertIn("github.com", args) + def test_metadata_reads_bypass_cached_responses(self): + """Request fresh metadata when checking state after remote mutations.""" + with patch("release.command", return_value="{}") as execute: + GitHub("example/cetl").api("pages") + self.assertIn("Cache-Control: no-cache", execute.call_args.args[0]) + + def test_draft_creation_returns_the_post_response(self): + """Retain the server-assigned release ID without another listing request.""" + github = GitHub("example/cetl") + record = {"id": 42, "tag_name": "v2.0.0", "draft": True, "prerelease": False, "body": "marker"} + with patch.object(github, "api", return_value=record) as request: + result = github.create_draft("v2.0.0", "a" * 40, "marker", {"html_url": "https://example.test/ci"}, "notes") + self.assertIs(result, record) + self.assertEqual(request.call_args.args, ("releases", "POST")) + self.assertIs(request.call_args.kwargs["prerelease"], False) + def test_pagination_keeps_later_pages(self): """Collect records beyond the first full API page.""" github = GitHub("example/cetl") @@ -296,7 +311,8 @@ def test_stable_version_comparison_uses_numeric_versions(self): github = GitHub("example/cetl") releases = [{"tag_name": tag, "draft": draft, "prerelease": prerelease} for tag, draft, prerelease in ( ("v2.9.0", False, False), ("v2.10.0", False, False), ("v3.0.0", True, False), - ("v4.0.0-rc.1", False, True), ("historical-non-semver", False, False))] + ("v4.0.0-rc.1", False, True), ("v4.0.0-rc.1", False, False), + ("historical-non-semver", False, False))] with patch.object(github, "pages", return_value=releases): self.assertEqual(max(github.published_stable_versions()), (2, 10, 0)) @@ -342,7 +358,7 @@ def __init__(self, git, docs, tag): """Initialize an accessible repository with passing CI and usable Pages setup.""" self.git, self.docs, self.tag = git, docs, tag super().__init__("example/cetl") - self.run: dict[str, Any] | None = successful_run(git.commit("HEAD"), tag) + self.run: dict[str, Any] | None = successful_run(git.commit("HEAD")) self.release_data: dict[str, Any] | None = None self.asset_data: dict[str, bytes] = {} self.writes: list[str] = [] @@ -352,6 +368,7 @@ def __init__(self, git, docs, tag): self.fail_docs = False self.other_versions = set() self.workflow_state = "active" + self.docs_workflow_state = "active" self.pages_type = "workflow" self.environment_policy = {"protected_branches": False, "custom_branch_policies": True} self.branch_policies = [{"name": "main", "type": "branch"}] @@ -385,25 +402,34 @@ def api(self, path, method="GET", **fields): return {"permissions": self.permissions, "default_branch": "main"} if path == "actions/workflows/cetlvast.yml": return {"state": self.workflow_state} + if path == "actions/workflows/release-docs.yml": + return {"state": self.docs_workflow_state} if path == "pages": return {"build_type": self.pages_type, "html_url": "https://example.test/cetl/"} if path == "environments/github-pages": return {"deployment_branch_policy": self.environment_policy} if path == "branches/main": return {"protected": self.main_protected} + if path == "releases/tags/" + self.tag: + if self.release_data is None: + raise ReleaseError("Release has not been published.") + return self.release_data.copy() + if path == "releases/1" and method == "GET": + assert self.release_data is not None + return self.release_data.copy() if path == "releases/1" and method == "PATCH": assert self.release_data is not None self.writes.append("publish") self.release_data["draft"] = fields["draft"] - return self.release_data + return self.release_data.copy() raise AssertionError((path, method, fields)) def release(self, tag): """Return release_data; each fixture represents a single release identity.""" return self.release_data - def runs(self, **filters): - """Return the docs run for dispatch queries, or the fixture's tag run otherwise.""" + def runs(self, workflow="cetlvast.yml", **filters): + """Return the docs run for dispatch queries, or the fixture's main run otherwise.""" if filters.get("event") == "workflow_dispatch": return [self.docs_run] if self.docs_run else [] return [self.run] if self.run else [] @@ -423,10 +449,11 @@ def asset_matches(self, asset, path): """Delegate byte/digest comparison to the production client implementation.""" return super().asset_matches(asset, path) - def create_draft(self, tag, commit, prerelease, marker, run, notes=None): + def create_draft(self, tag, commit, marker, run, notes=None): """Record draft creation and retain the classification and management marker.""" self.writes.append("draft") - self.release_data = {"id": 1, "draft": True, "tag_name": tag, "prerelease": prerelease, "body": marker} + self.release_data = {"id": 1, "draft": True, "tag_name": tag, "prerelease": False, "body": marker} + return self.release_data.copy() def cli(self, *args) -> str: """Simulate downloads, immutable uploads, and workflow dispatches. @@ -454,7 +481,7 @@ def cli(self, *args) -> str: elif args[:2] == ("workflow", "run"): self.writes.append("dispatch") if args[args.index("--ref") + 1] == "main": - self.docs_run = {"id": (self.docs_run or {"id": 200})["id"] + 1, "display_title": f"CETLVaSt / {self.tag}", + self.docs_run = {"id": (self.docs_run or {"id": 200})["id"] + 1, "display_title": f"CETL docs / {self.tag}", "status": "completed", "conclusion": "failure" if self.fail_docs else "success", "html_url": "https://example.test/docs"} else: @@ -499,9 +526,9 @@ def setUp(self): self.commit = self.git.commit("HEAD") self.docs = root / "html" self.docs.mkdir() - (self.docs / "index.html").write_text("Fixture built documentation") - self.args = argparse.Namespace(version="2.0.0", ref=self.commit, repo="example/cetl", remote="origin", - root=self.root, notes_file=None, dry_run=False, resume=False, timeout=1, poll=1) + (self.docs / "index.html").write_text('2.0.0') + self.args = argparse.Namespace(ref=self.commit, repo="example/cetl", remote="origin", + root=self.root, notes_file=None, dry_run=False, timeout=1, poll=1) self.github = SimulatedGitHub(self.git, self.docs, "v2.0.0") def cut(self): @@ -515,13 +542,8 @@ def add_tag(self, tag="v2.0.0"): self.git.run("tag", tag) self.git.run("push", "origin", f"refs/tags/{tag}") - def test_full_release_rehearsal_and_published_resume(self): - """Publish once and verify a repeated matching attempt performs no remote writes.""" - self.add_tag() - self.git.run("tag", "-d", "v2.0.0") # Cutting a release must not require a local tag. - self.args.resume = True - # The simulated CI checkout has the tag, just as actions/checkout fetch-depth: 0 does. - self.git.run("fetch", "--tags", "origin") + def test_full_release_rehearsal_and_repeat(self): + """Publish once and verify that repeating the command performs no remote writes.""" output = self.cut() self.assertIn("Documentation:", output) assert self.github.release_data is not None @@ -531,25 +553,17 @@ def test_full_release_rehearsal_and_published_resume(self): self.cut() self.assertEqual(self.github.writes, writes) - def test_new_tag_push_and_prerelease(self): - """Push and publish a prerelease while retaining the stable documentation site.""" - self.args.version = "2.0.0-rc.1" - self.github.tag = "v" + self.args.version - assert self.github.run is not None - self.github.run["head_branch"] = self.github.tag - # CI receives the pushed tag in its separate checkout. Simulate that fetch just before packaging. + def test_new_tag_is_created_after_verified_bundle_download(self): + """Keep the version untagged until verified CI artifacts have been downloaded.""" original_cli = self.github.cli def cli(*args): - """Fetch the pushed tag before emulating CI's separate packaging checkout.""" + """Check tag timing while emulating the verified bundle download.""" if args[:2] == ("run", "download"): - self.git.run("fetch", "--tags", "origin") + self.assertIsNone(self.git.remote_tag("origin", self.github.tag)) return original_cli(*args) self.github.cli = cli self.cut() self.assertEqual(self.git.remote_tag("origin", self.github.tag), self.commit) - assert self.github.release_data is not None - self.assertTrue(self.github.release_data["prerelease"]) - self.assertNotIn("dispatch", self.github.writes) def test_dry_run_has_no_remote_writes(self): """Complete preflight without pushing a tag or changing simulated GitHub state.""" @@ -566,7 +580,7 @@ def test_defaults_fetch_main_and_derive_header_version(self): new_commit = self.git.commit("HEAD") self.git.run("push", "origin", "main") self.git.run("checkout", "--detach", self.commit) - self.args.ref, self.args.version, self.args.dry_run = None, None, True + self.args.ref, self.args.dry_run = None, True output = self.cut() self.assertIn("Tag: v2.1.0", output) self.assertIn("Commit: " + new_commit, output) @@ -613,13 +627,6 @@ def test_protected_branch_pages_policy_rejects_unprotected_main(self): with self.assertRaisesRegex(ReleaseError, "Allow main"): self.cut() - def test_prerelease_does_not_require_pages_configuration(self): - """Allow prerelease preflight without a stable Pages deployment configuration.""" - self.args.version, self.args.dry_run = "2.0.0-rc.1", True - self.github.pages_type = "legacy" - self.github.branch_policies = [] - self.cut() - self.assertFalse(self.github.writes) def test_authentication_failure(self): """Stop on inaccessible repository metadata before creating remote release state.""" @@ -658,12 +665,20 @@ def test_unmerged_commit(self): self.cut() self.assertFalse(self.github.writes) - def test_mismatched_version_never_pushes(self): - """Reject a version differing from the selected header before pushing its tag.""" - self.args.version = "2.0.1" - with self.assertRaisesRegex(ReleaseError, "does not match"): + def test_mismatched_bundle_never_pushes_tag(self): + """Reject mismatched CI metadata before allocating a release tag.""" + original_cli = self.github.cli + def cli(*args): + """Inject incorrect release identity into a downloaded bundle.""" + result = original_cli(*args) + if args[:2] == ("run", "download"): + (Path(args[args.index("--dir") + 1]) / "release.json").write_text("{}") + return result + self.github.cli = cli + with self.assertRaisesRegex(ReleaseError, "metadata"): self.cut() - self.assertIsNone(self.git.remote_tag("origin", "v2.0.1")) + self.assertIsNone(self.git.remote_tag("origin", "v2.0.0")) + self.assertFalse(self.github.writes) def test_conflicting_remote_tag(self): """Preserve an existing remote tag when the requested source commit differs.""" @@ -673,86 +688,177 @@ def test_conflicting_remote_tag(self): self.git.run("commit", "-m", "Next commit") self.git.run("push", "origin", "main") self.args.ref = self.git.commit("HEAD") - self.args.resume = True with self.assertRaisesRegex(ReleaseError, "another commit"): self.cut() self.assertEqual(self.git.remote_tag("origin", "v2.0.0"), self.commit) - def test_existing_tag_requires_resume(self): - """Require explicit resume intent for an existing matching tag.""" + def test_matching_tag_continues_automatically(self): + """Continue an existing matching tag using the ordinary release command.""" self.add_tag() - with self.assertRaisesRegex(ReleaseError, "--resume"): - self.cut() + self.cut() + assert self.github.release_data is not None + self.assertFalse(self.github.release_data["draft"]) - def test_dry_run_for_existing_tag_prints_a_working_resume_command(self): - """Include resume intent and the exact commit in an existing-tag dry-run command.""" + def test_dry_run_pins_an_existing_release_commit(self): + """Print a command pinned to the existing release snapshot.""" self.add_tag() self.args.dry_run = True output = self.cut() release_line = next(line for line in output.splitlines() if line.startswith("Release this checked commit:")) - self.assertIn("--resume", release_line) self.assertIn("--ref " + self.commit, release_line) self.assertFalse(self.github.writes) - def test_resume_requires_pinned_version_and_commit(self): - """Reject resume options that could follow an advancing main branch.""" - self.args.resume, self.args.ref = True, None - with self.assertRaisesRegex(ReleaseError, "printed Resume command"): - self.cut() - self.assertIsNone(self.git.remote_tag("origin", "v2.0.0")) + def test_default_repeat_retains_the_tagged_snapshot_when_main_advances(self): + """Continue the selected release after later commits reach main.""" + self.cut() + before = list(self.github.writes) + (self.root / "README.md").write_text("Later documentation change") + self.git.run("add", ".") + self.git.run("commit", "-m", "Update documentation") + self.git.run("push", "origin", "main") + self.args.ref = None + self.github.run = None + self.assertIn("Commit: " + self.commit, self.cut()) + self.assertEqual(self.git.remote_tag("origin", "v2.0.0"), self.commit) + self.assertEqual(self.github.writes, before) - def test_failed_ci_cannot_create_draft_or_publish(self): - """Block draft creation and publication when tagged CI fails.""" - self.add_tag() - self.args.resume = True + def test_failed_ci_cannot_create_a_tag_or_release(self): + """Leave the version available when main verification fails.""" assert self.github.run is not None self.github.run["conclusion"] = "failure" with self.assertRaisesRegex(ReleaseError, "CI"): self.cut() + self.assertIsNone(self.git.remote_tag("origin", "v2.0.0")) self.assertFalse(self.github.writes) + def test_corrected_main_commit_can_use_the_unallocated_version(self): + """Release a corrected commit after failed CI without changing the version.""" + assert self.github.run is not None + self.github.run["conclusion"] = "failure" + with self.assertRaises(ReleaseError): + self.cut() + (self.root / "README.md").write_text("Corrected source") + self.git.run("add", ".") + self.git.run("commit", "-m", "Correct verification failure") + self.git.run("push", "origin", "main") + corrected = self.git.commit("HEAD") + self.args.ref = None + self.github.run = successful_run(corrected) + self.cut() + self.assertEqual(self.git.remote_tag("origin", "v2.0.0"), corrected) + self.assertEqual(self.github.writes.count("publish"), 1) + def test_unrelated_draft_cannot_be_adopted(self): """Refuse a draft lacking the matching script-management marker.""" self.add_tag() - self.args.resume = True - self.github.release_data = {"tag_name": "v2.0.0", "prerelease": False, "body": "Manual draft", "draft": True} + self.github.release_data = {"id": 1, "tag_name": "v2.0.0", "prerelease": False, "body": "Manual draft", "draft": True} with self.assertRaisesRegex(ReleaseError, "script-managed"): self.cut() self.assertFalse(self.github.writes) - def test_missing_release_after_draft_creation_stops_before_upload(self): - """Refuse uploads when a draft creation attempt leaves no visible release.""" - self.add_tag() - self.args.resume = True - with patch.object(self.github, "create_draft"): - with self.assertRaisesRegex(ReleaseError, "Release is missing after draft creation"): - self.cut() - self.assertNotIn("upload", self.github.writes) - self.assertNotIn("publish", self.github.writes) + def test_draft_creation_response_survives_release_list_delay(self): + """Publish from the creation response even when release listings remain stale.""" + with patch.object(self.github, "release", return_value=None): + self.cut() + assert self.github.release_data is not None + self.assertFalse(self.github.release_data["draft"]) + self.assertEqual(self.github.writes.count("draft"), 1) + self.assertEqual(self.github.writes.count("publish"), 1) + + def test_lost_creation_response_recovers_the_existing_draft(self): + """Continue a successful draft write whose response was lost in transit.""" + original_create = self.github.create_draft + def create(*args): + """Create the real fixture draft before simulating a transport failure.""" + original_create(*args) + raise ReleaseError("Creation response lost") + with patch.object(self.github, "create_draft", side_effect=create): + self.cut() + self.assertEqual(self.github.writes.count("draft"), 1) + self.assertEqual(self.github.writes.count("publish"), 1) + + def test_delayed_asset_listing_is_retried_before_publication(self): + """Wait for uploaded assets to become visible without uploading them twice.""" + original_assets = self.github.assets + hidden_reads = 0 + def assets(release): + """Hide one post-upload listing to model a delayed GitHub read.""" + nonlocal hidden_reads + if len(self.github.asset_data) == 5 and hidden_reads == 0: + hidden_reads += 1 + return [] + return original_assets(release) + with patch.object(self.github, "assets", side_effect=assets), patch("release.time.sleep"): + self.cut() + self.assertEqual(hidden_reads, 1) + self.assertEqual(self.github.writes.count("upload"), 5) + self.assertEqual(self.github.writes.count("publish"), 1) + + def test_delayed_publication_confirmation_is_retried_by_id(self): + """Confirm a completed publish through its ID when the PATCH response is stale.""" + original_api = self.github.api + def api(path, method="GET", **fields): + """Publish successfully but return the preceding draft state once.""" + result = original_api(path, method, **fields) + if method == "PATCH": + result = {**result, "draft": True} + return result + with patch.object(self.github, "api", side_effect=api), patch("release.time.sleep"): + self.cut() + self.assertEqual(self.github.writes.count("publish"), 1) + assert self.github.release_data is not None + self.assertFalse(self.github.release_data["draft"]) + + def test_lost_publication_response_is_confirmed_without_publishing_twice(self): + """Finish a successful publication when its response was lost in transit.""" + original_api = self.github.api + def api(path, method="GET", **fields): + """Commit the publication before simulating a failed response.""" + result = original_api(path, method, **fields) + if method == "PATCH": + raise ReleaseError("Publication response lost") + return result + with patch.object(self.github, "api", side_effect=api): + self.cut() + self.assertEqual(self.github.writes.count("publish"), 1) + self.assertIn("dispatch", self.github.writes) + + def test_repeat_refreshes_stale_draft_metadata_by_id(self): + """Recognize a completed publication even when discovery still reports a draft.""" + self.cut() + assert self.github.release_data is not None + stale_record = {**self.github.release_data, "draft": True} + before = list(self.github.writes) + with patch.object(self.github, "release", return_value=stale_record): + self.cut() + self.assertEqual(self.github.writes, before) + + def test_new_publication_can_deploy_before_release_list_catches_up(self): + """Use the known publication record when version listings still show older releases.""" + with patch.object(self.github, "published_stable_versions", return_value={(1, 0, 0)}): + self.cut() + self.assertIn("dispatch", self.github.writes) def test_remote_tag_movement_while_waiting_prevents_publication(self): """Recheck the remote tag after CI and reject movement before draft creation.""" self.add_tag() - self.args.resume = True with patch.object(self.git, "remote_tag", side_effect=[self.commit, "b" * 40]): with self.assertRaisesRegex(ReleaseError, "changed"): self.cut() self.assertFalse(self.github.writes) - def test_absent_ci_times_out_without_publication(self): - """Dispatch missing tagged CI and stop without publication when waiting expires.""" - self.add_tag() - self.args.resume = True + def test_absent_ci_times_out_without_allocating_a_version(self): + """Wait for main verification without creating a tag or a draft.""" self.github.run = None with patch("release.time.monotonic", side_effect=[0, 2]): with self.assertRaisesRegex(ReleaseError, "Timed out"): self.cut() - self.assertEqual(self.github.writes, ["dispatch"]) + self.assertIsNone(self.git.remote_tag("origin", "v2.0.0")) + self.assertFalse(self.github.writes) def test_interrupted_upload_resumes_without_overwrite(self): """Retain uploaded bytes and upload only missing assets on a matching retry.""" self.add_tag() - self.args.resume = True self.github.fail_after_upload = 2 with self.assertRaisesRegex(ReleaseError, "interrupted"): self.cut() @@ -770,7 +876,6 @@ def test_interrupted_upload_resumes_without_overwrite(self): def test_conflicting_asset_prevents_publication(self): """Preserve conflicting remote bytes and leave the release as a draft.""" self.add_tag() - self.args.resume = True self.github.fail_after_upload = 1 with self.assertRaises(ReleaseError): self.cut() @@ -786,7 +891,6 @@ def test_conflicting_asset_prevents_publication(self): def test_documentation_failure_can_recover_after_publication(self): """Retry failed documentation deployment without republishing the release.""" self.add_tag() - self.args.resume = True self.github.fail_docs = True with self.assertRaisesRegex(ReleaseError, "published, but documentation failed"): self.cut() @@ -800,7 +904,6 @@ def test_documentation_failure_can_recover_after_publication(self): def test_published_resume_works_without_ci_runs_or_artifacts(self): """Verify published assets after CI retention and the original notes file expire.""" self.add_tag() - self.args.resume = True self.cut() before = list(self.github.writes) self.github.run = None @@ -811,7 +914,6 @@ def test_published_resume_works_without_ci_runs_or_artifacts(self): def test_published_resume_detects_corrupt_actual_assets(self): """Reject corrupted published checksums without modifying remote release state.""" self.add_tag() - self.args.resume = True self.cut() self.github.asset_data["SHA256SUMS"] = b"corrupt" before = list(self.github.writes) @@ -822,7 +924,6 @@ def test_published_resume_detects_corrupt_actual_assets(self): def test_resume_reuses_pending_documentation_deployment(self): """Reuse an awaiting-approval deployment instead of dispatching another build.""" self.add_tag() - self.args.resume = True self.cut() assert self.github.docs_run is not None self.github.docs_run.update(status="waiting", conclusion=None) @@ -832,34 +933,85 @@ def test_resume_reuses_pending_documentation_deployment(self): self.cut() self.assertEqual(self.github.writes, before) - def test_new_release_cannot_downgrade_or_prerelease_a_stable_version(self): - """Require new stable and prerelease attempts to advance the published numeric core.""" + def test_new_release_must_advance_the_published_version(self): + """Reject a new release whose numeric version is already published.""" self.github.other_versions = {(2, 0, 0)} - for version in ("2.0.0", "2.0.0-rc.1"): - self.args.version = version - with self.assertRaisesRegex(ReleaseError, "advance"): - self.cut() - self.assertIsNone(self.git.remote_tag("origin", "v" + version)) + with self.assertRaisesRegex(ReleaseError, "advance"): + self.cut() + self.assertIsNone(self.git.remote_tag("origin", "v2.0.0")) def test_resume_old_release_does_not_roll_documentation_back(self): """Retain the newer stable site when an older published release is resumed.""" self.add_tag() - self.args.resume = True self.cut() before = list(self.github.writes) self.github.other_versions.add((2, 1, 0)) self.github.docs_run = None - self.assertIn("newer stable release", self.cut()) + self.assertIn("newer release", self.cut()) self.assertEqual(self.github.writes, before) - with self.assertRaisesRegex(ReleaseError, "latest published"): + with self.assertRaisesRegex(ReleaseError, "published"): require_latest_stable(self.github, "v2.0.0") def test_unpublished_tag_cannot_deploy_stable_documentation(self): """Require a published stable release before allowing a Pages deployment.""" self.add_tag() - with self.assertRaisesRegex(ReleaseError, "latest published"): + with self.assertRaisesRegex(ReleaseError, "published"): require_latest_stable(self.github, "v2.0.0") + def test_packaging_verified_main_does_not_require_a_tag(self): + """Build release artifacts before allocating the public version tag.""" + output = Path(self.temporary.name) / "untagged-bundle" + package(self.git, self.commit, "v2.0.0", self.docs, output) + self.assertEqual(len(check_bundle(output, self.commit, "v2.0.0")), 5) + self.assertEqual(self.git.run("tag", "--list"), "") + self.assertIsNone(self.git.remote_tag("origin", "v2.0.0")) + + def test_documentation_workflow_validates_a_merged_release_tag(self): + """Run the deployment workflow's identity check against an actual merged tag.""" + self.add_tag() + workflow = (SCRIPTS.parent / ".github/workflows/release-docs.yml").read_text() + script = _workflow_group(r"python3 - <<'PYTHON'\n(.*?)^ PYTHON", workflow, re.M | re.S) + output = Path(self.temporary.name) / "outputs" + env = {**os.environ, "PYTHONPATH": str(SCRIPTS), "RELEASE_TAG": "v2.0.0", "GITHUB_OUTPUT": str(output)} + result = subprocess.run([sys.executable, "-c", textwrap.dedent(script)], cwd=self.root, + env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("commit=" + self.commit, output.read_text()) + self.assertIn("version=2.0.0", output.read_text()) + + def test_documentation_workflow_rejects_an_unmerged_release_tag(self): + """Refuse deployment of a tag whose source is outside main history.""" + self.git.run("switch", "-c", "unmerged") + (self.root / "README.md").write_text("Unreviewed source") + self.git.run("add", ".") + self.git.run("commit", "-m", "Unmerged source") + self.git.run("tag", "v2.0.0") + self.git.run("switch", "main") + workflow = (SCRIPTS.parent / ".github/workflows/release-docs.yml").read_text() + script = _workflow_group(r"python3 - <<'PYTHON'\n(.*?)^ PYTHON", workflow, re.M | re.S) + output = Path(self.temporary.name) / "outputs" + env = {**os.environ, "PYTHONPATH": str(SCRIPTS), "RELEASE_TAG": "v2.0.0", "GITHUB_OUTPUT": str(output)} + result = subprocess.run([sys.executable, "-c", textwrap.dedent(script)], cwd=self.root, + env=env, capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(output.exists()) + + def test_published_documentation_is_extracted_and_version_checked(self): + """Deploy the checksummed documentation archive from the verified release.""" + root = Path(self.temporary.name) + package(self.git, self.commit, "v2.0.0", self.docs, root / "published") + html = extract_documentation(root / "published", self.commit, "v2.0.0", root / "site") + self.assertEqual(html, root / "site/cetl-2.0.0/html") + check_documentation((html / "index.html").read_text(), "2.0.0") + + def test_published_documentation_with_wrong_displayed_version_is_rejected(self): + """Refuse deployment when a valid bundle displays another version.""" + root = Path(self.temporary.name) + (self.docs / "index.html").write_text('1.0.0') + package(self.git, self.commit, "v2.0.0", self.docs, root / "published") + with self.assertRaisesRegex(ReleaseError, "Documentation displays"): + extract_documentation(root / "published", self.commit, "v2.0.0", root / "site") + def test_packaging_contents_reproducibility_and_corruption(self): """Verify archive layout, licenses, reproducible bytes, and manifest validation.""" self.add_tag() @@ -887,7 +1039,7 @@ def test_cli_validates_tagged_checkout_and_release_event(self): result = subprocess.run([sys.executable, str(SCRIPTS / "release_tools.py"), "validate"], cwd=self.root, env=env, capture_output=True, text=True) self.assertNotEqual(result.returncode, 0) - self.assertIn("prerelease flag", result.stderr) + self.assertIn("full release", result.stderr) event.write_text(json.dumps({"release": {"tag_name": "v2.0.0", "prerelease": False}})) result = subprocess.run([sys.executable, str(SCRIPTS / "release_tools.py"), "validate"], cwd=self.root, env=env, capture_output=True, text=True) From 0b9237f406f7c93b9ec8ae4ca1c510b0c7107025 Mon Sep 17 00:00:00 2001 From: Scott Dixon Date: Sat, 24 Oct 2026 14:57:21 -0700 Subject: [PATCH 2/3] Use release and asset IDs throughout recovery and documentation deployment --- .github/workflows/release-docs.yml | 15 +++++++-- scripts/release.py | 54 ++++++++++++++++++++++++------ scripts/tests/test_release.py | 43 ++++++++++++++++++++++-- 3 files changed, 95 insertions(+), 17 deletions(-) diff --git a/.github/workflows/release-docs.yml b/.github/workflows/release-docs.yml index 45cc194..876dbf6 100644 --- a/.github/workflows/release-docs.yml +++ b/.github/workflows/release-docs.yml @@ -8,6 +8,10 @@ on: description: Published release tag required: true type: string + release_id: + description: Release ID supplied by the release script + required: true + type: number permissions: contents: read @@ -62,6 +66,7 @@ jobs: GH_TOKEN: ${{ github.token }} RELEASE_TAG: ${{ steps.version.outputs.tag }} RELEASE_COMMIT: ${{ steps.version.outputs.commit }} + RELEASE_ID: ${{ inputs.release_id }} PYTHONPATH: scripts run: | python3 - <<'PYTHON' @@ -72,11 +77,12 @@ jobs: github = GitHub(os.environ['GITHUB_REPOSITORY']) tag, commit = os.environ['RELEASE_TAG'], os.environ['RELEASE_COMMIT'] - require_latest_stable(github, tag) + release = github.release_by_id(int(os.environ['RELEASE_ID'])) + require_latest_stable(github, tag, release) wait_for_release_run(github, commit, 300, 15) bundle = Path('release-bundle') bundle.mkdir() - github.cli('release', 'download', tag, '--dir', str(bundle)) + github.download_assets(release, bundle) extract_documentation(bundle, commit, tag, 'site') PYTHON - uses: actions/configure-pages@v6 @@ -87,12 +93,15 @@ jobs: env: GH_TOKEN: ${{ github.token }} RELEASE_TAG: ${{ steps.version.outputs.tag }} + RELEASE_ID: ${{ inputs.release_id }} PYTHONPATH: scripts run: | python3 - <<'PYTHON' import os from release import GitHub, require_latest_stable - require_latest_stable(GitHub(os.environ['GITHUB_REPOSITORY']), os.environ['RELEASE_TAG']) + github = GitHub(os.environ['GITHUB_REPOSITORY']) + release = github.release_by_id(int(os.environ['RELEASE_ID'])) + require_latest_stable(github, os.environ['RELEASE_TAG'], release) PYTHON - uses: actions/deploy-pages@v5 id: deployment diff --git a/scripts/release.py b/scripts/release.py index cc98d5c..8bc9d23 100644 --- a/scripts/release.py +++ b/scripts/release.py @@ -24,7 +24,7 @@ from urllib.error import HTTPError from urllib.request import Request, urlopen -from release_tools import (DOCS_WORKFLOW, Git, ReleaseError, WORKFLOW, check_bundle, check_documentation, check_run, command, +from release_tools import (DOCS_WORKFLOW, Git, ReleaseError, WORKFLOW, asset_names, check_bundle, check_documentation, check_run, command, parse_version, sha256) _PYTHON_MINIMUM = (3, 14) @@ -390,6 +390,38 @@ def create_draft(self, tag, commit, marker, run, notes=None): return self._api_object("releases", "POST", tag_name=tag, target_commitish=commit, name=tag, draft=True, prerelease=False, body=body) + def download_assets(self, release, directory): + """Download a release's complete bundle through immutable asset IDs. + + Args: + release (dict): Validated release record with id and tag_name. + directory (str | Path): Empty destination directory. + + Raises: + ReleaseError: Assets are missing, unexpected, incomplete, or cannot be downloaded. + OSError: The destination cannot be written. + + Uses asset endpoints instead of tag discovery so publication-index delays + cannot hide a release whose ID is already known. Callers verify the + resulting manifest and checksums with ``check_bundle``. + """ + version = release["tag_name"][1:] + parse_version(version) + assets = self.assets(release) + expected = asset_names(version) | {"SHA256SUMS"} + if len(assets) != len(expected) or {asset["name"] for asset in assets} != expected: + raise ReleaseError("Published release has missing or unexpected assets.") + directory = Path(directory) + directory.mkdir(parents=True, exist_ok=True) + if any(directory.iterdir()): + raise ReleaseError("Release download directory must be empty.") + for asset in assets: + data = command(["gh", "api", "--hostname", "github.com", "-H", "Accept: application/octet-stream", + f"repos/{self.repository}/releases/assets/{asset['id']}"], binary=True) + if len(data) != asset["size"]: + raise ReleaseError(f"Incomplete download: {asset['name']}; rerun the release command.") + (directory / asset["name"]).write_bytes(data) + def asset_matches(self, asset, path): """Compare a remote asset's uploaded bytes with a local file. @@ -497,20 +529,18 @@ def require_new_version(github, version): raise ReleaseError("A new release must advance the numeric version beyond published stable releases.") -def require_latest_stable(github, tag, release=None): +def require_latest_stable(github, tag, release): """Require a tag to identify the newest published stable release. Args: github (GitHub): Repository client providing release and version records. tag (str): Validated v-prefixed version being considered for deployment. - release (dict | None): Known publication response; otherwise read the - published release by its tag. + release (dict): Publication response or metadata read through its known ID. Raises: ReleaseError: The release is absent, draft, prerelease, older than the newest stable core, or remote state cannot be read. """ - release = release if release is not None else github._api_object(f"releases/tags/{tag}") versions = github.published_stable_versions() if (release.get("tag_name") != tag or release["draft"] or release["prerelease"] or any(prior > version_core(tag[1:]) for prior in versions)): @@ -603,13 +633,13 @@ def verify_published_assets(github, release, commit, tag): """ with tempfile.TemporaryDirectory(prefix="cetl-published-") as temporary: directory = Path(temporary) - github.cli("release", "download", tag, "--dir", str(directory)) + github.download_assets(release, directory) names = check_bundle(directory, commit, tag) sync_assets(github, release, tag, directory, names, upload=False) print("Published release assets verified.") -def deploy_docs(github, tag, timeout, poll): +def deploy_docs(github, tag, timeout, poll, release_id): """Deploy verified release documentation through the main-branch Pages environment. Args: @@ -617,6 +647,7 @@ def deploy_docs(github, tag, timeout, poll): tag (str): Published stable tag, checked by the caller before deployment. timeout (int | float): Polling budget in seconds. poll (int | float): Seconds between deployment-status polls. + release_id (int): Known published release ID passed to the deployment workflow. Raises: ReleaseError: API/dispatch access fails, the newest matching run fails, @@ -641,7 +672,8 @@ def deploy_docs(github, tag, timeout, poll): previous = latest["id"] - 1 # Resume an in-flight deployment instead of queueing another full build. else: previous = max((run["id"] for run in runs), default=0) - github.cli("workflow", "run", DOCS_WORKFLOW, "--ref", "main", "-f", f"tag={tag}") + github.cli("workflow", "run", DOCS_WORKFLOW, "--ref", "main", "-f", f"tag={tag}", + "-f", f"release_id={release_id}") deadline = time.monotonic() + timeout approval_reported = set() while time.monotonic() < deadline: @@ -800,7 +832,7 @@ def cut_release(args, git=None, github=None): finish_documentation(github, tag, version, args.timeout, args.poll, release) -def finish_documentation(github, tag, version, timeout, poll, release=None): +def finish_documentation(github, tag, version, timeout, poll, release): """Deploy and verify documentation for the latest published release. Args: @@ -809,7 +841,7 @@ def finish_documentation(github, tag, version, timeout, poll, release=None): version (str): Matching header version. timeout (int | float): Budget for each deployment/content check. poll (int | float): Seconds between checks. - release (dict | None): Known publication response, avoiding list visibility delays. + release (dict): Known publication response, avoiding list visibility delays. Raises: ReleaseError: Publication identity, deployment, or public HTML checks fail. @@ -820,7 +852,7 @@ def finish_documentation(github, tag, version, timeout, poll, release=None): print("Documentation: a newer release is published; its site is retained.") else: require_latest_stable(github, tag, release) - deploy_docs(github, tag, timeout, poll) + deploy_docs(github, tag, timeout, poll, release["id"]) github.verify_pages(version, timeout, poll) diff --git a/scripts/tests/test_release.py b/scripts/tests/test_release.py index e7001ff..5f83aa0 100644 --- a/scripts/tests/test_release.py +++ b/scripts/tests/test_release.py @@ -30,7 +30,7 @@ sys.path.insert(0, str(SCRIPTS)) from release import GitHub, check_local_tools, cut_release, require_latest_stable # noqa: E402 -from release_tools import (Git, HEADER, RELEASE_FILES, ReleaseError, check_bundle, check_documentation, check_run, command, # noqa: E402 +from release_tools import (Git, HEADER, RELEASE_FILES, asset_names, ReleaseError, check_bundle, check_documentation, check_run, command, # noqa: E402 extract_documentation, header_version, package, parse_version, repository_from_url, required_jobs, sha256, validate) @@ -276,6 +276,36 @@ def test_draft_creation_returns_the_post_response(self): self.assertEqual(request.call_args.args, ("releases", "POST")) self.assertIs(request.call_args.kwargs["prerelease"], False) + def test_published_asset_downloads_use_ids_without_tag_discovery(self): + """Download all bundle files directly even when tag discovery is unavailable.""" + github = GitHub("example/cetl") + files = {name: name.encode() for name in asset_names("2.0.0") | {"SHA256SUMS"}} + records = [{"id": index, "name": name, "size": len(data)} + for index, (name, data) in enumerate(files.items(), 1)] + by_id = {record["id"]: files[record["name"]] for record in records} + def download(args, **kwargs): + """Serve immutable bytes for the requested asset ID.""" + self.assertTrue(kwargs["binary"]) + self.assertIn("Accept: application/octet-stream", args) + return by_id[int(args[-1].rsplit("/", 1)[1])] + with tempfile.TemporaryDirectory() as directory, \ + patch.object(github, "assets", return_value=records), \ + patch.object(github, "release", side_effect=AssertionError("Tag lookup attempted")), \ + patch("release.command", side_effect=download): + github.download_assets({"id": 42, "tag_name": "v2.0.0"}, directory) + self.assertEqual({path.name: path.read_bytes() for path in Path(directory).iterdir()}, files) + + def test_unexpected_asset_names_cannot_escape_the_download_directory(self): + """Reject hostile asset paths before writing files or downloading bytes.""" + github = GitHub("example/cetl") + with tempfile.TemporaryDirectory() as directory, \ + patch.object(github, "assets", return_value=[{"name": "../escape", "id": 1, "size": 0}]), \ + patch("release.command") as download: + with self.assertRaisesRegex(ReleaseError, "missing or unexpected"): + github.download_assets({"id": 42, "tag_name": "v2.0.0"}, directory) + download.assert_not_called() + self.assertEqual(list(Path(directory).iterdir()), []) + def test_pagination_keeps_later_pages(self): """Collect records beyond the first full API page.""" github = GitHub("example/cetl") @@ -449,6 +479,11 @@ def asset_matches(self, asset, path): """Delegate byte/digest comparison to the production client implementation.""" return super().asset_matches(asset, path) + def download_assets(self, release, directory): + """Copy published fixture bytes without tag discovery or network access.""" + for name, data in self.asset_data.items(): + (Path(directory) / name).write_bytes(data) + def create_draft(self, tag, commit, marker, run, notes=None): """Record draft creation and retain the classification and management marker.""" self.writes.append("draft") @@ -950,13 +985,15 @@ def test_resume_old_release_does_not_roll_documentation_back(self): self.assertIn("newer release", self.cut()) self.assertEqual(self.github.writes, before) with self.assertRaisesRegex(ReleaseError, "published"): - require_latest_stable(self.github, "v2.0.0") + record = self.github.release_data or {"tag_name": "v2.0.0", "draft": True, "prerelease": False} + require_latest_stable(self.github, "v2.0.0", record) def test_unpublished_tag_cannot_deploy_stable_documentation(self): """Require a published stable release before allowing a Pages deployment.""" self.add_tag() with self.assertRaisesRegex(ReleaseError, "published"): - require_latest_stable(self.github, "v2.0.0") + record = self.github.release_data or {"tag_name": "v2.0.0", "draft": True, "prerelease": False} + require_latest_stable(self.github, "v2.0.0", record) def test_packaging_verified_main_does_not_require_a_tag(self): """Build release artifacts before allocating the public version tag.""" From bafcf5f9c63181ccee334c9550d19291bc4a84a9 Mon Sep 17 00:00:00 2001 From: Scott Dixon Date: Sat, 24 Oct 2026 14:59:53 -0700 Subject: [PATCH 3/3] Keep digest fallback downloads independent of release discovery --- scripts/release.py | 15 +++++++++------ scripts/tests/test_release.py | 17 +++++++++++++---- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/scripts/release.py b/scripts/release.py index 8bc9d23..c8b1677 100644 --- a/scripts/release.py +++ b/scripts/release.py @@ -416,18 +416,21 @@ def download_assets(self, release, directory): if any(directory.iterdir()): raise ReleaseError("Release download directory must be empty.") for asset in assets: - data = command(["gh", "api", "--hostname", "github.com", "-H", "Accept: application/octet-stream", - f"repos/{self.repository}/releases/assets/{asset['id']}"], binary=True) + data = self._asset_bytes(asset) if len(data) != asset["size"]: raise ReleaseError(f"Incomplete download: {asset['name']}; rerun the release command.") (directory / asset["name"]).write_bytes(data) + def _asset_bytes(self, asset): + """Read immutable asset bytes by ID through authenticated GitHub CLI access.""" + return command(["gh", "api", "--hostname", "github.com", "-H", "Accept: application/octet-stream", + f"repos/{self.repository}/releases/assets/{asset['id']}"], binary=True) + def asset_matches(self, asset, path): """Compare a remote asset's uploaded bytes with a local file. Args: - asset (dict): Asset record with state and size, optional digest, and - release_tag for the download fallback. + asset (dict): Asset record with id, state, size, and optional digest. path (Path): Local file whose name identifies the remote asset. Returns: @@ -444,7 +447,7 @@ def asset_matches(self, asset, path): return asset["digest"] == "sha256:" + sha256(path) # Older GitHub assets may not have a server-side digest. Compare the actual bytes. with tempfile.TemporaryDirectory(prefix="cetl-asset-") as directory: - self.cli("release", "download", asset["release_tag"], "--pattern", path.name, "--dir", directory) + (Path(directory) / path.name).write_bytes(self._asset_bytes(asset)) return sha256(Path(directory) / path.name) == sha256(path) @@ -573,7 +576,7 @@ def sync_assets(github, release, tag, directory, names, upload): if {asset["name"] for asset in existing} - set(names): raise ReleaseError("Release contains unexpected assets; refusing to overwrite it.") for asset in existing: - if not github.asset_matches({**asset, "release_tag": tag}, directory / asset["name"]): + if not github.asset_matches(asset, directory / asset["name"]): raise ReleaseError(f"Existing asset differs: {asset['name']}; no assets were overwritten.") missing = set(names) - {asset["name"] for asset in existing} if missing and not upload: diff --git a/scripts/tests/test_release.py b/scripts/tests/test_release.py index 5f83aa0..04b78ef 100644 --- a/scripts/tests/test_release.py +++ b/scripts/tests/test_release.py @@ -295,6 +295,19 @@ def download(args, **kwargs): github.download_assets({"id": 42, "tag_name": "v2.0.0"}, directory) self.assertEqual({path.name: path.read_bytes() for path in Path(directory).iterdir()}, files) + def test_asset_without_a_server_digest_is_compared_by_id(self): + """Verify downloaded bytes when GitHub omits an asset's digest.""" + github = GitHub("example/cetl") + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "asset" + path.write_bytes(b"correct") + record = {"id": 42, "state": "uploaded", "size": 7} + with patch("release.command", return_value=b"correct") as download: + self.assertTrue(github.asset_matches(record, path)) + self.assertTrue(download.call_args.args[0][-1].endswith("/releases/assets/42")) + with patch("release.command", return_value=b"changed"): + self.assertFalse(github.asset_matches(record, path)) + def test_unexpected_asset_names_cannot_escape_the_download_directory(self): """Reject hostile asset paths before writing files or downloading bytes.""" github = GitHub("example/cetl") @@ -440,10 +453,6 @@ def api(self, path, method="GET", **fields): return {"deployment_branch_policy": self.environment_policy} if path == "branches/main": return {"protected": self.main_protected} - if path == "releases/tags/" + self.tag: - if self.release_data is None: - raise ReleaseError("Release has not been published.") - return self.release_data.copy() if path == "releases/1" and method == "GET": assert self.release_data is not None return self.release_data.copy()