diff --git a/.devcontainer/toolshed-amd64/devcontainer.json b/.devcontainer/toolshed-amd64/devcontainer.json index ba5504ae..e2372e80 100644 --- a/.devcontainer/toolshed-amd64/devcontainer.json +++ b/.devcontainer/toolshed-amd64/devcontainer.json @@ -1,6 +1,6 @@ { "name": "linux development environment (linux/amd64)", - "image": "ghcr.io/opencyphal/toolshed:ts24.4.3", + "image": "ghcr.io/opencyphal/toolshed:ts26.4.5", "workspaceFolder": "/repo", "workspaceMount": "source=${localWorkspaceFolder},target=/repo,type=bind,consistency=delegated", "mounts": [ diff --git a/.devcontainer/toolshed-arm64/devcontainer.json b/.devcontainer/toolshed-arm64/devcontainer.json index 8e704285..b628f385 100644 --- a/.devcontainer/toolshed-arm64/devcontainer.json +++ b/.devcontainer/toolshed-arm64/devcontainer.json @@ -1,6 +1,6 @@ { "name": "linux development environment (linux/arm64)", - "image": "ghcr.io/opencyphal/toolshed:ts24.4.3", + "image": "ghcr.io/opencyphal/toolshed:ts26.4.5", "workspaceFolder": "/repo", "workspaceMount": "source=${localWorkspaceFolder},target=/repo,type=bind,consistency=delegated", "mounts": [ diff --git a/.github/workflows/cetlvast.yml b/.github/workflows/cetlvast.yml index 1e81e429..e29efed5 100644 --- a/.github/workflows/cetlvast.yml +++ b/.github/workflows/cetlvast.yml @@ -1,4 +1,12 @@ name: "CETLVaSt" +run-name: CETLVaSt / ${{ inputs.docs_ref || github.ref_name }} + +permissions: + contents: read + +concurrency: + group: cetlvast-${{ github.ref }}-${{ inputs.docs_ref }} + cancel-in-progress: false on: push: # Further filtering is done in the jobs. @@ -11,13 +19,50 @@ on: workflow_dispatch: inputs: docs_ref: - description: "Tag or commit to build the docs from instead of the dispatched ref. Set by release-docs." + description: "Release tag to verify in full and deploy from main (stable releases only)." required: false default: "" jobs: + version: + runs-on: ubuntu-latest + outputs: + commit: ${{ steps.version.outputs.commit }} + tag: ${{ steps.version.outputs.tag }} + version: ${{ steps.version.outputs.version }} + prerelease: ${{ steps.version.outputs.prerelease }} + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ inputs.docs_ref || github.event.release.tag_name || github.sha }} + fetch-depth: 0 + - name: Validate library and release versions + id: version + env: + DOCS_REF: ${{ inputs.docs_ref }} + run: | + if [ -n "$DOCS_REF" ]; then + python3 scripts/release_tools.py validate --tag "$DOCS_REF" + else + python3 scripts/release_tools.py validate + fi + + release-tests: + runs-on: ubuntu-latest + needs: [version] + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ needs.version.outputs.commit }} + - uses: actions/setup-python@v7 + with: + python-version: '3.14' + - name: Test release tooling and rehearse in disposable Git repositories + run: python3 -m unittest discover -s scripts/tests -v + cache-warmup: if: > + startsWith(github.ref, 'refs/tags/v') || github.event_name == 'release' || github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.message, '#verification') || @@ -27,10 +72,11 @@ jobs: (github.event_name == 'pull_request') runs-on: ubuntu-latest container: ghcr.io/opencyphal/toolshed:ts26.4.5 + needs: [version] steps: - uses: actions/checkout@v7 with: - ref: ${{ inputs.docs_ref }} + ref: ${{ needs.version.outputs.commit }} - name: Cache ext modules id: cetlvast-ext uses: actions/cache@v6 @@ -43,25 +89,27 @@ jobs: run: cmake --workflow --preset manual-simple-workflow-online verification-amd64: - # Skipped in the docs-only dispatch that release-docs starts. if: > - !inputs.docs_ref && ( + startsWith(github.ref, 'refs/tags/v') || github.event_name == 'release' || github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.message, '#verification') || contains(github.ref, '/main') || contains(github.ref, '/issue/') || - (github.event_name == 'pull_request')) + (github.event_name == 'pull_request') runs-on: ubuntu-latest container: ghcr.io/opencyphal/toolshed:ts26.4.5 needs: - cache-warmup + - version strategy: matrix: toolchain: ["gcc-native-32", "clang-native"] standard: ["cpp-14", "cpp-17", "cpp-20", "cpp-23"] steps: - uses: actions/checkout@v7 + with: + ref: ${{ needs.version.outputs.commit }} - name: Cache ext modules uses: actions/cache@v6 with: @@ -82,27 +130,30 @@ jobs: cetlvast/build/compile_commands.json cetlvast/build/*/**/gcovr_html/*.* if-no-files-found: error + overwrite: true verification-arm64: - # Skipped in the docs-only dispatch that release-docs starts. if: > - !inputs.docs_ref && ( + startsWith(github.ref, 'refs/tags/v') || github.event_name == 'release' || github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.message, '#verification') || contains(github.ref, '/main') || contains(github.ref, '/issue/') || - (github.event_name == 'pull_request')) + (github.event_name == 'pull_request') runs-on: ubuntu-24.04-arm container: ghcr.io/opencyphal/toolshed:ts26.4.5 needs: - cache-warmup + - version strategy: matrix: toolchain: ["gcc-native", "clang-native"] standard: ["cpp-14", "cpp-17", "cpp-20"] steps: - uses: actions/checkout@v7 + with: + ref: ${{ needs.version.outputs.commit }} - name: Cache ext modules uses: actions/cache@v6 with: @@ -123,23 +174,24 @@ jobs: cetlvast/build/compile_commands.json cetlvast/build/*/**/gcovr_html/*.* if-no-files-found: error + overwrite: true # Compilers the current toolshed image no longer ships run here, in the last image that has them. verification-legacy-arm64: - # Skipped in the docs-only dispatch that release-docs starts. if: > - !inputs.docs_ref && ( + startsWith(github.ref, 'refs/tags/v') || github.event_name == 'release' || github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.message, '#verification') || contains(github.ref, '/main') || contains(github.ref, '/issue/') || - (github.event_name == 'pull_request')) + (github.event_name == 'pull_request') runs-on: ubuntu-24.04-arm # ts26.4.x dropped g++-7. Keep this pinned to ts24.4.3 rather than following the other jobs. container: ghcr.io/opencyphal/toolshed:ts24.4.3 needs: - cache-warmup + - version strategy: matrix: # GCC 7 is still what much of the Arduino ecosystem ships (#149). It only supports C++14 here. @@ -147,6 +199,8 @@ jobs: standard: ["cpp-14"] steps: - uses: actions/checkout@v7 + with: + ref: ${{ needs.version.outputs.commit }} - name: Cache ext modules uses: actions/cache@v6 with: @@ -167,11 +221,14 @@ jobs: cetlvast/build/compile_commands.json cetlvast/build/*/**/gcovr_html/*.* if-no-files-found: error + overwrite: true docs: if: > + startsWith(github.ref, 'refs/tags/v') || github.event_name == 'release' || github.event_name == 'workflow_dispatch' || + contains(github.event.head_commit.message, '#verification') || contains(github.event.head_commit.message, '#docs') || contains(github.ref, '/main') || contains(github.ref, '/issue/') || @@ -180,23 +237,11 @@ jobs: container: ghcr.io/opencyphal/toolshed:ts26.4.5 needs: - cache-warmup + - version steps: - uses: actions/checkout@v7 with: - ref: ${{ inputs.docs_ref }} - - name: check-release-version - if: ${{ github.event_name == 'release' && !github.event.release.prerelease }} - env: - RELEASE_TAG: ${{ github.event.release.tag_name }} - run: | - header=include/cetl/cetl.hpp - major=$(sed -n 's/^#define CETL_VERSION_MAJOR \([0-9]*\)$/\1/p' "$header") - minor=$(sed -n 's/^#define CETL_VERSION_MINOR \([0-9]*\)$/\1/p' "$header") - patch=$(sed -n 's/^#define CETL_VERSION_PATCH \([0-9]*\)$/\1/p' "$header") - if [ "$RELEASE_TAG" != "v$major.$minor.$patch" ]; then - echo "::error file=$header::Release $RELEASE_TAG does not match CETL_VERSION_* in $header (v$major.$minor.$patch)." - exit 1 - fi + ref: ${{ needs.version.outputs.commit }} - name: Cache ext modules uses: actions/cache@v6 with: @@ -205,40 +250,126 @@ jobs: fail-on-cache-miss: true - name: doc-gen-configure working-directory: cetlvast - run: cmake --preset configure-gcc-native-cpp-14-offline + env: + CETL_RELEASE_VERSION: ${{ needs.version.outputs.version }} + run: cmake --preset configure-gcc-native-cpp-14-offline -DCETL_PROJECT_VERSION="$CETL_RELEASE_VERSION" - name: doc-gen-build working-directory: cetlvast/build run: ninja docs + - name: Verify the version displayed by the built documentation + env: + CETL_RELEASE_VERSION: ${{ needs.version.outputs.version }} + run: >- + python3 scripts/release_tools.py verify-docs --version "$CETL_RELEASE_VERSION" + --path cetlvast/build/suites/docs/html/index.html - name: Setup Pages - if: ${{ github.event_name != 'pull_request' }} + if: ${{ github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' }} uses: actions/configure-pages@v6 - name: Upload docs - if: ${{ github.event_name != 'pull_request' }} + if: ${{ github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' }} uses: actions/upload-pages-artifact@v5 with: path: "cetlvast/build/suites/docs/html/" - # A release run's ref is its tag, which the github-pages environment does not allow to deploy. Instead, start a - # dispatch run on main that builds the docs from the release tag and deploys them. - release-docs: - if: ${{ github.event_name == 'release' && !github.event.release.prerelease }} - needs: - - docs - permissions: - actions: write # to dispatch this workflow + - name: Save built release documentation + uses: actions/upload-artifact@v7 + with: + name: release-docs + path: cetlvast/build/suites/docs/html/ + include-hidden-files: true + if-no-files-found: error + overwrite: true + + release-ready: + if: >- + always() && ( + startsWith(github.ref, 'refs/tags/v') || + github.event_name == 'release' || + github.event_name == 'workflow_dispatch' || + contains(github.event.head_commit.message, '#verification') || + contains(github.ref, '/main') || + contains(github.ref, '/issue/') || + github.event_name == 'pull_request') runs-on: ubuntu-latest + needs: [version, release-tests, cache-warmup, verification-amd64, verification-arm64, verification-legacy-arm64, docs] steps: - - name: Dispatch docs deployment from main + - name: Require every release dependency to succeed env: - GH_TOKEN: ${{ github.token }} - RELEASE_TAG: ${{ github.event.release.tag_name }} - run: > - gh workflow run cetlvast.yml --repo "$GITHUB_REPOSITORY" --ref main -f docs_ref="$RELEASE_TAG" + RESULTS: ${{ toJSON(needs) }} + run: | + python3 - <<'PYTHON' + import json + import os + results = json.loads(os.environ['RESULTS']) + failed = [name for name, job in results.items() if job['result'] != 'success'] + if failed: + raise SystemExit('Required verification did not succeed: ' + ', '.join(failed)) + PYTHON + + release-assets: + runs-on: ubuntu-latest + needs: [version, release-ready] + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ needs.version.outputs.commit }} + fetch-depth: 0 + - uses: actions/download-artifact@v8 + with: + name: release-docs + path: release-docs + - name: Package the verified source, headers, and documentation + env: + RELEASE_TAG: ${{ needs.version.outputs.tag }} + RELEASE_COMMIT: ${{ needs.version.outputs.commit }} + LIBRARY_VERSION: ${{ needs.version.outputs.version }} + run: | + # PR/branch runs rehearse packaging using a local tag; it is never pushed. + if [ -z "$RELEASE_TAG" ]; then + RELEASE_TAG="v$LIBRARY_VERSION-rc.$GITHUB_RUN_ID" + git tag "$RELEASE_TAG" "$RELEASE_COMMIT" + fi + python3 scripts/release_tools.py package --tag "$RELEASE_TAG" --commit "$RELEASE_COMMIT" \ + --docs release-docs --output release-bundle + python3 - <<'PYTHON' + import json + import os + from pathlib import Path + version = json.loads(Path('release-bundle/release.json').read_text())['version'] + with open(os.environ['GITHUB_ENV'], 'a') as stream: + stream.write('PACKAGED_VERSION=' + version + '\n') + PYTHON + - name: Compile consumers using only the packaged headers + run: | + mkdir consumer + tar -xzf "release-bundle/cetl-$PACKAGED_VERSION-headers.tar.gz" -C consumer + cat > consumer/main.cpp <<'CPP' + #include + int main() + { + const cetl::pf17::optional value{42}; + return value.value() == 42 ? 0 : 1; + } + CPP + for standard in c++14 c++20; do + c++ -std="$standard" -Wall -Wextra -Werror -pedantic \ + -I "consumer/cetl-$PACKAGED_VERSION/include" consumer/main.cpp -o consumer/test + ./consumer/test + done + - uses: actions/upload-artifact@v7 + with: + name: release-bundle + path: release-bundle/ + if-no-files-found: error + overwrite: true + retention-days: 30 deploy-docs: - if: ${{ github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' }} - needs: - - docs + if: ${{ github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && needs.version.outputs.tag != '' && needs.version.outputs.prerelease == 'false' }} + needs: [version, release-ready, release-assets] + concurrency: + group: cetl-stable-pages + cancel-in-progress: false # Grant GITHUB_TOKEN the permissions required to make a Pages deployment permissions: contents: read @@ -253,6 +384,20 @@ jobs: # Specify runner + deployment step runs-on: ubuntu-latest steps: + - uses: actions/checkout@v7 + with: + ref: ${{ needs.version.outputs.commit }} + - name: Prevent unpublished releases or older retries from replacing stable docs + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ needs.version.outputs.tag }} + PYTHONPATH: scripts + run: | + python3 - <<'PYTHON' + import os + from release import GitHub, require_latest_stable + require_latest_stable(GitHub(os.environ['GITHUB_REPOSITORY']), os.environ['RELEASE_TAG']) + PYTHON - name: Deploy to GitHub Pages id: deployment uses: actions/deploy-pages@v5 diff --git a/.vscode/settings.json b/.vscode/settings.json index 54531f56..a13867e2 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -51,5 +51,8 @@ "files.associations": { "*.in": "cmake", "*.tcc": "cpp" - } + }, + "cSpell.words": [ + "anaphors" + ] } diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..bda6e483 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,148 @@ +# General Project Rules + +## GitHub Actions + +Before writing or editing a workflow, look up each action's latest release and the runtime its +`action.yml` declares; never pin a version from memory. Dependabot covers `github-actions`. + +## Prefer Python over Shell Scripts + +Shell scripts suffer from multiple compatibility issues including different shells on +different platforms using different system tools. Python is Python. Prefer using Python +where scripting is needed. + +### Scripting Use Refinement + +Only use scripting where another technology does not already provide adequate automation +capabilities. For example, if working in a build system prefer using that build system's +native capabilities first and Python only if those capabilities are not suitable. + +# Prose Rules + +Use these rules when writing natural language. + +## No Stupid Writing Rule + +Be Precise and Concise When Writing + +### Do not pontificate. + +> ❌ The obj backend never emits these attributes: it compiles the C or C++ it generates as part of its own pipeline, and warnings there would be about code the user never sees. + +> ✅ The obj backend never emits these attributes. + +### Do not prevaricate. + +> ❌ There are ways this is implemented and these ways include several which are dangerous and should be used with caution. + +> ✅ Do not use raw pointers. + +### Do not ramble + +Where the banner is self explanatory: + +> ❌ Each page carries a gating mode banner. Structural means coverage was inferred from registered test names; behavioral means a cell counts as covered only if a matching test actually executed and passed. The published pages are structural: the docs build compiles the compiler but does not run the test suite. The behavioral run is a release gate in CI, where the generators consume the suite's JUnit results and fail the build on a regression. Read the banner rather than assuming. + +> ✅ Each page carries a gating mode banner. + +### Do not be inane. + +Do not assume the user is an idiot. + +> ❌ ... are written by the report generators and rebuilt whenever the site is published. Editing them by hand accomplishes nothing — the next build overwrites the file. + +> ✅ ... are written by the report generators and rebuilt whenever the site is published. + +## No Kindergarten Headings + +> ❌ What is in it + +This talks down to an audience that is technical, highly educated, and whom appreciate concise, precise prose. + +> ✅ Contents + +## No Click-Bait Teasers + +> ❌ The one fact that makes this easy +> You can use cyanoacrylate to tack the workpiece down first. + +This is click bait. Do _not_ use anything adjacent to "this one weird trick..." or any other tagline meant to titillate or excite a user into reading something. People will read what they need to read. We are not getting paid to make them navigate through our docs. + +> ✅ Use cyanoacrylate to tack the workpiece down first. + +Write the fact, not the teaser. In some cases, where something really should be emphasized for scanability, use an emoji sigil rather than smarmy text: + +> 💡 Tip: Use cyanoacrylate to tack the workpiece down first. + +## Anaphor rules + +In code comments, the use of anaphors must consider the value of paragraphs that are internally complete as readers are often using searches for specific knowledge that does not require the full context of the text above it. Technical writing is not BuzzFeed. + +> ❌ There is no affordance for it. + +There is no antecedent in the previous sentence. It might be in a title but we probably grepped for this using newlines as a delimiter. + +> ✅ There is no affordance for the menu item. + +In this version, the real subject survives delineation. Prefer this. + +# Documentation Rules + +Use these rules when the natural language is documentation of the project, code, or +any other technical instruction. + +Prose Rules also apply to documentation where the documentation uses natural language. + +## No documentation ghosts (no bananas rule) + +When you **remove** something, remove it cleanly — don't leave behind a note that it +used to be there or a negative explaining its absence. The deletion does not need to +be written down. + +Turning a removal into a stated negative weakens the logic of the text: + +> V1: We want bananas and oranges for the table. +> *(remove bananas)* +> ❌ V2: We want oranges but not bananas for the table. + +"Not bananas" is bizarre on its face — it invites the reader to ask *why bananas? why +not apples? Does excluding bananas imply apples are fine?* A clean elision carries +no such baggage: + +> ✅ V2: We want oranges for the table. + +**The exception — a strong reappearance signal.** Keep the negative only when there +is a concrete, logical force that would otherwise pull the removed thing back: a +common recommendation, an obvious-looking default, something the wider world (or a +model's priors) will keep suggesting. Then the negative is doing real work — +pre-empting the wrong answer — and should carry its *reason*: + +> ✅ Use apple juice when sick. We tried orange juice — the usual cold remedy — and it +> did nothing for us. + +Here "we tried orange juice" earns its place because the OJ-for-a-cold prior is +strong and would otherwise creep back in. Absent that kind of pull, prefer the +clean elision. + +### Oracle variation + +Ghosts can also appear as future predictions that have no embodiment. For example: + +> ❌ The current UI uses blue and red colours. + +"Current" suggests there is a future UI that does not use blue and red colours yet this +plan is not visible and the user is left wondering, "is this UI deprecated? Should I +be using it? Is blue and red a temporary decision?". + +> ✅ The UI uses blue and red colours. + +Here we are providing actionable documentation. Yes, the UI is supported. Yes, you +should continue using blue and red colours if you want to be consistent with the +documented GUI. + +## CLIs Are Self Documenting + +Do not add markdown or other copy-and-paste documentation for CLI flags when `--help` is +adequate and authoritative. There are times when an example of using the CLI to perform +some action is relevant but this is only in service of documenting some other concern +where the CLI can be used, _not_ in documenting the CLI itself. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..43c994c2 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9f7eec6b..06810de9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -107,13 +107,14 @@ and manually run it. ### TLDR -See [devcontainer.json](.devcontainer/toolshed/devcontainer.json) for the current value of x for `ts24.4.x`. +The [AMD64](.devcontainer/toolshed-amd64/devcontainer.json) and +[ARM64](.devcontainer/toolshed-arm64/devcontainer.json) devcontainers use Toolshed `ts26.4.5`, matching the main CI jobs. ``` -docker pull ghcr.io/opencyphal/toolshed:ts24.4.x +docker pull ghcr.io/opencyphal/toolshed:ts26.4.5 git clone {this repo} cd CETL -docker run --rm -it -v ${PWD}:/repo ghcr.io/opencyphal/toolshed:ts24.4.x +docker run --rm -it -v ${PWD}:/repo ghcr.io/opencyphal/toolshed:ts26.4.5 cd cetlvast cmake --workflow --preset workflow-clang-native-cpp-14-online ``` @@ -122,7 +123,7 @@ cmake --workflow --preset workflow-clang-native-cpp-14-online 1. Pull the OpenCyphal dev-container used for CETL: ``` -docker pull ghcr.io/opencyphal/toolshed:ts24.4.x +docker pull ghcr.io/opencyphal/toolshed:ts26.4.5 ``` 2. Clone CETL, cd into the repo, and launch an interactive terminal session of the dev container. This command will mount the current directory (`${PWD}`) in @@ -133,7 +134,7 @@ building. ``` git clone {this repo} cd CETL -docker run --rm -it -v ${PWD}:/repo ghcr.io/opencyphal/toolshed:ts24.4.x +docker run --rm -it -v ${PWD}:/repo ghcr.io/opencyphal/toolshed:ts26.4.5 ``` 3. See available configurations ``` @@ -160,7 +161,7 @@ To ensure that the formatting matches the expectations of the CI suite, invoke Clang-Format of the correct version from the container (be sure to use the correct image tag): ``` -docker run --rm -v ${PWD}:/repo ghcr.io/opencyphal/toolshed:ts24.4.x /bin/sh -c 'cd cetlvast && cmake --preset configure-clang-native-cpp-17-offline && cd build && ninja danger-danger-cetlvast-clang-format-in-place' +docker run --rm -v ${PWD}:/repo ghcr.io/opencyphal/toolshed:ts26.4.5 /bin/sh -c 'cd cetlvast && cmake --preset configure-clang-native-cpp-17-offline && cd build && ninja danger-danger-cetlvast-clang-format-in-place' ``` # `issue/*` and hashtag-based CI triggering @@ -175,6 +176,13 @@ will force the CI to execute jobs named `verification` and `docs`. Note that if the job you requested is dependent on other jobs that are not triggered, it will not run. +Version/header validation and release-tool tests run independently of hashtags. +The `#verification` trigger includes documentation and release archive checks. +Version tags and release dispatches always run full verification. A release +documentation dispatch also verifies the complete release; it is not a docs-only shortcut. +See [Releasing CETL](RELEASING.md) for the maintainer Python CLI and recovery instructions. +The CMake `release` target is a legacy build/test alias and does not publish a release. + # IDE-specific notes ## CLion diff --git a/README.md b/README.md index 21f82f2c..6c6693c8 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,8 @@ Also under each "cetl/pfXX" folder will be a "cetlpf.hpp" header. These headers so by violating certain AUTOSAR-14 rules. We recommend using the CETL polyfill types directly in code that adheres to AUTOSAR-14. -There are currently three ways to take a dependency on CETL (see [issue #9](https://github.com/OpenCyphal/CETL/issues/9) -for a future 4th way): +There are currently three ways to take a dependency on CETL. See +[Releasing CETL](RELEASING.md) for the release process: 1. **Download the [release zip/tar from github](https://github.com/OpenCyphal/CETL/releases)** – This, of course, provides no strong link back to the CETL source and test artifacts associated with it but you have full control over it and can diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 00000000..8977ff67 --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,36 @@ +# Releasing CETL + +Merge the reviewed changes and version bump into `main`. Choose patch versions for +compatible fixes, minor versions for compatible additions, and major versions for +incompatible changes. Documentation/test-only changes do not require a version bump. + +```sh +python3 scripts/release.py --dry-run +``` + +Run the printed **Release this checked commit** command to publish exactly the +version and commit that passed preflight. The script fetches remote `main`, reads +the header version, checks the tools/access/workflow/Pages setup, waits for full CI, +verifies the archives, publishes the release, and verifies the deployed documentation. +Failures report the corrective action. No manual test, checksum, or site-version +check is required. + +For a prerelease, supply its version explicitly: + +```sh +python3 scripts/release.py --version 2.0.0-rc.1 --dry-run +``` + +Use `--ref COMMIT` to select a particular merged commit and `--notes-file PATH` for +reviewed release notes; otherwise GitHub generates the notes. See `--help` for +forks, alternate remotes, and wait limits. + +## Recovery + +Run the printed **Resume command** after fixing the reported problem. It retains +matching tags/assets, finishes missing uploads, and retries documentation deployment. +Published releases are verified from their actual assets, so expired CI artifacts +do not require manual recovery. Conflicting tags or asset contents are refused. + +For a transient CI failure, run the full-run retry command printed in the error, +then resume. A source fix needs a new reviewed commit and a new version/tag. diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 00000000..febcae4d --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,6 @@ +{ + "include": ["scripts"], + "extraPaths": ["scripts"], + "pythonVersion": "3.14", + "typeCheckingMode": "standard" +} diff --git a/scripts/release.py b/scripts/release.py new file mode 100644 index 00000000..77433c6f --- /dev/null +++ b/scripts/release.py @@ -0,0 +1,821 @@ +#!/usr/bin/env python3 +"""Publish verified CETL releases using Git and authenticated GitHub CLI access. + +Preflight selects a merged commit and validates the release configuration. +Publication waits for tagged CI, verifies assets, and deploys stable documentation. +Matching attempts can resume after interruption; completed remote operations are +preserved. Python and the standard library implement the orchestration; +``release_tools`` supplies shared validation and packaging contracts. +""" + +import argparse +from fnmatch import fnmatchcase +import json +import os +from pathlib import Path +import re +import shlex +import subprocess +import sys +import tempfile +import time +from typing import Any +from urllib.parse import urlencode +from urllib.error import HTTPError +from urllib.request import Request, urlopen + +from release_tools import (Git, ReleaseError, WORKFLOW, check_bundle, check_documentation, check_run, command, + parse_version, sha256) + +_PYTHON_MINIMUM = (3, 14) + +def check_local_tools(python_version=None): + """Require the supported Python version and Git/GitHub CLI operations. + + Args: + python_version (tuple[int, ...] | None): Version tuple to check; None uses + sys.version_info. Supplying a tuple supports isolated preflight tests. + + Raises: + ReleaseError: Python is older than the minimum supported version, an executable + is missing, or a required GitHub CLI help command is unavailable. + + Executes local version/help commands; repository access is checked separately. + """ + if (python_version or sys.version_info) < _PYTHON_MINIMUM: + raise ReleaseError( + f"Run the release script with Python {_PYTHON_MINIMUM[0]}.{_PYTHON_MINIMUM[1]} or newer." + ) + command(["git", "--version"]) + command(["gh", "--version"]) + for operation in (("run", "download"), ("release", "upload"), ("workflow", "run")): + try: + command(["gh", *operation, "--help"]) + except ReleaseError as ex: + raise ReleaseError("Update GitHub CLI from https://cli.github.com/ and retry; " + f"gh {' '.join(operation)} is unavailable.") from ex + + +def release_command(args, root, version, commit, resume=False): + """Format a shell command pinned to a previously checked release revision. + + Args: + args (argparse.Namespace): Parsed options containing repo, remote, + timeout, poll, and the optional notes_file Path. + root (str | Path): Checkout directory for the printed command. + version (str): Validated release version without the v prefix. + commit (str): Full commit object ID, retained even if main advances. + resume (bool): Include the option for continuing an existing attempt. + + Returns: + str: Command using this interpreter/script, platform-appropriate quoting, + and an absolute notes path when notes_file is supplied. + """ + parts = [sys.executable, str(Path(__file__).resolve()), "--root", str(root), "--repo", args.repo, + "--remote", args.remote, "--version", version, "--ref", commit, + "--timeout", str(args.timeout), "--poll", str(args.poll)] + if args.notes_file: + parts.extend(["--notes-file", str(args.notes_file.resolve())]) + if resume: + parts.append("--resume") + return subprocess.list2cmdline(parts) if os.name == "nt" else shlex.join(parts) + + +class GitHub: + """Access a github.com repository through the authenticated ``gh`` executable. + + Args: + repository (str): OWNER/NAME repository identity. + + Attributes: + repository (str): Identity used for every API and CLI request. + + API records retain GitHub's JSON field names. Methods propagate ReleaseError + for command failures and ValueError for malformed JSON responses. Read + methods query remote state; ``cli``, ``api``, and ``create_draft`` can mutate + it. Authentication comes from GitHub CLI's configuration or environment. + """ + + def __init__(self, repository): + """Store the repository identity without issuing a network request.""" + self.repository = repository + + def cli(self, *args): + """Execute a repository-scoped GitHub CLI command on github.com. + + Args: + *args (str): Subcommand and arguments; the repository option is added. + + Returns: + str: Decoded, stripped stdout from the command. + + Raises: + ReleaseError: The command is unavailable, fails, or times out. + + The supplied subcommand controls downloads, uploads, or other mutations. + """ + try: + return command(["gh", *args, "--repo", "github.com/" + self.repository]) + except ReleaseError as ex: + raise ReleaseError(f"gh {' '.join(args[:2])} failed. Run gh auth status --hostname github.com; " + "release operations need Contents read/write and Actions read/write access. " + "Then retry the printed resume command.") from ex + + def api(self, path, method="GET", **fields): + """Issue a repository-relative REST request and decode its JSON response. + + Args: + path (str): Path beneath repos/OWNER/NAME, including any query string; + an empty string requests repository metadata. + method (str): HTTP method; defaults to a read-only GET. + **fields: Request fields. Booleans are typed JSON values; other + supplied values are passed as strings to GitHub CLI. + + Returns: + dict | list | None: Parsed response, or None for an empty response. + + Raises: + ReleaseError: GitHub CLI cannot complete the request. + ValueError: A nonempty response is invalid JSON. + + Write methods may change remote state before a transport failure occurs. + """ + args = ["gh", "api", "--hostname", "github.com", "--method", method, + f"repos/{self.repository}/{path}".rstrip("/")] + for key, value in fields.items(): + args.extend(["-F" if isinstance(value, bool) else "-f", + f"{key}={str(value).lower() if isinstance(value, bool) else value}"]) + try: + result = command(args) + except ReleaseError as ex: + raise ReleaseError(f"GitHub {method} {path or 'repository access'} failed. " + "Run gh auth login --hostname github.com or fix GH_TOKEN/SSO access. " + "The credential needs Contents read/write and Actions read/write " + "(plus Pages read for stable documentation).") from ex + return json.loads(result) if result else None + + def _api_object(self, path: str, method: str = "GET", **fields) -> dict[str, Any]: + """Require an object response from a metadata endpoint, or raise ReleaseError.""" + result = self.api(path, method, **fields) + if not isinstance(result, dict): + raise ReleaseError(f"GitHub {method} {path or 'repository metadata'} returned no JSON object.") + return result + + def preflight(self, stable): + """Check workflow availability and, when needed, Pages deployment setup. + + Args: + stable (bool): Require Actions-based Pages and an environment policy + permitting main when True; otherwise check only CETLVaSt. + + Raises: + ReleaseError: Configuration is inaccessible, CETLVaSt is disabled, + Pages uses another build source, or main cannot deploy. + + Performs reads and reports setup fixes without changing repository policy. + """ + workflow = self._api_object(f"actions/workflows/{WORKFLOW}") + if workflow.get("state") != "active": + raise ReleaseError(f"Enable CETLVaSt: gh workflow enable {WORKFLOW} --repo {self.repository}") + if not stable: + return + try: + site = self._api_object("pages") + environment = self._api_object("environments/github-pages") + except ReleaseError as ex: + raise ReleaseError(f"Cannot read Pages setup. Check credential Pages/Actions read access and " + f"https://github.com/{self.repository}/settings/pages and " + f"https://github.com/{self.repository}/settings/environments.") from ex + if site.get("build_type") != "workflow": + raise ReleaseError(f"Select GitHub Actions as the Pages build source at " + f"https://github.com/{self.repository}/settings/pages.") + policy = environment.get("deployment_branch_policy") or {} + if policy.get("protected_branches"): + allowed = self._api_object("branches/main").get("protected", False) + elif policy.get("custom_branch_policies"): + policies = self.pages("environments/github-pages/deployment-branch-policies", "branch_policies") + allowed = any(item.get("type", "branch") == "branch" and fnmatchcase("main", item["name"]) + for item in policies) + else: + allowed = True + if not allowed: + raise ReleaseError(f"Allow main to deploy to github-pages at " + f"https://github.com/{self.repository}/settings/environments.") + + def verify_pages(self, version, timeout, poll): + """Wait for the public Pages HTML to display the expected release version. + + Args: + version (str): Expected Doxygen project number, without the tag's v. + timeout (int | float): Polling budget in seconds. + poll (int | float): Seconds to sleep between unsuccessful requests. + + Raises: + ReleaseError: Pages metadata cannot be read, or network/content + verification remains unsuccessful when the budget expires. + + Reads at most two MiB per HTML response with a 15-second request timeout. + Public HTML requests carry no GitHub credentials. Progress and successful + verification are printed; the last failure supplies recovery guidance. + """ + url = self._api_object("pages")["html_url"] + print(f"Checking published documentation: {url} ({version})") + deadline = time.monotonic() + timeout + last_error = "No documentation response received." + while time.monotonic() < deadline: + try: + with urlopen(Request(url, headers={"Cache-Control": "no-cache"}), timeout=15) as response: + check_documentation(response.read(2 * 1024 * 1024).decode("utf-8"), version) + print(f"Published documentation verified: {url} ({version})") + return + except ReleaseError as ex: + last_error = str(ex) + except HTTPError as ex: + last_error = f"Documentation request returned HTTP {ex.code}." + except (OSError, ValueError) as ex: + last_error = f"Could not read documentation ({type(ex).__name__}); check network/TLS access." + time.sleep(poll) + raise ReleaseError(f"Release is published, but documentation verification failed at {url}. {last_error} " + "Fix access or allow Pages/CDN propagation, then retry the printed resume command.") + + def pages(self, path, key=None): + """Collect a REST collection across pages of up to 100 records. + + Args: + path (str): Repository-relative collection path, optionally with a + query string, without pagination parameters. + key (str | None): Collection field in an object response; None means + that each response is itself a list. + + Returns: + list: Records in API page order, including an empty collection. + + Raises: + ReleaseError: A request fails or 1,000 full pages are exhausted. + """ + items = [] + for page in range(1, 1001): + separator = "&" if "?" in path else "?" + result = self.api(f"{path}{separator}per_page=100&page={page}") + if key: + if not isinstance(result, dict): + raise ReleaseError(f"GitHub collection {path} returned no JSON object.") + batch = result.get(key) + else: + batch = result + if not isinstance(batch, list): + raise ReleaseError(f"GitHub collection {path} returned no JSON array.") + items.extend(batch) + if len(batch) < 100: + return items + raise ReleaseError("GitHub pagination limit exceeded.") + + def release(self, tag): + """Find a release, including visible drafts, by exact tag name. + + Args: + tag (str): v-prefixed tag identifying the release. + + Returns: + dict | None: GitHub release record, or None after a successful + complete listing finds no matching tag. + + Raises: + ReleaseError: Listing fails; permission errors are not treated as + evidence that the release is absent. + """ + return next((release for release in self.pages("releases") if release["tag_name"] == tag), None) + + def published_stable_versions(self): + """Collect numeric versions from published, stable, vX.Y.Z releases. + + Returns: + set[tuple[int, int, int]]: Version cores suitable for numeric ordering. + Drafts, prereleases, and nonconforming historical tags are ignored. + + Raises: + ReleaseError: The release listing cannot be completed. + """ + versions = set() + for release in self.pages("releases"): + if not release["draft"] and not release["prerelease"]: + match = re.fullmatch(r"v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)", release["tag_name"]) + if match: + versions.add(tuple(map(int, match.groups()))) + return versions + + def runs(self, **filters): + """List CETLVaSt workflow runs matching GitHub's REST filter fields. + + Args: + **filters: Run-list parameters such as head_sha, event, and branch; + values are URL-encoded before requesting every result page. + + Returns: + list[dict]: GitHub Actions run records, in API order. + + Raises: + ReleaseError: The workflow run listing cannot be completed. + """ + return self.pages(f"actions/workflows/{WORKFLOW}/runs?{urlencode(filters)}", "workflow_runs") + + def jobs(self, run): + """List jobs belonging to exactly the attempt identified by a run record. + + Args: + run (dict): Actions record containing id and run_attempt. + + Returns: + list[dict]: Attempt-specific job records with names and conclusions. + + Raises: + ReleaseError: The job listing cannot be completed. + + Successful jobs from earlier attempts cannot satisfy a later attempt. + """ + return self.pages(f"actions/runs/{run['id']}/attempts/{run['run_attempt']}/jobs", "jobs") + + def assets(self, release): + """List all assets attached to a release. + + Args: + release (dict): GitHub release record containing id. + + Returns: + list[dict]: Asset records, including upload state, size, and any digest. + + Raises: + ReleaseError: The asset listing cannot be completed. + """ + return self.pages(f"releases/{release['id']}/assets") + + def create_draft(self, tag, commit, prerelease, marker, run, notes=None): + """Create a draft with source identity and successful verification evidence. + + Args: + tag (str): Existing v-prefixed remote tag; also the release title. + commit (str): Full source commit object ID. + prerelease (bool): Classification already validated against the tag. + marker (str): Script-management marker used to identify matching retries. + run (dict): Verified Actions run containing html_url. + notes (str | None): Reviewed body text, or None to generate GitHub notes. + + Raises: + ReleaseError: Note generation or release creation fails. + + Writes a remote draft. Callers must check for an existing release first + and re-read remote state after an interrupted creation request. + """ + if notes is None: + notes = self._api_object("releases/generate-notes", "POST", tag_name=tag, target_commitish=commit)["body"] + body = f"{notes}\n\nSource commit: `{commit}`\n\nVerification: {run['html_url']}\n\n{marker}\n" + self.api("releases", "POST", tag_name=tag, target_commitish=commit, name=tag, + draft=True, prerelease=prerelease, body=body) + + def asset_matches(self, asset, path): + """Compare a remote asset's uploaded bytes with a local file. + + Args: + asset (dict): Asset record with state and size, optional digest, and + release_tag for the download fallback. + path (Path): Local file whose name identifies the remote asset. + + Returns: + bool: True when upload state, size, and SHA-256 match. Assets lacking + a server digest are downloaded into a temporary directory. + + Raises: + ReleaseError: A fallback download fails. + OSError: The local or downloaded file cannot be read. + """ + if asset.get("state") != "uploaded" or asset.get("size") != path.stat().st_size: + return False + if asset.get("digest"): + return asset["digest"] == "sha256:" + sha256(path) + # Older GitHub assets may not have a server-side digest. Compare the actual bytes. + with tempfile.TemporaryDirectory(prefix="cetl-asset-") as directory: + self.cli("release", "download", asset["release_tag"], "--pattern", path.name, "--dir", directory) + return sha256(Path(directory) / path.name) == sha256(path) + + +def wait_for_release_run(github, commit, tag, timeout, poll): + """Wait for the newest tagged push/dispatch run and validate its full gate. + + Args: + github (GitHub): Repository client used to list runs and attempt jobs. + commit (str): Full release commit object ID. + tag (str): Exact v-prefixed release tag. + timeout (int | float): Polling budget in seconds. + poll (int | float): Seconds between polls while a run is absent/incomplete. + + Returns: + dict: Completed successful run accepted by ``check_run``. + + Raises: + ReleaseError: API access fails, a completed run fails validation, or the + budget expires. A failed gate includes a full-run retry command. + """ + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + candidates = [run for run in github.runs(head_sha=commit) + if run["head_branch"] == tag and run["event"] in ("push", "workflow_dispatch")] + if candidates: + run = max(candidates, key=lambda item: item["id"]) + if run["status"] == "completed": + try: + check_run(run, github.jobs(run), commit, tag) + except ReleaseError as ex: + raise ReleaseError(f"{ex} See {run['html_url']}. For a transient failure run " + f"gh run rerun {run['id']} --repo {github.repository} " + "(the entire run), then use the printed resume command.") from ex + print(f"Verification: {run['html_url']}") + return run + time.sleep(poll) + raise ReleaseError("Timed out waiting for release CI; leave the tag/draft intact and retry with --resume.") + + +def check_draft(release, tag, prerelease, marker): + """Require a release to belong to the same script-managed attempt. + + Args: + release (dict): Draft or published release with tag_name, prerelease, + and body fields. Publication state is checked by the caller. + tag (str): Expected exact tag name. + prerelease (bool): Expected release classification. + marker (str): Management marker that must appear in the release body. + + Raises: + ReleaseError: The tag, classification, or management marker disagrees. + """ + if (release.get("tag_name") != tag or release.get("prerelease") != prerelease + or marker not in (release.get("body") or "")): + raise ReleaseError("Existing release is not a matching script-managed release; refusing to change it.") + + +def version_core(version): + """Extract an orderable numeric core from a supported version string. + + Args: + version (str): Stable or prerelease version without a v prefix. + + Returns: + tuple[int, int, int]: Major, minor, and patch as integers. + + Raises: + ReleaseError: The version fails the shared grammar. + """ + return tuple(map(int, parse_version(version).groups()[:3])) + + +def require_new_version(github, version): + """Require a candidate's numeric core to exceed every published stable core. + + Args: + github (GitHub): Repository client used to list published stable versions. + version (str): Candidate stable or prerelease version without v. + + Raises: + ReleaseError: A published stable core is greater than or equal to the + candidate's core, or the remote listing cannot be completed. + """ + if any(prior >= version_core(version) for prior in github.published_stable_versions()): + raise ReleaseError("A new release must advance the numeric version beyond published stable releases.") + + +def require_latest_stable(github, tag): + """Require a tag to identify the newest published stable release. + + Args: + github (GitHub): Repository client providing release and version records. + tag (str): Validated v-prefixed version being considered for deployment. + + Raises: + ReleaseError: The release is absent, draft, prerelease, older than the + newest stable core, or remote state cannot be read. + """ + release = github.release(tag) + versions = github.published_stable_versions() + if (not release or release["draft"] or release["prerelease"] or not versions + or version_core(tag[1:]) != max(versions)): + raise ReleaseError("Only the latest published stable release may replace the documentation site.") + + +def sync_assets(github, release, tag, directory, names, upload): + """Verify existing assets and optionally upload missing expected files. + + Args: + github (GitHub): Client used to list, compare, and upload assets. + release (dict): Target release record containing id. + tag (str): Exact release tag used for downloads and uploads. + directory (Path): Local directory containing the verified bundle files. + names (Collection[str]): Complete collection of expected asset filenames. + upload (bool): Permit missing assets to be uploaded when True; otherwise + require the release to contain the complete bundle already. + + Raises: + ReleaseError: Names are duplicated/unexpected, existing bytes differ, + required assets are missing, or an upload/comparison fails. + OSError: A file needed for comparison cannot be read. + + Existing files are checked before uploads and never overwritten. Interrupted + uploads may leave a partial remote set for a matching attempt to resume. + """ + existing = github.assets(release) + if len({asset["name"] for asset in existing}) != len(existing): + raise ReleaseError("Release contains duplicate asset names.") + if {asset["name"] for asset in existing} - set(names): + raise ReleaseError("Release contains unexpected assets; refusing to overwrite it.") + for asset in existing: + if not github.asset_matches({**asset, "release_tag": tag}, directory / asset["name"]): + raise ReleaseError(f"Existing asset differs: {asset['name']}; no assets were overwritten.") + missing = set(names) - {asset["name"] for asset in existing} + if missing and not upload: + raise ReleaseError("Release is missing required assets.") + for name in sorted(missing): + # No --clobber: concurrent attempts must fail safely rather than replace bytes. + github.cli("release", "upload", tag, str(directory / name)) + + +def verify_published_assets(github, release, commit, tag): + """Verify a published release directly, independent of CI artifact retention. + + Args: + github (GitHub): Client used for asset downloads and remote comparisons. + release (dict): Published release record containing id. + commit (str): Expected full source commit object ID. + tag (str): Expected validated v-prefixed release version. + + Raises: + ReleaseError: Download, bundle validation, or remote comparison fails. + OSError: Temporary assets cannot be read or written. + ValueError: The downloaded manifest is invalid JSON. + + Downloads into a temporary directory, prints successful verification, and + leaves the published release unchanged. + """ + with tempfile.TemporaryDirectory(prefix="cetl-published-") as temporary: + directory = Path(temporary) + github.cli("release", "download", tag, "--dir", str(directory)) + names = check_bundle(directory, commit, tag) + sync_assets(github, release, tag, directory, names, upload=False) + print("Published release assets verified.") + + +def deploy_docs(github, tag, timeout, poll): + """Complete a main-branch documentation deployment for a published stable tag. + + Args: + github (GitHub): Client used to find/dispatch CETLVaSt and inspect jobs. + tag (str): Published stable tag, checked by the caller before deployment. + timeout (int | float): Polling budget in seconds. + poll (int | float): Seconds between deployment-status polls. + + Raises: + ReleaseError: API/dispatch access fails, the newest matching run fails, + its deployment job is unsuccessful, or the budget expires. + + Reuses a successful or in-flight matching dispatch. Otherwise starts a run + on main with the release tag as docs_ref. Prints environment-approval links + when required; site-content verification belongs to ``finish_documentation``. + """ + title = f"CETLVaSt / {tag}" + runs = [run for run in github.runs(event="workflow_dispatch", branch="main") + if run["display_title"] == title] + # A completed successful deployment makes --resume a no-op. + latest = max(runs, key=lambda run: run["id"]) if runs else None + if latest is not None: + if latest["status"] == "completed" and latest["conclusion"] == "success": + jobs = github.jobs(latest) + if any(job["name"] == "deploy-docs" and job["conclusion"] == "success" for job in jobs): + print(f"Documentation: {latest['html_url']}") + return + if latest is not None and latest["status"] != "completed": + previous = latest["id"] - 1 # Resume an in-flight deployment instead of queueing another full build. + else: + previous = max((run["id"] for run in runs), default=0) + github.cli("workflow", "run", WORKFLOW, "--ref", "main", "-f", f"docs_ref={tag}") + deadline = time.monotonic() + timeout + approval_reported = set() + while time.monotonic() < deadline: + runs = [run for run in github.runs(event="workflow_dispatch", branch="main") + if run["display_title"] == title and run["id"] > previous] + if runs: + run = max(runs, key=lambda item: item["id"]) + if run["status"] == "waiting" and run["id"] not in approval_reported: + print(f"GitHub environment approval required: {run['html_url']}") + approval_reported.add(run["id"]) + if run["status"] == "completed": + jobs = github.jobs(run) + if run["conclusion"] != "success" or not any( + job["name"] == "deploy-docs" and job["conclusion"] == "success" for job in jobs + ): + raise ReleaseError(f"Release is published, but documentation failed: {run['html_url']}; use --resume.") + print(f"Documentation: {run['html_url']}") + return + time.sleep(poll) + raise ReleaseError("Release is published, but documentation timed out; use --resume to recover.") + + +def cut_release(args, git=None, github=None): + """Preflight, publish, or resume a CETL release and its stable documentation. + + Args: + args (argparse.Namespace): Options produced by ``main``: root (Path), + repo and remote (str), version and ref (str or None), notes_file + (Path or None), dry_run and resume (bool), and positive timeout and + poll durations in seconds. Omitted version/ref select the freshly + fetched main commit and its header version; resume requires both. + git (Git | None): Checkout adapter; None constructs Git(args.root). + github (GitHub | None): Repository adapter; None constructs GitHub(args.repo). + Tests may inject adapters implementing the same operations. + + Raises: + ReleaseError: Preflight, version/ref policy, CI, asset, publication, or + deployment checks fail. Matching remote state is retained for retry. + OSError: Notes, temporary bundle files, or subprocesses are inaccessible. + ValueError: Remote or bundle JSON cannot be decoded. + KeyError: A service record lacks a required field. + + Fetches remote main and prints commands pinned to the selected revision. + Dry runs stop before remote writes. Publication may push a tag, dispatch CI, + create/upload/publish a release, and deploy Pages. Resuming a published + release verifies its actual assets before finishing documentation. + """ + check_local_tools() + if args.resume and (not args.version or not args.ref): + raise ReleaseError("Use the printed Resume command: --resume requires explicit --version and --ref " + "so an advancing main branch cannot change the release being resumed.") + if args.version: + parse_version(args.version) + if not re.fullmatch(r"[\w.-]+/[\w.-]+", args.repo) or args.remote.startswith("-"): + raise ReleaseError("Expected --repo OWNER/NAME and a named Git remote.") + git = git or Git(args.root) + github = github or GitHub(args.repo) + if git.run("status", "--porcelain"): + raise ReleaseError("The checkout must be clean, including untracked files. " + "Run git status --short and commit or stash the reported changes.") + if git.remote_repository(args.remote).lower() != args.repo.lower(): + raise ReleaseError("Git remote does not match --repo.") + repository = github._api_object("") + if repository.get("archived") or not repository.get("permissions", {}).get("push"): + raise ReleaseError("GitHub credentials require repository write access. " + "Run gh auth login --hostname github.com with a maintainer account.") + if repository["default_branch"] != "main": + raise ReleaseError("The supported release branch is main; review the policy before changing it.") + git.run("fetch", "--no-tags", args.remote, "refs/heads/main:refs/remotes/" + args.remote + "/main") + main_commit = git.commit(f"refs/remotes/{args.remote}/main") + commit = git.commit(args.ref) if args.ref else main_commit + try: + git.run("merge-base", "--is-ancestor", commit, f"refs/remotes/{args.remote}/main") + except ReleaseError as ex: + raise ReleaseError("Release commit must already be merged into the remote main branch.") from ex + for target in {main_commit, commit}: + git.require_release_files(target) + version = args.version or git.validate(commit)["version"] + tag = "v" + version + metadata = git.validate(commit, tag) + print(f"Repository: {args.repo}\nTag: {tag}\nCommit: {commit}\nPrerelease: {metadata['prerelease']}") + print("Resume command: " + release_command(args, git.root, version, commit, resume=True)) + remote_commit = git.remote_tag(args.remote, tag) + if remote_commit and remote_commit != commit: + raise ReleaseError(f"Remote tag {tag} points to another commit; refusing to move it.") + local_tags = git.run("tag", "--list", tag) + if local_tags and git.commit(f"refs/tags/{tag}") != commit: + raise ReleaseError(f"Local tag {tag} points to another commit.") + release = github.release(tag) + notes = args.notes_file.read_text(encoding="utf-8") if args.notes_file and not release else None + runs = github.runs(head_sha=commit) # Check Actions read access before pushing a tag. + marker = f"" + if release: + if not release["draft"] and marker not in (release.get("body") or "") and not args.resume: + raise ReleaseError(f"Version {version} is already published. Merge a reviewed version bump into main " + "before cutting a new release.") + check_draft(release, tag, metadata["prerelease"], marker) + if not remote_commit: + raise ReleaseError("A release exists without its remote tag; manual investigation is required.") + if not release or release["draft"]: + require_new_version(github, version) + deploy_stable = not metadata["prerelease"] and not any( + prior > version_core(version) for prior in github.published_stable_versions()) + github.preflight(deploy_stable) + if (release or remote_commit) and not args.resume and not args.dry_run: + raise ReleaseError("Release/tag already exists; inspect it and use --resume for a matching attempt.") + if not remote_commit: + git.run("push", "--dry-run", args.remote, f"{commit}:refs/tags/{tag}") + if args.dry_run: + print("Dry run: push immutable tag; await full CETLVaSt; download and verify bundle; create/resume draft; " + "upload matching assets; publish; deploy stable documentation. No remote writes performed.") + print("Release this checked commit: " + release_command( + args, git.root, version, commit, resume=bool(release or remote_commit or args.resume))) + return + if release and not release["draft"]: + verify_published_assets(github, release, commit, tag) + print(f"Release: https://github.com/{args.repo}/releases/tag/{tag}") + finish_documentation(github, tag, version, metadata["prerelease"], args.timeout, args.poll) + return + if not remote_commit: + git.run("push", args.remote, f"{commit}:refs/tags/{tag}") + elif not any(run["head_branch"] == tag and run["event"] in ("push", "workflow_dispatch") + for run in runs): + github.cli("workflow", "run", WORKFLOW, "--ref", tag) + run = wait_for_release_run(github, commit, tag, args.timeout, args.poll) + with tempfile.TemporaryDirectory(prefix="cetl-release-") as temporary: + directory = Path(temporary) + github.cli("run", "download", str(run["id"]), "--name", "release-bundle", "--dir", str(directory)) + names = check_bundle(directory, commit, tag) + if git.remote_tag(args.remote, tag) != commit: + raise ReleaseError("Remote tag changed while waiting for CI.") + # Refresh after the wait: a concurrent attempt may have created or published the draft. + release = github.release(tag) + if not release: + github.create_draft(tag, commit, metadata["prerelease"], marker, run, notes) + release = github.release(tag) + if release is None: + raise ReleaseError("Release is missing after draft creation; retry the printed resume command.") + check_draft(release, tag, metadata["prerelease"], marker) + sync_assets(github, release, tag, directory, names, upload=release["draft"]) + release = github.release(tag) + if release is None: + raise ReleaseError("Release disappeared after asset verification; retry the printed resume command.") + check_draft(release, tag, metadata["prerelease"], marker) + sync_assets(github, release, tag, directory, names, upload=False) + if release["draft"]: + require_new_version(github, version) + if git.remote_tag(args.remote, tag) != commit: + raise ReleaseError("Remote tag changed before publication.") + # Never edit a published release, including on a retry. + github.api(f"releases/{release['id']}", "PATCH", draft=False, make_latest="false" if metadata["prerelease"] else "true") + release = github.release(tag) + if release is None: + raise ReleaseError("Release is missing after publication; retry the printed resume command.") + check_draft(release, tag, metadata["prerelease"], marker) + if release["draft"]: + raise ReleaseError("GitHub still reports a draft; retry the printed resume command.") + sync_assets(github, release, tag, directory, names, upload=False) + print(f"Release: https://github.com/{args.repo}/releases/tag/{tag}") + finish_documentation(github, tag, version, metadata["prerelease"], args.timeout, args.poll) + + +def finish_documentation(github, tag, version, prerelease, timeout, poll): + """Deploy and verify documentation when a release is the latest stable one. + + Args: + github (GitHub): Repository and Pages client. + tag (str): Published, validated release tag. + version (str): Corresponding version without v. + prerelease (bool): Classification validated against the tag. + timeout (int | float): Separate polling budget for deployment and HTML checks. + poll (int | float): Seconds between status/content checks. + + Raises: + ReleaseError: The latest-stable check, deployment, or public HTML + verification fails, or GitHub state cannot be read. + + Prints a skip reason for prereleases and releases superseded by a newer + stable version. Eligible releases may dispatch a deployment through main. + """ + if prerelease: + print("Documentation: built and attached; prereleases do not replace the stable Pages site.") + else: + if any(prior > version_core(version) for prior in github.published_stable_versions()): + print("Documentation: a newer stable release is published; its site is retained.") + else: + require_latest_stable(github, tag) + deploy_docs(github, tag, timeout, poll) + github.verify_pages(version, timeout, poll) + + +def main(): + """Parse maintainer CLI arguments and run the release operation. + + Reads sys.argv and prints progress or recovery diagnostics. ``cut_release`` + defines filesystem/network effects; argparse's help defines CLI options. + + Raises: + SystemExit: Help/argument parsing exits, a release error exits with + status 1, or keyboard interruption exits with status 130. + """ + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--version", help="Override header version, e.g. X.Y.Z-rc.1 (without v)") + parser.add_argument("--ref", help="Commit/ref merged into remote main; defaults to freshly fetched main") + parser.add_argument("--repo", default="OpenCyphal/CETL") + parser.add_argument("--remote", default="origin") + parser.add_argument("--root", type=Path, default=Path.cwd()) + parser.add_argument("--notes-file", type=Path, help="Reviewed release notes; otherwise generate notes from GitHub") + parser.add_argument("--dry-run", action="store_true", help="Perform preflight reads, without remote writes") + parser.add_argument("--resume", action="store_true", help="Resume only an identical tag/script-managed release") + parser.add_argument("--timeout", type=int, default=7200, help="Seconds to wait for each CI/deployment phase") + parser.add_argument("--poll", type=int, default=15, help="Seconds between GitHub status reads") + args = parser.parse_args() + if args.timeout < 1 or args.poll < 1: + parser.error("--timeout and --poll must be positive") + try: + cut_release(args) + except KeyboardInterrupt: + parser.exit(130, "Release interrupted. Use the printed Resume command to continue.\n") + except (ReleaseError, OSError, ValueError, KeyError) as ex: + parser.exit(1, f"Release stopped: {ex}\nFix the reported problem and retry; existing tags/assets are preserved.\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/release_tools.py b/scripts/release_tools.py new file mode 100644 index 00000000..7d5cb045 --- /dev/null +++ b/scripts/release_tools.py @@ -0,0 +1,570 @@ +#!/usr/bin/env python3 +"""Validate CETL releases and build their distribution archives. + +The maintainer CLI and CI share the version, verification-job, documentation, +and bundle contracts defined here. Git reads use committed content; packaging +writes local archives, a release manifest, and checksums. The module uses the +Python standard library and the Git executable. ``main`` provides the CI +entry point and writes validated metadata to GitHub Actions outputs. +""" + +import argparse +from collections.abc import Iterable, Mapping, Sequence +import gzip +import hashlib +from html.parser import HTMLParser +import io +import json +import os +from pathlib import Path +import re +import subprocess +import tarfile +from typing import Any, Literal, overload + + +class ReleaseError(RuntimeError): + """Report a failed release check or operation to the maintainer. + + The message describes the failed condition or corrective action. CLI entry + points catch this exception and exit unsuccessfully. Operations completed + before the exception, including remote writes, may need to be resumed. + """ + + +VERSION = re.compile(r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(?:-(alpha|beta|rc)\.([1-9][0-9]*))?") +HEADER = "include/cetl/cetl.hpp" +WORKFLOW = "cetlvast.yml" +RELEASE_FILES = ("scripts/release.py", "scripts/release_tools.py", "scripts/tests/test_release.py", + f".github/workflows/{WORKFLOW}") + + +def parse_version(value): + """Parse a supported stable or prerelease version without a tag prefix. + + Args: + value (str): X.Y.Z or X.Y.Z-{alpha,beta,rc}.N, with decimal components, + no leading zeros, and a positive prerelease sequence number. + + Returns: + re.Match: Five groups: major, minor, patch, stage, and sequence number. + The final two groups are None for stable versions. + + Raises: + ReleaseError: The entire string does not match the version grammar. + """ + match = VERSION.fullmatch(value) + if not match: + raise ReleaseError("Expected X.Y.Z or X.Y.Z-{alpha,beta,rc}.N (no leading zeros; N >= 1).") + return match + + +def header_version(text): + """Read the numeric version from CETL's public header. + + Args: + text (str): Header source containing CETL_VERSION_MAJOR, MINOR, and PATCH. + + Returns: + str: The normalized X.Y.Z version. + + Raises: + ReleaseError: A macro is missing, repeated, or not a bare nonnegative + decimal integer without leading zeros. + """ + parts = [] + for part in ("MAJOR", "MINOR", "PATCH"): + definitions = re.findall(r"^\s*#\s*define[ \t]+CETL_VERSION_" + part + r"\b([^\n]*)", text, re.M) + if len(definitions) != 1 or not re.fullmatch(r"[ \t]+(0|[1-9][0-9]*)[ \t]*", definitions[0]): + raise ReleaseError(f"{HEADER}: CETL_VERSION_{part} must be defined exactly once as a decimal integer.") + parts.append(definitions[0].strip()) + return ".".join(parts) + + +def validate(text, tag=None, prerelease=None): + """Check agreement between header macros, a tag, and release classification. + + Args: + text (str): CETL header source to inspect. + tag (str | None): A v-prefixed version; None uses the header's version. + prerelease (bool | None): Expected GitHub prerelease flag, or None to + derive the classification from the version suffix. + + Returns: + dict: ``version`` without v, ``tag`` (empty when omitted), and the + boolean ``prerelease`` classification. + + Raises: + ReleaseError: Header/version syntax is invalid, the tag's numeric core + differs from the header, or the supplied classification disagrees. + """ + core = header_version(text) + version = tag[1:] if tag and tag.startswith("v") else core + if tag and not tag.startswith("v"): + raise ReleaseError("Release tags must start with v.") + match = parse_version(version) + if ".".join(match.groups()[:3]) != core: + raise ReleaseError(f"Tag {tag} does not match {HEADER} ({core}).") + is_prerelease = match.group(4) is not None + if prerelease is not None and prerelease != is_prerelease: + raise ReleaseError("GitHub's prerelease flag does not match the tag suffix.") + return {"version": version, "tag": tag or "", "prerelease": is_prerelease} + + +@overload +def command(args: Sequence[str], cwd: str | Path | None = None, binary: Literal[False] = False) -> str: ... + + +@overload +def command(args: Sequence[str], cwd: str | Path | None = None, *, binary: Literal[True]) -> bytes: ... + + +@overload +def command(args: Sequence[str], cwd: str | Path | None = None, binary: bool = False) -> str | bytes: ... + + +def command(args: Sequence[str], cwd: str | Path | None = None, binary: bool = False) -> str | bytes: + """Run an argument vector with captured output and a 120-second timeout. + + Args: + args (Sequence[str]): Executable and arguments, passed without a shell. + cwd (str | Path | None): Working directory; None inherits the caller's. + binary (bool): Return stdout bytes instead of decoded, stripped text. + + Returns: + str | bytes: Captured stdout in the requested representation. + + Raises: + ReleaseError: The executable is missing, times out, or exits nonzero. + OSError: Another process-launch error occurs. + UnicodeDecodeError: Text stdout cannot be decoded as UTF-8. + + Subprocess diagnostics omit arguments and stderr, which may contain + credentials. The invoked command determines filesystem and network effects. + """ + try: + result = subprocess.run(args, cwd=cwd, capture_output=True, check=False, timeout=120) + except FileNotFoundError as ex: + hint = {"git": "Install Git and rerun the release command.", + "gh": "Install GitHub CLI from https://cli.github.com/ and run gh auth login --hostname github.com."} + raise ReleaseError(f"Required executable is missing: {args[0]}. " + hint.get(args[0], "Install it and retry.")) from ex + except subprocess.TimeoutExpired as ex: + raise ReleaseError(f"{args[0]} timed out; inspect remote state before retrying.") from ex + if result.returncode: + raise ReleaseError(f"{args[0]} failed (exit {result.returncode}); check authentication, permissions, refs, and connectivity.") + return result.stdout if binary else result.stdout.decode().strip() + + +class Git: + """Run Git operations against one checkout using captured subprocess output. + + Args: + root (str | Path): Checkout directory, resolved to an absolute path. + + Attributes: + root (Path): Working directory used by every Git subprocess. + + Construction stores the path; individual operations check repository state. + Methods propagate ReleaseError from failed Git commands. + """ + + def __init__(self, root): + """Store the resolved checkout path without opening or changing it.""" + self.root = Path(root).resolve() + + @overload + def run(self, *args: str, binary: Literal[False] = False) -> str: ... + + @overload + def run(self, *args: str, binary: Literal[True]) -> bytes: ... + + @overload + def run(self, *args: str, binary: bool = False) -> str | bytes: ... + + def run(self, *args: str, binary: bool = False) -> str | bytes: + """Execute Git in this checkout with the output contract of ``command``. + + Args: + *args (str): Git subcommand and arguments, passed without a shell. + binary (bool): Return raw stdout bytes when True. + + Returns: + str | bytes: Captured stdout, stripped when returned as text. + + Raises: + ReleaseError: Git is unavailable, times out, or exits nonzero. + + Mutations and network access depend on the requested subcommand. + """ + return command(["git", *args], cwd=self.root, binary=binary) + + def commit(self, ref): + """Resolve a revision to its full commit object ID. + + Args: + ref (str): Git revision expression; annotated tags are peeled. + + Returns: + str: The full commit object ID. + + Raises: + ReleaseError: The revision is missing or cannot resolve to a commit. + """ + # --end-of-options prevents a ref supplied at the CLI from becoming a Git option. + return self.run("rev-parse", "--verify", "--end-of-options", ref + "^{commit}") + + def validate(self, commit, tag=None, prerelease=None): + """Validate a committed header using ``validate``'s metadata contract. + + Args: + commit (str): Commit or revision containing the header. + tag (str | None): Expected v-prefixed release version. + prerelease (bool | None): Expected GitHub prerelease classification. + + Returns: + dict: Validated ``version``, ``tag``, and ``prerelease`` fields. + + Raises: + ReleaseError: Git cannot read the header or its metadata disagrees. + """ + return validate(self.run("show", f"{commit}:{HEADER}"), tag, prerelease) + + def require_release_files(self, commit): + """Require every path in RELEASE_FILES to exist in a committed tree. + + Args: + commit (str): Revision to inspect for release scripts and workflow. + + Raises: + ReleaseError: Git cannot find or inspect a required path. + """ + for path in RELEASE_FILES: + try: + self.run("cat-file", "-e", f"{commit}:{path}") + except ReleaseError as ex: + raise ReleaseError(f"{path} is missing from {commit}. Merge the release tooling into main " + "and select a commit containing it.") from ex + + def remote_repository(self, remote): + """Identify a remote whose fetch and push URLs name the same repository. + + Args: + remote (str): Configured Git remote name. + + Returns: + str: GitHub OWNER/NAME, preserving the fetch URL's spelling. + + Raises: + ReleaseError: The remote is unavailable, a URL is unsupported, or + fetch and push identities differ when compared without case. + """ + fetch = repository_from_url(self.run("remote", "get-url", remote)) + push = repository_from_url(self.run("remote", "get-url", "--push", remote)) + if fetch.lower() != push.lower(): + raise ReleaseError("Fetch and push remotes must identify the same GitHub repository.") + return fetch + + def remote_tag(self, remote, tag): + """Query a remote tag without fetching it into the checkout. + + Args: + remote (str): Git remote name. + tag (str): Exact tag name, including the release's v prefix. + + Returns: + str | None: Peeled target object ID for an annotated tag, direct + object ID for a lightweight tag, or None when the tag is absent. + + Raises: + ReleaseError: Git cannot query the remote. + """ + refs = dict(line.split()[::-1] for line in self.run( + "ls-remote", "--tags", remote, f"refs/tags/{tag}", f"refs/tags/{tag}^{{}}" + ).splitlines()) + return refs.get(f"refs/tags/{tag}^{{}}", refs.get(f"refs/tags/{tag}")) + + +def repository_from_url(url): + """Extract the repository identity from a supported github.com remote URL. + + Args: + url (str): HTTPS, SCP-style git@github.com, or ssh://git@github.com URL. + + Returns: + str: OWNER/NAME with an optional .git suffix and trailing slash removed. + + Raises: + ReleaseError: The URL uses another host, embeds HTTPS credentials, + contains URL parameters, or does not identify one repository. + """ + match = re.fullmatch(r"(?:https://github\.com/|git@github\.com:|ssh://git@github\.com/)([\w.-]+/[\w.-]+?)(?:\.git)?/?", url) + if not match: + raise ReleaseError("Use a github.com SSH or HTTPS remote without embedded credentials.") + return match.group(1) + + +def required_jobs(): + """Return the set of exact job names required for a release CI attempt. + + The set contains validation, tooling tests, dependency preparation, docs, + the release gate, asset packaging, and each compiler/standard matrix entry. + The workflow contract test checks this list against CETLVaSt's matrix. + + Returns: + set[str]: Job names whose conclusion must be success. + """ + names = {"version", "release-tests", "cache-warmup", "docs", "release-ready", "release-assets"} + for architecture, compilers, standards in ( + ("amd64", ("gcc-native-32", "clang-native"), (14, 17, 20, 23)), + ("arm64", ("gcc-native", "clang-native"), (14, 17, 20)), + ("legacy-arm64", ("gcc7-native",), (14,)), + ): + names.update(f"verification-{architecture} ({compiler}, cpp-{standard})" + for compiler in compilers for standard in standards) + return names + + +def check_run(run: Mapping[str, Any], jobs: Iterable[Mapping[str, Any]], commit: str, tag: str) -> None: + """Require successful CETLVaSt verification of one exact release revision. + + Args: + run (dict): Actions run with head_sha, head_branch, event, path, status, + and conclusion fields. + jobs (Iterable[dict]): Jobs from this run's current attempt, each with + name and conclusion fields. The caller must isolate the attempt. + commit (str): Expected full commit object ID. + tag (str): Expected v-prefixed tag used as the run's head_branch. + + Raises: + ReleaseError: The run identifies another workflow/revision/event, is + unsuccessful or incomplete, or lacks a successful required job. + """ + if (run.get("head_sha") != commit or run.get("head_branch") != tag + or run.get("event") not in ("push", "workflow_dispatch") + or run.get("path") != f".github/workflows/{WORKFLOW}"): + raise ReleaseError("CI must be CETLVaSt on the exact release tag and commit (push or manual dispatch).") + if run.get("status") != "completed" or run.get("conclusion") != "success": + raise ReleaseError("Release CI has not completed successfully.") + successful = {job["name"] for job in jobs if job.get("conclusion") == "success"} + missing = required_jobs() - successful + if missing: + raise ReleaseError("Required release jobs did not succeed: " + ", ".join(sorted(missing))) + + +def asset_names(version): + """Return the archive and manifest filenames covered by SHA256SUMS. + + Args: + version (str): Validated version without the v prefix. + + Returns: + set[str]: Source, headers, and docs archive names plus release.json. + SHA256SUMS itself is outside this set. + """ + return {f"cetl-{version}-{kind}.tar.gz" for kind in ("source", "headers", "docs")} | {"release.json"} + + +def sha256(path): + """Hash a file in chunks of at most one MiB. + + Args: + path (str | Path): File to read as bytes. + + Returns: + str: Lowercase hexadecimal SHA-256 digest. + + Raises: + OSError: The file cannot be opened or read. + """ + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def check_documentation(html, version): + """Require Doxygen's displayed project number to equal a release version. + + Args: + html (str): Decoded HTML containing CETL's projectnumber span. + version (str): Expected version text, including any prerelease suffix. + + Raises: + ReleaseError: The displayed version is absent or differs after stripping + surrounding whitespace. Incidental version text is ignored. + """ + class ProjectNumber(HTMLParser): + """Collect decoded text from CETL's projectnumber span.""" + + def __init__(self): + """Initialize the capture flag and collected text fragments.""" + super().__init__() + self.recording = False + self.values = [] + + def handle_starttag(self, tag, attrs): + """Start capturing at an element with the projectnumber ID.""" + if dict(attrs).get("id") == "projectnumber": + self.recording = True + + def handle_endtag(self, tag): + """End capture at a closing span, matching CETL's Doxygen template.""" + if tag == "span": + self.recording = False + + def handle_data(self, data): + """Append decoded text while the project number is being captured.""" + if self.recording: + self.values.append(data) + + parser = ProjectNumber() + parser.feed(html) + actual = "".join(parser.values).strip() + if actual != version: + raise ReleaseError(f"Documentation displays {actual or '(no version)'}, expected {version}.") + + +def package(git, commit, tag, docs, output): + """Write reproducible release archives, a manifest, and their checksums. + + Args: + git (Git): Checkout containing the selected commit and local tag. + commit (str): Full commit object ID to archive. + tag (str): Matching v-prefixed release version. + docs (str | Path): Built HTML directory containing index.html. + output (str | Path): Empty or nonexistent destination directory. + + Raises: + ReleaseError: Version/tag checks fail, the documentation entry point is + missing, the output is nonempty, or documentation contains symlinks. + OSError: A directory or archive cannot be read or written. + + Archives share a cetl-VERSION root. The header and documentation archives + include LICENSE. Archive metadata and ordering are normalized for repeatable + bytes from identical inputs. Failures may leave partial local output. + """ + metadata = git.validate(commit, tag) + if git.commit(f"refs/tags/{tag}") != commit: + raise ReleaseError("Packaging commit differs from the local release tag.") + docs, output = Path(docs), Path(output) + if not (docs / "index.html").is_file(): + raise ReleaseError("Built documentation must contain index.html.") + if output.exists() and any(output.iterdir()): + raise ReleaseError("Package output directory must be empty.") + output.mkdir(parents=True, exist_ok=True) + version = metadata["version"] + prefix = f"cetl-{version}/" + for kind, paths in (("source", []), ("headers", ["include", "LICENSE"])): + archive = git.run("archive", "--format=tar", f"--prefix={prefix}", commit, *paths, binary=True) + with (output / f"cetl-{version}-{kind}.tar.gz").open("wb") as stream: + with gzip.GzipFile(fileobj=stream, mode="wb", filename="", mtime=0) as compressed: + compressed.write(archive) + with (output / f"cetl-{version}-docs.tar.gz").open("wb") as stream: + with gzip.GzipFile(fileobj=stream, mode="wb", filename="", mtime=0) as compressed: + with tarfile.open(fileobj=compressed, mode="w") as archive: + for path in sorted(docs.rglob("*")): + if path.is_symlink(): + raise ReleaseError("Documentation must not contain symlinks.") + if path.is_file(): + data = path.read_bytes() + info = tarfile.TarInfo(prefix + "html/" + path.relative_to(docs).as_posix()) + info.size, info.mode = len(data), 0o644 + archive.addfile(info, io.BytesIO(data)) + license_data = git.run("show", f"{commit}:LICENSE", binary=True) + info = tarfile.TarInfo(prefix + "LICENSE") + info.size, info.mode = len(license_data), 0o644 + archive.addfile(info, io.BytesIO(license_data)) + manifest = {**metadata, "commit": commit} + (output / "release.json").write_text(json.dumps(manifest, indent=2) + "\n") + (output / "SHA256SUMS").write_text("".join( + f"{sha256(output / name)} {name}\n" for name in sorted(asset_names(version)) + )) + + +def check_bundle(directory, commit, tag): + """Check a bundle's exact filenames, release identity, and file checksums. + + Args: + directory (str | Path): Unpacked directory produced by ``package``. + commit (str): Expected full source commit object ID. + tag (str): Expected validated v-prefixed release version. + + Returns: + list[str]: Sorted filenames, including SHA256SUMS, for asset syncing. + + Raises: + ReleaseError: Files are missing/unexpected, metadata differs, or a + checksum differs. Archive contents are not extracted by this check. + ValueError: The manifest is invalid JSON. + OSError: A bundle file cannot be read. + """ + directory = Path(directory) + version = tag[1:] + expected = asset_names(version) + if {path.name for path in directory.iterdir()} != expected | {"SHA256SUMS"}: + raise ReleaseError("Release bundle contains missing or unexpected files.") + manifest = json.loads((directory / "release.json").read_text()) + if manifest != {"version": version, "tag": tag, "prerelease": "-" in version, "commit": commit}: + raise ReleaseError("Release bundle metadata does not match the requested release.") + sums = (directory / "SHA256SUMS").read_text() + if sums != "".join(f"{sha256(directory / name)} {name}\n" for name in sorted(expected)): + raise ReleaseError("Release bundle checksums do not match its contents.") + return sorted(expected | {"SHA256SUMS"}) + + +def main(): + """Run the validation, packaging, or built-documentation CLI operation. + + Arguments come from sys.argv. Validation checks HEAD against tag/release + event metadata from GitHub's environment, prints JSON, and appends fields to + GITHUB_OUTPUT when configured. Packaging writes the selected local bundle; + documentation verification reads the supplied HTML file. + + Raises: + SystemExit: Argument parsing exits, or an operation fails with status 1. + """ + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="operation", required=True) + check = subparsers.add_parser("validate", help="Validate checkout and GitHub event metadata.") + check.add_argument("--tag") + check.add_argument("--prerelease", choices=("true", "false")) + build = subparsers.add_parser("package", help="Package the validated tagged commit and built documentation.") + build.add_argument("--tag", required=True) + build.add_argument("--commit", required=True) + build.add_argument("--docs", required=True) + build.add_argument("--output", required=True) + docs = subparsers.add_parser("verify-docs", help="Check the version displayed by built documentation.") + docs.add_argument("--version", required=True) + docs.add_argument("--path", type=Path, required=True) + args = parser.parse_args() + git = Git(Path.cwd()) + try: + if args.operation == "verify-docs": + parse_version(args.version) + check_documentation(args.path.read_text(encoding="utf-8"), args.version) + return + commit = git.commit("HEAD") + if args.operation == "package": + package(git, git.commit(args.commit), args.tag, args.docs, args.output) + return + tag, prerelease = args.tag, None if args.prerelease is None else args.prerelease == "true" + event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text()) if os.getenv("GITHUB_EVENT_PATH") else {} + if os.getenv("GITHUB_EVENT_NAME") == "release": + tag, prerelease = event["release"]["tag_name"], event["release"]["prerelease"] + elif os.getenv("GITHUB_REF", "").startswith("refs/tags/"): + tag = os.environ["GITHUB_REF"][len("refs/tags/"):] + if tag and git.commit(f"refs/tags/{tag}") != commit: + raise ReleaseError("Checkout does not match the requested tag's commit.") + result = {**git.validate(commit, tag, prerelease), "commit": commit} + print(json.dumps(result, indent=2)) + if os.getenv("GITHUB_OUTPUT"): + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as stream: + for key, value in result.items(): + stream.write(f"{key}={str(value).lower() if isinstance(value, bool) else value}\n") + except (ReleaseError, ValueError, OSError, KeyError) as ex: + parser.exit(1, f"Release validation failed: {ex}\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/tests/test_release.py b/scripts/tests/test_release.py new file mode 100644 index 00000000..eeddba00 --- /dev/null +++ b/scripts/tests/test_release.py @@ -0,0 +1,914 @@ +"""Test release contracts and rehearse publication against disposable Git repositories. + +Pure checks exercise version, transport, CI-gate, and documentation contracts. +Repository rehearsals use real Git subprocesses and a local bare remote, with +SimulatedGitHub supplying API state and failure injection. GitHub/network access +is replaced by fixtures or mocks; temporary repositories and assets are cleaned +up by unittest. Run the suite through unittest discovery. +""" + +import argparse +import ast +import contextlib +import hashlib +import io +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +from typing import Any +import unittest +from unittest.mock import patch + +SCRIPTS = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(SCRIPTS)) + +from release import GitHub, check_local_tools, cut_release, require_latest_stable # noqa: E402 +from release_tools import (Git, HEADER, RELEASE_FILES, ReleaseError, check_bundle, check_documentation, check_run, command, # noqa: E402 + header_version, package, parse_version, repository_from_url, + required_jobs, sha256, validate) + +HEADER_TEXT = "#define CETL_VERSION_MAJOR 2\n#define CETL_VERSION_MINOR 0\n#define CETL_VERSION_PATCH 0\n" + + +def _workflow_group(pattern: str, workflow: str, flags: int = 0) -> str: + """Extract a required workflow field, failing explicitly when its pattern is absent.""" + match = re.search(pattern, workflow, flags) + if match is None: + raise AssertionError(f"Required workflow pattern did not match: {pattern}") + return match.group(1) + + +class VersionTests(unittest.TestCase): + """Check the accepted version grammar, header definitions, and remote identities.""" + + def test_stable_and_prerelease(self): + """Accept matching stable versions and each supported prerelease stage.""" + self.assertEqual(header_version(HEADER_TEXT), "2.0.0") + self.assertFalse(validate(HEADER_TEXT, "v2.0.0", False)["prerelease"]) + for stage in ("alpha", "beta", "rc"): + self.assertTrue(validate(HEADER_TEXT, f"v2.0.0-{stage}.1", True)["prerelease"]) + + def test_invalid_versions(self): + """Reject ambiguous components, unsupported suffixes, and malformed version text.""" + for version in ("02.0.0", "2.00.0", "2.0", "v2.0.0", "2.0.0-rc.0", "2.0.0-rc.01", + "2.0.0-dev.1", "2.0.0+build", "2.0.0\n", "-n", "2.0.0-rc.1/branch"): + with self.subTest(version=version), self.assertRaises(ReleaseError): + parse_version(version) + + def test_bad_header_definitions(self): + """Require each version macro exactly once as a bare decimal integer.""" + for header in (HEADER_TEXT.replace("#define CETL_VERSION_MAJOR 2\n", ""), + HEADER_TEXT + "#define CETL_VERSION_MAJOR 2\n", + HEADER_TEXT.replace("MAJOR 2", "MAJOR (2)"), + HEADER_TEXT.replace("MAJOR 2", "MAJOR 02"), + HEADER_TEXT.replace("MAJOR 2", "MAJOR -2"), + HEADER_TEXT.replace("MAJOR 2", "MAJOR 2 garbage")): + with self.subTest(header=header), self.assertRaises(ReleaseError): + header_version(header) + + def test_misaligned_tags_and_flags(self): + """Reject tag/header disagreements and inconsistent prerelease flags.""" + for tag, flag in (("v2.0.1", None), ("2.0.0", None), ("v2.0.0", True), ("v2.0.0-rc.1", False)): + with self.subTest(tag=tag, flag=flag), self.assertRaises(ReleaseError): + validate(HEADER_TEXT, tag, flag) + + def test_github_remote_formats(self): + """Accept supported GitHub remotes and reject credentials or another host.""" + for url in ("https://github.com/OpenCyphal/CETL.git", "git@github.com:OpenCyphal/CETL.git", + "ssh://git@github.com/OpenCyphal/CETL", "https://github.com/OpenCyphal/CETL/"): + self.assertEqual(repository_from_url(url), "OpenCyphal/CETL") + for url in ("https://token@github.com/OpenCyphal/CETL", "https://github.com.evil/CETL", + "/tmp/origin.git", "https://github.com/OpenCyphal/CETL?token=secret"): + with self.assertRaises(ReleaseError): + repository_from_url(url) + + +def successful_run(commit: str, tag: str) -> dict[str, Any]: + """Build a successful tagged-push Actions record for gate tests. + + Args: + commit (str): Source object ID exposed as head_sha. + tag (str): Exact release tag exposed as head_branch. + + Returns: + dict: Mutable run fixture identifying CETLVaSt, attempt 1, and run 123. + """ + return {"id": 123, "run_attempt": 1, "head_sha": commit, "head_branch": tag, + "event": "push", "path": ".github/workflows/cetlvast.yml", + "status": "completed", "conclusion": "success", "html_url": "https://example.test/runs/123"} + + +class GateTests(unittest.TestCase): + """Check that release gates require the exact workflow, revision, and job set.""" + + def setUp(self): + """Create an accepted run and one successful record for every required job.""" + self.workflow_run = successful_run("a" * 40, "v2.0.0") + self.jobs = [{"name": name, "conclusion": "success"} for name in sorted(required_jobs())] + + def check(self): + """Validate the mutable run/job fixtures against the fixed release identity.""" + check_run(self.workflow_run, self.jobs, "a" * 40, "v2.0.0") + + def test_success(self): + """Accept a successful run containing every required release job.""" + self.check() + + def test_missing_or_unsuccessful_job(self): + """Reject a missing job and every non-successful required-job conclusion.""" + for result in (None, "skipped", "failure", "cancelled", "timed_out"): + with self.subTest(result=result): + jobs = self.jobs[:-1] if result is None else [*self.jobs[:-1], {**self.jobs[-1], "conclusion": result}] + with self.assertRaises(ReleaseError): + check_run(self.workflow_run, jobs, "a" * 40, "v2.0.0") + + def test_wrong_commit_event_workflow_tag_or_state(self): + """Reject a run whose identity or completion state differs from the release.""" + for key, value in (("head_sha", "b" * 40), ("event", "pull_request"), ("head_branch", "main"), + ("path", ".github/workflows/other.yml"), ("status", "in_progress"), + ("conclusion", "cancelled")): + with self.subTest(key=key), self.assertRaises(ReleaseError): + check_run({**self.workflow_run, key: value}, self.jobs, "a" * 40, "v2.0.0") + + def test_matrix_and_gate_contract_matches_workflow(self): + """Keep required job names and aggregate dependencies aligned with CETLVaSt.""" + # Parse only the inline arrays we own; no third-party YAML dependency is needed by the tooling. + workflow = (SCRIPTS.parent / ".github/workflows/cetlvast.yml").read_text() + for architecture in ("amd64", "arm64", "legacy-arm64"): + section = _workflow_group( + rf"^ verification-{architecture}:\n(.*?)(?=^ [a-z]|\Z)", workflow, re.M | re.S + ) + compilers = ast.literal_eval(_workflow_group(r"toolchain: (\[.*\])", section)) + standards = ast.literal_eval(_workflow_group(r"standard: (\[.*\])", section)) + expected = {f"verification-{architecture} ({compiler}, {standard})" + for compiler in compilers for standard in standards} + self.assertEqual(expected, {name for name in required_jobs() if name.startswith(f"verification-{architecture} (")}) + self.assertIn("startsWith(github.ref, 'refs/tags/v')", section) + self.assertNotIn("!inputs.docs_ref", section) + gate = _workflow_group(r"^ release-ready:\n(.*?)(?=^ [a-z])", workflow, re.M | re.S) + dependencies = _workflow_group(r"needs: \[(.*)\]", gate).split(", ") + self.assertEqual(set(dependencies), {"version", "release-tests", "cache-warmup", "docs", + "verification-amd64", "verification-arm64", "verification-legacy-arm64"}) + self.assertIn("job['result'] != 'success'", gate) + + def test_full_release_gate_triggers_cover_all_dependencies(self): + """Require every full-gate trigger to enable each conditional dependency.""" + workflow = (SCRIPTS.parent / ".github/workflows/cetlvast.yml").read_text() + gate = _workflow_group(r"^ release-ready:\n(.*?)(?=^ [a-z])", workflow, re.M | re.S) + condition_pattern = r"^ if: >-?\n(.*?)(?=^ \S)" + # Each predicate is an OR term; always() lets the gate inspect failed dependencies. + predicate_pattern = r"(?:startsWith|contains)\([^()\n]*\)|github\.event_name == '[^']+'" + condition = _workflow_group(condition_pattern, gate, re.M | re.S) + gate_triggers = set(re.findall(predicate_pattern, condition)) + self.assertTrue(gate_triggers) + dependencies = _workflow_group(r"needs: \[(.*)\]", gate).split(", ") + for dependency in dependencies: + with self.subTest(dependency=dependency): + section = _workflow_group(rf"^ {re.escape(dependency)}:\n(.*?)(?=^ [a-z]|\Z)", + workflow, re.M | re.S) + if not re.search(r"^ if:", section, re.M): + continue + condition = _workflow_group(condition_pattern, section, re.M | re.S) + missing = gate_triggers - set(re.findall(predicate_pattern, condition)) + self.assertFalse(missing, f"{dependency} skips full-gate triggers: {sorted(missing)}") + + +class GitHubTransportTests(unittest.TestCase): + """Check CLI/API encoding, local prerequisites, diagnostics, and public HTML reads.""" + + def test_local_tools_require_python_314(self): + """Reject unsupported Python versions before launching any subprocess.""" + for version in ((3, 10), (3, 13)): + with self.subTest(version=version), patch("release.command") as execute: + with self.assertRaisesRegex(ReleaseError, r"Python 3\.14"): + check_local_tools(version) + execute.assert_not_called() + + def test_old_github_cli_prints_upgrade_instruction(self): + """Report an upgrade action when a required GitHub CLI operation is unavailable.""" + with patch("release.command", side_effect=["git version", "gh version", ReleaseError("unsupported")]): + with self.assertRaisesRegex(ReleaseError, "Update GitHub CLI"): + check_local_tools((3, 14)) + + def test_missing_tool_prints_install_instruction(self): + """Report installation guidance when the GitHub CLI executable is missing.""" + with patch("release_tools.subprocess.run", side_effect=FileNotFoundError): + with self.assertRaisesRegex(ReleaseError, "Install GitHub CLI"): + command(["gh", "--version"]) + + def test_api_error_explains_authentication_without_echoing_credentials(self): + """Replace transport diagnostics with authentication guidance that omits secrets.""" + with patch("release.command", side_effect=ReleaseError("secret-token")): + with self.assertRaisesRegex(ReleaseError, "gh auth login") as caught: + GitHub("example/cetl").api("") + self.assertNotIn("secret-token", str(caught.exception)) + + def test_documentation_version_reads_visible_doxygen_number(self): + """Match the displayed project number rather than incidental page text.""" + check_documentation(' 2.0.0', "2.0.0") + for html in ('1.0.0', '

2.0.0 elsewhere in the page

'): + with self.assertRaisesRegex(ReleaseError, "Documentation displays"): + check_documentation(html, "2.0.0") + + def test_site_verification_fetches_actual_published_html(self): + """Fetch public site HTML without attaching GitHub credentials.""" + github = GitHub("example/cetl") + response = io.BytesIO(b'2.0.0') + with patch.object(github, "api", return_value={"html_url": "https://example.test/cetl/"}), \ + patch("release.urlopen", return_value=response) as fetch: + with contextlib.redirect_stdout(io.StringIO()): + github.verify_pages("2.0.0", 1, 1) + self.assertEqual(fetch.call_args.args[0].full_url, "https://example.test/cetl/") + self.assertIsNone(fetch.call_args.args[0].get_header("Authorization")) + + def test_wrong_published_site_version_fails_with_resume_instruction(self): + """Report the displayed/expected version mismatch and a recovery action.""" + github = GitHub("example/cetl") + response = io.BytesIO(b'1.0.0') + with patch.object(github, "api", return_value={"html_url": "https://example.test/cetl/"}), \ + patch("release.urlopen", return_value=response), \ + patch("release.time.monotonic", side_effect=[0, 0, 2]), patch("release.time.sleep"), \ + contextlib.redirect_stdout(io.StringIO()): + with self.assertRaisesRegex(ReleaseError, "Documentation displays 1.0.0, expected 2.0.0.*printed resume command"): + github.verify_pages("2.0.0", 1, 1) + + def test_site_network_failure_is_distinguished_from_wrong_version(self): + """Identify network access failures without exposing raw transport details.""" + github = GitHub("example/cetl") + with patch.object(github, "api", return_value={"html_url": "https://example.test/cetl/"}), \ + patch("release.urlopen", side_effect=OSError("sensitive transport details")), \ + patch("release.time.monotonic", side_effect=[0, 0, 2]), patch("release.time.sleep"), \ + contextlib.redirect_stdout(io.StringIO()): + with self.assertRaisesRegex(ReleaseError, "check network/TLS access") as caught: + github.verify_pages("2.0.0", 1, 1) + self.assertNotIn("sensitive transport details", str(caught.exception)) + + def test_cli_pins_host_and_api_encodes_booleans(self): + """Scope commands to github.com and preserve API field values and boolean types.""" + github = GitHub("example/cetl") + with patch("release.command", return_value="{}") as execute: + github.cli("release", "upload", "v2.0.0", "/tmp/notes") + self.assertEqual(execute.call_args.args[0][-2:], ["--repo", "github.com/example/cetl"]) + github.api("releases/1", "PATCH", draft=False, body="literal $() and `quotes`\nnext line") + args = execute.call_args.args[0] + self.assertIn("draft=false", args) + self.assertIn("body=literal $() and `quotes`\nnext line", args) + self.assertIn("github.com", args) + + def test_pagination_keeps_later_pages(self): + """Collect records beyond the first full API page.""" + github = GitHub("example/cetl") + with patch.object(github, "api", side_effect=[{"jobs": list(range(100))}, {"jobs": [100]}]): + self.assertEqual(github.pages("actions/runs/123/jobs?filter=latest", "jobs"), list(range(101))) + + def test_permission_error_is_not_a_missing_release(self): + """Propagate listing failures instead of treating inaccessible releases as absent.""" + github = GitHub("example/cetl") + with patch.object(github, "api", side_effect=ReleaseError("Forbidden")): + with self.assertRaisesRegex(ReleaseError, "Forbidden"): + github.release("v2.0.0") + + def test_unexpected_metadata_response_stops_preflight(self): + """Reject empty or non-object metadata before reading workflow configuration.""" + github = GitHub("example/cetl") + for response in (None, [], "not an object"): + with self.subTest(response=response), patch.object(github, "api", return_value=response): + with self.assertRaisesRegex(ReleaseError, "returned no JSON object"): + github.preflight(stable=False) + + def test_malformed_collection_does_not_hide_missing_records(self): + """Reject malformed collections instead of treating their records as absent.""" + github = GitHub("example/cetl") + for response, key in ((None, None), ({}, None), ([], "jobs"), ({}, "jobs"), + ({"jobs": None}, "jobs"), ({"jobs": {}}, "jobs")): + with self.subTest(response=response, key=key), patch.object(github, "api", return_value=response): + with self.assertRaisesRegex(ReleaseError, "returned no JSON (object|array)"): + github.pages("actions/runs/123/jobs", key) + + def test_stable_version_comparison_uses_numeric_versions(self): + """Order published stable versions numerically and ignore other release records.""" + github = GitHub("example/cetl") + releases = [{"tag_name": tag, "draft": draft, "prerelease": prerelease} for tag, draft, prerelease in ( + ("v2.9.0", False, False), ("v2.10.0", False, False), ("v3.0.0", True, False), + ("v4.0.0-rc.1", False, True), ("historical-non-semver", False, False))] + with patch.object(github, "pages", return_value=releases): + self.assertEqual(max(github.published_stable_versions()), (2, 10, 0)) + + def test_subprocess_errors_do_not_echo_credentials(self): + """Keep captured subprocess stderr out of reported command failures.""" + with patch("release_tools.subprocess.run", return_value=subprocess.CompletedProcess( + ["gh"], 1, stdout=b"", stderr=b"secret-token" + )): + with self.assertRaises(ReleaseError) as caught: + command(["gh", "api", "user"]) + self.assertNotIn("secret-token", str(caught.exception)) + + +class LocalGit(Git): + """Use a local bare remote while supplying the rehearsal's GitHub identity. + + Construction and Git operations follow ``Git``. Overriding only repository + identity lets release checks use real commits, refs, archives, and pushes + without a hosted repository. + """ + + def remote_repository(self, remote): + """Return example/cetl for the fixture remote, bypassing GitHub URL parsing.""" + return "example/cetl" + + +class SimulatedGitHub(GitHub): + """Implement the release client's test interface with mutable in-memory state. + + Args: + git (LocalGit): Fixture checkout used to build actual release archives. + docs (Path): Fixture HTML directory supplied to ``package``. + tag (str): Release tag shared by the fixture's run and release. + + Tests adjust run, release_data, docs_run, policy, and site_version fields to + model remote state. asset_data maps filenames to uploaded bytes; writes + records draft/upload/publish/dispatch events. fail_after_upload interrupts + after the specified upload count, and fail_docs controls deployment failure. + Unsupported API paths and CLI operations raise AssertionError. + """ + + def __init__(self, git, docs, tag): + """Initialize an accessible repository with passing CI and usable Pages setup.""" + self.git, self.docs, self.tag = git, docs, tag + super().__init__("example/cetl") + self.run: dict[str, Any] | None = successful_run(git.commit("HEAD"), tag) + self.release_data: dict[str, Any] | None = None + self.asset_data: dict[str, bytes] = {} + self.writes: list[str] = [] + self.fail_after_upload: int | None = None + self.permissions = {"push": True} + self.docs_run: dict[str, Any] | None = None + self.fail_docs = False + self.other_versions = set() + self.workflow_state = "active" + self.pages_type = "workflow" + self.environment_policy = {"protected_branches": False, "custom_branch_policies": True} + self.branch_policies = [{"name": "main", "type": "branch"}] + self.main_protected = True + self.site_version = "2.0.0" + + def preflight(self, stable): + """Run the production preflight against simulated workflow and policy records.""" + return super().preflight(stable) + + def pages(self, path, key=None): + """Supply the configured deployment branch policies for preflight checks.""" + if path == "environments/github-pages/deployment-branch-policies": + return self.branch_policies + raise AssertionError(path) + + def verify_pages(self, version, timeout, poll): + """Check site_version through the real HTML validator without polling.""" + check_documentation(f'{self.site_version}', version) + + def published_stable_versions(self): + """Combine other_versions with the fixture release's core when published stable.""" + versions = set(self.other_versions) + if self.release_data and not self.release_data["draft"] and not self.release_data["prerelease"]: + versions.add(tuple(map(int, self.tag[1:].split(".")))) + return versions + + def api(self, path, method="GET", **fields): + """Return configuration records or apply and record the fixture's publish PATCH.""" + if path == "": + return {"permissions": self.permissions, "default_branch": "main"} + if path == "actions/workflows/cetlvast.yml": + return {"state": self.workflow_state} + if path == "pages": + return {"build_type": self.pages_type, "html_url": "https://example.test/cetl/"} + if path == "environments/github-pages": + return {"deployment_branch_policy": self.environment_policy} + if path == "branches/main": + return {"protected": self.main_protected} + if path == "releases/1" and method == "PATCH": + assert self.release_data is not None + self.writes.append("publish") + self.release_data["draft"] = fields["draft"] + return self.release_data + raise AssertionError((path, method, fields)) + + def release(self, tag): + """Return release_data; each fixture represents a single release identity.""" + return self.release_data + + def runs(self, **filters): + """Return the docs run for dispatch queries, or the fixture's tag run otherwise.""" + if filters.get("event") == "workflow_dispatch": + return [self.docs_run] if self.docs_run else [] + return [self.run] if self.run else [] + + def jobs(self, run): + """Supply a deployment result for docs_run, or a complete successful gate.""" + if run is self.docs_run: + return [{"name": "deploy-docs", "conclusion": "failure" if self.fail_docs else "success"}] + return [{"name": name, "conclusion": "success"} for name in required_jobs()] + + def assets(self, release): + """Expose stored asset bytes as uploaded records with sizes and SHA-256 digests.""" + return [{"name": name, "state": "uploaded", "size": len(data), + "digest": "sha256:" + hashlib.sha256(data).hexdigest()} for name, data in self.asset_data.items()] + + def asset_matches(self, asset, path): + """Delegate byte/digest comparison to the production client implementation.""" + return super().asset_matches(asset, path) + + def create_draft(self, tag, commit, prerelease, marker, run, notes=None): + """Record draft creation and retain the classification and management marker.""" + self.writes.append("draft") + self.release_data = {"id": 1, "draft": True, "tag_name": tag, "prerelease": prerelease, "body": marker} + + def cli(self, *args) -> str: + """Simulate downloads, immutable uploads, and workflow dispatches. + + Run downloads package the fixture's real Git tree; release downloads + write saved asset bytes. Uploads record bytes before optional failure + injection, reproducing an interrupted operation's partial remote state. + Main dispatches create a completed docs run controlled by fail_docs. + """ + if args[:2] == ("run", "download"): + assert self.run is not None + package(self.git, self.run["head_sha"], self.tag, self.docs, Path(args[args.index("--dir") + 1])) + elif args[:2] == ("release", "upload"): + path = Path(args[3]) + if path.name in self.asset_data: + raise AssertionError("Attempted to overwrite an asset") + self.asset_data[path.name] = path.read_bytes() + self.writes.append("upload") + if self.fail_after_upload == len(self.asset_data): + raise ReleaseError("Simulated interrupted upload") + elif args[:2] == ("release", "download"): + directory = Path(args[args.index("--dir") + 1]) + for name, data in self.asset_data.items(): + (directory / name).write_bytes(data) + elif args[:2] == ("workflow", "run"): + self.writes.append("dispatch") + if args[args.index("--ref") + 1] == "main": + self.docs_run = {"id": (self.docs_run or {"id": 200})["id"] + 1, "display_title": f"CETLVaSt / {self.tag}", + "status": "completed", "conclusion": "failure" if self.fail_docs else "success", + "html_url": "https://example.test/docs"} + else: + raise AssertionError(args) + return "" + + +class RepositoryTests(unittest.TestCase): + """Rehearse publication and recovery with real Git and a simulated GitHub service.""" + + def setUp(self): + """Create a clean main checkout, bare remote, tooling files, and release options. + + Register repository cleanup and replace local executable preflight; its + behavior is tested separately by GitHubTransportTests. + """ + tools = patch("release.check_local_tools") + tools.start() + self.addCleanup(tools.stop) + self.temporary = tempfile.TemporaryDirectory(prefix="cetl-rehearsal-") + self.addCleanup(self.temporary.cleanup) + root = Path(self.temporary.name) + self.root = root / "checkout" + self.root.mkdir() + self.git = LocalGit(self.root) + self.git.run("init", "--initial-branch=main") + self.git.run("config", "user.name", "CETL release rehearsal") + self.git.run("config", "user.email", "release@example.test") + (self.root / HEADER).parent.mkdir(parents=True) + (self.root / HEADER).write_text(HEADER_TEXT) + (self.root / "LICENSE").write_text("Fixture license\n") + (self.root / "README.md").write_text("Fixture source\n") + for path in RELEASE_FILES: + destination = self.root / path + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(SCRIPTS.parent / path, destination) + self.git.run("add", ".") + self.git.run("commit", "-m", "Release fixture") + command(["git", "init", "--bare", "--initial-branch=main", str(root / "remote.git")]) + self.git.run("remote", "add", "origin", str(root / "remote.git")) + self.git.run("push", "origin", "main") + self.commit = self.git.commit("HEAD") + self.docs = root / "html" + self.docs.mkdir() + (self.docs / "index.html").write_text("Fixture built documentation") + self.args = argparse.Namespace(version="2.0.0", ref=self.commit, repo="example/cetl", remote="origin", + root=self.root, notes_file=None, dry_run=False, resume=False, timeout=1, poll=1) + self.github = SimulatedGitHub(self.git, self.docs, "v2.0.0") + + def cut(self): + """Run the release with mutable fixture options and return captured progress text.""" + with contextlib.redirect_stdout(io.StringIO()) as output: + cut_release(self.args, self.git, self.github) + return output.getvalue() + + def add_tag(self, tag="v2.0.0"): + """Create a lightweight tag at fixture HEAD and push it to the bare remote.""" + self.git.run("tag", tag) + self.git.run("push", "origin", f"refs/tags/{tag}") + + def test_full_release_rehearsal_and_published_resume(self): + """Publish once and verify a repeated matching attempt performs no remote writes.""" + self.add_tag() + self.git.run("tag", "-d", "v2.0.0") # Cutting a release must not require a local tag. + self.args.resume = True + # The simulated CI checkout has the tag, just as actions/checkout fetch-depth: 0 does. + self.git.run("fetch", "--tags", "origin") + output = self.cut() + self.assertIn("Documentation:", output) + assert self.github.release_data is not None + self.assertFalse(self.github.release_data["draft"]) + self.assertEqual(self.git.remote_tag("origin", "v2.0.0"), self.commit) + writes = list(self.github.writes) + self.cut() + self.assertEqual(self.github.writes, writes) + + def test_new_tag_push_and_prerelease(self): + """Push and publish a prerelease while retaining the stable documentation site.""" + self.args.version = "2.0.0-rc.1" + self.github.tag = "v" + self.args.version + assert self.github.run is not None + self.github.run["head_branch"] = self.github.tag + # CI receives the pushed tag in its separate checkout. Simulate that fetch just before packaging. + original_cli = self.github.cli + def cli(*args): + """Fetch the pushed tag before emulating CI's separate packaging checkout.""" + if args[:2] == ("run", "download"): + self.git.run("fetch", "--tags", "origin") + return original_cli(*args) + self.github.cli = cli + self.cut() + self.assertEqual(self.git.remote_tag("origin", self.github.tag), self.commit) + assert self.github.release_data is not None + self.assertTrue(self.github.release_data["prerelease"]) + self.assertNotIn("dispatch", self.github.writes) + + def test_dry_run_has_no_remote_writes(self): + """Complete preflight without pushing a tag or changing simulated GitHub state.""" + self.args.dry_run = True + self.assertIn("No remote writes", self.cut()) + self.assertFalse(self.github.writes) + self.assertIsNone(self.git.remote_tag("origin", "v2.0.0")) + + def test_defaults_fetch_main_and_derive_header_version(self): + """Pin commands to freshly fetched main even when the local checkout is older.""" + (self.root / HEADER).write_text(HEADER_TEXT.replace("MINOR 0", "MINOR 1")) + self.git.run("add", ".") + self.git.run("commit", "-m", "Prepare version 2.1.0") + new_commit = self.git.commit("HEAD") + self.git.run("push", "origin", "main") + self.git.run("checkout", "--detach", self.commit) + self.args.ref, self.args.version, self.args.dry_run = None, None, True + output = self.cut() + self.assertIn("Tag: v2.1.0", output) + self.assertIn("Commit: " + new_commit, output) + self.assertIn("--ref " + new_commit, output) + self.assertIn("Release this checked commit:", output) + self.assertFalse(self.github.writes) + self.assertIsNone(self.git.remote_tag("origin", "v2.1.0")) + + def test_missing_workflow_on_main_stops_before_tag_creation(self): + """Reject a main commit missing release tooling before publishing any tag.""" + self.git.run("rm", ".github/workflows/cetlvast.yml") + self.git.run("commit", "-m", "Remove workflow") + self.git.run("push", "origin", "main") + self.args.ref = self.git.commit("HEAD") + with self.assertRaisesRegex(ReleaseError, "Merge the release tooling"): + self.cut() + self.assertIsNone(self.git.remote_tag("origin", "v2.0.0")) + + def test_disabled_workflow_stops_before_tag_creation(self): + """Reject disabled CETLVaSt with an enable command before pushing a tag.""" + self.github.workflow_state = "disabled_manually" + with self.assertRaisesRegex(ReleaseError, "gh workflow enable"): + self.cut() + self.assertIsNone(self.git.remote_tag("origin", "v2.0.0")) + + def test_pages_must_use_actions_before_publishing_stable(self): + """Reject a stable release when Pages uses the legacy build source.""" + self.github.pages_type = "legacy" + with self.assertRaisesRegex(ReleaseError, "Pages build source"): + self.cut() + self.assertIsNone(self.git.remote_tag("origin", "v2.0.0")) + + def test_tag_only_pages_policy_does_not_allow_main(self): + """Require branch deployment permission even when tag deployment is unrestricted.""" + self.github.branch_policies = [{"name": "*", "type": "tag"}] + with self.assertRaisesRegex(ReleaseError, "Allow main"): + self.cut() + self.assertFalse(self.github.writes) + + def test_protected_branch_pages_policy_rejects_unprotected_main(self): + """Reject an unprotected main under a protected-branches-only Pages policy.""" + self.github.environment_policy = {"protected_branches": True, "custom_branch_policies": False} + self.github.main_protected = False + with self.assertRaisesRegex(ReleaseError, "Allow main"): + self.cut() + + def test_prerelease_does_not_require_pages_configuration(self): + """Allow prerelease preflight without a stable Pages deployment configuration.""" + self.args.version, self.args.dry_run = "2.0.0-rc.1", True + self.github.pages_type = "legacy" + self.github.branch_policies = [] + self.cut() + self.assertFalse(self.github.writes) + + def test_authentication_failure(self): + """Stop on inaccessible repository metadata before creating remote release state.""" + with patch.object(self.github, "api", side_effect=ReleaseError("Authentication failed")): + with self.assertRaisesRegex(ReleaseError, "Authentication"): + self.cut() + self.assertIsNone(self.git.remote_tag("origin", "v2.0.0")) + + def test_read_only_credentials(self): + """Refuse publication when repository permissions lack push access.""" + self.github.permissions = {"push": False} + with self.assertRaisesRegex(ReleaseError, "write access"): + self.cut() + self.assertFalse(self.github.writes) + + def test_dirty_checkout(self): + """Refuse release preparation when the checkout contains an untracked file.""" + (self.root / "untracked").write_text("untracked") + with self.assertRaisesRegex(ReleaseError, "clean"): + self.cut() + + def test_wrong_repository(self): + """Reject a requested repository that disagrees with the Git remote identity.""" + self.args.repo = "other/cetl" + with self.assertRaisesRegex(ReleaseError, "does not match"): + self.cut() + + def test_unmerged_commit(self): + """Require the selected commit to be an ancestor of remote main.""" + self.git.run("switch", "-c", "unmerged") + (self.root / "new").write_text("unmerged") + self.git.run("add", ".") + self.git.run("commit", "-m", "Not merged") + self.args.ref = self.git.commit("HEAD") + with self.assertRaises(ReleaseError): + self.cut() + self.assertFalse(self.github.writes) + + def test_mismatched_version_never_pushes(self): + """Reject a version differing from the selected header before pushing its tag.""" + self.args.version = "2.0.1" + with self.assertRaisesRegex(ReleaseError, "does not match"): + self.cut() + self.assertIsNone(self.git.remote_tag("origin", "v2.0.1")) + + def test_conflicting_remote_tag(self): + """Preserve an existing remote tag when the requested source commit differs.""" + self.add_tag() + (self.root / "new").write_text("new") + self.git.run("add", ".") + self.git.run("commit", "-m", "Next commit") + self.git.run("push", "origin", "main") + self.args.ref = self.git.commit("HEAD") + self.args.resume = True + with self.assertRaisesRegex(ReleaseError, "another commit"): + self.cut() + self.assertEqual(self.git.remote_tag("origin", "v2.0.0"), self.commit) + + def test_existing_tag_requires_resume(self): + """Require explicit resume intent for an existing matching tag.""" + self.add_tag() + with self.assertRaisesRegex(ReleaseError, "--resume"): + self.cut() + + def test_dry_run_for_existing_tag_prints_a_working_resume_command(self): + """Include resume intent and the exact commit in an existing-tag dry-run command.""" + self.add_tag() + self.args.dry_run = True + output = self.cut() + release_line = next(line for line in output.splitlines() if line.startswith("Release this checked commit:")) + self.assertIn("--resume", release_line) + self.assertIn("--ref " + self.commit, release_line) + self.assertFalse(self.github.writes) + + def test_resume_requires_pinned_version_and_commit(self): + """Reject resume options that could follow an advancing main branch.""" + self.args.resume, self.args.ref = True, None + with self.assertRaisesRegex(ReleaseError, "printed Resume command"): + self.cut() + self.assertIsNone(self.git.remote_tag("origin", "v2.0.0")) + + def test_failed_ci_cannot_create_draft_or_publish(self): + """Block draft creation and publication when tagged CI fails.""" + self.add_tag() + self.args.resume = True + assert self.github.run is not None + self.github.run["conclusion"] = "failure" + with self.assertRaisesRegex(ReleaseError, "CI"): + self.cut() + self.assertFalse(self.github.writes) + + def test_unrelated_draft_cannot_be_adopted(self): + """Refuse a draft lacking the matching script-management marker.""" + self.add_tag() + self.args.resume = True + self.github.release_data = {"tag_name": "v2.0.0", "prerelease": False, "body": "Manual draft", "draft": True} + with self.assertRaisesRegex(ReleaseError, "script-managed"): + self.cut() + self.assertFalse(self.github.writes) + + def test_missing_release_after_draft_creation_stops_before_upload(self): + """Refuse uploads when a draft creation attempt leaves no visible release.""" + self.add_tag() + self.args.resume = True + with patch.object(self.github, "create_draft"): + with self.assertRaisesRegex(ReleaseError, "Release is missing after draft creation"): + self.cut() + self.assertNotIn("upload", self.github.writes) + self.assertNotIn("publish", self.github.writes) + + def test_remote_tag_movement_while_waiting_prevents_publication(self): + """Recheck the remote tag after CI and reject movement before draft creation.""" + self.add_tag() + self.args.resume = True + with patch.object(self.git, "remote_tag", side_effect=[self.commit, "b" * 40]): + with self.assertRaisesRegex(ReleaseError, "changed"): + self.cut() + self.assertFalse(self.github.writes) + + def test_absent_ci_times_out_without_publication(self): + """Dispatch missing tagged CI and stop without publication when waiting expires.""" + self.add_tag() + self.args.resume = True + self.github.run = None + with patch("release.time.monotonic", side_effect=[0, 2]): + with self.assertRaisesRegex(ReleaseError, "Timed out"): + self.cut() + self.assertEqual(self.github.writes, ["dispatch"]) + + def test_interrupted_upload_resumes_without_overwrite(self): + """Retain uploaded bytes and upload only missing assets on a matching retry.""" + self.add_tag() + self.args.resume = True + self.github.fail_after_upload = 2 + with self.assertRaisesRegex(ReleaseError, "interrupted"): + self.cut() + assert self.github.release_data is not None + self.assertTrue(self.github.release_data["draft"]) + saved = dict(self.github.asset_data) + self.github.fail_after_upload = None + self.cut() + assert self.github.release_data is not None + self.assertFalse(self.github.release_data["draft"]) + for name, data in saved.items(): + self.assertEqual(self.github.asset_data[name], data) + self.assertEqual(self.github.writes.count("upload"), 5) + + def test_conflicting_asset_prevents_publication(self): + """Preserve conflicting remote bytes and leave the release as a draft.""" + self.add_tag() + self.args.resume = True + self.github.fail_after_upload = 1 + with self.assertRaises(ReleaseError): + self.cut() + name = next(iter(self.github.asset_data)) + self.github.asset_data[name] = b"wrong bytes" + self.github.fail_after_upload = None + with self.assertRaisesRegex(ReleaseError, "differs"): + self.cut() + assert self.github.release_data is not None + self.assertTrue(self.github.release_data["draft"]) + self.assertEqual(self.github.asset_data[name], b"wrong bytes") + + def test_documentation_failure_can_recover_after_publication(self): + """Retry failed documentation deployment without republishing the release.""" + self.add_tag() + self.args.resume = True + self.github.fail_docs = True + with self.assertRaisesRegex(ReleaseError, "published, but documentation failed"): + self.cut() + assert self.github.release_data is not None + self.assertFalse(self.github.release_data["draft"]) + self.github.fail_docs = False + self.cut() + self.assertEqual(self.github.writes.count("publish"), 1) + self.assertEqual(self.github.writes.count("dispatch"), 2) + + def test_published_resume_works_without_ci_runs_or_artifacts(self): + """Verify published assets after CI retention and the original notes file expire.""" + self.add_tag() + self.args.resume = True + self.cut() + before = list(self.github.writes) + self.github.run = None + self.args.notes_file = Path(self.temporary.name) / "deleted-original-release-notes.md" + self.cut() + self.assertEqual(self.github.writes, before) + + def test_published_resume_detects_corrupt_actual_assets(self): + """Reject corrupted published checksums without modifying remote release state.""" + self.add_tag() + self.args.resume = True + self.cut() + self.github.asset_data["SHA256SUMS"] = b"corrupt" + before = list(self.github.writes) + with self.assertRaisesRegex(ReleaseError, "checksums"): + self.cut() + self.assertEqual(self.github.writes, before) + + def test_resume_reuses_pending_documentation_deployment(self): + """Reuse an awaiting-approval deployment instead of dispatching another build.""" + self.add_tag() + self.args.resume = True + self.cut() + assert self.github.docs_run is not None + self.github.docs_run.update(status="waiting", conclusion=None) + before = list(self.github.writes) + with patch("release.time.monotonic", side_effect=[0, 2]): + with self.assertRaisesRegex(ReleaseError, "documentation timed out"): + self.cut() + self.assertEqual(self.github.writes, before) + + def test_new_release_cannot_downgrade_or_prerelease_a_stable_version(self): + """Require new stable and prerelease attempts to advance the published numeric core.""" + self.github.other_versions = {(2, 0, 0)} + for version in ("2.0.0", "2.0.0-rc.1"): + self.args.version = version + with self.assertRaisesRegex(ReleaseError, "advance"): + self.cut() + self.assertIsNone(self.git.remote_tag("origin", "v" + version)) + + def test_resume_old_release_does_not_roll_documentation_back(self): + """Retain the newer stable site when an older published release is resumed.""" + self.add_tag() + self.args.resume = True + self.cut() + before = list(self.github.writes) + self.github.other_versions.add((2, 1, 0)) + self.github.docs_run = None + self.assertIn("newer stable release", self.cut()) + self.assertEqual(self.github.writes, before) + with self.assertRaisesRegex(ReleaseError, "latest published"): + require_latest_stable(self.github, "v2.0.0") + + def test_unpublished_tag_cannot_deploy_stable_documentation(self): + """Require a published stable release before allowing a Pages deployment.""" + self.add_tag() + with self.assertRaisesRegex(ReleaseError, "latest published"): + require_latest_stable(self.github, "v2.0.0") + + def test_packaging_contents_reproducibility_and_corruption(self): + """Verify archive layout, licenses, reproducible bytes, and manifest validation.""" + self.add_tag() + first, second = Path(self.temporary.name) / "first", Path(self.temporary.name) / "second" + for output in (first, second): + package(self.git, self.commit, "v2.0.0", self.docs, output) + self.assertEqual(len(check_bundle(output, self.commit, "v2.0.0")), 5) + self.assertEqual({p.name: sha256(p) for p in first.iterdir()}, {p.name: sha256(p) for p in second.iterdir()}) + with tarfile.open(first / "cetl-2.0.0-headers.tar.gz") as archive: + files = {member.name for member in archive if member.isfile()} + self.assertEqual(files, {"cetl-2.0.0/LICENSE", "cetl-2.0.0/" + HEADER}) + with tarfile.open(first / "cetl-2.0.0-docs.tar.gz") as archive: + self.assertIn("cetl-2.0.0/html/index.html", archive.getnames()) + self.assertIn("cetl-2.0.0/LICENSE", archive.getnames()) + (first / "release.json").write_text("{}") + with self.assertRaisesRegex(ReleaseError, "metadata"): + check_bundle(first, self.commit, "v2.0.0") + + def test_cli_validates_tagged_checkout_and_release_event(self): + """Check release-event prerelease metadata through the actual validation CLI.""" + self.add_tag() + event = Path(self.temporary.name) / "event.json" + event.write_text(json.dumps({"release": {"tag_name": "v2.0.0", "prerelease": True}})) + env = {**os.environ, "GITHUB_EVENT_NAME": "release", "GITHUB_EVENT_PATH": str(event)} + result = subprocess.run([sys.executable, str(SCRIPTS / "release_tools.py"), "validate"], + cwd=self.root, env=env, capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("prerelease flag", result.stderr) + event.write_text(json.dumps({"release": {"tag_name": "v2.0.0", "prerelease": False}})) + result = subprocess.run([sys.executable, str(SCRIPTS / "release_tools.py"), "validate"], + cwd=self.root, env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + + + def test_bundle_checksums_and_extra_files_are_rejected(self): + """Reject a changed archive digest and unexpected files in a release bundle.""" + self.add_tag() + output = Path(self.temporary.name) / "bundle" + package(self.git, self.commit, "v2.0.0", self.docs, output) + archive = output / "cetl-2.0.0-headers.tar.gz" + original = archive.read_bytes() + archive.write_bytes(original + b"corruption") + with self.assertRaisesRegex(ReleaseError, "checksums"): + check_bundle(output, self.commit, "v2.0.0") + archive.write_bytes(original) + (output / "unexpected").write_text("extra") + with self.assertRaisesRegex(ReleaseError, "unexpected"): + check_bundle(output, self.commit, "v2.0.0") + + +if __name__ == "__main__": + unittest.main()