From bb295efde8eaff3e4997d15e80bb5e78e347af14 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 06:50:21 +0100 Subject: [PATCH 1/3] mason: fall back to uncached CI when sccache cannot start Start sccache explicitly with three bounded attempts in every cache-enabled job, including reusable suites and Windows release jobs. Use the documented file-only startup timeout and clear the job-wide Rust wrapper on failure. Keep the cache action's setup and post-stat failures non-fatal, and do not re-enable the measurement wrapper after fallback. Add a workflow-free stubbed startup regression test and run it in CI guards. Before the helper fix, failed_startup_runs_uncached failed with startup step exited 2, expected 0. The final Linux dry suite passes all 3 cases; deleting the GITHUB_ENV wrapper reset reddens only failed_startup_runs_uncached, and restoring the helper returns all 3 cases to green with an empty git diff. No existing test contract was rewritten. Validated all 10 workflows with actionlint 1.7.12 and both scripts with ShellCheck 0.11.0. Auxiliary Python workflow suites and root Biome lint could not start after repeated Linux capacity refusals over an hour, including one-CPU retries; Windows verification was unreachable. No release-note fragment for this CI-only change. Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- .github/workflows/_e2e-suite.yml | 12 ++++ .github/workflows/_unit-suite.yml | 45 +++++++++++++++ .github/workflows/release.yml | 16 ++++++ .github/workflows/tests.yml | 8 ++- scripts/ci-sccache-start.sh | 26 +++++++++ scripts/test-ci-sccache-start.sh | 95 +++++++++++++++++++++++++++++++ 6 files changed, 201 insertions(+), 1 deletion(-) create mode 100644 scripts/ci-sccache-start.sh create mode 100644 scripts/test-ci-sccache-start.sh diff --git a/.github/workflows/_e2e-suite.yml b/.github/workflows/_e2e-suite.yml index c61987f52b..cc1984ed77 100644 --- a/.github/workflows/_e2e-suite.yml +++ b/.github/workflows/_e2e-suite.yml @@ -57,6 +57,10 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 with: @@ -443,6 +447,10 @@ jobs: targets: x86_64-pc-windows-msvc - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 with: @@ -524,6 +532,10 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh # cache-bin: false makes rust-cache safe on macOS: the old failure was # the cached ~/.cargo/bin shim overwriting the fresh cargo shim (which diff --git a/.github/workflows/_unit-suite.yml b/.github/workflows/_unit-suite.yml index 06bd5a7c7e..cc9e6997fb 100644 --- a/.github/workflows/_unit-suite.yml +++ b/.github/workflows/_unit-suite.yml @@ -65,6 +65,10 @@ jobs: python3 scripts/check-path-deps.py --self-test - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 with: @@ -186,6 +190,10 @@ jobs: components: rustfmt - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 with: @@ -220,6 +228,7 @@ jobs: - name: CI workflow and compiler-cache invariant guards run: | + bash scripts/test-ci-sccache-start.sh python3 -m unittest discover -s scripts/lib -p test_workflow_gates.py -v python3 -m unittest discover -s scripts/lib -p test_ci_rustc_cache.py -v python3 -m unittest discover -s scripts/lib -p test_rust_gate_collection.py -v @@ -560,6 +569,10 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 with: @@ -585,6 +598,10 @@ jobs: - uses: actions/checkout@v5 - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 with: shared-key: ci-storm-thin @@ -619,6 +636,10 @@ jobs: - uses: dtolnay/rust-toolchain@1.92 - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 with: @@ -660,6 +681,10 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: ./.github/actions/setup-nextest @@ -722,6 +747,10 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: ./.github/actions/setup-nextest @@ -787,6 +816,10 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: ./.github/actions/setup-nextest @@ -981,6 +1014,10 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 with: @@ -1041,6 +1078,10 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 with: @@ -1105,6 +1146,10 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a312e48341..c00eb53e46 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -173,6 +173,10 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh # cache-bin: false keeps a cached ~/.cargo/bin from overwriting the fresh # cargo shim, as in build-darwin-x64. @@ -245,6 +249,10 @@ jobs: targets: x86_64-apple-darwin - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 with: @@ -404,6 +412,10 @@ jobs: targets: aarch64-pc-windows-msvc - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 with: @@ -446,6 +458,10 @@ jobs: targets: x86_64-pc-windows-msvc - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ff3e51f787..9431648fe6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -136,6 +136,7 @@ jobs: run: | python3 --version python3 scripts/test-ci-rust-quality.py + bash scripts/test-ci-sccache-start.sh python3 -m unittest discover -s scripts/lib -p test_workflow_gates.py -v python3 -m unittest discover -s scripts/lib -p test_ci_rustc_cache.py -v python3 -m unittest discover -s scripts/lib -p test_rust_gate_collection.py -v @@ -207,6 +208,10 @@ jobs: - uses: dtolnay/rust-toolchain@stable id: rust - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh # Executables include the commit's build provenance. Never restore a # fallback binary from another SHA or compiler, even with identical source. - uses: actions/cache/restore@v4 @@ -237,7 +242,8 @@ jobs: if: steps.release-binary.outputs.cache-hit != 'true' run: cargo build --profile ci-storm --locked -p agent-file-tools --bin aft env: - RUSTC_WRAPPER: ${{ github.workspace }}/scripts/ci-rustc-cache.py + # The measurement wrapper invokes sccache too; do not re-enable it after fallback. + RUSTC_WRAPPER: ${{ env.RUSTC_WRAPPER != '' && format('{0}/scripts/ci-rustc-cache.py', github.workspace) || '' }} - name: Verify search-quality executable can self-report run: target/ci-storm/aft --version - uses: ./.github/actions/upload-artifact-retry diff --git a/scripts/ci-sccache-start.sh b/scripts/ci-sccache-start.sh new file mode 100644 index 0000000000..987d21b416 --- /dev/null +++ b/scripts/ci-sccache-start.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +# sccache v0.19.0 has a file-only startup timeout (milliseconds), not an env knob. +# Give slow runners 30 seconds per attempt without changing an explicit config. +# https://github.com/mozilla/sccache/blob/v0.19.0/docs/Configuration.md +if [[ -z "${SCCACHE_CONF:-}" ]]; then + export SCCACHE_CONF="$RUNNER_TEMP/aft-sccache.toml" + printf 'server_startup_timeout_ms = 30000\n' > "$SCCACHE_CONF" + printf 'SCCACHE_CONF=%s\n' "$SCCACHE_CONF" >> "$GITHUB_ENV" +fi + +for attempt in 1 2 3; do + echo "Starting compiler cache (attempt $attempt/3)" + if sccache --start-server; then + exit 0 + fi + if [[ "$attempt" != 3 ]]; then + sleep 2 + fi +done + +# An empty wrapper disables Cargo's job-level RUSTC_WRAPPER in subsequent steps. +unset RUSTC_WRAPPER +printf 'RUSTC_WRAPPER=\n' >> "$GITHUB_ENV" +echo '::warning::sccache server did not start after 3 attempts; this job runs uncached.' diff --git a/scripts/test-ci-sccache-start.sh b/scripts/test-ci-sccache-start.sh new file mode 100644 index 0000000000..d0ecb7db79 --- /dev/null +++ b/scripts/test-ci-sccache-start.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +mkdir -p "$repo_root/target/scratch" +scratch=$(mktemp -d "$repo_root/target/scratch/ci-sccache.XXXXXX") +trap 'rm -rf "$scratch"' EXIT +mkdir -p "$scratch/bin" "$scratch/runner temp" + +cat > "$scratch/bin/sccache" <<'STUB' +#!/usr/bin/env bash +set -euo pipefail +[[ "$*" == "--start-server" ]] +[[ $(< "$SCCACHE_CONF") == 'server_startup_timeout_ms = 30000' ]] +attempt=$(< "$SCCACHE_TEST_ATTEMPTS") +attempt=$((attempt + 1)) +printf '%s\n' "$attempt" > "$SCCACHE_TEST_ATTEMPTS" +if (( attempt <= SCCACHE_TEST_FAILURES )); then + echo 'sccache: error: Timed out waiting for server startup.' >&2 + exit 2 +fi +STUB +cat > "$scratch/bin/sleep" <<'STUB' +#!/usr/bin/env bash +exit 0 +STUB +chmod +x "$scratch/bin/sccache" "$scratch/bin/sleep" + +run_startup() { + local failures=$1 + export SCCACHE_TEST_ATTEMPTS="$scratch/attempts" + export SCCACHE_TEST_FAILURES="$failures" + export GITHUB_ENV="$scratch/github env" + printf '0\n' > "$SCCACHE_TEST_ATTEMPTS" + : > "$GITHUB_ENV" + step_status=0 + PATH="$scratch/bin:$PATH" RUSTC_WRAPPER=sccache SCCACHE_CONF='' RUNNER_TEMP="$scratch/runner temp" \ + bash "$repo_root/scripts/ci-sccache-start.sh" > "$scratch/output" 2>&1 || step_status=$? +} + +apply_job_env() { + local name value + while IFS='=' read -r name value; do + export "$name=$value" + done < "$GITHUB_ENV" +} + +failed_startup_runs_uncached() { + run_startup 99 + [[ "$step_status" == 0 ]] || { echo "startup step exited $step_status, expected 0"; return 1; } + [[ $(< "$SCCACHE_TEST_ATTEMPTS") == 3 ]] || { echo 'expected exactly three startup attempts'; return 1; } + local RUSTC_WRAPPER=sccache + # GitHub applies $GITHUB_ENV before the next step; Cargo treats an empty wrapper as disabled. + apply_job_env + [[ -z "$RUSTC_WRAPPER" ]] || { echo 'next step would still use sccache'; return 1; } + local warnings=0 line + while IFS= read -r line; do + if [[ "$line" == ::warning::*uncached* ]]; then + warnings=$((warnings + 1)) + fi + done < "$scratch/output" + [[ "$warnings" == 1 ]] || { echo "expected one uncached warning, got $warnings"; return 1; } +} + +successful_startup_keeps_wrapper() { + run_startup 0 + [[ "$step_status" == 0 && $(< "$SCCACHE_TEST_ATTEMPTS") == 1 ]] || return 1 + local RUSTC_WRAPPER=sccache + apply_job_env + [[ "$RUSTC_WRAPPER" == sccache && $(< "$SCCACHE_CONF") == 'server_startup_timeout_ms = 30000' ]] +} + +transient_failure_recovers() { + run_startup 2 + [[ "$step_status" == 0 && $(< "$SCCACHE_TEST_ATTEMPTS") == 3 ]] || return 1 + local RUSTC_WRAPPER=sccache + apply_job_env + [[ "$RUSTC_WRAPPER" == sccache ]] +} + +passed=0 +failed=0 +for test in failed_startup_runs_uncached successful_startup_keeps_wrapper transient_failure_recovers; do + if [[ $# == 0 || "$1" == "$test" ]]; then + if "$test"; then + echo "PASS $test" + passed=$((passed + 1)) + else + echo "FAIL $test" + failed=$((failed + 1)) + fi + fi +done +echo "$passed passed; $failed failed" +(( passed > 0 && failed == 0 )) From 69a962b248899d152978985ee2005528b5784a43 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 07:40:47 +0100 Subject: [PATCH 2/3] release notes and search-quality descriptor for ci-sccache-fallback Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- .../slice-descriptors/land-ci-sccache-fallback.json | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 benchmarks/aft-search/slice-descriptors/land-ci-sccache-fallback.json diff --git a/benchmarks/aft-search/slice-descriptors/land-ci-sccache-fallback.json b/benchmarks/aft-search/slice-descriptors/land-ci-sccache-fallback.json new file mode 100644 index 0000000000..ebd574711e --- /dev/null +++ b/benchmarks/aft-search/slice-descriptors/land-ci-sccache-fallback.json @@ -0,0 +1,8 @@ +{ + "targeted_mechanism": "none", + "kind": "harness", + "slice_class": "non_ranking", + "fixtures": [ + "harness-goldens" + ] +} From efaa5bca36464d89af175c717c6923a421001346 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Mon, 26 Oct 2026 07:45:38 +0100 Subject: [PATCH 3/3] Test that an explicit sccache config is left alone Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- scripts/test-ci-sccache-start.sh | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/scripts/test-ci-sccache-start.sh b/scripts/test-ci-sccache-start.sh index d0ecb7db79..81cb02559f 100644 --- a/scripts/test-ci-sccache-start.sh +++ b/scripts/test-ci-sccache-start.sh @@ -11,7 +11,7 @@ cat > "$scratch/bin/sccache" <<'STUB' #!/usr/bin/env bash set -euo pipefail [[ "$*" == "--start-server" ]] -[[ $(< "$SCCACHE_CONF") == 'server_startup_timeout_ms = 30000' ]] +[[ $(< "$SCCACHE_CONF") == "$SCCACHE_TEST_EXPECTED_CONF" ]] attempt=$(< "$SCCACHE_TEST_ATTEMPTS") attempt=$((attempt + 1)) printf '%s\n' "$attempt" > "$SCCACHE_TEST_ATTEMPTS" @@ -28,13 +28,19 @@ chmod +x "$scratch/bin/sccache" "$scratch/bin/sleep" run_startup() { local failures=$1 + # An explicit config path stands for a runner that already chose its sccache settings. + local explicit_conf=${2:-} + export SCCACHE_TEST_EXPECTED_CONF='server_startup_timeout_ms = 30000' + if [[ -n "$explicit_conf" ]]; then + SCCACHE_TEST_EXPECTED_CONF=$(< "$explicit_conf") + fi export SCCACHE_TEST_ATTEMPTS="$scratch/attempts" export SCCACHE_TEST_FAILURES="$failures" export GITHUB_ENV="$scratch/github env" printf '0\n' > "$SCCACHE_TEST_ATTEMPTS" : > "$GITHUB_ENV" step_status=0 - PATH="$scratch/bin:$PATH" RUSTC_WRAPPER=sccache SCCACHE_CONF='' RUNNER_TEMP="$scratch/runner temp" \ + PATH="$scratch/bin:$PATH" RUSTC_WRAPPER=sccache SCCACHE_CONF="$explicit_conf" RUNNER_TEMP="$scratch/runner temp" \ bash "$repo_root/scripts/ci-sccache-start.sh" > "$scratch/output" 2>&1 || step_status=$? } @@ -78,9 +84,25 @@ transient_failure_recovers() { [[ "$RUSTC_WRAPPER" == sccache ]] } +explicit_config_is_kept() { + local custom="$scratch/custom sccache.toml" + printf '[cache.disk]\ndir = "/runner/cache"\n' > "$custom" + local before + before=$(< "$custom") + rm -f "$scratch/runner temp/aft-sccache.toml" + run_startup 0 "$custom" + [[ "$step_status" == 0 && $(< "$SCCACHE_TEST_ATTEMPTS") == 1 ]] || { echo 'startup with an explicit config failed'; return 1; } + [[ $(< "$custom") == "$before" ]] || { echo 'explicit config file was changed'; return 1; } + [[ ! -e "$scratch/runner temp/aft-sccache.toml" ]] || { echo 'a replacement config was written'; return 1; } + local entry + while IFS= read -r entry; do + [[ "$entry" != SCCACHE_CONF=* ]] || { echo "job env replaces the config path: $entry"; return 1; } + done < "$GITHUB_ENV" +} + passed=0 failed=0 -for test in failed_startup_runs_uncached successful_startup_keeps_wrapper transient_failure_recovers; do +for test in failed_startup_runs_uncached successful_startup_keeps_wrapper transient_failure_recovers explicit_config_is_kept; do if [[ $# == 0 || "$1" == "$test" ]]; then if "$test"; then echo "PASS $test"