diff --git a/.github/workflows/_e2e-suite.yml b/.github/workflows/_e2e-suite.yml index c61987f52b..cc1984ed77 100644 --- a/.github/workflows/_e2e-suite.yml +++ b/.github/workflows/_e2e-suite.yml @@ -57,6 +57,10 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 with: @@ -443,6 +447,10 @@ jobs: targets: x86_64-pc-windows-msvc - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 with: @@ -524,6 +532,10 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh # cache-bin: false makes rust-cache safe on macOS: the old failure was # the cached ~/.cargo/bin shim overwriting the fresh cargo shim (which diff --git a/.github/workflows/_unit-suite.yml b/.github/workflows/_unit-suite.yml index 06bd5a7c7e..cc9e6997fb 100644 --- a/.github/workflows/_unit-suite.yml +++ b/.github/workflows/_unit-suite.yml @@ -65,6 +65,10 @@ jobs: python3 scripts/check-path-deps.py --self-test - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 with: @@ -186,6 +190,10 @@ jobs: components: rustfmt - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 with: @@ -220,6 +228,7 @@ jobs: - name: CI workflow and compiler-cache invariant guards run: | + bash scripts/test-ci-sccache-start.sh python3 -m unittest discover -s scripts/lib -p test_workflow_gates.py -v python3 -m unittest discover -s scripts/lib -p test_ci_rustc_cache.py -v python3 -m unittest discover -s scripts/lib -p test_rust_gate_collection.py -v @@ -560,6 +569,10 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 with: @@ -585,6 +598,10 @@ jobs: - uses: actions/checkout@v5 - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 with: shared-key: ci-storm-thin @@ -619,6 +636,10 @@ jobs: - uses: dtolnay/rust-toolchain@1.92 - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 with: @@ -660,6 +681,10 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: ./.github/actions/setup-nextest @@ -722,6 +747,10 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: ./.github/actions/setup-nextest @@ -787,6 +816,10 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: ./.github/actions/setup-nextest @@ -981,6 +1014,10 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 with: @@ -1041,6 +1078,10 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 with: @@ -1105,6 +1146,10 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a312e48341..c00eb53e46 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -173,6 +173,10 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh # cache-bin: false keeps a cached ~/.cargo/bin from overwriting the fresh # cargo shim, as in build-darwin-x64. @@ -245,6 +249,10 @@ jobs: targets: x86_64-apple-darwin - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 with: @@ -404,6 +412,10 @@ jobs: targets: aarch64-pc-windows-msvc - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 with: @@ -446,6 +458,10 @@ jobs: targets: x86_64-pc-windows-msvc - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ff3e51f787..9431648fe6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -136,6 +136,7 @@ jobs: run: | python3 --version python3 scripts/test-ci-rust-quality.py + bash scripts/test-ci-sccache-start.sh python3 -m unittest discover -s scripts/lib -p test_workflow_gates.py -v python3 -m unittest discover -s scripts/lib -p test_ci_rustc_cache.py -v python3 -m unittest discover -s scripts/lib -p test_rust_gate_collection.py -v @@ -207,6 +208,10 @@ jobs: - uses: dtolnay/rust-toolchain@stable id: rust - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + - name: Start compiler cache (fall back to uncached) + shell: bash + run: bash scripts/ci-sccache-start.sh # Executables include the commit's build provenance. Never restore a # fallback binary from another SHA or compiler, even with identical source. - uses: actions/cache/restore@v4 @@ -237,7 +242,8 @@ jobs: if: steps.release-binary.outputs.cache-hit != 'true' run: cargo build --profile ci-storm --locked -p agent-file-tools --bin aft env: - RUSTC_WRAPPER: ${{ github.workspace }}/scripts/ci-rustc-cache.py + # The measurement wrapper invokes sccache too; do not re-enable it after fallback. + RUSTC_WRAPPER: ${{ env.RUSTC_WRAPPER != '' && format('{0}/scripts/ci-rustc-cache.py', github.workspace) || '' }} - name: Verify search-quality executable can self-report run: target/ci-storm/aft --version - uses: ./.github/actions/upload-artifact-retry diff --git a/benchmarks/aft-search/slice-descriptors/land-ci-sccache-fallback.json b/benchmarks/aft-search/slice-descriptors/land-ci-sccache-fallback.json new file mode 100644 index 0000000000..ebd574711e --- /dev/null +++ b/benchmarks/aft-search/slice-descriptors/land-ci-sccache-fallback.json @@ -0,0 +1,8 @@ +{ + "targeted_mechanism": "none", + "kind": "harness", + "slice_class": "non_ranking", + "fixtures": [ + "harness-goldens" + ] +} diff --git a/scripts/ci-sccache-start.sh b/scripts/ci-sccache-start.sh new file mode 100644 index 0000000000..987d21b416 --- /dev/null +++ b/scripts/ci-sccache-start.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +# sccache v0.19.0 has a file-only startup timeout (milliseconds), not an env knob. +# Give slow runners 30 seconds per attempt without changing an explicit config. +# https://github.com/mozilla/sccache/blob/v0.19.0/docs/Configuration.md +if [[ -z "${SCCACHE_CONF:-}" ]]; then + export SCCACHE_CONF="$RUNNER_TEMP/aft-sccache.toml" + printf 'server_startup_timeout_ms = 30000\n' > "$SCCACHE_CONF" + printf 'SCCACHE_CONF=%s\n' "$SCCACHE_CONF" >> "$GITHUB_ENV" +fi + +for attempt in 1 2 3; do + echo "Starting compiler cache (attempt $attempt/3)" + if sccache --start-server; then + exit 0 + fi + if [[ "$attempt" != 3 ]]; then + sleep 2 + fi +done + +# An empty wrapper disables Cargo's job-level RUSTC_WRAPPER in subsequent steps. +unset RUSTC_WRAPPER +printf 'RUSTC_WRAPPER=\n' >> "$GITHUB_ENV" +echo '::warning::sccache server did not start after 3 attempts; this job runs uncached.' diff --git a/scripts/test-ci-sccache-start.sh b/scripts/test-ci-sccache-start.sh new file mode 100644 index 0000000000..81cb02559f --- /dev/null +++ b/scripts/test-ci-sccache-start.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +mkdir -p "$repo_root/target/scratch" +scratch=$(mktemp -d "$repo_root/target/scratch/ci-sccache.XXXXXX") +trap 'rm -rf "$scratch"' EXIT +mkdir -p "$scratch/bin" "$scratch/runner temp" + +cat > "$scratch/bin/sccache" <<'STUB' +#!/usr/bin/env bash +set -euo pipefail +[[ "$*" == "--start-server" ]] +[[ $(< "$SCCACHE_CONF") == "$SCCACHE_TEST_EXPECTED_CONF" ]] +attempt=$(< "$SCCACHE_TEST_ATTEMPTS") +attempt=$((attempt + 1)) +printf '%s\n' "$attempt" > "$SCCACHE_TEST_ATTEMPTS" +if (( attempt <= SCCACHE_TEST_FAILURES )); then + echo 'sccache: error: Timed out waiting for server startup.' >&2 + exit 2 +fi +STUB +cat > "$scratch/bin/sleep" <<'STUB' +#!/usr/bin/env bash +exit 0 +STUB +chmod +x "$scratch/bin/sccache" "$scratch/bin/sleep" + +run_startup() { + local failures=$1 + # An explicit config path stands for a runner that already chose its sccache settings. + local explicit_conf=${2:-} + export SCCACHE_TEST_EXPECTED_CONF='server_startup_timeout_ms = 30000' + if [[ -n "$explicit_conf" ]]; then + SCCACHE_TEST_EXPECTED_CONF=$(< "$explicit_conf") + fi + export SCCACHE_TEST_ATTEMPTS="$scratch/attempts" + export SCCACHE_TEST_FAILURES="$failures" + export GITHUB_ENV="$scratch/github env" + printf '0\n' > "$SCCACHE_TEST_ATTEMPTS" + : > "$GITHUB_ENV" + step_status=0 + PATH="$scratch/bin:$PATH" RUSTC_WRAPPER=sccache SCCACHE_CONF="$explicit_conf" RUNNER_TEMP="$scratch/runner temp" \ + bash "$repo_root/scripts/ci-sccache-start.sh" > "$scratch/output" 2>&1 || step_status=$? +} + +apply_job_env() { + local name value + while IFS='=' read -r name value; do + export "$name=$value" + done < "$GITHUB_ENV" +} + +failed_startup_runs_uncached() { + run_startup 99 + [[ "$step_status" == 0 ]] || { echo "startup step exited $step_status, expected 0"; return 1; } + [[ $(< "$SCCACHE_TEST_ATTEMPTS") == 3 ]] || { echo 'expected exactly three startup attempts'; return 1; } + local RUSTC_WRAPPER=sccache + # GitHub applies $GITHUB_ENV before the next step; Cargo treats an empty wrapper as disabled. + apply_job_env + [[ -z "$RUSTC_WRAPPER" ]] || { echo 'next step would still use sccache'; return 1; } + local warnings=0 line + while IFS= read -r line; do + if [[ "$line" == ::warning::*uncached* ]]; then + warnings=$((warnings + 1)) + fi + done < "$scratch/output" + [[ "$warnings" == 1 ]] || { echo "expected one uncached warning, got $warnings"; return 1; } +} + +successful_startup_keeps_wrapper() { + run_startup 0 + [[ "$step_status" == 0 && $(< "$SCCACHE_TEST_ATTEMPTS") == 1 ]] || return 1 + local RUSTC_WRAPPER=sccache + apply_job_env + [[ "$RUSTC_WRAPPER" == sccache && $(< "$SCCACHE_CONF") == 'server_startup_timeout_ms = 30000' ]] +} + +transient_failure_recovers() { + run_startup 2 + [[ "$step_status" == 0 && $(< "$SCCACHE_TEST_ATTEMPTS") == 3 ]] || return 1 + local RUSTC_WRAPPER=sccache + apply_job_env + [[ "$RUSTC_WRAPPER" == sccache ]] +} + +explicit_config_is_kept() { + local custom="$scratch/custom sccache.toml" + printf '[cache.disk]\ndir = "/runner/cache"\n' > "$custom" + local before + before=$(< "$custom") + rm -f "$scratch/runner temp/aft-sccache.toml" + run_startup 0 "$custom" + [[ "$step_status" == 0 && $(< "$SCCACHE_TEST_ATTEMPTS") == 1 ]] || { echo 'startup with an explicit config failed'; return 1; } + [[ $(< "$custom") == "$before" ]] || { echo 'explicit config file was changed'; return 1; } + [[ ! -e "$scratch/runner temp/aft-sccache.toml" ]] || { echo 'a replacement config was written'; return 1; } + local entry + while IFS= read -r entry; do + [[ "$entry" != SCCACHE_CONF=* ]] || { echo "job env replaces the config path: $entry"; return 1; } + done < "$GITHUB_ENV" +} + +passed=0 +failed=0 +for test in failed_startup_runs_uncached successful_startup_keeps_wrapper transient_failure_recovers explicit_config_is_kept; do + if [[ $# == 0 || "$1" == "$test" ]]; then + if "$test"; then + echo "PASS $test" + passed=$((passed + 1)) + else + echo "FAIL $test" + failed=$((failed + 1)) + fi + fi +done +echo "$passed passed; $failed failed" +(( passed > 0 && failed == 0 ))