land: ci-sccache-fallback

.github/workflows/_e2e-suite.yml

@@ -57,6 +57,10 @@ jobs:
       - uses: dtolnay/rust-toolchain@stable
 
       - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11
+        continue-on-error: true
+      - name: Start compiler cache (fall back to uncached)
+        shell: bash
+        run: bash scripts/ci-sccache-start.sh
 
       - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
         with:
@@ -443,6 +447,10 @@ jobs:
           targets: x86_64-pc-windows-msvc
 
       - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11
+        continue-on-error: true
+      - name: Start compiler cache (fall back to uncached)
+        shell: bash
+        run: bash scripts/ci-sccache-start.sh
 
       - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
         with:
@@ -524,6 +532,10 @@ jobs:
       - uses: dtolnay/rust-toolchain@stable
 
       - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11
+        continue-on-error: true
+      - name: Start compiler cache (fall back to uncached)
+        shell: bash
+        run: bash scripts/ci-sccache-start.sh
 
       # cache-bin: false makes rust-cache safe on macOS: the old failure was
       # the cached ~/.cargo/bin shim overwriting the fresh cargo shim (which

.github/workflows/_unit-suite.yml

@@ -65,6 +65,10 @@ jobs:
           python3 scripts/check-path-deps.py --self-test
 
       - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11
+        continue-on-error: true
+      - name: Start compiler cache (fall back to uncached)
+        shell: bash
+        run: bash scripts/ci-sccache-start.sh
 
       - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
         with:
@@ -186,6 +190,10 @@ jobs:
           components: rustfmt
 
       - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11
+        continue-on-error: true
+      - name: Start compiler cache (fall back to uncached)
+        shell: bash
+        run: bash scripts/ci-sccache-start.sh
 
       - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
         with:
@@ -220,6 +228,7 @@ jobs:
 
       - name: CI workflow and compiler-cache invariant guards
         run: |
+          bash scripts/test-ci-sccache-start.sh
           python3 -m unittest discover -s scripts/lib -p test_workflow_gates.py -v
           python3 -m unittest discover -s scripts/lib -p test_ci_rustc_cache.py -v
           python3 -m unittest discover -s scripts/lib -p test_rust_gate_collection.py -v
@@ -560,6 +569,10 @@ jobs:
       - uses: dtolnay/rust-toolchain@stable
 
       - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11
+        continue-on-error: true
+      - name: Start compiler cache (fall back to uncached)
+        shell: bash
+        run: bash scripts/ci-sccache-start.sh
 
       - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
         with:
@@ -585,6 +598,10 @@ jobs:
       - uses: actions/checkout@v5
       - uses: dtolnay/rust-toolchain@stable
       - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11
+        continue-on-error: true
+      - name: Start compiler cache (fall back to uncached)
+        shell: bash
+        run: bash scripts/ci-sccache-start.sh
       - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
         with:
           shared-key: ci-storm-thin
@@ -619,6 +636,10 @@ jobs:
       - uses: dtolnay/rust-toolchain@1.92
 
       - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11
+        continue-on-error: true
+      - name: Start compiler cache (fall back to uncached)
+        shell: bash
+        run: bash scripts/ci-sccache-start.sh
 
       - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
         with:
@@ -660,6 +681,10 @@ jobs:
       - uses: dtolnay/rust-toolchain@stable
 
       - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11
+        continue-on-error: true
+      - name: Start compiler cache (fall back to uncached)
+        shell: bash
+        run: bash scripts/ci-sccache-start.sh
 
       - uses: ./.github/actions/setup-nextest
 
@@ -722,6 +747,10 @@ jobs:
       - uses: dtolnay/rust-toolchain@stable
 
       - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11
+        continue-on-error: true
+      - name: Start compiler cache (fall back to uncached)
+        shell: bash
+        run: bash scripts/ci-sccache-start.sh
 
       - uses: ./.github/actions/setup-nextest
 
@@ -787,6 +816,10 @@ jobs:
       - uses: dtolnay/rust-toolchain@stable
 
       - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11
+        continue-on-error: true
+      - name: Start compiler cache (fall back to uncached)
+        shell: bash
+        run: bash scripts/ci-sccache-start.sh
 
       - uses: ./.github/actions/setup-nextest
 
@@ -981,6 +1014,10 @@ jobs:
       - uses: dtolnay/rust-toolchain@stable
 
       - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11
+        continue-on-error: true
+      - name: Start compiler cache (fall back to uncached)
+        shell: bash
+        run: bash scripts/ci-sccache-start.sh
 
       - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
         with:
@@ -1041,6 +1078,10 @@ jobs:
       - uses: dtolnay/rust-toolchain@stable
 
       - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11
+        continue-on-error: true
+      - name: Start compiler cache (fall back to uncached)
+        shell: bash
+        run: bash scripts/ci-sccache-start.sh
 
       - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
         with:
@@ -1105,6 +1146,10 @@ jobs:
       - uses: dtolnay/rust-toolchain@stable
 
       - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11
+        continue-on-error: true
+      - name: Start compiler cache (fall back to uncached)
+        shell: bash
+        run: bash scripts/ci-sccache-start.sh
 
       - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
         with:

.github/workflows/release.yml

@@ -173,6 +173,10 @@ jobs:
       - uses: dtolnay/rust-toolchain@stable
 
       - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11
+        continue-on-error: true
+      - name: Start compiler cache (fall back to uncached)
+        shell: bash
+        run: bash scripts/ci-sccache-start.sh
 
       # cache-bin: false keeps a cached ~/.cargo/bin from overwriting the fresh
       # cargo shim, as in build-darwin-x64.
@@ -245,6 +249,10 @@ jobs:
           targets: x86_64-apple-darwin
 
       - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11
+        continue-on-error: true
+      - name: Start compiler cache (fall back to uncached)
+        shell: bash
+        run: bash scripts/ci-sccache-start.sh
 
       - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
         with:
@@ -404,6 +412,10 @@ jobs:
           targets: aarch64-pc-windows-msvc
 
       - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11
+        continue-on-error: true
+      - name: Start compiler cache (fall back to uncached)
+        shell: bash
+        run: bash scripts/ci-sccache-start.sh
 
       - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
         with:
@@ -446,6 +458,10 @@ jobs:
           targets: x86_64-pc-windows-msvc
 
       - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11
+        continue-on-error: true
+      - name: Start compiler cache (fall back to uncached)
+        shell: bash
+        run: bash scripts/ci-sccache-start.sh
 
       - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
         with:

.github/workflows/tests.yml

@@ -136,6 +136,7 @@ jobs:
         run: |
           python3 --version
           python3 scripts/test-ci-rust-quality.py
+          bash scripts/test-ci-sccache-start.sh
           python3 -m unittest discover -s scripts/lib -p test_workflow_gates.py -v
           python3 -m unittest discover -s scripts/lib -p test_ci_rustc_cache.py -v
           python3 -m unittest discover -s scripts/lib -p test_rust_gate_collection.py -v
@@ -207,6 +208,10 @@ jobs:
       - uses: dtolnay/rust-toolchain@stable
         id: rust
       - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11
+        continue-on-error: true
+      - name: Start compiler cache (fall back to uncached)
+        shell: bash
+        run: bash scripts/ci-sccache-start.sh
       # Executables include the commit's build provenance. Never restore a
       # fallback binary from another SHA or compiler, even with identical source.
       - uses: actions/cache/restore@v4
@@ -237,7 +242,8 @@ jobs:
         if: steps.release-binary.outputs.cache-hit != 'true'
         run: cargo build --profile ci-storm --locked -p agent-file-tools --bin aft
         env:
-          RUSTC_WRAPPER: ${{ github.workspace }}/scripts/ci-rustc-cache.py
+          # The measurement wrapper invokes sccache too; do not re-enable it after fallback.
+          RUSTC_WRAPPER: ${{ env.RUSTC_WRAPPER != '' && format('{0}/scripts/ci-rustc-cache.py', github.workspace) || '' }}
       - name: Verify search-quality executable can self-report
         run: target/ci-storm/aft --version
       - uses: ./.github/actions/upload-artifact-retry

benchmarks/aft-search/slice-descriptors/land-ci-sccache-fallback.json

@@ -0,0 +1,8 @@
+{
+  "targeted_mechanism": "none",
+  "kind": "harness",
+  "slice_class": "non_ranking",
+  "fixtures": [
+    "harness-goldens"
+  ]
+}

scripts/ci-sccache-start.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# sccache v0.19.0 has a file-only startup timeout (milliseconds), not an env knob.
+# Give slow runners 30 seconds per attempt without changing an explicit config.
+# https://github.com/mozilla/sccache/blob/v0.19.0/docs/Configuration.md
+if [[ -z "${SCCACHE_CONF:-}" ]]; then
+    export SCCACHE_CONF="$RUNNER_TEMP/aft-sccache.toml"
+    printf 'server_startup_timeout_ms = 30000\n' > "$SCCACHE_CONF"
+    printf 'SCCACHE_CONF=%s\n' "$SCCACHE_CONF" >> "$GITHUB_ENV"
+fi
+
+for attempt in 1 2 3; do
+    echo "Starting compiler cache (attempt $attempt/3)"
+    if sccache --start-server; then
+        exit 0
+    fi
+    if [[ "$attempt" != 3 ]]; then
+        sleep 2
+    fi
+done
+
+# An empty wrapper disables Cargo's job-level RUSTC_WRAPPER in subsequent steps.
+unset RUSTC_WRAPPER
+printf 'RUSTC_WRAPPER=\n' >> "$GITHUB_ENV"
+echo '::warning::sccache server did not start after 3 attempts; this job runs uncached.'

scripts/test-ci-sccache-start.sh

@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+mkdir -p "$repo_root/target/scratch"
+scratch=$(mktemp -d "$repo_root/target/scratch/ci-sccache.XXXXXX")
+trap 'rm -rf "$scratch"' EXIT
+mkdir -p "$scratch/bin" "$scratch/runner temp"
+
+cat > "$scratch/bin/sccache" <<'STUB'
+#!/usr/bin/env bash
+set -euo pipefail
+[[ "$*" == "--start-server" ]]
+[[ $(< "$SCCACHE_CONF") == "$SCCACHE_TEST_EXPECTED_CONF" ]]
+attempt=$(< "$SCCACHE_TEST_ATTEMPTS")
+attempt=$((attempt + 1))
+printf '%s\n' "$attempt" > "$SCCACHE_TEST_ATTEMPTS"
+if (( attempt <= SCCACHE_TEST_FAILURES )); then
+    echo 'sccache: error: Timed out waiting for server startup.' >&2
+    exit 2
+fi
+STUB
+cat > "$scratch/bin/sleep" <<'STUB'
+#!/usr/bin/env bash
+exit 0
+STUB
+chmod +x "$scratch/bin/sccache" "$scratch/bin/sleep"
+
+run_startup() {
+    local failures=$1
+    # An explicit config path stands for a runner that already chose its sccache settings.
+    local explicit_conf=${2:-}
+    export SCCACHE_TEST_EXPECTED_CONF='server_startup_timeout_ms = 30000'
+    if [[ -n "$explicit_conf" ]]; then
+        SCCACHE_TEST_EXPECTED_CONF=$(< "$explicit_conf")
+    fi
+    export SCCACHE_TEST_ATTEMPTS="$scratch/attempts"
+    export SCCACHE_TEST_FAILURES="$failures"
+    export GITHUB_ENV="$scratch/github env"
+    printf '0\n' > "$SCCACHE_TEST_ATTEMPTS"
+    : > "$GITHUB_ENV"
+    step_status=0
+    PATH="$scratch/bin:$PATH" RUSTC_WRAPPER=sccache SCCACHE_CONF="$explicit_conf" RUNNER_TEMP="$scratch/runner temp" \
+        bash "$repo_root/scripts/ci-sccache-start.sh" > "$scratch/output" 2>&1 || step_status=$?
+}
+
+apply_job_env() {
+    local name value
+    while IFS='=' read -r name value; do
+        export "$name=$value"
+    done < "$GITHUB_ENV"
+}
+
+failed_startup_runs_uncached() {
+    run_startup 99
+    [[ "$step_status" == 0 ]] || { echo "startup step exited $step_status, expected 0"; return 1; }
+    [[ $(< "$SCCACHE_TEST_ATTEMPTS") == 3 ]] || { echo 'expected exactly three startup attempts'; return 1; }
+    local RUSTC_WRAPPER=sccache
+    # GitHub applies $GITHUB_ENV before the next step; Cargo treats an empty wrapper as disabled.
+    apply_job_env
+    [[ -z "$RUSTC_WRAPPER" ]] || { echo 'next step would still use sccache'; return 1; }
+    local warnings=0 line
+    while IFS= read -r line; do
+        if [[ "$line" == ::warning::*uncached* ]]; then
+            warnings=$((warnings + 1))
+        fi
+    done < "$scratch/output"
+    [[ "$warnings" == 1 ]] || { echo "expected one uncached warning, got $warnings"; return 1; }
+}
+
+successful_startup_keeps_wrapper() {
+    run_startup 0
+    [[ "$step_status" == 0 && $(< "$SCCACHE_TEST_ATTEMPTS") == 1 ]] || return 1
+    local RUSTC_WRAPPER=sccache
+    apply_job_env
+    [[ "$RUSTC_WRAPPER" == sccache && $(< "$SCCACHE_CONF") == 'server_startup_timeout_ms = 30000' ]]
+}
+
+transient_failure_recovers() {
+    run_startup 2
+    [[ "$step_status" == 0 && $(< "$SCCACHE_TEST_ATTEMPTS") == 3 ]] || return 1
+    local RUSTC_WRAPPER=sccache
+    apply_job_env
+    [[ "$RUSTC_WRAPPER" == sccache ]]
+}
+
+explicit_config_is_kept() {
+    local custom="$scratch/custom sccache.toml"
+    printf '[cache.disk]\ndir = "/runner/cache"\n' > "$custom"
+    local before
+    before=$(< "$custom")
+    rm -f "$scratch/runner temp/aft-sccache.toml"
+    run_startup 0 "$custom"
+    [[ "$step_status" == 0 && $(< "$SCCACHE_TEST_ATTEMPTS") == 1 ]] || { echo 'startup with an explicit config failed'; return 1; }
+    [[ $(< "$custom") == "$before" ]] || { echo 'explicit config file was changed'; return 1; }
+    [[ ! -e "$scratch/runner temp/aft-sccache.toml" ]] || { echo 'a replacement config was written'; return 1; }
+    local entry
+    while IFS= read -r entry; do
+        [[ "$entry" != SCCACHE_CONF=* ]] || { echo "job env replaces the config path: $entry"; return 1; }
+    done < "$GITHUB_ENV"
+}
+
+passed=0
+failed=0
+for test in failed_startup_runs_uncached successful_startup_keeps_wrapper transient_failure_recovers explicit_config_is_kept; do
+    if [[ $# == 0 || "$1" == "$test" ]]; then
+        if "$test"; then
+            echo "PASS $test"
+            passed=$((passed + 1))
+        else
+            echo "FAIL $test"
+            failed=$((failed + 1))
+        fi
+    fi
+done
+echo "$passed passed; $failed failed"
+(( passed > 0 && failed == 0 ))