ci: cut unit-test duplication, and stop a flaky download failing the build

.github/dependabot.yml

@@ -0,0 +1,29 @@
+# sbt is not a supported ecosystem, so JVM dependencies in build.sbt still need
+# updating by hand. These cover the parts Dependabot can reach.
+
+version: 2
+updates:
+  # Action pins age out quietly: the checkout and setup-java pins currently in
+  # use target Node 20, which the runners have already deprecated.
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    groups:
+      actions:
+        patterns:
+          - "*"
+
+  - package-ecosystem: "npm"
+    directory: "/ui"
+    schedule:
+      interval: "weekly"
+    groups:
+      ui-dependencies:
+        patterns:
+          - "*"
+
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/license-check.yml

@@ -0,0 +1,52 @@
+# Dependency licences can only change when the build definition changes, so this
+# runs on the build files rather than on every pull request. It used to be a step
+# inside the unit test matrix, where it cost ~4.5 minutes per leg on every change.
+
+name: License Check
+
+on:
+  push:
+    branches: [ "main", "branch-*" ]
+    paths:
+      - build.sbt
+      - version.sbt
+      - project/**
+      - .github/workflows/license-check.yml
+  pull_request:
+    branches: [ "main", "branch-*" ]
+    paths:
+      - build.sbt
+      - version.sbt
+      - project/**
+      - .github/workflows/license-check.yml
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  license-check:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    steps:
+    - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+    - name: Set up JDK 17
+      uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4.8.0
+      with:
+        java-version: '17'
+        distribution: 'temurin'
+        # Caches ~/.sbt, ~/.ivy2/cache and ~/.cache/coursier.
+        cache: 'sbt'
+
+    - name: Cache the sbt launcher
+      uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+      with:
+        path: build/sbt-launch-*.jar
+        key: sbt-launcher-${{ hashFiles('project/build.properties') }}
+
+    - name: Run license check
+      run: build/sbt licenseCheck

.github/workflows/lint-check.yml

@@ -2,13 +2,18 @@ name: Code Format Check
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ "main", "branch-*" ]
   pull_request:
-    branches: [ "main" ]
+    branches: [ "main", "branch-*" ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
   lint:
     runs-on: ubuntu-latest
+    timeout-minutes: 15
     permissions:
       contents: read
       pull-requests: read
@@ -20,13 +25,16 @@ jobs:
         uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
         with:
           node-version: '18'
+          cache: 'yarn'
+          cache-dependency-path: ui/yarn.lock
 
       - name: Run lint check
         run: |
           (cd ui && yarn install --frozen-lockfile && yarn test:format)
 
   java-fmt:
     runs-on: ubuntu-latest
+    timeout-minutes: 15
     permissions:
       contents: read
       pull-requests: read
@@ -39,6 +47,14 @@ jobs:
         with:
           java-version: '17'
           distribution: 'temurin'
+          # Caches ~/.sbt, ~/.ivy2/cache and ~/.cache/coursier.
+          cache: 'sbt'
+
+      - name: Cache the sbt launcher
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: build/sbt-launch-*.jar
+          key: sbt-launcher-${{ hashFiles('project/build.properties') }}
 
       - name: Run Java format check
         run: |

.github/workflows/python-client-tests.yml

@@ -9,9 +9,14 @@ on:
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-latest
+    timeout-minutes: 30
 
     steps:
     - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
@@ -20,6 +25,14 @@ jobs:
       with:
         java-version: '17'
         distribution: 'temurin'
+        # Caches ~/.sbt, ~/.ivy2/cache and ~/.cache/coursier.
+        cache: 'sbt'
+
+    - name: Cache the sbt launcher
+      uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+      with:
+        path: build/sbt-launch-*.jar
+        key: sbt-launcher-${{ hashFiles('project/build.properties') }}
 
     - name: Install uv
       run: bash dev/install-uv.sh

.github/workflows/ui-tests.yml

@@ -2,19 +2,24 @@ name: UI Tests
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ "main", "branch-*" ]
     paths:
       - 'ui/**'
       - '.github/workflows/ui-tests.yml'
   pull_request:
-    branches: [ "main" ]
+    branches: [ "main", "branch-*" ]
     paths:
       - 'ui/**'
       - '.github/workflows/ui-tests.yml'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   ui-test:
     runs-on: ubuntu-latest
+    timeout-minutes: 20
     permissions:
       contents: read
       pull-requests: read

.github/workflows/unit-tests.yml

@@ -13,12 +13,80 @@ on:
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
-  test:
+  # Everything that does not depend on sparkVersion, run once.
+  #
+  # Only `spark` and `integrationTests` read sparkVersion (build.sbt), so these
+  # modules produce identical results for every Spark version. Running them
+  # inside the matrix duplicated ~7 minutes of tests, plus a ~4 minute
+  # licenseCheck, on every pull request.
+  #
+  # This list must stay in sync with root's .aggregate(...) in build.sbt, minus
+  # `spark`. A module added there and not added here is silently never tested.
+  core:
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    name: core
+
+    steps:
+    - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+    - name: Set up JDK 17
+      uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4.8.0
+      with:
+        java-version: '17'
+        distribution: 'temurin'
+        # Caches ~/.sbt, ~/.ivy2/cache and ~/.cache/coursier.
+        cache: 'sbt'
+
+    - name: Cache the sbt launcher
+      uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+      with:
+        path: build/sbt-launch-*.jar
+        key: sbt-launcher-${{ hashFiles('project/build.properties') }}
+
+    - name: Build and verify the local deployment.
+      run: build/sbt clean package publishM2
+
+    - name: Run tests with coverage
+      run: >
+        build/sbt -J-Xmx2G
+        +serverModels/jacoco
+        +client/jacoco
+        +pythonClient/jacoco
+        +server/jacoco
+        +cli/jacoco
+        +controlApi/jacoco
+        +controlModels/jacoco
+        +apiDocs/jacoco
+
+    # jacoco writes a report per module and nothing consumed them before, so a
+    # coverage run produced numbers nobody could read.
+    - name: Upload coverage reports
+      if: always()
+      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+      with:
+        name: jacoco-core
+        path: '**/jacoco/report/**'
+        if-no-files-found: warn
+        retention-days: 7
+
+  # Only the Spark-dependent modules are matrixed.
+  #
+  # The job name is load-bearing: `test (Spark 4.0.0)` and `test (Spark 4.1.0)`
+  # are required status checks on main. Renaming them blocks every pull request
+  # on checks that never report.
+  spark:
     runs-on: ubuntu-latest
+    timeout-minutes: 45
 
     name: test (Spark ${{ matrix.spark-version }})
     strategy:
+      # A flake in one leg must not cancel the other leg's results.
+      fail-fast: false
       matrix:
         spark-version: ["4.0.0", "4.1.0"]
 
@@ -29,17 +97,30 @@ jobs:
       with:
         java-version: '17'
         distribution: 'temurin'
+        # Caches ~/.sbt, ~/.ivy2/cache and ~/.cache/coursier.
+        cache: 'sbt'
+
+    - name: Cache the sbt launcher
+      uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+      with:
+        path: build/sbt-launch-*.jar
+        key: sbt-launcher-${{ hashFiles('project/build.properties') }}
 
       # 'integrationTests' is not in root project, since we avoid running it via `./build/sbt test`,
       # Here, we explicitly compile `integrationTests` to catch compilation issues early.
     - name: Compile integration tests to verify compilation and checkstyle issues.
       run: build/sbt -DsparkVersion=${{ matrix.spark-version }} integrationTests/test:compile
 
-    - name: Build and verify the local deployment.
-      run: build/sbt -DsparkVersion=${{ matrix.spark-version }} clean package publishM2
-
     - name: Run tests with coverage
-      run: build/sbt -DsparkVersion=${{ matrix.spark-version }} -J-Xmx2G +jacoco
+      run: build/sbt -DsparkVersion=${{ matrix.spark-version }} -J-Xmx2G +spark/jacoco
 
-    - name: Run license check
-      run: build/sbt -DsparkVersion=${{ matrix.spark-version }} licenseCheck
+    # jacoco writes a report per module and nothing consumed them before, so a
+    # coverage run produced numbers nobody could read.
+    - name: Upload coverage reports
+      if: always()
+      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+      with:
+        name: jacoco-spark-${{ matrix.spark-version }}
+        path: '**/jacoco/report/**'
+        if-no-files-found: warn
+        retention-days: 7

build/sbt-launch-lib.bash

@@ -58,7 +58,13 @@ acquire_sbt_jar () {
     printf 'Attempting to fetch sbt from %s\n' "${URL1}"
     JAR_DL="${JAR}.part"
     if [ $(command -v curl) ]; then
-      curl --fail --location --silent ${URL1} > "${JAR_DL}" &&\
+      # Retry: a single transient error from the artifact repository must not
+      # fail the build. --retry alone covers 5xx and timeouts; --retry-all-errors
+      # extends that to connection resets. --show-error keeps the HTTP status
+      # visible when every attempt fails.
+      curl --fail --location --silent --show-error \
+        --retry 5 --retry-all-errors --connect-timeout 10 \
+        ${URL1} > "${JAR_DL}" &&\
         mv "${JAR_DL}" "${JAR}"
     elif [ $(command -v wget) ]; then
       wget --quiet ${URL1} -O "${JAR_DL}" &&\

connectors/spark/src/test/resources/log4j2.properties

@@ -0,0 +1,18 @@
+# Test logging for the Spark connector.
+#
+# Spark logs at INFO to stderr, which sbt tags [error]. In a passing CI job that
+# produced 226,580 of 240,773 log lines, so a real failure was unfindable and the
+# log archive ran to ~49MB. Keep the framework at WARN and Unity Catalog's own
+# loggers at INFO, so test diagnostics survive.
+
+rootLogger.level = warn
+rootLogger.appenderRef.stderr.ref = stderr
+
+appender.stderr.type = Console
+appender.stderr.name = stderr
+appender.stderr.target = SYSTEM_ERR
+appender.stderr.layout.type = PatternLayout
+appender.stderr.layout.pattern = %d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
+
+logger.unitycatalog.name = io.unitycatalog
+logger.unitycatalog.level = info