diff --git a/.semaphore/live-tests-pr.yml b/.semaphore/live-tests-pr.yml new file mode 100644 index 0000000000..12effe93b8 --- /dev/null +++ b/.semaphore/live-tests-pr.yml @@ -0,0 +1,40 @@ +version: v1.0 +name: Relevant Live Tests (PR, manual) +agent: + machine: + type: s1-prod-ubuntu24-04-amd64-2 + +# A newer push to the PR cancels an in-flight scoped run (unlike nightly, which never cancels). +auto_cancel: + running: + when: "true" + +global_job_config: + prologue: + commands: + - checkout + - sem-version go $(cat .go-version) + - export PATH=$(go env GOPATH)/bin:$PATH + +# Scoped runs are a subset of the 24h nightly. Tune once you see real durations. +execution_time_limit: + hours: 8 + +blocks: + - name: "Relevant Live Tests" + task: + jobs: + - name: "Detect affected groups & run" + commands: + - . vault-sem-get-secret v1/ci/kv/apif/cli/live-testing-data + - | + set -e + GROUPS="$(BASE_REF=origin/main bash scripts/live-test-affected-groups.sh)" + echo "Affected live-test groups: $GROUPS" + if [ "$GROUPS" = "NONE" ]; then + echo "Nothing with live coverage changed; skipping live tests." + elif [ "$GROUPS" = "all" ]; then + make live-test + else + make live-test CLI_LIVE_TEST_GROUPS="$GROUPS" + fi diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 820ac353fe..aa50d2e7d6 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -9,8 +9,11 @@ auto_cancel: running: when: "branch != 'main'" +# Raised from 1h so the merge-queue live-test block below (which only runs on +# gh-readonly-queue/* branches) has room; normal PR/branch runs finish quickly and +# are unaffected -- this is a ceiling, not a fixed duration. execution_time_limit: - hours: 1 + hours: 12 blocks: - name: linux/amd64 @@ -99,6 +102,40 @@ blocks: commands: - test-results publish unit-test-report.xml integration-test-report.xml -N "windows/amd64" --ignore-missing + # Automatic pre-merge gate: GitHub merge queue builds a temporary + # gh-readonly-queue/main/* branch (main + the queued PR(s)); this block runs ONLY + # there, so the affected live tests must pass before the PR merges -- no manual + # click. On normal PR/branch runs the `run/when` skips it, so this pipeline's + # required check passes fast and the PR can enter the queue. + # Requires: merge queue enabled on main, and this pipeline's check kept required. + # If Semaphore does NOT build gh-readonly-queue branches, do NOT make it required + # (the queue would deadlock waiting for a check that never runs). + - name: "Relevant Live Tests (merge queue)" + dependencies: ["linux/amd64"] + run: + when: "branch =~ '^gh-readonly-queue/'" + task: + jobs: + - name: "Detect & run affected live tests" + execution_time_limit: + hours: 8 + commands: + - checkout + - sem-version go $(cat .go-version) + - export PATH=$(go env GOPATH)/bin:$PATH + - . vault-sem-get-secret v1/ci/kv/apif/cli/live-testing-data + - | + set -e + GROUPS="$(BASE_REF=origin/main bash scripts/live-test-affected-groups.sh)" + echo "Affected live-test groups: $GROUPS" + if [ "$GROUPS" = "NONE" ]; then + echo "Nothing with live coverage changed; skipping live tests." + elif [ "$GROUPS" = "all" ]; then + make live-test + else + make live-test CLI_LIVE_TEST_GROUPS="$GROUPS" + fi + after_pipeline: task: jobs: @@ -116,3 +153,5 @@ promotions: pipeline_file: ".semaphore/macos.yml" - name: "Run live integration tests" pipeline_file: ".semaphore/live-tests.yml" + - name: "Run relevant live tests (PR)" + pipeline_file: ".semaphore/live-tests-pr.yml" diff --git a/scripts/live-test-affected-groups.sh b/scripts/live-test-affected-groups.sh new file mode 100755 index 0000000000..bdbd26254e --- /dev/null +++ b/scripts/live-test-affected-groups.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# +# Prints the live-test groups relevant to this PR's diff, as ONE line: +# NONE nothing with live coverage changed -> skip +# all a shared change -> run everything +# kafka,flink run just these groups +# +# It maps each changed command folder (internal/) to its live-test group +# (the table below). Shared code (pkg/, cmd/, go.mod) runs everything; folders +# with no live tests run nothing. You only touch the table when a brand-new +# live-test GROUP is added. +# +# Env: BASE_REF (default origin/main), CHANGED (override the diff, for tests). + +set -euo pipefail +BASE_REF="${BASE_REF:-origin/main}" + +# internal/ -> its live-test group(s); empty = no live coverage. +group_for_dir() { + case "$1" in + kafka) echo kafka ;; + schema-registry) echo schema_registry ;; + connect) echo connect ;; + flink) echo flink ;; + iam) echo iam,core ;; # rbac(iam) + service-account(core) + login|logout) echo auth ;; + api-key|environment|organization) echo core ;; + rtce) echo rtce ;; + *) echo "" ;; + esac +} + +changed="${CHANGED:-}" +if [ -z "$changed" ]; then + git fetch -q origin "${BASE_REF#origin/}" 2>/dev/null || true + base=$(git merge-base HEAD "$BASE_REF" 2>/dev/null || echo "$BASE_REF") + changed=$(git diff --name-only "$base" HEAD 2>/dev/null || true) +fi +[ -n "${changed// }" ] || { echo NONE; exit 0; } + +groups="" +add() { + local x + for x in $(echo "$1" | tr ',' ' '); do + case ",$groups," in *",$x,"*) ;; *) groups="${groups:+$groups,}$x";; esac + done +} + +while IFS= read -r f; do + [ -n "$f" ] || continue + case "$f" in + go.mod|go.sum|pkg/*|cmd/*) echo all; exit 0 ;; # shared code -> everything + internal/*) + d=${f#internal/}; d=${d%%/*} + add "$(group_for_dir "$d")" ;; + *) : ;; # docs, mock, ci -> ignore + esac +done <