ci: stabilize release/3.16 CI environment

.github/workflows/build.yml

@@ -25,7 +25,7 @@ jobs:
       fail-fast: false
       matrix:
         platform:
-          - ubuntu-latest
+          - ubuntu-24.04
         os_name:
           - linux_openresty
         test_dir:

.github/workflows/check-changelog.yml

@@ -12,7 +12,7 @@ on:
 
 jobs:
   check-changelog:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:

.github/workflows/cli.yml

@@ -25,7 +25,7 @@ jobs:
       fail-fast: false
       matrix:
         platform:
-          - ubuntu-latest
+          - ubuntu-24.04
         job_name:
           - linux_apisix_current_luarocks
           - linux_apisix_current_luarocks_in_customed_nginx

.github/workflows/close-unresponded.yml

@@ -13,7 +13,7 @@ jobs:
     permissions:
       issues: write  # for actions/stale to close stale issues
     name: Prune Unresponded
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 10
 
     steps:

.github/workflows/code-lint.yml

@@ -12,7 +12,7 @@ permissions:
 
 jobs:
   lint:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 10
     steps:
     - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -33,7 +33,7 @@ jobs:
         make lint
 
   sc-lint:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 5
     steps:
       - name: Checkout code

.github/workflows/doc-lint.yml

@@ -19,7 +19,7 @@ permissions:
 jobs:
   markdownlint:
     name: 🍇 Markdown
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 1
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -46,7 +46,7 @@ jobs:
 
   Chinse-Copywriting-lint:
     name: Chinese Copywriting
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 1
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

.github/workflows/docker-standalone.yml

@@ -27,7 +27,7 @@ permissions:
 
 jobs:
   docker-test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 30
 
     steps:

.github/workflows/kubernetes-ci.yml

@@ -25,7 +25,7 @@ jobs:
       fail-fast: false
       matrix:
         platform:
-          - ubuntu-latest
+          - ubuntu-24.04
         os_name:
           - linux_openresty
 

.github/workflows/license-checker.yml

@@ -26,7 +26,7 @@ on:
 
 jobs:
   check-license:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 3
 
     steps:

.github/workflows/link-check.yml

@@ -29,7 +29,7 @@ concurrency:
 
 jobs:
   check-links:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

.github/workflows/lint.yml

@@ -8,7 +8,7 @@ permissions:
 jobs:
   misc:
     name: misc checker
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Check out code.
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -25,7 +25,7 @@ jobs:
           bash ./utils/check-plugins-code.sh
 
   ci-eclint:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 5
 
     steps:

.github/workflows/push-dev-image-on-commit.yml

@@ -93,7 +93,7 @@ jobs:
   merge-tags:
     needs: build-test-push
     if: github.ref == 'refs/heads/master'
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Login to Docker Hub
         uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0

.github/workflows/redhat-ci.yaml

@@ -18,7 +18,7 @@ permissions:
 jobs:
   test_apisix:
     name: run ci on redhat ubi
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 90
     strategy:
       fail-fast: false

.github/workflows/semantic.yml

@@ -10,7 +10,7 @@ on:
 jobs:
   main:
     name: Validate PR title
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Check out repository code
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

.github/workflows/source-install.yml

@@ -25,7 +25,7 @@ jobs:
       fail-fast: false
       matrix:
         platform:
-          - ubuntu-latest
+          - ubuntu-24.04
         os_platform:
           - ubuntu
           - redhat

.github/workflows/stale.yml

@@ -14,7 +14,7 @@ jobs:
       issues: write  # for actions/stale to close stale issues
       pull-requests: write  # for actions/stale to close stale PRs
     name: Prune Stale
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 10
 
     steps:

.github/workflows/tars-ci.yml

@@ -25,7 +25,7 @@ jobs:
       fail-fast: false
       matrix:
         platform:
-          - ubuntu-latest
+          - ubuntu-24.04
         os_name:
           - linux_openresty
 

.github/workflows/update-labels.yml

@@ -12,7 +12,7 @@ on:
 jobs:
   issue_commented:
     if: github.event.issue && !github.event.issue.pull_request && github.event.comment.user.login == github.event.issue.user.login && contains(github.event.issue.labels.*.name, 'wait for update') && !contains(github.event.issue.labels.*.name, 'user responded')
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: update labels when user responds
         uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
@@ -33,7 +33,7 @@ jobs:
 
   pr_commented:
     if: github.event.issue && github.event.issue.pull_request && github.event.comment.user.login == github.event.issue.user.login && (contains(github.event.issue.labels.*.name, 'wait for update') || contains(github.event.issue.labels.*.name, 'discuss') || contains(github.event.issue.labels.*.name, 'need test cases')) && !contains(github.event.issue.labels.*.name, 'user responded')
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: update label when user responds
         uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
@@ -48,7 +48,7 @@ jobs:
 
   pr_review_commented:
     if: github.event.pull_request && github.event.comment.user.login == github.event.pull_request.user.login && (contains(github.event.pull_request.labels.*.name, 'wait for update') || contains(github.event.pull_request.labels.*.name, 'discuss') || contains(github.event.issue.labels.*.name, 'need test cases')) && !contains(github.event.pull_request.labels.*.name, 'user responded')
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: update label when user responds
         uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0

ci/common.sh

@@ -108,7 +108,10 @@ install_nodejs () {
     export PNPM_HOME="/pnpm"
     export PATH="$PNPM_HOME:$PATH"
     corepack enable pnpm
-    pnpm setup
+    # Pin the pnpm used outside t/, which has its own packageManager pin.
+    # pnpm 12 rejects setup under sudo, while these maintenance branches run
+    # this installer through sudo in CI. Keep the last version known to work.
+    corepack prepare pnpm@11.25.0 --activate
 }
 
 install_brotli () {

docker/debian-dev/Dockerfile

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-FROM debian:bullseye-slim AS build
+FROM debian/eol:bullseye AS build
 
 ARG ENABLE_PROXY=false
 ARG CODE_PATH
@@ -39,7 +39,7 @@ RUN set -x \
     && cp -r deps ${ENV_INST_LUADIR} \
     && make install
 
-FROM debian:bullseye-slim
+FROM debian/eol:bullseye
 
 ARG ENTRYPOINT_PATH=./docker-entrypoint.sh
 ARG INSTALL_BROTLI=./install-brotli.sh

t/cli/common.sh

@@ -39,5 +39,29 @@ exit_if_not_customed_nginx() {
     openresty -V 2>&1 | grep apisix-nginx-module || exit 0
 }
 
+# wait_for_tcp <host> <port> [timeout_secs]
+# Poll until the port accepts TCP; default timeout 10s. Bash-only (/dev/tcp, local).
+wait_for_tcp() {
+    if [ -z "${BASH_VERSION:-}" ]; then
+        echo "wait_for_tcp: requires bash" >&2
+        return 2
+    fi
+    local host="$1"
+    local port="$2"
+    local timeout="${3:-10}"
+    local deadline=$(( $(date +%s) + timeout ))
+    { set +x; } 2>/dev/null
+    while [ "$(date +%s)" -lt "$deadline" ]; do
+        if (exec 3<>/dev/tcp/"$host"/"$port") 2>/dev/null; then
+            set -x
+            return 0
+        fi
+        sleep 0.1
+    done
+    set -x
+    echo "wait_for_tcp: ${host}:${port} not accepting connections after ${timeout}s" >&2
+    return 1
+}
+
 rm logs/error.log || true # clear previous error log
 unset APISIX_PROFILE

t/cli/test_dns.sh

@@ -143,7 +143,8 @@ nginx_config:
 " > conf/config.yaml
 
 make run
-sleep 0.5
+wait_for_tcp 127.0.0.1 9180
+wait_for_tcp 127.0.0.1 9100
 admin_key=$(yq '.deployment.admin.admin_key[0].key' conf/config.yaml | sed 's/"//g')
 curl -v -k -i -m 20 -o /dev/null -s -X PUT http://127.0.0.1:9180/apisix/admin/stream_routes/1 \
     -H "X-API-KEY: $admin_key" \
@@ -158,7 +159,14 @@ curl -v -k -i -m 20 -o /dev/null -s -X PUT http://127.0.0.1:9180/apisix/admin/st
         }
     }'
 
-curl http://127.0.0.1:9100 || true
+# Fire probes over a 10s window to cover etcd->stream-worker propagation.
+deadline=$(( $(date +%s) + 10 ))
+{ set +x; } 2>/dev/null
+while [ "$(date +%s)" -lt "$deadline" ]; do
+    curl -s --connect-timeout 1 --max-time 2 http://127.0.0.1:9100 >/dev/null 2>&1 || true
+    sleep 0.2
+done
+set -x
 make stop
 sleep 0.1 # wait for logs output
 

t/cli/test_etcd_sync_event_handle.sh

@@ -89,7 +89,18 @@ etcdctl --endpoints=127.0.0.1:2379 --user=root:apache-api6-sync put /apisix/rout
 etcdctl --endpoints=127.0.0.1:2379 --user=root:apache-api6-sync auth disable
 etcdctl --endpoints=127.0.0.1:2379 user delete root
 etcdctl --endpoints=127.0.0.1:2379 role delete root
-sleep 5 # wait resync by watch
+# Wait for resync by watch. The fixed `sleep 5` was racy on slow runners — the etcd
+# auth-toggle + watch-reconnect + bulk-event-apply can take longer, so poll route /1
+# until the fault-injection plugin is applied (status 204).
+deadline=$(( $(date +%s) + 30 ))
+{ set +x; } 2>/dev/null
+while [ "$(date +%s)" -lt "$deadline" ]; do
+    if [ "$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:9080/1)" = "204" ]; then
+        break
+    fi
+    sleep 0.5
+done
+set -x
 
 # Test request
 # All but the intentionally incoming misconfigurations should be applied,

t/cli/test_prometheus_stream.sh

@@ -36,7 +36,8 @@ plugin_attr:
 " > conf/config.yaml
 
 make run
-sleep 0.5
+wait_for_tcp 127.0.0.1 9180
+wait_for_tcp 127.0.0.1 9100
 
 admin_key=$(yq '.deployment.admin.admin_key[0].key' conf/config.yaml | sed 's/"//g')
 curl -v -k -i -m 20 -o /dev/null -s -X PUT http://127.0.0.1:9180/apisix/admin/stream_routes/1 \
@@ -55,11 +56,22 @@ curl -v -k -i -m 20 -o /dev/null -s -X PUT http://127.0.0.1:9180/apisix/admin/st
         }
     }'
 
-curl http://127.0.0.1:9100 || true
-sleep 1 # wait for sync
-
-out="$(curl http://127.0.0.1:9091/apisix/prometheus/metrics)"
-if ! echo "$out" | grep "apisix_stream_connection_total{route=\"1\"} 1" > /dev/null; then
+# Retry trigger + read: route-propagation and exporter-cache populate are both async.
+# Counter can exceed 1 across retries, so match any positive integer.
+ok=0
+deadline=$(( $(date +%s) + 20 ))
+{ set +x; } 2>/dev/null
+while [ "$(date +%s)" -lt "$deadline" ]; do
+    curl -s --connect-timeout 1 --max-time 2 http://127.0.0.1:9100 >/dev/null 2>&1 || true
+    if curl -s --connect-timeout 1 --max-time 2 http://127.0.0.1:9091/apisix/prometheus/metrics \
+         | grep -qE 'apisix_stream_connection_total\{route="1"\} [1-9][0-9]*'; then
+        ok=1
+        break
+    fi
+    sleep 0.5
+done
+set -x
+if [ "$ok" -ne 1 ]; then
     echo "failed: prometheus can't work in stream subsystem"
     exit 1
 fi
@@ -83,13 +95,25 @@ plugin_attr:
 " > conf/config.yaml
 
 make run
-sleep 0.5
-
-curl http://127.0.0.1:9100 || true
-sleep 1 # wait for sync
-
-out="$(curl http://127.0.0.1:9091/apisix/prometheus/metrics)"
-if ! echo "$out" | grep "apisix_stream_connection_total{route=\"1\"} 1" > /dev/null; then
+wait_for_tcp 127.0.0.1 9100
+
+# Same retry pattern as the first block; `|| true` on the curl cmd-subst so curl
+# failure mid-loop doesn't trip `set -e`.
+ok=0
+deadline=$(( $(date +%s) + 20 ))
+out=""
+{ set +x; } 2>/dev/null
+while [ "$(date +%s)" -lt "$deadline" ]; do
+    curl -s --connect-timeout 1 --max-time 2 http://127.0.0.1:9100 >/dev/null 2>&1 || true
+    out="$(curl -s --connect-timeout 1 --max-time 2 http://127.0.0.1:9091/apisix/prometheus/metrics || true)"
+    if echo "$out" | grep -qE 'apisix_stream_connection_total\{route="1"\} [1-9][0-9]*'; then
+        ok=1
+        break
+    fi
+    sleep 0.5
+done
+set -x
+if [ "$ok" -ne 1 ]; then
     echo "failed: prometheus can't work in stream subsystem"
     exit 1
 fi