ci: download the test models from Hugging Face per feature, drop the model cache

.github/actions/fetch-models/action.yml

@@ -0,0 +1,34 @@
+# SPDX-FileCopyrightText: 2026 Bernard Ladenthin <bernard.ladenthin@gmail.com>
+#
+# SPDX-License-Identifier: MIT OR Apache-2.0
+
+# Downloads the models a job needs from Hugging Face -- the one place a job gets its models. A job
+# names the features it needs (the feature column of .github/models.csv) and gets exactly their
+# files; .github/fetch-models.sh pins each to a Hugging Face commit and checks its SHA-256, so a
+# missing or changed model fails here instead of letting a model-backed test self-skip. The token
+# makes the downloads count against the account's rate limit instead of the runner IP's shared one
+# (which answered 429); without it (fork pull requests) the download is anonymous. There is no
+# GitHub Actions cache in between any more: one 6.1 GB entry restored by about 20 jobs per run went
+# over the account's cache egress ("Egress is over the account limit") and the repository's 10 GB.
+# Needs the repository checked out.
+name: Fetch GGUF models
+description: Download the models of the named features of .github/models.csv from Hugging Face and verify their SHA-256.
+inputs:
+  features:
+    description: Space-separated features of .github/models.csv (draft, tts, ...); empty means every model.
+    required: false
+    default: ''
+  hf-token:
+    description: Hugging Face token (secrets.HF_TOKEN); empty downloads anonymously.
+    required: false
+    default: ''
+runs:
+  using: composite
+  steps:
+    - name: Download and verify the models (.github/models.csv)
+      shell: bash
+      env:
+        HF_TOKEN: ${{ inputs.hf-token }}
+        FEATURES: ${{ inputs.features }}
+      # $FEATURES unquoted on purpose: one argument per feature.
+      run: bash .github/fetch-models.sh $FEATURES

.github/buildcheck/models.py

@@ -1,24 +1,71 @@
 # SPDX-FileCopyrightText: 2026 Bernard Ladenthin <bernard.ladenthin@gmail.com>
 #
 # SPDX-License-Identifier: MIT OR Apache-2.0
-"""The GGUF file names publish.yml's `env:` names (for the smoke scripts, the Android emulator jobs
-and the integration jobs outside the llama module), checked against .github/models.csv -- the list
-the download-models job fetches and validate-models.sh requires. A name the list lacks is a model
-no job downloads: its consumer would fail or self-skip. (The llama module's tests default to the
-list itself; TestConstantsTest checks that side.)
+"""The CI model set, .github/models.csv: its rows (filename,feature,sha256,url, the format
+fetch-models.sh reads), and the names publish.yml uses from it -- the GGUF file names of its `env:`
+(for the smoke scripts, the Android emulator jobs and the integration jobs outside the llama module)
+and the features its jobs ask fetch-models for. A name or feature the list lacks is a model no job
+downloads: its consumer would fail or self-skip. (The llama module's tests default to the list
+itself; TestConstantsTest checks that side.)
 """
 
 import re
 
 ENV_MODEL = re.compile(r'^  ([A-Z0-9_]+):\s*"?([^"\s]+\.gguf)"?\s*$', re.M)
+FEATURES_INPUT = re.compile(r"^ +features: *(.*?) *$", re.M)
+FILENAME = re.compile(r"^[A-Za-z0-9._-]+\.gguf$")
+FEATURE = re.compile(r"^[a-z]+$")
+SHA256 = re.compile(r"^[0-9a-f]{64}$")
+# A file of a Hugging Face model repository at a commit: resolve/main would let the file change
+# under the test set, and the SHA-256 would then fail every job at once.
+PINNED_URL = re.compile(r"^https://huggingface\.co/[^/]+/[^/]+/resolve/[0-9a-f]{40}/(?:.+/)?([^/]+)$")
+
+
+def rows(models_csv_text):
+    return [[field.strip() for field in line.split(",")] for line in models_csv_text.splitlines()
+            if line.strip() and not line.startswith("#")]
 
 
 def filenames(models_csv_text):
-    return {line.split(",", 1)[0].strip() for line in models_csv_text.splitlines()
-            if line.strip() and not line.startswith("#")}
+    return {fields[0] for fields in rows(models_csv_text)}
+
+
+def features(models_csv_text):
+    return {fields[1] for fields in rows(models_csv_text) if len(fields) > 1}
+
+
+def check_rows(models_csv_text):
+    failures, seen = [], set()
+    for fields in rows(models_csv_text):
+        if len(fields) != 4:
+            failures.append(f"models.csv row {','.join(fields)!r} is not filename,feature,sha256,url")
+            continue
+        name, feature, sha256, url = fields
+        if not FILENAME.match(name):
+            failures.append(f"models.csv: {name!r} is no plain .gguf file name")
+        if name in seen:
+            failures.append(f"models.csv: {name} is listed twice")
+        seen.add(name)
+        if not FEATURE.match(feature):
+            failures.append(f"models.csv: {name}: feature {feature!r} is not one lower-case word")
+        if not SHA256.match(sha256):
+            failures.append(f"models.csv: {name}: {sha256!r} is no lower-case SHA-256")
+        pinned = PINNED_URL.match(url)
+        if not pinned:
+            failures.append(f"models.csv: {name}: {url} is no Hugging Face URL pinned to a commit "
+                            "(https://huggingface.co/<owner>/<repo>/resolve/<40 hex>/<file>)")
+        elif pinned.group(1) != name:
+            failures.append(f"models.csv: {name}: the URL names another file, {pinned.group(1)}")
+    return failures
 
 
 def check(models_csv_text, workflow_text):
-    listed = filenames(models_csv_text)
-    return [f"publish.yml env {name}={value} is not a filename of .github/models.csv -- no job downloads it"
-            for name, value in ENV_MODEL.findall(workflow_text) if value not in listed]
+    listed, known = filenames(models_csv_text), features(models_csv_text)
+    failures = check_rows(models_csv_text)
+    failures += [f"publish.yml env {name}={value} is not a filename of .github/models.csv -- no job downloads it"
+                 for name, value in ENV_MODEL.findall(workflow_text) if value not in listed]
+    failures += [f"publish.yml asks fetch-models for the feature {feature!r}, which no row of "
+                 ".github/models.csv has"
+                 for line in FEATURES_INPUT.findall(workflow_text) for feature in line.split()
+                 if feature not in known]
+    return failures

.github/buildcheck/tests/test_models.py

@@ -7,7 +7,11 @@
 from buildcheck import models, natives
 from buildcheck.tests.helpers import REPO
 
-CSV = "# comment\na.gguf,https://x/a\nb.gguf,https://x/b\n"
+COMMIT = "0123456789abcdef0123456789abcdef01234567"
+SHA = "ab" * 32
+CSV = ("# comment\n"
+       f"a.gguf,draft,{SHA},https://huggingface.co/o/r/resolve/{COMMIT}/a.gguf\n"
+       f"b.gguf,tts,{SHA},https://huggingface.co/o/r/resolve/{COMMIT}/sub/b.gguf\n")
 
 
 class ModelsTest(unittest.TestCase):
@@ -19,6 +23,33 @@ def test_env_names_must_be_listed(self):
                          ["publish.yml env C_MODEL_NAME=c.gguf is not a filename of .github/models.csv "
                           "-- no job downloads it"])
 
+    def test_requested_features_must_be_listed(self):
+        workflow = "      - uses: ./.github/actions/fetch-models\n        with:\n          features: draft tts\n"
+        self.assertEqual(models.check(CSV, workflow), [])
+        self.assertEqual(models.check(CSV, workflow.replace("tts", "tts vison")),
+                         ["publish.yml asks fetch-models for the feature 'vison', which no row of "
+                          ".github/models.csv has"])
+
+    def test_a_row_has_four_fields(self):
+        self.assertEqual(models.check_rows("a.gguf,https://huggingface.co/o/r/resolve/main/a.gguf\n"),
+                         ["models.csv row 'a.gguf,https://huggingface.co/o/r/resolve/main/a.gguf' is not "
+                          "filename,feature,sha256,url"])
+
+    def test_the_url_is_pinned_to_a_commit_and_names_the_file(self):
+        unpinned = CSV.replace(f"resolve/{COMMIT}/a.gguf", "resolve/main/a.gguf")
+        self.assertEqual(len(models.check_rows(unpinned)), 1)
+        self.assertIn("pinned to a commit", models.check_rows(unpinned)[0])
+        other = CSV.replace(f"{COMMIT}/a.gguf", f"{COMMIT}/c.gguf")
+        self.assertEqual(models.check_rows(other), ["models.csv: a.gguf: the URL names another file, c.gguf"])
+
+    def test_hash_feature_and_duplicates(self):
+        self.assertEqual(models.check_rows(CSV.replace(SHA, "AB" * 32, 1)),
+                         [f"models.csv: a.gguf: {'AB' * 32!r} is no lower-case SHA-256"])
+        self.assertEqual(models.check_rows(CSV.replace(",draft,", ",Draft model,")),
+                         ["models.csv: a.gguf: feature 'Draft model' is not one lower-case word"])
+        first = CSV.splitlines()[1]
+        self.assertEqual(models.check_rows(CSV + first + "\n"), ["models.csv: a.gguf is listed twice"])
+
     def test_the_repository_passes(self):
         self.assertEqual(models.check(natives.read(REPO, ".github/models.csv"),
                                       natives.read(REPO, ".github/workflows/publish.yml")), [])

.github/fetch-models.sh

@@ -0,0 +1,103 @@
+#!/bin/bash
+
+# SPDX-FileCopyrightText: 2026 Bernard Ladenthin <bernard.ladenthin@gmail.com>
+#
+# SPDX-License-Identifier: MIT OR Apache-2.0
+
+# Downloads models of the CI set (.github/models.csv) into models/ and checks each against the
+# SHA-256 of its row -- the one place CI gets a model, and the same command works locally.
+#
+# Usage: fetch-models.sh [feature ...]
+#   no argument   every model of the list
+#   feature ...   only the rows whose feature column names one of them (draft, tts, vision, ...);
+#                 a feature no row has is an error, so a typo cannot fetch nothing and pass
+#
+# Environment:
+#   HF_TOKEN     optional Hugging Face token (CI: the HF_TOKEN secret, absent on fork pull
+#                requests). With it the downloads count against that account's rate limit instead
+#                of the shared one of the runner's IP address, which answered 429 before.
+#   MODELS_DIR   target directory (default: models, relative to the working directory)
+#
+# A file that is already there with the right SHA-256 is kept, so a local models/ directory is
+# reused; one with another hash is downloaded again. Downloads land in <file>.part and are moved
+# into place only after the hash matched. Runs under Linux, macOS (shasum) and Git Bash on Windows.
+
+set -euo pipefail
+
+MODELS_CSV="$(dirname "$0")/models.csv"
+DEST="${MODELS_DIR:-models}"
+[ -f "$MODELS_CSV" ] || { echo "ERROR: model manifest not found: $MODELS_CSV" >&2; exit 1; }
+
+sha256_of() {
+    if command -v sha256sum > /dev/null 2>&1; then
+        sha256sum "$1" | cut -d' ' -f1
+    else
+        shasum -a 256 "$1" | cut -d' ' -f1
+    fi
+}
+
+# Read the whole manifest first: curl must not inherit a loop's stdin.
+NAMES=() FEATURES=() HASHES=() URLS=()
+while IFS=, read -r name feature hash url rest; do
+    name="${name%$'\r'}"
+    case "$name" in '' | \#*) continue ;; esac
+    url="${url%$'\r'}"
+    if [ -z "$url" ] || [ -n "${rest:-}" ]; then
+        echo "ERROR: $MODELS_CSV: row '$name' is not filename,feature,sha256,url" >&2
+        exit 1
+    fi
+    NAMES+=("$name") FEATURES+=("$feature") HASHES+=("$hash") URLS+=("$url")
+done < "$MODELS_CSV"
+[ "${#NAMES[@]}" -gt 0 ] || { echo "ERROR: no models parsed from $MODELS_CSV" >&2; exit 1; }
+
+for wanted in "$@"; do
+    found=no
+    for feature in "${FEATURES[@]}"; do [ "$feature" = "$wanted" ] && found=yes; done
+    [ "$found" = yes ] || { echo "ERROR: no model of $MODELS_CSV has the feature '$wanted'" >&2; exit 1; }
+done
+
+CURL_AUTH=()
+if [ -n "${HF_TOKEN:-}" ]; then
+    # The header goes through a file, so the token is not on curl's command line. curl sends a
+    # custom Authorization header only to the host it was given, not to the CDN it is redirected to.
+    HEADER_FILE="$(mktemp)"
+    trap 'rm -f "$HEADER_FILE"' EXIT
+    printf 'Authorization: Bearer %s\n' "$HF_TOKEN" > "$HEADER_FILE"
+    CURL_AUTH=(-H "@$HEADER_FILE")
+    echo "Hugging Face: with a token (HF_TOKEN)"
+else
+    echo "Hugging Face: anonymous (no HF_TOKEN; the runner's shared per-IP rate limit applies)"
+fi
+
+mkdir -p "$DEST"
+fetched=0 kept=0
+for i in "${!NAMES[@]}"; do
+    name="${NAMES[$i]}" feature="${FEATURES[$i]}" hash="${HASHES[$i]}" url="${URLS[$i]}"
+    if [ "$#" -gt 0 ]; then
+        selected=no
+        for wanted in "$@"; do [ "$feature" = "$wanted" ] && selected=yes; done
+        [ "$selected" = yes ] || continue
+    fi
+    file="$DEST/$name"
+    if [ -f "$file" ]; then
+        if [ "$(sha256_of "$file")" = "$hash" ]; then
+            echo "- $name ($feature): present, SHA-256 matches"
+            kept=$((kept + 1))
+            continue
+        fi
+        echo "- $name ($feature): present with another SHA-256, downloading again"
+    fi
+    start=$SECONDS
+    curl -L --proto =https --proto-redir =https --fail -sS --retry 5 --retry-all-errors \
+        --connect-timeout 30 ${CURL_AUTH[@]+"${CURL_AUTH[@]}"} -o "$file.part" "$url"
+    actual="$(sha256_of "$file.part")"
+    if [ "$actual" != "$hash" ]; then
+        rm -f "$file.part"
+        echo "ERROR: $name: SHA-256 $actual, the manifest says $hash ($url)" >&2
+        exit 1
+    fi
+    mv -f "$file.part" "$file"
+    echo "- $name ($feature): downloaded, $(wc -c < "$file" | tr -d ' ') bytes in $((SECONDS - start)) s, SHA-256 matches"
+    fetched=$((fetched + 1))
+done
+echo "models: $fetched downloaded, $kept already present (${*:-all features})"