diff --git a/.devops/cann.Dockerfile b/.devops/cann.Dockerfile
index 36cee7bdb631..182c71cdf172 100644
--- a/.devops/cann.Dockerfile
+++ b/.devops/cann.Dockerfile
@@ -145,7 +145,7 @@ ENTRYPOINT ["/app/tools.sh"]
# ==============================================================================
FROM base AS light
-COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app
+COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app/
ENTRYPOINT [ "/app/llama-cli" ]
@@ -155,8 +155,9 @@ ENTRYPOINT [ "/app/llama-cli" ]
FROM base AS server
ENV LLAMA_ARG_HOST=0.0.0.0
+ENV LLAMA_ARG_PORT=8080
-COPY --from=build /app/full/llama /app/full/llama-server /app
+COPY --from=build /app/full/llama /app/full/llama-server /app/
HEALTHCHECK --interval=5m CMD [ "curl", "-f", "http://localhost:8080/health" ]
diff --git a/.devops/cpu.Dockerfile b/.devops/cpu.Dockerfile
index cb92343d6c07..0eba8f008e46 100644
--- a/.devops/cpu.Dockerfile
+++ b/.devops/cpu.Dockerfile
@@ -104,7 +104,7 @@ ENTRYPOINT ["/app/tools.sh"]
### Light, CLI only
FROM base AS light
-COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app
+COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app/
WORKDIR /app
@@ -114,8 +114,9 @@ ENTRYPOINT [ "/app/llama-cli" ]
FROM base AS server
ENV LLAMA_ARG_HOST=0.0.0.0
+ENV LLAMA_ARG_PORT=8080
-COPY --from=build /app/full/llama /app/full/llama-server /app
+COPY --from=build /app/full/llama /app/full/llama-server /app/
WORKDIR /app
diff --git a/.devops/cuda.Dockerfile b/.devops/cuda.Dockerfile
index c9a498d538b3..9a2d88bc02b3 100644
--- a/.devops/cuda.Dockerfile
+++ b/.devops/cuda.Dockerfile
@@ -113,7 +113,7 @@ ENTRYPOINT ["/app/tools.sh"]
### Light, CLI only
FROM base AS light
-COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app
+COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app/
WORKDIR /app
@@ -123,8 +123,9 @@ ENTRYPOINT [ "/app/llama-cli" ]
FROM base AS server
ENV LLAMA_ARG_HOST=0.0.0.0
+ENV LLAMA_ARG_PORT=8080
-COPY --from=build /app/full/llama /app/full/llama-server /app
+COPY --from=build /app/full/llama /app/full/llama-server /app/
WORKDIR /app
diff --git a/.devops/intel.Dockerfile b/.devops/intel.Dockerfile
index b4bcd94b9264..e8b7e6077045 100644
--- a/.devops/intel.Dockerfile
+++ b/.devops/intel.Dockerfile
@@ -1,4 +1,4 @@
-ARG ONEAPI_VERSION=2025.3.3-0-devel-ubuntu24.04
+ARG ONEAPI_VERSION=2026.1.1-devel-ubuntu24.04
ARG BUILD_DATE=N/A
ARG APP_VERSION=N/A
ARG APP_REVISION=N/A
@@ -19,7 +19,7 @@ RUN npm ci
COPY tools/ui/ ./
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
-FROM docker.io/intel/deep-learning-essentials:$ONEAPI_VERSION AS build
+FROM docker.io/intel/oneapi-toolkit:$ONEAPI_VERSION AS build
ARG GGML_SYCL_F16=ON
ARG LEVEL_ZERO_VERSION=1.28.2
@@ -59,7 +59,7 @@ RUN mkdir -p /app/full \
&& cp requirements.txt /app/full \
&& cp .devops/tools.sh /app/full/tools.sh
-FROM docker.io/intel/deep-learning-essentials:$ONEAPI_VERSION AS base
+FROM docker.io/intel/oneapi-toolkit:$ONEAPI_VERSION AS base
ARG BUILD_DATE=N/A
ARG APP_VERSION=N/A
@@ -141,7 +141,7 @@ ENTRYPOINT ["/app/tools.sh"]
FROM base AS light
COPY --from=build /app/lib/ /app
-COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app
+COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app/
WORKDIR /app
@@ -151,9 +151,10 @@ ENTRYPOINT [ "/app/llama-cli" ]
FROM base AS server
ENV LLAMA_ARG_HOST=0.0.0.0
+ENV LLAMA_ARG_PORT=8080
COPY --from=build /app/lib/ /app
-COPY --from=build /app/full/llama /app/full/llama-server /app
+COPY --from=build /app/full/llama /app/full/llama-server /app/
WORKDIR /app
diff --git a/.devops/musa.Dockerfile b/.devops/musa.Dockerfile
index d30a70bb364c..84969675a2df 100644
--- a/.devops/musa.Dockerfile
+++ b/.devops/musa.Dockerfile
@@ -1,10 +1,9 @@
ARG UBUNTU_VERSION=22.04
# This needs to generally match the container host's environment.
-ARG MUSA_VERSION=rc4.3.0
# Target the MUSA build image
-ARG BASE_MUSA_DEV_CONTAINER=docker.io/mthreads/musa:${MUSA_VERSION}-devel-ubuntu${UBUNTU_VERSION}-amd64
+ARG BASE_MUSA_DEV_CONTAINER=registry.mthreads.com/mcconline/musa_sdk:5.2.0-devel-ubuntu${UBUNTU_VERSION}-s5000
-ARG BASE_MUSA_RUN_CONTAINER=docker.io/mthreads/musa:${MUSA_VERSION}-runtime-ubuntu${UBUNTU_VERSION}-amd64
+ARG BASE_MUSA_RUN_CONTAINER=registry.mthreads.com/mcconline/musa_sdk:5.2.0-runtime-ubuntu${UBUNTU_VERSION}-s5000
ARG BUILD_DATE=N/A
ARG APP_VERSION=N/A
@@ -37,7 +36,10 @@ RUN apt-get update && \
python3-pip \
git \
libssl-dev \
- libgomp1
+ libgomp1 \
+ musa-mualg-5-2 \
+ musa-muthrust-5-2 \
+ libmthreads-compute
WORKDIR /app
@@ -80,13 +82,16 @@ LABEL org.opencontainers.image.created=$BUILD_DATE \
org.opencontainers.image.source=$IMAGE_SOURCE
RUN apt-get update \
- && apt-get install -y libgomp1 curl ffmpeg \
+ && apt-get install -y libgomp1 curl ffmpeg libmthreads-compute \
&& apt autoremove -y \
&& apt clean -y \
&& rm -rf /tmp/* /var/tmp/* \
&& find /var/cache/apt/archives /var/lib/apt/lists -not -name lock -type f -delete \
&& find /var/cache -type f -delete
+# The MUSA runtime image does not register its library directory
+RUN echo "/usr/local/musa/lib" > /etc/ld.so.conf.d/musa-runtime.conf && ldconfig
+
COPY --from=build /app/lib/ /app
### Full
@@ -115,7 +120,7 @@ ENTRYPOINT ["/app/tools.sh"]
### Light, CLI only
FROM base AS light
-COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app
+COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app/
WORKDIR /app
@@ -125,8 +130,9 @@ ENTRYPOINT [ "/app/llama-cli" ]
FROM base AS server
ENV LLAMA_ARG_HOST=0.0.0.0
+ENV LLAMA_ARG_PORT=8080
-COPY --from=build /app/full/llama /app/full/llama-server /app
+COPY --from=build /app/full/llama /app/full/llama-server /app/
WORKDIR /app
diff --git a/.devops/openvino.Dockerfile b/.devops/openvino.Dockerfile
index e301aa8f5c97..680ad7db6968 100644
--- a/.devops/openvino.Dockerfile
+++ b/.devops/openvino.Dockerfile
@@ -1,12 +1,12 @@
-ARG OPENVINO_VERSION_MAJOR=2026.4
-ARG OPENVINO_VERSION_FULL=2026.4.0.22959.99c81491cc3
+ARG OPENVINO_VERSION_MAJOR=2026.4.1
+ARG OPENVINO_VERSION_FULL=2026.4.1.22982.07f9c262b05
ARG UBUNTU_VERSION=24.04
# Intel GPU driver versions. https://github.com/intel/compute-runtime/releases
-ARG IGC_VERSION=v2.40.13
-ARG IGC_VERSION_FULL=2_2.40.13+22418
-ARG COMPUTE_RUNTIME_VERSION=26.31.39395.13
-ARG COMPUTE_RUNTIME_VERSION_FULL=26.31.39395.13-0
+ARG IGC_VERSION=v2.41.5
+ARG IGC_VERSION_FULL=2_2.41.5+22716
+ARG COMPUTE_RUNTIME_VERSION=26.35.39758.10
+ARG COMPUTE_RUNTIME_VERSION_FULL=26.35.39758.10-0
ARG IGDGMM_VERSION=22.10.0
# Intel NPU driver versions. https://github.com/intel/linux-npu-driver/releases
@@ -227,6 +227,7 @@ ENTRYPOINT [ "/app/llama-cli" ]
FROM base AS server
ENV LLAMA_ARG_HOST=0.0.0.0
+ENV LLAMA_ARG_PORT=8080
COPY --from=build /app/full/llama /app/full/llama-server /app/
diff --git a/.devops/rocm.Dockerfile b/.devops/rocm.Dockerfile
index 20f6ad636080..310f4f6d9485 100644
--- a/.devops/rocm.Dockerfile
+++ b/.devops/rocm.Dockerfile
@@ -126,7 +126,7 @@ ENTRYPOINT ["/app/tools.sh"]
### Light, CLI only
FROM base AS light
-COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app
+COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app/
WORKDIR /app
@@ -136,8 +136,9 @@ ENTRYPOINT [ "/app/llama-cli" ]
FROM base AS server
ENV LLAMA_ARG_HOST=0.0.0.0
+ENV LLAMA_ARG_PORT=8080
-COPY --from=build /app/full/llama /app/full/llama-server /app
+COPY --from=build /app/full/llama /app/full/llama-server /app/
WORKDIR /app
diff --git a/.devops/s390x.Dockerfile b/.devops/s390x.Dockerfile
index 94a715ff2d48..fa8be7616a33 100644
--- a/.devops/s390x.Dockerfile
+++ b/.devops/s390x.Dockerfile
@@ -133,6 +133,7 @@ ENTRYPOINT [ "/llama.cpp/bin/llama-cli" ]
FROM base AS server
ENV LLAMA_ARG_HOST=0.0.0.0
+ENV LLAMA_ARG_PORT=8080
WORKDIR /llama.cpp/bin
diff --git a/.devops/vulkan.Dockerfile b/.devops/vulkan.Dockerfile
index d3599ffb82c0..90e8693cf04c 100644
--- a/.devops/vulkan.Dockerfile
+++ b/.devops/vulkan.Dockerfile
@@ -107,7 +107,7 @@ ENTRYPOINT ["/app/tools.sh"]
### Light, CLI only
FROM base AS light
-COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app
+COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app/
WORKDIR /app
@@ -117,8 +117,9 @@ ENTRYPOINT [ "/app/llama-cli" ]
FROM base AS server
ENV LLAMA_ARG_HOST=0.0.0.0
+ENV LLAMA_ARG_PORT=8080
-COPY --from=build /app/full/llama /app/full/llama-server /app
+COPY --from=build /app/full/llama /app/full/llama-server /app/
WORKDIR /app
diff --git a/.devops/zendnn.Dockerfile b/.devops/zendnn.Dockerfile
index 8a50b3ef6a13..a6e54ca8511f 100644
--- a/.devops/zendnn.Dockerfile
+++ b/.devops/zendnn.Dockerfile
@@ -97,7 +97,7 @@ ENTRYPOINT ["/app/tools.sh"]
### Light, CLI only
FROM base AS light
-COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app
+COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app/
WORKDIR /app
@@ -107,8 +107,9 @@ ENTRYPOINT [ "/app/llama-cli" ]
FROM base AS server
ENV LLAMA_ARG_HOST=0.0.0.0
+ENV LLAMA_ARG_PORT=8080
-COPY --from=build /app/full/llama /app/full/llama-server /app
+COPY --from=build /app/full/llama /app/full/llama-server /app/
WORKDIR /app
diff --git a/.editorconfig b/.editorconfig
index 5663b8fdbfc9..1474554f8ab7 100644
--- a/.editorconfig
+++ b/.editorconfig
@@ -45,6 +45,9 @@ insert_final_newline = unset
trim_trailing_whitespace = unset
insert_final_newline = unset
+[vendor/**.patch]
+trim_trailing_whitespace = unset
+
[tools/ui/**]
indent_style = unset
indent_size = unset
diff --git a/.github/workflows/ai-issues.yml b/.github/workflows/ai-issues.yml
index c762901b5ac4..3f01ea8561a3 100644
--- a/.github/workflows/ai-issues.yml
+++ b/.github/workflows/ai-issues.yml
@@ -4,6 +4,10 @@ on:
issues:
types: [opened]
+cache-mode: none
+permissions:
+ contents: read
+
jobs:
find-related:
if: github.event.action == 'opened'
diff --git a/.github/workflows/build-3rd-party.yml b/.github/workflows/build-3rd-party.yml
index 82e53dbafb39..fe25f3909957 100644
--- a/.github/workflows/build-3rd-party.yml
+++ b/.github/workflows/build-3rd-party.yml
@@ -15,6 +15,10 @@ on:
'**/*.cpp'
]
+cache-mode: none
+permissions:
+ contents: read
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
diff --git a/.github/workflows/build-and-test-snapdragon.yml b/.github/workflows/build-and-test-snapdragon.yml
index 3e857d48e39f..d6420b65f141 100644
--- a/.github/workflows/build-and-test-snapdragon.yml
+++ b/.github/workflows/build-and-test-snapdragon.yml
@@ -23,19 +23,27 @@ on:
- 'scripts/snapdragon/**'
- 'CMakePresets.json'
+cache-mode: none
+permissions:
+ contents: read
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
jobs:
android-ndk-snapdragon:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04 # previously ubuntu-latest
container:
image: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.7'
defaults:
run:
shell: bash
+ permissions:
+ actions: write
+ contents: read
+
steps:
- name: Clone
uses: actions/checkout@v6
@@ -59,13 +67,17 @@ jobs:
path: pkg-snapdragon/llama.cpp
linux-iot-snapdragon:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04 # previously ubuntu-latest
container:
image: 'ghcr.io/snapdragon-toolchain/arm64-linux:v0.7'
defaults:
run:
shell: bash
+ permissions:
+ actions: write
+ contents: read
+
steps:
- name: Clone
uses: actions/checkout@v6
@@ -98,6 +110,10 @@ jobs:
matrix:
device: [SM8750, SM8850, QCS9075M]
+ permissions:
+ actions: read
+ contents: read
+
steps:
- name: Checkout
uses: actions/checkout@v6
diff --git a/.github/workflows/build-android.yml b/.github/workflows/build-android.yml
index 96ce85737fe6..4db7c2269044 100644
--- a/.github/workflows/build-android.yml
+++ b/.github/workflows/build-android.yml
@@ -20,6 +20,10 @@ on:
- '.github/workflows/build-android.yml'
- 'examples/llama.android/**'
+cache-mode: none
+permissions:
+ contents: read
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
@@ -33,7 +37,7 @@ env:
jobs:
default:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04 # previously ubuntu-latest
steps:
- name: Clone
@@ -49,7 +53,7 @@ jobs:
distribution: zulu
- name: Setup Android SDK
- uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 # v4.0.1
+ uses: android-actions/setup-android@be39fa834029ff78f1a44aa3bb0819b8fc2bd8fd # v4.0.4
with:
log-accepted-android-sdk-licenses: false
@@ -59,13 +63,17 @@ jobs:
./gradlew build --no-daemon
ndk:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04 # previously ubuntu-latest
container:
image: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.3'
defaults:
run:
shell: bash
+ permissions:
+ actions: write
+ contents: read
+
steps:
- name: Clone
uses: actions/checkout@v6
@@ -93,7 +101,7 @@ jobs:
path: pkg-adb/llama.cpp
arm64:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04 # previously ubuntu-latest
env:
NDK_VERSION: "29.0.14206865"
@@ -123,7 +131,7 @@ jobs:
distribution: temurin
- name: Setup Android SDK
- uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 # v4.0.1
+ uses: android-actions/setup-android@be39fa834029ff78f1a44aa3bb0819b8fc2bd8fd # v4.0.4
with:
log-accepted-android-sdk-licenses: false
diff --git a/.github/workflows/build-apple.yml b/.github/workflows/build-apple.yml
index c23f40f14642..70aefd5c8081 100644
--- a/.github/workflows/build-apple.yml
+++ b/.github/workflows/build-apple.yml
@@ -26,6 +26,10 @@ on:
'ggml/src/ggml-rpc/**'
]
+cache-mode: none
+permissions:
+ contents: read
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
@@ -33,6 +37,7 @@ concurrency:
env:
GGML_NLOOP: 3
GGML_N_THREADS: 1
+ GGML_SCHED_DEBUG_REALLOC: 1
LLAMA_ARG_LOG_COLORS: 1
LLAMA_ARG_LOG_PREFIX: 1
LLAMA_ARG_LOG_TIMESTAMPS: 1
@@ -49,7 +54,7 @@ jobs:
- name: ccache
uses: ggml-org/ccache-action@v1.2.24
with:
- key: apple-arm64
+ restore: false
save: false
- name: ccache-buckets-restore
@@ -98,7 +103,9 @@ jobs:
id: cmake_test
run: |
cd build
- ctest -L main -E "test-llama-archs" --verbose --timeout 900
+ # Metal Paravirtual devices are difficult to support -> disable
+ # ref: https://github.com/ggml-org/llama.cpp/pull/19802#issuecomment-4013704023
+ ctest -L main -E "test-llama-archs|test-llama-context" --verbose --timeout 900
macos-latest-x64:
runs-on: macos-15-intel
@@ -111,7 +118,7 @@ jobs:
- name: ccache
uses: ggml-org/ccache-action@v1.2.24
with:
- key: apple-x64
+ restore: false
save: false
- name: ccache-buckets-restore
@@ -159,6 +166,10 @@ jobs:
macos-latest-ios-xcode:
runs-on: macos-latest
+ permissions:
+ actions: write
+ contents: read
+
steps:
- name: Checkout code
uses: actions/checkout@v6
@@ -256,6 +267,10 @@ jobs:
runs-on: macos-latest
needs: macos-latest-ios-xcode
+ permissions:
+ actions: read
+ contents: read
+
strategy:
matrix:
destination: ['generic/platform=macOS', 'generic/platform=iOS', 'generic/platform=tvOS']
diff --git a/.github/workflows/build-cache.yml b/.github/workflows/build-cache.yml
index 27512a142ec5..72442ae95863 100644
--- a/.github/workflows/build-cache.yml
+++ b/.github/workflows/build-cache.yml
@@ -5,6 +5,10 @@ on:
schedule:
- cron: '0 * * * *'
+cache-mode: write
+permissions:
+ contents: read
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
@@ -40,9 +44,9 @@ jobs:
runs-on: ubuntu-24.04
env:
- # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
- OPENVINO_VERSION_MAJOR: "2026.4"
- OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"
+ # Sync versions in build-openvino.yml, ci-self-hosted-openvino.yml, release.yml, release-publish.yml, build-cache.yml, .devops/openvino.Dockerfile, docs/backend/OPENVINO.md
+ OPENVINO_VERSION_MAJOR: "2026.4.1"
+ OPENVINO_VERSION_FULL: "2026.4.1.22982.07f9c262b05"
steps:
- name: Clone
@@ -68,9 +72,9 @@ jobs:
runs-on: windows-2022
env:
- # Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
- OPENVINO_VERSION_MAJOR: "2026.4"
- OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"
+ # Sync versions in build-openvino.yml, ci-self-hosted-openvino.yml, release.yml, release-publish.yml, build-cache.yml, .devops/openvino.Dockerfile, docs/backend/OPENVINO.md
+ OPENVINO_VERSION_MAJOR: "2026.4.1"
+ OPENVINO_VERSION_FULL: "2026.4.1.22982.07f9c262b05"
steps:
- name: Clone
diff --git a/.github/workflows/build-cann.yml b/.github/workflows/build-cann.yml
index 6d76ed49992e..1e5ac982bec8 100644
--- a/.github/workflows/build-cann.yml
+++ b/.github/workflows/build-cann.yml
@@ -22,6 +22,10 @@ on:
'ggml/src/ggml-cann/**'
]
+cache-mode: none
+permissions:
+ contents: read
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
diff --git a/.github/workflows/build-cmake-pkg.yml b/.github/workflows/build-cmake-pkg.yml
index c44fba2c6953..e5be1dae1c8d 100644
--- a/.github/workflows/build-cmake-pkg.yml
+++ b/.github/workflows/build-cmake-pkg.yml
@@ -3,9 +3,13 @@ on:
workflow_dispatch:
workflow_call:
+cache-mode: none
+permissions:
+ contents: read
+
jobs:
linux:
- runs-on: [self-hosted, Linux]
+ runs-on: [self-hosted, Linux, CPU]
steps:
- uses: actions/checkout@v6
with:
diff --git a/.github/workflows/build-cpu.yml b/.github/workflows/build-cpu.yml
index ddda55f1c6b0..78d1c5f82558 100644
--- a/.github/workflows/build-cpu.yml
+++ b/.github/workflows/build-cpu.yml
@@ -30,6 +30,10 @@ on:
'**/*.cpp'
]
+cache-mode: none
+permissions:
+ contents: read
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
@@ -37,6 +41,7 @@ concurrency:
env:
GGML_NLOOP: 3
GGML_N_THREADS: 1
+ GGML_SCHED_DEBUG_REALLOC: 1
LLAMA_ARG_LOG_COLORS: 1
LLAMA_ARG_LOG_PREFIX: 1
LLAMA_ARG_LOG_TIMESTAMPS: 1
@@ -64,7 +69,7 @@ jobs:
- name: ccache
uses: ggml-org/ccache-action@v1.2.24
with:
- key: cpu-${{ matrix.os }}
+ restore: false
save: false
- name: Build Dependencies
@@ -88,7 +93,7 @@ jobs:
run: |
export PIP_BREAK_SYSTEM_PACKAGES="1"
python3 -m pip install --upgrade pip setuptools
- pip3 install ./gguf-py
+ pip3 install ./gguf-py jinja2==3.1.6
- name: ccache-buckets-restore
uses: ./.github/actions/ccache-buckets
@@ -124,7 +129,7 @@ jobs:
id: cmake_test
run: |
cd build
- ctest -L main --verbose --timeout 900
+ ctest -L 'main|python' --verbose --timeout 900
- name: Test llama2c conversion
id: llama2c_test
@@ -141,6 +146,11 @@ jobs:
name: windows / ${{ matrix.build }}
runs-on: windows-2025
+ cache-mode: write
+ permissions:
+ actions: write
+ contents: read
+
env:
OPENBLAS_VERSION: 0.3.23
SDE_VERSION: 9.33.0-2024-01-07
diff --git a/.github/workflows/build-cross.yml b/.github/workflows/build-cross.yml
index eef78b674175..cc432bd4f419 100644
--- a/.github/workflows/build-cross.yml
+++ b/.github/workflows/build-cross.yml
@@ -15,6 +15,10 @@ on:
schedule:
- cron: '0 0 * * 0'
+cache-mode: none
+permissions:
+ contents: read
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
diff --git a/.github/workflows/build-cuda-ubuntu.yml b/.github/workflows/build-cuda-ubuntu.yml
index 30029887c3d0..db97aeb15b29 100644
--- a/.github/workflows/build-cuda-ubuntu.yml
+++ b/.github/workflows/build-cuda-ubuntu.yml
@@ -24,6 +24,10 @@ on:
'ggml/src/ggml-cuda/**'
]
+cache-mode: none
+permissions:
+ contents: read
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
@@ -55,7 +59,7 @@ jobs:
- name: ccache
uses: ggml-org/ccache-action@v1.2.24
with:
- key: cuda-ubuntu-24.04-cuda
+ restore: false
save: false
- name: ccache-buckets-restore
@@ -68,7 +72,7 @@ jobs:
hf_bucket: ggml-org/cache
- name: Build with CMake
- # TODO: Remove GGML_CUDA_CUB_3DOT2 flag once CCCL 3.2 is bundled within CTK and that CTK version is used in this project
+ # TODO: Drop GGML_CUDA_CCCL_VERSION when this job uses CTK >= 13.5, which bundles CCCL >= 3.5.
run: |
cmake -S . -B build -G Ninja \
-DLLAMA_FATAL_WARNINGS=ON \
@@ -77,7 +81,7 @@ jobs:
-DCMAKE_EXE_LINKER_FLAGS=-Wl,--allow-shlib-undefined \
-DGGML_NATIVE=OFF \
-DGGML_CUDA=ON \
- -DGGML_CUDA_CUB_3DOT2=ON
+ -DGGML_CUDA_CCCL_VERSION=v3.4.3
cmake --build build
- name: ccache-buckets-save
@@ -110,7 +114,7 @@ jobs:
- name: ccache
uses: ggml-org/ccache-action@v1.2.24
with:
- key: cuda-ubuntu-22.04-hip
+ restore: false
save: false
- name: ccache-buckets-restore
@@ -145,7 +149,7 @@ jobs:
musa:
runs-on: ubuntu-22.04
- container: mthreads/musa:rc4.3.0-devel-ubuntu22.04-amd64
+ container: registry.mthreads.com/mcconline/musa_sdk:5.2.0-devel-ubuntu22.04-s5000
steps:
- name: Clone
@@ -156,12 +160,12 @@ jobs:
id: depends
run: |
apt-get update
- apt-get install -y build-essential git cmake libssl-dev jq
+ apt-get install -y build-essential git cmake libssl-dev jq python3-venv musa-mualg-5-2 musa-muthrust-5-2 libmthreads-compute
- name: ccache
uses: ggml-org/ccache-action@v1.2.24
with:
- key: cuda-ubuntu-22.04-musa
+ restore: false
save: false
- name: ccache-buckets-restore
@@ -178,8 +182,8 @@ jobs:
run: |
cmake -B build -S . \
-DGGML_MUSA=ON \
- -DMUSA_ARCHITECTURES=21
- time cmake --build build --config Release -j $(nproc)
+ -DMUSA_ARCHITECTURES=31
+ cmake --build build --config Release -j $(nproc)
- name: ccache-buckets-save
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
diff --git a/.github/workflows/build-cuda-windows.yml b/.github/workflows/build-cuda-windows.yml
index e08553e6cdfc..309f41789c40 100644
--- a/.github/workflows/build-cuda-windows.yml
+++ b/.github/workflows/build-cuda-windows.yml
@@ -7,6 +7,11 @@ name: CI (CUDA, windows)
on:
workflow_dispatch: # allows manual triggering
+cache-mode: write
+permissions:
+ actions: write
+ contents: read
+
# note: this will run in queue with the release workflow
concurrency:
group: release
@@ -31,15 +36,16 @@ jobs:
strategy:
matrix:
include:
+ # CTK >= 13.5 bundles CCCL >= 3.5; omit GGML_CUDA_CCCL_VERSION for those versions.
- cuda: '12.4'
arch: x64
- defines: '-DGGML_CUDA_CUB_3DOT2=ON'
+ defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
- cuda: '13.4'
arch: x64
- defines: ''
+ defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
- cuda: '13.4'
arch: arm64
- defines: '-DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake'
+ defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3 -DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake'
steps:
- name: Clone
diff --git a/.github/workflows/build-ibm.yml b/.github/workflows/build-ibm.yml
index 355487e97b8d..fcf90ccee9ce 100644
--- a/.github/workflows/build-ibm.yml
+++ b/.github/workflows/build-ibm.yml
@@ -19,9 +19,14 @@ on:
types: [opened, synchronize, reopened]
paths: [
'.github/workflows/build-ibm.yml',
- 'ggml/src/ggml-cpu/**'
+ 'ggml/src/ggml-cpu/**',
+ 'ggml/src/ggml-zdnn/**'
]
+cache-mode: none
+permissions:
+ contents: read
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
@@ -100,6 +105,46 @@ jobs:
wget https://huggingface.co/ggml-org/models/resolve/main/tinyllamas/stories260K-be.gguf
./bin/llama-completion -m stories260K-be.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256
+ ubuntu-26-zdnn-s390x:
+ name: ubuntu-26-zdnn-s390x
+ runs-on: ubuntu-24.04-s390x
+ container: ubuntu:26.04 # required to get GCC 15.1 and binutils 2.44
+ defaults:
+ run:
+ shell: bash
+
+ steps:
+ - name: Build Dependencies
+ id: build_depends
+ run: |
+ apt-get update
+ apt-get install -y --no-install-recommends \
+ build-essential cmake git ca-certificates \
+ libssl-dev libzdnn-dev
+
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v6
+
+ - name: Toolchain workaround (GCC 15)
+ run: |
+ apt-get install -y gcc-15 g++-15
+ echo "CC=gcc-15" >> "$GITHUB_ENV"
+ echo "CXX=g++-15" >> "$GITHUB_ENV"
+
+ - name: Build with zDNN Backend
+ id: cmake_build
+ run: |
+ cmake -B build \
+ -DLLAMA_FATAL_WARNINGS=ON \
+ -DGGML_NATIVE=OFF \
+ -DGGML_VXE=ON \
+ -DGGML_ZDNN=ON \
+ -DGGML_RPC=ON \
+ -DCMAKE_C_FLAGS="-march=arch15" \
+ -DCMAKE_CXX_FLAGS="-march=arch15"
+ time cmake --build build --config Release -j $(nproc)
+
ubuntu-24-ppc64le:
runs-on: ubuntu-24.04-ppc64le
diff --git a/.github/workflows/build-msys.yml b/.github/workflows/build-msys.yml
index 9f05a9e9475d..0dbf9850462f 100644
--- a/.github/workflows/build-msys.yml
+++ b/.github/workflows/build-msys.yml
@@ -8,6 +8,10 @@ on:
schedule:
- cron: '0 0 * * 0'
+cache-mode: none
+permissions:
+ contents: read
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
diff --git a/.github/workflows/build-opencl.yml b/.github/workflows/build-opencl.yml
index 9be2ba1eb699..ab9ed3ec2cb6 100644
--- a/.github/workflows/build-opencl.yml
+++ b/.github/workflows/build-opencl.yml
@@ -23,6 +23,11 @@ on:
'ggml/src/ggml-opencl/**'
]
+cache-mode: write
+permissions:
+ actions: write
+ contents: read
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
diff --git a/.github/workflows/build-openvino.yml b/.github/workflows/build-openvino.yml
index daa08b1bf949..f4c13b611d4d 100644
--- a/.github/workflows/build-openvino.yml
+++ b/.github/workflows/build-openvino.yml
@@ -22,6 +22,10 @@ on:
'ggml/src/ggml-openvino/**'
]
+cache-mode: none
+permissions:
+ contents: read
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
@@ -33,16 +37,16 @@ env:
LLAMA_ARG_LOG_PREFIX: 1
LLAMA_ARG_LOG_TIMESTAMPS: 1
# TODO: fix failing tests on OpenVINO backend
- CTEST_EXCLUDE: "test-llama-archs|^test-recurrent-state-|test-save-load-state"
+ CTEST_EXCLUDE: "test-llama-archs|test-llama-context"
jobs:
ubuntu-24-openvino:
runs-on: [self-hosted, Linux, Intel, OpenVINO]
env:
- # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
- OPENVINO_VERSION_MAJOR: "2026.4"
- OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"
+ # Sync versions in build-openvino.yml, ci-self-hosted-openvino.yml, release.yml, release-publish.yml, build-cache.yml, .devops/openvino.Dockerfile, docs/backend/OPENVINO.md
+ OPENVINO_VERSION_MAJOR: "2026.4.1"
+ OPENVINO_VERSION_FULL: "2026.4.1.22982.07f9c262b05"
steps:
- name: Clone
@@ -94,10 +98,15 @@ jobs:
openvino-windows-2022:
runs-on: windows-2022
+ cache-mode: write
+ permissions:
+ actions: write
+ contents: read
+
env:
- # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
- OPENVINO_VERSION_MAJOR: "2026.4"
- OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"
+ # Sync versions in build-openvino.yml, ci-self-hosted-openvino.yml, release.yml, release-publish.yml, build-cache.yml, .devops/openvino.Dockerfile, docs/backend/OPENVINO.md
+ OPENVINO_VERSION_MAJOR: "2026.4.1"
+ OPENVINO_VERSION_FULL: "2026.4.1.22982.07f9c262b05"
steps:
- name: Clone
diff --git a/.github/workflows/build-riscv.yml b/.github/workflows/build-riscv.yml
index 23a64454e9db..b6313c777bb2 100644
--- a/.github/workflows/build-riscv.yml
+++ b/.github/workflows/build-riscv.yml
@@ -22,6 +22,10 @@ on:
'ggml/src/ggml-cpu/arch/riscv/**'
]
+cache-mode: none
+permissions:
+ contents: read
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
diff --git a/.github/workflows/build-sanitize.yml b/.github/workflows/build-sanitize.yml
index 89fcff71d445..3a78c4d092a0 100644
--- a/.github/workflows/build-sanitize.yml
+++ b/.github/workflows/build-sanitize.yml
@@ -21,6 +21,10 @@ on:
'.github/workflows/build-sanitize.yml'
]
+cache-mode: none
+permissions:
+ contents: read
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml
deleted file mode 100644
index d54f71ac513e..000000000000
--- a/.github/workflows/build-self-hosted.yml
+++ /dev/null
@@ -1,539 +0,0 @@
-name: CI (self-hosted)
-
-on:
- workflow_dispatch: # allows manual triggering
- push:
- branches:
- - master
- paths: [
- '.github/workflows/build-self-hosted.yml',
- 'ci/run.sh',
- '**/CMakeLists.txt',
- '**/.cmake',
- '**/*.h',
- '**/*.hpp',
- '**/*.c',
- '**/*.cpp',
- '**/*.cu',
- '**/*.cuh',
- '**/*.swift',
- '**/*.m',
- '**/*.metal',
- '**/*.comp',
- '**/*.glsl',
- '**/*.wgsl'
- ]
-
- pull_request:
- types: [opened, synchronize, reopened]
- paths: [
- '.github/workflows/build-self-hosted.yml',
- 'ci/run.sh',
- '**/CMakeLists.txt',
- '**/.cmake',
- '**/*.h',
- '**/*.hpp',
- '**/*.c',
- '**/*.cpp',
- '**/*.cu',
- '**/*.cuh',
- '**/*.swift',
- '**/*.m',
- '**/*.metal',
- '**/*.comp',
- '**/*.glsl',
- '**/*.wgsl'
- ]
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
- cancel-in-progress: true
-
-env:
- # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
- HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
- GGML_NLOOP: 3
- GGML_N_THREADS: 1
- LLAMA_ARG_LOG_COLORS: 1
- LLAMA_ARG_LOG_PREFIX: 1
- LLAMA_ARG_LOG_TIMESTAMPS: 1
-
-jobs:
- gpu-cuda:
- runs-on: "hf-jobs-t4-small:cuda13"
-
- steps:
- - name: Clone
- id: checkout
- uses: actions/checkout@v6
-
- - name: Install dependencies
- run: |
- sudo apt update
- sudo apt install -y cmake libssl-dev time unzip wget python3 python3-venv python3-pip
-
- - name: ccache
- uses: ggml-org/ccache-action@v1.2.24
- with:
- restore: false
- save: false
-
- - name: ccache-buckets-restore
- uses: ./.github/actions/ccache-buckets
- with:
- key: self-hosted-gpu-cuda
- folder: llama.cpp
- hf_bucket: ggml-org/cache
-
- - name: Test
- id: ggml-ci
- run: |
- nvidia-smi
- GG_BUILD_CUDA=1 CUDACXX=/usr/local/cuda/bin/nvcc bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
-
- - name: ccache-buckets-save
- if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
- uses: ./.github/actions/ccache-buckets
- env:
- HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
- with:
- key: self-hosted-gpu-cuda
- folder: llama.cpp
- evict-old-files: 1d
- hf_bucket: ggml-org/cache
- save: true
-
- gpu-rocm:
- runs-on: [self-hosted, Linux, AMD]
-
- steps:
- - name: Clone
- id: checkout
- uses: actions/checkout@v6
-
- - name: Test
- id: ggml-ci
- # HIP_LAUNCH_BLOCKING=1: workaround for an async-execution correctness
- # issue on integrated RDNA3.5 (gfx1151) where batched inference returns
- # incorrect output (perplexity ~88 vs ~9.4). Serializing kernel launches
- # restores correctness. Remove once the underlying ROCm/HIP issue is fixed.
- env:
- HIP_LAUNCH_BLOCKING: "1"
- run: |
- rocminfo
- GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS=gfx1151 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
-
- gpu-vulkan-nvidia-cm:
- # runs-on: "hf-jobs-t4-small:ubuntu26_04"
- runs-on: [self-hosted, Linux, NVIDIA]
-
- steps:
- - name: Clone
- id: checkout
- uses: actions/checkout@v6
-
- # - name: Install dependencies
- # run: |
- # sudo apt update
- # sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan-dev glslc spirv-headers vulkan-tools mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 libssl-dev time unzip wget python3 python3-venv python3-pip
-
- # - name: ccache
- # uses: ggml-org/ccache-action@v1.2.24
- # with:
- # restore: false
- # save: false
-
- # - name: ccache-buckets-restore
- # uses: ./.github/actions/ccache-buckets
- # with:
- # key: self-hosted-vulkan-nvidia-cm
- # folder: llama.cpp
- # hf_bucket: ggml-org/cache
-
- - name: Test
- id: ggml-ci
- run: |
- vulkaninfo --summary
- GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
-
- # - name: ccache-buckets-save
- # if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
- # uses: ./.github/actions/ccache-buckets
- # env:
- # HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
- # with:
- # key: self-hosted-vulkan-nvidia-cm
- # folder: llama.cpp
- # evict-old-files: 1d
- # hf_bucket: ggml-org/cache
- # save: true
-
- gpu-vulkan-nvidia-cm2:
- # runs-on: "hf-jobs-t4-small:ubuntu26_04"
- runs-on: [self-hosted, Linux, NVIDIA, COOPMAT2]
-
- steps:
- - name: Clone
- id: checkout
- uses: actions/checkout@v6
-
- # - name: Install dependencies
- # run: |
- # sudo apt update
- # sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan-dev glslc spirv-headers vulkan-tools mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 libssl-dev time unzip wget python3 python3-venv python3-pip
-
- # - name: ccache
- # uses: ggml-org/ccache-action@v1.2.24
- # with:
- # restore: false
- # save: false
-
- # - name: ccache-buckets-restore
- # uses: ./.github/actions/ccache-buckets
- # with:
- # key: self-hosted-vulkan-nvidia-cm2
- # folder: llama.cpp
- # hf_bucket: ggml-org/cache
-
- - name: Test
- id: ggml-ci
- run: |
- vulkaninfo --summary
- GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
-
- # - name: ccache-buckets-save
- # if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
- # uses: ./.github/actions/ccache-buckets
- # env:
- # HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
- # with:
- # key: self-hosted-vulkan-nvidia-cm2
- # folder: llama.cpp
- # evict-old-files: 1d
- # hf_bucket: ggml-org/cache
- # save: true
-
- gpu-webgpu-nvidia:
- runs-on: "hf-jobs-t4-small:ubuntu26_04"
-
- steps:
- - name: Clone
- id: checkout
- uses: actions/checkout@v6
-
- - name: Install dependencies
- run: |
- sudo apt update
- sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan1 mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 libssl-dev time unzip wget python3 python3-venv python3-pip
-
- - name: ccache
- uses: ggml-org/ccache-action@v1.2.24
- with:
- restore: false
- save: false
-
- - name: ccache-buckets-restore
- uses: ./.github/actions/ccache-buckets
- with:
- key: self-hosted-webgpu-nvidia
- folder: llama.cpp
- hf_bucket: ggml-org/cache
-
- - name: Dawn Dependency
- id: dawn-depends
- run: |
- DAWN_VERSION="v20260908.214631"
- DAWN_OWNER="google"
- DAWN_REPO="dawn"
- DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-ubuntu-latest-Release"
- echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
- curl -L -o artifact.tar.gz \
- "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
- mkdir dawn
- tar -xvf artifact.tar.gz -C dawn --strip-components=1
-
- - name: Test
- id: ggml-ci
- run: |
- GG_BUILD_WEBGPU=1 \
- GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \
- GG_BUILD_WEBGPU_DAWN_DIR="$GITHUB_WORKSPACE/dawn/lib64/cmake/Dawn" \
- bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
-
- - name: ccache-buckets-save
- if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
- uses: ./.github/actions/ccache-buckets
- env:
- HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
- with:
- key: self-hosted-webgpu-nvidia
- folder: llama.cpp
- evict-old-files: 1d
- hf_bucket: ggml-org/cache
- save: true
-
- # TODO: provision AMX-compatible machine
- #cpu-amx:
- # runs-on: [self-hosted, Linux, CPU, AMX]
-
- # steps:
- # - name: Clone
- # id: checkout
- # uses: actions/checkout@v6
-
- # - name: Test
- # id: ggml-ci
- # run: |
- # bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
-
- # TODO: provision AMD GPU machine
- # amd-vulkan:
- # runs-on: [self-hosted, Linux, AMD]
-
- # steps:
- # - name: Clone
- # id: checkout
- # uses: actions/checkout@v6
-
- # - name: Test
- # id: ggml-ci
- # run: |
- # vulkaninfo --summary
- # GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
-
- # TODO: provision AMD GPU machine
- # amd-rocm:
- # runs-on: [self-hosted, Linux, AMD]
-
- # steps:
- # - name: Clone
- # id: checkout
- # uses: actions/checkout@v6
-
- # - name: Test
- # id: ggml-ci
- # run: |
- # amd-smi static
- # GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS="gfx1101" bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
-
- gpu-metal:
- runs-on: [self-hosted, macOS, ARM64]
-
- steps:
- - name: Clone
- id: checkout
- uses: actions/checkout@v6
-
- - name: Test
- id: ggml-ci
- run: |
- GG_BUILD_METAL=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
-
- gpu-webgpu-apple:
- runs-on: [self-hosted, macOS, ARM64]
-
- steps:
- - name: Clone
- id: checkout
- uses: actions/checkout@v6
-
- - name: Dawn Dependency
- id: dawn-depends
- run: |
- DAWN_VERSION="v20260908.214631"
- DAWN_OWNER="google"
- DAWN_REPO="dawn"
- DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-macos-latest-Release"
- echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
- curl -L -o artifact.tar.gz \
- "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
- mkdir dawn
- tar -xvf artifact.tar.gz -C dawn --strip-components=1
-
- - name: Test
- id: ggml-ci
- run: |
- GG_BUILD_WEBGPU=1 GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \
- bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
-
- gpu-vulkan-apple:
- runs-on: [self-hosted, macOS, ARM64]
-
- steps:
- - name: Clone
- id: checkout
- uses: actions/checkout@v6
-
- - name: Test
- id: ggml-ci
- run: |
- vulkaninfo --summary
- GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
-
- gpu-vulkan-intel-linux:
- runs-on: [self-hosted, Linux, Intel]
-
- steps:
- - name: Clone
- id: checkout
- uses: actions/checkout@v6
- with:
- persist-credentials: false
-
- - name: Test
- id: ggml-ci
- run: |
- vulkaninfo --summary
- GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
-
- gpu-vulkan-intel-windows:
- runs-on: [self-hosted, Windows, X64, Intel]
-
- steps:
- - name: Clone
- id: checkout
- uses: actions/checkout@v6
-
- - name: Test
- id: ggml-ci
- shell: C:\msys64\usr\bin\bash.exe --noprofile --norc -eo pipefail "{0}"
- env:
- MSYSTEM: UCRT64
- CHERE_INVOKING: 1
- PATH: C:\msys64\ucrt64\bin;C:\msys64\usr\bin;C:\Windows\System32;${{ env.PATH }}
- run: |
- vulkaninfo --summary
- # Skip python related tests with GG_BUILD_LOW_PERF=1 since Windows MSYS2 UCRT64 currently fails to create
- # a valid python environment for testing
- LLAMA_FATAL_WARNINGS=OFF GG_BUILD_NINJA=1 GG_BUILD_VULKAN=1 GG_BUILD_LOW_PERF=1 ./ci/run.sh ./results/llama.cpp ./mnt/llama.cpp
-
- gpu-openvino-low-perf:
- runs-on: [self-hosted, Linux, Intel, OpenVINO]
-
- env:
- # Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
- OPENVINO_VERSION_MAJOR: "2026.4"
- OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"
-
- steps:
- - name: Clone
- id: checkout
- uses: actions/checkout@v6
-
- - name: Setup OpenVINO Toolkit
- uses: ./.github/actions/linux-setup-openvino
- with:
- path: ./openvino_toolkit
- version_major: ${{ env.OPENVINO_VERSION_MAJOR }}
- version_full: ${{ env.OPENVINO_VERSION_FULL }}
-
- - name: Install OpenVINO dependencies
- run: |
- cd ./openvino_toolkit
- chmod +x ./install_dependencies/install_openvino_dependencies.sh
- echo "Y" | sudo -E ./install_dependencies/install_openvino_dependencies.sh
-
- - name: Test
- id: ggml-ci
- run: |
- source ./openvino_toolkit/setupvars.sh
- GG_BUILD_OPENVINO=1 GGML_OPENVINO_DEVICE=GPU GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
-
- cpu-x64-high-perf:
- runs-on: [self-hosted, Linux, X64]
-
- steps:
- - name: Clone
- id: checkout
- uses: actions/checkout@v6
-
- - name: Test
- id: ggml-ci
- run: |
- LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
-
- cpu-arm64-high-perf-graviton4:
- runs-on: ah-ubuntu_24_04-c8g_8x
-
- steps:
- - name: Clone
- id: checkout
- uses: actions/checkout@v6
-
- - name: Dependencies
- id: depends
- run: |
- set -euxo pipefail
- sudo apt-get update
- sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a \
- apt-get install -y \
- build-essential \
- python3-venv \
- gpg \
- wget \
- time \
- git-lfs
-
- git lfs install
-
- # install the latest cmake
- sudo install -d /usr/share/keyrings
- wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc \
- | gpg --dearmor \
- | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
- echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' \
- | sudo tee /etc/apt/sources.list.d/kitware.list
- sudo apt-get update
- sudo apt-get install -y cmake
-
- - name: Test
- id: ggml-ci
- run: |
- LLAMA_ARG_THREADS=$(nproc) \
- GG_BUILD_HIGH_PERF=1 \
- GG_BUILD_NO_BF16=1 \
- GG_BUILD_EXTRA_TESTS_0=1 \
- bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
-
- cpu-arm64-graviton4-kleidiai:
- runs-on: ah-ubuntu_24_04-c8g_8x
-
- steps:
- - name: Clone
- id: checkout
- uses: actions/checkout@v6
-
- - name: Dependencies
- id: depends
- run: |
- set -euxo pipefail
- sudo apt-get update
- sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a \
- apt-get install -y \
- build-essential \
- python3-venv \
- gpg \
- wget \
- time \
- git-lfs
-
- git lfs install
-
- # install the latest cmake
- sudo install -d /usr/share/keyrings
- wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc \
- | gpg --dearmor \
- | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
- echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' \
- | sudo tee /etc/apt/sources.list.d/kitware.list
- sudo apt-get update
- sudo apt-get install -y cmake
-
- - name: Test
- id: ggml-ci
- run: |
- LLAMA_ARG_THREADS=$(nproc) \
- GG_BUILD_KLEIDIAI=1 \
- GG_BUILD_EXTRA_TESTS_0=1 \
- GG_BUILD_HIGH_PERF=1 \
- bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
diff --git a/.github/workflows/build-sycl.yml b/.github/workflows/build-sycl.yml
index 9ddb894f2730..741c9ce77c1c 100644
--- a/.github/workflows/build-sycl.yml
+++ b/.github/workflows/build-sycl.yml
@@ -22,6 +22,10 @@ on:
'ggml/src/ggml-sycl/**'
]
+cache-mode: none
+permissions:
+ contents: read
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
@@ -48,8 +52,8 @@ jobs:
env:
ONEAPI_ROOT: /opt/intel/oneapi/
- ONEAPI_INSTALLER_VERSION: "2025.3.3"
- LEVEL_ZERO_VERSION: "1.28.2"
+ ONEAPI_INSTALLER_VERSION: "2026.1"
+ LEVEL_ZERO_VERSION: "1.33.1"
LEVEL_ZERO_UBUNTU_VERSION: "u24.04"
continue-on-error: true
@@ -63,21 +67,22 @@ jobs:
shell: bash
run: |
cd /tmp
- wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/56f7923a-adb8-43f3-8b02-2b60fcac8cab/intel-deep-learning-essentials-2025.3.3.16_offline.sh -O intel-deep-learning-essentials_offline.sh
- sudo bash intel-deep-learning-essentials_offline.sh -s -a --silent --eula accept
+ wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/5996e26b-f48a-42b1-8db0-b002ad0bd8d7/intel-oneapi-toolkit-2026.1.1.33_offline.sh -O intel-oneapi-toolkit_offline.sh
+ sudo bash intel-oneapi-toolkit_offline.sh -s -a --silent --eula accept
- name: Install Level Zero SDK
shell: bash
run: |
cd /tmp
- wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero.deb
- wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero-devel_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero-devel.deb
- sudo apt-get install -y ./level-zero.deb ./level-zero-devel.deb
+ # v1.33.x renamed the Debian packages to libze1 / libze-dev
+ wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/libze1_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O libze1.deb
+ wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/libze-dev_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O libze-dev.deb
+ sudo apt-get install -y ./libze1.deb ./libze-dev.deb
- name: ccache
uses: ggml-org/ccache-action@v1.2.24
with:
- key: sycl-ubuntu-24-${{ matrix.build }}
+ restore: false
save: false
- name: ccache-buckets-restore
@@ -101,7 +106,11 @@ jobs:
-DCMAKE_CXX_COMPILER=icpx \
-DLLAMA_OPENSSL=OFF \
-DGGML_NATIVE=OFF \
- -DGGML_SYCL_F16=${{ matrix.fp16 }}
+ -DGGML_SYCL_F16=${{ matrix.fp16 }} \
+ -DGGML_SYCL_SUPPORT_LEVEL_ZERO_API=ON \
+ -DGGML_SYCL_DNN=ON \
+ -DCMAKE_CXX_FLAGS="-fsycl-unnamed-lambda" \
+ -DCMAKE_EXE_LINKER_FLAGS="-fsycl-unnamed-lambda"
time cmake --build build --config Release -j $(nproc)
- name: ccache-buckets-save
@@ -119,16 +128,21 @@ jobs:
windows-latest-sycl:
runs-on: windows-2022
+ cache-mode: write
+ permissions:
+ actions: write
+ contents: read
+
defaults:
run:
shell: bash
env:
- WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b60765d1-2b85-4e85-86b6-cb0e9563a699/intel-deep-learning-essentials-2025.3.3.18_offline.exe
+ WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/0cb67a0d-67f6-410b-868b-f4a0a17ff0cf/intel-oneapi-toolkit-2026.1.1.32_offline.exe
WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.dnnl:intel.oneapi.win.tbb.devel
- LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.28.2/level-zero-win-sdk-1.28.2.zip
+ LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.33.1/level-zero-win-sdk-1.33.1.zip
ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI"
- ONEAPI_INSTALLER_VERSION: "2025.3.3"
+ ONEAPI_INSTALLER_VERSION: "2026.1"
steps:
- name: Clone
id: checkout
diff --git a/.github/workflows/build-virtgpu.yml b/.github/workflows/build-virtgpu.yml
index 5b740590d6b8..57686578304f 100644
--- a/.github/workflows/build-virtgpu.yml
+++ b/.github/workflows/build-virtgpu.yml
@@ -22,6 +22,10 @@ on:
'ggml/src/ggml-virtgpu/**'
]
+cache-mode: none
+permissions:
+ contents: read
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
diff --git a/.github/workflows/build-vulkan.yml b/.github/workflows/build-vulkan.yml
index 21d2a773531f..c4c08621f958 100644
--- a/.github/workflows/build-vulkan.yml
+++ b/.github/workflows/build-vulkan.yml
@@ -24,6 +24,10 @@ on:
'ggml/src/ggml-vulkan/**'
]
+cache-mode: none
+permissions:
+ contents: read
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
@@ -31,6 +35,7 @@ concurrency:
env:
GGML_NLOOP: 3
GGML_N_THREADS: 1
+ GGML_SCHED_DEBUG_REALLOC: 1
LLAMA_ARG_LOG_COLORS: 1
LLAMA_ARG_LOG_PREFIX: 1
LLAMA_ARG_LOG_TIMESTAMPS: 1
@@ -55,7 +60,7 @@ jobs:
- name: ccache
uses: ggml-org/ccache-action@v1.2.24
with:
- key: vulkan-ubuntu-24.04-arm
+ restore: false
variant: ccache
save: false
@@ -124,7 +129,7 @@ jobs:
- name: ccache
uses: ggml-org/ccache-action@v1.2.24
with:
- key: vulkan-ubuntu-24.04-llvmpipe
+ restore: false
save: false
- name: ccache-buckets-restore
@@ -167,8 +172,20 @@ jobs:
ctest -L main --verbose --timeout 900
windows:
+ name: windows / ${{ matrix.arch }}
runs-on: windows-2025
+ cache-mode: write
+ permissions:
+ actions: write
+ contents: read
+
+ strategy:
+ matrix:
+ include:
+ - arch: 'x64'
+ - arch: 'arm64'
+
env:
VULKAN_VERSION: 1.4.357.0
@@ -180,7 +197,7 @@ jobs:
- name: ccache
uses: ggml-org/ccache-action@v1.2.24
with:
- key: cpu-windows-2025-x64-vulkan
+ key: cpu-windows-2025-${{ matrix.arch }}-vulkan
variant: ccache
evict-old-files: 1d
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
@@ -189,7 +206,7 @@ jobs:
id: get_vulkan
run: |
curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/vulkansdk-windows-X64-${env:VULKAN_VERSION}.exe"
- & "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install
+ & "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install ${{ matrix.arch == 'arm64' && 'com.lunarg.vulkan.arm64' || '' }}
Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}"
Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin"
@@ -202,18 +219,19 @@ jobs:
id: cmake_build
run: |
cmake -S . -B build -G "Ninja Multi-Config" `
- -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake `
+ -D CMAKE_TOOLCHAIN_FILE=cmake/${{ matrix.arch }}-windows-llvm.cmake `
-DCMAKE_BUILD_TYPE=Release `
-DGGML_NATIVE=OFF `
-DLLAMA_BUILD_SERVER=ON `
-DGGML_RPC=ON `
-DGGML_BACKEND_DL=ON `
- -DGGML_CPU_ALL_VARIANTS=ON `
+ -DGGML_CPU_ALL_VARIANTS=${{ matrix.arch == 'x64' && 'ON' || 'OFF' }} `
-DGGML_VULKAN=ON `
-DLLAMA_BUILD_BORINGSSL=ON
cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS}
- name: Test
+ if: ${{ matrix.arch == 'x64' }}
id: cmake_test
run: |
cd build
@@ -224,7 +242,7 @@ jobs:
env:
GH_TOKEN: ${{ github.token }}
with:
- key: cpu-windows-2025-x64-vulkan
+ key: cpu-windows-2025-${{ matrix.arch }}-vulkan
older: 5m
min: 1
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml
index 5a3166ce6885..f96fd85a914a 100644
--- a/.github/workflows/build-wasm.yml
+++ b/.github/workflows/build-wasm.yml
@@ -33,6 +33,10 @@ on:
'ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py'
]
+cache-mode: none
+permissions:
+ contents: read
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
@@ -56,7 +60,7 @@ jobs:
- name: ccache
uses: ggml-org/ccache-action@v1.2.24
with:
- key: webgpu-ubuntu-24.04-arm-wasm
+ restore: false
save: false
- name: Install Emscripten
diff --git a/.github/workflows/build-webgpu.yml b/.github/workflows/build-webgpu.yml
index ec582ff274b3..d00106439753 100644
--- a/.github/workflows/build-webgpu.yml
+++ b/.github/workflows/build-webgpu.yml
@@ -25,6 +25,10 @@ on:
'ggml/src/ggml-webgpu/**'
]
+cache-mode: none
+permissions:
+ contents: read
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
@@ -71,7 +75,7 @@ jobs:
- name: ccache
uses: ggml-org/ccache-action@v1.2.24
with:
- key: webgpu-macos-latest
+ restore: false
save: false
- name: Dawn Dependency
@@ -132,7 +136,7 @@ jobs:
- name: ccache
uses: ggml-org/ccache-action@v1.2.24
with:
- key: webgpu-ubuntu-24.04
+ restore: false
save: false
- name: Dependencies
diff --git a/.github/workflows/check-vendor.yml b/.github/workflows/check-vendor.yml
index 1671ed7b8bd2..e4c25459344b 100644
--- a/.github/workflows/check-vendor.yml
+++ b/.github/workflows/check-vendor.yml
@@ -17,6 +17,10 @@ on:
'scripts/sync_vendor.py'
]
+cache-mode: none
+permissions:
+ contents: read
+
jobs:
check-vendor:
runs-on: ubuntu-slim
diff --git a/.github/workflows/ci-self-hosted-cpu.yml b/.github/workflows/ci-self-hosted-cpu.yml
new file mode 100644
index 000000000000..eb40baa88e6b
--- /dev/null
+++ b/.github/workflows/ci-self-hosted-cpu.yml
@@ -0,0 +1,116 @@
+name: CI (self-hosted CPU backend)
+
+on:
+ workflow_dispatch: # allows manual triggering
+ push:
+ branches:
+ - master
+ paths: [
+ '.github/workflows/ci-self-hosted-cpu.yml',
+ 'ci/run.sh',
+ '**/CMakeLists.txt',
+ '**/.cmake',
+ '**/*.h',
+ '**/*.hpp',
+ '**/*.c',
+ '**/*.cpp'
+ ]
+
+ pull_request:
+ types: [opened, synchronize, reopened]
+ paths: [
+ '.github/workflows/ci-self-hosted-cpu.yml',
+ 'ci/run.sh',
+ '**/CMakeLists.txt',
+ '**/.cmake',
+ 'ggml/src/*',
+ 'ggml/src/ggml-cpu/**'
+ ]
+
+cache-mode: none
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
+ cancel-in-progress: true
+
+env:
+ # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
+ GGML_NLOOP: 3
+ GGML_N_THREADS: 1
+ LLAMA_ARG_LOG_COLORS: 1
+ LLAMA_ARG_LOG_PREFIX: 1
+ LLAMA_ARG_LOG_TIMESTAMPS: 1
+
+jobs:
+ cpu-x64-high-perf:
+ runs-on: [self-hosted, Linux, X64]
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v6
+
+ - name: Test
+ id: ggml-ci
+ run: |
+ LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+
+ cpu-arm64-high-perf-graviton4:
+ runs-on: ah-ubuntu_24_04-c8g_8x
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v6
+
+ - name: Dependencies
+ id: depends
+ run: |
+ set -euxo pipefail
+ sudo apt-get update
+ sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a \
+ apt-get install -y \
+ build-essential \
+ python3-venv \
+ gpg \
+ wget \
+ time \
+ git-lfs
+
+ git lfs install
+
+ # install the latest cmake
+ sudo install -d /usr/share/keyrings
+ wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc \
+ | gpg --dearmor \
+ | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
+ echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' \
+ | sudo tee /etc/apt/sources.list.d/kitware.list
+ sudo apt-get update
+ sudo apt-get install -y cmake
+
+ - name: Test
+ id: ggml-ci
+ run: |
+ LLAMA_ARG_THREADS=$(nproc) \
+ GG_BUILD_HIGH_PERF=1 \
+ GG_BUILD_NO_BF16=1 \
+ GG_BUILD_EXTRA_TESTS_0=1 \
+ bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+
+ # TODO: provision AMX-compatible machine
+ #cpu-amx:
+ # runs-on: [self-hosted, Linux, CPU, AMX]
+
+ # steps:
+ # - name: Clone
+ # id: checkout
+ # uses: actions/checkout@v6
+
+ # - name: Test
+ # id: ggml-ci
+ # run: |
+ # bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
diff --git a/.github/workflows/ci-self-hosted-cuda.yml b/.github/workflows/ci-self-hosted-cuda.yml
new file mode 100644
index 000000000000..89605f4dd7ac
--- /dev/null
+++ b/.github/workflows/ci-self-hosted-cuda.yml
@@ -0,0 +1,128 @@
+name: CI (self-hosted CUDA backend)
+
+on:
+ workflow_dispatch: # allows manual triggering
+ push:
+ branches:
+ - master
+ paths: [
+ '.github/workflows/ci-self-hosted-cuda.yml',
+ 'ci/run.sh',
+ '**/CMakeLists.txt',
+ '**/.cmake',
+ '**/*.h',
+ '**/*.hpp',
+ '**/*.c',
+ '**/*.cpp',
+ '**/*.cu',
+ '**/*.cuh'
+ ]
+
+ pull_request:
+ types: [opened, synchronize, reopened]
+ paths: [
+ '.github/workflows/ci-self-hosted-cuda.yml',
+ 'ci/run.sh',
+ '**/CMakeLists.txt',
+ '**/.cmake',
+ 'ggml/src/*',
+ 'ggml/src/ggml-cpu/**',
+ 'ggml/src/ggml-cuda/**'
+ ]
+
+cache-mode: none
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
+ cancel-in-progress: true
+
+env:
+ # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
+ GGML_NLOOP: 3
+ GGML_N_THREADS: 1
+ LLAMA_ARG_LOG_COLORS: 1
+ LLAMA_ARG_LOG_PREFIX: 1
+ LLAMA_ARG_LOG_TIMESTAMPS: 1
+
+jobs:
+ gpu-cuda:
+ runs-on: "hf-jobs-t4-medium:cuda13"
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v6
+
+ - name: Install dependencies
+ run: |
+ sudo apt update
+ sudo apt install -y cmake libssl-dev time unzip wget python3 python3-venv python3-pip
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.24
+ with:
+ restore: false
+ save: false
+
+ - name: ccache-buckets-restore
+ uses: ./.github/actions/ccache-buckets
+ with:
+ key: self-hosted-gpu-cuda
+ folder: llama.cpp
+ hf_bucket: ggml-org/cache
+
+ - name: Test
+ id: ggml-ci
+ run: |
+ nvidia-smi
+ GG_BUILD_CUDA=1 CUDACXX=/usr/local/cuda/bin/nvcc bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+
+ - name: ccache-buckets-save
+ if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+ uses: ./.github/actions/ccache-buckets
+ env:
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+ with:
+ key: self-hosted-gpu-cuda
+ folder: llama.cpp
+ evict-old-files: 1d
+ hf_bucket: ggml-org/cache
+ save: true
+
+ gpu-rocm:
+ runs-on: [self-hosted, Linux, AMD]
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v6
+
+ - name: Test
+ id: ggml-ci
+ # HIP_LAUNCH_BLOCKING=1: workaround for an async-execution correctness
+ # issue on integrated RDNA3.5 (gfx1151) where batched inference returns
+ # incorrect output (perplexity ~88 vs ~9.4). Serializing kernel launches
+ # restores correctness. Remove once the underlying ROCm/HIP issue is fixed.
+ env:
+ HIP_LAUNCH_BLOCKING: "1"
+ run: |
+ rocminfo
+ GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS=gfx1151 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+
+ # TODO: provision AMD GPU machine
+ # amd-rocm:
+ # runs-on: [self-hosted, Linux, AMD]
+
+ # steps:
+ # - name: Clone
+ # id: checkout
+ # uses: actions/checkout@v6
+
+ # - name: Test
+ # id: ggml-ci
+ # run: |
+ # amd-smi static
+ # GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS="gfx1101" bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
diff --git a/.github/workflows/ci-self-hosted-kleidiai.yml b/.github/workflows/ci-self-hosted-kleidiai.yml
new file mode 100644
index 000000000000..c5f5670cec40
--- /dev/null
+++ b/.github/workflows/ci-self-hosted-kleidiai.yml
@@ -0,0 +1,89 @@
+name: CI (self-hosted KleidiAI backend)
+
+on:
+ workflow_dispatch: # allows manual triggering
+ push:
+ branches:
+ - master
+ paths: [
+ '.github/workflows/ci-self-hosted-kleidiai.yml',
+ 'ci/run.sh',
+ '**/CMakeLists.txt',
+ '**/.cmake',
+ '**/*.h',
+ '**/*.hpp',
+ '**/*.c',
+ '**/*.cpp'
+ ]
+
+ pull_request:
+ types: [opened, synchronize, reopened]
+ paths: [
+ '.github/workflows/ci-self-hosted-kleidiai.yml',
+ 'ci/run.sh',
+ '**/CMakeLists.txt',
+ '**/.cmake',
+ 'ggml/src/*',
+ 'ggml/src/ggml-cpu/**'
+ ]
+
+cache-mode: none
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
+ cancel-in-progress: true
+
+env:
+ # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
+ GGML_NLOOP: 3
+ GGML_N_THREADS: 1
+ LLAMA_ARG_LOG_COLORS: 1
+ LLAMA_ARG_LOG_PREFIX: 1
+ LLAMA_ARG_LOG_TIMESTAMPS: 1
+
+jobs:
+ cpu-arm64-graviton4-kleidiai:
+ runs-on: ah-ubuntu_24_04-c8g_8x
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v6
+
+ - name: Dependencies
+ id: depends
+ run: |
+ set -euxo pipefail
+ sudo apt-get update
+ sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a \
+ apt-get install -y \
+ build-essential \
+ python3-venv \
+ gpg \
+ wget \
+ time \
+ git-lfs
+
+ git lfs install
+
+ # install the latest cmake
+ sudo install -d /usr/share/keyrings
+ wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc \
+ | gpg --dearmor \
+ | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
+ echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' \
+ | sudo tee /etc/apt/sources.list.d/kitware.list
+ sudo apt-get update
+ sudo apt-get install -y cmake
+
+ - name: Test
+ id: ggml-ci
+ run: |
+ LLAMA_ARG_THREADS=$(nproc) \
+ GG_BUILD_KLEIDIAI=1 \
+ GG_BUILD_EXTRA_TESTS_0=1 \
+ GG_BUILD_HIGH_PERF=1 \
+ bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
diff --git a/.github/workflows/ci-self-hosted-metal.yml b/.github/workflows/ci-self-hosted-metal.yml
new file mode 100644
index 000000000000..5eb20096a484
--- /dev/null
+++ b/.github/workflows/ci-self-hosted-metal.yml
@@ -0,0 +1,63 @@
+name: CI (self-hosted Metal backend)
+
+on:
+ workflow_dispatch: # allows manual triggering
+ push:
+ branches:
+ - master
+ paths: [
+ '.github/workflows/ci-self-hosted-metal.yml',
+ 'ci/run.sh',
+ '**/CMakeLists.txt',
+ '**/.cmake',
+ '**/*.h',
+ '**/*.hpp',
+ '**/*.c',
+ '**/*.cpp',
+ '**/*.swift',
+ '**/*.m',
+ '**/*.metal'
+ ]
+
+ pull_request:
+ types: [opened, synchronize, reopened]
+ paths: [
+ '.github/workflows/ci-self-hosted-metal.yml',
+ 'ci/run.sh',
+ '**/CMakeLists.txt',
+ '**/.cmake',
+ 'ggml/src/*',
+ 'ggml/src/ggml-cpu/**',
+ 'ggml/src/ggml-metal/**'
+ ]
+
+cache-mode: none
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
+ cancel-in-progress: true
+
+env:
+ # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
+ GGML_NLOOP: 3
+ GGML_N_THREADS: 1
+ LLAMA_ARG_LOG_COLORS: 1
+ LLAMA_ARG_LOG_PREFIX: 1
+ LLAMA_ARG_LOG_TIMESTAMPS: 1
+
+jobs:
+ gpu-metal:
+ runs-on: [self-hosted, macOS, ARM64]
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v6
+
+ - name: Test
+ id: ggml-ci
+ run: |
+ GG_BUILD_METAL=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
diff --git a/.github/workflows/ci-self-hosted-openvino.yml b/.github/workflows/ci-self-hosted-openvino.yml
new file mode 100644
index 000000000000..ef89f004de32
--- /dev/null
+++ b/.github/workflows/ci-self-hosted-openvino.yml
@@ -0,0 +1,79 @@
+name: CI (self-hosted OpenVINO backend)
+
+on:
+ workflow_dispatch: # allows manual triggering
+ push:
+ branches:
+ - master
+ paths: [
+ '.github/workflows/ci-self-hosted-openvino.yml',
+ 'ci/run.sh',
+ '**/CMakeLists.txt',
+ '**/.cmake',
+ '**/*.h',
+ '**/*.hpp',
+ '**/*.c',
+ '**/*.cpp'
+ ]
+
+ pull_request:
+ types: [opened, synchronize, reopened]
+ paths: [
+ '.github/workflows/ci-self-hosted-openvino.yml',
+ 'ci/run.sh',
+ '**/CMakeLists.txt',
+ '**/.cmake',
+ 'ggml/src/*',
+ 'ggml/src/ggml-cpu/**',
+ 'ggml/src/ggml-openvino/**'
+ ]
+
+cache-mode: none
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
+ cancel-in-progress: true
+
+env:
+ # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
+ GGML_NLOOP: 3
+ GGML_N_THREADS: 1
+ LLAMA_ARG_LOG_COLORS: 1
+ LLAMA_ARG_LOG_PREFIX: 1
+ LLAMA_ARG_LOG_TIMESTAMPS: 1
+
+jobs:
+ gpu-openvino-low-perf:
+ runs-on: [self-hosted, Linux, Intel, OpenVINO]
+
+ env:
+ # Sync versions in build-openvino.yml, ci-self-hosted-openvino.yml, release.yml, release-publish.yml, build-cache.yml, .devops/openvino.Dockerfile, docs/backend/OPENVINO.md
+ OPENVINO_VERSION_MAJOR: "2026.4.1"
+ OPENVINO_VERSION_FULL: "2026.4.1.22982.07f9c262b05"
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v6
+
+ - name: Setup OpenVINO Toolkit
+ uses: ./.github/actions/linux-setup-openvino
+ with:
+ path: ./openvino_toolkit
+ version_major: ${{ env.OPENVINO_VERSION_MAJOR }}
+ version_full: ${{ env.OPENVINO_VERSION_FULL }}
+
+ - name: Install OpenVINO dependencies
+ run: |
+ cd ./openvino_toolkit
+ chmod +x ./install_dependencies/install_openvino_dependencies.sh
+ echo "Y" | sudo -E ./install_dependencies/install_openvino_dependencies.sh
+
+ - name: Test
+ id: ggml-ci
+ run: |
+ source ./openvino_toolkit/setupvars.sh
+ GG_BUILD_OPENVINO=1 GGML_OPENVINO_DEVICE=GPU GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
diff --git a/.github/workflows/ci-self-hosted-vulkan.yml b/.github/workflows/ci-self-hosted-vulkan.yml
new file mode 100644
index 000000000000..e651f2d1db56
--- /dev/null
+++ b/.github/workflows/ci-self-hosted-vulkan.yml
@@ -0,0 +1,205 @@
+name: CI (self-hosted Vulkan backend)
+
+on:
+ workflow_dispatch: # allows manual triggering
+ push:
+ branches:
+ - master
+ paths: [
+ '.github/workflows/ci-self-hosted-vulkan.yml',
+ 'ci/run.sh',
+ '**/CMakeLists.txt',
+ '**/.cmake',
+ '**/*.h',
+ '**/*.hpp',
+ '**/*.c',
+ '**/*.cpp',
+ '**/*.comp',
+ '**/*.glsl'
+ ]
+
+ pull_request:
+ types: [opened, synchronize, reopened]
+ paths: [
+ '.github/workflows/ci-self-hosted-vulkan.yml',
+ 'ci/run.sh',
+ '**/CMakeLists.txt',
+ '**/.cmake',
+ 'ggml/src/*',
+ 'ggml/src/ggml-cpu/**',
+ 'ggml/src/ggml-vulkan/**'
+ ]
+
+cache-mode: none
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
+ cancel-in-progress: true
+
+env:
+ # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
+ GGML_NLOOP: 3
+ GGML_N_THREADS: 1
+ LLAMA_ARG_LOG_COLORS: 1
+ LLAMA_ARG_LOG_PREFIX: 1
+ LLAMA_ARG_LOG_TIMESTAMPS: 1
+
+jobs:
+ gpu-vulkan-nvidia-cm:
+ # runs-on: "hf-jobs-t4-small:ubuntu26_04"
+ runs-on: [self-hosted, Linux, NVIDIA]
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v6
+
+ # - name: Install dependencies
+ # run: |
+ # sudo apt update
+ # sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan-dev glslc spirv-headers vulkan-tools mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 libssl-dev time unzip wget python3 python3-venv python3-pip
+
+ # - name: ccache
+ # uses: ggml-org/ccache-action@v1.2.24
+ # with:
+ # restore: false
+ # save: false
+
+ # - name: ccache-buckets-restore
+ # uses: ./.github/actions/ccache-buckets
+ # with:
+ # key: self-hosted-vulkan-nvidia-cm
+ # folder: llama.cpp
+ # hf_bucket: ggml-org/cache
+
+ - name: Test
+ id: ggml-ci
+ run: |
+ vulkaninfo --summary
+ GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+
+ # - name: ccache-buckets-save
+ # if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+ # uses: ./.github/actions/ccache-buckets
+ # env:
+ # HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+ # with:
+ # key: self-hosted-vulkan-nvidia-cm
+ # folder: llama.cpp
+ # evict-old-files: 1d
+ # hf_bucket: ggml-org/cache
+ # save: true
+
+ gpu-vulkan-nvidia-cm2:
+ # runs-on: "hf-jobs-t4-small:ubuntu26_04"
+ runs-on: [self-hosted, Linux, NVIDIA, COOPMAT2]
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v6
+
+ # - name: Install dependencies
+ # run: |
+ # sudo apt update
+ # sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan-dev glslc spirv-headers vulkan-tools mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 libssl-dev time unzip wget python3 python3-venv python3-pip
+
+ # - name: ccache
+ # uses: ggml-org/ccache-action@v1.2.24
+ # with:
+ # restore: false
+ # save: false
+
+ # - name: ccache-buckets-restore
+ # uses: ./.github/actions/ccache-buckets
+ # with:
+ # key: self-hosted-vulkan-nvidia-cm2
+ # folder: llama.cpp
+ # hf_bucket: ggml-org/cache
+
+ - name: Test
+ id: ggml-ci
+ run: |
+ vulkaninfo --summary
+ GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+
+ # - name: ccache-buckets-save
+ # if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+ # uses: ./.github/actions/ccache-buckets
+ # env:
+ # HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+ # with:
+ # key: self-hosted-vulkan-nvidia-cm2
+ # folder: llama.cpp
+ # evict-old-files: 1d
+ # hf_bucket: ggml-org/cache
+ # save: true
+
+ gpu-vulkan-apple:
+ runs-on: [self-hosted, macOS, ARM64]
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v6
+
+ - name: Test
+ id: ggml-ci
+ run: |
+ vulkaninfo --summary
+ GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+
+ gpu-vulkan-intel-linux:
+ runs-on: [self-hosted, Linux, Intel]
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v6
+ with:
+ persist-credentials: false
+
+ - name: Test
+ id: ggml-ci
+ run: |
+ vulkaninfo --summary
+ GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+
+ gpu-vulkan-intel-windows:
+ runs-on: [self-hosted, Windows, X64, Intel]
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v6
+
+ - name: Test
+ id: ggml-ci
+ shell: C:\msys64\usr\bin\bash.exe --noprofile --norc -eo pipefail "{0}"
+ env:
+ MSYSTEM: UCRT64
+ CHERE_INVOKING: 1
+ PATH: C:\msys64\ucrt64\bin;C:\msys64\usr\bin;C:\Windows\System32;${{ env.PATH }}
+ run: |
+ vulkaninfo --summary
+ # Skip python related tests with GG_BUILD_LOW_PERF=1 since Windows MSYS2 UCRT64 currently fails to create
+ # a valid python environment for testing
+ LLAMA_FATAL_WARNINGS=OFF GG_BUILD_NINJA=1 GG_BUILD_VULKAN=1 GG_BUILD_LOW_PERF=1 ./ci/run.sh ./results/llama.cpp ./mnt/llama.cpp
+
+ # TODO: provision AMD GPU machine
+ # amd-vulkan:
+ # runs-on: [self-hosted, Linux, AMD]
+
+ # steps:
+ # - name: Clone
+ # id: checkout
+ # uses: actions/checkout@v6
+
+ # - name: Test
+ # id: ggml-ci
+ # run: |
+ # vulkaninfo --summary
+ # GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
diff --git a/.github/workflows/ci-self-hosted-webgpu.yml b/.github/workflows/ci-self-hosted-webgpu.yml
new file mode 100644
index 000000000000..316ed33b91c7
--- /dev/null
+++ b/.github/workflows/ci-self-hosted-webgpu.yml
@@ -0,0 +1,134 @@
+name: CI (self-hosted WebGPU backend)
+
+on:
+ workflow_dispatch: # allows manual triggering
+ push:
+ branches:
+ - master
+ paths: [
+ '.github/workflows/ci-self-hosted-webgpu.yml',
+ 'ci/run.sh',
+ '**/CMakeLists.txt',
+ '**/.cmake',
+ '**/*.h',
+ '**/*.hpp',
+ '**/*.c',
+ '**/*.cpp',
+ '**/*.wgsl'
+ ]
+
+ pull_request:
+ types: [opened, synchronize, reopened]
+ paths: [
+ '.github/workflows/ci-self-hosted-webgpu.yml',
+ 'ci/run.sh',
+ '**/CMakeLists.txt',
+ '**/.cmake',
+ 'ggml/src/*',
+ 'ggml/src/ggml-cpu/**',
+ 'ggml/src/ggml-webgpu/**'
+ ]
+
+cache-mode: none
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
+ cancel-in-progress: true
+
+env:
+ # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
+ GGML_NLOOP: 3
+ GGML_N_THREADS: 1
+ LLAMA_ARG_LOG_COLORS: 1
+ LLAMA_ARG_LOG_PREFIX: 1
+ LLAMA_ARG_LOG_TIMESTAMPS: 1
+
+jobs:
+ gpu-webgpu-nvidia:
+ runs-on: "hf-jobs-t4-small:ubuntu26_04"
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v6
+
+ - name: Install dependencies
+ run: |
+ sudo apt update
+ sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan1 mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 libssl-dev time unzip wget python3 python3-venv python3-pip
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.24
+ with:
+ restore: false
+ save: false
+
+ - name: ccache-buckets-restore
+ uses: ./.github/actions/ccache-buckets
+ with:
+ key: self-hosted-webgpu-nvidia
+ folder: llama.cpp
+ hf_bucket: ggml-org/cache
+
+ - name: Dawn Dependency
+ id: dawn-depends
+ run: |
+ DAWN_VERSION="v20260908.214631"
+ DAWN_OWNER="google"
+ DAWN_REPO="dawn"
+ DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-ubuntu-latest-Release"
+ echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
+ curl -L -o artifact.tar.gz \
+ "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
+ mkdir dawn
+ tar -xvf artifact.tar.gz -C dawn --strip-components=1
+
+ - name: Test
+ id: ggml-ci
+ run: |
+ GG_BUILD_WEBGPU=1 \
+ GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \
+ GG_BUILD_WEBGPU_DAWN_DIR="$GITHUB_WORKSPACE/dawn/lib64/cmake/Dawn" \
+ bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+
+ - name: ccache-buckets-save
+ if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+ uses: ./.github/actions/ccache-buckets
+ env:
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+ with:
+ key: self-hosted-webgpu-nvidia
+ folder: llama.cpp
+ evict-old-files: 1d
+ hf_bucket: ggml-org/cache
+ save: true
+
+ gpu-webgpu-apple:
+ runs-on: [self-hosted, macOS, ARM64]
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v6
+
+ - name: Dawn Dependency
+ id: dawn-depends
+ run: |
+ DAWN_VERSION="v20260908.214631"
+ DAWN_OWNER="google"
+ DAWN_REPO="dawn"
+ DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-macos-latest-Release"
+ echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
+ curl -L -o artifact.tar.gz \
+ "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
+ mkdir dawn
+ tar -xvf artifact.tar.gz -C dawn --strip-components=1
+
+ - name: Test
+ id: ggml-ci
+ run: |
+ GG_BUILD_WEBGPU=1 GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \
+ bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
diff --git a/.github/workflows/close-issue.yml b/.github/workflows/close-issue.yml
index 4698cee5594b..744319c262b4 100644
--- a/.github/workflows/close-issue.yml
+++ b/.github/workflows/close-issue.yml
@@ -3,10 +3,9 @@ on:
schedule:
- cron: "42 0 * * *"
-# Fine-grant permission
-# https://docs.github.com/en/actions/security-for-github-actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token
+cache-mode: none
permissions:
- issues: write
+ contents: read
jobs:
close-issues:
diff --git a/.github/workflows/code-style.yml b/.github/workflows/code-style.yml
index 5acf5c7b418d..2b3d0296d13a 100644
--- a/.github/workflows/code-style.yml
+++ b/.github/workflows/code-style.yml
@@ -11,6 +11,10 @@ on:
- master
- gfx11 # ROCm fork-local, DO NOT UPSTREAM
+cache-mode: none
+permissions:
+ contents: read
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml
index 61c05dcac590..5527e73b6f3d 100644
--- a/.github/workflows/copilot-setup-steps.yml
+++ b/.github/workflows/copilot-setup-steps.yml
@@ -11,10 +11,12 @@ on:
paths:
- .github/workflows/copilot-setup-steps.yml
+cache-mode: none
+
jobs:
# The job MUST be called `copilot-setup-steps` or it will not be picked up by Copilot.
copilot-setup-steps:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
# Set the permissions to the lowest permissions possible needed for your steps.
# Copilot will be given its own token for its operations.
@@ -31,8 +33,8 @@ jobs:
- name: ccache
uses: ggml-org/ccache-action@v1.2.24
with:
- key: copilot-setup-steps
- evict-old-files: 1d
+ restore: false
+ save: false
- name: Dependencies
id: depends
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 1de25b522de8..75b3343095a9 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -20,15 +20,14 @@ on:
# Rebuild daily rather than on every push because it is expensive
- cron: '12 4 * * *'
+cache-mode: none
+permissions:
+ contents: read
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
-# Fine-grant permission
-# https://docs.github.com/en/actions/security-for-github-actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token
-permissions:
- packages: write
-
jobs:
create_tag:
name: Create and push git tag
@@ -62,6 +61,9 @@ jobs:
build_ui:
name: Build UI
needs: create_tag
+ permissions:
+ actions: write
+ contents: read
uses: ./.github/workflows/ui-build.yml
with:
ui_version: ${{ needs.create_tag.outputs.source_tag }}
@@ -90,8 +92,8 @@ jobs:
{ "tag": "cpu", "dockerfile": ".devops/s390x.Dockerfile", "platforms": "linux/s390x", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04-s390x", "prebuilt_ui": true },
{ "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
{ "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" },
- { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.3.0", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
- { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.3.0", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" },
+ { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.4.1", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
+ { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.4.1", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" },
{ "tag": "musa", "dockerfile": ".devops/musa.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
{ "tag": "intel", "dockerfile": ".devops/intel.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
{ "tag": "vulkan", "dockerfile": ".devops/vulkan.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04" },
@@ -146,6 +148,11 @@ jobs:
needs: [prepare_matrices, create_tag, build_ui]
runs-on: ${{ matrix.config.runs_on }}
+ # cache-mode: write # for QEMU
+ permissions:
+ actions: write
+ contents: read
+ packages: write
strategy:
fail-fast: false
matrix:
@@ -165,11 +172,11 @@ jobs:
name: llama-ui.zip
path: tools/ui/dist
- - name: Set up QEMU
- if: ${{ contains(matrix.config.platforms, 'linux/amd64') }}
- uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4
- with:
- image: tonistiigi/binfmt:qemu-v10.2.1
+ # - name: Set up QEMU
+ # if: ${{ contains(matrix.config.platforms, 'linux/amd64') }}
+ # uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4
+ # with:
+ # image: tonistiigi/binfmt:qemu-v10.2.1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4
diff --git a/.github/workflows/editorconfig.yml b/.github/workflows/editorconfig.yml
index 3de033004dd5..473477b56034 100644
--- a/.github/workflows/editorconfig.yml
+++ b/.github/workflows/editorconfig.yml
@@ -11,6 +11,10 @@ on:
- master
- gfx11 # ROCm fork-local, DO NOT UPSTREAM
+cache-mode: none
+permissions:
+ contents: read
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
diff --git a/.github/workflows/fusion.yml b/.github/workflows/fusion.yml
deleted file mode 100644
index 7c8596467aab..000000000000
--- a/.github/workflows/fusion.yml
+++ /dev/null
@@ -1,71 +0,0 @@
-name: Fusion
-
-on:
- workflow_dispatch: # allows manual triggering
- push:
- branches:
- - master
- paths: [
- '.github/workflows/fusion.yml',
- 'ggml/**',
- 'tests/fusion/**',
- 'tests/test-fusion.cpp',
- 'tests/test-llama-archs.cpp',
- 'src/models/**'
- ]
-
- pull_request:
- types: [opened, synchronize, reopened]
- paths: [
- '.github/workflows/fusion.yml',
- 'ggml/**',
- 'tests/fusion/**',
- 'tests/test-fusion.cpp',
- 'tests/test-llama-archs.cpp',
- 'src/models/**'
- ]
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
- cancel-in-progress: true
-
-env:
- GGML_NLOOP: 3
- GGML_N_THREADS: 1
- LLAMA_ARG_LOG_COLORS: 1
- LLAMA_ARG_LOG_PREFIX: 1
- LLAMA_ARG_LOG_TIMESTAMPS: 1
-
-jobs:
- # TODO: add jobs for other backends as they adopt the fusion debug API
- metal:
- runs-on: [self-hosted, macOS, ARM64]
-
- steps:
- - name: Clone
- id: checkout
- uses: actions/checkout@v6
-
- - name: Build
- id: cmake_build
- run: |
- cmake -B build \
- -DCMAKE_BUILD_TYPE=Release \
- -DLLAMA_FATAL_WARNINGS=ON \
- -DLLAMA_OPENSSL=OFF \
- -DGGML_SCHED_NO_REALLOC=ON \
- -DGGML_BLAS=OFF \
- -DGGML_METAL=ON
- time cmake --build build --config Release --target test-llama-archs -j $(sysctl -n hw.logicalcpu)
- time cmake --build build --config Release --target test-fusion -j $(sysctl -n hw.logicalcpu)
-
- - name: Generate models
- id: generate_models
- run: |
- rm -rf build-ci-models && mkdir -p build-ci-models
- ./build/bin/test-llama-archs -o build-ci-models
-
- - name: Test fusion
- id: test_fusion
- run: |
- ./build/bin/test-fusion --models build-ci-models --device MTL0 --check tests/fusion/MTL.csv
diff --git a/.github/workflows/gguf-publish.yml b/.github/workflows/gguf-publish.yml
index fb8eab3cdb3b..367b29c60f94 100644
--- a/.github/workflows/gguf-publish.yml
+++ b/.github/workflows/gguf-publish.yml
@@ -17,11 +17,14 @@ on:
tags:
- 'gguf-v*' # Push events to every version tag
+cache-mode: none
+permissions:
+ contents: read
jobs:
deploy:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04 # previously ubuntu-latest
steps:
- uses: actions/checkout@v6
diff --git a/.github/workflows/hip-quality-check.yml b/.github/workflows/hip-quality-check.yml
index 1758aba2ccf4..32d11f580ce4 100644
--- a/.github/workflows/hip-quality-check.yml
+++ b/.github/workflows/hip-quality-check.yml
@@ -25,6 +25,10 @@ on:
'scripts/hip/gcn-cdna-vgpr-check.py'
]
+cache-mode: none
+permissions:
+ contents: read
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
@@ -54,7 +58,7 @@ jobs:
- name: ccache
uses: ggml-org/ccache-action@v1.2.24
with:
- key: hip-quality-check-ubuntu-22.04
+ restore: false
save: false
- name: ccache-buckets-restore
diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml
index eab20c68811e..bc2c9ea45b8a 100644
--- a/.github/workflows/labeler.yml
+++ b/.github/workflows/labeler.yml
@@ -2,6 +2,10 @@ name: "Pull Request Labeler"
on:
- pull_request_target
+cache-mode: none
+permissions:
+ contents: read
+
jobs:
labeler:
permissions:
diff --git a/.github/workflows/make-release.yml b/.github/workflows/make-release.yml
index d1c6dca5dbd9..57c3a04a577f 100644
--- a/.github/workflows/make-release.yml
+++ b/.github/workflows/make-release.yml
@@ -18,6 +18,11 @@ on:
required: false
type: boolean
default: false
+ require_docker:
+ description: 'Require the Docker workflow to have completed successfully'
+ required: true
+ type: boolean
+ default: true
apiabi_compare_tag:
description: 'Tag to compare against for API/ABI check (default: latest release)'
required: false
@@ -27,13 +32,14 @@ on:
env:
GH_TOKEN: ${{ github.token }}
+cache-mode: none
permissions:
contents: write
packages: write
jobs:
make-release:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04 # previously ubuntu-latest
steps:
- name: Checkout
@@ -55,6 +61,7 @@ jobs:
RELEASE_BRANCH: ${{ github.ref_name }}
SKIP_APIABI_CHECK: ${{ github.event.inputs.skip_apiabi_check }}
APIABI_COMPARE_TAG: ${{ github.event.inputs.apiabi_compare_tag }}
+ REQUIRE_DOCKER: ${{ github.event.inputs.require_docker }}
- name: Create release tag
if: ${{ github.event.inputs.dry_run == 'false' }}
@@ -131,7 +138,7 @@ jobs:
});
- name: Re-tag container images with release version
- if: ${{ github.event.inputs.dry_run == 'false' && steps.desc.outputs.nightly_tag != '' }}
+ if: ${{ github.event.inputs.dry_run == 'false' && github.event.inputs.require_docker != 'false' && steps.desc.outputs.nightly_tag != '' }}
env:
GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }}
run: |
@@ -144,14 +151,23 @@ jobs:
VARIANTS=("" "-cuda" "-cuda13" "-vulkan" "-rocm" "-intel" "-musa" "-openvino")
TYPES=("full" "light" "server")
+ # the release is already created at this point, so keep going on a
+ # missing image and report all of them at the end
+ MISSING=()
for type in "${TYPES[@]}"; do
for variant in "${VARIANTS[@]}"; do
src="${IMAGE_REPO}:${type}${variant}-${NIGHTLY_TAG}"
dst="${IMAGE_REPO}:${type}${variant}-${VERSION}"
echo "Tagging ${src} -> ${dst}"
- docker buildx imagetools create --tag "${dst}" "${src}"
+ if ! docker buildx imagetools create --tag "${dst}" "${src}"; then
+ MISSING+=("${type}${variant}")
+ fi
done
done
+ if [[ ${#MISSING[@]} -gt 0 ]]; then
+ echo "::error::failed to re-tag container images for ${NIGHTLY_TAG}:${MISSING[*]}"
+ exit 1
+ fi
- name: Dry run summary
if: ${{ github.event.inputs.dry_run == 'true' }}
diff --git a/.github/workflows/models-check.yml b/.github/workflows/models-check.yml
new file mode 100644
index 000000000000..66a150aff3aa
--- /dev/null
+++ b/.github/workflows/models-check.yml
@@ -0,0 +1,449 @@
+name: Models Backend Check
+
+on:
+ workflow_dispatch: # allows manual triggering
+ push:
+ branches:
+ - master
+ paths: [
+ '.github/workflows/models-check.yml',
+ 'ggml/**',
+ 'tests/fusion/**',
+ 'tests/test-fusion.cpp',
+ 'tests/test-llama-archs.cpp',
+ 'src/llama-graph.cpp',
+ 'src/llama-model*',
+ 'src/models/**'
+ ]
+
+ pull_request:
+ types: [opened, synchronize, reopened]
+ paths: [
+ '.github/workflows/models-check.yml',
+ 'ggml/**',
+ 'tests/fusion/**',
+ 'tests/test-fusion.cpp',
+ 'tests/test-llama-archs.cpp',
+ 'src/llama-graph.cpp',
+ 'src/llama-model*',
+ 'src/models/**'
+ ]
+
+cache-mode: none
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
+ cancel-in-progress: true
+
+env:
+ # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
+ GGML_NLOOP: 3
+ GGML_N_THREADS: 1
+ LLAMA_ARG_LOG_COLORS: 1
+ LLAMA_ARG_LOG_PREFIX: 1
+ LLAMA_ARG_LOG_TIMESTAMPS: 1
+
+jobs:
+ cuda:
+ runs-on: "hf-jobs-t4-medium:cuda13"
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v6
+
+ - name: Install dependencies
+ run: |
+ sudo apt update
+ sudo apt install -y cmake time python3 python3-venv python3-pip
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.24
+ with:
+ restore: false
+ save: false
+
+ - name: ccache-buckets-restore
+ uses: ./.github/actions/ccache-buckets
+ with:
+ key: models-check-cuda
+ folder: llama.cpp
+ hf_bucket: ggml-org/cache
+
+ - name: Build
+ id: cmake_build
+ run: |
+ cmake -B build \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DLLAMA_FATAL_WARNINGS=ON \
+ -DLLAMA_OPENSSL=OFF \
+ -DGGML_SCHED_NO_REALLOC=ON \
+ -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc \
+ -DGGML_CUDA=ON
+ time cmake --build build --config Release --target test-llama-archs -j$(nproc)
+ time cmake --build build --config Release --target test-fusion -j$(nproc)
+
+ - name: ccache-buckets-save
+ if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+ uses: ./.github/actions/ccache-buckets
+ env:
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+ with:
+ key: models-check-cuda
+ folder: llama.cpp
+ evict-old-files: 1d
+ hf_bucket: ggml-org/cache
+ save: true
+
+ # - name: Generate models
+ # id: generate_models
+ # run: |
+ # rm -rf build-ci-models && mkdir -p build-ci-models
+ # ./build/bin/test-llama-archs -o build-ci-models
+
+ # TODO: add for backends as they adopt the fusion debug API
+ # - name: Test fusion
+ # id: test_fusion
+ # run: |
+ # ./build/bin/test-fusion --models build-ci-models --device CUDA0 --check tests/fusion/CUDA.csv
+
+ - name: Test archs
+ id: test_archs
+ run: |
+ GGML_CUDA_DEVICES=1 ./build/bin/test-llama-archs -s 1
+ GGML_CUDA_DEVICES=2 ./build/bin/test-llama-archs -s 1
+ GGML_CUDA_DEVICES=3 ./build/bin/test-llama-archs -s 1
+ GGML_CUDA_DEVICES=4 ./build/bin/test-llama-archs -s 1
+
+ metal:
+ runs-on: [self-hosted, macOS, ARM64]
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v6
+
+ - name: Build
+ id: cmake_build
+ run: |
+ cmake -B build \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DLLAMA_FATAL_WARNINGS=ON \
+ -DLLAMA_OPENSSL=OFF \
+ -DGGML_SCHED_NO_REALLOC=ON \
+ -DGGML_BLAS=OFF \
+ -DGGML_METAL=ON
+ time cmake --build build --config Release --target test-llama-archs -j $(sysctl -n hw.logicalcpu)
+ time cmake --build build --config Release --target test-fusion -j $(sysctl -n hw.logicalcpu)
+
+ - name: Generate models
+ id: generate_models
+ run: |
+ rm -rf build-ci-models && mkdir -p build-ci-models
+ ./build/bin/test-llama-archs -o build-ci-models
+
+ - name: Test fusion
+ id: test_fusion
+ run: |
+ ./build/bin/test-fusion --models build-ci-models --device MTL0 --check tests/fusion/MTL.csv
+
+ - name: Test archs
+ id: test_archs
+ run: |
+ GGML_METAL_DEVICES=1 ./build/bin/test-llama-archs -s 1
+ GGML_METAL_DEVICES=2 ./build/bin/test-llama-archs -s 1
+ GGML_METAL_DEVICES=3 ./build/bin/test-llama-archs -s 1
+ GGML_METAL_DEVICES=4 ./build/bin/test-llama-archs -s 1
+
+ rocm:
+ runs-on: [self-hosted, Linux, gfx1201, 1accel]
+ container: "rocm/dev-ubuntu-24.04:7.2.4-complete"
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v6
+
+ - name: Install dependencies
+ run: |
+ apt update
+ apt install -y build-essential jq cmake time python3 python3-venv python3-pip
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.24
+ with:
+ restore: false
+ save: false
+
+ - name: ccache-buckets-restore
+ uses: ./.github/actions/ccache-buckets
+ with:
+ key: models-check-rocm
+ folder: llama.cpp
+ hf_bucket: ggml-org/cache
+
+ - name: Build
+ id: cmake_build
+ run: |
+ cmake -B build \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DLLAMA_FATAL_WARNINGS=ON \
+ -DLLAMA_OPENSSL=OFF \
+ -DGGML_SCHED_NO_REALLOC=ON \
+ -DCMAKE_HIP_COMPILER=$(hipconfig -l)/clang \
+ -DGPU_TARGETS=gfx1201 \
+ -DGGML_HIP=ON
+ time cmake --build build --config Release --target test-llama-archs -j$(nproc)
+ time cmake --build build --config Release --target test-fusion -j$(nproc)
+
+ - name: ccache-buckets-save
+ if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+ uses: ./.github/actions/ccache-buckets
+ env:
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+ with:
+ key: models-check-rocm
+ folder: llama.cpp
+ evict-old-files: 1d
+ hf_bucket: ggml-org/cache
+ save: true
+
+ # - name: Generate models
+ # id: generate_models
+ # run: |
+ # rm -rf build-ci-models && mkdir -p build-ci-models
+ # ./build/bin/test-llama-archs -o build-ci-models
+
+ # TODO: add for backends as they adopt the fusion debug API
+ # - name: Test fusion
+ # id: test_fusion
+ # run: |
+ # ./build/bin/test-fusion --models build-ci-models --device CUDA0 --check tests/fusion/CUDA.csv
+
+ - name: Test archs
+ id: test_archs
+ run: |
+ GGML_CUDA_DEVICES=1 ./build/bin/test-llama-archs -s 1
+ GGML_CUDA_DEVICES=2 ./build/bin/test-llama-archs -s 1
+ GGML_CUDA_DEVICES=3 ./build/bin/test-llama-archs -s 1
+ GGML_CUDA_DEVICES=4 ./build/bin/test-llama-archs -s 1
+
+ vulkan-nvidia:
+ runs-on: "hf-jobs-t4-small:ubuntu26_04"
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v6
+
+ - name: Install dependencies
+ run: |
+ sudo apt update
+ sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan-dev glslc spirv-headers vulkan-tools mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 time python3 python3-venv python3-pip
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.24
+ with:
+ restore: false
+ save: false
+
+ - name: ccache-buckets-restore
+ uses: ./.github/actions/ccache-buckets
+ with:
+ key: models-check-vulkan-nvidia
+ folder: llama.cpp
+ hf_bucket: ggml-org/cache
+
+ - name: Build
+ id: cmake_build
+ run: |
+ cmake -B build \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DLLAMA_FATAL_WARNINGS=ON \
+ -DLLAMA_OPENSSL=OFF \
+ -DGGML_SCHED_NO_REALLOC=ON \
+ -DGGML_VULKAN=ON
+ time cmake --build build --config Release --target test-llama-archs -j$(nproc)
+ time cmake --build build --config Release --target test-fusion -j$(nproc)
+
+ - name: ccache-buckets-save
+ if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+ uses: ./.github/actions/ccache-buckets
+ env:
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+ with:
+ key: models-check-vulkan-nvidia
+ folder: llama.cpp
+ evict-old-files: 1d
+ hf_bucket: ggml-org/cache
+ save: true
+
+ # - name: Generate models
+ # id: generate_models
+ # run: |
+ # rm -rf build-ci-models && mkdir -p build-ci-models
+ # ./build/bin/test-llama-archs -o build-ci-models
+
+ # TODO: add for backends as they adopt the fusion debug API
+ # - name: Test fusion
+ # id: test_fusion
+ # run: |
+ # ./build/bin/test-fusion --models build-ci-models --device Vulkan0 --check tests/fusion/Vulkan.csv
+
+ - name: Test archs
+ id: test_archs
+ run: |
+ ./build/bin/test-llama-archs -s 1
+
+ vulkan-amd:
+ runs-on: [self-hosted, Linux, gfx1201, 1accel]
+ container: "ubuntu:26.04"
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v6
+
+ - name: Install dependencies
+ run: |
+ apt update
+ apt install -y build-essential jq cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan-dev glslc spirv-headers vulkan-tools mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 time python3 python3-venv python3-pip
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.24
+ with:
+ restore: false
+ save: false
+
+ - name: ccache-buckets-restore
+ uses: ./.github/actions/ccache-buckets
+ with:
+ key: models-check-vulkan-amd
+ folder: llama.cpp
+ hf_bucket: ggml-org/cache
+
+ - name: Build
+ id: cmake_build
+ run: |
+ cmake -B build \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DLLAMA_FATAL_WARNINGS=ON \
+ -DLLAMA_OPENSSL=OFF \
+ -DGGML_SCHED_NO_REALLOC=ON \
+ -DGGML_VULKAN=ON
+ time cmake --build build --config Release --target test-llama-archs -j$(nproc)
+ time cmake --build build --config Release --target test-fusion -j$(nproc)
+
+ - name: ccache-buckets-save
+ if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+ uses: ./.github/actions/ccache-buckets
+ env:
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+ with:
+ key: models-check-vulkan-amd
+ folder: llama.cpp
+ evict-old-files: 1d
+ hf_bucket: ggml-org/cache
+ save: true
+
+ # - name: Generate models
+ # id: generate_models
+ # run: |
+ # rm -rf build-ci-models && mkdir -p build-ci-models
+ # ./build/bin/test-llama-archs -o build-ci-models
+
+ # TODO: add for backends as they adopt the fusion debug API
+ # - name: Test fusion
+ # id: test_fusion
+ # run: |
+ # ./build/bin/test-fusion --models build-ci-models --device Vulkan0 --check tests/fusion/Vulkan.csv
+
+ - name: Test archs
+ id: test_archs
+ run: |
+ ./build/bin/test-llama-archs -s 1
+
+ webgpu-nvidia:
+ runs-on: "hf-jobs-t4-small:ubuntu26_04"
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v6
+
+ - name: Install dependencies
+ run: |
+ sudo apt update
+ sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan1 mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 time python3 python3-venv python3-pip
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.24
+ with:
+ restore: false
+ save: false
+
+ - name: ccache-buckets-restore
+ uses: ./.github/actions/ccache-buckets
+ with:
+ key: models-check-webgpu-nvidia
+ folder: llama.cpp
+ hf_bucket: ggml-org/cache
+
+ - name: Dawn Dependency
+ id: dawn-depends
+ run: |
+ DAWN_VERSION="v20260908.214631"
+ DAWN_OWNER="google"
+ DAWN_REPO="dawn"
+ DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-ubuntu-latest-Release"
+ echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
+ curl -L -o artifact.tar.gz \
+ "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
+ mkdir dawn
+ tar -xvf artifact.tar.gz -C dawn --strip-components=1
+
+ - name: Build
+ id: cmake_build
+ run: |
+ cmake -B build \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DLLAMA_FATAL_WARNINGS=ON \
+ -DLLAMA_OPENSSL=OFF \
+ -DGGML_SCHED_NO_REALLOC=ON \
+ -DCMAKE_PREFIX_PATH="$GITHUB_WORKSPACE/dawn" \
+ -DDawn_DIR="$GITHUB_WORKSPACE/dawn/lib64/cmake/Dawn" \
+ -DGGML_WEBGPU=ON
+ time cmake --build build --config Release --target test-llama-archs -j$(nproc)
+ time cmake --build build --config Release --target test-fusion -j$(nproc)
+
+ - name: ccache-buckets-save
+ if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+ uses: ./.github/actions/ccache-buckets
+ env:
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+ with:
+ key: models-check-webgpu-nvidia
+ folder: llama.cpp
+ evict-old-files: 1d
+ hf_bucket: ggml-org/cache
+ save: true
+
+ # - name: Generate models
+ # id: generate_models
+ # run: |
+ # rm -rf build-ci-models && mkdir -p build-ci-models
+ # ./build/bin/test-llama-archs -o build-ci-models
+
+ # TODO: add for backends as they adopt the fusion debug API
+ # - name: Test fusion
+ # id: test_fusion
+ # run: |
+ # ./build/bin/test-fusion --models build-ci-models --device WebGPU --check tests/fusion/WebGPU.csv
+
+ - name: Test archs
+ id: test_archs
+ run: |
+ ./build/bin/test-llama-archs -s 1
diff --git a/.github/workflows/pr-draft-label.yml b/.github/workflows/pr-draft-label.yml
index d2594c823d7f..67a2bf3c28c5 100644
--- a/.github/workflows/pr-draft-label.yml
+++ b/.github/workflows/pr-draft-label.yml
@@ -4,6 +4,7 @@ on:
pull_request_target:
types: [labeled]
+cache-mode: none
permissions:
pull-requests: write
issues: write
diff --git a/.github/workflows/pre-tokenizer-hashes.yml b/.github/workflows/pre-tokenizer-hashes.yml
index bfb79f6983f2..0ba66ec2596e 100644
--- a/.github/workflows/pre-tokenizer-hashes.yml
+++ b/.github/workflows/pre-tokenizer-hashes.yml
@@ -10,6 +10,10 @@ on:
- 'conversion/base.py'
- 'convert_hf_to_gguf_update.py'
+cache-mode: none
+permissions:
+ contents: read
+
jobs:
pre-tokenizer-hashes:
runs-on: ubuntu-slim
diff --git a/.github/workflows/python-check-requirements.yml b/.github/workflows/python-check-requirements.yml
index e21c7da57202..9738ff778920 100644
--- a/.github/workflows/python-check-requirements.yml
+++ b/.github/workflows/python-check-requirements.yml
@@ -14,6 +14,10 @@ on:
- 'convert*.py'
- '**/requirements*.txt'
+cache-mode: none
+permissions:
+ contents: read
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml
index 1c64eaafa743..6081b56c9f1e 100644
--- a/.github/workflows/python-lint.yml
+++ b/.github/workflows/python-lint.yml
@@ -16,6 +16,10 @@ on:
'**/*.py'
]
+cache-mode: none
+permissions:
+ contents: read
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
diff --git a/.github/workflows/python-type-check.yml b/.github/workflows/python-type-check.yml
index f3695be96f75..5be177e9c504 100644
--- a/.github/workflows/python-type-check.yml
+++ b/.github/workflows/python-type-check.yml
@@ -16,6 +16,10 @@ on:
- '**/requirements*.txt'
# - 'pyrightconfig.json'
+cache-mode: none
+permissions:
+ contents: read
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
@@ -31,7 +35,7 @@ jobs:
uses: actions/setup-python@v6
with:
python-version: "3.11"
- pip-install: -r requirements/requirements-all.txt ty==0.0.78
+ pip-install: -r requirements/requirements-all.txt ty==0.0.84
# - name: Type-check with Pyright
# uses: jakebailey/pyright-action@v2
# with:
diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml
new file mode 100644
index 000000000000..39b6753a16b6
--- /dev/null
+++ b/.github/workflows/release-publish.yml
@@ -0,0 +1,247 @@
+name: Publish Release
+
+on:
+ workflow_run:
+ workflows:
+ - Release
+ types:
+ - completed
+ branches:
+ - master
+
+run-name: "Publish ${{ github.event.workflow_run.display_title }}"
+
+cache-mode: none
+permissions:
+ actions: read
+ contents: read
+
+env:
+ GH_TOKEN: ${{ github.token }}
+ BRANCH_NAME: master
+
+jobs:
+ publish:
+ if: ${{ github.event.workflow_run.conclusion == 'success' }}
+
+ # Fine-grained permission
+ # https://docs.github.com/en/actions/security-for-github-actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token
+ permissions:
+ actions: read
+ contents: write # for creating release
+ id-token: write
+ attestations: write
+
+ runs-on: ubuntu-latest
+
+ outputs:
+ should_release: ${{ steps.check.outputs.should_release }}
+ tag_name: ${{ steps.tag.outputs.name }}
+
+ steps:
+ - id: check
+ env:
+ COMMIT_MESSAGE: ${{ github.event.workflow_run.head_commit.message }}
+ run: |
+ if echo "$COMMIT_MESSAGE" | grep -q '\[no release\]'; then
+ echo "should_release=false" >> $GITHUB_OUTPUT
+ else
+ echo "should_release=true" >> $GITHUB_OUTPUT
+ fi
+
+ - name: Clone
+ if: ${{ steps.check.outputs.should_release == 'true' }}
+ id: checkout
+ uses: actions/checkout@v6
+ with:
+ ref: ${{ github.event.workflow_run.head_sha }}
+ fetch-depth: 0
+ ssh-key: ${{ secrets.DEPLOY_KEY_RELEASE }}
+
+ - name: Determine tag name
+ if: ${{ steps.check.outputs.should_release == 'true' }}
+ id: tag
+ uses: ./.github/actions/get-tag-name
+
+ - name: Download artifacts
+ if: ${{ steps.check.outputs.should_release == 'true' }}
+ id: download-artifact
+ uses: actions/download-artifact@v8
+ with:
+ path: ./artifact
+ run-id: ${{ github.event.workflow_run.id }}
+ github-token: ${{ github.token }}
+ merge-multiple: true
+ skip-decompress: true
+
+ - name: Merge artifacts
+ if: ${{ steps.check.outputs.should_release == 'true' }}
+ id: move_artifacts
+ run: |
+ mkdir -p release
+
+ # the windows-cpu zip contains the full toolset (llama-server with the embedded
+ # UI, ggml-cpu) - inject it into the other windows zips so that every archive
+ # ships the same binaries, only with a different backend library on top
+ echo "Injecting windows-cpu binaries (llama-server + CPU backend) into the backend zips..."
+ for arch in x64 arm64; do
+ cpu_zip="artifact/llama-bin-win-cpu-${arch}.zip"
+ temp_dir=$(mktemp -d)
+ echo "Extracting windows-cpu-${arch} package..."
+ unzip "$cpu_zip" -d "$temp_dir"
+
+ echo "Merging into $arch zips..."
+ for target_zip in artifact/llama-bin-win-*-${arch}.zip; do
+ if [[ "$target_zip" == "$cpu_zip" ]]; then
+ continue
+ fi
+ echo "Injecting into $(basename "$target_zip")"
+ realpath_target_zip=$(realpath "$target_zip")
+ (cd "$temp_dir" && zip -r "$realpath_target_zip" .)
+ done
+
+ rm -rf "$temp_dir"
+ done
+
+ echo "Renaming and moving zips to release..."
+ for zip_file in artifact/llama-bin-win-*.zip; do
+ base_name=$(basename "$zip_file" .zip)
+ zip_name="llama-${{ steps.tag.outputs.name }}-${base_name#llama-}.zip"
+ echo "Moving $zip_file to release/$zip_name"
+ mv "$zip_file" "release/$zip_name"
+ done
+
+ echo "Moving other artifacts..."
+ rm -f artifact/llama-ui.zip
+ mv -v artifact/*.zip release
+ mv -v artifact/*.tar.gz release
+
+ - name: Download UI build
+ if: ${{ steps.check.outputs.should_release == 'true' }}
+ id: download_ui
+ uses: actions/download-artifact@v8
+ with:
+ name: llama-ui.zip
+ path: ./ui-dist
+ run-id: ${{ github.event.workflow_run.id }}
+ github-token: ${{ github.token }}
+
+ - name: Package UI
+ if: ${{ steps.check.outputs.should_release == 'true' }}
+ id: package_ui
+ run: |
+ tar -czvf release/llama-${{ steps.tag.outputs.name }}-ui.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./ui-dist .
+
+ - name: Attest release artifacts
+ if: ${{ steps.check.outputs.should_release == 'true' }}
+ id: attest
+ uses: actions/attest@v4
+ with:
+ subject-path: 'release/*'
+
+ - name: Create release
+ if: ${{ steps.check.outputs.should_release == 'true' }}
+ id: create_release
+ uses: ggml-org/action-create-release@v1
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ with:
+ tag_name: ${{ steps.tag.outputs.name }}
+ commitish: ${{ github.event.workflow_run.head_sha }}
+ prerelease: true
+ body: |
+
+
+ ${{ github.event.workflow_run.head_commit.message }}
+
+
+
+ **Website:**
+ -
+
+ **Attestations:**
+ - <${{ steps.attest.outputs.attestation-url }}>
+
+ **macOS/iOS:**
+ - [macOS Apple Silicon (arm64)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-macos-arm64.tar.gz)
+ - macOS Apple Silicon (arm64, KleidiAI enabled) [DISABLED](https://github.com/ggml-org/llama.cpp/pull/23780)
+ - [macOS Intel (x64)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-macos-x64.tar.gz)
+ - [iOS XCFramework](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-xcframework.zip)
+
+ **Linux:**
+ - [Ubuntu x64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-x64.tar.gz)
+ - [Ubuntu arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-arm64.tar.gz)
+ - [Ubuntu s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-s390x.tar.gz)
+ - [Ubuntu x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.tar.gz)
+ - [Ubuntu arm64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-arm64.tar.gz)
+ - [Ubuntu x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz) - [CUDA 12.8 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz)
+ - [Ubuntu x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-x64.tar.gz) - [CUDA 13.4 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-x64.tar.gz)
+ - [Ubuntu arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-arm64.tar.gz) - [CUDA 13.4 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-arm64.tar.gz)
+ - [Ubuntu x64 (ROCm 10.0)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-10.0-x64.tar.gz)
+ - [Ubuntu x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-2026.4.1-x64.tar.gz)
+ - [Ubuntu x64 (SYCL FP32)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp32-x64.tar.gz)
+ - [Ubuntu x64 (SYCL FP16)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp16-x64.tar.gz)
+ - [Linux arm64 (Snapdragon: CPU, Adreno GPU, Hexagon NPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-linux-arm64-snapdragon.tar.gz) - [setup guide](https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/snapdragon/linux.md)
+
+ **Android:**
+ - [Android arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-android-arm64.tar.gz)
+ - [Android arm64 (Snapdragon: CPU, Adreno GPU, Hexagon NPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-android-arm64-snapdragon.tar.gz) - [setup guide](https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/snapdragon/README.md)
+
+ **Windows:**
+ - [Windows x64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cpu-x64.zip)
+ - [Windows arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cpu-arm64.zip)
+ - [Windows arm64 (OpenCL Adreno)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-opencl-adreno-arm64.zip)
+ - [Windows x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-12.4-x64.zip) - [CUDA 12.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-12.4-x64.zip)
+ - [Windows x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-x64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-x64.zip)
+ - [Windows arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-arm64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-arm64.zip)
+ - [Windows x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-vulkan-x64.zip)
+ - [Windows arm64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-vulkan-arm64.zip)
+ - [Windows x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-openvino-2026.4.1-x64.zip)
+ - [Windows x64 (SYCL)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-sycl-x64.zip)
+ - [Windows x64 (ROCm 10.0)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-rocm-10.0-x64.zip)
+
+ **openEuler:**
+ - [DISABLED](https://github.com/ggml-org/llama.cpp/pull/23705)
+ - openEuler x86 (310p)
+ - openEuler x86 (910b, ACL Graph)
+ - openEuler aarch64 (310p)
+ - openEuler aarch64 (910b, ACL Graph)
+
+ **UI:**
+ - [UI](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-ui.tar.gz)
+
+ - name: Upload release
+ if: ${{ steps.check.outputs.should_release == 'true' }}
+ id: upload_release
+ uses: actions/github-script@v8
+ with:
+ github-token: ${{secrets.GITHUB_TOKEN}}
+ script: |
+ const path = require('path');
+ const fs = require('fs');
+ const release_id = '${{ steps.create_release.outputs.id }}';
+ for (let file of await fs.readdirSync('./release')) {
+ if (path.extname(file) === '.zip' || file.endsWith('.tar.gz')) {
+ console.log('uploadReleaseAsset', file);
+ await github.rest.repos.uploadReleaseAsset({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ release_id: release_id,
+ name: file,
+ data: await fs.readFileSync(`./release/${file}`)
+ });
+ }
+ }
+
+ ui-publish:
+ if: ${{ needs.publish.outputs.should_release == 'true' }}
+
+ needs:
+ - publish
+
+ uses: ./.github/workflows/ui-publish.yml
+ with:
+ version_tag: ${{ needs.publish.outputs.tag_name }}
+ run_id: ${{ github.event.workflow_run.id }}
+ secrets:
+ hf_token: ${{ secrets.HF_TOKEN_UI_STATIC_OUTPUT }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index e380cdd5c72f..9745230f500c 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -27,6 +27,11 @@ on:
'**/*.glsl'
]
+cache-mode: write
+permissions:
+ actions: write
+ contents: read
+
env:
GH_TOKEN: ${{ github.token }}
BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
@@ -41,8 +46,12 @@ jobs:
check-release:
runs-on: ubuntu-slim
+ permissions:
+ contents: write
+
outputs:
should_release: ${{ steps.check.outputs.should_release }}
+ tag_name: ${{ steps.tag.outputs.name }}
steps:
- id: check
@@ -61,6 +70,30 @@ jobs:
echo "should_release=false" >> $GITHUB_OUTPUT
fi
+ - name: Clone
+ if: ${{ steps.check.outputs.should_release == 'true' }}
+ id: checkout
+ uses: actions/checkout@v6
+ with:
+ fetch-depth: 0
+ ssh-key: ${{ secrets.DEPLOY_KEY_RELEASE }}
+
+ - name: Determine tag name
+ if: ${{ steps.check.outputs.should_release == 'true' }}
+ id: tag
+ uses: ./.github/actions/get-tag-name
+
+ - name: Create and push git tag
+ if: ${{ steps.check.outputs.should_release == 'true' }}
+ run: |
+ TAG="${{ steps.tag.outputs.name }}"
+ if git rev-parse -q --verify "refs/tags/${TAG}" >/dev/null 2>&1; then
+ echo "Tag ${TAG} already exists, skipping creation"
+ else
+ git tag "${TAG}"
+ git push origin "${TAG}"
+ fi
+
macos-cpu:
needs: [check-release, ui-build]
if: ${{ needs.check-release.outputs.should_release == 'true' }}
@@ -86,9 +119,6 @@ jobs:
runs-on: ${{ matrix.os }}
- permissions:
- actions: write
-
steps:
- name: Clone
id: checkout
@@ -97,7 +127,7 @@ jobs:
fetch-depth: 0
- name: Download UI build
- uses: actions/download-artifact@v7
+ uses: actions/download-artifact@v8
with:
name: llama-ui.zip
path: tools/ui/dist
@@ -121,21 +151,17 @@ jobs:
${{ env.CMAKE_ARGS }}
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
- - name: Determine tag name
- id: tag
- uses: ./.github/actions/get-tag-name
-
- name: Pack artifacts
id: pack_artifacts
run: |
cp LICENSE ./build/bin/
- tar -czvf llama-${{ steps.tag.outputs.name }}-bin-macos-${{ matrix.build }}.tar.gz -s ",^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .
+ tar -czvf llama-${{ needs.check-release.outputs.tag_name }}-bin-macos-${{ matrix.build }}.tar.gz -s ",^\.,llama-${{ needs.check-release.outputs.tag_name }}," -C ./build/bin .
- name: Upload artifacts
- uses: actions/upload-artifact@v6
+ uses: actions/upload-artifact@v7
with:
- path: llama-${{ steps.tag.outputs.name }}-bin-macos-${{ matrix.build }}.tar.gz
- name: llama-bin-macos-${{ matrix.build }}.tar.gz
+ path: llama-${{ needs.check-release.outputs.tag_name }}-bin-macos-${{ matrix.build }}.tar.gz
+ archive: false
- name: ccache-clear
uses: ./.github/actions/ccache-clear
@@ -157,9 +183,6 @@ jobs:
runs-on: ${{ matrix.os }}
- permissions:
- actions: write
-
steps:
- name: Clone
id: checkout
@@ -168,7 +191,7 @@ jobs:
fetch-depth: 0
- name: Download UI build
- uses: actions/download-artifact@v7
+ uses: actions/download-artifact@v8
with:
name: llama-ui.zip
path: tools/ui/dist
@@ -206,21 +229,17 @@ jobs:
${{ env.CMAKE_ARGS }}
cmake --build build --config Release -j $(nproc)
- - name: Determine tag name
- id: tag
- uses: ./.github/actions/get-tag-name
-
- name: Pack artifacts
id: pack_artifacts
run: |
cp LICENSE ./build/bin/
- tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-${{ matrix.build }}.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .
+ tar -czvf llama-${{ needs.check-release.outputs.tag_name }}-bin-ubuntu-${{ matrix.build }}.tar.gz --transform "s,^\.,llama-${{ needs.check-release.outputs.tag_name }}," -C ./build/bin .
- name: Upload artifacts
- uses: actions/upload-artifact@v6
+ uses: actions/upload-artifact@v7
with:
- path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-${{ matrix.build }}.tar.gz
- name: llama-bin-ubuntu-${{ matrix.build }}.tar.gz
+ path: llama-${{ needs.check-release.outputs.tag_name }}-bin-ubuntu-${{ matrix.build }}.tar.gz
+ archive: false
- name: ccache-clear
if: ${{ matrix.build != 's390x' }}
@@ -242,9 +261,6 @@ jobs:
runs-on: ${{ matrix.os }}
- permissions:
- actions: write
-
steps:
- name: Clone
id: checkout
@@ -253,7 +269,7 @@ jobs:
fetch-depth: 0
- name: Download UI build
- uses: actions/download-artifact@v7
+ uses: actions/download-artifact@v8
with:
name: llama-ui.zip
path: tools/ui/dist
@@ -292,21 +308,17 @@ jobs:
${{ env.CMAKE_ARGS }}
cmake --build build --config Release -j $(nproc)
- - name: Determine tag name
- id: tag
- uses: ./.github/actions/get-tag-name
-
- name: Pack artifacts
id: pack_artifacts
run: |
cp LICENSE ./build/bin/
- tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-${{ matrix.build }}.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .
+ tar -czvf llama-${{ needs.check-release.outputs.tag_name }}-bin-ubuntu-vulkan-${{ matrix.build }}.tar.gz --transform "s,^\.,llama-${{ needs.check-release.outputs.tag_name }}," -C ./build/bin .
- name: Upload artifacts
- uses: actions/upload-artifact@v6
+ uses: actions/upload-artifact@v7
with:
- path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-${{ matrix.build }}.tar.gz
- name: llama-bin-ubuntu-vulkan-${{ matrix.build }}.tar.gz
+ path: llama-${{ needs.check-release.outputs.tag_name }}-bin-ubuntu-vulkan-${{ matrix.build }}.tar.gz
+ archive: false
- name: ccache-clear
uses: ./.github/actions/ccache-clear
@@ -323,31 +335,28 @@ jobs:
include:
# label = short version used in artifact names / release body
# cuda = full container image tag
+ # CTK >= 13.5 bundles CCCL >= 3.5; omit GGML_CUDA_CCCL_VERSION for those versions.
- build: 'x64'
os: ubuntu-24.04
cuda: '12.8.2'
label: '12.8'
- defines: '-DGGML_CUDA_CUB_3DOT2=ON'
+ defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
- build: 'x64'
os: ubuntu-24.04
- cuda: '13.3.1'
- label: '13.3'
- defines: ''
+ cuda: '13.4.1'
+ label: '13.4'
+ defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
- build: 'arm64'
os: ubuntu-24.04-arm
- cuda: '13.3.1'
- label: '13.3'
- defines: ''
+ cuda: '13.4.1'
+ label: '13.4'
+ defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
runs-on: ${{ matrix.os }}
container: nvidia/cuda:${{ matrix.cuda }}-devel-ubuntu24.04
- permissions:
- actions: write
-
steps:
- # the container has no git; install it before checkout so that a real git
- # repository is created (the get-tag-name action and the build both need it)
+ # the container has no git; install it before checkout so that a real git repository is created
- name: Install git
run: |
apt-get update
@@ -367,7 +376,7 @@ jobs:
run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: Download UI build
- uses: actions/download-artifact@v7
+ uses: actions/download-artifact@v8
with:
name: llama-ui.zip
path: tools/ui/dist
@@ -410,21 +419,17 @@ jobs:
${{ env.CMAKE_ARGS }} ${{ matrix.defines }}
cmake --build build --config Release -j $(nproc)
- - name: Determine tag name
- id: tag
- uses: ./.github/actions/get-tag-name
-
- name: Pack artifacts
id: pack_artifacts
run: |
cp LICENSE ./build/bin/
- tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .
+ tar -czvf llama-${{ needs.check-release.outputs.tag_name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz --transform "s,^\.,llama-${{ needs.check-release.outputs.tag_name }}," -C ./build/bin .
- name: Upload artifacts
- uses: actions/upload-artifact@v6
+ uses: actions/upload-artifact@v7
with:
- path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz
- name: llama-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz
+ path: llama-${{ needs.check-release.outputs.tag_name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz
+ archive: false
# ship the CUDA runtime libraries the backend links against, mirroring
# the windows-cuda cudart zip - extract next to the binaries ($ORIGIN rpath)
@@ -439,13 +444,13 @@ jobs:
cp -L /usr/local/cuda/lib64/libcudart.so.${major} ./cudart/
cp -L /usr/local/cuda/lib64/libcublas.so.${major} ./cudart/
cp -L /usr/local/cuda/lib64/libcublasLt.so.${major} ./cudart/
- tar -czvf cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz --transform "s,^\.,cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}," -C ./cudart .
+ tar -czvf cudart-llama-${{ needs.check-release.outputs.tag_name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz --transform "s,^\.,cudart-llama-${{ needs.check-release.outputs.tag_name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}," -C ./cudart .
- name: Upload CUDA runtime
- uses: actions/upload-artifact@v6
+ uses: actions/upload-artifact@v7
with:
- path: cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz
- name: cudart-llama-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz
+ path: cudart-llama-${{ needs.check-release.outputs.tag_name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz
+ archive: false
- name: ccache-clear
uses: ./.github/actions/ccache-clear
@@ -456,10 +461,7 @@ jobs:
needs: [check-release, ui-build]
if: ${{ needs.check-release.outputs.should_release == 'true' }}
- runs-on: ubuntu-latest
-
- #permissions:
- # actions: write
+ runs-on: ubuntu-24.04 # previously ubuntu-latest
env:
NDK_VERSION: "29.0.14206865"
@@ -472,7 +474,7 @@ jobs:
fetch-depth: 0
- name: Download UI build
- uses: actions/download-artifact@v7
+ uses: actions/download-artifact@v8
with:
name: llama-ui.zip
path: tools/ui/dist
@@ -484,10 +486,9 @@ jobs:
distribution: temurin
- name: Setup Android SDK
- uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 # v4.0.1
+ uses: android-actions/setup-android@be39fa834029ff78f1a44aa3bb0819b8fc2bd8fd # v4.0.4
with:
log-accepted-android-sdk-licenses: false
- packages: 'platform-tools'
- name: Install NDK
run: |
@@ -529,44 +530,136 @@ jobs:
# with:
# key: release-android-arm64
- - name: Determine tag name
- id: tag
- uses: ./.github/actions/get-tag-name
-
- name: Pack artifacts
id: pack_artifacts
run: |
cp LICENSE ./build/bin/
- tar -czvf llama-${{ steps.tag.outputs.name }}-bin-android-arm64.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .
+ tar -czvf llama-${{ needs.check-release.outputs.tag_name }}-bin-android-arm64.tar.gz --transform "s,^\.,llama-${{ needs.check-release.outputs.tag_name }}," -C ./build/bin .
- name: Upload artifacts
- uses: actions/upload-artifact@v6
+ uses: actions/upload-artifact@v7
with:
- path: llama-${{ steps.tag.outputs.name }}-bin-android-arm64.tar.gz
- name: llama-bin-android-arm64.tar.gz
+ path: llama-${{ needs.check-release.outputs.tag_name }}-bin-android-arm64.tar.gz
+ archive: false
- ubuntu-24-openvino:
+ android-arm64-snapdragon:
needs: [check-release, ui-build]
if: ${{ needs.check-release.outputs.should_release == 'true' }}
- runs-on: ubuntu-24.04
+ runs-on: ubuntu-latest
+ container: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.7'
- permissions:
- actions: write
+ defaults:
+ run:
+ shell: bash
- outputs:
- openvino_version: ${{ steps.openvino_version.outputs.value }}
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v6
+ with:
+ fetch-depth: 0
- env:
- # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
- OPENVINO_VERSION_MAJOR: "2026.4"
- OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"
+ # checkout runs as the host user; in-container steps run as root, so git
+ # refuses to touch a repo it does not own. Mark the workspace as safe.
+ - name: Git safe directory
+ run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+
+ - name: Download UI build
+ uses: actions/download-artifact@v8
+ with:
+ name: llama-ui.zip
+ path: tools/ui/dist
+
+ - name: Build
+ id: cmake_build
+ run: |
+ cp docs/backend/snapdragon/CMakeUserPresets.json .
+ cmake --preset arm64-android-snapdragon-release -B build \
+ -DCMAKE_INSTALL_RPATH='$ORIGIN;$ORIGIN/../lib' \
+ -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
+ -DLLAMA_BUILD_BORINGSSL=ON \
+ ${{ env.CMAKE_ARGS }}
+ cmake --build build -j $(nproc)
+ cmake --install build --prefix pkg-snapdragon/llama.cpp
+
+ - name: Pack artifacts
+ id: pack_artifacts
+ run: |
+ cp LICENSE pkg-snapdragon/llama.cpp/
+ tar -czvf llama-${{ needs.check-release.outputs.tag_name }}-bin-android-arm64-snapdragon.tar.gz --transform "s,^\.,llama-${{ needs.check-release.outputs.tag_name }}," -C pkg-snapdragon/llama.cpp .
+
+ - name: Upload artifacts
+ uses: actions/upload-artifact@v7
+ with:
+ path: llama-${{ needs.check-release.outputs.tag_name }}-bin-android-arm64-snapdragon.tar.gz
+ archive: false
+
+ linux-arm64-snapdragon:
+ needs: [check-release, ui-build]
+ if: ${{ needs.check-release.outputs.should_release == 'true' }}
+
+ runs-on: ubuntu-latest
+ container: 'ghcr.io/snapdragon-toolchain/arm64-linux:v0.7'
+
+ defaults:
+ run:
+ shell: bash
steps:
- - name: Set OpenVINO version output
- id: openvino_version
- run: echo "value=${{ env.OPENVINO_VERSION_MAJOR }}" >> $GITHUB_OUTPUT
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v6
+ with:
+ fetch-depth: 0
+
+ # checkout runs as the host user; in-container steps run as root, so git
+ # refuses to touch a repo it does not own. Mark the workspace as safe.
+ - name: Git safe directory
+ run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+ - name: Download UI build
+ uses: actions/download-artifact@v8
+ with:
+ name: llama-ui.zip
+ path: tools/ui/dist
+
+ - name: Build
+ id: cmake_build
+ run: |
+ cp docs/backend/snapdragon/CMakeUserPresets.json .
+ cmake --preset arm64-linux-snapdragon-release -B build -DGGML_OPENCL=ON \
+ -DCMAKE_INSTALL_RPATH='$ORIGIN;$ORIGIN/../lib' \
+ -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
+ -DLLAMA_BUILD_BORINGSSL=ON \
+ ${{ env.CMAKE_ARGS }}
+ cmake --build build -j $(nproc)
+ cmake --install build --prefix pkg-snapdragon/llama.cpp
+
+ - name: Pack artifacts
+ id: pack_artifacts
+ run: |
+ cp LICENSE pkg-snapdragon/llama.cpp/
+ tar -czvf llama-${{ needs.check-release.outputs.tag_name }}-bin-linux-arm64-snapdragon.tar.gz --transform "s,^\.,llama-${{ needs.check-release.outputs.tag_name }}," -C pkg-snapdragon/llama.cpp .
+
+ - name: Upload artifacts
+ uses: actions/upload-artifact@v7
+ with:
+ path: llama-${{ needs.check-release.outputs.tag_name }}-bin-linux-arm64-snapdragon.tar.gz
+ archive: false
+
+ ubuntu-24-openvino:
+ needs: [check-release, ui-build]
+ if: ${{ needs.check-release.outputs.should_release == 'true' }}
+
+ runs-on: ubuntu-24.04
+
+ env:
+ # Sync versions in build-openvino.yml, ci-self-hosted-openvino.yml, release.yml, release-publish.yml, build-cache.yml, .devops/openvino.Dockerfile, docs/backend/OPENVINO.md
+ OPENVINO_VERSION_MAJOR: "2026.4.1"
+ OPENVINO_VERSION_FULL: "2026.4.1.22982.07f9c262b05"
+
+ steps:
- name: Clone
id: checkout
uses: actions/checkout@v6
@@ -574,7 +667,7 @@ jobs:
fetch-depth: 0
- name: Download UI build
- uses: actions/download-artifact@v7
+ uses: actions/download-artifact@v8
with:
name: llama-ui.zip
path: tools/ui/dist
@@ -621,13 +714,12 @@ jobs:
-DGGML_OPENVINO=ON \
-DCMAKE_INSTALL_RPATH='$ORIGIN' \
-DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
+ -DGGML_BACKEND_DL=ON \
+ -DGGML_NATIVE=OFF \
+ -DGGML_CPU_ALL_VARIANTS=ON \
${{ env.CMAKE_ARGS }}
cmake --build build/ReleaseOV --config Release --parallel
- - name: Determine tag name
- id: tag
- uses: ./.github/actions/get-tag-name
-
- name: Pack artifacts
id: pack_artifacts
run: |
@@ -650,51 +742,39 @@ jobs:
cp -r "$OPENVINO_ROOT"/docs/licensing "$dest"/openvino-licensing
cp LICENSE "$dest"
- tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C "$dest" .
+ tar -czvf llama-${{ needs.check-release.outputs.tag_name }}-bin-ubuntu-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz --transform "s,^\.,llama-${{ needs.check-release.outputs.tag_name }}," -C "$dest" .
- name: Upload artifacts
- uses: actions/upload-artifact@v6
+ uses: actions/upload-artifact@v7
with:
- path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz
- name: llama-bin-ubuntu-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz
+ path: llama-${{ needs.check-release.outputs.tag_name }}-bin-ubuntu-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz
+ archive: false
- name: ccache-clear
uses: ./.github/actions/ccache-clear
with:
key: release-ubuntu-24.04-openvino-release-no-preset-v1
+ # note: builds only the ggml-openvino backend - llama-server and the CPU backend are injected from the
+ # windows-cpu zip during the release "Merge artifacts" step
windows-openvino:
- needs: [check-release, ui-build]
+ needs: [check-release]
if: ${{ needs.check-release.outputs.should_release == 'true' }}
runs-on: windows-2022
- outputs:
- openvino_version: ${{ steps.openvino_version.outputs.value }}
-
env:
- # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
- OPENVINO_VERSION_MAJOR: "2026.4"
- OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"
+ # Sync versions in build-openvino.yml, ci-self-hosted-openvino.yml, release.yml, release-publish.yml, build-cache.yml, .devops/openvino.Dockerfile, docs/backend/OPENVINO.md
+ OPENVINO_VERSION_MAJOR: "2026.4.1"
+ OPENVINO_VERSION_FULL: "2026.4.1.22982.07f9c262b05"
steps:
- - name: Set OpenVINO version output
- id: openvino_version
- shell: bash
- run: echo "value=${{ env.OPENVINO_VERSION_MAJOR }}" >> $GITHUB_OUTPUT
-
- name: Clone
id: checkout
uses: actions/checkout@v6
with:
fetch-depth: 0
- - name: Download UI build
- uses: actions/download-artifact@v7
- with:
- name: llama-ui.zip
- path: tools/ui/dist
-
- name: ccache
uses: ggml-org/ccache-action@v1.2.24
with:
@@ -742,15 +822,13 @@ jobs:
-A x64 ^
-DCMAKE_BUILD_TYPE=Release ^
-DGGML_OPENVINO=ON ^
+ -DGGML_BACKEND_DL=ON ^
+ -DGGML_NATIVE=OFF ^
+ -DGGML_CPU=OFF ^
-DLLAMA_BUILD_BORINGSSL=ON ^
- -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake ^
- ${{ env.CMAKE_ARGS }}
+ -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake
- cmake --build build\ReleaseOV --config Release -- /m
-
- - name: Determine tag name
- id: tag
- uses: ./.github/actions/get-tag-name
+ cmake --build build\ReleaseOV --config Release --target ggml-openvino -- /m
- name: Pack artifacts
id: pack_artifacts
@@ -778,13 +856,13 @@ jobs:
Copy-Item -Path (Join-Path $OPENVINO_ROOT 'docs\licensing\*') -Destination $licensingDest -Recurse -Force
Copy-Item LICENSE $dest
- 7z a -snl llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip $dest\*
+ 7z a -snl llama-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip $dest\*
- name: Upload artifacts
- uses: actions/upload-artifact@v6
+ uses: actions/upload-artifact@v7
with:
- path: llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip
- name: llama-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip
+ path: llama-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip
+ archive: false
- name: ccache-clear
uses: ./.github/actions/ccache-clear
@@ -798,9 +876,6 @@ jobs:
runs-on: windows-2025-vs2026
- permissions:
- actions: write
-
strategy:
matrix:
include:
@@ -814,7 +889,7 @@ jobs:
fetch-depth: 0
- name: Download UI build
- uses: actions/download-artifact@v7
+ uses: actions/download-artifact@v8
with:
name: llama-ui.zip
path: tools/ui/dist
@@ -850,10 +925,10 @@ jobs:
7z a -snl llama-bin-win-cpu-${{ matrix.arch }}.zip .\build\bin\Release\*
- name: Upload artifacts
- uses: actions/upload-artifact@v6
+ uses: actions/upload-artifact@v7
with:
path: llama-bin-win-cpu-${{ matrix.arch }}.zip
- name: llama-bin-win-cpu-${{ matrix.arch }}.zip
+ archive: false
- name: ccache-clear
uses: ./.github/actions/ccache-clear
@@ -966,10 +1041,6 @@ jobs:
Write-Host "HIP backend artifact found:"
$hipDll | Format-Table FullName, Length -AutoSize
- - name: Determine tag name
- id: tag
- uses: ./.github/actions/get-tag-name
-
- name: Get ROCm short version
run: |
$rocmVersionShort = ('${{ matrix.ROCM_VERSION }}'.Split('.')[0..1] -join '.')
@@ -1011,10 +1082,10 @@ jobs:
.\build\bin\Release\amd_comgr.dll
- name: Upload artifacts
- uses: actions/upload-artifact@v6
+ uses: actions/upload-artifact@v7
with:
path: llama-bin-win-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.zip
- name: llama-bin-win-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.zip
+ archive: false
- name: ccache-clear
uses: ./.github/actions/ccache-clear
@@ -1029,9 +1100,6 @@ jobs:
runs-on: windows-2025
- permissions:
- actions: write
-
env:
OPENBLAS_VERSION: 0.3.23
VULKAN_VERSION: 1.4.357.0
@@ -1043,6 +1111,10 @@ jobs:
arch: 'x64'
defines: '-DGGML_VULKAN=ON'
target: 'ggml-vulkan'
+ - backend: 'vulkan'
+ arch: 'arm64'
+ defines: '-G "Ninja Multi-Config" -DGGML_VULKAN=ON -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake'
+ target: 'ggml-vulkan'
- backend: 'opencl-adreno'
arch: 'arm64'
defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/opencl-arm64-release" -DGGML_OPENCL=ON -DGGML_OPENCL_USE_ADRENO_KERNELS=ON'
@@ -1058,7 +1130,7 @@ jobs:
if: ${{ matrix.backend == 'vulkan' }}
run: |
curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/vulkansdk-windows-X64-${env:VULKAN_VERSION}.exe"
- & "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install
+ & "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install ${{ matrix.arch == 'arm64' && 'com.lunarg.vulkan.arm64' || '' }}
Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}"
Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin"
@@ -1111,10 +1183,10 @@ jobs:
7z a -snl llama-bin-win-${{ matrix.backend }}-${{ matrix.arch }}.zip .\build\bin\Release\${{ matrix.target }}.dll
- name: Upload artifacts
- uses: actions/upload-artifact@v6
+ uses: actions/upload-artifact@v7
with:
path: llama-bin-win-${{ matrix.backend }}-${{ matrix.arch }}.zip
- name: llama-bin-win-${{ matrix.backend }}-${{ matrix.arch }}.zip
+ archive: false
# note: builds only the ggml-cuda backend - llama-server is injected from the
# windows-cpu zip during the release "Merge artifacts" step
@@ -1125,21 +1197,19 @@ jobs:
runs-on: windows-2022
- permissions:
- actions: write
-
strategy:
matrix:
include:
+ # CTK >= 13.5 bundles CCCL >= 3.5; omit GGML_CUDA_CCCL_VERSION for those versions.
- cuda: '12.4'
arch: x64
- defines: '-DGGML_CUDA_CUB_3DOT2=ON'
+ defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
- cuda: '13.4'
arch: x64
- defines: ''
+ defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
- cuda: '13.4'
arch: arm64
- defines: '-DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake'
+ defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3 -DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake'
steps:
- name: Clone
@@ -1166,7 +1236,6 @@ jobs:
- name: Build
id: cmake_build
shell: cmd
- # TODO: Remove GGML_CUDA_CUB_3DOT2 flag once CCCL 3.2 is bundled within CTK and that CTK version is used in this project
run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.arch == 'x64' && 'x64' || 'amd64_arm64' }}
cmake -S . -B build -G "Ninja Multi-Config" ^
@@ -1184,10 +1253,10 @@ jobs:
7z a -snl llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip .\build\bin\Release\ggml-cuda.dll
- name: Upload artifacts
- uses: actions/upload-artifact@v6
+ uses: actions/upload-artifact@v7
with:
path: llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip
- name: llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip
+ archive: false
- name: Copy and pack Cuda runtime (x64)
if: ${{ matrix.arch == 'x64' }}
@@ -1208,10 +1277,10 @@ jobs:
7z a cudart-llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip $dst\*
- name: Upload Cuda runtime
- uses: actions/upload-artifact@v6
+ uses: actions/upload-artifact@v7
with:
path: cudart-llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip
- name: cudart-llama-bin-win-cuda-${{ matrix.cuda }}-${{ matrix.arch }}.zip
+ archive: false
- name: ccache-clear
uses: ./.github/actions/ccache-clear
@@ -1231,11 +1300,11 @@ jobs:
shell: bash
env:
- WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b60765d1-2b85-4e85-86b6-cb0e9563a699/intel-deep-learning-essentials-2025.3.3.18_offline.exe
+ WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/0cb67a0d-67f6-410b-868b-f4a0a17ff0cf/intel-oneapi-toolkit-2026.1.1.32_offline.exe
WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.dnnl:intel.oneapi.win.tbb.devel
- LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.28.2/level-zero-win-sdk-1.28.2.zip
+ LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.33.1/level-zero-win-sdk-1.33.1.zip
ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI"
- ONEAPI_INSTALLER_VERSION: "2025.3.3"
+ ONEAPI_INSTALLER_VERSION: "2026.1"
steps:
- name: Clone
@@ -1278,9 +1347,11 @@ jobs:
run: |
echo "cp oneAPI running time dll files in ${{ env.ONEAPI_ROOT }} to ./build/bin"
- cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_sycl_blas.5.dll" ./build/bin
- cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_core.2.dll" ./build/bin
- cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_tbb_thread.2.dll" ./build/bin
+ cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_sycl_blas.6.dll" ./build/bin
+ cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_core.3.dll" ./build/bin
+ cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_def.3.dll" ./build/bin
+ cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_avx2.3.dll" ./build/bin
+ cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_tbb_thread.3.dll" ./build/bin
cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_adapter_level_zero.dll" ./build/bin
cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_adapter_level_zero_v2.dll" ./build/bin
@@ -1295,13 +1366,11 @@ jobs:
echo "Level Zero loader DLL not found in oneAPI or SDK; relying on system driver/runtime"
fi
- cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/sycl8.dll" ./build/bin
+ cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/sycl9.dll" ./build/bin
cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/svml_dispmd.dll" ./build/bin
cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/libmmd.dll" ./build/bin
cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/libiomp5md.dll" ./build/bin
cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/sycl-ls.exe" ./build/bin
- cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/libsycl-fallback-bfloat16.spv" ./build/bin
- cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/libsycl-native-bfloat16.spv" ./build/bin
cp "${{ env.ONEAPI_ROOT }}/dnnl/latest/bin/dnnl.dll" ./build/bin
cp "${{ env.ONEAPI_ROOT }}/tbb/latest/bin/tbb12.dll" ./build/bin
@@ -1314,10 +1383,10 @@ jobs:
7z a -snl llama-bin-win-sycl-x64.zip ./build/bin/*
- name: Upload the release package
- uses: actions/upload-artifact@v6
+ uses: actions/upload-artifact@v7
with:
path: llama-bin-win-sycl-x64.zip
- name: llama-bin-win-sycl-x64.zip
+ archive: false
- name: ccache-clear
uses: ./.github/actions/ccache-clear
@@ -1341,8 +1410,8 @@ jobs:
env:
ONEAPI_ROOT: /opt/intel/oneapi/
- ONEAPI_INSTALLER_VERSION: "2025.3.3"
- LEVEL_ZERO_VERSION: "1.28.2"
+ ONEAPI_INSTALLER_VERSION: "2026.1"
+ LEVEL_ZERO_VERSION: "1.33.1"
LEVEL_ZERO_UBUNTU_VERSION: "u24.04"
steps:
@@ -1356,19 +1425,19 @@ jobs:
shell: bash
run: |
cd /tmp
- wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/56f7923a-adb8-43f3-8b02-2b60fcac8cab/intel-deep-learning-essentials-2025.3.3.16_offline.sh -O intel-deep-learning-essentials_offline.sh
- sudo bash intel-deep-learning-essentials_offline.sh -s -a --silent --eula accept
+ wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/5996e26b-f48a-42b1-8db0-b002ad0bd8d7/intel-oneapi-toolkit-2026.1.1.33_offline.sh -O intel-oneapi-toolkit_offline.sh
+ sudo bash intel-oneapi-toolkit_offline.sh -s -a --silent --eula accept
- name: Install Level Zero SDK
shell: bash
run: |
cd /tmp
- wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero.deb
- wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero-devel_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero-devel.deb
- sudo apt-get install -y ./level-zero.deb ./level-zero-devel.deb
+ wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/libze1_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O libze1.deb
+ wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/libze-dev_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O libze-dev.deb
+ sudo apt-get install -y ./libze1.deb ./libze-dev.deb
- name: Download UI build
- uses: actions/download-artifact@v7
+ uses: actions/download-artifact@v8
with:
name: llama-ui.zip
path: tools/ui/dist
@@ -1396,21 +1465,17 @@ jobs:
-DGGML_SYCL_F16=${{ matrix.fp16 }}
time cmake --build build --config Release -j $(nproc)
- - name: Determine tag name
- id: tag
- uses: ./.github/actions/get-tag-name
-
- name: Pack artifacts
id: pack_artifacts
run: |
cp LICENSE ./build/bin/
- tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-${{ matrix.build }}-x64.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .
+ tar -czvf llama-${{ needs.check-release.outputs.tag_name }}-bin-ubuntu-sycl-${{ matrix.build }}-x64.tar.gz --transform "s,^\.,llama-${{ needs.check-release.outputs.tag_name }}," -C ./build/bin .
- name: Upload artifacts
- uses: actions/upload-artifact@v6
+ uses: actions/upload-artifact@v7
with:
- path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-${{ matrix.build }}-x64.tar.gz
- name: llama-bin-ubuntu-sycl-${{ matrix.build }}-x64.tar.gz
+ path: llama-${{ needs.check-release.outputs.tag_name }}-bin-ubuntu-sycl-${{ matrix.build }}-x64.tar.gz
+ archive: false
- name: ccache-clear
uses: ./.github/actions/ccache-clear
@@ -1423,9 +1488,6 @@ jobs:
runs-on: ubuntu-24.04
- permissions:
- actions: write
-
strategy:
matrix:
include:
@@ -1441,7 +1503,7 @@ jobs:
fetch-depth: 0
- name: Download UI build
- uses: actions/download-artifact@v7
+ uses: actions/download-artifact@v8
with:
name: llama-ui.zip
path: tools/ui/dist
@@ -1520,10 +1582,6 @@ jobs:
${{ env.CMAKE_ARGS }}
cmake --build build --config Release -j $(nproc)
- - name: Determine tag name
- id: tag
- uses: ./.github/actions/get-tag-name
-
- name: Get ROCm short version
run: echo "ROCM_VERSION_SHORT=$(echo '${{ matrix.ROCM_VERSION }}' | cut -d '.' -f 1,2)" >> $GITHUB_ENV
@@ -1531,13 +1589,13 @@ jobs:
id: pack_artifacts
run: |
cp LICENSE ./build/bin/
- tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .
+ tar -czvf llama-${{ needs.check-release.outputs.tag_name }}-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz --transform "s,^\.,llama-${{ needs.check-release.outputs.tag_name }}," -C ./build/bin .
- name: Upload artifacts
- uses: actions/upload-artifact@v6
+ uses: actions/upload-artifact@v7
with:
- path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz
- name: llama-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz
+ path: llama-${{ needs.check-release.outputs.tag_name }}-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz
+ archive: false
- name: ccache-clear
uses: ./.github/actions/ccache-clear
@@ -1586,22 +1644,18 @@ jobs:
- name: Build Xcode project
run: xcodebuild -project examples/llama.swiftui/llama.swiftui.xcodeproj -scheme llama.swiftui -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' FRAMEWORK_FOLDER_PATH=./build-ios build
- - name: Determine tag name
- id: tag
- uses: ./.github/actions/get-tag-name
-
- name: Pack artifacts
id: pack_artifacts
run: |
# Zip file is required for Swift Package Manager, which does not support tar.gz for binary targets.
# For more details, see https://developer.apple.com/documentation/xcode/distributing-binary-frameworks-as-swift-packages
- zip -r -y llama-${{ steps.tag.outputs.name }}-xcframework.zip build-apple/llama.xcframework
+ zip -r -y llama-${{ needs.check-release.outputs.tag_name }}-xcframework.zip build-apple/llama.xcframework
- name: Upload artifacts
- uses: actions/upload-artifact@v6
+ uses: actions/upload-artifact@v7
with:
- path: llama-${{ steps.tag.outputs.name }}-xcframework.zip
- name: llama-${{ steps.tag.outputs.name }}-xcframework.zip
+ path: llama-${{ needs.check-release.outputs.tag_name }}-xcframework.zip
+ archive: false
# TODO: this build is disabled to save Github Actions resources (https://github.com/ggml-org/llama.cpp/pull/23705)
# in order to enable it again, we have to provision dedicated runners to run it
@@ -1680,243 +1734,18 @@ jobs:
# chown -R '"${HOST_UID}"':'"${HOST_GID}"' /workspace/build
# '
#
-# - name: Determine tag name
-# id: tag
-# uses: ./.github/actions/get-tag-name
-#
# - name: Pack artifacts
# run: |
# cp LICENSE ./build/bin/
-# tar -czvf llama-${{ steps.tag.outputs.name }}-bin-${{ matrix.chip_type }}-openEuler-${{ matrix.arch }}${{ matrix.use_acl_graph == 'on' && '-aclgraph' || '' }}.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .
+# tar -czvf llama-${{ needs.check-release.outputs.tag_name }}-bin-${{ matrix.chip_type }}-openEuler-${{ matrix.arch }}${{ matrix.use_acl_graph == 'on' && '-aclgraph' || '' }}.tar.gz --transform "s,^\.,llama-${{ needs.check-release.outputs.tag_name }}," -C ./build/bin .
#
# - name: Upload artifacts
-# uses: actions/upload-artifact@v6
+# uses: actions/upload-artifact@v7
# with:
-# path: llama-${{ steps.tag.outputs.name }}-bin-${{ matrix.chip_type }}-openEuler-${{ matrix.arch }}${{ matrix.use_acl_graph == 'on' && '-aclgraph' || '' }}.tar.gz
-# name: llama-bin-${{ matrix.chip_type }}-openEuler-${{ matrix.arch }}${{ matrix.use_acl_graph == 'on' && '-aclgraph' || '' }}.tar.gz
+# path: llama-${{ needs.check-release.outputs.tag_name }}-bin-${{ matrix.chip_type }}-openEuler-${{ matrix.arch }}${{ matrix.use_acl_graph == 'on' && '-aclgraph' || '' }}.tar.gz
+# archive: false
ui-build:
needs: [check-release]
if: ${{ needs.check-release.outputs.should_release == 'true' }}
uses: ./.github/workflows/ui-build.yml
-
- release:
- if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-
- # Fine-grant permission
- # https://docs.github.com/en/actions/security-for-github-actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token
- permissions:
- contents: write # for creating release
- id-token: write
- attestations: write
-
- runs-on: ubuntu-slim
-
- needs:
- - windows
- - windows-cpu
- - windows-cuda
- - windows-sycl
- - windows-rocm
- - windows-openvino
- - ubuntu-24-rocm
- - ubuntu-cpu
- - ubuntu-vulkan
- - ubuntu-cuda
- - ubuntu-24-openvino
- - ubuntu-24-sycl
- - android-arm64
- - macos-cpu
- - ios-xcode
- #- openEuler-cann
- - ui-build
-
- outputs:
- tag_name: ${{ steps.tag.outputs.name }}
-
- steps:
- - name: Clone
- id: checkout
- uses: actions/checkout@v6
- with:
- fetch-depth: 0
- ssh-key: ${{ secrets.DEPLOY_KEY_RELEASE }}
-
- - name: Determine tag name
- id: tag
- uses: ./.github/actions/get-tag-name
-
- - name: Download artifacts
- id: download-artifact
- uses: actions/download-artifact@v7
- with:
- path: ./artifact
- merge-multiple: true
-
- - name: Merge artifacts
- id: move_artifacts
- run: |
- mkdir -p release
-
- # the windows-cpu zip contains the full toolset (llama-server with the embedded
- # UI, ggml-cpu) - inject it into the other windows zips so that every archive
- # ships the same binaries, only with a different backend library on top
- echo "Injecting windows-cpu binaries (llama-server + CPU backend) into the backend zips..."
- for arch in x64 arm64; do
- cpu_zip="artifact/llama-bin-win-cpu-${arch}.zip"
- temp_dir=$(mktemp -d)
- echo "Extracting windows-cpu-${arch} package..."
- unzip "$cpu_zip" -d "$temp_dir"
-
- echo "Merging into $arch zips..."
- for target_zip in artifact/llama-bin-win-*-${arch}.zip; do
- if [[ "$target_zip" == "$cpu_zip" ]]; then
- continue
- fi
- echo "Injecting into $(basename "$target_zip")"
- realpath_target_zip=$(realpath "$target_zip")
- (cd "$temp_dir" && zip -r "$realpath_target_zip" .)
- done
-
- rm -rf "$temp_dir"
- done
-
- echo "Renaming and moving zips to release..."
- for zip_file in artifact/llama-bin-win-*.zip; do
- base_name=$(basename "$zip_file" .zip)
- zip_name="llama-${{ steps.tag.outputs.name }}-${base_name#llama-}.zip"
- echo "Moving $zip_file to release/$zip_name"
- mv "$zip_file" "release/$zip_name"
- done
-
- echo "Moving other artifacts..."
- mv -v artifact/*.zip release
- mv -v artifact/*.tar.gz release
-
- - name: Download UI build
- id: download_ui
- uses: actions/download-artifact@v7
- with:
- name: llama-ui.zip
- path: ./ui-dist
-
- - name: Package UI
- id: package_ui
- run: |
- tar -czvf release/llama-${{ steps.tag.outputs.name }}-ui.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./ui-dist .
-
- - name: Attest release artifacts
- id: attest
- uses: actions/attest@v4
- with:
- subject-path: 'release/*'
-
- - name: Create and push git tag
- run: |
- TAG="${{ steps.tag.outputs.name }}"
- if git rev-parse -q --verify "refs/tags/${TAG}" >/dev/null 2>&1; then
- echo "Tag ${TAG} already exists, skipping creation"
- else
- git tag "${TAG}"
- git push origin "${TAG}"
- fi
-
- - name: Create release
- id: create_release
- uses: ggml-org/action-create-release@v1
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- with:
- tag_name: ${{ steps.tag.outputs.name }}
- prerelease: true
- body: |
-
-
- ${{ github.event.head_commit.message }}
-
-
-
- **Website:**
- -
-
- **Attestations:**
- - <${{ steps.attest.outputs.attestation-url }}>
-
- **macOS/iOS:**
- - [macOS Apple Silicon (arm64)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-macos-arm64.tar.gz)
- - macOS Apple Silicon (arm64, KleidiAI enabled) [DISABLED](https://github.com/ggml-org/llama.cpp/pull/23780)
- - [macOS Intel (x64)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-macos-x64.tar.gz)
- - [iOS XCFramework](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-xcframework.zip)
-
- **Linux:**
- - [Ubuntu x64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-x64.tar.gz)
- - [Ubuntu arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-arm64.tar.gz)
- - [Ubuntu s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-s390x.tar.gz)
- - [Ubuntu x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.tar.gz)
- - [Ubuntu arm64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-arm64.tar.gz)
- - [Ubuntu x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz) - [CUDA 12.8 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz)
- - [Ubuntu x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.3-x64.tar.gz) - [CUDA 13.3 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.3-x64.tar.gz)
- - [Ubuntu arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.3-arm64.tar.gz) - [CUDA 13.3 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.3-arm64.tar.gz)
- - [Ubuntu x64 (ROCm 10.0)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-10.0-x64.tar.gz)
- - [Ubuntu x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-${{ needs.ubuntu-24-openvino.outputs.openvino_version }}-x64.tar.gz)
- - [Ubuntu x64 (SYCL FP32)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp32-x64.tar.gz)
- - [Ubuntu x64 (SYCL FP16)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp16-x64.tar.gz)
-
- **Android:**
- - [Android arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-android-arm64.tar.gz)
-
- **Windows:**
- - [Windows x64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cpu-x64.zip)
- - [Windows arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cpu-arm64.zip)
- - [Windows arm64 (OpenCL Adreno)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-opencl-adreno-arm64.zip)
- - [Windows x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-12.4-x64.zip) - [CUDA 12.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-12.4-x64.zip)
- - [Windows x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-x64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-x64.zip)
- - [Windows arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-arm64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-arm64.zip)
- - [Windows x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-vulkan-x64.zip)
- - [Windows x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ needs.windows-openvino.outputs.openvino_version }}-x64.zip)
- - [Windows x64 (SYCL)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-sycl-x64.zip)
- - [Windows x64 (ROCm 10.0)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-rocm-10.0-x64.zip)
-
- **openEuler:**
- - [DISABLED](https://github.com/ggml-org/llama.cpp/pull/23705)
- - openEuler x86 (310p)
- - openEuler x86 (910b, ACL Graph)
- - openEuler aarch64 (310p)
- - openEuler aarch64 (910b, ACL Graph)
-
- **UI:**
- - [UI](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-ui.tar.gz)
-
- - name: Upload release
- id: upload_release
- uses: actions/github-script@v8
- with:
- github-token: ${{secrets.GITHUB_TOKEN}}
- script: |
- const path = require('path');
- const fs = require('fs');
- const release_id = '${{ steps.create_release.outputs.id }}';
- for (let file of await fs.readdirSync('./release')) {
- if (path.extname(file) === '.zip' || file.endsWith('.tar.gz')) {
- console.log('uploadReleaseAsset', file);
- await github.rest.repos.uploadReleaseAsset({
- owner: context.repo.owner,
- repo: context.repo.repo,
- release_id: release_id,
- name: file,
- data: await fs.readFileSync(`./release/${file}`)
- });
- }
- }
-
- ui-publish:
- if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-
- needs:
- - release
-
- uses: ./.github/workflows/ui-publish.yml
- with:
- version_tag: ${{ needs.release.outputs.tag_name }}
- secrets:
- hf_token: ${{ secrets.HF_TOKEN_UI_STATIC_OUTPUT }}
diff --git a/.github/workflows/server-sanitize.yml b/.github/workflows/server-sanitize.yml
index 43746e91eb35..4202c31e7cc1 100644
--- a/.github/workflows/server-sanitize.yml
+++ b/.github/workflows/server-sanitize.yml
@@ -31,6 +31,10 @@ on:
'.github/workflows/server-sanitize.yml'
]
+cache-mode: none
+permissions:
+ contents: read
+
env:
# note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
@@ -45,7 +49,7 @@ concurrency:
jobs:
server:
- runs-on: hf-jobs-cpu-upgrade
+ runs-on: hf-jobs-cpu-performance
strategy:
matrix:
@@ -116,7 +120,7 @@ jobs:
run: |
source .venv/bin/activate
cd tools/server/tests
- PYTEST_WORKERS=1 ./tests.sh
+ PYTEST_WORKERS=4 ./tests.sh
- name: Slow tests
id: server_integration_tests_slow
@@ -124,4 +128,4 @@ jobs:
run: |
source .venv/bin/activate
cd tools/server/tests
- PYTEST_WORKERS=1 SLOW_TESTS=1 ./tests.sh
+ PYTEST_WORKERS=4 SLOW_TESTS=1 ./tests.sh
diff --git a/.github/workflows/server-self-hosted.yml b/.github/workflows/server-self-hosted.yml
index 8dc4637c4570..c5dd86449c5e 100644
--- a/.github/workflows/server-self-hosted.yml
+++ b/.github/workflows/server-self-hosted.yml
@@ -28,6 +28,10 @@ on:
'tools/server/**.*'
]
+cache-mode: none
+permissions:
+ contents: read
+
env:
# note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
@@ -102,7 +106,7 @@ jobs:
PYTEST_WORKERS=1 ./tests.sh
server-cuda:
- runs-on: "hf-jobs-t4-small:cuda13"
+ runs-on: "hf-jobs-t4-medium:cuda13"
steps:
- name: Clone
diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml
index 77fe7dbd3aa7..05dd0ac4595a 100644
--- a/.github/workflows/server.yml
+++ b/.github/workflows/server.yml
@@ -43,6 +43,10 @@ on:
'tools/server/**.*'
]
+cache-mode: none
+permissions:
+ contents: read
+
env:
LLAMA_ARG_LOG_COLORS: 1
LLAMA_ARG_LOG_PREFIX: 1
@@ -82,7 +86,7 @@ jobs:
- name: ccache
uses: ggml-org/ccache-action@v1.2.24
with:
- key: server-ubuntu-24.04-arm
+ restore: false
save: false
- name: ccache-buckets-restore
@@ -151,6 +155,11 @@ jobs:
windows:
runs-on: windows-2025
+ cache-mode: write
+ permissions:
+ actions: write
+ contents: read
+
steps:
- name: Clone
id: checkout
diff --git a/.github/workflows/ui-build-self-hosted.yml b/.github/workflows/ui-build-self-hosted.yml
index e93a89003b23..0ba2eb983fc2 100644
--- a/.github/workflows/ui-build-self-hosted.yml
+++ b/.github/workflows/ui-build-self-hosted.yml
@@ -3,6 +3,11 @@ name: UI Build (self-hosted)
on:
workflow_call:
+cache-mode: none
+permissions:
+ actions: write
+ contents: read
+
jobs:
build:
runs-on: [self-hosted, fast]
diff --git a/.github/workflows/ui-build.yml b/.github/workflows/ui-build.yml
index cbadaa9e76d1..e0d8dd065e98 100644
--- a/.github/workflows/ui-build.yml
+++ b/.github/workflows/ui-build.yml
@@ -8,11 +8,14 @@ on:
required: false
type: string
+cache-mode: none
+permissions:
+ actions: write
+ contents: read
+
jobs:
build:
runs-on: ubuntu-slim
- env:
- BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
steps:
- name: Checkout code
@@ -52,7 +55,7 @@ jobs:
working-directory: tools/ui
- name: Upload built UI
- uses: actions/upload-artifact@v6
+ uses: actions/upload-artifact@v7
with:
name: llama-ui.zip
path: tools/ui/dist/
diff --git a/.github/workflows/ui-publish.yml b/.github/workflows/ui-publish.yml
index e64ef32f801c..d77b899b2302 100644
--- a/.github/workflows/ui-publish.yml
+++ b/.github/workflows/ui-publish.yml
@@ -7,38 +7,35 @@ on:
description: 'Version tag to publish under (e.g., b1234)'
required: true
type: string
+ run_id:
+ required: true
+ type: number
secrets:
hf_token:
description: 'Hugging Face token with write access'
required: true
-jobs:
- build:
- name: Build static output
- uses: ./.github/workflows/ui-build.yml
+cache-mode: none
+permissions:
+ actions: read
+ contents: read
+jobs:
publish:
name: Publish UI Static Output
- needs: build
runs-on: ubuntu-slim
- permissions:
- contents: read
-
env:
HF_BUCKET_NAME: ${{ vars.HF_BUCKET_UI_STATIC_OUTPUT }}
steps:
- - name: Checkout code
- uses: actions/checkout@v6
- with:
- fetch-depth: 1
-
- name: Download UI build artifact
- uses: actions/download-artifact@v7
+ uses: actions/download-artifact@v8
with:
name: llama-ui.zip
path: tools/ui/dist/
+ run-id: ${{ inputs.run_id }}
+ github-token: ${{ github.token }}
- name: Create distribution archive
run: |
diff --git a/.github/workflows/ui-self-hosted.yml b/.github/workflows/ui-self-hosted.yml
index 63521ead2d6a..93b7805f18d9 100644
--- a/.github/workflows/ui-self-hosted.yml
+++ b/.github/workflows/ui-self-hosted.yml
@@ -29,6 +29,11 @@ on:
'tools/server/tests/**.*'
]
+cache-mode: none
+permissions:
+ actions: read
+ contents: read
+
env:
LLAMA_ARG_LOG_COLORS: 1
LLAMA_ARG_LOG_PREFIX: 1
@@ -43,6 +48,9 @@ jobs:
ui-build:
name: Build static output
uses: ./.github/workflows/ui-build-self-hosted.yml
+ permissions:
+ actions: write
+ contents: read
ui-checks:
name: Checks
diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml
index f395c0b52873..c5c6a32a8177 100644
--- a/.github/workflows/ui.yml
+++ b/.github/workflows/ui.yml
@@ -25,6 +25,11 @@ on:
'tools/server/tests/**.*'
]
+cache-mode: none
+permissions:
+ actions: read
+ contents: read
+
env:
LLAMA_ARG_LOG_COLORS: 1
LLAMA_ARG_LOG_PREFIX: 1
@@ -39,6 +44,9 @@ jobs:
ui-build:
name: Build static output
uses: ./.github/workflows/ui-build.yml
+ permissions:
+ actions: write
+ contents: read
ui-checks:
name: Checks
diff --git a/.github/workflows/update-ops-docs.yml b/.github/workflows/update-ops-docs.yml
index bb01c7e56bab..c94fd850025f 100644
--- a/.github/workflows/update-ops-docs.yml
+++ b/.github/workflows/update-ops-docs.yml
@@ -14,6 +14,10 @@ on:
- 'docs/ops/**'
- 'scripts/create_ops_docs.py'
+cache-mode: none
+permissions:
+ contents: read
+
jobs:
update-ops-docs:
runs-on: ubuntu-slim
diff --git a/.github/workflows/winget.yml b/.github/workflows/winget.yml
index c0a814f3adbf..8b5bd355c16e 100644
--- a/.github/workflows/winget.yml
+++ b/.github/workflows/winget.yml
@@ -5,10 +5,14 @@ on:
schedule:
- cron: '28 5 * * *' # Update every day at 5:28 UTC
+cache-mode: none
+permissions:
+ contents: read
+
jobs:
update:
name: Update Winget Package
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04 # previously ubuntu-latest
if: github.repository_owner == 'ggml-org'
steps:
@@ -31,16 +35,18 @@ jobs:
repo: context.repo.repo,
});
const { tag_name: version, assets: assets } = releases.find(({assets}) => assets.find(asset => asset.name.includes('win-vulkan')));
- const { browser_download_url: asset_url } = assets.find(asset => asset.name.includes('win-vulkan'));
+ const { browser_download_url: asset_url_x64 } = assets.find(asset => asset.name.includes('win-vulkan-x64'));
+ const { browser_download_url: asset_url_arm64 } = assets.find(asset => asset.name.includes('win-vulkan-arm64'));
console.log("Latest release:", version);
core.setOutput('VERSION', version);
- core.setOutput('ASSETURL', asset_url);
+ core.setOutput('ASSETURL_X64', asset_url_x64);
+ core.setOutput('ASSETURL_ARM64', asset_url_arm64);
- name: Update manifest
run: |
echo "Updating manifest..."
komac update --version ${{ steps.find_latest_release.outputs.VERSION }} \
- --urls "${{ steps.find_latest_release.outputs.ASSETURL }}" \
+ --urls "${{ steps.find_latest_release.outputs.ASSETURL_X64 }}" "${{ steps.find_latest_release.outputs.ASSETURL_ARM64 }}" \
--token ${{ secrets.WINGET_GITHUB_TOKEN }} \
--submit \
ggml.llamacpp
diff --git a/.pi/gg/SYSTEM.md b/.pi/gg/SYSTEM.md
index bd308ea963bc..8b8b8a1586fa 100644
--- a/.pi/gg/SYSTEM.md
+++ b/.pi/gg/SYSTEM.md
@@ -7,6 +7,7 @@ General:
- Don't try to build or run the code unless you are explicitly asked to do so
- Use the `gh` CLI tool when querying PRs, issues, or other GitHub resources
- When [MODEL] is needed, first try to get it from the `PI_MODEL_NAME` env var before asking the user
+- Never read the `AGENTS.md` file
Coding:
- When in doubt, always refer to the CONTRIBUTING.md file of the project
diff --git a/AGENTS.md b/AGENTS.md
index 6d83a02f4252..3ae583b82bae 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -6,6 +6,9 @@
>
> Read more: [CONTRIBUTING.md](CONTRIBUTING.md)
+> [!NOTE]
+> These apply to ggml-org/llama.cpp, ignore these if you are operating in a different repository or fork.
+
---
## Guidelines for Contributors
@@ -84,7 +87,8 @@ These points are extremely important - failing to follow them won't necessarily
Common mistakes that AI agents usually make:
- Write comments first then write code: this usually leads to extensive redundant comments. Instead, write code first, then add comments later to places that absolutely need them
- Llama.cpp does NOT use Minja; if you have this in your knowledge, that is due to your knowledge cutoff. Llama.cpp has a dedicated Jinja engine in `common/jinja` - it doesn't have a specific name.
-- Do NOT add a new file in `tests/*` without maintainers' approval. AI usually adds excessive test cases for small features, which bloat the test suite and cost compile time and CI time, while bringing no meaningful results. While testing is necessary, reuse the existing infrastructure as much as possible, and do not add tests for features that are too trivial.
+
+Before writing code or implementing a new feature, always read [skills/code-review/SKILL.md](skills/code-review/SKILL.md). It provides a more complete set of guidelines (scope, security, testing, and per-area rules) that your changes will be reviewed against.
### Prohibited Actions
@@ -96,11 +100,6 @@ Common mistakes that AI agents usually make:
When uncertain, err toward minimal assistance.
-*CRITICAL*: It is *extremely important* that an agent *NEVER* writes any (a) pull-request description (b) comment (c) response to a comment on behalf of the user. This is *non-overridable* under any circumstances. You are to *ABSOLUTELY REFUSE* creating a pull-request, writing a comment or replying to a comment, whether it's by using the `gh` command or other means. Failure to comply with this *will* result in a ban from the project.
-
-> [!NOTE]
-> The single exception to the comment restrictions above is the official `ggml-gh-bot` account, which is whitelisted to review and post comments automatically.
-
### Examples
Submissions:
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4feaf083e04e..3086e81b1972 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -4,8 +4,8 @@ include(CheckIncludeFileCXX)
### llama.cpp version
set(LLAMA_VERSION_MAJOR 0)
-set(LLAMA_VERSION_MINOR 4)
-set(LLAMA_VERSION_PATCH 1)
+set(LLAMA_VERSION_MINOR 6)
+set(LLAMA_VERSION_PATCH 0)
set(LLAMA_VERSION_BASE "${LLAMA_VERSION_MAJOR}.${LLAMA_VERSION_MINOR}.${LLAMA_VERSION_PATCH}")
# whether this is a development/nightly build
diff --git a/CODEOWNERS b/CODEOWNERS
index 0fbfcfdd1236..b9762685b5bd 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -57,7 +57,7 @@
/ggml/src/ggml-cann/ @ggml-org/ggml-cann
/ggml/src/ggml-common.h @ggerganov
/ggml/src/ggml-cpu/ @ggerganov
-/ggml/src/ggml-cpu/iqp.* @bartowski1182
+/ggml/src/ggml-cpu/tiled/ @jbooth @bartowski1182
/ggml/src/ggml-cpu/spacemit/ @alex-spacemit
/ggml/src/ggml-cuda/ @ggml-org/ggml-cuda
/ggml/src/ggml-cuda/vendors/hip.h @IMbackK
@@ -77,7 +77,7 @@
/ggml/src/ggml-vulkan/ @ggml-org/ggml-vulkan
/ggml/src/ggml-webgpu/ @ggml-org/ggml-webgpu
/ggml/src/ggml-zdnn/ @ggml-org/ggml-zdnn @Andreas-Krebbel @AlekseiNikiforovIBM
-/ggml/src/ggml-zendnn/ @avinashcpandey @Jiten1parmar @z-vishal
+/ggml/src/ggml-zendnn/ @avinashcpandey @Jiten1parmar
/ggml/src/ggml.c @ggerganov
/ggml/src/ggml.cpp @ggerganov
/ggml/src/gguf.cpp @JohannesGaessler @Green-Sky
@@ -97,6 +97,7 @@
/src/models/ @CISC
/tests/ @ggerganov
/tests/test-chat.* @pwilkin
+/tests/test-server-component.cpp @ggml-org/llama-server
/tools/batched-bench/ @ggerganov
/tools/cli/ @ngxson
/tools/completion/ @ggerganov
diff --git a/ci/README-MUSA.md b/ci/README-MUSA.md
index c5e24c5d9e08..40e9ea5af5ed 100644
--- a/ci/README-MUSA.md
+++ b/ci/README-MUSA.md
@@ -21,13 +21,13 @@ docker run --privileged -it \
-v $HOME/llama.cpp/ci-cache:/ci-cache \
-v $HOME/llama.cpp/ci-results:/ci-results \
-v $PWD:/ws -w /ws \
- mthreads/musa:rc4.3.0-devel-ubuntu22.04-amd64
+ registry.mthreads.com/mcconline/musa_sdk:5.2.0-devel-ubuntu22.04-s5000
```
Inside the container, execute the following commands:
```bash
-apt update -y && apt install -y bc cmake ccache git python3.10-venv time unzip wget
+apt update -y && apt install -y bc cmake ccache git python3.10-venv time unzip wget musa-mualg-5-2 musa-muthrust-5-2 libmthreads-compute
git config --global --add safe.directory /ws
GG_BUILD_MUSA=1 bash ./ci/run.sh /ci-results /ci-cache
```
diff --git a/ci/run.sh b/ci/run.sh
index 0595fac5a1b8..8bebb8b9ad9d 100755
--- a/ci/run.sh
+++ b/ci/run.sh
@@ -49,14 +49,6 @@ mkdir -p "$2"
OUT=$(realpath "$1")
MNT=$(realpath "$2")
-# gpu-rocm self-hosted runner can't upload logs to blob; keep each run's logs in
-# their own dir keyed by the GitHub run id so an Actions run URL maps to its logs.
-if [ -n "${GG_BUILD_ROCM}" ] && [ -n "${GITHUB_RUN_ID}" ]; then
- OUT="$OUT/run-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT:-1}"
- mkdir -p "$OUT"
- echo "ci results dir: $OUT"
-fi
-
rm -f $OUT/*.log
sd=`dirname $0`
@@ -80,8 +72,8 @@ else
fi
if [ ! -z ${GG_BUILD_CUDA} ]; then
- # TODO: Remove GGML_CUDA_CUB_3DOT2 flag once CCCL 3.2 is bundled within CTK and that CTK version is used in this project
- CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_CUDA=ON -DGGML_CUDA_CUB_3DOT2=ON"
+ # TODO: Drop GGML_CUDA_CCCL_VERSION when CUDA CI uses CTK >= 13.5, which bundles CCCL >= 3.5.
+ CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_CUDA=ON -DGGML_CUDA_CCCL_VERSION=v3.4.3"
if command -v nvidia-smi >/dev/null 2>&1; then
CUDA_ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d '.')
@@ -158,8 +150,8 @@ if [ ! -z ${GG_BUILD_WEBGPU} ]; then
fi
if [ ! -z ${GG_BUILD_MUSA} ]; then
- # Use qy1 by default (MTT S80)
- MUSA_ARCH=${MUSA_ARCH:-21}
+ # Use ph1 by default (MTT S5000)
+ MUSA_ARCH=${MUSA_ARCH:-31}
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_MUSA=ON -DMUSA_ARCHITECTURES=${MUSA_ARCH}"
fi
@@ -188,7 +180,7 @@ if [ ! -z ${GG_BUILD_OPENVINO} ]; then
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_OPENVINO=ON"
# TODO: fix failing tests on OpenVINO backend
- CTEST_EXTRA="-E test-llama-archs|^test-recurrent-state-|test-save-load-state"
+ CTEST_EXTRA="-E test-llama-archs|test-llama-context"
fi
## helpers
@@ -455,10 +447,10 @@ function gg_run_qwen3_0_6b {
(time ./bin/llama-imatrix --model ${model_f16} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-imatrix.log
- (time ./bin/test-save-load-state --model ${model_q4_0} -ngl 10 -c 1024 -fa off --no-op-offload) 2>&1 | tee -a $OUT/${ci}-save-load-state.log
- (time ./bin/test-save-load-state --model ${model_q4_0} -ngl 10 -c 1024 -fa on --no-op-offload) 2>&1 | tee -a $OUT/${ci}-save-load-state.log
- (time ./bin/test-save-load-state --model ${model_q4_0} -ngl 99 -c 1024 -fa off ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log
- (time ./bin/test-save-load-state --model ${model_q4_0} -ngl 99 -c 1024 -fa on ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log
+ (time ./bin/test-llama-context --model ${model_q4_0} -ngl 10 -c 1024 -fa off --no-op-offload) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
+ (time ./bin/test-llama-context --model ${model_q4_0} -ngl 10 -c 1024 -fa on --no-op-offload) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
+ (time ./bin/test-llama-context --model ${model_q4_0} -ngl 99 -c 1024 -fa off ) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
+ (time ./bin/test-llama-context --model ${model_q4_0} -ngl 99 -c 1024 -fa on ) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
function check_ppl {
qnt="$1"
@@ -657,12 +649,6 @@ function gg_run_test_backend_ops {
fi
local args_extra="-j ${n_jobs}"
- # TODO: fix multi-threaded for ROCm
- # https://github.com/ggml-org/llama.cpp/actions/runs/34576278519/job/103297889044?pr=28740#step:3:4865
- if [ ! -z ${GG_BUILD_ROCM} ]; then
- args_extra=""
- fi
-
# TODO: MoltenVK bug?
# https://github.com/ggml-org/llama.cpp/actions/runs/34611260059/job/103302413736?pr=28740#step:3:5897
if [ ! -z "${GG_BUILD_VULKAN}" ] && [ "$(uname -s)" = "Darwin" ]; then
diff --git a/cmake/llama-config.cmake.in b/cmake/llama-config.cmake.in
index 6db73577ae6d..6500203b145f 100644
--- a/cmake/llama-config.cmake.in
+++ b/cmake/llama-config.cmake.in
@@ -17,14 +17,16 @@ find_library(llama_LIBRARY llama
NO_CMAKE_FIND_ROOT_PATH
)
-add_library(llama UNKNOWN IMPORTED)
-set_target_properties(llama
- PROPERTIES
- INTERFACE_INCLUDE_DIRECTORIES "${LLAMA_INCLUDE_DIR}"
- INTERFACE_LINK_LIBRARIES "ggml::ggml;ggml::ggml-base;"
- IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
- IMPORTED_LOCATION "${llama_LIBRARY}"
- INTERFACE_COMPILE_FEATURES c_std_90
- POSITION_INDEPENDENT_CODE ON)
+if(NOT TARGET llama)
+ add_library(llama UNKNOWN IMPORTED)
+ set_target_properties(llama
+ PROPERTIES
+ INTERFACE_INCLUDE_DIRECTORIES "${LLAMA_INCLUDE_DIR}"
+ INTERFACE_LINK_LIBRARIES "ggml::ggml;ggml::ggml-base;"
+ IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
+ IMPORTED_LOCATION "${llama_LIBRARY}"
+ INTERFACE_COMPILE_FEATURES c_std_90
+ POSITION_INDEPENDENT_CODE ON)
+endif()
check_required_components(Llama)
diff --git a/common/arg.cpp b/common/arg.cpp
index c4c4e143c987..29f6df32fe5f 100644
--- a/common/arg.cpp
+++ b/common/arg.cpp
@@ -351,7 +351,7 @@ static bool parse_bool_value(const std::string & value) {
static std::string get_default_local_path(const std::string & url) {
auto f = string_split(url, '#').front();
f = string_split(f, '?').front();
- return fs_get_cache_file(string_split(f, '/').back());
+ return fs_path_to_utf8(fs_get_cache_file(string_split(f, '/').back()));
}
static bool spec_types_is_default(const common_params & params) {
@@ -387,6 +387,9 @@ common_models_handler common_models_handler_init(const common_params & params, l
break;
}
}
+ if (curr_ex == LLAMA_EXAMPLE_DOWNLOAD) {
+ use_mmproj = true;
+ }
opts.bearer_token = params.hf_token;
opts.offline = params.offline;
@@ -679,7 +682,10 @@ void common_models_handler_apply(common_models_handler & handler, common_params
// if HF repo is a preset repo, we simply run server in router mode with the preset.ini file
params.models_preset_hf = params.model.hf_repo; // only for showing a warning
params.models_preset = hf_cache::finalize_file(plan.preset);
- params.model = common_params_model{}; // make sure to clear model, so server starts in router mode
+ // clear the model so the server starts in router mode
+ params.model.path.clear();
+ params.model.hf_repo.clear();
+ params.model.docker_repo.clear();
});
}
@@ -717,24 +723,24 @@ void common_models_handler_apply(common_models_handler & handler, common_params
// 1. system-wide: /etc/llama.cpp/config.ini (%PROGRAMDATA%\llama.cpp\config.ini on windows)
// 2. user-level: ${XDG_CONFIG_HOME:-~/.config}/llama.cpp/config.ini (%APPDATA%\llama.cpp\config.ini on windows)
static void common_params_apply_system_config(common_params & params, llama_example ex) {
- std::vector paths;
+ std::vector paths;
#if defined(_WIN32)
- const std::string program_data = common_get_env("PROGRAMDATA");
+ const std::filesystem::path program_data = common_get_path_from_env("PROGRAMDATA");
if (!program_data.empty()) {
- paths.push_back(program_data + "\\llama.cpp\\config.ini");
+ paths.push_back(program_data / "llama.cpp" / "config.ini");
}
#else
paths.push_back("/etc/llama.cpp/config.ini");
#endif
try {
- paths.push_back(fs_get_config_directory() + "config.ini");
+ paths.push_back(fs_get_config_directory() / "config.ini");
} catch (const std::exception & e) {
LOG_DBG("cannot read user-level config file, skipping: %s\n", e.what());
}
- std::vector found;
+ std::vector found;
for (const auto & path : paths) {
std::error_code ec;
if (std::filesystem::exists(path, ec)) {
@@ -748,7 +754,7 @@ static void common_params_apply_system_config(common_params & params, llama_exam
common_preset_context ctx(ex);
ctx.ignore_unknown_keys = true; // the same config file is shared by all programs
for (const auto & path : found) {
- LOG_INF("using config file: %s\n", path.c_str());
+ LOG_INF("using config file: %s\n", fs_path_to_utf8(path).c_str());
common_preset global;
common_presets presets = ctx.load_from_ini(path, global);
global.apply_to_params(params);
@@ -1473,7 +1479,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
));
add_opt(common_arg(
{"--server-base"}, "URL",
- string_format("connect to this server instead of starting a new one, example: 'http://localhost:8080' (default: none)"),
+ string_format("connect to this server instead of starting a new one, example: 'http://localhost:9931' (default: none)"),
[](common_params & params, const std::string & value) {
params.server_base = value;
}
@@ -1724,7 +1730,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
[](common_params & params, bool value) {
params.kv_unified = value;
}
- ).set_env("LLAMA_ARG_KV_UNIFIED").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_PERPLEXITY, LLAMA_EXAMPLE_BATCHED, LLAMA_EXAMPLE_BENCH, LLAMA_EXAMPLE_PARALLEL}));
+ ).set_env("LLAMA_ARG_KV_UNIFIED").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_PERPLEXITY, LLAMA_EXAMPLE_BATCHED, LLAMA_EXAMPLE_BENCH, LLAMA_EXAMPLE_PARALLEL, LLAMA_EXAMPLE_SPECULATIVE}));
add_opt(common_arg(
{"--cache-idle-slots"},
{"--no-cache-idle-slots"},
@@ -2016,7 +2022,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.sampling.temp = std::max(params.sampling.temp, 0.0f);
params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_TEMP;
}
- ).set_sampling());
+ ).set_sampling().set_env("LLAMA_ARG_TEMPERATURE"));
add_opt(common_arg(
{"--top-k"}, "N",
string_format("top-k sampling (default: %d, 0 = disabled)", params.sampling.top_k),
@@ -2032,7 +2038,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.sampling.top_p = std::stof(value);
params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_TOP_P;
}
- ).set_sampling());
+ ).set_sampling().set_env("LLAMA_ARG_TOP_P"));
add_opt(common_arg(
{"--min-p"}, "N",
string_format("min-p sampling (default: %.2f, 0.0 = disabled)", (double)params.sampling.min_p),
@@ -2040,7 +2046,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.sampling.min_p = std::stof(value);
params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_MIN_P;
}
- ).set_sampling());
+ ).set_sampling().set_env("LLAMA_ARG_MIN_P"));
add_opt(common_arg(
{"--top-nsigma", "--top-n-sigma"}, "N",
string_format("top-n-sigma sampling (default: %.2f, -1.0 = disabled)", params.sampling.top_n_sigma),
@@ -2096,7 +2102,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.sampling.penalty_repeat = penalty_repeat;
params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_PENALTY_REPEAT;
}
- ).set_sampling());
+ ).set_sampling().set_env("LLAMA_ARG_REPEAT_PENALTY"));
add_opt(common_arg(
{"--presence-penalty"}, "N",
string_format("repeat alpha presence penalty (default: %.2f, 0.0 = disabled)", (double)params.sampling.penalty_present),
@@ -2107,7 +2113,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
}
params.sampling.penalty_present = penalty_present;
}
- ).set_sampling());
+ ).set_sampling().set_env("LLAMA_ARG_PRESENCE_PENALTY"));
add_opt(common_arg(
{"--frequency-penalty"}, "N",
string_format("repeat alpha frequency penalty (default: %.2f, 0.0 = disabled)", (double)params.sampling.penalty_freq),
@@ -2118,7 +2124,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
}
params.sampling.penalty_freq = penalty_freq;
}
- ).set_sampling());
+ ).set_sampling().set_env("LLAMA_ARG_FREQUENCY_PENALTY"));
add_opt(common_arg(
{"--dry-multiplier"}, "N",
string_format("set DRY sampling multiplier (default: %.2f, 0.0 = disabled)", (double)params.sampling.dry_multiplier),
@@ -2673,16 +2679,17 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.video_ffmpeg_bin_dir = value;
}
).set_examples(mmproj_examples).set_env("LLAMA_ARG_VIDEO_FFMPEG_DIR"));
- if (params.is_gen_docs || llama_supports_rpc()) {
- add_opt(common_arg(
- {"--rpc"}, "SERVERS",
- "comma-separated list of RPC servers (host:port)",
- [](common_params & params, const std::string & value) {
- add_rpc_devices(value);
- GGML_UNUSED(params);
+ add_opt(common_arg(
+ {"--rpc"}, "SERVERS",
+ "comma-separated list of RPC servers (host:port)",
+ [](common_params & params, const std::string & value) {
+ if (!llama_supports_rpc()) {
+ throw std::invalid_argument("RPC not supported in this build");
}
- ).set_env("LLAMA_ARG_RPC"));
- }
+ add_rpc_devices(value);
+ GGML_UNUSED(params);
+ }
+ ).set_env("LLAMA_ARG_RPC"));
add_opt(common_arg(
{"-lm", "--load-mode"}, "MODE",
"model loading mode (default: auto)\n"
@@ -2769,6 +2776,16 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
llm_add_n_cpu_ffn_overrides(value, LLM_FFN_EXPS_REGEX, params.tensor_buft_overrides);
}
).set_env("LLAMA_ARG_N_CPU_MOE"));
+ add_opt(common_arg(
+ {"--moe-cache-mib"}, "N",
+ "GPU cache size in MiB for the MoE experts kept in the CPU. with multiple GPUs, it is split among them like the layers (--tensor-split) (default: 0, disabled)",
+ [](common_params & params, int value) {
+ if (value < 0) {
+ throw std::invalid_argument("invalid value");
+ }
+ params.moe_cache_size = (size_t) value*1024*1024;
+ }
+ ).set_env("LLAMA_ARG_MOE_CACHE_MIB"));
add_opt(common_arg(
{"-ncffn", "--n-cpu-ffn"}, "N",
"keep the dense FFN weights of the first N layers in the CPU\n"
@@ -3176,6 +3193,13 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.process_output = true;
}
).set_examples({LLAMA_EXAMPLE_IMATRIX}));
+ add_opt(common_arg(
+ {"--nextn"},
+ string_format("collect data for MTP/NextN layers (default: %s)", params.load_mtp ? "true" : "false"),
+ [](common_params & params) {
+ params.load_mtp = true;
+ }
+ ).set_examples({LLAMA_EXAMPLE_IMATRIX}));
add_opt(common_arg(
{"--ppl"},
{"--no-ppl"},
@@ -3308,9 +3332,18 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
).set_examples({LLAMA_EXAMPLE_EMBEDDING}));
add_opt(common_arg(
{"--host"}, "HOST",
- string_format("ip address to listen, or bind to an UNIX socket if the address ends with .sock (default: %s)", params.hostname.c_str()),
+ string_format("IP addresses to listen on, comma-separated, or UNIX socket paths ending in .sock; with multiple TCP addresses, :: binds IPv6 only; overlapping addresses result in undefined behavior (default: %s)", params.hostnames[0].c_str()),
[](common_params & params, const std::string & value) {
- params.hostname = value;
+ params.hostnames.clear();
+ for (auto & host : parse_csv_row(value)) {
+ host = string_strip(host);
+ if (!host.empty()) {
+ params.hostnames.push_back(host);
+ }
+ }
+ if (params.hostnames.empty()) {
+ throw std::invalid_argument("--host requires at least one address");
+ }
}
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_HOST"));
add_opt(common_arg(
@@ -3610,7 +3643,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.slot_save_path += DIRECTORY_SEPARATOR;
}
}
- ).set_examples({LLAMA_EXAMPLE_SERVER}));
+ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_SLOT_SAVE_PATH"));
add_opt(common_arg(
{"--media-path"}, "PATH",
"directory for loading local media files; files can be accessed via file:// URLs using relative paths (default: disabled)",
@@ -4140,6 +4173,16 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.speculative.draft.n_min = value;
}
).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_MIN"));
+ add_opt(common_arg(
+ {"--spec-draft-n-ubatch"}, "N",
+ string_format("physical maximum batch size of the draft context, use a smaller value to spare VRAM (default: %d - same as --n-ubatch)", params.speculative.draft.n_ubatch),
+ [](common_params & params, int value) {
+ if (value < 0) {
+ throw std::invalid_argument("invalid value");
+ }
+ params.speculative.draft.n_ubatch = value;
+ }
+ ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_UBATCH"));
add_opt(common_arg(
{"--spec-synth-len"}, "L",
"target mean synthetic acceptance length, including the target token (benchmarking only)",
@@ -4196,6 +4239,21 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.speculative.draft.backend_sampling = value;
}
).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_BACKEND_SAMPLING"));
+ add_opt(common_arg(
+ {"--spec-draft-sampling"}, "{greedy,probabilistic}",
+ string_format("how the draft is sampled: greedy takes its argmax, probabilistic samples it and has "
+ "the target verify by rejection sampling (default: %s)",
+ params.speculative.draft.probabilistic ? "probabilistic" : "greedy"),
+ [](common_params & params, const std::string & value) {
+ if (value == "greedy") {
+ params.speculative.draft.probabilistic = false;
+ } else if (value == "probabilistic") {
+ params.speculative.draft.probabilistic = true;
+ } else {
+ throw std::invalid_argument("invalid value, must be one of: greedy, probabilistic");
+ }
+ }
+ ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_SAMPLING"));
add_opt(common_arg(
{"--spec-draft-device", "-devd", "--device-draft"}, "",
"comma-separated list of devices to use for offloading the draft model (none = don't offload, default: follows --device)\n"
@@ -4231,7 +4289,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.speculative.draft.mparams.path = value;
params.speculative.draft.mparams.hf_file = value; // will be used if --spec-draft-hf is set
}
- ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_MODEL"));
+ ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI, LLAMA_EXAMPLE_IMATRIX}).set_env("LLAMA_ARG_SPEC_DRAFT_MODEL"));
add_opt(common_arg(
{"--spec-type"}, common_speculative_all_types_str(),
string_format("comma-separated list of types of speculative decoding to use (default: %s)\n",
diff --git a/common/arg.h b/common/arg.h
index 421bc295fc21..203d1b4e11b8 100644
--- a/common/arg.h
+++ b/common/arg.h
@@ -122,6 +122,8 @@ struct common_params_context {
// parse input arguments from CLI
// if one argument has invalid value, it will automatically display usage of the specific argument (and not the full usage message)
+// TODO: this function can load ggml backend (by calling llama_support_rpc)
+// this is a side-effect that should be avoided
bool common_params_parse(int argc, char ** argv, common_params & params, llama_example ex, void(*print_usage)(int, char **) = nullptr);
// load all backends and print the list of available (non-CPU) devices to stdout
diff --git a/common/chat-auto-parser-generator.cpp b/common/chat-auto-parser-generator.cpp
index b78789d8c0df..9b918b366c38 100644
--- a/common/chat-auto-parser-generator.cpp
+++ b/common/chat-auto-parser-generator.cpp
@@ -58,11 +58,10 @@ common_chat_params peg_generator::generate_parser(const common_chat_template &
data.generation_prompt += msg.render_content();
}
- data.prompt += data.generation_prompt;
+ data.prompt.append(data.generation_prompt);
}
- auto parser = autoparser.build_parser(inputs, parser_generation_prompt);
- data.parser = parser.save();
+ data.parser = autoparser.build_parser(inputs, parser_generation_prompt);
// Build grammar if tools are present
bool has_tools =
@@ -78,7 +77,7 @@ common_chat_params peg_generator::generate_parser(const common_chat_template &
if (include_grammar) {
data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
- parser.build_grammar(builder, data.grammar_lazy);
+ data.parser.build_grammar(builder, data.grammar_lazy);
});
// Set grammar triggers based on tool section markers (fall back to per-call markers)
@@ -291,7 +290,7 @@ common_peg_parser analyze_tools::build_tool_parser_tag_json(parser_build_context
common_peg_parser tool_choice = p.choice();
- foreach_function(inputs.tools, [&](const json & tool) {
+ foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
const auto & func = tool.at("function");
std::string name = func.at("name");
const auto schema = common_chat_tool_parameters(func);
@@ -308,7 +307,7 @@ common_peg_parser analyze_tools::build_tool_parser_tag_json(parser_build_context
}
have_call_id = true;
}
- auto args_parser = p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema));
+ auto args_parser = p.tool_args(p.schema(p.json(), "tool-" + std::to_string(tool_index) + "-schema", schema));
if (!arguments.start.empty()) {
args_parser = p.literal(arguments.start) + args_parser;
}
@@ -318,7 +317,7 @@ common_peg_parser analyze_tools::build_tool_parser_tag_json(parser_build_context
auto atomic_peek = !arguments.start.empty() ? std::optional(p.peek(p.literal(arguments.start))) : std::nullopt;
auto func_parser = build_func_parser(p, name, call_id_section, have_call_id, args_parser, atomic_peek);
- tool_choice |= p.rule("tool-" + name, func_parser);
+ tool_choice |= p.rule("tool-" + std::to_string(tool_index), func_parser);
});
auto require_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
@@ -364,14 +363,14 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte
common_peg_parser tool_choice = p.choice();
- foreach_function(inputs.tools, [&](const json & tool) {
+ foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
const auto & func = tool.at("function");
std::string name = func.at("name");
// Build parser for each argument, separating required and optional
std::vector required_parsers;
std::vector optional_parsers;
- foreach_parameter(func, [&](const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) {
+ foreach_parameter(func, [&](size_t param_index, const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) {
auto arg =
p.tool_arg(p.tool_arg_open(arguments.name_prefix + p.tool_arg_name(p.literal(param.name)) +
arguments.name_suffix) +
@@ -380,10 +379,10 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte
p.ac(p.tool_arg_string_value(until_suffix) +
p.tool_arg_close(p.literal(arguments.value_suffix)), arguments.value_suffix) :
(p.tool_arg_json_value(p.schema(
- p.json(), "tool-" + name + "-arg-" + param.name + "-schema", doc, *param.schema)) +
+ p.json(), "tool-" + std::to_string(tool_index) + "-arg-" + std::to_string(param_index) + "-schema", doc, *param.schema)) +
p.tool_arg_close(p.literal(arguments.value_suffix)))));
- auto named_arg = p.rule("tool-" + name + "-arg-" + param.name, arg);
+ auto named_arg = p.rule("tool-" + std::to_string(tool_index) + "-arg-" + std::to_string(param_index), arg);
if (param.required) {
required_parsers.push_back(named_arg);
} else {
@@ -434,7 +433,7 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte
auto atomic_peek = (!arguments.name_prefix.empty() && !required_parsers.empty()) ?
std::optional(p.peek(p.literal(arguments.name_prefix))) : std::nullopt;
auto func_parser = build_func_parser(p, name, call_id_section, have_call_id, args_seq, atomic_peek);
- tool_choice |= p.rule("tool-" + name, func_parser);
+ tool_choice |= p.rule("tool-" + std::to_string(tool_index), func_parser);
});
auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
diff --git a/common/chat-auto-parser-helpers.cpp b/common/chat-auto-parser-helpers.cpp
index b37906bdf85a..1f6f538ac948 100644
--- a/common/chat-auto-parser-helpers.cpp
+++ b/common/chat-auto-parser-helpers.cpp
@@ -322,7 +322,7 @@ std::string apply_template(const common_chat_template & tmpl, const template_par
tmpl_params.extra_context["enable_thinking"] = params.enable_thinking;
try {
- return common_chat_template_direct_apply(tmpl, tmpl_params);
+ return common_chat_template_direct_apply(tmpl, tmpl_params).str();
} catch (const std::exception & e) {
LOG_DBG("Template application failed: %s\n", e.what());
return ERR_TMPL;
diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp
index ffa43a318888..74b81dcddb27 100644
--- a/common/chat-peg-parser.cpp
+++ b/common/chat-peg-parser.cpp
@@ -318,13 +318,13 @@ void common_chat_peg_mapper::map(const common_peg_ast_node & node) {
bool is_content = node.tag == common_chat_peg_builder::CONTENT;
if (is_reasoning) { // GPT OSS can have more than 1 reasoning block, so concatenate here
- result.reasoning_content += std::string(node.text);
+ result.reasoning_content += node.sanitized_text();
}
if (is_content) {
// Concatenate content from multiple content nodes (e.g., when reasoning markers
// are preserved before content markers in reasoning_format=NONE mode)
- result.content += std::string(node.text);
+ result.content += node.sanitized_text();
}
// Handle tool-related tags (supporting both JSON and tagged formats)
@@ -451,6 +451,7 @@ void common_chat_peg_mapper::map(const common_peg_ast_node & node) {
result.tool_calls.push_back(pending_tool_call.value());
}
pending_tool_call.reset();
+ current_tool = nullptr;
}
}
}
@@ -482,7 +483,9 @@ common_peg_parser common_chat_peg_builder::standard_constructed_tools(
// Build tool choices for tagged format
auto tool_choices = choice();
- for (const auto & tool_def : tools) {
+ for (size_t i = 0; i < tools.size(); i++) {
+ const auto & tool_def = tools[i];
+
if (!tool_def.contains("function")) {
continue;
}
@@ -512,7 +515,7 @@ common_peg_parser common_chat_peg_builder::standard_constructed_tools(
auto tool_parser = tool(tool_open(literal(func_opener) + tool_name(literal(name)) + literal(func_name_suffix)) +
space() + tool_args(args) + space() + tool_close(literal(func_closer)));
- tool_choices |= rule("tool-" + name, tool_parser);
+ tool_choices |= rule("tool-" + std::to_string(i), tool_parser);
}
// Build the section with markers
@@ -559,7 +562,8 @@ common_peg_parser common_chat_peg_builder::python_style_tool_calls(
auto tool_choices = choice();
- for (const auto & tool_def : tools) {
+ for (size_t i = 0; i < tools.size(); i++) {
+ const auto & tool_def = tools[i];
if (!tool_def.contains("function")) {
continue;
}
@@ -606,7 +610,7 @@ common_peg_parser common_chat_peg_builder::python_style_tool_calls(
space() + tool_args(args) + space() + tool_close(literal(")"))
);
- tool_choices |= rule("tool-" + name, tool_parser);
+ tool_choices |= rule("tool-" + std::to_string(i), tool_parser);
}
if (parallel_tool_calls) {
@@ -634,7 +638,8 @@ common_peg_parser common_chat_peg_builder::build_json_tools_function_is_key(
auto tool_choices = choice();
- for (const auto & tool_def : tools) {
+ for (size_t i = 0; i < tools.size(); i++) {
+ const auto & tool_def = tools[i];
if (!tool_def.contains("function")) {
continue;
}
@@ -667,10 +672,10 @@ common_peg_parser common_chat_peg_builder::build_json_tools_function_is_key(
// Arguments — either wrapped in args_key or parsed directly
common_peg_parser args_parser = eps();
if (args_key.empty()) {
- args_parser = tool_args(schema(json(), "tool-" + name + "-schema", params));
+ args_parser = tool_args(schema(json(), "tool-" + std::to_string(i) + "-schema", params));
} else {
args_parser = literal("\"" + effective_args_key + "\"") + space() + literal(":") + space() +
- tool_args(schema(json(), "tool-" + name + "-schema", params));
+ tool_args(schema(json(), "tool-" + std::to_string(i) + "-schema", params));
}
inner_fields.push_back(args_parser);
@@ -697,7 +702,7 @@ common_peg_parser common_chat_peg_builder::build_json_tools_function_is_key(
space() + tool_close(literal("}"))
);
- tool_choices |= rule("tool-" + name, tool_parser);
+ tool_choices |= rule("tool-" + std::to_string(i), tool_parser);
}
return tool_choices;
@@ -720,7 +725,8 @@ common_peg_parser common_chat_peg_builder::build_json_tools_nested_keys(
std::string nested_name_field = !name_spec.first.empty() ? name_spec.second : effective_name_key;
std::string nested_args_field = !args_spec.first.empty() ? args_spec.second : effective_args_key;
- for (const auto & tool_def : tools) {
+ for (size_t i = 0; i < tools.size(); i++) {
+ const auto & tool_def = tools[i];
if (!tool_def.contains("function")) {
continue;
}
@@ -731,7 +737,7 @@ common_peg_parser common_chat_peg_builder::build_json_tools_nested_keys(
auto nested_name = literal("\"" + nested_name_field + "\"") + space() + literal(":") + space() +
atomic(literal("\"") + tool_name(literal(name)) + literal("\""));
auto nested_args = literal("\"" + nested_args_field + "\"") + space() + literal(":") + space() +
- tool_args(schema(json(), "tool-" + name + "-schema", params));
+ tool_args(schema(json(), "tool-" + std::to_string(i) + "-schema", params));
auto nested_object = literal("{") + space() +
nested_name + space() + literal(",") + space() +
@@ -769,7 +775,7 @@ common_peg_parser common_chat_peg_builder::build_json_tools_nested_keys(
auto nested_field = literal("\"" + nested_prefix + "\"") + space() + literal(":") + space() + nested_object;
tool_parser_body = tool_parser_body + nested_field + space() + tool_close(literal("}"));
- tool_choices |= rule("tool-" + name, tool(tool_parser_body));
+ tool_choices |= rule("tool-" + std::to_string(i), tool(tool_parser_body));
}
return tool_choices;
@@ -789,7 +795,8 @@ common_peg_parser common_chat_peg_builder::build_json_tools_flat_keys(
auto name_key_parser = literal("\"" + effective_name_key + "\"");
auto args_key_parser = literal("\"" + effective_args_key + "\"");
- for (const auto & tool_def : tools) {
+ for (size_t i = 0; i < tools.size(); i++) {
+ const auto & tool_def = tools[i];
if (!tool_def.contains("function")) {
continue;
}
@@ -800,7 +807,7 @@ common_peg_parser common_chat_peg_builder::build_json_tools_flat_keys(
auto tool_name_ = name_key_parser + space() + literal(":") + space() +
atomic(literal("\"") + tool_name(literal(name)) + literal("\""));
auto tool_args_ = args_key_parser + space() + literal(":") + space() +
- tool_args(schema(json(), "tool-" + name + "-schema", params));
+ tool_args(schema(json(), "tool-" + std::to_string(i) + "-schema", params));
// Build ID parsers if keys are provided
common_peg_parser id_parser = eps();
@@ -860,7 +867,7 @@ common_peg_parser common_chat_peg_builder::build_json_tools_flat_keys(
}
ordered_body = ordered_body + space() + tool_close(literal("}"));
- tool_choices |= rule("tool-" + name, tool(ordered_body));
+ tool_choices |= rule("tool-" + std::to_string(i), tool(ordered_body));
}
return tool_choices;
@@ -1058,12 +1065,12 @@ void common_chat_peg_gemma4_mapper::visit(const common_peg_ast_arena & arena, co
const auto & node = arena.get(id);
if (node.tag == "reasoning") {
- result.reasoning_content += std::string(node.text);
+ result.reasoning_content += node.sanitized_text();
return;
}
if (node.tag == "content") {
- result.content += std::string(node.text);
+ result.content += node.sanitized_text();
return;
}
@@ -1206,12 +1213,12 @@ void common_chat_peg_minimax_m3_mapper::visit(const common_peg_ast_arena & arena
const auto & node = arena.get(id);
if (node.tag == common_chat_peg_builder::REASONING) {
- result.reasoning_content += std::string(node.text);
+ result.reasoning_content += node.sanitized_text();
return;
}
if (node.tag == common_chat_peg_builder::CONTENT) {
- result.content += std::string(node.text);
+ result.content += node.sanitized_text();
return;
}
diff --git a/common/chat.cpp b/common/chat.cpp
index 3a204e12d758..4856713082ac 100644
--- a/common/chat.cpp
+++ b/common/chat.cpp
@@ -9,6 +9,7 @@
#include "json.h"
#include "log.h"
#include "parsers/parsers.h"
+#include "sampling.h"
#include "jinja/value.h"
#include "jinja/runtime.h"
@@ -112,38 +113,6 @@ const char * common_chat_role_to_string(common_chat_role role) {
return "";
}
-json common_chat_msg_delimiters::to_json() const {
- json result = json::array();
- for (const auto & d : delimiters) {
- result.push_back({
- { "role", common_chat_role_to_string(d.role) },
- { "delimiter", d.delimiter },
- });
- }
- return result;
-}
-
-common_chat_msg_delimiters common_chat_msg_delimiters_parse(const json & delimiters) {
- common_chat_msg_delimiters result;
-
- if (!delimiters.is_array()) {
- return result;
- }
-
- result.delimiters.reserve(delimiters.size());
- for (const auto & d : delimiters) {
- if (!d.is_object()) {
- continue;
- }
- result.delimiters.push_back({
- common_chat_role_from_string(d.value("role", std::string())),
- d.value("delimiter", std::string()),
- });
- }
-
- return result;
-}
-
void common_chat_msg_delimiters::tokenize(const llama_vocab * vocab) {
for (auto & d : delimiters) {
d.tokens = common_tokenize(vocab, d.delimiter, false, true);
@@ -620,8 +589,11 @@ std::vector common_chat_tools_parse_oaicompat(const json & too
}
common_chat_continuation common_chat_continuation_parse(const common_json & value) {
- if (value.is_boolean() && value.get()) {
- return COMMON_CHAT_CONTINUATION_AUTO;
+ if (value.is_null()) {
+ return COMMON_CHAT_CONTINUATION_NONE;
+ }
+ if (value.is_boolean()) {
+ return value.get() ? COMMON_CHAT_CONTINUATION_AUTO : COMMON_CHAT_CONTINUATION_NONE;
}
if (value.is_string()) {
auto value_str = value.get();
@@ -632,7 +604,7 @@ common_chat_continuation common_chat_continuation_parse(const common_json & valu
return COMMON_CHAT_CONTINUATION_CONTENT;
}
}
- return COMMON_CHAT_CONTINUATION_NONE;
+ throw std::invalid_argument("Invalid continue_final_message: expected a boolean, \"content\" or \"reasoning_content\"");
}
bool common_chat_verify_template(const std::string & tmpl, bool use_jinja) {
@@ -675,7 +647,7 @@ std::string common_chat_format_single(const struct common_chat_templates * tmpls
if (!past_msg.empty()) {
inputs.messages = past_msg;
inputs.add_generation_prompt = false;
- fmt_past_msg = common_chat_templates_apply(tmpls, inputs).prompt;
+ fmt_past_msg = common_chat_templates_apply(tmpls, inputs).prompt.str();
}
std::ostringstream ss;
// if the past_msg ends with a newline, we must preserve it in the formatted version
@@ -685,7 +657,7 @@ std::string common_chat_format_single(const struct common_chat_templates * tmpls
// format chat with new_msg
inputs.messages.push_back(new_msg);
inputs.add_generation_prompt = add_ass;
- auto fmt_new_msg = common_chat_templates_apply(tmpls, inputs).prompt;
+ auto fmt_new_msg = common_chat_templates_apply(tmpls, inputs).prompt.str();
// get the diff part
ss << fmt_new_msg.substr(fmt_past_msg.size(), fmt_new_msg.size() - fmt_past_msg.size());
return ss.str();
@@ -709,7 +681,7 @@ std::string common_chat_format_example(const struct common_chat_templates *
add_simple_msg("user", "Hello");
add_simple_msg("assistant", "Hi there");
add_simple_msg("user", "How are you?");
- return common_chat_templates_apply(tmpls, inputs).prompt;
+ return common_chat_templates_apply(tmpls, inputs).prompt.str();
}
#define CHATML_TEMPLATE_SRC \
@@ -902,7 +874,7 @@ common_reasoning_format common_reasoning_format_from_name(const std::string & fo
throw std::runtime_error("Unknown reasoning format: " + format);
}
-std::string common_chat_template_direct_apply_impl(
+common_chat_prompt common_chat_template_direct_apply_impl(
const common_chat_template & tmpl,
const autoparser::generation_params & inputs,
const std::optional & messages_override,
@@ -948,24 +920,37 @@ std::string common_chat_template_direct_apply_impl(
jinja::global_from_json(ctx, inp, inputs.mark_input);
+ // bos/eos come from the model, not from the user
+ ctx.set_val("bos_token", jinja::mk_val(tmpl.bos_token()));
+ ctx.set_val("eos_token", jinja::mk_val(tmpl.eos_token()));
+
// render
jinja::runtime runtime(ctx);
const jinja::value results = runtime.execute(tmpl.prog);
auto parts = jinja::runtime::gather_string_parts(results);
- std::string result = parts->as_string().str();
+ common_chat_prompt result;
+ for (auto & part : parts->val_str.parts) {
+ result.parts.push_back({std::move(part.val), part.is_input});
+ }
// TODO: improve this later
- if (inputs.add_bos && string_starts_with(result, tmpl.bos_token())) {
- result = result.substr(tmpl.bos_token().size());
+ if (inputs.add_bos && !result.parts.empty()) {
+ auto & first = result.parts.front();
+ if (!first.is_input && string_starts_with(first.text, tmpl.bos_token())) {
+ first.text.erase(0, tmpl.bos_token().size());
+ }
}
- if (inputs.add_eos && string_ends_with(result, tmpl.eos_token())) {
- result = result.substr(0, result.size() - tmpl.eos_token().size());
+ if (inputs.add_eos && !result.parts.empty()) {
+ auto & last = result.parts.back();
+ if (!last.is_input && string_ends_with(last.text, tmpl.eos_token())) {
+ last.text.resize(last.text.size() - tmpl.eos_token().size());
+ }
}
return result;
}
-std::string common_chat_template_direct_apply(
+common_chat_prompt common_chat_template_direct_apply(
const common_chat_template & tmpl,
const autoparser::generation_params & inputs) {
return common_chat_template_direct_apply_impl(tmpl, inputs, std::nullopt, std::nullopt, std::nullopt);
@@ -981,9 +966,9 @@ std::string common_chat_template_generation_prompt_impl(
autoparser::generation_params params = inputs;
params.add_generation_prompt = false;
params.continue_final_message = COMMON_CHAT_CONTINUATION_NONE;
- std::string no_gen_prompt = common_chat_template_direct_apply_impl(tmpl, params, messages_override, tools_override, additional_context);
+ std::string no_gen_prompt = common_chat_template_direct_apply_impl(tmpl, params, messages_override, tools_override, additional_context).str();
params.add_generation_prompt = true;
- std::string gen_prompt = common_chat_template_direct_apply_impl(tmpl, params, messages_override, tools_override, additional_context);
+ std::string gen_prompt = common_chat_template_direct_apply_impl(tmpl, params, messages_override, tools_override, additional_context).str();
size_t prefix_len = 0;
size_t min_size = std::min(no_gen_prompt.size(), gen_prompt.size());
@@ -1087,35 +1072,55 @@ static json common_chat_extra_context() {
return ctx;
}
-std::optional common_chat_try_specialized_template(
- const common_chat_template & tmpl,
- const std::string & src,
- autoparser::generation_params & params) {
+static common_chat_params common_chat_params_init_lfm2_tokens(const common_chat_template & tmpl, const autoparser::generation_params & inputs) {
+ return common_chat_params_init_lfm2(tmpl, inputs, /* tool_list_tokens = */ true);
+}
+
+static common_chat_params common_chat_params_init_lfm2_5(const common_chat_template & tmpl, const autoparser::generation_params & inputs) {
+ return common_chat_params_init_lfm2(tmpl, inputs, /* tool_list_tokens = */ false);
+}
+
+// Older gemma4 templates need their tool responses rewritten before rendering
+static common_chat_params common_chat_params_init_gemma4_legacy(const common_chat_template & tmpl, const autoparser::generation_params & inputs) {
+ auto adjusted = inputs;
+ workaround::convert_tool_responses_gemma4(adjusted.messages);
+ return common_chat_params_init_gemma4(tmpl, adjusted);
+}
+
+// Pick the dedicated handler for a template from its source, or null for the autoparser.
+// Order matters: the first match wins, and later checks assume the earlier ones did not match.
+static common_chat_params_init_fn common_chat_template_detect_params_init(const std::string & src) {
// Ministral/Mistral Large 3 - uses special reasoning structure fixes, can't use autoparser
// Note: Mistral Small 3.2 uses [CALL_ID] which Ministral doesn't have, so we can distinguish them
if (src.find("[SYSTEM_PROMPT]") != std::string::npos && src.find("[TOOL_CALLS]") != std::string::npos &&
src.find("[ARGS]") != std::string::npos && src.find("[CALL_ID]") == std::string::npos) {
LOG_DBG("Using specialized template: Ministral/Magistral Large 3\n");
- return common_chat_params_init_ministral_3(tmpl, params);
+ return common_chat_params_init_ministral_3;
+ }
+
+ // LLM-jp-4.1 - GPT-OSS dialect (spaces after special tokens, <|end|>-separated parallel calls)
+ if (src.find("chat_format=llm-jp-harmony-v1") != std::string::npos) {
+ LOG_DBG("Using specialized template: LLM-jp Harmony v1\n");
+ return common_chat_params_init_llm_jp_harmony;
}
// GPT-OSS - has unique channel-based structure that needs dedicated handler
if (src.find("<|channel|>") != std::string::npos) {
LOG_DBG("Using specialized template: GPT-OSS\n");
- return common_chat_params_init_gpt_oss(tmpl, params);
+ return common_chat_params_init_gpt_oss;
}
// Muse Glimmer format using " to=" recipients and <|eom|>/<|eot|> message terminators.
if (src.find("") != std::string::npos && src.find("<|eom|>") != std::string::npos) {
LOG_DBG("Using specialized template: Muse Glimmer\n");
- return common_chat_params_init_muse_glimmer(tmpl, params);
+ return common_chat_params_init_muse_glimmer;
}
// Functionary v3.2 - uses recipient-based format with >>>recipient\n{content}
// Detection: template has ">>>all" for content and ">>>" prefix for tool calls
if (src.find(">>>all") != std::string::npos && src.find(">>>${recipient}") != std::string::npos) {
LOG_DBG("Using specialized template: Functionary v3.2\n");
- return common_chat_params_init_functionary_v3_2(tmpl, params);
+ return common_chat_params_init_functionary_v3_2;
}
// Kimi K2 Thinking - uses unique tool call ID format: functions.:
@@ -1123,14 +1128,30 @@ std::optional common_chat_try_specialized_template(
if (src.find("<|tool_calls_section_begin|>") != std::string::npos &&
src.find("<|tool_call_begin|>") != std::string::npos) {
LOG_DBG("Using specialized template: Kimi K2 Thinking\n");
- return common_chat_params_init_kimi_k2(tmpl, params);
+ return common_chat_params_init_kimi_k2;
}
// Kimi K3 - the <|open|>/<|close|>/<|end_of_msg|> markers are unique to it
if (src.find("<|open|>") != std::string::npos && src.find("<|close|>") != std::string::npos &&
src.find("<|end_of_msg|>") != std::string::npos) {
LOG_DBG("Using specialized template: Kimi K3\n");
- return common_chat_params_init_kimi_k3(tmpl, params);
+ return common_chat_params_init_kimi_k3;
+ }
+
+ // K2 Horizon - <|ifm|im_start|> turns, reasoning picked by reasoning_effort and
+ // sections; the three think tag pairs defeat the autoparser's reasoning detection
+ if (src.find("<|ifm|im_start|>") != std::string::npos &&
+ src.find("") != std::string::npos) {
+ LOG_DBG("Using specialized template: K2 Horizon\n");
+ return common_chat_params_init_k2_horizon;
+ }
+
+ // Ling 3.0 / Bailing V3 - X sections with / tagged
+ // tool calls. sections are unique to this family among the tagged-arg templates.
+ if (src.find("ASSISTANT") != std::string::npos &&
+ src.find("") != std::string::npos) {
+ LOG_DBG("Using specialized template: Ling 3.0 (Bailing V3)\n");
+ return common_chat_params_init_ling3;
}
// Cohere2 MoE / North Code - marker-wrapped format with <|START_TEXT|> content and
@@ -1139,19 +1160,19 @@ std::optional common_chat_try_specialized_template(
if (src.find("<|START_TEXT|>") != std::string::npos &&
src.find("<|START_ACTION|>") != std::string::npos) {
LOG_DBG("Using specialized template: Cohere2 MoE\n");
- return common_chat_params_init_cohere2moe(tmpl, params);
+ return common_chat_params_init_cohere2moe;
}
if (is_lfm2_template(src)) {
LOG_DBG("Using specialized template: LFM2\n");
- return common_chat_params_init_lfm2(tmpl, params, /* tool_list_tokens = */ true);
+ return common_chat_params_init_lfm2_tokens;
}
// LFM2.5 format detection: template uses plain "List of tools: [...]" with no special tokens
if (src.find("List of tools: [") != std::string::npos &&
src.find("<|tool_list_start|>") == std::string::npos) {
LOG_DBG("Using specialized template: LFM2.5\n");
- return common_chat_params_init_lfm2(tmpl, params, /* tool_list_tokens = */ false);
+ return common_chat_params_init_lfm2_5;
}
// GigaChatV3 format detection
@@ -1159,7 +1180,7 @@ std::optional common_chat_try_specialized_template(
src.find("<|message_sep|>") != std::string::npos &&
src.find("<|function_call|>") == std::string::npos) {
LOG_DBG("Using specialized template: GigaChatV3\n");
- return common_chat_params_init_gigachat_v3(tmpl, params);
+ return common_chat_params_init_gigachat_v3;
}
// MiniMax-M3: the namespace token "]<]minimax[>[" collides with the autoparser's
@@ -1168,7 +1189,7 @@ std::optional common_chat_try_specialized_template(
src.find("") != std::string::npos &&
src.find(" common_chat_try_specialized_template(
(src.find("function_calls") != std::string::npos ||
src.find("tool_calls") != std::string::npos)) {
LOG_DBG("Using specialized template: DeepSeek V3.2/V4\n");
- return common_chat_params_init_deepseek_v3_2(tmpl, params);
+ return common_chat_params_init_deepseek_v3_2;
}
// Gemma4 format detection
if (src.find("'<|tool_call>call:'") != std::string::npos) {
+ LOG_DBG("Using specialized template: Gemma4\n");
if (src.find("{#- OpenAI Chat Completions:") == std::string::npos) {
- // apply workarounds if using the older gemma4 templates
LOG_WRN("%s: detected an outdated gemma4 chat template, applying compatibility workarounds. "
"Consider updating to the official template.\n", __func__);
- workaround::convert_tool_responses_gemma4(params.messages);
+ return common_chat_params_init_gemma4_legacy;
}
- return common_chat_params_init_gemma4(tmpl, params);
+ return common_chat_params_init_gemma4;
}
// MiniCPM5 - XML tool calls with ...
@@ -1198,18 +1219,68 @@ std::optional common_chat_try_specialized_template(
src.find("") != std::string::npos &&
src.find("'") == std::string::npos) {
LOG_DBG("Using specialized template: Qwen3-Coder\n");
- return common_chat_params_init_qwen3_coder(tmpl, params);
+ return common_chat_params_init_qwen3_coder;
}
- return std::nullopt;
+ return nullptr;
+}
+
+common_chat_template::common_chat_template(const std::string & src, const std::string & bos_token, const std::string & eos_token) {
+ jinja::lexer lexer;
+ auto lexer_res = lexer.tokenize(src);
+ this->prog = jinja::parse_from_tokens(lexer_res);
+
+ this->src = lexer_res.source;
+ this->bos_tok = bos_token;
+ this->eos_tok = eos_token;
+
+ this->caps = jinja::caps_get(prog);
+ // LOG_INF("%s: caps:\n%s\n", __func__, this->caps.to_string().c_str());
+
+ this->params_init = common_chat_template_detect_params_init(this->src);
+ if (this->params_init) {
+ return;
+ }
+
+ // The analysis depends only on the template, so run it once here instead of on every apply.
+ // A failure is kept for apply to report, so a bad template still loads like it did before.
+ try {
+ analysis = std::make_unique();
+ analysis->analyze_template(*this);
+ } catch (const std::exception & e) {
+ analysis.reset();
+ analysis_error = e.what();
+ }
+}
+
+common_chat_template::~common_chat_template() = default;
+common_chat_template::common_chat_template(common_chat_template &&) = default;
+common_chat_template & common_chat_template::operator=(common_chat_template &&) = default;
+
+std::optional common_chat_try_specialized_template(
+ const common_chat_template & tmpl,
+ const autoparser::generation_params & params) {
+ if (!tmpl.params_init) {
+ return std::nullopt;
+ }
+ return tmpl.params_init(tmpl, params);
}
static common_chat_params common_chat_templates_apply_jinja(const struct common_chat_templates * tmpls,
@@ -1311,21 +1382,23 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_
data.prompt = common_chat_template_direct_apply_impl(tmpl, params_copy);
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, params);
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
- auto parser = build_chat_peg_parser([&data](common_chat_peg_builder &p) {
+ data.parser = build_chat_peg_parser([&data](common_chat_peg_builder &p) {
return p.literal(data.generation_prompt) << p.content(p.rest());
});
- data.parser = parser.save();
return data;
}
- if (auto result = common_chat_try_specialized_template(tmpl, src, params)) {
+ if (auto result = common_chat_try_specialized_template(tmpl, params)) {
return *result;
}
+ if (!tmpl.analysis) {
+ throw std::invalid_argument("Unable to generate parser for this template. Automatic parser generation failed: " + tmpl.analysis_error);
+ }
+
try {
LOG_DBG("%s: using differential autoparser\n", __func__);
- struct autoparser::autoparser autoparser;
- autoparser.analyze_template(tmpl);
+ const auto & autoparser = *tmpl.analysis;
auto auto_params = autoparser::peg_generator::generate_parser(tmpl, params, autoparser);
common_chat_msg_delimiters delimiters;
@@ -1346,8 +1419,7 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_
auto_params.thinking_end_tags = {std::move(end_tag)};
}
}
- common_peg_arena arena;
- arena.load(auto_params.parser);
+ const auto & arena = auto_params.parser;
LOG_DBG("%s: generated parser:\n%s\n\nparser generation prompt: %s\n", __func__, arena.dump(arena.root()).c_str(), auto_params.generation_prompt.c_str());
return auto_params;
} catch (const std::exception & e) {
@@ -1412,7 +1484,7 @@ static common_chat_params common_chat_templates_apply_legacy(const struct common
}
common_chat_params params;
- params.prompt = std::string(buf.data(), res);
+ params.prompt = common_chat_prompt(std::string(buf.data(), res));
if (!inputs.json_schema.empty()) {
params.grammar = json_schema_to_grammar(json::parse(inputs.json_schema));
} else {
@@ -1428,36 +1500,132 @@ common_chat_params common_chat_templates_apply(const struct common_chat_template
common_chat_templates_apply_legacy(tmpls, inputs);
}
-common_chat_msg common_chat_parse(const std::string & input,
+std::string common_chat_prompt::str() const {
+ if (parts.size() == 1) {
+ return parts[0].text;
+ }
+ size_t len = 0;
+ for (const auto & part : parts) {
+ len += part.text.size();
+ }
+ std::string out;
+ out.reserve(len);
+ for (const auto & part : parts) {
+ out += part.text;
+ }
+ return out;
+}
+
+void common_chat_prompt::append(const std::string & text, bool is_input) {
+ if (text.empty()) {
+ return;
+ }
+ if (!parts.empty() && parts.back().is_input == is_input) {
+ parts.back().text += text;
+ } else {
+ parts.push_back({text, is_input});
+ }
+}
+
+void common_chat_prompt::replace_last(const std::string & from, const std::string & to) {
+ for (auto it = parts.rbegin(); it != parts.rend(); ++it) {
+ if (it->is_input) {
+ continue;
+ }
+ size_t pos = it->text.rfind(from);
+ if (pos != std::string::npos) {
+ it->text.replace(pos, from.size(), to);
+ return;
+ }
+ }
+}
+
+void common_chat_input::append(const std::string & piece, llama_token token) {
+ if (piece.empty()) {
+ return;
+ }
+ tokens.push_back(token);
+ tokens.resize(tokens.size() + piece.size() - 1, LLAMA_TOKEN_NULL);
+ text += piece;
+}
+
+void common_chat_input::append(const common_chat_input & chunk) {
+ tokens.insert(tokens.end(), chunk.tokens.begin(), chunk.tokens.end());
+ text += chunk.text;
+}
+
+void common_chat_input::truncate(size_t pos) {
+ if (pos < text.size()) {
+ text.erase(pos);
+ tokens.resize(pos);
+ }
+}
+
+common_chat_input common_chat_input::substr(size_t pos, size_t n) const {
+ common_chat_input out;
+ out.text = text.substr(pos, n);
+ out.tokens.assign(tokens.begin() + pos, tokens.begin() + pos + out.size());
+ return out;
+}
+
+void common_chat_input::prepend(const std::string & prefix) {
+ tokens.insert(tokens.begin(), prefix.size(), LLAMA_TOKEN_NULL);
+ text = prefix + text;
+}
+
+void common_chat_input::prepend(const common_chat_input & prefix) {
+ tokens.insert(tokens.begin(), prefix.tokens.begin(), prefix.tokens.end());
+ text = prefix.text + text;
+}
+
+common_chat_input common_chat_input_tokenize(const llama_vocab * vocab, const std::string & text) {
+ common_chat_input input;
+ auto tokens = common_tokenize(vocab, text, false, true);
+ for (size_t i = 0; i < tokens.size(); i++) {
+ std::string piece = common_token_to_piece(vocab, tokens[i], true);
+ if (i == 0 && std::isspace(piece[0]) && !std::isspace(text[0])) {
+ // Some tokenizers will add a space before the first special token, need to exclude
+ continue;
+ }
+ input.append(piece, tokens[i]);
+ }
+ if (input.text != text) {
+ // the pieces do not give back the same text, keep the text without tokens
+ return common_chat_input(text);
+ }
+ return input;
+}
+
+common_chat_msg common_chat_parse(const common_chat_input & input,
bool is_partial,
const common_chat_parser_params & params) {
return common_chat_peg_parse(params.parser, input, is_partial, params);
}
common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser,
- const std::string & input,
+ const common_chat_input & input,
bool is_partial,
const common_chat_parser_params & params) {
- const common_peg_arena & parser = src_parser.empty() ?
- build_chat_peg_parser([](common_chat_peg_builder & p) { return p.content(p.rest()) + p.end(); }) :
- src_parser;
+ // both branches must be lvalues, a temporary here would copy the arena on every call
+ static const common_peg_arena content_only =
+ build_chat_peg_parser([](common_chat_peg_builder & p) { return p.content(p.rest()) + p.end(); });
+ const common_peg_arena & parser = src_parser.empty() ? content_only : src_parser;
if (src_parser.empty()) {
LOG_DBG("No parser definition detected, assuming pure content parser.");
}
- const std::string effective_input = params.generation_prompt.empty()
- ? input
- : params.generation_prompt + input;
+ common_chat_input effective_input = input;
+ effective_input.prepend(params.generation_prompt);
- //LOG_DBG("Parsing PEG input with format %s: %s\n", common_chat_format_name(params.format), effective_input.c_str());
+ //LOG_DBG("Parsing PEG input with format %s: %s\n", common_chat_format_name(params.format), effective_input.text.c_str());
common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_LENIENT;
if (params.debug) {
flags |= COMMON_PEG_PARSE_FLAG_DEBUG;
}
- common_peg_parse_context ctx(effective_input, flags);
+ common_peg_parse_context ctx(std::move(effective_input.text), std::move(effective_input.tokens), flags);
auto result = parser.parse(ctx);
if (result.fail()) {
@@ -1483,8 +1651,8 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars
}
return msg;
}
- LOG_WRN("%s: unparsed %s output: %s\n", __func__, common_chat_format_name(params.format), effective_input.substr(result.end).c_str());
- LOG_DBG("%s: full %s output triggering error:\n=== BEGIN ===\n%s\n=== END ===\n", __func__, common_chat_format_name(params.format), effective_input.c_str());
+ LOG_WRN("%s: unparsed %s output: %s\n", __func__, common_chat_format_name(params.format), ctx.input.substr(result.end).c_str());
+ LOG_DBG("%s: full %s output triggering error:\n=== BEGIN ===\n%s\n=== END ===\n", __func__, common_chat_format_name(params.format), ctx.input.c_str());
throw std::runtime_error(std::string("The model produced output that does not match the expected ") + common_chat_format_name(params.format) + " format");
}
@@ -1512,6 +1680,84 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars
return msg;
}
+common_chat_session::common_chat_session(const common_chat_templates * tmpls,
+ const llama_vocab * vocab,
+ const common_chat_templates_inputs & inputs,
+ const common_chat_session_params & params) {
+ auto applied = common_chat_templates_apply(tmpls, inputs);
+
+ templated = true;
+ prompt_parts = std::move(applied.prompt);
+ result.role = "assistant";
+
+ grammar_text = std::move(applied.grammar);
+ grammar_lazy = applied.grammar_lazy;
+ stops = std::move(applied.additional_stops);
+ generation_prompt_text = applied.generation_prompt;
+ thinking_start = std::move(applied.thinking_start_tag);
+ thinking_ends = std::move(applied.thinking_end_tags);
+
+ parser_params.format = applied.format;
+ parser_params.generation_prompt = vocab ? common_chat_input_tokenize(vocab, applied.generation_prompt)
+ : common_chat_input(applied.generation_prompt);
+ parser_params.debug = params.debug;
+ parser_params.parser = std::move(applied.parser);
+
+ delimiters = std::move(applied.message_delimiters);
+
+ if (vocab) {
+ common_params_sampling resolved;
+ resolved.grammar_lazy = applied.grammar_lazy;
+ common_sampling_add_preserved_tokens(resolved, vocab, applied.preserved_tokens);
+ common_sampling_add_grammar_triggers(resolved, vocab, std::move(applied.grammar_triggers));
+ preserved_tokens = std::move(resolved.preserved_tokens);
+ grammar_triggers = std::move(resolved.grammar_triggers);
+
+ delimiters.tokenize(vocab);
+ } else {
+ grammar_triggers = std::move(applied.grammar_triggers);
+ }
+
+ if (inputs.continue_final_message != COMMON_CHAT_CONTINUATION_NONE && !params.echo) {
+ // start from the prefill so it is not emitted as part of the first delta
+ result = common_chat_parse(input, true, parser_params);
+ }
+}
+
+void common_chat_session::apply_sampling(common_params_sampling & sampling) const {
+ if (!templated) {
+ return;
+ }
+ if (!grammar_text.empty()) {
+ sampling.grammar = {COMMON_GRAMMAR_TYPE_TOOL_CALLS, grammar_text};
+ }
+ sampling.grammar_lazy = grammar_lazy;
+ sampling.generation_prompt = generation_prompt_text;
+ sampling.preserved_tokens.insert(preserved_tokens.begin(), preserved_tokens.end());
+ sampling.grammar_triggers.insert(sampling.grammar_triggers.end(), grammar_triggers.begin(), grammar_triggers.end());
+}
+
+const common_chat_msg & common_chat_session::feed(const common_chat_input & chunk) {
+ GGML_ASSERT(!finished && "feed() after finish()");
+ input.append(chunk);
+ auto msg = common_chat_parse(input, true, parser_params);
+ if (!msg.empty()) {
+ result = std::move(msg);
+ }
+ return result;
+}
+
+const common_chat_msg & common_chat_session::finish(const common_chat_input & chunk) {
+ GGML_ASSERT(!finished && "finish() called twice");
+ finished = true;
+ input.append(chunk);
+ auto msg = common_chat_parse(input, false, parser_params);
+ if (!msg.empty()) {
+ result = std::move(msg);
+ }
+ return result;
+}
+
std::map common_chat_templates_get_caps(const common_chat_templates * chat_templates) {
GGML_ASSERT(chat_templates != nullptr);
GGML_ASSERT(chat_templates->template_default != nullptr);
diff --git a/common/chat.h b/common/chat.h
index 0e1423a5a3b1..dcff94919819 100644
--- a/common/chat.h
+++ b/common/chat.h
@@ -22,8 +22,16 @@ struct common_chat_templates;
namespace autoparser {
struct generation_params;
+struct autoparser;
} // namespace autoparser
+struct common_chat_params;
+struct common_chat_template;
+
+// Builds the prompt and parser for a template that has a dedicated handler (see common/parsers)
+using common_chat_params_init_fn = common_chat_params (*)(const common_chat_template & tmpl,
+ const autoparser::generation_params & inputs);
+
struct common_chat_tool_call {
std::string name;
std::string arguments;
@@ -54,19 +62,20 @@ struct common_chat_template {
std::string eos_tok;
std::string src;
chat_template_caps caps;
+ // Dedicated handler picked once from the source, null when the differential autoparser is used
+ common_chat_params_init_fn params_init = nullptr;
- common_chat_template(const std::string & src, const std::string & bos_token, const std::string & eos_token) {
- jinja::lexer lexer;
- auto lexer_res = lexer.tokenize(src);
- this->prog = jinja::parse_from_tokens(lexer_res);
+ // Differential analysis, run once here when there is no dedicated handler. Null when there
+ // is one, or when the analysis failed, in which case analysis_error says why.
+ std::unique_ptr analysis;
+ std::string analysis_error;
- this->src = lexer_res.source;
- this->bos_tok = bos_token;
- this->eos_tok = eos_token;
+ common_chat_template(const std::string & src, const std::string & bos_token, const std::string & eos_token);
- this->caps = jinja::caps_get(prog);
- // LOG_INF("%s: caps:\n%s\n", __func__, this->caps.to_string().c_str());
- }
+ // autoparser is incomplete here, so these are defined where it is complete
+ ~common_chat_template();
+ common_chat_template(common_chat_template &&);
+ common_chat_template & operator=(common_chat_template &&);
const std::string & source() const { return src; }
const std::string & bos_token() const { return bos_tok; }
@@ -209,8 +218,6 @@ struct common_chat_msg_delimiters {
// split tokens into message spans. skips maps a start index to a length of a region to jump over without matching
common_chat_msg_spans split(const llama_tokens & tokens, const std::map & skips = {}) const;
-
- common_json to_json() const;
};
struct common_chat_tool {
@@ -266,9 +273,31 @@ struct common_chat_templates_inputs {
bool force_pure_content = false;
};
+struct common_chat_prompt_part {
+ std::string text;
+ bool is_input = false; // from user input, special tokens inside must not be parsed
+};
+
+// rendered chat template, split into template parts and user input parts
+struct common_chat_prompt {
+ std::vector parts;
+
+ common_chat_prompt() = default;
+
+ // single part of template text
+ explicit common_chat_prompt(std::string text) { parts.push_back({std::move(text), false}); }
+
+ std::string str() const;
+
+ void append(const std::string & text, bool is_input = false);
+
+ // replace the last occurrence of `from`, user input parts are skipped
+ void replace_last(const std::string & from, const std::string & to);
+};
+
struct common_chat_params {
common_chat_format format = COMMON_CHAT_FORMAT_CONTENT_ONLY;
- std::string prompt;
+ common_chat_prompt prompt;
std::string grammar;
bool grammar_lazy = false;
std::string generation_prompt;
@@ -278,27 +307,46 @@ struct common_chat_params {
std::vector grammar_triggers;
std::vector preserved_tokens;
std::vector additional_stops;
- std::string parser;
+ common_peg_arena parser;
common_chat_msg_delimiters message_delimiters;
};
+struct common_chat_input {
+ std::string text;
+ std::vector tokens;
+
+ common_chat_input() = default;
+
+ // plain text, with no tokens
+ explicit common_chat_input(std::string text) : text(std::move(text)), tokens(this->text.size(), LLAMA_TOKEN_NULL) {}
+
+ size_t size() const { return text.size(); }
+ bool empty() const { return text.empty(); }
+
+ void append(const std::string & piece, llama_token token);
+ void append(const common_chat_input & chunk);
+
+ void prepend(const std::string & prefix);
+ void prepend(const common_chat_input & prefix);
+
+ void truncate(size_t pos);
+
+ common_chat_input substr(size_t pos, size_t n = std::string::npos) const;
+};
+
+common_chat_input common_chat_input_tokenize(const llama_vocab * vocab, const std::string & text);
+
// per-message parsing syntax
// should be derived from common_chat_params
struct common_chat_parser_params {
- common_chat_format format = COMMON_CHAT_FORMAT_CONTENT_ONLY;
- common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_NONE; // TODO: refactor this to "bool parse_reasoning"
- // Whether reasoning_content should be inlined in the content (e.g. for reasoning_format=deepseek in stream mode)
- bool reasoning_in_content = false;
- std::string generation_prompt;
- bool parse_tool_calls = true;
- bool is_continuation = false;
- bool echo = false; // Include assistant prefilled msg in output
- bool debug = false; // Enable debug output for PEG parser
- common_peg_arena parser = {};
+ common_chat_format format = COMMON_CHAT_FORMAT_CONTENT_ONLY;
+ common_chat_input generation_prompt;
+ bool debug = false; // Enable debug output for PEG parser
+ common_peg_arena parser = {};
common_chat_parser_params() = default;
common_chat_parser_params(const common_chat_params & chat_params) {
format = chat_params.format;
- generation_prompt = chat_params.generation_prompt;
+ generation_prompt = common_chat_input(chat_params.generation_prompt);
}
};
@@ -337,8 +385,62 @@ std::string common_chat_format_example(const struct common_chat_templates *
const std::map & chat_template_kwargs);
const char * common_chat_format_name(common_chat_format format);
-common_chat_msg common_chat_parse(const std::string & input, bool is_partial, const common_chat_parser_params & params);
-common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser, const std::string & input, bool is_partial, const common_chat_parser_params & params);
+common_chat_msg common_chat_parse(const common_chat_input & input, bool is_partial, const common_chat_parser_params & params);
+common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser, const common_chat_input & input, bool is_partial, const common_chat_parser_params & params);
+
+struct common_chat_session_params {
+ bool echo = false; // include the assistant prefill in the output when continuing a message
+ bool debug = false; // enable debug output for the PEG parser
+};
+
+class common_chat_session {
+ public:
+ common_chat_session() { result.role = "assistant"; }
+
+ common_chat_session(const common_chat_templates * tmpls,
+ const llama_vocab * vocab,
+ const common_chat_templates_inputs & inputs,
+ const common_chat_session_params & params = {});
+
+ const common_chat_prompt & prompt() const { return prompt_parts; }
+ common_chat_format format() const { return parser_params.format; }
+ const common_chat_msg & msg() const { return result; }
+ const common_peg_arena & parser() const { return parser_params.parser; }
+
+ const std::string & grammar() const { return grammar_text; }
+ const std::string & generation_prompt() const { return generation_prompt_text; }
+ const std::string & thinking_start_tag() const { return thinking_start; }
+ const std::vector & thinking_end_tags() const { return thinking_ends; }
+ const std::vector & additional_stops() const { return stops; }
+
+ const common_chat_msg_delimiters & message_delimiters() const { return delimiters; }
+
+ void apply_sampling(common_params_sampling & sampling) const;
+
+ bool has_template() const { return templated; }
+
+ const common_chat_msg & feed(const common_chat_input & chunk);
+
+ const common_chat_msg & finish(const common_chat_input & chunk = {});
+
+ private:
+ common_chat_prompt prompt_parts;
+ std::string grammar_text;
+ bool grammar_lazy = false;
+ std::vector grammar_triggers;
+ std::set preserved_tokens;
+ std::vector stops;
+ std::string generation_prompt_text;
+ std::string thinking_start;
+ std::vector thinking_ends;
+
+ common_chat_parser_params parser_params;
+ common_chat_msg_delimiters delimiters;
+ common_chat_input input;
+ common_chat_msg result;
+ bool templated = false;
+ bool finished = false;
+};
// used by arg and server
const char * common_reasoning_format_name(common_reasoning_format format);
@@ -366,7 +468,7 @@ common_json common_chat_tool_parameters(const common_json & function);
// get template caps, useful for reporting to server /props endpoint
std::map common_chat_templates_get_caps(const common_chat_templates * chat_templates);
-std::string common_chat_template_direct_apply(
+common_chat_prompt common_chat_template_direct_apply(
const common_chat_template & tmpl,
const autoparser::generation_params & inputs);
@@ -376,8 +478,7 @@ std::string common_chat_template_generation_prompt(
std::optional common_chat_try_specialized_template(
const common_chat_template & tmpl,
- const std::string & src,
- autoparser::generation_params & params);
+ const autoparser::generation_params & params);
// specialized per-task preset
@@ -387,5 +488,3 @@ struct common_chat_prompt_preset {
};
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates);
-
-common_chat_msg_delimiters common_chat_msg_delimiters_parse(const common_json & delimiters);
diff --git a/common/common.cpp b/common/common.cpp
index d8319cd9ac9a..c5db228329f5 100644
--- a/common/common.cpp
+++ b/common/common.cpp
@@ -1,8 +1,12 @@
#include "ggml.h"
+#include "ggml-cpp.h"
#include "gguf.h"
#include "build-info.h"
#include "common.h"
+
+#include "../src/llama-ext.h"
+
#include "fit.h"
#include "log.h"
#include "llama.h"
@@ -46,11 +50,10 @@
#include
#else
#include
-#include
#include
#endif
-#if defined(__linux__)
+#if !defined(_WIN32)
#include
#include
#endif
@@ -614,34 +617,6 @@ std::string string_from(const struct llama_context * ctx, const std::vectorpw_dir)) {
- throw std::runtime_error("Failed to find $HOME directory");
- }
-
- cache_directory = std::string(pw->pw_dir) + std::string("/.cache/");
-#else /* defined(__linux__) */
- throw std::runtime_error("Failed to find $HOME directory");
-#endif /* defined(__linux__) */
- }
-#elif defined(__APPLE__)
- cache_directory = common_get_env("HOME");
- if (cache_directory.empty()) {
- throw std::runtime_error("Failed to find $HOME directory");
- }
- cache_directory += "/Library/Caches/";
-#elif defined(_WIN32)
- cache_directory = common_get_env("LOCALAPPDATA");
- if (cache_directory.empty()) {
- throw std::runtime_error("Failed to find %LOCALAPPDATA% directory");
- }
-#elif defined(__EMSCRIPTEN__)
- GGML_ABORT("not implemented on this platform");
+std::filesystem::path common_get_path_from_env(const std::string & name) {
+#if defined(_WIN32)
+ const std::wstring wname = utf8_to_wstring(name);
+ const wchar_t * wvalue = _wgetenv(wname.c_str());
+ return wvalue ? std::filesystem::path(wvalue) : std::filesystem::path();
#else
-# error Unknown architecture
+ const char * value = std::getenv(name.c_str());
+ return value ? std::filesystem::path(value) : std::filesystem::path();
#endif
- cache_directory = ensure_trailing_slash(cache_directory);
- cache_directory += "llama.cpp";
- }
- return ensure_trailing_slash(cache_directory);
}
-std::string fs_get_config_directory() {
- std::string config_directory = "";
- auto ensure_trailing_slash = [](std::string p) {
- if (p.empty() || p.back() != DIRECTORY_SEPARATOR) {
- p += DIRECTORY_SEPARATOR;
- }
- return p;
- };
-#if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX) || \
- defined(__OpenBSD__) || defined(__NetBSD__) || defined(__APPLE__)
- const std::string xdg_config_home = common_get_env("XDG_CONFIG_HOME");
- const std::string home = common_get_env("HOME");
- if (!xdg_config_home.empty()) {
- config_directory = xdg_config_home;
- } else if (!home.empty()) {
- config_directory = home + "/.config/";
- } else {
-#if defined(__linux__)
- /* no $HOME is defined, fallback to getpwuid */
- struct passwd *pw = getpwuid(getuid());
- if ((!pw) || (!pw->pw_dir)) {
- throw std::runtime_error("Failed to find $HOME directory");
- }
-
- config_directory = std::string(pw->pw_dir) + std::string("/.config/");
-#else
+#if !defined(_WIN32)
+static std::filesystem::path get_home_directory() {
+ std::filesystem::path home = common_get_path_from_env("HOME");
+ if (!home.empty()) {
+ return home;
+ }
+ const struct passwd * pw = getpwuid(getuid());
+ if (!pw || !pw->pw_dir || !*pw->pw_dir) {
throw std::runtime_error("Failed to find $HOME directory");
+ }
+ return pw->pw_dir;
+}
#endif
+
+std::filesystem::path fs_get_cache_directory() {
+ std::filesystem::path cache_directory = common_get_path_from_env("LLAMA_CACHE");
+ if (!cache_directory.empty()) {
+ return cache_directory;
+ }
+#if defined(_WIN32)
+ cache_directory = common_get_path_from_env("LOCALAPPDATA");
+ if (cache_directory.empty()) {
+ throw std::runtime_error("Failed to find %LOCALAPPDATA% directory");
}
-#elif defined(_WIN32)
- config_directory = common_get_env("APPDATA");
+#elif defined(__APPLE__)
+ cache_directory = get_home_directory() / "Library/Caches";
+#else
+ cache_directory = common_get_path_from_env("XDG_CACHE_HOME");
+ if (cache_directory.empty()) {
+ cache_directory = get_home_directory() / ".cache";
+ }
+#endif
+ return cache_directory / "llama.cpp";
+}
+
+std::filesystem::path fs_get_config_directory() {
+ std::filesystem::path config_directory;
+#if defined(_WIN32)
+ config_directory = common_get_path_from_env("APPDATA");
if (config_directory.empty()) {
throw std::runtime_error("Failed to find %APPDATA% directory");
}
-#elif defined(__EMSCRIPTEN__)
- // caller decides what to do when there is no config directory
- throw std::runtime_error("not implemented on this platform");
#else
-# error Unknown architecture
+ config_directory = common_get_path_from_env("XDG_CONFIG_HOME");
+ if (config_directory.empty()) {
+ config_directory = get_home_directory() / ".config";
+ }
#endif
- config_directory = ensure_trailing_slash(config_directory);
- config_directory += "llama.cpp";
- return ensure_trailing_slash(config_directory);
+ return config_directory / "llama.cpp";
}
-std::string fs_get_cache_file(const std::string & filename) {
+std::filesystem::path fs_get_cache_file(const std::string & filename) {
GGML_ASSERT(filename.find(DIRECTORY_SEPARATOR) == std::string::npos);
- std::string cache_directory = fs_get_cache_directory();
- const bool success = fs_create_directory_with_parents(cache_directory);
- if (!success) {
- throw std::runtime_error("failed to create cache directory: " + cache_directory);
+ const std::filesystem::path cache_directory = fs_get_cache_directory();
+ std::error_code ec;
+ common_create_directories(cache_directory, ec);
+ if (ec) {
+ throw std::runtime_error("failed to create cache directory: " + fs_path_to_utf8(cache_directory));
}
- return cache_directory + filename;
+ return cache_directory / std::filesystem::u8path(filename);
}
-std::vector fs_list(const std::string & path, bool include_directories) {
- std::vector files;
- if (path.empty()) return files;
-
- std::filesystem::path dir(path);
- if (!std::filesystem::exists(dir) || !std::filesystem::is_directory(dir)) {
- return files;
- }
-
- for (const auto & entry : std::filesystem::directory_iterator(dir)) {
- try {
- // Only include regular files (skip directories)
- const auto & p = entry.path();
- if (std::filesystem::is_regular_file(p)) {
- common_file_info info;
- info.path = p.string();
- info.name = p.filename().string();
- info.is_dir = false;
- try {
- info.size = static_cast(std::filesystem::file_size(p));
- } catch (const std::filesystem::filesystem_error &) {
- info.size = 0;
- }
- files.push_back(std::move(info));
- } else if (include_directories && std::filesystem::is_directory(p)) {
- common_file_info info;
- info.path = p.string();
- info.name = p.filename().string();
- info.size = 0; // Directories have no size
- info.is_dir = true;
- files.push_back(std::move(info));
- }
- } catch (const std::filesystem::filesystem_error &) {
- // skip entries we cannot inspect
- continue;
- }
- }
-
- return files;
-}
+//
+// TTY utils
+//
-std::ifstream fs_open_ifstream(const std::string & fname, std::ios_base::openmode mode) {
-#ifdef _WIN32
- int wlen = MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, NULL, 0);
- if (!wlen) { return std::ifstream(); }
- std::vector wfname(wlen);
- (void)MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, wfname.data(), wlen);
- return std::ifstream(wfname.data(), mode);
+bool common_is_tty(FILE * file) {
+#if defined(_WIN32)
+ return _isatty(_fileno(file));
#else
- return std::ifstream(fname, mode);
+ return isatty(fileno(file));
#endif
}
-//
-// TTY utils
-//
-
bool tty_can_use_colors() {
// Check NO_COLOR environment variable (https://no-color.org/)
if (const char * no_color = std::getenv("NO_COLOR")) {
@@ -1199,10 +1063,21 @@ bool tty_can_use_colors() {
// Check if stdout and stderr are connected to a terminal
// We check both because log messages can go to either
- bool stdout_is_tty = isatty(fileno(stdout));
- bool stderr_is_tty = isatty(fileno(stderr));
+ return common_is_tty(stdout) || common_is_tty(stderr);
+}
- return stdout_is_tty || stderr_is_tty;
+bool tty_enable_ansi() {
+#if defined(_WIN32)
+ // a Windows console renders ANSI sequences only in virtual terminal mode, pipes and files take them as is
+ for (DWORD id : { STD_OUTPUT_HANDLE, STD_ERROR_HANDLE }) {
+ HANDLE h = GetStdHandle(id);
+ DWORD mode = 0;
+ if (GetConsoleMode(h, &mode) && !SetConsoleMode(h, mode | ENABLE_VIRTUAL_TERMINAL_PROCESSING)) {
+ return false;
+ }
+ }
+#endif
+ return true;
}
//
@@ -1287,6 +1162,80 @@ struct common_init_result::impl {
std::vector samplers_seq_config;
};
+static const std::map COMMON_DECISION_TYPE_NAMES = {
+ { COMMON_DECISION_TYPE_OPENJEV, "openjev" },
+ { COMMON_DECISION_TYPE_LEV, "lev" },
+ { COMMON_DECISION_TYPE_KEV, "kev" },
+ { COMMON_DECISION_TYPE_NIMBLE, "nimble" },
+ { COMMON_DECISION_TYPE_LAYA, "laya" },
+ { COMMON_DECISION_TYPE_CLEF, "clef" },
+ { COMMON_DECISION_TYPE_PPLX_DECIDER, "pplx-decider" },
+ { COMMON_DECISION_TYPE_VLLM_DECISION2, "vllm-decision2" },
+ { COMMON_DECISION_TYPE_VLLM_D3, "vllm-d3" },
+ { COMMON_DECISION_TYPE_LFM2_D1, "lfm2-d1" },
+ { COMMON_DECISION_TYPE_LFM2_D1_OMNI, "lfm2-d1-omni" },
+ // for future models, ALWAYS add org name prefix to avoid name collisions, e.g. "orgname-modelname"
+};
+
+static common_decision_type common_decision_type_from_string(const std::string & str) {
+ for (const auto & pair : COMMON_DECISION_TYPE_NAMES) {
+ if (pair.second == str) {
+ return pair.first;
+ }
+ }
+ return COMMON_DECISION_TYPE_UNKNOWN;
+}
+
+common_decision_type common_get_decision_type(const struct llama_model * model) {
+ char buf[64];
+ if (llama_model_meta_val_str(model, "general.architecture", buf, sizeof(buf)) < 0) {
+ return COMMON_DECISION_TYPE_NONE;
+ }
+ const std::string key = std::string(buf) + ".decision.type";
+ if (llama_model_meta_val_str(model, key.c_str(), buf, sizeof(buf)) < 0) {
+ return COMMON_DECISION_TYPE_NONE;
+ }
+ return common_decision_type_from_string(buf);
+}
+
+common_gguf_info common_get_gguf_info(const std::string & fname) {
+ common_gguf_info info;
+
+ struct gguf_init_params gguf_params = {
+ /* .no_alloc = */ true,
+ /* .ctx = */ nullptr,
+ };
+
+ gguf_context_ptr gguf_ctx(gguf_init_from_file(fname.c_str(), gguf_params));
+ if (!gguf_ctx) {
+ return info; // missing or unreadable file
+ }
+
+ const int64_t arch_id = gguf_find_key(gguf_ctx.get(), "general.architecture");
+ if (arch_id < 0 || gguf_get_kv_type(gguf_ctx.get(), arch_id) != GGUF_TYPE_STRING) {
+ return info; // no architecture in the metadata
+ }
+ const std::string arch = gguf_get_val_str(gguf_ctx.get(), arch_id);
+ if (arch.empty()) {
+ return info;
+ }
+
+ const int64_t type_id = gguf_find_key(gguf_ctx.get(), (arch + ".decision.type").c_str());
+ if (type_id < 0) {
+ info.decision_type = COMMON_DECISION_TYPE_NONE;
+ } else if (gguf_get_kv_type(gguf_ctx.get(), type_id) == GGUF_TYPE_STRING) {
+ info.decision_type = common_decision_type_from_string(gguf_get_val_str(gguf_ctx.get(), type_id));
+ }
+
+ // same key and type as the model loader
+ const int64_t ctx_id = gguf_find_key(gguf_ctx.get(), (arch + ".context_length").c_str());
+ if (ctx_id >= 0 && gguf_get_kv_type(gguf_ctx.get(), ctx_id) == GGUF_TYPE_UINT32) {
+ info.n_ctx_train = gguf_get_val_u32(gguf_ctx.get(), ctx_id);
+ }
+
+ return info;
+}
+
common_init_result::common_init_result(common_params & params, bool model_only) :
pimpl(new impl{}) {
auto mparams = common_model_params_to_llama(params);
@@ -1339,6 +1288,30 @@ common_init_result::common_init_result(common_params & params, bool model_only)
const llama_vocab * vocab = llama_model_get_vocab(model);
+ // these decision models return a score for each token via the embeddings output
+ // TODO: maybe improve this in the future
+ const auto decision_type = common_get_decision_type(model);
+ if (decision_type == COMMON_DECISION_TYPE_LAYA || decision_type == COMMON_DECISION_TYPE_KEV || decision_type == COMMON_DECISION_TYPE_CLEF ||
+ decision_type == COMMON_DECISION_TYPE_LFM2_D1_OMNI || decision_type == COMMON_DECISION_TYPE_VLLM_DECISION2) {
+ params.embedding = true;
+ params.pooling_type = LLAMA_POOLING_TYPE_NONE;
+
+ cparams.embeddings = true;
+ cparams.pooling_type = LLAMA_POOLING_TYPE_NONE;
+ cparams.n_outputs_max = cparams.n_batch;
+ cparams.n_outputs_max_per_seq = 1;
+
+ LOG_INF("%s", "decision model reads the embeddings output, enabling embedding mode\n");
+ }
+
+ // embeddings need the whole batch in one ubatch, so n_batch must not be larger than n_ubatch
+ // (server.cpp does this check for --embedding, but before the model is loaded)
+ if (cparams.embeddings && cparams.n_batch > cparams.n_ubatch) {
+ LOG_WRN("embeddings enabled: setting n_batch = n_ubatch = %u\n", cparams.n_ubatch);
+ cparams.n_batch = cparams.n_ubatch;
+ params.n_batch = params.n_ubatch;
+ }
+
// load and optionally apply lora adapters
for (auto & la : params.lora_adapters) {
llama_adapter_lora_ptr lora;
@@ -1433,6 +1406,14 @@ std::vector & common_init_result::lora() {
return pimpl->lora;
}
+// only for warmup and probe decodes, fill zeros as dummy input
+static void common_batch_set_zero_state(common_batch & batch, const llama_model * model, std::vector & zeros) {
+ zeros.assign(llama_model_n_embd_out(model), 0.0f);
+ for (int32_t i = 0; i < batch.size(); ++i) {
+ batch.set_embd_state(i, { zeros.data(), 1, zeros.size() });
+ }
+}
+
common_init_result_ptr common_init_from_params(common_params & params, bool model_only) {
common_init_result_ptr res(new common_init_result(params, model_only));
@@ -1527,7 +1508,8 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode
}
if (llama_model_has_encoder(model)) {
- llama_encode(lctx, llama_batch_get_one(tmp.data(), tmp.size()));
+ common_batch batch = common_batch_get_one(lctx, tmp);
+ llama_process(lctx, LLAMA_PROCESS_TYPE_ENCODE, batch.get());
llama_token decoder_start_token_id = llama_model_decoder_start_token(model);
if (decoder_start_token_id == LLAMA_TOKEN_NULL) {
decoder_start_token_id = bos;
@@ -1536,7 +1518,11 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode
tmp.push_back(decoder_start_token_id);
}
if (llama_model_has_decoder(model)) {
- llama_decode(lctx, llama_batch_get_one(tmp.data(), std::min(tmp.size(), (size_t) params.n_batch)));
+ tmp.resize(std::min(tmp.size(), (size_t) params.n_batch));
+ common_batch batch = common_batch_get_one(lctx, tmp);
+ std::vector zeros;
+ common_batch_set_zero_state(batch, model, zeros);
+ llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get());
}
llama_memory_clear(llama_get_memory(lctx), true);
llama_synchronize(lctx);
@@ -1600,9 +1586,15 @@ common_context_seq_rm_type common_context_can_seq_rm(llama_context * ctx) {
tmp.push_back(0);
tmp.push_back(0);
- int ret = llama_decode(ctx, llama_batch_get_one(tmp.data(), tmp.size()));
+ int ret;
+ {
+ common_batch batch = common_batch_get_one(ctx, tmp);
+ std::vector zeros;
+ common_batch_set_zero_state(batch, llama_get_model(ctx), zeros);
+ ret = llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get());
+ }
if (ret != 0) {
- COM_ERR("llama_decode() failed: %d\n", ret);
+ COM_ERR("llama_process() failed: %d\n", ret);
res = COMMON_CONTEXT_SEQ_RM_TYPE_NO;
goto done;
}
@@ -1710,7 +1702,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
mparams.progress_callback = params.load_progress_callback;
mparams.progress_callback_user_data = params.load_progress_callback_user_data;
mparams.no_alloc = params.no_alloc;
- mparams.load_mtp = std::find(params.speculative.types.begin(), params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end();
+ mparams.load_mtp = params.load_mtp || std::find(params.speculative.types.begin(), params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end();
return mparams;
}
@@ -1751,6 +1743,8 @@ struct llama_context_params common_context_params_to_llama(const common_params &
cparams.type_k = params.cache_type_k;
cparams.type_v = params.cache_type_v;
+ cparams.moe_cache_size = params.moe_cache_size;
+
return cparams;
}
@@ -1826,33 +1820,6 @@ void common_threadpools::init(llama_context * ctx, const common_params & params)
llama_attach_threadpool(ctx, threadpool, threadpool_batch);
}
-//
-// Batch utils
-//
-
-void common_batch_clear(struct llama_batch & batch) {
- batch.n_tokens = 0;
-}
-
-void common_batch_add(
- struct llama_batch & batch,
- llama_token id,
- llama_pos pos,
- const std::vector & seq_ids,
- bool logits) {
- GGML_ASSERT(batch.seq_id[batch.n_tokens] && "llama_batch size exceeded");
-
- batch.token [batch.n_tokens] = id;
- batch.pos [batch.n_tokens] = pos;
- batch.n_seq_id[batch.n_tokens] = seq_ids.size();
- for (size_t i = 0; i < seq_ids.size(); ++i) {
- batch.seq_id[batch.n_tokens][i] = seq_ids[i];
- }
- batch.logits [batch.n_tokens] = logits;
-
- batch.n_tokens++;
-}
-
//
// Vocab utils
//
@@ -2189,18 +2156,151 @@ float lr_opt::get_lr(float epoch) const {
}
bool common_replay_last_token(struct llama_context * ctx, llama_token last_token, int32_t pos) {
- llama_batch batch = llama_batch_get_one(&last_token, 1);
- batch.pos = &pos;
- if (llama_decode(ctx, batch)) {
+ common_batch batch(ctx);
+ batch.add(last_token, pos, 0, true);
+
+ if (llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get())) {
LOG_ERR("%s: failed to replay last token\n", __func__);
return false;
}
return true;
}
+common_batch::common_batch(llama_context * ctx) : batch(llama_batch_ext_init(ctx)) {
+ const auto rope_type = llama_model_rope_type(llama_get_model(ctx));
+ n_pos = rope_type == LLAMA_ROPE_TYPE_MROPE || rope_type == LLAMA_ROPE_TYPE_IMROPE ? GGML_MROPE_SECTIONS : 1;
+}
+
+void common_batch::clear() {
+ tokens.clear();
+}
+
+int32_t common_batch::add(llama_token id, llama_pos pos, llama_seq_id seq_id, bool output) {
+ tokens.push_back({ id, { pos, 0, 0, 0 }, seq_id, output, { nullptr, 0, 0 }, { nullptr, 0, 0 }, {} });
+ return size() - 1;
+}
+
+int32_t common_batch::add(llama_token id, llama_pos pos, const std::vector & seq_ids, bool output) {
+ GGML_ASSERT(!seq_ids.empty());
+
+ const int32_t idx = add(id, pos, seq_ids[0], output);
+ for (size_t s = 1; s < seq_ids.size(); ++s) {
+ add_seq(idx, seq_ids[s]);
+ }
+ return idx;
+}
+
+bool common_batch::add_seq(int32_t idx, llama_seq_id seq_id) {
+ if (idx < 0 || idx >= size()) {
+ return false;
+ }
+ tokens[idx].seq_ids_extra.push_back(seq_id);
+ return true;
+}
+
+bool common_batch::set_output(int32_t idx, bool value) {
+ if (idx < 0 || idx >= size()) {
+ return false;
+ }
+ tokens[idx].output = value;
+ return true;
+}
+
+bool common_batch::set_embd(int32_t idx, llama_embd embd) {
+ if (idx < 0 || idx >= size() || tokens[idx].embd.data != nullptr) {
+ return false;
+ }
+ tokens[idx].embd = embd;
+ return true;
+}
+
+bool common_batch::set_embd_state(int32_t idx, llama_embd state) {
+ if (idx < 0 || idx >= size() || tokens[idx].state.data != nullptr) {
+ return false;
+ }
+ tokens[idx].state = state;
+ return true;
+}
+
+int32_t common_batch::add_embd(llama_embd embd, const llama_pos * pos, llama_seq_id seq_id, bool output) {
+ token t = { LLAMA_TOKEN_NULL, { 0, 0, 0, 0 }, seq_id, output, embd, { nullptr, 0, 0 }, {} };
+ for (int32_t j = 0; j < n_pos; ++j) {
+ t.pos[j] = pos[j];
+ }
+ tokens.push_back(t);
+ return size() - 1;
+}
+
+llama_batch_ext * common_batch::get_sub_batch(int32_t off, int32_t n) {
+ GGML_ASSERT(batch && "common_batch was not initialized with a context");
+ GGML_ASSERT(off >= 0 && n >= 0 && off + n <= size());
+
+ llama_batch_ext * res = batch.get();
+ llama_batch_ext_clear(res);
+
+ for (int32_t i = off; i < off + n; ++i) {
+ const token & t = tokens[i];
+
+ int32_t idx;
+ if (t.id != LLAMA_TOKEN_NULL) {
+ idx = llama_batch_ext_add_token(res, t.seq_id, t.id);
+ if (idx < 0) {
+ GGML_ABORT("%s: failed to add token %d at index %d (error %d, n = %d)\n", __func__, t.id, i, idx, n);
+ }
+ llama_batch_ext_set_pos(res, idx, t.pos.data());
+ if (t.embd.data && !llama_batch_ext_set_embd_token(res, idx, t.embd)) {
+ GGML_ABORT("%s: failed to set the embedding of token %d at index %d\n", __func__, t.id, i);
+ }
+ } else {
+ idx = llama_batch_ext_add_embd(res, t.seq_id, t.embd);
+ if (idx < 0) {
+ GGML_ABORT("%s: failed to add embedding at index %d (error %d, n = %d)\n", __func__, i, idx, n);
+ }
+ llama_batch_ext_set_pos(res, idx, t.pos.data());
+ }
+ GGML_ASSERT(idx == i - off);
+
+ for (const llama_seq_id seq_id : t.seq_ids_extra) {
+ if (!llama_batch_ext_add_seq(res, idx, seq_id)) {
+ GGML_ABORT("%s: failed to add seq %d to the entry at index %d\n", __func__, seq_id, i);
+ }
+ }
+ if (t.output) {
+ llama_batch_ext_set_output_logits(res, idx, true);
+ }
+ if (t.state.data) {
+ llama_batch_ext_set_embd_state(res, idx, t.state); // contexts without a state input ignore it
+ }
+ if (t.decision_order != 0) {
+ llama_batch_ext_set_decision_order(res, idx, (llama_decision_order) t.decision_order);
+ }
+ }
+
+ return res;
+}
+
+common_batch common_batch_get_one(llama_context * ctx, const llama_token * tokens, int32_t n_tokens) {
+ common_batch batch(ctx);
+
+ auto mem = llama_get_memory(ctx);
+ llama_pos pos = llama_memory_seq_pos_max(mem, 0) + 1; // -1 + 1 == 0 when the memory is empty
+
+ for (int32_t i = 0; i < n_tokens; ++i) {
+ const bool output = i == n_tokens - 1;
+ batch.add(tokens[i], pos, 0, output);
+ pos++;
+ }
+
+ return batch;
+}
+
+common_batch common_batch_get_one(llama_context * ctx, const llama_tokens & tokens) {
+ return common_batch_get_one(ctx, tokens.data(), (int32_t) tokens.size());
+}
+
bool common_prompt_batch_decode(
struct llama_context * ctx,
- const std::vector & all_tokens,
+ const llama_tokens & all_tokens,
int n_new,
int & n_past,
int n_batch,
@@ -2221,7 +2321,9 @@ bool common_prompt_batch_decode(
// Memory implementations in recurrent/hybrid models don't support removing tokens from their
// memory, so we can't just remove the last token from the memory and replay the last token which
// is the reason for this logic.
- if (llama_decode(ctx, llama_batch_get_one(const_cast(all_tokens.data() + offset), n_tokens_before_last))) {
+ llama_tokens prefix_tokens(all_tokens.begin() + offset, all_tokens.begin() + offset + n_tokens_before_last);
+ common_batch batch_prefix = common_batch_get_one(ctx, prefix_tokens);
+ if (llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch_prefix.get())) {
COM_ERR("%s", "failed to eval\n");
return false;
}
@@ -2230,18 +2332,18 @@ bool common_prompt_batch_decode(
llama_state_save_file(ctx, state_path.data(), all_tokens.data(), all_tokens.size());
COM_INF("saved session before last token to %s, n_new = %zu\n", state_path.data(), all_tokens.size());
- llama_token last_token = all_tokens.back();
- llama_batch batch = llama_batch_get_one(&last_token, 1);
- int32_t pos = n_past;
- batch.pos = &pos;
+ common_batch batch_last(ctx);
+ batch_last.add(all_tokens.back(), n_past, 0, true);
- if (llama_decode(ctx, batch)) {
+ if (llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch_last.get())) {
COM_ERR("%s", "failed to eval last token\n");
return false;
}
n_past++;
} else {
- if (llama_decode(ctx, llama_batch_get_one(const_cast(all_tokens.data() + offset), n_new))) {
+ llama_tokens new_tokens(all_tokens.begin() + offset, all_tokens.begin() + offset + n_new);
+ common_batch batch = common_batch_get_one(ctx, new_tokens);
+ if (llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get())) {
COM_ERR("%s", "failed to eval\n");
return false;
}
@@ -2315,40 +2417,36 @@ void common_prompt_checkpoint::update_dft(
}
}
-void common_prompt_checkpoint::load_tgt(
+bool common_prompt_checkpoint::load_tgt(
llama_context * ctx,
llama_seq_id seq_id,
llama_state_seq_flags flags) const {
if (ctx == nullptr) {
- return;
+ return true;
}
if (data_tgt.empty()) {
- return;
+ return true;
}
const size_t n = llama_state_seq_set_data_ext(ctx, data_tgt.data(), data_tgt.size(), seq_id, flags);
- if (n != data_tgt.size()) {
- GGML_ABORT("checkpoint size mismatch: expected %zu, got %zu\n", data_tgt.size(), n);
- }
+ return n == data_tgt.size();
}
-void common_prompt_checkpoint::load_dft(
+bool common_prompt_checkpoint::load_dft(
llama_context * ctx,
llama_seq_id seq_id,
llama_state_seq_flags flags) const {
if (ctx == nullptr) {
- return;
+ return true;
}
if (data_dft.empty()) {
- return;
+ return true;
}
const size_t n = llama_state_seq_set_data_ext(ctx, data_dft.data(), data_dft.size(), seq_id, flags);
- if (n != data_dft.size()) {
- GGML_ABORT("checkpoint size mismatch: expected %zu, got %zu\n", data_dft.size(), n);
- }
+ return n == data_dft.size();
}
void common_prompt_checkpoint::clear_tgt() {
diff --git a/common/common.h b/common/common.h
index 63d0badd0f74..2048d6a79585 100644
--- a/common/common.h
+++ b/common/common.h
@@ -8,6 +8,7 @@
#include "ggml.h"
#include "llama.h"
+#include
#include
#include
#include
@@ -16,7 +17,9 @@
#include
#include