Skip to content

ggml-hrx: fuse Lemonade model operations #128

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rsuderman merged 7 commits into from
Oct 1, 2026
Merged

ggml-hrx: fuse Lemonade model operations #128

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
12f086c
ggml-hrx: fuse Lemonade model operations
rsuderman Oct 1, 2026
4709e9c
ggml-hrx: fix MoE routing JIT catalog coverage
rsuderman Oct 1, 2026
0c822e7
Merge remote-tracking branch 'amdeco/hrx-graph-develop-v2'
rsuderman Oct 1, 2026
c7baa2b
ggml-hrx: consolidate Qwen MoE kernels
rsuderman Oct 1, 2026
a653d0c
ggml-hrx: add Lemonade publication fusions
rsuderman Oct 1, 2026
f602a42
ggml-hrx: add Lemonade compound fusions
rsuderman Oct 1, 2026
1b61a7d
hrx: consolidate configurable binary fusion kernels
rsuderman Oct 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ggml/src/ggml-hrx/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,7 @@ ggml_add_backend_library(ggml-hrx
dispatch/transient-allocator.h
dispatch_registration/common/dispatch-binary.cpp
dispatch_registration/common/dispatch-binary.h
dispatch_registration/common/dispatch-binary-common.h
dispatch_registration/common/dispatch-common.cpp
dispatch_registration/common/dispatch-common.h
dispatch_registration/common/dispatch-copy.cpp
Expand Down
120 changes: 62 additions & 58 deletions ggml/src/ggml-hrx/benchmarks/loom/qwen3_30b_a3b_q4_k_m.tg8.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,30 +130,26 @@
},
"kernel": "qwen3_moe:qwen3_moe_attention_qkv_postprocess_fused_decode",
"library_sources": [
"qwen3_moe/attention_postprocess_f32_f16.loom",
"qwen3_moe/motifs/attention_qkv_postprocess_fused_decode.loom",
"../loom-libs/motifs/attention/rmsnorm_neox_cache_f16.loom",
"qwen3_moe/model_config.loom",
"qwen3_moe/attention_qkv_quantized.loom",
"qwen3_moe/dense_linear_quantized_f16_wmma.loom",
"ggml/linear_q6k_q8_1_x4.loom",
"qwen3_moe/attention_prepare_quantized.loom",
"qwen3_moe/routed_linear_q4k_f16_wmma.loom",
"qwen3_moe/routed_gate_up_swiglu_q4k.loom",
"ggml/quantize_q8_1_x4.loom"
"../loom-libs/motifs/matmul/q4_q6_q8_1_x4_qkv.loom",
"../loom-libs/motifs/matmul/q4_q8_1_x4_decode.loom",
"../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom",
"../loom-libs/motifs/quantize_q8_1_x4_group.loom"
],
"primary_sources": [
"qwen3_moe/attention_qkv_postprocess_fused.loom"
],
"sources": [
"qwen3_moe/attention_qkv_postprocess_fused.loom",
"qwen3_moe/attention_postprocess_f32_f16.loom",
"qwen3_moe/motifs/attention_qkv_postprocess_fused_decode.loom",
"../loom-libs/motifs/attention/rmsnorm_neox_cache_f16.loom",
"qwen3_moe/model_config.loom",
"qwen3_moe/attention_qkv_quantized.loom",
"qwen3_moe/dense_linear_quantized_f16_wmma.loom",
"ggml/linear_q6k_q8_1_x4.loom",
"qwen3_moe/attention_prepare_quantized.loom",
"qwen3_moe/routed_linear_q4k_f16_wmma.loom",
"qwen3_moe/routed_gate_up_swiglu_q4k.loom",
"ggml/quantize_q8_1_x4.loom"
"../loom-libs/motifs/matmul/q4_q6_q8_1_x4_qkv.loom",
"../loom-libs/motifs/matmul/q4_q8_1_x4_decode.loom",
"../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom",
"../loom-libs/motifs/quantize_q8_1_x4_group.loom"
],
"symbol": "qwen3_moe_attention_qkv_postprocess_fused_decode",
"workload_parameters": [
Expand Down Expand Up @@ -228,24 +224,22 @@
},
"kernel": "qwen3_moe:qwen3_moe_dense_linear_q4k_q8_1_x4_next_q8",
"library_sources": [
"ggml/linear_q6k_q8_1_x4.loom",
"qwen3_moe/attention_prepare_quantized.loom",
"qwen3_moe/model_config.loom",
"qwen3_moe/routed_linear_q4k_f16_wmma.loom",
"qwen3_moe/routed_gate_up_swiglu_q4k.loom",
"ggml/quantize_q8_1_x4.loom"
"qwen3_moe/motifs/attention_prepare_target.loom",
"../loom-libs/motifs/matmul/q4_q8_1_x4_dense.loom",
"../loom-libs/motifs/matmul/q4_q8_1_x4_decode.loom",
"../loom-libs/motifs/normalization/rmsnorm_q8_1_x4.loom",
"../loom-libs/motifs/quantize_q8_1_x4_group.loom"
],
"primary_sources": [
"qwen3_moe/dense_linear_quantized_f16_wmma.loom"
"qwen3_moe/motifs/dense_linear_q4k_next_q8.loom"
],
"sources": [
"qwen3_moe/dense_linear_quantized_f16_wmma.loom",
"ggml/linear_q6k_q8_1_x4.loom",
"qwen3_moe/attention_prepare_quantized.loom",
"qwen3_moe/model_config.loom",
"qwen3_moe/routed_linear_q4k_f16_wmma.loom",
"qwen3_moe/routed_gate_up_swiglu_q4k.loom",
"ggml/quantize_q8_1_x4.loom"
"qwen3_moe/motifs/dense_linear_q4k_next_q8.loom",
"qwen3_moe/motifs/attention_prepare_target.loom",
"../loom-libs/motifs/matmul/q4_q8_1_x4_dense.loom",
"../loom-libs/motifs/matmul/q4_q8_1_x4_decode.loom",
"../loom-libs/motifs/normalization/rmsnorm_q8_1_x4.loom",
"../loom-libs/motifs/quantize_q8_1_x4_group.loom"
],
"symbol": "qwen3_moe_dense_linear_q4k_q8_1_x4_next_q8",
"workload_parameters": [
Expand All @@ -272,17 +266,19 @@
"kernel": "qwen3_moe:qwen3_moe_router_projection_top8_fused_decode_f32",
"library_sources": [
"qwen3_moe/model_config.loom",
"qwen3_moe/router_projection_f32.loom",
"qwen3_moe/router_top8_f32.loom"
"../loom-libs/motifs/routing/router_projection_storage.loom",
"../loom-libs/motifs/routing/router_topk_row.loom",
"../loom-libs/motifs/routing/router_projection_fused.loom"
],
"primary_sources": [
"qwen3_moe/router_projection_top8_fused_f32.loom"
],
"sources": [
"qwen3_moe/router_projection_top8_fused_f32.loom",
"qwen3_moe/model_config.loom",
"qwen3_moe/router_projection_f32.loom",
"qwen3_moe/router_top8_f32.loom"
"../loom-libs/motifs/routing/router_projection_storage.loom",
"../loom-libs/motifs/routing/router_topk_row.loom",
"../loom-libs/motifs/routing/router_projection_fused.loom"
],
"symbol": "qwen3_moe_router_projection_top8_fused_decode_f32",
"workload_parameters": [
Expand Down Expand Up @@ -387,7 +383,7 @@
"motifs/matmul_id/vector_weighted.loom",
"motifs/matmul_id/vector_q4k_q8.loom",
"motifs/matmul/q6_f32_decode.loom",
"../qwen_moe/qwen3_moe/attention_prepare_quantized.loom"
"motifs/normalization/rmsnorm_q8_1_x4.loom"
],
"primary_sources": [
"ops/matmul_id/vector_weighted.loom"
Expand All @@ -397,7 +393,7 @@
"motifs/matmul_id/vector_weighted.loom",
"motifs/matmul_id/vector_q4k_q8.loom",
"motifs/matmul/q6_f32_decode.loom",
"../qwen_moe/qwen3_moe/attention_prepare_quantized.loom"
"motifs/normalization/rmsnorm_q8_1_x4.loom"
],
"symbol": "ggml_mul_mat_id_vector_weighted_wave64_publish_q8",
"workload_parameters": [
Expand Down Expand Up @@ -447,30 +443,26 @@
},
"kernel": "qwen3_moe:qwen3_moe_attention_qkv_postprocess_fused_decode",
"library_sources": [
"qwen3_moe/attention_postprocess_f32_f16.loom",
"qwen3_moe/motifs/attention_qkv_postprocess_fused_decode.loom",
"../loom-libs/motifs/attention/rmsnorm_neox_cache_f16.loom",
"qwen3_moe/model_config.loom",
"qwen3_moe/attention_qkv_quantized.loom",
"qwen3_moe/dense_linear_quantized_f16_wmma.loom",
"ggml/linear_q6k_q8_1_x4.loom",
"qwen3_moe/attention_prepare_quantized.loom",
"qwen3_moe/routed_linear_q4k_f16_wmma.loom",
"qwen3_moe/routed_gate_up_swiglu_q4k.loom",
"ggml/quantize_q8_1_x4.loom"
"../loom-libs/motifs/matmul/q4_q6_q8_1_x4_qkv.loom",
"../loom-libs/motifs/matmul/q4_q8_1_x4_decode.loom",
"../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom",
"../loom-libs/motifs/quantize_q8_1_x4_group.loom"
],
"primary_sources": [
"qwen3_moe/attention_qkv_postprocess_fused.loom"
],
"sources": [
"qwen3_moe/attention_qkv_postprocess_fused.loom",
"qwen3_moe/attention_postprocess_f32_f16.loom",
"qwen3_moe/motifs/attention_qkv_postprocess_fused_decode.loom",
"../loom-libs/motifs/attention/rmsnorm_neox_cache_f16.loom",
"qwen3_moe/model_config.loom",
"qwen3_moe/attention_qkv_quantized.loom",
"qwen3_moe/dense_linear_quantized_f16_wmma.loom",
"ggml/linear_q6k_q8_1_x4.loom",
"qwen3_moe/attention_prepare_quantized.loom",
"qwen3_moe/routed_linear_q4k_f16_wmma.loom",
"qwen3_moe/routed_gate_up_swiglu_q4k.loom",
"ggml/quantize_q8_1_x4.loom"
"../loom-libs/motifs/matmul/q4_q6_q8_1_x4_qkv.loom",
"../loom-libs/motifs/matmul/q4_q8_1_x4_decode.loom",
"../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom",
"../loom-libs/motifs/quantize_q8_1_x4_group.loom"
],
"symbol": "qwen3_moe_attention_qkv_postprocess_fused_decode",
"workload_parameters": [
Expand Down Expand Up @@ -575,7 +567,7 @@
"motifs/matmul_id/vector_weighted.loom",
"motifs/matmul_id/vector_q4k_q8.loom",
"motifs/matmul/q6_f32_decode.loom",
"../qwen_moe/qwen3_moe/attention_prepare_quantized.loom"
"motifs/normalization/rmsnorm_q8_1_x4.loom"
],
"primary_sources": [
"ops/matmul_id/vector_weighted.loom"
Expand All @@ -585,7 +577,7 @@
"motifs/matmul_id/vector_weighted.loom",
"motifs/matmul_id/vector_q4k_q8.loom",
"motifs/matmul/q6_f32_decode.loom",
"../qwen_moe/qwen3_moe/attention_prepare_quantized.loom"
"motifs/normalization/rmsnorm_q8_1_x4.loom"
],
"symbol": "ggml_mul_mat_id_vector_weighted_wave32_publish_q8",
"workload_parameters": [
Expand Down Expand Up @@ -959,16 +951,22 @@
},
"kernel": "qwen3_moe:qwen3_moe_rmsnorm_f32_quantize_q8_1_x4",
"library_sources": [
"qwen3_moe/motifs/attention_prepare_target.loom",
"../loom-libs/motifs/normalization/rmsnorm_q8_1_x4.loom",
"qwen3_moe/model_config.loom",
"ggml/quantize_q8_1_x4.loom"
"ggml/quantize_q8_1_x4.loom",
"../loom-libs/motifs/quantize_q8_1_x4_group.loom"
],
"primary_sources": [
"qwen3_moe/attention_prepare_quantized.loom"
],
"sources": [
"qwen3_moe/attention_prepare_quantized.loom",
"qwen3_moe/motifs/attention_prepare_target.loom",
"../loom-libs/motifs/normalization/rmsnorm_q8_1_x4.loom",
"qwen3_moe/model_config.loom",
"ggml/quantize_q8_1_x4.loom"
"ggml/quantize_q8_1_x4.loom",
"../loom-libs/motifs/quantize_q8_1_x4_group.loom"
],
"symbol": "qwen3_moe_rmsnorm_f32_quantize_q8_1_x4",
"workload_parameters": [
Expand All @@ -993,14 +991,20 @@
},
"kernel": "qwen3_moe:ggml_linear_q6k_q8_1_x4",
"library_sources": [
"ggml/quantize_q8_1_x4.loom"
"../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom",
"../loom-libs/motifs/q6_k_f16.loom",
"ggml/quantize_q8_1_x4.loom",
"../loom-libs/motifs/quantize_q8_1_x4_group.loom"
],
"primary_sources": [
"ggml/linear_q6k_q8_1_x4.loom"
],
"sources": [
"ggml/linear_q6k_q8_1_x4.loom",
"ggml/quantize_q8_1_x4.loom"
"../loom-libs/motifs/matmul/q6_q8_1_x4_decode.loom",
"../loom-libs/motifs/q6_k_f16.loom",
"ggml/quantize_q8_1_x4.loom",
"../loom-libs/motifs/quantize_q8_1_x4_group.loom"
],
"symbol": "ggml_linear_q6k_q8_1_x4",
"workload_parameters": [
Expand Down
154 changes: 154 additions & 0 deletions ggml/src/ggml-hrx/dispatch_registration/common/dispatch-binary-common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
#pragma once

#include "../dispatch-registry.h"
#include "graph/op-params.h"

#include <cstdint>
#include <string>

namespace ggml::hrx {

inline constexpr uint32_t common_binary_kind_bit(BinaryKind kind) {
return uint32_t{ 1 } << static_cast<uint32_t>(kind);
}

inline constexpr uint32_t kCommonArithmeticBinaryKinds =
common_binary_kind_bit(BinaryKind::Add) | common_binary_kind_bit(BinaryKind::Sub) |
common_binary_kind_bit(BinaryKind::Mul) | common_binary_kind_bit(BinaryKind::Div);

inline constexpr uint32_t kCommonGatedBinaryKinds =
common_binary_kind_bit(BinaryKind::SwiGLU) | common_binary_kind_bit(BinaryKind::GeGLU) |
common_binary_kind_bit(BinaryKind::RegLU) | common_binary_kind_bit(BinaryKind::GeGLUErf) |
common_binary_kind_bit(BinaryKind::GeGLUQuick);

inline bool common_binary_kind_allowed(BinaryKind kind, uint32_t allowed_kinds) {
return binary_kind_supported(kind) && (allowed_kinds & common_binary_kind_bit(kind)) != 0;
}

inline bool common_binary_kind_requires_order(BinaryKind kind) {
return kind == BinaryKind::Sub || kind == BinaryKind::Div;
}

inline bool common_binary_kind_matches_graph_op(BinaryKind kind, ggml_op op) {
switch (kind) {
case BinaryKind::Add:
return op == GGML_OP_ADD;
case BinaryKind::Sub:
return op == GGML_OP_SUB;
case BinaryKind::Mul:
return op == GGML_OP_MUL;
case BinaryKind::Div:
return op == GGML_OP_DIV;
case BinaryKind::SwiGLU:
case BinaryKind::GeGLU:
case BinaryKind::RegLU:
case BinaryKind::GeGLUErf:
case BinaryKind::GeGLUQuick:
return op == GGML_OP_GLU;
}
return false;
}

inline bool common_arithmetic_kind_from_graph_op(ggml_op op, BinaryKind & kind) {
switch (op) {
case GGML_OP_ADD:
kind = BinaryKind::Add;
return true;
case GGML_OP_SUB:
kind = BinaryKind::Sub;
return true;
case GGML_OP_MUL:
kind = BinaryKind::Mul;
return true;
case GGML_OP_DIV:
kind = BinaryKind::Div;
return true;
default:
return false;
}
}

inline bool common_fused_binary_kind_from_params(const OpParams & params, BinaryKind & kind) {
const BinaryParams * binary_params = op_params_as<BinaryParams>(params);
if (binary_params != nullptr && binary_kind_supported(binary_params->op)) {
kind = binary_params->op;
return true;
}

const GluParams * glu_params = op_params_as<GluParams>(params);
if (glu_params == nullptr) {
return false;
}

switch (glu_params->op) {
case GGML_GLU_OP_REGLU:
kind = BinaryKind::RegLU;
return true;
case GGML_GLU_OP_SWIGLU:
kind = BinaryKind::SwiGLU;
return true;
case GGML_GLU_OP_GEGLU:
kind = BinaryKind::GeGLU;
return true;
case GGML_GLU_OP_GEGLU_ERF:
kind = BinaryKind::GeGLUErf;
return true;
case GGML_GLU_OP_GEGLU_QUICK:
kind = BinaryKind::GeGLUQuick;
return true;
default:
return false;
}
}

struct CommonBinaryMatch {
const GraphNode * node = nullptr;
ValueId operand = ValueId(-1);
BinaryKind kind = BinaryKind::Add;
bool producer_is_lhs = true;

bool matched() const { return node != nullptr && operand.value >= 0; }
};

inline CommonBinaryMatch common_match_binary_consumer(const GraphNode * node,
ValueId producer,
uint32_t allowed_kinds) {
CommonBinaryMatch match;
if (node == nullptr || node->inputs.size() != 2) {
return match;
}

BinaryKind kind;
const bool has_kind =
common_fused_binary_kind_from_params(node->params, kind) ||
(std::holds_alternative<std::monostate>(node->params) && common_arithmetic_kind_from_graph_op(node->op, kind));
if (!has_kind || !common_binary_kind_matches_graph_op(kind, node->op) ||
!common_binary_kind_allowed(kind, allowed_kinds)) {
return match;
}

const bool producer_is_lhs = node->inputs[0] == producer;
const bool producer_is_rhs = node->inputs[1] == producer;
if (producer_is_lhs == producer_is_rhs) {
return match;
}

match.node = node;
match.operand = producer_is_lhs ? node->inputs[1] : node->inputs[0];
match.kind = kind;
match.producer_is_lhs = producer_is_lhs;
return match;
}

inline void common_set_binary_compile_parameters(KernelSpecialization & kernel,
const char * op_key,
const char * lhs_key,
BinaryKind kind,
bool producer_is_lhs) {
kernel.compile_parameters.emplace(op_key, std::to_string(binary_kind_config_value(kind)));
if (lhs_key != nullptr) {
kernel.compile_parameters.emplace(lhs_key, producer_is_lhs ? "1" : "0");
}
}

} // namespace ggml::hrx
Loading
Loading