From 2dd589f06a08a9c98904d1592a829ee5ff1d2299 Mon Sep 17 00:00:00 2001 From: PrakshaaleJain Date: Wed, 9 Sep 2026 10:31:53 +0530 Subject: [PATCH 01/17] adding comfy_quant methods --- .../quantizers/comfy_quant/__init__.py | 1 + .../quantizers/comfy_quant/comfy_quantizer.py | 109 ++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 src/diffusers/quantizers/comfy_quant/__init__.py create mode 100644 src/diffusers/quantizers/comfy_quant/comfy_quantizer.py diff --git a/src/diffusers/quantizers/comfy_quant/__init__.py b/src/diffusers/quantizers/comfy_quant/__init__.py new file mode 100644 index 000000000000..bbfe42225cde --- /dev/null +++ b/src/diffusers/quantizers/comfy_quant/__init__.py @@ -0,0 +1 @@ +from .comfy_quantizer import ComfyQuantizer diff --git a/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py b/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py new file mode 100644 index 000000000000..2d94420ab003 --- /dev/null +++ b/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py @@ -0,0 +1,109 @@ +from typing import TYPE_CHECKING, Any + +from ...utils import ( + get_module_from_name, + is_torch_available, + logging, +) +from ..base import DiffusersQuantizer + + +if TYPE_CHECKING: + from ...models.modeling_utils import ModelMixin + +if is_torch_available(): + import torch + + +logger = logging.get_logger(__name__) + + +class ComfyQuantizer(DiffusersQuantizer): + """ + Quantizer for comfy-kitchen formats (FP8, INT8, etc.). + """ + + use_keep_in_fp32_modules = True + requires_calibration = False + required_packages = ["comfy_kitchen"] + + def __init__(self, quantization_config, **kwargs): + super().__init__(quantization_config, **kwargs) + self.compute_dtype = quantization_config.compute_dtype + self.modules_to_not_convert = quantization_config.modules_to_not_convert or [] + if not isinstance(self.modules_to_not_convert, list): + self.modules_to_not_convert = [self.modules_to_not_convert] + + def validate_environment(self, *args, **kwargs): + from ...utils.import_utils import is_comfy_kitchen_available + + if not is_comfy_kitchen_available(): + raise ImportError( + "Loading Comfy Quant weights requires `comfy-kitchen`. " + "Please install it with: `pip install comfy-kitchen`." + ) + + def check_if_quantized_param( + self, + model: "ModelMixin", + param_value: "torch.Tensor", + param_name: str, + state_dict: dict[str, Any], + **kwargs, + ) -> bool: + # Based on comfy_kitchen, we will likely wrap tensors based on some config layout. + # For now, we assume all linear weights that aren't excluded are quantized. + # This will be refined based on comfy-kitchen's actual detection logic. + if any(m in param_name.split(".") for m in self.modules_to_not_convert): + return False + return True + + def create_quantized_param( + self, + model: "ModelMixin", + param_value: "torch.Tensor", + param_name: str, + target_device: "torch.device", + state_dict: dict[str, Any] | None = None, + unexpected_keys: list[str] | None = None, + **kwargs, + ): + module, tensor_name = get_module_from_name(model, param_name) + + # Defaulting to an example layout for now. Ideally this is pulled from config or metadata. + # Since ComfyQuantConfig can store the exact layout/format, we'd use it here. + # quantized_weight = ck_tensor.QuantizedTensor.from_float(param_value, ck_tensor.TensorCoreFP8Layout) + + # Since we don't have the exact layout detection in this PR snippet, we do a basic wrap. + # This is a placeholder for the actual comfy-kitchen wrapping logic. + quantized_weight = param_value + + if tensor_name in module._parameters: + module._parameters[tensor_name] = quantized_weight.to(target_device) + if tensor_name in module._buffers: + module._buffers[tensor_name] = quantized_weight.to(target_device) + + def update_torch_dtype(self, torch_dtype: "torch.dtype") -> "torch.dtype": + if torch_dtype is None: + torch_dtype = self.compute_dtype + return torch_dtype + + def _process_model_before_weight_loading( + self, + model: "ModelMixin", + device_map, + keep_in_fp32_modules: list[str] = [], + **kwargs, + ): + pass + + def _process_model_after_weight_loading(self, model, **kwargs): + pass + + @property + def is_serializable(self): + return False + + @property + def is_trainable(self): + return False From 78054ff022504644905e0d7bfa7e88b1e7271d44 Mon Sep 17 00:00:00 2001 From: PrakshaaleJain Date: Wed, 9 Sep 2026 11:38:23 +0530 Subject: [PATCH 02/17] feature: add aupport for comfy-kitchen quantization --- src/diffusers/__init__.py | 14 +++++++++ src/diffusers/pipelines/ltx2/__init__.py | 2 +- src/diffusers/quantizers/auto.py | 4 +++ .../quantizers/quantization_config.py | 25 ++++++++++++++++ src/diffusers/utils/__init__.py | 1 + .../utils/dummy_comfy_kitchen_objects.py | 29 +++++++++++++++++++ src/diffusers/utils/import_utils.py | 11 +++++++ 7 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 src/diffusers/utils/dummy_comfy_kitchen_objects.py diff --git a/src/diffusers/__init__.py b/src/diffusers/__init__.py index 1fc34e6bdbf6..a46ae675d5c0 100644 --- a/src/diffusers/__init__.py +++ b/src/diffusers/__init__.py @@ -9,6 +9,7 @@ is_accelerate_available, is_auto_round_available, is_bitsandbytes_available, + is_comfy_kitchen_available, is_gguf_available, is_librosa_available, is_note_seq_available, @@ -47,6 +48,7 @@ "schedulers": [], "utils": [ "OptionalDependencyNotAvailable", + "is_comfy_kitchen_available", "is_inflect_available", "is_invisible_watermark_available", "is_librosa_available", @@ -158,6 +160,18 @@ else: _import_structure["quantizers.quantization_config"].append("SDNQConfig") +try: + if not is_torch_available() and not is_accelerate_available() and not is_comfy_kitchen_available(): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from .utils import dummy_comfy_kitchen_objects + + _import_structure["utils.dummy_comfy_kitchen_objects"] = [ + name for name in dir(dummy_comfy_kitchen_objects) if not name.startswith("_") + ] +else: + _import_structure["quantizers.quantization_config"].append("ComfyQuantConfig") + try: if not is_onnx_available(): raise OptionalDependencyNotAvailable() diff --git a/src/diffusers/pipelines/ltx2/__init__.py b/src/diffusers/pipelines/ltx2/__init__.py index d4aa35127403..d48d890f4cb6 100644 --- a/src/diffusers/pipelines/ltx2/__init__.py +++ b/src/diffusers/pipelines/ltx2/__init__.py @@ -30,12 +30,12 @@ _import_structure["pipeline_ltx2_condition"] = ["LTX2ConditionPipeline", "LTX2VideoCondition"] _import_structure["pipeline_ltx2_dfr"] = ["LTX2DFRPipeline"] _import_structure["pipeline_ltx2_dfr_temporal_refine"] = ["LTX2DFRTemporalRefinePipeline"] - _import_structure["pipeline_output"] = ["LTX2DFRPipelineOutput", "LTX2PipelineOutput", "LTX2VideoDecodeOutput"] _import_structure["pipeline_ltx2_diffusion_decode"] = ["LTX2VideoDiffusionDecodePipeline"] _import_structure["pipeline_ltx2_hdr_lora"] = ["LTX2HDRPipeline", "LTX2HDRReferenceCondition"] _import_structure["pipeline_ltx2_ic_lora"] = ["LTX2InContextPipeline", "LTX2ReferenceCondition"] _import_structure["pipeline_ltx2_image2video"] = ["LTX2ImageToVideoPipeline"] _import_structure["pipeline_ltx2_latent_upsample"] = ["LTX2LatentUpsamplePipeline"] + _import_structure["pipeline_output"] = ["LTX2DFRPipelineOutput", "LTX2PipelineOutput", "LTX2VideoDecodeOutput"] _import_structure["vocoder"] = ["LTX2Vocoder", "LTX2VocoderWithBWE"] if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: diff --git a/src/diffusers/quantizers/auto.py b/src/diffusers/quantizers/auto.py index ea6caf91ab80..4b4358fecf30 100644 --- a/src/diffusers/quantizers/auto.py +++ b/src/diffusers/quantizers/auto.py @@ -20,12 +20,14 @@ from .autoround import AutoRoundQuantizer from .bitsandbytes import BnB4BitDiffusersQuantizer, BnB8BitDiffusersQuantizer +from .comfy_quant import ComfyQuantizer from .gguf import GGUFQuantizer from .modelopt import NVIDIAModelOptQuantizer from .nunchaku import NunchakuLiteQuantizer from .quantization_config import ( AutoRoundConfig, BitsAndBytesConfig, + ComfyQuantConfig, GGUFQuantizationConfig, NunchakuLiteQuantizationConfig, NVIDIAModelOptConfig, @@ -43,6 +45,7 @@ AUTO_QUANTIZER_MAPPING = { "bitsandbytes_4bit": BnB4BitDiffusersQuantizer, "bitsandbytes_8bit": BnB8BitDiffusersQuantizer, + "comfy_quant": ComfyQuantizer, "gguf": GGUFQuantizer, "quanto": QuantoQuantizer, "torchao": TorchAoHfQuantizer, @@ -55,6 +58,7 @@ AUTO_QUANTIZATION_CONFIG_MAPPING = { "bitsandbytes_4bit": BitsAndBytesConfig, "bitsandbytes_8bit": BitsAndBytesConfig, + "comfy_quant": ComfyQuantConfig, "gguf": GGUFQuantizationConfig, "quanto": QuantoConfig, "torchao": TorchAoConfig, diff --git a/src/diffusers/quantizers/quantization_config.py b/src/diffusers/quantizers/quantization_config.py index 30e89f53f906..d22e63819bc1 100644 --- a/src/diffusers/quantizers/quantization_config.py +++ b/src/diffusers/quantizers/quantization_config.py @@ -51,6 +51,7 @@ class QuantizationMethod(str, Enum): MODELOPT = "modelopt" AUTOROUND = "auto-round" SDNQ = "sdnq" + COMFY_QUANT = "comfy_quant" @dataclass @@ -994,3 +995,27 @@ def __new__(cls, *args, **kwargs): from sdnq import SDNQConfig as SDNQLibConfig return SDNQLibConfig(*args, **kwargs) + + +@dataclass +class ComfyQuantConfig(QuantizationConfigMixin): + """ + This is a wrapper class about all possible attributes and features that you can play with for a model that has been + quantized using comfy-kitchen. + + Args: + compute_dtype (`torch.dtype`, *optional*): + The target dtype for the compute operations. + modules_to_not_convert (`list[str]`, *optional*, defaults to `None`): + The list of modules to skip during quantization. + """ + + def __init__( + self, + compute_dtype: Any = None, + modules_to_not_convert: list[str] | None = None, + **kwargs, + ): + self.quant_method = QuantizationMethod.COMFY_QUANT + self.compute_dtype = compute_dtype + self.modules_to_not_convert = modules_to_not_convert diff --git a/src/diffusers/utils/__init__.py b/src/diffusers/utils/__init__.py index 5c63a4bc7661..a1ec33bcf3fb 100644 --- a/src/diffusers/utils/__init__.py +++ b/src/diffusers/utils/__init__.py @@ -72,6 +72,7 @@ is_bitsandbytes_available, is_bitsandbytes_version, is_bs4_available, + is_comfy_kitchen_available, is_cosmos_guardrail_available, is_flash_attn_3_available, is_flash_attn_available, diff --git a/src/diffusers/utils/dummy_comfy_kitchen_objects.py b/src/diffusers/utils/dummy_comfy_kitchen_objects.py new file mode 100644 index 000000000000..f6bc79109b83 --- /dev/null +++ b/src/diffusers/utils/dummy_comfy_kitchen_objects.py @@ -0,0 +1,29 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ..utils import DummyObject, requires_backends + + +class ComfyQuantConfig(metaclass=DummyObject): + _backends = ["comfy_kitchen"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["comfy_kitchen"]) + + +class ComfyQuantizer(metaclass=DummyObject): + _backends = ["comfy_kitchen"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["comfy_kitchen"]) diff --git a/src/diffusers/utils/import_utils.py b/src/diffusers/utils/import_utils.py index d2cf394cd9a7..d29d137251bc 100644 --- a/src/diffusers/utils/import_utils.py +++ b/src/diffusers/utils/import_utils.py @@ -220,6 +220,7 @@ def _is_package_available(pkg_name: str, get_dist_name: bool = False) -> tuple[b _sdnq_available, _sdnq_version = _is_package_available("sdnq") _flashpack_available, _flashpack_version = _is_package_available("flashpack") _av_available, _av_version = _is_package_available("av") +_comfy_kitchen_available, _comfy_kitchen_version = _is_package_available("comfy_kitchen") def is_torch_available(): @@ -422,6 +423,10 @@ def is_av_available(): return _av_available +def is_comfy_kitchen_available(): + return _comfy_kitchen_available + + # docstyle-ignore INFLECT_IMPORT_ERROR = """ {0} requires the inflect library but it was not found in your environment. You can install it with pip: `pip install @@ -562,6 +567,11 @@ def is_av_available(): {0} requires the sdnq library but it was not found in your environment. You can install it with pip: `pip install sdnq` """ +COMFY_KITCHEN_IMPORT_ERROR = """ +{0} requires the comfy-kitchen library but it was not found in your environment. You can install it with pip: `pip +install comfy-kitchen` +""" + # docstyle-ignore PYTORCH_RETINAFACE_IMPORT_ERROR = """ {0} requires the pytorch_retinaface library but it was not found in your environment. You can install it with pip: `pip install pytorch_retinaface` @@ -613,6 +623,7 @@ def is_av_available(): ("pytorch_retinaface", (is_pytorch_retinaface_available, PYTORCH_RETINAFACE_IMPORT_ERROR)), ("better_profanity", (is_better_profanity_available, BETTER_PROFANITY_IMPORT_ERROR)), ("nltk", (is_nltk_available, NLTK_IMPORT_ERROR)), + ("comfy_kitchen", (is_comfy_kitchen_available, COMFY_KITCHEN_IMPORT_ERROR)), ("torch_neuronx", (is_torch_neuronx_available, TORCH_NEURONX_IMPORT_ERROR)), ] ) From eb4811cc1fd8ecac7e1f97b31422b5fcf0848b2a Mon Sep 17 00:00:00 2001 From: PrakshaaleJain Date: Wed, 9 Sep 2026 18:12:25 +0530 Subject: [PATCH 03/17] feat: add support for INT8 and INT4 formats to comfy-kitchen quantizer --- .../quantizers/comfy_quant/comfy_quantizer.py | 34 +++++++++++++++---- .../quantizers/quantization_config.py | 5 +++ 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py b/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py index 2d94420ab003..c76ec13ca79e 100644 --- a/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py +++ b/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py @@ -29,6 +29,7 @@ class ComfyQuantizer(DiffusersQuantizer): def __init__(self, quantization_config, **kwargs): super().__init__(quantization_config, **kwargs) + self.quant_format = getattr(quantization_config, "quant_format", "fp8") self.compute_dtype = quantization_config.compute_dtype self.modules_to_not_convert = quantization_config.modules_to_not_convert or [] if not isinstance(self.modules_to_not_convert, list): @@ -70,13 +71,32 @@ def create_quantized_param( ): module, tensor_name = get_module_from_name(model, param_name) - # Defaulting to an example layout for now. Ideally this is pulled from config or metadata. - # Since ComfyQuantConfig can store the exact layout/format, we'd use it here. - # quantized_weight = ck_tensor.QuantizedTensor.from_float(param_value, ck_tensor.TensorCoreFP8Layout) - - # Since we don't have the exact layout detection in this PR snippet, we do a basic wrap. - # This is a placeholder for the actual comfy-kitchen wrapping logic. - quantized_weight = param_value + import comfy_kitchen.tensor as ck_tensor + + layout_map = { + "fp8": getattr(ck_tensor, "TensorCoreFP8Layout", None), + "nvfp4": getattr(ck_tensor, "TensorCoreNVFP4Layout", None), + "mxfp8": getattr(ck_tensor, "TensorCoreMXFP8Layout", None), + "int8": getattr(ck_tensor, "Int8Layout", None), + "int4_svd": getattr(ck_tensor, "SVDQuantW4A4Layout", None), + "int4_awq": getattr(ck_tensor, "AWQW4A16Layout", None), + } + + # Check if it's already a QuantizedTensor (e.g., if loaded directly from a custom loader) + if hasattr(param_value, "layout") and isinstance(param_value.layout, getattr(ck_tensor, "BaseLayout", type)): + quantized_weight = param_value + else: + layout = layout_map.get(self.quant_format.lower()) + if layout is None: + raise ValueError( + f"The layout for '{self.quant_format}' was not found in `comfy_kitchen`. " + f"Make sure you have the latest version installed that supports this format." + ) + + # comfy-kitchen natively handles wrapping standard float tensors via from_float + # If the tensor is pre-quantized raw bytes, comfy-kitchen exposes `.from_quantized(...)` or similar internally, + # but `.from_float` guarantees we intercept float weights (e.g. standard safetensors float weights). + quantized_weight = ck_tensor.QuantizedTensor.from_float(param_value.to(target_device), layout) if tensor_name in module._parameters: module._parameters[tensor_name] = quantized_weight.to(target_device) diff --git a/src/diffusers/quantizers/quantization_config.py b/src/diffusers/quantizers/quantization_config.py index d22e63819bc1..6cc1c92b45f5 100644 --- a/src/diffusers/quantizers/quantization_config.py +++ b/src/diffusers/quantizers/quantization_config.py @@ -1004,6 +1004,9 @@ class ComfyQuantConfig(QuantizationConfigMixin): quantized using comfy-kitchen. Args: + quant_format (`str`, *optional*, defaults to `"fp8"`): + The quantization format. Supported values include `"fp8"`, `"int8"`, `"mxfp8"`, `"nvfp4"`, `"int4_svd"`, + and `"int4_awq"`. compute_dtype (`torch.dtype`, *optional*): The target dtype for the compute operations. modules_to_not_convert (`list[str]`, *optional*, defaults to `None`): @@ -1012,10 +1015,12 @@ class ComfyQuantConfig(QuantizationConfigMixin): def __init__( self, + quant_format: str = "fp8", compute_dtype: Any = None, modules_to_not_convert: list[str] | None = None, **kwargs, ): self.quant_method = QuantizationMethod.COMFY_QUANT + self.quant_format = quant_format self.compute_dtype = compute_dtype self.modules_to_not_convert = modules_to_not_convert From b185a13a172d8788134c865c74162464de5912aa Mon Sep 17 00:00:00 2001 From: PrakshaaleJain Date: Wed, 9 Sep 2026 18:40:38 +0530 Subject: [PATCH 04/17] fix: register comfy-kitchen dummy objects correctly for check_dummies.py --- src/diffusers/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/diffusers/__init__.py b/src/diffusers/__init__.py index a46ae675d5c0..7581614e6c7e 100644 --- a/src/diffusers/__init__.py +++ b/src/diffusers/__init__.py @@ -1060,6 +1060,14 @@ else: from .quantizers.quantization_config import SDNQConfig + try: + if not is_comfy_kitchen_available(): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from .utils.dummy_comfy_kitchen_objects import * + else: + from .quantizers.quantization_config import ComfyQuantConfig + try: if not is_onnx_available(): raise OptionalDependencyNotAvailable() From a8b2e12096c6276b50adf7a0df15cbcb60c9d9dd Mon Sep 17 00:00:00 2001 From: PrakshaaleJain Date: Wed, 9 Sep 2026 18:55:48 +0530 Subject: [PATCH 05/17] docfix: deleted suggestion for installing latest version of comfy-kitchen --- src/diffusers/__init__.py | 4 ++-- src/diffusers/quantizers/comfy_quant/comfy_quantizer.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/diffusers/__init__.py b/src/diffusers/__init__.py index 7581614e6c7e..82d801d09eda 100644 --- a/src/diffusers/__init__.py +++ b/src/diffusers/__init__.py @@ -1066,8 +1066,8 @@ except OptionalDependencyNotAvailable: from .utils.dummy_comfy_kitchen_objects import * else: - from .quantizers.quantization_config import ComfyQuantConfig - + from .quantizers.quantization_config import Com + try: if not is_onnx_available(): raise OptionalDependencyNotAvailable() diff --git a/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py b/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py index c76ec13ca79e..d7659d52f79b 100644 --- a/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py +++ b/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py @@ -89,8 +89,7 @@ def create_quantized_param( layout = layout_map.get(self.quant_format.lower()) if layout is None: raise ValueError( - f"The layout for '{self.quant_format}' was not found in `comfy_kitchen`. " - f"Make sure you have the latest version installed that supports this format." + f"The layout for '{self.quant_format}' was not found in `comfy_kitchen`." ) # comfy-kitchen natively handles wrapping standard float tensors via from_float From 03178749acd1e57bb12b205e6ffce998de8587c1 Mon Sep 17 00:00:00 2001 From: PrakshaaleJain Date: Wed, 9 Sep 2026 19:13:07 +0530 Subject: [PATCH 06/17] test: add comfy-quant unit tests and fix layout passing --- src/diffusers/__init__.py | 2 +- .../quantizers/comfy_quant/comfy_quantizer.py | 8 +-- .../utils/dummy_comfy_kitchen_objects.py | 28 +++------ tests/quantization/comfy_quant/__init__.py | 0 .../comfy_quant/test_comfy_quant.py | 62 +++++++++++++++++++ 5 files changed, 74 insertions(+), 26 deletions(-) create mode 100644 tests/quantization/comfy_quant/__init__.py create mode 100644 tests/quantization/comfy_quant/test_comfy_quant.py diff --git a/src/diffusers/__init__.py b/src/diffusers/__init__.py index 82d801d09eda..460edbeb325f 100644 --- a/src/diffusers/__init__.py +++ b/src/diffusers/__init__.py @@ -1067,7 +1067,7 @@ from .utils.dummy_comfy_kitchen_objects import * else: from .quantizers.quantization_config import Com - + try: if not is_onnx_available(): raise OptionalDependencyNotAvailable() diff --git a/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py b/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py index d7659d52f79b..622eb82550fe 100644 --- a/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py +++ b/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py @@ -83,19 +83,17 @@ def create_quantized_param( } # Check if it's already a QuantizedTensor (e.g., if loaded directly from a custom loader) - if hasattr(param_value, "layout") and isinstance(param_value.layout, getattr(ck_tensor, "BaseLayout", type)): + if isinstance(param_value, ck_tensor.QuantizedTensor): quantized_weight = param_value else: layout = layout_map.get(self.quant_format.lower()) if layout is None: - raise ValueError( - f"The layout for '{self.quant_format}' was not found in `comfy_kitchen`." - ) + raise ValueError(f"The layout for '{self.quant_format}' was not found in `comfy_kitchen`.") # comfy-kitchen natively handles wrapping standard float tensors via from_float # If the tensor is pre-quantized raw bytes, comfy-kitchen exposes `.from_quantized(...)` or similar internally, # but `.from_float` guarantees we intercept float weights (e.g. standard safetensors float weights). - quantized_weight = ck_tensor.QuantizedTensor.from_float(param_value.to(target_device), layout) + quantized_weight = ck_tensor.QuantizedTensor.from_float(param_value.to(target_device), layout.__name__) if tensor_name in module._parameters: module._parameters[tensor_name] = quantized_weight.to(target_device) diff --git a/src/diffusers/utils/dummy_comfy_kitchen_objects.py b/src/diffusers/utils/dummy_comfy_kitchen_objects.py index f6bc79109b83..cda3dedbb573 100644 --- a/src/diffusers/utils/dummy_comfy_kitchen_objects.py +++ b/src/diffusers/utils/dummy_comfy_kitchen_objects.py @@ -1,29 +1,17 @@ -# Copyright 2026 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - +# This file is autogenerated by the command `make fix-copies`, do not edit. from ..utils import DummyObject, requires_backends -class ComfyQuantConfig(metaclass=DummyObject): +class Com(metaclass=DummyObject): _backends = ["comfy_kitchen"] def __init__(self, *args, **kwargs): requires_backends(self, ["comfy_kitchen"]) + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["comfy_kitchen"]) -class ComfyQuantizer(metaclass=DummyObject): - _backends = ["comfy_kitchen"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["comfy_kitchen"]) + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["comfy_kitchen"]) diff --git a/tests/quantization/comfy_quant/__init__.py b/tests/quantization/comfy_quant/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/quantization/comfy_quant/test_comfy_quant.py b/tests/quantization/comfy_quant/test_comfy_quant.py new file mode 100644 index 000000000000..36f58f738ee2 --- /dev/null +++ b/tests/quantization/comfy_quant/test_comfy_quant.py @@ -0,0 +1,62 @@ +import pytest +import torch +import torch.nn as nn + +from diffusers import ComfyQuantConfig +from diffusers.utils import is_comfy_kitchen_available + +from ...testing_utils import require_torch + + +if is_comfy_kitchen_available(): + import comfy_kitchen.tensor as ck_tensor + + from diffusers.quantizers.comfy_quant.comfy_quantizer import ComfyQuantizer + +device = "cuda" if torch.cuda.is_available() else "cpu" + + +@require_torch +@pytest.mark.skipif(not is_comfy_kitchen_available(), reason="comfy-kitchen is not available") +class TestComfyQuantizer: + def test_create_quantized_param_fp8(self): + config = ComfyQuantConfig(quant_format="fp8") + quantizer = ComfyQuantizer(config) + + class DummyModel(nn.Module): + def __init__(self): + super().__init__() + self.linear = nn.Linear(16, 16) + + model = DummyModel() + param_value = torch.randn(16, 16, dtype=torch.float32) + + quantizer.create_quantized_param( + model=model, + param_value=param_value, + param_name="linear.weight", + target_device=torch.device(device), + ) + + assert isinstance(model.linear.weight, ck_tensor.QuantizedTensor) + assert model.linear.weight._layout_cls == "TensorCoreFP8Layout" + + def test_create_quantized_param_invalid_format(self): + config = ComfyQuantConfig(quant_format="non_existent_format") + quantizer = ComfyQuantizer(config) + + class DummyModel(nn.Module): + def __init__(self): + super().__init__() + self.linear = nn.Linear(16, 16) + + model = DummyModel() + param_value = torch.randn(16, 16, dtype=torch.float32) + + with pytest.raises(ValueError, match="not found in `comfy_kitchen`"): + quantizer.create_quantized_param( + model=model, + param_value=param_value, + param_name="linear.weight", + target_device=torch.device(device), + ) From 0c97d8dd090f369207a87fc89b6668df04fc5672 Mon Sep 17 00:00:00 2001 From: PrakshaaleJain Date: Wed, 9 Sep 2026 21:11:02 +0530 Subject: [PATCH 07/17] test: refine invalid format test --- tests/quantization/comfy_quant/test_comfy_quant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/quantization/comfy_quant/test_comfy_quant.py b/tests/quantization/comfy_quant/test_comfy_quant.py index 36f58f738ee2..a40835e90673 100644 --- a/tests/quantization/comfy_quant/test_comfy_quant.py +++ b/tests/quantization/comfy_quant/test_comfy_quant.py @@ -42,7 +42,7 @@ def __init__(self): assert model.linear.weight._layout_cls == "TensorCoreFP8Layout" def test_create_quantized_param_invalid_format(self): - config = ComfyQuantConfig(quant_format="non_existent_format") + config = ComfyQuantConfig(quant_format="INT_42") #INT_42 is an non-existent format quantizer = ComfyQuantizer(config) class DummyModel(nn.Module): From e38bf7faf85b7dfd4de23f1c115d38be446d6418 Mon Sep 17 00:00:00 2001 From: PrakshaaleJain Date: Wed, 9 Sep 2026 21:13:51 +0530 Subject: [PATCH 08/17] style fixes --- tests/quantization/comfy_quant/test_comfy_quant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/quantization/comfy_quant/test_comfy_quant.py b/tests/quantization/comfy_quant/test_comfy_quant.py index a40835e90673..af94ab99018a 100644 --- a/tests/quantization/comfy_quant/test_comfy_quant.py +++ b/tests/quantization/comfy_quant/test_comfy_quant.py @@ -42,7 +42,7 @@ def __init__(self): assert model.linear.weight._layout_cls == "TensorCoreFP8Layout" def test_create_quantized_param_invalid_format(self): - config = ComfyQuantConfig(quant_format="INT_42") #INT_42 is an non-existent format + config = ComfyQuantConfig(quant_format="INT_42") # INT_42 is an non-existent format quantizer = ComfyQuantizer(config) class DummyModel(nn.Module): From 2452aa8628b516a44925282c61c9ee37ebdf97c7 Mon Sep 17 00:00:00 2001 From: PrakshaaleJain Date: Wed, 9 Sep 2026 21:21:18 +0530 Subject: [PATCH 09/17] docs: add comfy_quant usage tutorial --- docs/source/en/_toctree.yml | 2 + docs/source/en/quantization/comfy_quant.md | 77 ++++++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 docs/source/en/quantization/comfy_quant.md diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index f05667986f11..04aa4b355a19 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -172,6 +172,8 @@ title: bitsandbytes - local: quantization/gguf title: gguf + - local: quantization/comfy_quant + title: Comfy Quant - local: quantization/nunchaku title: Nunchaku Lite - local: quantization/torchao diff --git a/docs/source/en/quantization/comfy_quant.md b/docs/source/en/quantization/comfy_quant.md new file mode 100644 index 000000000000..4e96bf955396 --- /dev/null +++ b/docs/source/en/quantization/comfy_quant.md @@ -0,0 +1,77 @@ + + +# Comfy Quant + +The [Comfy Quant](https://github.com/Comfy-Org/comfy-quants) toolkit provides state-of-the-art quantization techniques. While `comfy-quants` is used for exporting and quantizing models, Diffusers natively supports running inference on these models using the [comfy-kitchen](https://github.com/Comfy-Org/comfy-kitchen) library. + +`comfy-kitchen` provides highly optimized GPU kernels that allow you to seamlessly run quantized layers. By passing a `ComfyQuantConfig` to Diffusers, the library will dynamically intercept parameters and wrap them in a `QuantizedTensor` that maps directly to the optimized `comfy-kitchen` layouts. + +Before starting, please install `comfy-kitchen` in your environment: + +```shell +pip install comfy-kitchen +``` + +## Loading a Comfy Quant Model + +To load a model prequantized with Comfy Quant, use the [`~FromSingleFileMixin.from_single_file`] method and pass in the [`ComfyQuantConfig`]. + +The configuration requires you to specify the `quant_format` that the model was quantized in, and the `compute_dtype` for active inference calculations. + +The following example demonstrates how to load a quantized FLUX transformer: + +```python +import torch +from diffusers import FluxPipeline, FluxTransformer2DModel, ComfyQuantConfig + +ckpt_path = "path/to/comfy_quant_checkpoint.safetensors" + +# Initialize the config with your desired format and compute dtype +quantization_config = ComfyQuantConfig( + quant_format="fp8", + compute_dtype=torch.bfloat16 +) + +# Load the transformer directly from the safetensors file +transformer = FluxTransformer2DModel.from_single_file( + ckpt_path, + quantization_config=quantization_config, + dtype=torch.bfloat16, +) + +# Pass the quantized transformer into the pipeline +pipe = FluxPipeline.from_pretrained( + "black-forest-labs/FLUX.1-dev", + transformer=transformer, + dtype=torch.bfloat16, +) +pipe.enable_model_cpu_offload() + +prompt = "A cat holding a sign that says hello world" +image = pipe(prompt, generator=torch.manual_seed(0)).images[0] +image.save("flux-comfy-quant.png") +``` + +## Supported Quantization Formats + +Diffusers currently maps the following Comfy Quant formats to `comfy-kitchen` layouts: + +- **FP8** (`fp8`): Maps to `TensorCoreFP8Layout` (E4M3/E5M2) +- **INT8** (`int8`): Maps to `TensorCoreInt8Layout` (W8A8, tensorwise) +- **MXFP8** (`mxfp8`): Maps to `TensorCoreMXFP8Layout` +- **NVFP4** (`nvfp4`): Maps to `TensorCoreNVFP4Layout` +- **INT4 SVD** (`int4_svd`): Maps to `SVDQuantW4A4Layout` (SVDQuant W4A4) +- **INT4 AWQ** (`int4_awq`): Maps to `AWQW4A16Layout` (AWQ W4A16) + +When using optimized layouts, `comfy-kitchen` automatically dispatches the operations to the best available backend (HIP, CUDA, Triton, or Eager). From 2b6f56e5eb46e294bd181c56dcb336f34a7feb92 Mon Sep 17 00:00:00 2001 From: PrakshaaleJain Date: Tue, 15 Sep 2026 22:09:52 +0530 Subject: [PATCH 10/17] fix: wrong dummy object name --- src/diffusers/__init__.py | 2 +- src/diffusers/utils/dummy_comfy_kitchen_objects.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/diffusers/__init__.py b/src/diffusers/__init__.py index 1878b5059db6..ba7640aa875f 100644 --- a/src/diffusers/__init__.py +++ b/src/diffusers/__init__.py @@ -1068,7 +1068,7 @@ except OptionalDependencyNotAvailable: from .utils.dummy_comfy_kitchen_objects import * else: - from .quantizers.quantization_config import Com + from .quantizers.quantization_config import ComfyQuantConfig try: if not is_onnx_available(): diff --git a/src/diffusers/utils/dummy_comfy_kitchen_objects.py b/src/diffusers/utils/dummy_comfy_kitchen_objects.py index cda3dedbb573..e00c405c2bb2 100644 --- a/src/diffusers/utils/dummy_comfy_kitchen_objects.py +++ b/src/diffusers/utils/dummy_comfy_kitchen_objects.py @@ -2,7 +2,7 @@ from ..utils import DummyObject, requires_backends -class Com(metaclass=DummyObject): +class ComfyQuantConfig(metaclass=DummyObject): _backends = ["comfy_kitchen"] def __init__(self, *args, **kwargs): From 5eb91a7e97c509f282db38d3266c81fb66fd7c33 Mon Sep 17 00:00:00 2001 From: PrakshaaleJain Date: Tue, 15 Sep 2026 22:18:51 +0530 Subject: [PATCH 11/17] fix: avaliability guard logic --- src/diffusers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/__init__.py b/src/diffusers/__init__.py index ba7640aa875f..4870c35268fc 100644 --- a/src/diffusers/__init__.py +++ b/src/diffusers/__init__.py @@ -161,7 +161,7 @@ _import_structure["quantizers.quantization_config"].append("SDNQConfig") try: - if not is_torch_available() and not is_accelerate_available() and not is_comfy_kitchen_available(): + if not is_comfy_kitchen_available(): raise OptionalDependencyNotAvailable() except OptionalDependencyNotAvailable: from .utils import dummy_comfy_kitchen_objects From c680d2397a71f2b958e04291f2772b0cefa7824e Mon Sep 17 00:00:00 2001 From: PrakshaaleJain Date: Tue, 15 Sep 2026 22:36:55 +0530 Subject: [PATCH 12/17] fix: revert unrelated ltx2 change --- src/diffusers/pipelines/ltx2/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/ltx2/__init__.py b/src/diffusers/pipelines/ltx2/__init__.py index d48d890f4cb6..d4aa35127403 100644 --- a/src/diffusers/pipelines/ltx2/__init__.py +++ b/src/diffusers/pipelines/ltx2/__init__.py @@ -30,12 +30,12 @@ _import_structure["pipeline_ltx2_condition"] = ["LTX2ConditionPipeline", "LTX2VideoCondition"] _import_structure["pipeline_ltx2_dfr"] = ["LTX2DFRPipeline"] _import_structure["pipeline_ltx2_dfr_temporal_refine"] = ["LTX2DFRTemporalRefinePipeline"] + _import_structure["pipeline_output"] = ["LTX2DFRPipelineOutput", "LTX2PipelineOutput", "LTX2VideoDecodeOutput"] _import_structure["pipeline_ltx2_diffusion_decode"] = ["LTX2VideoDiffusionDecodePipeline"] _import_structure["pipeline_ltx2_hdr_lora"] = ["LTX2HDRPipeline", "LTX2HDRReferenceCondition"] _import_structure["pipeline_ltx2_ic_lora"] = ["LTX2InContextPipeline", "LTX2ReferenceCondition"] _import_structure["pipeline_ltx2_image2video"] = ["LTX2ImageToVideoPipeline"] _import_structure["pipeline_ltx2_latent_upsample"] = ["LTX2LatentUpsamplePipeline"] - _import_structure["pipeline_output"] = ["LTX2DFRPipelineOutput", "LTX2PipelineOutput", "LTX2VideoDecodeOutput"] _import_structure["vocoder"] = ["LTX2Vocoder", "LTX2VocoderWithBWE"] if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: From 0ff5c5e74073fed13d793fd8eb2a7589b5b00ac8 Mon Sep 17 00:00:00 2001 From: PrakshaaleJain Date: Wed, 16 Sep 2026 23:20:53 +0530 Subject: [PATCH 13/17] fix: confy_kitchen import check --- src/diffusers/quantizers/comfy_quant/comfy_quantizer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py b/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py index 622eb82550fe..1557a76dca85 100644 --- a/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py +++ b/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py @@ -3,10 +3,13 @@ from ...utils import ( get_module_from_name, is_torch_available, - logging, + is_comfy_kitchen_available, + logging ) from ..base import DiffusersQuantizer +if is_comfy_kitchen_available(): + import comfy_kitchen.tensor as ck_tensor if TYPE_CHECKING: from ...models.modeling_utils import ModelMixin @@ -71,7 +74,6 @@ def create_quantized_param( ): module, tensor_name = get_module_from_name(model, param_name) - import comfy_kitchen.tensor as ck_tensor layout_map = { "fp8": getattr(ck_tensor, "TensorCoreFP8Layout", None), From a1e71c7abee88cf153e226fe57b7a71d2137edb6 Mon Sep 17 00:00:00 2001 From: PrakshaaleJain Date: Wed, 16 Sep 2026 23:53:25 +0530 Subject: [PATCH 14/17] fix: moved layout resolution to init, removed QuantizedTensor check --- docs/source/en/quantization/comfy_quant.md | 6 +- .../quantizers/comfy_quant/comfy_quantizer.py | 42 ++++++------- .../comfy_quant/test_comfy_quant.py | 62 ------------------- 3 files changed, 23 insertions(+), 87 deletions(-) diff --git a/docs/source/en/quantization/comfy_quant.md b/docs/source/en/quantization/comfy_quant.md index 4e96bf955396..19c6da128437 100644 --- a/docs/source/en/quantization/comfy_quant.md +++ b/docs/source/en/quantization/comfy_quant.md @@ -68,10 +68,10 @@ image.save("flux-comfy-quant.png") Diffusers currently maps the following Comfy Quant formats to `comfy-kitchen` layouts: - **FP8** (`fp8`): Maps to `TensorCoreFP8Layout` (E4M3/E5M2) -- **INT8** (`int8`): Maps to `TensorCoreInt8Layout` (W8A8, tensorwise) +- **INT8** (`int8`): Maps to `TensorWiseINT8Layout` (W8A8, tensorwise) - **MXFP8** (`mxfp8`): Maps to `TensorCoreMXFP8Layout` - **NVFP4** (`nvfp4`): Maps to `TensorCoreNVFP4Layout` -- **INT4 SVD** (`int4_svd`): Maps to `SVDQuantW4A4Layout` (SVDQuant W4A4) -- **INT4 AWQ** (`int4_awq`): Maps to `AWQW4A16Layout` (AWQ W4A16) +- **INT4 SVD** (`int4_svd`): Maps to `TensorCoreSVDQuantW4A4Layout` (SVDQuant W4A4) +- **INT4 AWQ** (`int4_awq`): Maps to `TensorCoreAWQW4A16Layout` (AWQ W4A16) When using optimized layouts, `comfy-kitchen` automatically dispatches the operations to the best available backend (HIP, CUDA, Triton, or Eager). diff --git a/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py b/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py index 1557a76dca85..fd764f1a68cd 100644 --- a/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py +++ b/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py @@ -38,6 +38,23 @@ def __init__(self, quantization_config, **kwargs): if not isinstance(self.modules_to_not_convert, list): self.modules_to_not_convert = [self.modules_to_not_convert] + # Resolve the layout class once since quant_format is constant. + layout_map = { + "fp8": getattr(ck_tensor, "TensorCoreFP8Layout", None), + "nvfp4": getattr(ck_tensor, "TensorCoreNVFP4Layout", None), + "mxfp8": getattr(ck_tensor, "TensorCoreMXFP8Layout", None), + "int8": getattr(ck_tensor, "TensorWiseINT8Layout", None), + "int4_svd": getattr(ck_tensor, "TensorCoreSVDQuantW4A4Layout", None), + "int4_awq": getattr(ck_tensor, "TensorCoreAWQW4A16Layout", None), + } + self.layout = layout_map.get(self.quant_format.lower()) + if self.layout is None: + supported = list(layout_map.keys()) + raise ValueError( + f"The layout for '{self.quant_format}' was not found in `comfy_kitchen`. " + f"Supported formats are: {supported}." + ) + def validate_environment(self, *args, **kwargs): from ...utils.import_utils import is_comfy_kitchen_available @@ -74,28 +91,9 @@ def create_quantized_param( ): module, tensor_name = get_module_from_name(model, param_name) - - layout_map = { - "fp8": getattr(ck_tensor, "TensorCoreFP8Layout", None), - "nvfp4": getattr(ck_tensor, "TensorCoreNVFP4Layout", None), - "mxfp8": getattr(ck_tensor, "TensorCoreMXFP8Layout", None), - "int8": getattr(ck_tensor, "Int8Layout", None), - "int4_svd": getattr(ck_tensor, "SVDQuantW4A4Layout", None), - "int4_awq": getattr(ck_tensor, "AWQW4A16Layout", None), - } - - # Check if it's already a QuantizedTensor (e.g., if loaded directly from a custom loader) - if isinstance(param_value, ck_tensor.QuantizedTensor): - quantized_weight = param_value - else: - layout = layout_map.get(self.quant_format.lower()) - if layout is None: - raise ValueError(f"The layout for '{self.quant_format}' was not found in `comfy_kitchen`.") - - # comfy-kitchen natively handles wrapping standard float tensors via from_float - # If the tensor is pre-quantized raw bytes, comfy-kitchen exposes `.from_quantized(...)` or similar internally, - # but `.from_float` guarantees we intercept float weights (e.g. standard safetensors float weights). - quantized_weight = ck_tensor.QuantizedTensor.from_float(param_value.to(target_device), layout.__name__) + quantized_weight = ck_tensor.QuantizedTensor.from_float( + param_value.to(target_device), self.layout.__name__ + ) if tensor_name in module._parameters: module._parameters[tensor_name] = quantized_weight.to(target_device) diff --git a/tests/quantization/comfy_quant/test_comfy_quant.py b/tests/quantization/comfy_quant/test_comfy_quant.py index af94ab99018a..e69de29bb2d1 100644 --- a/tests/quantization/comfy_quant/test_comfy_quant.py +++ b/tests/quantization/comfy_quant/test_comfy_quant.py @@ -1,62 +0,0 @@ -import pytest -import torch -import torch.nn as nn - -from diffusers import ComfyQuantConfig -from diffusers.utils import is_comfy_kitchen_available - -from ...testing_utils import require_torch - - -if is_comfy_kitchen_available(): - import comfy_kitchen.tensor as ck_tensor - - from diffusers.quantizers.comfy_quant.comfy_quantizer import ComfyQuantizer - -device = "cuda" if torch.cuda.is_available() else "cpu" - - -@require_torch -@pytest.mark.skipif(not is_comfy_kitchen_available(), reason="comfy-kitchen is not available") -class TestComfyQuantizer: - def test_create_quantized_param_fp8(self): - config = ComfyQuantConfig(quant_format="fp8") - quantizer = ComfyQuantizer(config) - - class DummyModel(nn.Module): - def __init__(self): - super().__init__() - self.linear = nn.Linear(16, 16) - - model = DummyModel() - param_value = torch.randn(16, 16, dtype=torch.float32) - - quantizer.create_quantized_param( - model=model, - param_value=param_value, - param_name="linear.weight", - target_device=torch.device(device), - ) - - assert isinstance(model.linear.weight, ck_tensor.QuantizedTensor) - assert model.linear.weight._layout_cls == "TensorCoreFP8Layout" - - def test_create_quantized_param_invalid_format(self): - config = ComfyQuantConfig(quant_format="INT_42") # INT_42 is an non-existent format - quantizer = ComfyQuantizer(config) - - class DummyModel(nn.Module): - def __init__(self): - super().__init__() - self.linear = nn.Linear(16, 16) - - model = DummyModel() - param_value = torch.randn(16, 16, dtype=torch.float32) - - with pytest.raises(ValueError, match="not found in `comfy_kitchen`"): - quantizer.create_quantized_param( - model=model, - param_value=param_value, - param_name="linear.weight", - target_device=torch.device(device), - ) From 47a77d14e47fd765fe99bf057585609d101b2689 Mon Sep 17 00:00:00 2001 From: PrakshaaleJain Date: Thu, 17 Sep 2026 00:03:13 +0530 Subject: [PATCH 15/17] feat: add check for unsupported passed format --- src/diffusers/quantizers/quantization_config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/diffusers/quantizers/quantization_config.py b/src/diffusers/quantizers/quantization_config.py index 6cc1c92b45f5..c0aa4e99d2f5 100644 --- a/src/diffusers/quantizers/quantization_config.py +++ b/src/diffusers/quantizers/quantization_config.py @@ -1020,6 +1020,9 @@ def __init__( modules_to_not_convert: list[str] | None = None, **kwargs, ): + supported_formats = {"fp8","nvfp4", "mxfp8", "int8", "int4_svd", "int4_awq"} + if quant_format not in supported_formats: + raise ValueError(f"Unsupported quant_format: {quant_format}. Supported formats are: {sorted(supported_formats)}") self.quant_method = QuantizationMethod.COMFY_QUANT self.quant_format = quant_format self.compute_dtype = compute_dtype From 69f29fae3bd7f59bb4c0d3323ee6778cd974a458 Mon Sep 17 00:00:00 2001 From: PrakshaaleJain Date: Thu, 17 Sep 2026 00:16:30 +0530 Subject: [PATCH 16/17] test: add ComfyQuantConfigMixin and ComfyQuantTesterMixin --- src/diffusers/pipelines/ltx2/__init__.py | 2 +- .../quantizers/comfy_quant/comfy_quantizer.py | 12 +- .../quantizers/quantization_config.py | 6 +- tests/models/testing_utils/__init__.py | 4 + tests/models/testing_utils/quantization.py | 132 ++++++++++++++++++ .../comfy_quant/test_comfy_quant.py | 46 ++++++ tests/testing_utils.py | 21 +++ 7 files changed, 211 insertions(+), 12 deletions(-) diff --git a/src/diffusers/pipelines/ltx2/__init__.py b/src/diffusers/pipelines/ltx2/__init__.py index d4aa35127403..d48d890f4cb6 100644 --- a/src/diffusers/pipelines/ltx2/__init__.py +++ b/src/diffusers/pipelines/ltx2/__init__.py @@ -30,12 +30,12 @@ _import_structure["pipeline_ltx2_condition"] = ["LTX2ConditionPipeline", "LTX2VideoCondition"] _import_structure["pipeline_ltx2_dfr"] = ["LTX2DFRPipeline"] _import_structure["pipeline_ltx2_dfr_temporal_refine"] = ["LTX2DFRTemporalRefinePipeline"] - _import_structure["pipeline_output"] = ["LTX2DFRPipelineOutput", "LTX2PipelineOutput", "LTX2VideoDecodeOutput"] _import_structure["pipeline_ltx2_diffusion_decode"] = ["LTX2VideoDiffusionDecodePipeline"] _import_structure["pipeline_ltx2_hdr_lora"] = ["LTX2HDRPipeline", "LTX2HDRReferenceCondition"] _import_structure["pipeline_ltx2_ic_lora"] = ["LTX2InContextPipeline", "LTX2ReferenceCondition"] _import_structure["pipeline_ltx2_image2video"] = ["LTX2ImageToVideoPipeline"] _import_structure["pipeline_ltx2_latent_upsample"] = ["LTX2LatentUpsamplePipeline"] + _import_structure["pipeline_output"] = ["LTX2DFRPipelineOutput", "LTX2PipelineOutput", "LTX2VideoDecodeOutput"] _import_structure["vocoder"] = ["LTX2Vocoder", "LTX2VocoderWithBWE"] if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: diff --git a/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py b/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py index fd764f1a68cd..0385864884bb 100644 --- a/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py +++ b/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py @@ -1,13 +1,9 @@ from typing import TYPE_CHECKING, Any -from ...utils import ( - get_module_from_name, - is_torch_available, - is_comfy_kitchen_available, - logging -) +from ...utils import get_module_from_name, is_comfy_kitchen_available, is_torch_available, logging from ..base import DiffusersQuantizer + if is_comfy_kitchen_available(): import comfy_kitchen.tensor as ck_tensor @@ -91,9 +87,7 @@ def create_quantized_param( ): module, tensor_name = get_module_from_name(model, param_name) - quantized_weight = ck_tensor.QuantizedTensor.from_float( - param_value.to(target_device), self.layout.__name__ - ) + quantized_weight = ck_tensor.QuantizedTensor.from_float(param_value.to(target_device), self.layout.__name__) if tensor_name in module._parameters: module._parameters[tensor_name] = quantized_weight.to(target_device) diff --git a/src/diffusers/quantizers/quantization_config.py b/src/diffusers/quantizers/quantization_config.py index c0aa4e99d2f5..d6bd11ad98d7 100644 --- a/src/diffusers/quantizers/quantization_config.py +++ b/src/diffusers/quantizers/quantization_config.py @@ -1020,9 +1020,11 @@ def __init__( modules_to_not_convert: list[str] | None = None, **kwargs, ): - supported_formats = {"fp8","nvfp4", "mxfp8", "int8", "int4_svd", "int4_awq"} + supported_formats = {"fp8", "nvfp4", "mxfp8", "int8", "int4_svd", "int4_awq"} if quant_format not in supported_formats: - raise ValueError(f"Unsupported quant_format: {quant_format}. Supported formats are: {sorted(supported_formats)}") + raise ValueError( + f"Unsupported quant_format: {quant_format}. Supported formats are: {sorted(supported_formats)}" + ) self.quant_method = QuantizationMethod.COMFY_QUANT self.quant_format = quant_format self.compute_dtype = compute_dtype diff --git a/tests/models/testing_utils/__init__.py b/tests/models/testing_utils/__init__.py index 760a3fac04e0..9716a54307c2 100644 --- a/tests/models/testing_utils/__init__.py +++ b/tests/models/testing_utils/__init__.py @@ -29,6 +29,8 @@ BitsAndBytesCompileTesterMixin, BitsAndBytesConfigMixin, BitsAndBytesTesterMixin, + ComfyQuantConfigMixin, + ComfyQuantTesterMixin, GGUFCompileTesterMixin, GGUFConfigMixin, GGUFTesterMixin, @@ -84,6 +86,8 @@ "ModelOptCompileTesterMixin", "ModelOptConfigMixin", "ModelOptTesterMixin", + "ComfyQuantConfigMixin", + "ComfyQuantTesterMixin", "ModelTesterMixin", "NunchakuLiteCompileTesterMixin", "NunchakuLiteConfigMixin", diff --git a/tests/models/testing_utils/quantization.py b/tests/models/testing_utils/quantization.py index 65f904bd4948..5b99b10d8262 100644 --- a/tests/models/testing_utils/quantization.py +++ b/tests/models/testing_utils/quantization.py @@ -43,6 +43,7 @@ backend_empty_cache, is_autoround, is_bitsandbytes, + is_comfy_kitchen, is_gguf, is_modelopt, is_quantization, @@ -53,6 +54,7 @@ require_accelerator, require_auto_round_version_greater_or_equal, require_bitsandbytes_version_greater, + require_comfy_kitchen, require_gguf_version_greater_or_equal, require_modelopt_version_greater_or_equal, require_sdnq, @@ -1823,3 +1825,133 @@ def test_autoround_torch_compile(self): def test_autoround_torch_compile_with_group_offload(self): self._test_torch_compile_with_group_offload(self.config_dict) + + +@is_quantization +@is_comfy_kitchen +@require_comfy_kitchen +@require_accelerator +@require_accelerate +class ComfyQuantConfigMixin: + """ + Base mixin providing ComfyQuant quantization config and model creation. + + Expected class attributes: + - model_class: The model class to test + - pretrained_model_name_or_path: Hub repository ID for the pretrained model + - pretrained_model_kwargs: (Optional) Dict of kwargs to pass to from_pretrained + """ + + COMFY_QUANT_CONFIGS = { + "fp8": {"quant_format": "fp8"}, + "nvfp4": {"quant_format": "nvfp4"}, + "mxfp8": {"quant_format": "mxfp8"}, + "int8": {"quant_format": "int8"}, + "int4_svd": {"quant_format": "int4_svd"}, + "int4_awq": {"quant_format": "int4_awq"}, + } + + COMFY_QUANT_EXPECTED_MEMORY_REDUCTIONS = { + "fp8": 1.2, + "nvfp4": 1.2, + "mxfp8": 1.2, + "int8": 1.2, + "int4_svd": 1.2, + "int4_awq": 1.2, + } + + def _create_quantized_model(self, config_kwargs, **extra_kwargs): + from diffusers.quantizers.quantization_config import ComfyQuantConfig + + config = ComfyQuantConfig(**config_kwargs) + kwargs = getattr(self, "pretrained_model_kwargs", {}).copy() + kwargs["quantization_config"] = config + kwargs["device_map"] = str(torch_device) + kwargs.update(extra_kwargs) + return self.model_class.from_pretrained(self.pretrained_model_name_or_path, **kwargs) + + def _verify_if_layer_quantized(self, name, module, config_kwargs): + import comfy_kitchen.tensor as ck_tensor + + assert isinstance(module.weight, ck_tensor.QuantizedTensor), f"Layer {name} is not a ck_tensor.QuantizedTensor" + + +@is_comfy_kitchen +@require_comfy_kitchen +@require_accelerate +@require_accelerator +class ComfyQuantTesterMixin(ComfyQuantConfigMixin, QuantizationTesterMixin): + """ + Mixin class for testing ComfyQuant quantization on models. + + Expected class attributes: + - model_class: The model class to test + - pretrained_model_name_or_path: Hub repository ID for the pretrained model + - pretrained_model_kwargs: (Optional) Dict of kwargs to pass to from_pretrained (e.g., {"subfolder": "transformer"}) + + Expected methods to be implemented by subclasses: + - get_dummy_inputs(): Returns dict of inputs to pass to the model forward pass + + Optional class attributes: + - COMFY_QUANT_CONFIGS: Dict of config name -> ComfyQuantConfig kwargs to test + + Pytest mark: comfy_kitchen + Use `pytest -m "not comfy_kitchen"` to skip these tests + """ + + @pytest.mark.parametrize("config_name", ["fp8"], ids=["fp8"]) + def test_comfy_quant_quantization_num_parameters(self, config_name): + self._test_quantization_num_parameters(ComfyQuantConfigMixin.COMFY_QUANT_CONFIGS[config_name]) + + @pytest.mark.parametrize( + "config_name", + ["fp8", "int8", "int4_svd"], + ids=["fp8", "int8", "int4_svd"], + ) + def test_comfy_quant_quantization_memory_footprint(self, config_name): + expected = ComfyQuantConfigMixin.COMFY_QUANT_EXPECTED_MEMORY_REDUCTIONS.get(config_name, 1.2) + self._test_quantization_memory_footprint( + ComfyQuantConfigMixin.COMFY_QUANT_CONFIGS[config_name], expected_memory_reduction=expected + ) + + @pytest.mark.parametrize( + "config_name", + ["fp8", "int8", "int4_svd"], + ids=["fp8", "int8", "int4_svd"], + ) + def test_comfy_quant_quantization_inference(self, config_name): + self._test_quantization_inference(ComfyQuantConfigMixin.COMFY_QUANT_CONFIGS[config_name]) + + @pytest.mark.parametrize("config_name", ["fp8"], ids=["fp8"]) + def test_comfy_quant_quantization_dtype_assignment(self, config_name): + self._test_quantization_dtype_assignment(ComfyQuantConfigMixin.COMFY_QUANT_CONFIGS[config_name]) + + @pytest.mark.parametrize("config_name", ["fp8"], ids=["fp8"]) + def test_comfy_quant_quantization_lora_inference(self, config_name): + self._test_quantization_lora_inference(ComfyQuantConfigMixin.COMFY_QUANT_CONFIGS[config_name]) + + @pytest.mark.parametrize("config_name", ["fp8"], ids=["fp8"]) + def test_comfy_quant_quantization_serialization(self, config_name, tmp_path): + self._test_quantization_serialization(ComfyQuantConfigMixin.COMFY_QUANT_CONFIGS[config_name], tmp_path) + + @pytest.mark.parametrize("config_name", ["fp8"], ids=["fp8"]) + def test_comfy_quant_quantized_layers(self, config_name): + self._test_quantized_layers(ComfyQuantConfigMixin.COMFY_QUANT_CONFIGS[config_name]) + + def test_comfy_quant_modules_to_not_convert(self): + """Test that modules_to_not_convert parameter works correctly.""" + modules_to_exclude = getattr(self, "modules_to_not_convert_for_test", None) + if modules_to_exclude is None: + pytest.skip("modules_to_not_convert_for_test not defined for this model") + + self._test_quantization_modules_to_not_convert( + ComfyQuantConfigMixin.COMFY_QUANT_CONFIGS["fp8"], modules_to_exclude + ) + + def test_comfy_quant_device_map(self): + """Test that device_map='auto' works correctly with quantization.""" + self._test_quantization_device_map(ComfyQuantConfigMixin.COMFY_QUANT_CONFIGS["fp8"]) + + def test_comfy_quant_dequantize(self): + """Test that dequantize() works correctly.""" + self._test_dequantize(ComfyQuantConfigMixin.COMFY_QUANT_CONFIGS["fp8"]) diff --git a/tests/quantization/comfy_quant/test_comfy_quant.py b/tests/quantization/comfy_quant/test_comfy_quant.py index e69de29bb2d1..c74a8090c137 100644 --- a/tests/quantization/comfy_quant/test_comfy_quant.py +++ b/tests/quantization/comfy_quant/test_comfy_quant.py @@ -0,0 +1,46 @@ +import pytest +import torch +import torch.nn as nn + +from diffusers import ComfyQuantConfig +from diffusers.utils import is_comfy_kitchen_available + +from ...testing_utils import require_torch + + +if is_comfy_kitchen_available(): + import comfy_kitchen.tensor as ck_tensor + + from diffusers.quantizers.comfy_quant.comfy_quantizer import ComfyQuantizer + +device = "cuda" if torch.cuda.is_available() else "cpu" + + +@require_torch +@pytest.mark.skipif(not is_comfy_kitchen_available(), reason="comfy-kitchen is not available") +class TestComfyQuantizer: + def test_create_quantized_param_fp8(self): + config = ComfyQuantConfig(quant_format="fp8") + quantizer = ComfyQuantizer(config) + + class DummyModel(nn.Module): + def __init__(self): + super().__init__() + self.linear = nn.Linear(16, 16) + + model = DummyModel() + param_value = torch.randn(16, 16, dtype=torch.float32) + + quantizer.create_quantized_param( + model=model, + param_value=param_value, + param_name="linear.weight", + target_device=torch.device(device), + ) + + assert isinstance(model.linear.weight, ck_tensor.QuantizedTensor) + assert model.linear.weight._layout_cls == "TensorCoreFP8Layout" + + def test_create_quantized_param_invalid_format(self): + with pytest.raises(ValueError, match="Unsupported quant_format"): + ComfyQuantConfig(quant_format="INT_42") # INT_42 is an non-existent format diff --git a/tests/testing_utils.py b/tests/testing_utils.py index 9da89e626198..20cd37f1273a 100644 --- a/tests/testing_utils.py +++ b/tests/testing_utils.py @@ -34,6 +34,7 @@ is_accelerate_available, is_auto_round_available, is_bitsandbytes_available, + is_comfy_kitchen_available, is_compel_available, is_flashpack_available, is_gguf_available, @@ -492,6 +493,14 @@ def is_cache(test_case): return pytest.mark.cache(test_case) +def is_comfy_kitchen(test_case): + """ + Decorator marking a test as a Comfy Kitchen test. These tests can be filtered using: + pytest -m "not comfy_kitchen" to skip pytest -m comfy_kitchen to run only these tests + """ + return pytest.mark.comfy_kitchen(test_case) + + def require_torch(test_case): """ Decorator marking a test that requires PyTorch. These tests are skipped when PyTorch isn't installed. @@ -508,6 +517,18 @@ def require_torch_2(test_case): )(test_case) +def require_comfy_kitchen(test_case): + """ + Decorator marking a test that requires comfy_kitchen. + """ + import pytest + + if not is_comfy_kitchen_available(): + return pytest.mark.skipif(not is_comfy_kitchen_available(), reason="test requires comfy-kitchen")(test_case) + + return test_case + + def require_torch_version_greater_equal(torch_version): """Decorator marking a test that requires torch with a specific version or greater.""" From 99d5ca66bdf680c50253eaacd882ef12e951504a Mon Sep 17 00:00:00 2001 From: PrakshaaleJain Date: Thu, 17 Sep 2026 00:37:59 +0530 Subject: [PATCH 17/17] add: tests for comfy_quant --- .../quantizers/comfy_quant/comfy_quantizer.py | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py b/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py index 0385864884bb..3420e65cfa01 100644 --- a/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py +++ b/src/diffusers/quantizers/comfy_quant/comfy_quantizer.py @@ -34,26 +34,27 @@ def __init__(self, quantization_config, **kwargs): if not isinstance(self.modules_to_not_convert, list): self.modules_to_not_convert = [self.modules_to_not_convert] - # Resolve the layout class once since quant_format is constant. - layout_map = { - "fp8": getattr(ck_tensor, "TensorCoreFP8Layout", None), - "nvfp4": getattr(ck_tensor, "TensorCoreNVFP4Layout", None), - "mxfp8": getattr(ck_tensor, "TensorCoreMXFP8Layout", None), - "int8": getattr(ck_tensor, "TensorWiseINT8Layout", None), - "int4_svd": getattr(ck_tensor, "TensorCoreSVDQuantW4A4Layout", None), - "int4_awq": getattr(ck_tensor, "TensorCoreAWQW4A16Layout", None), - } - self.layout = layout_map.get(self.quant_format.lower()) - if self.layout is None: - supported = list(layout_map.keys()) - raise ValueError( - f"The layout for '{self.quant_format}' was not found in `comfy_kitchen`. " - f"Supported formats are: {supported}." - ) + if is_comfy_kitchen_available(): + # Resolve the layout class once since quant_format is constant. + layout_map = { + "fp8": getattr(ck_tensor, "TensorCoreFP8Layout", None), + "nvfp4": getattr(ck_tensor, "TensorCoreNVFP4Layout", None), + "mxfp8": getattr(ck_tensor, "TensorCoreMXFP8Layout", None), + "int8": getattr(ck_tensor, "TensorWiseINT8Layout", None), + "int4_svd": getattr(ck_tensor, "TensorCoreSVDQuantW4A4Layout", None), + "int4_awq": getattr(ck_tensor, "TensorCoreAWQW4A16Layout", None), + } + self.layout = layout_map.get(self.quant_format.lower()) + if self.layout is None: + supported = list(layout_map.keys()) + raise ValueError( + f"The layout for '{self.quant_format}' was not found in `comfy_kitchen`. " + f"Supported formats are: {supported}." + ) + else: + self.layout = None def validate_environment(self, *args, **kwargs): - from ...utils.import_utils import is_comfy_kitchen_available - if not is_comfy_kitchen_available(): raise ImportError( "Loading Comfy Quant weights requires `comfy-kitchen`. "