Skip to content

llmcompressor.entrypoints.model_free.converter

Classes:

ModelFreePtqConverter

ModelFreePtqConverter(
    config: QuantizationConfig,
    weight_names: Iterable[str] | None = None,
)

Bases: Converter

Converter that quantizes and compresses safetensors checkpoints without loading a model. Implements the compressed-tensors Converter protocol so it can be chained with other converters (e.g. a dequantizer) and used with convert_checkpoint. For microscale schemes, it needs the checkpoint's weight_names to resolve fused weights; model_free_ptq passes them.

Parameters:

  • config (QuantizationConfig) –

    quantization config to apply

  • weight_names (Iterable[str] | None, default: None ) –

    names of all tensors in the checkpoint. Required to resolve fused weights for microscale schemes, because a layer's fused group depends on its sibling layers. For example, Kimi g_proj is fused with delta attention q/k/v/b/f_a projections on some layers and with MLA q_a/kv_a projections on others

Methods:

  • get_dependencies –

    For microscale schemes, return fused partner tensor names that must be

  • process –

    Quantize and compress all tensors. Device is inferred from the tensors.

  • update_config –

    Build or merge the final QuantizationConfig in COMPRESSED status.

  • validate –

    Validate tensor shapes and simulate the compressed state-dict layout.

Source code in src/llmcompressor/entrypoints/model_free/converter.py
def __init__(
    self,
    config: QuantizationConfig,
    weight_names: Iterable[str] | None = None,
):
    """
    :param config: quantization config to apply
    :param weight_names: names of all tensors in the checkpoint. Required
        to resolve fused weights for microscale schemes, because a layer's
        fused group depends on its sibling layers. For example, Kimi
        `g_proj` is fused with delta attention q/k/v/b/f_a projections on
        some layers and with MLA q_a/kv_a projections on others
    """
    self.config = config

    # primary weight name -> names of the fused partners it loads
    self._fused_partners: dict[str, set[str]] | None = None
    if weight_names is not None:
        self._fused_partners = {}
        for fused_group in get_fused_names(weight_names):
            primary, *partners = fused_group.values()
            self._fused_partners[primary] = set(partners)

get_dependencies

get_dependencies(weight_name: str) -> set[str]

For microscale schemes, return fused partner tensor names that must be loaded together to compute a shared global scale. Only the primary weight of a fused group has dependencies, so the shard owning the primary loads the whole group. Standard schemes have no cross-tensor dependencies.

Source code in src/llmcompressor/entrypoints/model_free/converter.py
def get_dependencies(self, weight_name: str) -> set[str]:
    """
    For microscale schemes, return fused partner tensor names that must be
    loaded together to compute a shared global scale. Only the primary
    weight of a fused group has dependencies, so the shard owning the
    primary loads the whole group. Standard schemes have no cross-tensor
    dependencies.
    """
    if not has_microscale_scheme(self.config):
        return set()

    if self._fused_partners is None:
        raise ValueError(
            "Resolving fused weights for microscale schemes requires the "
            "checkpoint's weight names. Please construct "
            "ModelFreePtqConverter with `weight_names`"
        )

    return set(self._fused_partners.get(weight_name, ()))

process

process(
    tensors: dict[str, Tensor],
) -> dict[str, torch.Tensor]

Quantize and compress all tensors. Device is inferred from the tensors.

Source code in src/llmcompressor/entrypoints/model_free/converter.py
def process(self, tensors: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]:
    """Quantize and compress all tensors. Device is inferred from the tensors."""
    tensors = split_fused_moe_experts(tensors)
    if has_microscale_scheme(self.config):
        return self._process_microscale(tensors)
    return self._process_standard(tensors)

update_config

update_config(
    config: QuantizationConfig | None,
) -> QuantizationConfig

Build or merge the final QuantizationConfig in COMPRESSED status. When chained after a dequantizer (config=None), creates from scratch. When seeded with a pre-existing checkpoint config, merges into it.

Source code in src/llmcompressor/entrypoints/model_free/converter.py
def update_config(self, config: QuantizationConfig | None) -> QuantizationConfig:
    """
    Build or merge the final QuantizationConfig in COMPRESSED status.
    When chained after a dequantizer (config=None), creates from scratch.
    When seeded with a pre-existing checkpoint config, merges into it.
    """
    unique_formats = {
        s.format for s in self.config.config_groups.values() if s.format is not None
    }
    if len(unique_formats) == 1:
        format_ = next(iter(unique_formats))
    elif len(unique_formats) > 1:
        format_ = "mixed-precision"
    else:
        format_ = "dense"

    new_config = QuantizationConfig(
        config_groups=self.config.config_groups,
        kv_cache_scheme=self.config.kv_cache_scheme,
        ignore=list(self.config.ignore) if self.config.ignore else [],
        quantization_status=QuantizationStatus.COMPRESSED,
        format=format_,
    )
    if config is None:
        return new_config
    config.merge(new_config)
    return config

validate

validate(
    tensors: dict[str, Tensor],
) -> dict[str, torch.Tensor]

Validate tensor shapes and simulate the compressed state-dict layout.

Converter validation runs on meta tensors before conversion. Some calibration observers need tensor values, so validation initializes each quantized module with meta tensors and compresses it without calibration. This verifies the same shape and compression compatibility checks as :meth:process while leaving observers to run only during the real conversion passes.

Source code in src/llmcompressor/entrypoints/model_free/converter.py
def validate(self, tensors: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]:
    """
    Validate tensor shapes and simulate the compressed state-dict layout.

    Converter validation runs on meta tensors before conversion. Some
    calibration observers need tensor values, so validation initializes each
    quantized module with meta tensors and compresses it without calibration.
    This verifies the same shape and compression compatibility checks as
    :meth:`process` while leaving observers to run only during the real
    conversion passes.
    """
    tensors = split_fused_moe_experts(tensors)
    output = dict(tensors)
    for module_name, name, scheme in _match_tensors_to_schemes(
        tensors, self.config
    ):
        validate_weight_for_quantization(tensors[name], scheme, name)
        module = initialize_quantized_linear(
            tensors[name], scheme, torch.device("meta")
        )
        compress_module(module)

        del output[name]
        prefix = module_name + "."
        for key, value in module.state_dict(prefix=prefix).items():
            output[key] = value

    return output

split_fused_moe_experts

split_fused_moe_experts(
    tensors: dict[str, Tensor],
) -> dict[str, torch.Tensor]

Find fused MoE experts (with gate_up_proj/down_proj). Split them from 3D tensors into individual 2D expert tensors.

Args: tensors: Dictionary of loaded tensors from safetensors file

Returns: split_tensors: New dictionary with split expert weights

Source code in src/llmcompressor/entrypoints/model_free/converter.py
def split_fused_moe_experts(
    tensors: dict[str, torch.Tensor],
) -> dict[str, torch.Tensor]:
    """
    Find fused MoE experts (with gate_up_proj/down_proj).
    Split them from 3D tensors into individual 2D expert tensors.

    Args:
        tensors: Dictionary of loaded tensors from safetensors file

    Returns:
        split_tensors: New dictionary with split expert weights
    """
    split_tensors = {}

    params_to_split = {
        # If a 3D gate_up_proj layer is found, split it into a
        # 2D gate_proj and up_proj layer for each expert
        "gate_up_proj": ["gate_proj", "up_proj"],
        # If a 3D down_proj layer is found, split it into a
        # 2D down_proj layer for each expert
        "down_proj": ["down_proj"],
    }

    for name, tensor in tensors.items():
        keys_to_split = [key for key in params_to_split if key in name]
        if len(keys_to_split) >= 2:
            raise ValueError(f"Found multiple keys matching {name}: {keys_to_split}")

        elif len(keys_to_split) == 1 and tensor.ndim == 3:
            unsplit_name = keys_to_split[0]
            split_names = params_to_split[unsplit_name]

            # Get number of experts
            num_experts = tensor.shape[0]

            if tensor.shape[1] % len(split_names) != 0:
                raise ValueError(
                    f"{unsplit_name} expects a second dimension divisible by "
                    f"{len(split_names)} but got shape: {tensor.shape}"
                )

            # Split into experts
            intermediate_size = tensor.shape[1] // len(split_names)
            for expert_idx in range(num_experts):
                expert_tensor = tensor[expert_idx]
                # Split into layers
                split_layers = expert_tensor.split(intermediate_size, dim=0)
                for split_name, split_layer in zip(split_names, split_layers):
                    key = name.replace(unsplit_name, f"{expert_idx}.{split_name}")
                    if not key.endswith(".weight"):
                        key = f"{key}.weight"
                    split_tensors[key] = split_layer

            logger.info(f"Split {name} into {num_experts} experts")

        else:
            # Non-MoE or non-3D tensors, keep as is
            split_tensors[name] = tensor

    return split_tensors