llmcompressor.entrypoints.model_free.converter

Classes:

ModelFreePtqConverter –

Converter that quantizes and compresses safetensors checkpoints without

ModelFreePtqConverter

ModelFreePtqConverter(config: QuantizationConfig)

Bases: Converter

Converter that quantizes and compresses safetensors checkpoints without loading a model. Implements the compressed-tensors Converter protocol so it can be chained with other converters (e.g. a dequantizer) and used with convert_checkpoint.

Methods:

get_dependencies –

For microscale schemes, return fused partner tensor names that must be
process –

Quantize and compress all tensors. Device is inferred from the tensors.
update_config –

Build or merge the final QuantizationConfig in COMPRESSED status.

Source code in src/llmcompressor/entrypoints/model_free/converter.py

def __init__(self, config: QuantizationConfig):
    self.config = config

get_dependencies

get_dependencies(weight_name: str) -> set[str]

For microscale schemes, return fused partner tensor names that must be loaded together to compute a shared global scale. Standard schemes have no cross-tensor dependencies.

Source code in src/llmcompressor/entrypoints/model_free/converter.py

def get_dependencies(self, weight_name: str) -> set[str]:
    """
    For microscale schemes, return fused partner tensor names that must be
    loaded together to compute a shared global scale. Standard schemes have
    no cross-tensor dependencies.
    """
    if not has_microscale_scheme(self.config):
        return set()

    deps = set()
    for primary_pattern, partner_templates in DEFAULT_FUSED_MAPPINGS.items():
        match = re.match(primary_pattern, weight_name)
        if match is None:
            continue
        for template in partner_templates:
            deps.add(template.format(**match.groupdict()))
    return deps

process

process(
    tensors: dict[str, Tensor],
) -> dict[str, torch.Tensor]

Quantize and compress all tensors. Device is inferred from the tensors.

Source code in src/llmcompressor/entrypoints/model_free/converter.py

def process(self, tensors: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]:
    """Quantize and compress all tensors. Device is inferred from the tensors."""
    tensors = split_fused_moe_experts(tensors)
    if has_microscale_scheme(self.config):
        return self._process_microscale(tensors)
    return self._process_standard(tensors)

update_config

update_config(
    config: QuantizationConfig | None,
) -> QuantizationConfig

Build or merge the final QuantizationConfig in COMPRESSED status. When chained after a dequantizer (config=None), creates from scratch. When seeded with a pre-existing checkpoint config, merges into it.

Source code in src/llmcompressor/entrypoints/model_free/converter.py

def update_config(self, config: QuantizationConfig | None) -> QuantizationConfig:
    """
    Build or merge the final QuantizationConfig in COMPRESSED status.
    When chained after a dequantizer (config=None), creates from scratch.
    When seeded with a pre-existing checkpoint config, merges into it.
    """
    unique_formats = {
        s.format for s in self.config.config_groups.values() if s.format is not None
    }
    if len(unique_formats) == 1:
        format_ = next(iter(unique_formats))
    elif len(unique_formats) > 1:
        format_ = "mixed-precision"
    else:
        format_ = "dense"

    new_config = QuantizationConfig(
        config_groups=self.config.config_groups,
        kv_cache_scheme=self.config.kv_cache_scheme,
        ignore=list(self.config.ignore) if self.config.ignore else [],
        quantization_status=QuantizationStatus.COMPRESSED,
        format=format_,
    )
    if config is None:
        return new_config
    config.merge(new_config)
    return config

split_fused_moe_experts

split_fused_moe_experts(
    tensors: dict[str, Tensor],
) -> dict[str, torch.Tensor]

Find fused MoE experts (with gate_up_proj/down_proj). Split them from 3D tensors into individual 2D expert tensors.

Args: tensors: Dictionary of loaded tensors from safetensors file

Returns: split_tensors: New dictionary with split expert weights

Source code in src/llmcompressor/entrypoints/model_free/converter.py

def split_fused_moe_experts(
    tensors: dict[str, torch.Tensor],
) -> dict[str, torch.Tensor]:
    """
    Find fused MoE experts (with gate_up_proj/down_proj).
    Split them from 3D tensors into individual 2D expert tensors.

    Args:
        tensors: Dictionary of loaded tensors from safetensors file

    Returns:
        split_tensors: New dictionary with split expert weights
    """
    split_tensors = {}

    params_to_split = {
        # If a 3D gate_up_proj layer is found, split it into a
        # 2D gate_proj and up_proj layer for each expert
        "gate_up_proj": ["gate_proj", "up_proj"],
        # If a 3D down_proj layer is found, split it into a
        # 2D down_proj layer for each expert
        "down_proj": ["down_proj"],
    }

    for name, tensor in tensors.items():
        keys_to_split = [key for key in params_to_split if key in name]
        if len(keys_to_split) >= 2:
            raise ValueError(f"Found multiple keys matching {name}: {keys_to_split}")

        elif len(keys_to_split) == 1 and tensor.ndim == 3:
            unsplit_name = keys_to_split[0]
            split_names = params_to_split[unsplit_name]

            # Get number of experts
            num_experts = tensor.shape[0]

            if tensor.shape[1] % len(split_names) != 0:
                raise ValueError(
                    f"{unsplit_name} expects a second dimension divisible by "
                    f"{len(split_names)} but got shape: {tensor.shape}"
                )

            # Split into experts
            intermediate_size = tensor.shape[1] // len(split_names)
            for expert_idx in range(num_experts):
                expert_tensor = tensor[expert_idx]
                # Split into layers
                split_layers = expert_tensor.split(intermediate_size, dim=0)
                for split_name, split_layer in zip(split_names, split_layers):
                    key = name.replace(unsplit_name, f"{expert_idx}.{split_name}")
                    if not key.endswith(".weight"):
                        key = f"{key}.weight"
                    split_tensors[key] = split_layer

            logger.info(f"Split {name} into {num_experts} experts")

        else:
            # Non-MoE or non-3D tensors, keep as is
            split_tensors[name] = tensor

    return split_tensors