Skip to content

llmcompressor.transformers.compression.mtp

Checkpoint handling for MTP layers supported by Transformers.

Functions:

has_mtp_in_checkpoint

has_mtp_in_checkpoint(model: PreTrainedModel) -> bool

Whether the model config declares MTP layers.

Source code in src/llmcompressor/transformers/compression/mtp.py
def has_mtp_in_checkpoint(model: PreTrainedModel) -> bool:
    """Whether the model config declares MTP layers."""
    text_config = model.config.get_text_config()
    return any(
        getattr(text_config, name, 0)
        for name in (
            "num_mtp_layers",
            "mtp_num_hidden_layers",
            "num_nextn_predict_layers",
        )
    )

load_with_mtp_model

load_with_mtp_model(
    model_cls: type[PreTrainedModel] = AutoModelForCausalLM,
)

Attach MTP before the surrounding loaders convert and offload the model.

Source code in src/llmcompressor/transformers/compression/mtp.py
@contextmanager
def load_with_mtp_model(
    model_cls: type[PreTrainedModel] = AutoModelForCausalLM,
):
    """Attach MTP before the surrounding loaders convert and offload the model."""
    original_from_pretrained = model_cls.from_pretrained

    @classmethod
    @wraps(original_from_pretrained)
    def from_pretrained(cls, *args, **kwargs):
        model = original_from_pretrained(*args, **kwargs)
        if hasattr(model, "mtp") or not has_mtp_in_checkpoint(model):
            return model
        from .mtp_compat import get_mtp_config, load_mtp_model

        try:
            if not _mtp_patterns(model):
                raise ValueError("No registered MTP checkpoint patterns")
            mtp_config = get_mtp_config(model)
            if (
                getattr(mtp_config, "first_k_dense_replace", 0) > 0
                and getattr(mtp_config, "mlp_layer_types", None) is None
            ):
                raise ValueError(
                    "This Transformers version constructs dense MTP layers instead "
                    "of MoE layers. Upgrade Transformers to quantize this model's MTP."
                )
        except ValueError as error:
            validate_mtp_copy_source(model)
            logger.warning(
                f"Cannot load MTP for {type(model).__name__}: {error}. "
                "MTP weights will be copied unchanged when saving. "
                f"To quantize them explicitly, see {FALLBACK_EXAMPLE}."
            )
            return model

        # Upstream MtpModel does not dequantize source FP8/packed tensors.
        validate_mtp_copy_source(model)
        device_map = {"": "meta"} if kwargs.get("device_map") == "meta" else None
        backbone_modules = set(model.modules())
        model.mtp = load_mtp_model(model, mtp_config, device_map=device_map)

        # Keep only MTP-specific mappings here. The surrounding MoE loader owns
        # the backbone mappings and finishes configuring them after this returns.
        model._mtp_weight_conversions = [
            conv
            for conv in model.mtp._weight_conversions
            if conv not in model._weight_conversions
        ]
        # The backbone owns these modules. Keep MTP's references without
        # registering the same modules twice for offloading and serialization.
        for name in ("embed_tokens", "shared_head", "rotary_emb"):
            shared = getattr(model.mtp, name, None)
            if shared is not None and shared in backbone_modules:
                model.mtp._modules.pop(name)
                object.__setattr__(model.mtp, name, shared)
        # MtpModel derives embeddings from the backbone, including copied ones.
        model._keys_to_ignore_on_save = set(model._keys_to_ignore_on_save or ()) | {
            "mtp.embed_tokens.weight"
        }
        return model

    with patch_attr(model_cls, "from_pretrained", from_pretrained):
        yield

save_mtp_tensors

save_mtp_tensors(
    model: PreTrainedModel,
    destination: str,
    *,
    source_mtp: tuple[
        dict[str, tuple[str, str | None]], list[str]
    ]
    | None,
) -> None

Preserve unloaded MTP using an already validated source.

Source code in src/llmcompressor/transformers/compression/mtp.py
def save_mtp_tensors(
    model: PreTrainedModel,
    destination: str,
    *,
    source_mtp: tuple[dict[str, tuple[str, str | None]], list[str]] | None,
) -> None:
    """Preserve unloaded MTP using an already validated source."""
    if source_mtp is None:
        return
    weights, patterns = source_mtp
    if not weights:
        message = "Model config indicates MTP, but no MTP checkpoint weights were found"
        if not patterns:
            message += f". For unsupported FP8 layouts, see {FALLBACK_EXAMPLE}."
        logger.warning(message)
        return
    save_mtp_tensors_to_checkpoint(
        model.name_or_path,
        destination,
        source_weight_map={name: shard for name, (shard, _) in weights.items()},
    )

validate_mtp_copy_source

validate_mtp_copy_source(
    model: PreTrainedModel,
) -> (
    tuple[dict[str, tuple[str, str | None]], list[str]]
    | None
)

Return source MTP weights and patterns only if the weights are unquantized.

Return None when the model config does not declare MTP. If MTP is declared but no weights are found, return an empty weight map with the patterns; save_mtp_tensors warns when saving this source.

Raise ValueError for source-quantized MTP weights, which cannot be copied without preserving their serving scheme.

Source code in src/llmcompressor/transformers/compression/mtp.py
def validate_mtp_copy_source(
    model: PreTrainedModel,
) -> tuple[dict[str, tuple[str, str | None]], list[str]] | None:
    """Return source MTP weights and patterns only if the weights are unquantized.

    Return None when the model config does not declare MTP. If MTP is declared
    but no weights are found, return an empty weight map with the patterns;
    save_mtp_tensors warns when saving this source.

    Raise ValueError for source-quantized MTP weights, which cannot be copied
    without preserving their serving scheme.
    """
    if not has_mtp_in_checkpoint(model):
        return None

    weights, patterns = _mtp_weights(model)
    qparams = set(QuantizationMetadata.all_qparam_names()) | {
        "weight_packed",
        "weight_scale_inv",
    }
    quantized = any(
        name.rpartition(".")[-1] in qparams
        or (dtype is not None and dtype.startswith("F8"))
        or (
            name.endswith(".weight")
            and dtype in ("I8", "U8")
            and f"{name.removesuffix('.weight')}.scale" in weights
        )
        for name, (_, dtype) in weights.items()
    )
    if quantized:
        raise ValueError(
            "Cannot copy source-quantized MTP weights without their serving "
            f"scheme. Convert them first; see {FALLBACK_EXAMPLE}."
        )
    return weights, patterns