Skip to content

llmcompressor.modifiers.transform.awq.dynamic_mappings

Dynamic AWQ mapping builders for hybrid attention models.

Models with hybrid attention (mix of full self-attention and linear/Gated DeltaNet attention) need layer-index-specific AWQ mappings that vary by model size. This module provides runtime detection and mapping generation for such architectures (e.g. Qwen3Next, Qwen3.5).

Functions:

build_gemma4_mappings

build_gemma4_mappings(
    model: PreTrainedModel,
) -> list[AWQMapping] | None

Build AWQ mappings for Gemma4ForConditionalGeneration and Gemma4UnifiedForConditionalGeneration.

Gemma4 has hybrid attention with two types of layers: - sliding_attention layers: have separate k_proj and v_proj (GQA) - full_attention layers: share k=v projection (no separate v_proj)

For sliding_attention layers with GQA, the v_proj -> o_proj mapping is skipped since it would hit a GQA dimension mismatch. For full_attention layers, k and v are shared, so the k_proj -> o_proj mapping is also skipped since altering k_proj would change the attention behavior.

Source code in src/llmcompressor/modifiers/transform/awq/dynamic_mappings.py
def build_gemma4_mappings(model: PreTrainedModel) -> list[AWQMapping] | None:
    """
    Build AWQ mappings for Gemma4ForConditionalGeneration and
    Gemma4UnifiedForConditionalGeneration.

    Gemma4 has hybrid attention with two types of layers:
    - sliding_attention layers: have separate k_proj and v_proj (GQA)
    - full_attention layers: share k=v projection (no separate v_proj)

    For sliding_attention layers with GQA, the v_proj -> o_proj mapping is
    skipped since it would hit a GQA dimension mismatch. For full_attention
    layers, k and v are shared, so the k_proj -> o_proj mapping is also
    skipped since altering k_proj would change the attention behavior.
    """
    config = model.config
    text_config = config.get_text_config()
    layer_types = getattr(text_config, "layer_types", None)
    num_layers = getattr(text_config, "num_hidden_layers", None)

    if layer_types is None or num_layers is None:
        return None

    # Find sliding and full attention layer indices
    sliding_indices = [
        i for i, t in enumerate(layer_types[:num_layers]) if t == "sliding_attention"
    ]
    full_indices = [
        i for i, t in enumerate(layer_types[:num_layers]) if t == "full_attention"
    ]

    if not sliding_indices and not full_indices:
        return None

    mappings = []

    # Sliding attention layers: input_layernorm -> q/k/v_proj
    # Balance layers must also be scoped to sliding layers only to avoid
    # matching full attention layers which don't have v_proj
    if sliding_indices:
        sliding_re = "|".join(str(i) for i in sliding_indices)
        mappings.append(
            AWQMapping(
                f"re:.*layers\\.({sliding_re})\\.input_layernorm$",
                [
                    f"re:.*layers\\.({sliding_re})\\.self_attn\\.q_proj$",
                    f"re:.*layers\\.({sliding_re})\\.self_attn\\.k_proj$",
                    f"re:.*layers\\.({sliding_re})\\.self_attn\\.v_proj$",
                ],
            )
        )
        # No v_proj -> o_proj mapping due to GQA dimension mismatch is unsupported

    # Full attention layers: input_layernorm -> q/k_proj (k=v, no separate v_proj)
    # Balance layers scoped to full attention layers only
    # Note: can't do k_proj -> o_proj mapping since that would alter the attention
    if full_indices:
        full_re = "|".join(str(i) for i in full_indices)
        mappings.append(
            AWQMapping(
                f"re:.*layers\\.({full_re})\\.input_layernorm$",
                [
                    f"re:.*layers\\.({full_re})\\.self_attn\\.q_proj$",
                    f"re:.*layers\\.({full_re})\\.self_attn\\.k_proj$",
                ],
            )
        )
        # No k_proj -> o_proj mapping, altering k_proj would change attention behavior

    all_re = "|".join(str(i) for i in range(num_layers))
    is_moe = bool(getattr(text_config, "enable_moe_block", False))

    # Dense MLP: pre_feedforward_layernorm -> mlp gate/up_proj
    mappings.append(
        AWQMapping(
            f"re:.*layers\\.({all_re})\\.pre_feedforward_layernorm$",
            [
                f"re:.*layers\\.({all_re})\\.mlp\\.gate_proj$",
                f"re:.*layers\\.({all_re})\\.mlp\\.up_proj$",
            ],
        )
    )

    # MoE experts: pre_feedforward_layernorm_2 -> experts gate/up_proj
    if is_moe:
        mappings.append(
            AWQMapping(
                f"re:.*layers\\.({all_re})\\.pre_feedforward_layernorm_2$",
                [
                    f"re:.*layers\\.({all_re})\\.experts\\..*\\.gate_proj$",
                    f"re:.*layers\\.({all_re})\\.experts\\..*\\.up_proj$",
                ],
            )
        )

    mappings.append(AWQMapping("re:.*up_proj$", ["re:.*down_proj$"]))

    logger.info(
        f"Built dynamic Gemma4 AWQ mappings: "
        f"{len(sliding_indices)} sliding-attention layers, "
        f"{len(full_indices)} full-attention layers (k=v), MoE: {is_moe}"
    )

    return mappings

build_hybrid_attention_mappings

build_hybrid_attention_mappings(
    model: Module,
) -> list[AWQMapping] | None

Dynamically build AWQ mappings for models with hybrid attention (full self-attention + linear/Gated DeltaNet attention), such as Qwen3Next and Qwen3.5.

Reads layer_types from the model config to determine which layers use full vs linear attention, then inspects the model's module names to detect the correct linear attention projection names and MLP structure.

Returns None if the model is not a hybrid attention model.

Source code in src/llmcompressor/modifiers/transform/awq/dynamic_mappings.py
def build_hybrid_attention_mappings(model: Module) -> list[AWQMapping] | None:
    """
    Dynamically build AWQ mappings for models with hybrid attention
    (full self-attention + linear/Gated DeltaNet attention), such as
    Qwen3Next and Qwen3.5.

    Reads layer_types from the model config to determine which layers use
    full vs linear attention, then inspects the model's module names to
    detect the correct linear attention projection names and MLP structure.

    Returns None if the model is not a hybrid attention model.
    """
    result = get_hybrid_attention_config(model)
    if result is None:
        return None

    layer_types, num_layers = result

    full_indices = [i for i in range(num_layers) if layer_types[i] == "full_attention"]
    linear_indices = [
        i for i in range(num_layers) if layer_types[i] == "linear_attention"
    ]

    if not full_indices or not linear_indices:
        logger.warning(
            "Hybrid attention model detected but could not find indices for "
            "both full and linear attention layers. Falling back."
        )
        return None

    full_re = "|".join(str(i) for i in full_indices)
    linear_re = "|".join(str(i) for i in linear_indices)

    linear_proj_names = _detect_linear_attn_projections(model)
    is_moe = is_moe_model(model)

    mappings = []

    # Full attention layers: input_layernorm -> q/k/v_proj
    mappings.append(
        AWQMapping(
            f"re:.*layers\\.({full_re})\\.input_layernorm$",
            [
                "re:.*self_attn.q_proj$",
                "re:.*self_attn.k_proj$",
                "re:.*self_attn.v_proj$",
            ],
        )
    )

    # Linear attention layers: input_layernorm -> linear_attn projections
    if linear_proj_names:
        mappings.append(
            AWQMapping(
                f"re:.*layers\\.({linear_re})\\.input_layernorm$",
                [f"re:.*linear_attn.{p}$" for p in linear_proj_names],
            )
        )

    # MLP mappings depend on whether the model uses MoE
    if is_moe:
        mappings.append(
            AWQMapping(
                "re:.*post_attention_layernorm$",
                [
                    # TODO: should add "re:.*mlp.gate.weight$" but is a Parameter
                    "re:.*mlp.experts.*.gate_proj$",
                    "re:.*mlp.experts.*.up_proj$",
                    "re:.*mlp.shared_expert_gate$",
                    "re:.*mlp.shared_expert.gate_proj$",
                    "re:.*mlp.shared_expert.up_proj$",
                ],
            )
        )
    else:
        mappings.append(
            AWQMapping(
                "re:.*post_attention_layernorm$",
                ["re:.*gate_proj$", "re:.*up_proj$"],
            )
        )

    mappings.append(AWQMapping("re:.*up_proj$", ["re:.*down_proj$"]))

    logger.info(
        f"Built dynamic hybrid attention AWQ mappings: "
        f"{len(full_indices)} full-attention layers, "
        f"{len(linear_indices)} linear-attention layers, "
        f"linear projections: {linear_proj_names}, MoE: {is_moe}"
    )

    return mappings

build_step3p5_mappings

build_step3p5_mappings(
    model: Module,
) -> list[AWQMapping] | None

Dynamically build AWQ mappings for Step3p5 models.

Step-3.5-Flash uses dense FFN layers early in the stack and MoE FFN layers later in the stack. The dense and MoE post-attention mappings must be layer-index-specific so AWQ only groups each norm with balance layers that exist in the same decoder layer.

Source code in src/llmcompressor/modifiers/transform/awq/dynamic_mappings.py
def build_step3p5_mappings(model: Module) -> list[AWQMapping] | None:
    """
    Dynamically build AWQ mappings for Step3p5 models.

    Step-3.5-Flash uses dense FFN layers early in the stack and MoE FFN layers
    later in the stack. The dense and MoE post-attention mappings must be
    layer-index-specific so AWQ only groups each norm with balance layers that
    exist in the same decoder layer.
    """
    dense_indices, moe_indices = _detect_step3p5_ffn_layer_indices(model)

    if not dense_indices and not moe_indices:
        logger.warning(
            "Step3p5 model detected but dense/MoE FFN layer indices could not be "
            "inferred. Falling back."
        )
        return None

    mappings = [
        AWQMapping(
            "re:.*input_layernorm$",
            [
                "re:.*self_attn.q_proj$",
                "re:.*self_attn.k_proj$",
                "re:.*self_attn.v_proj$",
                "re:.*self_attn.g_proj$",
            ],
        ),
        AWQMapping("re:.*self_attn.v_proj$", ["re:.*self_attn.o_proj$"]),
    ]

    if dense_indices:
        dense_re = "|".join(str(i) for i in dense_indices)
        mappings.append(
            AWQMapping(
                f"re:.*layers\\.({dense_re})\\.post_attention_layernorm$",
                [
                    "re:.*mlp.gate_proj$",
                    "re:.*mlp.up_proj$",
                ],
            )
        )

    if moe_indices:
        moe_re = "|".join(str(i) for i in moe_indices)
        mappings.append(
            AWQMapping(
                f"re:.*layers\\.({moe_re})\\.post_attention_layernorm$",
                [
                    "re:.*moe.gate$",
                    "re:.*moe.gate_proj$",
                    "re:.*moe.up_proj$",
                    "re:.*share_expert.gate_proj$",
                    "re:.*share_expert.up_proj$",
                ],
            )
        )

    # The packed moe.up_proj -> moe.down_proj path is intentionally excluded
    # because AWQ's smooth-layer update assumes a 1D/2D smooth weight.
    mappings.append(
        AWQMapping(
            "re:.*(mlp|share_expert).up_proj$",
            ["re:.*(mlp|share_expert).down_proj$"],
        )
    )

    logger.info(
        f"Built dynamic Step3p5 AWQ mappings: "
        f"{len(dense_indices)} dense layers, {len(moe_indices)} MoE layers"
    )

    return mappings

get_layer_mappings_from_model

get_layer_mappings_from_model(
    model: Module,
) -> list[AWQMapping]

Infer AWQ mappings from a model. Checks the dynamic mapping registry first (for models needing runtime-generated mappings), then falls back to the static registry, then to default mappings.

Parameters:

  • model (Module) –

    the model to infer mappings for

Returns:

  • list[AWQMapping] –

    list of AWQMapping for the model

Source code in src/llmcompressor/modifiers/transform/awq/dynamic_mappings.py
def get_layer_mappings_from_model(model: Module) -> list[AWQMapping]:
    """
    Infer AWQ mappings from a model. Checks the dynamic mapping registry
    first (for models needing runtime-generated mappings), then falls back
    to the static registry, then to default mappings.

    :param model: the model to infer mappings for
    :return: list of AWQMapping for the model
    """
    model_name = model.__class__.__name__

    if model_name in AWQ_DYNAMIC_MAPPING_REGISTRY:
        mappings = AWQ_DYNAMIC_MAPPING_REGISTRY[model_name](model)
        if mappings is not None:
            return mappings

    if model_name in AWQ_MAPPING_REGISTRY:
        return AWQ_MAPPING_REGISTRY[model_name]

    logger.info(
        f"Architecture {model_name} not found in mappings. "
        f"Using default mappings: {default_mappings}"
    )
    return default_mappings