vllm.config.model_arch ¶

@dataclass(config=ConfigDict(arbitrary_types_allowed=True))
class ModelArchitectureConfig:
    """
    Configuration for model architecture that required by vLLM runtime
    """

    architectures: list[str]
    """List of model architecture class names (e.g., ['LlamaForCausalLM']).
       It can be None upon calling `vllm_config.with_hf_config(config.text_config)`"""

    model_type: str
    """Model type identifier (e.g., 'llama', 'gpt_oss')."""

    text_model_type: str | None
    """Text model type identifier (e.g., 'llama4_text')."""

    hidden_size: int
    """Hidden size of the model."""

    total_num_hidden_layers: int
    """Number of hidden layers in the model."""

    total_num_attention_heads: int
    """Number of attention heads in the model."""

    head_size: int
    """Head dimension of the model."""

    vocab_size: int
    """Vocabulary size of the model."""

    total_num_kv_heads: int
    """Number of key value heads in the model."""

    num_experts: int
    """Number of experts in the model."""

    num_experts_per_token: int
    """Number of routed experts selected per token."""

    quantization_config: dict[str, Any] | None
    """Quantization configuration dictionary containing quantization parameters."""

    is_deepseek_mla: bool
    """Whether the model is a DeepSeek MLA model."""

    is_mm_prefix_lm: bool
    """Whether the model uses image bidirectional attention."""

    rswa_window: int | None
    """Reference Sliding Window Attention window size (None disables R-SWA)."""

    derived_max_model_len_and_key: tuple[float, str | None]
    """Derived maximum model length and key from the hf config."""

    per_layer_overrides: "list[dict[str, Any]] | None" = None
    """Per-layer values for the fields that vary, `None` unless some field does.

    One dict per layer, holding only the fields whose value differs from the
    whole-model value above. Everything else is read from the whole-model config,
    so later edits to it are visible through `self[layer_idx]`."""

    def __getitem__(self, layer_idx: int) -> "ModelArchitectureConfig":
        """ModelArchitectureConfig for a specific layer.

        Returns `self` when no field varies by layer, so callers never need to
        branch on heterogeneity. Mirrors `PreTrainedConfig.per_layer_config[i]`.
        """
        if layer_idx < 0:
            # Homogeneous configs would ignore a negative index and heterogeneous
            # ones would wrap, so an off-by-one would only show up on the models
            # this exists for. Reject it for both.
            raise IndexError(f"layer index must not be negative, got {layer_idx}")
        if self.per_layer_overrides is None:
            return self
        layer = copy(self)
        object.__setattr__(layer, "per_layer_overrides", None)
        for name, value in self.per_layer_overrides[layer_idx].items():
            object.__setattr__(layer, name, value)
        return layer

    @classmethod
    def from_layers(
        cls, layers: "list[ModelArchitectureConfig]"
    ) -> "ModelArchitectureConfig":
        """Whole-model config for a checkpoint whose layers differ.

        Fields that agree across layers are taken as they are. Fields that differ
        are collapsed with `max`, so buffers are sized for the largest layer, and
        the differing values are kept per layer. No field is named here: which
        ones vary is whatever the checkpoint says.
        """
        if not layers:
            raise ValueError("a model must have at least one layer")

        merged: dict[str, Any] = {}
        overrides: list[dict[str, Any]] = [{} for _ in layers]
        for f in dataclass_fields(cls):
            if f.name == "per_layer_overrides":
                continue
            values = [getattr(layer, f.name) for layer in layers]
            if all(value == values[0] for value in values):
                merged[f.name] = values[0]
                continue
            # `bool` is an `int`, so an exact type check is what keeps a varying
            # flag from collapsing to `any`. `is_deepseek_mla` doing that would
            # make `use_mla` true model wide, and `get_num_kv_heads` then returns
            # 1 for every layer, discarding the overrides built here.
            if not all(type(value) in (int, float) for value in values):
                raise ValueError(
                    f"{f.name!r} varies across layers and has no whole-model "
                    f"value: {sorted(set(map(repr, values)))}. Only numeric "
                    f"fields collapse (with `max`, to size buffers for the "
                    f"largest layer); give this one an explicit rule in "
                    f"ModelArchitectureConfig.from_layers."
                )
            merged[f.name] = max(values)
            for override, value in zip(overrides, values):
                if value != merged[f.name]:
                    override[f.name] = value

        if len(layers) != merged["total_num_hidden_layers"]:
            raise ValueError(
                f"got {len(layers)} per-layer configs for a model with "
                f"{merged['total_num_hidden_layers']} layers"
            )
        # A checkpoint can be heterogeneous over attributes vLLM never reads, in
        # which case there is nothing to keep the layers apart for.
        return cls(**merged, per_layer_overrides=overrides if any(overrides) else None)