Skip to content

vllm.models.hy_v4.nvidia.dfly

Classes:

HYV4DFlyFeedForward

Bases: HYV4FeedForward

Keep the module identity while waiting for the streamed FFN width.

Source code in vllm/models/hy_v4/nvidia/dfly.py
class HYV4DFlyFeedForward(HYV4FeedForward):
    """Keep the module identity while waiting for the streamed FFN width."""

    def __init__(
        self,
        hidden_size: int,
        intermediate_size: int,
        hidden_act: str,
        quant_config: QuantizationConfig | None = None,
        reduce_results: bool = True,
        prefix: str = "",
    ):
        nn.Module.__init__(self)
        self._kwargs: dict[str, Any] = dict(
            hidden_size=hidden_size,
            hidden_act=hidden_act,
            quant_config=quant_config,
            reduce_results=reduce_results,
            prefix=prefix,
        )
        self.register_buffer("_placement", torch.empty(0), persistent=False)

    def materialize(self, intermediate_size: int) -> None:
        with (
            torch.device(self._placement.device),
            set_default_torch_dtype(self._placement.dtype),
        ):
            mlp = HYV4FeedForward(intermediate_size=intermediate_size, **self._kwargs)
        self.gate_up_proj = mlp.gate_up_proj
        self.down_proj = mlp.down_proj
        self.act_fn = mlp.act_fn

HYV4DFlyQuantizationConfig

Bases: CompressedTensorsConfig

Match draft linears against their checkpoint paths, without rewriting regexes.

Source code in vllm/models/hy_v4/nvidia/dfly.py
class HYV4DFlyQuantizationConfig(CompressedTensorsConfig):
    """Match draft linears against their checkpoint paths, without rewriting regexes."""

    model_prefix: str

    @classmethod
    def for_model(
        cls, source: CompressedTensorsConfig, model_prefix: str
    ) -> "HYV4DFlyQuantizationConfig":
        if source.config is None:
            raise ValueError("DFly requires serialized compressed-tensors metadata")
        config = cls.from_config(deepcopy(source.config))
        assert isinstance(config, cls)
        config.model_prefix = f"{model_prefix}."
        config.packed_modules_mapping = source.packed_modules_mapping.copy()
        return config

    def get_scheme_dict(self, layer: nn.Module, layer_name: str | None = None):
        if layer_name is not None and layer_name.startswith(self.model_prefix):
            local_name = layer_name.removeprefix(self.model_prefix)
            local_name = local_name.removeprefix("layers.")
            layer_name = f"model.dfly_layers.{local_name}"
        scheme = super().get_scheme_dict(layer, layer_name)
        if scheme is not None:
            weights = scheme["weights"]
            if not isinstance(weights, QuantizationArgs) or weights.num_bits != 8:
                raise ValueError("HYV4 DFly draft layers require BF16 or blockwise FP8")
        return scheme