Skip to content

vllm.models.hy_v4_flash.nvidia.model

HYV4 Flash backbone using the public HYV4 checkpoint contract.

Classes:

HYV4FlashForCausalLM

Bases: HYV4ForCausalLM

Methods:

Source code in vllm/models/hy_v4_flash/nvidia/model.py
class HYV4FlashForCausalLM(HYV4ForCausalLM):
    model_cls = HYV4FlashModel
    supports_lora = False  # type: ignore[assignment]
    supports_pp = False  # type: ignore[assignment]
    requires_raw_input_tokens = True
    packed_modules_mapping = {"gate_up_proj": ["gate_proj", "up_proj"]}
    hf_to_vllm_mapper = WeightsMapper(
        orig_to_new_stacked={
            ".mlp.gate_proj": (".mlp.gate_up_proj", 0),
            ".mlp.up_proj": (".mlp.gate_up_proj", 1),
            ".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0),
            ".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1),
        },
        orig_to_new_substr={".compressed_indexer.": ".indexer."},
    )

    def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
        validate_runtime(vllm_config)
        super().__init__(vllm_config=vllm_config, prefix=prefix)
        self.token_lookback_depth = self.model.token_lookback_depth
        self._weight_loading_depth = 0
        self._seen_weights: set[str] = set()
        self._loaded_weights: set[str] = set()
        self._load_config = vllm_config.load_config

    @staticmethod
    def get_model_state_cls():
        from vllm.models.deepseek_v41.nvidia.model_state import DeepseekV41ModelState

        return DeepseekV41ModelState

    def forward(
        self,
        input_ids,
        positions,
        intermediate_tensors=None,
        inputs_embeds=None,
        lookback_token_ids=None,
    ):
        return self.model(
            input_ids,
            positions,
            intermediate_tensors,
            inputs_embeds,
            lookback_token_ids,
        )

    @contextmanager
    def weight_loading_session(self):
        """Audit the complete stream, including interleaved VL tower weights."""
        outermost = self._weight_loading_depth == 0
        if outermost:
            self.model.loaded_parts.clear()
            self._seen_weights.clear()
            self._loaded_weights.clear()
        self._weight_loading_depth += 1
        try:
            yield
            if outermost:
                from vllm.model_executor.model_loader.default_loader import (
                    DefaultModelLoader,
                )

                required = {
                    name
                    for name, _ in self.named_parameters()
                    if name.endswith((".weight", ".w13_weight", ".w2_weight"))
                }
                missing = required - self._loaded_weights
                if missing:
                    raise ValueError(
                        f"Missing Flash checkpoint weights: {sorted(missing)}"
                    )
                DefaultModelLoader(self._load_config).track_weights_loading(
                    self, self._loaded_weights.copy(), quantized=True
                )
                self.model.verify_loaded_parts()
        finally:
            self._weight_loading_depth -= 1
            if outermost:
                self._seen_weights.clear()
                self._loaded_weights.clear()

    def _target_weights(self, weights):
        for name, value in weights:
            if name in self._seen_weights:
                raise ValueError(f"Duplicate Flash checkpoint weight: {name}")
            self._seen_weights.add(name)
            engram = re.fullmatch(
                r"model\.layers\.(\d+)\.(engram|engram_layer)\.(.+)", name
            )
            if engram is not None:
                layer_number, prefix, leaf = engram.groups()
                layer_id = int(layer_number) + (prefix == "engram_layer")
                if layer_id not in (self.config.engram_layer_ids or ()):
                    raise ValueError(f"Undeclared Engram checkpoint weight: {name}")
                name = f"model.layers.{layer_id - 1}.engram_layer.{leaf}"
            skip = False
            if name.startswith("model.layers."):
                layer = name.removeprefix("model.layers.").split(".", 1)[0]
                first = self.config.num_hidden_layers
                if layer.isdigit() and first <= int(layer) < (
                    first + self.config.num_nextn_predict_layers
                ):
                    continue
            for prefix, count in (
                ("model.mtp_layers.", self.config.num_nextn_predict_layers),
                (
                    "model.dfly_layers.",
                    getattr(self.config, "dfly_num_hidden_layers", 0),
                ),
            ):
                if name.startswith(prefix):
                    local = name[len(prefix) :]
                    layer = local.removeprefix("layers.").split(".", 1)[0]
                    declared = count > 0 and layer.isdigit() and 0 <= int(layer) < count
                    if prefix == "model.dfly_layers." and count > 0:
                        declared |= local.startswith(
                            (
                                "mtp_fusion.",
                                "hidden_correction.",
                                "final_norm.",
                                "norm.",
                                "embed_tokens.",
                                "confidence_head.",
                            )
                        )
                    if not declared:
                        raise ValueError(f"Undeclared auxiliary Flash weight: {name}")
                    skip = True
                    break
            if not skip:
                yield name, value

    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
        with self.weight_loading_session():
            loaded = AutoWeightsLoader(self).load_weights(
                self._target_weights(weights), mapper=self.hf_to_vllm_mapper
            )
            self._loaded_weights.update(loaded)
            return loaded

weight_loading_session()

Audit the complete stream, including interleaved VL tower weights.

Source code in vllm/models/hy_v4_flash/nvidia/model.py
@contextmanager
def weight_loading_session(self):
    """Audit the complete stream, including interleaved VL tower weights."""
    outermost = self._weight_loading_depth == 0
    if outermost:
        self.model.loaded_parts.clear()
        self._seen_weights.clear()
        self._loaded_weights.clear()
    self._weight_loading_depth += 1
    try:
        yield
        if outermost:
            from vllm.model_executor.model_loader.default_loader import (
                DefaultModelLoader,
            )

            required = {
                name
                for name, _ in self.named_parameters()
                if name.endswith((".weight", ".w13_weight", ".w2_weight"))
            }
            missing = required - self._loaded_weights
            if missing:
                raise ValueError(
                    f"Missing Flash checkpoint weights: {sorted(missing)}"
                )
            DefaultModelLoader(self._load_config).track_weights_loading(
                self, self._loaded_weights.copy(), quantized=True
            )
            self.model.verify_loaded_parts()
    finally:
        self._weight_loading_depth -= 1
        if outermost:
            self._seen_weights.clear()
            self._loaded_weights.clear()