Skip to content

vllm.models.hy_v4.nvidia.vision

Classes:

HYV4VisionPatchEmbed

Bases: Module

Methods:

Source code in vllm/models/hy_v4/nvidia/vision.py
class HYV4VisionPatchEmbed(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.config = config
        self.position_edge = config.max_image_size // config.patch_size
        self.patch_embedding = nn.Conv2d(
            config.num_channels,
            config.hidden_size,
            config.patch_size,
            stride=config.patch_size,
            bias=True,
        )
        self.position_embedding = nn.Embedding(
            self.position_edge**2 + 1, config.hidden_size, dtype=torch.float32
        )

    def _patch_conv(self, pixel_values: torch.Tensor) -> torch.Tensor:
        config = self.config
        patches = pixel_values.reshape(
            -1, config.num_channels, config.patch_size, config.patch_size
        )
        return self.patch_embedding(patches).flatten(2).squeeze(-1).unsqueeze(0)

    def compute_pos_embeds(
        self, grid_thw: list[list[int]], dtype: torch.dtype
    ) -> torch.Tensor:
        """Interpolated position embedding per patch, `(num_patches, hidden)`."""
        positions = (
            self.position_embedding.weight[1:]
            .float()
            .reshape(1, self.position_edge, self.position_edge, self.config.hidden_size)
            .permute(0, 3, 1, 2)
        )
        positional = []
        for t, h, w in grid_thw:
            resized = F.interpolate(
                positions,
                scale_factor=(
                    (h + 0.1) / self.position_edge,
                    (w + 0.1) / self.position_edge,
                ),
                mode=self.config.interpolate_mode,
                align_corners=False,
            )
            positional.append(
                resized.flatten(2).transpose(1, 2).to(dtype).repeat(1, t, 1)
            )
        return torch.cat(positional, dim=1).squeeze(0)

    def forward(self, pixel_values, grid_thw):
        patches = self._patch_conv(pixel_values)
        pos_embeds = self.compute_pos_embeds(grid_thw, patches.dtype)
        return patches + pos_embeds.unsqueeze(0)

    def forward_with_pos_embeds(
        self, pixel_values: torch.Tensor, pos_embeds: torch.Tensor
    ) -> torch.Tensor:
        """Packed path: patch conv plus precomputed position embeddings."""
        return self._patch_conv(pixel_values) + pos_embeds.unsqueeze(0)

compute_pos_embeds(grid_thw, dtype)

Interpolated position embedding per patch, (num_patches, hidden).

Source code in vllm/models/hy_v4/nvidia/vision.py
def compute_pos_embeds(
    self, grid_thw: list[list[int]], dtype: torch.dtype
) -> torch.Tensor:
    """Interpolated position embedding per patch, `(num_patches, hidden)`."""
    positions = (
        self.position_embedding.weight[1:]
        .float()
        .reshape(1, self.position_edge, self.position_edge, self.config.hidden_size)
        .permute(0, 3, 1, 2)
    )
    positional = []
    for t, h, w in grid_thw:
        resized = F.interpolate(
            positions,
            scale_factor=(
                (h + 0.1) / self.position_edge,
                (w + 0.1) / self.position_edge,
            ),
            mode=self.config.interpolate_mode,
            align_corners=False,
        )
        positional.append(
            resized.flatten(2).transpose(1, 2).to(dtype).repeat(1, t, 1)
        )
    return torch.cat(positional, dim=1).squeeze(0)

forward_with_pos_embeds(pixel_values, pos_embeds)

Packed path: patch conv plus precomputed position embeddings.

Source code in vllm/models/hy_v4/nvidia/vision.py
def forward_with_pos_embeds(
    self, pixel_values: torch.Tensor, pos_embeds: torch.Tensor
) -> torch.Tensor:
    """Packed path: patch conv plus precomputed position embeddings."""
    return self._patch_conv(pixel_values) + pos_embeds.unsqueeze(0)

HYV4VisionTransformer

Bases: Module

Methods:

Source code in vllm/models/hy_v4/nvidia/vision.py
class HYV4VisionTransformer(nn.Module):
    def __init__(
        self,
        config,
        quant_config=None,
        prefix="",
        input_norm: nn.Module | None = None,
    ):
        super().__init__()
        self.config = config
        self.input_norm = input_norm or IdentityInputNorm()
        self.embeddings = HYV4VisionPatchEmbed(config)
        self.layers = nn.ModuleList(
            [
                HYV4VisionBlock(config, quant_config, f"{prefix}.layers.{i}")
                for i in range(config.num_hidden_layers)
            ]
        )
        self.perceive = HYV4VisionPatchMerger(config)
        self.attn_backend = get_vit_attn_backend(
            head_size=config.hidden_size // config.num_attention_heads,
            dtype=torch.get_default_dtype(),
        )
        use_dp = is_vit_use_data_parallel(config.num_attention_heads)
        self.tp_size = 1 if use_dp else get_tensor_model_parallel_world_size()
        self.rotary_pos_emb = None
        if config.use_2d_rope:
            if config.rope_2d_percent != 1.0:
                raise ValueError("HYV4 vision supports only full 2D RoPE")
            self.rotary_pos_emb = get_rope(
                head_size=config.hidden_size // config.num_attention_heads,
                max_position=self.embeddings.position_edge + 1,
                is_neox_style=True,
                rope_parameters={
                    "rope_theta": config.rope_2d_theta,
                    "partial_rotary_factor": 0.5,
                },
                dtype=torch.float32,
            )

    @property
    def dtype(self):
        return self.embeddings.patch_embedding.weight.dtype

    @property
    def device(self):
        return self.embeddings.patch_embedding.weight.device

    def _rope_cos_sin(
        self, grid_thw: list[list[int]]
    ) -> tuple[torch.Tensor, torch.Tensor]:
        assert self.rotary_pos_emb is not None
        max_grid = max(max(h, w) for _, h, w in grid_thw)
        cos_table, sin_table = self.rotary_pos_emb.get_cos_sin(max_grid)
        indices = [
            torch.stack(
                (torch.arange(h).repeat_interleave(w), torch.arange(w).repeat(h)),
                dim=-1,
            ).repeat(t, 1)
            for t, h, w in grid_thw
        ]
        indices = torch.cat(indices).to(cos_table.device)
        return cos_table[indices].flatten(1), sin_table[indices].flatten(1)

    def _encode(
        self, pixel_values: torch.Tensor, grid_thw: list[list[int]]
    ) -> torch.Tensor:
        """Patch embedding plus ViT layers over the packed patch sequence."""
        config = self.config
        lengths = [h * w for t, h, w in grid_thw for _ in range(t)]
        if sum(lengths) != pixel_values.shape[0] or not lengths or min(lengths) <= 0:
            raise ValueError(
                "HYV4 pixel_values length does not match positive grid_thw"
            )
        hidden = self.embeddings(
            self.input_norm(pixel_values.to(device=self.device), self.dtype), grid_thw
        )
        cu = np.cumsum([0, *lengths], dtype=np.int32)
        max_seq = torch.tensor(
            MMEncoderAttention.compute_max_seqlen(self.attn_backend, cu),
            dtype=torch.int32,
        )
        seq_lens = MMEncoderAttention.maybe_compute_seq_lens(
            self.attn_backend, cu, self.device
        )
        cu_seqlens = MMEncoderAttention.maybe_recompute_cu_seqlens(
            self.attn_backend, cu, config.hidden_size, self.tp_size, self.device
        )
        cos = sin = None
        if self.rotary_pos_emb is not None:
            cos, sin = self._rope_cos_sin(grid_thw)
        for layer in self.layers:
            hidden = layer(
                hidden,
                cu_seqlens=cu_seqlens,
                max_seqlen=max_seq,
                cos=cos,
                sin=sin,
                sequence_lengths=seq_lens,
            )
        return hidden

    def _iter_item_states(
        self, hidden: torch.Tensor, grid_thw: list[list[int]]
    ) -> Iterable[torch.Tensor]:
        """Per-item `(frames, h, w, hidden)` slices with temporal padding."""
        config = self.config
        offset = 0
        for t, h, w in grid_thw:
            states = hidden[:, offset : offset + t * h * w].reshape(t, h, w, -1)
            offset += t * h * w
            if t > 1 and t % config.temporal_patch_size:
                pad = config.temporal_patch_size - t % config.temporal_patch_size
                states = torch.cat((states, states[-1:].expand(pad, -1, -1, -1)), dim=0)
            yield states

    def forward(
        self, pixel_values: torch.Tensor, grid_thw: list[list[int]]
    ) -> list[torch.Tensor]:
        hidden = self._encode(pixel_values, grid_thw)
        return [
            self.perceive(states) for states in self._iter_item_states(hidden, grid_thw)
        ]

    def prepare_encoder_cudagraph_metadata(
        self,
        grid_thw: list[list[int]],
        *,
        device: torch.device,
        max_seqlen_override: int | None = None,
    ) -> dict[str, torch.Tensor | None]:
        """Precompute the packed-forward inputs that depend only on grids."""
        grid_thw = [[int(v) for v in grid] for grid in grid_thw]
        metadata: dict[str, torch.Tensor | None] = {}
        metadata["pos_embeds"] = self.embeddings.compute_pos_embeds(
            grid_thw, self.dtype
        ).to(device)
        if self.rotary_pos_emb is not None:
            cos, sin = self._rope_cos_sin(grid_thw)
            metadata["rope_cos"] = cos.to(device)
            metadata["rope_sin"] = sin.to(device)
        else:
            metadata["rope_cos"] = metadata["rope_sin"] = None
        lengths = [h * w for t, h, w in grid_thw for _ in range(t)]
        cu = np.cumsum([0, *lengths], dtype=np.int32)
        max_seqlen = (
            max_seqlen_override
            if max_seqlen_override is not None
            else MMEncoderAttention.compute_max_seqlen(self.attn_backend, cu)
        )
        metadata["max_seqlen"] = torch.tensor(max_seqlen, dtype=torch.int32)
        metadata["cu_seqlens"] = MMEncoderAttention.maybe_recompute_cu_seqlens(
            self.attn_backend, cu, self.config.hidden_size, self.tp_size, device
        )
        return metadata

    def forward_packed(
        self,
        patches: torch.Tensor,
        pos_embeds: torch.Tensor,
        rope_cos: torch.Tensor | None,
        rope_sin: torch.Tensor | None,
        cu_seqlens: torch.Tensor,
        max_seqlen: torch.Tensor,
    ) -> torch.Tensor:
        """Static-shape packed ViT forward for encoder CUDA graph capture/replay.

        `patches` holds the *embedded* patches (input norm + patch conv
        applied eagerly at replay time): both conv2d kernels (patch embed and
        merger) are cudnn calls whose results are not invariant to input
        shape (measured on GB200), so they cannot run on padded buffers
        without changing results bit-wise. Everything left in the graph —
        pos-embed add, per-layer LayerNorm/GEMM/RoPE/FlashAttn-varlen — is
        row-invariant and verified bitwise-identical to the eager path.

        All grid-dependent host logic must be precomputed via
        `prepare_encoder_cudagraph_metadata`. Returns the padded packed
        hidden states `(1, num_patch_slots, hidden_size)`; the per-item
        merger then runs eagerly on the real rows (in
        `postprocess_encoder_output`).
        """
        hidden = patches.unsqueeze(0) + pos_embeds.unsqueeze(0)
        for layer in self.layers:
            hidden = layer(
                hidden,
                cu_seqlens=cu_seqlens,
                max_seqlen=max_seqlen,
                cos=rope_cos,
                sin=rope_sin,
                sequence_lengths=None,
            )
        return hidden

    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
        params = dict(self.named_parameters())
        loaded = set()
        for name, weight in weights:
            for projection, shard in (
                ("q_proj", "q"),
                ("k_proj", "k"),
                ("v_proj", "v"),
            ):
                if f".{projection}." in name:
                    name = name.replace(f".{projection}.", ".qkv.")
                    param = params[name]
                    param.weight_loader(param, weight, shard)
                    break
            else:
                param = params[name]
                getattr(param, "weight_loader", default_weight_loader)(param, weight)
            loaded.add(name)
        return loaded

_encode(pixel_values, grid_thw)

Patch embedding plus ViT layers over the packed patch sequence.

Source code in vllm/models/hy_v4/nvidia/vision.py
def _encode(
    self, pixel_values: torch.Tensor, grid_thw: list[list[int]]
) -> torch.Tensor:
    """Patch embedding plus ViT layers over the packed patch sequence."""
    config = self.config
    lengths = [h * w for t, h, w in grid_thw for _ in range(t)]
    if sum(lengths) != pixel_values.shape[0] or not lengths or min(lengths) <= 0:
        raise ValueError(
            "HYV4 pixel_values length does not match positive grid_thw"
        )
    hidden = self.embeddings(
        self.input_norm(pixel_values.to(device=self.device), self.dtype), grid_thw
    )
    cu = np.cumsum([0, *lengths], dtype=np.int32)
    max_seq = torch.tensor(
        MMEncoderAttention.compute_max_seqlen(self.attn_backend, cu),
        dtype=torch.int32,
    )
    seq_lens = MMEncoderAttention.maybe_compute_seq_lens(
        self.attn_backend, cu, self.device
    )
    cu_seqlens = MMEncoderAttention.maybe_recompute_cu_seqlens(
        self.attn_backend, cu, config.hidden_size, self.tp_size, self.device
    )
    cos = sin = None
    if self.rotary_pos_emb is not None:
        cos, sin = self._rope_cos_sin(grid_thw)
    for layer in self.layers:
        hidden = layer(
            hidden,
            cu_seqlens=cu_seqlens,
            max_seqlen=max_seq,
            cos=cos,
            sin=sin,
            sequence_lengths=seq_lens,
        )
    return hidden

_iter_item_states(hidden, grid_thw)

Per-item (frames, h, w, hidden) slices with temporal padding.

Source code in vllm/models/hy_v4/nvidia/vision.py
def _iter_item_states(
    self, hidden: torch.Tensor, grid_thw: list[list[int]]
) -> Iterable[torch.Tensor]:
    """Per-item `(frames, h, w, hidden)` slices with temporal padding."""
    config = self.config
    offset = 0
    for t, h, w in grid_thw:
        states = hidden[:, offset : offset + t * h * w].reshape(t, h, w, -1)
        offset += t * h * w
        if t > 1 and t % config.temporal_patch_size:
            pad = config.temporal_patch_size - t % config.temporal_patch_size
            states = torch.cat((states, states[-1:].expand(pad, -1, -1, -1)), dim=0)
        yield states

forward_packed(patches, pos_embeds, rope_cos, rope_sin, cu_seqlens, max_seqlen)

Static-shape packed ViT forward for encoder CUDA graph capture/replay.

patches holds the embedded patches (input norm + patch conv applied eagerly at replay time): both conv2d kernels (patch embed and merger) are cudnn calls whose results are not invariant to input shape (measured on GB200), so they cannot run on padded buffers without changing results bit-wise. Everything left in the graph — pos-embed add, per-layer LayerNorm/GEMM/RoPE/FlashAttn-varlen — is row-invariant and verified bitwise-identical to the eager path.

All grid-dependent host logic must be precomputed via prepare_encoder_cudagraph_metadata. Returns the padded packed hidden states (1, num_patch_slots, hidden_size); the per-item merger then runs eagerly on the real rows (in postprocess_encoder_output).

Source code in vllm/models/hy_v4/nvidia/vision.py
def forward_packed(
    self,
    patches: torch.Tensor,
    pos_embeds: torch.Tensor,
    rope_cos: torch.Tensor | None,
    rope_sin: torch.Tensor | None,
    cu_seqlens: torch.Tensor,
    max_seqlen: torch.Tensor,
) -> torch.Tensor:
    """Static-shape packed ViT forward for encoder CUDA graph capture/replay.

    `patches` holds the *embedded* patches (input norm + patch conv
    applied eagerly at replay time): both conv2d kernels (patch embed and
    merger) are cudnn calls whose results are not invariant to input
    shape (measured on GB200), so they cannot run on padded buffers
    without changing results bit-wise. Everything left in the graph —
    pos-embed add, per-layer LayerNorm/GEMM/RoPE/FlashAttn-varlen — is
    row-invariant and verified bitwise-identical to the eager path.

    All grid-dependent host logic must be precomputed via
    `prepare_encoder_cudagraph_metadata`. Returns the padded packed
    hidden states `(1, num_patch_slots, hidden_size)`; the per-item
    merger then runs eagerly on the real rows (in
    `postprocess_encoder_output`).
    """
    hidden = patches.unsqueeze(0) + pos_embeds.unsqueeze(0)
    for layer in self.layers:
        hidden = layer(
            hidden,
            cu_seqlens=cu_seqlens,
            max_seqlen=max_seqlen,
            cos=rope_cos,
            sin=rope_sin,
            sequence_lengths=None,
        )
    return hidden

prepare_encoder_cudagraph_metadata(grid_thw, *, device, max_seqlen_override=None)

Precompute the packed-forward inputs that depend only on grids.

Source code in vllm/models/hy_v4/nvidia/vision.py
def prepare_encoder_cudagraph_metadata(
    self,
    grid_thw: list[list[int]],
    *,
    device: torch.device,
    max_seqlen_override: int | None = None,
) -> dict[str, torch.Tensor | None]:
    """Precompute the packed-forward inputs that depend only on grids."""
    grid_thw = [[int(v) for v in grid] for grid in grid_thw]
    metadata: dict[str, torch.Tensor | None] = {}
    metadata["pos_embeds"] = self.embeddings.compute_pos_embeds(
        grid_thw, self.dtype
    ).to(device)
    if self.rotary_pos_emb is not None:
        cos, sin = self._rope_cos_sin(grid_thw)
        metadata["rope_cos"] = cos.to(device)
        metadata["rope_sin"] = sin.to(device)
    else:
        metadata["rope_cos"] = metadata["rope_sin"] = None
    lengths = [h * w for t, h, w in grid_thw for _ in range(t)]
    cu = np.cumsum([0, *lengths], dtype=np.int32)
    max_seqlen = (
        max_seqlen_override
        if max_seqlen_override is not None
        else MMEncoderAttention.compute_max_seqlen(self.attn_backend, cu)
    )
    metadata["max_seqlen"] = torch.tensor(max_seqlen, dtype=torch.int32)
    metadata["cu_seqlens"] = MMEncoderAttention.maybe_recompute_cu_seqlens(
        self.attn_backend, cu, self.config.hidden_size, self.tp_size, device
    )
    return metadata