Skip to content

vllm.model_executor.models.encoder_cudagraph_utils

Reusable implementations of the SupportsEncoderCudaGraph protocol.

Classes:

Qwen2VLEncoderCudaGraphMixin

Bases: SupportsEncoderCudaGraph

Encoder CUDA graph support for Qwen2-VL-style vision towers.

Inputs are concatenated patches (pixel_values or pixel_values_videos) with one (t, h, w) row per item in image_grid_thw or video_grid_thw. self.visual must provide spatial_merge_size, prepare_encoder_metadata() and accept encoder_metadata in its forward; the capture inputs also read patch_embed and input_norm. Hosts still implement get_encoder_cudagraph_config().

This class holds only the glue to the encoder CUDA graph manager; model-specific metadata belongs in the vision tower. Do not add per-model branches here: a model that differs overrides the protocol method itself.

Source code in vllm/model_executor/models/encoder_cudagraph_utils.py
class Qwen2VLEncoderCudaGraphMixin(SupportsEncoderCudaGraph):
    """Encoder CUDA graph support for Qwen2-VL-style vision towers.

    Inputs are concatenated patches (`pixel_values` or
    `pixel_values_videos`) with one `(t, h, w)` row per item in
    `image_grid_thw` or `video_grid_thw`. `self.visual` must provide
    `spatial_merge_size`, `prepare_encoder_metadata()` and accept
    `encoder_metadata` in its forward; the capture inputs also read
    `patch_embed` and `input_norm`. Hosts still implement
    `get_encoder_cudagraph_config()`.

    This class holds only the glue to the encoder CUDA graph manager;
    model-specific metadata belongs in the vision tower. Do not add
    per-model branches here: a model that differs overrides the protocol
    method itself.
    """

    visual: Any

    def get_input_modality(self, mm_kwargs: dict[str, Any]) -> str:
        if "image_grid_thw" in mm_kwargs:
            return "image"
        elif "video_grid_thw" in mm_kwargs:
            return "video"
        raise AssertionError("This line should be unreachable.")

    def _get_pixel_values_by_modality(self, mm_kwargs: dict[str, Any]) -> torch.Tensor:
        if self.get_input_modality(mm_kwargs) == "image":
            return mm_kwargs["pixel_values"]
        return mm_kwargs["pixel_values_videos"]

    def _get_grid_thw_by_modality(self, mm_kwargs: dict[str, Any]) -> list[list[int]]:
        grid_thw = mm_kwargs[f"{self.get_input_modality(mm_kwargs)}_grid_thw"]
        if not isinstance(grid_thw, list):
            grid_thw = grid_thw.tolist()
        return grid_thw

    def get_encoder_cudagraph_item_specs(
        self,
        mm_kwargs: dict[str, Any],
    ) -> list[EncoderItemSpec]:
        m = self.visual.spatial_merge_size
        return [
            EncoderItemSpec(
                input_size=t * h * w,
                output_tokens=t * (h // m) * (w // m),
            )
            for t, h, w in self._get_grid_thw_by_modality(mm_kwargs)
        ]

    def select_encoder_cudagraph_items(
        self,
        mm_kwargs: dict[str, Any],
        indices: list[int],
    ) -> dict[str, Any]:
        grid_thw = self._get_grid_thw_by_modality(mm_kwargs)
        pixel_values = self._get_pixel_values_by_modality(mm_kwargs)
        modality = self.get_input_modality(mm_kwargs)
        pixel_values_key = (
            "pixel_values" if modality == "image" else "pixel_values_videos"
        )
        grid_thw_key = f"{modality}_grid_thw"

        if len(indices) == 0:
            return {pixel_values_key: pixel_values[:0], grid_thw_key: []}

        cum_patches = [0]
        for t, h, w in grid_thw:
            cum_patches.append(cum_patches[-1] + t * h * w)

        selected_pv = torch.cat(
            [pixel_values[cum_patches[i] : cum_patches[i + 1]] for i in indices]
        )
        return {
            pixel_values_key: selected_pv,
            grid_thw_key: [grid_thw[i] for i in indices],
        }

    def prepare_encoder_cudagraph_capture_inputs(
        self,
        token_budget: int,
        max_batch_size: int,
        max_frames_per_batch: int,
        device: torch.device,
        dtype: torch.dtype,
        path: str = "default",
        axis_keys: tuple[Hashable, ...] | None = None,
    ) -> EncoderCudaGraphCaptureInputs:
        m = self.visual.spatial_merge_size
        # Ceil so one item using the full budget fits the captured buffers.
        per_item_output = (token_budget + max_batch_size - 1) // max_batch_size
        frames_per_item = max_frames_per_batch // max_batch_size
        if frames_per_item > 1:
            # Video layout (T > 1) so cu_seqlens, with one entry per frame, is
            # sized for video replays. Ceil keeps frames_per_item *
            # tokens_per_frame >= per_item_output.
            tokens_per_frame = (
                per_item_output + frames_per_item - 1
            ) // frames_per_item
            grid_config = [
                [frames_per_item, m, tokens_per_frame * m]
                for _ in range(max_batch_size)
            ]
        else:
            grid_config = [[1, m, per_item_output * m] for _ in range(max_batch_size)]

        # Overwritten on replay; uint8 when normalization is fused on-device.
        patch_embed = self.visual.patch_embed
        dummy_pixel_values = torch.zeros(
            sum(t * h * w for t, h, w in grid_config),
            patch_embed.proj.in_channels
            * patch_embed.temporal_patch_size
            * patch_embed.patch_size
            * patch_embed.patch_size,
            device=device,
            dtype=self.visual.input_norm.input_dtype or dtype,
        )
        # max_seqlen is baked into the graph at capture, so it must cover one
        # item consuming the full budget.
        metadata = self.visual.prepare_encoder_metadata(
            grid_config,
            max_batch_size=max_batch_size,
            max_frames_per_batch=max_frames_per_batch,
            max_seqlen_override=token_budget * m**2,
            device=device,
        )
        # Image and video pixel_values share the [num_patches, C*T*P*P] layout,
        # so one capture buffer serves both modalities.
        return EncoderCudaGraphCaptureInputs(
            values=metadata | {"pixel_values": dummy_pixel_values}
        )

    def prepare_encoder_cudagraph_replay_buffers(
        self,
        mm_kwargs: dict[str, Any],
        max_batch_size: int,
        max_frames_per_batch: int,
        path: str = "default",
    ) -> EncoderCudaGraphReplayBuffers:
        grid_thw_list = self._get_grid_thw_by_modality(mm_kwargs)
        if self.get_input_modality(mm_kwargs) == "image":
            metadata = self.visual.prepare_encoder_metadata(
                grid_thw_list,
                max_batch_size=max_batch_size,
            )
        else:
            metadata = self.visual.prepare_encoder_metadata(
                grid_thw_list,
                max_frames_per_batch=max_frames_per_batch,
            )
        return EncoderCudaGraphReplayBuffers(
            values=metadata
            | {"pixel_values": self._get_pixel_values_by_modality(mm_kwargs)}
        )

    def encoder_cudagraph_forward(
        self,
        inputs: dict[str, torch.Tensor],
        path: str = "default",
    ) -> torch.Tensor:
        pixel_values = inputs.pop("pixel_values")
        return self.visual(pixel_values, None, encoder_metadata=inputs)

    def encoder_eager_forward(
        self,
        mm_kwargs: dict[str, Any],
        path: str = "default",
    ) -> torch.Tensor:
        pixel_values = self._get_pixel_values_by_modality(mm_kwargs)
        grid_thw = self._get_grid_thw_by_modality(mm_kwargs)
        return self.visual(pixel_values, grid_thw)