class HYV4VLEncoderCudaGraphMixin:
"""`SupportsEncoderCudaGraph` for `HYV4VLForConditionalGeneration`.
Expects `self.visual`/`self.config`/`self.model_config` from the
host model.
"""
supports_encoder_cudagraph: ClassVar[Literal[True]] = True
# Provided by the host model; declared here so mixin methods type-check.
visual: "HYV4VisionTransformer | None"
config: "HYV4VLConfig"
model_config: "ModelConfig"
multimodal_config: "MultiModalConfig | None"
# -- helpers --
def get_input_modality(self, mm_kwargs: dict[str, Any]) -> str:
if "image_grid_thw" in mm_kwargs:
return "image"
if "video_grid_thw" in mm_kwargs:
return "video"
raise ValueError("HYV4 encoder CUDA graph inputs need a grid_thw key")
@staticmethod
def _get_grid_list(mm_kwargs: dict[str, Any], key: str) -> list[list[int]]:
grid = mm_kwargs[key]
if isinstance(grid, torch.Tensor):
grid = grid.tolist()
return [[int(v) for v in row] for row in grid]
def _get_pixel_values(self, mm_kwargs: dict[str, Any]) -> torch.Tensor:
return mm_kwargs[self._pixel_key(self.get_input_modality(mm_kwargs))]
@staticmethod
def _pixel_key(modality: str) -> str:
return "pixel_values" if modality == "image" else "pixel_values_videos"
# -- SupportsEncoderCudaGraph protocol --
def get_max_frames_per_video(self) -> int:
info = MULTIMODAL_REGISTRY.get_processing_info(self.model_config)
assert isinstance(info, HYV4VLProcessingInfo)
return info.get_max_frames_per_video()
def get_encoder_cudagraph_config(self) -> EncoderCudaGraphConfig:
if self.visual is None:
raise ValueError(
"cudagraph_mm_encoder requires the HYV4 vision tower, but it "
"was stubbed out because both image and video limits are 0. "
"Disable cudagraph_mm_encoder."
)
backend = self.visual.attn_backend
# FlashInfer reads max_seqlen on the host and TORCH_SDPA calls
# .tolist() on CUDA cu_seqlens; only FLASH_ATTN is capture-safe here.
if backend != AttentionBackendEnum.FLASH_ATTN:
raise ValueError(
f"cudagraph_mm_encoder is not supported with the "
f"{backend.name} ViT attention backend for HYV4 VL. Set "
"--mm-encoder-attn-backend FLASH_ATTN or disable "
"cudagraph_mm_encoder."
)
buffer_keys = ["patches", "pos_embeds"]
if self.visual.rotary_pos_emb is not None:
buffer_keys += ["rope_cos", "rope_sin"]
buffer_keys += ["cu_seqlens", "max_seqlen"]
# FlashAttn varlen pays a fixed per-sequence scheduling cost even for
# zero-length padding sequences (~0.14us/seq/layer measured on
# GB200/FA4), so the frame-slot count baked into a captured graph
# taxes every replay. Capture separate image and video graphs: image
# graphs need only one slot per item, video graphs cover multi-frame
# batches (bounded by the frame charge in the item specs).
video_enabled = (
self.multimodal_config is None
or self.multimodal_config.get_limit_per_prompt("video") > 0
)
capture_axes = (("image", "video"),) if video_enabled else ()
return EncoderCudaGraphConfig(
modalities=["image", "video"],
buffer_keys=buffer_keys,
out_hidden_size=self.config.text_config.hidden_size,
max_frames_per_video=self.get_max_frames_per_video(),
capture_axes=capture_axes,
)
def get_encoder_cudagraph_budget_range(
self,
vllm_config: "VllmConfig",
) -> tuple[int, int]:
# Budgets are in patches. Min: comfortably above one smallest image
# (and >= the auto-inferred max_batch_size = 64 via the manager's
# max_batch_size <= min_budget invariant). Max: one batched-prefill
# worth of vision tokens, converted to patches.
vision = self.config.vision_config
patches_per_token = (
vision.temporal_patch_size
* vision.spatial_merge_size**2
* vision.spatial_patch_size**2
)
min_budget = 64
max_budget = (
min(
vllm_config.scheduler_config.max_num_batched_tokens,
vllm_config.model_config.max_model_len,
)
* patches_per_token
)
return (min_budget, max_budget)
def get_encoder_cudagraph_item_specs(
self,
mm_kwargs: dict[str, Any],
) -> list[EncoderItemSpec]:
modality = self.get_input_modality(mm_kwargs)
vision = self.config.vision_config
eff_merge = vision.spatial_merge_size * vision.spatial_patch_size
specs = []
for grid in self._get_grid_list(mm_kwargs, f"{modality}_grid_thw"):
t, h, w = grid
patches = t * h * w
path_tokens = patches
if modality == "video":
# Charge frames against the budget so the packer splits
# frame-heavy batches; captured video graphs carry
# budget // (_FRAME_CHARGE * eff_merge**2) cu_seqlens slots.
path_tokens = max(patches, _FRAME_CHARGE * eff_merge * eff_merge * t)
specs.append(
EncoderItemSpec(
input_size=patches,
output_tokens=vision_token_count(grid, vision, modality),
path_output_tokens={"default": path_tokens},
)
)
return specs
def select_encoder_cudagraph_items(
self,
mm_kwargs: dict[str, Any],
indices: list[int],
) -> dict[str, Any]:
modality = self.get_input_modality(mm_kwargs)
grid_key = f"{modality}_grid_thw"
grids = self._get_grid_list(mm_kwargs, grid_key)
pixel_values = self._get_pixel_values(mm_kwargs)
if len(indices) == 0:
return {
self._pixel_key(modality): pixel_values[:0],
grid_key: torch.zeros((0, 3), dtype=torch.int64),
ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG: (modality,),
}
cum_patches = [0]
for t, h, w in grids:
cum_patches.append(cum_patches[-1] + t * h * w)
return {
self._pixel_key(modality): torch.cat(
[pixel_values[cum_patches[i] : cum_patches[i + 1]] for i in indices]
),
grid_key: torch.tensor([grids[i] for i in indices], dtype=torch.int64),
ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG: (modality,),
}
def prepare_encoder_cudagraph_capture_inputs(
self,
token_budget: int,
max_batch_size: int,
max_frames_per_batch: int,
device: torch.device,
dtype: torch.dtype,
path: str = "default",
axis_keys: tuple[Hashable, ...] | None = None,
) -> EncoderCudaGraphCaptureInputs:
if self.visual is None:
raise RuntimeError(
"cudagraph_mm_encoder requires the HYV4 vision tower, but it "
"was stubbed out because the image/video limits are 0. "
"Disable cudagraph_mm_encoder."
)
visual = self.visual
vision = self.config.vision_config
eff_merge = vision.spatial_merge_size * vision.spatial_patch_size
patches_per_cell = eff_merge**2
# Replay buffers are global and flat: the dummy grids only fix buffer
# *sizes*, their per-item layout is irrelevant at replay. Size them
# tightly against the worst-case replay batch of `token_budget`
# patches and at most `max_batch_size` items, with every dummy frame
# grid small enough for the position-embedding / 2D-RoPE tables
# (both sized position_edge).
side = visual.embeddings.position_edge // eff_merge
if side < 1:
raise ValueError(
f"HYV4 vision position table (edge "
f"{visual.embeddings.position_edge}) is smaller than one "
f"merge cell ({eff_merge}); cannot capture encoder CUDA graphs."
)
patches_per_frame_cap = side * side * patches_per_cell
axis = axis_keys[0] if axis_keys else None
if axis == "image" or (axis is None and max_frames_per_batch == 0):
# One frame per dummy item; add dummy items until each stays
# within the position-table capacity.
n_items = max(max_batch_size, -(-token_budget // patches_per_frame_cap))
t_per_item = [1] * n_items
cells = -(-token_budget // (n_items * patches_per_cell))
else:
# Video-capable graph (the default when no capture axes are set).
# The frame charge in the item specs bounds replay frames to
# budget // (_FRAME_CHARGE * patches_per_cell).
frames = max(
max_batch_size,
-(-token_budget // (_FRAME_CHARGE * patches_per_cell)),
-(-token_budget // patches_per_frame_cap),
)
cells = -(-token_budget // (frames * patches_per_cell))
base, rem = divmod(frames, max_batch_size)
t_per_item = [base + (1 if i < rem else 0) for i in range(max_batch_size)]
# Split cells into rows/cols that fit the position/RoPE tables.
if cells <= side:
rows, cols = 1, cells
else:
cols = side
rows = -(-cells // side)
assert rows <= side, "cells is capped by side**2 by construction"
grids = [[t, rows * eff_merge, cols * eff_merge] for t in t_per_item]
total_patches = sum(t_per_item) * rows * cols * patches_per_cell
# max_seqlen is baked into the graph at capture time; cover the worst
# replay case (one item's frame filling the whole budget) as well as
# the dummy frame length.
max_seqlen_override = max(token_budget, cells * patches_per_cell)
metadata = visual.prepare_encoder_cudagraph_metadata(
grids,
device=device,
max_seqlen_override=max_seqlen_override,
)
# The buffer holds *embedded* patches (input norm + patch conv are
# applied eagerly at replay time, because cudnn conv results are not
# invariant to input shape and would break bitwise parity).
patches = torch.zeros(
total_patches,
vision.hidden_size,
device=device,
dtype=dtype,
)
values: dict[str, torch.Tensor] = {"patches": patches}
values.update({k: v for k, v in metadata.items() if v is not None})
return EncoderCudaGraphCaptureInputs(values=values)
def prepare_encoder_cudagraph_replay_buffers(
self,
mm_kwargs: dict[str, Any],
max_batch_size: int,
max_frames_per_batch: int,
path: str = "default",
) -> EncoderCudaGraphReplayBuffers:
modality = self.get_input_modality(mm_kwargs)
grids = self._get_grid_list(mm_kwargs, f"{modality}_grid_thw")
pixel_values = self._get_pixel_values(mm_kwargs)
assert self.visual is not None
# Run the fused input norm and patch conv eagerly, replicating the
# eager call exactly (cudnn conv is not batch-shape invariant, so the
# conv must see the real unpadded batch to stay bitwise-identical).
patches = self.visual.embeddings._patch_conv(
self.visual.input_norm(pixel_values, self.visual.dtype)
).squeeze(0)
# Repeating the final cumulative offset leaves padding rows outside
# every sequence, as in the other packed vision encoders.
metadata = self.visual.prepare_encoder_cudagraph_metadata(
grids,
device=pixel_values.device,
)
metadata.pop("max_seqlen")
values: dict[str, torch.Tensor | None] = {"patches": patches}
values.update(metadata)
return EncoderCudaGraphReplayBuffers(values=values)
def encoder_cudagraph_forward(
self,
inputs: dict[str, torch.Tensor],
path: str = "default",
) -> torch.Tensor:
assert self.visual is not None
return self.visual.forward_packed(
inputs["patches"],
inputs["pos_embeds"],
inputs.get("rope_cos"),
inputs.get("rope_sin"),
inputs["cu_seqlens"],
inputs["max_seqlen"],
).squeeze(0)
def encoder_eager_forward(
self,
mm_kwargs: dict[str, Any],
path: str = "default",
) -> torch.Tensor:
modality = self.get_input_modality(mm_kwargs)
grids = self._get_grid_list(mm_kwargs, f"{modality}_grid_thw")
pixel_values = self._get_pixel_values(mm_kwargs)
assert self.visual is not None
if not grids:
return torch.zeros(
(1, 0, self.config.vision_config.hidden_size),
device=self.visual.device,
dtype=self.visual.dtype,
)
return self.visual._encode(pixel_values, grids)
def postprocess_encoder_output(
self,
outputs: dict[str, torch.Tensor],
indices: list[int],
per_item_out_tokens: list[int],
dest: dict[int, torch.Tensor] | list[torch.Tensor | None],
clone: bool = False,
batch_mm_kwargs: dict[str, Any] | None = None,
) -> None:
if not indices:
return
assert batch_mm_kwargs is not None
assert self.visual is not None
hidden = outputs["default"]
modality = self.get_input_modality(batch_mm_kwargs)
grids = self._get_grid_list(batch_mm_kwargs, f"{modality}_grid_thw")
if hidden.dim() == 2:
hidden = hidden.unsqueeze(0)
# The merger runs eagerly on the original per-item shapes (cudnn's
# merge Conv2d is not shape-invariant, so this is what keeps the
# outputs bitwise identical to the eager path). The reads of the
# graph's output buffer happen here, synchronously, before the
# manager may replay the graph again; every output row is freshly
# allocated by the merger.
for idx, states in zip(indices, self.visual._iter_item_states(hidden, grids)):
dest[idx] = self.visual.perceive(states)