Skip to content

vllm.models.hy_v4_flash.nvidia.indexpool_cache

Engine-owned pool pages and speculative tail rings for Flash IndexPool.

Classes:

IndexPoolMetadataBuilder

Bases: AttentionMetadataBuilder[IndexPoolMetadata]

Methods:

Source code in vllm/models/hy_v4_flash/nvidia/indexpool_cache.py
class IndexPoolMetadataBuilder(AttentionMetadataBuilder[IndexPoolMetadata]):
    _cudagraph_support = AttentionCGSupport.ALWAYS

    def __init__(self, kv_cache_spec, layer_names, vllm_config, device):
        super().__init__(kv_cache_spec, layer_names, vllm_config, device)
        self.max_model_len = vllm_config.model_config.max_model_len

    def build(self, common_prefix_len, common_attn_metadata, fast_build=False):
        cm = common_attn_metadata
        return IndexPoolMetadata(
            query_start_loc=cm.query_start_loc[: cm.num_reqs + 1],
            seq_lens=cm.seq_lens[: cm.num_reqs],
            block_table=cm.block_table_tensor[: cm.num_reqs],
            num_actual_tokens=cm.num_actual_tokens,
            max_seq_len=cm.max_seq_len,
        )

    def build_for_cudagraph_capture(self, common_attn_metadata):
        """Reserve scoring width for replay beyond the dummy's short context."""
        metadata = self.build(0, common_attn_metadata)
        return replace(metadata, max_seq_len=self.max_model_len)

build_for_cudagraph_capture(common_attn_metadata)

Reserve scoring width for replay beyond the dummy's short context.

Source code in vllm/models/hy_v4_flash/nvidia/indexpool_cache.py
def build_for_cudagraph_capture(self, common_attn_metadata):
    """Reserve scoring width for replay beyond the dummy's short context."""
    metadata = self.build(0, common_attn_metadata)
    return replace(metadata, max_seq_len=self.max_model_len)