Skip to content

vllm.v1.hisparse.layout

Functions:

_lay_out_hisparse_groups(vllm_config, mla_specs, regular_groups)

Lay out the groups to fill the largest regular page. When a block-outer layout's pool block is not whole indexer pages, either the block grows to whole pages or the pages spread out to divide the block, whichever takes less memory per request. Before the layout is resolved, only the per-layer specs are used.

Source code in vllm/v1/hisparse/layout.py
def _lay_out_hisparse_groups(
    vllm_config: VllmConfig,
    mla_specs: dict[str, KVCacheLayerConfig],
    regular_groups: list[KVCacheGroup],
) -> list[KVCacheGroup]:
    """Lay out the groups to fill the largest regular page. When a block-outer
    layout's pool block is not whole indexer pages, either the block grows to
    whole pages or the pages spread out to divide the block, whichever takes
    less memory per request. Before the layout is resolved, only the per-layer
    specs are used."""
    from vllm.v1.core.kv_cache_planning import _pool_bytes_per_block

    block_bytes = max((g.page_size_bytes for g in regular_groups), default=0)
    groups = _fill_hisparse_groups(vllm_config, mla_specs, regular_groups, block_bytes)
    if not block_bytes or vllm_config.cache_config.kv_cache_layout is None:
        return groups
    if _pool_bytes_per_block(vllm_config, groups) == block_bytes:
        return groups
    other_groups = [
        group for group in groups if group.role is not KVCacheGroupRole.HISPARSE_INDEXER
    ]
    candidates = [
        groups,
        _fill_hisparse_groups(
            vllm_config,
            mla_specs,
            regular_groups,
            _pool_bytes_per_block(vllm_config, other_groups),
            spread_indexer_pages=True,
        ),
    ]
    return min(
        candidates,
        key=lambda groups: (
            _pool_bytes_per_block(vllm_config, groups)
            * sum(
                group.max_allocated_blocks_per_request(vllm_config)
                for group in groups
                if not group.host_resident
            )
        ),
    )

create_hisparse_layout(vllm_config, groups, host_budget)

Size the host pool for groups laid out by get_hisparse_kv_cache_groups.

Source code in vllm/v1/hisparse/layout.py
def create_hisparse_layout(
    vllm_config: VllmConfig,
    groups: list[KVCacheGroup],
    host_budget: int,
) -> HiSparseLayout:
    """Size the host pool for groups laid out by `get_hisparse_kv_cache_groups`."""
    (source_group,) = [group for group in groups if group.host_resident]
    shared_host_pool = use_shared_hisparse_host_pool(vllm_config)
    host_block_stride = get_hisparse_host_block_stride(
        source_group.page_size_bytes,
        use_shared_host_pool=shared_host_pool,
    )
    host_num_blocks = host_budget // host_block_stride
    if host_num_blocks <= 0:
        raise ValueError("HiSparse has no allocatable host blocks.")
    # Every computed page needs a host block, so one request at max_model_len
    # must fit alongside the pool's null block and a copy-on-write tail.
    gpu_block_size = source_group.block_size
    min_host_blocks = cdiv(vllm_config.model_config.max_model_len, gpu_block_size) + 2
    if host_num_blocks < min_host_blocks:
        raise ValueError(
            f"HiSparse host pool has {host_num_blocks} blocks but max_model_len "
            f"needs {min_host_blocks}; increase host_pool_gib."
        )

    return HiSparseLayout(
        source_group=source_group,
        device_groups=[group for group in groups if not group.host_resident],
        host_num_blocks=host_num_blocks,
        host_block_stride=host_block_stride,
        shared_host_pool=shared_host_pool,
    )

get_hisparse_kv_cache_groups(vllm_config, layer_configs)

The groups HiSparse allocates: the host source group, then the GPU groups sharing one device pool. Derived resident/hot specs in layer_configs (see resolve_hisparse_specs) are laid out again from the attention specs they belong to.

Source code in vllm/v1/hisparse/layout.py
def get_hisparse_kv_cache_groups(
    vllm_config: VllmConfig, layer_configs: dict[str, KVCacheLayerConfig]
) -> list[KVCacheGroup] | None:
    """The groups HiSparse allocates: the host source group, then the GPU
    groups sharing one device pool. Derived resident/hot specs in
    `layer_configs` (see `resolve_hisparse_specs`) are laid out again
    from the attention specs they belong to."""
    attention_config = getattr(vllm_config, "attention_config", None)
    if attention_config is None or attention_config.hisparse_config is None:
        return None

    mla_specs = {
        name: lc for name, lc in layer_configs.items() if is_hisparse_source_layer(lc)
    }
    other_specs = {
        name: lc
        for name, lc in layer_configs.items()
        if name not in mla_specs and not isinstance(lc.spec, HiSparsePageSpec)
    }
    if not mla_specs:
        return None

    from vllm.v1.core.kv_cache_planning import get_kv_cache_groups

    regular_groups = (
        get_kv_cache_groups(vllm_config, other_specs) if other_specs else []
    )
    return _lay_out_hisparse_groups(vllm_config, mla_specs, regular_groups)

get_hisparse_page_layer_configs(vllm_config, mla_specs)

The resident and hot layers HiSparse derives from its source layers. They are per-layer, so they don't depend on the KV cache layout that groups them.

Source code in vllm/v1/hisparse/layout.py
def get_hisparse_page_layer_configs(
    vllm_config: VllmConfig, mla_specs: dict[str, KVCacheLayerConfig]
) -> dict[str, KVCacheLayerConfig]:
    """The resident and hot layers HiSparse derives from its source layers.
    They are per-layer, so they don't depend on the KV cache layout that
    groups them."""
    source_specs, _ = _partition_hisparse_specs(
        {name: lc.spec for name, lc in mla_specs.items()}
    )
    block_sizes = {spec.block_size for spec in source_specs.values()}
    if len(block_sizes) != 1:
        raise ValueError("HiSparse requires one resolved GPU block size.")
    gpu_block_size = block_sizes.pop()
    config = ResolvedHiSparseConfig.from_vllm_config(
        vllm_config,
        max(
            spec.topk_buffer_width
            if spec.topk_buffer_width is not None
            else vllm_config.model_config.hf_text_config.index_topk
            for spec in source_specs.values()
        ),
        gpu_block_size,
    )
    assert config is not None
    resident_manager = HiSparseResidentManager.config()
    hot_manager = HiSparseHotManager.config(
        blocks_per_request=cdiv(config.device_buffer_size, gpu_block_size)
    )
    layer_configs: dict[str, KVCacheLayerConfig] = {}
    for name, spec in source_specs.items():
        layer_configs[f"{name}{HISPARSE_RESIDENT_SUFFIX}"] = KVCacheLayerConfig(
            resident_manager,
            HiSparsePageSpec(block_size=gpu_block_size, page_size=spec.page_size_bytes),
        )
        layer_configs[f"{name}{HISPARSE_HOT_SUFFIX}"] = KVCacheLayerConfig(
            hot_manager,
            HiSparseHotPageSpec(
                block_size=gpu_block_size,
                page_size=spec.page_size_bytes,
                block_stride_alignment=spec.block_stride_alignment,
            ),
        )
    return layer_configs

get_hisparse_steady_state_concurrency(vllm_config, kv_cache_config)

Max concurrency at max_model_len once running requests read from host.

A request reading from host pins only its active tail of each resident group, but admitting one still takes its full in-flight window, so the bound is all-but-one requests at steady state plus one being admitted.

Source code in vllm/v1/hisparse/layout.py
def get_hisparse_steady_state_concurrency(
    vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
) -> float:
    """Max concurrency at max_model_len once running requests read from host.

    A request reading from host pins only its active tail of each resident
    group, but admitting one still takes its full in-flight window, so the
    bound is all-but-one requests at steady state plus one being admitted.
    """
    admission_blocks = 0
    steady_blocks = 0
    host_blocks = 0
    for group in kv_cache_config.kv_cache_groups:
        required = group.max_allocated_blocks_per_request(vllm_config)
        if group.host_resident:
            host_blocks += required
            continue
        admission_blocks += required
        if issubclass(group.manager_cls, HiSparseResidentManager):
            required = min(required, ACTIVE_TAIL_PAGES)
        steady_blocks += required
    assert kv_cache_config.hisparse_host_num_blocks is not None
    num_blocks = kv_cache_config.num_blocks
    if num_blocks < admission_blocks:
        gpu_concurrency = num_blocks / admission_blocks
    else:
        gpu_concurrency = 1 + (num_blocks - admission_blocks) / steady_blocks
    return min(gpu_concurrency, kv_cache_config.hisparse_host_num_blocks / host_blocks)

is_hisparse_source_layer(layer_config)

Whether HiSparse backs this layer's KV with host memory: full-attention sparse MLA and its indexer.

Source code in vllm/v1/hisparse/layout.py
def is_hisparse_source_layer(layer_config: KVCacheLayerConfig) -> bool:
    """Whether HiSparse backs this layer's KV with host memory: full-attention
    sparse MLA and its indexer."""
    return isinstance(layer_config.spec, MLAAttentionSpec) and issubclass(
        layer_config.manager_cls, FullAttentionManager
    )