Skip to content

vllm.models.hy_v4.nvidia.dfly_target_reuse

Classes:

HYV4DFlyTargetReuse

Source code in vllm/models/hy_v4/nvidia/dfly_target_reuse.py
class HYV4DFlyTargetReuse:
    def __init__(
        self, target: nn.Module, draft: HYV4DFlyForCausalLM, vllm_config: VllmConfig
    ):
        config = draft.config
        if vllm_config.cache_config.cache_dtype not in (
            "auto",
            "bfloat16",
            "fp8_ds_mla",
        ):
            raise ValueError("DFly requires BF16 or fp8_ds_mla MLA KV cache")
        language_model = (
            target.get_language_model()
            if hasattr(target, "get_language_model")
            else target
        )
        target_layers = language_model.model.layers
        self.index_layers: dict[int, HYV4MLAAttention] = {}
        self.layer_pairs: list[tuple[HYV4MLAAttention, HYV4MLAAttention]] = []
        self.aliases: dict[str, str] = {}
        self.draft = draft
        self.config = config

        def target_attention(index: int) -> HYV4MLAAttention:
            attn = target_layers[index].self_attn
            if not isinstance(attn, HYV4MLAAttention):
                raise ValueError(f"DFly target layer {index} is not HYV4 MLA")
            return attn

        for index in config.target_index_layer:
            attn = target_attention(index)
            if (
                attn.indexer is None
                or attn.skip_topk
                or attn.topk_indices_buffer is None
            ):
                raise ValueError(
                    f"DFly target index layer {index} must run its own indexer"
                )
            if attn.topk_indices_buffer.shape[-1] != config.target_index_topk:
                raise ValueError(
                    f"DFly target index layer {index} has incompatible top-k width"
                )
            self.index_layers[index] = attn
        for layer, index in zip(draft.model.layers, config.target_kv_layers):
            source, dest = target_attention(index), layer.self_attn
            if type(source.rotary_emb) is not type(dest.rotary_emb):
                raise ValueError(f"DFly target KV layer {index} uses incompatible RoPE")
            if source.num_local_heads != dest.num_local_heads:
                raise ValueError("DFly target and draft must use matching attention TP")
            storage_specs: list[MLAAttentionSpec] = []
            for attn in (source, dest):
                spec = attn.mla_attn.get_kv_cache_config(vllm_config).spec
                if not isinstance(spec, MLAAttentionSpec):
                    raise ValueError("DFly requires MLA KV cache specs")
                customized_spec = customize_attention_spec(
                    attn.mla_attn.get_attn_backend(), spec, attn.mla_attn.layer_name
                )
                if not isinstance(customized_spec, MLAAttentionSpec):
                    raise ValueError("DFly backend must preserve the MLA KV cache spec")
                storage_specs.append(
                    replace(
                        customized_spec,
                        topk_buffer_width=None,
                        is_index_group_leader=False,
                    )
                )
            if storage_specs[0] != storage_specs[1]:
                raise ValueError(
                    f"DFly target KV layer {index} has incompatible cache spec"
                )
            if (
                source.learnable_sink
                and not source.mla_attn.get_attn_backend().supports_sink()
            ):
                raise ValueError("DFly target backend must apply its learnable sink")
            self.layer_pairs.append((source, dest))
            self.aliases[dest.mla_attn.layer_name] = source.mla_attn.layer_name
        if len(set(self.aliases.values())) != len(self.aliases):
            raise ValueError("DFly needs a distinct target KV layer per draft layer")
        # One stacked snapshot buffer, so one launch builds every draft layer's
        # indices; each draft layer reads the slot of its target index layer.
        index_layer_ids = list(self.index_layers)
        first = next(iter(self.index_layers.values())).topk_indices_buffer
        assert first is not None
        self.snapshots = torch.full(
            (len(index_layer_ids), *first.shape),
            -1,
            dtype=first.dtype,
            device=first.device,
        )
        for slot, index in enumerate(index_layer_ids):
            self.index_layers[index].topk_snapshot = self.snapshots[slot]
        self.layer_snapshot_slots = torch.tensor(
            [index_layer_ids.index(i) for i in config.target_index_layer],
            dtype=torch.int32,
            device=first.device,
        )
        self._set_storage_aliases(True)

    def _set_storage_aliases(self, enabled: bool) -> None:
        """Toggle the KV-sharing alias on the draft layers.

        The alias makes KV cache allocation bind each draft layer to its target
        layer's storage, but MLA also skips the KV write for sharing layers. The
        draft must write its query block KV, so the alias is only kept while the
        KV cache is being allocated and bound.
        """
        for source, dest in self.layer_pairs:
            dest.mla_attn.kv_sharing_target_layer_name = (
                source.mla_attn.layer_name if enabled else None
            )

    def validate_groups(self, config: KVCacheConfig) -> None:
        self._set_storage_aliases(True)
        groups = {
            name: i
            for i, group in enumerate(config.kv_cache_groups)
            for name in group.layer_names
        }
        for draft, target in self.aliases.items():
            if draft not in groups or groups[draft] != groups.get(target):
                raise ValueError(
                    f"DFly alias {draft} and {target} must share a KV group"
                )

    def validate_storage(self) -> None:
        for source, dest in self.layer_pairs:
            a, b = source.mla_attn.kv_cache, dest.mla_attn.kv_cache
            if (
                a.numel() == 0
                or a.data_ptr() != b.data_ptr()
                or a.shape != b.shape
                or a.stride() != b.stride()
                or a.dtype != b.dtype
            ):
                raise ValueError(
                    "DFly query and target KV must alias the same local layout"
                )
        self._set_storage_aliases(False)

    def update_indices(
        self,
        active: torch.Tensor,
        source_rows: torch.Tensor,
        anchors: torch.Tensor,
        padded_batch: int,
    ) -> None:
        build_dfly_reuse_indices(
            self.draft.model.topk_indices,
            self.snapshots,
            self.layer_snapshot_slots,
            active,
            source_rows,
            anchors,
            self.config.target_index_topk,
            self.config.context_local_window or 0,
            self.draft.model.query_width,
            padded_batch,
        )

_set_storage_aliases(enabled)

Toggle the KV-sharing alias on the draft layers.

The alias makes KV cache allocation bind each draft layer to its target layer's storage, but MLA also skips the KV write for sharing layers. The draft must write its query block KV, so the alias is only kept while the KV cache is being allocated and bound.

Source code in vllm/models/hy_v4/nvidia/dfly_target_reuse.py
def _set_storage_aliases(self, enabled: bool) -> None:
    """Toggle the KV-sharing alias on the draft layers.

    The alias makes KV cache allocation bind each draft layer to its target
    layer's storage, but MLA also skips the KV write for sharing layers. The
    draft must write its query block KV, so the alias is only kept while the
    KV cache is being allocated and bound.
    """
    for source, dest in self.layer_pairs:
        dest.mla_attn.kv_sharing_target_layer_name = (
            source.mla_attn.layer_name if enabled else None
        )