Skip to content

vllm.models.hy_v4.nvidia.dfly_inputs

Fused DFly draft input preparation.

Three launches replace the per-step torch ops: one per request for the query block and sampling state, one per KV group for the query slots, and one for the sparse indices of every draft layer.

Functions:

_build_dfly_reuse_indices_kernel(out_ptr, out_layer_stride, out_row_stride, snapshot_ptr, snapshot_slot_stride, snapshot_row_stride, layer_snapshot_slots_ptr, active_ptr, source_rows_ptr, anchors_ptr, TARGET_TOPK, LOCAL_WINDOW, QUERY_WIDTH, OUT_WIDTH, BLOCK_SIZE)

Per query row: target top-k history, local window, then causal block.

Source code in vllm/models/hy_v4/nvidia/dfly_inputs.py
@triton.jit
def _build_dfly_reuse_indices_kernel(
    out_ptr,
    out_layer_stride,
    out_row_stride,
    snapshot_ptr,
    snapshot_slot_stride,
    snapshot_row_stride,
    layer_snapshot_slots_ptr,
    active_ptr,
    source_rows_ptr,
    anchors_ptr,
    TARGET_TOPK: tl.constexpr,
    LOCAL_WINDOW: tl.constexpr,
    QUERY_WIDTH: tl.constexpr,
    OUT_WIDTH: tl.constexpr,
    BLOCK_SIZE: tl.constexpr,
):
    """Per query row: target top-k history, local window, then causal block."""
    req_idx = tl.program_id(0)
    query_offset = tl.program_id(1)
    layer = tl.program_id(2)

    active = tl.load(active_ptr + req_idx)
    anchor = tl.load(anchors_ptr + req_idx)
    source_row = tl.load(source_rows_ptr + req_idx).to(tl.int64)
    slot = tl.load(layer_snapshot_slots_ptr + layer).to(tl.int64)
    history_ptr = (
        snapshot_ptr + slot * snapshot_slot_stride + source_row * snapshot_row_stride
    )
    out_row = (
        out_ptr
        + layer * out_layer_stride
        + (req_idx * QUERY_WIDTH + query_offset).to(tl.int64) * out_row_stride
    )
    window_start = anchor - LOCAL_WINDOW
    for i in range(0, OUT_WIDTH, BLOCK_SIZE):
        cols = i + tl.arange(0, BLOCK_SIZE)
        is_history = cols < TARGET_TOPK
        history = tl.load(history_ptr + cols, mask=is_history & active, other=-1)
        history = tl.where((history >= 0) & (history < window_start), history, -1)
        window_col = cols - TARGET_TOPK
        is_window = (window_col >= 0) & (window_col < LOCAL_WINDOW)
        window = tl.maximum(window_start + window_col, -1)
        block_col = window_col - LOCAL_WINDOW
        is_causal = (block_col >= 0) & (block_col <= query_offset)
        value = tl.where(
            is_history,
            history,
            tl.where(is_window, window, tl.where(is_causal, anchor + block_col, -1)),
        )
        value = tl.where(active, value, -1)
        tl.store(out_row + cols, value.to(tl.int32), mask=cols < OUT_WIDTH)

build_dfly_reuse_indices(out, snapshots, layer_snapshot_slots, active, source_rows, anchors, target_topk, local_window, query_width, padded_batch)

Fill out[layer, :padded_batch * query_width] for every draft layer.

Source code in vllm/models/hy_v4/nvidia/dfly_inputs.py
def build_dfly_reuse_indices(
    out: torch.Tensor,
    snapshots: torch.Tensor,
    layer_snapshot_slots: torch.Tensor,
    active: torch.Tensor,
    source_rows: torch.Tensor,
    anchors: torch.Tensor,
    target_topk: int,
    local_window: int,
    query_width: int,
    padded_batch: int,
) -> None:
    """Fill `out[layer, :padded_batch * query_width]` for every draft layer."""
    num_layers, _, out_width = out.shape
    assert out.stride(-1) == 1 and snapshots.stride(-1) == 1
    _build_dfly_reuse_indices_kernel[(padded_batch, query_width, num_layers)](
        out,
        out.stride(0),
        out.stride(1),
        snapshots,
        snapshots.stride(0),
        snapshots.stride(1),
        layer_snapshot_slots,
        active,
        source_rows,
        anchors,
        TARGET_TOPK=target_topk,
        LOCAL_WINDOW=local_window,
        QUERY_WIDTH=query_width,
        OUT_WIDTH=out_width,
        BLOCK_SIZE=1024,
    )

prepare_dfly_inputs(*, input_ids, positions, is_padding, query_start_loc, seq_lens, active, source_rows, anchors, target_hidden_out, mrope_positions, draft_idx_mapping, out_temperature, out_seeds, target_query_start_loc, target_positions, idx_mapping, num_sampled, num_rejected, last_sampled, target_hidden, mrope_delta, temperature, seeds, max_model_len, mask_token_id, query_width, num_reqs, padded_batch)

Fill the query block of padded_batch requests. Rows past num_reqs are inactive padding, and num_reqs=0 gives an all-padding block for dummy runs and capture.

Source code in vllm/models/hy_v4/nvidia/dfly_inputs.py
def prepare_dfly_inputs(
    *,
    input_ids: torch.Tensor,
    positions: torch.Tensor,
    is_padding: torch.Tensor,
    query_start_loc: torch.Tensor,
    seq_lens: torch.Tensor,
    active: torch.Tensor,
    source_rows: torch.Tensor,
    anchors: torch.Tensor,
    target_hidden_out: torch.Tensor,
    mrope_positions: torch.Tensor | None,
    draft_idx_mapping: torch.Tensor,
    out_temperature: torch.Tensor,
    out_seeds: torch.Tensor,
    target_query_start_loc: torch.Tensor,
    target_positions: torch.Tensor,
    idx_mapping: torch.Tensor,
    num_sampled: torch.Tensor,
    num_rejected: torch.Tensor,
    last_sampled: torch.Tensor,
    target_hidden: torch.Tensor,
    mrope_delta: torch.Tensor | None,
    temperature: torch.Tensor,
    seeds: torch.Tensor,
    max_model_len: torch.Tensor,
    mask_token_id: int,
    query_width: int,
    num_reqs: int,
    padded_batch: int,
) -> None:
    """Fill the query block of `padded_batch` requests. Rows past `num_reqs`
    are inactive padding, and `num_reqs=0` gives an all-padding block for
    dummy runs and capture."""
    if num_reqs > 0:
        assert target_hidden.shape[-1] == target_hidden_out.shape[-1], (
            "DFly fusion requires the target's final hidden states"
        )
    _prepare_dfly_inputs_kernel[(padded_batch,)](
        input_ids,
        positions,
        is_padding,
        query_start_loc,
        seq_lens,
        active,
        source_rows,
        anchors,
        target_hidden_out,
        target_hidden_out.stride(0),
        mrope_positions,
        mrope_positions.stride(0) if mrope_positions is not None else 0,
        draft_idx_mapping,
        out_temperature,
        out_seeds,
        target_query_start_loc,
        target_positions,
        idx_mapping,
        num_sampled,
        num_rejected,
        last_sampled,
        last_sampled.stride(0),
        target_hidden,
        target_hidden.stride(0),
        mrope_delta,
        temperature,
        seeds,
        max_model_len,
        mask_token_id,
        target_hidden_out.shape[-1],
        num_reqs,
        padded_batch,
        QUERY_WIDTH=query_width,
        QUERY_BLOCK=triton.next_power_of_2(query_width),
        MROPE_DIMS=mrope_positions.shape[0] if mrope_positions is not None else 0,
        HAS_MROPE_DELTA=mrope_delta is not None,
        HIDDEN_BLOCK=1024,
    )