class Glm5NextLinearAttention(GatedDeltaNetAttention):
head_dim: int
num_heads: int
conv_size: int
def get_kv_cache_checkpoint_alignment(self, vllm_config: VllmConfig) -> int | None:
return kda_prefill_checkpoint_alignment(self.kda_prefill_backend)
def get_state_dtype(
self,
) -> tuple[torch.dtype, torch.dtype]:
if self.model_config is None or self.cache_config is None:
raise ValueError("model_config and cache_config must be set")
return MambaStateDtypeCalculator.kda_state_dtype(
self.model_config.dtype,
self.cache_config.mamba_cache_dtype,
self.cache_config.mamba_ssm_cache_dtype,
)
def get_state_shape(
self,
) -> tuple[tuple[int, ...], tuple[int, ...]]:
# conv_state width must include num_spec so the spec-decode conv update
# (causal_conv1d_update with num_accepted_tokens + max_query_len) can
# slide the window across the draft-verify tokens without reading past
# the allocated width. Matches qwen_gdn_linear_attn.get_state_shape.
return MambaStateShapeCalculator.kda_state_shape(
self.tp_size,
self.num_heads,
self.head_dim,
conv_kernel_size=self.conv_size,
num_spec=self.num_spec,
)
def __init__(
self,
config: Glm5NextTextConfig,
vllm_config: VllmConfig,
prefix: str = "",
) -> None:
super().__init__(config, vllm_config, prefix)
self.head_dim = config.linear_head_dim
self.num_heads = config.linear_num_heads
self.conv_size = config.linear_conv_kernel_dim
assert self.num_heads % self.tp_size == 0
self.local_num_heads = divide(self.num_heads, self.tp_size)
projection_size = self.head_dim * self.num_heads
self.local_projection_size = divide(projection_size, self.tp_size)
_, recurrent_state_shape = self.get_state_shape()
_, recurrent_state_dtype = self.get_state_dtype()
scatter_states.register_warmup(
state_shape=recurrent_state_shape,
state_dtype=recurrent_state_dtype,
indices_dtype=torch.int32,
max_num_tokens=vllm_config.scheduler_config.max_num_seqs,
)
# Merge q, k, v, b, f_a, g_a projections into one GEMM (6→1 launches).
# Order matches checkpoint's fused_qkvbfg_a_proj convention.
# Shards 4 (f_a) and 5 (g_a) are replicated across TP ranks.
self.in_proj_qkvbfg_a = _Glm5NextMergedColumnParallelLinear(
self.hidden_size,
[
projection_size, # q (shard 0)
projection_size, # k (shard 1)
projection_size, # v (shard 2)
self.num_heads, # b (shard 3)
self.head_dim, # f_a (shard 4, replicated)
self.head_dim, # g_a (shard 5, replicated)
],
replicated_shard_ids=(4, 5),
tp_size=self.tp_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.in_proj_qkvbfg_a",
)
self.f_b_proj = ColumnParallelLinear(
self.head_dim,
projection_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.f_b_proj",
)
self.dt_bias = nn.Parameter(
torch.empty(divide(projection_size, self.tp_size), dtype=torch.float32)
)
set_weight_attrs(self.dt_bias, {"weight_loader": sharded_weight_loader(0)})
self.conv1d = _fused_qkv_conv1d(
self.conv_size,
projection_size,
self.tp_size,
self.tp_rank,
prefix=f"{prefix}.conv1d",
)
self.A_log = nn.Parameter(
torch.empty(1, 1, self.local_num_heads, 1, dtype=torch.float32)
)
set_weight_attrs(self.A_log, {"weight_loader": sharded_weight_loader(2)})
self.g_b_proj = ColumnParallelLinear(
self.head_dim,
projection_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.g_b_proj",
)
self.o_norm = FusedRMSNormGated(
self.head_dim,
activation="sigmoid",
enforce_enable=current_platform.is_rocm(),
)
self.o_proj = RowParallelLinear(
projection_size,
self.hidden_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.o_proj",
)
compilation_config = get_current_vllm_config().compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
# Checkpoints store A_log as 1-D; the model parameter is 4-D.
def _a_log_weight_loader(param, loaded_weight):
if loaded_weight.dim() == 1:
loaded_weight = loaded_weight.view([1, 1, -1, 1])
return sharded_weight_loader(2)(param, loaded_weight)
self.A_log.weight_loader = _a_log_weight_loader
# GLM-5.3-Flash uses a bounded sigmoid gate instead of the default
# unbounded softplus gate.
self.kda_safe_gate = True
self.kda_lower_bound = config.linear_lower_bound
# Process-global conv-state layout, resolved once here instead of on
# every _forward call (it reads an env-derived flag each time).
self._conv_state_dim_first = is_conv_state_dim_first()
additional_config = vllm_config.additional_config
conv_state_dtype, recurrent_state_dtype = self.get_state_dtype()
self.kda_prefill_backend = _resolve_kda_prefill_backend(
additional_config.get("kda_prefill_backend", "auto")
if isinstance(additional_config, dict)
else "auto",
self.head_dim,
vllm_config.model_config.dtype,
self.kda_lower_bound,
recurrent_state_dtype,
)
if self.kda_prefill_backend == "triton":
l2norm_fwd_block.register_warmup(
dtype=vllm_config.model_config.dtype,
dim=self.head_dim,
)
# One option (`--kda-decode-backend`) for every decode row: with
# speculative decoding they are draft-verify rows of 1 + num_spec
# tokens, otherwise single-token rows.
self.kda_decode_backend = _resolve_kda_decode_backend(
additional_config.get("kda_decode_backend", "auto")
if isinstance(additional_config, dict)
else "auto",
self.local_num_heads,
self.head_dim,
self.conv_size,
1 + self.num_spec,
vllm_config.model_config.dtype,
conv_state_dtype,
recurrent_state_dtype,
self.kda_lower_bound,
)
# FlashInfer's [3, width, P] fp32 conv weight and fp32 norm weight,
# built in process_weights_after_loading.
self._fi_conv_weight: torch.Tensor | None = None
self._fi_norm_weight: torch.Tensor | None = None
self._flashkda_buffer_specs: (
tuple[tuple[tuple[int, ...], torch.dtype], ...] | None
) = None
self._triton_checkpoint_spec: TritonKDACheckpointSpec | None = None
self._checkpoint_exporter: FlashKDAPrefillCheckpointExporter | None = None
if kda_prefill_checkpoint_alignment(self.kda_prefill_backend) is not None:
self._checkpoint_exporter = FlashKDAPrefillCheckpointExporter(
state_len=self.conv_size - 1
)
if self.kda_prefill_backend == "triton":
self._triton_checkpoint_spec = reserve_triton_kda_checkpoint_buffer(
vllm_config,
self.local_num_heads,
self.head_dim,
self.get_state_dtype()[1],
)
if self.kda_prefill_backend == "flashkda":
import vllm._flashkda_C # noqa: F401
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
max_seqs = vllm_config.scheduler_config.max_num_seqs
workspace_size = torch.ops._flashkda_C.get_workspace_size(
max_tokens, self.local_num_heads, max_seqs
)
self._flashkda_buffer_specs = (
(
(max_seqs, self.local_num_heads, self.head_dim, self.head_dim),
self.get_state_dtype()[1],
),
(
(max_seqs, self.local_num_heads, self.head_dim, self.head_dim),
self.get_state_dtype()[1],
),
((workspace_size,), torch.uint8),
# Output buffer for steps that also carry spec-decode tokens:
# the non-spec tokens are then scattered by non_spec_token_indx.
(
(1, max_tokens, self.local_num_heads, self.head_dim),
vllm_config.model_config.dtype,
),
)
def _flashkda_prefill(
self,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
g: torch.Tensor,
beta: torch.Tensor,
initial_state: torch.Tensor,
cu_seqlens: torch.Tensor,
out: torch.Tensor | None,
checkpoint: MambaPrefillCheckpointMetadata | None = None,
raw_qkv: torch.Tensor | None = None,
conv_state: torch.Tensor | None = None,
recurrent_state: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Fused KDA chunked prefill (FlashKDA). Takes the raw gate logits `g`
and raw `beta` logits, l2-normalizes q/k in-kernel and applies the
bounded gate `lower_bound * sigmoid(exp(A_log) * (g + dt_bias))`,
matching `chunk_kda_with_fused_gate(..., safe_gate=True)`. Writes the
attention output into `out` (a workspace buffer when `None`) and
returns `(out, final_state)`."""
assert self._flashkda_buffer_specs is not None
final_state, checkpoint_state, workspace, workspace_out = (
current_workspace_manager().get_simultaneous(*self._flashkda_buffer_specs)
)
final_state = final_state[: initial_state.shape[0]]
checkpoint_state = (
checkpoint_state[: initial_state.shape[0]]
if checkpoint is not None
else None
)
if out is None:
out = workspace_out[:, : q.shape[1]]
# FlashKDA accepts row-strided q/k/v (contiguous H/D rows, aligned
# token stride), so the packed q|k|v views are passed without a copy.
torch.ops._flashkda_C.fwd(
q,
k,
v,
g.contiguous(),
beta,
self.head_dim**-0.5,
out,
workspace,
self.A_log.view(-1),
self.dt_bias.view(-1, self.head_dim),
self.kda_lower_bound,
initial_state.contiguous(),
final_state,
cu_seqlens.contiguous(),
checkpoint_state,
checkpoint.checkpoint_offsets.contiguous()
if checkpoint is not None
else None,
)
if checkpoint is not None:
assert self._checkpoint_exporter is not None
assert raw_qkv is not None and conv_state is not None
assert recurrent_state is not None
self._checkpoint_exporter.export(
checkpoint,
raw_qkv=raw_qkv,
conv_state=conv_state,
recurrent_checkpoint=checkpoint_state,
recurrent_state=recurrent_state,
cu_seqlens=cu_seqlens,
)
return out, final_state
def forward(
self,
hidden_states: torch.Tensor,
positions: torch.Tensor,
) -> torch.Tensor:
num_tokens = hidden_states.size(0)
# One merged GEMM for q, k, v, b, f_a, g_a (replaces 6 separate GEMMs).
projected = self.in_proj_qkvbfg_a(hidden_states)[0]
qkv, beta_raw, f_a, g_a = projected.split(
[
3 * self.local_projection_size,
self.local_num_heads,
self.head_dim,
self.head_dim,
],
dim=-1,
)
# Beta stays raw (bf16) here: the recurrent kernel sigmoids it in fp32
# at load (SIGMOID_BETA), and only the chunked prefill path needs the
# pre-computed fp32 sigmoid — computed lazily in _forward. Pure decode
# / spec-verify steps then skip the _cast_sigmoid kernel and its fp32
# intermediate entirely.
beta = beta_raw.unsqueeze(0)
g1 = self.f_b_proj(f_a)[0]
g1 = g1.reshape(1, -1, self.local_num_heads, self.head_dim)
g_proj_states = self.g_b_proj(g_a)[0]
# Must stay 3D: rms_norm_gated reads H from g.shape[-2].
g2 = g_proj_states.reshape(-1, self.local_num_heads, self.head_dim)
core_attn_out = torch.empty(
(1, num_tokens, self.local_num_heads, self.head_dim),
dtype=hidden_states.dtype,
device=hidden_states.device,
)
# Call the decorated eager break directly so host-side prefill branches
# are not captured by PIECEWISE CUDA graphs.
self._forward(
qkv_proj_states=qkv,
g1=g1,
beta=beta,
core_attn_out=core_attn_out,
g2=g2,
)
if not self._norm_in_forward:
core_attn_out = self.o_norm(core_attn_out, g2)
core_attn_out = core_attn_out.reshape(core_attn_out.size(1), -1)
return self.o_proj(core_attn_out)[0]
@property
def _norm_in_forward(self) -> bool:
"""FlashInfer's fused decode applies o_norm itself, so with it
`_forward` returns normed output for every step type: a breakable CUDA
graph replays the code after `_forward` as captured, whatever the live
step."""
return self.kda_decode_backend == "flashinfer"
def process_weights_after_loading(self) -> None:
"""Build FlashInfer's conv and norm weights from the loaded weights;
reloads refill them in place."""
if not self._norm_in_forward:
return
conv_weight = self.conv1d.weight.view(
3, self.local_projection_size, self.conv_size
)
refill_derived(
self, "_fi_conv_weight", conv_weight.transpose(1, 2).contiguous()
)
refill_derived(self, "_fi_norm_weight", self.o_norm.weight.float().contiguous())
def _flashinfer_decode_rows(
self,
metadata: GDNAttentionMetadata,
qkv: torch.Tensor,
g1: torch.Tensor,
beta: torch.Tensor,
g2: torch.Tensor,
out: torch.Tensor,
conv_state: torch.Tensor,
state: torch.Tensor,
) -> int:
"""Runs the step's leading decode rows through FlashInfer's fused
decode and returns their token count (0 if it runs none). Without
speculative decoding these are the decode rows of decode and mixed
steps (vLLM orders decodes first); with it, every row of a step of
draft-verify rows only."""
if self.kda_decode_backend != "flashinfer":
return 0
if self.num_spec:
if (
metadata.spec_sequence_masks is None
or metadata.num_spec_decodes == 0
or metadata.num_decodes
or metadata.num_prefills
):
return 0
assert metadata.spec_state_indices_tensor is not None
assert metadata.spec_query_start_loc is not None
n = metadata.num_actual_tokens
self._flashinfer_decode(
x=qkv,
conv_state=conv_state,
raw_gate=g1,
raw_beta=beta,
state_indices=metadata.spec_state_indices_tensor,
state=state,
output_gate=g2[:n],
output=out[:, :n],
query_start_loc=metadata.spec_query_start_loc,
num_accepted_tokens=metadata.num_accepted_tokens,
)
return n
n = metadata.num_decode_tokens
# A mixed step with prefix-cache checkpoints runs on the chunked path.
if n == 0 or (metadata.num_prefills and metadata.checkpoint is not None):
return 0
assert metadata.non_spec_state_indices_tensor is not None
self._flashinfer_decode(
x=qkv[:n],
conv_state=conv_state,
raw_gate=g1[:, :n],
raw_beta=beta[:, :n],
state_indices=metadata.non_spec_state_indices_tensor[:n],
state=state,
output_gate=g2[:n],
output=out[:, :n],
)
return n
def _flashinfer_decode(
self,
x: torch.Tensor,
conv_state: torch.Tensor,
raw_gate: torch.Tensor,
raw_beta: torch.Tensor,
state_indices: torch.Tensor,
state: torch.Tensor,
output_gate: torch.Tensor,
output: torch.Tensor,
query_start_loc: torch.Tensor | None = None,
num_accepted_tokens: torch.Tensor | None = None,
) -> None:
"""FlashInfer fused KDA decode: conv + SiLU with the conv-state update,
the recurrence and the gated RMSNorm, written into `output`. Rows
with a non-positive slot are null rows. With `query_start_loc` the
rows are draft-verify rows (`state_indices` [N, 1 + num_spec])."""
assert self._fi_conv_weight is not None
assert self._fi_norm_weight is not None
kwargs = dict(
x=x,
weight=self._fi_conv_weight,
conv_state=conv_state,
raw_gate=raw_gate,
raw_beta=raw_beta,
A_log=self.A_log.view(-1),
dt_bias=self.dt_bias,
state=state,
output_gate=output_gate,
norm_weight=self._fi_norm_weight,
lower_bound=self.kda_lower_bound,
norm_eps=self.o_norm.eps,
output=output,
)
if query_start_loc is None:
flashinfer_fused_kda_decode(state_indices=state_indices, **kwargs)
else:
flashinfer_packed_fused_kda_decode(
state_indices=state_indices.contiguous(),
query_start_loc=query_start_loc,
num_accepted_tokens=num_accepted_tokens,
**kwargs,
)
def _flashinfer_prefill(
self,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
g: torch.Tensor,
beta: torch.Tensor,
state: torch.Tensor,
state_indices: torch.Tensor,
has_initial_state: torch.Tensor,
cu_seqlens: torch.Tensor,
metadata: GDNAttentionMetadata,
out: torch.Tensor | None,
) -> torch.Tensor:
"""FlashInfer's CuTe DSL chunked prefill from the raw gate and beta
logits, reading and writing the recurrent states in place at
`state_indices`. Writes `out` (a new buffer when `None`) and returns
it."""
# FlashInfer has no has-initial-state flag: zero the slots of fresh
# sequences first (the others zero the unused null slot).
state.index_fill_(
0, torch.where(has_initial_state, NULL_BLOCK_ID, state_indices).long(), 0
)
if out is None:
out = torch.empty_like(q)
# With speculative decoding the slots are column 0 of the
# [N, 1 + num_spec] block table, which FlashInfer rejects.
state_indices = state_indices.contiguous()
if q.size(1) == 1:
# FlashInfer's prefill needs two tokens; one is a decode step.
fused_recurrent_kda(
q=q,
k=k,
v=v,
g=g,
beta=beta,
initial_state=state,
use_qk_l2norm_in_kernel=True,
cu_seqlens=cu_seqlens,
ssm_state_indices=state_indices,
out=out,
sigmoid_beta=True,
a_log=self.A_log,
g_bias=self.dt_bias,
compute_gate=True,
lower_bound=self.kda_lower_bound,
)
return out
assert self.kda_lower_bound is not None
flashinfer_recurrent_kda(
q=q.contiguous(),
k=k.contiguous(),
v=v.contiguous(),
g=g.contiguous(),
beta=beta.contiguous(),
A_log=self.A_log.view(-1),
dt_bias=self.dt_bias,
scale=q.shape[-1] ** -0.5,
initial_state=state,
output_final_state=True,
use_qk_l2norm_in_kernel=True,
use_gate_in_kernel=True,
lower_bound=self.kda_lower_bound,
cu_seqlens=_int64_cu_seqlens(metadata, cu_seqlens),
ssm_state_indices=state_indices,
output=out,
beta_is_logit=True,
backend="cute-dsl",
)
return out
@eager_break_during_capture
def _forward(
self,
qkv_proj_states: torch.Tensor,
g1: torch.Tensor,
beta: torch.Tensor,
core_attn_out: torch.Tensor,
g2: torch.Tensor,
) -> None:
forward_context = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
if attn_metadata_raw is None:
return
assert isinstance(attn_metadata_raw, dict)
attn_metadata_narrowed = attn_metadata_raw.get(self.prefix)
if attn_metadata_narrowed is None:
# Profile/warmup dummy runs may omit mamba-family metadata.
return
assert isinstance(attn_metadata_narrowed, GDNAttentionMetadata)
has_initial_state = attn_metadata_narrowed.has_initial_state
non_spec_query_start_loc = attn_metadata_narrowed.non_spec_query_start_loc
non_spec_state_indices_tensor = (
attn_metadata_narrowed.non_spec_state_indices_tensor
) # noqa: E501
num_actual_tokens = attn_metadata_narrowed.num_actual_tokens
# Spec-decode metadata (all None when speculative decoding is disabled).
spec_sequence_masks = attn_metadata_narrowed.spec_sequence_masks
spec_query_start_loc = attn_metadata_narrowed.spec_query_start_loc
spec_state_indices_tensor = attn_metadata_narrowed.spec_state_indices_tensor
spec_token_indx = attn_metadata_narrowed.spec_token_indx
non_spec_token_indx = attn_metadata_narrowed.non_spec_token_indx
num_accepted_tokens = attn_metadata_narrowed.num_accepted_tokens
num_spec_decodes = attn_metadata_narrowed.num_spec_decodes
use_spec = spec_sequence_masks is not None and num_spec_decodes > 0
# Safe-gate checkpoints use the bounded sigmoid variant.
safe_gate = self.kda_safe_gate
lower_bound = self.kda_lower_bound
constant_caches = self.kv_cache
qkv_proj_states = qkv_proj_states[:num_actual_tokens]
g1 = g1[:, :num_actual_tokens]
beta = beta[:, :num_actual_tokens]
(conv_state, recurrent_state) = constant_caches
# conv_state must be (..., dim, width-1) for the conv kernels.
# DS layout stores it that way directly; SD layout needs a transpose.
# Layout is process-global and resolved once at init (see __init__).
if not self._conv_state_dim_first:
conv_state = conv_state.transpose(-1, -2)
# One merged short-conv over q|k|v instead of three separate calls. The
# 1D conv is independent per channel, so concatenating q/k/v along the
# channel dim and running a single causal_conv1d is bit-identical to
# three calls. conv_state is already stored as the merged q|k|v state.
conv_weights = self.conv1d.weight.view(
self.conv1d.weight.size(0), self.conv1d.weight.size(-1)
)
conv_bias = self.conv1d.bias
num_fused_decodes = self._flashinfer_decode_rows(
attn_metadata_narrowed,
qkv_proj_states,
g1,
beta,
g2,
core_attn_out,
conv_state,
recurrent_state,
)
if num_fused_decodes and attn_metadata_narrowed.num_prefills == 0:
return
# Split projections / gating into spec (draft-verify) and non-spec token
# groups when speculative decoding is active. Spec tokens carry
# num_spec+1 recurrent-state columns each and are advanced with
# num_accepted_tokens for rejection-sampling rollback; non-spec tokens
# are one-per-request. Mirrors olmo_gdn_linear_attn.py. Projections are
# [n, *] (token dim 0); g1/beta are [1, n, h, d] (token dim 1).
if use_spec:
# In a pure spec-verify step (no non-spec tokens) the metadata
# builder sets spec_token_indx = arange(num_actual_tokens), making
# the index_select calls below identity copies. Skip them on this
# steady-state decode hot path. The outputs alias the inputs here;
# the downstream conv/recurrent kernels read them without mutating
# in place, so the aliasing is safe.
if non_spec_token_indx is None or non_spec_token_indx.numel() == 0:
qkv_spec = qkv_proj_states
g1_spec = g1
beta_spec = beta
else:
qkv_spec = qkv_proj_states.index_select(0, spec_token_indx)
g1_spec = g1.index_select(1, spec_token_indx)
beta_spec = beta.index_select(1, spec_token_indx)
if non_spec_token_indx is not None and non_spec_token_indx.numel() > 0:
qkv_ns = qkv_proj_states.index_select(0, non_spec_token_indx)
g1_ns = g1.index_select(1, non_spec_token_indx)
beta_ns = beta.index_select(1, non_spec_token_indx)
else:
qkv_ns = g1_ns = beta_ns = None
else:
qkv_spec = g1_spec = beta_spec = None
qkv_ns, g1_ns, beta_ns = qkv_proj_states, g1, beta
# --- causal conv1d: spec (draft-verify) path ---
if use_spec:
assert spec_state_indices_tensor is not None
assert num_accepted_tokens is not None
conv_idx = spec_state_indices_tensor[:, 0][:num_spec_decodes]
conv_mql = spec_state_indices_tensor.size(-1)
qkv_spec = causal_conv1d_update(
qkv_spec,
conv_state,
conv_weights,
conv_bias,
activation="silu",
conv_state_indices=conv_idx,
num_accepted_tokens=num_accepted_tokens,
query_start_loc=spec_query_start_loc,
max_query_len=conv_mql,
)
q_spec, k_spec, v_spec = qkv_spec.split(self.local_projection_size, dim=-1)
# --- causal conv1d: non-spec path (prefill or plain decode) ---
raw_qkv_ns = qkv_ns
q_ns = k_ns = v_ns = None
conv_cache_indices = non_spec_state_indices_tensor
prefill_query_start_loc = non_spec_query_start_loc
prefill_state_indices = non_spec_state_indices_tensor
prefill_has_initial_state = has_initial_state
if num_fused_decodes:
# Mixed step: the conv skips the decode rows FlashInfer ran (null
# slots) and the prefill kernel sees the builder's prefill tail.
assert non_spec_state_indices_tensor is not None
conv_cache_indices = non_spec_state_indices_tensor.clone()
conv_cache_indices[:num_fused_decodes] = NULL_BLOCK_ID
prefill_query_start_loc = attn_metadata_narrowed.prefill_query_start_loc
prefill_state_indices = attn_metadata_narrowed.prefill_state_indices
prefill_has_initial_state = attn_metadata_narrowed.prefill_has_initial_state
if attn_metadata_narrowed.num_prefills > 0:
assert qkv_ns is not None
qkv_ns = causal_conv1d_fn(
qkv_ns.transpose(0, 1),
conv_weights,
conv_bias,
activation="silu",
conv_states=conv_state,
has_initial_state=has_initial_state,
cache_indices=conv_cache_indices,
query_start_loc=non_spec_query_start_loc,
metadata=attn_metadata_narrowed,
).transpose(0, 1)
q_ns, k_ns, v_ns = qkv_ns.split(self.local_projection_size, dim=-1)
elif attn_metadata_narrowed.num_decodes > 0:
assert non_spec_state_indices_tensor is not None
decode_conv_indices = non_spec_state_indices_tensor[
: attn_metadata_narrowed.num_decodes
]
qkv_ns = causal_conv1d_update(
qkv_ns,
conv_state,
conv_weights,
conv_bias,
activation="silu",
conv_state_indices=decode_conv_indices,
)
q_ns, k_ns, v_ns = qkv_ns.split(self.local_projection_size, dim=-1)
def _rearr(x):
return x.reshape(1, -1, self.local_num_heads, self.head_dim)
# --- core attention: spec (draft-verify) path ---
core_attn_out_spec = None
# In a pure spec-verify step (no non-spec tokens) the recurrent kernel
# can write straight into the layer output buffer, skipping the
# fresh allocation + copy below. Mixed steps must scatter via
# spec_token_indx, so they keep the kernel-managed output.
spec_out = (
core_attn_out[0, :num_actual_tokens].unsqueeze(0)
if non_spec_token_indx is None or non_spec_token_indx.numel() == 0
else None
)
if use_spec:
assert spec_state_indices_tensor is not None
assert num_accepted_tokens is not None
assert spec_query_start_loc is not None
# Gate computed inside the recurrent kernel (COMPUTE_GATE) from
# raw g1 — replicates fused_kda_gate's arithmetic bit-for-bit and
# skips its launch + fp32 [n, H, D] intermediate per layer.
core_attn_out_spec, _ = fused_recurrent_kda(
q=_rearr(q_spec),
k=_rearr(k_spec),
v=_rearr(v_spec),
g=g1_spec,
beta=beta_spec,
initial_state=recurrent_state,
use_qk_l2norm_in_kernel=True,
cu_seqlens=spec_query_start_loc[: num_spec_decodes + 1],
ssm_state_indices=spec_state_indices_tensor,
num_accepted_tokens=num_accepted_tokens,
out=spec_out,
sigmoid_beta=True,
a_log=self.A_log,
g_bias=self.dt_bias,
compute_gate=True,
lower_bound=lower_bound,
)
# --- core attention: non-spec path (prefill or plain decode) ---
core_attn_out_non_spec = None
# Only the plain-decode recurrent kernel can write straight into the
# layer output buffer; the chunked prefill kernel cannot, so this
# stays None there and the merge copy below runs as before.
ns_out = None
if attn_metadata_narrowed.num_prefills > 0:
assert q_ns is not None and k_ns is not None and v_ns is not None
assert g1_ns is not None and beta_ns is not None
assert prefill_query_start_loc is not None
assert prefill_state_indices is not None
assert prefill_has_initial_state is not None
# Without the decode rows FlashInfer ran (none outside mixed steps).
q_ns, k_ns, v_ns = (t[num_fused_decodes:] for t in (q_ns, k_ns, v_ns))
g1_ns = g1_ns[:, num_fused_decodes:]
beta_ns = beta_ns[:, num_fused_decodes:]
q_p, k_p, v_p = _rearr(q_ns), _rearr(k_ns), _rearr(v_ns)
if self.kda_prefill_backend != "triton" and not use_spec:
# FlashInfer / FlashKDA write a non-spec step straight into the
# layer output (dense token order, no merge copy); a step with
# spec-decode tokens gets its own buffer, scattered below.
ns_out = core_attn_out[:, num_fused_decodes:num_actual_tokens]
if self.kda_prefill_backend == "flashinfer":
# In place on the state pool: no gather / scatter.
core_attn_out_non_spec = self._flashinfer_prefill(
q=q_p,
k=k_p,
v=v_p,
g=g1_ns,
beta=beta_ns,
state=recurrent_state,
state_indices=prefill_state_indices,
has_initial_state=prefill_has_initial_state,
cu_seqlens=prefill_query_start_loc,
metadata=attn_metadata_narrowed,
out=ns_out,
)
else:
initial_state = gather_initial_states(
recurrent_state, prefill_state_indices, prefill_has_initial_state
)
if self.kda_prefill_backend == "flashkda":
core_attn_out_non_spec, last_recurrent_state = (
self._flashkda_prefill(
q=q_p,
k=k_p,
v=v_p,
g=g1_ns,
beta=beta_ns,
initial_state=initial_state,
cu_seqlens=prefill_query_start_loc,
out=ns_out,
checkpoint=attn_metadata_narrowed.checkpoint,
raw_qkv=raw_qkv_ns,
conv_state=conv_state,
recurrent_state=recurrent_state,
)
)
else:
checkpoint = attn_metadata_narrowed.checkpoint
checkpoint_state, checkpoint_offsets = (
get_triton_kda_checkpoint_buffer(
checkpoint,
self._triton_checkpoint_spec,
initial_state.shape[0],
)
)
core_attn_out_non_spec, last_recurrent_state = (
chunk_kda_with_fused_gate(
q=q_p,
k=k_p,
v=v_p,
raw_g=g1_ns,
# Chunk path wants the pre-sigmoided fp32 beta (its
# kernels don't sigmoid); beta_ns is raw bf16.
beta=_cast_sigmoid(beta_ns.squeeze(0)).unsqueeze(0),
A_log=self.A_log,
g_bias=self.dt_bias,
initial_state=initial_state,
output_final_state=True,
use_qk_l2norm_in_kernel=True,
cu_seqlens=prefill_query_start_loc,
safe_gate=safe_gate,
lower_bound=lower_bound,
checkpoint_state=checkpoint_state,
checkpoint_offsets=checkpoint_offsets,
)
)
if checkpoint is not None:
assert self._checkpoint_exporter is not None
self._checkpoint_exporter.export(
checkpoint,
raw_qkv=raw_qkv_ns,
conv_state=conv_state,
recurrent_checkpoint=checkpoint_state,
recurrent_state=recurrent_state,
cu_seqlens=prefill_query_start_loc,
)
# Init cache
scatter_states(
recurrent_state, last_recurrent_state, prefill_state_indices
)
elif attn_metadata_narrowed.num_decodes > 0:
assert non_spec_query_start_loc is not None
assert non_spec_state_indices_tensor is not None
# Plain decode step (no spec tokens): token order is dense, so the
# kernel can write straight into the layer output buffer. A mixed
# step scatters non-spec output via non_spec_token_indx instead.
# Gate computed in-kernel (COMPUTE_GATE), beta sigmoided in-kernel.
if not use_spec:
ns_out = spec_out
core_attn_out_non_spec, _ = fused_recurrent_kda(
q=_rearr(q_ns),
k=_rearr(k_ns),
v=_rearr(v_ns),
g=g1_ns,
beta=beta_ns,
initial_state=recurrent_state,
use_qk_l2norm_in_kernel=True,
cu_seqlens=non_spec_query_start_loc[
: attn_metadata_narrowed.num_decodes + 1
],
ssm_state_indices=non_spec_state_indices_tensor,
out=ns_out,
sigmoid_beta=True,
a_log=self.A_log,
g_bias=self.dt_bias,
compute_gate=True,
lower_bound=lower_bound,
)
# --- merge spec / non-spec outputs back into token order ---
if use_spec and core_attn_out_non_spec is not None:
assert core_attn_out_spec is not None
core_attn_out.index_copy_(1, spec_token_indx, core_attn_out_spec)
core_attn_out.index_copy_(1, non_spec_token_indx, core_attn_out_non_spec)
elif use_spec:
assert core_attn_out_spec is not None
if spec_out is None:
core_attn_out[0, :num_actual_tokens] = core_attn_out_spec.squeeze(0)
else:
assert core_attn_out_non_spec is not None
if ns_out is None:
core_attn_out[0, num_fused_decodes:num_actual_tokens] = (
core_attn_out_non_spec[0, : num_actual_tokens - num_fused_decodes]
)
if self._norm_in_forward:
# Rows from the fused decode kernel are normed already.
live_out = core_attn_out[:, num_fused_decodes:num_actual_tokens]
normed = self.o_norm(live_out, g2[num_fused_decodes:num_actual_tokens])
if normed.data_ptr() != live_out.data_ptr():
live_out.copy_(normed)