Skip to content

vllm.models.glm5next.common.kda

GLM-5.3-Flash KDA layer with separate convolutions and a bounded safe gate.

Classes:

Glm5NextLinearAttention

Bases: GatedDeltaNetAttention

Methods:

Source code in vllm/models/glm5next/common/kda.py
 350
 351
 352
 353
 354
 355
 356
 357
 358
 359
 360
 361
 362
 363
 364
 365
 366
 367
 368
 369
 370
 371
 372
 373
 374
 375
 376
 377
 378
 379
 380
 381
 382
 383
 384
 385
 386
 387
 388
 389
 390
 391
 392
 393
 394
 395
 396
 397
 398
 399
 400
 401
 402
 403
 404
 405
 406
 407
 408
 409
 410
 411
 412
 413
 414
 415
 416
 417
 418
 419
 420
 421
 422
 423
 424
 425
 426
 427
 428
 429
 430
 431
 432
 433
 434
 435
 436
 437
 438
 439
 440
 441
 442
 443
 444
 445
 446
 447
 448
 449
 450
 451
 452
 453
 454
 455
 456
 457
 458
 459
 460
 461
 462
 463
 464
 465
 466
 467
 468
 469
 470
 471
 472
 473
 474
 475
 476
 477
 478
 479
 480
 481
 482
 483
 484
 485
 486
 487
 488
 489
 490
 491
 492
 493
 494
 495
 496
 497
 498
 499
 500
 501
 502
 503
 504
 505
 506
 507
 508
 509
 510
 511
 512
 513
 514
 515
 516
 517
 518
 519
 520
 521
 522
 523
 524
 525
 526
 527
 528
 529
 530
 531
 532
 533
 534
 535
 536
 537
 538
 539
 540
 541
 542
 543
 544
 545
 546
 547
 548
 549
 550
 551
 552
 553
 554
 555
 556
 557
 558
 559
 560
 561
 562
 563
 564
 565
 566
 567
 568
 569
 570
 571
 572
 573
 574
 575
 576
 577
 578
 579
 580
 581
 582
 583
 584
 585
 586
 587
 588
 589
 590
 591
 592
 593
 594
 595
 596
 597
 598
 599
 600
 601
 602
 603
 604
 605
 606
 607
 608
 609
 610
 611
 612
 613
 614
 615
 616
 617
 618
 619
 620
 621
 622
 623
 624
 625
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
class Glm5NextLinearAttention(GatedDeltaNetAttention):
    head_dim: int
    num_heads: int
    conv_size: int

    def get_kv_cache_checkpoint_alignment(self, vllm_config: VllmConfig) -> int | None:
        return kda_prefill_checkpoint_alignment(self.kda_prefill_backend)

    def get_state_dtype(
        self,
    ) -> tuple[torch.dtype, torch.dtype]:
        if self.model_config is None or self.cache_config is None:
            raise ValueError("model_config and cache_config must be set")
        return MambaStateDtypeCalculator.kda_state_dtype(
            self.model_config.dtype,
            self.cache_config.mamba_cache_dtype,
            self.cache_config.mamba_ssm_cache_dtype,
        )

    def get_state_shape(
        self,
    ) -> tuple[tuple[int, ...], tuple[int, ...]]:
        # conv_state width must include num_spec so the spec-decode conv update
        # (causal_conv1d_update with num_accepted_tokens + max_query_len) can
        # slide the window across the draft-verify tokens without reading past
        # the allocated width. Matches qwen_gdn_linear_attn.get_state_shape.
        return MambaStateShapeCalculator.kda_state_shape(
            self.tp_size,
            self.num_heads,
            self.head_dim,
            conv_kernel_size=self.conv_size,
            num_spec=self.num_spec,
        )

    def __init__(
        self,
        config: Glm5NextTextConfig,
        vllm_config: VllmConfig,
        prefix: str = "",
    ) -> None:
        super().__init__(config, vllm_config, prefix)

        self.head_dim = config.linear_head_dim
        self.num_heads = config.linear_num_heads
        self.conv_size = config.linear_conv_kernel_dim
        assert self.num_heads % self.tp_size == 0
        self.local_num_heads = divide(self.num_heads, self.tp_size)

        projection_size = self.head_dim * self.num_heads
        self.local_projection_size = divide(projection_size, self.tp_size)

        _, recurrent_state_shape = self.get_state_shape()
        _, recurrent_state_dtype = self.get_state_dtype()
        scatter_states.register_warmup(
            state_shape=recurrent_state_shape,
            state_dtype=recurrent_state_dtype,
            indices_dtype=torch.int32,
            max_num_tokens=vllm_config.scheduler_config.max_num_seqs,
        )

        # Merge q, k, v, b, f_a, g_a projections into one GEMM (6→1 launches).
        # Order matches checkpoint's fused_qkvbfg_a_proj convention.
        # Shards 4 (f_a) and 5 (g_a) are replicated across TP ranks.
        self.in_proj_qkvbfg_a = _Glm5NextMergedColumnParallelLinear(
            self.hidden_size,
            [
                projection_size,  # q (shard 0)
                projection_size,  # k (shard 1)
                projection_size,  # v (shard 2)
                self.num_heads,  # b (shard 3)
                self.head_dim,  # f_a (shard 4, replicated)
                self.head_dim,  # g_a (shard 5, replicated)
            ],
            replicated_shard_ids=(4, 5),
            tp_size=self.tp_size,
            bias=False,
            quant_config=self.quant_config,
            prefix=f"{prefix}.in_proj_qkvbfg_a",
        )

        self.f_b_proj = ColumnParallelLinear(
            self.head_dim,
            projection_size,
            bias=False,
            quant_config=self.quant_config,
            prefix=f"{prefix}.f_b_proj",
        )
        self.dt_bias = nn.Parameter(
            torch.empty(divide(projection_size, self.tp_size), dtype=torch.float32)
        )

        set_weight_attrs(self.dt_bias, {"weight_loader": sharded_weight_loader(0)})

        self.conv1d = _fused_qkv_conv1d(
            self.conv_size,
            projection_size,
            self.tp_size,
            self.tp_rank,
            prefix=f"{prefix}.conv1d",
        )

        self.A_log = nn.Parameter(
            torch.empty(1, 1, self.local_num_heads, 1, dtype=torch.float32)
        )
        set_weight_attrs(self.A_log, {"weight_loader": sharded_weight_loader(2)})

        self.g_b_proj = ColumnParallelLinear(
            self.head_dim,
            projection_size,
            bias=False,
            quant_config=self.quant_config,
            prefix=f"{prefix}.g_b_proj",
        )
        self.o_norm = FusedRMSNormGated(
            self.head_dim,
            activation="sigmoid",
            enforce_enable=current_platform.is_rocm(),
        )
        self.o_proj = RowParallelLinear(
            projection_size,
            self.hidden_size,
            bias=False,
            quant_config=self.quant_config,
            prefix=f"{prefix}.o_proj",
        )

        compilation_config = get_current_vllm_config().compilation_config
        if prefix in compilation_config.static_forward_context:
            raise ValueError(f"Duplicate layer name: {prefix}")
        compilation_config.static_forward_context[prefix] = self

        # Checkpoints store A_log as 1-D; the model parameter is 4-D.
        def _a_log_weight_loader(param, loaded_weight):
            if loaded_weight.dim() == 1:
                loaded_weight = loaded_weight.view([1, 1, -1, 1])
            return sharded_weight_loader(2)(param, loaded_weight)

        self.A_log.weight_loader = _a_log_weight_loader

        # GLM-5.3-Flash uses a bounded sigmoid gate instead of the default
        # unbounded softplus gate.
        self.kda_safe_gate = True
        self.kda_lower_bound = config.linear_lower_bound
        # Process-global conv-state layout, resolved once here instead of on
        # every _forward call (it reads an env-derived flag each time).
        self._conv_state_dim_first = is_conv_state_dim_first()

        additional_config = vllm_config.additional_config
        conv_state_dtype, recurrent_state_dtype = self.get_state_dtype()
        self.kda_prefill_backend = _resolve_kda_prefill_backend(
            additional_config.get("kda_prefill_backend", "auto")
            if isinstance(additional_config, dict)
            else "auto",
            self.head_dim,
            vllm_config.model_config.dtype,
            self.kda_lower_bound,
            recurrent_state_dtype,
        )
        if self.kda_prefill_backend == "triton":
            l2norm_fwd_block.register_warmup(
                dtype=vllm_config.model_config.dtype,
                dim=self.head_dim,
            )

        # One option (`--kda-decode-backend`) for every decode row: with
        # speculative decoding they are draft-verify rows of 1 + num_spec
        # tokens, otherwise single-token rows.
        self.kda_decode_backend = _resolve_kda_decode_backend(
            additional_config.get("kda_decode_backend", "auto")
            if isinstance(additional_config, dict)
            else "auto",
            self.local_num_heads,
            self.head_dim,
            self.conv_size,
            1 + self.num_spec,
            vllm_config.model_config.dtype,
            conv_state_dtype,
            recurrent_state_dtype,
            self.kda_lower_bound,
        )
        # FlashInfer's [3, width, P] fp32 conv weight and fp32 norm weight,
        # built in process_weights_after_loading.
        self._fi_conv_weight: torch.Tensor | None = None
        self._fi_norm_weight: torch.Tensor | None = None

        self._flashkda_buffer_specs: (
            tuple[tuple[tuple[int, ...], torch.dtype], ...] | None
        ) = None
        self._triton_checkpoint_spec: TritonKDACheckpointSpec | None = None
        self._checkpoint_exporter: FlashKDAPrefillCheckpointExporter | None = None
        if kda_prefill_checkpoint_alignment(self.kda_prefill_backend) is not None:
            self._checkpoint_exporter = FlashKDAPrefillCheckpointExporter(
                state_len=self.conv_size - 1
            )
        if self.kda_prefill_backend == "triton":
            self._triton_checkpoint_spec = reserve_triton_kda_checkpoint_buffer(
                vllm_config,
                self.local_num_heads,
                self.head_dim,
                self.get_state_dtype()[1],
            )
        if self.kda_prefill_backend == "flashkda":
            import vllm._flashkda_C  # noqa: F401

            max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
            max_seqs = vllm_config.scheduler_config.max_num_seqs
            workspace_size = torch.ops._flashkda_C.get_workspace_size(
                max_tokens, self.local_num_heads, max_seqs
            )
            self._flashkda_buffer_specs = (
                (
                    (max_seqs, self.local_num_heads, self.head_dim, self.head_dim),
                    self.get_state_dtype()[1],
                ),
                (
                    (max_seqs, self.local_num_heads, self.head_dim, self.head_dim),
                    self.get_state_dtype()[1],
                ),
                ((workspace_size,), torch.uint8),
                # Output buffer for steps that also carry spec-decode tokens:
                # the non-spec tokens are then scattered by non_spec_token_indx.
                (
                    (1, max_tokens, self.local_num_heads, self.head_dim),
                    vllm_config.model_config.dtype,
                ),
            )

    def _flashkda_prefill(
        self,
        q: torch.Tensor,
        k: torch.Tensor,
        v: torch.Tensor,
        g: torch.Tensor,
        beta: torch.Tensor,
        initial_state: torch.Tensor,
        cu_seqlens: torch.Tensor,
        out: torch.Tensor | None,
        checkpoint: MambaPrefillCheckpointMetadata | None = None,
        raw_qkv: torch.Tensor | None = None,
        conv_state: torch.Tensor | None = None,
        recurrent_state: torch.Tensor | None = None,
    ) -> tuple[torch.Tensor, torch.Tensor]:
        """Fused KDA chunked prefill (FlashKDA). Takes the raw gate logits `g`
        and raw `beta` logits, l2-normalizes q/k in-kernel and applies the
        bounded gate `lower_bound * sigmoid(exp(A_log) * (g + dt_bias))`,
        matching `chunk_kda_with_fused_gate(..., safe_gate=True)`. Writes the
        attention output into `out` (a workspace buffer when `None`) and
        returns `(out, final_state)`."""
        assert self._flashkda_buffer_specs is not None
        final_state, checkpoint_state, workspace, workspace_out = (
            current_workspace_manager().get_simultaneous(*self._flashkda_buffer_specs)
        )
        final_state = final_state[: initial_state.shape[0]]
        checkpoint_state = (
            checkpoint_state[: initial_state.shape[0]]
            if checkpoint is not None
            else None
        )
        if out is None:
            out = workspace_out[:, : q.shape[1]]
        # FlashKDA accepts row-strided q/k/v (contiguous H/D rows, aligned
        # token stride), so the packed q|k|v views are passed without a copy.
        torch.ops._flashkda_C.fwd(
            q,
            k,
            v,
            g.contiguous(),
            beta,
            self.head_dim**-0.5,
            out,
            workspace,
            self.A_log.view(-1),
            self.dt_bias.view(-1, self.head_dim),
            self.kda_lower_bound,
            initial_state.contiguous(),
            final_state,
            cu_seqlens.contiguous(),
            checkpoint_state,
            checkpoint.checkpoint_offsets.contiguous()
            if checkpoint is not None
            else None,
        )
        if checkpoint is not None:
            assert self._checkpoint_exporter is not None
            assert raw_qkv is not None and conv_state is not None
            assert recurrent_state is not None
            self._checkpoint_exporter.export(
                checkpoint,
                raw_qkv=raw_qkv,
                conv_state=conv_state,
                recurrent_checkpoint=checkpoint_state,
                recurrent_state=recurrent_state,
                cu_seqlens=cu_seqlens,
            )
        return out, final_state

    def forward(
        self,
        hidden_states: torch.Tensor,
        positions: torch.Tensor,
    ) -> torch.Tensor:
        num_tokens = hidden_states.size(0)
        # One merged GEMM for q, k, v, b, f_a, g_a (replaces 6 separate GEMMs).
        projected = self.in_proj_qkvbfg_a(hidden_states)[0]
        qkv, beta_raw, f_a, g_a = projected.split(
            [
                3 * self.local_projection_size,
                self.local_num_heads,
                self.head_dim,
                self.head_dim,
            ],
            dim=-1,
        )

        # Beta stays raw (bf16) here: the recurrent kernel sigmoids it in fp32
        # at load (SIGMOID_BETA), and only the chunked prefill path needs the
        # pre-computed fp32 sigmoid — computed lazily in _forward. Pure decode
        # / spec-verify steps then skip the _cast_sigmoid kernel and its fp32
        # intermediate entirely.
        beta = beta_raw.unsqueeze(0)
        g1 = self.f_b_proj(f_a)[0]
        g1 = g1.reshape(1, -1, self.local_num_heads, self.head_dim)

        g_proj_states = self.g_b_proj(g_a)[0]
        # Must stay 3D: rms_norm_gated reads H from g.shape[-2].
        g2 = g_proj_states.reshape(-1, self.local_num_heads, self.head_dim)

        core_attn_out = torch.empty(
            (1, num_tokens, self.local_num_heads, self.head_dim),
            dtype=hidden_states.dtype,
            device=hidden_states.device,
        )
        # Call the decorated eager break directly so host-side prefill branches
        # are not captured by PIECEWISE CUDA graphs.
        self._forward(
            qkv_proj_states=qkv,
            g1=g1,
            beta=beta,
            core_attn_out=core_attn_out,
            g2=g2,
        )
        if not self._norm_in_forward:
            core_attn_out = self.o_norm(core_attn_out, g2)
        core_attn_out = core_attn_out.reshape(core_attn_out.size(1), -1)
        return self.o_proj(core_attn_out)[0]

    @property
    def _norm_in_forward(self) -> bool:
        """FlashInfer's fused decode applies o_norm itself, so with it
        `_forward` returns normed output for every step type: a breakable CUDA
        graph replays the code after `_forward` as captured, whatever the live
        step."""
        return self.kda_decode_backend == "flashinfer"

    def process_weights_after_loading(self) -> None:
        """Build FlashInfer's conv and norm weights from the loaded weights;
        reloads refill them in place."""
        if not self._norm_in_forward:
            return
        conv_weight = self.conv1d.weight.view(
            3, self.local_projection_size, self.conv_size
        )
        refill_derived(
            self, "_fi_conv_weight", conv_weight.transpose(1, 2).contiguous()
        )
        refill_derived(self, "_fi_norm_weight", self.o_norm.weight.float().contiguous())

    def _flashinfer_decode_rows(
        self,
        metadata: GDNAttentionMetadata,
        qkv: torch.Tensor,
        g1: torch.Tensor,
        beta: torch.Tensor,
        g2: torch.Tensor,
        out: torch.Tensor,
        conv_state: torch.Tensor,
        state: torch.Tensor,
    ) -> int:
        """Runs the step's leading decode rows through FlashInfer's fused
        decode and returns their token count (0 if it runs none). Without
        speculative decoding these are the decode rows of decode and mixed
        steps (vLLM orders decodes first); with it, every row of a step of
        draft-verify rows only."""
        if self.kda_decode_backend != "flashinfer":
            return 0
        if self.num_spec:
            if (
                metadata.spec_sequence_masks is None
                or metadata.num_spec_decodes == 0
                or metadata.num_decodes
                or metadata.num_prefills
            ):
                return 0
            assert metadata.spec_state_indices_tensor is not None
            assert metadata.spec_query_start_loc is not None
            n = metadata.num_actual_tokens
            self._flashinfer_decode(
                x=qkv,
                conv_state=conv_state,
                raw_gate=g1,
                raw_beta=beta,
                state_indices=metadata.spec_state_indices_tensor,
                state=state,
                output_gate=g2[:n],
                output=out[:, :n],
                query_start_loc=metadata.spec_query_start_loc,
                num_accepted_tokens=metadata.num_accepted_tokens,
            )
            return n
        n = metadata.num_decode_tokens
        # A mixed step with prefix-cache checkpoints runs on the chunked path.
        if n == 0 or (metadata.num_prefills and metadata.checkpoint is not None):
            return 0
        assert metadata.non_spec_state_indices_tensor is not None
        self._flashinfer_decode(
            x=qkv[:n],
            conv_state=conv_state,
            raw_gate=g1[:, :n],
            raw_beta=beta[:, :n],
            state_indices=metadata.non_spec_state_indices_tensor[:n],
            state=state,
            output_gate=g2[:n],
            output=out[:, :n],
        )
        return n

    def _flashinfer_decode(
        self,
        x: torch.Tensor,
        conv_state: torch.Tensor,
        raw_gate: torch.Tensor,
        raw_beta: torch.Tensor,
        state_indices: torch.Tensor,
        state: torch.Tensor,
        output_gate: torch.Tensor,
        output: torch.Tensor,
        query_start_loc: torch.Tensor | None = None,
        num_accepted_tokens: torch.Tensor | None = None,
    ) -> None:
        """FlashInfer fused KDA decode: conv + SiLU with the conv-state update,
        the recurrence and the gated RMSNorm, written into `output`. Rows
        with a non-positive slot are null rows. With `query_start_loc` the
        rows are draft-verify rows (`state_indices` [N, 1 + num_spec])."""
        assert self._fi_conv_weight is not None
        assert self._fi_norm_weight is not None
        kwargs = dict(
            x=x,
            weight=self._fi_conv_weight,
            conv_state=conv_state,
            raw_gate=raw_gate,
            raw_beta=raw_beta,
            A_log=self.A_log.view(-1),
            dt_bias=self.dt_bias,
            state=state,
            output_gate=output_gate,
            norm_weight=self._fi_norm_weight,
            lower_bound=self.kda_lower_bound,
            norm_eps=self.o_norm.eps,
            output=output,
        )
        if query_start_loc is None:
            flashinfer_fused_kda_decode(state_indices=state_indices, **kwargs)
        else:
            flashinfer_packed_fused_kda_decode(
                state_indices=state_indices.contiguous(),
                query_start_loc=query_start_loc,
                num_accepted_tokens=num_accepted_tokens,
                **kwargs,
            )

    def _flashinfer_prefill(
        self,
        q: torch.Tensor,
        k: torch.Tensor,
        v: torch.Tensor,
        g: torch.Tensor,
        beta: torch.Tensor,
        state: torch.Tensor,
        state_indices: torch.Tensor,
        has_initial_state: torch.Tensor,
        cu_seqlens: torch.Tensor,
        metadata: GDNAttentionMetadata,
        out: torch.Tensor | None,
    ) -> torch.Tensor:
        """FlashInfer's CuTe DSL chunked prefill from the raw gate and beta
        logits, reading and writing the recurrent states in place at
        `state_indices`. Writes `out` (a new buffer when `None`) and returns
        it."""
        # FlashInfer has no has-initial-state flag: zero the slots of fresh
        # sequences first (the others zero the unused null slot).
        state.index_fill_(
            0, torch.where(has_initial_state, NULL_BLOCK_ID, state_indices).long(), 0
        )
        if out is None:
            out = torch.empty_like(q)
        # With speculative decoding the slots are column 0 of the
        # [N, 1 + num_spec] block table, which FlashInfer rejects.
        state_indices = state_indices.contiguous()
        if q.size(1) == 1:
            # FlashInfer's prefill needs two tokens; one is a decode step.
            fused_recurrent_kda(
                q=q,
                k=k,
                v=v,
                g=g,
                beta=beta,
                initial_state=state,
                use_qk_l2norm_in_kernel=True,
                cu_seqlens=cu_seqlens,
                ssm_state_indices=state_indices,
                out=out,
                sigmoid_beta=True,
                a_log=self.A_log,
                g_bias=self.dt_bias,
                compute_gate=True,
                lower_bound=self.kda_lower_bound,
            )
            return out
        assert self.kda_lower_bound is not None
        flashinfer_recurrent_kda(
            q=q.contiguous(),
            k=k.contiguous(),
            v=v.contiguous(),
            g=g.contiguous(),
            beta=beta.contiguous(),
            A_log=self.A_log.view(-1),
            dt_bias=self.dt_bias,
            scale=q.shape[-1] ** -0.5,
            initial_state=state,
            output_final_state=True,
            use_qk_l2norm_in_kernel=True,
            use_gate_in_kernel=True,
            lower_bound=self.kda_lower_bound,
            cu_seqlens=_int64_cu_seqlens(metadata, cu_seqlens),
            ssm_state_indices=state_indices,
            output=out,
            beta_is_logit=True,
            backend="cute-dsl",
        )
        return out

    @eager_break_during_capture
    def _forward(
        self,
        qkv_proj_states: torch.Tensor,
        g1: torch.Tensor,
        beta: torch.Tensor,
        core_attn_out: torch.Tensor,
        g2: torch.Tensor,
    ) -> None:
        forward_context = get_forward_context()
        attn_metadata_raw = forward_context.attn_metadata

        if attn_metadata_raw is None:
            return

        assert isinstance(attn_metadata_raw, dict)
        attn_metadata_narrowed = attn_metadata_raw.get(self.prefix)
        if attn_metadata_narrowed is None:
            # Profile/warmup dummy runs may omit mamba-family metadata.
            return
        assert isinstance(attn_metadata_narrowed, GDNAttentionMetadata)
        has_initial_state = attn_metadata_narrowed.has_initial_state
        non_spec_query_start_loc = attn_metadata_narrowed.non_spec_query_start_loc
        non_spec_state_indices_tensor = (
            attn_metadata_narrowed.non_spec_state_indices_tensor
        )  # noqa: E501
        num_actual_tokens = attn_metadata_narrowed.num_actual_tokens
        # Spec-decode metadata (all None when speculative decoding is disabled).
        spec_sequence_masks = attn_metadata_narrowed.spec_sequence_masks
        spec_query_start_loc = attn_metadata_narrowed.spec_query_start_loc
        spec_state_indices_tensor = attn_metadata_narrowed.spec_state_indices_tensor
        spec_token_indx = attn_metadata_narrowed.spec_token_indx
        non_spec_token_indx = attn_metadata_narrowed.non_spec_token_indx
        num_accepted_tokens = attn_metadata_narrowed.num_accepted_tokens
        num_spec_decodes = attn_metadata_narrowed.num_spec_decodes
        use_spec = spec_sequence_masks is not None and num_spec_decodes > 0
        # Safe-gate checkpoints use the bounded sigmoid variant.
        safe_gate = self.kda_safe_gate
        lower_bound = self.kda_lower_bound
        constant_caches = self.kv_cache

        qkv_proj_states = qkv_proj_states[:num_actual_tokens]
        g1 = g1[:, :num_actual_tokens]
        beta = beta[:, :num_actual_tokens]

        (conv_state, recurrent_state) = constant_caches
        # conv_state must be (..., dim, width-1) for the conv kernels.
        # DS layout stores it that way directly; SD layout needs a transpose.
        # Layout is process-global and resolved once at init (see __init__).
        if not self._conv_state_dim_first:
            conv_state = conv_state.transpose(-1, -2)

        # One merged short-conv over q|k|v instead of three separate calls. The
        # 1D conv is independent per channel, so concatenating q/k/v along the
        # channel dim and running a single causal_conv1d is bit-identical to
        # three calls. conv_state is already stored as the merged q|k|v state.
        conv_weights = self.conv1d.weight.view(
            self.conv1d.weight.size(0), self.conv1d.weight.size(-1)
        )
        conv_bias = self.conv1d.bias

        num_fused_decodes = self._flashinfer_decode_rows(
            attn_metadata_narrowed,
            qkv_proj_states,
            g1,
            beta,
            g2,
            core_attn_out,
            conv_state,
            recurrent_state,
        )
        if num_fused_decodes and attn_metadata_narrowed.num_prefills == 0:
            return

        # Split projections / gating into spec (draft-verify) and non-spec token
        # groups when speculative decoding is active. Spec tokens carry
        # num_spec+1 recurrent-state columns each and are advanced with
        # num_accepted_tokens for rejection-sampling rollback; non-spec tokens
        # are one-per-request. Mirrors olmo_gdn_linear_attn.py. Projections are
        # [n, *] (token dim 0); g1/beta are [1, n, h, d] (token dim 1).
        if use_spec:
            # In a pure spec-verify step (no non-spec tokens) the metadata
            # builder sets spec_token_indx = arange(num_actual_tokens), making
            # the index_select calls below identity copies. Skip them on this
            # steady-state decode hot path. The outputs alias the inputs here;
            # the downstream conv/recurrent kernels read them without mutating
            # in place, so the aliasing is safe.
            if non_spec_token_indx is None or non_spec_token_indx.numel() == 0:
                qkv_spec = qkv_proj_states
                g1_spec = g1
                beta_spec = beta
            else:
                qkv_spec = qkv_proj_states.index_select(0, spec_token_indx)
                g1_spec = g1.index_select(1, spec_token_indx)
                beta_spec = beta.index_select(1, spec_token_indx)
            if non_spec_token_indx is not None and non_spec_token_indx.numel() > 0:
                qkv_ns = qkv_proj_states.index_select(0, non_spec_token_indx)
                g1_ns = g1.index_select(1, non_spec_token_indx)
                beta_ns = beta.index_select(1, non_spec_token_indx)
            else:
                qkv_ns = g1_ns = beta_ns = None
        else:
            qkv_spec = g1_spec = beta_spec = None
            qkv_ns, g1_ns, beta_ns = qkv_proj_states, g1, beta

        # --- causal conv1d: spec (draft-verify) path ---
        if use_spec:
            assert spec_state_indices_tensor is not None
            assert num_accepted_tokens is not None
            conv_idx = spec_state_indices_tensor[:, 0][:num_spec_decodes]
            conv_mql = spec_state_indices_tensor.size(-1)
            qkv_spec = causal_conv1d_update(
                qkv_spec,
                conv_state,
                conv_weights,
                conv_bias,
                activation="silu",
                conv_state_indices=conv_idx,
                num_accepted_tokens=num_accepted_tokens,
                query_start_loc=spec_query_start_loc,
                max_query_len=conv_mql,
            )
            q_spec, k_spec, v_spec = qkv_spec.split(self.local_projection_size, dim=-1)

        # --- causal conv1d: non-spec path (prefill or plain decode) ---
        raw_qkv_ns = qkv_ns
        q_ns = k_ns = v_ns = None
        conv_cache_indices = non_spec_state_indices_tensor
        prefill_query_start_loc = non_spec_query_start_loc
        prefill_state_indices = non_spec_state_indices_tensor
        prefill_has_initial_state = has_initial_state
        if num_fused_decodes:
            # Mixed step: the conv skips the decode rows FlashInfer ran (null
            # slots) and the prefill kernel sees the builder's prefill tail.
            assert non_spec_state_indices_tensor is not None
            conv_cache_indices = non_spec_state_indices_tensor.clone()
            conv_cache_indices[:num_fused_decodes] = NULL_BLOCK_ID
            prefill_query_start_loc = attn_metadata_narrowed.prefill_query_start_loc
            prefill_state_indices = attn_metadata_narrowed.prefill_state_indices
            prefill_has_initial_state = attn_metadata_narrowed.prefill_has_initial_state
        if attn_metadata_narrowed.num_prefills > 0:
            assert qkv_ns is not None
            qkv_ns = causal_conv1d_fn(
                qkv_ns.transpose(0, 1),
                conv_weights,
                conv_bias,
                activation="silu",
                conv_states=conv_state,
                has_initial_state=has_initial_state,
                cache_indices=conv_cache_indices,
                query_start_loc=non_spec_query_start_loc,
                metadata=attn_metadata_narrowed,
            ).transpose(0, 1)
            q_ns, k_ns, v_ns = qkv_ns.split(self.local_projection_size, dim=-1)
        elif attn_metadata_narrowed.num_decodes > 0:
            assert non_spec_state_indices_tensor is not None
            decode_conv_indices = non_spec_state_indices_tensor[
                : attn_metadata_narrowed.num_decodes
            ]
            qkv_ns = causal_conv1d_update(
                qkv_ns,
                conv_state,
                conv_weights,
                conv_bias,
                activation="silu",
                conv_state_indices=decode_conv_indices,
            )
            q_ns, k_ns, v_ns = qkv_ns.split(self.local_projection_size, dim=-1)

        def _rearr(x):
            return x.reshape(1, -1, self.local_num_heads, self.head_dim)

        # --- core attention: spec (draft-verify) path ---
        core_attn_out_spec = None
        # In a pure spec-verify step (no non-spec tokens) the recurrent kernel
        # can write straight into the layer output buffer, skipping the
        # fresh allocation + copy below. Mixed steps must scatter via
        # spec_token_indx, so they keep the kernel-managed output.
        spec_out = (
            core_attn_out[0, :num_actual_tokens].unsqueeze(0)
            if non_spec_token_indx is None or non_spec_token_indx.numel() == 0
            else None
        )
        if use_spec:
            assert spec_state_indices_tensor is not None
            assert num_accepted_tokens is not None
            assert spec_query_start_loc is not None
            # Gate computed inside the recurrent kernel (COMPUTE_GATE) from
            # raw g1 — replicates fused_kda_gate's arithmetic bit-for-bit and
            # skips its launch + fp32 [n, H, D] intermediate per layer.
            core_attn_out_spec, _ = fused_recurrent_kda(
                q=_rearr(q_spec),
                k=_rearr(k_spec),
                v=_rearr(v_spec),
                g=g1_spec,
                beta=beta_spec,
                initial_state=recurrent_state,
                use_qk_l2norm_in_kernel=True,
                cu_seqlens=spec_query_start_loc[: num_spec_decodes + 1],
                ssm_state_indices=spec_state_indices_tensor,
                num_accepted_tokens=num_accepted_tokens,
                out=spec_out,
                sigmoid_beta=True,
                a_log=self.A_log,
                g_bias=self.dt_bias,
                compute_gate=True,
                lower_bound=lower_bound,
            )

        # --- core attention: non-spec path (prefill or plain decode) ---
        core_attn_out_non_spec = None
        # Only the plain-decode recurrent kernel can write straight into the
        # layer output buffer; the chunked prefill kernel cannot, so this
        # stays None there and the merge copy below runs as before.
        ns_out = None
        if attn_metadata_narrowed.num_prefills > 0:
            assert q_ns is not None and k_ns is not None and v_ns is not None
            assert g1_ns is not None and beta_ns is not None
            assert prefill_query_start_loc is not None
            assert prefill_state_indices is not None
            assert prefill_has_initial_state is not None
            # Without the decode rows FlashInfer ran (none outside mixed steps).
            q_ns, k_ns, v_ns = (t[num_fused_decodes:] for t in (q_ns, k_ns, v_ns))
            g1_ns = g1_ns[:, num_fused_decodes:]
            beta_ns = beta_ns[:, num_fused_decodes:]
            q_p, k_p, v_p = _rearr(q_ns), _rearr(k_ns), _rearr(v_ns)
            if self.kda_prefill_backend != "triton" and not use_spec:
                # FlashInfer / FlashKDA write a non-spec step straight into the
                # layer output (dense token order, no merge copy); a step with
                # spec-decode tokens gets its own buffer, scattered below.
                ns_out = core_attn_out[:, num_fused_decodes:num_actual_tokens]
            if self.kda_prefill_backend == "flashinfer":
                # In place on the state pool: no gather / scatter.
                core_attn_out_non_spec = self._flashinfer_prefill(
                    q=q_p,
                    k=k_p,
                    v=v_p,
                    g=g1_ns,
                    beta=beta_ns,
                    state=recurrent_state,
                    state_indices=prefill_state_indices,
                    has_initial_state=prefill_has_initial_state,
                    cu_seqlens=prefill_query_start_loc,
                    metadata=attn_metadata_narrowed,
                    out=ns_out,
                )
            else:
                initial_state = gather_initial_states(
                    recurrent_state, prefill_state_indices, prefill_has_initial_state
                )
                if self.kda_prefill_backend == "flashkda":
                    core_attn_out_non_spec, last_recurrent_state = (
                        self._flashkda_prefill(
                            q=q_p,
                            k=k_p,
                            v=v_p,
                            g=g1_ns,
                            beta=beta_ns,
                            initial_state=initial_state,
                            cu_seqlens=prefill_query_start_loc,
                            out=ns_out,
                            checkpoint=attn_metadata_narrowed.checkpoint,
                            raw_qkv=raw_qkv_ns,
                            conv_state=conv_state,
                            recurrent_state=recurrent_state,
                        )
                    )
                else:
                    checkpoint = attn_metadata_narrowed.checkpoint
                    checkpoint_state, checkpoint_offsets = (
                        get_triton_kda_checkpoint_buffer(
                            checkpoint,
                            self._triton_checkpoint_spec,
                            initial_state.shape[0],
                        )
                    )
                    core_attn_out_non_spec, last_recurrent_state = (
                        chunk_kda_with_fused_gate(
                            q=q_p,
                            k=k_p,
                            v=v_p,
                            raw_g=g1_ns,
                            # Chunk path wants the pre-sigmoided fp32 beta (its
                            # kernels don't sigmoid); beta_ns is raw bf16.
                            beta=_cast_sigmoid(beta_ns.squeeze(0)).unsqueeze(0),
                            A_log=self.A_log,
                            g_bias=self.dt_bias,
                            initial_state=initial_state,
                            output_final_state=True,
                            use_qk_l2norm_in_kernel=True,
                            cu_seqlens=prefill_query_start_loc,
                            safe_gate=safe_gate,
                            lower_bound=lower_bound,
                            checkpoint_state=checkpoint_state,
                            checkpoint_offsets=checkpoint_offsets,
                        )
                    )
                    if checkpoint is not None:
                        assert self._checkpoint_exporter is not None
                        self._checkpoint_exporter.export(
                            checkpoint,
                            raw_qkv=raw_qkv_ns,
                            conv_state=conv_state,
                            recurrent_checkpoint=checkpoint_state,
                            recurrent_state=recurrent_state,
                            cu_seqlens=prefill_query_start_loc,
                        )
                # Init cache
                scatter_states(
                    recurrent_state, last_recurrent_state, prefill_state_indices
                )
        elif attn_metadata_narrowed.num_decodes > 0:
            assert non_spec_query_start_loc is not None
            assert non_spec_state_indices_tensor is not None
            # Plain decode step (no spec tokens): token order is dense, so the
            # kernel can write straight into the layer output buffer. A mixed
            # step scatters non-spec output via non_spec_token_indx instead.
            # Gate computed in-kernel (COMPUTE_GATE), beta sigmoided in-kernel.
            if not use_spec:
                ns_out = spec_out
            core_attn_out_non_spec, _ = fused_recurrent_kda(
                q=_rearr(q_ns),
                k=_rearr(k_ns),
                v=_rearr(v_ns),
                g=g1_ns,
                beta=beta_ns,
                initial_state=recurrent_state,
                use_qk_l2norm_in_kernel=True,
                cu_seqlens=non_spec_query_start_loc[
                    : attn_metadata_narrowed.num_decodes + 1
                ],
                ssm_state_indices=non_spec_state_indices_tensor,
                out=ns_out,
                sigmoid_beta=True,
                a_log=self.A_log,
                g_bias=self.dt_bias,
                compute_gate=True,
                lower_bound=lower_bound,
            )

        # --- merge spec / non-spec outputs back into token order ---
        if use_spec and core_attn_out_non_spec is not None:
            assert core_attn_out_spec is not None
            core_attn_out.index_copy_(1, spec_token_indx, core_attn_out_spec)
            core_attn_out.index_copy_(1, non_spec_token_indx, core_attn_out_non_spec)
        elif use_spec:
            assert core_attn_out_spec is not None
            if spec_out is None:
                core_attn_out[0, :num_actual_tokens] = core_attn_out_spec.squeeze(0)
        else:
            assert core_attn_out_non_spec is not None
            if ns_out is None:
                core_attn_out[0, num_fused_decodes:num_actual_tokens] = (
                    core_attn_out_non_spec[0, : num_actual_tokens - num_fused_decodes]
                )
        if self._norm_in_forward:
            # Rows from the fused decode kernel are normed already.
            live_out = core_attn_out[:, num_fused_decodes:num_actual_tokens]
            normed = self.o_norm(live_out, g2[num_fused_decodes:num_actual_tokens])
            if normed.data_ptr() != live_out.data_ptr():
                live_out.copy_(normed)

_norm_in_forward property

FlashInfer's fused decode applies o_norm itself, so with it _forward returns normed output for every step type: a breakable CUDA graph replays the code after _forward as captured, whatever the live step.

_flashinfer_decode(x, conv_state, raw_gate, raw_beta, state_indices, state, output_gate, output, query_start_loc=None, num_accepted_tokens=None)

FlashInfer fused KDA decode: conv + SiLU with the conv-state update, the recurrence and the gated RMSNorm, written into output. Rows with a non-positive slot are null rows. With query_start_loc the rows are draft-verify rows (state_indices [N, 1 + num_spec]).

Source code in vllm/models/glm5next/common/kda.py
def _flashinfer_decode(
    self,
    x: torch.Tensor,
    conv_state: torch.Tensor,
    raw_gate: torch.Tensor,
    raw_beta: torch.Tensor,
    state_indices: torch.Tensor,
    state: torch.Tensor,
    output_gate: torch.Tensor,
    output: torch.Tensor,
    query_start_loc: torch.Tensor | None = None,
    num_accepted_tokens: torch.Tensor | None = None,
) -> None:
    """FlashInfer fused KDA decode: conv + SiLU with the conv-state update,
    the recurrence and the gated RMSNorm, written into `output`. Rows
    with a non-positive slot are null rows. With `query_start_loc` the
    rows are draft-verify rows (`state_indices` [N, 1 + num_spec])."""
    assert self._fi_conv_weight is not None
    assert self._fi_norm_weight is not None
    kwargs = dict(
        x=x,
        weight=self._fi_conv_weight,
        conv_state=conv_state,
        raw_gate=raw_gate,
        raw_beta=raw_beta,
        A_log=self.A_log.view(-1),
        dt_bias=self.dt_bias,
        state=state,
        output_gate=output_gate,
        norm_weight=self._fi_norm_weight,
        lower_bound=self.kda_lower_bound,
        norm_eps=self.o_norm.eps,
        output=output,
    )
    if query_start_loc is None:
        flashinfer_fused_kda_decode(state_indices=state_indices, **kwargs)
    else:
        flashinfer_packed_fused_kda_decode(
            state_indices=state_indices.contiguous(),
            query_start_loc=query_start_loc,
            num_accepted_tokens=num_accepted_tokens,
            **kwargs,
        )

_flashinfer_decode_rows(metadata, qkv, g1, beta, g2, out, conv_state, state)

Runs the step's leading decode rows through FlashInfer's fused decode and returns their token count (0 if it runs none). Without speculative decoding these are the decode rows of decode and mixed steps (vLLM orders decodes first); with it, every row of a step of draft-verify rows only.

Source code in vllm/models/glm5next/common/kda.py
def _flashinfer_decode_rows(
    self,
    metadata: GDNAttentionMetadata,
    qkv: torch.Tensor,
    g1: torch.Tensor,
    beta: torch.Tensor,
    g2: torch.Tensor,
    out: torch.Tensor,
    conv_state: torch.Tensor,
    state: torch.Tensor,
) -> int:
    """Runs the step's leading decode rows through FlashInfer's fused
    decode and returns their token count (0 if it runs none). Without
    speculative decoding these are the decode rows of decode and mixed
    steps (vLLM orders decodes first); with it, every row of a step of
    draft-verify rows only."""
    if self.kda_decode_backend != "flashinfer":
        return 0
    if self.num_spec:
        if (
            metadata.spec_sequence_masks is None
            or metadata.num_spec_decodes == 0
            or metadata.num_decodes
            or metadata.num_prefills
        ):
            return 0
        assert metadata.spec_state_indices_tensor is not None
        assert metadata.spec_query_start_loc is not None
        n = metadata.num_actual_tokens
        self._flashinfer_decode(
            x=qkv,
            conv_state=conv_state,
            raw_gate=g1,
            raw_beta=beta,
            state_indices=metadata.spec_state_indices_tensor,
            state=state,
            output_gate=g2[:n],
            output=out[:, :n],
            query_start_loc=metadata.spec_query_start_loc,
            num_accepted_tokens=metadata.num_accepted_tokens,
        )
        return n
    n = metadata.num_decode_tokens
    # A mixed step with prefix-cache checkpoints runs on the chunked path.
    if n == 0 or (metadata.num_prefills and metadata.checkpoint is not None):
        return 0
    assert metadata.non_spec_state_indices_tensor is not None
    self._flashinfer_decode(
        x=qkv[:n],
        conv_state=conv_state,
        raw_gate=g1[:, :n],
        raw_beta=beta[:, :n],
        state_indices=metadata.non_spec_state_indices_tensor[:n],
        state=state,
        output_gate=g2[:n],
        output=out[:, :n],
    )
    return n

_flashinfer_prefill(q, k, v, g, beta, state, state_indices, has_initial_state, cu_seqlens, metadata, out)

FlashInfer's CuTe DSL chunked prefill from the raw gate and beta logits, reading and writing the recurrent states in place at state_indices. Writes out (a new buffer when None) and returns it.

Source code in vllm/models/glm5next/common/kda.py
def _flashinfer_prefill(
    self,
    q: torch.Tensor,
    k: torch.Tensor,
    v: torch.Tensor,
    g: torch.Tensor,
    beta: torch.Tensor,
    state: torch.Tensor,
    state_indices: torch.Tensor,
    has_initial_state: torch.Tensor,
    cu_seqlens: torch.Tensor,
    metadata: GDNAttentionMetadata,
    out: torch.Tensor | None,
) -> torch.Tensor:
    """FlashInfer's CuTe DSL chunked prefill from the raw gate and beta
    logits, reading and writing the recurrent states in place at
    `state_indices`. Writes `out` (a new buffer when `None`) and returns
    it."""
    # FlashInfer has no has-initial-state flag: zero the slots of fresh
    # sequences first (the others zero the unused null slot).
    state.index_fill_(
        0, torch.where(has_initial_state, NULL_BLOCK_ID, state_indices).long(), 0
    )
    if out is None:
        out = torch.empty_like(q)
    # With speculative decoding the slots are column 0 of the
    # [N, 1 + num_spec] block table, which FlashInfer rejects.
    state_indices = state_indices.contiguous()
    if q.size(1) == 1:
        # FlashInfer's prefill needs two tokens; one is a decode step.
        fused_recurrent_kda(
            q=q,
            k=k,
            v=v,
            g=g,
            beta=beta,
            initial_state=state,
            use_qk_l2norm_in_kernel=True,
            cu_seqlens=cu_seqlens,
            ssm_state_indices=state_indices,
            out=out,
            sigmoid_beta=True,
            a_log=self.A_log,
            g_bias=self.dt_bias,
            compute_gate=True,
            lower_bound=self.kda_lower_bound,
        )
        return out
    assert self.kda_lower_bound is not None
    flashinfer_recurrent_kda(
        q=q.contiguous(),
        k=k.contiguous(),
        v=v.contiguous(),
        g=g.contiguous(),
        beta=beta.contiguous(),
        A_log=self.A_log.view(-1),
        dt_bias=self.dt_bias,
        scale=q.shape[-1] ** -0.5,
        initial_state=state,
        output_final_state=True,
        use_qk_l2norm_in_kernel=True,
        use_gate_in_kernel=True,
        lower_bound=self.kda_lower_bound,
        cu_seqlens=_int64_cu_seqlens(metadata, cu_seqlens),
        ssm_state_indices=state_indices,
        output=out,
        beta_is_logit=True,
        backend="cute-dsl",
    )
    return out

_flashkda_prefill(q, k, v, g, beta, initial_state, cu_seqlens, out, checkpoint=None, raw_qkv=None, conv_state=None, recurrent_state=None)

Fused KDA chunked prefill (FlashKDA). Takes the raw gate logits g and raw beta logits, l2-normalizes q/k in-kernel and applies the bounded gate lower_bound * sigmoid(exp(A_log) * (g + dt_bias)), matching chunk_kda_with_fused_gate(..., safe_gate=True). Writes the attention output into out (a workspace buffer when None) and returns (out, final_state).

Source code in vllm/models/glm5next/common/kda.py
def _flashkda_prefill(
    self,
    q: torch.Tensor,
    k: torch.Tensor,
    v: torch.Tensor,
    g: torch.Tensor,
    beta: torch.Tensor,
    initial_state: torch.Tensor,
    cu_seqlens: torch.Tensor,
    out: torch.Tensor | None,
    checkpoint: MambaPrefillCheckpointMetadata | None = None,
    raw_qkv: torch.Tensor | None = None,
    conv_state: torch.Tensor | None = None,
    recurrent_state: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
    """Fused KDA chunked prefill (FlashKDA). Takes the raw gate logits `g`
    and raw `beta` logits, l2-normalizes q/k in-kernel and applies the
    bounded gate `lower_bound * sigmoid(exp(A_log) * (g + dt_bias))`,
    matching `chunk_kda_with_fused_gate(..., safe_gate=True)`. Writes the
    attention output into `out` (a workspace buffer when `None`) and
    returns `(out, final_state)`."""
    assert self._flashkda_buffer_specs is not None
    final_state, checkpoint_state, workspace, workspace_out = (
        current_workspace_manager().get_simultaneous(*self._flashkda_buffer_specs)
    )
    final_state = final_state[: initial_state.shape[0]]
    checkpoint_state = (
        checkpoint_state[: initial_state.shape[0]]
        if checkpoint is not None
        else None
    )
    if out is None:
        out = workspace_out[:, : q.shape[1]]
    # FlashKDA accepts row-strided q/k/v (contiguous H/D rows, aligned
    # token stride), so the packed q|k|v views are passed without a copy.
    torch.ops._flashkda_C.fwd(
        q,
        k,
        v,
        g.contiguous(),
        beta,
        self.head_dim**-0.5,
        out,
        workspace,
        self.A_log.view(-1),
        self.dt_bias.view(-1, self.head_dim),
        self.kda_lower_bound,
        initial_state.contiguous(),
        final_state,
        cu_seqlens.contiguous(),
        checkpoint_state,
        checkpoint.checkpoint_offsets.contiguous()
        if checkpoint is not None
        else None,
    )
    if checkpoint is not None:
        assert self._checkpoint_exporter is not None
        assert raw_qkv is not None and conv_state is not None
        assert recurrent_state is not None
        self._checkpoint_exporter.export(
            checkpoint,
            raw_qkv=raw_qkv,
            conv_state=conv_state,
            recurrent_checkpoint=checkpoint_state,
            recurrent_state=recurrent_state,
            cu_seqlens=cu_seqlens,
        )
    return out, final_state

process_weights_after_loading()

Build FlashInfer's conv and norm weights from the loaded weights; reloads refill them in place.

Source code in vllm/models/glm5next/common/kda.py
def process_weights_after_loading(self) -> None:
    """Build FlashInfer's conv and norm weights from the loaded weights;
    reloads refill them in place."""
    if not self._norm_in_forward:
        return
    conv_weight = self.conv1d.weight.view(
        3, self.local_projection_size, self.conv_size
    )
    refill_derived(
        self, "_fi_conv_weight", conv_weight.transpose(1, 2).contiguous()
    )
    refill_derived(self, "_fi_norm_weight", self.o_norm.weight.float().contiguous())

_Glm5NextMergedColumnParallelLinear

Bases: MergedColumnParallelLinear

Merged projection with multiple replicated output shards.

Extends K3's _KimiGDNMergedColumnParallelLinear to support two replicated shards (f_a, g_a) instead of one. Pre-multiplies each replicated entry's output_size by tp_size so the per-rank shard divides back to the full size, and forces tp_rank=0 during weight loading for replicated shards.

Source code in vllm/models/glm5next/common/kda.py
class _Glm5NextMergedColumnParallelLinear(MergedColumnParallelLinear):
    """Merged projection with multiple replicated output shards.

    Extends K3's `_KimiGDNMergedColumnParallelLinear` to support two
    replicated shards (f_a, g_a) instead of one. Pre-multiplies each
    replicated entry's output_size by tp_size so the per-rank shard
    divides back to the full size, and forces tp_rank=0 during weight
    loading for replicated shards.
    """

    def __init__(
        self,
        input_size: int,
        output_sizes: list[int],
        replicated_shard_ids: tuple[int, ...],
        tp_size: int,
        **kwargs,
    ) -> None:
        self.replicated_shard_ids = set(replicated_shard_ids)
        output_sizes = output_sizes.copy()
        for sid in self.replicated_shard_ids:
            output_sizes[sid] *= tp_size
        super().__init__(input_size, output_sizes, **kwargs)

    def weight_loader(
        self,
        param: nn.Parameter,
        loaded_weight: torch.Tensor,
        loaded_shard_id: tuple[int, ...] | int | None = None,
    ) -> None:
        tp_rank = self.tp_rank
        param_tp_rank = getattr(param, "tp_rank", None)
        if loaded_shard_id in self.replicated_shard_ids:
            self.tp_rank = 0
            if param_tp_rank is not None:
                param.tp_rank = 0
        try:
            super().weight_loader(param, loaded_weight, loaded_shard_id)
        finally:
            self.tp_rank = tp_rank
            if param_tp_rank is not None:
                param.tp_rank = param_tp_rank

    def weight_loader_v2(
        self,
        param: nn.Parameter,
        loaded_weight: torch.Tensor,
        loaded_shard_id: tuple[int, ...] | int | None = None,
    ) -> None:
        tp_rank = self.tp_rank
        param_tp_rank = getattr(param, "tp_rank", None)
        if loaded_shard_id in self.replicated_shard_ids:
            self.tp_rank = 0
            if param_tp_rank is not None:
                param.tp_rank = 0
        try:
            super().weight_loader_v2(param, loaded_weight, loaded_shard_id)
        finally:
            self.tp_rank = tp_rank
            if param_tp_rank is not None:
                param.tp_rank = param_tp_rank

_cast_sigmoid(x)

Fuse the fp32 cast + sigmoid into one Inductor kernel.

Source code in vllm/models/glm5next/common/kda.py
@torch.compile(
    dynamic=True,
    backend=current_platform.simple_compile_backend,
    options=maybe_disable_graph_partition(current_platform.simple_compile_backend),
)
def _cast_sigmoid(x: torch.Tensor) -> torch.Tensor:
    """Fuse the fp32 cast + sigmoid into one Inductor kernel."""
    return x.float().sigmoid()

_fused_qkv_conv1d(conv_size, projection_size, tp_size, tp_rank, prefix)

The q|k|v short convs as one [3 * C_local, 1, K] parameter.

Checkpoint q/k/v_conv1d weights load as shards 0/1/2 (see stacked_params_mapping). The conv kernels read this parameter directly, so every weight write (load, refit, offload, IPC cache) lands in it and its address stays fixed for cudagraphs (#55087).

Source code in vllm/models/glm5next/common/kda.py
def _fused_qkv_conv1d(
    conv_size: int, projection_size: int, tp_size: int, tp_rank: int, prefix: str
) -> ColumnParallelLinear:
    """The q|k|v short convs as one `[3 * C_local, 1, K]` parameter.

    Checkpoint `q/k/v_conv1d` weights load as shards 0/1/2 (see
    `stacked_params_mapping`). The conv kernels read this parameter directly,
    so every weight write (load, refit, offload, IPC cache) lands in it and its
    address stays fixed for cudagraphs (#55087).
    """
    conv1d = ColumnParallelLinear(
        input_size=conv_size,
        output_size=3 * projection_size,
        bias=False,
        params_dtype=torch.float32,
        prefix=prefix,
    )
    conv1d.weight.data = conv1d.weight.data.unsqueeze(1)
    delattr(conv1d.weight, "weight_loader")
    set_weight_attrs(
        conv1d.weight,
        {
            "weight_loader": _make_fused_conv1d_weight_loader(
                [projection_size] * 3, tp_size, tp_rank
            )
        },
    )
    return conv1d

_int64_cu_seqlens(metadata, cu_seqlens)

cu_seqlens as int64, converted once per step and shared by every KDA layer of the step (cached on the step's metadata). FlashInfer's CuTe DSL prefill would otherwise convert int32 offsets to a new tensor in each layer, and its few-sequence route reads the offsets to the host, memoized by tensor identity: a shared tensor makes that one blocking read per step instead of one per layer.

Source code in vllm/models/glm5next/common/kda.py
def _int64_cu_seqlens(
    metadata: GDNAttentionMetadata, cu_seqlens: torch.Tensor
) -> torch.Tensor:
    """`cu_seqlens` as int64, converted once per step and shared by every KDA
    layer of the step (cached on the step's metadata). FlashInfer's CuTe DSL
    prefill would otherwise convert int32 offsets to a new tensor in each
    layer, and its few-sequence route reads the offsets to the host, memoized
    by tensor identity: a shared tensor makes that one blocking read per step
    instead of one per layer."""
    cached = getattr(metadata, "_kda_int64_cu_seqlens", None)
    if cached is None or cached[0] is not cu_seqlens:
        cached = (cu_seqlens, cu_seqlens.to(torch.int64))
        metadata._kda_int64_cu_seqlens = cached  # type: ignore[attr-defined]
    return cached[1]

_is_flashinfer_kda_decode_supported(num_heads, head_dim, conv_size, query_len, dtype, conv_state_dtype, state_dtype, lower_bound)

Whether FlashInfer's fused KDA decode (query_len 1) or its packed draft-verify decode (query_len 1 + num_spec) runs this layer.

Source code in vllm/models/glm5next/common/kda.py
def _is_flashinfer_kda_decode_supported(
    num_heads: int,
    head_dim: int,
    conv_size: int,
    query_len: int,
    dtype: torch.dtype,
    conv_state_dtype: torch.dtype,
    state_dtype: torch.dtype,
    lower_bound: float | None,
) -> bool:
    """Whether FlashInfer's fused KDA decode (`query_len` 1) or its packed
    draft-verify decode (`query_len` 1 + num_spec) runs this layer."""
    heads: tuple[int, ...]
    if query_len == 1:
        has_kernel = has_flashinfer_fused_kda_decode()
        heads = (8, 12, 16, 24, 32, 48, 64, 96)
    else:
        # The packed kernel supports only the SD conv-state layout.
        has_kernel = (
            has_flashinfer_packed_fused_kda_decode() and not is_conv_state_dim_first()
        )
        heads = (12, 16, 24, 32, 48, 64, 96)
    return (
        has_kernel
        and current_platform.is_cuda()
        and current_platform.is_device_capability_family(100)
        and num_heads in heads
        and head_dim == 128
        and conv_size == 4
        and 1 <= query_len <= 8
        and dtype == torch.bfloat16
        and conv_state_dtype == torch.bfloat16
        and state_dtype in (torch.float32, torch.bfloat16)
        and lower_bound is not None
        and lower_bound < 0
    )

_is_flashinfer_kda_prefill_supported(head_dim, dtype, state_dtype, lower_bound)

Whether FlashInfer's CuTe DSL chunked KDA prefill runs this layer.

Source code in vllm/models/glm5next/common/kda.py
def _is_flashinfer_kda_prefill_supported(
    head_dim: int,
    dtype: torch.dtype,
    state_dtype: torch.dtype,
    lower_bound: float | None,
) -> bool:
    """Whether FlashInfer's CuTe DSL chunked KDA prefill runs this layer."""
    if not current_platform.is_cuda() or not has_flashinfer_recurrent_kda():
        return False
    capability = current_platform.get_device_capability()
    if capability is None:
        return False
    return (
        (capability.major, capability.minor) in ((10, 0), (10, 3), (10, 7))
        and head_dim == 128
        and dtype == torch.bfloat16
        and state_dtype in (torch.float32, torch.bfloat16)
        and lower_bound is not None
        and lower_bound < 0
    )

_resolve_kda_decode_backend(backend, num_heads, head_dim, conv_size, query_len, dtype, conv_state_dtype, state_dtype, lower_bound)

Pick the decode kernels: FlashInfer's fused KDA decode (conv + SiLU, the recurrence and the gated RMSNorm in one launch per layer) or the Triton conv, recurrence and norm kernels. backend comes from additional_config.kda_decode_backend (--kda-decode-backend: auto / flashinfer / triton) and covers plain and draft-verify rows; auto picks FlashInfer when it supports the layer. query_len is 1 for plain decode rows and 1 + num_speculative_tokens for draft-verify rows.

Source code in vllm/models/glm5next/common/kda.py
def _resolve_kda_decode_backend(
    backend: str,
    num_heads: int,
    head_dim: int,
    conv_size: int,
    query_len: int,
    dtype: torch.dtype,
    conv_state_dtype: torch.dtype,
    state_dtype: torch.dtype,
    lower_bound: float | None,
) -> str:
    """Pick the decode kernels: FlashInfer's fused KDA decode (conv + SiLU, the
    recurrence and the gated RMSNorm in one launch per layer) or the Triton
    conv, recurrence and norm kernels. `backend` comes from
    `additional_config.kda_decode_backend` (`--kda-decode-backend`: auto /
    flashinfer / triton) and covers plain and draft-verify rows; `auto` picks
    FlashInfer when it supports the layer. `query_len` is 1 for plain decode
    rows and 1 + num_speculative_tokens for draft-verify rows."""
    if backend not in ("auto", "flashinfer", "triton"):
        raise ValueError(f"Unsupported GLM-5.3 KDA decode backend: {backend}")
    if backend == "triton":
        return "triton"
    if _is_flashinfer_kda_decode_supported(
        num_heads,
        head_dim,
        conv_size,
        query_len,
        dtype,
        conv_state_dtype,
        state_dtype,
        lower_bound,
    ):
        logger.info_once(
            "Using FlashInfer fused KDA decode (%d tokens per row).", query_len
        )
        return "flashinfer"
    if backend == "flashinfer":
        raise RuntimeError(
            "FlashInfer fused KDA decode requires CUDA SM10x, 8/12/16/24/32/48/"
            "64/96 local heads (12-96 and the SD conv-state layout for "
            "draft-verify rows, at most 7 speculative tokens), head_dim 128, "
            "conv width 4, bfloat16 activations and conv state, a float32 or "
            "bfloat16 recurrent state and a bounded gate."
        )
    return "triton"

_resolve_kda_prefill_backend(backend, head_dim, dtype, lower_bound, state_dtype=torch.float32)

Pick the chunked-prefill kernel: FlashKDA (fused CUDA, ~2-4x faster on SM90/SM10x/SM12x for bf16, head_dim 128 and a bounded gate), FlashInfer's CuTe DSL kernel (opt-in; updates the state pool in place) or the Triton chunk_kda_with_fused_gate path. backend comes from additional_config.kda_prefill_backend (auto / triton / flashkda / flashinfer).

Source code in vllm/models/glm5next/common/kda.py
def _resolve_kda_prefill_backend(
    backend: str,
    head_dim: int,
    dtype: torch.dtype,
    lower_bound: float | None,
    state_dtype: torch.dtype = torch.float32,
) -> str:
    """Pick the chunked-prefill kernel: FlashKDA (fused CUDA, ~2-4x faster on
    SM90/SM10x/SM12x for bf16, head_dim 128 and a bounded gate), FlashInfer's
    CuTe DSL kernel (opt-in; updates the state pool in place) or the Triton
    `chunk_kda_with_fused_gate` path. `backend` comes from
    `additional_config.kda_prefill_backend` (auto / triton / flashkda /
    flashinfer)."""
    if backend not in ("auto", "triton", "flashkda", "flashinfer"):
        raise ValueError(f"Unsupported KDA prefill backend: {backend}")
    if backend == "flashinfer":
        if not _is_flashinfer_kda_prefill_supported(
            head_dim, dtype, state_dtype, lower_bound
        ):
            raise RuntimeError(
                "FlashInfer KDA prefill requires CUDA SM100/SM103/SM107, "
                "bfloat16 activations, a float32 or bfloat16 recurrent state, "
                "head_dim=128 and a bounded KDA gate."
            )
        logger.info_once("Using FlashInfer KDA prefill.")
        return "flashinfer"
    capability = current_platform.get_device_capability()
    supported = (
        current_platform.is_cuda()
        and capability is not None
        and capability.major in (9, 10, 12)
        and head_dim == 128
        and dtype == torch.bfloat16
        and lower_bound is not None
    )
    if backend == "flashkda" and not supported:
        raise RuntimeError(
            "FlashKDA requires CUDA SM90/SM10x/SM12x, bfloat16, head_dim=128 "
            "and a bounded KDA gate."
        )
    return "flashkda" if supported and backend != "triton" else "triton"