Skip to content

vllm.models.inkling2.nvidia.ops.moe_sm90

Inkling2 routed experts on Hopper: Humming expert GEMMs (bf16 activations, NVFP4 / MXFP8 weights) in place of the TRT-LLM-gen ones.

Rows stay in the flat (token * topk + slot) order: the gate/up GEMM gathers each row's token, the down GEMM reads and writes row by row, and ops.moe.combine sums them with the identity src2dst (-1: not local).

Classes:

  • HummingExperts –

    Humming tensors and kernel configs of the two expert projections.

Functions:

  • select_experts –

    The router and top-k of ops.moe.route, without its layout.

  • swiglu –

    activation.swiglu_quantize's SiTU of the interleaved (gate, up)

HummingExperts

Humming tensors and kernel configs of the two expert projections.

Parameters:

  • weights

    (dict[str, tuple[Tensor, Tensor, Tensor | None]]) –

    Per projection (w13, w2), the row-major checkpoint weights [E, N, K / pack], block scales [E, N, K / group] and, for NVFP4, the per-expert global scales [E] (else None).

  • shapes

    (tuple[tuple[int, int], tuple[int, int]]) –

    (N, K) of each projection.

  • nvfp4

    (bool) –

    NVFP4 (else MXFP8) weights.

Methods:

  • __call__ –

    bf16 [T * topk, H] down-projection rows of the local slots of

Source code in vllm/models/inkling2/nvidia/ops/moe_sm90.py
class HummingExperts:
    """Humming tensors and kernel configs of the two expert projections.

    Args:
        weights: Per projection (`w13`, `w2`), the row-major checkpoint
            weights `[E, N, K / pack]`, block scales `[E, N, K / group]`
            and, for NVFP4, the per-expert global scales `[E]` (else None).
        shapes: `(N, K)` of each projection.
        nvfp4: NVFP4 (else MXFP8) weights.

    """

    def __init__(
        self,
        weights: dict[str, tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]],
        shapes: tuple[tuple[int, int], tuple[int, int]],
        nvfp4: bool,
    ) -> None:
        from vllm.utils.humming import (
            GemmType,
            HummingInputSchema,
            ModeloptMxfp8WeightSchema,
            ModeloptNvfp4WeightSchema,
            get_heuristics_config,
            prepare_and_transform_humming_tensors,
        )

        self.configs = {}
        self.tensors: dict[str, dict[str, torch.Tensor]] = {}
        self.tunings: dict[str, list] = {}
        self.tuning_strs: dict[str, str] = {}
        for name, (n, k) in zip(("w13", "w2"), shapes):
            weight, scale, global_scale = weights[name]
            num_experts = weight.shape[0]
            tensors = {"weight": weight, "weight_scale": scale}
            if nvfp4:
                source = ModeloptNvfp4WeightSchema()
                tensors["weight_scale_2"] = global_scale
            else:
                source = ModeloptMxfp8WeightSchema()
            schema, tensors = source.convert_humming(
                tensors=tensors,
                shape_n_stacks=[n],
                shape_k_stacks=[k],
                param_dtype=torch.bfloat16,
                num_experts=num_experts,
            )
            config, tensors = prepare_and_transform_humming_tensors(
                tensors=tensors,
                shape_n=n,
                shape_k=k,
                pad_n_to_multiple=256,
                pad_k_to_multiple=128,
                input_schema=HummingInputSchema(),
                weight_schema=schema,
                num_experts=num_experts,
                torch_dtype=torch.bfloat16,
                device=weight.device,
            )
            assert config.shape_n == n and config.shape_k == k, (n, k)
            tuning = get_heuristics_config(
                layer_config=config,
                use_f16_accum=False,
                use_batch_invariant=envs.VLLM_BATCH_INVARIANT,
                gemm_type=GemmType.INDEXED,
            )
            # Stream-K adds the K slices into the output with bf16 atomics,
            # whose order (and so the result) changes from run to run.
            for _, _, kernel_config in tuning:
                kernel_config["use_stream_k"] = False
            self.configs[name] = config
            self.tensors[name] = tensors
            self.tunings[name] = tuning
            self.tuning_strs[name] = json.dumps(tuning)
        self.num_experts = num_experts
        self.compute_config = json.dumps(
            {
                "use_batch_invariant": envs.VLLM_BATCH_INVARIANT,
                "use_f16_accum": False,
                "gemm_type": GemmType.INDEXED.value,
            }
        )
        self.locks = torch.zeros(2048, dtype=torch.int32, device=weight.device)

    def _block_m(self, name: str, rows: int) -> int:
        for lower, upper, config in self.tunings[name]:
            if lower < rows <= upper:
                return config["block_shape"][0]
        return _DEFAULT_BLOCK_M

    def _gemm(
        self,
        name: str,
        inputs: torch.Tensor,
        rows: int,
        alignment: tuple[torch.Tensor, torch.Tensor, torch.Tensor],
        top_k: int,
        valid_rows: int,
    ) -> torch.Tensor:
        from vllm.utils.humming import humming_forward

        config = self.configs[name]
        tensors = self.tensors[name]
        out = torch.empty(
            (rows, config.shape_n), dtype=torch.bfloat16, device=inputs.device
        )
        sorted_ids, expert_ids, num_tokens_padded = alignment
        return humming_forward(
            config,
            inputs=inputs,
            weight=tensors["weight"],
            weight_scale=tensors.get("weight_scale"),
            weight_scale_2=tensors.get("weight_scale_2"),
            outputs=out,
            locks=self.locks,
            sorted_ids=sorted_ids,
            expert_ids=expert_ids,
            num_tokens_padded=num_tokens_padded,
            top_k=top_k,
            valid_shape_m=valid_rows,
            compute_config=self.compute_config,
            tuning_config=self.tuning_strs[name],
        )

    def __call__(
        self,
        x: torch.Tensor,
        ids: torch.Tensor,
        weights: torch.Tensor,
        src2dst: torch.Tensor,
        num_global_experts: int,
        gate_cap: float,
        up_cap: float,
    ) -> torch.Tensor:
        """bf16 `[T * topk, H]` down-projection rows of the local slots of
        `ids` (`select_experts`); the other rows are left unset."""
        tokens, topk = ids.shape
        rows = tokens * topk
        # Expected local rows, which pick the kernel configs.
        valid_rows = -(-rows * self.num_experts // num_global_experts)
        block_m = {name: self._block_m(name, valid_rows) for name in ("w13", "w2")}
        alignments = {
            size: moe_align_block_size(
                ids, size, self.num_experts, ignore_invalid_experts=True
            )[:3]
            for size in set(block_m.values())
        }
        gate_up = self._gemm(
            "w13", x, rows, alignments[block_m["w13"]], topk, valid_rows
        )
        activated = swiglu(gate_up, weights, src2dst, gate_cap, up_cap)
        del gate_up
        return self._gemm(
            "w2", activated, rows, alignments[block_m["w2"]], 1, valid_rows
        )

__call__(x, ids, weights, src2dst, num_global_experts, gate_cap, up_cap)

bf16 [T * topk, H] down-projection rows of the local slots of ids (select_experts); the other rows are left unset.

Source code in vllm/models/inkling2/nvidia/ops/moe_sm90.py
def __call__(
    self,
    x: torch.Tensor,
    ids: torch.Tensor,
    weights: torch.Tensor,
    src2dst: torch.Tensor,
    num_global_experts: int,
    gate_cap: float,
    up_cap: float,
) -> torch.Tensor:
    """bf16 `[T * topk, H]` down-projection rows of the local slots of
    `ids` (`select_experts`); the other rows are left unset."""
    tokens, topk = ids.shape
    rows = tokens * topk
    # Expected local rows, which pick the kernel configs.
    valid_rows = -(-rows * self.num_experts // num_global_experts)
    block_m = {name: self._block_m(name, valid_rows) for name in ("w13", "w2")}
    alignments = {
        size: moe_align_block_size(
            ids, size, self.num_experts, ignore_invalid_experts=True
        )[:3]
        for size in set(block_m.values())
    }
    gate_up = self._gemm(
        "w13", x, rows, alignments[block_m["w13"]], topk, valid_rows
    )
    activated = swiglu(gate_up, weights, src2dst, gate_cap, up_cap)
    del gate_up
    return self._gemm(
        "w2", activated, rows, alignments[block_m["w2"]], 1, valid_rows
    )

_select_kernel(Partials, Bias, ExpertBounds, IsPadding, Ids, Weights, Src2Dst, T, route_scale, E, E_POW2, GROUPS, TOPK, SORT)

Local expert ids (-1: not local), routing weights and identity src2dst of the flat rows of one token.

Source code in vllm/models/inkling2/nvidia/ops/moe_sm90.py
@triton.jit
def _select_kernel(
    Partials,
    Bias,
    ExpertBounds,
    IsPadding,
    Ids,
    Weights,
    Src2Dst,
    T,
    route_scale,
    E: tl.constexpr,
    E_POW2: tl.constexpr,
    GROUPS: tl.constexpr,
    TOPK: tl.constexpr,
    SORT: tl.constexpr,
):
    """Local expert ids (-1: not local), routing weights and identity
    `src2dst` of the flat rows of one token."""
    token = tl.program_id(0) + tl.arange(0, 1)
    bias = _load_bias(Bias, E, E_POW2)
    ids, weights = _select_experts(
        Partials, bias, token, T, route_scale, E, E_POW2, GROUPS, TOPK, SORT
    )
    padding = None
    if IsPadding is not None:
        padding = tl.load(IsPadding + token)
    local = _local_experts(
        ids, tl.load(ExpertBounds), tl.load(ExpertBounds + 1), padding
    )
    rows = token[:, None] * TOPK + tl.arange(0, TOPK)[None, :]
    tl.store(Ids + rows, local)
    tl.store(Weights + rows, weights)
    tl.store(Src2Dst + rows, tl.where(local >= 0, rows, -1))

_swiglu_kernel(GateUp, Weights, Src2Dst, Out, N, BLOCK, GATE_CAP, UP_CAP)

bf16 softcapped SwiGLU of a local row times its routing weight.

Source code in vllm/models/inkling2/nvidia/ops/moe_sm90.py
@triton.jit
def _swiglu_kernel(
    GateUp,
    Weights,
    Src2Dst,
    Out,
    N: tl.constexpr,
    BLOCK: tl.constexpr,
    GATE_CAP: tl.constexpr,
    UP_CAP: tl.constexpr,
):
    """bf16 softcapped SwiGLU of a local row times its routing weight."""
    row = tl.program_id(0)
    if tl.load(Src2Dst + row) < 0:
        return
    start = tl.program_id(1) * BLOCK
    gamma = tl.load(Weights + row)
    values = _situ(GateUp, row, start, gamma, N, BLOCK, GATE_CAP, UP_CAP)
    tl.store(Out + row * N + start + tl.arange(0, BLOCK), values.to(tl.bfloat16))

select_experts(x, gate_weight, bias, topk, route_scale, expert_bounds, is_padding=None)

The router and top-k of ops.moe.route, without its layout.

Returns the local expert id of each token's slots [T, topk] (-1: not one of the local experts expert_bounds, or a padding token), their routing weights [T * topk] and the identity src2dst [T * topk] of the local rows (-1 elsewhere).

Source code in vllm/models/inkling2/nvidia/ops/moe_sm90.py
def select_experts(
    x: torch.Tensor,
    gate_weight: torch.Tensor,
    bias: torch.Tensor,
    topk: int,
    route_scale: float,
    expert_bounds: torch.Tensor,
    is_padding: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
    """The router and top-k of `ops.moe.route`, without its layout.

    Returns the local expert id of each token's slots `[T, topk]` (-1: not
    one of the local experts `expert_bounds`, or a padding token), their
    routing weights `[T * topk]` and the identity `src2dst` `[T * topk]`
    of the local rows (-1 elsewhere).
    """
    tokens = x.shape[0]
    experts = gate_weight.shape[0]
    partials = k_group_partials(x, gate_weight)
    ids = torch.empty((tokens, topk), dtype=torch.int32, device=x.device)
    weights = torch.empty(tokens * topk, dtype=torch.float32, device=x.device)
    src2dst = torch.empty(tokens * topk, dtype=torch.int32, device=x.device)
    _select_kernel[(tokens,)](
        partials,
        bias,
        expert_bounds,
        is_padding,
        ids,
        weights,
        src2dst,
        tokens,
        route_scale,
        E=experts,
        E_POW2=triton.next_power_of_2(experts),
        GROUPS=partials.shape[0],
        TOPK=topk,
        SORT=tokens <= 128,
        num_warps=4 if tokens <= 256 else 1,
    )
    return ids, weights, src2dst

swiglu(gate_up, weights, src2dst, gate_cap, up_cap)

activation.swiglu_quantize's SiTU of the interleaved (gate, up) columns times the routing weights, in bf16, for the local rows (the others are left unset).

Source code in vllm/models/inkling2/nvidia/ops/moe_sm90.py
def swiglu(
    gate_up: torch.Tensor,
    weights: torch.Tensor,
    src2dst: torch.Tensor,
    gate_cap: float,
    up_cap: float,
) -> torch.Tensor:
    """`activation.swiglu_quantize`'s SiTU of the interleaved (gate, up)
    columns times the routing weights, in bf16, for the local rows (the
    others are left unset)."""
    rows, n = gate_up.shape[0], gate_up.shape[1] // 2
    block = 256 if n % 256 == 0 else 128
    assert n % block == 0
    out = torch.empty((rows, n), dtype=torch.bfloat16, device=gate_up.device)
    _swiglu_kernel[(rows, n // block)](
        gate_up,
        weights,
        src2dst,
        out,
        N=n,
        BLOCK=block,
        GATE_CAP=gate_cap,
        UP_CAP=up_cap,
        num_warps=2,
    )
    return out