Skip to content

llmcompressor.utils.triton_utils

Low-level Triton helpers shared by observer kernels.

Classes:

  • TritonDType –

    Shared Triton dtype codes; FP8 and E8M0 are used for scales only.

Functions:

TritonDType

Bases: IntEnum

Shared Triton dtype codes; FP8 and E8M0 are used for scales only.

calculate_candidate_scale

calculate_candidate_scale(
    min_value,
    max_value,
    q_min,
    q_max,
    SCALE_ROUND_TYPE: constexpr,
    QUANT_TYPE: constexpr,
    SYMMETRIC: constexpr,
    COMPUTE_DTYPE: constexpr,
)

Derive and round a candidate scale from candidate min/max bounds.

Source code in src/llmcompressor/utils/triton_utils.py
@triton.jit
def calculate_candidate_scale(
    min_value,
    max_value,
    q_min,
    q_max,
    SCALE_ROUND_TYPE: tl.constexpr,
    QUANT_TYPE: tl.constexpr,
    SYMMETRIC: tl.constexpr,
    COMPUTE_DTYPE: tl.constexpr,
):
    """Derive and round a candidate scale from candidate min/max bounds."""
    if SCALE_ROUND_TYPE == _TRITON_E8M0:
        # MX/E8M0 operates on the group absmax before element-format scaling.
        return round_candidate_scale(
            tl.maximum(tl.abs(min_value), tl.abs(max_value)),
            SCALE_ROUND_TYPE=SCALE_ROUND_TYPE,
            QUANT_TYPE=QUANT_TYPE,
        )
    if SYMMETRIC:
        scale = tl.maximum(tl.abs(min_value), tl.abs(max_value)) / (
            (q_max - q_min) * 0.5
        )
    else:
        scale = (max_value - min_value) / (q_max - q_min)
    scale = round_candidate_scale(
        scale, SCALE_ROUND_TYPE=SCALE_ROUND_TYPE, QUANT_TYPE=QUANT_TYPE
    )
    if SCALE_ROUND_TYPE == _TRITON_FP32:
        return round_fp32_as_dtype(scale, COMPUTE_DTYPE)
    return scale

calculate_observer_error

calculate_observer_error(
    values,
    quantized,
    importance,
    group_absmax,
    norm: constexpr,
    HAS_IMPORTANCE: constexpr,
    HAS_MAGNITUDE_NORMALIZATION: constexpr,
    COMPUTE_DTYPE: constexpr,
)

Calculate MSE or importance-weighted QDQ error for observer search.

Source code in src/llmcompressor/utils/triton_utils.py
@triton.jit
def calculate_observer_error(
    values,
    quantized,
    importance,
    group_absmax,
    norm: tl.constexpr,
    HAS_IMPORTANCE: tl.constexpr,
    HAS_MAGNITUDE_NORMALIZATION: tl.constexpr,
    COMPUTE_DTYPE: tl.constexpr,
):
    """Calculate MSE or importance-weighted QDQ error for observer search."""
    error = calculate_qdq_error(
        values,
        quantized,
        norm,
        COMPUTE_DTYPE=COMPUTE_DTYPE,
    )
    if HAS_IMPORTANCE:
        error = error * importance
    if HAS_MAGNITUDE_NORMALIZATION:
        error = error * tl.sqrt(values * values + group_absmax * group_absmax)
    return error

calculate_qdq_error

calculate_qdq_error(
    values,
    quantized,
    norm: constexpr,
    COMPUTE_DTYPE: constexpr,
)

Compute QDQ powers in FP32 after rounding the difference to input dtype.

Source code in src/llmcompressor/utils/triton_utils.py
@triton.jit
def calculate_qdq_error(
    values,
    quantized,
    norm: tl.constexpr,
    COMPUTE_DTYPE: tl.constexpr,
):
    """Compute QDQ powers in FP32 after rounding the difference to input dtype."""
    # Eager fake_quantize rounds its QDQ result and the observed values in their
    # storage dtype before subtraction, then rounds the difference again. Keep
    # that order here so the selected grid point matches eager.
    quantized = round_fp32_as_dtype(quantized, COMPUTE_DTYPE)
    values = round_fp32_as_dtype(values, COMPUTE_DTYPE)
    diff = round_fp32_as_dtype(tl.abs(quantized - values), COMPUTE_DTYPE)
    error_norm = round_fp32_as_dtype(tl.full((), norm, tl.float32), COMPUTE_DTYPE)
    return tldevice.pow(diff, error_norm)

is_xpu_target

is_xpu_target()

Report a non-XPU target when Triton has no constexpr functions.

Source code in src/llmcompressor/utils/triton_utils.py
@triton.jit
def is_xpu_target():
    """Report a non-XPU target when Triton has no constexpr functions."""
    return False

observer_triton_req

observer_triton_req(
    observed, args, *unused_args, **unused_kwargs
) -> bool

Check whether observer inputs use a format supported by the shared kernels.

Source code in src/llmcompressor/utils/triton_utils.py
def observer_triton_req(observed, args, *unused_args, **unused_kwargs) -> bool:
    """Check whether observer inputs use a format supported by the shared kernels."""
    if not triton_req(observed):
        return False
    if observed.dtype not in (torch.float32, torch.float16, torch.bfloat16):
        return False
    if args.type == QuantizationType.INT:
        return args.num_bits <= 8
    if args.type == QuantizationType.FLOAT:
        return args.num_bits in (4, 8)
    return False

quantize_dequantize

quantize_dequantize(
    values,
    scale,
    zero_point,
    q_min,
    q_max,
    QUANT_TYPE: constexpr,
    NUM_BITS: constexpr,
    HAS_ZP: constexpr,
    COMPUTE_DTYPE: constexpr,
    USE_FP8_E4B15: constexpr = False,
)

GPTQ-style QDQ shared by MSE and importance-weighted observer kernels.

Source code in src/llmcompressor/utils/triton_utils.py
@triton.jit
def quantize_dequantize(
    values,
    scale,
    zero_point,
    q_min,
    q_max,
    QUANT_TYPE: tl.constexpr,
    NUM_BITS: tl.constexpr,
    HAS_ZP: tl.constexpr,
    COMPUTE_DTYPE: tl.constexpr,
    USE_FP8_E4B15: tl.constexpr = False,
):
    """GPTQ-style QDQ shared by MSE and importance-weighted observer kernels."""
    normalized = tldevice.div_rn(values, scale)
    normalized = round_fp32_as_dtype(normalized, COMPUTE_DTYPE)
    if HAS_ZP:
        # fake_quantize adds the zero point in the input dtype
        normalized += zero_point
        normalized = round_fp32_as_dtype(normalized, COMPUTE_DTYPE)
    normalized = tl.clamp(normalized, q_min, q_max)
    if QUANT_TYPE == 0:
        rounded = tldevice.rint(normalized)
    elif NUM_BITS == 4:
        rounded = _round_to_fp4(normalized)
    else:
        if USE_FP8_E4B15:
            rounded = (normalized * 0.00390625).to(tl.float8e4b15).to(
                tl.float32
            ) * 256.0
        else:
            rounded = round_fp32_as_dtype(normalized, _TRITON_FP8_E4M3)
    if HAS_ZP:
        rounded -= zero_point
    return tldevice.mul_rn(rounded, scale)

round_candidate_scale

round_candidate_scale(
    scale,
    SCALE_ROUND_TYPE: constexpr,
    QUANT_TYPE: constexpr,
)

Round a floating candidate scale as compressed-tensors qparams do.

Source code in src/llmcompressor/utils/triton_utils.py
@triton.jit
def round_candidate_scale(
    scale, SCALE_ROUND_TYPE: tl.constexpr, QUANT_TYPE: tl.constexpr
):
    """Round a floating candidate scale as compressed-tensors qparams do."""
    if SCALE_ROUND_TYPE == _TRITON_FP16:
        return round_fp32_as_dtype(tl.minimum(scale, 65504.0), _TRITON_FP16)
    if SCALE_ROUND_TYPE == _TRITON_BF16:
        return round_fp32_as_dtype(scale, _TRITON_BF16)
    if SCALE_ROUND_TYPE == _TRITON_FP8_E4M3:
        return round_fp32_as_dtype(tl.minimum(scale, 448.0), _TRITON_FP8_E4M3)
    if SCALE_ROUND_TYPE == _TRITON_E8M0:
        # E8M0 stores the power-of-two scale after accounting for the element
        # format's largest exponent (FP4: 2, FP8: 8).
        is_zero = scale == 0.0
        safe_scale = tl.where(is_zero, 1.0, scale)
        exponent = tl.floor(tldevice.log2(safe_scale))
        power_2 = tldevice.exp2(exponent)
        exponent += tl.where(safe_scale >= 1.75 * power_2, 1.0, 0.0)
        offset = 2.0 if QUANT_TYPE == 1 else 8.0
        encoded = tl.clamp(exponent - offset + 127.0, 0.0, 255.0)
        encoded = tl.where(is_zero, 0.0, encoded)
        return tldevice.exp2(encoded - 127.0)
    return scale

round_fp32_as_dtype

round_fp32_as_dtype(value, COMPUTE_DTYPE: constexpr)

Round an FP32 value to the requested dtype and return it as FP32.

Source code in src/llmcompressor/utils/triton_utils.py
@triton.jit
def round_fp32_as_dtype(value, COMPUTE_DTYPE: tl.constexpr):
    """Round an FP32 value to the requested dtype and return it as FP32."""
    tl.static_assert(value.dtype == tl.float32, "value must be FP32")

    if COMPUTE_DTYPE == _TRITON_FP8_E4M3:
        return value.to(tl.float8e4nv).to(tl.float32)

    if COMPUTE_DTYPE == _TRITON_FP16 and not is_xpu_target():
        return value.to(tl.float16).to(tl.float32)

    if COMPUTE_DTYPE == _TRITON_BF16 and not is_xpu_target():
        return value.to(tl.bfloat16).to(tl.float32)

    if COMPUTE_DTYPE == _TRITON_FP16 and is_xpu_target():
        half = value.to(tl.float16)
        # The identity FMA forces the Intel backend to materialize the
        # FP16 value; a plain cast is optimized away in arithmetic chains.
        return tldevice.fma(
            half,
            tl.full((), 1.0, tl.float16),
            tl.full((), 0.0, tl.float16),
        ).to(tl.float32)

    if COMPUTE_DTYPE == _TRITON_BF16 and is_xpu_target():
        # Intel Triton promotes BF16 arithmetic to FP32, even across an
        # explicit BF16 cast, so round through the FP32 bit representation.
        bits = value.to(tl.int32, bitcast=True)
        least_significant_bit = (bits >> 16) & 1
        bits = bits + 0x7FFF + least_significant_bit
        bits = bits & -65536
        return bits.to(tl.float32, bitcast=True)

    return value

scale_round_config

scale_round_config(
    dtype: dtype | None,
) -> tuple[int, float]

Return the Triton scale rounding mode and minimum valid scale.

Source code in src/llmcompressor/utils/triton_utils.py
def scale_round_config(dtype: torch.dtype | None) -> tuple[int, float]:
    """Return the Triton scale rounding mode and minimum valid scale."""
    if dtype is None or dtype == torch.float32:
        return TritonDType.FP32.value, torch.finfo(torch.float32).eps
    if dtype == torch.float16:
        return TritonDType.FP16.value, torch.finfo(torch.float16).eps
    if dtype == torch.bfloat16:
        return TritonDType.BF16.value, torch.finfo(torch.bfloat16).eps
    if dtype == torch.float8_e4m3fn:
        return TritonDType.FP8_E4M3.value, 0.125
    if dtype == torch.uint8:
        return TritonDType.E8M0.value, 2.0**-127
    raise ValueError(f"Unsupported Triton scale dtype: {dtype}")