class HYV4DFlySpeculatorConfig(DFlySpeculatorConfig):
"""HYV4's configuration and implementation of DFly speculative decoding."""
speculator_class = "vllm.models.hy_v4.nvidia.dfly_speculator.HYV4DFlySpeculator"
target_model_type = "hy_v4"
@classmethod
def derive_draft_hf_config(cls, hf_config, *, target_config):
return _derive_dfly_config(
hf_config,
target_config=target_config,
target_model_type=cls.target_model_type,
)
@classmethod
def target_rope_parameters(cls, target_text_config, draft_config):
return target_text_config.rope_parameters
@classmethod
def verify_target_parallel_config(cls, spec: "SpeculativeConfig") -> None:
for field in (
"pipeline_parallel_size",
"prefill_context_parallel_size",
"decode_context_parallel_size",
):
if getattr(spec.target_parallel_config, field) != 1:
raise ValueError(f"DFly requires {field}=1")
@classmethod
def draft_hf_overrides(
cls, spec: "SpeculativeConfig"
) -> Callable[[PreTrainedConfig], PreTrainedConfig]:
target = spec.target_model_config
if spec.model != target.model:
raise ValueError("DFly weights must be in the target checkpoint")
target_config = deepcopy(target.hf_config)
return partial(cls.derive_draft_hf_config, target_config=target_config)
@classmethod
def adapt_draft_model_config(cls, spec: "SpeculativeConfig") -> None:
target = spec.target_model_config
draft = spec.draft_model_config
draft.model_weights = target.model_weights
config = draft.hf_config
if config.model_type != "hy_v4_dfly":
raise ValueError("method='dfly' requires a HYV4 DFly checkpoint")
if target.hf_text_config.model_type != cls.target_model_type:
raise ValueError(f"DFly requires a {cls.target_model_type} target model")
if str(target.dtype) != "torch.bfloat16":
raise ValueError("DFly requires a BF16 target dtype")
checkpoint_quantized = target.quantization in ("fp8", "compressed-tensors")
if checkpoint_quantized:
_validate_checkpoint_quantization(
target.model_arch_config.quantization_config
)
elif target.quantization is not None:
from vllm.config.quantization import QuantizationConfigArgs, QuantSpec
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Static128BlockSym,
)
quantization_config = target.quantization_config
if isinstance(quantization_config, dict):
quantization_config = QuantizationConfigArgs(**quantization_config)
if target.quantization != "online" or quantization_config != (
QuantizationConfigArgs(moe=QuantSpec(weight=kFp8Static128BlockSym))
):
raise ValueError(
"DFly supports only unquantized targets, checkpoint blockwise "
"FP8/MXFP4 or online MoE-only fp8_per_block quantization"
)
if checkpoint_quantized:
if draft.quantization != target.quantization or spec.quantization not in (
None,
target.quantization,
):
raise ValueError("DFly must use the checkpoint's quantization")
_validate_checkpoint_quantization(
draft.model_arch_config.quantization_config
)
elif draft.quantization is not None or spec.quantization is not None:
raise ValueError("DFly does not support quantized draft weights")
parallel = spec.target_parallel_config
cls.verify_target_parallel_config(spec)
if spec.draft_tensor_parallel_size not in (None, parallel.tensor_parallel_size):
raise ValueError("DFly draft and target must use the same TP size")
for size in (
config.num_attention_heads,
config.dfly_intermediate_size,
config.hidden_correction_intermediate_size,
):
if size is not None and size % parallel.tensor_parallel_size:
raise ValueError(
"DFly heads and FFN dimensions must be divisible by TP"
)
for field in ("vocab_size", "hidden_size", "kv_lora_rank", "qk_rope_head_dim"):
if getattr(config, field) != getattr(target.hf_text_config, field):
raise ValueError(f"DFly and target must have the same {field}")
if config.target_hidden_size != target.get_hidden_size():
raise ValueError("DFly target_hidden_size disagrees with the target")
if config.target_index_topk != target.hf_text_config.index_topk:
raise ValueError("DFly target_index_topk disagrees with the target")
if config.rope_parameters != cls.target_rope_parameters(
target.hf_text_config, config
):
raise ValueError("DFly and target must use identical RoPE parameters")
layers = target.hf_text_config.num_hidden_layers
if config.target_num_hidden_layers not in (None, layers):
raise ValueError("DFly target_num_hidden_layers disagrees with the target")
indices = config.target_index_layer
if isinstance(indices, int) and not isinstance(indices, bool):
indices = [indices] * config.num_hidden_layers
for name, selected in (
("target_index_layer", indices),
("target_kv_layers", config.target_kv_layers),
):
if (
not isinstance(selected, list)
or len(selected) != config.num_hidden_layers
or any(type(i) is not int or not 1 <= i < layers for i in selected)
):
raise ValueError(
f"DFly {name} needs one valid target layer per draft layer"
)
if len(set(config.target_kv_layers)) != config.num_hidden_layers:
raise ValueError("DFly requires distinct target KV layers")
config.target_index_layer = indices
super().adapt_draft_model_config(spec)
@classmethod
def apply_options(cls, spec: "SpeculativeConfig") -> None:
if spec.kv_cache_dtype not in (None, "auto", "bfloat16", "fp8_ds_mla"):
raise ValueError("DFly requires BF16 or fp8_ds_mla draft MLA KV cache")
if spec.attention_backend is not None:
raise ValueError("DFly does not support a draft attention backend override")
super().apply_options(spec)