vllm.v1.spec_decode.vocab_mapping ¶

def _detect_space_prefix(tokenizer) -> tuple[str, ...]:
    """Detect the space-prefix character(s) by tokenizing a literal space.

    Different tokenizer families mark word-initial spaces differently:
    BPE uses 'Ġ' (U+0120), SentencePiece uses '▁' (U+2581). Probing at
    runtime avoids hardcoding assumptions and correctly handles mixed-family
    pairs (e.g. BPE draft + SentencePiece target).
    """
    try:
        space_ids = tokenizer.encode(" a", add_special_tokens=False)
        if space_ids:
            tok_str = tokenizer.convert_ids_to_tokens(space_ids[0])
            if (
                isinstance(tok_str, str)
                and len(tok_str) > 1
                and tok_str.endswith("a")
                and tok_str[0] not in (" ", "	")
            ):
                return (tok_str[:-1],)
    except Exception:
        pass
    # Fallback: cover both BPE (Ġ U+0120) and SentencePiece (▁ U+2581)
    return ("\u0120", "\u2581")

def _get_unk_token_id(tokenizer, role: str) -> int:
    """Return a safe fallback token ID for out-of-intersection tokens.

    Preferred: unk_token_id → eos_token_id → ValueError.
    Checking with ``is not None`` is required because token ID 0 is a valid
    (and common) unk ID on many tokenizers; using ``or 0`` would silently
    mishandle those cases.
    """
    unk = getattr(tokenizer, "unk_token_id", None)
    if unk is not None:
        return unk
    eos = getattr(tokenizer, "eos_token_id", None)
    if eos is not None:
        logger.warning(
            "VocabMapping: %s has no unk_token_id; "
            "falling back to eos_token_id=%d for out-of-intersection tokens",
            role,
            eos,
        )
        return eos
    raise ValueError(
        f"VocabMapping: {role} has neither unk_token_id nor eos_token_id; "
        "cannot safely map out-of-intersection tokens"
    )