Skip to content

vllm.platforms.xpu

logger module-attribute 

logger = init_logger(__name__)

XPUPlatform

Bases: Platform

Source code in vllm/platforms/xpu.py 
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
class XPUPlatform(Platform):
    _enum = PlatformEnum.XPU
    device_name: str = "xpu"
    device_type: str = "xpu"
    dispatch_key: str = "XPU"
    # Intel XPU's device key is "GPU" for Ray.
    # see https://github.com/ray-project/ray/blob/6a5eb5865eeb9ccf058a79b44f107e327e360673/python/ray/_private/accelerators/intel_gpu.py#L20 # noqa: E501
    ray_device_key: str = "GPU"
    device_control_env_var: str = "ONEAPI_DEVICE_SELECTOR"

    @classmethod
    def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
                             dtype: torch.dtype, kv_cache_dtype: Optional[str],
                             block_size: int, use_v1: bool,
                             use_mla: bool) -> str:
        if selected_backend != _Backend.IPEX:
            logger.info("Cannot use %s backend on XPU.", selected_backend)
        logger.info("Using IPEX attention backend.")
        return "vllm.attention.backends.ipex_attn.IpexAttnBackend"

    @classmethod
    def get_device_capability(
        cls,
        device_id: int = 0,
    ) -> Optional[DeviceCapability]:
        # capacity format differs from cuda's and will cause unexpected
        # failure, so use None directly
        return None

    @classmethod
    def get_device_name(cls, device_id: int = 0) -> str:
        return torch.xpu.get_device_name(device_id)

    @classmethod
    def get_device_total_memory(cls, device_id: int = 0) -> int:
        device_props = torch.xpu.get_device_properties(device_id)
        return device_props.total_memory

    @classmethod
    def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
        return True

    @classmethod
    def inference_mode(cls):
        return torch.no_grad()

    @classmethod
    def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
        cache_config = vllm_config.cache_config
        if cache_config and cache_config.block_size is None:
            cache_config.block_size = 16

        # check and update model config
        model_config = vllm_config.model_config
        if model_config.dtype == torch.bfloat16:
            bf16_supported = cls.device_support_bf16()
            if not bf16_supported:
                logger.warning(
                    "bfloat16 is only supported on Intel Data Center GPU, "
                    "Intel Arc GPU is not supported yet. Your device is %s,"
                    " which is not supported. will fallback to float16",
                    cls.get_device_name())
                model_config.dtype = torch.float16
        if not model_config.enforce_eager:
            logger.warning(
                "CUDA graph is not supported on XPU, fallback to the eager "
                "mode.")
            model_config.enforce_eager = True

        if vllm_config.speculative_config is not None:
            raise NotImplementedError(
                "XPU does not support speculative decoding")

        if vllm_config.device_config is not None:
            assert vllm_config.device_config.device_type == "xpu"

        # check and update parallel config
        parallel_config = vllm_config.parallel_config
        if parallel_config.worker_cls == "auto":
            parallel_config.worker_cls = "vllm.worker.xpu_worker.XPUWorker"

        if parallel_config.distributed_executor_backend is None:
            parallel_config.distributed_executor_backend = "ray"
        elif parallel_config.distributed_executor_backend == "mp":
            # FIXME(kunshang):
            # spawn needs calling `if __name__ == '__main__':``
            # fork is not supported for xpu start new process.
            logger.error(
                "Both start methods (spawn and fork) have issue "
                "on XPU if you use mp backend, setting it to ray instead.")
            parallel_config.distributed_executor_backend = "ray"

        elif parallel_config.distributed_executor_backend != "ray":
            logger.warning(
                "%s is not supported on XPU, fallback to ray distributed"
                " executor backend.",
                parallel_config.distributed_executor_backend)
            parallel_config.distributed_executor_backend = "ray"

        if vllm_config.model_config and vllm_config.model_config.use_mla:
            logger.info(
                "MLA is enabled on a non-GPU platform; forcing chunked "
                "prefill and prefix caching to be disabled.")
            vllm_config.scheduler_config.enable_chunked_prefill = False
            vllm_config.scheduler_config.chunked_prefill_enabled = False
            vllm_config.scheduler_config.max_num_batched_tokens = max(
                vllm_config.scheduler_config.max_model_len,
                DEFAULT_MAX_NUM_BATCHED_TOKENS)

    @classmethod
    def is_pin_memory_available(cls):
        logger.warning("Pin memory is not supported on XPU.")
        return False

    @classmethod
    def get_current_memory_usage(cls,
                                 device: Optional[torch.types.Device] = None
                                 ) -> float:
        torch.xpu.reset_peak_memory_stats(device)
        return torch.xpu.max_memory_allocated(device)

    @classmethod
    def device_support_bf16(cls) -> bool:
        device_name = cls.get_device_name().lower()
        if device_name.count("arc") > 0:
            return False
        elif device_name.count("data center gpu") > 0:
            return True
        else:
            logger.warning("Unknown device name %s, always use float16",
                           device_name)
            return False

    @classmethod
    def get_device_communicator_cls(cls) -> str:
        return "vllm.distributed.device_communicators.xpu_communicator.XpuCommunicator"  # noqa

_enum class-attribute instance-attribute 

_enum = XPU

device_control_env_var class-attribute instance-attribute 

device_control_env_var: str = 'ONEAPI_DEVICE_SELECTOR'

device_name class-attribute instance-attribute 

device_name: str = 'xpu'

device_type class-attribute instance-attribute 

device_type: str = 'xpu'

dispatch_key class-attribute instance-attribute 

dispatch_key: str = 'XPU'

ray_device_key class-attribute instance-attribute 

ray_device_key: str = 'GPU'

check_and_update_config classmethod 

check_and_update_config(vllm_config: VllmConfig) -> None
Source code in vllm/platforms/xpu.py 
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
@classmethod
def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
    cache_config = vllm_config.cache_config
    if cache_config and cache_config.block_size is None:
        cache_config.block_size = 16

    # check and update model config
    model_config = vllm_config.model_config
    if model_config.dtype == torch.bfloat16:
        bf16_supported = cls.device_support_bf16()
        if not bf16_supported:
            logger.warning(
                "bfloat16 is only supported on Intel Data Center GPU, "
                "Intel Arc GPU is not supported yet. Your device is %s,"
                " which is not supported. will fallback to float16",
                cls.get_device_name())
            model_config.dtype = torch.float16
    if not model_config.enforce_eager:
        logger.warning(
            "CUDA graph is not supported on XPU, fallback to the eager "
            "mode.")
        model_config.enforce_eager = True

    if vllm_config.speculative_config is not None:
        raise NotImplementedError(
            "XPU does not support speculative decoding")

    if vllm_config.device_config is not None:
        assert vllm_config.device_config.device_type == "xpu"

    # check and update parallel config
    parallel_config = vllm_config.parallel_config
    if parallel_config.worker_cls == "auto":
        parallel_config.worker_cls = "vllm.worker.xpu_worker.XPUWorker"

    if parallel_config.distributed_executor_backend is None:
        parallel_config.distributed_executor_backend = "ray"
    elif parallel_config.distributed_executor_backend == "mp":
        # FIXME(kunshang):
        # spawn needs calling `if __name__ == '__main__':``
        # fork is not supported for xpu start new process.
        logger.error(
            "Both start methods (spawn and fork) have issue "
            "on XPU if you use mp backend, setting it to ray instead.")
        parallel_config.distributed_executor_backend = "ray"

    elif parallel_config.distributed_executor_backend != "ray":
        logger.warning(
            "%s is not supported on XPU, fallback to ray distributed"
            " executor backend.",
            parallel_config.distributed_executor_backend)
        parallel_config.distributed_executor_backend = "ray"

    if vllm_config.model_config and vllm_config.model_config.use_mla:
        logger.info(
            "MLA is enabled on a non-GPU platform; forcing chunked "
            "prefill and prefix caching to be disabled.")
        vllm_config.scheduler_config.enable_chunked_prefill = False
        vllm_config.scheduler_config.chunked_prefill_enabled = False
        vllm_config.scheduler_config.max_num_batched_tokens = max(
            vllm_config.scheduler_config.max_model_len,
            DEFAULT_MAX_NUM_BATCHED_TOKENS)

device_support_bf16 classmethod 

device_support_bf16() -> bool
Source code in vllm/platforms/xpu.py 
141
142
143
144
145
146
147
148
149
150
151
@classmethod
def device_support_bf16(cls) -> bool:
    device_name = cls.get_device_name().lower()
    if device_name.count("arc") > 0:
        return False
    elif device_name.count("data center gpu") > 0:
        return True
    else:
        logger.warning("Unknown device name %s, always use float16",
                       device_name)
        return False

get_attn_backend_cls classmethod 

get_attn_backend_cls(
    selected_backend: _Backend,
    head_size: int,
    dtype: dtype,
    kv_cache_dtype: Optional[str],
    block_size: int,
    use_v1: bool,
    use_mla: bool,
) -> str
Source code in vllm/platforms/xpu.py 
30
31
32
33
34
35
36
37
38
@classmethod
def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
                         dtype: torch.dtype, kv_cache_dtype: Optional[str],
                         block_size: int, use_v1: bool,
                         use_mla: bool) -> str:
    if selected_backend != _Backend.IPEX:
        logger.info("Cannot use %s backend on XPU.", selected_backend)
    logger.info("Using IPEX attention backend.")
    return "vllm.attention.backends.ipex_attn.IpexAttnBackend"

get_current_memory_usage classmethod 

get_current_memory_usage(
    device: Optional[Device] = None,
) -> float
Source code in vllm/platforms/xpu.py 
134
135
136
137
138
139
@classmethod
def get_current_memory_usage(cls,
                             device: Optional[torch.types.Device] = None
                             ) -> float:
    torch.xpu.reset_peak_memory_stats(device)
    return torch.xpu.max_memory_allocated(device)

get_device_capability classmethod 

get_device_capability(
    device_id: int = 0,
) -> Optional[DeviceCapability]
Source code in vllm/platforms/xpu.py 
40
41
42
43
44
45
46
47
@classmethod
def get_device_capability(
    cls,
    device_id: int = 0,
) -> Optional[DeviceCapability]:
    # capacity format differs from cuda's and will cause unexpected
    # failure, so use None directly
    return None

get_device_communicator_cls classmethod 

get_device_communicator_cls() -> str
Source code in vllm/platforms/xpu.py 
153
154
155
@classmethod
def get_device_communicator_cls(cls) -> str:
    return "vllm.distributed.device_communicators.xpu_communicator.XpuCommunicator"  # noqa

get_device_name classmethod 

get_device_name(device_id: int = 0) -> str
Source code in vllm/platforms/xpu.py 
49
50
51
@classmethod
def get_device_name(cls, device_id: int = 0) -> str:
    return torch.xpu.get_device_name(device_id)

get_device_total_memory classmethod 

get_device_total_memory(device_id: int = 0) -> int
Source code in vllm/platforms/xpu.py 
53
54
55
56
@classmethod
def get_device_total_memory(cls, device_id: int = 0) -> int:
    device_props = torch.xpu.get_device_properties(device_id)
    return device_props.total_memory

inference_mode classmethod 

inference_mode()
Source code in vllm/platforms/xpu.py 
62
63
64
@classmethod
def inference_mode(cls):
    return torch.no_grad()

is_async_output_supported classmethod 

is_async_output_supported(
    enforce_eager: Optional[bool],
) -> bool
Source code in vllm/platforms/xpu.py 
58
59
60
@classmethod
def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
    return True

is_pin_memory_available classmethod 

is_pin_memory_available()
Source code in vllm/platforms/xpu.py 
129
130
131
132
@classmethod
def is_pin_memory_available(cls):
    logger.warning("Pin memory is not supported on XPU.")
    return False