Skip to content

vllm_gaudi.envs

VLLM_HPU_FORCE_CHANNEL_FP8 module-attribute 

VLLM_HPU_FORCE_CHANNEL_FP8: bool = True

VLLM_HPU_HETERO_KV_LAYOUT module-attribute 

VLLM_HPU_HETERO_KV_LAYOUT: bool = False

VLLM_HPU_MULTI_MODEL_CONFIG module-attribute 

VLLM_HPU_MULTI_MODEL_CONFIG: Optional[str] = None

VLLM_HPU_NIXL_JOINT_KV module-attribute 

VLLM_HPU_NIXL_JOINT_KV: bool = False

VLLM_HPU_NIXL_STAGING_SLOTS module-attribute 

VLLM_HPU_NIXL_STAGING_SLOTS: int = 0

VLLM_NIXL_ABORT_REQUEST_TIMEOUT module-attribute 

VLLM_NIXL_ABORT_REQUEST_TIMEOUT: float = 300.0

VLLM_NIXL_SIDE_CHANNEL_HOST module-attribute 

VLLM_NIXL_SIDE_CHANNEL_HOST: str = 'localhost'

VLLM_NIXL_SIDE_CHANNEL_PORT module-attribute 

VLLM_NIXL_SIDE_CHANNEL_PORT: int = 5600

VLLM_USE_HPU_CONTIGUOUS_CACHE_FETCH module-attribute 

VLLM_USE_HPU_CONTIGUOUS_CACHE_FETCH: bool = True

environment_variables module-attribute 

environment_variables: dict[str, Callable[[], Any]] = {
    "VLLM_USE_HPU_CONTIGUOUS_CACHE_FETCH": lambda: (
        os.environ.get("VLLM_CONTIGUOUS_PA", "true").lower()
        in ("1", "true")
    ),
    "VLLM_HPU_FORCE_CHANNEL_FP8": lambda: (
        os.environ.get(
            "VLLM_HPU_FORCE_CHANNEL_FP8", "true"
        ).lower()
        in ("1", "true")
        and os.environ.get("QUANT_CONFIG", None) is None
    ),
    "VLLM_HPU_HETERO_KV_LAYOUT": lambda: (
        os.environ.get(
            "VLLM_HPU_HETERO_KV_LAYOUT", "false"
        ).lower()
        in ("0", "false")
    ),
    "VLLM_HPU_MULTI_MODEL_CONFIG": lambda: os.environ.get(
        "VLLM_HPU_MULTI_MODEL_CONFIG", None
    ),
    "VLLM_NIXL_ABORT_REQUEST_TIMEOUT": lambda: float(
        os.environ.get(
            "VLLM_NIXL_ABORT_REQUEST_TIMEOUT", "300"
        )
    ),
    "VLLM_NIXL_SIDE_CHANNEL_HOST": lambda: os.environ.get(
        "VLLM_NIXL_SIDE_CHANNEL_HOST", "localhost"
    ),
    "VLLM_NIXL_SIDE_CHANNEL_PORT": lambda: int(
        os.environ.get(
            "VLLM_NIXL_SIDE_CHANNEL_PORT", "5600"
        )
    ),
    "VLLM_HPU_NIXL_JOINT_KV": lambda: (
        os.environ.get(
            "VLLM_HPU_NIXL_JOINT_KV", "false"
        ).lower()
        in ("1", "true")
    ),
    "VLLM_HPU_NIXL_STAGING_SLOTS": lambda: int(
        os.environ.get("VLLM_HPU_NIXL_STAGING_SLOTS", "0")
    ),
}

__dir__ 

__dir__()
Source code in vllm_gaudi/envs.py 
85
86
def __dir__():
    return list(environment_variables.keys())

__getattr__ 

__getattr__(name: str)
Source code in vllm_gaudi/envs.py 
78
79
80
81
82
def __getattr__(name: str):
    # lazy evaluation of environment variables
    if name in environment_variables:
        return environment_variables[name]()
    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")