vllm_gaudi.entrypoints.openai.multi_model_api_server

OpenAI-compatible API server with Gaudi multi-model switching support.

args module-attribute

args = parse_args()

logger module-attribute

logger = init_logger(
    "vllm_gaudi.entrypoints.openai.multi_model_api_server"
)

parser module-attribute

parser = make_arg_parser(
    FlexibleArgumentParser(
        description="vLLM OpenAI-Compatible RESTful API server (Gaudi multi-model)."
    )
)

FrontendSettings dataclass

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

@dataclass
class FrontendSettings:
    enable_auto_tool_choice: bool
    tool_call_parser: str | None
    chat_template: str | None

chat_template instance-attribute

chat_template: str | None

enable_auto_tool_choice instance-attribute

enable_auto_tool_choice: bool

tool_call_parser instance-attribute

tool_call_parser: str | None

__init__

__init__(
    enable_auto_tool_choice: bool,
    tool_call_parser: str | None,
    chat_template: str | None,
) -> None

ModelFrontendOverrides dataclass

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

@dataclass
class ModelFrontendOverrides:
    enable_auto_tool_choice: bool | None = None
    tool_call_parser: str | None = None
    chat_template: str | None = None

chat_template class-attribute instance-attribute

chat_template: str | None = None

enable_auto_tool_choice class-attribute instance-attribute

enable_auto_tool_choice: bool | None = None

tool_call_parser class-attribute instance-attribute

tool_call_parser: str | None = None

__init__

__init__(
    enable_auto_tool_choice: bool | None = None,
    tool_call_parser: str | None = None,
    chat_template: str | None = None,
) -> None

ModelSwitchRequest

Bases: BaseModel

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

class ModelSwitchRequest(BaseModel):
    model: str = Field(..., description="Target model name")
    drain_timeout: int = Field(60, ge=0)

drain_timeout class-attribute instance-attribute

drain_timeout: int = Field(60, ge=0)

model class-attribute instance-attribute

model: str = Field(..., description='Target model name')

ModelSwitchResponse

Bases: BaseModel

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

class ModelSwitchResponse(BaseModel):
    previous_model: str | None
    current_model: str
    switched: bool
    duration_ms: float
    reconfigure_ms: float | None = None
    memory_before_mb: float | None = None
    memory_after_unload_mb: float | None = None
    freed_memory_mb: float | None = None
    stash_memory_after_mb: float | None = None

current_model instance-attribute

current_model: str

duration_ms instance-attribute

duration_ms: float

freed_memory_mb class-attribute instance-attribute

freed_memory_mb: float | None = None

memory_after_unload_mb class-attribute instance-attribute

memory_after_unload_mb: float | None = None

memory_before_mb class-attribute instance-attribute

memory_before_mb: float | None = None

previous_model instance-attribute

previous_model: str | None

reconfigure_ms class-attribute instance-attribute

reconfigure_ms: float | None = None

stash_memory_after_mb class-attribute instance-attribute

stash_memory_after_mb: float | None = None

switched instance-attribute

switched: bool

MultiModelConfigLoadResult

Bases: NamedTuple

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

class MultiModelConfigLoadResult(NamedTuple):
    model_configs: dict[str, AsyncEngineArgs]
    default_model: str
    model_frontend_overrides: dict[str, ModelFrontendOverrides]
    model_quant_configs: dict[str, str | None]

default_model instance-attribute

default_model: str

model_configs instance-attribute

model_configs: dict[str, AsyncEngineArgs]

model_frontend_overrides instance-attribute

model_frontend_overrides: dict[str, ModelFrontendOverrides]

model_quant_configs instance-attribute

model_quant_configs: dict[str, str | None]

MultiModelEngineClient

Bases: EngineClient

EngineClient adapter for MultiModelAsyncLLM.

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

class MultiModelEngineClient(EngineClient):
    """EngineClient adapter for MultiModelAsyncLLM."""

    def __init__(self, manager: MultiModelAsyncLLM):
        self.manager = manager

    @property
    def _engine(self):
        return self.manager.engine

    @property
    def vllm_config(self):
        return self._engine.vllm_config

    @property
    def model_config(self):
        return self._engine.model_config

    @property
    def renderer(self):
        return self._engine.renderer

    @property
    def input_processor(self):
        return self._engine.input_processor

    @property
    def is_running(self) -> bool:
        return self._engine.is_running

    @property
    def is_stopped(self) -> bool:
        return self._engine.is_stopped

    @property
    def errored(self) -> bool:
        return self._engine.errored

    @property
    def dead_error(self) -> BaseException:
        return self._engine.dead_error

    def generate(self, *args, **kwargs):
        return self._engine.generate(*args, **kwargs)

    def encode(self, *args, **kwargs):
        return self._engine.encode(*args, **kwargs)

    async def abort(self, request_id):
        await self._engine.abort(request_id)

    async def is_tracing_enabled(self) -> bool:
        return await self._engine.is_tracing_enabled()

    async def do_log_stats(self) -> None:
        await self._engine.do_log_stats()

    async def check_health(self) -> None:
        await self._engine.check_health()

    async def start_profile(self) -> None:
        await self._engine.start_profile()

    async def stop_profile(self) -> None:
        await self._engine.stop_profile()

    async def reset_mm_cache(self) -> None:
        await self._engine.reset_mm_cache()

    async def reset_encoder_cache(self) -> None:
        await self._engine.reset_encoder_cache()

    async def reset_prefix_cache(self, reset_running_requests: bool = False, reset_connector: bool = False) -> bool:
        return await self._engine.reset_prefix_cache(
            reset_running_requests=reset_running_requests,
            reset_connector=reset_connector,
        )

    async def sleep(self, level: int = 1, mode: str = "abort") -> None:
        await self._engine.sleep(level=level, mode=mode)

    async def wake_up(self, tags: list[str] | None = None) -> None:
        await self._engine.wake_up(tags=tags)

    async def notify_kv_transfer_request_rejected(
        self,
        request_id: str,
        kv_transfer_params: dict[str, Any],
        *,
        data_parallel_rank: int | None = None,
    ) -> None:
        await self._engine.notify_kv_transfer_request_rejected(
            request_id=request_id,
            kv_transfer_params=kv_transfer_params,
            data_parallel_rank=data_parallel_rank,
        )

    async def is_sleeping(self) -> bool:
        return await self._engine.is_sleeping()

    async def add_lora(self, lora_request) -> bool:
        return await self._engine.add_lora(lora_request)

    async def pause_generation(
        self,
        *,
        mode: str = "abort",
        wait_for_inflight_requests: bool = False,
        clear_cache: bool = True,
    ) -> None:
        await self._engine.pause_generation(
            mode=mode,
            wait_for_inflight_requests=wait_for_inflight_requests,
            clear_cache=clear_cache,
        )

    async def resume_generation(self) -> None:
        await self._engine.resume_generation()

    async def is_paused(self) -> bool:
        return await self._engine.is_paused()

    def shutdown(self, timeout: float | None = None) -> None:
        self.manager.shutdown()

    async def scale_elastic_ep(self, new_data_parallel_size: int, drain_timeout: int = 300) -> None:
        await self._engine.scale_elastic_ep(
            new_data_parallel_size=new_data_parallel_size,
            drain_timeout=drain_timeout,
        )

    async def collective_rpc(
            self,
            method: str,
            timeout: float | None = None,
            args: tuple = (),
            kwargs: dict | None = None,
    ):
        return await self._engine.collective_rpc(
            method=method,
            timeout=timeout,
            args=args,
            kwargs=kwargs,
        )

    async def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
        return await self._engine.get_supported_tasks()

    async def init_weight_transfer_engine(self, request) -> None:
        await self._engine.init_weight_transfer_engine(request)

    async def update_weights(self, request) -> None:
        await self._engine.update_weights(request)

_engine property

_engine

dead_error property

dead_error: BaseException

errored property

errored: bool

input_processor property

input_processor

is_running property

is_running: bool

is_stopped property

is_stopped: bool

manager instance-attribute

manager = manager

model_config property

model_config

renderer property

renderer

vllm_config property

vllm_config

__init__

__init__(manager: MultiModelAsyncLLM)

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

def __init__(self, manager: MultiModelAsyncLLM):
    self.manager = manager

abort async

abort(request_id)

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

async def abort(self, request_id):
    await self._engine.abort(request_id)

add_lora async

add_lora(lora_request) -> bool

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

async def add_lora(self, lora_request) -> bool:
    return await self._engine.add_lora(lora_request)

check_health async

check_health() -> None

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

async def check_health(self) -> None:
    await self._engine.check_health()

collective_rpc async

collective_rpc(
    method: str,
    timeout: float | None = None,
    args: tuple = (),
    kwargs: dict | None = None,
)

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

async def collective_rpc(
        self,
        method: str,
        timeout: float | None = None,
        args: tuple = (),
        kwargs: dict | None = None,
):
    return await self._engine.collective_rpc(
        method=method,
        timeout=timeout,
        args=args,
        kwargs=kwargs,
    )

do_log_stats async

do_log_stats() -> None

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

async def do_log_stats(self) -> None:
    await self._engine.do_log_stats()

encode

encode(*args, **kwargs)

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

def encode(self, *args, **kwargs):
    return self._engine.encode(*args, **kwargs)

generate

generate(*args, **kwargs)

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

def generate(self, *args, **kwargs):
    return self._engine.generate(*args, **kwargs)

get_supported_tasks async

get_supported_tasks() -> tuple[SupportedTask, ...]

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

async def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
    return await self._engine.get_supported_tasks()

init_weight_transfer_engine async

init_weight_transfer_engine(request) -> None

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

async def init_weight_transfer_engine(self, request) -> None:
    await self._engine.init_weight_transfer_engine(request)

is_paused async

is_paused() -> bool

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

async def is_paused(self) -> bool:
    return await self._engine.is_paused()

is_sleeping async

is_sleeping() -> bool

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

async def is_sleeping(self) -> bool:
    return await self._engine.is_sleeping()

is_tracing_enabled async

is_tracing_enabled() -> bool

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

async def is_tracing_enabled(self) -> bool:
    return await self._engine.is_tracing_enabled()

notify_kv_transfer_request_rejected async

notify_kv_transfer_request_rejected(
    request_id: str,
    kv_transfer_params: dict[str, Any],
    *,
    data_parallel_rank: int | None = None,
) -> None

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

async def notify_kv_transfer_request_rejected(
    self,
    request_id: str,
    kv_transfer_params: dict[str, Any],
    *,
    data_parallel_rank: int | None = None,
) -> None:
    await self._engine.notify_kv_transfer_request_rejected(
        request_id=request_id,
        kv_transfer_params=kv_transfer_params,
        data_parallel_rank=data_parallel_rank,
    )

pause_generation async

pause_generation(
    *,
    mode: str = "abort",
    wait_for_inflight_requests: bool = False,
    clear_cache: bool = True,
) -> None

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

async def pause_generation(
    self,
    *,
    mode: str = "abort",
    wait_for_inflight_requests: bool = False,
    clear_cache: bool = True,
) -> None:
    await self._engine.pause_generation(
        mode=mode,
        wait_for_inflight_requests=wait_for_inflight_requests,
        clear_cache=clear_cache,
    )

reset_encoder_cache async

reset_encoder_cache() -> None

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

async def reset_encoder_cache(self) -> None:
    await self._engine.reset_encoder_cache()

reset_mm_cache async

reset_mm_cache() -> None

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

async def reset_mm_cache(self) -> None:
    await self._engine.reset_mm_cache()

reset_prefix_cache async

reset_prefix_cache(
    reset_running_requests: bool = False,
    reset_connector: bool = False,
) -> bool

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

async def reset_prefix_cache(self, reset_running_requests: bool = False, reset_connector: bool = False) -> bool:
    return await self._engine.reset_prefix_cache(
        reset_running_requests=reset_running_requests,
        reset_connector=reset_connector,
    )

resume_generation async

resume_generation() -> None

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

async def resume_generation(self) -> None:
    await self._engine.resume_generation()

scale_elastic_ep async

scale_elastic_ep(
    new_data_parallel_size: int, drain_timeout: int = 300
) -> None

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

async def scale_elastic_ep(self, new_data_parallel_size: int, drain_timeout: int = 300) -> None:
    await self._engine.scale_elastic_ep(
        new_data_parallel_size=new_data_parallel_size,
        drain_timeout=drain_timeout,
    )

shutdown

shutdown(timeout: float | None = None) -> None

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

def shutdown(self, timeout: float | None = None) -> None:
    self.manager.shutdown()

sleep async

sleep(level: int = 1, mode: str = 'abort') -> None

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

async def sleep(self, level: int = 1, mode: str = "abort") -> None:
    await self._engine.sleep(level=level, mode=mode)

start_profile async

start_profile() -> None

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

async def start_profile(self) -> None:
    await self._engine.start_profile()

stop_profile async

stop_profile() -> None

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

async def stop_profile(self) -> None:
    await self._engine.stop_profile()

update_weights async

update_weights(request) -> None

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

async def update_weights(self, request) -> None:
    await self._engine.update_weights(request)

wake_up async

wake_up(tags: list[str] | None = None) -> None

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

async def wake_up(self, tags: list[str] | None = None) -> None:
    await self._engine.wake_up(tags=tags)

MultiModelServingModels

Bases: OpenAIServingModels

OpenAI model registry that exposes all models but validates active only.

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

class MultiModelServingModels(OpenAIServingModels):
    """OpenAI model registry that exposes all models but validates active only."""

    def __init__(
        self,
        engine_client: EngineClient,
        all_model_paths: dict[str, BaseModelPath],
        active_model_name: str,
        model_max_lens: dict[str, int],
        *,
        lora_modules: list | None = None,
    ):
        self._all_model_paths = all_model_paths
        self._model_max_lens = model_max_lens
        active_model = all_model_paths[active_model_name]
        super().__init__(
            engine_client=engine_client,
            base_model_paths=[active_model],
            lora_modules=lora_modules,
        )

    def is_base_model(self, model_name) -> bool:
        if not self.base_model_paths:
            return False
        return self.base_model_paths[0].name == model_name

    async def show_available_models(self):
        from vllm.entrypoints.openai.engine.protocol import (
            ModelCard,
            ModelList,
            ModelPermission,
        )

        model_cards = []
        for name, base_model in self._all_model_paths.items():
            max_model_len = self._model_max_lens.get(name, self.model_config.max_model_len)
            model_cards.append(
                ModelCard(
                    id=name,
                    max_model_len=max_model_len,
                    root=base_model.model_path,
                    permission=[ModelPermission()],
                ))
        lora_cards = [
            ModelCard(
                id=lora.lora_name,
                root=lora.path,
                parent=lora.base_model_name if lora.base_model_name else self.base_model_paths[0].name,
                permission=[ModelPermission()],
            ) for lora in self.lora_requests.values()
        ]
        model_cards.extend(lora_cards)
        return ModelList(data=model_cards)

_all_model_paths instance-attribute

_all_model_paths = all_model_paths

_model_max_lens instance-attribute

_model_max_lens = model_max_lens

__init__

__init__(
    engine_client: EngineClient,
    all_model_paths: dict[str, BaseModelPath],
    active_model_name: str,
    model_max_lens: dict[str, int],
    *,
    lora_modules: list | None = None,
)

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

def __init__(
    self,
    engine_client: EngineClient,
    all_model_paths: dict[str, BaseModelPath],
    active_model_name: str,
    model_max_lens: dict[str, int],
    *,
    lora_modules: list | None = None,
):
    self._all_model_paths = all_model_paths
    self._model_max_lens = model_max_lens
    active_model = all_model_paths[active_model_name]
    super().__init__(
        engine_client=engine_client,
        base_model_paths=[active_model],
        lora_modules=lora_modules,
    )

is_base_model

is_base_model(model_name) -> bool

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

def is_base_model(self, model_name) -> bool:
    if not self.base_model_paths:
        return False
    return self.base_model_paths[0].name == model_name

show_available_models async

show_available_models()

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

async def show_available_models(self):
    from vllm.entrypoints.openai.engine.protocol import (
        ModelCard,
        ModelList,
        ModelPermission,
    )

    model_cards = []
    for name, base_model in self._all_model_paths.items():
        max_model_len = self._model_max_lens.get(name, self.model_config.max_model_len)
        model_cards.append(
            ModelCard(
                id=name,
                max_model_len=max_model_len,
                root=base_model.model_path,
                permission=[ModelPermission()],
            ))
    lora_cards = [
        ModelCard(
            id=lora.lora_name,
            root=lora.path,
            parent=lora.base_model_name if lora.base_model_name else self.base_model_paths[0].name,
            permission=[ModelPermission()],
        ) for lora in self.lora_requests.values()
    ]
    model_cards.extend(lora_cards)
    return ModelList(data=model_cards)

_attach_multi_model_router

_attach_multi_model_router(app: FastAPI) -> None

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

def _attach_multi_model_router(app: FastAPI) -> None:
    if not envs.VLLM_SERVER_DEV_MODE:
        logger.warning("The /v1/models/switch endpoint is disabled. Set VLLM_SERVER_DEV_MODE=1 to enable it.")
        return

    router = APIRouter()

    @router.post("/v1/models/switch", response_model=ModelSwitchResponse)
    async def switch_model(request: ModelSwitchRequest, raw_request: Request):
        manager: MultiModelAsyncLLM = raw_request.app.state.multi_model_manager
        engine_client: EngineClient = raw_request.app.state.multi_model_engine_client
        all_model_paths = raw_request.app.state.multi_model_all_model_paths
        model_max_lens = raw_request.app.state.multi_model_max_lens
        model_frontend_overrides = raw_request.app.state.multi_model_frontend_overrides
        args = raw_request.app.state.args
        supported_tasks = raw_request.app.state.supported_tasks

        if request.model not in manager.available_models:
            raise HTTPException(status_code=404, detail="Model not found")

        start = time.perf_counter()
        previous_model = manager.current_model
        switch_metrics = await manager.switch_model(
            request.model,
            drain_timeout=request.drain_timeout,
        )

        await _init_multi_model_state(
            engine_client,
            raw_request.app.state,
            args,
            supported_tasks,
            all_model_paths=all_model_paths,
            model_max_lens=model_max_lens,
            active_model_name=manager.current_model or request.model,
            model_frontend_overrides=model_frontend_overrides,
        )
        duration_ms = (time.perf_counter() - start) * 1000.0
        reconfigure_ms = None
        memory_before_mb = None
        memory_after_unload_mb = None
        freed_mb = None
        stash_memory_after_mb = None
        if isinstance(switch_metrics, dict):
            reconfigure_s = switch_metrics.get("reconfigure_s")
            if isinstance(reconfigure_s, (int, float)):
                reconfigure_ms = float(reconfigure_s) * 1000.0
            raw_before_mb = switch_metrics.get("memory_before_mb")
            raw_after_unload_mb = switch_metrics.get("memory_after_unload_mb")
            raw_freed_mb = switch_metrics.get("freed_memory_mb")
            raw_stash_after_mb = switch_metrics.get("stash_memory_after_mb")
            if isinstance(raw_before_mb, (int, float)):
                memory_before_mb = float(raw_before_mb)
            if isinstance(raw_after_unload_mb, (int, float)):
                memory_after_unload_mb = float(raw_after_unload_mb)
            if isinstance(raw_freed_mb, (int, float)):
                freed_mb = float(raw_freed_mb)
            if isinstance(raw_stash_after_mb, (int, float)):
                stash_memory_after_mb = float(raw_stash_after_mb)

        return ModelSwitchResponse(
            previous_model=previous_model,
            current_model=manager.current_model or request.model,
            switched=previous_model != manager.current_model,
            duration_ms=duration_ms,
            reconfigure_ms=reconfigure_ms,
            memory_before_mb=memory_before_mb,
            memory_after_unload_mb=memory_after_unload_mb,
            freed_memory_mb=freed_mb,
            stash_memory_after_mb=stash_memory_after_mb,
        )

    app.include_router(router)

_build_model_registry

_build_model_registry(
    manager: MultiModelAsyncLLM,
) -> tuple[dict[str, BaseModelPath], dict[str, int]]

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

def _build_model_registry(manager: MultiModelAsyncLLM) -> tuple[dict[str, BaseModelPath], dict[str, int]]:
    all_model_paths: dict[str, BaseModelPath] = {}
    model_max_lens: dict[str, int] = {}
    for name, vllm_config in manager.get_all_vllm_configs().items():
        model_path = vllm_config.model_config.model
        all_model_paths[name] = BaseModelPath(name=name, model_path=model_path)
        model_max_lens[name] = vllm_config.model_config.max_model_len
    return all_model_paths, model_max_lens

_init_multi_model_state async

_init_multi_model_state(
    engine_client: EngineClient,
    state,
    args: Namespace,
    supported_tasks: tuple[SupportedTask, ...],
    *,
    all_model_paths: dict[str, BaseModelPath],
    model_max_lens: dict[str, int],
    active_model_name: str,
    model_frontend_overrides: dict[
        str, ModelFrontendOverrides
    ],
) -> None

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

async def _init_multi_model_state(
    engine_client: EngineClient,
    state,
    args: Namespace,
    supported_tasks: tuple[SupportedTask, ...],
    *,
    all_model_paths: dict[str, BaseModelPath],
    model_max_lens: dict[str, int],
    active_model_name: str,
    model_frontend_overrides: dict[str, ModelFrontendOverrides],
) -> None:
    request_logger = RequestLogger(max_log_len=args.max_log_len) if args.enable_log_requests else None

    state.request_logger = request_logger
    state.engine_client = engine_client
    state.log_stats = not args.disable_log_stats
    state.vllm_config = engine_client.vllm_config

    default_mm_loras = (engine_client.vllm_config.lora_config.default_mm_loras
                        if engine_client.vllm_config.lora_config is not None else {})
    lora_modules = process_lora_modules(args.lora_modules, default_mm_loras)
    state.openai_serving_models = MultiModelServingModels(
        engine_client=engine_client,
        all_model_paths=all_model_paths,
        active_model_name=active_model_name,
        model_max_lens=model_max_lens,
        lora_modules=lora_modules,
    )
    await state.openai_serving_models.init_static_loras()

    frontend_settings = _resolve_frontend_settings(args, model_frontend_overrides, active_model_name)
    resolved_chat_template = load_chat_template(frontend_settings.chat_template)

    state.openai_serving_render = OpenAIServingRender(
        model_config=engine_client.model_config,
        renderer=engine_client.renderer,
        model_registry=state.openai_serving_models.registry,
        request_logger=request_logger,
        chat_template=resolved_chat_template,
        chat_template_content_format=args.chat_template_content_format,
        trust_request_chat_template=args.trust_request_chat_template,
        enable_auto_tools=frontend_settings.enable_auto_tool_choice,
        exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
        tool_parser=frontend_settings.tool_call_parser,
        default_chat_template_kwargs=args.default_chat_template_kwargs,
        log_error_stack=args.log_error_stack,
    )

    state.openai_serving_tokenization = OpenAIServingTokenization(
        engine_client,
        state.openai_serving_models,
        state.openai_serving_render,
        request_logger=request_logger,
        chat_template=resolved_chat_template,
        chat_template_content_format=args.chat_template_content_format,
        default_chat_template_kwargs=args.default_chat_template_kwargs,
        trust_request_chat_template=args.trust_request_chat_template,
    )

    if "generate" in supported_tasks:
        from vllm.entrypoints.openai.generate.api_router import init_generate_state

        state_args = Namespace(**vars(args))
        state_args.enable_auto_tool_choice = frontend_settings.enable_auto_tool_choice
        state_args.tool_call_parser = frontend_settings.tool_call_parser
        state_args.chat_template = frontend_settings.chat_template
        await init_generate_state(engine_client, state, state_args, request_logger, supported_tasks)

    if "transcription" in supported_tasks:
        from vllm.entrypoints.openai.speech_to_text.api_router import (
            init_transcription_state, )

        init_transcription_state(engine_client, state, args, request_logger, supported_tasks)

    if "realtime" in supported_tasks:
        from vllm.entrypoints.openai.realtime.api_router import init_realtime_state

        init_realtime_state(engine_client, state, args, request_logger, supported_tasks)

    if any(task in POOLING_TASKS for task in supported_tasks):
        from vllm.entrypoints.pooling import init_pooling_state

        init_pooling_state(engine_client, state, args, request_logger, supported_tasks)

    state.enable_server_load_tracking = args.enable_server_load_tracking
    state.server_load_metrics = 0

_load_multi_model_config

_load_multi_model_config(
    path: str,
) -> MultiModelConfigLoadResult

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

def _load_multi_model_config(path: str, ) -> MultiModelConfigLoadResult:
    with open(path) as f:
        data = yaml.safe_load(f)

    if not isinstance(data, dict):
        raise ValueError("Multi-model config must be a YAML/JSON object.")

    raw_models = data.get("models")
    if not isinstance(raw_models, dict) or not raw_models:
        raise ValueError("Multi-model config requires a non-empty 'models' mapping.")

    config_dir = os.path.dirname(os.path.abspath(path))
    model_configs: dict[str, AsyncEngineArgs] = {}
    model_frontend_overrides: dict[str, ModelFrontendOverrides] = {}
    model_quant_configs: dict[str, str | None] = {}
    for name, raw_cfg in raw_models.items():
        if not isinstance(raw_cfg, dict):
            raise ValueError(f"Model config for '{name}' must be a mapping.")
        if "model" not in raw_cfg:
            raise ValueError(f"Model config for '{name}' must include 'model'.")

        raw_cfg_copy = dict(raw_cfg)
        overrides = ModelFrontendOverrides()
        if "enable_auto_tool_choice" in raw_cfg_copy:
            enable_auto_tool_choice = raw_cfg_copy.pop("enable_auto_tool_choice")
            if not isinstance(enable_auto_tool_choice, bool):
                raise ValueError(f"Model '{name}' field 'enable_auto_tool_choice' must be a boolean.")
            overrides.enable_auto_tool_choice = enable_auto_tool_choice

        if "tool_call_parser" in raw_cfg_copy:
            tool_call_parser = raw_cfg_copy.pop("tool_call_parser")
            if tool_call_parser is not None and not isinstance(tool_call_parser, str):
                raise ValueError(f"Model '{name}' field 'tool_call_parser' must be a string.")
            overrides.tool_call_parser = tool_call_parser

        if "chat_template" in raw_cfg_copy:
            chat_template = raw_cfg_copy.pop("chat_template")
            if chat_template is not None and not isinstance(chat_template, str):
                raise ValueError(f"Model '{name}' field 'chat_template' must be a string.")
            if isinstance(chat_template, str):
                chat_template = _normalize_chat_template_path(config_dir, chat_template)
            overrides.chat_template = chat_template

        model_frontend_overrides[name] = overrides

        if "quant_config" in raw_cfg_copy:
            quant_config = raw_cfg_copy.pop("quant_config")
            if quant_config is not None and not isinstance(quant_config, str):
                raise ValueError(f"Model '{name}' field 'quant_config' must be a string path.")
            if isinstance(quant_config, str) and not os.path.isabs(quant_config):
                quant_config = os.path.abspath(os.path.join(config_dir, quant_config))
            model_quant_configs[name] = quant_config

        try:
            model_configs[name] = AsyncEngineArgs(**raw_cfg_copy)
        except TypeError as e:
            raise ValueError(f"Invalid config for '{name}': {e}") from e

    default_model = data.get("default_model") or os.environ.get("MODEL")
    if default_model is None:
        default_model = next(iter(model_configs.keys()))
    if default_model not in model_configs:
        raise ValueError(f"Default model '{default_model}' not found in config models: {list(model_configs.keys())}")

    return MultiModelConfigLoadResult(
        model_configs=model_configs,
        default_model=default_model,
        model_frontend_overrides=model_frontend_overrides,
        model_quant_configs=model_quant_configs,
    )

_normalize_chat_template_path

_normalize_chat_template_path(
    config_dir: str, chat_template: str
) -> str

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

def _normalize_chat_template_path(config_dir: str, chat_template: str) -> str:
    if os.path.isabs(chat_template):
        return chat_template
    return os.path.abspath(os.path.join(config_dir, chat_template))

_resolve_frontend_settings

_resolve_frontend_settings(
    args: Namespace,
    model_frontend_overrides: dict[
        str, ModelFrontendOverrides
    ],
    active_model_name: str,
) -> FrontendSettings

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

def _resolve_frontend_settings(
    args: Namespace,
    model_frontend_overrides: dict[str, ModelFrontendOverrides],
    active_model_name: str,
) -> FrontendSettings:
    model_overrides = model_frontend_overrides.get(active_model_name, ModelFrontendOverrides())
    enable_auto_tool_choice = (model_overrides.enable_auto_tool_choice
                               if model_overrides.enable_auto_tool_choice is not None else args.enable_auto_tool_choice)
    tool_call_parser = (model_overrides.tool_call_parser
                        if model_overrides.tool_call_parser is not None else args.tool_call_parser)
    chat_template = model_overrides.chat_template if model_overrides.chat_template is not None else args.chat_template
    return FrontendSettings(
        enable_auto_tool_choice=enable_auto_tool_choice,
        tool_call_parser=tool_call_parser,
        chat_template=chat_template,
    )

_resolve_multi_model_config_path

_resolve_multi_model_config_path() -> str | None

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

def _resolve_multi_model_config_path() -> str | None:
    return os.environ.get("VLLM_HPU_MULTI_MODEL_CONFIG")

_run_multi_model_server async

_run_multi_model_server(args: Namespace) -> None

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

async def _run_multi_model_server(args: Namespace) -> None:
    decorate_logs("APIServer")

    listen_address, sock = setup_server(args)
    await _run_multi_model_server_worker(listen_address, sock, args)

_run_multi_model_server_worker async

_run_multi_model_server_worker(
    listen_address: str,
    sock,
    args: Namespace,
    **uvicorn_kwargs,
) -> None

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

async def _run_multi_model_server_worker(
    listen_address: str,
    sock,
    args: Namespace,
    **uvicorn_kwargs,
) -> None:
    if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
        ToolParserManager.import_tool_parser(args.tool_parser_plugin)

    if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3:
        ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin)

    log_config = get_uvicorn_log_config(args)
    if log_config is not None:
        uvicorn_kwargs["log_config"] = log_config

    async with build_multi_model_engine_client(args) as (
            engine_client,
            manager,
            all_model_paths,
            model_max_lens,
            model_frontend_overrides,
    ):
        try:
            supported_tasks = await engine_client.get_supported_tasks()
            logger.info("Supported tasks: %s", supported_tasks)

            app = build_app(args, supported_tasks)
            app.state.multi_model_manager = manager
            app.state.multi_model_engine_client = engine_client
            app.state.multi_model_all_model_paths = all_model_paths
            app.state.multi_model_max_lens = model_max_lens
            app.state.multi_model_frontend_overrides = model_frontend_overrides
            app.state.supported_tasks = supported_tasks
            app.state.args = args

            await _init_multi_model_state(
                engine_client,
                app.state,
                args,
                supported_tasks,
                all_model_paths=all_model_paths,
                model_max_lens=model_max_lens,
                active_model_name=manager.current_model or args.model,
                model_frontend_overrides=model_frontend_overrides,
            )
            _attach_multi_model_router(app)

            logger.info("Starting vLLM multi-model API server on %s", listen_address)
            shutdown_task = await serve_http(
                app,
                sock=sock,
                enable_ssl_refresh=args.enable_ssl_refresh,
                host=args.host,
                port=args.port,
                log_level=args.uvicorn_log_level,
                access_log=not args.disable_uvicorn_access_log,
                timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
                ssl_keyfile=args.ssl_keyfile,
                ssl_certfile=args.ssl_certfile,
                ssl_ca_certs=args.ssl_ca_certs,
                ssl_cert_reqs=args.ssl_cert_reqs,
                ssl_ciphers=args.ssl_ciphers,
                h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
                h11_max_header_count=args.h11_max_header_count,
                **uvicorn_kwargs,
            )

            await shutdown_task
        finally:
            sock.close()

_should_use_multi_model

_should_use_multi_model() -> bool

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

def _should_use_multi_model() -> bool:
    return _resolve_multi_model_config_path() is not None

_validate_model_frontend_overrides

_validate_model_frontend_overrides(
    args: Namespace,
    model_frontend_overrides: dict[
        str, ModelFrontendOverrides
    ],
) -> None

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

def _validate_model_frontend_overrides(
    args: Namespace,
    model_frontend_overrides: dict[str, ModelFrontendOverrides],
) -> None:
    valid_tool_parsers = ToolParserManager.list_registered()
    for model_name, override in model_frontend_overrides.items():
        effective_enable_auto = (override.enable_auto_tool_choice
                                 if override.enable_auto_tool_choice is not None else args.enable_auto_tool_choice)
        effective_tool_parser = (override.tool_call_parser
                                 if override.tool_call_parser is not None else args.tool_call_parser)

        if effective_enable_auto and not effective_tool_parser:
            raise ValueError(f"Model '{model_name}' enables auto tool choice but no tool_call_parser is set.")

        if effective_enable_auto and effective_tool_parser and effective_tool_parser not in valid_tool_parsers:
            raise ValueError(f"Model '{model_name}' has invalid tool_call_parser='{effective_tool_parser}'. "
                             f"Valid options: {valid_tool_parsers}")

        if override.chat_template is not None:
            # load_chat_template validates template content/format up front.
            load_chat_template(override.chat_template)

build_multi_model_engine_client async

build_multi_model_engine_client(
    args: Namespace,
    *,
    usage_context: UsageContext = OPENAI_API_SERVER,
) -> AsyncIterator[
    tuple[
        MultiModelEngineClient,
        MultiModelAsyncLLM,
        dict[str, BaseModelPath],
        dict[str, int],
        dict[str, ModelFrontendOverrides],
    ]
]

Source code in vllm_gaudi/entrypoints/openai/multi_model_api_server.py

@asynccontextmanager
async def build_multi_model_engine_client(
    args: Namespace,
    *,
    usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,
) -> AsyncIterator[tuple[
        MultiModelEngineClient,
        MultiModelAsyncLLM,
        dict[str, BaseModelPath],
        dict[str, int],
        dict[str, ModelFrontendOverrides],
]]:
    config_path = _resolve_multi_model_config_path()
    if not config_path:
        raise ValueError("A multi-model config path must be set when multi-model mode is enabled. "
                         "Supported env var: VLLM_HPU_MULTI_MODEL_CONFIG.")

    config = _load_multi_model_config(config_path)
    _validate_model_frontend_overrides(args, config.model_frontend_overrides)

    manager = MultiModelAsyncLLM(
        config.model_configs,
        usage_context=usage_context,
        disable_log_stats=args.disable_log_stats,
        enable_log_requests=args.enable_log_requests,
        model_quant_configs=config.model_quant_configs,
    )

    await manager.initialize(config.default_model)
    engine_client = MultiModelEngineClient(manager)
    await engine_client.reset_mm_cache()

    all_model_paths, model_max_lens = _build_model_registry(manager)

    try:
        yield engine_client, manager, all_model_paths, model_max_lens, config.model_frontend_overrides
    finally:
        manager.shutdown()