Skip to content

vllm.entrypoints.cli.serve

logger module-attribute 

logger = init_logger(__name__)

ServeSubcommand

Bases: CLISubcommand

The serve subcommand for the vLLM CLI.

Source code in vllm/entrypoints/cli/serve.py 
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
class ServeSubcommand(CLISubcommand):
    """The `serve` subcommand for the vLLM CLI. """

    def __init__(self):
        self.name = "serve"
        super().__init__()

    @staticmethod
    def cmd(args: argparse.Namespace) -> None:
        # If model is specified in CLI (as positional arg), it takes precedence
        if hasattr(args, 'model_tag') and args.model_tag is not None:
            args.model = args.model_tag

        if args.headless:
            run_headless(args)
        else:
            uvloop.run(run_server(args))

    def validate(self, args: argparse.Namespace) -> None:
        validate_parsed_serve_args(args)

    def subparser_init(
            self,
            subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
        serve_parser = subparsers.add_parser(
            "serve",
            help="Start the vLLM OpenAI Compatible API server.",
            description="Start the vLLM OpenAI Compatible API server.",
            usage="vllm serve [model_tag] [options]")
        serve_parser.add_argument("model_tag",
                                  type=str,
                                  nargs='?',
                                  help="The model tag to serve "
                                  "(optional if specified in config)")
        serve_parser.add_argument(
            "--headless",
            action='store_true',
            default=False,
            help="Run in headless mode. See multi-node data parallel "
            "documentation for more details.")
        serve_parser.add_argument(
            '--data-parallel-start-rank',
            '-dpr',
            type=int,
            default=0,
            help='Starting data parallel rank for secondary nodes.')
        serve_parser.add_argument(
            "--config",
            type=str,
            default='',
            required=False,
            help="Read CLI options from a config file."
            "Must be a YAML with the following options:"
            "https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html#cli-reference"
        )

        serve_parser = make_arg_parser(serve_parser)
        show_filtered_argument_or_group_from_help(serve_parser)
        serve_parser.epilog = VLLM_SERVE_PARSER_EPILOG
        return serve_parser

name instance-attribute 

name = 'serve'

__init__ 

__init__()
Source code in vllm/entrypoints/cli/serve.py 
29
30
31
def __init__(self):
    self.name = "serve"
    super().__init__()

cmd staticmethod 

cmd(args: Namespace) -> None
Source code in vllm/entrypoints/cli/serve.py 
33
34
35
36
37
38
39
40
41
42
@staticmethod
def cmd(args: argparse.Namespace) -> None:
    # If model is specified in CLI (as positional arg), it takes precedence
    if hasattr(args, 'model_tag') and args.model_tag is not None:
        args.model = args.model_tag

    if args.headless:
        run_headless(args)
    else:
        uvloop.run(run_server(args))

subparser_init 

subparser_init(
    subparsers: _SubParsersAction,
) -> FlexibleArgumentParser
Source code in vllm/entrypoints/cli/serve.py 
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
def subparser_init(
        self,
        subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
    serve_parser = subparsers.add_parser(
        "serve",
        help="Start the vLLM OpenAI Compatible API server.",
        description="Start the vLLM OpenAI Compatible API server.",
        usage="vllm serve [model_tag] [options]")
    serve_parser.add_argument("model_tag",
                              type=str,
                              nargs='?',
                              help="The model tag to serve "
                              "(optional if specified in config)")
    serve_parser.add_argument(
        "--headless",
        action='store_true',
        default=False,
        help="Run in headless mode. See multi-node data parallel "
        "documentation for more details.")
    serve_parser.add_argument(
        '--data-parallel-start-rank',
        '-dpr',
        type=int,
        default=0,
        help='Starting data parallel rank for secondary nodes.')
    serve_parser.add_argument(
        "--config",
        type=str,
        default='',
        required=False,
        help="Read CLI options from a config file."
        "Must be a YAML with the following options:"
        "https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html#cli-reference"
    )

    serve_parser = make_arg_parser(serve_parser)
    show_filtered_argument_or_group_from_help(serve_parser)
    serve_parser.epilog = VLLM_SERVE_PARSER_EPILOG
    return serve_parser

validate 

validate(args: Namespace) -> None
Source code in vllm/entrypoints/cli/serve.py 
44
45
def validate(self, args: argparse.Namespace) -> None:
    validate_parsed_serve_args(args)

cmd_init 

cmd_init() -> list[CLISubcommand]
Source code in vllm/entrypoints/cli/serve.py 
88
89
def cmd_init() -> list[CLISubcommand]:
    return [ServeSubcommand()]

run_headless 

run_headless(args: Namespace)
Source code in vllm/entrypoints/cli/serve.py 
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
def run_headless(args: argparse.Namespace):

    # Create the EngineConfig.
    engine_args = AsyncEngineArgs.from_cli_args(args)
    usage_context = UsageContext.OPENAI_API_SERVER
    vllm_config = engine_args.create_engine_config(usage_context=usage_context)

    if not envs.VLLM_USE_V1:
        raise RuntimeError("Headless mode is only supported for V1")

    parallel_config = vllm_config.parallel_config
    local_engine_count = parallel_config.data_parallel_size_local
    host = parallel_config.data_parallel_master_ip
    port = engine_args.data_parallel_rpc_port  # add to config too
    input_address = get_tcp_uri(host, port)

    if local_engine_count <= 0:
        raise RuntimeError("data_parallel_size_local must be > 0 in "
                           "headless mode")

    # Catch SIGTERM and SIGINT to allow graceful shutdown.
    def signal_handler(signum, frame):
        logger.debug("Received %d signal.", signum)
        raise SystemExit

    signal.signal(signal.SIGTERM, signal_handler)
    signal.signal(signal.SIGINT, signal_handler)

    logger.info(
        "Launching %d data parallel engine(s) in headless mode, "
        "with head node address %s.", local_engine_count, input_address)

    # Create the engines.
    engine_manager = CoreEngineProcManager(
        target_fn=EngineCoreProc.run_engine_core,
        local_engine_count=local_engine_count,
        start_index=args.data_parallel_start_rank,
        local_start_index=0,
        vllm_config=vllm_config,
        on_head_node=False,
        input_address=input_address,
        executor_class=Executor.get_class(vllm_config),
        log_stats=not engine_args.disable_log_stats,
    )

    try:
        engine_manager.join_first()
    finally:
        logger.info("Shutting down.")
        engine_manager.close()