Skip to content

vllm.v1.request

_FINISHED_REASON_MAP module-attribute 

_FINISHED_REASON_MAP = {
    FINISHED_STOPPED: STOP,
    FINISHED_LENGTH_CAPPED: LENGTH,
    FINISHED_ABORTED: ABORT,
    FINISHED_IGNORED: LENGTH,
}

Request

Source code in vllm/v1/request.py 
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
class Request:

    def __init__(
        self,
        request_id: str,
        prompt_token_ids: list[int],
        multi_modal_inputs: Optional[list[MultiModalKwargs]],
        multi_modal_hashes: Optional[list[str]],
        multi_modal_placeholders: Optional[list[PlaceholderRange]],
        sampling_params: SamplingParams,
        eos_token_id: Optional[int],
        arrival_time: float,
        lora_request: Optional["LoRARequest"] = None,
        structured_output_request: Optional["StructuredOutputRequest"] = None,
        cache_salt: Optional[str] = None,
    ) -> None:
        self.request_id = request_id
        self.sampling_params = sampling_params
        # Because of LoRA, the eos token id can be different for each request.
        self.eos_token_id = eos_token_id
        self.lora_request = lora_request
        self.structured_output_request = structured_output_request

        self.status = (RequestStatus.WAITING_FOR_FSM
                       if sampling_params.guided_decoding is not None else
                       RequestStatus.WAITING)
        self.events: list[EngineCoreEvent] = []
        self.stop_reason: Union[int, str, None] = None
        assert sampling_params.max_tokens is not None
        self.max_tokens = sampling_params.max_tokens

        self.prompt_token_ids = prompt_token_ids
        self.num_prompt_tokens = len(self.prompt_token_ids)
        self._output_token_ids: list[int] = []
        self._all_token_ids: list[int] = self.prompt_token_ids.copy()
        self.spec_token_ids: list[int] = []
        self.num_computed_tokens = 0
        self.cache_salt: Optional[str] = cache_salt

        # Multi-modal related
        self.mm_positions = multi_modal_placeholders or []
        self.mm_inputs = multi_modal_inputs or []
        self.mm_hashes: list[str] = multi_modal_hashes or []
        self.num_encoder_inputs = len(self.mm_inputs)
        self.has_encoder_inputs = self.num_encoder_inputs > 0

        # P/D: Connector-specific KV transfer parameters.
        kv_params = (None if sampling_params.extra_args is None else
                     sampling_params.extra_args.get("kv_transfer_params"))
        self.kv_transfer_params: Optional[dict[str, Any]] = kv_params

        # Sanity check
        assert len(self.mm_inputs) == len(self.mm_positions)
        if self.mm_hashes:
            assert len(self.mm_inputs) == len(self.mm_hashes)

        # Read-only views
        # Prevent directly appending to these lists since
        # they should also be updated simultaneously.
        self.output_token_ids = ConstantList(self._output_token_ids)
        self.all_token_ids = ConstantList(self._all_token_ids)

        # State
        # The number of tokens with prefix cache hits.
        self.num_cached_tokens = -1

    @classmethod
    def from_engine_core_request(cls, request: EngineCoreRequest) -> "Request":
        if request.mm_inputs is not None:
            assert isinstance(request.mm_inputs, list)
            assert is_list_of(request.mm_inputs, MultiModalKwargs), (
                "mm_inputs was not updated in EngineCore.add_request")

        return cls(
            request_id=request.request_id,
            prompt_token_ids=request.prompt_token_ids,
            multi_modal_inputs=request.mm_inputs,
            multi_modal_hashes=request.mm_hashes,
            multi_modal_placeholders=request.mm_placeholders,
            sampling_params=request.sampling_params,
            eos_token_id=request.eos_token_id,
            arrival_time=request.arrival_time,
            lora_request=request.lora_request,
            structured_output_request=StructuredOutputRequest(
                sampling_params=request.sampling_params),
            cache_salt=request.cache_salt,
        )

    def append_output_token_ids(
        self,
        token_ids: Union[int, list[int]],
    ) -> None:
        if isinstance(token_ids, int):
            self._output_token_ids.append(token_ids)
            self._all_token_ids.append(token_ids)
        else:
            self._output_token_ids.extend(token_ids)
            self._all_token_ids.extend(token_ids)

    @property
    def num_tokens(self) -> int:
        return len(self._all_token_ids)

    @property
    def num_tokens_with_spec(self) -> int:
        return len(self._all_token_ids) + len(self.spec_token_ids)

    @property
    def num_output_tokens(self) -> int:
        return len(self._output_token_ids)

    def is_finished(self) -> bool:
        return RequestStatus.is_finished(self.status)

    def get_finished_reason(self) -> Union[FinishReason, None]:
        return RequestStatus.get_finished_reason(self.status)

    def get_num_encoder_tokens(self, input_id: int) -> int:
        assert input_id < len(self.mm_positions)
        num_tokens = self.mm_positions[input_id].length
        return num_tokens

    @property
    def use_structured_output(self) -> bool:
        return self.sampling_params.guided_decoding is not None

    def record_event(
        self,
        event_type: EngineCoreEventType,
        timestamp: Optional[float] = None,
    ) -> None:
        self.events.append(EngineCoreEvent.new_event(event_type, timestamp))

    def take_events(self) -> Optional[list[EngineCoreEvent]]:
        if not self.events:
            return None
        events, self.events = self.events, []
        return events

_all_token_ids instance-attribute 

_all_token_ids: list[int] = copy()

_output_token_ids instance-attribute 

_output_token_ids: list[int] = []

all_token_ids instance-attribute 

all_token_ids = ConstantList(_all_token_ids)

cache_salt instance-attribute 

cache_salt: Optional[str] = cache_salt

eos_token_id instance-attribute 

eos_token_id = eos_token_id

events instance-attribute 

events: list[EngineCoreEvent] = []

has_encoder_inputs instance-attribute 

has_encoder_inputs = num_encoder_inputs > 0

kv_transfer_params instance-attribute 

kv_transfer_params: Optional[dict[str, Any]] = kv_params

lora_request instance-attribute 

lora_request = lora_request

max_tokens instance-attribute 

max_tokens = max_tokens

mm_hashes instance-attribute 

mm_hashes: list[str] = multi_modal_hashes or []

mm_inputs instance-attribute 

mm_inputs = multi_modal_inputs or []

mm_positions instance-attribute 

mm_positions = multi_modal_placeholders or []

num_cached_tokens instance-attribute 

num_cached_tokens = -1

num_computed_tokens instance-attribute 

num_computed_tokens = 0

num_encoder_inputs instance-attribute 

num_encoder_inputs = len(mm_inputs)

num_output_tokens property 

num_output_tokens: int

num_prompt_tokens instance-attribute 

num_prompt_tokens = len(prompt_token_ids)

num_tokens property 

num_tokens: int

num_tokens_with_spec property 

num_tokens_with_spec: int

output_token_ids instance-attribute 

output_token_ids = ConstantList(_output_token_ids)

prompt_token_ids instance-attribute 

prompt_token_ids = prompt_token_ids

request_id instance-attribute 

request_id = request_id

sampling_params instance-attribute 

sampling_params = sampling_params

spec_token_ids instance-attribute 

spec_token_ids: list[int] = []

status instance-attribute 

status = (
    WAITING_FOR_FSM
    if guided_decoding is not None
    else WAITING
)

stop_reason instance-attribute 

stop_reason: Union[int, str, None] = None

structured_output_request instance-attribute 

structured_output_request = structured_output_request

use_structured_output property 

use_structured_output: bool

__init__ 

__init__(
    request_id: str,
    prompt_token_ids: list[int],
    multi_modal_inputs: Optional[list[MultiModalKwargs]],
    multi_modal_hashes: Optional[list[str]],
    multi_modal_placeholders: Optional[
        list[PlaceholderRange]
    ],
    sampling_params: SamplingParams,
    eos_token_id: Optional[int],
    arrival_time: float,
    lora_request: Optional[LoRARequest] = None,
    structured_output_request: Optional[
        StructuredOutputRequest
    ] = None,
    cache_salt: Optional[str] = None,
) -> None
Source code in vllm/v1/request.py 
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
def __init__(
    self,
    request_id: str,
    prompt_token_ids: list[int],
    multi_modal_inputs: Optional[list[MultiModalKwargs]],
    multi_modal_hashes: Optional[list[str]],
    multi_modal_placeholders: Optional[list[PlaceholderRange]],
    sampling_params: SamplingParams,
    eos_token_id: Optional[int],
    arrival_time: float,
    lora_request: Optional["LoRARequest"] = None,
    structured_output_request: Optional["StructuredOutputRequest"] = None,
    cache_salt: Optional[str] = None,
) -> None:
    self.request_id = request_id
    self.sampling_params = sampling_params
    # Because of LoRA, the eos token id can be different for each request.
    self.eos_token_id = eos_token_id
    self.lora_request = lora_request
    self.structured_output_request = structured_output_request

    self.status = (RequestStatus.WAITING_FOR_FSM
                   if sampling_params.guided_decoding is not None else
                   RequestStatus.WAITING)
    self.events: list[EngineCoreEvent] = []
    self.stop_reason: Union[int, str, None] = None
    assert sampling_params.max_tokens is not None
    self.max_tokens = sampling_params.max_tokens

    self.prompt_token_ids = prompt_token_ids
    self.num_prompt_tokens = len(self.prompt_token_ids)
    self._output_token_ids: list[int] = []
    self._all_token_ids: list[int] = self.prompt_token_ids.copy()
    self.spec_token_ids: list[int] = []
    self.num_computed_tokens = 0
    self.cache_salt: Optional[str] = cache_salt

    # Multi-modal related
    self.mm_positions = multi_modal_placeholders or []
    self.mm_inputs = multi_modal_inputs or []
    self.mm_hashes: list[str] = multi_modal_hashes or []
    self.num_encoder_inputs = len(self.mm_inputs)
    self.has_encoder_inputs = self.num_encoder_inputs > 0

    # P/D: Connector-specific KV transfer parameters.
    kv_params = (None if sampling_params.extra_args is None else
                 sampling_params.extra_args.get("kv_transfer_params"))
    self.kv_transfer_params: Optional[dict[str, Any]] = kv_params

    # Sanity check
    assert len(self.mm_inputs) == len(self.mm_positions)
    if self.mm_hashes:
        assert len(self.mm_inputs) == len(self.mm_hashes)

    # Read-only views
    # Prevent directly appending to these lists since
    # they should also be updated simultaneously.
    self.output_token_ids = ConstantList(self._output_token_ids)
    self.all_token_ids = ConstantList(self._all_token_ids)

    # State
    # The number of tokens with prefix cache hits.
    self.num_cached_tokens = -1

append_output_token_ids 

append_output_token_ids(
    token_ids: Union[int, list[int]],
) -> None
Source code in vllm/v1/request.py 
106
107
108
109
110
111
112
113
114
115
def append_output_token_ids(
    self,
    token_ids: Union[int, list[int]],
) -> None:
    if isinstance(token_ids, int):
        self._output_token_ids.append(token_ids)
        self._all_token_ids.append(token_ids)
    else:
        self._output_token_ids.extend(token_ids)
        self._all_token_ids.extend(token_ids)

from_engine_core_request classmethod 

from_engine_core_request(
    request: EngineCoreRequest,
) -> Request
Source code in vllm/v1/request.py 
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
@classmethod
def from_engine_core_request(cls, request: EngineCoreRequest) -> "Request":
    if request.mm_inputs is not None:
        assert isinstance(request.mm_inputs, list)
        assert is_list_of(request.mm_inputs, MultiModalKwargs), (
            "mm_inputs was not updated in EngineCore.add_request")

    return cls(
        request_id=request.request_id,
        prompt_token_ids=request.prompt_token_ids,
        multi_modal_inputs=request.mm_inputs,
        multi_modal_hashes=request.mm_hashes,
        multi_modal_placeholders=request.mm_placeholders,
        sampling_params=request.sampling_params,
        eos_token_id=request.eos_token_id,
        arrival_time=request.arrival_time,
        lora_request=request.lora_request,
        structured_output_request=StructuredOutputRequest(
            sampling_params=request.sampling_params),
        cache_salt=request.cache_salt,
    )

get_finished_reason 

get_finished_reason() -> Union[FinishReason, None]
Source code in vllm/v1/request.py 
132
133
def get_finished_reason(self) -> Union[FinishReason, None]:
    return RequestStatus.get_finished_reason(self.status)

get_num_encoder_tokens 

get_num_encoder_tokens(input_id: int) -> int
Source code in vllm/v1/request.py 
135
136
137
138
def get_num_encoder_tokens(self, input_id: int) -> int:
    assert input_id < len(self.mm_positions)
    num_tokens = self.mm_positions[input_id].length
    return num_tokens

is_finished 

is_finished() -> bool
Source code in vllm/v1/request.py 
129
130
def is_finished(self) -> bool:
    return RequestStatus.is_finished(self.status)

record_event 

record_event(
    event_type: EngineCoreEventType,
    timestamp: Optional[float] = None,
) -> None
Source code in vllm/v1/request.py 
144
145
146
147
148
149
def record_event(
    self,
    event_type: EngineCoreEventType,
    timestamp: Optional[float] = None,
) -> None:
    self.events.append(EngineCoreEvent.new_event(event_type, timestamp))

take_events 

take_events() -> Optional[list[EngineCoreEvent]]
Source code in vllm/v1/request.py 
151
152
153
154
155
def take_events(self) -> Optional[list[EngineCoreEvent]]:
    if not self.events:
        return None
    events, self.events = self.events, []
    return events

RequestStatus

Bases: IntEnum

Status of a request.

Source code in vllm/v1/request.py 
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
class RequestStatus(enum.IntEnum):
    """Status of a request."""
    WAITING = enum.auto()
    WAITING_FOR_FSM = enum.auto()
    WAITING_FOR_REMOTE_KVS = enum.auto()
    RUNNING = enum.auto()
    PREEMPTED = enum.auto()
    # Note: anything after PREEMPTED will be considered
    # as a finished status.
    FINISHED_STOPPED = enum.auto()
    FINISHED_LENGTH_CAPPED = enum.auto()
    FINISHED_ABORTED = enum.auto()
    FINISHED_IGNORED = enum.auto()

    @staticmethod
    def is_finished(status: "RequestStatus") -> bool:
        return status > RequestStatus.PREEMPTED

    @staticmethod
    def get_finished_reason(
            status: "RequestStatus") -> Union[FinishReason, None]:
        return _FINISHED_REASON_MAP.get(status)

FINISHED_ABORTED class-attribute instance-attribute 

FINISHED_ABORTED = auto()

FINISHED_IGNORED class-attribute instance-attribute 

FINISHED_IGNORED = auto()

FINISHED_LENGTH_CAPPED class-attribute instance-attribute 

FINISHED_LENGTH_CAPPED = auto()

FINISHED_STOPPED class-attribute instance-attribute 

FINISHED_STOPPED = auto()

PREEMPTED class-attribute instance-attribute 

PREEMPTED = auto()

RUNNING class-attribute instance-attribute 

RUNNING = auto()

WAITING class-attribute instance-attribute 

WAITING = auto()

WAITING_FOR_FSM class-attribute instance-attribute 

WAITING_FOR_FSM = auto()

WAITING_FOR_REMOTE_KVS class-attribute instance-attribute 

WAITING_FOR_REMOTE_KVS = auto()

get_finished_reason staticmethod 

get_finished_reason(
    status: RequestStatus,
) -> Union[FinishReason, None]
Source code in vllm/v1/request.py 
176
177
178
179
@staticmethod
def get_finished_reason(
        status: "RequestStatus") -> Union[FinishReason, None]:
    return _FINISHED_REASON_MAP.get(status)

is_finished staticmethod 

is_finished(status: RequestStatus) -> bool
Source code in vllm/v1/request.py 
172
173
174
@staticmethod
def is_finished(status: "RequestStatus") -> bool:
    return status > RequestStatus.PREEMPTED