Skip to content

vllm.multimodal.profiling

_I module-attribute 

_I = TypeVar('_I', bound=BaseProcessingInfo)

logger module-attribute 

logger = init_logger(__name__)

BaseDummyInputsBuilder

Bases: ABC, Generic[_I]

Abstract base class that constructs the dummy data to profile multi-modal models.

Source code in vllm/multimodal/profiling.py 
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
class BaseDummyInputsBuilder(ABC, Generic[_I]):
    """
    Abstract base class that constructs the dummy data to profile
    multi-modal models.
    """

    def __init__(self, info: _I) -> None:
        super().__init__()

        self.info = info

    @abstractmethod
    def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
        """
        Build the text input corresponding to `mm_counts`.
        """
        raise NotImplementedError

    @abstractmethod
    def get_dummy_mm_data(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
    ) -> MultiModalDataDict:
        """
        Build the multimodal input which, after processing, results in
        the maximum possible number of placeholder tokens.
        """
        raise NotImplementedError

    def get_dummy_processor_inputs(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
    ) -> ProcessorInputs:
        """
        Build the input which, after processing, results in
        the maximum possible number of placeholder tokens.
        """
        dummy_text = self.get_dummy_text(mm_counts)
        dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts)

        return ProcessorInputs(prompt=dummy_text, mm_data=dummy_mm_data)

    def _get_dummy_audios(
        self,
        *,
        length: int,
        num_audios: int,
    ) -> list[npt.NDArray]:
        if num_audios == 0:
            return []
        audio = np.zeros((length, ))
        return [audio] * num_audios

    def _get_dummy_images(
        self,
        *,
        width: int,
        height: int,
        num_images: int,
    ) -> list[Image.Image]:
        if num_images == 0:
            return []
        image = Image.new("RGB", (width, height), color=255)
        return [image] * num_images

    def _get_dummy_videos(
        self,
        *,
        width: int,
        height: int,
        num_frames: int,
        num_videos: int,
    ) -> list[npt.NDArray]:
        if num_videos == 0:
            return []
        video = np.full((num_frames, width, height, 3), 255)
        return [video] * num_videos

info instance-attribute 

info = info

__init__ 

__init__(info: _I) -> None
Source code in vllm/multimodal/profiling.py 
58
59
60
61
def __init__(self, info: _I) -> None:
    super().__init__()

    self.info = info

_get_dummy_audios 

_get_dummy_audios(
    *, length: int, num_audios: int
) -> list[NDArray]
Source code in vllm/multimodal/profiling.py 
 96
 97
 98
 99
100
101
102
103
104
105
def _get_dummy_audios(
    self,
    *,
    length: int,
    num_audios: int,
) -> list[npt.NDArray]:
    if num_audios == 0:
        return []
    audio = np.zeros((length, ))
    return [audio] * num_audios

_get_dummy_images 

_get_dummy_images(
    *, width: int, height: int, num_images: int
) -> list[Image]
Source code in vllm/multimodal/profiling.py 
107
108
109
110
111
112
113
114
115
116
117
def _get_dummy_images(
    self,
    *,
    width: int,
    height: int,
    num_images: int,
) -> list[Image.Image]:
    if num_images == 0:
        return []
    image = Image.new("RGB", (width, height), color=255)
    return [image] * num_images

_get_dummy_videos 

_get_dummy_videos(
    *,
    width: int,
    height: int,
    num_frames: int,
    num_videos: int,
) -> list[NDArray]
Source code in vllm/multimodal/profiling.py 
119
120
121
122
123
124
125
126
127
128
129
130
def _get_dummy_videos(
    self,
    *,
    width: int,
    height: int,
    num_frames: int,
    num_videos: int,
) -> list[npt.NDArray]:
    if num_videos == 0:
        return []
    video = np.full((num_frames, width, height, 3), 255)
    return [video] * num_videos

get_dummy_mm_data abstractmethod 

get_dummy_mm_data(
    seq_len: int, mm_counts: Mapping[str, int]
) -> MultiModalDataDict

Build the multimodal input which, after processing, results in the maximum possible number of placeholder tokens.

Source code in vllm/multimodal/profiling.py 
70
71
72
73
74
75
76
77
78
79
80
@abstractmethod
def get_dummy_mm_data(
    self,
    seq_len: int,
    mm_counts: Mapping[str, int],
) -> MultiModalDataDict:
    """
    Build the multimodal input which, after processing, results in
    the maximum possible number of placeholder tokens.
    """
    raise NotImplementedError

get_dummy_processor_inputs 

get_dummy_processor_inputs(
    seq_len: int, mm_counts: Mapping[str, int]
) -> ProcessorInputs

Build the input which, after processing, results in the maximum possible number of placeholder tokens.

Source code in vllm/multimodal/profiling.py 
82
83
84
85
86
87
88
89
90
91
92
93
94
def get_dummy_processor_inputs(
    self,
    seq_len: int,
    mm_counts: Mapping[str, int],
) -> ProcessorInputs:
    """
    Build the input which, after processing, results in
    the maximum possible number of placeholder tokens.
    """
    dummy_text = self.get_dummy_text(mm_counts)
    dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts)

    return ProcessorInputs(prompt=dummy_text, mm_data=dummy_mm_data)

get_dummy_text abstractmethod 

get_dummy_text(mm_counts: Mapping[str, int]) -> str

Build the text input corresponding to mm_counts.

Source code in vllm/multimodal/profiling.py 
63
64
65
66
67
68
@abstractmethod
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
    """
    Build the text input corresponding to `mm_counts`.
    """
    raise NotImplementedError

DummyDecoderData

Bases: NamedTuple

Dummy data used for profiling.

Source code in vllm/multimodal/profiling.py 
41
42
43
44
45
46
class DummyDecoderData(NamedTuple):
    """Dummy data used for profiling."""

    prompt_token_ids: list[int]
    multi_modal_data: MultiModalKwargs
    multi_modal_placeholders: MultiModalPlaceholderDict

multi_modal_data instance-attribute 

multi_modal_data: MultiModalKwargs

multi_modal_placeholders instance-attribute 

multi_modal_placeholders: MultiModalPlaceholderDict

prompt_token_ids instance-attribute 

prompt_token_ids: list[int]

DummyEncoderData

Bases: NamedTuple

Dummy data used for profiling.

Source code in vllm/multimodal/profiling.py 
35
36
37
38
class DummyEncoderData(NamedTuple):
    """Dummy data used for profiling."""

    prompt_token_ids: list[int]

prompt_token_ids instance-attribute 

prompt_token_ids: list[int]

MultiModalProfiler

Bases: Generic[_I]

Contains code for running memory profiling for multi-modal models.

Source code in vllm/multimodal/profiling.py 
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
class MultiModalProfiler(Generic[_I]):
    """
    Contains code for running memory profiling for multi-modal models.
    """

    def __init__(
        self,
        processor: BaseMultiModalProcessor[_I],
    ) -> None:
        super().__init__()

        self.processor = processor

    @property
    def processing_info(self) -> BaseProcessingInfo:
        return self.processor.info

    @property
    def dummy_inputs(self) -> BaseDummyInputsBuilder[_I]:
        return self.processor.dummy_inputs

    def get_mm_limits(self) -> Mapping[str, int]:
        return self.processing_info.get_allowed_mm_limits()

    def _get_dummy_mm_inputs(
        self,
        seq_len: int,
        mm_counts: Optional[Mapping[str, int]] = None,
    ) -> MultiModalInputs:
        if mm_counts is None:
            mm_counts = self.get_mm_limits()

        factory = self.dummy_inputs
        processor_inputs = factory.get_dummy_processor_inputs(
            seq_len, mm_counts)

        return self.processor.apply(
            prompt=processor_inputs.prompt,
            mm_data=processor_inputs.mm_data,
            hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
        )

    def _get_mm_num_tokens(
        self,
        mm_inputs: MultiModalInputs,
    ) -> Mapping[str, int]:
        placeholders_by_modality = mm_inputs["mm_placeholders"]

        return {
            modality: sum(item.get_num_embeds() for item in placeholders)
            for modality, placeholders in placeholders_by_modality.items()
        }

    def get_encoder_dummy_data(
        self,
        seq_len: int,
        mm_counts: Optional[Mapping[str, int]] = None,
    ) -> DummyEncoderData:
        mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts)
        mm_inputs = cast(MultiModalEncDecInputs, mm_inputs)

        # For encoder-decoder models, use encoder prompt token ids instead of
        # decoder prompt to construct dummy seq_data for encoder profiling.
        encoder_prompt_token_ids = mm_inputs["encoder_prompt_token_ids"]

        total_len = len(encoder_prompt_token_ids)

        processor = cast(EncDecMultiModalProcessor, self.processor)
        if processor.pad_dummy_encoder_prompt:
            num_tokens_to_pad = max(total_len, seq_len) - total_len
            encoder_prompt_token_ids.extend([0] * num_tokens_to_pad)
        # NOTE: Whisper allows total_len > seq_len.
        elif total_len > seq_len and not envs.VLLM_USE_V1:
            # `max_num_batched_tokens` is defined by `SchedulerConfig`
            logger.warning_once(
                "The encoder sequence length used for profiling (max_num_batched_tokens / max_num_seqs = %d) "  # noqa: E501
                "is too short to hold the multi-modal embeddings in the worst case (%d tokens in total, out of which %s are reserved for multi-modal embeddings). "  # noqa: E501
                "This may cause certain multi-modal inputs to fail during inference, even when the input text is short. "  # noqa: E501
                "To avoid this, you should increase `max_model_len`, reduce `max_num_seqs`, and/or reduce `mm_counts`.",  # noqa: E501
                seq_len,
                total_len,
                str(self._get_mm_num_tokens(mm_inputs)),
            )

        return DummyEncoderData(encoder_prompt_token_ids)

    def get_decoder_dummy_data(
        self,
        seq_len: int,
        mm_counts: Optional[Mapping[str, int]] = None,
    ) -> DummyDecoderData:
        mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts)

        prompt_token_ids = mm_inputs["prompt_token_ids"]
        total_len = len(prompt_token_ids)

        # V0 does not support chunked prefill.
        if total_len > seq_len and not envs.VLLM_USE_V1:
            # `max_num_batched_tokens` is defined by `SchedulerConfig`
            logger.warning_once(
                "The sequence length used for profiling (max_num_batched_tokens / max_num_seqs = %d) "  # noqa: E501
                "is too short to hold the multi-modal embeddings in the worst case (%d tokens in total, out of which %s are reserved for multi-modal embeddings). "  # noqa: E501
                "This may cause certain multi-modal inputs to fail during inference, even when the input text is short. "  # noqa: E501
                "To avoid this, you should increase `max_model_len`, reduce `max_num_seqs`, and/or reduce `mm_counts`.",  # noqa: E501
                seq_len,
                total_len,
                str(self._get_mm_num_tokens(mm_inputs)),
            )

        if total_len < seq_len:
            prompt_token_ids.extend([0] * (seq_len - total_len))

        return DummyDecoderData(
            prompt_token_ids=prompt_token_ids,
            multi_modal_data=mm_inputs["mm_kwargs"],
            multi_modal_placeholders=mm_inputs["mm_placeholders"],
        )

    def get_mm_max_tokens(
        self,
        seq_len: int,
        mm_counts: Optional[Mapping[str, int]] = None,
    ) -> Mapping[str, int]:
        mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts)

        return self._get_mm_num_tokens(mm_inputs)

dummy_inputs property 

dummy_inputs: BaseDummyInputsBuilder[_I]

processing_info property 

processing_info: BaseProcessingInfo

processor instance-attribute 

processor = processor

__init__ 

__init__(processor: BaseMultiModalProcessor[_I]) -> None
Source code in vllm/multimodal/profiling.py 
138
139
140
141
142
143
144
def __init__(
    self,
    processor: BaseMultiModalProcessor[_I],
) -> None:
    super().__init__()

    self.processor = processor

_get_dummy_mm_inputs 

_get_dummy_mm_inputs(
    seq_len: int,
    mm_counts: Optional[Mapping[str, int]] = None,
) -> MultiModalInputs
Source code in vllm/multimodal/profiling.py 
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
def _get_dummy_mm_inputs(
    self,
    seq_len: int,
    mm_counts: Optional[Mapping[str, int]] = None,
) -> MultiModalInputs:
    if mm_counts is None:
        mm_counts = self.get_mm_limits()

    factory = self.dummy_inputs
    processor_inputs = factory.get_dummy_processor_inputs(
        seq_len, mm_counts)

    return self.processor.apply(
        prompt=processor_inputs.prompt,
        mm_data=processor_inputs.mm_data,
        hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
    )

_get_mm_num_tokens 

_get_mm_num_tokens(
    mm_inputs: MultiModalInputs,
) -> Mapping[str, int]
Source code in vllm/multimodal/profiling.py 
175
176
177
178
179
180
181
182
183
184
def _get_mm_num_tokens(
    self,
    mm_inputs: MultiModalInputs,
) -> Mapping[str, int]:
    placeholders_by_modality = mm_inputs["mm_placeholders"]

    return {
        modality: sum(item.get_num_embeds() for item in placeholders)
        for modality, placeholders in placeholders_by_modality.items()
    }

get_decoder_dummy_data 

get_decoder_dummy_data(
    seq_len: int,
    mm_counts: Optional[Mapping[str, int]] = None,
) -> DummyDecoderData
Source code in vllm/multimodal/profiling.py 
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
def get_decoder_dummy_data(
    self,
    seq_len: int,
    mm_counts: Optional[Mapping[str, int]] = None,
) -> DummyDecoderData:
    mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts)

    prompt_token_ids = mm_inputs["prompt_token_ids"]
    total_len = len(prompt_token_ids)

    # V0 does not support chunked prefill.
    if total_len > seq_len and not envs.VLLM_USE_V1:
        # `max_num_batched_tokens` is defined by `SchedulerConfig`
        logger.warning_once(
            "The sequence length used for profiling (max_num_batched_tokens / max_num_seqs = %d) "  # noqa: E501
            "is too short to hold the multi-modal embeddings in the worst case (%d tokens in total, out of which %s are reserved for multi-modal embeddings). "  # noqa: E501
            "This may cause certain multi-modal inputs to fail during inference, even when the input text is short. "  # noqa: E501
            "To avoid this, you should increase `max_model_len`, reduce `max_num_seqs`, and/or reduce `mm_counts`.",  # noqa: E501
            seq_len,
            total_len,
            str(self._get_mm_num_tokens(mm_inputs)),
        )

    if total_len < seq_len:
        prompt_token_ids.extend([0] * (seq_len - total_len))

    return DummyDecoderData(
        prompt_token_ids=prompt_token_ids,
        multi_modal_data=mm_inputs["mm_kwargs"],
        multi_modal_placeholders=mm_inputs["mm_placeholders"],
    )

get_encoder_dummy_data 

get_encoder_dummy_data(
    seq_len: int,
    mm_counts: Optional[Mapping[str, int]] = None,
) -> DummyEncoderData
Source code in vllm/multimodal/profiling.py 
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
def get_encoder_dummy_data(
    self,
    seq_len: int,
    mm_counts: Optional[Mapping[str, int]] = None,
) -> DummyEncoderData:
    mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts)
    mm_inputs = cast(MultiModalEncDecInputs, mm_inputs)

    # For encoder-decoder models, use encoder prompt token ids instead of
    # decoder prompt to construct dummy seq_data for encoder profiling.
    encoder_prompt_token_ids = mm_inputs["encoder_prompt_token_ids"]

    total_len = len(encoder_prompt_token_ids)

    processor = cast(EncDecMultiModalProcessor, self.processor)
    if processor.pad_dummy_encoder_prompt:
        num_tokens_to_pad = max(total_len, seq_len) - total_len
        encoder_prompt_token_ids.extend([0] * num_tokens_to_pad)
    # NOTE: Whisper allows total_len > seq_len.
    elif total_len > seq_len and not envs.VLLM_USE_V1:
        # `max_num_batched_tokens` is defined by `SchedulerConfig`
        logger.warning_once(
            "The encoder sequence length used for profiling (max_num_batched_tokens / max_num_seqs = %d) "  # noqa: E501
            "is too short to hold the multi-modal embeddings in the worst case (%d tokens in total, out of which %s are reserved for multi-modal embeddings). "  # noqa: E501
            "This may cause certain multi-modal inputs to fail during inference, even when the input text is short. "  # noqa: E501
            "To avoid this, you should increase `max_model_len`, reduce `max_num_seqs`, and/or reduce `mm_counts`.",  # noqa: E501
            seq_len,
            total_len,
            str(self._get_mm_num_tokens(mm_inputs)),
        )

    return DummyEncoderData(encoder_prompt_token_ids)

get_mm_limits 

get_mm_limits() -> Mapping[str, int]
Source code in vllm/multimodal/profiling.py 
154
155
def get_mm_limits(self) -> Mapping[str, int]:
    return self.processing_info.get_allowed_mm_limits()

get_mm_max_tokens 

get_mm_max_tokens(
    seq_len: int,
    mm_counts: Optional[Mapping[str, int]] = None,
) -> Mapping[str, int]
Source code in vllm/multimodal/profiling.py 
251
252
253
254
255
256
257
258
def get_mm_max_tokens(
    self,
    seq_len: int,
    mm_counts: Optional[Mapping[str, int]] = None,
) -> Mapping[str, int]:
    mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts)

    return self._get_mm_num_tokens(mm_inputs)

ProcessorInputs dataclass

Represents the keyword arguments to vllm.multimodal.processing.BaseMultiModalProcessor.apply.

Source code in vllm/multimodal/profiling.py 
24
25
26
27
28
29
30
31
32
@dataclass
class ProcessorInputs:
    """
    Represents the keyword arguments to
    [`vllm.multimodal.processing.BaseMultiModalProcessor.apply`][].
    """
    prompt: Union[str, list[int]]
    mm_data: MultiModalDataDict
    hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)

hf_processor_mm_kwargs class-attribute instance-attribute 

hf_processor_mm_kwargs: Mapping[str, object] = field(
    default_factory=dict
)

mm_data instance-attribute 

mm_data: MultiModalDataDict

prompt instance-attribute 

prompt: Union[str, list[int]]

__init__ 

__init__(
    prompt: Union[str, list[int]],
    mm_data: MultiModalDataDict,
    hf_processor_mm_kwargs: Mapping[str, object] = dict(),
) -> None