Skip to content

vllm.multimodal.parse

ModalityDataParser module-attribute 

ModalityDataParser: TypeAlias = Callable[
    [ModalityData[Any]],
    Optional[ModalityDataItems[Any, Any]],
]

_D module-attribute 

_D = TypeVar('_D', bound=ModalityDataItems[Any, Any])

_I module-attribute 

_I = TypeVar('_I')

_T module-attribute 

_T = TypeVar('_T')

AudioEmbeddingItems

Bases: EmbeddingItems

Source code in vllm/multimodal/parse.py 
191
192
193
194
class AudioEmbeddingItems(EmbeddingItems):

    def __init__(self, data: Union[torch.Tensor, list[torch.Tensor]]) -> None:
        super().__init__(data, "audio")

__init__ 

__init__(data: Union[Tensor, list[Tensor]]) -> None
Source code in vllm/multimodal/parse.py 
193
194
def __init__(self, data: Union[torch.Tensor, list[torch.Tensor]]) -> None:
    super().__init__(data, "audio")

AudioProcessorItems

Bases: ProcessorBatchItems[HfAudioItem]

Source code in vllm/multimodal/parse.py 
181
182
183
184
185
186
187
188
class AudioProcessorItems(ProcessorBatchItems[HfAudioItem]):

    def __init__(self, data: Sequence[HfAudioItem]) -> None:
        super().__init__(data, "audio")

    def get_audio_length(self, item_idx: int) -> int:
        audio = self.get(item_idx)
        return len(audio)

__init__ 

__init__(data: Sequence[HfAudioItem]) -> None
Source code in vllm/multimodal/parse.py 
183
184
def __init__(self, data: Sequence[HfAudioItem]) -> None:
    super().__init__(data, "audio")

get_audio_length 

get_audio_length(item_idx: int) -> int
Source code in vllm/multimodal/parse.py 
186
187
188
def get_audio_length(self, item_idx: int) -> int:
    audio = self.get(item_idx)
    return len(audio)

DictEmbeddingItems

Bases: ModalityDataItems[Mapping[str, Tensor], Mapping[str, Tensor]]

Base class for data items that are expressed as a dictionary of tensors.

Usually, the dictionary keys correspond to the outputs of HF processor.

Source code in vllm/multimodal/parse.py 
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
class DictEmbeddingItems(ModalityDataItems[Mapping[str, torch.Tensor],
                                           Mapping[str, torch.Tensor]]):
    """
    Base class for data items that are expressed as a dictionary of tensors.

    Usually, the dictionary keys correspond to the outputs of HF processor.
    """

    def __init__(
        self,
        data: Mapping[str, torch.Tensor],
        modality: str,
        required_fields: set[str],
        fields_factory: Callable[
            [Mapping[str, torch.Tensor]],
            Mapping[str, MultiModalFieldConfig],
        ],
    ) -> None:
        from transformers.feature_extraction_utils import BatchFeature

        super().__init__(data, modality)

        missing_required_data_keys = required_fields - data.keys()
        if missing_required_data_keys:
            data_keys = set(data.keys())
            msg = (f"The data should contain the fields: {required_fields}, "
                   f"but only found the following keys: {data_keys}")
            raise ValueError(msg)

        fields_config = fields_factory(data)
        missing_required_fields = required_fields - fields_config.keys()
        if missing_required_fields:
            fields = set(fields_config.keys())
            msg = f"{required_fields=} should be a subset of {fields=}"
            raise ValueError(msg)

        self.fields_config = fields_config
        self.required_fields = required_fields

        self._kwargs = MultiModalKwargs.from_hf_inputs(
            BatchFeature(dict(data)),
            fields_config,
        )

    def get_count(self) -> int:
        return self._kwargs.get_item_count(self.modality)

    def get(self, index: int) -> Mapping[str, torch.Tensor]:
        return {
            k: v.data
            for k, v in self._kwargs.get_item(self.modality, index).items()
        }

    def get_processor_data(self) -> Mapping[str, object]:
        return {}

    def get_passthrough_data(self) -> Mapping[str, object]:
        return self.data

_kwargs instance-attribute 

_kwargs = from_hf_inputs(
    BatchFeature(dict(data)), fields_config
)

fields_config instance-attribute 

fields_config = fields_config

required_fields instance-attribute 

required_fields = required_fields

__init__ 

__init__(
    data: Mapping[str, Tensor],
    modality: str,
    required_fields: set[str],
    fields_factory: Callable[
        [Mapping[str, Tensor]],
        Mapping[str, MultiModalFieldConfig],
    ],
) -> None
Source code in vllm/multimodal/parse.py 
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
def __init__(
    self,
    data: Mapping[str, torch.Tensor],
    modality: str,
    required_fields: set[str],
    fields_factory: Callable[
        [Mapping[str, torch.Tensor]],
        Mapping[str, MultiModalFieldConfig],
    ],
) -> None:
    from transformers.feature_extraction_utils import BatchFeature

    super().__init__(data, modality)

    missing_required_data_keys = required_fields - data.keys()
    if missing_required_data_keys:
        data_keys = set(data.keys())
        msg = (f"The data should contain the fields: {required_fields}, "
               f"but only found the following keys: {data_keys}")
        raise ValueError(msg)

    fields_config = fields_factory(data)
    missing_required_fields = required_fields - fields_config.keys()
    if missing_required_fields:
        fields = set(fields_config.keys())
        msg = f"{required_fields=} should be a subset of {fields=}"
        raise ValueError(msg)

    self.fields_config = fields_config
    self.required_fields = required_fields

    self._kwargs = MultiModalKwargs.from_hf_inputs(
        BatchFeature(dict(data)),
        fields_config,
    )

get 

get(index: int) -> Mapping[str, Tensor]
Source code in vllm/multimodal/parse.py 
168
169
170
171
172
def get(self, index: int) -> Mapping[str, torch.Tensor]:
    return {
        k: v.data
        for k, v in self._kwargs.get_item(self.modality, index).items()
    }

get_count 

get_count() -> int
Source code in vllm/multimodal/parse.py 
165
166
def get_count(self) -> int:
    return self._kwargs.get_item_count(self.modality)

get_passthrough_data 

get_passthrough_data() -> Mapping[str, object]
Source code in vllm/multimodal/parse.py 
177
178
def get_passthrough_data(self) -> Mapping[str, object]:
    return self.data

get_processor_data 

get_processor_data() -> Mapping[str, object]
Source code in vllm/multimodal/parse.py 
174
175
def get_processor_data(self) -> Mapping[str, object]:
    return {}

EmbeddingItems

Bases: ModalityDataItems[Union[Tensor, list[Tensor]], Tensor]

Base class for data items that are expressed as a batched embedding tensor, or a list of embedding tensors (one per item).

Source code in vllm/multimodal/parse.py 
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
class EmbeddingItems(ModalityDataItems[Union[torch.Tensor, list[torch.Tensor]],
                                       torch.Tensor]):
    """
    Base class for data items that are expressed as a batched embedding tensor,
    or a list of embedding tensors (one per item).
    """

    def get_count(self) -> int:
        return len(self.data)

    def get(self, index: int) -> torch.Tensor:
        return self.data[index]

    def get_processor_data(self) -> Mapping[str, object]:
        return {}

    def get_passthrough_data(self) -> Mapping[str, object]:
        return {f"{self.modality}_embeds": self.data}

    def get_feature_size(self, item_idx: int) -> int:
        return len(self.get(item_idx))

get 

get(index: int) -> Tensor
Source code in vllm/multimodal/parse.py 
108
109
def get(self, index: int) -> torch.Tensor:
    return self.data[index]

get_count 

get_count() -> int
Source code in vllm/multimodal/parse.py 
105
106
def get_count(self) -> int:
    return len(self.data)

get_feature_size 

get_feature_size(item_idx: int) -> int
Source code in vllm/multimodal/parse.py 
117
118
def get_feature_size(self, item_idx: int) -> int:
    return len(self.get(item_idx))

get_passthrough_data 

get_passthrough_data() -> Mapping[str, object]
Source code in vllm/multimodal/parse.py 
114
115
def get_passthrough_data(self) -> Mapping[str, object]:
    return {f"{self.modality}_embeds": self.data}

get_processor_data 

get_processor_data() -> Mapping[str, object]
Source code in vllm/multimodal/parse.py 
111
112
def get_processor_data(self) -> Mapping[str, object]:
    return {}

ImageEmbeddingItems

Bases: EmbeddingItems

Source code in vllm/multimodal/parse.py 
219
220
221
222
class ImageEmbeddingItems(EmbeddingItems):

    def __init__(self, data: Union[torch.Tensor, list[torch.Tensor]]) -> None:
        super().__init__(data, "image")

__init__ 

__init__(data: Union[Tensor, list[Tensor]]) -> None
Source code in vllm/multimodal/parse.py 
221
222
def __init__(self, data: Union[torch.Tensor, list[torch.Tensor]]) -> None:
    super().__init__(data, "image")

ImageProcessorItems

Bases: ProcessorBatchItems[HfImageItem]

Source code in vllm/multimodal/parse.py 
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
class ImageProcessorItems(ProcessorBatchItems[HfImageItem]):

    def __init__(self, data: Sequence[HfImageItem]) -> None:
        super().__init__(data, "image")

    def get_image_size(self, item_idx: int) -> ImageSize:
        image = self.get(item_idx)

        if isinstance(image, PILImage.Image):
            return ImageSize(*image.size)
        if isinstance(image, (np.ndarray, torch.Tensor)):
            _, h, w = image.shape
            return ImageSize(w, h)

        assert_never(image)

__init__ 

__init__(data: Sequence[HfImageItem]) -> None
Source code in vllm/multimodal/parse.py 
204
205
def __init__(self, data: Sequence[HfImageItem]) -> None:
    super().__init__(data, "image")

get_image_size 

get_image_size(item_idx: int) -> ImageSize
Source code in vllm/multimodal/parse.py 
207
208
209
210
211
212
213
214
215
216
def get_image_size(self, item_idx: int) -> ImageSize:
    image = self.get(item_idx)

    if isinstance(image, PILImage.Image):
        return ImageSize(*image.size)
    if isinstance(image, (np.ndarray, torch.Tensor)):
        _, h, w = image.shape
        return ImageSize(w, h)

    assert_never(image)

ImageSize

Bases: NamedTuple

Source code in vllm/multimodal/parse.py 
197
198
199
class ImageSize(NamedTuple):
    width: int
    height: int

height instance-attribute 

height: int

width instance-attribute 

width: int

ModalityDataItems

Bases: ABC, Generic[_T, _I]

Represents data items for a modality in MultiModalDataItems.

Source code in vllm/multimodal/parse.py 
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
class ModalityDataItems(ABC, Generic[_T, _I]):
    """
    Represents data items for a modality in
    [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems].
    """

    def __init__(self, data: _T, modality: str) -> None:
        super().__init__()

        self.data = data
        self.modality = modality

    def __repr__(self) -> str:
        return (f"{type(self).__name__}(modality={self.modality!r}, "
                f"len={len(self)})")

    def __len__(self) -> int:
        return self.get_count()

    def __getitem__(self, index: int) -> _I:
        return self.get(index)

    if TYPE_CHECKING:
        # Auto-generated
        def __iter__(self) -> Iterator[_I]:
            ...

    @abstractmethod
    def get_count(self) -> int:
        """Get the number of data items."""
        raise NotImplementedError

    @abstractmethod
    def get(self, index: int) -> _I:
        """Get a data item by its index."""
        raise NotImplementedError

    def get_all(self) -> list[_I]:
        """Get all data items."""
        return [self.get(idx) for idx in range(self.get_count())]

    @abstractmethod
    def get_processor_data(self) -> Mapping[str, object]:
        """Get the data to pass to the HF processor."""
        raise NotImplementedError

    @abstractmethod
    def get_passthrough_data(self) -> Mapping[str, object]:
        """Get the data to pass directly to the model."""
        raise NotImplementedError

data instance-attribute 

data = data

modality instance-attribute 

modality = modality

__getitem__ 

__getitem__(index: int) -> _I
Source code in vllm/multimodal/parse.py 
49
50
def __getitem__(self, index: int) -> _I:
    return self.get(index)

__init__ 

__init__(data: _T, modality: str) -> None
Source code in vllm/multimodal/parse.py 
36
37
38
39
40
def __init__(self, data: _T, modality: str) -> None:
    super().__init__()

    self.data = data
    self.modality = modality

__iter__ 

__iter__() -> Iterator[_I]
Source code in vllm/multimodal/parse.py 
54
55
def __iter__(self) -> Iterator[_I]:
    ...

__len__ 

__len__() -> int
Source code in vllm/multimodal/parse.py 
46
47
def __len__(self) -> int:
    return self.get_count()

__repr__ 

__repr__() -> str
Source code in vllm/multimodal/parse.py 
42
43
44
def __repr__(self) -> str:
    return (f"{type(self).__name__}(modality={self.modality!r}, "
            f"len={len(self)})")

get abstractmethod 

get(index: int) -> _I

Get a data item by its index.

Source code in vllm/multimodal/parse.py 
62
63
64
65
@abstractmethod
def get(self, index: int) -> _I:
    """Get a data item by its index."""
    raise NotImplementedError

get_all 

get_all() -> list[_I]

Get all data items.

Source code in vllm/multimodal/parse.py 
67
68
69
def get_all(self) -> list[_I]:
    """Get all data items."""
    return [self.get(idx) for idx in range(self.get_count())]

get_count abstractmethod 

get_count() -> int

Get the number of data items.

Source code in vllm/multimodal/parse.py 
57
58
59
60
@abstractmethod
def get_count(self) -> int:
    """Get the number of data items."""
    raise NotImplementedError

get_passthrough_data abstractmethod 

get_passthrough_data() -> Mapping[str, object]

Get the data to pass directly to the model.

Source code in vllm/multimodal/parse.py 
76
77
78
79
@abstractmethod
def get_passthrough_data(self) -> Mapping[str, object]:
    """Get the data to pass directly to the model."""
    raise NotImplementedError

get_processor_data abstractmethod 

get_processor_data() -> Mapping[str, object]

Get the data to pass to the HF processor.

Source code in vllm/multimodal/parse.py 
71
72
73
74
@abstractmethod
def get_processor_data(self) -> Mapping[str, object]:
    """Get the data to pass to the HF processor."""
    raise NotImplementedError

MultiModalDataItems

Bases: UserDict[str, ModalityDataItems[Any, Any]]

As MultiModalDataDict, but normalized such that each entry corresponds to a list.

Source code in vllm/multimodal/parse.py 
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):
    """
    As [`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict], but
    normalized such that each entry corresponds to a list.
    """

    def get_count(self, modality: str, *, strict: bool = True) -> int:
        """
        Get the number of data items belonging to a modality.

        If `strict=False`, return `0` instead of raising [`KeyError`][]
        even if the modality is not found.
        """
        if modality not in self:
            if strict:
                available_modalities = set(self.keys())
                raise KeyError(f"Modality {modality!r} not found. "
                               f"Available modalities: {available_modalities}")

            return 0

        return self[modality].get_count()

    def get_all_counts(self) -> Mapping[str, int]:
        """Get the number of items belonging to each modality."""
        return {m: items.get_count() for m, items in self.items()}

    def get_items(
        self,
        modality: str,
        typ: Union[type[_D], tuple[type[_D], ...]],
    ) -> _D:
        """
        Get the data items belonging to a modality,
        requiring that they belong to a certain type.
        """
        if modality not in self:
            available_modalities = set(self.keys())
            raise KeyError(f"Modality {modality!r} not found. "
                           f"Available modalities: {available_modalities}")

        items = self[modality]
        if not isinstance(items, typ):
            raise TypeError(f"Invalid type of data items for {modality=}. "
                            f"Expected type: {typ}, but "
                            f"found type: {type(items)}")

        return items  # type: ignore[return-value]

get_all_counts 

get_all_counts() -> Mapping[str, int]

Get the number of items belonging to each modality.

Source code in vllm/multimodal/parse.py 
277
278
279
def get_all_counts(self) -> Mapping[str, int]:
    """Get the number of items belonging to each modality."""
    return {m: items.get_count() for m, items in self.items()}

get_count 

get_count(modality: str, *, strict: bool = True) -> int

Get the number of data items belonging to a modality.

If strict=False, return 0 instead of raising KeyError even if the modality is not found.

Source code in vllm/multimodal/parse.py 
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
def get_count(self, modality: str, *, strict: bool = True) -> int:
    """
    Get the number of data items belonging to a modality.

    If `strict=False`, return `0` instead of raising [`KeyError`][]
    even if the modality is not found.
    """
    if modality not in self:
        if strict:
            available_modalities = set(self.keys())
            raise KeyError(f"Modality {modality!r} not found. "
                           f"Available modalities: {available_modalities}")

        return 0

    return self[modality].get_count()

get_items 

get_items(
    modality: str,
    typ: Union[type[_D], tuple[type[_D], ...]],
) -> _D

Get the data items belonging to a modality, requiring that they belong to a certain type.

Source code in vllm/multimodal/parse.py 
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
def get_items(
    self,
    modality: str,
    typ: Union[type[_D], tuple[type[_D], ...]],
) -> _D:
    """
    Get the data items belonging to a modality,
    requiring that they belong to a certain type.
    """
    if modality not in self:
        available_modalities = set(self.keys())
        raise KeyError(f"Modality {modality!r} not found. "
                       f"Available modalities: {available_modalities}")

    items = self[modality]
    if not isinstance(items, typ):
        raise TypeError(f"Invalid type of data items for {modality=}. "
                        f"Expected type: {typ}, but "
                        f"found type: {type(items)}")

    return items  # type: ignore[return-value]

MultiModalDataParser

Parses MultiModalDataDict into MultiModalDataItems.

Parameters:

Name Type Description Default
target_sr float

Enables automatic resampling of audio items to the model's expected sampling rate.

 None
Source code in vllm/multimodal/parse.py 
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
class MultiModalDataParser:
    """
    Parses [`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict]
    into [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems].

    Args:
        target_sr (float, optional): Enables automatic resampling of audio
            items to the model's expected sampling rate.
    """

    def __init__(
        self,
        *,
        target_sr: Optional[float] = None,
        audio_resample_method: Literal["librosa", "scipy"] = "librosa",
    ) -> None:
        super().__init__()

        self.audio_resampler = AudioResampler(
            target_sr=target_sr,
            method=audio_resample_method,
        )

    def _is_embeddings(
            self, data: object
    ) -> TypeGuard[Union[torch.Tensor, list[torch.Tensor]]]:
        if isinstance(data, torch.Tensor):
            return data.ndim == 3
        if is_list_of(data, torch.Tensor):
            return data[0].ndim == 2

        return False

    def _is_empty(self, data: object) -> TypeGuard[None]:
        if isinstance(data, list):
            return len(data) == 0
        if isinstance(data, (np.ndarray, torch.Tensor)):
            return data.size == 0

        return False

    def _get_audio_with_sr(
        self,
        audio: AudioItem,
    ) -> tuple[np.ndarray, Optional[float]]:
        if isinstance(audio, tuple):
            return audio
        if isinstance(audio, list):
            return np.array(audio), None
        if isinstance(audio, np.ndarray):
            return audio, None
        if isinstance(audio, torch.Tensor):
            return audio.numpy(), None

        assert_never(audio)

    def _parse_audio_data(
        self,
        data: ModalityData[AudioItem],
    ) -> Optional[ModalityDataItems[Any, Any]]:
        # also check single audio item with sampling rate
        if self._is_empty(data) or (isinstance(data, tuple)
                                    and self._is_empty(data[0])):
            return None

        if self._is_embeddings(data):
            return AudioEmbeddingItems(data)

        if (is_list_of(data, float)
                or isinstance(data,
                              (np.ndarray, torch.Tensor)) and data.ndim == 1
                or isinstance(data, tuple)):
            data_items = [data]
        elif isinstance(data, (np.ndarray, torch.Tensor)):
            data_items = [elem for elem in data]
        else:
            data_items = data

        new_audios = list[np.ndarray]()
        for data_item in data_items:
            audio, orig_sr = self._get_audio_with_sr(data_item)
            if orig_sr is None:
                new_audio = audio
            else:
                new_audio = self.audio_resampler.resample(audio,
                                                          orig_sr=orig_sr)

            new_audios.append(new_audio)

        return AudioProcessorItems(new_audios)

    def _parse_image_data(
        self,
        data: ModalityData[ImageItem],
    ) -> Optional[ModalityDataItems[Any, Any]]:
        if self._is_empty(data):
            return None

        if self._is_embeddings(data):
            return ImageEmbeddingItems(data)

        if (isinstance(data, PILImage.Image)
                or isinstance(data,
                              (np.ndarray, torch.Tensor)) and data.ndim == 3):
            data_items = [data]
        elif isinstance(data, (np.ndarray, torch.Tensor)):
            data_items = [elem for elem in data]
        else:
            data_items = data

        return ImageProcessorItems(data_items)

    def _parse_video_data(
        self,
        data: ModalityData[VideoItem],
    ) -> Optional[ModalityDataItems[Any, Any]]:
        if self._is_empty(data):
            return None

        if self._is_embeddings(data):
            return VideoEmbeddingItems(data)

        if (is_list_of(data, PILImage.Image)
                or isinstance(data,
                              (np.ndarray, torch.Tensor)) and data.ndim == 4):
            data_items = [data]
        elif isinstance(data, (np.ndarray, torch.Tensor)):
            data_items = [elem for elem in data]
        else:
            data_items = data

        return VideoProcessorItems(data_items)

    def _get_subparsers(self) -> Mapping[str, ModalityDataParser]:
        return {
            "audio": self._parse_audio_data,
            "image": self._parse_image_data,
            "video": self._parse_video_data,
        }

    def parse_mm_data(self,
                      mm_data: MultiModalDataDict) -> MultiModalDataItems:
        subparsers = self._get_subparsers()

        mm_items = MultiModalDataItems()
        for k, v in mm_data.items():
            if k not in subparsers:
                raise ValueError(f"Unsupported modality: {k}")

            # ignore empty embedding data
            if (parsed_data := subparsers[k](v)) is not None:
                mm_items[k] = parsed_data

        return mm_items

audio_resampler instance-attribute 

audio_resampler = AudioResampler(
    target_sr=target_sr, method=audio_resample_method
)

__init__ 

__init__(
    *,
    target_sr: Optional[float] = None,
    audio_resample_method: Literal[
        "librosa", "scipy"
    ] = "librosa",
) -> None
Source code in vllm/multimodal/parse.py 
318
319
320
321
322
323
324
325
326
327
328
329
def __init__(
    self,
    *,
    target_sr: Optional[float] = None,
    audio_resample_method: Literal["librosa", "scipy"] = "librosa",
) -> None:
    super().__init__()

    self.audio_resampler = AudioResampler(
        target_sr=target_sr,
        method=audio_resample_method,
    )

_get_audio_with_sr 

_get_audio_with_sr(
    audio: AudioItem,
) -> tuple[ndarray, Optional[float]]
Source code in vllm/multimodal/parse.py 
349
350
351
352
353
354
355
356
357
358
359
360
361
362
def _get_audio_with_sr(
    self,
    audio: AudioItem,
) -> tuple[np.ndarray, Optional[float]]:
    if isinstance(audio, tuple):
        return audio
    if isinstance(audio, list):
        return np.array(audio), None
    if isinstance(audio, np.ndarray):
        return audio, None
    if isinstance(audio, torch.Tensor):
        return audio.numpy(), None

    assert_never(audio)

_get_subparsers 

_get_subparsers() -> Mapping[str, ModalityDataParser]
Source code in vllm/multimodal/parse.py 
441
442
443
444
445
446
def _get_subparsers(self) -> Mapping[str, ModalityDataParser]:
    return {
        "audio": self._parse_audio_data,
        "image": self._parse_image_data,
        "video": self._parse_video_data,
    }

_is_embeddings 

_is_embeddings(
    data: object,
) -> TypeGuard[Union[Tensor, list[Tensor]]]
Source code in vllm/multimodal/parse.py 
331
332
333
334
335
336
337
338
339
def _is_embeddings(
        self, data: object
) -> TypeGuard[Union[torch.Tensor, list[torch.Tensor]]]:
    if isinstance(data, torch.Tensor):
        return data.ndim == 3
    if is_list_of(data, torch.Tensor):
        return data[0].ndim == 2

    return False

_is_empty 

_is_empty(data: object) -> TypeGuard[None]
Source code in vllm/multimodal/parse.py 
341
342
343
344
345
346
347
def _is_empty(self, data: object) -> TypeGuard[None]:
    if isinstance(data, list):
        return len(data) == 0
    if isinstance(data, (np.ndarray, torch.Tensor)):
        return data.size == 0

    return False

_parse_audio_data 

_parse_audio_data(
    data: ModalityData[AudioItem],
) -> Optional[ModalityDataItems[Any, Any]]
Source code in vllm/multimodal/parse.py 
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
def _parse_audio_data(
    self,
    data: ModalityData[AudioItem],
) -> Optional[ModalityDataItems[Any, Any]]:
    # also check single audio item with sampling rate
    if self._is_empty(data) or (isinstance(data, tuple)
                                and self._is_empty(data[0])):
        return None

    if self._is_embeddings(data):
        return AudioEmbeddingItems(data)

    if (is_list_of(data, float)
            or isinstance(data,
                          (np.ndarray, torch.Tensor)) and data.ndim == 1
            or isinstance(data, tuple)):
        data_items = [data]
    elif isinstance(data, (np.ndarray, torch.Tensor)):
        data_items = [elem for elem in data]
    else:
        data_items = data

    new_audios = list[np.ndarray]()
    for data_item in data_items:
        audio, orig_sr = self._get_audio_with_sr(data_item)
        if orig_sr is None:
            new_audio = audio
        else:
            new_audio = self.audio_resampler.resample(audio,
                                                      orig_sr=orig_sr)

        new_audios.append(new_audio)

    return AudioProcessorItems(new_audios)

_parse_image_data 

_parse_image_data(
    data: ModalityData[ImageItem],
) -> Optional[ModalityDataItems[Any, Any]]
Source code in vllm/multimodal/parse.py 
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
def _parse_image_data(
    self,
    data: ModalityData[ImageItem],
) -> Optional[ModalityDataItems[Any, Any]]:
    if self._is_empty(data):
        return None

    if self._is_embeddings(data):
        return ImageEmbeddingItems(data)

    if (isinstance(data, PILImage.Image)
            or isinstance(data,
                          (np.ndarray, torch.Tensor)) and data.ndim == 3):
        data_items = [data]
    elif isinstance(data, (np.ndarray, torch.Tensor)):
        data_items = [elem for elem in data]
    else:
        data_items = data

    return ImageProcessorItems(data_items)

_parse_video_data 

_parse_video_data(
    data: ModalityData[VideoItem],
) -> Optional[ModalityDataItems[Any, Any]]
Source code in vllm/multimodal/parse.py 
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
def _parse_video_data(
    self,
    data: ModalityData[VideoItem],
) -> Optional[ModalityDataItems[Any, Any]]:
    if self._is_empty(data):
        return None

    if self._is_embeddings(data):
        return VideoEmbeddingItems(data)

    if (is_list_of(data, PILImage.Image)
            or isinstance(data,
                          (np.ndarray, torch.Tensor)) and data.ndim == 4):
        data_items = [data]
    elif isinstance(data, (np.ndarray, torch.Tensor)):
        data_items = [elem for elem in data]
    else:
        data_items = data

    return VideoProcessorItems(data_items)

parse_mm_data 

parse_mm_data(
    mm_data: MultiModalDataDict,
) -> MultiModalDataItems
Source code in vllm/multimodal/parse.py 
448
449
450
451
452
453
454
455
456
457
458
459
460
461
def parse_mm_data(self,
                  mm_data: MultiModalDataDict) -> MultiModalDataItems:
    subparsers = self._get_subparsers()

    mm_items = MultiModalDataItems()
    for k, v in mm_data.items():
        if k not in subparsers:
            raise ValueError(f"Unsupported modality: {k}")

        # ignore empty embedding data
        if (parsed_data := subparsers[k](v)) is not None:
            mm_items[k] = parsed_data

    return mm_items

ProcessorBatchItems

Bases: ModalityDataItems[Sequence[_T], _T]

Base class for data items that are arranged in a list.

Source code in vllm/multimodal/parse.py 
82
83
84
85
86
87
88
89
90
91
92
93
94
95
class ProcessorBatchItems(ModalityDataItems[Sequence[_T], _T]):
    """Base class for data items that are arranged in a list."""

    def get_count(self) -> int:
        return len(self.data)

    def get(self, index: int) -> _T:
        return self.data[index]

    def get_processor_data(self) -> Mapping[str, object]:
        return {f"{self.modality}s": self.data}

    def get_passthrough_data(self) -> Mapping[str, object]:
        return {}

get 

get(index: int) -> _T
Source code in vllm/multimodal/parse.py 
88
89
def get(self, index: int) -> _T:
    return self.data[index]

get_count 

get_count() -> int
Source code in vllm/multimodal/parse.py 
85
86
def get_count(self) -> int:
    return len(self.data)

get_passthrough_data 

get_passthrough_data() -> Mapping[str, object]
Source code in vllm/multimodal/parse.py 
94
95
def get_passthrough_data(self) -> Mapping[str, object]:
    return {}

get_processor_data 

get_processor_data() -> Mapping[str, object]
Source code in vllm/multimodal/parse.py 
91
92
def get_processor_data(self) -> Mapping[str, object]:
    return {f"{self.modality}s": self.data}

VideoEmbeddingItems

Bases: EmbeddingItems

Source code in vllm/multimodal/parse.py 
245
246
247
248
class VideoEmbeddingItems(EmbeddingItems):

    def __init__(self, data: Union[torch.Tensor, list[torch.Tensor]]) -> None:
        super().__init__(data, "video")

__init__ 

__init__(data: Union[Tensor, list[Tensor]]) -> None
Source code in vllm/multimodal/parse.py 
247
248
def __init__(self, data: Union[torch.Tensor, list[torch.Tensor]]) -> None:
    super().__init__(data, "video")

VideoProcessorItems

Bases: ProcessorBatchItems[HfVideoItem]

Source code in vllm/multimodal/parse.py 
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
class VideoProcessorItems(ProcessorBatchItems[HfVideoItem]):

    def __init__(self, data: Sequence[HfVideoItem]) -> None:
        super().__init__(data, "video")

    def get_num_frames(self, item_idx: int) -> int:
        return len(self.get(item_idx))

    def get_frame_size(self, item_idx: int) -> ImageSize:
        image = self.get(item_idx)[0]  # Assume that the video isn't empty

        if isinstance(image, PILImage.Image):
            return ImageSize(*image.size)
        if isinstance(image, (np.ndarray, torch.Tensor)):
            _, h, w = image.shape
            return ImageSize(w, h)

        assert_never(image)

__init__ 

__init__(data: Sequence[HfVideoItem]) -> None
Source code in vllm/multimodal/parse.py 
227
228
def __init__(self, data: Sequence[HfVideoItem]) -> None:
    super().__init__(data, "video")

get_frame_size 

get_frame_size(item_idx: int) -> ImageSize
Source code in vllm/multimodal/parse.py 
233
234
235
236
237
238
239
240
241
242
def get_frame_size(self, item_idx: int) -> ImageSize:
    image = self.get(item_idx)[0]  # Assume that the video isn't empty

    if isinstance(image, PILImage.Image):
        return ImageSize(*image.size)
    if isinstance(image, (np.ndarray, torch.Tensor)):
        _, h, w = image.shape
        return ImageSize(w, h)

    assert_never(image)

get_num_frames 

get_num_frames(item_idx: int) -> int
Source code in vllm/multimodal/parse.py 
230
231
def get_num_frames(self, item_idx: int) -> int:
    return len(self.get(item_idx))