Skip to content

vllm.multimodal

Modules:

Name Description
audio
base
hasher
image
inputs
parse
processing
profiling
registry
utils
video

BatchedTensorInputs module-attribute 

BatchedTensorInputs: TypeAlias = Mapping[str, NestedTensors]

A dictionary containing nested tensors which have been batched via MultiModalKwargs.batch.

MULTIMODAL_REGISTRY module-attribute 

MULTIMODAL_REGISTRY = MultiModalRegistry()

The global MultiModalRegistry is used by model runners to dispatch data processing according to the target model.

Info

mm_processing

ModalityData module-attribute 

ModalityData: TypeAlias = Union[_T, list[_T]]

Either a single data item, or a list of data items.

The number of data items allowed per modality is restricted by --limit-mm-per-prompt.

MultiModalDataDict module-attribute 

MultiModalDataDict: TypeAlias = Mapping[
    str, ModalityData[Any]
]

A dictionary containing an entry for each modality type to input.

The built-in modalities are defined by MultiModalDataBuiltins.

MultiModalHashDict module-attribute 

MultiModalHashDict = Mapping[str, list[str]]

A dictionary containing hashes for items in each modality.

MultiModalPlaceholderDict module-attribute 

MultiModalPlaceholderDict: TypeAlias = Mapping[
    str, Sequence[PlaceholderRange]
]

A dictionary containing placeholder ranges for each modality.

NestedTensors module-attribute 

NestedTensors: TypeAlias = Union[
    list["NestedTensors"],
    list["torch.Tensor"],
    "torch.Tensor",
    tuple["torch.Tensor", ...],
]

Uses a list instead of a tensor if the dimensions of each element do not match.

__all__ module-attribute 

__all__ = [
    "BatchedTensorInputs",
    "ModalityData",
    "MultiModalDataBuiltins",
    "MultiModalDataDict",
    "MultiModalHashDict",
    "MultiModalHasher",
    "MultiModalKwargs",
    "MultiModalPlaceholderDict",
    "MultiModalPlaceholderMap",
    "NestedTensors",
    "MULTIMODAL_REGISTRY",
    "MultiModalRegistry",
]

MultiModalDataBuiltins

Bases: TypedDict

Type annotations for modality types predefined by vLLM.

Source code in vllm/multimodal/inputs.py 
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
@final
class MultiModalDataBuiltins(TypedDict, total=False):
    """Type annotations for modality types predefined by vLLM."""

    image: ModalityData[ImageItem]
    """The input image(s)."""

    video: ModalityData[VideoItem]
    """The input video(s)."""

    audio: ModalityData[AudioItem]
    """The input audio(s)."""

audio instance-attribute 

audio: ModalityData[AudioItem]

The input audio(s).

image instance-attribute 

image: ModalityData[ImageItem]

The input image(s).

video instance-attribute 

video: ModalityData[VideoItem]

The input video(s).

MultiModalHasher

Source code in vllm/multimodal/hasher.py 
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
class MultiModalHasher:

    @classmethod
    def serialize_item(cls, obj: object) -> Union[bytes, memoryview]:
        # Simple cases
        if isinstance(obj, str):
            return obj.encode("utf-8")
        if isinstance(obj, (bytes, memoryview)):
            return obj
        if isinstance(obj, (int, float)):
            return np.array(obj).tobytes()

        if isinstance(obj, Image.Image):
            return cls.item_to_bytes(
                "image", np.asarray(convert_image_mode(obj, "RGBA")))
        if isinstance(obj, torch.Tensor):
            return cls.item_to_bytes("tensor", obj.numpy())
        if isinstance(obj, np.ndarray):
            # If the array is non-contiguous, we need to copy it first
            arr_data = obj.data if obj.flags.c_contiguous else obj.tobytes()
            return cls.item_to_bytes("ndarray", {
                "dtype": obj.dtype.str,
                "shape": obj.shape,
                "data": arr_data,
            })

        logger.warning(
            "No serialization method found for %s. "
            "Falling back to pickle.", type(obj))

        return pickle.dumps(obj)

    @classmethod
    def item_to_bytes(
        cls,
        key: str,
        obj: object,
    ) -> bytes:
        return b''.join(kb + vb for kb, vb in cls.iter_item_to_bytes(key, obj))

    @classmethod
    def iter_item_to_bytes(
        cls,
        key: str,
        obj: object,
    ) -> Iterable[tuple[bytes, Union[bytes, memoryview]]]:
        # Recursive cases
        if isinstance(obj, (list, tuple)):
            for i, elem in enumerate(obj):
                yield from cls.iter_item_to_bytes(f"{key}.{i}", elem)
        elif isinstance(obj, dict):
            for k, v in obj.items():
                yield from cls.iter_item_to_bytes(f"{key}.{k}", v)
        else:
            key_bytes = key.encode("utf-8")
            value_bytes = cls.serialize_item(obj)
            yield key_bytes, value_bytes

    @classmethod
    def hash_kwargs(cls, **kwargs: object) -> str:
        hasher = blake3()

        for k, v in kwargs.items():
            for k_bytes, v_bytes in cls.iter_item_to_bytes(k, v):
                hasher.update(k_bytes)
                hasher.update(v_bytes)

        return hasher.hexdigest()

hash_kwargs classmethod 

hash_kwargs(**kwargs: object) -> str
Source code in vllm/multimodal/hasher.py 
82
83
84
85
86
87
88
89
90
91
@classmethod
def hash_kwargs(cls, **kwargs: object) -> str:
    hasher = blake3()

    for k, v in kwargs.items():
        for k_bytes, v_bytes in cls.iter_item_to_bytes(k, v):
            hasher.update(k_bytes)
            hasher.update(v_bytes)

    return hasher.hexdigest()

item_to_bytes classmethod 

item_to_bytes(key: str, obj: object) -> bytes
Source code in vllm/multimodal/hasher.py 
56
57
58
59
60
61
62
@classmethod
def item_to_bytes(
    cls,
    key: str,
    obj: object,
) -> bytes:
    return b''.join(kb + vb for kb, vb in cls.iter_item_to_bytes(key, obj))

iter_item_to_bytes classmethod 

iter_item_to_bytes(
    key: str, obj: object
) -> Iterable[tuple[bytes, Union[bytes, memoryview]]]
Source code in vllm/multimodal/hasher.py 
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
@classmethod
def iter_item_to_bytes(
    cls,
    key: str,
    obj: object,
) -> Iterable[tuple[bytes, Union[bytes, memoryview]]]:
    # Recursive cases
    if isinstance(obj, (list, tuple)):
        for i, elem in enumerate(obj):
            yield from cls.iter_item_to_bytes(f"{key}.{i}", elem)
    elif isinstance(obj, dict):
        for k, v in obj.items():
            yield from cls.iter_item_to_bytes(f"{key}.{k}", v)
    else:
        key_bytes = key.encode("utf-8")
        value_bytes = cls.serialize_item(obj)
        yield key_bytes, value_bytes

serialize_item classmethod 

serialize_item(obj: object) -> Union[bytes, memoryview]
Source code in vllm/multimodal/hasher.py 
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
@classmethod
def serialize_item(cls, obj: object) -> Union[bytes, memoryview]:
    # Simple cases
    if isinstance(obj, str):
        return obj.encode("utf-8")
    if isinstance(obj, (bytes, memoryview)):
        return obj
    if isinstance(obj, (int, float)):
        return np.array(obj).tobytes()

    if isinstance(obj, Image.Image):
        return cls.item_to_bytes(
            "image", np.asarray(convert_image_mode(obj, "RGBA")))
    if isinstance(obj, torch.Tensor):
        return cls.item_to_bytes("tensor", obj.numpy())
    if isinstance(obj, np.ndarray):
        # If the array is non-contiguous, we need to copy it first
        arr_data = obj.data if obj.flags.c_contiguous else obj.tobytes()
        return cls.item_to_bytes("ndarray", {
            "dtype": obj.dtype.str,
            "shape": obj.shape,
            "data": arr_data,
        })

    logger.warning(
        "No serialization method found for %s. "
        "Falling back to pickle.", type(obj))

    return pickle.dumps(obj)

MultiModalKwargs

Bases: UserDict[str, NestedTensors]

A dictionary that represents the keyword arguments to torch.nn.Module.forward.

The metadata items enables us to obtain the keyword arguments corresponding to each data item in MultiModalDataItems, via get_item and get_items.

Source code in vllm/multimodal/inputs.py 
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
class MultiModalKwargs(UserDict[str, NestedTensors]):
    """
    A dictionary that represents the keyword arguments to
    [`torch.nn.Module.forward`][].

    The metadata `items` enables us to obtain the keyword arguments
    corresponding to each data item in
    [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems], via
    [`get_item`][vllm.multimodal.inputs.MultiModalKwargs.get_item] and
    [`get_items`][vllm.multimodal.inputs.MultiModalKwargs.get_items].
    """

    @staticmethod
    def from_hf_inputs(
        hf_inputs: "BatchFeature",
        config_by_key: Mapping[str, MultiModalFieldConfig],
    ):
        # NOTE: This skips fields in `hf_inputs` that are not in `config_by_key`
        # We assume that those fields are not used in vLLM
        elems_by_key = dict[str, Sequence[MultiModalFieldElem]]()
        keys_by_modality = defaultdict[str, set[str]](set)
        for key, config in config_by_key.items():
            batch = hf_inputs.get(key)
            if batch is not None:
                elems = config.build_elems(key, batch)
                if len(elems) > 0:
                    elems_by_key[key] = elems
                    keys_by_modality[config.modality].add(key)

        items = list[MultiModalKwargsItem]()
        for modality, keys in keys_by_modality.items():
            elems_in_modality = {k: elems_by_key[k] for k in keys}
            batch_sizes = {k: len(v) for k, v in elems_in_modality.items()}

            if len(set(batch_sizes.values())) > 1:
                raise ValueError(
                    f"Cannot merge different batch sizes for {modality=}! "
                    f"Found: {batch_sizes=}")

            batch_size = next(iter(batch_sizes.values()))
            for item_idx in range(batch_size):
                elems = [v[item_idx] for v in elems_in_modality.values()]
                items.append(MultiModalKwargsItem.from_elems(elems))

        return MultiModalKwargs.from_items(items)

    @staticmethod
    def from_items(items: Sequence[MultiModalKwargsItem]):
        """Construct a new
        [`MultiModalKwargs`][vllm.multimodal.inputs.MultiModalKwargs]
        from multiple items."""
        elems_by_key = defaultdict[str, list[MultiModalFieldElem]](list)
        for item in items:
            for key, elem in item.items():
                elems_by_key[key].append(elem)

        data = {
            key: elems[0].field.reduce_data(elems)
            for key, elems in elems_by_key.items() if len(elems) > 0
        }

        return MultiModalKwargs(data, items=items)

    def __init__(
        self,
        data: Mapping[str, NestedTensors],
        *,
        items: Optional[Sequence[MultiModalKwargsItem]] = None,
    ) -> None:
        super().__init__(data)

        items_by_modality = full_groupby(items or [], key=lambda x: x.modality)
        self._items_by_modality = dict(items_by_modality)

    @property
    def modalities(self):
        return self._items_by_modality.keys()

    @staticmethod
    def _try_stack(nested_tensors: NestedTensors,
                   pin_memory: bool = False) -> NestedTensors:
        """
        Stack the inner dimensions that have the same shape in
        a nested list of tensors.

        Thus, a dimension represented by a list means that the inner
        dimensions are different for each element along that dimension.
        """
        if isinstance(nested_tensors, torch.Tensor):
            return nested_tensors

        # TODO: Remove these once all models have been migrated
        if isinstance(nested_tensors, np.ndarray):
            return torch.from_numpy(nested_tensors)
        if isinstance(nested_tensors, (int, float)):
            return torch.tensor(nested_tensors)

        stacked = [
            MultiModalKwargs._try_stack(t, pin_memory) for t in nested_tensors
        ]
        if not is_list_of(stacked, torch.Tensor, check="all"):
            # Only tensors (not lists) can be stacked.
            return stacked

        tensors_ = cast(list[torch.Tensor], stacked)
        if len(tensors_) == 1:
            # An optimization when `tensors_` contains only one tensor:
            # - produce exactly same result as `torch.stack(tensors_)`
            # - will achieve zero-copy if the tensor is contiguous
            return tensors_[0].unsqueeze(0).contiguous()

        if any(t.shape != tensors_[0].shape for t in tensors_):
            # The tensors have incompatible shapes and can't be stacked.
            return tensors_

        outputs = torch.empty(len(tensors_),
                              *tensors_[0].shape,
                              dtype=tensors_[0].dtype,
                              device=tensors_[0].device,
                              pin_memory=pin_memory)
        return torch.stack(tensors_, out=outputs)

    @staticmethod
    def batch(inputs_list: list["MultiModalKwargs"],
              pin_memory: bool = False) -> BatchedTensorInputs:
        """
        Batch multiple inputs together into a dictionary.

        The resulting dictionary has the same keys as the inputs.
        If the corresponding value from each input is a tensor and they all
        share the same shape, the output value is a single batched tensor;
        otherwise, the output value is a list containing the original value
        from each input.
        """
        if len(inputs_list) == 0:
            return {}

        # We need to consider the case where each item in the batch
        # contains different modalities (i.e. different keys).
        item_lists = defaultdict[str, list[NestedTensors]](list)

        for inputs in inputs_list:
            for k, v in inputs.items():
                item_lists[k].append(v)

        return {
            k: MultiModalKwargs._try_stack(item_list, pin_memory)
            for k, item_list in item_lists.items()
        }

    @staticmethod
    def as_kwargs(
        batched_inputs: BatchedTensorInputs,
        *,
        device: torch.types.Device,
    ) -> BatchedTensorInputs:
        json_inputs = cast(JSONTree[torch.Tensor], batched_inputs)

        json_mapped = json_map_leaves(
            lambda x: x.to(device=device, non_blocking=True),
            json_inputs,
        )

        return cast(BatchedTensorInputs, json_mapped)

    def __delitem__(self, key: str) -> None:
        super().__delitem__(key)

        for items in self._items_by_modality.values():
            for item in items:
                item.pop(key, None)

    def __eq__(self, other: object) -> bool:
        if not isinstance(other, self.__class__):
            return False
        if self._items_by_modality != other._items_by_modality:
            return False

        ks = self.keys()
        return (ks == other.keys()
                and all(nested_tensors_equal(self[k], other[k]) for k in ks))

    def _validate_modality(self, method_name: str, modality: str) -> None:
        if not self._items_by_modality:
            raise RuntimeError(
                f"`{method_name}` is not supported when "
                "MultiModalKwargs is not initialized with `items`")

        if modality not in self._items_by_modality:
            available_modalities = set(self._items_by_modality.keys())
            raise KeyError(f"Modality {modality!r} not found. "
                           f"Available modalities: {available_modalities}")

    def get_item_count(self, modality: str) -> int:
        """Get the number of items belonging to a modality."""
        self._validate_modality("get_item_count", modality)
        return len(self._items_by_modality[modality])

    def get_item(self, modality: str, item_index: int) -> MultiModalKwargsItem:
        """
        Get the keyword arguments corresponding to an item identified by
        its modality and index.
        """
        self._validate_modality("get_item", modality)
        return self._items_by_modality[modality][item_index]

    def get_items(self, modality: str) -> Sequence[MultiModalKwargsItem]:
        """
        Get the keyword arguments corresponding to each item belonging to
        a modality.
        """
        self._validate_modality("get_items", modality)
        return self._items_by_modality[modality]

_items_by_modality instance-attribute 

_items_by_modality = dict(items_by_modality)

modalities property 

modalities

__delitem__ 

__delitem__(key: str) -> None
Source code in vllm/multimodal/inputs.py 
769
770
771
772
773
774
def __delitem__(self, key: str) -> None:
    super().__delitem__(key)

    for items in self._items_by_modality.values():
        for item in items:
            item.pop(key, None)

__eq__ 

__eq__(other: object) -> bool
Source code in vllm/multimodal/inputs.py 
776
777
778
779
780
781
782
783
784
def __eq__(self, other: object) -> bool:
    if not isinstance(other, self.__class__):
        return False
    if self._items_by_modality != other._items_by_modality:
        return False

    ks = self.keys()
    return (ks == other.keys()
            and all(nested_tensors_equal(self[k], other[k]) for k in ks))

__init__ 

__init__(
    data: Mapping[str, NestedTensors],
    *,
    items: Optional[Sequence[MultiModalKwargsItem]] = None,
) -> None
Source code in vllm/multimodal/inputs.py 
667
668
669
670
671
672
673
674
675
676
def __init__(
    self,
    data: Mapping[str, NestedTensors],
    *,
    items: Optional[Sequence[MultiModalKwargsItem]] = None,
) -> None:
    super().__init__(data)

    items_by_modality = full_groupby(items or [], key=lambda x: x.modality)
    self._items_by_modality = dict(items_by_modality)

_try_stack staticmethod 

_try_stack(
    nested_tensors: NestedTensors, pin_memory: bool = False
) -> NestedTensors

Stack the inner dimensions that have the same shape in a nested list of tensors.

Thus, a dimension represented by a list means that the inner dimensions are different for each element along that dimension.

Source code in vllm/multimodal/inputs.py 
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
@staticmethod
def _try_stack(nested_tensors: NestedTensors,
               pin_memory: bool = False) -> NestedTensors:
    """
    Stack the inner dimensions that have the same shape in
    a nested list of tensors.

    Thus, a dimension represented by a list means that the inner
    dimensions are different for each element along that dimension.
    """
    if isinstance(nested_tensors, torch.Tensor):
        return nested_tensors

    # TODO: Remove these once all models have been migrated
    if isinstance(nested_tensors, np.ndarray):
        return torch.from_numpy(nested_tensors)
    if isinstance(nested_tensors, (int, float)):
        return torch.tensor(nested_tensors)

    stacked = [
        MultiModalKwargs._try_stack(t, pin_memory) for t in nested_tensors
    ]
    if not is_list_of(stacked, torch.Tensor, check="all"):
        # Only tensors (not lists) can be stacked.
        return stacked

    tensors_ = cast(list[torch.Tensor], stacked)
    if len(tensors_) == 1:
        # An optimization when `tensors_` contains only one tensor:
        # - produce exactly same result as `torch.stack(tensors_)`
        # - will achieve zero-copy if the tensor is contiguous
        return tensors_[0].unsqueeze(0).contiguous()

    if any(t.shape != tensors_[0].shape for t in tensors_):
        # The tensors have incompatible shapes and can't be stacked.
        return tensors_

    outputs = torch.empty(len(tensors_),
                          *tensors_[0].shape,
                          dtype=tensors_[0].dtype,
                          device=tensors_[0].device,
                          pin_memory=pin_memory)
    return torch.stack(tensors_, out=outputs)

_validate_modality 

_validate_modality(method_name: str, modality: str) -> None
Source code in vllm/multimodal/inputs.py 
786
787
788
789
790
791
792
793
794
795
def _validate_modality(self, method_name: str, modality: str) -> None:
    if not self._items_by_modality:
        raise RuntimeError(
            f"`{method_name}` is not supported when "
            "MultiModalKwargs is not initialized with `items`")

    if modality not in self._items_by_modality:
        available_modalities = set(self._items_by_modality.keys())
        raise KeyError(f"Modality {modality!r} not found. "
                       f"Available modalities: {available_modalities}")

as_kwargs staticmethod 

as_kwargs(
    batched_inputs: BatchedTensorInputs, *, device: Device
) -> BatchedTensorInputs
Source code in vllm/multimodal/inputs.py 
754
755
756
757
758
759
760
761
762
763
764
765
766
767
@staticmethod
def as_kwargs(
    batched_inputs: BatchedTensorInputs,
    *,
    device: torch.types.Device,
) -> BatchedTensorInputs:
    json_inputs = cast(JSONTree[torch.Tensor], batched_inputs)

    json_mapped = json_map_leaves(
        lambda x: x.to(device=device, non_blocking=True),
        json_inputs,
    )

    return cast(BatchedTensorInputs, json_mapped)

batch staticmethod 

batch(
    inputs_list: list[MultiModalKwargs],
    pin_memory: bool = False,
) -> BatchedTensorInputs

Batch multiple inputs together into a dictionary.

The resulting dictionary has the same keys as the inputs. If the corresponding value from each input is a tensor and they all share the same shape, the output value is a single batched tensor; otherwise, the output value is a list containing the original value from each input.

Source code in vllm/multimodal/inputs.py 
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
@staticmethod
def batch(inputs_list: list["MultiModalKwargs"],
          pin_memory: bool = False) -> BatchedTensorInputs:
    """
    Batch multiple inputs together into a dictionary.

    The resulting dictionary has the same keys as the inputs.
    If the corresponding value from each input is a tensor and they all
    share the same shape, the output value is a single batched tensor;
    otherwise, the output value is a list containing the original value
    from each input.
    """
    if len(inputs_list) == 0:
        return {}

    # We need to consider the case where each item in the batch
    # contains different modalities (i.e. different keys).
    item_lists = defaultdict[str, list[NestedTensors]](list)

    for inputs in inputs_list:
        for k, v in inputs.items():
            item_lists[k].append(v)

    return {
        k: MultiModalKwargs._try_stack(item_list, pin_memory)
        for k, item_list in item_lists.items()
    }

from_hf_inputs staticmethod 

from_hf_inputs(
    hf_inputs: BatchFeature,
    config_by_key: Mapping[str, MultiModalFieldConfig],
)
Source code in vllm/multimodal/inputs.py 
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
@staticmethod
def from_hf_inputs(
    hf_inputs: "BatchFeature",
    config_by_key: Mapping[str, MultiModalFieldConfig],
):
    # NOTE: This skips fields in `hf_inputs` that are not in `config_by_key`
    # We assume that those fields are not used in vLLM
    elems_by_key = dict[str, Sequence[MultiModalFieldElem]]()
    keys_by_modality = defaultdict[str, set[str]](set)
    for key, config in config_by_key.items():
        batch = hf_inputs.get(key)
        if batch is not None:
            elems = config.build_elems(key, batch)
            if len(elems) > 0:
                elems_by_key[key] = elems
                keys_by_modality[config.modality].add(key)

    items = list[MultiModalKwargsItem]()
    for modality, keys in keys_by_modality.items():
        elems_in_modality = {k: elems_by_key[k] for k in keys}
        batch_sizes = {k: len(v) for k, v in elems_in_modality.items()}

        if len(set(batch_sizes.values())) > 1:
            raise ValueError(
                f"Cannot merge different batch sizes for {modality=}! "
                f"Found: {batch_sizes=}")

        batch_size = next(iter(batch_sizes.values()))
        for item_idx in range(batch_size):
            elems = [v[item_idx] for v in elems_in_modality.values()]
            items.append(MultiModalKwargsItem.from_elems(elems))

    return MultiModalKwargs.from_items(items)

from_items staticmethod 

from_items(items: Sequence[MultiModalKwargsItem])

Construct a new MultiModalKwargs from multiple items.

Source code in vllm/multimodal/inputs.py 
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
@staticmethod
def from_items(items: Sequence[MultiModalKwargsItem]):
    """Construct a new
    [`MultiModalKwargs`][vllm.multimodal.inputs.MultiModalKwargs]
    from multiple items."""
    elems_by_key = defaultdict[str, list[MultiModalFieldElem]](list)
    for item in items:
        for key, elem in item.items():
            elems_by_key[key].append(elem)

    data = {
        key: elems[0].field.reduce_data(elems)
        for key, elems in elems_by_key.items() if len(elems) > 0
    }

    return MultiModalKwargs(data, items=items)

get_item 

get_item(
    modality: str, item_index: int
) -> MultiModalKwargsItem

Get the keyword arguments corresponding to an item identified by its modality and index.

Source code in vllm/multimodal/inputs.py 
802
803
804
805
806
807
808
def get_item(self, modality: str, item_index: int) -> MultiModalKwargsItem:
    """
    Get the keyword arguments corresponding to an item identified by
    its modality and index.
    """
    self._validate_modality("get_item", modality)
    return self._items_by_modality[modality][item_index]

get_item_count 

get_item_count(modality: str) -> int

Get the number of items belonging to a modality.

Source code in vllm/multimodal/inputs.py 
797
798
799
800
def get_item_count(self, modality: str) -> int:
    """Get the number of items belonging to a modality."""
    self._validate_modality("get_item_count", modality)
    return len(self._items_by_modality[modality])

get_items 

get_items(modality: str) -> Sequence[MultiModalKwargsItem]

Get the keyword arguments corresponding to each item belonging to a modality.

Source code in vllm/multimodal/inputs.py 
810
811
812
813
814
815
816
def get_items(self, modality: str) -> Sequence[MultiModalKwargsItem]:
    """
    Get the keyword arguments corresponding to each item belonging to
    a modality.
    """
    self._validate_modality("get_items", modality)
    return self._items_by_modality[modality]

MultiModalPlaceholderMap

Relates multi-modal embeddings to their corresponding placeholders.

Note: This is only used in V0.

Source code in vllm/multimodal/base.py 
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
class MultiModalPlaceholderMap:
    """
    Relates multi-modal embeddings to their corresponding placeholders.

    Note: This is only used in V0.
    """

    class IndexMap(NamedTuple):
        src: list[int]
        dest: list[int]

    src_ranges: list[range]
    """
    The indices of the multi-modal embeddings that will replace the
    corresponding placeholder embeddings pointed to by ``dest_ranges``.
    """

    src_len: int
    """
    The total number of flattened multi-modal embeddings.
    """

    dest_ranges: list[range]
    """
    The indices of the placeholder embeddings that will be replaced by the
    multimodal embeddings.
    """

    dest_len: int
    """
    The total number of embeddings in the destination tensor.
    """

    def __init__(self):
        self.src_ranges = []
        self.src_len = 0
        self.dest_ranges = []
        self.dest_len = 0

    @classmethod
    def from_seq_group(
        cls, seq_group: "SequenceGroupMetadata", positions: range
    ) -> tuple[MultiModalKwargs, dict[str, "MultiModalPlaceholderMap"]]:
        """
        Returns the multi-modal items that intersect with the portion of a
        prompt (``seq_group``) represented by ``positions``, as well as a
        ``MultiModalPlaceholderMap`` that relates the multi-modal embedding
        vectors to their corresponding placeholders.

        Examples:

        ```
        Prompt:    |AAAA BBBB What's in these images?|
        Positions: |.................................|

            images      = [A, B]
            src_ranges  = [(0, 4), (4, 8)]
            dest_ranges = [(0, 4), (5, 9)]

        Prompt:    |AAAA BBBB What's in these images?|
        Positions: |  .....                          |

            images      = [A, B]
            src_ranges  = [(2, 4), (4, 6)]
            dest_ranges = [(0, 2), (3, 5)]

        Prompt:    |AAAA BBBB What's in these images?|
        Positions: |     .........                   |

            images      = [B]
            src_ranges  = [(0, 4)]
            dest_ranges = [(0, 4)]

        Prompt:    |AAAA BBBB What's in these images?|
        Positions: |          .......................|

            images      = []
            src_ranges  = []
            dest_ranges = []
        ```
        """
        seq_mm_data = seq_group.multi_modal_data
        seq_mm_placeholders = seq_group.multi_modal_placeholders

        if not seq_mm_data or not seq_mm_placeholders:
            return MultiModalKwargs({}), {}

        placeholder_maps = dict[str, MultiModalPlaceholderMap]()

        for modality, placeholders in seq_mm_placeholders.items():
            placeholder_map = MultiModalPlaceholderMap()

            if positions:
                placeholder_map.append_items_from_seq_group(
                    positions,
                    # Dummy, since we don't care about intersecting items
                    [None] * len(placeholders),
                    placeholders,
                )

            placeholder_maps[modality] = placeholder_map

        return seq_mm_data, placeholder_maps

    def append_items_from_seq_group(
        self,
        positions: range,
        multi_modal_items: list[_T],
        multi_modal_placeholders: Sequence[PlaceholderRange],
    ) -> list[_T]:
        """
        Adds the multi-modal items that intersect ```positions`` to this
        placeholder map and returns the intersecting items.
        """
        intersecting_items = []

        if len(multi_modal_items) != len(multi_modal_placeholders):
            raise ValueError(
                "Multi-modal placeholders and items must have the same length."
            )
        for placeholder_dict, mm_item in zip(multi_modal_placeholders,
                                             multi_modal_items):
            placeholder = range(
                placeholder_dict.offset,
                placeholder_dict.offset + placeholder_dict.length,
            )
            intersection = range(
                max(positions.start, placeholder.start),
                min(positions.stop, placeholder.stop),
            )

            if not intersection:
                # Skip this multi-modal item.
                continue

            token_embedding_range = range(
                intersection.start - positions.start,
                intersection.stop - positions.start,
            )

            multimodal_embedding_range = range(
                intersection.start - placeholder.start + self.src_len,
                intersection.stop - placeholder.start + self.src_len,
            )

            intersecting_items.append(mm_item)
            self.dest_ranges.append(token_embedding_range)
            self.src_ranges.append(multimodal_embedding_range)
            self.src_len += len(placeholder)

        self.dest_len += len(positions)
        return intersecting_items

    def extend(self, other: "MultiModalPlaceholderMap"):
        """
        Adds the placeholders from another ``MultiModalPlaceholderMap`` to this
        instance based on the source and destination tensors being
        concatenated.
        """

        self.src_ranges.extend(
            range(self.src_len + r.start, self.src_len + r.stop)
            for r in other.src_ranges)
        self.src_len += other.src_len
        self.dest_ranges.extend(
            range(self.dest_len + r.start, self.dest_len + r.stop)
            for r in other.dest_ranges)
        self.dest_len += other.dest_len

    def index_map(self) -> "IndexMap":
        """
        Finalizes the placeholder map into lists of indices that can be used to
        index the source and destination tensors.
        """

        src_indices = [i for r in self.src_ranges for i in r]
        dest_indices = [i for r in self.dest_ranges for i in r]

        if len(src_indices) != len(dest_indices):
            raise ValueError(
                f"The number of source ({len(src_indices)}) and destination "
                f"indices ({len(dest_indices)}) must be the same.")

        return self.IndexMap(src=src_indices, dest=dest_indices)

dest_len instance-attribute 

dest_len: int = 0

The total number of embeddings in the destination tensor.

dest_ranges instance-attribute 

dest_ranges: list[range] = []

The indices of the placeholder embeddings that will be replaced by the multimodal embeddings.

src_len instance-attribute 

src_len: int = 0

The total number of flattened multi-modal embeddings.

src_ranges instance-attribute 

src_ranges: list[range] = []

The indices of the multi-modal embeddings that will replace the corresponding placeholder embeddings pointed to by dest_ranges.

IndexMap

Bases: NamedTuple

Source code in vllm/multimodal/base.py 
24
25
26
class IndexMap(NamedTuple):
    src: list[int]
    dest: list[int]

dest instance-attribute 

dest: list[int]

src instance-attribute 

src: list[int]

__init__ 

__init__()
Source code in vllm/multimodal/base.py 
50
51
52
53
54
def __init__(self):
    self.src_ranges = []
    self.src_len = 0
    self.dest_ranges = []
    self.dest_len = 0

append_items_from_seq_group 

append_items_from_seq_group(
    positions: range,
    multi_modal_items: list[_T],
    multi_modal_placeholders: Sequence[PlaceholderRange],
) -> list[_T]

Adds the multi-modal items that intersect `positions to this placeholder map and returns the intersecting items.

Source code in vllm/multimodal/base.py 
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
def append_items_from_seq_group(
    self,
    positions: range,
    multi_modal_items: list[_T],
    multi_modal_placeholders: Sequence[PlaceholderRange],
) -> list[_T]:
    """
    Adds the multi-modal items that intersect ```positions`` to this
    placeholder map and returns the intersecting items.
    """
    intersecting_items = []

    if len(multi_modal_items) != len(multi_modal_placeholders):
        raise ValueError(
            "Multi-modal placeholders and items must have the same length."
        )
    for placeholder_dict, mm_item in zip(multi_modal_placeholders,
                                         multi_modal_items):
        placeholder = range(
            placeholder_dict.offset,
            placeholder_dict.offset + placeholder_dict.length,
        )
        intersection = range(
            max(positions.start, placeholder.start),
            min(positions.stop, placeholder.stop),
        )

        if not intersection:
            # Skip this multi-modal item.
            continue

        token_embedding_range = range(
            intersection.start - positions.start,
            intersection.stop - positions.start,
        )

        multimodal_embedding_range = range(
            intersection.start - placeholder.start + self.src_len,
            intersection.stop - placeholder.start + self.src_len,
        )

        intersecting_items.append(mm_item)
        self.dest_ranges.append(token_embedding_range)
        self.src_ranges.append(multimodal_embedding_range)
        self.src_len += len(placeholder)

    self.dest_len += len(positions)
    return intersecting_items

extend 

extend(other: MultiModalPlaceholderMap)

Adds the placeholders from another MultiModalPlaceholderMap to this instance based on the source and destination tensors being concatenated.

Source code in vllm/multimodal/base.py 
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
def extend(self, other: "MultiModalPlaceholderMap"):
    """
    Adds the placeholders from another ``MultiModalPlaceholderMap`` to this
    instance based on the source and destination tensors being
    concatenated.
    """

    self.src_ranges.extend(
        range(self.src_len + r.start, self.src_len + r.stop)
        for r in other.src_ranges)
    self.src_len += other.src_len
    self.dest_ranges.extend(
        range(self.dest_len + r.start, self.dest_len + r.stop)
        for r in other.dest_ranges)
    self.dest_len += other.dest_len

from_seq_group classmethod 

from_seq_group(
    seq_group: SequenceGroupMetadata, positions: range
) -> tuple[
    MultiModalKwargs, dict[str, MultiModalPlaceholderMap]
]

Returns the multi-modal items that intersect with the portion of a prompt (seq_group) represented by positions, as well as a MultiModalPlaceholderMap that relates the multi-modal embedding vectors to their corresponding placeholders.

Examples:

Prompt:    |AAAA BBBB What's in these images?|
Positions: |.................................|

    images      = [A, B]
    src_ranges  = [(0, 4), (4, 8)]
    dest_ranges = [(0, 4), (5, 9)]

Prompt:    |AAAA BBBB What's in these images?|
Positions: |  .....                          |

    images      = [A, B]
    src_ranges  = [(2, 4), (4, 6)]
    dest_ranges = [(0, 2), (3, 5)]

Prompt:    |AAAA BBBB What's in these images?|
Positions: |     .........                   |

    images      = [B]
    src_ranges  = [(0, 4)]
    dest_ranges = [(0, 4)]

Prompt:    |AAAA BBBB What's in these images?|
Positions: |          .......................|

    images      = []
    src_ranges  = []
    dest_ranges = []
Source code in vllm/multimodal/base.py 
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
@classmethod
def from_seq_group(
    cls, seq_group: "SequenceGroupMetadata", positions: range
) -> tuple[MultiModalKwargs, dict[str, "MultiModalPlaceholderMap"]]:
    """
    Returns the multi-modal items that intersect with the portion of a
    prompt (``seq_group``) represented by ``positions``, as well as a
    ``MultiModalPlaceholderMap`` that relates the multi-modal embedding
    vectors to their corresponding placeholders.

    Examples:

    ```
    Prompt:    |AAAA BBBB What's in these images?|
    Positions: |.................................|

        images      = [A, B]
        src_ranges  = [(0, 4), (4, 8)]
        dest_ranges = [(0, 4), (5, 9)]

    Prompt:    |AAAA BBBB What's in these images?|
    Positions: |  .....                          |

        images      = [A, B]
        src_ranges  = [(2, 4), (4, 6)]
        dest_ranges = [(0, 2), (3, 5)]

    Prompt:    |AAAA BBBB What's in these images?|
    Positions: |     .........                   |

        images      = [B]
        src_ranges  = [(0, 4)]
        dest_ranges = [(0, 4)]

    Prompt:    |AAAA BBBB What's in these images?|
    Positions: |          .......................|

        images      = []
        src_ranges  = []
        dest_ranges = []
    ```
    """
    seq_mm_data = seq_group.multi_modal_data
    seq_mm_placeholders = seq_group.multi_modal_placeholders

    if not seq_mm_data or not seq_mm_placeholders:
        return MultiModalKwargs({}), {}

    placeholder_maps = dict[str, MultiModalPlaceholderMap]()

    for modality, placeholders in seq_mm_placeholders.items():
        placeholder_map = MultiModalPlaceholderMap()

        if positions:
            placeholder_map.append_items_from_seq_group(
                positions,
                # Dummy, since we don't care about intersecting items
                [None] * len(placeholders),
                placeholders,
            )

        placeholder_maps[modality] = placeholder_map

    return seq_mm_data, placeholder_maps

index_map 

index_map() -> IndexMap

Finalizes the placeholder map into lists of indices that can be used to index the source and destination tensors.

Source code in vllm/multimodal/base.py 
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
def index_map(self) -> "IndexMap":
    """
    Finalizes the placeholder map into lists of indices that can be used to
    index the source and destination tensors.
    """

    src_indices = [i for r in self.src_ranges for i in r]
    dest_indices = [i for r in self.dest_ranges for i in r]

    if len(src_indices) != len(dest_indices):
        raise ValueError(
            f"The number of source ({len(src_indices)}) and destination "
            f"indices ({len(dest_indices)}) must be the same.")

    return self.IndexMap(src=src_indices, dest=dest_indices)

MultiModalRegistry

A registry that dispatches data processing according to the model.

Source code in vllm/multimodal/registry.py 
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
class MultiModalRegistry:
    """
    A registry that dispatches data processing according to the model.
    """

    def __init__(self) -> None:
        self._processor_factories = ClassRegistry[nn.Module,
                                                  _ProcessorFactories]()

        self._processing_cache = ProcessingCache(VLLM_MM_INPUT_CACHE_GIB)

    def reset_processor_cache(self) -> bool:
        """Reset the multi-modal processing cache."""
        self._processing_cache.reset()

        return True  # Success

    @deprecated("Legacy input processor/mapper pipeline has been removed. "
                "Please update your model runner to use "
                "`seq_group_metadata.multi_modal_data` directly without "
                "further processing.")
    def create_input_mapper(self, model_config: "ModelConfig"):
        return lambda data, mm_processor_kwargs: data

    def get_max_tokens_per_item_by_modality(
        self,
        model_config: "ModelConfig",
    ) -> Mapping[str, int]:
        """
        Get the maximum number of tokens per data item from each modality based
        on underlying model configuration.
        """
        if not model_config.is_multimodal_model:
            return {}

        processor = self.create_processor(model_config, disable_cache=False)
        profiler = MultiModalProfiler(processor)

        seq_len = model_config.max_model_len
        mm_limits = self.get_mm_limits_per_prompt(model_config)

        return profiler.get_mm_max_tokens(
            seq_len,
            {
                modality: 1
                for modality, limit in mm_limits.items() if limit > 0
            },
        )

    def get_max_tokens_per_item_by_nonzero_modality(
        self,
        model_config: "ModelConfig",
    ) -> Mapping[str, int]:
        """
        Get the maximum number of tokens per data item from each modality based
        on underlying model configuration, excluding modalities that user
        explicitly disabled via `limit_mm_per_prompt`.

        Note:
            This is currently directly used only in V1 for profiling the memory
            usage of a model.
        """
        mm_limits = self.get_mm_limits_per_prompt(model_config)

        return {
            key: max_tokens_per_mm_item
            for key, max_tokens_per_mm_item in
            self.get_max_tokens_per_item_by_modality(model_config).items()
            if mm_limits[key] > 0
        }

    def get_max_tokens_by_modality(
        self,
        model_config: "ModelConfig",
    ) -> Mapping[str, int]:
        """
        Get the maximum number of tokens from each modality
        for profiling the memory usage of a model.
        """
        mm_limits = self.get_mm_limits_per_prompt(model_config)

        return {
            key: mm_limits[key] * max_tokens_per_mm_item
            for key, max_tokens_per_mm_item in
            self.get_max_tokens_per_item_by_modality(model_config).items()
        }

    def get_max_multimodal_tokens(self, model_config: "ModelConfig") -> int:
        """
        Get the maximum number of multi-modal tokens
        for profiling the memory usage of a model.
        """
        return sum(self.get_max_tokens_by_modality(model_config).values())

    @deprecated("Legacy input processor/mapper pipeline has been removed. "
                "Please update your model runner to use "
                "`seq_group_metadata.multi_modal_data` directly without "
                "further processing.")
    def init_mm_limits_per_prompt(
        self,
        model_config: "ModelConfig",
    ) -> None:
        pass

    def get_mm_limits_per_prompt(
        self,
        model_config: "ModelConfig",
    ) -> Mapping[str, int]:
        """
        Get the maximum number of multi-modal input instances for each modality
        that are allowed per prompt for a model class.
        """
        if not model_config.is_multimodal_model:
            return {}

        processor = self.create_processor(model_config, disable_cache=False)
        profiler = MultiModalProfiler(processor)
        return profiler.get_mm_limits()

    def register_processor(
        self,
        processor: MultiModalProcessorFactory[_I],
        *,
        info: ProcessingInfoFactory[_I],
        dummy_inputs: DummyInputsBuilderFactory[_I],
    ):
        """
        Register a multi-modal processor to a model class. The processor
        is constructed lazily, hence a factory method should be passed.

        When the model receives multi-modal data, the provided function is
        invoked to transform the data into a dictionary of model inputs.
        """

        def wrapper(model_cls: N) -> N:
            if self._processor_factories.contains(model_cls, strict=True):
                logger.warning(
                    "Model class %s already has a multi-modal processor "
                    "registered to %s. It is overwritten by the new one.",
                    model_cls, self)

            self._processor_factories[model_cls] = _ProcessorFactories(
                info=info,
                dummy_inputs=dummy_inputs,
                processor=processor,
            )

            return model_cls

        return wrapper

    def _get_model_cls(self, model_config: "ModelConfig"):
        # Avoid circular import
        from vllm.model_executor.model_loader import get_model_architecture

        model_cls, _ = get_model_architecture(model_config)
        return model_cls

    @deprecated("Legacy input processor/mapper pipeline has been removed. "
                "Please update your model runner to use "
                "`seq_group_metadata.multi_modal_data` directly without "
                "further processing.")
    def has_processor(self, model_config: "ModelConfig") -> bool:
        return True

    def create_processor(
        self,
        model_config: "ModelConfig",
        *,
        tokenizer: Optional[AnyTokenizer] = None,
        disable_cache: Optional[bool] = None,
    ) -> BaseMultiModalProcessor[BaseProcessingInfo]:
        """
        Create a multi-modal processor for a specific model and tokenizer.
        """
        if not model_config.is_multimodal_model:
            raise ValueError(f"{model_config.model} is not a multimodal model")

        if tokenizer is None:
            tokenizer = cached_tokenizer_from_config(model_config)
        if disable_cache is None:
            mm_config = model_config.get_multimodal_config()
            disable_cache = mm_config.disable_mm_preprocessor_cache

        model_cls = self._get_model_cls(model_config)
        factories = self._processor_factories[model_cls]

        ctx = InputProcessingContext(model_config, tokenizer)
        cache = None if disable_cache else self._processing_cache

        return factories.build_processor(ctx, cache=cache)

    def get_decoder_dummy_data(
        self,
        model_config: "ModelConfig",
        seq_len: int,
        mm_counts: Optional[Mapping[str, int]] = None,
    ) -> DummyDecoderData:
        """
        Create dummy data for profiling the memory usage of a model.

        The model is identified by ``model_config``.
        """
        processor = self.create_processor(model_config, disable_cache=False)
        profiler = MultiModalProfiler(processor)
        dummy_data = profiler.get_decoder_dummy_data(seq_len, mm_counts)

        # Having more tokens is over-conservative but otherwise fine
        token_ids = dummy_data.prompt_token_ids
        if len(token_ids) < seq_len:
            raise AssertionError(
                f"Expected at least {seq_len} dummy tokens for profiling, "
                f"but found {len(token_ids)} tokens instead.")

        return dummy_data

    def get_encoder_dummy_data(
        self,
        model_config: "ModelConfig",
        seq_len: int,
        mm_counts: Optional[Mapping[str, int]] = None,
    ) -> DummyEncoderData:
        """
        Create dummy data for profiling the memory usage of a model.

        The model is identified by ``model_config``.
        """
        processor = self.create_processor(model_config, disable_cache=False)
        profiler = MultiModalProfiler(processor)
        dummy_data = profiler.get_encoder_dummy_data(seq_len, mm_counts)

        # Having more tokens is over-conservative but otherwise fine
        token_ids = dummy_data.prompt_token_ids
        if len(token_ids) < seq_len:
            logger.warning_once(
                "Expected at least %d dummy encoder tokens for profiling, but found %d tokens instead.",  # noqa: E501
                seq_len,
                len(token_ids),
            )

        return dummy_data

_processing_cache instance-attribute 

_processing_cache = ProcessingCache(VLLM_MM_INPUT_CACHE_GIB)

_processor_factories instance-attribute 

_processor_factories = ClassRegistry[
    Module, _ProcessorFactories
]()

__init__ 

__init__() -> None
Source code in vllm/multimodal/registry.py 
 96
 97
 98
 99
100
def __init__(self) -> None:
    self._processor_factories = ClassRegistry[nn.Module,
                                              _ProcessorFactories]()

    self._processing_cache = ProcessingCache(VLLM_MM_INPUT_CACHE_GIB)

_get_model_cls 

_get_model_cls(model_config: ModelConfig)
Source code in vllm/multimodal/registry.py 
242
243
244
245
246
247
def _get_model_cls(self, model_config: "ModelConfig"):
    # Avoid circular import
    from vllm.model_executor.model_loader import get_model_architecture

    model_cls, _ = get_model_architecture(model_config)
    return model_cls

create_input_mapper 

create_input_mapper(model_config: ModelConfig)
Source code in vllm/multimodal/registry.py 
108
109
110
111
112
113
@deprecated("Legacy input processor/mapper pipeline has been removed. "
            "Please update your model runner to use "
            "`seq_group_metadata.multi_modal_data` directly without "
            "further processing.")
def create_input_mapper(self, model_config: "ModelConfig"):
    return lambda data, mm_processor_kwargs: data

create_processor 

create_processor(
    model_config: ModelConfig,
    *,
    tokenizer: Optional[AnyTokenizer] = None,
    disable_cache: Optional[bool] = None,
) -> BaseMultiModalProcessor[BaseProcessingInfo]

Create a multi-modal processor for a specific model and tokenizer.

Source code in vllm/multimodal/registry.py 
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
def create_processor(
    self,
    model_config: "ModelConfig",
    *,
    tokenizer: Optional[AnyTokenizer] = None,
    disable_cache: Optional[bool] = None,
) -> BaseMultiModalProcessor[BaseProcessingInfo]:
    """
    Create a multi-modal processor for a specific model and tokenizer.
    """
    if not model_config.is_multimodal_model:
        raise ValueError(f"{model_config.model} is not a multimodal model")

    if tokenizer is None:
        tokenizer = cached_tokenizer_from_config(model_config)
    if disable_cache is None:
        mm_config = model_config.get_multimodal_config()
        disable_cache = mm_config.disable_mm_preprocessor_cache

    model_cls = self._get_model_cls(model_config)
    factories = self._processor_factories[model_cls]

    ctx = InputProcessingContext(model_config, tokenizer)
    cache = None if disable_cache else self._processing_cache

    return factories.build_processor(ctx, cache=cache)

get_decoder_dummy_data 

get_decoder_dummy_data(
    model_config: ModelConfig,
    seq_len: int,
    mm_counts: Optional[Mapping[str, int]] = None,
) -> DummyDecoderData

Create dummy data for profiling the memory usage of a model.

The model is identified by model_config.

Source code in vllm/multimodal/registry.py 
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
def get_decoder_dummy_data(
    self,
    model_config: "ModelConfig",
    seq_len: int,
    mm_counts: Optional[Mapping[str, int]] = None,
) -> DummyDecoderData:
    """
    Create dummy data for profiling the memory usage of a model.

    The model is identified by ``model_config``.
    """
    processor = self.create_processor(model_config, disable_cache=False)
    profiler = MultiModalProfiler(processor)
    dummy_data = profiler.get_decoder_dummy_data(seq_len, mm_counts)

    # Having more tokens is over-conservative but otherwise fine
    token_ids = dummy_data.prompt_token_ids
    if len(token_ids) < seq_len:
        raise AssertionError(
            f"Expected at least {seq_len} dummy tokens for profiling, "
            f"but found {len(token_ids)} tokens instead.")

    return dummy_data

get_encoder_dummy_data 

get_encoder_dummy_data(
    model_config: ModelConfig,
    seq_len: int,
    mm_counts: Optional[Mapping[str, int]] = None,
) -> DummyEncoderData

Create dummy data for profiling the memory usage of a model.

The model is identified by model_config.

Source code in vllm/multimodal/registry.py 
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
def get_encoder_dummy_data(
    self,
    model_config: "ModelConfig",
    seq_len: int,
    mm_counts: Optional[Mapping[str, int]] = None,
) -> DummyEncoderData:
    """
    Create dummy data for profiling the memory usage of a model.

    The model is identified by ``model_config``.
    """
    processor = self.create_processor(model_config, disable_cache=False)
    profiler = MultiModalProfiler(processor)
    dummy_data = profiler.get_encoder_dummy_data(seq_len, mm_counts)

    # Having more tokens is over-conservative but otherwise fine
    token_ids = dummy_data.prompt_token_ids
    if len(token_ids) < seq_len:
        logger.warning_once(
            "Expected at least %d dummy encoder tokens for profiling, but found %d tokens instead.",  # noqa: E501
            seq_len,
            len(token_ids),
        )

    return dummy_data

get_max_multimodal_tokens 

get_max_multimodal_tokens(model_config: ModelConfig) -> int

Get the maximum number of multi-modal tokens for profiling the memory usage of a model.

Source code in vllm/multimodal/registry.py 
178
179
180
181
182
183
def get_max_multimodal_tokens(self, model_config: "ModelConfig") -> int:
    """
    Get the maximum number of multi-modal tokens
    for profiling the memory usage of a model.
    """
    return sum(self.get_max_tokens_by_modality(model_config).values())

get_max_tokens_by_modality 

get_max_tokens_by_modality(
    model_config: ModelConfig,
) -> Mapping[str, int]

Get the maximum number of tokens from each modality for profiling the memory usage of a model.

Source code in vllm/multimodal/registry.py 
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
def get_max_tokens_by_modality(
    self,
    model_config: "ModelConfig",
) -> Mapping[str, int]:
    """
    Get the maximum number of tokens from each modality
    for profiling the memory usage of a model.
    """
    mm_limits = self.get_mm_limits_per_prompt(model_config)

    return {
        key: mm_limits[key] * max_tokens_per_mm_item
        for key, max_tokens_per_mm_item in
        self.get_max_tokens_per_item_by_modality(model_config).items()
    }

get_max_tokens_per_item_by_modality 

get_max_tokens_per_item_by_modality(
    model_config: ModelConfig,
) -> Mapping[str, int]

Get the maximum number of tokens per data item from each modality based on underlying model configuration.

Source code in vllm/multimodal/registry.py 
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
def get_max_tokens_per_item_by_modality(
    self,
    model_config: "ModelConfig",
) -> Mapping[str, int]:
    """
    Get the maximum number of tokens per data item from each modality based
    on underlying model configuration.
    """
    if not model_config.is_multimodal_model:
        return {}

    processor = self.create_processor(model_config, disable_cache=False)
    profiler = MultiModalProfiler(processor)

    seq_len = model_config.max_model_len
    mm_limits = self.get_mm_limits_per_prompt(model_config)

    return profiler.get_mm_max_tokens(
        seq_len,
        {
            modality: 1
            for modality, limit in mm_limits.items() if limit > 0
        },
    )

get_max_tokens_per_item_by_nonzero_modality 

get_max_tokens_per_item_by_nonzero_modality(
    model_config: ModelConfig,
) -> Mapping[str, int]

Get the maximum number of tokens per data item from each modality based on underlying model configuration, excluding modalities that user explicitly disabled via limit_mm_per_prompt.

Note

This is currently directly used only in V1 for profiling the memory usage of a model.

Source code in vllm/multimodal/registry.py 
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
def get_max_tokens_per_item_by_nonzero_modality(
    self,
    model_config: "ModelConfig",
) -> Mapping[str, int]:
    """
    Get the maximum number of tokens per data item from each modality based
    on underlying model configuration, excluding modalities that user
    explicitly disabled via `limit_mm_per_prompt`.

    Note:
        This is currently directly used only in V1 for profiling the memory
        usage of a model.
    """
    mm_limits = self.get_mm_limits_per_prompt(model_config)

    return {
        key: max_tokens_per_mm_item
        for key, max_tokens_per_mm_item in
        self.get_max_tokens_per_item_by_modality(model_config).items()
        if mm_limits[key] > 0
    }

get_mm_limits_per_prompt 

get_mm_limits_per_prompt(
    model_config: ModelConfig,
) -> Mapping[str, int]

Get the maximum number of multi-modal input instances for each modality that are allowed per prompt for a model class.

Source code in vllm/multimodal/registry.py 
195
196
197
198
199
200
201
202
203
204
205
206
207
208
def get_mm_limits_per_prompt(
    self,
    model_config: "ModelConfig",
) -> Mapping[str, int]:
    """
    Get the maximum number of multi-modal input instances for each modality
    that are allowed per prompt for a model class.
    """
    if not model_config.is_multimodal_model:
        return {}

    processor = self.create_processor(model_config, disable_cache=False)
    profiler = MultiModalProfiler(processor)
    return profiler.get_mm_limits()

has_processor 

has_processor(model_config: ModelConfig) -> bool
Source code in vllm/multimodal/registry.py 
249
250
251
252
253
254
@deprecated("Legacy input processor/mapper pipeline has been removed. "
            "Please update your model runner to use "
            "`seq_group_metadata.multi_modal_data` directly without "
            "further processing.")
def has_processor(self, model_config: "ModelConfig") -> bool:
    return True

init_mm_limits_per_prompt 

init_mm_limits_per_prompt(
    model_config: ModelConfig,
) -> None
Source code in vllm/multimodal/registry.py 
185
186
187
188
189
190
191
192
193
@deprecated("Legacy input processor/mapper pipeline has been removed. "
            "Please update your model runner to use "
            "`seq_group_metadata.multi_modal_data` directly without "
            "further processing.")
def init_mm_limits_per_prompt(
    self,
    model_config: "ModelConfig",
) -> None:
    pass

register_processor 

register_processor(
    processor: MultiModalProcessorFactory[_I" optional hover>_I],
    *,
    info: ProcessingInfoFactory[_I" optional hover>_I],
    dummy_inputs: DummyInputsBuilderFactory[_I],
)

Register a multi-modal processor to a model class. The processor is constructed lazily, hence a factory method should be passed.

When the model receives multi-modal data, the provided function is invoked to transform the data into a dictionary of model inputs.

Source code in vllm/multimodal/registry.py 
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
def register_processor(
    self,
    processor: MultiModalProcessorFactory[_I],
    *,
    info: ProcessingInfoFactory[_I],
    dummy_inputs: DummyInputsBuilderFactory[_I],
):
    """
    Register a multi-modal processor to a model class. The processor
    is constructed lazily, hence a factory method should be passed.

    When the model receives multi-modal data, the provided function is
    invoked to transform the data into a dictionary of model inputs.
    """

    def wrapper(model_cls: N) -> N:
        if self._processor_factories.contains(model_cls, strict=True):
            logger.warning(
                "Model class %s already has a multi-modal processor "
                "registered to %s. It is overwritten by the new one.",
                model_cls, self)

        self._processor_factories[model_cls] = _ProcessorFactories(
            info=info,
            dummy_inputs=dummy_inputs,
            processor=processor,
        )

        return model_cls

    return wrapper

reset_processor_cache 

reset_processor_cache() -> bool

Reset the multi-modal processing cache.

Source code in vllm/multimodal/registry.py 
102
103
104
105
106
def reset_processor_cache(self) -> bool:
    """Reset the multi-modal processing cache."""
    self._processing_cache.reset()

    return True  # Success