Skip to content

vllm.multimodal.hasher

MultiModalHashDict module-attribute 

MultiModalHashDict = Mapping[str, list[str]]

A dictionary containing hashes for items in each modality.

logger module-attribute 

logger = init_logger(__name__)

MultiModalHasher

Source code in vllm/multimodal/hasher.py 
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
class MultiModalHasher:

    @classmethod
    def serialize_item(cls, obj: object) -> bytes:
        # Simple cases
        if isinstance(obj, str):
            return obj.encode("utf-8")
        if isinstance(obj, bytes):
            return obj
        if isinstance(obj, (int, float)):
            return np.array(obj).tobytes()

        if isinstance(obj, Image.Image):
            return cls.item_to_bytes(
                "image", np.asarray(convert_image_mode(obj, "RGBA")))
        if isinstance(obj, torch.Tensor):
            return cls.item_to_bytes("tensor", obj.numpy())
        if isinstance(obj, np.ndarray):
            return cls.item_to_bytes(
                "ndarray", {
                    "dtype": obj.dtype.str,
                    "shape": obj.shape,
                    "data": obj.tobytes(),
                })

        logger.warning(
            "No serialization method found for %s. "
            "Falling back to pickle.", type(obj))

        return pickle.dumps(obj)

    @classmethod
    def item_to_bytes(
        cls,
        key: str,
        obj: object,
    ) -> bytes:
        return b''.join(kb + vb for kb, vb in cls.iter_item_to_bytes(key, obj))

    @classmethod
    def iter_item_to_bytes(
        cls,
        key: str,
        obj: object,
    ) -> Iterable[tuple[bytes, bytes]]:
        # Recursive cases
        if isinstance(obj, (list, tuple)):
            for i, elem in enumerate(obj):
                yield from cls.iter_item_to_bytes(f"{key}.{i}", elem)
        elif isinstance(obj, dict):
            for k, v in obj.items():
                yield from cls.iter_item_to_bytes(f"{key}.{k}", v)
        else:
            key_bytes = cls.serialize_item(key)
            value_bytes = cls.serialize_item(obj)
            yield key_bytes, value_bytes

    @classmethod
    def hash_kwargs(cls, **kwargs: object) -> str:
        hasher = blake3()

        for k, v in kwargs.items():
            for k_bytes, v_bytes in cls.iter_item_to_bytes(k, v):
                hasher.update(k_bytes)
                hasher.update(v_bytes)

        return hasher.hexdigest()

    @classmethod
    def hash_prompt_mm_data(
            cls, prompt: "TokensPrompt") -> Optional["MultiModalHashDict"]:
        """Hash multimodal data in the user input prompt if they exist."""

        if "multi_modal_data" not in prompt:
            return None

        mm_data = prompt["multi_modal_data"]
        if not mm_data:
            # mm_data can be None or an empty dict.
            return None

        mm_items = {
            modality: items if isinstance(items, list) else [items]
            for modality, items in mm_data.items()
        }

        mm_hashes = {
            modality: [cls.hash_kwargs(**{modality: item}) for item in items]
            for modality, items in mm_items.items()
        }

        return mm_hashes

hash_kwargs classmethod 

hash_kwargs(**kwargs: object) -> str
Source code in vllm/multimodal/hasher.py 
83
84
85
86
87
88
89
90
91
92
@classmethod
def hash_kwargs(cls, **kwargs: object) -> str:
    hasher = blake3()

    for k, v in kwargs.items():
        for k_bytes, v_bytes in cls.iter_item_to_bytes(k, v):
            hasher.update(k_bytes)
            hasher.update(v_bytes)

    return hasher.hexdigest()

hash_prompt_mm_data classmethod 

hash_prompt_mm_data(
    prompt: TokensPrompt,
) -> Optional[MultiModalHashDict]

Hash multimodal data in the user input prompt if they exist.

Source code in vllm/multimodal/hasher.py 
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
@classmethod
def hash_prompt_mm_data(
        cls, prompt: "TokensPrompt") -> Optional["MultiModalHashDict"]:
    """Hash multimodal data in the user input prompt if they exist."""

    if "multi_modal_data" not in prompt:
        return None

    mm_data = prompt["multi_modal_data"]
    if not mm_data:
        # mm_data can be None or an empty dict.
        return None

    mm_items = {
        modality: items if isinstance(items, list) else [items]
        for modality, items in mm_data.items()
    }

    mm_hashes = {
        modality: [cls.hash_kwargs(**{modality: item}) for item in items]
        for modality, items in mm_items.items()
    }

    return mm_hashes

item_to_bytes classmethod 

item_to_bytes(key: str, obj: object) -> bytes
Source code in vllm/multimodal/hasher.py 
57
58
59
60
61
62
63
@classmethod
def item_to_bytes(
    cls,
    key: str,
    obj: object,
) -> bytes:
    return b''.join(kb + vb for kb, vb in cls.iter_item_to_bytes(key, obj))

iter_item_to_bytes classmethod 

iter_item_to_bytes(
    key: str, obj: object
) -> Iterable[tuple[bytes, bytes]]
Source code in vllm/multimodal/hasher.py 
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
@classmethod
def iter_item_to_bytes(
    cls,
    key: str,
    obj: object,
) -> Iterable[tuple[bytes, bytes]]:
    # Recursive cases
    if isinstance(obj, (list, tuple)):
        for i, elem in enumerate(obj):
            yield from cls.iter_item_to_bytes(f"{key}.{i}", elem)
    elif isinstance(obj, dict):
        for k, v in obj.items():
            yield from cls.iter_item_to_bytes(f"{key}.{k}", v)
    else:
        key_bytes = cls.serialize_item(key)
        value_bytes = cls.serialize_item(obj)
        yield key_bytes, value_bytes

serialize_item classmethod 

serialize_item(obj: object) -> bytes
Source code in vllm/multimodal/hasher.py 
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
@classmethod
def serialize_item(cls, obj: object) -> bytes:
    # Simple cases
    if isinstance(obj, str):
        return obj.encode("utf-8")
    if isinstance(obj, bytes):
        return obj
    if isinstance(obj, (int, float)):
        return np.array(obj).tobytes()

    if isinstance(obj, Image.Image):
        return cls.item_to_bytes(
            "image", np.asarray(convert_image_mode(obj, "RGBA")))
    if isinstance(obj, torch.Tensor):
        return cls.item_to_bytes("tensor", obj.numpy())
    if isinstance(obj, np.ndarray):
        return cls.item_to_bytes(
            "ndarray", {
                "dtype": obj.dtype.str,
                "shape": obj.shape,
                "data": obj.tobytes(),
            })

    logger.warning(
        "No serialization method found for %s. "
        "Falling back to pickle.", type(obj))

    return pickle.dumps(obj)