Skip to content

vllm.v1.core.encoder_cache_manager

logger module-attribute 

logger = init_logger(__name__)

EncoderCacheManager

Source code in vllm/v1/core/encoder_cache_manager.py 
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
class EncoderCacheManager:

    def __init__(self, cache_size: int):
        self.cache_size = cache_size
        self.num_free_slots = cache_size
        # req_id -> cached input ids
        self.cached: dict[str, set[int]] = {}
        # list of [req_id, input_id]
        self.freed: list[tuple[str, int]] = []

    def has_cache(self, request: Request, input_id: int) -> bool:
        req_id = request.request_id
        return req_id in self.cached and input_id in self.cached[req_id]

    def can_allocate(self, request: Request, input_id: int) -> bool:
        num_tokens = request.get_num_encoder_tokens(input_id)
        return num_tokens <= self.num_free_slots

    def allocate(self, request: Request, input_id: int) -> None:
        req_id = request.request_id
        if req_id not in self.cached:
            self.cached[req_id] = set()
        self.cached[req_id].add(input_id)
        self.num_free_slots -= request.get_num_encoder_tokens(input_id)

    def get_cached_input_ids(self, request: Request) -> set[int]:
        return self.cached.get(request.request_id, set())

    def free_encoder_input(self, request: Request, input_id: int) -> None:
        """Free a single encoder input id for the request."""
        req_id = request.request_id
        if req_id not in self.cached:
            return

        self.cached[req_id].discard(input_id)
        if len(self.cached[req_id]) == 0:
            del self.cached[req_id]
        self.num_free_slots += request.get_num_encoder_tokens(input_id)
        self.freed.append((req_id, input_id))

    def free(self, request: Request) -> None:
        """Free all cached input ids for the request."""
        input_ids = self.get_cached_input_ids(request).copy()
        for input_id in input_ids:
            self.free_encoder_input(request, input_id)

    def get_freed_ids(self) -> list[tuple[str, int]]:
        freed = self.freed
        self.freed = []
        return freed

cache_size instance-attribute 

cache_size = cache_size

cached instance-attribute 

cached: dict[str, set[int]] = {}

freed instance-attribute 

freed: list[tuple[str, int]] = []

num_free_slots instance-attribute 

num_free_slots = cache_size

__init__ 

__init__(cache_size: int)
Source code in vllm/v1/core/encoder_cache_manager.py 
17
18
19
20
21
22
23
def __init__(self, cache_size: int):
    self.cache_size = cache_size
    self.num_free_slots = cache_size
    # req_id -> cached input ids
    self.cached: dict[str, set[int]] = {}
    # list of [req_id, input_id]
    self.freed: list[tuple[str, int]] = []

allocate 

allocate(request: Request, input_id: int) -> None
Source code in vllm/v1/core/encoder_cache_manager.py 
33
34
35
36
37
38
def allocate(self, request: Request, input_id: int) -> None:
    req_id = request.request_id
    if req_id not in self.cached:
        self.cached[req_id] = set()
    self.cached[req_id].add(input_id)
    self.num_free_slots -= request.get_num_encoder_tokens(input_id)

can_allocate 

can_allocate(request: Request, input_id: int) -> bool
Source code in vllm/v1/core/encoder_cache_manager.py 
29
30
31
def can_allocate(self, request: Request, input_id: int) -> bool:
    num_tokens = request.get_num_encoder_tokens(input_id)
    return num_tokens <= self.num_free_slots

free 

free(request: Request) -> None

Free all cached input ids for the request.

Source code in vllm/v1/core/encoder_cache_manager.py 
55
56
57
58
59
def free(self, request: Request) -> None:
    """Free all cached input ids for the request."""
    input_ids = self.get_cached_input_ids(request).copy()
    for input_id in input_ids:
        self.free_encoder_input(request, input_id)

free_encoder_input 

free_encoder_input(request: Request, input_id: int) -> None

Free a single encoder input id for the request.

Source code in vllm/v1/core/encoder_cache_manager.py 
43
44
45
46
47
48
49
50
51
52
53
def free_encoder_input(self, request: Request, input_id: int) -> None:
    """Free a single encoder input id for the request."""
    req_id = request.request_id
    if req_id not in self.cached:
        return

    self.cached[req_id].discard(input_id)
    if len(self.cached[req_id]) == 0:
        del self.cached[req_id]
    self.num_free_slots += request.get_num_encoder_tokens(input_id)
    self.freed.append((req_id, input_id))

get_cached_input_ids 

get_cached_input_ids(request: Request) -> set[int]
Source code in vllm/v1/core/encoder_cache_manager.py 
40
41
def get_cached_input_ids(self, request: Request) -> set[int]:
    return self.cached.get(request.request_id, set())

get_freed_ids 

get_freed_ids() -> list[tuple[str, int]]
Source code in vllm/v1/core/encoder_cache_manager.py 
61
62
63
64
def get_freed_ids(self) -> list[tuple[str, int]]:
    freed = self.freed
    self.freed = []
    return freed

has_cache 

has_cache(request: Request, input_id: int) -> bool
Source code in vllm/v1/core/encoder_cache_manager.py 
25
26
27
def has_cache(self, request: Request, input_id: int) -> bool:
    req_id = request.request_id
    return req_id in self.cached and input_id in self.cached[req_id]

_compute_encoder_budget_multimodal 

_compute_encoder_budget_multimodal(
    model_config: ModelConfig,
    scheduler_config: SchedulerConfig,
    mm_registry: MultiModalRegistry,
) -> tuple[int, int]

Compute the encoder cache budget based on the model and scheduler configurations for a multimodal model.

Parameters:

Name Type Description Default
model_config ModelConfig

Model configuration.

 required
scheduler_config SchedulerConfig

Scheduler configuration.

 required
mm_registry MultiModalRegistry

Provides information about the token cost.

 required

Returns:

Type Description
int
  • Compute budget for encoder execution, in unit of number of tokens in the input sequence.
int
  • Space budget for encoder cache size, in unit of number of tokens in the input sequence.
Source code in vllm/v1/core/encoder_cache_manager.py 
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
def _compute_encoder_budget_multimodal(
    model_config: "ModelConfig",
    scheduler_config: "SchedulerConfig",
    mm_registry: MultiModalRegistry,
) -> tuple[int, int]:
    """Compute the encoder cache budget based on the model and scheduler 
    configurations for a multimodal model.

    Args:
        model_config: Model configuration.
        scheduler_config: Scheduler configuration.
        mm_registry: Provides information about the token cost.

    Returns:
        - Compute budget for encoder execution, in unit of number of tokens 
            in the input sequence.
        - Space budget for encoder cache size, in unit of number of tokens 
            in the input sequence.
    """

    max_tokens_by_modality_dict = mm_registry \
        .get_max_tokens_per_item_by_nonzero_modality(model_config)

    if not max_tokens_by_modality_dict:
        logger.warning(
            "All non-text modalities supported by the model have been "
            "explicitly disabled via limit_mm_per_prompt. Encoder cache will "
            "not be initialized.")
        return 0, 0

    _, max_tokens_per_mm_item = max(max_tokens_by_modality_dict.items(),
                                    key=lambda item: item[1])

    if (scheduler_config.disable_chunked_mm_input and max_tokens_per_mm_item
            > scheduler_config.max_num_batched_tokens):
        raise ValueError(
            "Chunked MM input disabled but max_tokens_per_mm_item "
            f"({max_tokens_per_mm_item}) is larger than max_num_batched_tokens"
            f" ({scheduler_config.max_num_batched_tokens}). Please increase "
            "max_num_batched_tokens.")

    encoder_compute_budget = max(scheduler_config.max_num_encoder_input_tokens,
                                 max_tokens_per_mm_item)
    encoder_cache_size = max(scheduler_config.encoder_cache_size,
                             max_tokens_per_mm_item)

    return encoder_compute_budget, encoder_cache_size

compute_encoder_budget 

compute_encoder_budget(
    model_config: ModelConfig,
    scheduler_config: SchedulerConfig,
    mm_registry: MultiModalRegistry,
) -> tuple[int, int]

Compute the encoder cache budget based on the model and scheduler configurations.

Parameters:

Name Type Description Default
model_config ModelConfig

Model configuration.

 required
scheduler_config SchedulerConfig

Scheduler configuration.

 required
mm_registry MultiModalRegistry

Provides information about the token cost.

 required

Returns:

Type Description
int
  • Compute budget for encoder execution, in unit of number of tokens in the input sequence.
int
  • Space budget for encoder cache size, in unit of number of tokens in the input sequence.
Source code in vllm/v1/core/encoder_cache_manager.py 
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
def compute_encoder_budget(
    model_config: "ModelConfig",
    scheduler_config: "SchedulerConfig",
    mm_registry: MultiModalRegistry,
) -> tuple[int, int]:
    """Compute the encoder cache budget based on the model and scheduler 
    configurations.

    Args:
        model_config: Model configuration.
        scheduler_config: Scheduler configuration.
        mm_registry: Provides information about the token cost.

    Returns:
        - Compute budget for encoder execution, in unit of number of tokens 
            in the input sequence.
        - Space budget for encoder cache size, in unit of number of tokens 
            in the input sequence.
    """

    if not model_config.is_multimodal_model:
        return 0, 0

    # TODO: handle encoder-decoder models once we support them.
    (
        encoder_compute_budget,
        encoder_cache_size,
    ) = _compute_encoder_budget_multimodal(
        model_config,
        scheduler_config,
        mm_registry,
    )

    return encoder_compute_budget, encoder_cache_size