Persist and reuse the memory-profiling result across engine boots.
On startup, vLLM measures how much GPU memory the KV cache can use and computes the --kv-cache-memory value that reproduces that allocation. For a fixed (model, config, hardware, library) combination the result is deterministic, yet it is re-measured on every boot.
When VLLM_ENABLE_STARTUP_PLAN=1, each worker persists that value under {VLLM_CACHE_ROOT}/startup_plan/ (regenerable derived state, alongside the torch.compile cache), keyed by a fingerprint of everything the value depends on, and later boots apply it automatically -- skipping the memory-profiling measurement and the CUDA-graph memory estimation pass -- if and only if the fingerprint matches and the device has at least as much free memory as when the plan was recorded. On any mismatch the worker falls back to full profiling, so a stale plan costs nothing and is never trusted.
Functions:
_applicable_kv_cache_memory_bytes(plan, current_free_memory)
The apply-time OOM-safety gate.
The recorded value is only valid if the device has at least as much free memory now as when the plan was measured (co-tenants, leaked allocations, or MIG changes all reduce it). Outside that envelope, return None and let the caller re-profile.
Source code in vllm/v1/worker/startup_plan.py
| def _applicable_kv_cache_memory_bytes(
plan: dict, current_free_memory: int
) -> int | None:
"""The apply-time OOM-safety gate.
The recorded value is only valid if the device has at least as much
free memory now as when the plan was measured (co-tenants, leaked
allocations, or MIG changes all reduce it). Outside that envelope,
return None and let the caller re-profile.
"""
kv_bytes = plan.get("kv_cache_memory_bytes")
baseline = plan.get("free_memory_baseline")
if not isinstance(kv_bytes, int) or not isinstance(baseline, int):
return None
if kv_bytes <= 0:
return None
if current_free_memory < baseline:
logger.info(
"Startup plan not applied: current free memory (%.2f GiB) is "
"below the recorded baseline (%.2f GiB); falling back to full "
"memory profiling.",
current_free_memory / (1 << 30),
baseline / (1 << 30),
)
return None
return kv_bytes
|
_load_plan(fingerprint)
Load a plan for this fingerprint; None if absent or unreadable.
Source code in vllm/v1/worker/startup_plan.py
| def _load_plan(fingerprint: str) -> dict | None:
"""Load a plan for this fingerprint; None if absent or unreadable."""
path = _plan_path(fingerprint)
try:
with open(path) as f:
plan = json.load(f)
except FileNotFoundError:
return None
except (OSError, json.JSONDecodeError) as e:
logger.warning("Ignoring unreadable startup plan %s: %s", path, e)
return None
if (
plan.get("schema") != PLAN_SCHEMA_VERSION
or plan.get("fingerprint") != fingerprint
):
return None
return plan
|
_plan_path(fingerprint)
Plans are regenerable derived state, so they live under the standard vLLM cache root (like the torch.compile cache) and relocate with VLLM_CACHE_ROOT instead of needing a location knob of their own.
Source code in vllm/v1/worker/startup_plan.py
| def _plan_path(fingerprint: str) -> str:
"""Plans are regenerable derived state, so they live under the standard
vLLM cache root (like the torch.compile cache) and relocate with
``VLLM_CACHE_ROOT`` instead of needing a location knob of their own."""
# VLLM_CACHE_ROOT is already user-expanded by envs.py.
return os.path.join(
envs.VLLM_CACHE_ROOT, "startup_plan", f"startup_plan_{fingerprint}.json"
)
|
compute_plan_fingerprint(vllm_config, rank, world_size)
Hash everything the profiled KV-cache memory value depends on.
VllmConfig.compute_hash() covers the vLLM version and the model, cache, parallel, and compilation configs, but deliberately contains no device identity (DeviceConfig.compute_hash is empty), so device name, total memory, compute capability, and the torch/CUDA build are added here. The vLLM version is also pinned as an explicit factor so version invalidation holds no matter how compute_hash evolves. Rank is included because per-rank memory use differs under TP/PP. Driver-only changes are not part of the key; the free-memory gate at apply time bounds the residual risk.
Source code in vllm/v1/worker/startup_plan.py
| def compute_plan_fingerprint(
vllm_config: VllmConfig, rank: int, world_size: int
) -> str:
"""Hash everything the profiled KV-cache memory value depends on.
``VllmConfig.compute_hash()`` covers the vLLM version and the model,
cache, parallel, and compilation configs, but deliberately contains no
device identity (``DeviceConfig.compute_hash`` is empty), so device
name, total memory, compute capability, and the torch/CUDA build are
added here. The vLLM version is also pinned as an explicit factor so
version invalidation holds no matter how ``compute_hash`` evolves.
Rank is included because per-rank memory use differs under TP/PP.
Driver-only changes are not part of the key; the free-memory gate at
apply time bounds the residual risk.
"""
# Imported here (as VllmConfig.compute_hash does) to avoid a cycle with
# the top-level vllm package.
from vllm import __version__ as vllm_version
capability = current_platform.get_device_capability()
factors = {
"schema": PLAN_SCHEMA_VERSION,
"vllm": vllm_version,
"vllm_config": vllm_config.compute_hash(),
"device_name": current_platform.get_device_name(),
"device_total_memory": current_platform.get_device_total_memory(),
"device_capability": str(capability) if capability else "",
"torch": torch.__version__,
"cuda": torch.version.cuda or "",
"rank": rank,
"world_size": world_size,
}
digest = hashlib.sha256(json.dumps(factors, sort_keys=True).encode()).hexdigest()
return digest[:16]
|
maybe_apply_startup_plan(worker)
If enabled and --kv-cache-memory was not set explicitly, apply a persisted plan by setting worker.cache_config.kv_cache_memory_bytes. No-op unless VLLM_ENABLE_STARTUP_PLAN=1.
Source code in vllm/v1/worker/startup_plan.py
| def maybe_apply_startup_plan(worker: "Worker") -> None:
"""If enabled and ``--kv-cache-memory`` was not set explicitly, apply a
persisted plan by setting ``worker.cache_config.kv_cache_memory_bytes``.
No-op unless ``VLLM_ENABLE_STARTUP_PLAN=1``."""
if (
not envs.VLLM_ENABLE_STARTUP_PLAN
or worker.cache_config.kv_cache_memory_bytes is not None
):
return
fingerprint = compute_plan_fingerprint(
worker.vllm_config, worker.rank, worker.parallel_config.world_size
)
plan = _load_plan(fingerprint)
if plan is None:
return
current_free_memory = worker.init_snapshot.free_memory
kv_bytes = _applicable_kv_cache_memory_bytes(plan, current_free_memory)
if kv_bytes is None:
return
logger.info(
"Applying persisted startup plan (fingerprint %s): "
"kv_cache_memory_bytes=%d (%.2f GiB), recorded free-memory "
"baseline %.2f GiB, current %.2f GiB. Memory profiling will "
"be skipped.",
fingerprint,
kv_bytes,
kv_bytes / (1 << 30),
plan["free_memory_baseline"] / (1 << 30),
current_free_memory / (1 << 30),
)
worker.cache_config.kv_cache_memory_bytes = kv_bytes
|
maybe_save_startup_plan(worker, kv_cache_memory_bytes)
Atomically persist this boot's profiling result for future boots. No-op unless VLLM_ENABLE_STARTUP_PLAN=1; failures are logged, never raised.
Source code in vllm/v1/worker/startup_plan.py
| def maybe_save_startup_plan(worker: "Worker", kv_cache_memory_bytes: int) -> None:
"""Atomically persist this boot's profiling result for future boots.
No-op unless ``VLLM_ENABLE_STARTUP_PLAN=1``; failures are logged,
never raised."""
if not envs.VLLM_ENABLE_STARTUP_PLAN:
return
fingerprint = compute_plan_fingerprint(
worker.vllm_config, worker.rank, worker.parallel_config.world_size
)
path = _plan_path(fingerprint)
try:
os.makedirs(os.path.dirname(path), exist_ok=True)
payload = {
"schema": PLAN_SCHEMA_VERSION,
"fingerprint": fingerprint,
"kv_cache_memory_bytes": int(kv_cache_memory_bytes),
"free_memory_baseline": int(worker.init_snapshot.free_memory),
}
tmp = f"{path}.tmp.{os.getpid()}"
with open(tmp, "w") as f:
json.dump(payload, f)
os.replace(tmp, path)
logger.info("Saved startup plan to %s", path)
except OSError as e:
logger.warning("Failed to save startup plan to %s: %s", path, e)
|