Skip to content

vllm.model_executor.model_loader.weight_cache.weight_seed.loader

Fill a daemon's model with a source replica's post-processed weights.

Classes:

  • PeerModelLoader –

    Loads a daemon's model from a source replica's weight cache daemon.

PeerModelLoader

Bases: BaseModelLoader

Loads a daemon's model from a source replica's weight cache daemon.

Source code in vllm/model_executor/model_loader/weight_cache/weight_seed/loader.py
class PeerModelLoader(BaseModelLoader):
    """Loads a daemon's model from a source replica's weight cache daemon."""

    def __init__(self, load_config: LoadConfig):
        super().__init__(load_config)
        extra_config = copy(load_config.model_loader_extra_config or {})
        self.is_draft = bool(extra_config.pop("is_draft", False))
        if extra_config:
            raise ValueError(
                f"Unexpected extra config keys for load format "
                f"{load_config.load_format}: {sorted(extra_config)}"
            )

    def download_model(self, model_config: ModelConfig) -> None:
        return None

    def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None:
        raise NotImplementedError(
            "Standalone weight reload is not supported for load format 'peer_cache' yet"
        )

    @instrument(span_name="Load model")
    def load_model(
        self,
        vllm_config: VllmConfig,
        model_config: ModelConfig,
        prefix: str = "",
        *,
        payload: WeightsSeedPayload,
        transport: SeedTransport,
    ) -> nn.Module:
        check_weight_cache_loader_support(
            vllm_config, model_config, is_draft=self.is_draft
        )
        manifest = payload["manifest"]
        mover = get_mover(payload["seed"]["backend"])
        device = torch.device(
            current_platform.device_type, torch.accelerator.current_device_index()
        )
        tensors = mover.fill(manifest, payload["seed"], device)
        acknowledge_lease(transport)
        kinds: dict[str, TensorKind] = {
            name: "param" if view["is_param"] else "buffer"
            for name, view in manifest["tensors"].items()
        }
        model = build_model_from_processed_weights(
            vllm_config,
            model_config,
            prefix,
            tensors,
            kinds,
            aliases=dict(payload["aliases"]),
            attrs=dict(payload["attrs"]),
            load_config=self.load_config,
        )
        logger.info(
            "Loaded %d tensors from a peer daemon over %s",
            len(tensors),
            payload["seed"]["backend"],
        )
        return model