vllm_omni.diffusion.models.bagel.pipeline_bagel ¶

cfg_img_scale: float = 1.5

cfg_interval: tuple = (0.4, 1.0)

cfg_renorm_min: float = 0.0

cfg_renorm_type: str = 'global'

cfg_text_scale: float = 4.0

num_timesteps: int = 50

timestep_shift: float = 3.0

bagel = Bagel(
    language_model=self.language_model,
    vit_model=self.vit_model,
    parallel_config=parallel_config,
    quant_config=quant_config,
    prefix="bagel",
    config=BagelConfig(
        llm_config=llm_config,
        vae_config=vae_cfg,
        vit_config=vit_cfg,
        vit_max_num_patch_per_side=int(
            bagel_cfg.get("vit_max_num_patch_per_side", 70)
        ),
        connector_act=str(
            bagel_cfg.get(
                "connector_act", "gelu_pytorch_tanh"
            )
        ),
        interpolate_pos=bool(
            bagel_cfg.get("interpolate_pos", False)
        ),
        latent_patch_size=int(
            bagel_cfg.get("latent_patch_size", 2)
        ),
        max_latent_size=int(
            bagel_cfg.get("max_latent_size", 32)
        ),
        timestep_shift=float(
            bagel_cfg.get("timestep_shift", 1.0)
        ),
    ),
)

device = get_local_device()

image_processor = SiglipImageProcessor.from_pretrained(
    model_path, local_files_only=True
)

language_model = Qwen2MoTForCausalLM(
    llm_config,
    parallel_config=parallel_config,
    quant_config=quant_config,
    prefix="bagel.language_model",
)

od_config = od_config

scheduler: object | None = None

scheduler_kwargs: dict = {}

tokenizer = AutoTokenizer.from_pretrained(
    model_path,
    local_files_only=True,
    trust_remote_code=True,
)

transformer = self.language_model.model

vae = DistributedAutoEncoder(ae_params)

vit_model = SiglipVisionModel(vit_conf)

weights_sources = [
    DiffusersPipelineLoader.ComponentSource(
        model_or_path=od_config.model,
        subfolder=None,
        revision=od_config.revision,
        prefix="",
        fall_back_to_pt=False,
    )
]

vision_model = vision_model.vision_model