vllm_omni.diffusion.models.hunyuan_video ¶

color_format = 'RGB'

current_timestep

device = get_local_device()

feature_extractor = from_pretrained(
    model,
    subfolder="feature_extractor",
    local_files_only=local_files_only,
)

guidance_scale

image_encoder = to(device)

num_channels_latents = (
    latent_channels if hasattr(vae, "config") else 32
)

num_timesteps

od_config = od_config

prompt_template_encode_start_idx = 108

scheduler = from_pretrained(
    model,
    subfolder="scheduler",
    local_files_only=local_files_only,
)

support_image_input = True

system_message = "You are a helpful assistant. Describe the video by detailing the following aspects:         1. The main content and theme of the video.         2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects.         3. Actions, events, behaviors temporal relationships, physical movement changes of the objects.         4. background environment, light, style and atmosphere.         5. camera angles, movements, and transitions used in the video."

text_encoder = to(device)

text_encoder_2 = to(dtype=dtype, device=device)

tokenizer = from_pretrained(
    model,
    subfolder="tokenizer",
    local_files_only=local_files_only,
)

tokenizer_2 = from_pretrained(
    model,
    subfolder="tokenizer_2",
    local_files_only=local_files_only,
)

tokenizer_2_max_length = 256

tokenizer_max_length = 1000

transformer = HunyuanVideo15Transformer3DModel(
    od_config=od_config, **transformer_kwargs
)

use_meanflow = getattr(
    tf_model_config, "use_meanflow", False
)

vae = to(device)

vae_scale_factor_spatial = (
    spatial_compression_ratio
    if hasattr(vae, "spatial_compression_ratio")
    else 16
)

vae_scale_factor_temporal = (
    temporal_compression_ratio
    if hasattr(vae, "temporal_compression_ratio")
    else 4
)

vision_num_semantic_tokens = 729

vision_states_dim = 1152

weights_sources = [
    ComponentSource(
        model_or_path=model,
        subfolder="transformer",
        revision=None,
        prefix="transformer.",
        fall_back_to_pt=True,
    ),
    ComponentSource(
        model_or_path=model,
        subfolder="text_encoder_2",
        revision=None,
        prefix="text_encoder_2.",
        fall_back_to_pt=True,
    ),
]

current_timestep

device = get_local_device()

guidance_scale

num_channels_latents = (
    latent_channels if hasattr(vae, "config") else 32
)

num_timesteps

od_config = od_config

prompt_template_encode_start_idx = 108

scheduler = from_pretrained(
    model,
    subfolder="scheduler",
    local_files_only=local_files_only,
)

system_message = "You are a helpful assistant. Describe the video by detailing the following aspects:         1. The main content and theme of the video.         2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects.         3. Actions, events, behaviors temporal relationships, physical movement changes of the objects.         4. background environment, light, style and atmosphere.         5. camera angles, movements, and transitions used in the video."

text_encoder = to(device)

text_encoder_2 = to(dtype=dtype, device=device)

tokenizer = from_pretrained(
    model,
    subfolder="tokenizer",
    local_files_only=local_files_only,
)

tokenizer_2 = from_pretrained(
    model,
    subfolder="tokenizer_2",
    local_files_only=local_files_only,
)

tokenizer_2_max_length = 256

tokenizer_max_length = 1000

transformer = HunyuanVideo15Transformer3DModel(
    od_config=od_config, **transformer_kwargs
)

use_meanflow = getattr(
    tf_model_config, "use_meanflow", False
)

vae = to(device)

vae_scale_factor_spatial = (
    spatial_compression_ratio
    if hasattr(vae, "spatial_compression_ratio")
    else 16
)

vae_scale_factor_temporal = (
    temporal_compression_ratio
    if hasattr(vae, "temporal_compression_ratio")
    else 4
)

vision_num_semantic_tokens = 729

vision_states_dim = 1152

weights_sources = [
    ComponentSource(
        model_or_path=model,
        subfolder="transformer",
        revision=None,
        prefix="transformer.",
        fall_back_to_pt=True,
    ),
    ComponentSource(
        model_or_path=model,
        subfolder="text_encoder_2",
        revision=None,
        prefix="text_encoder_2.",
        fall_back_to_pt=True,
    ),
]

cond_type_embed = Embedding(3, inner_dim)

context_embedder = HunyuanVideo15TokenRefiner(
    text_embed_dim,
    num_attention_heads,
    attention_head_dim,
    num_layers=num_refiner_layers,
)

context_embedder_2 = HunyuanVideo15ByT5TextProjection(
    text_embed_2_dim, 2048, inner_dim
)

image_embedder = HunyuanVideo15ImageProjection(
    image_embed_dim, inner_dim
)

in_channels = in_channels

inner_dim = inner_dim

norm_out = AdaLayerNormContinuous(
    inner_dim,
    inner_dim,
    elementwise_affine=False,
    eps=1e-06,
)

out_channels = out_channels or in_channels

packed_modules_mapping = {
    "to_qkv": ["to_q", "to_k", "to_v"],
    "add_kv_proj": [
        "add_q_proj",
        "add_k_proj",
        "add_v_proj",
    ],
}

parallel_config = parallel_config

patch_size = patch_size

patch_size_t = patch_size_t

proj_out = Linear(
    inner_dim,
    patch_size_t * patch_size * patch_size * out_channels,
)

rope = HunyuanVideo15RotaryPosEmbed(
    patch_size,
    patch_size_t,
    list(rope_axes_dim),
    rope_theta,
)

time_embed = HunyuanVideo15TimeEmbedding(
    inner_dim, use_meanflow=use_meanflow
)

transformer_blocks = ModuleList(
    [
        (
            HunyuanVideo15TransformerBlock(
                num_attention_heads,
                attention_head_dim,
                mlp_ratio=mlp_ratio,
                qk_norm=qk_norm,
            )
        )
        for _ in (range(num_layers))
    ]
)

x_embedder = HunyuanVideo15PatchEmbed(
    (patch_size_t, patch_size, patch_size),
    in_channels,
    inner_dim,
)