vllm_omni.diffusion.models.internvla_a1 ¶

action_in_proj = Linear(max_action_dim, hidden_size)

action_out_proj = Linear(hidden_size, max_action_dim)

action_time_mlp_in = Linear(2 * hidden_size, hidden_size)

action_time_mlp_out = Linear(hidden_size, hidden_size)

config = config

cosmos = ImageTokenizer(
    checkpoint_enc=str(cosmos_encoder_path),
    checkpoint_dec=str(cosmos_decoder_path),
    device=device,
)

cosmos_in_proj = Conv2d(
    vae_dim, hidden_size, kernel_size=1, stride=1, padding=0
)

cosmos_out_layer_norm = LayerNorm(hidden_size)

cosmos_out_proj = Linear(hidden_size, vae_dim)

downsample_conv = Conv2d(
    hidden_size,
    hidden_size,
    kernel_size=ds,
    stride=ds,
    padding=0,
)

qwen3_vl_with_expert = Qwen3VLWithExpertModel(
    vlm_config, action_expert_config, precision=dtype
)

state_proj = Linear(max_state_dim, hidden_size)

upsample_conv = ConvTranspose2d(
    hidden_size,
    hidden_size,
    kernel_size=ds,
    stride=ds,
    padding=0,
)

action_expert_variant: str = 'qwen3_28l'

attn_implementation: str = 'eager'

chunk_size: int = 50

compile_mode: str = 'max-autotune'

compile_model: bool = False

device: str = 'cuda'

dtype: str = 'bfloat16'

empty_cameras: int = 0

enable_regional_compile: bool = False

enable_suffix_static_context_optimization: bool = False

freeze_vision_encoder: bool = False

gradient_checkpointing: bool = False

hidden_size: int = 128

image_history: int = 2

image_resolution: tuple[int, int] = (224, 224)

input_features: dict[str, Any] = field(default_factory=dict)

intermediate_size: int = 256

lambda_gen: float = 0.01

max_action_dim: int = 32

max_period: float = 4.0

max_state_dim: int = 32

min_period: float = 0.004

n_action_steps: int = 50

num_attention_heads: int = 4

num_cameras: int = 3

num_hidden_layers: int = 2

num_inference_steps: int = 10

output_features: dict[str, Any] = field(
    default_factory=dict
)

pixel_feature_dim: int = 48

qwen3_vl_variant: str = 'qwen3_vl_28l'

regional_compile_dynamic: bool = True

scale_factor: int = 8

time_sampling_beta_alpha: float = 1.5

time_sampling_beta_beta: float = 1.0

time_sampling_offset: float = 0.001

time_sampling_scale: float = 0.999

tokenizer_max_length: int = 48

train_expert_only: bool = False

train_vlm_only: bool = False

type: str = 'internvla_a1'

vocab_size: int = 256

config = _build_config(od_config)

enable_warmup = (
    bool(enable_warmup)
    if isinstance(enable_warmup, bool)
    else False
)

model_dir = model

od_config = od_config

policy = _initialize_policy()

prefix = prefix

processor_model_name = str(
    get("processor_model_name", DEFAULT_QWEN3_VL_MODEL)
)

strict_load = bool(get('strict_load', False))

config = config

input_builder = Qwen3VLInputBuilder(
    processor_model_name=processor_model_name,
    max_length=tokenizer_max_length,
)

model = InternVLAA1(
    config,
    cosmos_encoder_path=cosmos_encoder_path,
    cosmos_decoder_path=cosmos_decoder_path,
)

action_mode: str = 'delta'

processor_model_name: str = DEFAULT_QWEN3_VL_MODEL