vllm_omni.config ¶

cache_config: OmniStageCacheConfig = field(
    default_factory=OmniStageCacheConfig
)

cfg_kv_collect_func: str | None

compilation_config: CompilationConfig | None = None

connector_config: OmniStageConnectorConfig = field(
    default_factory=OmniStageConnectorConfig
)

custom_process_input_func: str | None

custom_process_next_stage_input_func: str | None

engine_output_type: str | None

final_output: bool

final_output_type: str | None

hf_config_name: str | None

input_sources: list[int]

is_comprehension: bool

load_config: OmniStageLoadConfig = field(
    default_factory=OmniStageLoadConfig
)

model_config: OmniStageModelConfig = field(
    default_factory=OmniStageModelConfig
)

model_stage: str

parallel_config: OmniStageParallelConfig = field(
    default_factory=OmniStageParallelConfig
)

pooling_config: OmniStagePoolingConfig = field(
    default_factory=OmniStagePoolingConfig
)

profiler_config: ProfilerConfig | None = None

prompt_expand_func: str | None

prompt_transform_func: str | None

quantization_config: _QuantizationConfigType = None

requires_multimodal_data: bool

runtime_config: OmniStageRuntimeConfig = field(
    default_factory=OmniStageRuntimeConfig
)

sampling_constraints: dict[str, Any]

scheduler_cls: str | None

scheduler_config: OmniStageSchedulerConfig = field(
    default_factory=OmniStageSchedulerConfig
)

stage_id: int

stage_pipeline_config: StagePipelineConfig

stage_type: StageType

worker_type: str | None

active_stream_window: int = 0

async_chunk: bool = True

connectors: dict[str, Any] | None = None

cuda_mps: bool = False

custom_voice_dir: str | None = None

data_parallel_size: int | None = None

distributed_executor_backend: str | None = None

dtype: str | None = None

duplex_session: DuplexSessionRuntimeConfig = field(
    default_factory=DuplexSessionRuntimeConfig
)

edges: list[dict[str, Any]] | None = None

enable_chunked_prefill: bool | None = None

enable_prefix_caching: bool | None = None

model_runner: Literal['v1', 'v2'] = 'v1'

pipeline: str | None = None

pipeline_parallel_size: int | None = None

platforms: dict[str, Any] | None = None

quantization: str | None = None

session_mode: str = 'turn'

stages: list[StageDeployConfig] = field(
    default_factory=list
)

trust_remote_code: bool | None = None

active_stream_window: int = 0

architectures: list[str]

async_chunk: bool = False

codec_frame_rate_hz: float | None = None

custom_process_next_stage_input_func: str | None = None

duplex_max_sessions: int = 1

embedding_size

enable_sleep_mode: bool = False

engine_output_type: str | None = None

final_output: bool = False

has_sampling_extra_args: bool = False

hf_config_name: str | None = None

model_arch: str | None = None

model_stage: str = 'thinker'

omni_kv_config: dict | None = None

pooling_output_decoder: str | None = None

registry

requires_full_payload_input: bool = False

retains_state_across_chunks: bool = False

sampling_extra_args_keys: tuple[str, ...] = ()

session_mode: str = 'turn'

silence_ban_frames: int = 0

stage_connector_config: dict[str, Any] = field(
    default_factory=lambda: {
        "name": "SharedMemoryConnector",
        "extra": {},
    }
)

stage_id: int = 0

subtalker_sampling_params: dict[str, Any] | None = None

supports_native_mrv2_data_plane: bool = False

task_type: str | None = None

use_v2_model_runner: bool = False

uses_mrope: bool

worker_type: str | None = None

disable_hybrid_kv_cache_manager: bool | None = None

enable_prefix_caching: bool | None = None

gpu_memory_utilization: float | None = Field(
    default=None, gt=0.0, le=1.0
)

kv_cache_memory_bytes: int | None = Field(
    default=None, ge=0
)

mamba_ssm_cache_dtype: str | None = None

mm_processor_cache_gb: float | None = Field(
    default=None, ge=0.0
)

async_chunk: bool = False

input_connectors: dict[str, Any] | None = None

kv_transfer_config: KVTransferConfig | None = None

omni_kv_config: dict[str, Any] | None = None

output_connectors: dict[str, Any] | None = None

stage_connector: dict[str, Any] = field(
    default_factory=lambda: {
        "name": "SharedMemoryConnector",
        "extra": {},
    }
)

allgather_degree: int = Field(default=1, ge=1)

cfg_parallel_size: int = Field(default=1, ge=1)

hsdp_replicate_size: int = Field(default=1, ge=1)

hsdp_shard_size: int = -1

mask_sp_padding: bool = False

ring_degree: int = Field(default=1, ge=1)

sequence_parallel_size: int = Field(
    default=1, ge=1, init=False
)

text_encoder_tp_size: int = Field(default=1, ge=1)

ulysses_a2a_permute: bool = False

ulysses_degree: int = Field(default=1, ge=1)

ulysses_mode: str = 'strict'

use_hsdp: bool = False

vae_parallel_mode: str = 'tile'

vae_patch_parallel_size: int = Field(default=1, ge=1)

config_format: str | None = None

skip_mm_profiling: bool | None = None

skip_tokenizer_init: bool = False

tokenizer: str | None = None

tokenizer_mode: str = 'auto'

active_stream_window: int = Field(default=0, ge=0)

allowed_local_media_path: str | None = None

allowed_media_domains: list[str] | None = None

attention_backend: Any = None

attention_config: Any = None

code_revision: str | None = None

codec_frame_rate_hz: float | None = None

custom_voice_dir: str | None = None

default_sampling_params: dict[str, Any] | None = None

disable_autocast: bool = False

dtype: Any = 'auto'

duplex_max_sessions: int = Field(default=1, ge=1)

enable_broadcast_weight_load: bool = False

enable_flashinfer_autotune: bool | None = None

enable_multithread_weight_load: bool = True

enable_prompt_embeds: bool | None = None

enable_sleep_mode: bool = False

enforce_eager: bool = False

final_output: bool = False

generation_config: str | None = None

has_sampling_extra_args: bool = False

hf_config_path: str | None = None

hf_overrides: Any = None

hf_token: bool | str | None = None

interleave_mm_strings: bool | None = None

limit_mm_per_prompt: dict[str, Any] | None = None

logits_processors: list[str | type] | None = None

logprobs_mode: str | None = None

max_cudagraph_capture_size: int | None = Field(
    default=None, ge=0
)

max_logprobs: int | None = None

media_io_kwargs: dict[str, Any] | None = None

mm_processor_cache_type: str | None = None

mm_processor_kwargs: dict[str, Any] | None = None

model: str | None = None

model_arch: str | None = None

model_subdir: str | None = None

moe_backend: str = 'auto'

num_weight_load_threads: int = Field(default=4, ge=1)

override_generation_config: dict[str, Any] | None = None

requires_full_payload_input: bool = False

revision: str | None = None

seed: int | None = None

served_model_name: str | list[str] | None = None

session_mode: str = 'turn'

silence_ban_frames: int = 0

subtalker_sampling_params: dict[str, Any] | None = None

supports_native_mrv2_data_plane: bool = False

task_type: str | None = None

tokenizer_revision: str | None = None

tokenizer_subdir: str | None = None

trust_remote_code: bool = False

use_v2_model_runner: bool = False

data_parallel_rank: int | None = Field(default=None, ge=0)

data_parallel_rpc_port: int | None = None

data_parallel_size_local: int | None = Field(
    default=None, ge=0
)

worker_cls: str | None = None

world_size_across_dp: int

additional_config: dict[str, Any] | None = None

cuda_mps: bool = False

devices: str | None = None

distributed_executor_backend: Any = None

env: dict[str, Any] | None = None

log_level: str = 'info'

log_stats: bool = False

num_gpus: int = Field(default=1, ge=1)

num_replicas: int = Field(default=1, ge=1)

worker_cls: str | None = None

async_scheduling: bool | None = None

enable_chunked_prefill: bool | None = None

max_model_len: int | None = Field(default=None, ge=-1)

max_num_batched_tokens: int | None = Field(
    default=None, ge=1
)

max_num_seqs: int | None = Field(default=None, ge=1)

default_deploy_config_name: str | None = None

default_session_mode: str | None = None

diffusers_class_aliases: tuple[str, ...] = ()

diffusers_class_name: str | None = None

duplex_control_enabled: bool = False

duplex_plugin: str | None = None

duplex_runtime_extension: str | None = None

duplex_serving_adapter: str | None = None

endpoint_restrictions: tuple[EndpointRestriction, ...] = ()

hf_architectures: tuple[str, ...] = ()

hf_config_predicate: Callable[[Any], bool] | None = None

model_arch: str = ''

model_type: str

stage_cli_aliases: dict[str, tuple[int, str]] = field(
    default_factory=dict
)

stages: tuple[StagePipelineConfig, ...] = ()

custom_process_input_func: str | None = None

final_output: bool = False

final_output_type: str | None = None

hf_config_name: str | None = None

input_sources: list[int] = field(default_factory=list)

is_comprehension: bool = False

model_stage: str

runtime_overrides: dict[str, Any] = field(
    default_factory=dict
)

sampling_constraints: dict[str, Any] = field(
    default_factory=dict
)

scheduler_cls: str | None = None

session_mode: str = 'turn'

stage_id: int

stage_type: StageType = StageType.LLM

worker_type: str | None = None

yaml_engine_args: dict[str, Any] = field(
    default_factory=dict
)

yaml_extras: dict[str, Any] = field(default_factory=dict)

yaml_runtime: dict[str, Any] = field(default_factory=dict)

allgather_degree: int | None = None

async_chunk: bool | None = None

async_scheduling: bool | None = None

auxiliary_text_encoder: str | None = None

boundary_ratio: float | None = None

cache_backend: str | None = None

cache_config: dict[str, Any] | None = None

cfg_parallel_size: int | None = None

compilation_config: dict[str, Any] | None = None

config_format: str | None = None

default_pooling_params: dict[str, Any] | None = None

default_sampling_params: dict[str, Any] | None = None

devices: str | None = None

diffusers_call_kwargs: dict[str, Any] | None = None

diffusers_load_kwargs: dict[str, Any] | None = None

diffusion_attention_backend: str | None = None

diffusion_attention_config: dict[str, Any] | None = None

diffusion_compile_dynamic: bool | None = None

diffusion_compile_granularity: str | None = None

diffusion_kv_cache_dtype: str | None = None

diffusion_kv_cache_skip_layers: str | None = None

diffusion_kv_cache_skip_steps: str | None = None

diffusion_load_format: str | None = None

diffusion_offload_config: dict[str, Any] | None = None

diffusion_quantization_config: str | None = None

disable_hybrid_kv_cache_manager: bool | None = None

dlo_host_registration_limit_gib: float | None = None

dlo_resident_layers: int | None = None

dlo_use_allgather: bool | None = None

enable_broadcast_weight_load: bool | None = None

enable_cache_dit_summary: bool | None = None

enable_cpu_offload: bool | None = None

enable_diffusion_pipeline_profiler: bool | None = None

enable_distributed_layerwise_offload: bool | None = None

enable_expert_parallel: bool | None = None

enable_flashinfer_autotune: bool | None = None

enable_layerwise_offload: bool | None = None

enable_multithread_weight_load: bool | None = None

enforce_eager: bool | None = None

engine_extras: dict[str, Any] = field(default_factory=dict)

env: dict[str, Any] | None = None

fa_deterministic: bool | None = None

fastvideo_vsa_topk: int | None = None

flow_shift: float | None = None

gpu_memory_utilization: float | None = None

host_weight_runtime_mode: str | None = None

host_weight_runtime_root: str | None = None

hsdp_replicate_size: int | None = None

hsdp_shard_size: int | None = None

input_connectors: dict[str, str] | None = None

load_format: str | None = None

lora_backend: str | None = None

lora_path: str | list[str] | None = None

lora_scale: float | None = None

mamba_ssm_cache_dtype: str | None = None

max_generated_image_size: int | None = None

max_model_len: int | None = None

max_num_batched_tokens: int | None = None

max_num_seqs: int | None = None

mm_processor_cache_gb: float | None = None

model_class_name: str | None = None

num_replicas: int = 1