vllm_omni.model_executor.models.indextts2.gpt.perceiver ¶

dropout = dropout

attend = Attend(dropout=dropout)

cross_attn_include_queries = cross_attn_include_queries

heads = heads

scale = dim_head ** -0.5

to_kv = nn.Linear(dim_context, dim_inner * 2, bias=False)

to_out = nn.Linear(dim_inner, dim, bias=False)

to_q = nn.Linear(dim, dim_inner, bias=False)

latents = nn.Parameter(torch.randn(num_latents, dim))

layers = nn.ModuleList([])

norm = RMSNorm(dim)

proj_context = (
    nn.Linear(dim_context, dim)
    if dim_context != dim
    else nn.Identity()
)

gamma = nn.Parameter(torch.ones(dim)) if scale else None

scale = dim ** 0.5