vllm_omni.model_executor.models.covo_audio.token2wav ¶

activations = nn.ModuleList(
    [
        (
            AliasFreeActivation1d(
                activation=SnakeBeta(
                    channels,
                    alpha_logscale=h.snake_logscale,
                )
            )
        )
        for _ in (range(self.num_layers))
    ]
)

convs1 = nn.ModuleList(
    [
        weight_norm(
            Conv1d(
                channels,
                channels,
                kernel_size,
                1,
                dilation=dilation[0],
                causal=causal,
            )
        ),
        weight_norm(
            Conv1d(
                channels,
                channels,
                kernel_size,
                1,
                dilation=dilation[1],
                causal=causal,
            )
        ),
        weight_norm(
            Conv1d(
                channels,
                channels,
                kernel_size,
                1,
                dilation=dilation[2],
                causal=causal,
            )
        ),
    ]
)

convs2 = nn.ModuleList(
    [
        weight_norm(
            Conv1d(
                channels,
                channels,
                kernel_size,
                1,
                dilation=1,
                causal=causal,
            )
        ),
        weight_norm(
            Conv1d(
                channels,
                channels,
                kernel_size,
                1,
                dilation=1,
                causal=causal,
            )
        ),
        weight_norm(
            Conv1d(
                channels,
                channels,
                kernel_size,
                1,
                dilation=1,
                causal=causal,
            )
        ),
    ]
)

h = h

num_layers = len(self.convs1) + len(self.convs2)

activation_post = AliasFreeActivation1d(
    activation=activation_post
)

config = h

conv_post = weight_norm(Conv1d(ch, 1, 7, 1, causal=causal))

conv_pre = weight_norm(
    Conv1d(
        h.latent_dim,
        h.upsample_initial_channel,
        7,
        1,
        causal=False,
    )
)

h = h

hop_size = np.prod(h.downsample_rates)

num_kernels = len(h.resblock_kernel_sizes)

num_upsamples = len(h.upsample_rates)

resblocks = nn.ModuleList()

ups = nn.ModuleList()

activation = (
    activation if activation is not None else nn.Identity()
)

bn = nn.BatchNorm1d(out_channels) if bn else nn.Identity()

causal = causal

in_channels = in_channels

left_padding = dilation * (kernel_size - 1)

transpose = input_transpose

conv = Conv1d(
    hidden_size,
    hidden_size,
    kernel_size,
    groups=groups,
    input_transpose=True,
    activation=nn.GELU(),
)

causal = causal

stride = stride

transpose = input_transpose

force_drop = force_drop

inplace = inplace

p = p

output_dim = embedding_dim

pad_id = pad_id

adaLN_modulation = nn.Sequential(
    nn.SiLU(),
    nn.Linear(hidden_size, 2 * hidden_size, bias=True),
)

linear = Linear(hidden_size, output_size, bias=True)

norm = nn.LayerNorm(
    hidden_size, elementwise_affine=False, eps=1e-06
)

activation = (
    activation if activation is not None else nn.Identity()
)

output_dim = out_channels

act = act_layer()

drop = Dropout(dropout)

fc1 = Linear(hidden_size, ffn_hidden_size)

fc2 = Linear(ffn_hidden_size, hidden_size)

attn_drop = Dropout(attn_drop)

cpu_config = AttentionConfig(True, True, True)

cuda_config = AttentionConfig(True, True, True)

head_dim = hidden_size // num_heads

k_norm = (
    norm_layer(self.head_dim) if qk_norm else nn.Identity()
)

k_proj = Linear(hidden_size, hidden_size, bias=qkv_bias)

max_position_embeddings = max_position_embeddings

num_heads = num_heads

o_dropout = Dropout(dropout)

o_proj = Linear(hidden_size, hidden_size)

q_norm = (
    norm_layer(self.head_dim) if qk_norm else nn.Identity()
)

q_proj = Linear(hidden_size, hidden_size, bias=qkv_bias)

rotary = RotaryEmbedding(self.head_dim)

rotary_bias = rotary_bias

scale = self.head_dim ** -0.5

v_proj = Linear(hidden_size, hidden_size, bias=qkv_bias)

device

frequency_embedding_size = frequency_embedding_size

mlp = nn.Sequential(
    Linear(
        frequency_embedding_size, hidden_size, bias=True
    ),
    nn.SiLU(),
    Linear(hidden_size, hidden_size, bias=True),
)

config = config

global_mean_var = getattr(config, 'global_mean_var', None)

token2latent = Token2latentFlowMatchingWithEmbed(
    config.token2latent
)

trainable_module = ['wavegan', 'token2latent']

upsample_factor = self.token2latent.config.get(
    "upsample_factor", 1
)

wav_input_sr = config.get('wav_input_sr', 24000)

wavegan = BigVGANFlowVAE(config.wavegan)

wavegan_hop_size = np.prod(config.wavegan.downsample_rates)

blocks = nn.ModuleList()

cond_proj = Linear(
    self.model_dim + self.spkr_embed_dim, self.model_dim
)

config = config

conv_embed = ConvPositionEmbed(
    hidden_size=self.model_dim, kernel_size=31, groups=16
)

model_dim = config.transformer.hidden_size

output_layer = FinalLayer(
    config.transformer.hidden_size, self.target_dim
)

spkr_embed_dim = config.get('spkr_embed_dim', 512)

target_dim = config.z_dim

time_embedder = TimestepEmbedder(self.model_dim)

token_input_dim = config.get(
    "token_input_dim", self.model_dim
)

token_pad_id = -1

token_proj = nn.Sequential(
    Linear(self.token_input_dim, self.model_dim, bias=True),
    ConvTranspose1d(
        self.model_dim,
        self.model_dim,
        stride=config.upsample_factor,
        kernel_size=config.upsample_factor * 2,
    ),
)

transformer_input_proj = Linear(
    self.model_dim + self.target_dim * 2, self.model_dim
)

token_embedding = EmbeddingTable(
    num_embeddings=self.vocab_size,
    embedding_dim=self.token_input_dim,
    pad_id=self.token_pad_id,
)

vocab_size = config.token_vocab_size

attn = attention

ffn = ffn

modulation = modulation

modulation_layer = nn.Sequential(
    nn.SiLU(),
    Linear(hidden_size, 6 * hidden_size, bias=True),
)

norm1 = nn.LayerNorm(
    hidden_size, elementwise_affine=not modulation, eps=eps
)

norm2 = nn.LayerNorm(
    hidden_size, elementwise_affine=not modulation, eps=eps
)