[Model] Optimize MOSS-TTS Local batched execution and streaming codec
Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
1 parent
677265b64b
commit
251ef4f48f
12 files changed
+608
-28
No files matched your search
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
# Copyright 2026 OpenMOSS and the HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
@@ -801,7 +804,16 @@ class MossAudioTokenizerMultiheadAttention(StreamingModule):
|
||||
else:
|
||||
attn_bias = None
|
||||
|
||||
x = F.scaled_dot_product_attention(q, k, v, attn_bias, dropout_p=0.0)
|
||||
streaming_attention = getattr(self, "_streaming_attention", None)
|
||||
if (
|
||||
streaming_attention is not None
|
||||
and attn_bias is not None
|
||||
and q.dtype == torch.bfloat16
|
||||
and q.shape[-1] == 64
|
||||
):
|
||||
x = streaming_attention(q, k, v, attn_bias)
|
||||
else:
|
||||
x = F.scaled_dot_product_attention(q, k, v, attn_bias, dropout_p=0.0)
|
||||
x = x.transpose(1, 2).reshape(B, T, self.embed_dim)
|
||||
x = apply_weights_per_step(self.out_projs, self.weights_per_step_schedule, x, offset_cpu)
|
||||
|
||||
|
||||
@@ -145,15 +145,22 @@ class _MossCodecStreamSession:
|
||||
terminal_slots = terminal_slots or set()
|
||||
if not terminal_slots.issubset(slot_codes):
|
||||
raise ValueError("terminal_slots must be a subset of the decode batch slots.")
|
||||
step_lengths = {int(codes.shape[1]) for codes in slot_codes.values()}
|
||||
if len(step_lengths) != 1:
|
||||
raise ValueError(f"MOSS codec streaming step needs uniform T, got {sorted(step_lengths)}")
|
||||
(step_t,) = step_lengths
|
||||
step_t = max(int(codes.shape[1]) for codes in slot_codes.values())
|
||||
if any(int(codes.shape[1]) != step_t and slot not in terminal_slots for slot, codes in slot_codes.items()):
|
||||
raise ValueError("Only terminal codec rows may be padded to a larger step length")
|
||||
slots = list(slot_codes)
|
||||
if any(slot not in self._leased_slots for slot in slots):
|
||||
raise RuntimeError(f"Streaming decode references an unleased state slot: {slots}")
|
||||
codes_step = torch.stack(
|
||||
[slot_codes[slot].to(device=self._device, dtype=torch.long) for slot in slots],
|
||||
[
|
||||
torch.nn.functional.pad(
|
||||
slot_codes[slot].to(device=self._device, dtype=torch.long),
|
||||
(0, step_t - int(slot_codes[slot].shape[1])),
|
||||
)
|
||||
if int(slot_codes[slot].shape[1]) != step_t
|
||||
else slot_codes[slot].to(device=self._device, dtype=torch.long)
|
||||
for slot in slots
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
state_slot_ids = torch.tensor(slots, device=self._device, dtype=torch.long)
|
||||
@@ -197,9 +204,9 @@ class _MossCodecStreamSession:
|
||||
self._reset_slot_ids(terminal_slot_ids)
|
||||
|
||||
audio = audio_tensor.detach().to("cpu", torch.float32)
|
||||
audio_length = step_t * self._samples_per_frame
|
||||
out: dict[int, torch.Tensor] = {}
|
||||
for row, slot in enumerate(slots):
|
||||
audio_length = int(slot_codes[slot].shape[1]) * self._samples_per_frame
|
||||
out[slot] = audio[row, ..., :audio_length].contiguous()
|
||||
return out
|
||||
|
||||
@@ -518,6 +525,8 @@ class MossTTSCodecDecoder(nn.Module):
|
||||
outputs: dict[int, torch.Tensor] = {}
|
||||
grouped: dict[int, list[tuple[int, str, int, torch.Tensor, bool]]] = {}
|
||||
max_step_frames = max(1, int(self._stream_max_step_frames))
|
||||
graph = session._cudagraph_wrapper
|
||||
coalesce_tails = graph is not None and max(graph.batch_sizes, default=0) >= self._stream_state_capacity
|
||||
|
||||
for output_index, request_id, codes_nq_t, finished in items:
|
||||
slot = self._stream_req_slots.get(request_id)
|
||||
@@ -548,9 +557,13 @@ class MossTTSCodecDecoder(nn.Module):
|
||||
)
|
||||
continue
|
||||
|
||||
grouped.setdefault(int(codes_nq_t.shape[1]), []).append(
|
||||
(output_index, request_id, slot, codes_nq_t, finished)
|
||||
)
|
||||
frame_count = int(codes_nq_t.shape[1])
|
||||
# Only combine tails that already used the same padded graph.
|
||||
# Preserve exact-size graphs (including T=1) and the eager path.
|
||||
group_frames = frame_count
|
||||
if coalesce_tails and finished:
|
||||
group_frames = graph._select_frame_size(frame_count, allow_padding=True) or frame_count
|
||||
grouped.setdefault(group_frames, []).append((output_index, request_id, slot, codes_nq_t, finished))
|
||||
|
||||
for group in grouped.values():
|
||||
plan = {slot: codes_nq_t for _, _, slot, codes_nq_t, _ in group}
|
||||
@@ -756,6 +769,17 @@ class MossTTSCodecDecoder(nn.Module):
|
||||
codec.eval()
|
||||
if device.type != "cpu":
|
||||
codec.decoder.to(dtype=torch.bfloat16)
|
||||
attention_backend = getattr(self.vllm_config.model_config.hf_config, "codec_attention_backend", "sdpa")
|
||||
if attention_backend != "sdpa":
|
||||
if attention_backend != "triton" or device.type != "cuda":
|
||||
raise ValueError(f"Unsupported codec attention backend/device: {attention_backend}/{device.type}")
|
||||
from vllm_omni.model_executor.models.moss_tts.audio_tokenizer_v2 import MossAudioTokenizerMultiheadAttention
|
||||
from vllm_omni.model_executor.models.moss_tts.streaming_attention import masked_attention
|
||||
|
||||
for module in codec.decoder.modules():
|
||||
if isinstance(module, MossAudioTokenizerMultiheadAttention):
|
||||
module._streaming_attention = masked_attention
|
||||
logger.info("Enabled Triton masked attention for the streaming codec decoder")
|
||||
build_decode_lut = getattr(codec.quantizer, "build_decode_lut", None)
|
||||
if callable(build_decode_lut):
|
||||
lut_dtype = torch.bfloat16 if device.type == "cuda" else torch.float32
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
# Copyright 2026 OpenMOSS and the vLLM-Omni team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License").
|
||||
@@ -1671,20 +1674,19 @@ class MossTTSLocalTalkerForGeneration(nn.Module):
|
||||
|
||||
self._batch_state = [(info["audio_state"] if isinstance(info, dict) else {}) for info in info_dicts]
|
||||
self._batch_state_spans = kwargs.get("request_token_spans")
|
||||
should_values: list[torch.Tensor] = []
|
||||
have_mtp_output = False
|
||||
should_codes: list[torch.Tensor] = []
|
||||
should_indices: list[int] = []
|
||||
per_req_codes: list[torch.Tensor] = []
|
||||
have_codes = False
|
||||
for info in info_dicts:
|
||||
for index, info in enumerate(info_dicts):
|
||||
audio_codes = (info.get("audio_codes", {}) or {}) if isinstance(info, dict) else {}
|
||||
current = audio_codes.pop("current", None)
|
||||
if isinstance(current, torch.Tensor) and current.numel() > 0:
|
||||
codes = current.to(device=hidden.device, dtype=torch.long)
|
||||
if codes.dim() == 1:
|
||||
codes = codes.unsqueeze(0)
|
||||
should_continue = codes.ne(self.audio_pad_token_id).any(dim=-1)
|
||||
should_values.append(should_continue.reshape(-1)[:1])
|
||||
have_mtp_output = True
|
||||
should_codes.append(codes[:1])
|
||||
should_indices.append(index)
|
||||
# Do not boolean-index a CUDA tensor here. Even for the
|
||||
# common single-row case, ``codes[should_continue]`` must
|
||||
# discover a dynamic output size on the host and therefore
|
||||
@@ -1694,9 +1696,16 @@ class MossTTSLocalTalkerForGeneration(nn.Module):
|
||||
per_req_codes.append(codes)
|
||||
have_codes = True
|
||||
else:
|
||||
should_values.append(torch.ones((1,), device=hidden.device, dtype=torch.bool))
|
||||
per_req_codes.append(hidden.new_empty((0, self.n_vq), dtype=torch.long))
|
||||
self._batch_should_continue = torch.cat(should_values, dim=0) if have_mtp_output and should_values else None
|
||||
self._batch_should_continue = None
|
||||
if should_codes:
|
||||
active = torch.cat(should_codes, dim=0).ne(self.audio_pad_token_id).any(dim=-1)
|
||||
if len(should_codes) == len(info_dicts):
|
||||
self._batch_should_continue = active
|
||||
else:
|
||||
self._batch_should_continue = torch.ones(len(info_dicts), device=hidden.device, dtype=torch.bool)
|
||||
indices = torch.tensor(should_indices, device=hidden.device, dtype=torch.long)
|
||||
self._batch_should_continue.index_copy_(0, indices, active)
|
||||
|
||||
if not have_codes:
|
||||
return OmniOutput(text_hidden_states=hidden, multimodal_outputs={})
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Masked streaming attention preserving ring-cache position semantics."""
|
||||
|
||||
import torch
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _attention(
|
||||
q_ptr,
|
||||
k_ptr,
|
||||
v_ptr,
|
||||
mask_ptr,
|
||||
out_ptr,
|
||||
qs0: tl.constexpr,
|
||||
qs1: tl.constexpr,
|
||||
qs2: tl.constexpr,
|
||||
qs3: tl.constexpr,
|
||||
ks0: tl.constexpr,
|
||||
ks1: tl.constexpr,
|
||||
ks2: tl.constexpr,
|
||||
ks3: tl.constexpr,
|
||||
vs0: tl.constexpr,
|
||||
vs1: tl.constexpr,
|
||||
vs2: tl.constexpr,
|
||||
vs3: tl.constexpr,
|
||||
ms0: tl.constexpr,
|
||||
ms2: tl.constexpr,
|
||||
ms3: tl.constexpr,
|
||||
num_heads: tl.constexpr,
|
||||
q_len: tl.constexpr,
|
||||
kv_len: tl.constexpr,
|
||||
head_dim: tl.constexpr,
|
||||
block_m: tl.constexpr,
|
||||
block_n: tl.constexpr,
|
||||
):
|
||||
rows = tl.program_id(0) * block_m + tl.arange(0, block_m)
|
||||
bh = tl.program_id(1)
|
||||
batch, head = bh // num_heads, bh % num_heads
|
||||
dims = tl.arange(0, head_dim)
|
||||
q = tl.load(
|
||||
q_ptr + batch * qs0 + head * qs1 + rows[:, None] * qs2 + dims[None, :] * qs3,
|
||||
mask=rows[:, None] < q_len,
|
||||
other=0,
|
||||
)
|
||||
acc = tl.full((block_m, head_dim), 0.0, tl.float32)
|
||||
maximum = tl.full((block_m,), -float("inf"), tl.float32)
|
||||
denominator = tl.full((block_m,), 0.0, tl.float32)
|
||||
for first in range(tl.cdiv(kv_len, block_n)):
|
||||
cols = first * block_n + tl.arange(0, block_n)
|
||||
k = tl.load(
|
||||
k_ptr + batch * ks0 + head * ks1 + cols[None, :] * ks2 + dims[:, None] * ks3,
|
||||
mask=cols[None, :] < kv_len,
|
||||
other=0,
|
||||
)
|
||||
allowed = tl.load(
|
||||
mask_ptr + batch * ms0 + rows[:, None] * ms2 + cols[None, :] * ms3,
|
||||
mask=(rows[:, None] < q_len) & (cols[None, :] < kv_len),
|
||||
other=0,
|
||||
)
|
||||
score = tl.dot(q, k).to(tl.float32) * (head_dim**-0.5)
|
||||
score = tl.where(allowed & (cols[None, :] < kv_len), score, -float("inf"))
|
||||
next_max = tl.maximum(maximum, tl.max(score, axis=1))
|
||||
safe_max = tl.where(next_max == -float("inf"), 0.0, next_max)
|
||||
rescale = tl.exp(maximum - safe_max)
|
||||
p = tl.exp(score - safe_max[:, None])
|
||||
denominator = denominator * rescale + tl.sum(p, axis=1)
|
||||
v = tl.load(
|
||||
v_ptr + batch * vs0 + head * vs1 + cols[:, None] * vs2 + dims[None, :] * vs3,
|
||||
mask=cols[:, None] < kv_len,
|
||||
other=0,
|
||||
)
|
||||
acc = acc * rescale[:, None] + tl.dot(p.to(v.dtype), v)
|
||||
maximum = next_max
|
||||
out = acc / tl.where(denominator > 0, denominator, 1.0)[:, None]
|
||||
tl.store(
|
||||
out_ptr + ((batch * num_heads + head) * q_len + rows[:, None]) * head_dim + dims[None, :],
|
||||
out,
|
||||
mask=rows[:, None] < q_len,
|
||||
)
|
||||
|
||||
|
||||
@torch.library.custom_op("moss_streaming::masked_attention", mutates_args=())
|
||||
def masked_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
|
||||
b, h, t, d = q.shape
|
||||
out = torch.empty(q.shape, device=q.device, dtype=q.dtype)
|
||||
# Small query tiles expose parallelism for short streaming chunks. Larger
|
||||
# tiles reuse K/V for steady chunks. Keep 64-key softmax blocks to retain
|
||||
# the same accumulation granularity across both paths.
|
||||
bm = 16 if t <= 32 else 64
|
||||
num_warps = 8 if k.shape[2] <= 256 else 4
|
||||
_attention[(triton.cdiv(t, bm), b * h)](
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
mask,
|
||||
out,
|
||||
*q.stride(),
|
||||
*k.stride(),
|
||||
*v.stride(),
|
||||
mask.stride(0),
|
||||
mask.stride(2),
|
||||
mask.stride(3),
|
||||
h,
|
||||
t,
|
||||
k.shape[2],
|
||||
d,
|
||||
bm,
|
||||
64,
|
||||
num_warps=num_warps,
|
||||
num_stages=2,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@masked_attention.register_fake
|
||||
def _(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
|
||||
return torch.empty(q.shape, device=q.device, dtype=q.dtype)
|
||||
Reference in new issue
Block a user