[Model] Optimize MOSS-TTS Local batched execution and streaming codec

Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
Sy03 committed 2026-09-07 16:20:49 +08:00
1 parent 677265b64b
commit 251ef4f48f
12 files changed
+608 -28

No files matched your search

@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
# Copyright 2026 OpenMOSS and the HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -801,7 +804,16 @@ class MossAudioTokenizerMultiheadAttention(StreamingModule):
else:
attn_bias = None
x = F.scaled_dot_product_attention(q, k, v, attn_bias, dropout_p=0.0)
streaming_attention = getattr(self, "_streaming_attention", None)
if (
streaming_attention is not None
and attn_bias is not None
and q.dtype == torch.bfloat16
and q.shape[-1] == 64
):
x = streaming_attention(q, k, v, attn_bias)
else:
x = F.scaled_dot_product_attention(q, k, v, attn_bias, dropout_p=0.0)
x = x.transpose(1, 2).reshape(B, T, self.embed_dim)
x = apply_weights_per_step(self.out_projs, self.weights_per_step_schedule, x, offset_cpu)
@@ -145,15 +145,22 @@ class _MossCodecStreamSession:
terminal_slots = terminal_slots or set()
if not terminal_slots.issubset(slot_codes):
raise ValueError("terminal_slots must be a subset of the decode batch slots.")
step_lengths = {int(codes.shape[1]) for codes in slot_codes.values()}
if len(step_lengths) != 1:
raise ValueError(f"MOSS codec streaming step needs uniform T, got {sorted(step_lengths)}")
(step_t,) = step_lengths
step_t = max(int(codes.shape[1]) for codes in slot_codes.values())
if any(int(codes.shape[1]) != step_t and slot not in terminal_slots for slot, codes in slot_codes.items()):
raise ValueError("Only terminal codec rows may be padded to a larger step length")
slots = list(slot_codes)
if any(slot not in self._leased_slots for slot in slots):
raise RuntimeError(f"Streaming decode references an unleased state slot: {slots}")
codes_step = torch.stack(
[slot_codes[slot].to(device=self._device, dtype=torch.long) for slot in slots],
[
torch.nn.functional.pad(
slot_codes[slot].to(device=self._device, dtype=torch.long),
(0, step_t - int(slot_codes[slot].shape[1])),
)
if int(slot_codes[slot].shape[1]) != step_t
else slot_codes[slot].to(device=self._device, dtype=torch.long)
for slot in slots
],
dim=1,
)
state_slot_ids = torch.tensor(slots, device=self._device, dtype=torch.long)
@@ -197,9 +204,9 @@ class _MossCodecStreamSession:
self._reset_slot_ids(terminal_slot_ids)
audio = audio_tensor.detach().to("cpu", torch.float32)
audio_length = step_t * self._samples_per_frame
out: dict[int, torch.Tensor] = {}
for row, slot in enumerate(slots):
audio_length = int(slot_codes[slot].shape[1]) * self._samples_per_frame
out[slot] = audio[row, ..., :audio_length].contiguous()
return out
@@ -518,6 +525,8 @@ class MossTTSCodecDecoder(nn.Module):
outputs: dict[int, torch.Tensor] = {}
grouped: dict[int, list[tuple[int, str, int, torch.Tensor, bool]]] = {}
max_step_frames = max(1, int(self._stream_max_step_frames))
graph = session._cudagraph_wrapper
coalesce_tails = graph is not None and max(graph.batch_sizes, default=0) >= self._stream_state_capacity
for output_index, request_id, codes_nq_t, finished in items:
slot = self._stream_req_slots.get(request_id)
@@ -548,9 +557,13 @@ class MossTTSCodecDecoder(nn.Module):
)
continue
grouped.setdefault(int(codes_nq_t.shape[1]), []).append(
(output_index, request_id, slot, codes_nq_t, finished)
)
frame_count = int(codes_nq_t.shape[1])
# Only combine tails that already used the same padded graph.
# Preserve exact-size graphs (including T=1) and the eager path.
group_frames = frame_count
if coalesce_tails and finished:
group_frames = graph._select_frame_size(frame_count, allow_padding=True) or frame_count
grouped.setdefault(group_frames, []).append((output_index, request_id, slot, codes_nq_t, finished))
for group in grouped.values():
plan = {slot: codes_nq_t for _, _, slot, codes_nq_t, _ in group}
@@ -756,6 +769,17 @@ class MossTTSCodecDecoder(nn.Module):
codec.eval()
if device.type != "cpu":
codec.decoder.to(dtype=torch.bfloat16)
attention_backend = getattr(self.vllm_config.model_config.hf_config, "codec_attention_backend", "sdpa")
if attention_backend != "sdpa":
if attention_backend != "triton" or device.type != "cuda":
raise ValueError(f"Unsupported codec attention backend/device: {attention_backend}/{device.type}")
from vllm_omni.model_executor.models.moss_tts.audio_tokenizer_v2 import MossAudioTokenizerMultiheadAttention
from vllm_omni.model_executor.models.moss_tts.streaming_attention import masked_attention
for module in codec.decoder.modules():
if isinstance(module, MossAudioTokenizerMultiheadAttention):
module._streaming_attention = masked_attention
logger.info("Enabled Triton masked attention for the streaming codec decoder")
build_decode_lut = getattr(codec.quantizer, "build_decode_lut", None)
if callable(build_decode_lut):
lut_dtype = torch.bfloat16 if device.type == "cuda" else torch.float32
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
# Copyright 2026 OpenMOSS and the vLLM-Omni team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
@@ -1671,20 +1674,19 @@ class MossTTSLocalTalkerForGeneration(nn.Module):
self._batch_state = [(info["audio_state"] if isinstance(info, dict) else {}) for info in info_dicts]
self._batch_state_spans = kwargs.get("request_token_spans")
should_values: list[torch.Tensor] = []
have_mtp_output = False
should_codes: list[torch.Tensor] = []
should_indices: list[int] = []
per_req_codes: list[torch.Tensor] = []
have_codes = False
for info in info_dicts:
for index, info in enumerate(info_dicts):
audio_codes = (info.get("audio_codes", {}) or {}) if isinstance(info, dict) else {}
current = audio_codes.pop("current", None)
if isinstance(current, torch.Tensor) and current.numel() > 0:
codes = current.to(device=hidden.device, dtype=torch.long)
if codes.dim() == 1:
codes = codes.unsqueeze(0)
should_continue = codes.ne(self.audio_pad_token_id).any(dim=-1)
should_values.append(should_continue.reshape(-1)[:1])
have_mtp_output = True
should_codes.append(codes[:1])
should_indices.append(index)
# Do not boolean-index a CUDA tensor here. Even for the
# common single-row case, ``codes[should_continue]`` must
# discover a dynamic output size on the host and therefore
@@ -1694,9 +1696,16 @@ class MossTTSLocalTalkerForGeneration(nn.Module):
per_req_codes.append(codes)
have_codes = True
else:
should_values.append(torch.ones((1,), device=hidden.device, dtype=torch.bool))
per_req_codes.append(hidden.new_empty((0, self.n_vq), dtype=torch.long))
self._batch_should_continue = torch.cat(should_values, dim=0) if have_mtp_output and should_values else None
self._batch_should_continue = None
if should_codes:
active = torch.cat(should_codes, dim=0).ne(self.audio_pad_token_id).any(dim=-1)
if len(should_codes) == len(info_dicts):
self._batch_should_continue = active
else:
self._batch_should_continue = torch.ones(len(info_dicts), device=hidden.device, dtype=torch.bool)
indices = torch.tensor(should_indices, device=hidden.device, dtype=torch.long)
self._batch_should_continue.index_copy_(0, indices, active)
if not have_codes:
return OmniOutput(text_hidden_states=hidden, multimodal_outputs={})
@@ -0,0 +1,119 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Masked streaming attention preserving ring-cache position semantics."""
import torch
from vllm.triton_utils import tl, triton
@triton.jit
def _attention(
q_ptr,
k_ptr,
v_ptr,
mask_ptr,
out_ptr,
qs0: tl.constexpr,
qs1: tl.constexpr,
qs2: tl.constexpr,
qs3: tl.constexpr,
ks0: tl.constexpr,
ks1: tl.constexpr,
ks2: tl.constexpr,
ks3: tl.constexpr,
vs0: tl.constexpr,
vs1: tl.constexpr,
vs2: tl.constexpr,
vs3: tl.constexpr,
ms0: tl.constexpr,
ms2: tl.constexpr,
ms3: tl.constexpr,
num_heads: tl.constexpr,
q_len: tl.constexpr,
kv_len: tl.constexpr,
head_dim: tl.constexpr,
block_m: tl.constexpr,
block_n: tl.constexpr,
):
rows = tl.program_id(0) * block_m + tl.arange(0, block_m)
bh = tl.program_id(1)
batch, head = bh // num_heads, bh % num_heads
dims = tl.arange(0, head_dim)
q = tl.load(
q_ptr + batch * qs0 + head * qs1 + rows[:, None] * qs2 + dims[None, :] * qs3,
mask=rows[:, None] < q_len,
other=0,
)
acc = tl.full((block_m, head_dim), 0.0, tl.float32)
maximum = tl.full((block_m,), -float("inf"), tl.float32)
denominator = tl.full((block_m,), 0.0, tl.float32)
for first in range(tl.cdiv(kv_len, block_n)):
cols = first * block_n + tl.arange(0, block_n)
k = tl.load(
k_ptr + batch * ks0 + head * ks1 + cols[None, :] * ks2 + dims[:, None] * ks3,
mask=cols[None, :] < kv_len,
other=0,
)
allowed = tl.load(
mask_ptr + batch * ms0 + rows[:, None] * ms2 + cols[None, :] * ms3,
mask=(rows[:, None] < q_len) & (cols[None, :] < kv_len),
other=0,
)
score = tl.dot(q, k).to(tl.float32) * (head_dim**-0.5)
score = tl.where(allowed & (cols[None, :] < kv_len), score, -float("inf"))
next_max = tl.maximum(maximum, tl.max(score, axis=1))
safe_max = tl.where(next_max == -float("inf"), 0.0, next_max)
rescale = tl.exp(maximum - safe_max)
p = tl.exp(score - safe_max[:, None])
denominator = denominator * rescale + tl.sum(p, axis=1)
v = tl.load(
v_ptr + batch * vs0 + head * vs1 + cols[:, None] * vs2 + dims[None, :] * vs3,
mask=cols[:, None] < kv_len,
other=0,
)
acc = acc * rescale[:, None] + tl.dot(p.to(v.dtype), v)
maximum = next_max
out = acc / tl.where(denominator > 0, denominator, 1.0)[:, None]
tl.store(
out_ptr + ((batch * num_heads + head) * q_len + rows[:, None]) * head_dim + dims[None, :],
out,
mask=rows[:, None] < q_len,
)
@torch.library.custom_op("moss_streaming::masked_attention", mutates_args=())
def masked_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
b, h, t, d = q.shape
out = torch.empty(q.shape, device=q.device, dtype=q.dtype)
# Small query tiles expose parallelism for short streaming chunks. Larger
# tiles reuse K/V for steady chunks. Keep 64-key softmax blocks to retain
# the same accumulation granularity across both paths.
bm = 16 if t <= 32 else 64
num_warps = 8 if k.shape[2] <= 256 else 4
_attention[(triton.cdiv(t, bm), b * h)](
q,
k,
v,
mask,
out,
*q.stride(),
*k.stride(),
*v.stride(),
mask.stride(0),
mask.stride(2),
mask.stride(3),
h,
t,
k.shape[2],
d,
bm,
64,
num_warps=num_warps,
num_stages=2,
)
return out
@masked_attention.register_fake
def _(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
return torch.empty(q.shape, device=q.device, dtype=q.dtype)