Merge branch 'main' into dev/vllm-align

This commit is contained in:
Gao Han authored and GitHub committed 2026-09-10 11:23:08 +08:00
commit 0ae9bdd8c2
54 files changed
+1659 -500

No files matched your search

@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
# Copyright 2026 OpenMOSS and the HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -801,7 +804,16 @@ class MossAudioTokenizerMultiheadAttention(StreamingModule):
else:
attn_bias = None
x = F.scaled_dot_product_attention(q, k, v, attn_bias, dropout_p=0.0)
streaming_attention = getattr(self, "_streaming_attention", None)
if (
streaming_attention is not None
and attn_bias is not None
and q.dtype == torch.bfloat16
and q.shape[-1] == 64
):
x = streaming_attention(q, k, v, attn_bias)
else:
x = F.scaled_dot_product_attention(q, k, v, attn_bias, dropout_p=0.0)
x = x.transpose(1, 2).reshape(B, T, self.embed_dim)
x = apply_weights_per_step(self.out_projs, self.weights_per_step_schedule, x, offset_cpu)
@@ -145,15 +145,22 @@ class _MossCodecStreamSession:
terminal_slots = terminal_slots or set()
if not terminal_slots.issubset(slot_codes):
raise ValueError("terminal_slots must be a subset of the decode batch slots.")
step_lengths = {int(codes.shape[1]) for codes in slot_codes.values()}
if len(step_lengths) != 1:
raise ValueError(f"MOSS codec streaming step needs uniform T, got {sorted(step_lengths)}")
(step_t,) = step_lengths
step_t = max(int(codes.shape[1]) for codes in slot_codes.values())
if any(int(codes.shape[1]) != step_t and slot not in terminal_slots for slot, codes in slot_codes.items()):
raise ValueError("Only terminal codec rows may be padded to a larger step length")
slots = list(slot_codes)
if any(slot not in self._leased_slots for slot in slots):
raise RuntimeError(f"Streaming decode references an unleased state slot: {slots}")
codes_step = torch.stack(
[slot_codes[slot].to(device=self._device, dtype=torch.long) for slot in slots],
[
torch.nn.functional.pad(
slot_codes[slot].to(device=self._device, dtype=torch.long),
(0, step_t - int(slot_codes[slot].shape[1])),
)
if int(slot_codes[slot].shape[1]) != step_t
else slot_codes[slot].to(device=self._device, dtype=torch.long)
for slot in slots
],
dim=1,
)
state_slot_ids = torch.tensor(slots, device=self._device, dtype=torch.long)
@@ -197,9 +204,9 @@ class _MossCodecStreamSession:
self._reset_slot_ids(terminal_slot_ids)
audio = audio_tensor.detach().to("cpu", torch.float32)
audio_length = step_t * self._samples_per_frame
out: dict[int, torch.Tensor] = {}
for row, slot in enumerate(slots):
audio_length = int(slot_codes[slot].shape[1]) * self._samples_per_frame
out[slot] = audio[row, ..., :audio_length].contiguous()
return out
@@ -518,6 +525,8 @@ class MossTTSCodecDecoder(nn.Module):
outputs: dict[int, torch.Tensor] = {}
grouped: dict[int, list[tuple[int, str, int, torch.Tensor, bool]]] = {}
max_step_frames = max(1, int(self._stream_max_step_frames))
graph = session._cudagraph_wrapper
coalesce_tails = graph is not None and max(graph.batch_sizes, default=0) >= self._stream_state_capacity
for output_index, request_id, codes_nq_t, finished in items:
slot = self._stream_req_slots.get(request_id)
@@ -548,9 +557,13 @@ class MossTTSCodecDecoder(nn.Module):
)
continue
grouped.setdefault(int(codes_nq_t.shape[1]), []).append(
(output_index, request_id, slot, codes_nq_t, finished)
)
frame_count = int(codes_nq_t.shape[1])
# Only combine tails that already used the same padded graph.
# Preserve exact-size graphs (including T=1) and the eager path.
group_frames = frame_count
if coalesce_tails and finished:
group_frames = graph._select_frame_size(frame_count, allow_padding=True) or frame_count
grouped.setdefault(group_frames, []).append((output_index, request_id, slot, codes_nq_t, finished))
for group in grouped.values():
plan = {slot: codes_nq_t for _, _, slot, codes_nq_t, _ in group}
@@ -756,6 +769,17 @@ class MossTTSCodecDecoder(nn.Module):
codec.eval()
if device.type != "cpu":
codec.decoder.to(dtype=torch.bfloat16)
attention_backend = getattr(self.vllm_config.model_config.hf_config, "codec_attention_backend", "sdpa")
if attention_backend != "sdpa":
if attention_backend != "triton" or device.type != "cuda":
raise ValueError(f"Unsupported codec attention backend/device: {attention_backend}/{device.type}")
from vllm_omni.model_executor.models.moss_tts.audio_tokenizer_v2 import MossAudioTokenizerMultiheadAttention
from vllm_omni.model_executor.models.moss_tts.streaming_attention import masked_attention
for module in codec.decoder.modules():
if isinstance(module, MossAudioTokenizerMultiheadAttention):
module._streaming_attention = masked_attention
logger.info("Enabled Triton masked attention for the streaming codec decoder")
build_decode_lut = getattr(codec.quantizer, "build_decode_lut", None)
if callable(build_decode_lut):
lut_dtype = torch.bfloat16 if device.type == "cuda" else torch.float32
@@ -1,3 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
# Copyright 2026 OpenMOSS and the vLLM-Omni team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
@@ -101,8 +103,13 @@ class _MossTTSLocalAttention(nn.Module):
odd = x[..., 1::2]
return torch.stack((-odd, even), dim=-1).reshape_as(x)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""``hidden_states``: ``(B, S, H)``. Re-prefills with a fresh causal mask over ``[0, S)`` every call."""
def forward(
self,
hidden_states: torch.Tensor,
kv_cache: tuple[torch.Tensor, torch.Tensor] | None = None,
position: int = 0,
) -> torch.Tensor:
"""Run a causal prefix, or one new token using frame-local K/V."""
batch_size, seq_len, _ = hidden_states.shape
qkv = self.c_attn(hidden_states)
query, key, value = qkv.split(self.embed_dim, dim=-1)
@@ -110,14 +117,22 @@ class _MossTTSLocalAttention(nn.Module):
key = key.view(batch_size, seq_len, self.n_head, self.head_dim)
value = value.view(batch_size, seq_len, self.n_head, self.head_dim)
cos, sin = self._rope_cos_sin(seq_len, hidden_states.device, hidden_states.dtype)
cos, sin = self._rope_cos_sin(position + seq_len, hidden_states.device, hidden_states.dtype)
cos, sin = cos[:, position:], sin[:, position:]
query = query * cos + self._rotate_half(query) * sin
key = key * cos + self._rotate_half(key) * sin
query = query.transpose(1, 2)
key = key.transpose(1, 2)
value = value.transpose(1, 2)
attn_output = F.scaled_dot_product_attention(query, key, value, is_causal=True)
if kv_cache is not None:
assert seq_len == 1, "Cached Local Depth execution consumes one token at a time"
k_cache, v_cache = kv_cache
k_cache[:, :, position : position + 1].copy_(key)
v_cache[:, :, position : position + 1].copy_(value)
key = k_cache[:, :, : position + 1]
value = v_cache[:, :, : position + 1]
attn_output = F.scaled_dot_product_attention(query, key, value, is_causal=kv_cache is None)
attn_output = attn_output.transpose(1, 2).reshape(batch_size, seq_len, self.embed_dim)
return self.c_proj(attn_output)
@@ -141,8 +156,13 @@ class _MossTTSLocalBlock(nn.Module):
self.ln_2 = nn.LayerNorm(hidden_size, eps=eps)
self.mlp = _MossTTSLocalMLP(hidden_size, inner_size)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
hidden_states = hidden_states + self.attn(self.ln_1(hidden_states))
def forward(
self,
hidden_states: torch.Tensor,
kv_cache: tuple[torch.Tensor, torch.Tensor] | None = None,
position: int = 0,
) -> torch.Tensor:
hidden_states = hidden_states + self.attn(self.ln_1(hidden_states), kv_cache, position)
hidden_states = hidden_states + self.mlp(self.ln_2(hidden_states))
return hidden_states
@@ -156,11 +176,9 @@ class MossTTSLocalDepthTransformer(nn.Module):
and codebook-0's head (``audio_lm_heads[0]``) simultaneously.
- codebooks 1..n_vq-1 are sampled sequentially: each sampled code is
re-embedded (``audio_embeddings[c]``) and appended as the next
position, re-prefilling the block over the growing (<=n_vq) sequence
with a fresh causal mask each call -- mathematically identical to
incremental KV-cache decoding since attention is strictly causal and
only the last position is ever read, but avoids any cache plumbing
given the trivially short sequence length.
position. K/V for earlier positions are reused within the frame, so
each position's projections and MLP run once. A new frame overwrites
every cache position before reading it, including CUDA Graph replay.
"""
def __init__(self, gpt2_config, hidden_size: int | None = None) -> None:
@@ -174,11 +192,13 @@ class MossTTSLocalDepthTransformer(nn.Module):
self.ln_f = nn.LayerNorm(self.hidden_size, eps=eps)
self._compiled_forward_prefix = None
def _forward_prefix(self, seq_embeds: torch.Tensor) -> torch.Tensor:
hidden_states = seq_embeds
for block in self.h:
hidden_states = block(hidden_states)
return self.ln_f(hidden_states)
def _forward_prefix(
self,
seq_embeds: torch.Tensor,
kv_cache: tuple[torch.Tensor, torch.Tensor] | None = None,
position: int = 0,
) -> torch.Tensor:
return self.ln_f(self.h[0](seq_embeds, kv_cache, position))
def setup_compile(self) -> None:
if self._compiled_forward_prefix is not None:
@@ -189,14 +209,18 @@ class MossTTSLocalDepthTransformer(nn.Module):
return
self._compiled_forward_prefix = torch.compile(
self._forward_prefix,
dynamic=False,
# Share batch/position shapes instead of exhausting Dynamo's
# recompilation budget while warming request-size graph buckets.
dynamic=True,
options={"epilogue_fusion": False},
)
logger.info("MOSS-TTS local depth prefix enabled with torch.compile")
logger.info("MOSS-TTS local depth frame-local KV execution enabled with torch.compile")
def _run_prefix(self, seq_embeds: torch.Tensor) -> torch.Tensor:
def _run_prefix(
self, seq_embeds: torch.Tensor, kv_cache: tuple[torch.Tensor, torch.Tensor], position: int
) -> torch.Tensor:
forward_prefix = self._compiled_forward_prefix or self._forward_prefix
return forward_prefix(seq_embeds)
return forward_prefix(seq_embeds, kv_cache, position)
@torch.no_grad()
def generate_frame(
@@ -240,9 +264,13 @@ class MossTTSLocalDepthTransformer(nn.Module):
for block in self.h:
block.attn.prepare_rope_cache(n_vq, backbone_last_hidden.device, dtype)
embeds = backbone_last_hidden.new_zeros((batch_size, n_vq, self.hidden_size), dtype=dtype)
embeds[:, 0, :] = backbone_last_hidden.to(dtype)
hidden = self._run_prefix(embeds[:, :1, :])
attn = self.h[0].attn
cache_shape = (batch_size, attn.n_head, n_vq, attn.head_dim)
kv_cache = (
backbone_last_hidden.new_empty(cache_shape, dtype=dtype),
backbone_last_hidden.new_empty(cache_shape, dtype=dtype),
)
hidden = self._run_prefix(backbone_last_hidden[:, None, :].to(dtype), kv_cache, 0)
local_hidden = hidden[:, 0, :]
binary_logits = local_text_lm_head(local_hidden).float()
@@ -292,9 +320,9 @@ class MossTTSLocalDepthTransformer(nn.Module):
codes[:, channel_index] = channel_token
if channel_index + 1 < n_vq:
embeds[:, channel_index + 1, :] = audio_embeddings[channel_index](channel_token).to(dtype)
hidden = self._run_prefix(embeds[:, : channel_index + 2, :])
local_hidden = hidden[:, channel_index + 1, :]
embeds = audio_embeddings[channel_index](channel_token).to(dtype)[:, None, :]
hidden = self._run_prefix(embeds, kv_cache, channel_index + 1)
local_hidden = hidden[:, 0, :]
return should_continue, codes
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
# Copyright 2026 OpenMOSS and the vLLM-Omni team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
@@ -1671,20 +1674,19 @@ class MossTTSLocalTalkerForGeneration(nn.Module):
self._batch_state = [(info["audio_state"] if isinstance(info, dict) else {}) for info in info_dicts]
self._batch_state_spans = kwargs.get("request_token_spans")
should_values: list[torch.Tensor] = []
have_mtp_output = False
should_codes: list[torch.Tensor] = []
should_indices: list[int] = []
per_req_codes: list[torch.Tensor] = []
have_codes = False
for info in info_dicts:
for index, info in enumerate(info_dicts):
audio_codes = (info.get("audio_codes", {}) or {}) if isinstance(info, dict) else {}
current = audio_codes.pop("current", None)
if isinstance(current, torch.Tensor) and current.numel() > 0:
codes = current.to(device=hidden.device, dtype=torch.long)
if codes.dim() == 1:
codes = codes.unsqueeze(0)
should_continue = codes.ne(self.audio_pad_token_id).any(dim=-1)
should_values.append(should_continue.reshape(-1)[:1])
have_mtp_output = True
should_codes.append(codes[:1])
should_indices.append(index)
# Do not boolean-index a CUDA tensor here. Even for the
# common single-row case, ``codes[should_continue]`` must
# discover a dynamic output size on the host and therefore
@@ -1694,9 +1696,19 @@ class MossTTSLocalTalkerForGeneration(nn.Module):
per_req_codes.append(codes)
have_codes = True
else:
should_values.append(torch.ones((1,), device=hidden.device, dtype=torch.bool))
per_req_codes.append(hidden.new_empty((0, self.n_vq), dtype=torch.long))
self._batch_should_continue = torch.cat(should_values, dim=0) if have_mtp_output and should_values else None
self._batch_should_continue = None
if should_codes:
active = torch.cat(should_codes, dim=0).ne(self.audio_pad_token_id).any(dim=-1)
if len(should_codes) == len(info_dicts):
self._batch_should_continue = active
else:
self._batch_should_continue = torch.ones(len(info_dicts), device=hidden.device, dtype=torch.bool)
# Keep mixed prefill/decode output packing asynchronous.
indices = torch.tensor(should_indices, device="cpu", dtype=torch.long).to(
hidden.device, non_blocking=True
)
self._batch_should_continue.index_copy_(0, indices, active)
if not have_codes:
return OmniOutput(text_hidden_states=hidden, multimodal_outputs={})
@@ -1,8 +1,13 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
# Copyright 2026 OpenMOSS and the vLLM-Omni team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
"""Pipeline topology for all MOSS-TTS variants (2-stage: talker → codec)."""
from vllm.sampling_params import RequestOutputKind
from vllm_omni.config.stage_config import (
PipelineConfig,
StageExecutionType,
@@ -100,6 +105,10 @@ MOSS_TTS_LOCAL_PIPELINE = PipelineConfig(
sampling_constraints={
"detokenize": False,
"stop_token_ids": [151645],
# The worker connector streams codes directly to the codec.
# This internal stage only needs to publish its terminal
# result; codec audio output remains incremental.
"output_kind": RequestOutputKind.FINAL_ONLY,
},
),
StagePipelineConfig(
@@ -0,0 +1,119 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Masked streaming attention preserving ring-cache position semantics."""
import torch
from vllm.triton_utils import tl, triton
@triton.jit
def _attention(
q_ptr,
k_ptr,
v_ptr,
mask_ptr,
out_ptr,
qs0: tl.constexpr,
qs1: tl.constexpr,
qs2: tl.constexpr,
qs3: tl.constexpr,
ks0: tl.constexpr,
ks1: tl.constexpr,
ks2: tl.constexpr,
ks3: tl.constexpr,
vs0: tl.constexpr,
vs1: tl.constexpr,
vs2: tl.constexpr,
vs3: tl.constexpr,
ms0: tl.constexpr,
ms2: tl.constexpr,
ms3: tl.constexpr,
num_heads: tl.constexpr,
q_len: tl.constexpr,
kv_len: tl.constexpr,
head_dim: tl.constexpr,
block_m: tl.constexpr,
block_n: tl.constexpr,
):
rows = tl.program_id(0) * block_m + tl.arange(0, block_m)
bh = tl.program_id(1)
batch, head = bh // num_heads, bh % num_heads
dims = tl.arange(0, head_dim)
q = tl.load(
q_ptr + batch * qs0 + head * qs1 + rows[:, None] * qs2 + dims[None, :] * qs3,
mask=rows[:, None] < q_len,
other=0,
)
acc = tl.full((block_m, head_dim), 0.0, tl.float32)
maximum = tl.full((block_m,), -float("inf"), tl.float32)
denominator = tl.full((block_m,), 0.0, tl.float32)
for first in range(tl.cdiv(kv_len, block_n)):
cols = first * block_n + tl.arange(0, block_n)
k = tl.load(
k_ptr + batch * ks0 + head * ks1 + cols[None, :] * ks2 + dims[:, None] * ks3,
mask=cols[None, :] < kv_len,
other=0,
)
allowed = tl.load(
mask_ptr + batch * ms0 + rows[:, None] * ms2 + cols[None, :] * ms3,
mask=(rows[:, None] < q_len) & (cols[None, :] < kv_len),
other=0,
)
score = tl.dot(q, k).to(tl.float32) * (head_dim**-0.5)
score = tl.where(allowed & (cols[None, :] < kv_len), score, -float("inf"))
next_max = tl.maximum(maximum, tl.max(score, axis=1))
safe_max = tl.where(next_max == -float("inf"), 0.0, next_max)
rescale = tl.exp(maximum - safe_max)
p = tl.exp(score - safe_max[:, None])
denominator = denominator * rescale + tl.sum(p, axis=1)
v = tl.load(
v_ptr + batch * vs0 + head * vs1 + cols[:, None] * vs2 + dims[None, :] * vs3,
mask=cols[:, None] < kv_len,
other=0,
)
acc = acc * rescale[:, None] + tl.dot(p.to(v.dtype), v)
maximum = next_max
out = acc / tl.where(denominator > 0, denominator, 1.0)[:, None]
tl.store(
out_ptr + ((batch * num_heads + head) * q_len + rows[:, None]) * head_dim + dims[None, :],
out,
mask=rows[:, None] < q_len,
)
@torch.library.custom_op("moss_streaming::masked_attention", mutates_args=())
def masked_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
b, h, t, d = q.shape
out = torch.empty(q.shape, device=q.device, dtype=q.dtype)
# Small query tiles expose parallelism for short streaming chunks. Larger
# tiles reuse K/V for steady chunks. Keep 64-key softmax blocks to retain
# the same accumulation granularity across both paths.
bm = 16 if t <= 32 else 64
num_warps = 8 if k.shape[2] <= 256 else 4
_attention[(triton.cdiv(t, bm), b * h)](
q,
k,
v,
mask,
out,
*q.stride(),
*k.stride(),
*v.stride(),
mask.stride(0),
mask.stride(2),
mask.stride(3),
h,
t,
k.shape[2],
d,
bm,
64,
num_warps=num_warps,
num_stages=2,
)
return out
@masked_attention.register_fake
def _(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
return torch.empty(q.shape, device=q.device, dtype=q.dtype)