[Feature] Support MOSS-TTS-Local-v1.5 Streaming (#4804)

Signed-off-by: gcanlin <canlinguosdu@gmail.com>
This commit is contained in:
Canlin Guo authored and GitHub committed 2026-07-03 16:00:30 +08:00
1 parent d8db832fc2
commit a8b2bf11d2
15 files changed
+3222 -189

No files matched your search

File diff suppressed because it is too large. Load diff
@@ -0,0 +1,359 @@
# Copyright 2026 OpenMOSS and the HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""MossAudioTokenizer model configuration"""
from typing import Any
from transformers.configuration_utils import PreTrainedConfig
from transformers.utils import logging
logger = logging.get_logger(__name__)
class MossAudioTokenizerConfig(PreTrainedConfig):
r"""
This is the configuration class to store the configuration of a [`MossAudioTokenizerModel`]. It is used to
instantiate a MossAudioTokenizer model according to the specified arguments, defining the model architecture.
Instantiating a configuration with the defaults will yield a similar configuration to that of the
[VoiceAgentGroup/moss_audio_tokenizer](https://huggingface.co/VoiceAgentGroup/moss_audio_tokenizer) architecture.
Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
documentation from [`PreTrainedConfig`] for more information.
Args:
sampling_rate (`int`, *optional*, defaults to 24000):
The sampling rate at which the audio waveform should be digitalized expressed in hertz (Hz).
downsample_rate (`int`, *optional*, defaults to 1920):
Total downsampling rate from waveform to tokens.
causal_transformer_context_duration (`float`, *optional*, defaults to 10.0):
Context duration in seconds for causal transformer.
encoder_kwargs (`list[dict]`, *optional*):
List of encoder module configurations. Each dict specifies a module type and its parameters.
decoder_kwargs (`list[dict]`, *optional*):
List of decoder module configurations in execution order.
quantizer_type (`str`, *optional*, defaults to `"rvq"`):
Quantizer type. Options include `"rvq"`, `"spec_rvq"`, `"rlfq"`, `"random_prefix_rlfq"`.
quantizer_kwargs (`dict`, *optional*):
Configuration for the quantizer including `input_dim`, `rvq_dim`, `output_dim`, `num_quantizers`,
`codebook_size`, and `codebook_dim`.
Example:
```python
>>> from transformers import MossAudioTokenizerModel, MossAudioTokenizerConfig
>>> # Initializing a MossAudioTokenizer style configuration
>>> configuration = MossAudioTokenizerConfig()
>>> # Initializing a model (with random weights) from the configuration
>>> model = MossAudioTokenizerModel(configuration)
>>> # Accessing the model configuration
>>> configuration = model.config
```
"""
model_type = "moss-audio-tokenizer"
# Backward-compatible alias used by some checkpoints.
attribute_map = {"sample_rate": "sampling_rate"}
sampling_rate: int
downsample_rate: int
causal_transformer_context_duration: float
encoder_kwargs: list[dict[str, Any]]
decoder_kwargs: list[dict[str, Any]]
number_channels: int
enable_channel_interleave: bool
attention_implementation: str
compute_dtype: str
codec_weight_dtype: str
quantizer_type: str
quantizer_kwargs: dict[str, Any]
def __init__(
self,
version: str | None = None,
sampling_rate: int = 48000,
downsample_rate: int = 3840,
causal_transformer_context_duration: float = 10.0,
encoder_kwargs: list[dict[str, Any]] | None = None,
decoder_kwargs: list[dict[str, Any]] | None = None,
number_channels: int = 2,
enable_channel_interleave: bool = True,
attention_implementation: str = "sdpa",
compute_dtype: str = "fp32",
codec_weight_dtype: str = "fp32",
quantizer_type: str = "rlfq",
quantizer_kwargs: dict[str, Any] | None = None,
**kwargs,
):
# Some checkpoints might include an incorrect/legacy `model_type` (e.g. "speech_tokenizer").
# We drop it to avoid overriding the class-level `model_type`.
kwargs.pop("model_type", None)
if "channels_numbers" in kwargs:
number_channels = kwargs.pop("channels_numbers")
if "enable_channel_interleave" in kwargs:
enable_channel_interleave = kwargs.pop("enable_channel_interleave")
if "attention_backend" in kwargs and attention_implementation == "sdpa":
attention_implementation = kwargs.pop("attention_backend")
if "codec_compute_dtype" in kwargs and compute_dtype == "fp32":
compute_dtype = kwargs.pop("codec_compute_dtype")
if "codec_load_dtype" in kwargs and codec_weight_dtype == "fp32":
codec_weight_dtype = kwargs.pop("codec_load_dtype")
# `version` is accepted for compatibility but not used in modeling.
self.version = version
self.sampling_rate = sampling_rate
self.downsample_rate = downsample_rate
self.causal_transformer_context_duration = causal_transformer_context_duration
self.number_channels = number_channels
self.enable_channel_interleave = enable_channel_interleave
self.attention_implementation = attention_implementation
self.compute_dtype = compute_dtype
self.codec_weight_dtype = codec_weight_dtype
# Default encoder configuration
if encoder_kwargs is None:
encoder_kwargs = [
{
"module_type": "PatchedPretransform",
"patch_size": 240,
},
{
"module_type": "Transformer",
"input_dimension": 240,
"output_dimension": 384,
"d_model": 768,
"num_heads": 12,
"num_layers": 12,
"dim_feedforward": 3072,
"causal": True,
"norm": "layer_norm",
"positional_embedding": "rope",
"max_period": 10000,
"gating": "none",
"layer_scale": 0.01,
"conv_layout": True,
},
{
"module_type": "PatchedPretransform",
"patch_size": 2,
},
{
"module_type": "Transformer",
"input_dimension": 768,
"output_dimension": 384,
"d_model": 768,
"num_heads": 12,
"num_layers": 12,
"dim_feedforward": 3072,
"causal": True,
"norm": "layer_norm",
"positional_embedding": "rope",
"max_period": 10000,
"gating": "none",
"layer_scale": 0.01,
"conv_layout": True,
},
{
"module_type": "PatchedPretransform",
"patch_size": 2,
},
{
"module_type": "Transformer",
"input_dimension": 768,
"output_dimension": 640,
"d_model": 768,
"num_heads": 12,
"num_layers": 12,
"dim_feedforward": 3072,
"causal": True,
"norm": "layer_norm",
"positional_embedding": "rope",
"max_period": 10000,
"gating": "none",
"layer_scale": 0.01,
"conv_layout": True,
},
{
"module_type": "PatchedPretransform",
"patch_size": 2,
},
{
"module_type": "Transformer",
"input_dimension": 1280,
"output_dimension": 768,
"d_model": 1280,
"num_heads": 20,
"num_layers": 32,
"dim_feedforward": 5120,
"causal": True,
"norm": "layer_norm",
"positional_embedding": "rope",
"max_period": 10000,
"gating": "none",
"layer_scale": 0.01,
"conv_layout": True,
},
]
self.encoder_kwargs = encoder_kwargs
# Default decoder configuration (execution order)
if decoder_kwargs is None:
decoder_kwargs = [
{
"module_type": "Transformer",
"input_dimension": 768,
"output_dimension": 1280,
"d_model": 1280,
"num_heads": 20,
"num_layers": 32,
"dim_feedforward": 5120,
"causal": True,
"norm": "layer_norm",
"positional_embedding": "rope",
"max_period": 10000,
"gating": "none",
"layer_scale": 0.01,
"conv_layout": True,
},
{
"module_type": "PatchedPretransform",
"patch_size": 2,
},
{
"module_type": "Transformer",
"input_dimension": 640,
"output_dimension": 768,
"d_model": 768,
"num_heads": 12,
"num_layers": 12,
"dim_feedforward": 3072,
"causal": True,
"norm": "layer_norm",
"positional_embedding": "rope",
"max_period": 10000,
"gating": "none",
"layer_scale": 0.01,
"conv_layout": True,
},
{
"module_type": "PatchedPretransform",
"patch_size": 2,
},
{
"module_type": "Transformer",
"input_dimension": 384,
"output_dimension": 768,
"d_model": 768,
"num_heads": 12,
"num_layers": 12,
"dim_feedforward": 3072,
"causal": True,
"norm": "layer_norm",
"positional_embedding": "rope",
"max_period": 10000,
"gating": "none",
"layer_scale": 0.01,
"conv_layout": True,
},
{
"module_type": "PatchedPretransform",
"patch_size": 2,
},
{
"module_type": "Transformer",
"input_dimension": 384,
"output_dimension": 768,
"d_model": 768,
"num_heads": 12,
"num_layers": 12,
"dim_feedforward": 3072,
"causal": True,
"norm": "layer_norm",
"positional_embedding": "rope",
"max_period": 10000,
"gating": "none",
"layer_scale": 0.01,
"conv_layout": True,
},
{
"module_type": "PatchedPretransform",
"patch_size": 2,
},
{
"module_type": "Transformer",
"input_dimension": 384,
"output_dimension": 240,
"d_model": 768,
"num_heads": 12,
"num_layers": 12,
"dim_feedforward": 3072,
"causal": True,
"norm": "layer_norm",
"positional_embedding": "rope",
"max_period": 10000,
"gating": "none",
"layer_scale": 0.01,
"conv_layout": True,
},
{
"module_type": "PatchedPretransform",
"patch_size": 240,
},
]
self.decoder_kwargs = decoder_kwargs
# Default quantizer configuration
if quantizer_kwargs is None:
quantizer_kwargs = {
"input_dim": 768,
"rvq_dim": 512,
"output_dim": 768,
"num_quantizers": 32,
"codebook_size": 1024,
"codebook_dim": 8,
"quantizer_type": "rlfq",
}
# Handle quantizer_type from kwargs or config
kw_qtype = quantizer_kwargs.get("quantizer_type", None)
if kw_qtype is not None:
self.quantizer_type = kw_qtype
else:
self.quantizer_type = quantizer_type
quantizer_kwargs["quantizer_type"] = quantizer_type
self.quantizer_kwargs = quantizer_kwargs
super().__init__(**kwargs)
@property
def num_quantizers(self) -> int:
"""Return the number of quantizers from quantizer_kwargs."""
return self.quantizer_kwargs.get("num_quantizers", 32)
@property
def codebook_size(self) -> int:
"""Return the codebook size from quantizer_kwargs."""
return self.quantizer_kwargs.get("codebook_size", 4096)
@property
def frame_rate(self) -> float:
"""Return the frame rate (tokens per second)."""
return self.sampling_rate / self.downsample_rate
__all__ = ["MossAudioTokenizerConfig"]
@@ -5,6 +5,7 @@
from __future__ import annotations
import contextlib
from collections.abc import Iterable
from typing import Any
@@ -19,6 +20,12 @@ from vllm_omni.model_executor.models.moss_tts.audio_tokenizer import (
MossAudioTokenizerConfig,
MossAudioTokenizerModel,
)
from vllm_omni.model_executor.models.moss_tts.audio_tokenizer_v2 import (
MossAudioTokenizerModel as MossAudioTokenizerV2Model,
)
from vllm_omni.model_executor.models.moss_tts.configuration_moss_audio_tokenizer_v2 import (
MossAudioTokenizerConfig as MossAudioTokenizerV2Config,
)
from vllm_omni.model_executor.models.moss_tts.moss_codec_cudagraph import (
MossTTSCUDAGraphCodecWrapper,
)
@@ -27,6 +34,96 @@ from vllm_omni.model_executor.models.output_templates import OmniOutput
logger = init_logger(__name__)
class _MossCodecStreamSession:
"""Persistent streaming decode session for vendored MOSS-Audio-Tokenizer-v2."""
def __init__(
self,
codec: nn.Module,
*,
stream_slots: int,
n_vq: int,
) -> None:
self._codec = codec
self._stream_slots = int(stream_slots)
self._batch_size = self._stream_slots
self._n_vq = int(n_vq)
self._device = next(codec.parameters()).device
self._free_stream_slots = list(range(self._stream_slots))
self._exit_stack = contextlib.ExitStack()
self._closed = False
with torch.no_grad():
self._exit_stack.enter_context(codec.streaming(self._batch_size))
def acquire(self) -> int | None:
if not self._free_stream_slots:
return None
return self._free_stream_slots.pop()
def release(self, slot: int) -> None:
if self._closed:
return
self.reset_slots([slot])
self._free_stream_slots.append(slot)
def reset_slots(self, slots: list[int]) -> None:
if not slots:
return
reset_mask = torch.zeros(self._batch_size, dtype=torch.bool, device=self._device)
reset_mask[slots] = True
def _reset(module: nn.Module) -> None:
state = getattr(module, "_streaming_state", None)
if state is not None:
state.reset(reset_mask.to(state.device))
with torch.no_grad():
self._codec.apply(_reset)
def close(self) -> None:
if self._closed:
return
with torch.no_grad():
self._exit_stack.close()
self._closed = True
@torch.no_grad()
def step(self, slot_codes: dict[int, torch.Tensor]) -> dict[int, torch.Tensor]:
if not slot_codes:
return {}
step_lengths = {int(codes.shape[1]) for codes in slot_codes.values()}
if len(step_lengths) != 1:
raise ValueError(f"MOSS codec streaming step needs uniform T, got {sorted(step_lengths)}")
(step_t,) = step_lengths
codes_step = torch.zeros(
self._n_vq,
self._batch_size,
step_t,
dtype=torch.long,
device=self._device,
)
codes_lengths = torch.zeros(self._batch_size, dtype=torch.long, device=self._device)
exec_mask = torch.zeros(self._batch_size, dtype=torch.bool, device=self._device)
for slot, codes in slot_codes.items():
codes_step[:, slot, :] = codes.to(device=self._device, dtype=torch.long)
codes_lengths[slot] = int(codes.shape[1])
exec_mask[slot] = True
self._codec._set_streaming_exec_mask(exec_mask)
result = self._codec._decode_frame(codes_step, codes_lengths)
if result.audio is None:
return {}
audio = result.audio.detach().to("cpu", torch.float32)
lengths = result.audio_lengths.detach().to("cpu") if result.audio_lengths is not None else None
out: dict[int, torch.Tensor] = {}
for slot in slot_codes:
wav = audio[slot]
if lengths is not None:
wav = wav[..., : int(lengths[slot].item())]
out[slot] = wav.contiguous()
return out
class MossTTSCodecDecoder(nn.Module):
"""Stage-1 decoder for all MOSS-TTS variants.
@@ -75,6 +172,12 @@ class MossTTSCodecDecoder(nn.Module):
self._cuda_graph_wrapper: MossTTSCUDAGraphCodecWrapper | None = None
self._n_channels: int = 1
self._sr_tensor = torch.tensor(self._OUTPUT_SAMPLE_RATE, dtype=torch.int32)
self._stream_session: _MossCodecStreamSession | None = None
self._stream_slots: int = self._connector_int("codec_stream_slots", default=0)
self._stream_max_step_frames: int = self._connector_int("codec_max_step_frames", default=100)
self._stream_req_slots: dict[str, int] = {}
self._stream_pending_codes: dict[str, list[torch.Tensor]] = {}
self._stream_starved_reqs: set[str] = set()
# ------------------------------------------------------------------
# vLLM-Omni stubs (codec has no AR loop)
@@ -112,9 +215,9 @@ class MossTTSCodecDecoder(nn.Module):
chunk transfer adapter assigns those to ``request.prompt_token_ids``,
which arrives here as ``input_ids`` concatenated across all requests.
Per-request slice boundaries are computed from
``kwargs["num_scheduled_tokens"]`` (a list of token counts, one per
request). ``runtime_additional_information`` carries per-request
metadata such as ``left_context_size``.
``kwargs["seq_token_counts"]`` (token counts, one per request).
``runtime_additional_information`` carries per-request metadata such as
``left_context_size``.
Returns
-------
@@ -123,8 +226,8 @@ class MossTTSCodecDecoder(nn.Module):
multimodal_outputs["sr"] — list of scalar int32 tensors
"""
sr_tensor = self._sr_tensor
empty = torch.zeros((0,), dtype=torch.float32)
info_list: list[dict[str, Any]] = runtime_additional_information or [{}]
empty = self._empty_audio()
info_list: list[dict[str, Any]] = list(runtime_additional_information or [{}])
num_req = max(len(info_list), 1)
if self._codec is None:
@@ -140,24 +243,47 @@ class MossTTSCodecDecoder(nn.Module):
audios: list[torch.Tensor] = [empty] * num_req
srs: list[torch.Tensor] = [sr_tensor] * num_req
device = next(self._codec.parameters()).device
streaming_work: list[tuple[int, str, torch.Tensor, bool]] = []
if input_ids is None or input_ids.numel() == 0:
for i, wav in self._finish_empty_streaming_requests(info_list).items():
audios[i] = wav.reshape(-1) if wav.ndim == 1 or int(wav.shape[0]) == 1 else wav
return OmniOutput(
text_hidden_states=None,
multimodal_outputs={"model_outputs": audios, "sr": srs},
)
# ``input_ids`` is concatenated across all requests; split by
# query_start_loc-style offsets from ``num_scheduled_tokens`` in
# kwargs if available, else assume one request.
# ``input_ids`` is concatenated across all requests. vLLM-Omni runners
# pass the per-request lengths via the shared code2wav contract
# ``seq_token_counts``.
ids_flat = input_ids.reshape(-1).to(dtype=torch.long)
num_scheduled_tokens = kwargs.get("num_scheduled_tokens")
if isinstance(num_scheduled_tokens, (list, tuple)) and len(num_scheduled_tokens) == num_req:
offsets = [0]
for n in num_scheduled_tokens:
offsets.append(offsets[-1] + int(n))
else:
offsets = [0, int(ids_flat.shape[0])]
token_counts = self._normalize_seq_token_counts(kwargs.get("seq_token_counts"))
if token_counts is None:
raise RuntimeError(
"MossTTS codec requires seq_token_counts; otherwise concatenated "
"codec tokens cannot be split per request."
)
if sum(token_counts) != int(ids_flat.shape[0]):
raise RuntimeError(
"MossTTS codec seq_token_counts mismatch: "
f"counts={token_counts}, sum={sum(token_counts)}, input_tokens={int(ids_flat.shape[0])}."
)
num_req = len(token_counts)
if len(info_list) < num_req:
info_list.extend({} for _ in range(num_req - len(info_list)))
elif len(info_list) > num_req:
info_list = info_list[:num_req]
if len(audios) < num_req:
audios.extend(empty for _ in range(num_req - len(audios)))
srs.extend(sr_tensor for _ in range(num_req - len(srs)))
elif len(audios) > num_req:
audios = audios[:num_req]
srs = srs[:num_req]
offsets = [0]
for n in token_counts:
offsets.append(offsets[-1] + int(n))
for i, info in enumerate(info_list):
if i + 1 >= len(offsets):
@@ -165,6 +291,14 @@ class MossTTSCodecDecoder(nn.Module):
seg = ids_flat[offsets[i] : offsets[i + 1]]
if seg.numel() == 0:
continue
meta = (info.get("meta", {}) if isinstance(info, dict) else {}) or {}
finished = bool(meta.get("stream_finished", meta.get("finished", False)))
streaming_enabled = bool(meta.get("codec_streaming", False))
code_flat_numel = meta.get("code_flat_numel")
if streaming_enabled and finished and code_flat_numel is not None and int(code_flat_numel) == 0:
for _, wav in self._finish_empty_streaming_requests([info]).items():
audios[i] = wav.reshape(-1) if wav.ndim == 1 or int(wav.shape[0]) == 1 else wav
continue
if seg.numel() % self._n_vq != 0:
logger.warning(
"MossTTS codec input length %d not divisible by n_vq %d; skipping.",
@@ -181,14 +315,19 @@ class MossTTSCodecDecoder(nn.Module):
codebook_size = self._codec.config.codebook_size
codes_nq_t = codes_nq_t.clamp_(0, int(codebook_size) - 1)
meta = (info.get("meta", {}) if isinstance(info, dict) else {}) or {}
left_ctx = self._runtime_value(info, meta, "left_context_size", 0)
left_ctx = meta.get("left_context_size", 0)
if isinstance(left_ctx, (list, tuple)):
left_ctx = int(left_ctx[0]) if left_ctx else 0
elif isinstance(left_ctx, torch.Tensor):
left_ctx = int(left_ctx.reshape(-1)[0].item()) if left_ctx.numel() else 0
left_ctx = int(left_ctx)
req_key = self._runtime_request_key(info, meta, i)
if streaming_enabled:
streaming_work.append((i, req_key, codes_nq_t, finished))
continue
if self._cuda_graph_wrapper is not None:
out = self._cuda_graph_wrapper.decode(codes_nq_t)
else:
@@ -216,19 +355,246 @@ class MossTTSCodecDecoder(nn.Module):
)
wav = wav[..., trim:]
audios[i] = wav.reshape(-1) if self._n_channels == 1 else wav
audios[i] = wav.reshape(-1) if wav.ndim == 1 or int(wav.shape[0]) == 1 else wav
if streaming_work:
for i, wav in self._decode_streaming_batch(streaming_work).items():
audios[i] = wav.reshape(-1) if wav.ndim == 1 or int(wav.shape[0]) == 1 else wav
return OmniOutput(
text_hidden_states=None,
multimodal_outputs={"model_outputs": audios, "sr": srs},
)
def _finish_empty_streaming_requests(self, info_list: list[dict[str, Any]]) -> dict[int, torch.Tensor]:
"""Release codec stream state for empty finish sentinels.
Stage-0 can finish on a step that emits no new audio frame. The stage
input processor forwards that as an empty payload with finished=true.
If a request never acquired a stream slot, do not offline-decode its
buffered codes here: streaming requests must only emit client deltas.
"""
session = self._stream_session
outputs: dict[int, torch.Tensor] = {}
if session is None:
return outputs
for i, info in enumerate(info_list):
if not isinstance(info, dict):
continue
meta = (info.get("meta", {}) or {}) if isinstance(info.get("meta", {}), dict) else {}
if not bool(meta.get("codec_streaming", False)):
continue
finished = bool(meta.get("stream_finished", meta.get("finished", False)))
if not finished:
continue
req_key = self._runtime_request_key(info, meta, i)
slot = self._stream_req_slots.get(req_key)
pending = req_key in self._stream_pending_codes
if slot is not None or pending:
if pending and slot is None:
logger.warning(
"MOSS codec stream request %s finished before a codec stream slot became available; "
"dropping buffered codes instead of offline-decoding a non-delta waveform.",
req_key,
)
self._finish_stream_request(req_key, session, slot)
return outputs
@staticmethod
def _runtime_value(info: Any, meta: dict[str, Any], name: str, default: Any = None) -> Any:
if name in meta:
return meta[name]
if isinstance(info, dict) and name in info:
return info[name]
def _normalize_seq_token_counts(value: Any) -> list[int] | None:
if value is None:
return None
if not isinstance(value, (list, tuple)):
raise TypeError(
"MossTTS codec expects seq_token_counts to be a list/tuple of per-request token counts, "
f"got {type(value).__name__}."
)
counts = [int(item) for item in value]
if not counts:
return None
for count in counts:
if count < 0:
raise ValueError(f"MossTTS codec seq_token_counts must be non-negative, got {counts}.")
return counts
def _runtime_request_key(self, info: Any, meta: dict[str, Any], index: int) -> str:
for value in (
meta.get("req_id"),
info.get("request_id") if isinstance(info, dict) else None,
):
if isinstance(value, (list, tuple)):
value = value[0] if value else None
if value is not None:
return str(value)
return f"moss-codec-stream-{index}"
def _empty_audio(self) -> torch.Tensor:
if self._n_channels > 1:
return torch.zeros((self._n_channels, 0), dtype=torch.float32)
return torch.zeros((0,), dtype=torch.float32)
def _ensure_stream_session(self) -> _MossCodecStreamSession | None:
if self._codec is None:
return None
if self._stream_session is not None:
return self._stream_session
slots = self._stream_slots
if slots <= 0:
scheduler_cfg = getattr(self.vllm_config, "scheduler_config", None)
slots = int(getattr(scheduler_cfg, "max_num_seqs", 1) or 1)
self._stream_session = _MossCodecStreamSession(
self._codec,
stream_slots=max(1, slots),
n_vq=self._n_vq,
)
return self._stream_session
def _decode_streaming_batch(
self,
items: list[tuple[int, str, torch.Tensor, bool]],
) -> dict[int, torch.Tensor]:
session = self._ensure_stream_session()
if session is None:
return {}
outputs: dict[int, torch.Tensor] = {}
grouped: dict[int, list[tuple[int, str, int, torch.Tensor, bool]]] = {}
max_step_frames = max(1, int(self._stream_max_step_frames))
for output_index, request_id, codes_nq_t, finished in items:
pending = self._stream_pending_codes.get(request_id)
slot = self._stream_req_slots.get(request_id)
if slot is None:
slot = session.acquire()
if slot is None:
self._append_stream_pending(request_id, codes_nq_t)
if request_id not in self._stream_starved_reqs:
logger.warning(
"MOSS codec streaming slots exhausted; buffering %s until a stream slot is available.",
request_id,
)
self._stream_starved_reqs.add(request_id)
if finished:
logger.warning(
"MOSS codec stream request %s finished before a codec stream slot became available; "
"dropping buffered codes instead of offline-decoding a non-delta waveform.",
request_id,
)
self._finish_stream_request(request_id, session, None)
continue
self._stream_req_slots[request_id] = slot
if pending:
self._append_stream_pending(request_id, codes_nq_t)
replay_codes = self._pop_stream_pending(request_id)
wav = self._decode_stream_slot_sequence(session, slot, replay_codes)
if wav is not None:
outputs[output_index] = wav
if finished:
self._finish_stream_request(request_id, session, slot)
continue
if int(codes_nq_t.shape[1]) > max_step_frames:
wav = self._decode_stream_slot_sequence(session, slot, codes_nq_t)
if wav is not None:
outputs[output_index] = wav
if finished:
self._finish_stream_request(request_id, session, slot)
continue
grouped.setdefault(int(codes_nq_t.shape[1]), []).append(
(output_index, request_id, slot, codes_nq_t, finished)
)
for group in grouped.values():
plan = {slot: codes_nq_t for _, _, slot, codes_nq_t, _ in group}
decoded = session.step(plan)
for output_index, request_id, slot, _, finished in group:
wav = decoded.get(slot)
if wav is not None:
outputs[output_index] = wav
if finished:
self._finish_stream_request(request_id, session, slot)
return outputs
def _append_stream_pending(self, request_id: str, codes_nq_t: torch.Tensor) -> None:
self._stream_pending_codes.setdefault(request_id, []).append(
codes_nq_t.detach().to("cpu", torch.long).contiguous()
)
def _pop_stream_pending(self, request_id: str) -> torch.Tensor:
pending = self._stream_pending_codes.pop(request_id, [])
if not pending:
return torch.empty((self._n_vq, 0), dtype=torch.long)
return torch.cat(pending, dim=1).contiguous()
def _decode_stream_slot_sequence(
self,
session: _MossCodecStreamSession,
slot: int,
codes_nq_t: torch.Tensor,
) -> torch.Tensor | None:
if codes_nq_t.numel() == 0:
return None
max_step_frames = max(1, int(self._stream_max_step_frames))
parts: list[torch.Tensor] = []
for start in range(0, int(codes_nq_t.shape[1]), max_step_frames):
chunk = codes_nq_t[:, start : start + max_step_frames]
decoded = session.step({slot: chunk})
wav = decoded.get(slot)
if wav is not None:
parts.append(wav)
if not parts:
return None
return torch.cat(parts, dim=-1) if len(parts) > 1 else parts[0]
def _finish_stream_request(
self,
request_id: str,
session: _MossCodecStreamSession,
slot: int | None,
) -> None:
if slot is not None:
session.release(slot)
self._stream_req_slots.pop(request_id, None)
self._stream_pending_codes.pop(request_id, None)
self._stream_starved_reqs.discard(request_id)
def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None:
"""Release codec streaming slots when requests finish outside payload flow.
Normal streaming completion releases slots from ``_decode_streaming_batch``
when the Stage-0 payload carries ``finished=True``. Client disconnects
and engine-side aborts can finish a request without delivering that
terminal payload, so the runner calls this hook from its finished-request
path to avoid leaking stream slots and buffered codes.
"""
session = self._stream_session
for req_id in finished_req_ids:
request_id = str(req_id)
slot = self._stream_req_slots.get(request_id)
has_state = (
slot is not None or request_id in self._stream_pending_codes or request_id in self._stream_starved_reqs
)
if not has_state:
continue
if session is not None:
self._finish_stream_request(request_id, session, slot)
else:
self._stream_req_slots.pop(request_id, None)
self._stream_pending_codes.pop(request_id, None)
self._stream_starved_reqs.discard(request_id)
def _connector_int(self, name: str, default: int = 0) -> int:
model_cfg = getattr(self.vllm_config, "model_config", None)
connector_cfg = getattr(model_cfg, "stage_connector_config", None)
if isinstance(connector_cfg, dict):
extra_cfg: dict | None = connector_cfg.get("extra", connector_cfg)
else:
extra_cfg = getattr(connector_cfg, "extra", None)
if isinstance(extra_cfg, dict) and name in extra_cfg:
return int(extra_cfg[name])
return default
# ------------------------------------------------------------------
@@ -272,14 +638,12 @@ class MossTTSCodecDecoder(nn.Module):
(".self_attn.out_projs.0.", ".attn.out_proj."),
(".linear1.", ".ff1."),
(".linear2.", ".ff2."),
# v2 (MOSS-Audio-Tokenizer-v2) uses different submodule names for
# the same attention/FFN sublayers — no trailing "s"/index on
# in_proj/out_proj, and an `ffn.{0,2}` Sequential instead of
# separate linear1/linear2 attributes.
(".self_attn.in_proj.", ".attn.in_proj."),
(".self_attn.out_proj.", ".attn.out_proj."),
(".ffn.0.", ".ff1."),
(".ffn.2.", ".ff2."),
# v2 checkpoint names use singular in_proj/out_proj and ffn.{0,2};
# the vendored module keeps the original MOSS layer names.
(".self_attn.in_proj.", ".self_attn.in_projs.0."),
(".self_attn.out_proj.", ".self_attn.out_projs.0."),
(".ffn.0.", ".linear1."),
(".ffn.2.", ".linear2."),
(".layer_scale_1.", ".ls1."),
(".layer_scale_2.", ".ls2."),
(".input_proj.", ".in_proj."),
@@ -292,21 +656,39 @@ class MossTTSCodecDecoder(nn.Module):
return name.replace(src, dst)
return name
loaded_total = 0
loaded_names: set[str] = set()
skipped: list[str] = []
shape_mismatches: list[tuple[str, str, tuple[int, ...], tuple[int, ...]]] = []
for name, tensor in codec_weights:
# Try direct name first (e.g. ``quantizer.input_proj.*`` exists
# under the same name in both layouts), then the remap (transformer
# submodules need ``.linear1.``→``.ff1.`` etc.).
tgt = name if name in params_dict else _remap(name)
if tgt in params_dict:
expected_shape = tuple(params_dict[tgt].shape)
actual_shape = tuple(tensor.shape)
if expected_shape != actual_shape:
shape_mismatches.append((name, tgt, actual_shape, expected_shape))
continue
default_weight_loader(params_dict[tgt], tensor)
loaded_total += 1
loaded_names.add(tgt)
else:
skipped.append(name)
missing = sorted(set(params_dict) - loaded_names)
if missing or skipped or shape_mismatches:
raise RuntimeError(
"MOSS Audio Tokenizer weights were not fully loaded: "
f"loaded={len(loaded_names)}/{len(params_dict)} "
f"missing={len(missing)} skipped={len(skipped)} "
f"shape_mismatches={len(shape_mismatches)}; "
f"first_missing={missing[:5]} "
f"first_skipped={skipped[:5]} "
f"first_shape_mismatches={shape_mismatches[:3]}"
)
logger.info(
"MOSS Audio Tokenizer weights: loaded=%d/%d skipped=%d (first skipped: %s)",
loaded_total,
len(loaded_names),
len(params_dict),
len(skipped),
skipped[:3] if skipped else "none",
@@ -316,7 +698,15 @@ class MossTTSCodecDecoder(nn.Module):
codec.to(device=device, dtype=torch.float32)
codec.eval()
self._codec = codec
self._n_channels = int(getattr(codec_cfg, "number_channels", 1) or 1)
inferred_channels = 2 if "v2" in codec_path.lower() else 1
self._n_channels = int(
getattr(
codec_cfg,
"number_channels",
getattr(codec_cfg, "num_channels", inferred_channels),
)
or inferred_channels
)
self._sr_tensor = torch.tensor(int(codec_cfg.sampling_rate), dtype=torch.int32)
logger.info(
@@ -336,16 +726,13 @@ class MossTTSCodecDecoder(nn.Module):
def _build_codec(self, codec_path: str) -> tuple[Any, nn.Module]:
try:
from transformers import AutoConfig, AutoModel
codec_cfg = AutoConfig.from_pretrained(codec_path, trust_remote_code=True)
codec = AutoModel.from_config(codec_cfg, trust_remote_code=True)
logger.info("Using MOSS Audio Tokenizer remote-code classes from %s", codec_path)
codec_cfg = MossAudioTokenizerV2Config.from_pretrained(codec_path)
codec = MossAudioTokenizerV2Model(codec_cfg)
logger.info("Using vendored MOSS Audio Tokenizer v2 classes from %s", codec_path)
return codec_cfg, codec
except Exception:
logger.exception(
"Failed to instantiate official MOSS Audio Tokenizer via HF remote code; "
"falling back to vendored codec."
"Failed to instantiate vendored MOSS Audio Tokenizer v2; falling back to legacy vendored codec."
)
codec_cfg = MossAudioTokenizerConfig.from_pretrained(codec_path)
@@ -126,6 +126,7 @@ def _sample_token(
top_k: int,
top_p: float,
do_sample: bool,
generator: torch.Generator | None = None,
) -> torch.Tensor:
"""Top-k + top-p sampling (matches upstream's ``sample_token`` for the
inference branch).
@@ -152,7 +153,7 @@ def _sample_token(
probs = F.softmax(logits, dim=-1)
flat = probs.reshape(-1, probs.shape[-1])
sampled = torch.multinomial(flat, num_samples=1).reshape(probs.shape[:-1])
sampled = torch.multinomial(flat, num_samples=1, generator=generator).reshape(probs.shape[:-1])
return sampled
@@ -160,6 +160,7 @@ class MossTTSLocalDepthTransformer(nn.Module):
text_top_p: float = 1.0,
repetition_penalty: float = 1.0,
history_per_codebook: list[list[int]] | None = None,
generator: torch.Generator | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Generate one audio frame for batch B.
@@ -183,7 +184,16 @@ class MossTTSLocalDepthTransformer(nn.Module):
local_hidden = hidden[:, 0, :]
binary_logits = local_text_lm_head(local_hidden).float()
binary_choice = _sample_token(binary_logits, text_temperature, text_top_k, text_top_p, do_sample)
# This is a binary continue/stop gate. The checkpoint expects sampling
# here; greedy argmax is biased toward "continue" and may never stop.
binary_choice = _sample_token(
binary_logits,
text_temperature,
text_top_k,
text_top_p,
do_sample,
generator=generator,
)
should_continue = binary_choice.eq(0)
import os as _os
@@ -209,7 +219,14 @@ class MossTTSLocalDepthTransformer(nn.Module):
pos = sel > 0
sel = torch.where(pos, sel / repetition_penalty, sel * repetition_penalty)
channel_logits.index_copy_(-1, hist_t, sel)
channel_token = _sample_token(channel_logits, temperature, top_k, top_p, do_sample)
channel_token = _sample_token(
channel_logits,
temperature,
top_k,
top_p,
do_sample,
generator=generator,
)
codes[:, channel_index] = channel_token
if channel_index + 1 < n_vq:
@@ -1139,26 +1139,19 @@ class MossTTSRealtimeTalkerForGeneration(nn.Module):
info["audio_codes"] = {"current": new_codes, "accumulated": updated_acc}
state["step"] = int(state.get("step", 0)) + 1
per_req_codes[i] = updated_acc
per_req_codes[i] = new_codes.unsqueeze(0)
have_codes = True
if not have_codes:
return OmniOutput(text_hidden_states=hidden, multimodal_outputs={})
# Emit one accumulated-codes tensor per request as a list; the splitter
# routes ``element[idx]`` to request idx, so each Stage-1 request decodes
# its own codes instead of request 0's.
# The realtime variant emits raw codes (no delay pattern), so we signal
# that to the chunk processor via a 1-D bool tensor in ``meta.finished``
# (already a tensor field on the schema). The processor reads its truth
# value as "skip de-delay" only for realtime; the delay path doesn't
# populate ``meta`` at all.
device = hidden.device
# Emit one current-frame tensor per request as a list; the splitter
# routes ``element[idx]`` to request idx, so each Stage-1 request receives
# only its own newly generated raw codec row.
return OmniOutput(
text_hidden_states=hidden,
multimodal_outputs={
"codes": {"audio": per_req_codes},
"meta": {"finished": torch.tensor([True], dtype=torch.bool, device=device)},
},
)
@@ -1266,7 +1259,7 @@ class MossTTSLocalTalkerForGeneration(nn.Module):
* Per-step audio generation runs ``MossTTSLocalDepthTransformer``
(1-layer GPT2-style local transformer, interleaved RoPE, ``n_vq=12``
codebooks -- structurally different from Realtime's Qwen3-style
``CodePredictorBaseModel``) inside ``make_omni_output``. Stop
``CodePredictorBaseModel``) in the runner-owned ``talker_mtp`` path. Stop
condition: the local transformer's binary head selects the
"audio_end" candidate (logits index 1).
"""
@@ -1327,6 +1320,8 @@ class MossTTSLocalTalkerForGeneration(nn.Module):
# Stacked embedding cache built after load_weights() for vectorised
# additive-fusion gathers (avoids an n_vq-iteration Python loop).
self._stacked_audio_emb_w: torch.Tensor | None = None
self.mtp_hidden_size = hidden_size
self.talker_mtp_accepts_req_infos = True
self.gpu_resident_buffer_keys: set[tuple[str, str]] = {
("audio_codes", "current"),
@@ -1384,6 +1379,15 @@ class MossTTSLocalTalkerForGeneration(nn.Module):
num_rows,
):
if state is not None and state.get("is_stopping"):
forced_im_end_count = int(state.get("_forced_im_end_count", 0))
if forced_im_end_count == 1:
logger.warning(
"MOSS-TTS Local request is still running after "
"im_end_token_id=%d was forced once; verify stage-0 "
"stop_token_ids/eos includes this id.",
self.im_end_token_id,
)
state["_forced_im_end_count"] = forced_im_end_count + 1
logits[row_start:row_end, self.im_end_token_id] = 0.0
else:
logits[row_start:row_end, self.audio_assistant_slot_token_id] = 0.0
@@ -1455,38 +1459,159 @@ class MossTTSLocalTalkerForGeneration(nn.Module):
else torch.full((self.n_vq,), self.audio_pad_token_id, dtype=torch.long, device=device)
)
max_new_frames_req = info_dict.get("max_new_frames")
if isinstance(max_new_frames_req, (list, tuple)) and max_new_frames_req:
max_new_frames_req = max_new_frames_req[0]
try:
max_new_frames = int(max_new_frames_req) if max_new_frames_req is not None else -1
except (TypeError, ValueError):
max_new_frames = -1
max_new_frames = info_dict.get("max_new_frames", [-1])[0]
info_update: dict[str, Any] = {
"audio_state": {"is_stopping": False, "step": 0, "max_new_frames": max_new_frames},
"audio_codes": {"current": current_codes},
"audio_state": {
"is_stopping": False,
"step": 0,
"max_new_frames": max_new_frames,
},
"audio_codes": {"current": current_codes, "emit": False},
"ref_offset": ref_offset + span_len,
}
return input_ids, embeds, info_update
# Decode step: input_ids is the forced text token from compute_logits.
# Decode step: return the plain forced text-token embedding. The local
# depth transformer runs in OmniGPUModelRunner.talker_mtp before the
# backbone forward; it adds the freshly sampled audio-code embedding
# and stores the generated code frame in model_intermediate_buffer.
text_embed = self.model.embed_tokens(input_ids.reshape(1)) # (1, H)
prev_codes = (info_dict.get("audio_codes", {}) or {}).get("current")
if isinstance(prev_codes, torch.Tensor) and prev_codes.numel() == self.n_vq:
codes = prev_codes.to(device=device, dtype=torch.long).unsqueeze(0) # (1, n_vq)
combined = text_embed + self._audio_embed(codes)
hs = info_dict.get("hidden_states", {}) or {}
last_hidden = hs.get("last")
if isinstance(last_hidden, torch.Tensor) and last_hidden.numel() > 0:
mtp_hidden = last_hidden.to(device=device, dtype=text_embed.dtype).reshape(1, -1)
else:
combined = text_embed
return input_ids, combined, {}
mtp_hidden = torch.zeros((1, self.hidden_size), device=device, dtype=text_embed.dtype)
return input_ids, text_embed, {"mtp_inputs": (mtp_hidden, torch.zeros_like(mtp_hidden))}
def postprocess(self, hidden_states: torch.Tensor, **_: Any) -> dict[str, Any]:
if hidden_states.numel() == 0:
return {}
return {"hidden_states": {"last": hidden_states[-1].detach()}}
@torch.inference_mode()
def talker_mtp(
self,
input_ids: torch.Tensor,
input_embeds: torch.Tensor,
last_talker_hidden: torch.Tensor,
text_step: torch.Tensor,
do_sample: bool | None = None,
temperature: float | None = None,
top_k: int | None = None,
top_p: float | None = None,
generator: torch.Generator | None = None,
req_infos: list[dict[str, Any]] | None = None,
**_: Any,
) -> tuple[torch.Tensor, None]:
bsz = int(input_embeds.shape[0])
dev = input_embeds.device
input_embeds_out = input_embeds.reshape(bsz, -1).clone()
req_infos = req_infos or [{} for _ in range(bsz)]
def write_update(info: dict[str, Any], update: dict[str, Any]) -> None:
for key, value in update.items():
if isinstance(value, dict):
target = info.setdefault(key, {})
target.update(value)
else:
info[key] = value
# Match MOSS-TTS Local v1.5's model-card/SGLang defaults. The generic
# stage SamplingParams are for the backbone token loop and should not
# silently retune the local depth transformer's audio-channel sampler.
audio_temperature = 1.7
audio_top_k = 25
audio_top_p = 0.8
do_sample_val = True if do_sample is None else bool(do_sample)
for i in range(bsz):
info = req_infos[i] if i < len(req_infos) and isinstance(req_infos[i], dict) else {}
state = dict(info.get("audio_state", {}) or {})
step = int(state.get("step", 0))
max_new_frames = int(state.get("max_new_frames", -1))
acc = (info.get("audio_codes", {}) or {}).get("accumulated")
if state.get("is_stopping"):
write_update(
info,
{
"audio_state": state,
"audio_codes": {
"current": input_embeds.new_empty((0, self.n_vq), dtype=torch.long),
"emit": False,
},
},
)
continue
rep_window = 50
if isinstance(acc, torch.Tensor) and acc.numel() > 0:
tail = acc[-rep_window:].long().cpu().tolist()
hist_per_cb = [[row[cb] for row in tail] for cb in range(self.n_vq)]
else:
hist_per_cb = [[] for _ in range(self.n_vq)]
should_continue_t, new_codes_b = self.local_transformer.generate_frame(
last_talker_hidden[i : i + 1],
self.audio_lm_heads,
self.audio_embeddings,
self.local_text_lm_head,
n_vq=self.n_vq,
do_sample=do_sample_val,
temperature=audio_temperature,
top_k=audio_top_k,
top_p=audio_top_p,
repetition_penalty=1.0,
history_per_codebook=hist_per_cb,
generator=generator,
)
should_continue = bool(should_continue_t[0].item())
new_codes = new_codes_b.squeeze(0).to(device=dev, dtype=torch.long)
if not should_continue or (0 <= max_new_frames <= step):
state["is_stopping"] = True
state["step"] = step + 1
write_update(
info,
{
"audio_state": state,
"audio_codes": {
"current": input_embeds.new_empty((0, self.n_vq), dtype=torch.long),
"accumulated": acc,
"emit": False,
},
},
)
continue
if isinstance(acc, torch.Tensor) and acc.numel() > 0:
updated_acc = torch.cat([acc.to(dev), new_codes.unsqueeze(0)], dim=0)
else:
updated_acc = new_codes.unsqueeze(0)
state["step"] = step + 1
current = new_codes.unsqueeze(0)
input_embeds_out[i : i + 1] = input_embeds_out[i : i + 1] + self._audio_embed(current).to(
dtype=input_embeds_out.dtype
)
write_update(
info,
{
"audio_state": state,
"audio_codes": {
"current": current,
"accumulated": updated_acc,
"emit": True,
},
},
)
return input_embeds_out, None
# ------------------------------------------------------------------
# Per-step audio generation via local transformer
# Package runner-generated audio frames
# ------------------------------------------------------------------
def make_omni_output(
@@ -1511,91 +1636,41 @@ class MossTTSLocalTalkerForGeneration(nn.Module):
self._batch_state = [(info["audio_state"] if isinstance(info, dict) else {}) for info in info_dicts]
self._batch_state_spans = kwargs.get("request_token_spans")
spans = kwargs.get("request_token_spans")
per_req_codes: list[torch.Tensor] = [hidden.new_empty((0, self.n_vq), dtype=torch.long) for _ in info_dicts]
per_req_codes: list[torch.Tensor] = []
have_codes = False
if hidden.numel() > 0 and info_dicts:
num_rows = hidden.shape[0]
if spans is None or len(spans) != len(info_dicts):
raise RuntimeError(
"MOSS-TTS make_omni_output requires request_token_spans "
f"for {len(info_dicts)} request infos, got {0 if spans is None else len(spans)}"
)
for i, info in enumerate(info_dicts):
if not isinstance(info, dict):
continue
row_start, row_end = spans[i]
row_end = min(int(row_end), num_rows)
if row_start >= row_end:
continue
state = info.get("audio_state", {}) or {}
if state.get("is_stopping"):
continue # already stopped -- no more audio frames
last_h = hidden[row_end - 1].unsqueeze(0) # (1, H)
# Sampling parameters mirror Realtime's defaults; repetition
# penalty applies over a 50-frame rolling window per codebook.
rep_window = 50
acc_for_hist = (info.get("audio_codes", {}) or {}).get("accumulated")
if isinstance(acc_for_hist, torch.Tensor) and acc_for_hist.numel() > 0:
tail = acc_for_hist[-rep_window:].long().cpu().tolist()
hist_per_cb = [[row[cb] for row in tail] for cb in range(self.n_vq)]
else:
hist_per_cb = [[] for _ in range(self.n_vq)]
should_continue_t, new_codes = self.local_transformer.generate_frame(
last_h,
self.audio_lm_heads,
self.audio_embeddings,
self.local_text_lm_head,
n_vq=self.n_vq,
do_sample=True,
temperature=0.8,
top_k=30,
top_p=0.6,
repetition_penalty=1.1,
history_per_codebook=hist_per_cb,
)
should_continue = bool(should_continue_t[0].item())
new_codes = new_codes.squeeze(0) # (n_vq,)
step = int(state.get("step", 0))
max_new_frames = int(state.get("max_new_frames", -1))
if not should_continue or (0 <= max_new_frames <= step):
state["is_stopping"] = True
state["step"] = step + 1
info["audio_codes"] = {
"current": new_codes,
"accumulated": (info.get("audio_codes", {}) or {}).get("accumulated"),
}
continue # don't append the stop frame to accumulated
acc = (info.get("audio_codes", {}) or {}).get("accumulated")
if isinstance(acc, torch.Tensor) and acc.numel() > 0:
updated_acc = torch.cat([acc.to(new_codes.device), new_codes.unsqueeze(0)], dim=0)
else:
updated_acc = new_codes.unsqueeze(0)
info["audio_codes"] = {"current": new_codes, "accumulated": updated_acc}
state["step"] = step + 1
per_req_codes[i] = updated_acc
for info in info_dicts:
current = (info.get("audio_codes", {}) or {}).get("current") if isinstance(info, dict) else None
emit = bool((info.get("audio_codes", {}) or {}).get("emit")) if isinstance(info, dict) else False
if emit and isinstance(current, torch.Tensor) and current.numel() > 0:
codes = current.to(device=hidden.device, dtype=torch.long)
if codes.dim() == 1:
codes = codes.unsqueeze(0)
per_req_codes.append(codes)
have_codes = True
else:
per_req_codes.append(hidden.new_empty((0, self.n_vq), dtype=torch.long))
if not have_codes:
return OmniOutput(text_hidden_states=hidden, multimodal_outputs={})
# Local-v1.5 emits raw codes (no delay pattern), so signal that to the
# chunk processor the same way Realtime does, via ``meta.finished``
# (read as "skip de-delay" by ``talker2codec_async_chunk``).
device = hidden.device
# Local-v1.5 emits the current raw code frame (no delay pattern); the
# pipeline routes those rows through ``talker2codec_raw_async_chunk``.
per_req_ref_codes: list[torch.Tensor] = []
for info in info_dicts:
ref_codes = (info.get("codes", {}) or {}).get("ref") if isinstance(info, dict) else None
if isinstance(ref_codes, torch.Tensor) and ref_codes.numel() > 0:
ref_codes = ref_codes.detach()
if ref_codes.dim() == 1 and ref_codes.numel() % self.n_vq == 0:
ref_codes = ref_codes.view(-1, self.n_vq)
if ref_codes.dim() == 2 and ref_codes.shape[1] == self.n_vq:
per_req_ref_codes.append(ref_codes.to(device=hidden.device, dtype=torch.long))
continue
per_req_ref_codes.append(hidden.new_empty((0, self.n_vq), dtype=torch.long))
return OmniOutput(
text_hidden_states=hidden,
multimodal_outputs={
"codes": {"audio": per_req_codes},
"meta": {"finished": torch.tensor([True], dtype=torch.bool, device=device)},
"codes": {"audio": per_req_codes, "ref": per_req_ref_codes},
},
)
@@ -31,7 +31,7 @@ MOSS_TTS_PIPELINE = PipelineConfig(
input_sources=(),
owns_tokenizer=True,
engine_output_type="latent",
async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2codec_async_chunk"),
async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2codec_delay_async_chunk"),
sampling_constraints={
"detokenize": False,
},
@@ -62,7 +62,7 @@ MOSS_TTS_REALTIME_PIPELINE = PipelineConfig(
input_sources=(),
owns_tokenizer=True,
engine_output_type="latent",
async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2codec_async_chunk"),
async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2codec_raw_async_chunk"),
sampling_constraints={"detokenize": False},
),
StagePipelineConfig(
@@ -91,8 +91,11 @@ MOSS_TTS_LOCAL_PIPELINE = PipelineConfig(
input_sources=(),
owns_tokenizer=True,
engine_output_type="latent",
async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2codec_async_chunk"),
sampling_constraints={"detokenize": False},
async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2codec_raw_async_chunk"),
sampling_constraints={
"detokenize": False,
"stop_token_ids": [151645],
},
),
StagePipelineConfig(
stage_id=1,