Make chunk_size and left_context_size configurable via YAML for async chunking (#1423)
Signed-off-by: Junhong Liu <98734602+LJH-LBJ@users.noreply.github.com> Signed-off-by: Junhong Liu <ljh_lbj@163.com>
This commit is contained in:
1 parent
b13761ed26
commit
742d845b8e
10 files changed
+113
-44
No files matched your search
@@ -5,6 +5,7 @@
|
||||
|
||||
from collections.abc import Iterable
|
||||
from functools import cached_property
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -160,6 +161,7 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
self.requires_raw_input_tokens = True
|
||||
|
||||
elif self.model_stage == "code2wav":
|
||||
self.enable_update_additional_information = True
|
||||
self.thinker = None
|
||||
self.talker = None
|
||||
# Initialize code2wav (codec codes → audio waveform)
|
||||
@@ -254,7 +256,7 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
codec: torch.Tensor | None = None,
|
||||
sampling_metadata: SamplingMetadata | None = None,
|
||||
logits_index: int | None = None,
|
||||
additional_information: dict[str, object] | None = None,
|
||||
runtime_additional_information: list[dict[str, Any]] | None = None,
|
||||
**kwargs: object,
|
||||
) -> torch.Tensor | IntermediateTensors | OmniOutput:
|
||||
"""
|
||||
@@ -359,7 +361,15 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
codes = input_ids_flatten.reshape(1, 16, -1)
|
||||
|
||||
# Generate audio from codec codes
|
||||
audio_tensors = self.generate_audio(codes, voice_type, seq_token_counts)
|
||||
# Get every request's left_context_size from runtime_additional_information (passed via kwargs)
|
||||
left_context_size = []
|
||||
if runtime_additional_information is not None:
|
||||
for info in runtime_additional_information:
|
||||
if "left_context_size" in info:
|
||||
left_context_size.append(info["left_context_size"])
|
||||
else:
|
||||
logger.debug("No additional_information provided to code2wav stage.")
|
||||
audio_tensors = self.generate_audio(codes, voice_type, left_context_size, seq_token_counts)
|
||||
|
||||
return audio_tensors
|
||||
|
||||
@@ -440,6 +450,7 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
self,
|
||||
code: torch.Tensor,
|
||||
voice_type: str,
|
||||
left_context_size: list[int] | None = None,
|
||||
seq_token_counts: list[int] | None = None,
|
||||
) -> list[torch.Tensor]:
|
||||
"""
|
||||
@@ -448,6 +459,7 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
Args:
|
||||
code: [batch, num_quantizers, T] - RVQ codec codes
|
||||
voice_type: Voice type (not used in Qwen3, kept for compatibility)
|
||||
left_context_size: Left context size for streaming decode
|
||||
seq_token_counts: Token count for each request in batch
|
||||
|
||||
Returns:
|
||||
@@ -471,10 +483,10 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
talker_codes = talker_codes.expand(1, 16, -1)
|
||||
|
||||
if self.vllm_config.model_config.async_chunk:
|
||||
# Only use left_context_size from additional information
|
||||
audio_tensors = self.code2wav.chunked_decode_streaming(
|
||||
talker_codes,
|
||||
chunk_size=25,
|
||||
left_context_size=25,
|
||||
left_context_size=left_context_size,
|
||||
seq_token_counts=seq_token_counts,
|
||||
)
|
||||
else:
|
||||
|
||||
@@ -213,8 +213,7 @@ class Qwen3OmniMoeCode2Wav(nn.Module):
|
||||
def chunked_decode_streaming(
|
||||
self,
|
||||
codes: torch.Tensor,
|
||||
chunk_size: int = 25,
|
||||
left_context_size: int = 25,
|
||||
left_context_size: list[int] | None = None,
|
||||
seq_token_counts: list[int] | None = None,
|
||||
) -> list[torch.Tensor]:
|
||||
"""
|
||||
@@ -222,9 +221,10 @@ class Qwen3OmniMoeCode2Wav(nn.Module):
|
||||
|
||||
Uses overlapping chunks with left context to avoid boundary artifacts.
|
||||
|
||||
No longer need chunk size here, which is different from chunked_decode
|
||||
|
||||
Args:
|
||||
codes: [batch, num_quantizers, seq_len] - num_quantizers-layer RVQ codes
|
||||
chunk_size: Number of codec frames per chunk
|
||||
left_context_size: Number of overlapping frames for context
|
||||
seq_token_counts: Token count for each request in batch
|
||||
|
||||
@@ -233,6 +233,13 @@ class Qwen3OmniMoeCode2Wav(nn.Module):
|
||||
codes. For ``batch_size == 1``, this is a list containing a
|
||||
single tensor with shape ``[1, waveform_len]``.
|
||||
"""
|
||||
if not (left_context_size and seq_token_counts and len(left_context_size) == len(seq_token_counts)):
|
||||
logger.warning(
|
||||
"chunked_decode_streaming: missing/invalid left_context_size or seq_token_counts; "
|
||||
"defaulting to left_context_size=zeros(len=codes.shape[0])."
|
||||
)
|
||||
left_context_size = [0] * codes.shape[0]
|
||||
# Decode chunk
|
||||
wavs = []
|
||||
batch_wav = self(codes)
|
||||
if seq_token_counts is not None:
|
||||
@@ -241,14 +248,10 @@ class Qwen3OmniMoeCode2Wav(nn.Module):
|
||||
# Fallback: assume all batch elements share the same sequence length.
|
||||
code_seq_lens = [codes.shape[-1]] * codes.shape[0]
|
||||
for idx, code_seq_len in enumerate(code_seq_lens):
|
||||
# TODO: need to optimize algorithms, current only support
|
||||
# chunk_size = left_context_size = 25
|
||||
if code_seq_len <= chunk_size:
|
||||
context_size = 0
|
||||
else:
|
||||
context_size = left_context_size
|
||||
# Remove context from output (context_size * total_upsample samples)
|
||||
wav_chunk = batch_wav[idx, :, context_size * self.total_upsample : code_seq_len * self.total_upsample]
|
||||
# Remove context from output (left_context_size * total_upsample samples)
|
||||
wav_chunk = batch_wav[
|
||||
idx, :, left_context_size[idx] * self.total_upsample : code_seq_len * self.total_upsample
|
||||
]
|
||||
wavs.append(wav_chunk)
|
||||
return wavs
|
||||
|
||||
|
||||
@@ -33,6 +33,7 @@ class Qwen3TTSCode2Wav(nn.Module):
|
||||
self.have_multimodal_outputs = True
|
||||
self.has_preprocess = False
|
||||
self.has_postprocess = False
|
||||
self.enable_update_additional_information = True
|
||||
# Generation-only stage (no logits / sampling).
|
||||
self.requires_raw_input_tokens = True
|
||||
|
||||
@@ -146,6 +147,7 @@ class Qwen3TTSCode2Wav(nn.Module):
|
||||
positions: torch.Tensor | None = None,
|
||||
intermediate_tensors: Any = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
runtime_additional_information: list[dict[str, Any]] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> OmniOutput:
|
||||
"""Decode codec codes into audio waveform.
|
||||
@@ -177,18 +179,28 @@ class Qwen3TTSCode2Wav(nn.Module):
|
||||
ids = input_ids.reshape(-1).to(dtype=torch.long)
|
||||
request_ids_list = self._split_request_ids(ids, kwargs.get("seq_token_counts"))
|
||||
|
||||
# Parse each request: extract ctx_frames, validate, reshape codes.
|
||||
# input_ids layout per request: [codec_context_frames, *flat_codes]
|
||||
# Parse each request: extract left_context_size, validate, reshape codes.
|
||||
# input_ids layout per request: [*flat_codes]
|
||||
# where flat_codes is codebook-major [q*F].
|
||||
parsed = [] # (ctx_frames, actual_frames)
|
||||
parsed = [] # (left_context_size, actual_frames)
|
||||
valid_codes = []
|
||||
valid_indices = []
|
||||
# Get left_context_size from runtime_additional_information (passed via kwargs)
|
||||
left_context_size = [0] * len(request_ids_list)
|
||||
if runtime_additional_information is not None:
|
||||
for i, info in enumerate(runtime_additional_information):
|
||||
if i >= len(left_context_size):
|
||||
break
|
||||
if "left_context_size" in info:
|
||||
left_context_size[i] = info["left_context_size"]
|
||||
else:
|
||||
logger.debug("No additional_information provided to code2wav stage.")
|
||||
for i, req_ids in enumerate(request_ids_list):
|
||||
if req_ids.numel() < 2:
|
||||
if req_ids.numel() < 1:
|
||||
parsed.append((0, 0))
|
||||
continue
|
||||
ctx_frames = int(req_ids[0].item())
|
||||
flat = req_ids[1:]
|
||||
ctx_frames = left_context_size[i]
|
||||
flat = req_ids
|
||||
n = flat.numel()
|
||||
# Warmup / dummy_run: not divisible by num_quantizers.
|
||||
if n == 0 or n % q != 0:
|
||||
|
||||
Reference in new issue
Block a user