Make chunk_size and left_context_size configurable via YAML for async chunking (#1423)

Signed-off-by: Junhong Liu <98734602+LJH-LBJ@users.noreply.github.com>
Signed-off-by: Junhong Liu <ljh_lbj@163.com>
This commit is contained in:
Junhong Liu authored and GitHub committed 2026-03-03 22:06:52 +08:00
1 parent b13761ed26
commit 742d845b8e
10 files changed
+113 -44

No files matched your search

@@ -5,6 +5,7 @@
from collections.abc import Iterable
from functools import cached_property
from typing import Any
import torch
import torch.nn as nn
@@ -160,6 +161,7 @@ class Qwen3OmniMoeForConditionalGeneration(
self.requires_raw_input_tokens = True
elif self.model_stage == "code2wav":
self.enable_update_additional_information = True
self.thinker = None
self.talker = None
# Initialize code2wav (codec codes → audio waveform)
@@ -254,7 +256,7 @@ class Qwen3OmniMoeForConditionalGeneration(
codec: torch.Tensor | None = None,
sampling_metadata: SamplingMetadata | None = None,
logits_index: int | None = None,
additional_information: dict[str, object] | None = None,
runtime_additional_information: list[dict[str, Any]] | None = None,
**kwargs: object,
) -> torch.Tensor | IntermediateTensors | OmniOutput:
"""
@@ -359,7 +361,15 @@ class Qwen3OmniMoeForConditionalGeneration(
codes = input_ids_flatten.reshape(1, 16, -1)
# Generate audio from codec codes
audio_tensors = self.generate_audio(codes, voice_type, seq_token_counts)
# Get every request's left_context_size from runtime_additional_information (passed via kwargs)
left_context_size = []
if runtime_additional_information is not None:
for info in runtime_additional_information:
if "left_context_size" in info:
left_context_size.append(info["left_context_size"])
else:
logger.debug("No additional_information provided to code2wav stage.")
audio_tensors = self.generate_audio(codes, voice_type, left_context_size, seq_token_counts)
return audio_tensors
@@ -440,6 +450,7 @@ class Qwen3OmniMoeForConditionalGeneration(
self,
code: torch.Tensor,
voice_type: str,
left_context_size: list[int] | None = None,
seq_token_counts: list[int] | None = None,
) -> list[torch.Tensor]:
"""
@@ -448,6 +459,7 @@ class Qwen3OmniMoeForConditionalGeneration(
Args:
code: [batch, num_quantizers, T] - RVQ codec codes
voice_type: Voice type (not used in Qwen3, kept for compatibility)
left_context_size: Left context size for streaming decode
seq_token_counts: Token count for each request in batch
Returns:
@@ -471,10 +483,10 @@ class Qwen3OmniMoeForConditionalGeneration(
talker_codes = talker_codes.expand(1, 16, -1)
if self.vllm_config.model_config.async_chunk:
# Only use left_context_size from additional information
audio_tensors = self.code2wav.chunked_decode_streaming(
talker_codes,
chunk_size=25,
left_context_size=25,
left_context_size=left_context_size,
seq_token_counts=seq_token_counts,
)
else:
@@ -213,8 +213,7 @@ class Qwen3OmniMoeCode2Wav(nn.Module):
def chunked_decode_streaming(
self,
codes: torch.Tensor,
chunk_size: int = 25,
left_context_size: int = 25,
left_context_size: list[int] | None = None,
seq_token_counts: list[int] | None = None,
) -> list[torch.Tensor]:
"""
@@ -222,9 +221,10 @@ class Qwen3OmniMoeCode2Wav(nn.Module):
Uses overlapping chunks with left context to avoid boundary artifacts.
No longer need chunk size here, which is different from chunked_decode
Args:
codes: [batch, num_quantizers, seq_len] - num_quantizers-layer RVQ codes
chunk_size: Number of codec frames per chunk
left_context_size: Number of overlapping frames for context
seq_token_counts: Token count for each request in batch
@@ -233,6 +233,13 @@ class Qwen3OmniMoeCode2Wav(nn.Module):
codes. For ``batch_size == 1``, this is a list containing a
single tensor with shape ``[1, waveform_len]``.
"""
if not (left_context_size and seq_token_counts and len(left_context_size) == len(seq_token_counts)):
logger.warning(
"chunked_decode_streaming: missing/invalid left_context_size or seq_token_counts; "
"defaulting to left_context_size=zeros(len=codes.shape[0])."
)
left_context_size = [0] * codes.shape[0]
# Decode chunk
wavs = []
batch_wav = self(codes)
if seq_token_counts is not None:
@@ -241,14 +248,10 @@ class Qwen3OmniMoeCode2Wav(nn.Module):
# Fallback: assume all batch elements share the same sequence length.
code_seq_lens = [codes.shape[-1]] * codes.shape[0]
for idx, code_seq_len in enumerate(code_seq_lens):
# TODO: need to optimize algorithms, current only support
# chunk_size = left_context_size = 25
if code_seq_len <= chunk_size:
context_size = 0
else:
context_size = left_context_size
# Remove context from output (context_size * total_upsample samples)
wav_chunk = batch_wav[idx, :, context_size * self.total_upsample : code_seq_len * self.total_upsample]
# Remove context from output (left_context_size * total_upsample samples)
wav_chunk = batch_wav[
idx, :, left_context_size[idx] * self.total_upsample : code_seq_len * self.total_upsample
]
wavs.append(wav_chunk)
return wavs
@@ -33,6 +33,7 @@ class Qwen3TTSCode2Wav(nn.Module):
self.have_multimodal_outputs = True
self.has_preprocess = False
self.has_postprocess = False
self.enable_update_additional_information = True
# Generation-only stage (no logits / sampling).
self.requires_raw_input_tokens = True
@@ -146,6 +147,7 @@ class Qwen3TTSCode2Wav(nn.Module):
positions: torch.Tensor | None = None,
intermediate_tensors: Any = None,
inputs_embeds: torch.Tensor | None = None,
runtime_additional_information: list[dict[str, Any]] | None = None,
**kwargs: Any,
) -> OmniOutput:
"""Decode codec codes into audio waveform.
@@ -177,18 +179,28 @@ class Qwen3TTSCode2Wav(nn.Module):
ids = input_ids.reshape(-1).to(dtype=torch.long)
request_ids_list = self._split_request_ids(ids, kwargs.get("seq_token_counts"))
# Parse each request: extract ctx_frames, validate, reshape codes.
# input_ids layout per request: [codec_context_frames, *flat_codes]
# Parse each request: extract left_context_size, validate, reshape codes.
# input_ids layout per request: [*flat_codes]
# where flat_codes is codebook-major [q*F].
parsed = [] # (ctx_frames, actual_frames)
parsed = [] # (left_context_size, actual_frames)
valid_codes = []
valid_indices = []
# Get left_context_size from runtime_additional_information (passed via kwargs)
left_context_size = [0] * len(request_ids_list)
if runtime_additional_information is not None:
for i, info in enumerate(runtime_additional_information):
if i >= len(left_context_size):
break
if "left_context_size" in info:
left_context_size[i] = info["left_context_size"]
else:
logger.debug("No additional_information provided to code2wav stage.")
for i, req_ids in enumerate(request_ids_list):
if req_ids.numel() < 2:
if req_ids.numel() < 1:
parsed.append((0, 0))
continue
ctx_frames = int(req_ids[0].item())
flat = req_ids[1:]
ctx_frames = left_context_size[i]
flat = req_ids
n = flat.numel()
# Warmup / dummy_run: not divisible by num_quantizers.
if n == 0 or n % q != 0: