[Feat]audio streaming input for async chunk (#3614)

Signed-off-by: CHEN <116010019@link.cuhk.edu.cn>
This commit is contained in:
ChenWenjing authored and GitHub committed 2026-05-31 17:39:25 +08:00
1 parent 6f4bd3e45c
commit 5dfdf584dd
20 files changed
+536 -184

No files matched your search

@@ -183,11 +183,6 @@ class Qwen3OmniMoeForConditionalGeneration(
# sync stall; build_mm_cpu handles the eventual D2H at payload time.
("codes", "audio"),
}
# Keys that need to be accumulated across streaming inputs
self.streaming_accumulated_keys: set[tuple[str, str]] = {
("embed", "prefill"),
("hidden_states", "output"),
}
elif self.model_stage == "code2wav":
multimodal_config.skip_mm_profiling = True
@@ -685,7 +680,8 @@ class Qwen3OmniMoeForConditionalGeneration(
is_prefill = bool(payload.get("_omni_is_prefill", span_len > 1))
if is_prefill:
num_computed_tokens = payload.get("_omni_num_computed_tokens")
if num_computed_tokens is not None:
request_resumable = meta.get("resumable", False)
if num_computed_tokens is not None and not request_resumable:
meta["num_processed_tokens"] = int(num_computed_tokens)
input_ids, input_embeds, update_dict = self.talker_preprocess_prefill(input_ids, input_embeds, payload)
code_predictor_codes = torch.zeros(
@@ -893,12 +893,12 @@ class Qwen3TTSPromptEmbedsBuilder:
* ``talker_prompt``: ``[prompt_len, hidden]`` prefill embedding.
* ``trailing_text_hidden``: ``[T, hidden]`` queue of text-side
embeddings consumed one-per-decode-step (streaming mode) or a
single pad row (non-streaming).
embeddings consumed one-per-decode-step (streaming mode) or a
single pad row (non-streaming).
* ``ref_code_len``: Number of ref codec frames consumed when the
Base task ran in in-context mode (``None`` otherwise).
Base task ran in in-context mode (``None`` otherwise).
* ``ref_code``: The ``[T, Q]`` ref codec tensor used for ICL
(``None`` outside Base / ICL).
(``None`` outside Base / ICL).
"""
config = self._config
talker_config = self._talker_config
@@ -261,6 +261,89 @@ def _resolve_tts_token_embedding(
# =========================
def _construct_thinker2talker_streaming_input_async_chunk(
is_finished: bool,
request,
thinker_emb,
thinker_hid,
transfer_manager,
) -> OmniPayloadStruct | None:
"""Build Thinker -> Talker payloads for realtime streaming input chunks.
A resumable realtime request reuses the same logical request id across
audio segments. The first streaming prefill chunk is cached and returns ``None`` so the
connector does not emit an incomplete downstream chunk. The following
decode chunk flushes that cached prefill together with the current Thinker
output, keeping Talker ids and tensor rows aligned.
"""
request_id = request.external_req_id
output_token_ids = request.output_token_ids
# Convert ConstantList to regular list for OmniSerializer serialization
output_token_ids = _ensure_list(output_token_ids)
speaker = extract_speaker_from_request(request)
language = extract_language_from_request(request)
finished = torch.tensor(is_finished, dtype=torch.bool)
emb_cpu = thinker_emb.detach().cpu()
hid_cpu = thinker_hid.detach().cpu()
if output_token_ids:
if thinker_emb.shape[0] > 1:
# if thinker_emb.shape[0] > 1, new streaming input segment is added
# and will transfer prefill embeddings and hidden states to talker.
new_prompt_len = thinker_emb.shape[0]
payload = OmniPayloadStruct(
meta=MetaStruct(finished=finished),
embed=EmbeddingsStruct(prefill=emb_cpu),
hidden_states=HiddenStatesStruct(output=hid_cpu),
ids=IdsStruct(
all=_ensure_list(request.all_token_ids[-new_prompt_len - 1 :]),
prompt=_ensure_list(request.prompt_token_ids[-new_prompt_len:]),
),
speaker=speaker,
language=language,
)
transfer_manager._pending_streaming_prefills[request_id] = to_dict(payload)
return None
else:
save_payload = transfer_manager._pending_streaming_prefills.pop(request_id, None)
if save_payload is not None:
saved_prefill = save_payload.get("embed", {}).get("prefill")
saved_output = save_payload.get("hidden_states", {}).get("output")
if isinstance(saved_prefill, torch.Tensor) and isinstance(saved_output, torch.Tensor):
return OmniPayloadStruct(
meta=MetaStruct(finished=finished),
embed=EmbeddingsStruct(prefill=torch.cat((saved_prefill, emb_cpu), dim=0)),
hidden_states=HiddenStatesStruct(output=torch.cat((saved_output, hid_cpu), dim=0)),
ids=IdsStruct(
all=save_payload.get("ids", {}).get("all"),
prompt=save_payload.get("ids", {}).get("prompt"),
),
speaker=speaker,
language=language,
)
return OmniPayloadStruct(
meta=MetaStruct(
finished=finished,
),
embed=EmbeddingsStruct(decode=emb_cpu),
hidden_states=HiddenStatesStruct(output=hid_cpu),
ids=IdsStruct(output=output_token_ids),
speaker=speaker,
language=language,
)
else:
if not is_finished:
# do not send async chunk mode placeholder token or embedding/hidden of the stop token
return None
return OmniPayloadStruct(
meta=MetaStruct(finished=finished),
embed=EmbeddingsStruct(decode=emb_cpu),
hidden_states=HiddenStatesStruct(output=hid_cpu),
speaker=speaker,
language=language,
)
@dataclass
class _Thinker2TalkerStreamingState:
last_prompt_len: int = 0
@@ -295,16 +378,16 @@ def _get_streaming_talker_tokens(
streaming_context: Any | None = None,
*,
clear_state: bool = False,
) -> tuple[list[int], list[int], list[int], list[int]]:
"""Return streaming token slices and merged token views for thinker->talker.
e.g. For the second streaming input request:
merged_sequences: [input_prompt 1, output_tokens 1[:-1], input_prompt 2, output_tokens 2]
thinker_input_ids: [input_prompt 1, output_tokens 1[:-1], input_prompt 2]
) -> tuple[list[int], list[int]]:
"""Return prompt/output token deltas for the current streaming segment.
In non-async-chunk streaming, Thinker's prompt may already include the
next input segment. Remove that new prompt tail before building the Talker
delta for the previous segment.
Returns:
inc_prompt: prompt token delta for this segment.
inc_output: output token delta for this segment.
merged_sequences: full thinker_sequences to send downstream.
thinker_input_ids: full thinker_input_ids paired with merged_sequences.
"""
state = _get_qwen3_streaming_state(request_id, streaming_context).thinker2talker
if new_prompt_len_snapshot:
@@ -314,15 +397,6 @@ def _get_streaming_talker_tokens(
inc_prompt = prompt_token_ids[state.last_prompt_len :]
inc_output = output_token_ids[state.last_output_len :]
delta_sequences = inc_prompt + inc_output
cached_sequences = state.merged_sequences
merged_sequences = cached_sequences + delta_sequences
thinker_input_ids = cached_sequences + inc_prompt
# Persist history for next segment. Drop the latest sampled token to keep
# thinker_input_ids / thinker_sequences alignment with next-step append.
cached_sequences.extend(delta_sequences[:-1])
state.last_prompt_len = cur_prompt_len
state.last_output_len = cur_output_len
@@ -332,7 +406,7 @@ def _get_streaming_talker_tokens(
state.last_output_len = 0
state.merged_sequences.clear()
return inc_prompt, inc_output, merged_sequences, thinker_input_ids
return inc_prompt, inc_output
def _get_streaming_codec_delta_len(
@@ -429,6 +503,10 @@ def thinker2talker_async_chunk(
transfer_manager.request_payload[request_id] = to_dict(payload)
return None
else:
if request.resumable:
return _construct_thinker2talker_streaming_input_async_chunk(
is_finished, request, thinker_emb, thinker_hid, transfer_manager
)
if thinker_emb.shape[0] > 1:
logger.warning(
"Unexpected multiple embeddings in thinker2talker_async_chunk for chunk_id %d: "
@@ -567,7 +645,7 @@ def thinker2talker(
output_ids = _ensure_list(output.cumulative_token_ids)
is_streaming_session = bool(getattr(streaming_context, "enabled", False))
if is_streaming_session:
prompt_token_ids, output_ids, thinker_sequences, thinker_input_ids = _get_streaming_talker_tokens(
prompt_token_ids, output_ids = _get_streaming_talker_tokens(
req_id,
prompt_token_ids,
output_ids,
@@ -575,9 +653,8 @@ def thinker2talker(
streaming_context,
clear_state=bool(getattr(thinker_output, "finished", False)),
)
else:
thinker_sequences = prompt_token_ids + output_ids
thinker_input_ids = prompt_token_ids
thinker_sequences = prompt_token_ids + output_ids
thinker_input_ids = prompt_token_ids
new_seq_length = len(prompt_token_ids + output_ids) - 1
thinker_mm_raw = getattr(output, "multimodal_output", None)
if not isinstance(thinker_mm_raw, dict):
@@ -683,7 +760,7 @@ def thinker2talker_token_only(
output_ids = _ensure_list(output.cumulative_token_ids)
is_streaming_session = bool(getattr(streaming_context, "enabled", False))
if is_streaming_session:
prompt_token_ids, output_ids, thinker_sequences, thinker_input_ids = _get_streaming_talker_tokens(
prompt_token_ids, output_ids = _get_streaming_talker_tokens(
req_id,
prompt_token_ids,
output_ids,
@@ -691,9 +768,8 @@ def thinker2talker_token_only(
streaming_context,
clear_state=bool(getattr(thinker_output, "finished", False)),
)
else:
thinker_sequences = prompt_token_ids + output_ids
thinker_input_ids = prompt_token_ids
thinker_sequences = prompt_token_ids + output_ids
thinker_input_ids = prompt_token_ids
info_for_len = {"ids": {"all": thinker_sequences, "prompt": thinker_input_ids}}
prompt_len = _compute_talker_prompt_ids_length(info_for_len, device="cpu")
@@ -759,6 +835,15 @@ def talker2code2wav_async_chunk(
if isinstance(code_predictor_codes, torch.Tensor):
if not code_predictor_codes.any():
return None
sampling_params = getattr(request, "sampling_params", None)
stop_token_ids = set(getattr(sampling_params, "stop_token_ids", None) or [])
stop_token_id = getattr(sampling_params, "stop_token_id", None)
if stop_token_id is not None:
stop_token_ids.add(stop_token_id)
first_codebook = int(code_predictor_codes[0, 0].item())
if first_codebook in stop_token_ids:
logger.debug("skip stop-token codec frame: first_codebook=%s", first_codebook)
return None
else:
code_tensor = torch.tensor(code_predictor_codes, dtype=torch.long)
if not code_tensor.any():