[Feat]audio streaming input for async chunk (#3614)
Signed-off-by: CHEN <116010019@link.cuhk.edu.cn>
This commit is contained in:
1 parent
6f4bd3e45c
commit
5dfdf584dd
20 files changed
+536
-184
No files matched your search
@@ -183,11 +183,6 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
# sync stall; build_mm_cpu handles the eventual D2H at payload time.
|
||||
("codes", "audio"),
|
||||
}
|
||||
# Keys that need to be accumulated across streaming inputs
|
||||
self.streaming_accumulated_keys: set[tuple[str, str]] = {
|
||||
("embed", "prefill"),
|
||||
("hidden_states", "output"),
|
||||
}
|
||||
|
||||
elif self.model_stage == "code2wav":
|
||||
multimodal_config.skip_mm_profiling = True
|
||||
@@ -685,7 +680,8 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
is_prefill = bool(payload.get("_omni_is_prefill", span_len > 1))
|
||||
if is_prefill:
|
||||
num_computed_tokens = payload.get("_omni_num_computed_tokens")
|
||||
if num_computed_tokens is not None:
|
||||
request_resumable = meta.get("resumable", False)
|
||||
if num_computed_tokens is not None and not request_resumable:
|
||||
meta["num_processed_tokens"] = int(num_computed_tokens)
|
||||
input_ids, input_embeds, update_dict = self.talker_preprocess_prefill(input_ids, input_embeds, payload)
|
||||
code_predictor_codes = torch.zeros(
|
||||
|
||||
@@ -893,12 +893,12 @@ class Qwen3TTSPromptEmbedsBuilder:
|
||||
|
||||
* ``talker_prompt``: ``[prompt_len, hidden]`` prefill embedding.
|
||||
* ``trailing_text_hidden``: ``[T, hidden]`` queue of text-side
|
||||
embeddings consumed one-per-decode-step (streaming mode) or a
|
||||
single pad row (non-streaming).
|
||||
embeddings consumed one-per-decode-step (streaming mode) or a
|
||||
single pad row (non-streaming).
|
||||
* ``ref_code_len``: Number of ref codec frames consumed when the
|
||||
Base task ran in in-context mode (``None`` otherwise).
|
||||
Base task ran in in-context mode (``None`` otherwise).
|
||||
* ``ref_code``: The ``[T, Q]`` ref codec tensor used for ICL
|
||||
(``None`` outside Base / ICL).
|
||||
(``None`` outside Base / ICL).
|
||||
"""
|
||||
config = self._config
|
||||
talker_config = self._talker_config
|
||||
|
||||
@@ -261,6 +261,89 @@ def _resolve_tts_token_embedding(
|
||||
# =========================
|
||||
|
||||
|
||||
def _construct_thinker2talker_streaming_input_async_chunk(
|
||||
is_finished: bool,
|
||||
request,
|
||||
thinker_emb,
|
||||
thinker_hid,
|
||||
transfer_manager,
|
||||
) -> OmniPayloadStruct | None:
|
||||
"""Build Thinker -> Talker payloads for realtime streaming input chunks.
|
||||
|
||||
A resumable realtime request reuses the same logical request id across
|
||||
audio segments. The first streaming prefill chunk is cached and returns ``None`` so the
|
||||
connector does not emit an incomplete downstream chunk. The following
|
||||
decode chunk flushes that cached prefill together with the current Thinker
|
||||
output, keeping Talker ids and tensor rows aligned.
|
||||
"""
|
||||
request_id = request.external_req_id
|
||||
output_token_ids = request.output_token_ids
|
||||
# Convert ConstantList to regular list for OmniSerializer serialization
|
||||
output_token_ids = _ensure_list(output_token_ids)
|
||||
speaker = extract_speaker_from_request(request)
|
||||
language = extract_language_from_request(request)
|
||||
finished = torch.tensor(is_finished, dtype=torch.bool)
|
||||
emb_cpu = thinker_emb.detach().cpu()
|
||||
hid_cpu = thinker_hid.detach().cpu()
|
||||
|
||||
if output_token_ids:
|
||||
if thinker_emb.shape[0] > 1:
|
||||
# if thinker_emb.shape[0] > 1, new streaming input segment is added
|
||||
# and will transfer prefill embeddings and hidden states to talker.
|
||||
new_prompt_len = thinker_emb.shape[0]
|
||||
payload = OmniPayloadStruct(
|
||||
meta=MetaStruct(finished=finished),
|
||||
embed=EmbeddingsStruct(prefill=emb_cpu),
|
||||
hidden_states=HiddenStatesStruct(output=hid_cpu),
|
||||
ids=IdsStruct(
|
||||
all=_ensure_list(request.all_token_ids[-new_prompt_len - 1 :]),
|
||||
prompt=_ensure_list(request.prompt_token_ids[-new_prompt_len:]),
|
||||
),
|
||||
speaker=speaker,
|
||||
language=language,
|
||||
)
|
||||
transfer_manager._pending_streaming_prefills[request_id] = to_dict(payload)
|
||||
return None
|
||||
else:
|
||||
save_payload = transfer_manager._pending_streaming_prefills.pop(request_id, None)
|
||||
if save_payload is not None:
|
||||
saved_prefill = save_payload.get("embed", {}).get("prefill")
|
||||
saved_output = save_payload.get("hidden_states", {}).get("output")
|
||||
if isinstance(saved_prefill, torch.Tensor) and isinstance(saved_output, torch.Tensor):
|
||||
return OmniPayloadStruct(
|
||||
meta=MetaStruct(finished=finished),
|
||||
embed=EmbeddingsStruct(prefill=torch.cat((saved_prefill, emb_cpu), dim=0)),
|
||||
hidden_states=HiddenStatesStruct(output=torch.cat((saved_output, hid_cpu), dim=0)),
|
||||
ids=IdsStruct(
|
||||
all=save_payload.get("ids", {}).get("all"),
|
||||
prompt=save_payload.get("ids", {}).get("prompt"),
|
||||
),
|
||||
speaker=speaker,
|
||||
language=language,
|
||||
)
|
||||
return OmniPayloadStruct(
|
||||
meta=MetaStruct(
|
||||
finished=finished,
|
||||
),
|
||||
embed=EmbeddingsStruct(decode=emb_cpu),
|
||||
hidden_states=HiddenStatesStruct(output=hid_cpu),
|
||||
ids=IdsStruct(output=output_token_ids),
|
||||
speaker=speaker,
|
||||
language=language,
|
||||
)
|
||||
else:
|
||||
if not is_finished:
|
||||
# do not send async chunk mode placeholder token or embedding/hidden of the stop token
|
||||
return None
|
||||
return OmniPayloadStruct(
|
||||
meta=MetaStruct(finished=finished),
|
||||
embed=EmbeddingsStruct(decode=emb_cpu),
|
||||
hidden_states=HiddenStatesStruct(output=hid_cpu),
|
||||
speaker=speaker,
|
||||
language=language,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Thinker2TalkerStreamingState:
|
||||
last_prompt_len: int = 0
|
||||
@@ -295,16 +378,16 @@ def _get_streaming_talker_tokens(
|
||||
streaming_context: Any | None = None,
|
||||
*,
|
||||
clear_state: bool = False,
|
||||
) -> tuple[list[int], list[int], list[int], list[int]]:
|
||||
"""Return streaming token slices and merged token views for thinker->talker.
|
||||
e.g. For the second streaming input request:
|
||||
merged_sequences: [input_prompt 1, output_tokens 1[:-1], input_prompt 2, output_tokens 2]
|
||||
thinker_input_ids: [input_prompt 1, output_tokens 1[:-1], input_prompt 2]
|
||||
) -> tuple[list[int], list[int]]:
|
||||
"""Return prompt/output token deltas for the current streaming segment.
|
||||
|
||||
In non-async-chunk streaming, Thinker's prompt may already include the
|
||||
next input segment. Remove that new prompt tail before building the Talker
|
||||
delta for the previous segment.
|
||||
|
||||
Returns:
|
||||
inc_prompt: prompt token delta for this segment.
|
||||
inc_output: output token delta for this segment.
|
||||
merged_sequences: full thinker_sequences to send downstream.
|
||||
thinker_input_ids: full thinker_input_ids paired with merged_sequences.
|
||||
"""
|
||||
state = _get_qwen3_streaming_state(request_id, streaming_context).thinker2talker
|
||||
if new_prompt_len_snapshot:
|
||||
@@ -314,15 +397,6 @@ def _get_streaming_talker_tokens(
|
||||
|
||||
inc_prompt = prompt_token_ids[state.last_prompt_len :]
|
||||
inc_output = output_token_ids[state.last_output_len :]
|
||||
delta_sequences = inc_prompt + inc_output
|
||||
cached_sequences = state.merged_sequences
|
||||
|
||||
merged_sequences = cached_sequences + delta_sequences
|
||||
thinker_input_ids = cached_sequences + inc_prompt
|
||||
|
||||
# Persist history for next segment. Drop the latest sampled token to keep
|
||||
# thinker_input_ids / thinker_sequences alignment with next-step append.
|
||||
cached_sequences.extend(delta_sequences[:-1])
|
||||
|
||||
state.last_prompt_len = cur_prompt_len
|
||||
state.last_output_len = cur_output_len
|
||||
@@ -332,7 +406,7 @@ def _get_streaming_talker_tokens(
|
||||
state.last_output_len = 0
|
||||
state.merged_sequences.clear()
|
||||
|
||||
return inc_prompt, inc_output, merged_sequences, thinker_input_ids
|
||||
return inc_prompt, inc_output
|
||||
|
||||
|
||||
def _get_streaming_codec_delta_len(
|
||||
@@ -429,6 +503,10 @@ def thinker2talker_async_chunk(
|
||||
transfer_manager.request_payload[request_id] = to_dict(payload)
|
||||
return None
|
||||
else:
|
||||
if request.resumable:
|
||||
return _construct_thinker2talker_streaming_input_async_chunk(
|
||||
is_finished, request, thinker_emb, thinker_hid, transfer_manager
|
||||
)
|
||||
if thinker_emb.shape[0] > 1:
|
||||
logger.warning(
|
||||
"Unexpected multiple embeddings in thinker2talker_async_chunk for chunk_id %d: "
|
||||
@@ -567,7 +645,7 @@ def thinker2talker(
|
||||
output_ids = _ensure_list(output.cumulative_token_ids)
|
||||
is_streaming_session = bool(getattr(streaming_context, "enabled", False))
|
||||
if is_streaming_session:
|
||||
prompt_token_ids, output_ids, thinker_sequences, thinker_input_ids = _get_streaming_talker_tokens(
|
||||
prompt_token_ids, output_ids = _get_streaming_talker_tokens(
|
||||
req_id,
|
||||
prompt_token_ids,
|
||||
output_ids,
|
||||
@@ -575,9 +653,8 @@ def thinker2talker(
|
||||
streaming_context,
|
||||
clear_state=bool(getattr(thinker_output, "finished", False)),
|
||||
)
|
||||
else:
|
||||
thinker_sequences = prompt_token_ids + output_ids
|
||||
thinker_input_ids = prompt_token_ids
|
||||
thinker_sequences = prompt_token_ids + output_ids
|
||||
thinker_input_ids = prompt_token_ids
|
||||
new_seq_length = len(prompt_token_ids + output_ids) - 1
|
||||
thinker_mm_raw = getattr(output, "multimodal_output", None)
|
||||
if not isinstance(thinker_mm_raw, dict):
|
||||
@@ -683,7 +760,7 @@ def thinker2talker_token_only(
|
||||
output_ids = _ensure_list(output.cumulative_token_ids)
|
||||
is_streaming_session = bool(getattr(streaming_context, "enabled", False))
|
||||
if is_streaming_session:
|
||||
prompt_token_ids, output_ids, thinker_sequences, thinker_input_ids = _get_streaming_talker_tokens(
|
||||
prompt_token_ids, output_ids = _get_streaming_talker_tokens(
|
||||
req_id,
|
||||
prompt_token_ids,
|
||||
output_ids,
|
||||
@@ -691,9 +768,8 @@ def thinker2talker_token_only(
|
||||
streaming_context,
|
||||
clear_state=bool(getattr(thinker_output, "finished", False)),
|
||||
)
|
||||
else:
|
||||
thinker_sequences = prompt_token_ids + output_ids
|
||||
thinker_input_ids = prompt_token_ids
|
||||
thinker_sequences = prompt_token_ids + output_ids
|
||||
thinker_input_ids = prompt_token_ids
|
||||
info_for_len = {"ids": {"all": thinker_sequences, "prompt": thinker_input_ids}}
|
||||
prompt_len = _compute_talker_prompt_ids_length(info_for_len, device="cpu")
|
||||
|
||||
@@ -759,6 +835,15 @@ def talker2code2wav_async_chunk(
|
||||
if isinstance(code_predictor_codes, torch.Tensor):
|
||||
if not code_predictor_codes.any():
|
||||
return None
|
||||
sampling_params = getattr(request, "sampling_params", None)
|
||||
stop_token_ids = set(getattr(sampling_params, "stop_token_ids", None) or [])
|
||||
stop_token_id = getattr(sampling_params, "stop_token_id", None)
|
||||
if stop_token_id is not None:
|
||||
stop_token_ids.add(stop_token_id)
|
||||
first_codebook = int(code_predictor_codes[0, 0].item())
|
||||
if first_codebook in stop_token_ids:
|
||||
logger.debug("skip stop-token codec frame: first_codebook=%s", first_codebook)
|
||||
return None
|
||||
else:
|
||||
code_tensor = torch.tensor(code_predictor_codes, dtype=torch.long)
|
||||
if not code_tensor.any():
|
||||
|
||||
Reference in new issue
Block a user