Signed-off-by: Megha Agarwal <agarwalmegha1308@gmail.com> Signed-off-by: meghaagr13 <divyanshsinghvi@gmail.com> Signed-off-by: Taichang Zhou <tzhouam@connect.ust.hk> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: meghaagr13 <divyanshsinghvi@gmail.com> Co-authored-by: Boao Shi <aoibosh@connect.hku.hk> Co-authored-by: Taichang Zhou <tzhouam@connect.ust.hk> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
49 files changed
+985
-327
No files matched your search
@@ -1,4 +1,4 @@
|
||||
from collections.abc import Iterable
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import replace
|
||||
from functools import cached_property
|
||||
from pathlib import Path
|
||||
@@ -220,7 +220,7 @@ class VoxtralTTSForConditionalGeneration(
|
||||
|
||||
def tts_postprocess(self, hidden_states: torch.Tensor, multimodal_outputs: object, **info_dict: object | None):
|
||||
update_dict = {}
|
||||
if isinstance(multimodal_outputs, dict) and "audio" in multimodal_outputs:
|
||||
if isinstance(multimodal_outputs, Mapping) and "audio" in multimodal_outputs:
|
||||
assert self.post_process_idx < len(multimodal_outputs["audio"]), (
|
||||
f"Expect {self.post_process_idx=} < {len(multimodal_outputs['audio'])=}"
|
||||
)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections import defaultdict
|
||||
from collections.abc import Mapping
|
||||
from contextlib import nullcontext
|
||||
from typing import Any
|
||||
|
||||
@@ -162,7 +163,7 @@ def text2flow(
|
||||
|
||||
def talker2code2wav_async_chunk(
|
||||
transfer_manager: Any,
|
||||
pooling_output: dict[str, Any] | None,
|
||||
multimodal_output: dict[str, Any] | None,
|
||||
request: Any,
|
||||
is_finished: bool = False,
|
||||
) -> OmniPayloadStruct | None:
|
||||
@@ -198,12 +199,14 @@ def talker2code2wav_async_chunk(
|
||||
value = _to_cpu_tensor(info_embed.get(key))
|
||||
if value is not None:
|
||||
prompt_payload[key] = value
|
||||
if isinstance(pooling_output, dict):
|
||||
po_embed = pooling_output.get("embed", {}) if isinstance(pooling_output.get("embed"), dict) else {}
|
||||
if isinstance(multimodal_output, Mapping):
|
||||
mm_embed = multimodal_output.get("embed", {})
|
||||
if not isinstance(mm_embed, Mapping):
|
||||
mm_embed = multimodal_output
|
||||
for key in cond_keys:
|
||||
if key in prompt_payload:
|
||||
continue
|
||||
value = _to_cpu_tensor(po_embed.get(key))
|
||||
value = _to_cpu_tensor(mm_embed.get(key))
|
||||
if value is not None:
|
||||
prompt_payload[key] = value
|
||||
# Drop any right-padding carried from batched talker emission so
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Stage input processor for Fish Speech S2 Pro: Slow AR → DAC Decoder."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -91,14 +92,14 @@ def _select_backlog_chunk_size(
|
||||
return backlog_chunk_size
|
||||
|
||||
|
||||
def _extract_last_frame(pooling_output: dict[str, Any]) -> torch.Tensor | None:
|
||||
"""Extract the last frame of audio codes from the pooling output."""
|
||||
audio_codes = pooling_output.get("audio_codes")
|
||||
def _extract_last_frame(multimodal_output: dict[str, Any]) -> torch.Tensor | None:
|
||||
"""Extract the last frame of audio codes from the multimodal output."""
|
||||
audio_codes = multimodal_output.get("audio_codes")
|
||||
if not isinstance(audio_codes, torch.Tensor) or audio_codes.numel() == 0:
|
||||
return None
|
||||
if audio_codes.ndim == 2:
|
||||
frame = audio_codes[-1]
|
||||
valid = pooling_output.get("audio_code_valid")
|
||||
valid = multimodal_output.get("audio_code_valid")
|
||||
if isinstance(valid, torch.Tensor) and valid.numel() > 0:
|
||||
is_valid = bool(valid.reshape(-1)[-1].item())
|
||||
elif valid is not None:
|
||||
@@ -146,7 +147,7 @@ def slow_ar_to_dac_decoder(
|
||||
|
||||
def slow_ar_to_dac_decoder_async_chunk(
|
||||
transfer_manager: Any,
|
||||
pooling_output: dict[str, Any] | None,
|
||||
multimodal_output: dict[str, Any] | None,
|
||||
request: Any,
|
||||
is_finished: bool = False,
|
||||
) -> OmniPayloadStruct | None:
|
||||
@@ -160,8 +161,8 @@ def slow_ar_to_dac_decoder_async_chunk(
|
||||
finished = bool(is_finished or request.is_finished())
|
||||
cfg = _get_connector_extra(transfer_manager)
|
||||
|
||||
if isinstance(pooling_output, dict):
|
||||
frame = _extract_last_frame(pooling_output)
|
||||
if isinstance(multimodal_output, Mapping):
|
||||
frame = _extract_last_frame(multimodal_output)
|
||||
if frame is not None:
|
||||
transfer_manager.code_prompt_token_ids[request_id].append(frame.detach())
|
||||
elif not finished:
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
import math
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -23,7 +24,7 @@ def _has_source_image(mm_data: Any) -> bool:
|
||||
- `img2img`: legacy single-image key
|
||||
- `images`: list or single image
|
||||
"""
|
||||
if not isinstance(mm_data, dict):
|
||||
if not isinstance(mm_data, Mapping):
|
||||
return False
|
||||
if mm_data.get("image") is not None:
|
||||
return True
|
||||
@@ -35,7 +36,7 @@ def _has_source_image(mm_data: Any) -> bool:
|
||||
|
||||
def _first_source_image(mm_data: Any) -> Any:
|
||||
"""Get first source image from normalized multimodal keys."""
|
||||
if not isinstance(mm_data, dict):
|
||||
if not isinstance(mm_data, Mapping):
|
||||
return None
|
||||
|
||||
image = mm_data.get("image")
|
||||
@@ -285,7 +286,7 @@ def ar2diffusion(
|
||||
|
||||
if hasattr(ar_output, "multimodal_output") and ar_output.multimodal_output:
|
||||
mm_output = ar_output.multimodal_output
|
||||
if isinstance(mm_output, dict) and mm_output.get("ids", {}).get("prior_image") is not None:
|
||||
if isinstance(mm_output, Mapping) and mm_output.get("ids", {}).get("prior_image") is not None:
|
||||
is_i2i = True
|
||||
_dt_mode = (time.perf_counter() - _t_mode) * 1000
|
||||
|
||||
@@ -322,7 +323,7 @@ def ar2diffusion(
|
||||
# Check ar_output (RequestOutput) for multimodal_output - this is the correct location
|
||||
if hasattr(ar_output, "multimodal_output") and ar_output.multimodal_output:
|
||||
mm_output = ar_output.multimodal_output
|
||||
if isinstance(mm_output, dict):
|
||||
if isinstance(mm_output, Mapping):
|
||||
raw_prior_image_ids = mm_output.get("ids", {}).get("prior_image")
|
||||
if raw_prior_image_ids is not None:
|
||||
# Handle different formats:
|
||||
@@ -348,7 +349,7 @@ def ar2diffusion(
|
||||
if hasattr(output, "multimodal_output") and output.multimodal_output:
|
||||
mm_output = output.multimodal_output
|
||||
logger.debug(f"[ar2diffusion] Request {i}: found multimodal_output on CompletionOutput (fallback)")
|
||||
if isinstance(mm_output, dict):
|
||||
if isinstance(mm_output, Mapping):
|
||||
raw_prior_image_ids = mm_output.get("ids", {}).get("prior_image")
|
||||
if raw_prior_image_ids is not None:
|
||||
if isinstance(raw_prior_image_ids, torch.Tensor):
|
||||
|
||||
@@ -188,7 +188,7 @@ def _to_cpu_tensor(value: Any) -> torch.Tensor | None:
|
||||
|
||||
def ar_to_dit_async_chunk(
|
||||
transfer_manager: Any,
|
||||
pooling_output: dict[str, Any] | None,
|
||||
multimodal_output: dict[str, Any] | None,
|
||||
request: Any,
|
||||
is_finished: bool = False,
|
||||
) -> OmniPayloadStruct | None:
|
||||
@@ -216,6 +216,7 @@ def ar_to_dit_async_chunk(
|
||||
if request_id is None:
|
||||
raise ValueError("GLM-TTS async chunk request is missing request id")
|
||||
finished = bool(is_finished or request.is_finished())
|
||||
pooling_output = multimodal_output
|
||||
|
||||
# Read connector chunk config (supports progressive list or single int)
|
||||
connector = getattr(transfer_manager, "connector", None)
|
||||
|
||||
@@ -176,7 +176,7 @@ def _extract_last_frame(pooling_output: OmniPayload) -> torch.Tensor | None:
|
||||
|
||||
def talker2code2wav_async_chunk(
|
||||
transfer_manager: Any,
|
||||
pooling_output: OmniPayload | None,
|
||||
multimodal_output: OmniPayload | None,
|
||||
request: Any,
|
||||
is_finished: bool = False,
|
||||
) -> OmniPayloadStruct | None:
|
||||
@@ -188,6 +188,7 @@ def talker2code2wav_async_chunk(
|
||||
"""
|
||||
request_id = request.external_req_id
|
||||
finished = bool(is_finished or request.is_finished())
|
||||
pooling_output = multimodal_output
|
||||
|
||||
if isinstance(pooling_output, dict):
|
||||
frame = _extract_last_frame(pooling_output)
|
||||
|
||||
@@ -214,7 +214,7 @@ def _extract_last_step_row(pooling_output: OmniPayload) -> torch.Tensor | None:
|
||||
|
||||
def talker2code2wav_async_chunk(
|
||||
transfer_manager: Any,
|
||||
pooling_output: OmniPayload | None,
|
||||
multimodal_output: OmniPayload | None,
|
||||
request: Any,
|
||||
is_finished: bool = False,
|
||||
) -> OmniPayloadStruct | None:
|
||||
@@ -244,6 +244,7 @@ def talker2code2wav_async_chunk(
|
||||
"""
|
||||
request_id = request.external_req_id
|
||||
finished = bool(is_finished or request.is_finished())
|
||||
pooling_output = multimodal_output
|
||||
|
||||
# Per-request rolling state. ``code_prompt_token_ids`` is the
|
||||
# framework-provided buffer; ``higgs_v3_emitted_frames`` tracks how many
|
||||
|
||||
@@ -12,6 +12,7 @@ signature pattern as glm_image.ar2diffusion.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from functools import lru_cache
|
||||
from typing import Any
|
||||
|
||||
@@ -208,7 +209,7 @@ def ar2diffusion(
|
||||
# Forward multimodal output from AR (if any)
|
||||
if hasattr(ar_output, "multimodal_output") and ar_output.multimodal_output:
|
||||
mm_output = ar_output.multimodal_output
|
||||
if isinstance(mm_output, dict):
|
||||
if isinstance(mm_output, Mapping):
|
||||
diffusion_input["extra"]["ar_multimodal_output"] = mm_output
|
||||
|
||||
# Forward sampling params
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Stage input processor for MammothModa2 (AR -> DiT)."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -35,7 +36,7 @@ def ar2dit(
|
||||
full_token_ids = prompt_token_ids + gen_token_ids
|
||||
|
||||
mm_output = getattr(completion_output, "multimodal_output", None)
|
||||
if not isinstance(mm_output, dict) or "latent" not in mm_output:
|
||||
if not isinstance(mm_output, Mapping) or "latent" not in mm_output:
|
||||
raise ValueError(
|
||||
"AR stage output missing latent multimodal output. "
|
||||
f"request_id={getattr(ar_output, 'request_id', None)}, "
|
||||
|
||||
@@ -137,19 +137,19 @@ def _to_code_tensor(codes: Any) -> torch.Tensor | None:
|
||||
|
||||
def llm2code2wav_async_chunk(
|
||||
transfer_manager: Any,
|
||||
pooling_output: OmniPayload,
|
||||
multimodal_output: OmniPayload | dict[str, Any],
|
||||
request: Any,
|
||||
is_finished: bool = False,
|
||||
) -> OmniPayloadStruct | None:
|
||||
"""
|
||||
Async chunk version: convert stage-0 pooling_output to code2wav payload (pooling / connector accumulation).
|
||||
Async chunk version: convert stage-0 multimodal_output to code2wav payload (pooling / connector accumulation).
|
||||
|
||||
Accumulates codes in connector per request_id,
|
||||
returns payload only when chunk_size is full or request is finished; returns None when waiting.
|
||||
"""
|
||||
# Null guard: chunk_transfer_adapter calls this every emit step
|
||||
# including no-output steps where pooling_output is None.
|
||||
if pooling_output is None or not isinstance(pooling_output, dict):
|
||||
# including no-output steps where multimodal_output is None.
|
||||
if multimodal_output is None or not isinstance(multimodal_output, dict):
|
||||
if is_finished:
|
||||
connector = getattr(transfer_manager, "connector", None)
|
||||
raw_cfg = getattr(connector, "config", {}) or {}
|
||||
@@ -187,10 +187,8 @@ def llm2code2wav_async_chunk(
|
||||
|
||||
# Text-only paths (e.g. modalities=["text"]) yield no codec pooling output;
|
||||
# stage-0 still drives the chunk transfer adapter, so treat None as "no codes
|
||||
# this step" rather than letting `.get()` raise AttributeError — an unhandled
|
||||
# error here drops the chunk, starves stage-1 of the finished payload, and
|
||||
# the stage subprocesses die before the final token is emitted.
|
||||
po_codes = pooling_output.get("codes", {}) if pooling_output is not None else {}
|
||||
# this step" rather than letting `.get()` raise AttributeError.
|
||||
po_codes = multimodal_output.get("codes", {}) if multimodal_output is not None else {}
|
||||
if "audio" not in po_codes:
|
||||
if is_finished:
|
||||
return _flush_remaining_codes(transfer_manager, request_id, chunk_size, left_context_size)
|
||||
|
||||
@@ -167,10 +167,11 @@ def _build_chunk_observability(
|
||||
|
||||
def llm2audio_vae_async_chunk(
|
||||
transfer_manager: Any,
|
||||
pooling_output: dict[str, Any] | None,
|
||||
multimodal_output: dict[str, Any] | None,
|
||||
request: Any,
|
||||
is_finished: bool = False,
|
||||
) -> OmniPayloadStruct | None:
|
||||
pooling_output = multimodal_output
|
||||
request_id = request.external_req_id
|
||||
chunk_id = int(transfer_manager.put_req_chunk[request_id])
|
||||
finished = bool(is_finished or request.is_finished())
|
||||
|
||||
@@ -92,7 +92,7 @@ def talker2codec(
|
||||
|
||||
def talker2codec_async_chunk(
|
||||
transfer_manager: Any,
|
||||
pooling_output: dict[str, Any] | None,
|
||||
multimodal_output: dict[str, Any] | None,
|
||||
request: Any,
|
||||
is_finished: bool = False,
|
||||
) -> OmniPayloadStruct | None:
|
||||
@@ -107,6 +107,7 @@ def talker2codec_async_chunk(
|
||||
signal "not enough data yet — wait for more frames".
|
||||
"""
|
||||
req_id: str = str(getattr(request, "request_id", id(request)))
|
||||
pooling_output = multimodal_output
|
||||
|
||||
# Initialise per-request accumulation state
|
||||
if not hasattr(transfer_manager, "_moss_tts_state"):
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
"""Stage input processor for Qwen3 Omni MoE: Thinker → Talker transition."""
|
||||
|
||||
import logging
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
@@ -190,7 +191,7 @@ def _get_prefill_multimodal_output(
|
||||
if not isinstance(by_req, dict):
|
||||
return None
|
||||
prefill_mm = by_req.get(request_id)
|
||||
return prefill_mm if isinstance(prefill_mm, dict) else None
|
||||
return prefill_mm if isinstance(prefill_mm, Mapping) else None
|
||||
|
||||
|
||||
def _merge_pd_embeddings(
|
||||
@@ -215,7 +216,7 @@ def _merge_pd_embeddings(
|
||||
p_emb = p_layers[int(_EMBED_LAYER_KEY)].detach().to(device=device, dtype=torch.float)
|
||||
p_hid = p_layers[int(_HIDDEN_LAYER_KEY)].detach().to(device=device, dtype=torch.float)
|
||||
except (KeyError, AttributeError, TypeError) as exc:
|
||||
available_keys = list(prefill_mm.keys()) if isinstance(prefill_mm, dict) else type(prefill_mm).__name__
|
||||
available_keys = list(prefill_mm.keys()) if isinstance(prefill_mm, Mapping) else type(prefill_mm).__name__
|
||||
logger.error(
|
||||
"_merge_pd_embeddings: failed to extract prefill embeddings (%s). "
|
||||
"Expected keys %r and %r, got: %s. "
|
||||
@@ -433,7 +434,7 @@ def _get_streaming_codec_delta_len(
|
||||
|
||||
def thinker2talker_async_chunk(
|
||||
transfer_manager: Any,
|
||||
pooling_output: OmniPayload,
|
||||
multimodal_output: OmniPayload | dict[str, Any],
|
||||
request: OmniEngineCoreRequest,
|
||||
is_finished: bool = False,
|
||||
) -> OmniPayloadStruct | None:
|
||||
@@ -446,13 +447,13 @@ def thinker2talker_async_chunk(
|
||||
|
||||
request_id = request.external_req_id
|
||||
chunk_id = transfer_manager.put_req_chunk[request_id]
|
||||
if not isinstance(pooling_output, dict):
|
||||
logger.debug("thinker2talker_async_chunk: skip non-dict pooling_output for req=%s", request_id)
|
||||
if not isinstance(multimodal_output, Mapping):
|
||||
logger.debug("thinker2talker_async_chunk: skip non-dict multimodal_output for req=%s", request_id)
|
||||
return None
|
||||
|
||||
thinker_hs = pooling_output.get("hidden_states", {})
|
||||
thinker_hs = multimodal_output.get("hidden_states", {})
|
||||
thinker_layers = thinker_hs.get("layers", {}) if isinstance(thinker_hs, dict) else {}
|
||||
thinker_embed_raw = pooling_output.get("embed", {})
|
||||
thinker_embed_raw = multimodal_output.get("embed", {})
|
||||
thinker_embed = thinker_embed_raw if isinstance(thinker_embed_raw, dict) else {}
|
||||
thinker_emb = _layer_tensor(thinker_layers, _EMBED_LAYER_KEY)
|
||||
thinker_hid = _layer_tensor(thinker_layers, _HIDDEN_LAYER_KEY)
|
||||
@@ -534,7 +535,7 @@ def thinker2talker_full_payload(
|
||||
) -> dict[str, Any] | None:
|
||||
"""Pack complete thinker output for the non-async connector path."""
|
||||
rid = getattr(request, "request_id", None)
|
||||
if not isinstance(pooling_output, dict):
|
||||
if not isinstance(pooling_output, Mapping):
|
||||
logger.warning(
|
||||
"thinker2talker_full_payload: pooling_output not a dict (type=%s) for req=%s; consumer wait gate may hang.",
|
||||
type(pooling_output).__name__,
|
||||
@@ -657,12 +658,12 @@ def thinker2talker(
|
||||
thinker_input_ids = prompt_token_ids
|
||||
new_seq_length = len(prompt_token_ids + output_ids) - 1
|
||||
thinker_mm_raw = getattr(output, "multimodal_output", None)
|
||||
if not isinstance(thinker_mm_raw, dict):
|
||||
if not isinstance(thinker_mm_raw, Mapping):
|
||||
logger.debug("thinker2talker: skip req=%s due to empty multimodal_output", req_id)
|
||||
continue
|
||||
thinker_mm: OmniPayload = thinker_mm_raw
|
||||
mm_hs = thinker_mm.get("hidden_states", {})
|
||||
mm_layers = mm_hs.get("layers", {}) if isinstance(mm_hs, dict) else {}
|
||||
mm_layers = mm_hs.get("layers", {}) if isinstance(mm_hs, Mapping) else {}
|
||||
emb_layer = _layer_tensor(mm_layers, _EMBED_LAYER_KEY)
|
||||
hid_layer = _layer_tensor(mm_layers, _HIDDEN_LAYER_KEY)
|
||||
if emb_layer is None or hid_layer is None:
|
||||
@@ -748,11 +749,11 @@ def thinker2talker_token_only(
|
||||
# hidden-state layers), do the same here so the worker buffer
|
||||
# presence agrees with the orchestrator's scheduling decision.
|
||||
thinker_mm_raw = getattr(output, "multimodal_output", None)
|
||||
if not isinstance(thinker_mm_raw, dict):
|
||||
if not isinstance(thinker_mm_raw, Mapping):
|
||||
logger.debug("thinker2talker_token_only: skip req=%s due to empty multimodal_output", req_id)
|
||||
continue
|
||||
mm_hs = thinker_mm_raw.get("hidden_states", {})
|
||||
mm_layers = mm_hs.get("layers", {}) if isinstance(mm_hs, dict) else {}
|
||||
mm_layers = mm_hs.get("layers", {}) if isinstance(mm_hs, Mapping) else {}
|
||||
if _layer_tensor(mm_layers, _EMBED_LAYER_KEY) is None or _layer_tensor(mm_layers, _HIDDEN_LAYER_KEY) is None:
|
||||
logger.debug("thinker2talker_token_only: skip req=%s due to missing hidden-state layers", req_id)
|
||||
continue
|
||||
@@ -801,16 +802,16 @@ def thinker2talker_token_only(
|
||||
|
||||
def talker2code2wav_async_chunk(
|
||||
transfer_manager: Any,
|
||||
pooling_output: OmniPayload,
|
||||
multimodal_output: OmniPayload | dict[str, Any],
|
||||
request: OmniEngineCoreRequest,
|
||||
is_finished: bool = False,
|
||||
) -> OmniPayloadStruct | None:
|
||||
"""
|
||||
Pooling version.
|
||||
Multimodal output version.
|
||||
"""
|
||||
if not isinstance(pooling_output, dict):
|
||||
if not isinstance(multimodal_output, Mapping):
|
||||
return None
|
||||
talker_codes = pooling_output.get("codes", {})
|
||||
talker_codes = multimodal_output.get("codes", {})
|
||||
if not isinstance(talker_codes, dict):
|
||||
return None
|
||||
code_predictor_codes = talker_codes.get("audio")
|
||||
@@ -884,7 +885,7 @@ def talker2code2wav_full_payload(
|
||||
) -> dict[str, Any] | None:
|
||||
"""Pack complete talker codec output for the non-async connector path."""
|
||||
rid = getattr(request, "request_id", None)
|
||||
if not isinstance(pooling_output, dict):
|
||||
if not isinstance(pooling_output, Mapping):
|
||||
logger.warning(
|
||||
"talker2code2wav_full_payload: pooling_output not a dict "
|
||||
"(type=%s) for req=%s; consumer wait gate may hang.",
|
||||
@@ -983,7 +984,7 @@ def talker2code2wav(
|
||||
if is_streaming_session:
|
||||
seq_len = _get_streaming_codec_delta_len(cur_seq_len, req_id, talker_output, streaming_context)
|
||||
mm_raw = getattr(output, "multimodal_output", None)
|
||||
if not isinstance(mm_raw, dict):
|
||||
if not isinstance(mm_raw, Mapping):
|
||||
logger.debug("talker2code2wav: skip req=%s due to empty multimodal_output", req_id)
|
||||
continue
|
||||
mm: OmniPayload = mm_raw
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Stage input processor for Qwen3-TTS: Talker -> Code2Wav."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -123,8 +124,8 @@ def talker2code2wav(
|
||||
return code2wav_inputs
|
||||
|
||||
|
||||
def _extract_last_frame(pooling_output: OmniPayload) -> torch.Tensor | None:
|
||||
audio_codes = pooling_output.get("codes", {}).get("audio")
|
||||
def _extract_last_frame(multimodal_output: OmniPayload | dict[str, Any]) -> torch.Tensor | None:
|
||||
audio_codes = multimodal_output.get("codes", {}).get("audio")
|
||||
if not isinstance(audio_codes, torch.Tensor) or audio_codes.numel() == 0:
|
||||
return None
|
||||
if audio_codes.ndim == 2:
|
||||
@@ -139,7 +140,7 @@ def _extract_last_frame(pooling_output: OmniPayload) -> torch.Tensor | None:
|
||||
|
||||
def talker2code2wav_async_chunk(
|
||||
transfer_manager: Any,
|
||||
pooling_output: OmniPayload | None,
|
||||
multimodal_output: OmniPayload | dict[str, Any] | None,
|
||||
request: Any,
|
||||
is_finished: bool = False,
|
||||
) -> OmniPayloadStruct | None:
|
||||
@@ -150,12 +151,12 @@ def talker2code2wav_async_chunk(
|
||||
request_payload = {}
|
||||
transfer_manager.request_payload = request_payload
|
||||
|
||||
if isinstance(pooling_output, dict):
|
||||
frame = _extract_last_frame(pooling_output)
|
||||
if isinstance(multimodal_output, Mapping):
|
||||
frame = _extract_last_frame(multimodal_output)
|
||||
if frame is not None:
|
||||
codec_codes = frame.cpu().tolist()
|
||||
transfer_manager.code_prompt_token_ids[request_id].append(codec_codes)
|
||||
ref_code = pooling_output.get("codes", {}).get("ref")
|
||||
ref_code = multimodal_output.get("codes", {}).get("ref")
|
||||
if isinstance(ref_code, torch.Tensor) and ref_code.numel() > 0 and request_payload.get(request_id) is None:
|
||||
request_payload[request_id] = ref_code.to(torch.long).cpu().contiguous()
|
||||
elif not finished:
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -35,8 +36,8 @@ def generator2tokenizer(
|
||||
return tokenizer_inputs
|
||||
|
||||
|
||||
def _extract_last_frame(pooling_output: OmniPayload) -> torch.Tensor | None:
|
||||
audio = pooling_output.get("audio")
|
||||
def _extract_last_frame(multimodal_output: OmniPayload | dict[str, Any]) -> torch.Tensor | None:
|
||||
audio = multimodal_output.get("audio")
|
||||
if not isinstance(audio, torch.Tensor) or audio.numel() == 0:
|
||||
return None
|
||||
return audio.flatten()
|
||||
@@ -44,20 +45,20 @@ def _extract_last_frame(pooling_output: OmniPayload) -> torch.Tensor | None:
|
||||
|
||||
def generator2tokenizer_async_chunk(
|
||||
transfer_manager: Any,
|
||||
pooling_output: OmniPayload,
|
||||
multimodal_output: OmniPayload | dict[str, Any],
|
||||
request: Any,
|
||||
is_finished: bool = False,
|
||||
) -> OmniPayloadStruct | None:
|
||||
request_id = request.external_req_id
|
||||
finished = bool(is_finished or request.is_finished())
|
||||
|
||||
if isinstance(pooling_output, dict):
|
||||
frame = _extract_last_frame(pooling_output)
|
||||
if isinstance(multimodal_output, Mapping):
|
||||
frame = _extract_last_frame(multimodal_output)
|
||||
if frame is not None:
|
||||
codec_codes = frame.cpu().tolist()
|
||||
transfer_manager.code_prompt_token_ids[request_id].append(codec_codes)
|
||||
elif not finished:
|
||||
# Some steps may not produce pooling_output. Only flush on finish.
|
||||
# Some steps may not produce multimodal_output. Only flush on finish.
|
||||
return None
|
||||
|
||||
connector = getattr(transfer_manager, "connector", None)
|
||||
|
||||
Reference in new issue
Block a user