[Refactor] Output Processor Phase 2: separate multimodal output channel (#1601) (#2744)

Signed-off-by: Megha Agarwal <agarwalmegha1308@gmail.com>
Signed-off-by: meghaagr13 <divyanshsinghvi@gmail.com>
Signed-off-by: Taichang Zhou <tzhouam@connect.ust.hk>
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
Co-authored-by: meghaagr13 <divyanshsinghvi@gmail.com>
Co-authored-by: Boao Shi <aoibosh@connect.hku.hk>
Co-authored-by: Taichang Zhou <tzhouam@connect.ust.hk>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
authored and GitHub committed 2026-06-12 15:19:14 +08:00
1 parent 1be1a71156
commit 8f869c8ea5
49 files changed
+985 -327

No files matched your search

@@ -1,4 +1,4 @@
from collections.abc import Iterable
from collections.abc import Iterable, Mapping
from dataclasses import replace
from functools import cached_property
from pathlib import Path
@@ -220,7 +220,7 @@ class VoxtralTTSForConditionalGeneration(
def tts_postprocess(self, hidden_states: torch.Tensor, multimodal_outputs: object, **info_dict: object | None):
update_dict = {}
if isinstance(multimodal_outputs, dict) and "audio" in multimodal_outputs:
if isinstance(multimodal_outputs, Mapping) and "audio" in multimodal_outputs:
assert self.post_process_idx < len(multimodal_outputs["audio"]), (
f"Expect {self.post_process_idx=} < {len(multimodal_outputs['audio'])=}"
)
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections import defaultdict
from collections.abc import Mapping
from contextlib import nullcontext
from typing import Any
@@ -162,7 +163,7 @@ def text2flow(
def talker2code2wav_async_chunk(
transfer_manager: Any,
pooling_output: dict[str, Any] | None,
multimodal_output: dict[str, Any] | None,
request: Any,
is_finished: bool = False,
) -> OmniPayloadStruct | None:
@@ -198,12 +199,14 @@ def talker2code2wav_async_chunk(
value = _to_cpu_tensor(info_embed.get(key))
if value is not None:
prompt_payload[key] = value
if isinstance(pooling_output, dict):
po_embed = pooling_output.get("embed", {}) if isinstance(pooling_output.get("embed"), dict) else {}
if isinstance(multimodal_output, Mapping):
mm_embed = multimodal_output.get("embed", {})
if not isinstance(mm_embed, Mapping):
mm_embed = multimodal_output
for key in cond_keys:
if key in prompt_payload:
continue
value = _to_cpu_tensor(po_embed.get(key))
value = _to_cpu_tensor(mm_embed.get(key))
if value is not None:
prompt_payload[key] = value
# Drop any right-padding carried from batched talker emission so
@@ -1,5 +1,6 @@
"""Stage input processor for Fish Speech S2 Pro: Slow AR → DAC Decoder."""
from collections.abc import Mapping
from typing import Any
import torch
@@ -91,14 +92,14 @@ def _select_backlog_chunk_size(
return backlog_chunk_size
def _extract_last_frame(pooling_output: dict[str, Any]) -> torch.Tensor | None:
"""Extract the last frame of audio codes from the pooling output."""
audio_codes = pooling_output.get("audio_codes")
def _extract_last_frame(multimodal_output: dict[str, Any]) -> torch.Tensor | None:
"""Extract the last frame of audio codes from the multimodal output."""
audio_codes = multimodal_output.get("audio_codes")
if not isinstance(audio_codes, torch.Tensor) or audio_codes.numel() == 0:
return None
if audio_codes.ndim == 2:
frame = audio_codes[-1]
valid = pooling_output.get("audio_code_valid")
valid = multimodal_output.get("audio_code_valid")
if isinstance(valid, torch.Tensor) and valid.numel() > 0:
is_valid = bool(valid.reshape(-1)[-1].item())
elif valid is not None:
@@ -146,7 +147,7 @@ def slow_ar_to_dac_decoder(
def slow_ar_to_dac_decoder_async_chunk(
transfer_manager: Any,
pooling_output: dict[str, Any] | None,
multimodal_output: dict[str, Any] | None,
request: Any,
is_finished: bool = False,
) -> OmniPayloadStruct | None:
@@ -160,8 +161,8 @@ def slow_ar_to_dac_decoder_async_chunk(
finished = bool(is_finished or request.is_finished())
cfg = _get_connector_extra(transfer_manager)
if isinstance(pooling_output, dict):
frame = _extract_last_frame(pooling_output)
if isinstance(multimodal_output, Mapping):
frame = _extract_last_frame(multimodal_output)
if frame is not None:
transfer_manager.code_prompt_token_ids[request_id].append(frame.detach())
elif not finished:
@@ -4,6 +4,7 @@
import math
import time
from collections.abc import Mapping
from typing import Any
import torch
@@ -23,7 +24,7 @@ def _has_source_image(mm_data: Any) -> bool:
- `img2img`: legacy single-image key
- `images`: list or single image
"""
if not isinstance(mm_data, dict):
if not isinstance(mm_data, Mapping):
return False
if mm_data.get("image") is not None:
return True
@@ -35,7 +36,7 @@ def _has_source_image(mm_data: Any) -> bool:
def _first_source_image(mm_data: Any) -> Any:
"""Get first source image from normalized multimodal keys."""
if not isinstance(mm_data, dict):
if not isinstance(mm_data, Mapping):
return None
image = mm_data.get("image")
@@ -285,7 +286,7 @@ def ar2diffusion(
if hasattr(ar_output, "multimodal_output") and ar_output.multimodal_output:
mm_output = ar_output.multimodal_output
if isinstance(mm_output, dict) and mm_output.get("ids", {}).get("prior_image") is not None:
if isinstance(mm_output, Mapping) and mm_output.get("ids", {}).get("prior_image") is not None:
is_i2i = True
_dt_mode = (time.perf_counter() - _t_mode) * 1000
@@ -322,7 +323,7 @@ def ar2diffusion(
# Check ar_output (RequestOutput) for multimodal_output - this is the correct location
if hasattr(ar_output, "multimodal_output") and ar_output.multimodal_output:
mm_output = ar_output.multimodal_output
if isinstance(mm_output, dict):
if isinstance(mm_output, Mapping):
raw_prior_image_ids = mm_output.get("ids", {}).get("prior_image")
if raw_prior_image_ids is not None:
# Handle different formats:
@@ -348,7 +349,7 @@ def ar2diffusion(
if hasattr(output, "multimodal_output") and output.multimodal_output:
mm_output = output.multimodal_output
logger.debug(f"[ar2diffusion] Request {i}: found multimodal_output on CompletionOutput (fallback)")
if isinstance(mm_output, dict):
if isinstance(mm_output, Mapping):
raw_prior_image_ids = mm_output.get("ids", {}).get("prior_image")
if raw_prior_image_ids is not None:
if isinstance(raw_prior_image_ids, torch.Tensor):
@@ -188,7 +188,7 @@ def _to_cpu_tensor(value: Any) -> torch.Tensor | None:
def ar_to_dit_async_chunk(
transfer_manager: Any,
pooling_output: dict[str, Any] | None,
multimodal_output: dict[str, Any] | None,
request: Any,
is_finished: bool = False,
) -> OmniPayloadStruct | None:
@@ -216,6 +216,7 @@ def ar_to_dit_async_chunk(
if request_id is None:
raise ValueError("GLM-TTS async chunk request is missing request id")
finished = bool(is_finished or request.is_finished())
pooling_output = multimodal_output
# Read connector chunk config (supports progressive list or single int)
connector = getattr(transfer_manager, "connector", None)
@@ -176,7 +176,7 @@ def _extract_last_frame(pooling_output: OmniPayload) -> torch.Tensor | None:
def talker2code2wav_async_chunk(
transfer_manager: Any,
pooling_output: OmniPayload | None,
multimodal_output: OmniPayload | None,
request: Any,
is_finished: bool = False,
) -> OmniPayloadStruct | None:
@@ -188,6 +188,7 @@ def talker2code2wav_async_chunk(
"""
request_id = request.external_req_id
finished = bool(is_finished or request.is_finished())
pooling_output = multimodal_output
if isinstance(pooling_output, dict):
frame = _extract_last_frame(pooling_output)
@@ -214,7 +214,7 @@ def _extract_last_step_row(pooling_output: OmniPayload) -> torch.Tensor | None:
def talker2code2wav_async_chunk(
transfer_manager: Any,
pooling_output: OmniPayload | None,
multimodal_output: OmniPayload | None,
request: Any,
is_finished: bool = False,
) -> OmniPayloadStruct | None:
@@ -244,6 +244,7 @@ def talker2code2wav_async_chunk(
"""
request_id = request.external_req_id
finished = bool(is_finished or request.is_finished())
pooling_output = multimodal_output
# Per-request rolling state. ``code_prompt_token_ids`` is the
# framework-provided buffer; ``higgs_v3_emitted_frames`` tracks how many
@@ -12,6 +12,7 @@ signature pattern as glm_image.ar2diffusion.
from __future__ import annotations
from collections.abc import Mapping
from functools import lru_cache
from typing import Any
@@ -208,7 +209,7 @@ def ar2diffusion(
# Forward multimodal output from AR (if any)
if hasattr(ar_output, "multimodal_output") and ar_output.multimodal_output:
mm_output = ar_output.multimodal_output
if isinstance(mm_output, dict):
if isinstance(mm_output, Mapping):
diffusion_input["extra"]["ar_multimodal_output"] = mm_output
# Forward sampling params
@@ -1,5 +1,6 @@
"""Stage input processor for MammothModa2 (AR -> DiT)."""
from collections.abc import Mapping
from typing import Any
import torch
@@ -35,7 +36,7 @@ def ar2dit(
full_token_ids = prompt_token_ids + gen_token_ids
mm_output = getattr(completion_output, "multimodal_output", None)
if not isinstance(mm_output, dict) or "latent" not in mm_output:
if not isinstance(mm_output, Mapping) or "latent" not in mm_output:
raise ValueError(
"AR stage output missing latent multimodal output. "
f"request_id={getattr(ar_output, 'request_id', None)}, "
@@ -137,19 +137,19 @@ def _to_code_tensor(codes: Any) -> torch.Tensor | None:
def llm2code2wav_async_chunk(
transfer_manager: Any,
pooling_output: OmniPayload,
multimodal_output: OmniPayload | dict[str, Any],
request: Any,
is_finished: bool = False,
) -> OmniPayloadStruct | None:
"""
Async chunk version: convert stage-0 pooling_output to code2wav payload (pooling / connector accumulation).
Async chunk version: convert stage-0 multimodal_output to code2wav payload (pooling / connector accumulation).
Accumulates codes in connector per request_id,
returns payload only when chunk_size is full or request is finished; returns None when waiting.
"""
# Null guard: chunk_transfer_adapter calls this every emit step
# including no-output steps where pooling_output is None.
if pooling_output is None or not isinstance(pooling_output, dict):
# including no-output steps where multimodal_output is None.
if multimodal_output is None or not isinstance(multimodal_output, dict):
if is_finished:
connector = getattr(transfer_manager, "connector", None)
raw_cfg = getattr(connector, "config", {}) or {}
@@ -187,10 +187,8 @@ def llm2code2wav_async_chunk(
# Text-only paths (e.g. modalities=["text"]) yield no codec pooling output;
# stage-0 still drives the chunk transfer adapter, so treat None as "no codes
# this step" rather than letting `.get()` raise AttributeError — an unhandled
# error here drops the chunk, starves stage-1 of the finished payload, and
# the stage subprocesses die before the final token is emitted.
po_codes = pooling_output.get("codes", {}) if pooling_output is not None else {}
# this step" rather than letting `.get()` raise AttributeError.
po_codes = multimodal_output.get("codes", {}) if multimodal_output is not None else {}
if "audio" not in po_codes:
if is_finished:
return _flush_remaining_codes(transfer_manager, request_id, chunk_size, left_context_size)
@@ -167,10 +167,11 @@ def _build_chunk_observability(
def llm2audio_vae_async_chunk(
transfer_manager: Any,
pooling_output: dict[str, Any] | None,
multimodal_output: dict[str, Any] | None,
request: Any,
is_finished: bool = False,
) -> OmniPayloadStruct | None:
pooling_output = multimodal_output
request_id = request.external_req_id
chunk_id = int(transfer_manager.put_req_chunk[request_id])
finished = bool(is_finished or request.is_finished())
@@ -92,7 +92,7 @@ def talker2codec(
def talker2codec_async_chunk(
transfer_manager: Any,
pooling_output: dict[str, Any] | None,
multimodal_output: dict[str, Any] | None,
request: Any,
is_finished: bool = False,
) -> OmniPayloadStruct | None:
@@ -107,6 +107,7 @@ def talker2codec_async_chunk(
signal "not enough data yet — wait for more frames".
"""
req_id: str = str(getattr(request, "request_id", id(request)))
pooling_output = multimodal_output
# Initialise per-request accumulation state
if not hasattr(transfer_manager, "_moss_tts_state"):
@@ -4,6 +4,7 @@
"""Stage input processor for Qwen3 Omni MoE: Thinker → Talker transition."""
import logging
from collections.abc import Mapping
from dataclasses import dataclass, field
from typing import Any
@@ -190,7 +191,7 @@ def _get_prefill_multimodal_output(
if not isinstance(by_req, dict):
return None
prefill_mm = by_req.get(request_id)
return prefill_mm if isinstance(prefill_mm, dict) else None
return prefill_mm if isinstance(prefill_mm, Mapping) else None
def _merge_pd_embeddings(
@@ -215,7 +216,7 @@ def _merge_pd_embeddings(
p_emb = p_layers[int(_EMBED_LAYER_KEY)].detach().to(device=device, dtype=torch.float)
p_hid = p_layers[int(_HIDDEN_LAYER_KEY)].detach().to(device=device, dtype=torch.float)
except (KeyError, AttributeError, TypeError) as exc:
available_keys = list(prefill_mm.keys()) if isinstance(prefill_mm, dict) else type(prefill_mm).__name__
available_keys = list(prefill_mm.keys()) if isinstance(prefill_mm, Mapping) else type(prefill_mm).__name__
logger.error(
"_merge_pd_embeddings: failed to extract prefill embeddings (%s). "
"Expected keys %r and %r, got: %s. "
@@ -433,7 +434,7 @@ def _get_streaming_codec_delta_len(
def thinker2talker_async_chunk(
transfer_manager: Any,
pooling_output: OmniPayload,
multimodal_output: OmniPayload | dict[str, Any],
request: OmniEngineCoreRequest,
is_finished: bool = False,
) -> OmniPayloadStruct | None:
@@ -446,13 +447,13 @@ def thinker2talker_async_chunk(
request_id = request.external_req_id
chunk_id = transfer_manager.put_req_chunk[request_id]
if not isinstance(pooling_output, dict):
logger.debug("thinker2talker_async_chunk: skip non-dict pooling_output for req=%s", request_id)
if not isinstance(multimodal_output, Mapping):
logger.debug("thinker2talker_async_chunk: skip non-dict multimodal_output for req=%s", request_id)
return None
thinker_hs = pooling_output.get("hidden_states", {})
thinker_hs = multimodal_output.get("hidden_states", {})
thinker_layers = thinker_hs.get("layers", {}) if isinstance(thinker_hs, dict) else {}
thinker_embed_raw = pooling_output.get("embed", {})
thinker_embed_raw = multimodal_output.get("embed", {})
thinker_embed = thinker_embed_raw if isinstance(thinker_embed_raw, dict) else {}
thinker_emb = _layer_tensor(thinker_layers, _EMBED_LAYER_KEY)
thinker_hid = _layer_tensor(thinker_layers, _HIDDEN_LAYER_KEY)
@@ -534,7 +535,7 @@ def thinker2talker_full_payload(
) -> dict[str, Any] | None:
"""Pack complete thinker output for the non-async connector path."""
rid = getattr(request, "request_id", None)
if not isinstance(pooling_output, dict):
if not isinstance(pooling_output, Mapping):
logger.warning(
"thinker2talker_full_payload: pooling_output not a dict (type=%s) for req=%s; consumer wait gate may hang.",
type(pooling_output).__name__,
@@ -657,12 +658,12 @@ def thinker2talker(
thinker_input_ids = prompt_token_ids
new_seq_length = len(prompt_token_ids + output_ids) - 1
thinker_mm_raw = getattr(output, "multimodal_output", None)
if not isinstance(thinker_mm_raw, dict):
if not isinstance(thinker_mm_raw, Mapping):
logger.debug("thinker2talker: skip req=%s due to empty multimodal_output", req_id)
continue
thinker_mm: OmniPayload = thinker_mm_raw
mm_hs = thinker_mm.get("hidden_states", {})
mm_layers = mm_hs.get("layers", {}) if isinstance(mm_hs, dict) else {}
mm_layers = mm_hs.get("layers", {}) if isinstance(mm_hs, Mapping) else {}
emb_layer = _layer_tensor(mm_layers, _EMBED_LAYER_KEY)
hid_layer = _layer_tensor(mm_layers, _HIDDEN_LAYER_KEY)
if emb_layer is None or hid_layer is None:
@@ -748,11 +749,11 @@ def thinker2talker_token_only(
# hidden-state layers), do the same here so the worker buffer
# presence agrees with the orchestrator's scheduling decision.
thinker_mm_raw = getattr(output, "multimodal_output", None)
if not isinstance(thinker_mm_raw, dict):
if not isinstance(thinker_mm_raw, Mapping):
logger.debug("thinker2talker_token_only: skip req=%s due to empty multimodal_output", req_id)
continue
mm_hs = thinker_mm_raw.get("hidden_states", {})
mm_layers = mm_hs.get("layers", {}) if isinstance(mm_hs, dict) else {}
mm_layers = mm_hs.get("layers", {}) if isinstance(mm_hs, Mapping) else {}
if _layer_tensor(mm_layers, _EMBED_LAYER_KEY) is None or _layer_tensor(mm_layers, _HIDDEN_LAYER_KEY) is None:
logger.debug("thinker2talker_token_only: skip req=%s due to missing hidden-state layers", req_id)
continue
@@ -801,16 +802,16 @@ def thinker2talker_token_only(
def talker2code2wav_async_chunk(
transfer_manager: Any,
pooling_output: OmniPayload,
multimodal_output: OmniPayload | dict[str, Any],
request: OmniEngineCoreRequest,
is_finished: bool = False,
) -> OmniPayloadStruct | None:
"""
Pooling version.
Multimodal output version.
"""
if not isinstance(pooling_output, dict):
if not isinstance(multimodal_output, Mapping):
return None
talker_codes = pooling_output.get("codes", {})
talker_codes = multimodal_output.get("codes", {})
if not isinstance(talker_codes, dict):
return None
code_predictor_codes = talker_codes.get("audio")
@@ -884,7 +885,7 @@ def talker2code2wav_full_payload(
) -> dict[str, Any] | None:
"""Pack complete talker codec output for the non-async connector path."""
rid = getattr(request, "request_id", None)
if not isinstance(pooling_output, dict):
if not isinstance(pooling_output, Mapping):
logger.warning(
"talker2code2wav_full_payload: pooling_output not a dict "
"(type=%s) for req=%s; consumer wait gate may hang.",
@@ -983,7 +984,7 @@ def talker2code2wav(
if is_streaming_session:
seq_len = _get_streaming_codec_delta_len(cur_seq_len, req_id, talker_output, streaming_context)
mm_raw = getattr(output, "multimodal_output", None)
if not isinstance(mm_raw, dict):
if not isinstance(mm_raw, Mapping):
logger.debug("talker2code2wav: skip req=%s due to empty multimodal_output", req_id)
continue
mm: OmniPayload = mm_raw
@@ -1,5 +1,6 @@
"""Stage input processor for Qwen3-TTS: Talker -> Code2Wav."""
from collections.abc import Mapping
from typing import Any
import torch
@@ -123,8 +124,8 @@ def talker2code2wav(
return code2wav_inputs
def _extract_last_frame(pooling_output: OmniPayload) -> torch.Tensor | None:
audio_codes = pooling_output.get("codes", {}).get("audio")
def _extract_last_frame(multimodal_output: OmniPayload | dict[str, Any]) -> torch.Tensor | None:
audio_codes = multimodal_output.get("codes", {}).get("audio")
if not isinstance(audio_codes, torch.Tensor) or audio_codes.numel() == 0:
return None
if audio_codes.ndim == 2:
@@ -139,7 +140,7 @@ def _extract_last_frame(pooling_output: OmniPayload) -> torch.Tensor | None:
def talker2code2wav_async_chunk(
transfer_manager: Any,
pooling_output: OmniPayload | None,
multimodal_output: OmniPayload | dict[str, Any] | None,
request: Any,
is_finished: bool = False,
) -> OmniPayloadStruct | None:
@@ -150,12 +151,12 @@ def talker2code2wav_async_chunk(
request_payload = {}
transfer_manager.request_payload = request_payload
if isinstance(pooling_output, dict):
frame = _extract_last_frame(pooling_output)
if isinstance(multimodal_output, Mapping):
frame = _extract_last_frame(multimodal_output)
if frame is not None:
codec_codes = frame.cpu().tolist()
transfer_manager.code_prompt_token_ids[request_id].append(codec_codes)
ref_code = pooling_output.get("codes", {}).get("ref")
ref_code = multimodal_output.get("codes", {}).get("ref")
if isinstance(ref_code, torch.Tensor) and ref_code.numel() > 0 and request_payload.get(request_id) is None:
request_payload[request_id] = ref_code.to(torch.long).cpu().contiguous()
elif not finished:
@@ -1,3 +1,4 @@
from collections.abc import Mapping
from typing import Any
import torch
@@ -35,8 +36,8 @@ def generator2tokenizer(
return tokenizer_inputs
def _extract_last_frame(pooling_output: OmniPayload) -> torch.Tensor | None:
audio = pooling_output.get("audio")
def _extract_last_frame(multimodal_output: OmniPayload | dict[str, Any]) -> torch.Tensor | None:
audio = multimodal_output.get("audio")
if not isinstance(audio, torch.Tensor) or audio.numel() == 0:
return None
return audio.flatten()
@@ -44,20 +45,20 @@ def _extract_last_frame(pooling_output: OmniPayload) -> torch.Tensor | None:
def generator2tokenizer_async_chunk(
transfer_manager: Any,
pooling_output: OmniPayload,
multimodal_output: OmniPayload | dict[str, Any],
request: Any,
is_finished: bool = False,
) -> OmniPayloadStruct | None:
request_id = request.external_req_id
finished = bool(is_finished or request.is_finished())
if isinstance(pooling_output, dict):
frame = _extract_last_frame(pooling_output)
if isinstance(multimodal_output, Mapping):
frame = _extract_last_frame(multimodal_output)
if frame is not None:
codec_codes = frame.cpu().tolist()
transfer_manager.code_prompt_token_ids[request_id].append(codec_codes)
elif not finished:
# Some steps may not produce pooling_output. Only flush on finish.
# Some steps may not produce multimodal_output. Only flush on finish.
return None
connector = getattr(transfer_manager, "connector", None)