[Core] Remove dead custom_process_input_func hooks in stage input processors (#4531)

Signed-off-by: Isaac Hung <manhei5001@gmail.com>
This commit is contained in:
Isaac Hung authored and GitHub committed 2026-06-29 05:17:25 +02:00
1 parent cd35572069
commit e9d4931781
11 files changed
+59 -191

No files matched your search

-1
View File
@@ -290,7 +290,6 @@ stage_args:
enable_prefix_caching: false
engine_output_type: latent
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
default_sampling_params:
temperature: 0.9
top_p: 0.8
@@ -921,7 +921,6 @@ def test_cosyvoice3_full_payload_replace_keys_present() -> None:
def test_ming_flash_omni_thinker2talker_token_only_smoke() -> None:
"""Smoke: ming_flash_omni token-only carries voice metadata."""
from vllm_omni.model_executor.stage_input_processors.ming_flash_omni import (
thinker2talker,
thinker2talker_token_only,
)
@@ -939,15 +938,14 @@ def test_ming_flash_omni_thinker2talker_token_only_smoke() -> None:
src = [_Wrap("hello world")]
prompt = _Prompt({"voice_name": "ZH_FEMALE", "prompt_text": "ref text"})
for func in (thinker2talker, thinker2talker_token_only):
out = func(src, prompt=prompt)
assert len(out) == 1
assert out[0]["prompt_token_ids"] == [0]
info = out[0]["additional_information"]
assert info["text"] == "hello world"
assert info["voice_name"] == "ZH_FEMALE"
assert info["prompt_text"] == "ref text"
assert info["ming_task"] == "omni"
out = thinker2talker_token_only(src, prompt=prompt)
assert len(out) == 1
assert out[0]["prompt_token_ids"] == [0]
info = out[0]["additional_information"]
assert info["text"] == "hello world"
assert info["voice_name"] == "ZH_FEMALE"
assert info["prompt_text"] == "ref text"
assert info["ming_task"] == "omni"
def test_qwen2_5_omni_thinker2talker_token_only_smoke() -> None:
+19 -11
View File
@@ -1060,7 +1060,11 @@ class TestQwen2_5OmniPipeline:
assert isinstance(s, StagePipelineConfig)
assert s.input_sources == (0,)
assert s.sampling_constraints["stop_token_ids"] == [8294]
assert s.custom_process_input_func is not None
# thinker2talker was removed: qwen2_5_omni has no async_chunk support,
# so sync_process_input_func always wins and custom_process_input_func
# was dead code.
assert s.custom_process_input_func is None
assert s.sync_process_input_func is not None
def test_code2wav(self):
p = StageConfigFactory.resolve_pipeline_config("qwen2_5_omni")
@@ -1191,10 +1195,14 @@ class TestMingFlashOmniPipeline:
# Per-stage model_arch override (Ming talker has its own self-contained LLM)
assert s.model_arch == "MingFlashOmniTalkerForConditionalGeneration"
assert s.tokenizer_subdir == "talker/llm"
assert s.custom_process_input_func is not None
# thinker2talker was removed: ming_flash_omni has no async_chunk support
# and both thinker2talker / thinker2talker_token_only called _build_talker_inputs
# identically, so custom_process_input_func was dead code.
assert s.custom_process_input_func is None
assert s.sync_process_input_func is not None
def test_talker_stage_processor_wiring_resolves(self):
"""The custom_process_input_func string must point to a real callable.
"""The sync_process_input_func string must point to a real callable.
Lazy string references only fail at first inference otherwise — this
catches typos in the pipeline declaration at import / registration time.
@@ -1204,7 +1212,7 @@ class TestMingFlashOmniPipeline:
s = p.get_stage(1)
assert isinstance(s, StagePipelineConfig)
module_path, _, attr = s.custom_process_input_func.rpartition(".")
module_path, _, attr = s.sync_process_input_func.rpartition(".")
module = importlib.import_module(module_path)
assert callable(getattr(module, attr))
@@ -1720,11 +1728,12 @@ class TestSentinelDefaultPrecedence:
)
def test_ming_flash_omni_topology(self):
"""Guard ming_flash_omni's PR3 cleanup: stage 0 has no full-payload
producer hook (the connector path was removed as fake -- arch is not
in ``_FULL_PAYLOAD_INPUT_STAGES``), and stage 1 still wires the
legacy ``thinker2talker`` (custom_process_input_func) plus the
``thinker2talker_token_only`` placeholder (sync_process_input_func).
"""Guard ming_flash_omni's SIP cleanup: stage 0 has no full-payload
producer hook (arch is not in ``_FULL_PAYLOAD_INPUT_STAGES``), and
stage 1 uses only ``thinker2talker_token_only`` (sync_process_input_func).
The dead ``thinker2talker`` (custom_process_input_func) was removed
because ming_flash_omni has no async_chunk support and both functions
called ``_build_talker_inputs`` identically.
Merge under either async_chunk mode must not re-introduce a
stage-0 full-payload hook."""
pipeline = StageConfigFactory.resolve_pipeline_config("ming_flash_omni")
@@ -1735,8 +1744,7 @@ class TestSentinelDefaultPrecedence:
"ming_flash_omni stage 0 must not declare a full-payload producer "
"(connector path is not active for this arch)."
)
assert stage1.custom_process_input_func is not None
assert stage1.custom_process_input_func.endswith("thinker2talker")
assert stage1.custom_process_input_func is None
assert stage1.sync_process_input_func is not None
assert stage1.sync_process_input_func.endswith("thinker2talker_token_only")
+8 -6
View File
@@ -180,15 +180,17 @@ def _first_defined(*values: Any) -> Any:
def _validate_async_chunk_support(pipeline: PipelineConfig, deploy: DeployConfig) -> None:
if deploy.async_chunk and not any(
stage.async_chunk_process_next_stage_input_func or stage.custom_process_next_stage_input_func
for stage in pipeline.stages
has_inter_stage_edges = any(stage.input_sources for stage in pipeline.stages)
if (
deploy.async_chunk
and has_inter_stage_edges
and not any(stage.async_chunk_process_next_stage_input_func for stage in pipeline.stages)
):
raise ValueError(
f"Pipeline {pipeline.model_type!r} has async_chunk=True in deploy but no stage "
"declares a next-stage input processor "
"(``async_chunk_process_next_stage_input_func`` or ``custom_process_next_stage_input_func``). "
"Either set async_chunk=False or implement an async-chunk processor on the pipeline."
"declares a dedicated async-chunk next-stage processor "
"(``async_chunk_process_next_stage_input_func``). "
"Either set async_chunk=False or implement an async-chunk producer on the pipeline."
)
+20 -13
View File
@@ -603,7 +603,8 @@ def _apply_platform_overrides(
if platform is None:
from vllm_omni.platforms import current_omni_platform
platform = current_omni_platform.device_name.lower()
device_name = current_omni_platform.device_name
platform = device_name.lower() if device_name is not None else None
if platform is None or deploy.platforms is None:
return deploy
platform_section = deploy.platforms.get(platform)
@@ -784,21 +785,27 @@ def merge_pipeline_deploy(
if len(pipeline.stages) <= 1:
deploy.async_chunk = False
# A pipeline supports async_chunk if any stage has either an explicit
# async-chunk-only processor slot OR a custom next-stage processor (some
# pipelines like qwen3_omni wire async-chunk processing directly through
# ``custom_process_next_stage_input_func``). Only raise when neither is
# present — that's the "user enabled async_chunk but pipeline has no
# inter-stage processing at all" case.
if deploy.async_chunk and not any(
ps.async_chunk_process_next_stage_input_func or ps.custom_process_next_stage_input_func
for ps in pipeline.stages
# async_chunk only applies to multi-stage pipelines: a pipeline with no
# consumer stages (every stage has empty input_sources) has no inter-stage
# edges, so async_chunk is a no-op and we skip the check entirely.
# For pipelines that DO have inter-stage edges, require a dedicated per-step
# async producer (``async_chunk_process_next_stage_input_func``).
# ``custom_process_next_stage_input_func`` is the full-payload / connector-path
# producer and does NOT imply async_chunk support — pipelines like qwen2_5_omni
# and covo_audio have it but removed their consumer-side ``custom_process_input_func``
# because they don't support async_chunk, so accepting them here would silently
# miswire the consumer stage instead of raising a clear error.
_has_inter_stage_edges = any(ps.input_sources for ps in pipeline.stages)
if (
deploy.async_chunk
and _has_inter_stage_edges
and not any(ps.async_chunk_process_next_stage_input_func for ps in pipeline.stages)
):
raise ValueError(
f"Pipeline {pipeline.model_type!r} has async_chunk=True in deploy but no stage "
"declares a next-stage input processor "
"(``async_chunk_process_next_stage_input_func`` or ``custom_process_next_stage_input_func``). "
"Either set async_chunk=False or implement an async-chunk processor on the pipeline."
"declares a dedicated async-chunk next-stage processor "
"(``async_chunk_process_next_stage_input_func``). "
"Either set async_chunk=False or implement an async-chunk producer on the pipeline."
)
result: list[StageConfig] = []
@@ -44,7 +44,6 @@ COVO_AUDIO_PIPELINE = PipelineConfig(
final_output=True,
final_output_type="audio",
engine_output_type="audio",
custom_process_input_func=f"{_PROC}.llm2code2wav",
sync_process_input_func=f"{_PROC}.llm2code2wav_token_only",
sampling_constraints={"detokenize": False},
),
@@ -6,8 +6,8 @@ Stage 0: Thinker — multimodal understanding + text generation.
Stage 1: Talker — text -> audio waveform via CFM + AudioVAE.
The thinker -> talker bridge passes the detokenized text rather than
hidden states through `ming_flash_omni.thinker2talker`; the talker has a
self-contained Qwen2 LLM that retokenizes the string itself.
hidden states; the talker has a self-contained Qwen2 LLM that retokenizes
the string itself.
"""
from vllm_omni.config.stage_config import (
@@ -57,7 +57,6 @@ MING_FLASH_OMNI_PIPELINE = PipelineConfig(
hf_config_name="talker_config",
engine_output_type="audio",
tokenizer_subdir="talker/llm",
custom_process_input_func=f"{_PROC}.thinker2talker",
sync_process_input_func=f"{_PROC}.thinker2talker_token_only",
),
),
@@ -38,7 +38,6 @@ QWEN2_5_OMNI_PIPELINE = PipelineConfig(
execution_type=StageExecutionType.LLM_AR,
input_sources=(0,),
engine_output_type="latent",
custom_process_input_func=f"{_PROC}.thinker2talker",
sync_process_input_func=f"{_PROC}.thinker2talker_token_only",
custom_process_next_stage_input_func=f"{_PROC}.talker2code2wav_full_payload",
sampling_constraints={
@@ -54,7 +53,6 @@ QWEN2_5_OMNI_PIPELINE = PipelineConfig(
final_output=True,
final_output_type="audio",
engine_output_type="audio",
custom_process_input_func=f"{_PROC}.talker2code2wav",
sync_process_input_func=f"{_PROC}.talker2code2wav_token_only",
sampling_constraints={"detokenize": True},
),
@@ -23,33 +23,6 @@ def _filter_audio_codes(token_ids: list[int]) -> list[int]:
return audio_codes
def llm2code2wav(
source_outputs: list[Any],
prompt: Any = None,
requires_multimodal_data: bool = False,
) -> list[OmniTokensPrompt]:
"""Legacy orchestrator-path builder (retained for async_chunk + back-compat).
The non-async-chunk path now goes through ``llm2code2wav_token_only`` +
worker connector + ``llm2code2wav_full_payload``.
"""
talker_outputs = source_outputs
code2wav_inputs = []
for talker_output in talker_outputs:
output = talker_output.outputs[0]
audio_codes = _filter_audio_codes(list(output.token_ids))
code2wav_inputs.append(
OmniTokensPrompt(
prompt_token_ids=audio_codes,
multi_modal_data=None,
mm_processor_kwargs=None,
)
)
return code2wav_inputs
def llm2code2wav_token_only(
source_outputs: list[Any],
prompt: Any = None,
@@ -500,25 +500,6 @@ def _build_talker_inputs(
return talker_inputs
def thinker2talker(
source_outputs: list[Any],
prompt: OmniTokensPrompt | TextPrompt | None = None,
_requires_multimodal_data: bool = False,
_streaming_context: Any | None = None,
) -> list[OmniTokensPrompt]:
"""Build talker stage inputs from thinker stage outputs."""
return _build_talker_inputs(source_outputs, prompt)
# ming_flash_omni is not in ``_OMNI_CONNECTOR_INIT_ARCHS`` or
# ``_FULL_PAYLOAD_INPUT_STAGES``, so the worker connector is not
# initialised for this arch and the consumer never waits on a connector
# payload. Data flows through ``additional_information`` written by
# ``thinker2talker_token_only`` (wired as ``sync_process_input_func``
# in the pipeline) or ``thinker2talker`` (wired as
# ``custom_process_input_func``).
def thinker2talker_token_only(
source_outputs: list[Any],
prompt: OmniTokensPrompt | TextPrompt | None = None,
@@ -531,29 +512,9 @@ def thinker2talker_token_only(
thinker2talker_token_only._is_sync_input = True
def thinker2talker_full_payload(
transfer_manager,
pooling_output,
request,
):
"""Producer-side payload builder — no-op.
ming_flash_omni's thinker emits no heavy tensor to ship via the
worker connector (the bridge passes text only, and speaker metadata
arrives through the USER request's additional_information).
ming_flash_omni is not in ``_OMNI_CONNECTOR_INIT_ARCHS`` so this
function is never invoked at runtime; it is retained for forward
compatibility with the connector path.
"""
del transfer_manager, pooling_output, request
return None
__all__ = [
"CFG_TEXT_SUFFIX",
"expand_cfg_prompts",
"thinker2imagegen",
"thinker2talker",
"thinker2talker_full_payload",
"thinker2talker_token_only",
]
@@ -20,78 +20,6 @@ TALKER_CODEC_START_TOKEN_ID = 8293
TALKER_CODEC_END_TOKEN_ID = 8294
def thinker2talker(
source_outputs,
prompt: OmniTokensPrompt | TextPrompt = None,
requires_multimodal_data: bool = False,
):
thinker_outputs = source_outputs
talker_inputs = []
if not isinstance(prompt, list):
prompt = [prompt]
multi_modal_data = {
thinker_output.request_id: p.get("multi_modal_data", None) for thinker_output, p in zip(thinker_outputs, prompt)
}
for i, thinker_output in enumerate(thinker_outputs):
output = thinker_output.outputs[0]
prompt_token_ids = thinker_output.prompt_token_ids
thinker_output_ids = output.cumulative_token_ids
prompt_token_ids_len = len(prompt_token_ids)
mm: OmniPayload = output.multimodal_output
latent = mm["latent"]
thinker_hidden_states = latent.clone().detach().to(latent.device)
decode_hidden = thinker_hidden_states[prompt_token_ids_len:].to(torch.float32)
prefill_hidden = thinker_hidden_states[:prompt_token_ids_len].to(torch.float32)
additional_information = to_dict(
OmniPayloadStruct(
hidden_states=HiddenStatesStruct(output=decode_hidden, output_shape=list(decode_hidden.shape)),
embed=EmbeddingsStruct(prefill=prefill_hidden, prefill_shape=list(prefill_hidden.shape)),
ids=IdsStruct(prompt=list(prompt_token_ids), output=list(thinker_output_ids)),
)
)
talker_inputs.append(
OmniTokensPrompt(
prompt_token_ids=[TALKER_CODEC_START_TOKEN_ID]
+ [TALKER_CODEC_PAD_TOKEN_ID] * (len(prompt_token_ids))
+ [TALKER_CODEC_END_TOKEN_ID],
additional_information=additional_information,
multi_modal_data=(
multi_modal_data[thinker_output.request_id]
if requires_multimodal_data and multi_modal_data is not None
else None
),
mm_processor_kwargs=None,
)
)
return talker_inputs
def talker2code2wav(
source_outputs,
_prompt: OmniTokensPrompt | TextPrompt = None,
_requires_multimodal_data: bool = False,
):
code2wav_inputs = []
for talker_output in source_outputs:
output = talker_output.outputs[0]
token_ids = list(output.cumulative_token_ids)
if token_ids and token_ids[0] == TALKER_CODEC_START_TOKEN_ID:
token_ids = token_ids[1:]
if token_ids and token_ids[-1] == TALKER_CODEC_END_TOKEN_ID:
token_ids = token_ids[:-1]
if not token_ids:
continue
code2wav_inputs.append(
OmniTokensPrompt(
prompt_token_ids=token_ids,
multi_modal_data=None,
mm_processor_kwargs=None,
)
)
return code2wav_inputs
# ============================================================================
# Worker-connector data plane (non-async-chunk path).
# Both transitions ship payloads via the worker connector
@@ -101,9 +29,7 @@ def talker2code2wav(
# packs an OmniPayload-shaped dict (embed.prefill /
# hidden_states.output / ids.prompt / ids.output) for the talker, which
# the talker's ``talker_preprocess`` reads from
# ``model_intermediate_buffer``. The shape matches what legacy
# ``thinker2talker`` writes into ``additional_information`` as a debug
# fallback; ``thinker2talker_token_only`` only allocates prompt slots.
# ``model_intermediate_buffer``.
# - talker->code2wav strips TALKER_CODEC_{START,END} boundary tokens
# and ships the codec token ids.
# ============================================================================
@@ -212,9 +138,7 @@ def talker2code2wav_full_payload(
# prefill+decode trajectory and packs an OmniPayload-shaped dict that
# the talker's ``talker_preprocess`` reads from
# ``model_intermediate_buffer``. ``thinker2talker_token_only`` only
# allocates the talker's codec prompt slots; legacy
# ``thinker2talker`` above remains as a debug fallback that bundles the
# same shape into ``additional_information``.
# allocates the talker's codec prompt slots.
# ============================================================================