From e9d4931781e119363fe1ecb0be7963a351c556ad Mon Sep 17 00:00:00 2001 From: Isaac Hung <83223650+Nughm3@users.noreply.github.com> Date: Mon, 29 Jun 2026 11:17:25 +0800 Subject: [PATCH] [Core] Remove dead custom_process_input_func hooks in stage input processors (#4531) Signed-off-by: Isaac Hung --- docs/configuration/stage_configs.md | 1 - .../test_qwen3_omni_streaming_helpers.py | 18 ++--- tests/test_config_factory.py | 30 ++++--- vllm_omni/config/omni_config.py | 14 ++-- vllm_omni/config/stage_config.py | 33 +++++--- .../models/covo_audio/pipeline.py | 1 - .../models/ming_flash_omni/pipeline.py | 5 +- .../models/qwen2_5_omni/pipeline.py | 2 - .../stage_input_processors/covo_audio.py | 27 ------- .../stage_input_processors/ming_flash_omni.py | 39 --------- .../stage_input_processors/qwen2_5_omni.py | 80 +------------------ 11 files changed, 59 insertions(+), 191 deletions(-) diff --git a/docs/configuration/stage_configs.md b/docs/configuration/stage_configs.md index 3a17f1c40..ff10b6854 100644 --- a/docs/configuration/stage_configs.md +++ b/docs/configuration/stage_configs.md @@ -290,7 +290,6 @@ stage_args: enable_prefix_caching: false engine_output_type: latent engine_input_source: [0] - custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker default_sampling_params: temperature: 0.9 top_p: 0.8 diff --git a/tests/model_executor/stage_input_processors/test_qwen3_omni_streaming_helpers.py b/tests/model_executor/stage_input_processors/test_qwen3_omni_streaming_helpers.py index 52ca29905..69b612a05 100644 --- a/tests/model_executor/stage_input_processors/test_qwen3_omni_streaming_helpers.py +++ b/tests/model_executor/stage_input_processors/test_qwen3_omni_streaming_helpers.py @@ -921,7 +921,6 @@ def test_cosyvoice3_full_payload_replace_keys_present() -> None: def test_ming_flash_omni_thinker2talker_token_only_smoke() -> None: """Smoke: ming_flash_omni token-only carries voice metadata.""" from vllm_omni.model_executor.stage_input_processors.ming_flash_omni import ( - thinker2talker, thinker2talker_token_only, ) @@ -939,15 +938,14 @@ def test_ming_flash_omni_thinker2talker_token_only_smoke() -> None: src = [_Wrap("hello world")] prompt = _Prompt({"voice_name": "ZH_FEMALE", "prompt_text": "ref text"}) - for func in (thinker2talker, thinker2talker_token_only): - out = func(src, prompt=prompt) - assert len(out) == 1 - assert out[0]["prompt_token_ids"] == [0] - info = out[0]["additional_information"] - assert info["text"] == "hello world" - assert info["voice_name"] == "ZH_FEMALE" - assert info["prompt_text"] == "ref text" - assert info["ming_task"] == "omni" + out = thinker2talker_token_only(src, prompt=prompt) + assert len(out) == 1 + assert out[0]["prompt_token_ids"] == [0] + info = out[0]["additional_information"] + assert info["text"] == "hello world" + assert info["voice_name"] == "ZH_FEMALE" + assert info["prompt_text"] == "ref text" + assert info["ming_task"] == "omni" def test_qwen2_5_omni_thinker2talker_token_only_smoke() -> None: diff --git a/tests/test_config_factory.py b/tests/test_config_factory.py index b285a99cc..63a67d652 100644 --- a/tests/test_config_factory.py +++ b/tests/test_config_factory.py @@ -1060,7 +1060,11 @@ class TestQwen2_5OmniPipeline: assert isinstance(s, StagePipelineConfig) assert s.input_sources == (0,) assert s.sampling_constraints["stop_token_ids"] == [8294] - assert s.custom_process_input_func is not None + # thinker2talker was removed: qwen2_5_omni has no async_chunk support, + # so sync_process_input_func always wins and custom_process_input_func + # was dead code. + assert s.custom_process_input_func is None + assert s.sync_process_input_func is not None def test_code2wav(self): p = StageConfigFactory.resolve_pipeline_config("qwen2_5_omni") @@ -1191,10 +1195,14 @@ class TestMingFlashOmniPipeline: # Per-stage model_arch override (Ming talker has its own self-contained LLM) assert s.model_arch == "MingFlashOmniTalkerForConditionalGeneration" assert s.tokenizer_subdir == "talker/llm" - assert s.custom_process_input_func is not None + # thinker2talker was removed: ming_flash_omni has no async_chunk support + # and both thinker2talker / thinker2talker_token_only called _build_talker_inputs + # identically, so custom_process_input_func was dead code. + assert s.custom_process_input_func is None + assert s.sync_process_input_func is not None def test_talker_stage_processor_wiring_resolves(self): - """The custom_process_input_func string must point to a real callable. + """The sync_process_input_func string must point to a real callable. Lazy string references only fail at first inference otherwise — this catches typos in the pipeline declaration at import / registration time. @@ -1204,7 +1212,7 @@ class TestMingFlashOmniPipeline: s = p.get_stage(1) assert isinstance(s, StagePipelineConfig) - module_path, _, attr = s.custom_process_input_func.rpartition(".") + module_path, _, attr = s.sync_process_input_func.rpartition(".") module = importlib.import_module(module_path) assert callable(getattr(module, attr)) @@ -1720,11 +1728,12 @@ class TestSentinelDefaultPrecedence: ) def test_ming_flash_omni_topology(self): - """Guard ming_flash_omni's PR3 cleanup: stage 0 has no full-payload - producer hook (the connector path was removed as fake -- arch is not - in ``_FULL_PAYLOAD_INPUT_STAGES``), and stage 1 still wires the - legacy ``thinker2talker`` (custom_process_input_func) plus the - ``thinker2talker_token_only`` placeholder (sync_process_input_func). + """Guard ming_flash_omni's SIP cleanup: stage 0 has no full-payload + producer hook (arch is not in ``_FULL_PAYLOAD_INPUT_STAGES``), and + stage 1 uses only ``thinker2talker_token_only`` (sync_process_input_func). + The dead ``thinker2talker`` (custom_process_input_func) was removed + because ming_flash_omni has no async_chunk support and both functions + called ``_build_talker_inputs`` identically. Merge under either async_chunk mode must not re-introduce a stage-0 full-payload hook.""" pipeline = StageConfigFactory.resolve_pipeline_config("ming_flash_omni") @@ -1735,8 +1744,7 @@ class TestSentinelDefaultPrecedence: "ming_flash_omni stage 0 must not declare a full-payload producer " "(connector path is not active for this arch)." ) - assert stage1.custom_process_input_func is not None - assert stage1.custom_process_input_func.endswith("thinker2talker") + assert stage1.custom_process_input_func is None assert stage1.sync_process_input_func is not None assert stage1.sync_process_input_func.endswith("thinker2talker_token_only") diff --git a/vllm_omni/config/omni_config.py b/vllm_omni/config/omni_config.py index 02b413064..c44bd3ea9 100644 --- a/vllm_omni/config/omni_config.py +++ b/vllm_omni/config/omni_config.py @@ -180,15 +180,17 @@ def _first_defined(*values: Any) -> Any: def _validate_async_chunk_support(pipeline: PipelineConfig, deploy: DeployConfig) -> None: - if deploy.async_chunk and not any( - stage.async_chunk_process_next_stage_input_func or stage.custom_process_next_stage_input_func - for stage in pipeline.stages + has_inter_stage_edges = any(stage.input_sources for stage in pipeline.stages) + if ( + deploy.async_chunk + and has_inter_stage_edges + and not any(stage.async_chunk_process_next_stage_input_func for stage in pipeline.stages) ): raise ValueError( f"Pipeline {pipeline.model_type!r} has async_chunk=True in deploy but no stage " - "declares a next-stage input processor " - "(``async_chunk_process_next_stage_input_func`` or ``custom_process_next_stage_input_func``). " - "Either set async_chunk=False or implement an async-chunk processor on the pipeline." + "declares a dedicated async-chunk next-stage processor " + "(``async_chunk_process_next_stage_input_func``). " + "Either set async_chunk=False or implement an async-chunk producer on the pipeline." ) diff --git a/vllm_omni/config/stage_config.py b/vllm_omni/config/stage_config.py index d1c728732..a269216da 100644 --- a/vllm_omni/config/stage_config.py +++ b/vllm_omni/config/stage_config.py @@ -603,7 +603,8 @@ def _apply_platform_overrides( if platform is None: from vllm_omni.platforms import current_omni_platform - platform = current_omni_platform.device_name.lower() + device_name = current_omni_platform.device_name + platform = device_name.lower() if device_name is not None else None if platform is None or deploy.platforms is None: return deploy platform_section = deploy.platforms.get(platform) @@ -784,21 +785,27 @@ def merge_pipeline_deploy( if len(pipeline.stages) <= 1: deploy.async_chunk = False - # A pipeline supports async_chunk if any stage has either an explicit - # async-chunk-only processor slot OR a custom next-stage processor (some - # pipelines like qwen3_omni wire async-chunk processing directly through - # ``custom_process_next_stage_input_func``). Only raise when neither is - # present — that's the "user enabled async_chunk but pipeline has no - # inter-stage processing at all" case. - if deploy.async_chunk and not any( - ps.async_chunk_process_next_stage_input_func or ps.custom_process_next_stage_input_func - for ps in pipeline.stages + # async_chunk only applies to multi-stage pipelines: a pipeline with no + # consumer stages (every stage has empty input_sources) has no inter-stage + # edges, so async_chunk is a no-op and we skip the check entirely. + # For pipelines that DO have inter-stage edges, require a dedicated per-step + # async producer (``async_chunk_process_next_stage_input_func``). + # ``custom_process_next_stage_input_func`` is the full-payload / connector-path + # producer and does NOT imply async_chunk support — pipelines like qwen2_5_omni + # and covo_audio have it but removed their consumer-side ``custom_process_input_func`` + # because they don't support async_chunk, so accepting them here would silently + # miswire the consumer stage instead of raising a clear error. + _has_inter_stage_edges = any(ps.input_sources for ps in pipeline.stages) + if ( + deploy.async_chunk + and _has_inter_stage_edges + and not any(ps.async_chunk_process_next_stage_input_func for ps in pipeline.stages) ): raise ValueError( f"Pipeline {pipeline.model_type!r} has async_chunk=True in deploy but no stage " - "declares a next-stage input processor " - "(``async_chunk_process_next_stage_input_func`` or ``custom_process_next_stage_input_func``). " - "Either set async_chunk=False or implement an async-chunk processor on the pipeline." + "declares a dedicated async-chunk next-stage processor " + "(``async_chunk_process_next_stage_input_func``). " + "Either set async_chunk=False or implement an async-chunk producer on the pipeline." ) result: list[StageConfig] = [] diff --git a/vllm_omni/model_executor/models/covo_audio/pipeline.py b/vllm_omni/model_executor/models/covo_audio/pipeline.py index 97053e328..f604bc57d 100644 --- a/vllm_omni/model_executor/models/covo_audio/pipeline.py +++ b/vllm_omni/model_executor/models/covo_audio/pipeline.py @@ -44,7 +44,6 @@ COVO_AUDIO_PIPELINE = PipelineConfig( final_output=True, final_output_type="audio", engine_output_type="audio", - custom_process_input_func=f"{_PROC}.llm2code2wav", sync_process_input_func=f"{_PROC}.llm2code2wav_token_only", sampling_constraints={"detokenize": False}, ), diff --git a/vllm_omni/model_executor/models/ming_flash_omni/pipeline.py b/vllm_omni/model_executor/models/ming_flash_omni/pipeline.py index 7ebbfd9d8..e416a117e 100644 --- a/vllm_omni/model_executor/models/ming_flash_omni/pipeline.py +++ b/vllm_omni/model_executor/models/ming_flash_omni/pipeline.py @@ -6,8 +6,8 @@ Stage 0: Thinker — multimodal understanding + text generation. Stage 1: Talker — text -> audio waveform via CFM + AudioVAE. The thinker -> talker bridge passes the detokenized text rather than -hidden states through `ming_flash_omni.thinker2talker`; the talker has a -self-contained Qwen2 LLM that retokenizes the string itself. +hidden states; the talker has a self-contained Qwen2 LLM that retokenizes +the string itself. """ from vllm_omni.config.stage_config import ( @@ -57,7 +57,6 @@ MING_FLASH_OMNI_PIPELINE = PipelineConfig( hf_config_name="talker_config", engine_output_type="audio", tokenizer_subdir="talker/llm", - custom_process_input_func=f"{_PROC}.thinker2talker", sync_process_input_func=f"{_PROC}.thinker2talker_token_only", ), ), diff --git a/vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py b/vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py index afd0a92a5..b4ed8efee 100644 --- a/vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py +++ b/vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py @@ -38,7 +38,6 @@ QWEN2_5_OMNI_PIPELINE = PipelineConfig( execution_type=StageExecutionType.LLM_AR, input_sources=(0,), engine_output_type="latent", - custom_process_input_func=f"{_PROC}.thinker2talker", sync_process_input_func=f"{_PROC}.thinker2talker_token_only", custom_process_next_stage_input_func=f"{_PROC}.talker2code2wav_full_payload", sampling_constraints={ @@ -54,7 +53,6 @@ QWEN2_5_OMNI_PIPELINE = PipelineConfig( final_output=True, final_output_type="audio", engine_output_type="audio", - custom_process_input_func=f"{_PROC}.talker2code2wav", sync_process_input_func=f"{_PROC}.talker2code2wav_token_only", sampling_constraints={"detokenize": True}, ), diff --git a/vllm_omni/model_executor/stage_input_processors/covo_audio.py b/vllm_omni/model_executor/stage_input_processors/covo_audio.py index 7b5ed8c26..a15cca012 100644 --- a/vllm_omni/model_executor/stage_input_processors/covo_audio.py +++ b/vllm_omni/model_executor/stage_input_processors/covo_audio.py @@ -23,33 +23,6 @@ def _filter_audio_codes(token_ids: list[int]) -> list[int]: return audio_codes -def llm2code2wav( - source_outputs: list[Any], - prompt: Any = None, - requires_multimodal_data: bool = False, -) -> list[OmniTokensPrompt]: - """Legacy orchestrator-path builder (retained for async_chunk + back-compat). - - The non-async-chunk path now goes through ``llm2code2wav_token_only`` + - worker connector + ``llm2code2wav_full_payload``. - """ - talker_outputs = source_outputs - code2wav_inputs = [] - - for talker_output in talker_outputs: - output = talker_output.outputs[0] - audio_codes = _filter_audio_codes(list(output.token_ids)) - code2wav_inputs.append( - OmniTokensPrompt( - prompt_token_ids=audio_codes, - multi_modal_data=None, - mm_processor_kwargs=None, - ) - ) - - return code2wav_inputs - - def llm2code2wav_token_only( source_outputs: list[Any], prompt: Any = None, diff --git a/vllm_omni/model_executor/stage_input_processors/ming_flash_omni.py b/vllm_omni/model_executor/stage_input_processors/ming_flash_omni.py index 0c38bb807..653e34777 100644 --- a/vllm_omni/model_executor/stage_input_processors/ming_flash_omni.py +++ b/vllm_omni/model_executor/stage_input_processors/ming_flash_omni.py @@ -500,25 +500,6 @@ def _build_talker_inputs( return talker_inputs -def thinker2talker( - source_outputs: list[Any], - prompt: OmniTokensPrompt | TextPrompt | None = None, - _requires_multimodal_data: bool = False, - _streaming_context: Any | None = None, -) -> list[OmniTokensPrompt]: - """Build talker stage inputs from thinker stage outputs.""" - return _build_talker_inputs(source_outputs, prompt) - - -# ming_flash_omni is not in ``_OMNI_CONNECTOR_INIT_ARCHS`` or -# ``_FULL_PAYLOAD_INPUT_STAGES``, so the worker connector is not -# initialised for this arch and the consumer never waits on a connector -# payload. Data flows through ``additional_information`` written by -# ``thinker2talker_token_only`` (wired as ``sync_process_input_func`` -# in the pipeline) or ``thinker2talker`` (wired as -# ``custom_process_input_func``). - - def thinker2talker_token_only( source_outputs: list[Any], prompt: OmniTokensPrompt | TextPrompt | None = None, @@ -531,29 +512,9 @@ def thinker2talker_token_only( thinker2talker_token_only._is_sync_input = True -def thinker2talker_full_payload( - transfer_manager, - pooling_output, - request, -): - """Producer-side payload builder — no-op. - - ming_flash_omni's thinker emits no heavy tensor to ship via the - worker connector (the bridge passes text only, and speaker metadata - arrives through the USER request's additional_information). - ming_flash_omni is not in ``_OMNI_CONNECTOR_INIT_ARCHS`` so this - function is never invoked at runtime; it is retained for forward - compatibility with the connector path. - """ - del transfer_manager, pooling_output, request - return None - - __all__ = [ "CFG_TEXT_SUFFIX", "expand_cfg_prompts", "thinker2imagegen", - "thinker2talker", - "thinker2talker_full_payload", "thinker2talker_token_only", ] diff --git a/vllm_omni/model_executor/stage_input_processors/qwen2_5_omni.py b/vllm_omni/model_executor/stage_input_processors/qwen2_5_omni.py index 225674360..42377faa8 100644 --- a/vllm_omni/model_executor/stage_input_processors/qwen2_5_omni.py +++ b/vllm_omni/model_executor/stage_input_processors/qwen2_5_omni.py @@ -20,78 +20,6 @@ TALKER_CODEC_START_TOKEN_ID = 8293 TALKER_CODEC_END_TOKEN_ID = 8294 -def thinker2talker( - source_outputs, - prompt: OmniTokensPrompt | TextPrompt = None, - requires_multimodal_data: bool = False, -): - thinker_outputs = source_outputs - talker_inputs = [] - if not isinstance(prompt, list): - prompt = [prompt] - multi_modal_data = { - thinker_output.request_id: p.get("multi_modal_data", None) for thinker_output, p in zip(thinker_outputs, prompt) - } - - for i, thinker_output in enumerate(thinker_outputs): - output = thinker_output.outputs[0] - prompt_token_ids = thinker_output.prompt_token_ids - thinker_output_ids = output.cumulative_token_ids - prompt_token_ids_len = len(prompt_token_ids) - mm: OmniPayload = output.multimodal_output - latent = mm["latent"] - thinker_hidden_states = latent.clone().detach().to(latent.device) - decode_hidden = thinker_hidden_states[prompt_token_ids_len:].to(torch.float32) - prefill_hidden = thinker_hidden_states[:prompt_token_ids_len].to(torch.float32) - additional_information = to_dict( - OmniPayloadStruct( - hidden_states=HiddenStatesStruct(output=decode_hidden, output_shape=list(decode_hidden.shape)), - embed=EmbeddingsStruct(prefill=prefill_hidden, prefill_shape=list(prefill_hidden.shape)), - ids=IdsStruct(prompt=list(prompt_token_ids), output=list(thinker_output_ids)), - ) - ) - talker_inputs.append( - OmniTokensPrompt( - prompt_token_ids=[TALKER_CODEC_START_TOKEN_ID] - + [TALKER_CODEC_PAD_TOKEN_ID] * (len(prompt_token_ids)) - + [TALKER_CODEC_END_TOKEN_ID], - additional_information=additional_information, - multi_modal_data=( - multi_modal_data[thinker_output.request_id] - if requires_multimodal_data and multi_modal_data is not None - else None - ), - mm_processor_kwargs=None, - ) - ) - return talker_inputs - - -def talker2code2wav( - source_outputs, - _prompt: OmniTokensPrompt | TextPrompt = None, - _requires_multimodal_data: bool = False, -): - code2wav_inputs = [] - for talker_output in source_outputs: - output = talker_output.outputs[0] - token_ids = list(output.cumulative_token_ids) - if token_ids and token_ids[0] == TALKER_CODEC_START_TOKEN_ID: - token_ids = token_ids[1:] - if token_ids and token_ids[-1] == TALKER_CODEC_END_TOKEN_ID: - token_ids = token_ids[:-1] - if not token_ids: - continue - code2wav_inputs.append( - OmniTokensPrompt( - prompt_token_ids=token_ids, - multi_modal_data=None, - mm_processor_kwargs=None, - ) - ) - return code2wav_inputs - - # ============================================================================ # Worker-connector data plane (non-async-chunk path). # Both transitions ship payloads via the worker connector @@ -101,9 +29,7 @@ def talker2code2wav( # packs an OmniPayload-shaped dict (embed.prefill / # hidden_states.output / ids.prompt / ids.output) for the talker, which # the talker's ``talker_preprocess`` reads from -# ``model_intermediate_buffer``. The shape matches what legacy -# ``thinker2talker`` writes into ``additional_information`` as a debug -# fallback; ``thinker2talker_token_only`` only allocates prompt slots. +# ``model_intermediate_buffer``. # - talker->code2wav strips TALKER_CODEC_{START,END} boundary tokens # and ships the codec token ids. # ============================================================================ @@ -212,9 +138,7 @@ def talker2code2wav_full_payload( # prefill+decode trajectory and packs an OmniPayload-shaped dict that # the talker's ``talker_preprocess`` reads from # ``model_intermediate_buffer``. ``thinker2talker_token_only`` only -# allocates the talker's codec prompt slots; legacy -# ``thinker2talker`` above remains as a debug fallback that bundles the -# same shape into ``additional_information``. +# allocates the talker's codec prompt slots. # ============================================================================