[Core] Remove dead custom_process_input_func hooks in stage input processors (#4531)
Signed-off-by: Isaac Hung <manhei5001@gmail.com>
This commit is contained in:
1 parent
cd35572069
commit
e9d4931781
11 files changed
+59
-191
No files matched your search
@@ -290,7 +290,6 @@ stage_args:
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: latent
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_p: 0.8
|
||||
|
||||
@@ -921,7 +921,6 @@ def test_cosyvoice3_full_payload_replace_keys_present() -> None:
|
||||
def test_ming_flash_omni_thinker2talker_token_only_smoke() -> None:
|
||||
"""Smoke: ming_flash_omni token-only carries voice metadata."""
|
||||
from vllm_omni.model_executor.stage_input_processors.ming_flash_omni import (
|
||||
thinker2talker,
|
||||
thinker2talker_token_only,
|
||||
)
|
||||
|
||||
@@ -939,15 +938,14 @@ def test_ming_flash_omni_thinker2talker_token_only_smoke() -> None:
|
||||
|
||||
src = [_Wrap("hello world")]
|
||||
prompt = _Prompt({"voice_name": "ZH_FEMALE", "prompt_text": "ref text"})
|
||||
for func in (thinker2talker, thinker2talker_token_only):
|
||||
out = func(src, prompt=prompt)
|
||||
assert len(out) == 1
|
||||
assert out[0]["prompt_token_ids"] == [0]
|
||||
info = out[0]["additional_information"]
|
||||
assert info["text"] == "hello world"
|
||||
assert info["voice_name"] == "ZH_FEMALE"
|
||||
assert info["prompt_text"] == "ref text"
|
||||
assert info["ming_task"] == "omni"
|
||||
out = thinker2talker_token_only(src, prompt=prompt)
|
||||
assert len(out) == 1
|
||||
assert out[0]["prompt_token_ids"] == [0]
|
||||
info = out[0]["additional_information"]
|
||||
assert info["text"] == "hello world"
|
||||
assert info["voice_name"] == "ZH_FEMALE"
|
||||
assert info["prompt_text"] == "ref text"
|
||||
assert info["ming_task"] == "omni"
|
||||
|
||||
|
||||
def test_qwen2_5_omni_thinker2talker_token_only_smoke() -> None:
|
||||
|
||||
@@ -1060,7 +1060,11 @@ class TestQwen2_5OmniPipeline:
|
||||
assert isinstance(s, StagePipelineConfig)
|
||||
assert s.input_sources == (0,)
|
||||
assert s.sampling_constraints["stop_token_ids"] == [8294]
|
||||
assert s.custom_process_input_func is not None
|
||||
# thinker2talker was removed: qwen2_5_omni has no async_chunk support,
|
||||
# so sync_process_input_func always wins and custom_process_input_func
|
||||
# was dead code.
|
||||
assert s.custom_process_input_func is None
|
||||
assert s.sync_process_input_func is not None
|
||||
|
||||
def test_code2wav(self):
|
||||
p = StageConfigFactory.resolve_pipeline_config("qwen2_5_omni")
|
||||
@@ -1191,10 +1195,14 @@ class TestMingFlashOmniPipeline:
|
||||
# Per-stage model_arch override (Ming talker has its own self-contained LLM)
|
||||
assert s.model_arch == "MingFlashOmniTalkerForConditionalGeneration"
|
||||
assert s.tokenizer_subdir == "talker/llm"
|
||||
assert s.custom_process_input_func is not None
|
||||
# thinker2talker was removed: ming_flash_omni has no async_chunk support
|
||||
# and both thinker2talker / thinker2talker_token_only called _build_talker_inputs
|
||||
# identically, so custom_process_input_func was dead code.
|
||||
assert s.custom_process_input_func is None
|
||||
assert s.sync_process_input_func is not None
|
||||
|
||||
def test_talker_stage_processor_wiring_resolves(self):
|
||||
"""The custom_process_input_func string must point to a real callable.
|
||||
"""The sync_process_input_func string must point to a real callable.
|
||||
|
||||
Lazy string references only fail at first inference otherwise — this
|
||||
catches typos in the pipeline declaration at import / registration time.
|
||||
@@ -1204,7 +1212,7 @@ class TestMingFlashOmniPipeline:
|
||||
|
||||
s = p.get_stage(1)
|
||||
assert isinstance(s, StagePipelineConfig)
|
||||
module_path, _, attr = s.custom_process_input_func.rpartition(".")
|
||||
module_path, _, attr = s.sync_process_input_func.rpartition(".")
|
||||
module = importlib.import_module(module_path)
|
||||
assert callable(getattr(module, attr))
|
||||
|
||||
@@ -1720,11 +1728,12 @@ class TestSentinelDefaultPrecedence:
|
||||
)
|
||||
|
||||
def test_ming_flash_omni_topology(self):
|
||||
"""Guard ming_flash_omni's PR3 cleanup: stage 0 has no full-payload
|
||||
producer hook (the connector path was removed as fake -- arch is not
|
||||
in ``_FULL_PAYLOAD_INPUT_STAGES``), and stage 1 still wires the
|
||||
legacy ``thinker2talker`` (custom_process_input_func) plus the
|
||||
``thinker2talker_token_only`` placeholder (sync_process_input_func).
|
||||
"""Guard ming_flash_omni's SIP cleanup: stage 0 has no full-payload
|
||||
producer hook (arch is not in ``_FULL_PAYLOAD_INPUT_STAGES``), and
|
||||
stage 1 uses only ``thinker2talker_token_only`` (sync_process_input_func).
|
||||
The dead ``thinker2talker`` (custom_process_input_func) was removed
|
||||
because ming_flash_omni has no async_chunk support and both functions
|
||||
called ``_build_talker_inputs`` identically.
|
||||
Merge under either async_chunk mode must not re-introduce a
|
||||
stage-0 full-payload hook."""
|
||||
pipeline = StageConfigFactory.resolve_pipeline_config("ming_flash_omni")
|
||||
@@ -1735,8 +1744,7 @@ class TestSentinelDefaultPrecedence:
|
||||
"ming_flash_omni stage 0 must not declare a full-payload producer "
|
||||
"(connector path is not active for this arch)."
|
||||
)
|
||||
assert stage1.custom_process_input_func is not None
|
||||
assert stage1.custom_process_input_func.endswith("thinker2talker")
|
||||
assert stage1.custom_process_input_func is None
|
||||
assert stage1.sync_process_input_func is not None
|
||||
assert stage1.sync_process_input_func.endswith("thinker2talker_token_only")
|
||||
|
||||
|
||||
@@ -180,15 +180,17 @@ def _first_defined(*values: Any) -> Any:
|
||||
|
||||
|
||||
def _validate_async_chunk_support(pipeline: PipelineConfig, deploy: DeployConfig) -> None:
|
||||
if deploy.async_chunk and not any(
|
||||
stage.async_chunk_process_next_stage_input_func or stage.custom_process_next_stage_input_func
|
||||
for stage in pipeline.stages
|
||||
has_inter_stage_edges = any(stage.input_sources for stage in pipeline.stages)
|
||||
if (
|
||||
deploy.async_chunk
|
||||
and has_inter_stage_edges
|
||||
and not any(stage.async_chunk_process_next_stage_input_func for stage in pipeline.stages)
|
||||
):
|
||||
raise ValueError(
|
||||
f"Pipeline {pipeline.model_type!r} has async_chunk=True in deploy but no stage "
|
||||
"declares a next-stage input processor "
|
||||
"(``async_chunk_process_next_stage_input_func`` or ``custom_process_next_stage_input_func``). "
|
||||
"Either set async_chunk=False or implement an async-chunk processor on the pipeline."
|
||||
"declares a dedicated async-chunk next-stage processor "
|
||||
"(``async_chunk_process_next_stage_input_func``). "
|
||||
"Either set async_chunk=False or implement an async-chunk producer on the pipeline."
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -603,7 +603,8 @@ def _apply_platform_overrides(
|
||||
if platform is None:
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
platform = current_omni_platform.device_name.lower()
|
||||
device_name = current_omni_platform.device_name
|
||||
platform = device_name.lower() if device_name is not None else None
|
||||
if platform is None or deploy.platforms is None:
|
||||
return deploy
|
||||
platform_section = deploy.platforms.get(platform)
|
||||
@@ -784,21 +785,27 @@ def merge_pipeline_deploy(
|
||||
if len(pipeline.stages) <= 1:
|
||||
deploy.async_chunk = False
|
||||
|
||||
# A pipeline supports async_chunk if any stage has either an explicit
|
||||
# async-chunk-only processor slot OR a custom next-stage processor (some
|
||||
# pipelines like qwen3_omni wire async-chunk processing directly through
|
||||
# ``custom_process_next_stage_input_func``). Only raise when neither is
|
||||
# present — that's the "user enabled async_chunk but pipeline has no
|
||||
# inter-stage processing at all" case.
|
||||
if deploy.async_chunk and not any(
|
||||
ps.async_chunk_process_next_stage_input_func or ps.custom_process_next_stage_input_func
|
||||
for ps in pipeline.stages
|
||||
# async_chunk only applies to multi-stage pipelines: a pipeline with no
|
||||
# consumer stages (every stage has empty input_sources) has no inter-stage
|
||||
# edges, so async_chunk is a no-op and we skip the check entirely.
|
||||
# For pipelines that DO have inter-stage edges, require a dedicated per-step
|
||||
# async producer (``async_chunk_process_next_stage_input_func``).
|
||||
# ``custom_process_next_stage_input_func`` is the full-payload / connector-path
|
||||
# producer and does NOT imply async_chunk support — pipelines like qwen2_5_omni
|
||||
# and covo_audio have it but removed their consumer-side ``custom_process_input_func``
|
||||
# because they don't support async_chunk, so accepting them here would silently
|
||||
# miswire the consumer stage instead of raising a clear error.
|
||||
_has_inter_stage_edges = any(ps.input_sources for ps in pipeline.stages)
|
||||
if (
|
||||
deploy.async_chunk
|
||||
and _has_inter_stage_edges
|
||||
and not any(ps.async_chunk_process_next_stage_input_func for ps in pipeline.stages)
|
||||
):
|
||||
raise ValueError(
|
||||
f"Pipeline {pipeline.model_type!r} has async_chunk=True in deploy but no stage "
|
||||
"declares a next-stage input processor "
|
||||
"(``async_chunk_process_next_stage_input_func`` or ``custom_process_next_stage_input_func``). "
|
||||
"Either set async_chunk=False or implement an async-chunk processor on the pipeline."
|
||||
"declares a dedicated async-chunk next-stage processor "
|
||||
"(``async_chunk_process_next_stage_input_func``). "
|
||||
"Either set async_chunk=False or implement an async-chunk producer on the pipeline."
|
||||
)
|
||||
|
||||
result: list[StageConfig] = []
|
||||
|
||||
@@ -44,7 +44,6 @@ COVO_AUDIO_PIPELINE = PipelineConfig(
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
engine_output_type="audio",
|
||||
custom_process_input_func=f"{_PROC}.llm2code2wav",
|
||||
sync_process_input_func=f"{_PROC}.llm2code2wav_token_only",
|
||||
sampling_constraints={"detokenize": False},
|
||||
),
|
||||
|
||||
@@ -6,8 +6,8 @@ Stage 0: Thinker — multimodal understanding + text generation.
|
||||
Stage 1: Talker — text -> audio waveform via CFM + AudioVAE.
|
||||
|
||||
The thinker -> talker bridge passes the detokenized text rather than
|
||||
hidden states through `ming_flash_omni.thinker2talker`; the talker has a
|
||||
self-contained Qwen2 LLM that retokenizes the string itself.
|
||||
hidden states; the talker has a self-contained Qwen2 LLM that retokenizes
|
||||
the string itself.
|
||||
"""
|
||||
|
||||
from vllm_omni.config.stage_config import (
|
||||
@@ -57,7 +57,6 @@ MING_FLASH_OMNI_PIPELINE = PipelineConfig(
|
||||
hf_config_name="talker_config",
|
||||
engine_output_type="audio",
|
||||
tokenizer_subdir="talker/llm",
|
||||
custom_process_input_func=f"{_PROC}.thinker2talker",
|
||||
sync_process_input_func=f"{_PROC}.thinker2talker_token_only",
|
||||
),
|
||||
),
|
||||
|
||||
@@ -38,7 +38,6 @@ QWEN2_5_OMNI_PIPELINE = PipelineConfig(
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(0,),
|
||||
engine_output_type="latent",
|
||||
custom_process_input_func=f"{_PROC}.thinker2talker",
|
||||
sync_process_input_func=f"{_PROC}.thinker2talker_token_only",
|
||||
custom_process_next_stage_input_func=f"{_PROC}.talker2code2wav_full_payload",
|
||||
sampling_constraints={
|
||||
@@ -54,7 +53,6 @@ QWEN2_5_OMNI_PIPELINE = PipelineConfig(
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
engine_output_type="audio",
|
||||
custom_process_input_func=f"{_PROC}.talker2code2wav",
|
||||
sync_process_input_func=f"{_PROC}.talker2code2wav_token_only",
|
||||
sampling_constraints={"detokenize": True},
|
||||
),
|
||||
|
||||
@@ -23,33 +23,6 @@ def _filter_audio_codes(token_ids: list[int]) -> list[int]:
|
||||
return audio_codes
|
||||
|
||||
|
||||
def llm2code2wav(
|
||||
source_outputs: list[Any],
|
||||
prompt: Any = None,
|
||||
requires_multimodal_data: bool = False,
|
||||
) -> list[OmniTokensPrompt]:
|
||||
"""Legacy orchestrator-path builder (retained for async_chunk + back-compat).
|
||||
|
||||
The non-async-chunk path now goes through ``llm2code2wav_token_only`` +
|
||||
worker connector + ``llm2code2wav_full_payload``.
|
||||
"""
|
||||
talker_outputs = source_outputs
|
||||
code2wav_inputs = []
|
||||
|
||||
for talker_output in talker_outputs:
|
||||
output = talker_output.outputs[0]
|
||||
audio_codes = _filter_audio_codes(list(output.token_ids))
|
||||
code2wav_inputs.append(
|
||||
OmniTokensPrompt(
|
||||
prompt_token_ids=audio_codes,
|
||||
multi_modal_data=None,
|
||||
mm_processor_kwargs=None,
|
||||
)
|
||||
)
|
||||
|
||||
return code2wav_inputs
|
||||
|
||||
|
||||
def llm2code2wav_token_only(
|
||||
source_outputs: list[Any],
|
||||
prompt: Any = None,
|
||||
|
||||
@@ -500,25 +500,6 @@ def _build_talker_inputs(
|
||||
return talker_inputs
|
||||
|
||||
|
||||
def thinker2talker(
|
||||
source_outputs: list[Any],
|
||||
prompt: OmniTokensPrompt | TextPrompt | None = None,
|
||||
_requires_multimodal_data: bool = False,
|
||||
_streaming_context: Any | None = None,
|
||||
) -> list[OmniTokensPrompt]:
|
||||
"""Build talker stage inputs from thinker stage outputs."""
|
||||
return _build_talker_inputs(source_outputs, prompt)
|
||||
|
||||
|
||||
# ming_flash_omni is not in ``_OMNI_CONNECTOR_INIT_ARCHS`` or
|
||||
# ``_FULL_PAYLOAD_INPUT_STAGES``, so the worker connector is not
|
||||
# initialised for this arch and the consumer never waits on a connector
|
||||
# payload. Data flows through ``additional_information`` written by
|
||||
# ``thinker2talker_token_only`` (wired as ``sync_process_input_func``
|
||||
# in the pipeline) or ``thinker2talker`` (wired as
|
||||
# ``custom_process_input_func``).
|
||||
|
||||
|
||||
def thinker2talker_token_only(
|
||||
source_outputs: list[Any],
|
||||
prompt: OmniTokensPrompt | TextPrompt | None = None,
|
||||
@@ -531,29 +512,9 @@ def thinker2talker_token_only(
|
||||
thinker2talker_token_only._is_sync_input = True
|
||||
|
||||
|
||||
def thinker2talker_full_payload(
|
||||
transfer_manager,
|
||||
pooling_output,
|
||||
request,
|
||||
):
|
||||
"""Producer-side payload builder — no-op.
|
||||
|
||||
ming_flash_omni's thinker emits no heavy tensor to ship via the
|
||||
worker connector (the bridge passes text only, and speaker metadata
|
||||
arrives through the USER request's additional_information).
|
||||
ming_flash_omni is not in ``_OMNI_CONNECTOR_INIT_ARCHS`` so this
|
||||
function is never invoked at runtime; it is retained for forward
|
||||
compatibility with the connector path.
|
||||
"""
|
||||
del transfer_manager, pooling_output, request
|
||||
return None
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CFG_TEXT_SUFFIX",
|
||||
"expand_cfg_prompts",
|
||||
"thinker2imagegen",
|
||||
"thinker2talker",
|
||||
"thinker2talker_full_payload",
|
||||
"thinker2talker_token_only",
|
||||
]
|
||||
@@ -20,78 +20,6 @@ TALKER_CODEC_START_TOKEN_ID = 8293
|
||||
TALKER_CODEC_END_TOKEN_ID = 8294
|
||||
|
||||
|
||||
def thinker2talker(
|
||||
source_outputs,
|
||||
prompt: OmniTokensPrompt | TextPrompt = None,
|
||||
requires_multimodal_data: bool = False,
|
||||
):
|
||||
thinker_outputs = source_outputs
|
||||
talker_inputs = []
|
||||
if not isinstance(prompt, list):
|
||||
prompt = [prompt]
|
||||
multi_modal_data = {
|
||||
thinker_output.request_id: p.get("multi_modal_data", None) for thinker_output, p in zip(thinker_outputs, prompt)
|
||||
}
|
||||
|
||||
for i, thinker_output in enumerate(thinker_outputs):
|
||||
output = thinker_output.outputs[0]
|
||||
prompt_token_ids = thinker_output.prompt_token_ids
|
||||
thinker_output_ids = output.cumulative_token_ids
|
||||
prompt_token_ids_len = len(prompt_token_ids)
|
||||
mm: OmniPayload = output.multimodal_output
|
||||
latent = mm["latent"]
|
||||
thinker_hidden_states = latent.clone().detach().to(latent.device)
|
||||
decode_hidden = thinker_hidden_states[prompt_token_ids_len:].to(torch.float32)
|
||||
prefill_hidden = thinker_hidden_states[:prompt_token_ids_len].to(torch.float32)
|
||||
additional_information = to_dict(
|
||||
OmniPayloadStruct(
|
||||
hidden_states=HiddenStatesStruct(output=decode_hidden, output_shape=list(decode_hidden.shape)),
|
||||
embed=EmbeddingsStruct(prefill=prefill_hidden, prefill_shape=list(prefill_hidden.shape)),
|
||||
ids=IdsStruct(prompt=list(prompt_token_ids), output=list(thinker_output_ids)),
|
||||
)
|
||||
)
|
||||
talker_inputs.append(
|
||||
OmniTokensPrompt(
|
||||
prompt_token_ids=[TALKER_CODEC_START_TOKEN_ID]
|
||||
+ [TALKER_CODEC_PAD_TOKEN_ID] * (len(prompt_token_ids))
|
||||
+ [TALKER_CODEC_END_TOKEN_ID],
|
||||
additional_information=additional_information,
|
||||
multi_modal_data=(
|
||||
multi_modal_data[thinker_output.request_id]
|
||||
if requires_multimodal_data and multi_modal_data is not None
|
||||
else None
|
||||
),
|
||||
mm_processor_kwargs=None,
|
||||
)
|
||||
)
|
||||
return talker_inputs
|
||||
|
||||
|
||||
def talker2code2wav(
|
||||
source_outputs,
|
||||
_prompt: OmniTokensPrompt | TextPrompt = None,
|
||||
_requires_multimodal_data: bool = False,
|
||||
):
|
||||
code2wav_inputs = []
|
||||
for talker_output in source_outputs:
|
||||
output = talker_output.outputs[0]
|
||||
token_ids = list(output.cumulative_token_ids)
|
||||
if token_ids and token_ids[0] == TALKER_CODEC_START_TOKEN_ID:
|
||||
token_ids = token_ids[1:]
|
||||
if token_ids and token_ids[-1] == TALKER_CODEC_END_TOKEN_ID:
|
||||
token_ids = token_ids[:-1]
|
||||
if not token_ids:
|
||||
continue
|
||||
code2wav_inputs.append(
|
||||
OmniTokensPrompt(
|
||||
prompt_token_ids=token_ids,
|
||||
multi_modal_data=None,
|
||||
mm_processor_kwargs=None,
|
||||
)
|
||||
)
|
||||
return code2wav_inputs
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Worker-connector data plane (non-async-chunk path).
|
||||
# Both transitions ship payloads via the worker connector
|
||||
@@ -101,9 +29,7 @@ def talker2code2wav(
|
||||
# packs an OmniPayload-shaped dict (embed.prefill /
|
||||
# hidden_states.output / ids.prompt / ids.output) for the talker, which
|
||||
# the talker's ``talker_preprocess`` reads from
|
||||
# ``model_intermediate_buffer``. The shape matches what legacy
|
||||
# ``thinker2talker`` writes into ``additional_information`` as a debug
|
||||
# fallback; ``thinker2talker_token_only`` only allocates prompt slots.
|
||||
# ``model_intermediate_buffer``.
|
||||
# - talker->code2wav strips TALKER_CODEC_{START,END} boundary tokens
|
||||
# and ships the codec token ids.
|
||||
# ============================================================================
|
||||
@@ -212,9 +138,7 @@ def talker2code2wav_full_payload(
|
||||
# prefill+decode trajectory and packs an OmniPayload-shaped dict that
|
||||
# the talker's ``talker_preprocess`` reads from
|
||||
# ``model_intermediate_buffer``. ``thinker2talker_token_only`` only
|
||||
# allocates the talker's codec prompt slots; legacy
|
||||
# ``thinker2talker`` above remains as a debug fallback that bundles the
|
||||
# same shape into ``additional_information``.
|
||||
# allocates the talker's codec prompt slots.
|
||||
# ============================================================================
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user