[Bugfix][Examples][MammothModa2] Serve T2I via LLM-typed DiT stage (#7199) + benchmark harness and profiler wiring (#7293)

Signed-off-by: quan-yi-ai <82917002+quan-yi-ai@users.noreply.github.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
quan-yi-aiandHongsheng Liu authored and GitHub committed 2026-10-05 02:42:58 +08:00
1 parent b9cf74352a
commit 65f5a1d5b3
12 files changed
+933 -71

No files matched your search

+138 -25
View File
@@ -28,7 +28,8 @@ from vllm.parser.utils import (
count_chat_history_tool_calls as get_history_tool_calls_cnt,
)
from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args
from vllm_omni.diffusion.utils.image_output import extract_images_from_outputs
from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args, ar_grid_max_tokens
from vllm_omni.entrypoints.async_omni import AsyncOmni
from vllm_omni.entrypoints.openai.diffusion_request_utils import (
apply_normalized_diffusion_request_extra_args,
@@ -48,8 +49,10 @@ from vllm_omni.metrics.modality import (
)
from vllm_omni.model_executor.models.minicpmo_4_5.pipeline import MINICPMO45_REFERENCE_AUDIO_KEY
from vllm_omni.model_extras import (
build_text_to_image_prompt,
get_extra_body_params,
get_extra_output_params,
should_init_extra_args_for_non_diffusion_stages,
)
try:
@@ -142,6 +145,7 @@ from vllm_omni.entrypoints.openai.stage_params import (
from vllm_omni.entrypoints.openai.utils import (
get_stage_type,
get_supported_speakers_from_hf_config,
is_image_generation_stage,
is_single_stage_diffusion,
parse_lora_request,
resolve_diffusion_od_config,
@@ -389,15 +393,74 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
params: frozenset[str] = frozenset()
try:
od_config = resolve_diffusion_od_config(self.engine_client, self._diffusion_engine)
if od_config is not None and getattr(od_config, "model_class_name", None):
params = get_extra_body_params(od_config.model_class_name)
try:
od_config = resolve_diffusion_od_config(self.engine_client, self._diffusion_engine)
except Exception:
od_config = None
model_class_name = getattr(od_config, "model_class_name", None) if od_config is not None else None
if model_class_name is None:
model_class_name = self._resolve_multistage_model_class_name(self.engine_client)
if model_class_name:
params = get_extra_body_params(model_class_name)
except Exception as e:
logger.warning("Failed to read model extra_body params: %s", e)
self._diffusion_extra_body_params = params
return params
def _resolve_multistage_model_class_name(self, engine: Any) -> str | None:
"""Resolve the multistage pipeline's diffusion model class name.
Prefers the diffusion od_config (single source of truth for the
registry spec); falls back to the wrapper model arch declared on the
final image-output stage so multistage wrappers without a diffusion
od_config (e.g. the legacy generation-LLM DiT topology MammothModa2
used before #7134) still resolve.
"""
cached = getattr(self, "_multistage_model_class_name", None)
if cached is not None:
return cached
model_class_name: str | None = None
try:
od_config = resolve_diffusion_od_config(self.engine_client, self._diffusion_engine)
if od_config is not None:
model_class_name = getattr(od_config, "model_class_name", None)
except Exception as e:
logger.warning("Failed to read diffusion od_config model class: %s", e)
if model_class_name is None:
for stage_cfg in getattr(engine, "stage_configs", None) or []:
if not is_image_generation_stage(stage_cfg):
continue
stage_engine_args = getattr(stage_cfg, "engine_args", None)
if stage_engine_args is None and hasattr(stage_cfg, "get"):
try:
stage_engine_args = stage_cfg.get("engine_args")
except Exception:
stage_engine_args = None
for attr in ("model_class_name", "model_arch"):
model_class_name = None
if isinstance(stage_engine_args, dict):
model_class_name = stage_engine_args.get(attr)
elif stage_engine_args is not None and hasattr(stage_engine_args, "get"):
try:
model_class_name = stage_engine_args.get(attr)
except Exception:
model_class_name = None
elif stage_engine_args is not None:
model_class_name = getattr(stage_engine_args, attr, None)
if model_class_name is None:
model_class_name = getattr(stage_cfg, attr, None)
if model_class_name is not None:
break
if model_class_name is not None:
break
if model_class_name is not None:
self._multistage_model_class_name = model_class_name
return model_class_name
def _normalize_diffusion_request_args(
self,
request: ChatCompletionRequest,
@@ -3129,6 +3192,31 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
else:
engine_prompt_data = {"image": reference_images}
ar_prompt_info: dict[str, Any] | None = None
model_class_name = self._resolve_multistage_model_class_name(engine)
# Topology-independent: dispatch is by resolved model class, not by
# stage_type. The registry returns the prompt untouched for models
# without a text_to_image_prompt_builder, and #7134 migrates
# MammothModa2's DiT to a classical "diffusion" stage while still
# needing this AR envelope, so gating on "no diffusion stage present"
# would silently skip the builder after that migration.
if (
model_class_name is not None
and not reference_images
and bot_task is None
and use_system_prompt is None
and custom_system_prompt is None
):
model_t2i_prompt = build_text_to_image_prompt(
model_class_name,
{"prompt": prompt, "negative_prompt": negative_prompt},
height=height,
width=width,
)
if model_t2i_prompt is not None and model_t2i_prompt.get("prompt") != prompt:
prompt = model_t2i_prompt["prompt"]
ar_prompt_info = model_t2i_prompt.get("additional_information")
prompt_token_ids: list[int] | None = None
system_prompt_type: str | None = None
build_kwargs: dict[str, Any] = {}
@@ -3201,6 +3289,8 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
engine_prompt["modalities"] = modalities
if negative_prompt is not None:
engine_prompt["negative_prompt"] = negative_prompt
if ar_prompt_info is not None:
engine_prompt["additional_information"] = ar_prompt_info
mm_processor_kwargs: dict[str, Any] = {}
if height is not None:
@@ -3234,6 +3324,26 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
if stage_type == "llm" and ar_stop_token_ids is not None:
default_stage_params.stop_token_ids = ar_stop_token_ids
if stage_type == "llm" and idx == 0 and ar_prompt_info is not None:
if should_init_extra_args_for_non_diffusion_stages(model_class_name):
extra_args = getattr(default_stage_params, "extra_args", None)
if extra_args is None:
extra_args = {}
default_stage_params.extra_args = extra_args
apply_declared_extra_args(
default_stage_params,
self._get_diffusion_extra_body_params(),
extra_body,
)
seed_for_ar = extra_body.get("seed")
if seed_for_ar is not None and hasattr(default_stage_params, "seed"):
default_stage_params.seed = seed_for_ar
ar_width = int((ar_prompt_info.get("ar_width") or [0])[0])
ar_height = int((ar_prompt_info.get("ar_height") or [0])[0])
ar_grid_budget = ar_grid_max_tokens(ar_width, ar_height)
if ar_grid_budget is not None and hasattr(default_stage_params, "max_tokens"):
default_stage_params.max_tokens = ar_grid_budget
if (
comprehension_idx is not None
and idx == comprehension_idx
@@ -3283,6 +3393,25 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
default_stage_params.lora_scale = lora_scale
except Exception as e: # pragma: no cover - safeguard
logger.warning("Failed to parse LoRA request: %s", e)
elif stage_type == "llm" and idx > 0 and is_image_generation_stage(stage_cfg):
if num_outputs_per_prompt is not None and num_outputs_per_prompt > 1:
raise ValueError(
"n > 1 is not supported for image generation stages "
"served on the LLM path; each request produces "
"exactly one image"
)
self._set_if_supported(
default_stage_params,
height=height,
width=width,
seed=seed,
generator_device=generator_device,
)
apply_declared_extra_args(
default_stage_params,
self._get_diffusion_extra_body_params(),
extra_body,
)
return engine_prompt, sampling_params_list
@@ -3473,32 +3602,16 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
request_id=request_id,
)
images = getattr(result, "images", [])
# Generation-LLM DiT stages (e.g. MammothModa2) attach the decoded
# image to ``multimodal_output`` on the completion instead of the
# ``images`` field, so fall back to the shared extractor used by the
# offline example path before giving up.
images = getattr(result, "images", None) or extract_images_from_outputs(result)
stage_durations = result.stage_durations
peak_memory_mb = result.peak_memory_mb
response_metrics = getattr(result, "metrics", None) if return_stage_metrics else None
cot_output = None
req_out = result
if req_out:
prompt_obj = getattr(req_out, "prompt", None)
if isinstance(prompt_obj, dict):
extra = prompt_obj.get("extra", {})
if isinstance(extra, dict):
ar_text = extra.get("ar_generated_text")
if isinstance(ar_text, str) and ar_text.strip():
cot_output = ar_text
req_out = result
if req_out:
prompt_obj = getattr(req_out, "prompt", None)
if isinstance(prompt_obj, dict):
extra = prompt_obj.get("extra", {})
if isinstance(extra, dict):
ar_text = extra.get("ar_generated_text")
if isinstance(ar_text, str) and ar_text.strip():
cot_output = ar_text
req_out = result
if req_out:
prompt_obj = getattr(req_out, "prompt", None)