[Bugfix][Examples][MammothModa2] Serve T2I via LLM-typed DiT stage (#7199) + benchmark harness and profiler wiring (#7293)

Signed-off-by: quan-yi-ai <82917002+quan-yi-ai@users.noreply.github.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
quan-yi-aiandHongsheng Liu authored and GitHub committed 2026-10-05 02:42:58 +08:00
1 parent b9cf74352a
commit 65f5a1d5b3
12 files changed
+933 -71

No files matched your search

+9 -5
View File
@@ -2167,6 +2167,8 @@ async def generate_images(
extra_body["system_prompt"] = request.system_prompt
if request.return_stage_metrics is not None:
extra_body["return_stage_metrics"] = request.return_stage_metrics
for extra_key, extra_value in (request.model_extra or {}).items():
extra_body.setdefault(extra_key, extra_value)
generation_result = await chat_handler.generate_diffusion_images(
prompt=request.prompt,
@@ -2426,18 +2428,20 @@ async def edit_images(
# 3.0 Init with system default values
app_state_args = getattr(raw_request.app.state, "args", None)
default_sample_param = getattr(app_state_args, "default_sampling_params", None)
# Currently only have one diffusion stage.
diffusion_stage_ids = [i for i, cfg in enumerate(stage_configs) if get_stage_type(cfg) == "diffusion"]
if not diffusion_stage_ids:
# Image edits (img2img) remain gated to classical diffusion stages;
# MammothModa2 is rejected at route level by its pipeline's endpoint
# restrictions (test_mammoth_moda2_shared_runtime.py).
image_stage_ids = [i for i, cfg in enumerate(stage_configs) if get_stage_type(cfg) == "diffusion"]
if not image_stage_ids:
raise HTTPException(
status_code=HTTPStatus.SERVICE_UNAVAILABLE.value,
detail="No diffusion stage found in multi-stage pipeline.",
)
diffusion_stage_id = diffusion_stage_ids[0]
image_stage_id = image_stage_ids[0]
apply_stage_default_sampling_params(
default_sample_param,
gen_params,
str(diffusion_stage_id),
str(image_stage_id),
)
_update_if_not_none(gen_params, "num_outputs_per_prompt", n)
# 3.1 Parse per-request LoRA (compatible with chat's extra_body.lora shape).
+6 -5
View File
@@ -17,7 +17,7 @@ from vllm_omni.entrypoints.openai.serving_audio_generate import OmniOpenAIServin
from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat
from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
from vllm_omni.entrypoints.openai.serving_video import OmniOpenAIServingVideo
from vllm_omni.entrypoints.openai.utils import get_stage_type
from vllm_omni.entrypoints.openai.utils import is_image_generation_stage
ENDPOINT_LOAD_METRICS_FORMAT_HEADER_LABEL = "endpoint-load-metrics-format"
@@ -57,7 +57,8 @@ def _get_engine_and_model(raw_request: Request):
detail="Multi-stage engine not initialized. Start server with a multi-stage omni model.",
)
# Check if there's a diffusion stage.
# Check if there's an image generation stage (classical diffusion, or a
# legacy generation-LLM DiT stage that emits images).
# Prefer app state (compat layer populated at startup), then fall back to
# the engine client's stage configs for refactored AsyncOmni paths.
stage_configs = getattr(raw_request.app.state, "stage_configs", None)
@@ -70,12 +71,12 @@ def _get_engine_and_model(raw_request: Request):
)
normalized_stage_configs = list(stage_configs)
has_diffusion_stage = any(get_stage_type(stage_cfg) == "diffusion" for stage_cfg in normalized_stage_configs)
has_image_stage = any(is_image_generation_stage(stage_cfg) for stage_cfg in normalized_stage_configs)
if not has_diffusion_stage:
if not has_image_stage:
raise HTTPException(
status_code=HTTPStatus.SERVICE_UNAVAILABLE.value,
detail="No diffusion stage found in multi-stage pipeline.",
detail="No image generation stage found in multi-stage pipeline.",
)
# Get server's loaded model name
@@ -18,7 +18,7 @@ from typing import Any, Literal
from fastapi import HTTPException
from fastapi.responses import StreamingResponse
from pydantic import BaseModel, Field, field_validator
from pydantic import BaseModel, ConfigDict, Field, field_validator
from vllm_omni.entrypoints.openai.image_api_utils import validate_layered_layers
@@ -59,6 +59,8 @@ class ImageGenerationRequest(BaseModel):
for advanced diffusion parameters.
"""
model_config = ConfigDict(extra="allow")
# Required fields
prompt: str = Field(..., description="Text description of the desired image(s)")
bot_task: str | None = Field(
+138 -25
View File
@@ -28,7 +28,8 @@ from vllm.parser.utils import (
count_chat_history_tool_calls as get_history_tool_calls_cnt,
)
from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args
from vllm_omni.diffusion.utils.image_output import extract_images_from_outputs
from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args, ar_grid_max_tokens
from vllm_omni.entrypoints.async_omni import AsyncOmni
from vllm_omni.entrypoints.openai.diffusion_request_utils import (
apply_normalized_diffusion_request_extra_args,
@@ -48,8 +49,10 @@ from vllm_omni.metrics.modality import (
)
from vllm_omni.model_executor.models.minicpmo_4_5.pipeline import MINICPMO45_REFERENCE_AUDIO_KEY
from vllm_omni.model_extras import (
build_text_to_image_prompt,
get_extra_body_params,
get_extra_output_params,
should_init_extra_args_for_non_diffusion_stages,
)
try:
@@ -142,6 +145,7 @@ from vllm_omni.entrypoints.openai.stage_params import (
from vllm_omni.entrypoints.openai.utils import (
get_stage_type,
get_supported_speakers_from_hf_config,
is_image_generation_stage,
is_single_stage_diffusion,
parse_lora_request,
resolve_diffusion_od_config,
@@ -389,15 +393,74 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
params: frozenset[str] = frozenset()
try:
od_config = resolve_diffusion_od_config(self.engine_client, self._diffusion_engine)
if od_config is not None and getattr(od_config, "model_class_name", None):
params = get_extra_body_params(od_config.model_class_name)
try:
od_config = resolve_diffusion_od_config(self.engine_client, self._diffusion_engine)
except Exception:
od_config = None
model_class_name = getattr(od_config, "model_class_name", None) if od_config is not None else None
if model_class_name is None:
model_class_name = self._resolve_multistage_model_class_name(self.engine_client)
if model_class_name:
params = get_extra_body_params(model_class_name)
except Exception as e:
logger.warning("Failed to read model extra_body params: %s", e)
self._diffusion_extra_body_params = params
return params
def _resolve_multistage_model_class_name(self, engine: Any) -> str | None:
"""Resolve the multistage pipeline's diffusion model class name.
Prefers the diffusion od_config (single source of truth for the
registry spec); falls back to the wrapper model arch declared on the
final image-output stage so multistage wrappers without a diffusion
od_config (e.g. the legacy generation-LLM DiT topology MammothModa2
used before #7134) still resolve.
"""
cached = getattr(self, "_multistage_model_class_name", None)
if cached is not None:
return cached
model_class_name: str | None = None
try:
od_config = resolve_diffusion_od_config(self.engine_client, self._diffusion_engine)
if od_config is not None:
model_class_name = getattr(od_config, "model_class_name", None)
except Exception as e:
logger.warning("Failed to read diffusion od_config model class: %s", e)
if model_class_name is None:
for stage_cfg in getattr(engine, "stage_configs", None) or []:
if not is_image_generation_stage(stage_cfg):
continue
stage_engine_args = getattr(stage_cfg, "engine_args", None)
if stage_engine_args is None and hasattr(stage_cfg, "get"):
try:
stage_engine_args = stage_cfg.get("engine_args")
except Exception:
stage_engine_args = None
for attr in ("model_class_name", "model_arch"):
model_class_name = None
if isinstance(stage_engine_args, dict):
model_class_name = stage_engine_args.get(attr)
elif stage_engine_args is not None and hasattr(stage_engine_args, "get"):
try:
model_class_name = stage_engine_args.get(attr)
except Exception:
model_class_name = None
elif stage_engine_args is not None:
model_class_name = getattr(stage_engine_args, attr, None)
if model_class_name is None:
model_class_name = getattr(stage_cfg, attr, None)
if model_class_name is not None:
break
if model_class_name is not None:
break
if model_class_name is not None:
self._multistage_model_class_name = model_class_name
return model_class_name
def _normalize_diffusion_request_args(
self,
request: ChatCompletionRequest,
@@ -3129,6 +3192,31 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
else:
engine_prompt_data = {"image": reference_images}
ar_prompt_info: dict[str, Any] | None = None
model_class_name = self._resolve_multistage_model_class_name(engine)
# Topology-independent: dispatch is by resolved model class, not by
# stage_type. The registry returns the prompt untouched for models
# without a text_to_image_prompt_builder, and #7134 migrates
# MammothModa2's DiT to a classical "diffusion" stage while still
# needing this AR envelope, so gating on "no diffusion stage present"
# would silently skip the builder after that migration.
if (
model_class_name is not None
and not reference_images
and bot_task is None
and use_system_prompt is None
and custom_system_prompt is None
):
model_t2i_prompt = build_text_to_image_prompt(
model_class_name,
{"prompt": prompt, "negative_prompt": negative_prompt},
height=height,
width=width,
)
if model_t2i_prompt is not None and model_t2i_prompt.get("prompt") != prompt:
prompt = model_t2i_prompt["prompt"]
ar_prompt_info = model_t2i_prompt.get("additional_information")
prompt_token_ids: list[int] | None = None
system_prompt_type: str | None = None
build_kwargs: dict[str, Any] = {}
@@ -3201,6 +3289,8 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
engine_prompt["modalities"] = modalities
if negative_prompt is not None:
engine_prompt["negative_prompt"] = negative_prompt
if ar_prompt_info is not None:
engine_prompt["additional_information"] = ar_prompt_info
mm_processor_kwargs: dict[str, Any] = {}
if height is not None:
@@ -3234,6 +3324,26 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
if stage_type == "llm" and ar_stop_token_ids is not None:
default_stage_params.stop_token_ids = ar_stop_token_ids
if stage_type == "llm" and idx == 0 and ar_prompt_info is not None:
if should_init_extra_args_for_non_diffusion_stages(model_class_name):
extra_args = getattr(default_stage_params, "extra_args", None)
if extra_args is None:
extra_args = {}
default_stage_params.extra_args = extra_args
apply_declared_extra_args(
default_stage_params,
self._get_diffusion_extra_body_params(),
extra_body,
)
seed_for_ar = extra_body.get("seed")
if seed_for_ar is not None and hasattr(default_stage_params, "seed"):
default_stage_params.seed = seed_for_ar
ar_width = int((ar_prompt_info.get("ar_width") or [0])[0])
ar_height = int((ar_prompt_info.get("ar_height") or [0])[0])
ar_grid_budget = ar_grid_max_tokens(ar_width, ar_height)
if ar_grid_budget is not None and hasattr(default_stage_params, "max_tokens"):
default_stage_params.max_tokens = ar_grid_budget
if (
comprehension_idx is not None
and idx == comprehension_idx
@@ -3283,6 +3393,25 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
default_stage_params.lora_scale = lora_scale
except Exception as e: # pragma: no cover - safeguard
logger.warning("Failed to parse LoRA request: %s", e)
elif stage_type == "llm" and idx > 0 and is_image_generation_stage(stage_cfg):
if num_outputs_per_prompt is not None and num_outputs_per_prompt > 1:
raise ValueError(
"n > 1 is not supported for image generation stages "
"served on the LLM path; each request produces "
"exactly one image"
)
self._set_if_supported(
default_stage_params,
height=height,
width=width,
seed=seed,
generator_device=generator_device,
)
apply_declared_extra_args(
default_stage_params,
self._get_diffusion_extra_body_params(),
extra_body,
)
return engine_prompt, sampling_params_list
@@ -3473,32 +3602,16 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
request_id=request_id,
)
images = getattr(result, "images", [])
# Generation-LLM DiT stages (e.g. MammothModa2) attach the decoded
# image to ``multimodal_output`` on the completion instead of the
# ``images`` field, so fall back to the shared extractor used by the
# offline example path before giving up.
images = getattr(result, "images", None) or extract_images_from_outputs(result)
stage_durations = result.stage_durations
peak_memory_mb = result.peak_memory_mb
response_metrics = getattr(result, "metrics", None) if return_stage_metrics else None
cot_output = None
req_out = result
if req_out:
prompt_obj = getattr(req_out, "prompt", None)
if isinstance(prompt_obj, dict):
extra = prompt_obj.get("extra", {})
if isinstance(extra, dict):
ar_text = extra.get("ar_generated_text")
if isinstance(ar_text, str) and ar_text.strip():
cot_output = ar_text
req_out = result
if req_out:
prompt_obj = getattr(req_out, "prompt", None)
if isinstance(prompt_obj, dict):
extra = prompt_obj.get("extra", {})
if isinstance(extra, dict):
ar_text = extra.get("ar_generated_text")
if isinstance(ar_text, str) and ar_text.strip():
cot_output = ar_text
req_out = result
if req_out:
prompt_obj = getattr(req_out, "prompt", None)
+28
View File
@@ -38,6 +38,34 @@ def get_stage_type(stage_cfg: Any) -> str:
return getattr(stage_cfg, "stage_type", "llm")
def is_image_generation_stage(stage_cfg: Any) -> bool:
"""Return whether a stage produces the pipeline's final image output.
Covers both classical diffusion stages (``stage_type == "diffusion"``) and
generation-LLM stages that emit images (``stage_type == "llm"`` with
``final_output_type == "image"`` — the legacy topology MammothModa2 used
before #7134 migrated it to the shared diffusion runtime). Serving
endpoints use this so image routes accept either topology.
"""
if get_stage_type(stage_cfg) == "diffusion":
return True
final_output_type = None
if isinstance(stage_cfg, dict):
final_output = stage_cfg.get("final_output", False)
final_output_type = stage_cfg.get("final_output_type")
elif hasattr(stage_cfg, "get"):
try:
final_output = stage_cfg.get("final_output", False)
final_output_type = stage_cfg.get("final_output_type")
except Exception:
final_output = getattr(stage_cfg, "final_output", False)
final_output_type = getattr(stage_cfg, "final_output_type", None)
else:
final_output = getattr(stage_cfg, "final_output", False)
final_output_type = getattr(stage_cfg, "final_output_type", None)
return bool(final_output) and final_output_type in {"image", "images"}
def is_video_generation_pipeline(stage_configs: list[Any] | None) -> bool:
"""Return whether a pipeline declares a final video output stage."""
for stage in stage_configs or ():