[Model] Add Ming-flash-omni-2.0 Image Generation (Diffusion) Stage (#2875)

Signed-off-by: ZhengWG <zwg0606@gmail.com>
Signed-off-by: Zheng Wengang <zwg0606@gmail.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: Yuanheng Zhao <54058983+yuanheng-zhao@users.noreply.github.com>
This commit is contained in:
authored and GitHub committed 2026-05-27 23:21:20 +08:00
1 parent 36dc32ff81
commit a698269e47
25 files changed
+2614 -42

No files matched your search

@@ -126,15 +126,21 @@ class MingFlashOmniForConditionalGeneration(
self.make_empty_intermediate_tensors = self.thinker.make_empty_intermediate_tensors
elif self.model_stage == "imagegen":
# TODO: Implement image generator stage
# Image generation is a separate diffusion stage; it does not run
# through this AR wrapper. See
# ``vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py``
# and ``model_arch: MingImagePipeline`` in the stage YAML.
raise NotImplementedError(
"Image generation stage is not yet implemented. Please use model_stage='thinker' for now."
"Image generation stage is not implemented in this AR wrapper. "
"Use stage_type: diffusion with model_arch: MingImagePipeline "
"for the imagegen stage (see stage_configs/ming_flash_omni.yaml)."
)
else:
raise ValueError(
f"Invalid model_stage: {self.model_stage!r}. Must be one of: 'thinker', 'imagegen'. "
f"For the talker stage, use MingFlashOmniTalkerForConditionalGeneration directly."
f"For the talker stage, use MingFlashOmniTalkerForConditionalGeneration directly. "
f"For image generation use stage_type: diffusion with model_arch: MingImagePipeline."
)
def forward(
@@ -177,9 +183,43 @@ class MingFlashOmniForConditionalGeneration(
raise NotImplementedError("get_mrope_input_positions not available on current stage")
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stripped = ((name.removeprefix("thinker."), value) for name, value in weights)
thinker_loaded = self.thinker.load_weights(stripped)
return add_prefix_to_loaded_weights(thinker_loaded, "thinker")
loaded_weights: set[str] = set()
thinker_weights: list[tuple[str, torch.Tensor]] = []
for name, value in weights:
if name.startswith("thinker."):
thinker_weights.append((name, value))
elif name.startswith("imagegen."):
# Imagegen weights are loaded by MingImagePipeline (diffusion
# stage) from its own checkpoint subfolders; silently drop
# them here so the AR wrapper does not try to load them.
continue
elif name.startswith("talker."):
# Talker weights are loaded by
# MingFlashOmniTalkerForConditionalGeneration directly; drop.
continue
else:
# Weights without prefix go to thinker by default
thinker_weights.append((name, value))
if self.model_stage == "thinker" and thinker_weights:
stripped = ((name.removeprefix("thinker."), value) for name, value in thinker_weights)
thinker_loaded = self.thinker.load_weights(stripped)
thinker_loaded = add_prefix_to_loaded_weights(thinker_loaded, "thinker")
loaded_weights.update(thinker_loaded)
# ``query_tokens_dict.*`` parameters are pre-loaded inside
# ``MingFlashOmniThinker.__init__`` from
# ``<model>/mlp/model.safetensors`` (not the main shard index),
# so vllm's default loader does not see them via the standard
# ``load_weights`` path. Report them as loaded here to satisfy the
# post-load completeness check.
query_tokens_module = getattr(self.thinker, "query_tokens_dict", None)
if query_tokens_module is not None:
for scale_name, _param in query_tokens_module.items():
loaded_weights.add(f"thinker.query_tokens_dict.{scale_name}")
return loaded_weights
def get_mm_mapping(self) -> MultiModelKeys:
return MultiModelKeys.from_string_field(
@@ -6,6 +6,7 @@
"""Ming-flash-omni-2.0 Thinker stage implementation (multimodal understanding)."""
import os
from collections.abc import Iterable, Iterator, Mapping, Sequence
from typing import Annotated, Any
@@ -113,7 +114,9 @@ class MingFlashOmniThinkerProcessingInfo(Qwen2VLProcessingInfo):
return 1
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": None, "video": None, "audio": None}
# ``img2img`` is the serving-layer alias for a reference image in the
# image-gen path; treated as a regular image by ``_MingMultiModalDataParser``.
return {"image": None, "video": None, "audio": None, "img2img": 1}
def get_mm_max_tokens_per_item(
self,
@@ -137,7 +140,14 @@ class MingFlashOmniThinkerProcessingInfo(Qwen2VLProcessingInfo):
return mm_max_tokens
def get_feature_extractor(self, **kwargs: object) -> MingWhisperFeatureExtractor:
def get_feature_extractor(self, **kwargs: object):
"""Return the audio feature extractor from the processor.
The processor may be loaded via trust_remote_code paths that return
a stock transformers ``WhisperFeatureExtractor`` rather than
vllm-omni's subclass ``MingWhisperFeatureExtractor``. We accept both
as long as the caller-needed attributes (``sampling_rate``) exist.
"""
hf_processor = self.get_hf_processor(**kwargs)
feature_extractor = hf_processor.audio_processor
assert isinstance(feature_extractor, MingWhisperFeatureExtractor)
@@ -145,13 +155,56 @@ class MingFlashOmniThinkerProcessingInfo(Qwen2VLProcessingInfo):
def get_data_parser(self):
feature_extractor = self.get_feature_extractor()
return MultiModalDataParser(
return _MingMultiModalDataParser(
target_sr=feature_extractor.sampling_rate,
target_channels=self.get_target_channels(),
expected_hidden_size=self._get_expected_hidden_size(),
)
class _MingMultiModalDataParser(MultiModalDataParser):
"""Rewrite ``img2img`` → ``image`` in-place so the thinker embeds the ref.
Matches Ming's upstream (``processing_bailingmm2.py::__call__``): when
``images`` is provided the image_processor runs → pixel_values + image_grid_thw,
and ``_expand_image_tokens`` expands ``<IMAGE>`` placeholders in the text
to ``<imagePatch>*N``. The same PIL then ALSO goes to the diffusion stage
as ``image_gen_pixel_values_reference``.
On vllm-omni's serving path serving_chat emits
``multi_modal_data={"img2img": PIL}`` (Bagel convention). We mutate the
caller's mm_data dict here — moving the PIL from the ``img2img`` key into
the ``image`` key — so that:
* vLLM's renderer validator sees consistent keys in
``mm_data`` and the parser output (``mm_data_items["image"]``)
* The thinker-side image path (``_call_hf_processor`` reads
``mm_data["images"]``, ``_get_prompt_updates`` replaces ``<IMAGE>``
placeholders, ``_get_mm_fields_config`` declares image field layout)
all run normally with the ref PIL acting as a regular image input
* ``thinker2imagegen`` still finds the ref PIL — but now under
``multi_modal_data["image"]`` rather than ``["img2img"]``.
A companion prepend in ``_apply_hf_processor_main`` ensures the prompt
text carries a matching ``<IMAGE>`` placeholder when needed.
"""
def parse_mm_data(self, mm_data):
if "img2img" in mm_data:
img2img = mm_data.pop("img2img")
if not isinstance(img2img, list):
img2img = [img2img]
existing = mm_data.get("image")
if existing is None:
existing_list: list[Any] = []
elif isinstance(existing, list):
existing_list = list(existing)
else:
existing_list = [existing]
mm_data["image"] = existing_list + img2img
return super().parse_mm_data(mm_data)
class MingFlashOmniThinkerDummyInputsBuilder(BaseDummyInputsBuilder[MingFlashOmniThinkerProcessingInfo]):
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
num_images = mm_counts.get("image", 0)
@@ -185,6 +238,8 @@ class MingFlashOmniThinkerDummyInputsBuilder(BaseDummyInputsBuilder[MingFlashOmn
audio_length = int(audio_duration * sample_rate)
num_img2img = mm_counts.get("img2img", 0)
mm_data: MultiModalDataDict = {
"image": self._get_dummy_images(
width=image_width,
@@ -200,6 +255,13 @@ class MingFlashOmniThinkerDummyInputsBuilder(BaseDummyInputsBuilder[MingFlashOmn
"audio": [(np.random.randn(audio_length).astype(np.float32), sample_rate) for _ in range(num_audios)],
}
if num_img2img > 0:
mm_data["img2img"] = self._get_dummy_images(
width=image_width,
height=image_height,
num_images=num_img2img,
)
return mm_data
@@ -396,6 +458,61 @@ class MingFlashOmniThinkerMultiModalProcessor(BaseMultiModalProcessor[MingFlashO
) -> bool:
return False
def _apply_hf_processor_main(
self,
prompt: str | list[int],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
*,
enable_hf_prompt_update: bool,
):
"""Apply Ming-specific prompt rewrites before delegating to vllm's
default text / multimodal processing pipeline.
This is the single point in the base MM-processor call graph where we
still have BOTH the full prompt and the request-level
``hf_processor_mm_kwargs`` together. Downstream, vllm's MM caching
design splits the work into
:meth:`_apply_hf_processor_text_only` (prompt, empty kwargs) and
:meth:`_apply_hf_processor_mm_only` (dummy text, real kwargs), so a
per-request flag like ``modalities`` cannot reach the real
tokenization from inside :meth:`_call_hf_processor`.
For Ming-flash-omni-2.0, image-generation requests carry
``mm_processor_kwargs["modalities"]`` set to ``["image"]`` (t2i) or
``["img2img"]`` (image edit). We expand the prompt here with the
``<image><imagePatch>*N</image>`` query-token block so the expanded
form flows through both the tokenization and (eventual) MM paths.
For img2img, we also prepend ``<IMAGE>`` so the thinker-side prompt
replacement can locate the reference-image placeholder.
"""
modalities = hf_processor_mm_kwargs.get("modalities") or []
is_image_gen = "image" in modalities or "img2img" in modalities
if isinstance(prompt, str) and is_image_gen:
from vllm_omni.model_executor.models.ming_flash_omni.prompt_utils import (
DEFAULT_NUM_QUERY_TOKENS,
maybe_expand_image_gen_prompt,
)
# img2img: prepend ``<IMAGE>`` so the ref-image placeholder survives
# MM caching (when the ref image is cache-warm, the thinker sees 0
# missing items and cannot inject the placeholder itself).
if "img2img" in modalities:
prompt = "<IMAGE>" + prompt
ig = getattr(self.info.ctx.model_config.hf_config, "image_gen_config", None)
num_query_tokens = getattr(ig, "num_query_tokens", DEFAULT_NUM_QUERY_TOKENS)
prompt = maybe_expand_image_gen_prompt(prompt, num_query_tokens=int(num_query_tokens))
return super()._apply_hf_processor_main(
prompt=prompt,
mm_items=mm_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tokenization_kwargs=tokenization_kwargs,
enable_hf_prompt_update=enable_hf_prompt_update,
)
def _call_hf_processor(
self,
prompt: str,
@@ -408,6 +525,11 @@ class MingFlashOmniThinkerMultiModalProcessor(BaseMultiModalProcessor[MingFlashO
We call the image/audio sub-processors directly (instead of going
through `MingFlashOmniProcessor.__call__`) so that the high-level
placeholder tokens remain **unexpanded** in the tokenized output.
NOTE: Image-gen prompt expansion is handled in
:meth:`_apply_hf_processor_main` — by the time we get here, vllm's
MM caching has already split text/mm processing and ``mm_kwargs``
is empty on the text-only branch.
"""
hf_processor = self.info.get_hf_processor()
tokenizer = self.info.get_tokenizer()
@@ -425,6 +547,8 @@ class MingFlashOmniThinkerMultiModalProcessor(BaseMultiModalProcessor[MingFlashO
videos = mm_data.get("videos", None)
if videos is not None:
# TODO: ``videos=`` on image_processor is deprecated since
# transformers v4.57 (removed in v5); migrate to Qwen2VLVideoProcessor.
video_outputs = hf_processor.image_processor(
images=None,
videos=videos,
@@ -487,8 +611,13 @@ class MingFlashOmniThinkerForConditionalGeneration(
return "<VIDEO>"
elif modality.startswith("audio"):
return "<AUDIO>"
elif modality == "img2img":
# Ming's img2img ref image is consumed only by the diffusion stage
# (``thinker2imagegen`` reads the raw PIL). No text-side placeholder
# is emitted for the thinker.
return None
raise ValueError("Only image, video, or audio modality is supported")
raise ValueError("Only image, video, audio, or img2img modality is supported")
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -548,10 +677,79 @@ class MingFlashOmniThinkerForConditionalGeneration(
)
logger.info("Initialized WhisperAudioEncoder and AudioProjector")
# Image-generation learnable query tokens. Stored as a ParameterDict
# keyed by scale name (e.g. "16x16" -> [256, hidden_size]) to mirror
# the upstream Ming layout in ``mlp/query_tokens_dict.<scale>``. These
# are substituted in at ``<imagePatch>`` token positions during
# text-to-image requests; image-comprehension requests continue to
# fill those positions with vision features as before.
self.query_tokens_dict = nn.ParameterDict()
self._load_image_gen_query_tokens(vllm_config.model_config.model)
# Expose interfaces
self.make_empty_intermediate_tensors = self.language_model.make_empty_intermediate_tensors
logger.info("MingFlashOmniThinker initialized with vision and audio towers")
logger.info(
f"MingFlashOmniThinker initialized with: "
f"vision={'yes' if self.vision else 'no'}, "
f"audio={'yes' if self.audio else 'no'}, "
f"image_gen_scales={list(self.query_tokens_dict.keys())}"
)
def _load_image_gen_query_tokens(self, model_path: str) -> None:
"""Load learnable query tokens used for image generation.
Expected layout (observed on inclusionAI/Ming-flash-omni-2.0):
<model_path>/mlp/model.safetensors
key ``query_tokens_dict.<scale>`` -> [num_tokens, hidden_size]
e.g. ``query_tokens_dict.16x16`` -> [256, 4096]
Missing files are logged as a warning and image-gen is disabled
(text-only inference still works).
"""
from pathlib import Path
p = Path(model_path) / "mlp" / "model.safetensors"
if not p.exists() and not os.path.exists(model_path):
from vllm_omni.model_executor.model_loader.weight_utils import (
download_weights_from_hf_specific,
)
try:
local_path = download_weights_from_hf_specific(model_path, None, ["mlp/*"])
p = Path(local_path) / "mlp" / "model.safetensors"
except Exception:
logger.exception("[MingFlashOmniThinker] failed to download mlp/ from %s", model_path)
if not p.exists():
logger.warning(
"[MingFlashOmniThinker] mlp/model.safetensors not found at %s; "
"text-to-image will be disabled (text-only inference still works).",
p,
)
return
try:
from safetensors.torch import load_file # type: ignore
state = load_file(str(p))
except Exception:
logger.exception("[MingFlashOmniThinker] failed to read %s", p)
return
loaded = 0
for key, tensor in state.items():
if not key.startswith("query_tokens_dict."):
continue
scale_name = key.split(".", 1)[1]
param = nn.Parameter(tensor.clone(), requires_grad=False)
self.query_tokens_dict[scale_name] = param
loaded += 1
logger.info(
"[MingFlashOmniThinker] loaded query_tokens_dict[%s] shape=%s dtype=%s",
scale_name,
tuple(tensor.shape),
tensor.dtype,
)
if loaded == 0:
logger.warning("[MingFlashOmniThinker] no query_tokens_dict.* keys in mlp/model.safetensors")
def extract_image_feature(self, pixel_values: torch.Tensor, grid_thw: torch.Tensor) -> torch.Tensor:
"""Extract and project image features.
@@ -732,14 +930,86 @@ class MingFlashOmniThinkerForConditionalGeneration(
) -> torch.Tensor:
inputs_embeds = self.language_model.model.word_embeddings(input_ids)
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
return inputs_embeds
# Step 1: if vision/audio features were extracted, merge them into
# inputs_embeds at their <patch> positions first. This fills ref-image
# and other normal vision-input positions with real features.
if multimodal_embeddings is not None and len(multimodal_embeddings) > 0:
assert is_multimodal is not None, "`is_multimodal` mask required when `multimodal_embeddings` provided"
inputs_embeds = _merge_multimodal_embeddings(
inputs_embeds=inputs_embeds,
multimodal_embeddings=multimodal_embeddings,
is_multimodal=is_multimodal,
)
assert is_multimodal is not None, "`is_multimodal` mask required when `multimodal_embeddings` provided"
return _merge_multimodal_embeddings(
inputs_embeds=inputs_embeds,
multimodal_embeddings=multimodal_embeddings,
is_multimodal=is_multimodal,
# Step 2: image-gen query tokens are appended at the tail of the prompt
# as a contiguous block of ``<imagePatch>`` tokens. Inject the learnable
# ``query_tokens_dict`` entries at those positions. Running AFTER step 1
# is safe because the query-token block is disjoint from ref-image
# patch positions (ref patches are earlier in the prompt, filled by
# step 1; query tokens occupy the trailing block and get overwritten
# here). Injecting always — including when multimodal embeddings are
# present — matches Ming's img2img-edit flow where BOTH a real image
# input AND the learnable query tokens need to reach the LLM.
if len(self.query_tokens_dict) > 0:
self._maybe_inject_image_gen_query_embeds(input_ids, inputs_embeds)
return inputs_embeds
def _maybe_inject_image_gen_query_embeds(
self,
input_ids: torch.Tensor,
inputs_embeds: torch.Tensor,
) -> None:
"""Overwrite the trailing ``<imagePatch>`` block with learnable query tokens.
The image-gen query-token block is appended at the tail of the prompt
by ``maybe_expand_image_gen_prompt`` as literal text
``<image><imagePatch>*N</image>``. After tokenization:
... <image_start_token> <imagePatch>*N <image_end_token>
is the exact trailing pattern. We verify that signature before
injecting — without the check, a pure vision-understanding request
whose final ref-image block ends in N or more ``<imagePatch>`` tokens
would be misidentified as image-gen and its vision features silently
overwritten. Decode-phase calls (``input_ids`` is one next-token)
naturally fail the signature check and return early.
"""
if input_ids.numel() == 0:
return
patch_token_id = self.config.image_patch_token
image_end_token_id = getattr(self.config, "image_end_token", None)
if image_end_token_id is None:
return
scale_items = list(self.query_tokens_dict.items())
if not scale_items:
return
scale_name, query_param = scale_items[-1]
num_query_tokens = int(query_param.shape[0])
flat = input_ids.view(-1)
L = int(flat.numel())
# Need at least num_query_tokens patch tokens + 1 <image_end> token.
if L < num_query_tokens + 1:
return
# Signature: the prompt must end with <image_end_token>, and the
# N positions immediately before it must all be <imagePatch>.
if int(flat[-1].item()) != image_end_token_id:
return
tail_start = L - 1 - num_query_tokens
tail_end = L - 1 # exclusive — last index is <image_end_token>
if not (flat[tail_start:tail_end] == patch_token_id).all():
return
query_embeds = query_param.to(dtype=inputs_embeds.dtype, device=inputs_embeds.device)
inputs_embeds[tail_start:tail_end] = query_embeds
logger.debug(
"[MingFlashOmniThinker] injected %d image-gen query embeddings (scale=%s) at positions [%d, %d)",
num_query_tokens,
scale_name,
tail_start,
tail_end,
)
def forward(
@@ -3,12 +3,86 @@
# Copyright (c) Ant Group. All rights reserved.
# Adapted from Ming repo's usage cookbook:
# https://github.com/inclusionAI/Ming/blob/3954fcb880ff5e61ff128bcf7f1ec344d46a6fe3/cookbook.ipynb
"""Shared prompt-building helpers for Ming-flash-omni standalone talker."""
"""Ming-flash-omni-2.0 prompt utilities.
Two unrelated families of helpers live in the same file because they
both belong to the Ming module and are tightly coupled to Ming-specific
prompt conventions:
1. **Image-gen query-token expansion** — string-level helpers used by
the thinker stage to mark the ``<image><imagePatch>*N</image>`` block
that gets substituted with learnable ``query_tokens_dict`` embeddings
during forward.
2. **TTS / talker caption builder** — JSON caption template for the
Ming-flash-omni standalone talker (TTS) stage.
"""
from __future__ import annotations
import copy
import json
from typing import Any
# ============================================================
# Image-gen query-token block (thinker stage)
# ============================================================
# Ming's thinker uses these tokens to mark a learnable image-generation
# query block inside the text prompt. The thinker substitutes its
# ``query_tokens_dict`` embeddings at each ``<imagePatch>`` position
# during forward; see
# ``MingFlashOmniThinker._maybe_inject_image_gen_query_embeds``.
_IMAGE_OPEN_TOKEN = "<image>"
_IMAGE_CLOSE_TOKEN = "</image>"
IMAGE_PATCH_TOKEN = "<imagePatch>"
# Default query-token count matches ``MingImageGenConfig(img_gen_scales=[16])``
# (16 * 16 = 256), which is what the released inclusionAI/Ming-flash-omni-2.0
# checkpoint ships.
DEFAULT_NUM_QUERY_TOKENS = 256
def maybe_expand_image_gen_prompt(
prompt: str,
num_query_tokens: int = DEFAULT_NUM_QUERY_TOKENS,
) -> str:
"""Append the ``<image><imagePatch>*N</image>`` suffix for text-to-image.
The thinker expects image-generation requests to end with an N-wide
block of ``<imagePatch>`` tokens (wrapped in ``<image>``/``</image>``)
— those positions get substituted with learnable
``query_tokens_dict`` embeddings during forward.
This helper is a no-op (returns the input unchanged) when:
* ``prompt`` is not a non-empty string, or
* the prompt already contains an ``<imagePatch>`` block (avoids
double expansion for tests / manual calls that pre-format the
prompt).
Args:
prompt: Raw user prompt text.
num_query_tokens: Total number of query tokens to emit. Defaults
to 256 (the released checkpoint's ``img_gen_scales=[16]``).
Returns:
The (possibly expanded) prompt text.
"""
if not isinstance(prompt, str) or not prompt:
return prompt
if IMAGE_PATCH_TOKEN in prompt:
return prompt
# TODO(multi-scale): single-block emission assumes img_gen_scales=[16].
suffix = _IMAGE_OPEN_TOKEN + (IMAGE_PATCH_TOKEN * num_query_tokens) + _IMAGE_CLOSE_TOKEN
return prompt + suffix
# ============================================================
# TTS / talker caption builder
# ============================================================
DEFAULT_PROMPT = "Please generate speech based on the following description.\n"
BASE_CAPTION_TEMPLATE: dict[str, Any] = {
@@ -48,3 +122,13 @@ def create_instruction(user_input: dict[str, Any]) -> str:
if key in item:
item[key] = value
return json.dumps(caption, ensure_ascii=False)
__all__ = [
"IMAGE_PATCH_TOKEN",
"DEFAULT_NUM_QUERY_TOKENS",
"maybe_expand_image_gen_prompt",
"DEFAULT_PROMPT",
"BASE_CAPTION_TEMPLATE",
"create_instruction",
]