diff --git a/examples/offline_inference/image_to_image/image_edit.py b/examples/offline_inference/image_to_image/image_edit.py index 299efb33b..12bd978a3 100644 --- a/examples/offline_inference/image_to_image/image_edit.py +++ b/examples/offline_inference/image_to_image/image_edit.py @@ -138,6 +138,15 @@ def parse_args() -> argparse.Namespace: required=True, help="Path(s) to input image file(s) (PNG, JPG, etc.). Can specify multiple images.", ) + parser.add_argument( + "--deploy-config", + type=str, + default=None, + help=( + "Path to a deploy YAML. Required for multi-stage image-edit pipelines " + "whose deploy config is not auto-loaded." + ), + ) parser.add_argument( "--prompt", type=str, @@ -349,7 +358,12 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--enforce-eager", action="store_true", - help="Disable torch.compile and force eager execution.", + default=None, + help=( + "Disable torch.compile and force eager execution. Left unset (None) " + "so it is only forwarded when explicitly given; " + "otherwise the per-stage deploy YAML value wins." + ), ) parser.add_argument( "--vae-use-slicing", @@ -468,7 +482,7 @@ def main(): } # Initialize Omni with appropriate pipeline - omni = Omni( + omni_kwargs: dict[str, Any] = dict( model=args.model, enable_layerwise_offload=args.enable_layerwise_offload, vae_use_slicing=args.vae_use_slicing, @@ -476,11 +490,15 @@ def main(): cache_backend=args.cache_backend, cache_config=cache_config, parallel_config=parallel_config, - enforce_eager=args.enforce_eager, enable_cpu_offload=args.enable_cpu_offload, enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler, profiler_config=args.profiler_config, ) + if args.enforce_eager is not None: + omni_kwargs["enforce_eager"] = args.enforce_eager + if args.deploy_config: + omni_kwargs["deploy_config"] = args.deploy_config + omni = Omni(**omni_kwargs) model_class_name = get_model_class_name(omni) declared_extra_body_params = get_extra_body_params(model_class_name) print("Pipeline loaded") diff --git a/examples/offline_inference/ming_flash_omni/README.md b/examples/offline_inference/ming_flash_omni/README.md index 7e7ab0a8f..e60efe88b 100644 --- a/examples/offline_inference/ming_flash_omni/README.md +++ b/examples/offline_inference/ming_flash_omni/README.md @@ -90,14 +90,40 @@ Pass `--deploy-config /path/to/your_deploy.yaml` to any of the commands above to ### Image generation (text-to-image / img2img) -The diffusion-stage image-generation path is currently only wired through -the online OpenAI-compatible chat endpoint (`/v1/chat/completions` with -`"modalities": ["image"]`). For payloads, optional knobs -(`steps`/`cfg`/`seed`/`byte5_text`/`negative_prompt`), and the img2img -reference-image flow, see the -[image-gen section in the online-serving README](../../online_serving/ming_flash_omni/README.md#image-generation-text-to-image). -`end2end.py` does not yet exercise the imagegen stage; that is tracked as -follow-up work. +Image generation is served through the standard task examples. +The diffusion-stage knobs are declared centrally in `vllm_omni/model_extras/ming_flash_omni.py` and routed via `--extra-body`. With deploy yaml assigned properly, the model can run through the shared example scripts like any other diffusion model: + +Text-to-image (offline): + +```bash +python examples/offline_inference/text_to_image/text_to_image.py \ + --model Jonathan1909/Ming-flash-omni-2.0 \ + --deploy-config vllm_omni/deploy/ming_flash_omni_image.yaml \ + --prompt "Please draw a cute cat." \ + --height 1024 \ + --width 1024 \ + --extra-body '{"steps": 30, "cfg": 2.0, "seed": 42}' \ + --output ming_flash_omni_t2i.png +``` + +Image-to-image (offline): +```bash +# Reference image: figures/cases/person_gen_05.png from the upstream Ming repo +# https://github.com/inclusionAI/Ming/blob/3954fcb880ff5e61ff128bcf7f1ec344d46a6fe3/examples/vllm_demo.py +wget https://raw.githubusercontent.com/inclusionAI/Ming/3954fcb880ff5e61ff128bcf7f1ec344d46a6fe3/figures/cases/person_gen_05.png + +python examples/offline_inference/image_to_image/image_edit.py \ + --model Jonathan1909/Ming-flash-omni-2.0 \ + --deploy-config vllm_omni/deploy/ming_flash_omni_image.yaml \ + --image person_gen_05.png \ + --prompt "Put a pair of sunglasses on the person." \ + --extra-args '{"steps": 30, "cfg": 2.0, "seed": 42}' \ + --output ming_flash_omni_i2i.png +``` + +For the online path and the full knob list +(`steps`/`cfg`/`seed`/`byte5_text`/`negative_prompt`/`height`/`width`), see the +[image-generation section in the recipe](../../../recipes/inclusionAI/Ming-flash-omni-2.0.md#image-generation-text-to-image--img2img). ## Online serving diff --git a/examples/offline_inference/text_to_image/text_to_image.py b/examples/offline_inference/text_to_image/text_to_image.py index 69f9e4fcb..a2236d8ca 100644 --- a/examples/offline_inference/text_to_image/text_to_image.py +++ b/examples/offline_inference/text_to_image/text_to_image.py @@ -69,7 +69,16 @@ def parse_args() -> argparse.Namespace: "--stage-configs-path", type=str, default=None, - help="Path to a YAML file containing stage configurations for Omni.", + help="[Deprecated] Path to a legacy stage_args-format YAML. Prefer --deploy-config.", + ) + parser.add_argument( + "--deploy-config", + type=str, + default=None, + help=( + "Path to a deploy YAML (new pipeline/stages format). Required for multi-stage " + "text-to-image pipelines whose deploy config is not auto-loaded." + ), ) parser.add_argument("--prompt", default="a cup of coffee on the table", help="Text prompt for image generation.") parser.add_argument( @@ -155,7 +164,12 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--enforce-eager", action="store_true", - help="Disable torch.compile and force eager execution.", + default=None, + help=( + "Disable torch.compile and force eager execution. Left unset (None) " + "so it is only forwarded when explicitly given; " + "otherwise the per-stage deploy YAML value wins." + ), ) parser.add_argument( "--enable-cpu-offload", @@ -221,8 +235,13 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--tensor-parallel-size", type=int, - default=1, - help="Number of GPUs used for tensor parallelism (TP) inside the DiT.", + default=None, + help=( + "Number of GPUs used for tensor parallelism (TP) inside the DiT. " + "Left unset so it is only forwarded when explicitly given; " + "otherwise the per-stage deploy YAML (or engine default of 1) wins. " + "Passing it always overrides the deploy YAML." + ), ) parser.add_argument( "--enable-expert-parallel", @@ -410,10 +429,8 @@ def main(): "ring_degree": args.ring_degree, "ulysses_mode": args.ulysses_mode, "cfg_parallel_size": args.cfg_parallel_size, - "tensor_parallel_size": args.tensor_parallel_size, "vae_patch_parallel_size": args.vae_patch_parallel_size, "enable_expert_parallel": args.enable_expert_parallel, - "enforce_eager": args.enforce_eager, "enable_cpu_offload": args.enable_cpu_offload, "mode": "text-to-image", "log_stats": args.log_stats, @@ -425,8 +442,14 @@ def main(): **lora_args, **quant_kwargs, } + if args.tensor_parallel_size is not None: + omni_kwargs["tensor_parallel_size"] = args.tensor_parallel_size + if args.enforce_eager is not None: + omni_kwargs["enforce_eager"] = args.enforce_eager if args.stage_configs_path: omni_kwargs["stage_configs_path"] = args.stage_configs_path + if args.deploy_config: + omni_kwargs["deploy_config"] = args.deploy_config if use_nextstep: # NextStep-1.1 requires explicit pipeline class omni_kwargs["model_class_name"] = "NextStep11Pipeline" @@ -451,8 +474,9 @@ def main(): print(f" Quantization: {args.quantization if args.quantization else 'None (BF16)'}") if ignored_layers: print(f" Ignored layers: {ignored_layers}") + tp_display = args.tensor_parallel_size if args.tensor_parallel_size is not None else "deploy/default" print( - f" Parallel configuration: tensor_parallel_size={args.tensor_parallel_size}, " + f" Parallel configuration: tensor_parallel_size={tp_display}, " f"ulysses_degree={args.ulysses_degree}, ulysses_mode={args.ulysses_mode}, " f"ring_degree={args.ring_degree}, cfg_parallel_size={args.cfg_parallel_size}, " f"vae_patch_parallel_size={args.vae_patch_parallel_size}, " diff --git a/examples/online_serving/ming_flash_omni/README.md b/examples/online_serving/ming_flash_omni/README.md index d60ba8156..9810841db 100644 --- a/examples/online_serving/ming_flash_omni/README.md +++ b/examples/online_serving/ming_flash_omni/README.md @@ -49,22 +49,14 @@ python examples/online_serving/openai_chat_completion_client_for_multimodal_gene --modalities text ``` -Parameterized curl wrapper in this directory: -```bash -bash run_curl_multimodal_generation.sh text -bash run_curl_multimodal_generation.sh use_image -bash run_curl_multimodal_generation.sh use_audio -bash run_curl_multimodal_generation.sh use_video -bash run_curl_multimodal_generation.sh use_mixed_modalities -bash run_curl_multimodal_generation.sh use_image_gen -``` +## Image generation (text-to-image / img2img) -## Image generation (text-to-image) +Ming-flash-omni-2.0 also exposes an image-generation (diffusion) stage. Launch with the image deploy YAML, which adds an image-gen stage behind the thinker. -Ming-flash-omni-2.0 also exposes an image-generation (diffusion) stage. -Launch with the image deploy YAML, which adds an image-gen stage behind -the thinker: +The image-generation stage is a standard vLLM-Omni diffusion pipeline (`MingImagePipeline`); its request knobs are declared in `vllm_omni/model_extras/ming_flash_omni.py` and routed through `extra_body`, so they no longer need a bespoke `sampling_params_list` recipe (that form is still available for per-stage thinker sampling — see below). + +### Launch ```bash vllm serve Jonathan1909/Ming-flash-omni-2.0 --omni \ @@ -74,32 +66,58 @@ vllm serve Jonathan1909/Ming-flash-omni-2.0 --omni \ --port 8091 ``` -Then request image output by passing `"modalities": ["image"]`: + +### Text-to-image + +Request image output with `"modalities": ["image"]`: ```bash curl -s http://127.0.0.1:8091/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "Jonathan1909/Ming-flash-omni-2.0", - "messages": [{"role": "user", "content": "Please draw a cute cat."}], - "modalities": ["image"] - }' | jq -r '.choices[0].message.content[0].image_url.url | split(",")[1]' | base64 -d > ming_imagegen.png + -H "Content-Type: application/json" \ + -d '{ + "model": "Jonathan1909/Ming-flash-omni-2.0", + "messages": [ + { + "role": "user", + "content": "Please draw a cute cat." + } + ], + "modalities": ["image"] + }' \ + | jq -r '.choices[0].message.content[0].image_url.url | split(",")[1]' \ + | base64 -d > ming_imagegen.png ``` -### Optional knobs +Pass generation knobs under a literal `extra_body` object (the OpenAI Python client's `extra_body=` kwarg produces the same request). Keys are filtered against the declared set and routed into every stage's `extra_args`: -Pass image-gen overrides as flat keys on the diffusion-stage `sampling_params_list[1].extra_args`: +```bash +curl http://127.0.0.1:8091/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "Jonathan1909/Ming-flash-omni-2.0", + "modalities": ["image"], + "messages": [ + { + "role": "user", + "content": "Draw a poster." + } + ], + "extra_body": { + "steps": 6, + "cfg": 1.5, + "height": 512, + "width": 512, + "seed": 123, + "byte5_text": ["理解与生成统一"], + "negative_prompt": "ugly, blurry, distorted" + } + }' \ + | jq -r '.choices[0].message.content[0].image_url.url | split(",")[1]' \ + | base64 -d > ming_imagegen_extra_body.png +``` -| Key | Default | Description | -| --- | --- | --- | -| `height` / `width` | from config (1024) | Output resolution (multiples of `vae_scale_factor * 2`, currently 16). | -| `steps` | 30 | Number of FlowMatchEuler denoise steps. | -| `cfg` | 2.0 | Classifier-free guidance scale. | -| `seed` | 42 | Per-request RNG seed (deterministic when ByT5 is also seed-stable). | -| `byte5_text` | (auto) | Override the glyph text for ByT5 enhancement; raw strings are auto-wrapped to Ming's `Text "...". ` format. | -| `negative_prompt` | empty | Real CFG negative conditioning (set on **stage-0 thinker** `extra_args` so `expand_cfg_prompts` spawns the companion). | - -Example with all the knobs: +NOTE: `extra_body` does **not** set the thinker's own sampling params. To tune the thinker (stage-0) sampling (`temperature` / `top_p` / `top_k` / `max_tokens`) or place knobs explicitly per stage, use `sampling_params_list` (`[thinker, imagegen]`). +`negative_prompt` must sit on the **stage-0** entry to trigger the real-CFG companion; the imagegen knobs go on **stage-1**: ```bash curl http://127.0.0.1:8091/v1/chat/completions \ @@ -143,27 +161,48 @@ curl http://127.0.0.1:8091/v1/chat/completions \ ### img2img (reference image) -Add an `image_url` content part to the user message; the parser routes it -into the diffusion stage as `extra[reference_image]`: +Add an `image_url` content part (a base64 data URL) to the user message; it is routed into the DiT stage as `extra[reference_image]`. base64-encode a local file and stream it through `jq` via stdin - piping `base64 -> jq -> curl` so that avoids the shell `ARG_MAX` limit that inlining a large base64 string in `-d '…'` hits. -```jsonc -"messages": [{ - "role": "user", - "content": [ - {"type": "text", "text": "Change the background to a sandy beach at sunset."}, - {"type": "image_url", "image_url": {"url": "data:image/png;base64,"}} - ] -}] +```bash +# Reference image: figures/cases/person_gen_05.png from the upstream Ming repo +# Check https://github.com/inclusionAI/Ming/blob/3954fcb880ff5e61ff128bcf7f1ec344d46a6fe3/examples/vllm_demo.py +wget https://raw.githubusercontent.com/inclusionAI/Ming/3954fcb880ff5e61ff128bcf7f1ec344d46a6fe3/figures/cases/person_gen_05.png + +base64 -w0 person_gen_05.png \ +| jq -Rs --arg prompt "Put a pair of sunglasses on the person." '{ + model: "Jonathan1909/Ming-flash-omni-2.0", + modalities: ["image"], + messages: [ + { + role: "user", + content: [ + { type: "text", text: $prompt }, + { type: "image_url", image_url: { url: ("data:image/png;base64," + (. | rtrimstr("\n"))) } } + ] + } + ] + }' \ +| curl http://127.0.0.1:8091/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d @- \ +| jq -r '.choices[0].message.content[0].image_url.url | split(",")[1]' \ +| base64 -d > ming_img2img.png ``` -### GPU layout +The reference image can also be a public URL (`"url": "https://…/photo.jpg"`) or the simplified `{"image": ""}` content-part form — see the [image-to-image request formats](../image_to_image/README.md#request-format). -The shipped `ming_flash_omni_image.yaml` allocates the thinker on GPUs 0–3 -(TP=4) and the diffusion stage on GPU 4 (TP=1). Copy the YAML and edit -`devices` per stage to relocate; with fewer GPUs available, drop -the thinker TP to 2 and run the diffusion stage on a free card. Image-gen -warmup takes roughly an extra 30–60 s on top of the thinker — set -`--stage-init-timeout 1800` if the default 300 s is too tight. +### Knobs (declared `extra_body` params) + +| Key | Default | Description | +| --- | --- | --- | +| `height` / `width` | 1024 | Output resolution (multiples of `vae_scale_factor * 2`, currently 16). | +| `steps` | 30 | Number of FlowMatchEuler denoise steps. | +| `cfg` | 2.0 | Classifier-free guidance scale. | +| `seed` | 42 | Per-request RNG seed. | +| `byte5_text` | (auto) | Glyph text for ByT5 enhancement; raw strings are auto-wrapped to Ming's `Text "…". ` format. Auto-extracted from quoted spans in the prompt when omitted. | +| `negative_prompt` | (empty) | Real CFG negative conditioning (text-to-image only). | + +For the offline `text_to_image.py` / `image_edit.py` scripts and the full knob reference, see the [image-generation section in the recipe](../../../recipes/inclusionAI/Ming-flash-omni-2.0.md#image-generation-text-to-image--img2img). ## Modality control diff --git a/examples/online_serving/ming_flash_omni/run_curl_multimodal_generation.sh b/examples/online_serving/ming_flash_omni/run_curl_multimodal_generation.sh deleted file mode 100755 index 92bb32c47..000000000 --- a/examples/online_serving/ming_flash_omni/run_curl_multimodal_generation.sh +++ /dev/null @@ -1,169 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Server port -PORT="${PORT:-8091}" -# Default query type -QUERY_TYPE="${1:-text}" - -# Validate query type -if [[ ! "$QUERY_TYPE" =~ ^(text|use_audio|use_image|use_video|use_mixed_modalities|use_image_gen)$ ]]; then - echo "Error: Invalid query type '$QUERY_TYPE'" - echo "Usage: $0 [text|use_audio|use_image|use_video|use_mixed_modalities|use_image_gen]" - echo " text: Text-only query" - echo " use_audio: Audio + Text query" - echo " use_image: Image + Text query" - echo " use_video: Video + Text query" - echo " use_mixed_modalities: Audio + Image + Video + Text query" - echo " use_image_gen: Text-to-image (diffusion stage) — saves the generated PNG locally" - exit 1 -fi - -# Define URLs for assets -MARY_HAD_LAMB_AUDIO_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/mary_had_lamb.ogg" -CHERRY_BLOSSOM_IMAGE_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg" -SAMPLE_VIDEO_URL="https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4" - -# Build user content based on query type -case "$QUERY_TYPE" in - text) - user_content='[ - { - "type": "text", - "text": "请详细介绍鹦鹉的生活习性。" - } - ]' - ;; - use_image) - user_content='[ - { - "type": "image_url", - "image_url": { - "url": "'"$CHERRY_BLOSSOM_IMAGE_URL"'" - } - }, - { - "type": "text", - "text": "Describe this image in detail." - } - ]' - ;; - use_audio) - user_content='[ - { - "type": "audio_url", - "audio_url": { - "url": "'"$MARY_HAD_LAMB_AUDIO_URL"'" - } - }, - { - "type": "text", - "text": "Please recognize the language of this speech and transcribe it. Format: oral." - } - ]' - ;; - use_video) - user_content='[ - { - "type": "video_url", - "video_url": { - "url": "'"$SAMPLE_VIDEO_URL"'" - } - }, - { - "type": "text", - "text": "Describe what is happening in this video." - } - ]' - ;; - use_mixed_modalities) - user_content='[ - { - "type": "image_url", - "image_url": { - "url": "'"$CHERRY_BLOSSOM_IMAGE_URL"'" - } - }, - { - "type": "audio_url", - "audio_url": { - "url": "'"$MARY_HAD_LAMB_AUDIO_URL"'" - } - }, - { - "type": "text", - "text": "Describe the image, and recognize the language of this speech and transcribe it. Format: oral" - } - ]' - ;; -esac - -echo "Running query type: $QUERY_TYPE" -echo "" - -if [[ "$QUERY_TYPE" == "use_image_gen" ]]; then - # Image-gen branch: dual-stage diffusion endpoint, modalities=["image"]. - # The shipped ming_flash_omni_image.yaml runs the thinker on cards 0-3 - # (TP=4) and the diffusion stage on card 4 (TP=1). See the image-gen - # section in README.md for knobs and img2img. - out_path="${OUT_PATH:-/tmp/ming_imagegen.png}" - request_body=$(cat <<'EOF' -{ - "model": "Jonathan1909/Ming-flash-omni-2.0", - "modalities": ["image"], - "messages": [ - {"role": "user", "content": "Draw a beautiful girl with short black hair and red dress."} - ] -} -EOF -) - output=$(curl -sS --retry 3 --retry-delay 3 --retry-connrefused \ - -X POST "http://localhost:${PORT}/v1/chat/completions" \ - -H "Content-Type: application/json" \ - -d "$request_body") - # Server returns base64-encoded PNG either as a data URL in - # ``choices[0].message.content[0].image_url.url`` or as a raw string. - b64=$(echo "$output" | jq -r '.choices[0].message.content - | if type == "array" - then (map(select(.image_url? != null)) | .[0].image_url.url // "") - elif type == "string" then . else "" end') - if [[ -z "$b64" ]]; then - echo "Error: no image returned. Raw response:" - echo "$output" | jq '.' - exit 1 - fi - if [[ "$b64" == data:image* ]]; then b64="${b64#*,}"; fi - printf '%s' "$b64" | base64 --decode > "$out_path" - echo "Saved generated image: $out_path ($(stat -c%s "$out_path" 2>/dev/null || stat -f%z "$out_path") bytes)" - exit 0 -fi - -request_body=$(cat < ming_imagegen.png +``` + +**Quick form — `extra_body`** (keys are filtered against the declared set and routed into every stage's `extra_args`). Convenient, but it does **not** let you set the thinker (stage-0) sampling params. Wrap the knobs in a literal `"extra_body"` object for raw curl; the OpenAI Python client's `extra_body=` kwarg produces the same request: + +```bash +curl http://127.0.0.1:8091/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "Jonathan1909/Ming-flash-omni-2.0", + "modalities": ["image"], + "messages": [ + { + "role": "user", + "content": "Draw a poster." + } + ], + "extra_body": { + "steps": 6, + "cfg": 1.5, + "height": 512, + "width": 512, + "seed": 123, + "byte5_text": ["理解与生成统一"], + "negative_prompt": "ugly, blurry, distorted" + } + }' \ + | jq -r '.choices[0].message.content[0].image_url.url | split(",")[1]' \ + | base64 -d > ming_imagegen_extra_body.png +``` + +**Full control — `sampling_params_list`** (one entry per stage: `[thinker, imagegen]`). +Use this when you need to tune the thinker's own sampling (`temperature` / `top_p` / `top_k` / `max_tokens`), or to place knobs explicitly per stage. +Note `negative_prompt` must sit on the **stage-0 thinker** `extra_args`; the imagegen-stage knobs (`steps` / `cfg` / `height` / `width` / `seed` / `byte5_text`) go on the **stage-1** entry: + +```bash +curl http://127.0.0.1:8091/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "Jonathan1909/Ming-flash-omni-2.0", + "modalities": ["image"], + "sampling_params_list": [ + { + "temperature": 0.4, + "top_p": 0.9, + "top_k": 1, + "max_tokens": 1, + "seed": 42, + "extra_args": { + "negative_prompt": "ugly, blurry, distorted" + } + }, + { + "seed": 42, + "extra_args": { + "steps": 6, + "cfg": 1.5, + "height": 512, + "width": 512, + "seed": 123, + "byte5_text": ["理解与生成统一"] + } + } + ], + "messages": [ + { + "role": "user", + "content": "Draw a poster." + } + ] + }' \ + | jq -r '.choices[0].message.content[0].image_url.url | split(",")[1]' \ + | base64 -d > ming_imagegen_knobs.png +``` + +### Knobs (declared `extra_body` params) + +| Key | Default | Description | +| --- | --- | --- | +| `height` / `width` | 1024 | Output resolution (multiples of `vae_scale_factor * 2`, currently 16). | +| `steps` | 30 | Number of FlowMatchEuler denoise steps. | +| `cfg` | 2.0 | Classifier-free guidance scale. | +| `seed` | 42 | Per-request RNG seed. | +| `byte5_text` | (auto) | Glyph text for ByT5 enhancement; raw strings are auto-wrapped to Ming's `Text "…". ` format. Auto-extracted from quoted spans in the prompt when omitted. | +| `negative_prompt` | (empty) | Real CFG negative conditioning. Spawns a CFG-text companion via `expand_cfg_prompts`; **online / text-to-image only** (offline uses Ming's default zero-negative). | + +For img2img, add an `image_url` content part to the user message (online) or pass `--image` (offline); the reference image is routed into the DiT stage as `extra[reference_image]`. + ### 1x H100 80GB — standalone TTS (talker only) The bundled `ming_flash_omni_tts.yaml` runs the talker on a single GPU and exposes the OpenAI `/v1/audio/speech` endpoint. diff --git a/tests/model_extras/test_model_extras.py b/tests/model_extras/test_model_extras.py index f3a09346c..f2db9ded0 100644 --- a/tests/model_extras/test_model_extras.py +++ b/tests/model_extras/test_model_extras.py @@ -126,6 +126,102 @@ def test_magi_human_extra_registry_declares_request_and_response_params() -> Non assert should_init_extra_args_for_non_diffusion_stages("MagiHumanPipeline") is False +@pytest.mark.core_model +@pytest.mark.cpu +def test_ming_flash_omni_extra_registry_declares_request_and_response_params() -> None: + assert get_extra_body_params("MingImagePipeline") == frozenset( + { + "height", + "width", + "steps", + "cfg", + "seed", + "byte5_text", + "negative_prompt", + } + ) + assert get_extra_output_params("MingImagePipeline") == frozenset() + assert should_init_extra_args_for_non_diffusion_stages("MingImagePipeline") is True + + +@pytest.mark.core_model +@pytest.mark.cpu +def test_ming_flash_omni_declared_extra_args_route_into_sampling_params() -> None: + params = OmniDiffusionSamplingParams() + declared = get_extra_body_params("MingImagePipeline") + apply_declared_extra_args( + params, + declared, + { + "steps": 6, + "cfg": 1.5, + "byte5_text": ["理解与生成统一"], + "negative_prompt": "ugly, blurry", + "unknown": "ignored", + }, + ) + assert params.extra_args == { + "steps": 6, + "cfg": 1.5, + "byte5_text": ["理解与生成统一"], + "negative_prompt": "ugly, blurry", + } + + +@pytest.mark.core_model +@pytest.mark.cpu +def test_ming_flash_omni_text_to_image_prompt_builder() -> None: + assert build_text_to_image_prompt( + "MingImagePipeline", + prompt="Please draw a cute cat.", + negative_prompt="ugly, blurry", + height=512, + width=768, + ) == { + "prompt": "Please draw a cute cat.", + "modalities": ["image"], + "mm_processor_kwargs": { + "modalities": ["image"], + "target_h": 512, + "target_w": 768, + }, + "negative_prompt": "ugly, blurry", + } + # target_h/w are omitted when height/width are not supplied so the + # pipeline's 1024x1024 default still applies. + assert build_text_to_image_prompt( + "MingImagePipeline", + prompt="Draw a poster.", + negative_prompt=None, + ) == { + "prompt": "Draw a poster.", + "modalities": ["image"], + "mm_processor_kwargs": {"modalities": ["image"]}, + } + + +@pytest.mark.core_model +@pytest.mark.cpu +def test_ming_flash_omni_image_to_image_prompt_builder() -> None: + dummy_image = Image.new("RGB", (64, 64)) + result = build_image_to_image_prompt( + "MingImagePipeline", + prompt="Change the background to a sandy beach at sunset.", + negative_prompt=None, + input_image=dummy_image, + height=256, + width=256, + ) + assert result["modalities"] == ["img2img"] + assert result["multi_modal_data"]["img2img"] is dummy_image + assert result["mm_processor_kwargs"] == { + "modalities": ["img2img"], + "target_h": 256, + "target_w": 256, + } + assert "negative_prompt" not in result + + @pytest.mark.core_model @pytest.mark.cpu def test_cosmos3_text_to_image_prompt_builder_selects_image_modality() -> None: diff --git a/vllm_omni/model_extras/ming_flash_omni.py b/vllm_omni/model_extras/ming_flash_omni.py new file mode 100644 index 000000000..8fece255d --- /dev/null +++ b/vllm_omni/model_extras/ming_flash_omni.py @@ -0,0 +1,66 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +from typing import Any + +from PIL import Image + + +def build_text_to_image_prompt( + prompt: str, + negative_prompt: str | None, + height: int | None = None, + width: int | None = None, +) -> dict[str, Any]: + text_prompt: dict[str, Any] = { + "prompt": prompt, + "modalities": ["image"], + "mm_processor_kwargs": {"modalities": ["image"]}, + } + if height is not None: + text_prompt["mm_processor_kwargs"]["target_h"] = height + if width is not None: + text_prompt["mm_processor_kwargs"]["target_w"] = width + if negative_prompt is not None: + text_prompt["negative_prompt"] = negative_prompt + return text_prompt + + +def build_image_to_image_prompt( + prompt: str, + negative_prompt: str | None, + input_image: Image.Image | list[Image.Image], + height: int | None = None, + width: int | None = None, +) -> dict[str, Any]: + img_prompt: dict[str, Any] = { + "prompt": prompt, + "modalities": ["img2img"], + "multi_modal_data": {"img2img": input_image}, + "mm_processor_kwargs": {"modalities": ["img2img"]}, + } + if height is not None: + img_prompt["mm_processor_kwargs"]["target_h"] = height + if width is not None: + img_prompt["mm_processor_kwargs"]["target_w"] = width + if negative_prompt is not None: + img_prompt["negative_prompt"] = negative_prompt + return img_prompt + + +# For Image-generation (diffusion-stage) +MING_FLASH_OMNI_EXTRA_BODY_PARAMS = frozenset( + { + "height", + "width", + "steps", + "cfg", + "seed", + "byte5_text", + "negative_prompt", + } +) +MING_FLASH_OMNI_EXTRA_OUTPUT_PARAMS = frozenset() +MING_FLASH_OMNI_INIT_EXTRA_ARGS_FOR_NON_DIFFUSION_STAGES = True diff --git a/vllm_omni/model_extras/registry.py b/vllm_omni/model_extras/registry.py index cec304213..7103991c7 100644 --- a/vllm_omni/model_extras/registry.py +++ b/vllm_omni/model_extras/registry.py @@ -46,6 +46,17 @@ from vllm_omni.model_extras.mammothmodal2_preview import ( from vllm_omni.model_extras.mammothmodal2_preview import ( build_text_to_image_prompt as build_mammothmoda2_text_to_image_prompt, ) +from vllm_omni.model_extras.ming_flash_omni import ( + MING_FLASH_OMNI_EXTRA_BODY_PARAMS, + MING_FLASH_OMNI_EXTRA_OUTPUT_PARAMS, + MING_FLASH_OMNI_INIT_EXTRA_ARGS_FOR_NON_DIFFUSION_STAGES, +) +from vllm_omni.model_extras.ming_flash_omni import ( + build_image_to_image_prompt as build_ming_flash_omni_image_to_image_prompt, +) +from vllm_omni.model_extras.ming_flash_omni import ( + build_text_to_image_prompt as build_ming_flash_omni_text_to_image_prompt, +) from vllm_omni.model_extras.sensenova_u1 import ( SENSENOVA_U1_EXTRA_BODY_PARAMS, SENSENOVA_U1_EXTRA_OUTPUT_PARAMS, @@ -165,6 +176,13 @@ _EXTRA_SPECS: dict[str, dict[str, Any]] = { "init_extra_args_for_non_diffusion_stages": MAMMOTHMODA2_PREVIEW_INIT_EXTRA_ARGS_FOR_NON_DIFFUSION_STAGES, "text_to_image_prompt_builder": build_mammothmoda2_text_to_image_prompt, }, + "MingImagePipeline": { + "extra_body_params": MING_FLASH_OMNI_EXTRA_BODY_PARAMS, + "extra_output_params": MING_FLASH_OMNI_EXTRA_OUTPUT_PARAMS, + "init_extra_args_for_non_diffusion_stages": MING_FLASH_OMNI_INIT_EXTRA_ARGS_FOR_NON_DIFFUSION_STAGES, + "text_to_image_prompt_builder": build_ming_flash_omni_text_to_image_prompt, + "image_to_image_prompt_builder": build_ming_flash_omni_image_to_image_prompt, + }, }