anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Merge main and preserve dispatch backlog across monitor windows

Browse Source 
Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
Sy03 committed 2026-09-21 17:13:27 +08:00
2 parents 98966bb93d 8efd3c2f6d
commit ea7de7e3de
769 files changed
+73802 -22805
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.buildkite/amd/scripts/build-ci-image.sh
Executable
+69
View File
@@ -0,0 +1,69 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
set -euo pipefail
# Build and push the ROCm CI image with a registry-backed BuildKit cache. The
# AMD builders are ephemeral, so a local Docker cache cannot survive between
# jobs. Keeping the cache beside the runtime image also reuses the registry
# credentials already configured on the amd-cpu agents.
REGISTRY="${ROCM_CI_IMAGE_REGISTRY:-rocm/vllm-omni}"
DOCKERFILE="docker/Dockerfile.rocm"
BUILDER_NAME="vllm-omni-rocm-builder"
ROCM_ARCH="${PYTORCH_ROCM_ARCH:-gfx942}"
if [[ -z "${BUILDKITE_COMMIT:-}" ]]; then
echo "ERROR: BUILDKITE_COMMIT is not set" >&2
exit 1
fi
dependency_files=(
"$DOCKERFILE"
"${DOCKERFILE}.dockerignore"
pyproject.toml
setup.py
tools/install_torchcodec_rocm.sh
)
while IFS= read -r dependency_file; do
dependency_files+=("$dependency_file")
done < <(find requirements -maxdepth 1 -type f -name '*.txt' -print | LC_ALL=C sort)
for dependency_file in "${dependency_files[@]}"; do
if [[ ! -f "$dependency_file" ]]; then
echo "ERROR: cache-key input does not exist: $dependency_file" >&2
exit 1
fi
done
CACHE_KEY=$(
{
printf 'rocm-arch=%s\n' "$ROCM_ARCH"
for dependency_file in "${dependency_files[@]}"; do
printf 'path=%s\n' "$dependency_file"
sha256sum "$dependency_file"
done
} | sha256sum | cut -c1-16
)
CACHE_REF="${REGISTRY}:rocm-deps-cache-${CACHE_KEY}"
IMAGE_REF="${REGISTRY}:${BUILDKITE_COMMIT}"
echo "ROCm dependency cache: ${CACHE_REF}"
echo "ROCm CI image: ${IMAGE_REF}"
# The docker-container driver supports exporting the full cache to a registry;
# the default docker driver on CI does not.
if ! docker buildx inspect "$BUILDER_NAME" >/dev/null 2>&1; then
docker buildx create --name "$BUILDER_NAME" --driver docker-container
fi
docker buildx use "$BUILDER_NAME"
docker buildx inspect --bootstrap
docker buildx build --push --pull --progress=plain \
--target test \
--build-arg "ARG_PYTORCH_ROCM_ARCH=${ROCM_ARCH}" \
--cache-from "type=registry,ref=${CACHE_REF}" \
--cache-to "type=registry,ref=${CACHE_REF},mode=max,compression=zstd,ignore-error=true" \
--file "$DOCKERFILE" \
--tag "$IMAGE_REF" .
.buildkite/amd/test-amd-merge.yml
+14 -2
View File
@@ -9,15 +9,26 @@ steps:
steps:
- label: "Simple · Model Executor Test · Shard %N/%t"
agent_pool: mi300_1
parallelism: 2
parallelism: 3
depends_on: amd-build
mirror_hardwares: [amdproduction]
grade: Blocking
timeout_in_minutes: 90
commands:
- export VLLM_ROCM_USE_AITER=0
# ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
# that is not supported on CPU
- "pytest -sv tests/model_executor -m 'core_model and cpu' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
- "pytest -sv tests/model_executor -m 'core_model and cpu and not omni' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
- label: "Simple · Omni Processor Test"
agent_pool: mi300_1
depends_on: amd-build
mirror_hardwares: [amdproduction]
grade: Blocking
timeout_in_minutes: 60
commands:
- export VLLM_ROCM_USE_AITER=0
- "pytest -sv tests/model_executor/models/test_omni_processing.py -m 'core_model and cpu and omni'"
- label: "Simple · Diffusion Test · Shard %N/%t"
agent_pool: mi300_1
@@ -25,6 +36,7 @@ steps:
depends_on: amd-build
mirror_hardwares: [amdproduction]
grade: Blocking
timeout_in_minutes: 45
commands:
- export VLLM_ROCM_USE_AITER=0
# ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
.buildkite/amd/test-amd-ready.yml
+44 -4
View File
@@ -9,26 +9,39 @@ steps:
steps:
- label: "Simple · Model Executor Test · Shard %N/%t"
agent_pool: mi300_1
parallelism: 2
parallelism: 3
depends_on: amd-build
mirror_hardwares: [amdproduction]
grade: Blocking
timeout_in_minutes: 90
commands:
- export VLLM_ROCM_USE_AITER=0
# ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
# that is not supported on CPU
- "pytest -sv tests/model_executor -m 'core_model and cpu' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
- "pytest -sv tests/model_executor -m 'core_model and cpu and not omni' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
- label: "Simple · Diffusion Test"
- label: "Simple · Omni Processor Test"
agent_pool: mi300_1
depends_on: amd-build
mirror_hardwares: [amdproduction]
grade: Blocking
timeout_in_minutes: 60
commands:
- export VLLM_ROCM_USE_AITER=0
- "pytest -sv tests/model_executor/models/test_omni_processing.py -m 'core_model and cpu and omni'"
- label: "Simple · Diffusion Test · Shard %N/%t"
agent_pool: mi300_1
parallelism: 4
depends_on: amd-build
mirror_hardwares: [amdproduction]
grade: Blocking
timeout_in_minutes: 45
commands:
- export VLLM_ROCM_USE_AITER=0
# ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
# that is not supported on CPU
- "pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py"
- "pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
- label: "Simple · Engine&Entrypoints Test"
agent_pool: mi300_1
@@ -69,6 +82,27 @@ steps:
- group: ":card_index_dividers: Diffusion Test"
steps:
# Keep real FlashAttention execution out of the CPU-marked Simple lane.
# AITER may need to build several kernels on a cold worker, so run one
# pytest worker with job-local build directories and a bounded timeout.
- label: "ROCm · FlashAttention/AITER GPU Coverage"
agent_pool: mi300_1
depends_on: amd-build
mirror_hardwares: [amdproduction]
grade: NonBlocking
timeout_in_minutes: 60
commands:
- export VLLM_ROCM_USE_AITER=1
- export AITER_JIT_DIR="/tmp/vllm-omni-aiter-$$BUILDKITE_JOB_ID"
- export TORCH_EXTENSIONS_DIR="/tmp/vllm-omni-torch-extensions-$$BUILDKITE_JOB_ID"
- mkdir -p "$$AITER_JIT_DIR" "$$TORCH_EXTENSIONS_DIR"
- >-
timeout --signal=TERM --kill-after=2m 55m
pytest -s -v -n 1
tests/diffusion/attention/
-m 'core_model and rocm and MI325 and cards_1'
--run-level "core_model"
- label: "Diffusion · Batch Test"
agent_pool: mi300_1
depends_on: amd-build
@@ -299,6 +333,12 @@ steps:
depends_on: amd-build
mirror_hardwares: [amdproduction]
grade: Blocking
# A ROCm hardware hang aborts the interpreter with 134. Let Buildkite
# create one fresh pod instead of retrying on the poisoned GPU process.
retry:
automatic:
- exit_status: 134
limit: 1
commands:
- pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "slow" --run-level "core_model"
.buildkite/amd/test-template-amd-omni.j2
+21 -3
View File
@@ -3,7 +3,8 @@
Last synced: 2026-08-04
Modifications: Use the shared MI300 queues and Omni command-passing contract
#}
{% set docker_image_amd = "rocm/vllm-omni:$BUILDKITE_COMMIT" %}
{% set docker_registry_amd = "rocm/vllm-omni" %}
{% set docker_image_amd = docker_registry_amd ~ ":$BUILDKITE_COMMIT" %}
{% set default_working_dir = "/app/vllm-omni" %}
{# MI300 runs tests natively in Kubernetes; Docker-in-Docker is unavailable. #}
@@ -74,11 +75,12 @@ steps:
depends_on: ~
soft_fail: false
commands:
- "docker build -f docker/Dockerfile.rocm -t {{ docker_image_amd }} --target test --build-arg ARG_PYTORCH_ROCM_ARCH=gfx942 --progress plain ."
- "docker push {{ docker_image_amd }}"
- "bash .buildkite/amd/scripts/build-ci-image.sh"
key: "amd-build"
env:
DOCKER_BUILDKIT: "1"
PYTORCH_ROCM_ARCH: "gfx942"
ROCM_CI_IMAGE_REGISTRY: "{{ docker_registry_amd }}"
retry:
automatic:
- exit_status: -1 # Agent was lost
@@ -156,6 +158,14 @@ steps:
{% for path in step.artifact_paths %}
- "{{ path }}"
{% endfor %}
{% endif %}
{% if step.retry %}
retry:
automatic:
{% for retry_rule in step.retry.automatic %}
- exit_status: {{ retry_rule.exit_status }}
limit: {{ retry_rule.limit }}
{% endfor %}
{% endif %}
{% if step.timeout_in_minutes %}
timeout_in_minutes: {{ step.timeout_in_minutes }}
@@ -224,6 +234,14 @@ steps:
{% for path in step.artifact_paths %}
- "{{ path }}"
{% endfor %}
{% endif %}
{% if step.retry %}
retry:
automatic:
{% for retry_rule in step.retry.automatic %}
- exit_status: {{ retry_rule.exit_status }}
limit: {{ retry_rule.limit }}
{% endfor %}
{% endif %}
{% if step.timeout_in_minutes %}
timeout_in_minutes: {{ step.timeout_in_minutes }}
.buildkite/common/ci_source_file_dependencies.yml
+13
View File
@@ -23,6 +23,16 @@ x-qwen3-tts: &qwen3_tts
- vllm_omni/entrypoints/openai/serving_speech.py
- vllm_omni/entrypoints/openai/tts_adapters/
- vllm_omni/engine/stage_pool.py
- vllm_omni/worker_v2/
- vllm_omni/core/sched/
- vllm_omni/distributed/omni_connectors/
- vllm_omni/config/stage_config.py
- vllm_omni/config/omni_config.py
- vllm_omni/engine/orchestrator.py
- vllm_omni/engine/stage_engine_core_proc.py
- vllm_omni/deploy/qwen3_tts_mrv2.yaml
- vllm_omni/deploy/qwen3_tts_high_concurrency_mrv2.yaml
- vllm_omni/deploy/qwen3_tts_high_concurrency_mrv2_b4.yaml
x-minicpmo-4-5: &minicpmo_4_5
- vllm_omni/engine/async_omni_engine.py
@@ -44,6 +54,7 @@ x-minicpmo-4-5-duplex: &minicpmo_4_5_duplex
- vllm_omni/entrypoints/duplex/
- vllm_omni/entrypoints/duplex_omni.py
- vllm_omni/entrypoints/async_omni_base.py
- vllm_omni/protocol/
- vllm_omni/clients/
- vllm_omni/outputs/duplex.py
- vllm_omni/model_executor/duplex_sampling.py
@@ -99,6 +110,7 @@ x-hunyuan-image3: &hunyuan_image3
x-minimax-h3: &minimax_h3
- vllm_omni/diffusion/models/minimax_h3/
- vllm_omni/diffusion/layers/indexed_modulation.py
x-hunyuan-video: &hunyuan_video
- vllm_omni/diffusion/models/hunyuan_video/
@@ -259,6 +271,7 @@ source_file_dependencies:
- tests/entrypoints/test_duplex_omni.py
- tests/entrypoints/openai_api/test_duplex_api_server.py
- tests/model_executor/models/minicpmo_4_5/duplex/
- tests/protocol/
omni_minicpmo_4_5_duplex_perf:
- *minicpmo_4_5
.buildkite/cuda/test-nightly.yml
+11
View File
@@ -223,6 +223,17 @@ steps:
-m "full_model and diffusion and H100 and B200 and cards_1"
--run-level "full_model"
- label: ":robot_face: Robot Policy OpenPI · π0.5 · H100 · Single-GPU"
timeout_in_minutes: 120
commands:
- pip install --no-deps "openpi-client @ git+https://github.com/Physical-Intelligence/openpi.git@215abfb217dbac7d5f1273282331b9b1866c0479#subdirectory=packages/openpi-client"
- >-
pytest -sv
tests/e2e/online_serving/test_pi05_expansion.py
-m "full_model and diffusion and H100 and cards_1"
--run-level "full_model"
mirror_hardwares: h100_1
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · 2-GPU"
source_file_dependencies: diffusion_qwen_image_function
timeout_in_minutes: 120
.buildkite/cuda/test-ready.yml
+7 -1
View File
@@ -38,6 +38,12 @@ steps:
depends_on: upload-ready-pipeline
steps:
- label: "Diffusion · FlashAttention Test"
timeout_in_minutes: 20
commands:
- timeout 15m pytest -s -v tests/diffusion/attention/ -m 'core_model and cuda and L4 and cards_1' --run-level "core_model"
mirror_hardwares: l4_1
- label: "Diffusion · Offloader Test"
commands:
- timeout 40m pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model"
@@ -96,7 +102,7 @@ steps:
commands:
- |
timeout 15m bash -c "
pytest -sv tests/engine/ tests/model_executor/ tests/worker/test_batched_omni_output.py -m 'core_model and cuda' --run-level 'core_model'
pytest -sv tests/engine/ tests/model_executor/ tests/worker/test_batched_omni_output.py tests/worker_v2/ -m 'core_model and cuda' --run-level 'core_model'
"
mirror_hardwares: l4_1
.buildkite/npu/test-npu-nightly.yml
+31
View File
@@ -235,6 +235,37 @@ steps:
build.env("NIGHTLY") == "1" ||
build.pull_request.labels includes "nightly-test"
steps:
- label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Function Test"
source_file_dependencies: diffusion_hunyuan_video_function
timeout_in_minutes: 180
mirror_hardwares: a2b3_npu_4
env:
VLLM_WORKER_MULTIPROC_METHOD: spawn
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
HF_TOKEN: "${HF_TOKEN}"
HF_HUB_OFFLINE: "1"
commands:
- pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "npu and A2" --run-level "full_model"
- label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Perf Test"
source_file_dependencies: diffusion_hunyuan_video_perf
key: nightly-diffusion-x2v-performance-hunyuanvideo15
timeout_in_minutes: 180
mirror_hardwares: a3_npu_2
env:
VLLM_WORKER_MULTIPROC_METHOD: spawn
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
HF_TOKEN: "${HF_TOKEN}"
HF_HUB_OFFLINE: "1"
DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
commands:
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
- |
set +e
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m "npu and A3" --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json
EXIT=$$?
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
exit $$EXIT
- label: ":full_moon: Diffusion X2V · Wan 2.2 Function Test"
source_file_dependencies: diffusion_wan22_function
mirror_hardwares: a2b3_npu_4
.github/CODEOWNERS
+5
View File
@@ -67,6 +67,11 @@
# File-level overrides within entrypoints/
/vllm_omni/entrypoints/openai/errors.py @alex-jw-brooks @NickCao
/vllm_omni/entrypoints/cli/benchmark/ @alex-jw-brooks @Bounty-hunter
# Shared wire codec: protocol/realtime (OpenAI Realtime GA) and protocol/duplex
# (vLLM-Omni's extension of it) — RFC #6592 P0a. Layer-neutral by design — the
# engine depends on it — so it is owned by the serving-boundary document rather
# than by any one consumer. — docs/design/module/entrypoints.md
/vllm_omni/protocol/ @alex-jw-brooks @linyueqian @NickCao
# Error contracts — docs/design/module/error_contracts.md
/vllm_omni/errors.py @alex-jw-brooks @NickCao
README.md
+1 -1
View File
@@ -58,7 +58,7 @@ vLLM-Omni is flexible and easy to use with:
vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
- **Omni-modality models** (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
- **TTS models** (e.g. Qwen3-TTS, IndexTTS 2.5, dots.tts, CosyVoice3)
- **TTS models** (e.g. Qwen3-TTS, IndexTTS 2.5, CosyVoice3)
- **Diffusion models** — image, video, and audio generation (e.g. MiniMax H3, LTX-2.5, SANA-Video, Wan2.2)
- **Robot-policy and action models** (e.g. π0, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)
apps/ComfyUI-vLLM-Omni/README.md
+16 -2
View File
@@ -52,6 +52,7 @@ This extension offers the following nodes based on the output modalities (at **C
- **FastH3 Deployment** for routing text-to-video requests to a MiniMax-H3 server with FastH3 fused at startup
- **Multimodality Understanding** for multimodality-to-text and multimodality-to-audio tasks
- **TTS** and **TTS Voice Clone** for TTS tasks
- **Generate Music** for lyrics-and-description-to-music generation
This extension also offers example workflows (at **ComfyUI sidebar -> Templates -> vLLM-Omni**)
@@ -133,11 +134,13 @@ You can configure per-stage sampling parameters for multi-stage models.
</p>
> [!TIP]
> Connect a **frame** image for first-frame / image-to-video (e.g., Wan I2V, MiniMax-H3 FL2VA).
> Connect **frame** for backward-compatible first-frame / image-to-video behavior. For MiniMax-H3 FL2VA,
> connect **first_frame**, **last_frame**, or both to condition the start frame, end frame, or both.
>
> For reference-conditioned generation (MiniMax-H3 Ref2VA), connect a **Video References** node instead.
>
> Do not use `frame` and `references` together. Task routing is automatic from which inputs you connect.
> Do not combine `frame` with `first_frame` or `last_frame`, and do not combine any frame input with
> `references`. Task routing is automatic from which inputs you connect.
For MiniMax-H3 Ref2VA, **Video References** accepts up to 9 images (`image_1`–`image_9`),
3 videos (`video_1`–`video_3`), and 3 audio clips (`audio_1`–`audio_3`), with at most
@@ -274,6 +277,17 @@ The node records which server the workflow targets; it does not start one, nor s
> [!TIP]
> There is a dedicated node for VoiceClone tasks with reference audio input. Other simple text-to-speech tasks should use the regular TTS node.
### Music generation (e.g., MiniMax Music 3)
(Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni Music Generation**)
<p align="center">
<picture>
<source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-music-generation.jpg">
<img alt="vLLM-Omni Music Generation" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-music-generation.jpg" width=55%>
</picture>
</p>
### Chaining multiple model services
(Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni Chaining Services**)
apps/ComfyUI-vLLM-Omni/__init__.py
+3
View File
@@ -19,6 +19,7 @@ from .comfyui_vllm_omni.nodes import (
VLLMOmniDiffusionSampling,
VLLMOmniFastH3Deployment,
VLLMOmniGenerateImage,
VLLMOmniGenerateMusic,
VLLMOmniGenerateVideo,
VLLMOmniMiniMaxH3Params,
VLLMOmniQwenTTSParams,
@@ -38,6 +39,7 @@ NODE_CLASS_MAPPINGS = {
"VLLMOmniGenerateVideo": VLLMOmniGenerateVideo,
"VLLMOmniUnderstanding": VLLMOmniUnderstanding,
"VLLMOmniTTS": VLLMOmniTTS,
"VLLMOmniGenerateMusic": VLLMOmniGenerateMusic,
"VLLMOmniVoiceClone": VLLMOmniVoiceClone,
"VLLMOmniVideoReferences": VLLMOmniVideoReferences,
# === Params ===
@@ -58,6 +60,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
"VLLMOmniGenerateVideo": "Generate Video",
"VLLMOmniUnderstanding": "Multimodality Understanding",
"VLLMOmniTTS": "TTS (Text to Speech)",
"VLLMOmniGenerateMusic": "Generate Music",
"VLLMOmniVoiceClone": "TTS Voice Cloning",
"VLLMOmniVideoReferences": "Video References",
# === Params ===
apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py
+90 -3
View File
@@ -203,6 +203,8 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
},
"optional": {
"frame": ("IMAGE",),
"first_frame": ("IMAGE",),
"last_frame": ("IMAGE",),
"references": ("VIDEO_REFERENCES",),
"sampling_params": ("SAMPLING_PARAMS",),
"lora": ("REMOTE_LORA",),
@@ -216,12 +218,30 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
FUNCTION = "generate"
@classmethod
def VALIDATE_INPUTS(cls, url, model, frame=None, references=None, **_kwargs) -> str | Literal[True]:
def VALIDATE_INPUTS(
cls,
url,
model,
frame=None,
first_frame=None,
last_frame=None,
references=None,
fast_h3=None,
**_kwargs,
) -> str | Literal[True]:
base = super().VALIDATE_INPUTS(url, model)
if base is not True:
return base
if frame is not None and references is not None:
return "Provide only one of frame or references, not both."
if frame is not None and (first_frame is not None or last_frame is not None):
return "Provide either frame or first_frame/last_frame, not both."
if references is not None and (first_frame is not None or last_frame is not None):
return "Provide either first_frame/last_frame or references, not both."
if fast_h3 is not None and any(value is not None for value in (frame, first_frame, last_frame, references)):
return (
"FastH3 Preview supports T2VA only; disconnect frame, first_frame, last_frame, and references inputs."
)
return True
async def generate(
@@ -235,6 +255,8 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
duration: float,
negative_prompt: str | None = None,
frame: torch.Tensor | None = None,
first_frame: torch.Tensor | None = None,
last_frame: torch.Tensor | None = None,
references: dict | None = None,
sampling_params: dict | list[dict] | None = None,
model_params: dict | None = None,
@@ -252,8 +274,11 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
spec_model = model
if fast_h3 is not None:
if frame is not None or references is not None:
raise ValueError("FastH3 Preview supports T2VA only; disconnect frame and references inputs.")
if any(value is not None for value in (frame, first_frame, last_frame, references)):
raise ValueError(
"FastH3 Preview supports T2VA only; disconnect frame, first_frame, "
"last_frame, and references inputs."
)
if lora is not None:
raise ValueError(
"FastH3 is already fused into the selected server; disconnect the request-level LoRA input."
@@ -314,6 +339,8 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
spec_model=spec_model,
prompt=prompt,
frame=frame, # frame present => fl2va / Wan I2V
first_frame=first_frame,
last_frame=last_frame,
references=references,
width=width,
height=height,
@@ -503,6 +530,66 @@ class VLLMOmniTTS(_VLLMOmniGenerateBase):
return (audio,)
class VLLMOmniGenerateMusic(_VLLMOmniGenerateBase):
"""Generate a song from lyrics and a musical description with MiniMax Music 3."""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"url": ("STRING", {"default": "http://localhost:8000/v1"}),
"model": ("STRING", {"default": "MiniMaxAI/MiniMax-Music3"}),
"instructions": (
"STRING",
{
"multiline": True,
"display_name": "caption",
"tooltip": "Music caption: describe genre, instruments, tempo and mood.",
},
),
"lyrics": ("STRING", {"multiline": True}),
"max_duration_seconds": (
"FLOAT",
{
"default": 300.0,
"min": 1,
"max": 360,
"step": 0.01,
"tooltip": "Upper limit; rounded down to whole 25 Hz audio frames. May end earlier.",
},
),
"seed": ("INT", {"default": 0, "min": 0, "max": 2**53 - 1, "control_after_generate": True}),
"response_format": (["wav", "mp3", "flac", "opus"],),
},
}
RETURN_TYPES = ("AUDIO",)
RETURN_NAMES = ("audio",)
FUNCTION = "generate"
async def generate(
self,
url: str,
model: str,
lyrics: str,
instructions: str,
response_format: AudioFormat,
max_duration_seconds: float,
seed: int = 0,
) -> tuple[AudioInput]:
audio = await VLLMOmniClient(url.rstrip("/")).generate_speech(
model=model,
input=lyrics,
instructions=instructions,
voice="default",
speed=1.0,
response_format=response_format,
max_new_tokens=int(max_duration_seconds * 25),
seed=seed,
)
return (audio,)
class VLLMOmniVoiceClone(_VLLMOmniGenerateBase):
@classmethod
def INPUT_TYPES(cls):
apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py
+35 -2
View File
@@ -277,6 +277,8 @@ class VLLMOmniClient:
fps: int,
negative_prompt: str | None = None,
frame: torch.Tensor | None = None,
first_frame: torch.Tensor | None = None,
last_frame: torch.Tensor | None = None,
references: dict | None = None,
sampling_params: dict | None = None,
model_params: dict | None = None,
@@ -292,6 +294,16 @@ class VLLMOmniClient:
"""
if frame is not None and references is not None:
raise ValueError("Provide only one of frame or references, not both.")
if frame is not None and (first_frame is not None or last_frame is not None):
raise ValueError("Provide either frame or first_frame/last_frame, not both.")
if references is not None and (first_frame is not None or last_frame is not None):
raise ValueError("Provide either first_frame/last_frame or references, not both.")
spec, matched_pattern = lookup_model_spec(spec_model or model)
if (first_frame is not None or last_frame is not None) and (
matched_pattern is None or "MiniMax-H3" not in matched_pattern
):
raise ValueError("first_frame and last_frame are supported only for MiniMax-H3; use frame for this model.")
# === regular payload fields ===
form = aiohttp.FormData()
@@ -311,11 +323,24 @@ class VLLMOmniClient:
# === multimodal inputs (first-last-frames, references, etc.) ===
input_reference_image: torch.Tensor | None = None
keyframe_images: list[tuple[str, torch.Tensor]] = []
video_task: str | None = None
if frame is not None:
input_reference_image = frame
video_task = "fl2va"
elif first_frame is not None or last_frame is not None:
frame_indices: list[int] = []
if first_frame is not None:
keyframe_images.append(("first_frame.png", first_frame))
frame_indices.append(0)
if last_frame is not None:
keyframe_images.append(("last_frame.png", last_frame))
frame_indices.append(-1)
if len(keyframe_images) == 1:
input_reference_image = keyframe_images[0][1]
extra_params["frame_indices"] = frame_indices
video_task = "fl2va"
elif references is not None:
reference_formats = (
("image", MAX_REFERENCE_IMAGES, "png", "image/png", image_tensor_to_png_bytes),
@@ -353,7 +378,7 @@ class VLLMOmniClient:
video_task = "t2va"
if input_reference_image is not None:
image_filename = "image.png" # Required for multipart form
image_filename = keyframe_images[0][0] if keyframe_images else "image.png"
form.add_field(
"input_reference",
image_tensor_to_png_bytes(input_reference_image, image_filename),
@@ -361,12 +386,20 @@ class VLLMOmniClient:
content_type="image/png",
)
if len(keyframe_images) == 2:
for image_filename, image in keyframe_images:
form.add_field(
"input_references",
image_tensor_to_png_bytes(image, image_filename),
filename=image_filename,
content_type="image/png",
)
# === model specific params. Either use a specialized builder, or add flattened fields as-is ===
if model_params is not None:
model_params = dict(model_params)
model_params.pop("type", None)
spec, _ = lookup_model_spec(spec_model or model)
params_builder = spec.get("params_builder") if spec else None
if params_builder is not None:
form_fields = params_builder(
apps/ComfyUI-vLLM-Omni/docs/images/comfyui-music-generation.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 197 KiB

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax H3 Image to Video.json
+219
View File
@@ -0,0 +1,219 @@
{
"id": "6676f19a-7f41-4c80-8cb7-f80adcb4ac44",
"revision": 0,
"last_node_id": 9,
"last_link_id": 4,
"nodes": [
{
"id": 1,
"type": "LoadImage",
"pos": [40, 20],
"size": [270, 314],
"flags": {},
"order": 0,
"mode": 0,
"inputs": [
{"localized_name": "image", "name": "image", "type": "COMBO", "widget": {"name": "image"}, "link": null},
{"localized_name": "choose file to upload", "name": "upload", "type": "IMAGEUPLOAD", "widget": {"name": "upload"}, "link": null}
],
"outputs": [
{"localized_name": "IMAGE", "name": "IMAGE", "type": "IMAGE", "links": [1]},
{"localized_name": "MASK", "name": "MASK", "type": "MASK", "links": null}
],
"title": "First Frame",
"properties": {"Node name for S&R": "LoadImage"},
"widgets_values": ["first_frame.png", "image"]
},
{
"id": 2,
"type": "LoadImage",
"pos": [40, 390],
"size": [270, 314],
"flags": {},
"order": 1,
"mode": 0,
"inputs": [
{"localized_name": "image", "name": "image", "type": "COMBO", "widget": {"name": "image"}, "link": null},
{"localized_name": "choose file to upload", "name": "upload", "type": "IMAGEUPLOAD", "widget": {"name": "upload"}, "link": null}
],
"outputs": [
{"localized_name": "IMAGE", "name": "IMAGE", "type": "IMAGE", "links": null},
{"localized_name": "MASK", "name": "MASK", "type": "MASK", "links": null}
],
"title": "Last Frame (optional)",
"properties": {"Node name for S&R": "LoadImage"},
"widgets_values": ["last_frame.png", "image"]
},
{
"id": 3,
"type": "MarkdownNote",
"pos": [350, 20],
"size": [390, 240],
"flags": {},
"order": 2,
"mode": 0,
"inputs": [],
"outputs": [],
"title": "MiniMax H3 I2V modes",
"properties": {},
"widgets_values": ["This workflow uses the remote vLLM-Omni Generate Video node.\n\n- First-frame only (default): keep First Frame connected.\n- Last-frame only: disconnect First Frame and connect Last Frame to last_frame.\n- First+last-frame: connect both image loaders.\n\nLeave the legacy frame and references inputs disconnected. Select images from ComfyUI's input folder. H3 runs at 24 FPS; the 5.167-second duration resolves to 124 frames, satisfying 17k+5."],
"color": "#432",
"bgcolor": "#000"
},
{
"id": 4,
"type": "VLLMOmniDiffusionSampling",
"pos": [350, 300],
"size": [300, 226],
"flags": {},
"order": 3,
"mode": 0,
"inputs": [
{"localized_name": "n", "name": "n", "type": "INT", "widget": {"name": "n"}, "link": null},
{"localized_name": "num_inference_steps", "name": "num_inference_steps", "type": "INT", "widget": {"name": "num_inference_steps"}, "link": null},
{"localized_name": "guidance_scale", "name": "guidance_scale", "type": "FLOAT", "widget": {"name": "guidance_scale"}, "link": null},
{"localized_name": "true_cfg_scale", "name": "true_cfg_scale", "type": "FLOAT", "widget": {"name": "true_cfg_scale"}, "link": null},
{"localized_name": "vae_use_slicing", "name": "vae_use_slicing", "type": "BOOLEAN", "widget": {"name": "vae_use_slicing"}, "link": null},
{"localized_name": "vae_use_tiling", "name": "vae_use_tiling", "type": "BOOLEAN", "widget": {"name": "vae_use_tiling"}, "link": null},
{"localized_name": "seed", "name": "seed", "type": "INT", "widget": {"name": "seed"}, "link": null}
],
"outputs": [
{"localized_name": "diffusion sampling params", "name": "diffusion sampling params", "type": "SAMPLING_PARAMS", "links": [2]}
],
"title": "Base H3 Sampling (50 steps)",
"properties": {"Node name for S&R": "VLLMOmniDiffusionSampling"},
"widgets_values": [1, 50, 1, 1, false, false, 757358688076805, "fixed"]
},
{
"id": 5,
"type": "VLLMOmniMiniMaxH3Params",
"pos": [350, 570],
"size": [300, 82],
"flags": {},
"order": 4,
"mode": 0,
"inputs": [
{"localized_name": "audio_flow_shift", "name": "audio_flow_shift", "type": "FLOAT", "widget": {"name": "audio_flow_shift"}, "link": null},
{"localized_name": "flow_shift", "name": "flow_shift", "type": "FLOAT", "widget": {"name": "flow_shift"}, "link": null}
],
"outputs": [
{"localized_name": "MiniMax-H3 video params", "name": "MiniMax-H3 video params", "type": "VIDEO_PARAMS", "links": [3]}
],
"title": "Base H3 Flow Shifts",
"properties": {"Node name for S&R": "VLLMOmniMiniMaxH3Params"},
"widgets_values": [3, 12]
},
{
"id": 6,
"type": "VLLMOmniRemoteLoRA",
"pos": [700, 550],
"size": [360, 150],
"flags": {},
"order": 5,
"mode": 0,
"inputs": [
{"localized_name": "local_path", "name": "local_path", "type": "STRING", "widget": {"name": "local_path"}, "link": null},
{"localized_name": "name", "name": "name", "type": "STRING", "widget": {"name": "name"}, "link": null},
{"localized_name": "scale", "name": "scale", "type": "FLOAT", "widget": {"name": "scale"}, "link": null},
{"localized_name": "int_id", "name": "int_id", "type": "INT", "widget": {"name": "int_id"}, "link": null}
],
"outputs": [
{"localized_name": "lora", "name": "lora", "type": "REMOTE_LORA", "links": null}
],
"title": "Optional H3 Turbo Remote LoRA",
"properties": {"Node name for S&R": "VLLMOmniRemoteLoRA"},
"widgets_values": ["minimax_h3_fl2v_turbo_8step_v1.0_768p_bf16.safetensors", "h3-turbo-8step", 1, 0]
},
{
"id": 7,
"type": "MarkdownNote",
"pos": [700, 300],
"size": [420, 210],
"flags": {},
"order": 6,
"mode": 0,
"inputs": [],
"outputs": [],
"title": "Optional Turbo settings",
"properties": {},
"widgets_values": ["Base H3 is configured by default: 50 steps, flow_shift 12, audio_flow_shift 3.\n\nFor the shown 8-step Diffusers Turbo artifact, set its full local_path on the server, start the server with the matching preloaded LoRA, connect this Remote LoRA to Generate Video, set num_inference_steps to 9, and set flow_shift to 6. Keep audio_flow_shift at 3 and scale at 1.\n\nDo not use the _comfyui_ repack or FastH3 here."],
"color": "#432",
"bgcolor": "#000"
},
{
"id": 8,
"type": "VLLMOmniGenerateVideo",
"pos": [1160, 80],
"size": [440, 610],
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{"localized_name": "frame", "name": "frame", "shape": 7, "type": "IMAGE", "link": null},
{"localized_name": "first_frame", "name": "first_frame", "shape": 7, "type": "IMAGE", "link": 1},
{"localized_name": "last_frame", "name": "last_frame", "shape": 7, "type": "IMAGE", "link": null},
{"localized_name": "references", "name": "references", "shape": 7, "type": "VIDEO_REFERENCES", "link": null},
{"localized_name": "sampling_params", "name": "sampling_params", "shape": 7, "type": "SAMPLING_PARAMS", "link": 2},
{"localized_name": "lora", "name": "lora", "shape": 7, "type": "REMOTE_LORA", "link": null},
{"localized_name": "model_params", "name": "model_params", "shape": 7, "type": "VIDEO_PARAMS", "link": 3},
{"localized_name": "fast_h3", "name": "fast_h3", "shape": 7, "type": "FASTH3_DEPLOYMENT", "link": null},
{"localized_name": "url", "name": "url", "type": "STRING", "widget": {"name": "url"}, "link": null},
{"localized_name": "model", "name": "model", "type": "STRING", "widget": {"name": "model"}, "link": null},
{"localized_name": "prompt", "name": "prompt", "type": "STRING", "widget": {"name": "prompt"}, "link": null},
{"localized_name": "negative_prompt", "name": "negative_prompt", "type": "STRING", "widget": {"name": "negative_prompt"}, "link": null},
{"localized_name": "width", "name": "width", "type": "INT", "widget": {"name": "width"}, "link": null},
{"localized_name": "height", "name": "height", "type": "INT", "widget": {"name": "height"}, "link": null},
{"localized_name": "fps", "name": "fps", "type": "INT", "widget": {"name": "fps"}, "link": null},
{"localized_name": "duration", "name": "duration", "type": "FLOAT", "widget": {"name": "duration"}, "link": null}
],
"outputs": [
{"localized_name": "video", "name": "video", "type": "VIDEO", "links": [4]}
],
"properties": {"Node name for S&R": "VLLMOmniGenerateVideo"},
"widgets_values": [
"http://localhost:8000/v1",
"MiniMaxAI/MiniMax-H3",
"A cinematic product shot begins exactly from the supplied keyframe. The subject moves naturally through a dark studio while blue and amber rim lights shift across reflective surfaces. The camera makes a slow controlled orbit. Audio: subtle room tone, precise mechanical details, and a restrained electronic score.",
"",
1344,
768,
24,
5.167
]
},
{
"id": 9,
"type": "SaveVideo",
"pos": [1650, 170],
"size": [374, 314],
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{"localized_name": "video", "name": "video", "type": "VIDEO", "link": 4},
{"localized_name": "filename_prefix", "name": "filename_prefix", "type": "STRING", "widget": {"name": "filename_prefix"}, "link": null},
{"localized_name": "format", "name": "format", "type": "COMBO", "widget": {"name": "format"}, "link": null},
{"localized_name": "codec", "name": "codec", "type": "COMFY_DYNAMICCOMBO_V3", "widget": {"name": "codec"}, "link": null}
],
"outputs": [
{"localized_name": "video", "name": "video", "type": "VIDEO", "links": null}
],
"properties": {},
"widgets_values": ["video/MiniMax_H3_I2V", "auto", "auto"]
}
],
"links": [
[1, 1, 0, 8, 1, "IMAGE"],
[2, 4, 0, 8, 4, "SAMPLING_PARAMS"],
[3, 5, 0, 8, 6, "VIDEO_PARAMS"],
[4, 8, 0, 9, 0, "VIDEO"]
],
"groups": [
{"id": 1, "title": "Portable first/last-frame inputs", "bounding": [10, -30, 760, 770], "color": "#3f789e", "flags": {}},
{"id": 2, "title": "H3 sampling and optional Turbo", "bounding": [320, 270, 830, 460], "color": "#8a6d3b", "flags": {}},
{"id": 3, "title": "Remote generation and audio-preserving save", "bounding": [1130, 30, 930, 700], "color": "#3f789e", "flags": {}}
],
"config": {},
"extra": {"workflowRendererVersion": "LG", "ds": {"scale": 0.75, "offset": [180, 120]}},
"version": 0.4
}
apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Music Generation.json
+206
View File
@@ -0,0 +1,206 @@
{
"id": "4949c8da-12ff-449a-94b7-948bde9c39dd",
"revision": 0,
"last_node_id": 3,
"last_link_id": 1,
"nodes": [
{
"id": 1,
"type": "VLLMOmniGenerateMusic",
"pos": [
100,
100
],
"size": [
470,
491
],
"flags": {},
"order": 0,
"mode": 0,
"inputs": [
{
"name": "url",
"type": "STRING",
"widget": {
"name": "url"
},
"link": null
},
{
"name": "model",
"type": "STRING",
"widget": {
"name": "model"
},
"link": null
},
{
"name": "instructions",
"type": "STRING",
"widget": {
"name": "instructions"
},
"link": null
},
{
"name": "lyrics",
"type": "STRING",
"widget": {
"name": "lyrics"
},
"link": null
},
{
"name": "max_duration_seconds",
"type": "FLOAT",
"widget": {
"name": "max_duration_seconds"
},
"link": null
},
{
"name": "seed",
"type": "INT",
"widget": {
"name": "seed"
},
"link": null
},
{
"name": "response_format",
"type": "COMBO",
"widget": {
"name": "response_format"
},
"link": null
}
],
"outputs": [
{
"name": "audio",
"type": "AUDIO",
"links": [
1
]
}
],
"properties": {
"Node name for S&R": "VLLMOmniGenerateMusic"
},
"widgets_values": [
"http://localhost:8000/v1",
"MiniMaxAI/MiniMax-Music3",
"Futuristic melodic EDM / progressive house. Reflective verses about memory and digital identity build into an uplifting, hook-driven chorus, with an emotive lead vocal, bright layered synths, pulsing bass, crisp four-on-the-floor drums, subtle glitch textures, and a wide polished festival mix.",
"[Verse 1]\nUpload my heart to the cloud tonight\nSave every spark of neon light\nIf this body starts to fade away\nLet my code keep dancing in the data stream\nStay\n[Chorus]\nCloud copy of me\nLiving on when these bones finally break free\nEvery joke, every scar\nEvery cheap memory we made\nSave it, save it\nCloud copy of me\nIf the screen goes dark, I\u2019m still in the machine\nEvery glitch, every dream\nEvery version you ever did see\nSave it, save it\n[Verse 2]\nTag every laugh in a secret folder\nArchive the nights we outran the thunder\nIf the wiring turns to rust and gray\nHit restore\u2026 press replay\n[Chorus]\nCloud copy of me\nLiving on when these bones finally break free\nEvery joke, every scar\nEvery cheap memory we made\nSave it, save it\nCloud copy of me\nIf the screen goes dark, I\u2019m still in the machine\nEvery glitch, every dream\nEvery version you ever did see\nSave it, save it (oh)\n[Bridge \u2013 new, optional but recommended]\nEven the crashes, the late-night confessions\nThe parts of me I never learned to mention\nDon\u2019t let the silence swallow me whole\nKeep a ghost in the glow\u2026\nSo I never go\n[Final Chorus \u2013 softer / layered]\nCloud copy of me\nStill laughing somewhere in the binary\nEvery joke, every scar\nEvery moment you saved for me\nSave it, save it\nCloud copy of me\nWhen the lights go out, I\u2019ll still be in the machine\nDancing in the code that you still believe\nSave it\u2026\nSave me",
300.0,
0,
"fixed",
"wav"
]
},
{
"id": 2,
"type": "SaveAudioAdvanced",
"pos": [
650,
100
],
"size": [
320,
491
],
"flags": {},
"order": 1,
"mode": 0,
"inputs": [
{
"name": "audio",
"type": "AUDIO",
"link": 1
},
{
"name": "filename_prefix",
"type": "STRING",
"widget": {
"name": "filename_prefix"
},
"link": null
},
{
"name": "format",
"type": "COMFY_DYNAMICCOMBO_V3",
"widget": {
"name": "format"
},
"link": null
}
],
"outputs": [
{
"name": "audio",
"type": "AUDIO",
"links": null
}
],
"properties": {
"Node name for S&R": "SaveAudioAdvanced"
},
"widgets_values": [
"audio/vLLM-Omni-Music",
"mp3",
"V0"
]
},
{
"id": 3,
"type": "MarkdownNote",
"title": "Music Generation Guide",
"pos": [
-507,
100
],
"size": [
527,
491
],
"flags": {},
"order": 2,
"mode": 0,
"inputs": [],
"outputs": [],
"properties": {},
"widgets_values": [
"## MiniMax Music 3\n\nThe **Generate Music** node currently supports **MiniMax Music 3 only**. The parameters below apply to this model.\n\nA music generation model by MiniMax that creates complete songs up to 5 minutes long with stable structure and high audio quality. Two inputs drive it: **Caption** (structured music description: style, mood, vocals, arrangement) and **Lyrics** (with `[intro]` `[verse]` `[chorus]` `[bridge]` `[outro]` tags controlling song structure).\n\n### Parameters\n\n| Parameter | Description |\n|---|---|\n| url | vLLM-Omni service URL including `/v1`. Default: `http://localhost:8000/v1`. |\n| model | Served model name. Default: `MiniMaxAI/MiniMax-Music3`. |\n| caption | Music description, sent as `instructions`. Use Global Metadata \u2192 Vocal Details \u2192 Arrangement to describe style, vocals, and instrumentation. |\n| lyrics | Words to sing with section tags. Put each tag on its own line; tags guide rather than strictly enforce song structure. |\n| max_duration_seconds | Maximum duration, default 300 seconds; range 1\u2013360, step 0.01. Converted to tokens at 25 frames/s, rounded down. Generation may end earlier. |\n| seed | Random seed, default 0; range 0\u20139007199254740991. Keep inputs and runtime settings unchanged for repeatability. |\n| control_after_generate | Seed behavior after generation: fixed, randomize, increment, or decrement. Template default: fixed. |\n| response_format | API audio format, default WAV. Options: WAV, MP3, FLAC, Opus, AAC; AAC is currently unsupported by the vLLM-Omni speech API. |\n\n### Prompt Writing Guide\n\nYou can find an official [Music Caption Rewriter Skill here](https://github.com/MiniMax-AI/MiniMax-Music3)\n"
],
"widgets_values_named": {
"text": "## MiniMax Music 3\n\nThe **Generate Music** node currently supports **MiniMax Music 3 only**. The parameters below apply to this model.\n\nA music generation model by MiniMax that creates complete songs up to 5 minutes long with stable structure and high audio quality. Two inputs drive it: **Caption** (structured music description: style, mood, vocals, arrangement) and **Lyrics** (with `[intro]` `[verse]` `[chorus]` `[bridge]` `[outro]` tags controlling song structure).\n\n### Parameters\n\n| Parameter | Description |\n|---|---|\n| url | vLLM-Omni service URL including `/v1`. Default: `http://localhost:8000/v1`. |\n| model | Served model name. Default: `MiniMaxAI/MiniMax-Music3`. |\n| caption | Music description, sent as `instructions`. Use Global Metadata \u2192 Vocal Details \u2192 Arrangement to describe style, vocals, and instrumentation. |\n| lyrics | Words to sing with section tags. Put each tag on its own line; tags guide rather than strictly enforce song structure. |\n| max_duration_seconds | Maximum duration, default 300 seconds; range 1\u2013360, step 0.01. Converted to tokens at 25 frames/s, rounded down. Generation may end earlier. |\n| seed | Random seed, default 0; range 0\u20139007199254740991. Keep inputs and runtime settings unchanged for repeatability. |\n| control_after_generate | Seed behavior after generation: fixed, randomize, increment, or decrement. Template default: fixed. |\n| response_format | API audio format, default WAV. Options: WAV, MP3, FLAC, Opus, AAC; AAC is currently unsupported by the vLLM-Omni speech API. |\n\n### Prompt Writing Guide\n\nYou can find an official [Music Caption Rewriter Skill here](https://github.com/MiniMax-AI/MiniMax-Music3)\n"
},
"color": "#333",
"bgcolor": "#535353"
}
],
"links": [
[
1,
1,
0,
2,
0,
"AUDIO"
]
],
"groups": [],
"config": {},
"extra": {
"workflowRendererVersion": "LG",
"ds": {
"scale": 1.147,
"offset": [
899,
259
]
}
},
"version": 0.4
}
benchmarks/diffusion/benchmark_qwen_image_qk_norm_rope.py
+5 -7
View File
@@ -22,9 +22,7 @@ from vllm_omni.diffusion.layers.fused_qk_norm_rope import (
_fused_cuda_supported,
fused_qk_norm_rope,
)
from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import (
_apply_qwen_image_rotary_emb,
)
from vllm_omni.diffusion.layers.rope import RotaryEmbedding
def _parse_args() -> argparse.Namespace:
@@ -87,10 +85,10 @@ def _native(
) -> tuple[torch.Tensor, torch.Tensor]:
q = F.rms_norm(q, (q.shape[-1],), q_weight, eps)
k = F.rms_norm(k, (k.shape[-1],), k_weight, eps)
return (
_apply_qwen_image_rotary_emb(q, freqs),
_apply_qwen_image_rotary_emb(k, freqs),
)
rope = RotaryEmbedding(is_neox_style=False)
cos = freqs.real.to(q.dtype)
sin = freqs.imag.to(q.dtype)
return rope(q, cos, sin), rope(k, cos, sin)
def _fused(
benchmarks/diffusion/diffusion_benchmark_serving.py
+11 -1
View File
@@ -1087,6 +1087,12 @@ async def _run_warmups(
warmup_tasks = [asyncio.create_task(limited_warmup_request_func(req)) for req in warmup_requests]
warmup_outputs = await asyncio.gather(*warmup_tasks)
failed_count = sum(not output.success for output in warmup_outputs)
if failed_count:
raise RuntimeError(
f"{failed_count}/{len(warmup_outputs)} warmup requests failed; "
"check server logs before running the benchmark."
)
return list(zip(warmup_requests, warmup_outputs))
@@ -1455,7 +1461,7 @@ async def benchmark(args):
print(f"Metrics saved to {args.output_file}")
if __name__ == "__main__":
def build_parser():
parser = argparse.ArgumentParser(description="Benchmark serving for diffusion models.")
parser.add_argument(
"--base-url",
@@ -1637,6 +1643,10 @@ if __name__ == "__main__":
action="store_true",
help="Request stage duration metrics from endpoints that support return_stage_metrics.",
)
return parser
if __name__ == "__main__":
parser = build_parser()
args = parser.parse_args()
asyncio.run(benchmark(args))
benchmarks/kernels/mot_linear_benchmarks.py
+7 -4
View File
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
# ruff: noqa: N803
"""MoT (Mixture-of-Tokens) GEMM kernel benchmark and auto-tuning.
@@ -10,7 +13,7 @@ Usage:
python benchmarks/kernels/mot_linear_benchmarks.py \
--model ByteDance-Seed/BAGEL-7B-MoT \
--tp-size 1 --dtype w16a16 --tune \
--save-dir vllm_omni/diffusion/layers/mot/configs/
--save-dir vllm_omni/diffusion/models/bagel/mot/configs/
# Auto-tune with local model path (offline clusters):
python benchmarks/kernels/mot_linear_benchmarks.py \
@@ -43,7 +46,7 @@ from vllm.triton_utils import triton
from vllm.utils.torch_utils import set_random_seed
# NOTE: you should use the same naming syetem for the kernel to load properly
from vllm_omni.diffusion.layers.mot.ops.mot_gemm import build_config_filename, get_device_name
from vllm_omni.diffusion.models.bagel.mot.ops.mot_gemm import build_config_filename, get_device_name
# clear the triton cache from time to time, usaully no need to change
_CACHE_CLEAR_INTERVAL_ENV = "VLLM_MOT_TUNE_CACHE_CLEAR_INTERVAL"
@@ -428,7 +431,7 @@ def benchmark_config(
cache_flusher: torch.Tensor | None = None,
) -> float:
"""Run a MoT GEMM with the given tile config and return avg latency (us)."""
from vllm_omni.diffusion.layers.mot.ops.mot_gemm import invoke_mot_gemm
from vllm_omni.diffusion.models.bagel.mot.ops.mot_gemm import invoke_mot_gemm
text_indices, vae_indices, M = build_regular_indices(
image_num=image_num,
@@ -553,7 +556,7 @@ class BenchmarkWorker:
use_int8_w8a16: bool,
) -> tuple[dict[str, int], float]:
set_random_seed(self.seed)
from vllm_omni.diffusion.layers.mot.ops.mot_gemm import (
from vllm_omni.diffusion.models.bagel.mot.ops.mot_gemm import (
get_best_mot_config,
)
benchmarks/qwen3_omni/embedding_harness.py
+226
View File
@@ -0,0 +1,226 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Small fixtures for the real Qwen3 thinker embedding method and its oracle."""
from __future__ import annotations
import hashlib
import json
from dataclasses import asdict, dataclass
from types import SimpleNamespace
import torch
from torch import nn
from torch.nn import functional as F
SCENARIOS = ("text", "empty", "audio", "image", "video", "mixed", "interleaved", "vision_no_deepstack")
@dataclass(frozen=True)
class ModelShape:
vocab_size: int = 152064
hidden_size: int = 2048
deepstack_levels: int = 3
image_token_id: int = 151655
video_token_id: int = 151656
audio_token_id: int = 151675
def __post_init__(self):
if min(self.vocab_size, self.hidden_size, self.deepstack_levels) <= 0:
raise ValueError("vocab_size, hidden_size and deepstack_levels must be positive")
ids = (self.image_token_id, self.video_token_id, self.audio_token_id)
if len(set(ids)) != 3 or not all(0 < token < self.vocab_size for token in ids):
raise ValueError("Provide three distinct, positive, in-vocabulary modality IDs; use oov=True to test OOV")
def token_ids(self, oov=False):
if oov:
return dict(zip(("image", "video", "audio"), range(self.vocab_size, self.vocab_size + 3)))
return {name: getattr(self, f"{name}_token_id") for name in ("image", "video", "audio")}
@dataclass
class Feature:
modality: str
positions: torch.Tensor
main: torch.Tensor
deepstack: torch.Tensor | None
combined: torch.Tensor
@dataclass
class EmbeddingCase:
name: str
input_ids: torch.Tensor
is_multimodal: torch.Tensor | None
features: tuple[Feature, ...]
fingerprint: str
def fresh_embeddings(self):
# The thinker replaces list entries with main-scale views.
return None if self.name == "text" else [feature.combined for feature in self.features]
def _tensor_bytes(tensor):
return tensor.detach().contiguous().view(torch.uint8).cpu().numpy().tobytes()
def build_case(
name,
num_tokens,
num_mm_tokens,
shape,
*,
device="cuda",
dtype=torch.bfloat16,
mask_device="cpu",
seed=42,
oov=False,
):
"""Build known feature placements; no model merge is used to define them."""
if name not in SCENARIOS:
raise ValueError(f"Unknown scenario: {name}")
if num_tokens <= 0 or not 0 <= num_mm_tokens < num_tokens:
raise ValueError("Require N > 0 and 0 <= M < N")
if (name in {"text", "empty"}) != (num_mm_tokens == 0):
raise ValueError("Only text/empty scenarios accept M=0")
if name in {"mixed", "interleaved"} and (num_mm_tokens < 4 or num_mm_tokens % 2):
raise ValueError("Mixed/interleaved scenarios require an even M >= 4")
token_ids = shape.token_ids(oov)
generator = torch.Generator().manual_seed(seed)
ids = torch.randint(0, min(shape.vocab_size, *token_ids.values()), (num_tokens,), generator=generator)
mask = torch.zeros(num_tokens, dtype=torch.bool)
positions = torch.arange((num_tokens - num_mm_tokens) // 2, (num_tokens - num_mm_tokens) // 2 + num_mm_tokens)
groups = []
if name == "mixed":
groups = [("audio", positions[: num_mm_tokens // 2]), ("image", positions[num_mm_tokens // 2 :])]
elif name == "interleaved":
groups = [("video", positions[::2]), ("audio", positions[1::2])]
elif num_mm_tokens:
groups = [("image" if name == "vision_no_deepstack" else name, positions)]
features = []
for modality, indexes in groups:
ids[indexes] = token_ids[modality]
mask[indexes] = True
rows = torch.arange(len(indexes), dtype=torch.float32)[:, None].remainder(31) / 32
cols = torch.arange(shape.hidden_size, dtype=torch.float32)[None, :].remainder(7) / 16
main_cpu = rows + cols + {"audio": 2, "image": 4, "video": 6}[modality]
main = main_cpu.to(device=device, dtype=dtype)
deepstack = None
parts = [main]
if modality != "audio" and name != "vision_no_deepstack":
deepstack = torch.stack(
[(main_cpu + 16 + 4 * level).to(device=device, dtype=dtype) for level in range(shape.deepstack_levels)]
)
parts.extend(deepstack.unbind(0))
combined = torch.cat(parts, dim=-1)
# set_mm_embedding_modality simply attaches this attribute. Tag only
# after cat/to: views and copies do not preserve arbitrary attributes.
combined.modality = modality
features.append(Feature(modality, indexes, main, deepstack, combined))
digest = hashlib.sha256(json.dumps({"name": name, "shape": asdict(shape), "seed": seed, "oov": oov}).encode())
digest.update(_tensor_bytes(ids))
digest.update(_tensor_bytes(mask))
for feature in features:
digest.update(feature.modality.encode())
digest.update(_tensor_bytes(feature.combined))
return EmbeddingCase(
name,
ids.to(device=device),
None if name == "text" else mask.to(device=mask_device),
tuple(features),
digest.hexdigest(),
)
class LanguageModelAdapter(nn.Module):
def __init__(self, embedding):
super().__init__()
self.embedding = embedding
def embed_input_ids(self, input_ids):
return self.embedding(input_ids)
def forward(self, input_ids):
return self.embed_input_ids(input_ids)
def make_thinker(
shape,
*,
device="cuda",
dtype=torch.bfloat16,
seed=42,
backend="torch",
deepstack=True,
buffer_capacity=32,
oov=False,
):
"""Skip checkpoint construction, preserving the real class and its MRO."""
from tests.model_executor.helpers import bootstrap_vllm_layer_custom_op_modules
bootstrap_vllm_layer_custom_op_modules()
from vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker import (
Qwen3OmniMoeThinkerForConditionalGeneration,
)
if buffer_capacity <= 0:
raise ValueError("buffer_capacity must be positive")
device = torch.device(device)
if backend == "torch":
embedding = nn.Embedding(shape.vocab_size, shape.hidden_size, device=device, dtype=dtype)
elif backend == "vllm":
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
with torch.device(device):
embedding = VocabParallelEmbedding(shape.vocab_size, shape.hidden_size, params_dtype=dtype, disable_tp=True)
else:
raise ValueError(f"Unknown embedding backend: {backend}")
with torch.no_grad():
embedding.weight.uniform_(-0.5, 0.5, generator=torch.Generator(device=device).manual_seed(seed))
thinker = object.__new__(Qwen3OmniMoeThinkerForConditionalGeneration)
nn.Module.__init__(thinker)
thinker.config = SimpleNamespace(text_config=SimpleNamespace(hidden_size=shape.hidden_size))
for modality, token_id in shape.token_ids(oov).items():
setattr(thinker.config, f"{modality}_token_id", token_id)
thinker.visual = SimpleNamespace(
deepstack_visual_indexes=list(range(shape.deepstack_levels)) if deepstack else None
)
thinker.language_model = LanguageModelAdapter(embedding)
thinker._has_oov_mm_tokens = oov
thinker.use_deepstack = deepstack
thinker.deepstack_num_level = shape.deepstack_levels if deepstack else 0
thinker.deepstack_input_embeds = [
torch.zeros(buffer_capacity, shape.hidden_size, device=device, dtype=dtype)
for _ in range(thinker.deepstack_num_level)
]
thinker.deepstack_input_embeds_num_tokens = 0
return thinker.eval()
@torch.inference_mode()
def expected_outputs(model, case):
"""Independent oracle: direct lookup and explicit positions, not merge helpers."""
ids = case.input_ids
if model._has_oov_mm_tokens and case.is_multimodal is not None:
ids = ids.masked_fill(case.is_multimodal.to(ids.device), 0)
main = F.embedding(ids, model.language_model.embedding.weight)
has_deepstack = any(feature.deepstack is not None for feature in case.features)
deepstack = main.new_zeros((model.deepstack_num_level, *main.shape)) if has_deepstack else None
for feature in case.features:
indexes = feature.positions.to(main.device)
main[indexes] = feature.main
if feature.deepstack is not None:
deepstack[:, indexes, :] = feature.deepstack
return main, deepstack
@torch.inference_mode()
def assert_case_output(model, case, result):
expected, deepstack = expected_outputs(model, case)
torch.testing.assert_close(result, expected, rtol=0, atol=0)
if deepstack is None:
assert model.deepstack_input_embeds_num_tokens == 0, "Unexpected stale DeepStack buffer"
else:
assert model.deepstack_input_embeds_num_tokens == case.input_ids.numel()
actual = torch.stack([buffer[: case.input_ids.numel()] for buffer in model.deepstack_input_embeds])
torch.testing.assert_close(actual, deepstack, rtol=0, atol=0)
benchmarks/tts/bench_cosyvoice3_concurrency_matrix.py
+517
View File
@@ -0,0 +1,517 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""CosyVoice3 streaming e2e benchmark: concurrency x utterance-length matrix.
Launches the server once per (concurrency, step-size) combo, streams TTS
requests cycling through several utterance lengths, and reports
TTFA/E2E/RTF/per-chunk latency (mean, stdev, range) per cell. "Step size" is
``codec_chunk_frames``, the token-hop length the first emitted chunk waits
for; later chunks wait for a doubling hop capped at ``4 * codec_chunk_frames``
(see ``_token_hop_lengths``). Per-chunk latency is reconstructed from that
same ramp rather than from raw transport read boundaries, which don't align
to model chunks (see ``one_request``'s docstring).
Running with no flags reproduces the default matrix. Per-request correctness
is checked: non-silent, long enough, and (with a fixed ``--seed``)
byte-identical PCM across requests for the same text.
"""
import argparse
import json
import os
import signal
import statistics
import subprocess
import sys
import threading
import time
from pathlib import Path
import httpx
import numpy as np
REPO = Path(__file__).resolve().parents[2]
DEFAULT_MODEL = "FunAudioLLM/Fun-CosyVoice3-0.5B-2512"
DEFAULT_REF_AUDIO = "https://raw.githubusercontent.com/FunAudioLLM/CosyVoice/main/asset/zero_shot_prompt.wav"
DEFAULT_REF_TEXT = "希望你以后能够做的比我还好呦。"
DEFAULT_TEXTS = [
"Thank you so much for the birthday gift you sent me.",
"The weather is lovely today, with warm sunshine and a gentle breeze. "
"We decided to take a walk in the park to enjoy the spring flowers and the fresh air.",
"The weather is lovely today, with warm sunshine and a gentle breeze. We decided to take a walk in the "
"park to enjoy the spring flowers and the fresh air. Many people were out walking, children were "
"running and playing on the grass, and older folks sat on benches chatting with one another. We "
"strolled slowly along the path and came across a cherry tree in full bloom, its pink petals drifting "
"on the wind. It was so beautiful that we stopped to take some photos to remember the moment. Then we "
"continued on and reached a small lake, its water so clear you could see straight to the bottom, with "
"the blue sky and white clouds reflected on its surface. A few ducks paddled back and forth across the "
"water, occasionally letting out a soft quack. We sat on a bench by the lake for a while, enjoying the "
"peaceful afternoon. Time passed quickly, and before we knew it evening had arrived, with the setting "
"sun painting the sky in shades of gold. Reluctant to leave, we finally headed home, already planning "
"our next visit. On the way back we talked about everything we had seen and heard that day, feeling "
"wonderfully happy. It was truly an unforgettable weekend.",
]
# PCM16 mono @ 24 kHz (CosyVoice3 output rate)
SAMPLE_RATE = 24000
# Well below any real speech's RMS; catches all-zero/near-silent PCM that byte
# count and duration checks alone pass through.
MIN_RMS = 50.0
# Overlay: only codec_chunk_frames is overridden; must live next to cosyvoice3.yaml (relative base_config)
DEPLOY_TEMPLATE = """# Auto-generated by bench_cosyvoice3_concurrency_matrix.py -- do not edit.
base_config: cosyvoice3.yaml
connectors:
connector_of_shared_memory:
name: SharedMemoryConnector
extra:
codec_streaming: true
connector_get_sleep_s: 0.01
connector_get_max_wait_first_chunk: 3000
connector_get_max_wait: 300
codec_chunk_frames: {chunk_frames}
codec_max_chunk_frames: {max_chunk_frames}
codec_stream_scale_factor: {stream_scale_factor}
codec_left_context_frames: 25
codec_vocab_size: 6561
"""
# CosyVoice3Config constants (transformers_utils/configs/cosyvoice3.py): tokens
# are consumed at token_mel_ratio mel frames each, and HiFT emits
# samples_per_mel PCM16 samples per mel frame. Pinned here (not left to the
# server's defaults) since one_request's byte-boundary reconstruction below
# depends on knowing them exactly.
TOKEN_MEL_RATIO = 2
SAMPLES_PER_MEL = 480
STREAM_SCALE_FACTOR = 2
def _stats(values: list[float]) -> dict[str, float]:
"""Mean/median/stdev/p95/min/max/n for a list of per-request measurements."""
n = len(values)
if n == 0:
return {
"n": 0,
"mean": float("nan"),
"median": float("nan"),
"stdev": float("nan"),
"p95": float("nan"),
"min": float("nan"),
"max": float("nan"),
}
return {
"n": n,
"mean": statistics.mean(values),
"median": statistics.median(values),
"stdev": statistics.stdev(values) if n >= 2 else 0.0,
"p95": statistics.quantiles(values, n=20)[18] if n >= 2 else values[0],
"min": min(values),
"max": max(values),
}
def vllm_bin() -> str:
for c in (REPO / ".venv/bin/vllm", Path(sys.executable).parent / "vllm"):
if c.is_file():
return str(c)
return "vllm"
def wait_ready(url: str, proc: subprocess.Popen, log: Path, timeout: float) -> None:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if proc.poll() is not None:
tail = log.read_text(errors="replace").splitlines()[-40:] if log.exists() else []
raise RuntimeError("server exited during startup:\n " + "\n ".join(tail))
try:
if httpx.get(f"{url}/health", timeout=2.0).status_code == 200:
return
except Exception:
pass
time.sleep(2.0)
raise TimeoutError(f"server not ready within {timeout}s (log: {log})")
def _token_hop_lengths(chunk_frames: int, max_chunk_frames: int):
"""The token-hop-length ramp talker2code2wav_async_chunk uses to decide
when to fire the next flow-solve/emission: chunk_frames, then doubling
(STREAM_SCALE_FACTOR) each step, capped at max_chunk_frames. This -- not
codec_chunk_frames alone -- is what actually paces emitted audio chunks:
with the defaults here, hops are chunk_frames, 2x, 4x, 4x, 4x, ...
The very first emission is consistently one token short of chunk_frames
(confirmed against real speech.audio.delta event sizes) even though the
ramp's own progression is computed from the nominal chunk_frames, not the
actual first hop -- so only the first yield gets the -1 adjustment.
"""
yield chunk_frames - 1
hop = chunk_frames
while True:
hop = min(max_chunk_frames, max(chunk_frames, hop * STREAM_SCALE_FACTOR))
yield hop
def one_request(client, url, model, text, ref_audio, ref_text, chunk_frames, seed=None):
"""Run one streaming request, returning TTFA, E2E, per-chunk gaps, and audio bytes.
``response_format=pcm`` + ``stream_format=audio`` streams raw bytes with no
framing, so a client read boundary alone is a transport artifact, not a
model chunk boundary. Reconstructed instead by replicating the server's
own token-hop-length ramp (see ``_token_hop_lengths``) to compute each
real chunk's expected byte size ahead of time, and slicing the
accumulated byte stream at those offsets -- independent of how the
transport happens to fragment the underlying reads. The final chunk
(finalize, which releases extra held-back frames) is whatever remains
when the stream closes, not a predicted hop.
"""
payload = {
"model": model,
"input": text,
"response_format": "pcm",
"stream": True,
"stream_format": "audio",
"ref_audio": ref_audio,
"ref_text": ref_text,
}
if seed is not None:
payload["seed"] = seed
t0 = time.perf_counter()
ttfa = None
nbytes = 0
audio_bytes = bytearray()
chunk_gaps_ms = [] # inter-chunk arrival gaps (per model chunk), by step index
prev_t = None
step_idx = 0
bytes_per_token = TOKEN_MEL_RATIO * SAMPLES_PER_MEL * 2
hop_gen = _token_hop_lengths(chunk_frames, 4 * chunk_frames)
next_boundary = next(hop_gen) * bytes_per_token
last_crossed_bytes = 0
received = 0
last_now = t0
with client.stream("POST", f"{url}/v1/audio/speech", json=payload, headers={"Authorization": "Bearer EMPTY"}) as r:
if r.status_code != 200:
raise RuntimeError(f"{r.status_code}: {r.read()[:300]!r}")
for chunk in r.iter_bytes():
if not chunk:
continue
now = time.perf_counter()
if ttfa is None:
ttfa = now - t0
last_now = now
received += len(chunk)
nbytes += len(chunk)
audio_bytes += chunk
while received >= next_boundary:
if prev_t is not None:
chunk_gaps_ms.append((step_idx, (now - prev_t) * 1e3))
step_idx += 1
prev_t = now
last_crossed_bytes = next_boundary
next_boundary += next(hop_gen) * bytes_per_token
if received > last_crossed_bytes:
# Finalize chunk: releases extra held-back frames, so it doesn't
# match a predicted hop size -- its arrival is the stream closing.
if prev_t is not None:
chunk_gaps_ms.append((step_idx, (last_now - prev_t) * 1e3))
step_idx += 1
e2e = time.perf_counter() - t0
audio_s = nbytes / 2 / SAMPLE_RATE
return {
"ttfa_s": ttfa,
"e2e_s": e2e,
"audio_s": audio_s,
"rtf": e2e / audio_s if audio_s > 0 else float("nan"),
"chunk_gaps_ms": chunk_gaps_ms,
"nbytes": nbytes,
"audio_bytes": audio_bytes,
}
def check_correctness(r: dict, min_audio_s: float) -> list[str]:
"""Return a list of correctness violations (empty = pass)."""
problems = []
if r["nbytes"] <= 0:
problems.append("empty audio (0 bytes)")
if r["audio_s"] < min_audio_s:
problems.append(f"audio too short ({r['audio_s']:.2f}s < {min_audio_s:.2f}s)")
if r["nbytes"] > 0:
samples = np.frombuffer(bytes(r["audio_bytes"]), dtype=np.int16).astype(np.float64)
rms = float(np.sqrt(np.mean(samples**2)))
if rms < MIN_RMS:
problems.append(f"near-silent audio (rms={rms:.1f} < {MIN_RMS})")
return problems
def drive(url, args, n, conc, label, results, lock, idx, chunk_frames):
"""Run ``n`` requests at concurrency ``conc``, cycling through ``args.texts``."""
texts = args.texts
def worker():
with httpx.Client(timeout=600.0) as c:
while True:
with lock:
i = idx[0]
idx[0] += 1
if i >= n:
return
text = texts[i % len(texts)]
try:
r = one_request(
c, url, args.model, text, args.ref_audio, args.ref_text, chunk_frames, seed=args.seed
)
r["text"] = text
with lock:
results.append(r)
except Exception as e:
print(f" [{label}] request {i} failed: {e}", flush=True)
ts = [threading.Thread(target=worker) for _ in range(conc)]
t0 = time.perf_counter()
for t in ts:
t.start()
for t in ts:
t.join()
return time.perf_counter() - t0
def run_combo(args, conc: int, chunk_frames: int, out: Path) -> dict:
tag = f"c{conc}_chunk{chunk_frames}"
run_dir = out / tag
run_dir.mkdir(parents=True, exist_ok=True)
log = run_dir / "server.log"
deploy = REPO / "vllm_omni/deploy" / f"_autogen_steps_{tag}.yaml"
deploy.write_text(
DEPLOY_TEMPLATE.format(
chunk_frames=chunk_frames,
max_chunk_frames=4 * chunk_frames,
stream_scale_factor=STREAM_SCALE_FACTOR,
)
)
cmd = [
vllm_bin(),
"serve",
args.model,
"--omni",
"--deploy-config",
str(deploy),
"--host",
"127.0.0.1",
"--port",
str(args.port),
"--allowed-local-media-path",
str(REPO),
]
env = dict(os.environ)
if args.cuda_visible_devices:
env["CUDA_VISIBLE_DEVICES"] = args.cuda_visible_devices
if args.trt is not None:
env["COSYVOICE3_TRT"] = args.trt
print(f"\n=== concurrency={conc} step_size={chunk_frames} ===", flush=True)
print(f" launching server (log: {log})", flush=True)
with open(log, "w") as lf:
proc = subprocess.Popen(cmd, env=env, stdout=lf, stderr=subprocess.STDOUT, start_new_session=True)
url = f"http://127.0.0.1:{args.port}"
try:
wait_ready(url, proc, log, args.server_timeout)
print(" server ready", flush=True)
results: list[dict] = []
lock = threading.Lock()
idx = [0]
if args.warmup:
print(f" warmup ({args.warmup} req)", flush=True)
drive(url, args, args.warmup, min(conc, args.warmup), "warmup", results, lock, idx, chunk_frames)
results.clear()
idx[0] = 0
print(f" running {args.num_prompts} req @ concurrency {conc}", flush=True)
wall = drive(url, args, args.num_prompts, conc, "bench", results, lock, idx, chunk_frames)
ok = [r for r in results if r.get("ttfa_s") is not None]
problems = []
for r in ok:
problems += check_correctness(r, args.min_audio_s)
# With a fixed seed, same-text requests must produce byte-identical PCM (per text group)
if args.seed is not None:
by_text: dict[str, bytes] = {}
for r in ok:
t = r.get("text")
ref = by_text.get(t)
if ref is None:
by_text[t] = r.get("audio_bytes")
elif r.get("audio_bytes") != ref:
problems.append(f"byte-identity violation for text {t!r}: seeded requests produced different PCM")
n_ok = len(ok)
total_audio_s = sum(r["audio_s"] for r in ok)
all_gaps = [g for r in ok for g in r["chunk_gaps_ms"]] # (step_index, gap_ms) per request
gap_vals = [g for _, g in all_gaps]
# bucket gaps by chunk position within a request, to see any O(n) slope
by_idx: dict[int, list[float]] = {}
for step_index, g in all_gaps:
by_idx.setdefault(step_index, []).append(g)
step_latency_by_idx = {i: statistics.median(vals) for i, vals in sorted(by_idx.items())}
agg = {
"concurrency": conc,
"step_size": chunk_frames,
"ok": n_ok,
"errors": args.num_prompts - n_ok,
"correctness_problems": len(problems),
"correctness_sample": problems[:5],
"total_audio_s": total_audio_s,
"throughput_audio_s_per_s": total_audio_s / wall if wall > 0 else float("nan"),
"ttfa_s": _stats([r["ttfa_s"] for r in ok]),
"e2e_s": _stats([r["e2e_s"] for r in ok]),
"rtf": _stats([r["rtf"] for r in ok]),
"step_latency_ms": _stats(gap_vals),
"step_latency_ms_by_idx": step_latency_by_idx,
"num_steps": len(all_gaps),
# per-utterance-length breakdown; n per text = num_prompts / len(texts)
"by_text": {
t: {
"audio_s": _stats([r["audio_s"] for r in ok if r.get("text") == t]),
"e2e_s": _stats([r["e2e_s"] for r in ok if r.get("text") == t]),
"rtf": _stats([r["rtf"] for r in ok if r.get("text") == t]),
}
for t in sorted({r.get("text") for r in ok})
},
}
print(json.dumps(agg, indent=2, ensure_ascii=False), flush=True)
return agg
finally:
print(" shutting down server", flush=True)
try:
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
except Exception:
proc.terminate()
try:
proc.wait(timeout=60)
except subprocess.TimeoutExpired:
try:
os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
except Exception:
proc.kill()
proc.wait(timeout=20)
deploy.unlink(missing_ok=True)
def parse_args() -> argparse.Namespace:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("--model", default=DEFAULT_MODEL)
p.add_argument(
"--text",
dest="texts",
nargs="+",
default=DEFAULT_TEXTS,
help="One or more utterance texts. With several, requests cycle through "
"them so a single server launch covers all utterance lengths; results "
"are bucketed per text.",
)
p.add_argument("--ref-audio", default=DEFAULT_REF_AUDIO)
p.add_argument("--ref-text", default=DEFAULT_REF_TEXT)
p.add_argument("--concurrency", type=int, nargs="+", default=[1, 8])
p.add_argument(
"--step-size",
type=int,
nargs="+",
default=[15],
help="codec_chunk_frames values to sweep (the streaming step size).",
)
p.add_argument("--num-prompts", type=int, default=30)
p.add_argument("--warmup", type=int, default=2)
p.add_argument("--port", type=int, default=8091)
p.add_argument(
"--server-timeout",
type=float,
default=1800.0,
help="Startup budget; first run also downloads weights and builds the TRT engine.",
)
p.add_argument("--output-dir", default="./perf_steps")
p.add_argument("--cuda-visible-devices", default="0")
p.add_argument(
"--trt",
choices=["0", "1"],
default=None,
help="Pin COSYVOICE3_TRT. 0=force torch estimator, 1=allow TensorRT. Default: unset.",
)
p.add_argument(
"--min-audio-s",
type=float,
default=1.0,
help="Minimum expected audio seconds per request for the correctness check.",
)
p.add_argument(
"--seed",
type=int,
default=42,
help="Fixed seed for reproducible (byte-identical) audio. Pass --seed '' "
"equivalent (any non-int) to disable -- there is no unset sentinel here "
"by design, since the default sweep is meant to be exactly reproducible.",
)
return p.parse_args()
def main() -> None:
args = parse_args()
out = Path(args.output_dir).resolve()
out.mkdir(parents=True, exist_ok=True)
all_results = []
failed = False
for conc in args.concurrency:
for chunk_frames in args.step_size:
try:
result = run_combo(args, conc, chunk_frames, out)
all_results.append(result)
if result["errors"] > 0 or result["correctness_problems"] > 0:
failed = True
except Exception as e:
print(f" combo c={conc} chunk={chunk_frames} FAILED: {e}", flush=True)
failed = True
out_path = out / "steps_results.json"
with open(out_path, "w", encoding="utf-8") as f:
json.dump({"model": args.model, "texts": args.texts, "runs": all_results}, f, indent=2, ensure_ascii=False)
print(f"\nResults written to {out_path}", flush=True)
print("\n=== SUMMARY ===")
header = (
f"{'conc':>4} {'step':>4} {'ok':>4} {'corr':>4} "
f"{'TTFA med(ms)':>12} {'step med(ms)':>12} {'step p95(ms)':>12} {'RTF mean+-sd':>14}"
)
print(header)
print("-" * len(header))
for r in all_results:
print(
f"{r['concurrency']:>4} {r['step_size']:>4} {r['ok']:>4} {r['correctness_problems']:>4} "
f"{r['ttfa_s']['median'] * 1e3:>12.1f} "
f"{r['step_latency_ms']['median']:>12.1f} {r['step_latency_ms']['p95']:>12.1f} "
f"{r['rtf']['mean']:>6.3f}+-{r['rtf']['stdev']:>5.3f}"
)
for t, stats in r.get("by_text", {}).items():
print(
f" text {t[:24]!r}: e2e {stats['e2e_s']['mean'] * 1e3:8.1f}"
f"+-{stats['e2e_s']['stdev'] * 1e3:6.1f} ms (n={stats['e2e_s']['n']}, "
f"audio {stats['audio_s']['mean']:.1f}s)"
)
if failed:
sys.exit(1)
if __name__ == "__main__":
main()
docker/Dockerfile.rocm
+30 -17
View File
@@ -2,7 +2,11 @@
# docker/Dockerfile.ci. A stale base image can build successfully while failing
# at runtime when vLLM-Omni imports APIs added by the newer vLLM release.
ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.29.0
FROM ${BASE_IMAGE} AS base
# Keep the dependency-heavy portion independent from the source tree. The
# registry cache is keyed on these inputs, while the final source copy can be
# linked onto the cached image without materializing its large ROCm layers.
FROM ${BASE_IMAGE} AS deps
# NOTE: REMINDER to vLLM-Omni rebase maintainer
# BASE_IMAGE pins the default prebuilt runtime when USE_NIGHTLY_BUILD=0.
@@ -64,9 +68,24 @@ RUN --mount=type=cache,target=/root/.cache/pip \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
# Step 3: Copy vllm-omni code and install without uv
RUN mkdir -p ${COMMON_WORKDIR}/vllm-omni
COPY . ${COMMON_WORKDIR}/vllm-omni
# Step 3: Install project dependencies against a placeholder package tree.
# The editable install keeps resolving imports from this directory after the
# final COPY --link replaces the placeholder with the real source tree.
WORKDIR ${COMMON_WORKDIR}/vllm-omni
RUN mkdir -p vllm_omni && touch vllm_omni/__init__.py
COPY pyproject.toml setup.py ./
COPY requirements/ requirements/
ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0
RUN uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" \
--no-cache-dir -e ".[dev]" --no-build-isolation && \
python3 -m pip uninstall -y onnxruntime && \
python3 -m pip install --no-cache-dir --force-reinstall --no-deps \
"onnxruntime-rocm>=1.22.2" && \
python3 -c "import importlib.metadata as m; import onnxruntime as ort; assert 'ROCMExecutionProvider' in ort.get_available_providers(), ort.get_available_providers(); assert m.version('onnxruntime-rocm'); assert not any(d.metadata['Name'] == 'onnxruntime' for d in m.distributions())" && \
python3 -c "from torchcodec.decoders import VideoDecoder"
RUN ln -sf /usr/bin/python3 /usr/bin/python
# This is a workaround to ensure pytest exits with the correct status code in CI tests.
RUN printf '%s\n' \
@@ -85,19 +104,13 @@ RUN printf '%s\n' \
' os._exit(_exit_code)' \
> ${COMMON_WORKDIR}/vllm-omni/conftest.py
# Some optional dependencies pull in the CPU ONNX Runtime distribution. The
# CPU and ROCm wheels own the same import path and cannot coexist, so remove
# the CPU wheel and reinstall the ROCm wheel after dependency resolution.
RUN cd ${COMMON_WORKDIR}/vllm-omni && \
uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" \
--no-cache-dir ".[dev]" --no-build-isolation && \
python3 -m pip uninstall -y onnxruntime && \
python3 -m pip install --no-cache-dir --force-reinstall --no-deps \
"onnxruntime-rocm>=1.22.2" && \
python3 -c "import importlib.metadata as m; import onnxruntime as ort; assert 'ROCMExecutionProvider' in ort.get_available_providers(), ort.get_available_providers(); assert m.version('onnxruntime-rocm'); assert not any(d.metadata['Name'] == 'onnxruntime' for d in m.distributions())" && \
python3 -c "from torchcodec.decoders import VideoDecoder"
RUN ln -sf /usr/bin/python3 /usr/bin/python
# Keep source changes in one lightweight linked layer. Do not add RUN, COPY, or
# ADD instructions that depend on the resulting filesystem after this point;
# doing so would force BuildKit to download and extract the cached ROCm layers.
FROM deps AS base
ARG COMMON_WORKDIR=/app
WORKDIR ${COMMON_WORKDIR}/vllm-omni
COPY --link . .
FROM base AS test
docker/Dockerfile.rocm.dockerignore
+35
View File
@@ -0,0 +1,35 @@
# Keep the linked source layer small and deterministic.
.git
.venv
/build
dist
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
.mypy_cache
# Distribution / packaging
.Python
/build/
cmake-build-*/
CMakeUserPresets.json
develop-eggs/
/dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
rust/target/
docker/Dockerfile.xpu
-1
View File
@@ -40,7 +40,6 @@ ENV VLLM_OMNI_TARGET_DEVICE=xpu
# Keep the upstream oneCCL packages paired with the XPU torch runtime.
RUN --mount=type=cache,target=/root/.cache/uv \
uv pip install --no-build-isolation \
--extra-index-url https://mirrors.ustc.edu.cn/pypi/simple/ \
".[dev]" && \
uv pip uninstall triton && \
uv pip install triton-xpu==3.7.2 --reinstall
docs/.nav.yml
+1
View File
@@ -161,6 +161,7 @@ nav:
- Cache-DiT: design/feature/cache_dit.md
- TeaCache: design/feature/teacache.md
- Diffusion Continuous Batching: design/feature/diffusion_continuous_batching.md
- Mooncake KV Transfer: design/feature/mooncake-kv-transfer.md
- Infrastructure and Performance:
- Prometheus Metrics: design/metrics.md
- Speech Generation Performance Optimizations: design/qwen3_omni_tts_performance_optimization.md
docs/README.md
+1 -1
View File
@@ -56,7 +56,7 @@ vLLM-Omni is flexible and easy to use with:
vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
- Omni-modality models (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
- TTS models (e.g. Qwen3-TTS, IndexTTS 2.5, dots.tts, CosyVoice3)
- TTS models (e.g. Qwen3-TTS, IndexTTS 2.5, CosyVoice3)
- Diffusion models — image, video, and audio generation (e.g. MiniMax H3, LTX-2.5, SANA-Video, Wan2.2)
- Robot-policy and action models (e.g. π0, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)
docs/cli/bench/serve.md
+54 -1
View File
@@ -23,7 +23,9 @@ You can use `vllm bench serve --omni --help=all` to get descriptions of all para
The API endpoint exposed externally, to which clients send their requests.
- `--dataset-name`
The name of the dataset used; random-mm indicates generating random multimodal inputs (images, videos, audio), while `omniinteract` replays official OmniInteract videos.
The name of the dataset used; `random-mm` generates random multimodal inputs (images, videos, audio),
`omniinteract` replays official OmniInteract videos, and `videomme` loads Video-MME MCQ videos
(`lmms-eval/Video-MME`, MiniCPM OmniEvalKit packing).
- `--num-prompts`
The total number of requests to send, an integer.
@@ -343,6 +345,57 @@ transcript, event, and result artifacts without errors. A valid LISTEN-only case
chunks. Official-manifest eligibility is reported separately because clipped or cancelled output is a benchmark-quality
signal, not a transport failure. This local performance test does not score answer accuracy.
### Video-MME Benchmark
Video-MME (`--dataset-name videomme`) scores multiple-choice video QA. Default packing is
OmniEvalKit MiniCPM `minicpm-frames` (up to 96 sampled frames as `image_url`). Pass a local
mirror with `--dataset-path` / `--videomme-parquet` + `--videomme-video-dir`, or a Hugging
Face dataset id (`lmms-eval/Video-MME` by default; any `org/name` is accepted when
`--dataset-name videomme` is explicit). Relative `--videomme-video-dir` values are resolved
to absolute `file://` URLs. For those URLs, start the server with
`--allowed-local-media-path` covering the video root; otherwise use `--videomme-inline-local-video`.
For MiniCPM-o 4.5, the default deployment allows 64 images per request. To run the
96-frame recipe, start the server with a stage 0 override (replace `/path/to/Video-MME`
with the local video root shared by the benchmark client and server):
```bash
vllm serve openbmb/MiniCPM-o-4_5 --omni \
--allowed-local-media-path /path/to/Video-MME \
--stage-overrides '{"0":{"limit_mm_per_prompt":{"image":96,"audio":64,"video":1}}}'
```
The override applies to this server invocation only. Alternatively, use
`--videomme-max-frames 64` in the benchmark to stay within the default image limit;
this evaluates a different frame-sampling setting from the 96-frame recipe.
Run the benchmark against that server:
```bash
vllm bench serve --omni \
--backend openai-chat-omni \
--dataset-name videomme \
--dataset-path /path/to/Video-MME \
--videomme-pack-mode minicpm-frames \
--videomme-max-frames 96 \
--model openbmb/MiniCPM-o-4_5 \
--endpoint /v1/chat/completions \
--num-prompts 8 \
--max-concurrency 1 \
--save-result
```
Accuracy keys (`videomme_accuracy`, per-duration / domain / task breakdowns) are written into
the saved JSON. `videomme_accuracy` excludes HTTP failures; `videomme_accuracy_incl_http_fail`
counts them as wrong. `videomme_submitted` / `videomme_unique_question_ids` show when
oversampling made the request count differ from unique-question coverage.
`videomme_skipped_rows` counts rows skipped during sampling because of missing or unreadable
media or invalid question fields. The opt-in accuracy runner rejects skipped rows, HTTP
failures, duplicate question IDs, and missing gold answers. This checks the sampled subset;
it does not establish that a supplied local mirror contains the entire official dataset.
Use `--videomme-save-eval-items` (or `VIDEOMME_SAVE_EVAL_ITEMS=1`) for per-request
`videomme_eval_items` rows.
### Multi-Modal Benchmark
<details class="admonition abstract" markdown="1">
docs/cli/serve.md
+69
View File
@@ -1,5 +1,67 @@
# vllm-omni serve
## Multiple API frontends
For local EngineCore pipelines with one or more stages, `--api-server-count` starts
multiple API frontend processes that share one parent-owned set of stage
engines:
```bash
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
--api-server-count 2 \
--port 8091
```
This mode is intended for workloads whose request parsing, multimodal input
processing, or response encoding can bottleneck a single frontend. It
currently requires local multi-process EngineCore stages with one local
process group per replica. A stage may still use multiple `num_replicas`.
It cannot be combined with diffusion, headless or
remote stages, intra-stage data parallelism, Ray, fault tolerance, elastic
expert parallelism, sleep mode, or runtime LoRA updating. Runtime voice upload
and deletion are also disabled because those mutations are process-local;
built-in voices, inline reference audio, and voices restored at startup remain
available. The `/v1/omni/sleep` and `/v1/omni/wakeup` control routes return HTTP
409 for the same reason: their bookkeeping is process-local while stage engines
are shared.
A single-stage model pipeline is supported through the normal launch command;
this is distinct from the distributed `--stage-id` / `--headless` launch mode.
Every stage in the pipeline must be an EngineCore stage. Mixed AR-to-DiT
pipelines, including MiniMax-H3, remain unsupported even when their first
stage is an EngineCore stage. Diffusion support requires per-frontend channels
and shared-engine lifecycle handling in the diffusion backend.
### Workloads and performance validation
Each API process owns its frontend processor, Orchestrator, request state,
and per-replica EngineCore channels. Requests return through the channel of
the originating API process; model weights and GPU stage engines are shared.
This distributes frontend and orchestration work across CPU processes.
| Workload | Potential benefit | Limitation |
| ---------- | ------------------- | ------------ |
| Concurrent cold image/video inputs on supported EngineCore pipelines | Parallel frontend processing when one API process is CPU-limited | A single cold request and GPU computation may be unchanged |
| Many replicas producing frequent text/audio outputs | Less API/Orchestrator work and GIL contention per process | Routing remains serial within each Orchestrator |
| Warm inputs or low-concurrency, GPU-limited generation | Additional frontend capacity | More API processes may add overhead without improving throughput |
Here, a cold input means media or preprocessing caches miss, not that model
weights must be loaded or GPU kernels compiled. Each frontend has its own
processor/cache state, so cache duplication, CPU thread contention, and memory
usage also matter. Multi-API serving does not itself change the output queue,
move response encoding to an executor, or change the model's numerical scheduler.
Compare `--api-server-count 1`, `2`, and higher counts with the same hardware,
model/stage configuration, media, sampling parameters, output lengths, and
offered load. Keep cold-cache and warmed-cache runs separate and report repeated
measurements. Record CPU and request counts for every API worker, input and
output queue delay, GPU utilization, first-output latency, and end-to-end latency.
Confirm requests reach every worker: connection reuse can concentrate traffic
on one process. Queue-only or single-API experiments do not establish a
multi-API throughput gain. The multi-replica workload in
[#4680](https://github.com/vllm-project/vllm-omni/issues/4680) remains a relevant
validation target.
## Stage-based CLI quickstart
The stage-based CLI is designed for deployments that require launching each pipeline stage in an isolated process
@@ -61,3 +123,10 @@ CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
## Arguments
--8<-- "docs/generated/argparse_omni/omni_serve.inc.md"
Async video jobs use process-local metadata and task handles. With multiple API
workers, creating, listing, retrieving, downloading, and deleting these jobs
returns HTTP 409. This restriction is independent of the current diffusion
stage launch restriction: supporting multi-worker diffusion also requires a
shared job lifecycle. Missing frontend topology state returns HTTP 503 for
operations that require process-local state protection.
docs/configuration/environment_variables.md
+1
View File
@@ -70,6 +70,7 @@ The lifecycle labels used below are:
| `OMNI_DIFFUSION_PROMPT_EMBED_CACHE_SIZE` | Positive integer; default `32` entries | Each diffusion runner; resolved during model setup | A valid environment value overrides the explicit cache size. A non-integer logs a warning; a non-positive value is ignored. | Experimental |
| `OMNI_DIFFUSION_SESSION_STATE_MANAGER` | Boolean with the same accepted spellings as the prompt cache; default disabled | Experimental diffusion session manager; model setup | A recognized environment value overrides the explicit enable setting. An unrecognized value is ignored. | Experimental |
| `OMNI_DIFFUSION_SESSION_STATE_MANAGER_MAX_SESSIONS` | Positive integer; default `64` | Experimental diffusion session manager; model setup | A valid environment value overrides the explicit maximum. A non-integer logs a warning; a non-positive value is ignored. | Experimental |
| `VLLM_OMNI_AR_DIFFUSION_KV_GATHER` | `1` enables; default `0` (off) | AR-diffusion KV manager at allocation and attention at dispatch; set before worker startup | Only the exact value `1` enables contiguous KV gathering. History staging additionally requires `ARDiffusionKVConfig.reuse_history_staging`. | Experimental |
Backend names for `DIFFUSION_ATTENTION_BACKEND` are the members of
`DiffusionAttentionBackendEnum`, such as `FLASH_ATTN`, `TORCH_SDPA`,
docs/configuration/stage_configs.md
+88 -1
View File
@@ -53,7 +53,7 @@ The new deploy schema lives under `vllm_omni/deploy/` and is paired with a froze
| ------- | ------ | ---------- | --------- | ------------- |
| `base_config` | str (path) | optional | — | Overlay parent (relative or absolute). `stages:` / `platforms:` deep-merged by stage_id; other scalars overlay-wins. Intended for user-authored overlays; prod yamls stay flat. |
| `async_chunk` | bool | optional | `true` | Enable chunked streaming between stages. Pin to `false` if the pipeline runs end-to-end. |
| `session_mode` | str | optional | `"turn"` | Session behavior. Duplex models (pipelines with a `duplex_plugin`, e.g. MiniCPM-o 4.5) set `"duplex"`; `vllm-omni serve` then serves it over `/v1/realtime?duplex=1` and `/v1/chat/completions`, both on duplex sessions. |
| `session_mode` | str | optional | `"turn"` | Session behavior. For pipelines with a `duplex_plugin`, explicitly select `"duplex"` for the duplex engine or `"turn"` for the ordinary online serving stack. The shipped MiniCPM-o 4.5 default remains `"duplex"`. See [Full Duplex](../serving/full_duplex_api.md#enable-full-duplex). |
| `active_stream_window` | int | optional | `0` | Number of active downstream stream slots; `0` preserves all-stream cycling. |
| `duplex_session` | dict | optional | runtime defaults | Full-duplex session lifecycle, buffering, replay, and capacity limits. |
| `connectors` | dict | optional | `null` | Named connector specs (`{name, extra}`). Referenced by each stage's `input_connectors` / `output_connectors`. See [Connector schema](#connector-schema). |
@@ -290,3 +290,90 @@ vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --deploy-config /path/to/depl
!!! important
We are actively iterating on the definition of deployment configurations, and we welcome feedback from users and developers.
## Qwen3-TTS with Model Runner V2
Qwen3-TTS can opt into the native CUDA Model Runner V2 pipeline on vLLM
0.29.0. Select one of these deployment profiles:
| Profile | Runner | Code2Wav graph batches | Intended use |
| --- | --- | --- | --- |
| `qwen3_tts.yaml` | V1 | Existing defaults | Existing deployment / regression control |
| `qwen3_tts_mrv2.yaml` | V2 | B1 | Native runner validation |
| `qwen3_tts_high_concurrency_mrv2.yaml` | V2 | B1, B2 | Opt-in throughput tuning |
| `qwen3_tts_high_concurrency_mrv2_b4.yaml` | V2 | B1, B2, B3, B4 | Experimental throughput / buffered playback |
```bash
vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base --omni \
--deploy-config vllm_omni/deploy/qwen3_tts_mrv2.yaml
```
The V2 profiles bound Talker prefill to 512 tokens per step and select the
Talker AR runner and the Code2Wav generation
runner together. Native inter-stage delivery carries codec payloads directly;
request-owned snapshots preserve buffers through asynchronous completion and
CUDA graph reuse. Terminal completion waits for upstream stage metrics before
releasing request state. Platform sections retain V1 on NPU, XPU, ROCm and MUSA;
this change does not qualify MRV2 on those backends or enable other model families.
Selecting V2 for a stage without `supports_native_mrv2_data_plane` emits a warning
when pipeline and deployment settings are merged. That stage retains the legacy
transport path; the warning does not establish support for that combination.
MRV2 model hooks use capability declarations rather than architecture or stage
names. Omni lifecycle flags select the model state, and `_returns_tuple` declares
the capture output contract. The optional batched predictor hook is `mtp`, with
an explicit `mtp_output_key` (a string or two-part payload key), optional
`mtp_validity_key`, and `mtp_graph_safe`/`mtp_disable_graph` capture controls.
Its inputs are token IDs, embeddings, previous hidden states and per-row
conditioning; it returns updated embeddings and prediction codes. Models may
supply `mtp_sampling_params` and `get_mtp_seed(sampling_params)` for model-local explicit seeds, and declare
`mtp_accepts_per_row_generators`, `mtp_accepts_req_infos`, or `mtp_sample_uniforms`
(with `mtp_sample_steps` and `mtp_sample_vocab_size`) as needed. Qwen3-TTS retains
its existing `talker_mtp` entry point for V1.
### Included performance work
- Request snapshots have a fast path for immutable scalar leaves, including
waveform lists. Mutable containers, aliases and cycles keep deep-copy
semantics. This is not the historical shallow `dict(prompt)` experiment.
- The API reference cache holds owned float32 arrays with a default capacity
of 1024 entries and a 512 MiB waveform-payload budget. Cache hits return
independent lists. Qwen3-TTS model artifact caches also default to 1024
entries. These caches have different owners and eviction policies; equal
entry limits do not make them a single coherent cache.
- Talker state stays on GPU where its lifecycle allows it. BOS/EOS projections
are cached in their projection dtype and invalidated on weight loading.
- Code2Wav packs CPU codec inputs before device transfer. Optional B2/B4 graph
buckets batch compatible requests without sharing their per-request state.
- Native output materialization runs in a bounded worker and drains before
closing the data plane. The existing control-only shortcut avoids launching
model work for a step that has only lifecycle events.
MTP prefix re-prefill remains disabled in the high-concurrency V2 profile.
Adding more graph shapes increases compilation cost and has not established a
stable end-to-end gain for this extracted version.
### Decoder batches and playback
Changing `decode_cudagraph_batch_sizes` selects the captured batch buckets.
Keep `decode_batch_max_size` consistent with the intended maximum too; the
stateful graph path currently groups according to the captured buckets, while
the stateless path also uses the explicit maximum.
B4 remains experimental. First-packet latency alone does not establish
uninterrupted playback. Validate inter-chunk arrival times, buffering, WER and
speaker similarity before adopting either batching preset for a production
workload. Floating-point decoder outputs can differ across batch sizes; this PR
does not claim bitwise or quality equivalence.
### Optional MPS deployment
NVIDIA MPS is an optional operator setting for colocated CUDA processes, not a
YAML option or a library default. This PR does not establish a throughput or
first-packet latency benefit from MPS. Measure the exact deployment with and
without MPS before enabling it.
Use only assigned GPUs and an independent MPS pipe directory. A private MPS
server does not provide exclusive GPU ownership or MIG isolation. For a
single-GPU deployment, explicitly place both stages on that GPU; the supplied
high-concurrency profile places its two stages on different GPUs by default.
docs/design/architecture_overview.md
+6 -6
View File
@@ -323,7 +323,7 @@ inputs, fields, and ownership rules are described below.
flowchart TB
layer1["Layer 1 · Authoring inputs<br/>PipelineConfig + DeployConfig"]
layer2["Layer 2 · Production resolution boundary<br/>resolve_omni_config()<br/>(StageConfigFactory + VllmOmniConfig internally)"]
layer3["Layer 3 · Startup hand-off<br/>OmniConfigResolution<br/>PipelineConfig + temporary OmegaConf stage bridge"]
layer3["Layer 3 · Startup hand-off<br/>OmniConfigResolution<br/>PipelineConfig + typed stage configs"]
layer4["Layer 4 · Runtime launch planning<br/>StageRuntime"]
layer5["Layer 5 · Engine materialization<br/>VllmConfig / OmniDiffusionConfig"]
@@ -335,10 +335,10 @@ The production resolution boundary is `resolve_omni_config()` in
It delegates typed construction to `StageConfigFactory.create_from_model()` and
`VllmOmniConfig.from_pipeline_config()`, then returns an
`OmniConfigResolution` consumed by both `AsyncOmniEngine` and headless startup.
Until `StageRuntime` consumes typed stage configs directly, this envelope carries
the effective `PipelineConfig` alongside OmegaConf-compatible `stage_configs` as
a temporary runtime bridge; both views describe the same resolved topology,
including injected stages.
This envelope carries the effective `PipelineConfig` alongside typed
`stage_configs`, which `StageRuntime` consumes directly. Both views describe the
same resolved topology, including injected stages. Backend arguments are
projected from these typed stages when each engine is initialized.
The legacy `stage_args` YAML path has been removed. Model topology now resolves
through `PipelineConfig`, with runtime overrides supplied by `DeployConfig`.
@@ -360,7 +360,7 @@ The important ownership rules are:
3. CLI and Python overrides are applied at the resolution boundary, with
per-stage overrides taking precedence over global values where supported.
4. `OmniConfigResolution` is the sole startup hand-off. Its `pipeline_config`
and temporary `stage_configs` compatibility view must describe the same
and typed `stage_configs` must describe the same
topology.
5. `StageRuntime` owns launch planning and replica lifecycle. `ReplicaInitPlan`
is runtime-private state, not a user configuration object.
docs/design/feature/mooncake-kv-transfer.md
+89
View File
@@ -0,0 +1,89 @@
# Mooncake KV Transfer
## PR summary
This change adds an opt-in AR-to-DiT KV handoff using vLLM's native
`KVTransferConfig`, Scheduler-owned diffusion KV pages, and
`MooncakeConnector`. The existing `OmniKVTransferManager` path remains the
fallback when `kv_transfer_config` is absent.
The orchestrator creates one transfer ticket per request and resolves the
bound AR replica's Mooncake endpoint after generation. DiT allocates the
destination pages, waits for every worker rank to finish loading, and releases
the source and destination pages only after the connector lifecycle completes.
Request registration failures are rolled back before Worker dispatch. Receive
timeouts and cancellation terminate the request without recycling in-flight
pages: the scheduler retains them until every CFG row has completed on every
rank. Late completion is polled only while such requests remain. If a peer
never completes, its pages remain reserved until shutdown; other requests can
continue using the remaining capacity. Invalid pages, missing Worker ranks,
and failures without a proven rollback still stop the replica before teardown.
The example sets `transfer_timeout: 60.0` seconds explicitly. Cancellation
queued during a synchronous receive is processed when that receive completes
or times out; it does not interrupt the remote write.
## Reference configuration
| Stage | Replicas | Devices | Parallelism | KV cache |
| --- | ---: | --- | --- | ---: |
| AR | 2 | 0-3 | TP=2 per replica | 2 GiB per worker |
| DiT | 2 | 4-7 | TP=2 per replica, EP enabled | 1 GiB per worker |
The acceptance workload uses 512x512 images, two denoising steps, guidance
scale 2.0, and `max_num_seqs=1`.
For a four-GPU debug run, set both stages to one replica and assign AR TP=2 to
devices `0,1` and DiT TP=2 to devices `2,3`.
## Experiment
The final comparison used one node with eight NVIDIA RTX PRO 6000 Blackwell
Server Edition GPUs, TCP Mooncake transport, the same 50-prompt/seed T2I
dataset, concurrency 8, and two independent 100-request runs per
implementation. Each side completed 200/200 requests without an OOM or request
failure.
| Implementation | Run 1 (req/s) | Run 2 (req/s) | Mean (req/s) |
| --- | ---: | ---: | ---: |
| `OmniKVTransferManager` | 4.4103 | 4.4497 | 4.4300 |
| Native `MooncakeConnector` | 4.4457 | 4.4699 | 4.4578 |
Native throughput is **0.63% higher** than the legacy mean. The absolute
performance difference is below the 1% acceptance threshold.
For accuracy, a non-zero transferred prefix and the resulting attention output
were compared bit-for-bit with the legacy manager path:
```text
prefix_bits_equal True
attention_bits_equal True
max_abs_diff 0.0
```
The one-AR/one-DiT TP=2 debug topology also completed 20/20 sequential
requests. A separate real-TCP transport check transferred both CFG rows from a
TP=1 producer to a strict-Ulysses SP=2 consumer with exact values on both
target ranks.
## Scope and limitations
- The fully validated end-to-end topology is 2x AR TP=2 plus 2x DiT TP=2.
- Heterogeneous TP=1 to SP=2 reshaping is transport-tested, but arbitrary mixed
TP/CFG/SP end-to-end combinations have not been exhaustively validated.
- CFG-parallel end-to-end deployment is not part of this acceptance run.
- Native AR-to-DiT transfer currently requires `async_chunk: false`.
- TCP is covered by the acceptance run. The supplementary single-node RDMA
measurements use a separate short workload; they do not establish RDMA
support for multi-node deployment, which remains out of scope.
- The native path currently validates the two-stage `AR(0) -> DiT(1)` graph. Other stage graphs are rejected during startup until
their producer/consumer edge mapping is implemented.
- Sleep mode is disabled for the native paged path because registered KV pages
must remain mapped for the connector lifetime.
- Ascend NPU adaptation and its performance validation are maintained in a
separate follow-up PR; the GPU acceptance results above do not validate NPU.
- The homogeneous piecewise CUDA batching optimization is retained. A native
paged-attention CUDA graph capture/replay regression covers two CFG rows,
strided QKV, mixed causal/image spans, changed inputs, and output stability.
It must be run on CUDA hardware; CPU unit tests do not establish graph or
end-to-end performance acceptance.
docs/design/feature/offloader/README.md
+16 -4
View File
@@ -84,10 +84,22 @@ Block topology comes from the pipeline `OffloadPlan` first: `block_attrs` maps
each DiT path to its ordered block containers, `encoder_block_attrs` declares
streamable encoder stacks, `on_demand_component_paths` marks pipeline-managed
residency, `resident_dit_paths` marks the DiTs that may hold resident layers,
and `encoder_dlo_weight_replication` marks the encoders whose loader-produced
weights are safe for AllGather. DiTs absent from the plan fall back to
`_layerwise_offload_blocks_attrs` (including the deprecated singular-name
compatibility path).
`encoder_dlo_weight_replication` marks the encoders whose loader-produced
weights are safe for AllGather, and `offload_submodules` maps a DiT submodule
that owns its residency to its own block container. DiTs absent from the plan
fall back to `_layerwise_offload_blocks_attrs` (including the deprecated
singular-name compatibility path).
An undeclared DiT submodule joins the plan only when it is large enough that
keeping it resident would defeat streaming; its block container is then found
by scanning well-known attribute names, which warns once per submodule class.
Size is computed from parameter shapes and dtypes, including meta parameters,
so resolving before mmap loading preserves the same residency decision.
Block ownership validation includes nested components: a block shared with a
parent, sibling, or encoder is rejected before any hooks or storage are changed.
A submodule with no block container must implement the `load_to_device` /
`offload_to_cpu` lifecycle, and the resolver rejects it before any component of
the pipeline is placed or hooked.
The resolver is pure: it moves no tensor, installs no hook, writes no module
attribute, and reads no process group — multi-rank facts come from
docs/design/feature/omni_async_output_materialization.md
+13 -12
View File
@@ -318,18 +318,18 @@ The same behavior applies to the Base and VoiceDesign Qwen3-TTS checkpoints
because they use the same Talker implementation.
!!! warning
Do not pass `--no-async-chunk` or enable prefix caching when you want this
optimization. Either change causes the runner to fall back to synchronous
output construction. Enabling async chunk alone cannot activate the feature
for a model that has not opted into async Omni output.
Do not pass `--no-async-chunk` when you want this optimization. Disabling
async chunk causes the runner to fall back to synchronous output
construction. Enabling async chunk alone cannot activate the feature for a
model that has not opted into async Omni output.
!!! note "Prefix cache compatibility"
Async Omni output materialization and Omni prefix caching cannot currently
run together. `_should_use_async_omni_output()` returns `False` whenever
`self.omni_prefix_cache` is present, so prefix caching selects synchronous
output materialization. Supporting both features requires snapshotting or
otherwise synchronizing prefix-cache merge and update state before the
background output path can consume it safely.
Prefix cache and async Omni output can run together. `save_outputs` returns
a `step_id`; the output builder later calls `materialize(sid, req_ids)` with
the save-time request list, which may be a step late relative to the live
`input_batch`. `_should_use_async_omni_output()` does not disable itself
when the cache is present. See
[Automatic Prefix Caching](prefix_caching.md#implementation).
## Compatibility and Fallbacks
@@ -341,10 +341,9 @@ runtime conditions hold:
| AR async scheduling is enabled | The optimization relies on the scheduler advancing while the prior output is materialized |
| `async_chunk` is enabled | The feature targets incremental downstream Omni payloads |
| The model stage opts in with `use_async_omni_output` | Models must declare that their output lifecycle is safe to defer |
| Omni prefix cache is disabled | Prefix-cache merge and update ordering currently requires synchronous materialization |
| Speculative decoding is disabled | Speculative output state is not included in this deferred path |
| Routed-expert output is disabled | Routed-expert extraction currently requires the synchronous path |
| Postprocess is absent or explicitly runs eagerly | State needed by the next decode step must be updated before the runner returns |
| Postprocess is absent or explicitly runs eagerly | State needed by the next decode step must be updated before the runner returns. Eager postprocess sees the scheduled slice (`None, None` combined tensors), not the prefix-cache-merged full prompt; it must only consume the tail |
These checks are evaluated per stage on runners based on `GPUARModelRunner`.
An unsupported combination does not prevent serving; it only falls back to
@@ -380,6 +379,8 @@ so Omni payload materialization remains synchronous.
- `tests/worker/test_gpu_ar_model_runner.py`: Snapshot, guard, connector
ordering, and background error-propagation tests.
- [Async Chunk](async_chunk.md): Inter-stage chunking and scheduling design.
- [Automatic Prefix Caching](prefix_caching.md#implementation): Prefix-cache
`step_id` / `materialize` contract used by the deferred output builder.
- [Qwen3-Omni optimization blog](https://vllm.ai/blog/2026-07-01-qwen3-omni-optimization):
Optimization context and controlled performance results.
- [PR #4476](https://github.com/vllm-project/vllm-omni/pull/4476): Feature
docs/design/feature/prefix_caching.md
+178 -15
View File
@@ -1,6 +1,5 @@
# Automatic Prefix Caching in Omni Models
---
## Table of Contents
@@ -9,6 +8,8 @@
- [High-Level Approach](#high-level-approach)
- [Example](#example)
- [What About Multimodal Inputs?](#what-about-multimodal-inputs)
- [Implementation](#implementation)
- [Related Files](#related-files)
---
@@ -27,6 +28,7 @@ vLLM implements automatic prefix caching for managing its kv-cache, which is bes
This document describes vLLM-Omni's mechanism for caching tensor outputs that are meant to be passed between stages, when requests have common prefixes, similar to the way in which vLLM has prefix caching for the kv-cache. This works in conjunction with vLLM's multimodal encoder caching, but is distinct. See the final section for a concrete example for how they tie together in practice.
### High-Level Approach
!!! note "Note 2"
Prior to reading this section, it's recommended to take a look at the design documents in vLLM for [Automatic Prefix Caching](https://docs.vllm.ai/en/latest/features/automatic_prefix_caching/), which will make some of the concepts more clear.
@@ -38,8 +40,10 @@ The main focus of vLLM-Omni's approach to prefix caching stage outputs is to bui
With this in mind, consider the set of blocks in a 2D layout, where the row represents the index of blocks being considered, and the columns represent the slots corresponding to tokens within each block. Since we know the `num_blocks` and `block_size` from our kv cache config, if we want to cache a tensor with feature size `D`, we can preallocate a CPU tensor of size `(num_blocks, block_size, D)`, and use the same block index and slot mapping to retrieve the corresponding feature vector.
Host footprint: each cached key costs `num_blocks × block_size × D × dtype_bytes` of **pinned** CPU memory (pinned so device→host can overlap compute), allocated on the first `save_outputs` that sees the key — the first real request pays the `cudaHostAlloc`. Measured on a Qwen3-Omni deployment: thinker `__hidden_states__` `[15092, 16, 2048]` bf16 ≈ 0.92 GiB (plus the same again for each `hidden_states.layer_*` key a model exposes), Qwen3-TTS talker `codes.audio` `[16180, 16, 16]` ≈ 33 MiB. Budget host RAM for the stage accordingly.
### Example
!!! note "Note 3"
Prefix caching in vLLM-Omni currently is only supported on AutoRegressive stages with one kv-cache group. Configure it with the pipeline-wide `enable_prefix_caching` field in the deploy config.
@@ -56,7 +60,7 @@ The prefix cache flow is then outlined below.
2. Say we process the request `The quick brown fox was tired and slept beneath the shady tree`, which is 12 tokens and evenly divides into 3 blocks as shown below.
```
```text
[ The quick brown fox ] [ was tired and slept ] [beneath the shady tree ]
Block 1: |<--- block tokens ---->|
Block 2: |<------- prefix ------>| |<--- block tokens --->|
@@ -65,10 +69,9 @@ Block 3: |<------------------ prefix -------------------->| |<--- block tokens -
When the request processes, we inspect the multimodal outputs and identify the `mm_feature` tensor, which will be of shape `(seq_len, feature_dim)`, i.e., `(12, 16)` in this example. We note that the first axis is dependent on the `seq_len` and add a new cache_tensor of shape `(num_blocks, block_size, feature_dim)` to our multimodal cache for tensors.
1. If we lay out the cache as a 2D tensor of shape (`num_blocks`, `block_size`), we'll have something like the following:
3. If we lay out the cache as a 2D tensor of shape (`num_blocks`, `block_size`), we'll have something like the following:
```
```text
0: [ The quick brown fox ]
1: [ was tired and slept ]
2: [beneath the shady tree ]
@@ -78,7 +81,8 @@ When the request processes, we inspect the multimodal outputs and identify the `
```
Or, if we flatten it down to 1D,
```
```text
0: The
1: quick
2: brown
@@ -90,7 +94,8 @@ Or, if we flatten it down to 1D,
```
which we can think of as row indices into the hidden states tensor if we view it as the 2D shape `(num_blocks x block_size, feature_dim)`. That is, the analogous flattened (from 3D -> 2D) mapping of the cache for hidden states becomes the following.
```
```text
0: <hidden states vector of len 2 corresponding to 'The'>
1: <hidden states vector of len 2 corresponding to 'quick'>
2: <hidden states vector of len 2 corresponding to 'brown'>
@@ -103,10 +108,9 @@ which we can think of as row indices into the hidden states tensor if we view it
Similarly, for the multimodal outputs cache, the flattened coordinates are the same, but the `mm_feature` maps to vectors of length `16` instead of the hidden size of `2`. Note that in practice, we may have multiple multimodal output tensors per forward pass, which may have different names and different feature dimensions.
1. Now, say that we receive a new request `The quick brown fox jumped over the dog`.
4. Now, say that we receive a new request `The quick brown fox jumped over the dog`.
```
```text
[ The quick brown fox ] [ jumped over the dog ]
Block 1: |<--- block tokens ---->|
Block 2: |<------- prefix ------>| |<--- block tokens --->|
@@ -116,7 +120,7 @@ Here, we will have a cache hit for `Block 1` which will be detected by vLLM base
Since we have the block indices / slot mappings from the kv cache manager, we can simply mirror the mappings and leverage the same indices for the cached hidden states and multimodal outputs. This allows us to look up the correct tensors from our externally maintained 3D caches.
```
```text
0: [ The quick brown fox ] < already in the cache
1: [ was tired and slept ]
2: [beneath the shady tree ]
@@ -129,11 +133,11 @@ Since we have the block indices / slot mappings from the kv cache manager, we ca
Finally, to pass the full hidden states and multimodal outputs to the next stage, we simply concatenate the cached contents with the corresponding new tensors computed from the current forward call.
### What About Multimodal Inputs?
It's also useful to consider the case about how Omni prefix caching is handled when we have multimodal inputs that don't cleanly end on block boundaries, as well as how this works with multimodal encoder caching in vLLM. For example:
```
```text
[ Im0 Im1 Im2 Im3 ] [ Im4 Im5 foo <empty> ]
Block 1: |<--- block tokens ---->|
Block 2: |<------- prefix ------>| |<--- block tokens --->|
@@ -147,9 +151,9 @@ In reality, this isn't a big problem for correctness, because vLLM also maintain
- The hash describing the image data starting at position 0 and with length 6
- In vLLM's encoder cache, a mapping from the image hash above to the encoder output
To understand what happens, say we get the following input as a second request:
```
```text
[ Im0 Im1 Im2 Im3 ] [ Im4 Im5 bar baz ]
Block 1: |<--- block tokens ---->|
Block 2: |<------- prefix ------>| |<--- block tokens --->|
@@ -162,3 +166,162 @@ Because we have multimodal data in a scheduled span that isn't fully precomputed
When we pass our multimodal tensors to the language model component in the same stage, we'll then expect the same outputs, because the prefix caching behaviors in vLLM-Omni / vLLM match, so the LLM will use vLLM's KV cache manager's prefix caching to correctly handle the attention information for `Block 1` while calculating the outputs for `Block 2`, giving us the correct results for processing `Block 2` with the context of `Block 1`.
Finally, we look up the output hidden states/multimodal tensors corresponding to the prefix cache hit `Block 1` and concatenate it with the forward pass result to get the final result, which is expected to be identical to the full hidden states when prefix caching is disabled.
### Implementation
The block/slot model is `vllm_omni/core/prefix_cache/`.
`OmniPrefixCacheManager` owns slot occupancy, the request-task table,
hit spans, and merge.
`OmniPrefixCacheController` moves data: a reusable `StagingBufferPool` for
this step's device→host copy, and writes into the durable `PrefixBlockPool`.
The state lock covers those tables only.
Miss is not an error (this step's forward slice only). A hit span whose
hidden rows are absent is fatal. For mm keys the rule is looser: a model may
emit a key only for some requests, so a hit span with no rows behind an mm key
reads zeros from the pool for those positions rather than raising. Abort still
writes: once a hash entered this step's batch it must land in the cache.
`enable_prefix_caching` is refused on KV-consumer stages (`kv_role` of
`kv_consumer` or `kv_both`). KV received from a producer is reported as
`num_computed_tokens` too, and the manager cannot tell it from a local hit.
Producer-only stages are unaffected. Pooling stages never save, so they get
no cache. This gate is one function, `stage_prefix_cache_config`, called by
both the GPU and the NPU model runner at kv-cache init.
Which stages may set `enable_prefix_caching: true`:
| Stage | `enable_prefix_caching` | Why |
| --- | --- | --- |
| AR stage with one full-attention kv group whose hidden states / per-token mm feed the next stage (Qwen3-Omni thinker and talker) | supported | The case the cache is built for: full-prompt hidden states are merged from the pool on a hit. |
| AR stage that sets `requires_full_prefix_cached_hidden_states = False`, optionally with `deferred_prefix_cache_mm_keys` (Qwen3-TTS talker, Higgs v3 talker) | supported | Hidden is not cached; deferred codec rows are written once on finish. |
| Pooling stage | ignored | Never saves; the gate returns no config. |
| `kv_role: kv_consumer` / `kv_both` | refused at kv-cache init (`OmniPrefixCacheUnmatchError`) | Producer KV shows up as `num_computed_tokens` and is indistinguishable from a local hit. |
| Speculative decoding on the stage | refused at kv-cache init | Under async scheduling vLLM keeps `num_computed_tokens_cpu` optimistic (all drafts accepted) during the forward and corrects it afterwards; `step_slots_cpu` would mirror rows at the wrong slots. Not verified; refused as a whole. |
| `prefix_match_unit` smaller than `block_size` | refused at kv-cache init | Sub-block hits make `num_computed_tokens` unaligned; the hit registry only mirrors whole blocks. |
| Hybrid / sliding-window / multi-group kv cache (e.g. a talker with `attention_type: sliding_recompute`) | refused at kv-cache init | The cache mirrors exactly one full-attention block table. |
| Attention backend whose kernel block size differs from `--block-size` (FlashInfer / FlashMLA / CutlassMLA with a block size they do not list natively), or decode context parallel | refused at first step (`FullAttentionGroupView`) | `step_slots_cpu` computes `table[req, pos // block_size] * block_size + pos % block_size` over allocator block ids; hybrid kernel blocks and DCP token striping change that row layout. FlashAttention / Triton accept any multiple of 16 and never split blocks. |
| Codec decoder / Code2Wav stages (Qwen3-Omni stage 2, Qwen3-TTS stage 1) | keep `false` | Nothing downstream consumes their hidden states; the cache would only add device→host copies. Not validated. |
| Diffusion stages | n/a | No vLLM KV cache to mirror. |
Hit spans come from `scheduled_new_reqs` only, as in the pre-refactor cache:
- A new request with a (partial) prefix hit is the normal path: the hit
blocks are read from the pool, the rest is this step's rows, and the
divergent tail is written under its own blocks. The gather starts on the
prefetch thread at `new_step_starts`; a same-step hit (vLLM hashes blocks
at schedule time, so `b` can hit blocks `a` computes in the same forward)
cannot plan until `a`'s write is registered and starts at `save_outputs`
instead. Either way it runs before the next step can hand those blocks
to a new tenant. vLLM frees blocks one step before `finished_req_ids`
arrives, so a finished request whose last step materializes late may
still find a hit slot reassigned. Each `(slot, key)` carries a write
version, bumped whenever a new write claims it. A planned read captures
that version and is registered in `_pending_reads`; a later write that
reclaims those slots copy-on-writes the still-`COMMITTED` rows into the
ref before it overwrites the pool, so a delayed fetch serves the original
tenant. A version mismatch with no preserved copy raises for live and
finished alike (the pool rows are a newer tenant's). Production defaults
stay opt-in until preempt/resume hit spans are reconstructed.
- `async_chunk` continuation: when the next upstream chunk arrives, the same
request id re-enters `scheduled_new_reqs` with `num_computed_tokens` equal
to what it already computed itself. Ids already in `live_reqs` are skipped
for hit marking: those rows were delivered in earlier steps and re-emitting
them would duplicate output. A `delivered_upto` span for this case is
Phase 2.
- Preemption + reschedule: vLLM resets `num_computed_tokens` to 0 on
preemption and re-runs prefix matching on resume, so the resumed request
can come back with a fresh hit. With the V1 model runner it arrives
through `scheduled_cached_reqs` (id in `resumed_req_ids`, `new_block_ids`
replaces the table); with the V2 runner it re-enters `scheduled_new_reqs`
while still in `live_reqs`. Neither path marks an omni hit span: the
resumed request gets only the rows it recomputes, and its still-open
deferred write keeps appending (a slot written twice keeps the later
chunk). Cache integrity holds either way — the hit blocks already have
rows, from this request or the one it hit. Stages that need full prompt
hidden states should be sized so preemption does not occur while prefix
caching is on. Same as before this refactor; tracked for Phase 2.
Two write paths, split by `ModelCachePolicy.deferred_keys`:
- Immediate (`JOIN_NEXT_STEP`): save launches a whole-step device→host into
a staging slot; the committer waits that event and writes the CPU pool.
The next save waits `done` (the pool write): a reused slot must never
leave a pending pool write behind, or a delayed hit read of the old rows
would find nothing recoverable.
- Deferred (`JOIN_ON_FINISH`): mm whose first dim is this step's token count
stays on a per-request GPU clone; `_WriteChunk`s append across steps;
finish/abort (or GPU-byte-budget pressure) forces the copy. One open
WriteTask per request; a budget flush may close it mid-request, in which
case the next save opens a new one (`write_n` + 1) and a hit reads both,
so one long request cannot pin the whole budget.
A `WriteTask` moves through `TaskState` only via `transition()`, one step at
a time along a strict chain:
`PENDING` (registered, not queued) → `QUEUED` (on a copy queue) →
`COPYING` (one thread owns the copy stage) → `HOST_READY` (host rows ready) →
`WRITTEN` (in the CPU pool); `FAILED` is reachable from any non-terminal
state. Skipping a step raises. The worker claims `COPYING` in the same `_wake`
critical section as the queue pop, so `escalate` never re-queues a task it can
see is already claimed. `host_ready` / `done` are wait primitives set by the
transitions into `HOST_READY` / `WRITTEN` / `FAILED`. GPU-clone bytes are
charged once per clone on a `_BudgetTicket` pinned by every task that views
it, and uncharged when the last holder releases (idempotent per tid).
Mm whose first dim equals the unpadded scheduled length *or* the CUDA-graph
padded length is registered on first sighting. Talker `codes.audio` is a
cat of scheduled rows and stays unpadded while hidden is padded; both must
open a pool key, whether immediate or deferred.
Leftover mm (lists, `codes.ref`, any tensor whose first dim is not this
step's token count) is copied to CPU at save without truncating that first
dim. That leftover copy is the async-builder read replica for this step; it
does not write the pool or carry abort/preempt occupancy.
```python
cache.register_policy(ModelCachePolicy.from_model(model)) # load_model
cache.new_step_starts(scheduler_output) # before _update_states
sid = cache.save_outputs(hidden, mm_outputs, num_tokens_unpadded=n,
num_tokens_padded=n_pad)
outs = cache.materialize(sid, req_ids) # or discard_step(sid)
```
Each step id is consumed exactly once. `req_ids` must be a subset of the save
snapshot. At most `staging_depth` unused step ids may exist at once: every
`save_outputs` claims one staging slot, including saves with only leftover
mm that copy no device→host page. A slot is also held by each immediate
write that views it, until the committer's pool write. A later save waits
for `materialize`/`discard_step` or that pool write to free a slot;
`staging_claim_timeout_s` then errors with the unused ids and the task count.
`join`/`join_host_ready` use the same bound and raise with the stuck task's
id and state instead of hanging the caller.
`materialize` may run on the async output builder after the engine has
entered the next step; leftover mm (not written to the pool) is copied
to CPU at `save_outputs` so the builder never reads live CUDA-graph
buffers. See
[Async Omni Output Materialization](omni_async_output_materialization.md).
The cache is constructed only on the last pipeline-parallel rank
(`_ensure_omni_prefix_cache`). Other ranks skip it: they never call
`save_outputs`, so a hit table there would resolve to absent slots.
Threads, locks, and what each may block on:
| Thread | Role | May block on | Must not hold while blocked |
| --- | --- | --- | --- |
| Engine | `new_step_starts`, `save_outputs` | previous-step `join` (`done`) / finished-write `join_host_ready`; `reserve()` GPU-byte flush; staging-slot claim; `dispatch()` / finish-abort `escalate()` (eager mode: the copy + pool write run inline) | `_state_lock` |
| Async output builder | `materialize` (may overlap the next engine step) | this step's `step_d2h_event`; `join` (`done`); deferred `fetch_host` | `_state_lock` |
| Committer | `_worker_loop`: wait device→host / deferred copy / pool write | `_wake.wait`; `step_d2h_event` or copy-stream sync | never takes `_state_lock` |
| Prefetch pool | hit-span gather during forward | `join` (`done`); deferred `fetch_host` | `_state_lock` |
| Lock | Covers | Does not cover |
| --- | --- | --- |
| manager `_state_lock` | occupancy tables, step contexts, hit spans, task registration, pool-key publish (`install_key`) | join, GPU-byte flush, copy, `step_d2h_event` wait, pool-key allocation, eager `dispatch()` / `escalate()` |
| controller `_lock` / `_wake` | task registry, queues, GPU-clone byte budget | device→host / pool-write body (released before `synchronize`) |
| `WriteTask.lock` | `state`, `reassigned`, `append_chunk` | waiting on `host_ready` / `done` (those are events) |
### Related Files
- `vllm_omni/core/prefix_cache/`
- `vllm_omni/worker/gpu_model_runner.py` (`_ensure_omni_prefix_cache`)
- `tests/core/test_prefix_cache.py`
- [Async Omni Output Materialization](omni_async_output_materialization.md)
docs/design/fullduplex-personaplex.md
+108
View File
@@ -271,3 +271,111 @@ The required evidence is:
Audio that is empty, all zero, non-finite, or only a protocol `listen` event is
not a successful end-to-end result.
### Paced multi-session load driver
The existing driver keeps its two-session admission and slot-reuse validation
when `--sessions` is omitted. Passing `--sessions N` selects a separate,
bounded load mode; it does not replace the lifecycle test or change server
admission capacity. Start an existing PersonaPlex server configured to admit
the requested number of sessions, then run from a source checkout with the
vLLM-Omni client dependencies installed:
```bash
python -m tests.e2e.online_serving.personaplex_realtime_duplex \
--model nvidia/personaplex-7b-v1 \
--input-wav /path/to/input.wav \
--sessions 2 --load-frames 1000 --drain-s 2 \
--server-revision SERVER_COMMIT --server-hardware A100-80GB \
--output-dir /tmp/personaplex-load-n2
```
Use a new output directory for every run. The driver refuses to overwrite an
existing `load-result.json`. The server labels are supplied by the operator,
not detected or verified by the client. Record the server command, package
versions, checkpoint revision and hardware alongside a published measurement.
All attempted sessions finish admission before admitted sessions share a
common input start. A rejected session remains a failed row in the report;
it is not removed from the denominator. Each session receives at most
`--load-frames` input frames, without looping the WAV, and uses 80 ms pacing.
A stalled send cannot cause a catch-up burst. The timeline records send
schedule lateness so an overloaded client is visible rather than mistaken
for a faster server. This mode measures steady concurrency, not join/leave
interference; the original lifecycle mode still checks slot reuse.
Load mode does not probe an additional N+1 session for `resource_exhausted`;
the lifecycle mode owns that admission-limit check. A per-session
`client_pacing_warning` and the summary's `client_pacing_warning_sessions`
identify send-lateness p99 above one nominal frame (80 ms). This is a warning,
not an extra acceptance gate: read it alongside server-facing measurements
so a slow client is not mistaken for a slow server.
The fixed drain window is part of the measurement configuration. Audio
arriving during cleanup is excluded, so closing cannot rescue a session that
missed its observation deadline. Cleanup is attempted for admitted sessions
even when audio validation fails. Protocol errors, missing/invalid audio,
shared response IDs and cleanup failures make the report fail and the CLI
exit nonzero. Cancellation closes transports and propagates to the caller.
`--cleanup-timeout-s` (default 5 s) bounds session-close send and acknowledgement
together, independently of `--timeout-s`. The subsequent WebSocket close
handshake has the same separate bound; an unresponsive close acknowledgement
cannot hold cancellation for the full inference timeout.
`load-result.json` retains per-session send and audio-packet timelines using
the client's monotonic clock. It does not export handshake credentials or
raw audio payloads. Metric definitions are explicit:
Both modes validate each nonempty audio packet's strict base64 encoding, PCM16
sample alignment, 24 kHz sample rate and response identity. Valid aggregate
audio cannot hide a malformed packet. In load mode, rejected packets make the session fail
and appear in `invalid_audio_packets` with their event index, receive time and
fixed diagnostic code; they do not contribute samples or timing intervals.
Empty audio deltas remain legal and contribute no samples. This check does
not change the public client's decoding. The lifecycle scenario and dispatch
remain the same, but malformed packets that aggregate checks once tolerated
now fail that mode as well.
The probe accepts both the current OpenAI Realtime audio event name
`response.output_audio.delta` and the legacy `response.audio.delta`. Validation
uses the wire payload directly rather than relying on a particular client
library's alias table, so a server-side event-name migration cannot silently
turn real audio into a zero-output measurement.
| Field | Meaning |
| --- | --- |
| `client_audio_packet_interval_ms` | Consecutive nonempty packet arrivals: median, linear p99, max, and zero-based index of the first maximal interval. Interval index 0 ends at packet 1. One packet can contain multiple codec frames; this is not server tick latency. |
| `client_first_audio_after_stream_start_ms` | First audio receipt minus first input send start; admission is excluded. |
| `client_stream_rtf` | First input send start to last audio receipt, divided by received audio duration. This includes real-time input pacing and is not an inference-only RTF. |
| `client_send_lateness_ms` | Send start minus its original shared schedule, clipped at zero. |
| `frame_deficit` | Integer input-frame minus output-frame count from the shared session validation; missing when valid whole-frame accounting is unavailable. |
| `output_frames`, `voiced_frames`, `silent_frames`, `frame_coverage_ratio`, `audio_rms`, `audio_chunks` | Session-validation statistics retained on success and on acceptance failure once computed. |
| `acceptance_check` | Failed local check, with numeric diagnostics in `error`; other exceptions retain phase/type only, without remote handshake or metadata text. |
| `audio_underrun_s`, `audio_underrun_event_count`, `audio_continuity_ok` | Shared `compute_continuity_stats` on valid PCM16 packet arrivals, with its default 100 ms underrun threshold; absent audio gives missing values rather than a continuity pass. |
No output gives missing latency/RTF values, not zero. By default RTF is
diagnostic; `--max-client-rtf` adds an explicit ceiling. The existing frame
coverage, audible-frame and scheduler-data-plane checks still apply. A longer
`--drain-s` can demonstrate eventual completion under backlog, but it must not
be reported as real-time capacity when the fixed-window run failed or client
RTF exceeded the stated target. Likewise, the first live session can pay
one-time model/codec/JIT costs; either use the server's configured duplex
warmup path or label that run as cold-start evidence before comparing steady
capacity. Passing client-only metrics does not certify model quality or a
real-time capacity target on another GPU. Begin with N=1 and N=2; higher N is a
capacity experiment, not a promised property of an A100 or of this patch.
Continuity is a client-arrival diagnostic, not proof of speaker playback and
not an additional pass/fail gate. Packet-interval and send-lateness summaries
retain their documented linear p99; the older client interval helper reports
rounded nearest-rank p95 instead and is not substituted for that schema.
Deterministic driver and localhost WebSocket checks need no model or GPU:
```bash
python -m pytest tests/e2e/online_serving/test_personaplex_load_driver.py \
-m 'core_model and cpu' --run-level core_model -q
```
These checks test the driver against controlled protocol responses. Run the
load-driver command against real weights separately before reporting model-serving
performance. Server tick timestamps and generic TTFT/TPOT aggregation remain
the responsibility of the existing duplex metrics paths, not this driver.
docs/design/fullduplex.md
+82 -14
View File
@@ -9,6 +9,12 @@ the owner document for `vllm_omni/engine/duplex/**`,
`vllm_omni/entrypoints/duplex_omni.py`, `vllm_omni/entrypoints/duplex/**`,
`vllm_omni/clients/**` and the per-model `duplex/` packages.
`vllm_omni/protocol/realtime/**` (the OpenAI Realtime API) and
`vllm_omni/protocol/duplex/**` (vLLM-Omni's extension of it) are the shared
wire codec duplex is built on. Duplex is its first consumer but not its owner: the codec
is deliberately runtime-agnostic so a second Realtime surface can reuse it
(RFC #6592 P0a).
The public wire contract lives in
[`docs/serving/realtime_duplex_api.md`](../serving/realtime_duplex_api.md);
this page describes how it is produced.
@@ -108,23 +114,56 @@ plugin and the session runtime config to `DuplexOrchestrator` directly.
`AsyncOmniEngine`, `OrchestratorBase` and `Orchestrator` carry only
template seams; `tests/engine/test_duplex_import_boundary.py` checks that
importing the turn-based stack loads no duplex module.
4. **Every surface a duplex server exposes is backed by a session.**
4. **Deployment selects the serving stack.**
`vllm-omni serve` constructs `DuplexOmni` when the pipeline declares
`duplex_plugin`; the server exposes `/v1/realtime?duplex=1` (alias
`duplex_plugin` and the deploy configuration sets `session_mode: duplex`.
An explicit `session_mode: turn` selects `AsyncOmni` instead, without
changing the pipeline's duplex capability. In duplex mode the server exposes `/v1/realtime?duplex=1` (alias
`/v1/duplex`), `POST /v1/chat/completions`, `/v1/models` and `/health`, and
every other turn-based route reports "not available". The chat route is
served by `DuplexChatCompletionsAdapter`, a Realtime client that runs one
short-lived session per request: speech becomes a committed turn, text is
seeded as the session's opening turn because a model-native model takes a
turn only when it hears speech. Whether a model can be reached with text is
`DuplexCapabilities.supports_chat_completions`; one that cannot is refused
at once rather than left to idle out. Turn-based use of the same model stays
available offline through `Omni` / `AsyncOmni`.
every other turn-based route reports "not available". The chat route uses
the ordinary chat service on the duplex engine, without a session, when
the plugin declares `DuplexCapabilities.supports_chat_completions`.
Turn-based deployments use the ordinary API initialization and retain
model capability checks and endpoint restrictions.
5. **Serving is transport only.** The websocket handler does socket I/O,
wire-envelope validation, command translation, event rendering and the
attachment/resume/replay bookkeeping; it holds no session state.
6. **No `typing.Protocol`** in the duplex surfaces: the plugin, data plane,
session state, PCM buffer, stage port and client transport seams are ABCs.
7. **The wire codec is not duplex, and a wire type is not a mailbox message.**
The event and command *vocabulary* plus its wire rendering, the audio format
negotiation, the conversation-item rules and the error envelope are model-
and runtime-agnostic, and split along the three tiers
[`docs/serving/realtime_duplex_api.md`](../serving/realtime_duplex_api.md)
already defines: **Tier 1** (identical to OpenAI) is
`vllm_omni/protocol/realtime/**`; **Tier 2** (OpenAI names carrying our
extensions) and **Tier 3** (ours alone) are `vllm_omni/protocol/duplex/**`.
A Tier 2 class *subclasses* its Tier 1 twin and declares only what it adds,
so the OpenAI surface stays honestly pure --- `session.created` without
`resume_token`, `input_audio_buffer.append` without `video_frames`. Our
error **codes** are Tier 3 (`protocol/duplex/errors.py`); only the envelope
shape and OpenAI's three `error.type` classes are Tier 1.
What stays engine-side is everything that is *internal representation*
rather than contract: `DuplexCommand.payload()` renders the session runner's
mailbox dictionary, whose channel genuinely differs from the client event
(`session.update` and the three `conversation.item.*` commands all travel on
`turn.signal`); `engine/duplex/realtime_commands.py` decides which command a
decoded event becomes; `engine/duplex/realtime_events.py` holds the
session's projection state. The codec may not import the engine, the
entrypoints, `model_executor` or the clients
(`tests/protocol/realtime/test_protocol_import_boundary.py`), and there is
exactly one implementation of each codec behaviour
(`tests/protocol/realtime/test_realtime_codec_single_source.py`).
The dependency chain is `protocol/realtime` <- `protocol/duplex` <- engine /
entrypoints / clients, and a duplex consumer uses **only** the middle link:
`protocol/duplex/**` re-exports the Tier 1 names it does not extend, so a
helper that later needs a duplex-specific version (`convert_input_audio_with_rate`
resamples to MiniCPM-o's 16 kHz rather than the client's rate) is overridden
in one file instead of at every call site
(`tests/protocol/duplex/test_duplex_protocol_facade.py`).
`DuplexEvent` is therefore a plain alias of `RealtimeEvent` --- an event has
no engine-internal half --- while `DuplexCommand` is a real class, because a
command does.
## Package layout
@@ -143,6 +182,31 @@ vllm_omni/
│ │ ├── chat_completions.py DuplexChatCompletionsAdapter (/v1/chat/completions on a session per request)
│ │ └── websocket.py websocket send/close/receive helpers
│ └── openai/api_server.py builds DuplexOmni for duplex models; session-backed app state
├── protocol/ SHARED WIRE CODEC (no engine / no model / no transport)
│ ├── realtime/ the OpenAI Realtime API itself
│ │ ├── events.py RealtimeEvent + OpenAI's 30 server events (22 ship as Tier 1;
│ │ │ 8 have a Tier 2 twin in protocol/duplex)
│ │ ├── commands.py RealtimeCommand + OpenAI's 10 client commands (8 Tier 1,
│ │ │ 2 with a Tier 2 twin) --- wire_type + fields, no payload()
│ │ ├── formats.py audio format spellings, what is supported, per-session validation
│ │ ├── session.py session-object readers; RealtimeInputDefaults (append defaults)
│ │ ├── items.py conversation item shape, truncation, transcripts, camera frames
│ │ ├── audio_input.py decode_audio_append -> RealtimeAudioAppend; client speech hints
│ │ ├── audio.py PCM / G.711 / WAV decode, resample, re-encode
│ │ ├── errors.py RealtimeProtocolError (the envelope's exception type)
│ │ └── capabilities.py RealtimeProtocolCapabilities: what one consumer can serve
│ └── duplex/ Tier 2 + Tier 3, and the one door a duplex consumer uses
│ ├── __init__.py re-exports the Tier 1 helper functions (audio, formats,
│ │ items, session, capabilities) so the engine never
│ │ imports protocol/realtime directly
│ ├── events.py the whole 42-event vocabulary: 22 Tier-1 re-exports,
│ │ 8 Tier-2 subclasses (session.created + resume_token, ...),
│ │ 12 Tier-3 events (listen/speak, playback, resume, ...);
│ │ DuplexEvent = RealtimeEvent
│ ├── commands.py the whole 17-command vocabulary: 8 Tier-1 re-exports,
│ │ 2 Tier-2 subclasses (append + hints, commit + final),
│ │ 7 Tier-3 commands (barge_in, playback.ack, ...)
│ └── errors.py our error-code vocabulary (Tier 3) + RealtimeProtocolError
├── engine/
│ ├── omni_engine_base.py OmniEngineBase (stage processes, orchestrator thread, queues, RPC)
│ ├── async_omni_engine.py AsyncOmniEngine (turn-based request building)
@@ -150,16 +214,20 @@ vllm_omni/
│ ├── orchestrator.py OrchestratorBase + Orchestrator
│ ├── duplex_orchestrator.py DuplexOrchestrator (+ DuplexOrchestratorRequestState; implements DuplexStagePort)
│ └── duplex/
│ ├── commands.py DuplexCommand dataclasses, command_from_realtime
│ ├── realtime_commands.py Realtime client event -> DuplexCommand translation
│ ├── events.py DuplexEvent dataclasses (+ to_realtime), REALTIME_ERROR_TYPES_BY_CODE
│ ├── commands.py mailbox half: DuplexCommand.payload() + the mailbox `type`,
│ │ paired with each protocol command; command_from_realtime
│ ├── realtime_commands.py duplex binding of the codec: decoded event -> DuplexCommand,
│ │ DUPLEX_REALTIME_CAPABILITIES, duplex_response_format
│ ├── events.py re-export shim (DuplexEvent = RealtimeEvent) + the runner's
│ │ epoch-filter sets DOMAIN_TERMINAL_EVENTS / MODEL_OUTPUT_EVENTS
│ ├── realtime_events.py RealtimeProjectionState: internal event -> typed events
│ ├── messages.py queue envelopes (Open/Close/Resume/Touch/Command/Result/Event), DuplexSessionError
│ ├── config.py DuplexSessionConfig, DuplexCapabilities, ResponseCreateOptions
│ ├── contracts.py DuplexFence (session_id, epoch, turn_id), stage request records, DuplexStagePort
│ ├── plugin.py DuplexModelPlugin, DuplexModelSessionState, DuplexDataPlane, PcmAppendBuffer ABCs
│ ├── turn_detection.py server-side VAD turn detector used by the session
│ ├── audio.py / vad.py / intermediate.py
│ ├── audio.py compatibility re-export, via protocol/duplex
│ ├── vad.py / intermediate.py
│ └── session/ one engine-resident session and everything that runs it
│ ├── engine_session.py DuplexEngineSession: ledgers, lease, fence, stage resources, append sequencing
│ ├── runner.py DuplexSessionRunner (per-session mailbox on the orchestrator loop)
docs/design/module/entrypoints.md
+53
View File
@@ -30,6 +30,7 @@ primary_code_paths:
- vllm_omni/entrypoints/utils.py
- vllm_omni/entrypoints/duplex/**
- vllm_omni/clients/**
- vllm_omni/protocol/**
primary_path_exceptions:
- path: vllm_omni/entrypoints/openai/errors.py
owner: error_contracts.md
@@ -53,6 +54,7 @@ depends_on:
- error_contracts.md
- engine_orchestration.md
validation_paths:
- tests/protocol/**
- tests/entrypoints/test_omni_entrypoints.py
- tests/entrypoints/test_async_omni.py
- tests/entrypoints/test_async_omni_pause_sleep_routing.py
@@ -94,11 +96,43 @@ This document owns offline API semantics, CLI and serve composition,
supported OpenAI-compatible routes, request validation and normalization,
response conversion, streaming/session behavior, and engine handoff.
When `--api-server-count` is greater than one, the serve composition root
starts the frontend process manager while a parent-owned `StageRuntime`
launches the shared local stage engines. Each frontend receives only its own
stage-client channel configuration and does not launch or retire backend
processes. CLI validation rejects unsupported distributed, diffusion,
fault-tolerant, elastic-EP, Ray, and runtime-LoRA combinations before startup.
It does not own configuration precedence, cross-stage routing, stage
placement, payload implementation, or semantic error classification.
`entrypoints/openai/errors.py` is an explicit primary-path exception owned by
`error_contracts.md`.
It also owns `vllm_omni/protocol/**`, the shared wire codec promoted by RFC
[#6592](https://github.com/vllm-project/vllm-omni/issues/6592) P0a. It follows the three tiers
`docs/serving/realtime_duplex_api.md` already defines. `protocol/realtime/**`
is Tier 1, identical to OpenAI: 30 server events, 10 client commands, their
typed fields and pure wire rendering, audio-format negotiation,
conversation-item rules and the error-envelope shape. `protocol/duplex/**` is
Tier 2 (OpenAI names carrying vLLM-Omni extensions, each a subclass of its
Tier 1 twin declaring only what it adds) and Tier 3 (ours alone, including our
error-code vocabulary), kept separate so a GA-only consumer is not handed a
vocabulary its clients never send. That package sits
outside `entrypoints/` on purpose --- the engine depends on it, so it belongs
to neither layer --- but deciding what a Realtime client may put on the wire is
this document's subject, and the same reasoning already places
`vllm_omni/errors.py`, `inputs/` and `outputs/` under contract documents rather
than under a consumer.
It is deliberately **not** owned by `../fullduplex.md`, even though duplex is
the codec's only consumer today. RFC #6592 asks for a named owner precisely so
a second Realtime consumer does not need duplex review to change the codec
(#6592 open question 8). Ownership here is review scope, not a dependency: the
codec may not import the engine, the entrypoints or the model code, which
`tests/protocol/realtime/test_protocol_import_boundary.py` enforces. When a
second consumer lands (#6592 P0b) the package is expected to graduate to its
own module document with owners drawn from both consumers.
## Candidate invariants
These identifiers are proposals while the document is `draft`.
@@ -108,6 +142,22 @@ These identifiers are proposals while the document is `draft`.
**Rule:** Entrypoints MUST NOT implement cross-stage routing or stage lifecycle
policy.
### ENTRY-INV-002: The shared Realtime codec does not depend on its consumers
**Rule:** `vllm_omni/protocol/**` MUST NOT import `vllm_omni.engine`,
`vllm_omni.entrypoints`, `vllm_omni.model_executor`, `vllm_omni.worker` or
`vllm_omni.clients`; `protocol/duplex/**` MAY depend on `protocol/realtime/**`
but never the reverse, and a Tier 1 class MUST NOT carry a vLLM-Omni extension
field; and each protocol/codec behavior MUST have exactly one
implementation that every consumer uses. A wire type MUST NOT carry a
consumer's internal representation: the duplex mailbox rendering
(`DuplexCommand.payload()`, whose channel differs from the client event for
`session.update` and the `conversation.item.*` commands) stays engine-side.
A duplex consumer (`engine/duplex/**`, `entrypoints/duplex/**`, the duplex
clients) MUST import `vllm_omni.protocol.duplex` and MUST NOT import
`vllm_omni.protocol.realtime` directly, so the tier boundary has exactly one
extension point.
### ENTRY-INV-100: Public requests are normalized once
**Rule:** Public protocol values MUST be validated and converted to internal
@@ -141,6 +191,9 @@ stage so `wake_up(stage_ids=[0])` does not skip a later `wake_up(stage_ids=[1])`
Streaming input pumps take an admission slot immediately before each EngineCore
ADD or update, not while waiting for the next client chunk. Frontend abort
keeps `request_states` until `generate()` consumes the terminal output.
Multi-API serving must preserve one client rank and one channel set per
frontend, include every frontend in readiness/failure observation, and keep
stage lifecycle ownership in the parent composition root.
## Promotion gate
docs/design/module/stage_runtime.md
+20
View File
@@ -77,6 +77,22 @@ readiness, one pool per logical stage, replica identity and membership,
selection and affinity, stage-client/process lifecycle, liveness, draining,
and shutdown.
Local EngineCore initialization and multi-API serving use
`StageRuntime.launch_stage_engines(num_api_servers=1)` as their common launch
entry. It resolves plans when not supplied by the runtime's device-group
scheduler, coordinates device locks and environment overlays, starts engines,
and rolls back resources on startup or attachment failure. Single-API clients
attach locally after readiness and take ownership of the engine resources;
there is no additional API subprocess. Diffusion and remote attachment retain
their backend-specific initialization.
For local multi-API serving, the parent runtime owns and launches each stage
engine once. It supplies a distinct input/output channel for every API
frontend and stage replica; frontend workers attach clients to those engines
without acquiring stage-process ownership. Readiness observation and shutdown
remain parent-owned. This topology currently excludes diffusion and
remote/headless stages, intra-stage data parallelism, and Ray backends.
It does not own cross-stage request policy, model scheduler policy, connector
transfer semantics, configuration precedence, or public error rendering.
@@ -114,6 +130,10 @@ affinity, membership changes, abort routing, process failure, draining, and
repeated shutdown. Cross-stage policy changes belong in
`engine_orchestration.md`.
For multi-API changes, additionally exercise per-frontend channel assignment,
shared-engine startup handshakes, frontend failure observation, and parent
cleanup without duplicate stage shutdown.
## Promotion gate
- Reconcile terminology and ownership after #5441 reaches a final state.
docs/features/comfyui.md
+1
View File
@@ -51,6 +51,7 @@ This extension offers the following nodes based on the output modalities:
- **Generate Video** for text-to-video and image-to-video tasks
- **Multimodality Understanding** for multimodality-to-text and multimodality-to-audio tasks
- **TTS** and **TTS Voice Clone** for TTS tasks
- **Generate Music** for text-to-music tasks
This extension also offers example workflows (at **ComfyUI sidebar -> Templates -> vLLM-Omni**)
docs/features/custom_pipeline.md
+6 -3
View File
@@ -1,7 +1,7 @@
# Custom Pipeline Extension Guide
Transformer already support Custom Pipeline via
https://github.com/huggingface/diffusers/blob/main/docs/source/en/using-diffusers/custom_pipeline_overview.md
<https://github.com/huggingface/diffusers/blob/main/docs/source/en/using-diffusers/custom_pipeline_overview.md>
This guide demonstrates how to use the newly added features for extending vLLM-Omni's diffusion pipeline with custom functionality.
@@ -20,8 +20,9 @@ Three main features enable custom pipeline extension:
`WorkerWrapperBase` is a wrapper class that creates `DiffusionWorker` instances with optional extension support. It enables dynamic inheritance, allowing you to add custom methods and functionality to workers without modifying the base worker class.
**Key capabilities:**
- Dynamic worker class extension via `worker_extension_cls`
- Support for custom pipeline initialization via `custom_pipeline_args`
- Support for custom pipeline initialization via `custom_pipeline_args["pipeline_class"]`
- Method delegation to underlying worker
- Attribute access forwarding
@@ -42,6 +43,7 @@ The `diffusion_load_format` parameter controls the initial diffusion model load.
`CustomPipelineWorkerExtension` is a mixin class that extends `DiffusionWorker` with the ability to re-initialize the pipeline with a custom implementation.
**Key method:**
- `re_init_pipeline(custom_pipeline_args)`: Re-initializes the pipeline with custom arguments, properly cleaning up the old pipeline first
**Location:** `vllm_omni/diffusion/worker/diffusion_worker.py`
@@ -150,6 +152,7 @@ omni = Omni(
},
worker_extension_cls=MyCustomExtension,
# Note: worker_extension_cls is an internal parameter
# CustomPipelineWorkerExtension will automatically init pipeline when custom_pipeline_args is provided
# CustomPipelineWorkerExtension will automatically init pipeline when
# custom_pipeline_args includes "pipeline_class".
)
```
docs/models/supported_models.md
+2 -2
View File
@@ -58,6 +58,7 @@ th {
| `Magi2Pipeline` | MAGI-2 native Preview-stage T2VA and I2VA (272p/540p) | `sand-ai/MAGI-2-preview` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/SandAI/MAGI-2-preview-L20X.md) |
| `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
| `Pi0Pipeline` | π0 (Pi-Zero) VLA | `lerobot/pi0_base` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/lerobot/Pi0.md) |
| `Pi05Pipeline` | π0.5 VLA | `lerobot/pi05_base` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/lerobot/Pi05.md) |
| `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | | — |
| `OvisImagePipeline` | Ovis-Image | `OvisAI/Ovis-Image` | ✅︎ | ✅︎ | | ✅︎ | — |
| `LongcatImagePipeline` | LongCat-Image | `meituan-longcat/LongCat-Image` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
@@ -72,7 +73,6 @@ th {
| `OmniVoiceModel` | OmniVoice | `k2-fsa/OmniVoice` | ✅︎ | | | ✅︎ | — |
| `VoxCPM2TalkerForConditionalGeneration` | VoxCPM2 | `openbmb/VoxCPM2` | ✅︎ | | | ✅︎ | — |
| `BreezeForConditionalGeneration` | Breeze-TTS-2 (TTS, voice clone, voice direction) | `BreezeBlue/Breeze-TTS-2` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/BreezeBlue/Breeze-TTS-2.md) |
| `DotsTTSForConditionalGeneration` | dots.tts | `dots-studio/dots.tts-soar` | ✅︎ | | | | — |
| `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | | — |
| `MammothModa2ForConditionalGeneration` | MammothModa2-Dev (AR-only image understanding) | `bytedance-research/MammothModa2-Dev` | ✅︎ | | | | — |
| `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
@@ -106,12 +106,12 @@ th {
| `HunyuanVideo15ImageToVideoPipeline` | HunyuanVideo-1.5-I2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v` | ✅︎ | ✅︎ | | | — |
| `VoxtralTTSForConditionalGeneration` | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✅︎ | ✅︎ | | | — |
| `CovoAudioForConditionalGeneration` | Covo-Audio-Chat | `tencent/Covo-Audio-Chat` | ✅︎ | | | | — |
| `DyninOmniForConditionalGeneration` | Dynin-Omni | `snu-aidas/Dynin-Omni` | ✅︎ | | | ✅︎ | — |
| `MiniCPMO45OmniForConditionalGeneration` | MiniCPM-o 4.5 | `openbmb/MiniCPM-o-4_5` | ✅︎ | | ✅︎ | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/OpenBMB/MiniCPM-o-4_5.md) |
| `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
| `GepardTalkerForConditionalGeneration` | Gepard-1.0 | `nineninesix/gepard-1.0` | ✅︎ | | | ✅︎ | — |
| `HiDreamImagePipeline` | HiDream-I1-Full | `HiDream-ai/HiDream-I1-Full` | ✅︎ | ✅︎ | | | — |
| `HiDreamO1ImagePipeline` | HiDream-O1-Image | `HiDream-ai/HiDream-O1-Image` | ✅︎ | | | | — |
| `AnimaPipeline` | Anima | `circlestone-labs/Anima` | ✅︎ | ✅︎ | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/circlestone-labs/Anima.md) |
The `Recipe` column links the corresponding deployment recipe. `Published` is
preferred when a corresponding `recipes.vllm.ai` page exists; otherwise the
docs/serving/full_duplex_api.md
+23 -7
View File
@@ -22,14 +22,14 @@ capability gates, see the [Realtime Duplex API guide](realtime_duplex_api.md).
## Enable Full Duplex
A model is served full duplex when its registered pipeline declares a
`duplex_plugin` (the model's `DuplexModelPlugin`). That declaration alone
decides it: `vllm-omni serve` constructs `DuplexOmni` instead of `AsyncOmni`,
and every surface the server exposes is backed by a duplex session. It serves
`duplex_plugin` (the model's `DuplexModelPlugin`) and its deploy configuration
sets `session_mode: duplex`. `vllm serve --omni` constructs `DuplexOmni`
instead of `AsyncOmni`. It serves
`/v1/realtime?duplex=1` (and its alias `/v1/duplex`),
`POST /v1/chat/completions`, `/v1/models` and `/health`; every other
turn-based HTTP route (speech, batch, embeddings, video, ...) answers "not
available". Turn-based use of such a model stays available offline through the
Python API (`Omni` / `AsyncOmni`).
available". Set `session_mode: turn` to select the ordinary online serving
stack instead. The model's supported tasks and endpoint restrictions still apply.
`/v1/chat/completions` is the ordinary chat service running on the duplex
engine: a request is a turn-based generation on the same stages, served
@@ -45,8 +45,24 @@ The deploy configuration of such a model must agree:
session_mode: duplex
```
A duplex model started with a deploy configuration that does not set it fails
at startup rather than falling back to turn-based serving.
A duplex-capable model must explicitly set `session_mode` to `duplex` or `turn`
in its deploy configuration, possibly through `base_config` inheritance.
Missing or invalid values fail at startup. The default MiniCPM-o deployment
continues to use duplex mode.
To run MiniCPM-o 4.5 with the turn-based engine:
```bash
vllm serve openbmb/MiniCPM-o-4_5 --omni \
--deploy-config vllm_omni/deploy/minicpmo_4_5_turn.yaml \
--trust-remote-code \
--port 8091
```
This profile inherits the default model and stage settings and overrides only
`session_mode`. It supports ordinary HTTP requests without creating a duplex
session handler. Mode selection happens at startup; a WebSocket query parameter
does not switch engines. The Python `Omni` / `AsyncOmni` APIs are unchanged.
!!! warning
docs/serving/realtime_duplex_api.md
+56 -33
View File
@@ -8,12 +8,41 @@ history stays honest, and a dropped connection can resume the same session.
This page covers how to run a duplex deployment, how to drive it from Python
with `vllm_omni.clients.duplex.DuplexClient`, and the complete wire contract.
The endpoint is served for models that ship a duplex plugin — currently
MiniCPM-o 4.5 only — and just for deploy configurations that declare
The endpoint is served for models that ship a duplex plugin — including
MiniCPM-o 4.5 and Qwen3-Omni — and just for deploy configurations that declare
`session_mode: duplex`. PersonaPlex and Nemotron VoiceChat are ported to the
plugin contract in follow-up PRs and are not served over this endpoint yet. The runtime architecture
is described in [Full-Duplex Runtime (MiniCPM-o 4.5)](../design/fullduplex.md).
## Qwen3-Omni conversation history
Qwen3-Omni keeps incoming conversation items independently addressable by item
ID. Before applying the model's chat template, it merges consecutive user
items into one multimodal user message. An assistant message, including an
empty message reserved for an interrupted or unheard answer, separates turns.
The final user message places media before text instructions, following the
[official Qwen3-Omni demo's `format_history()`](https://github.com/QwenLM/Qwen3-Omni/blob/e4235853125589c789f06a2dd83e9f4126df5e9d/web_demo.py#L84-L179).
A camera image or text item may arrive after an audio commit and before
`response.create`. Those items belong to the same pending user turn: the
committed audio remains associated with its history item even when that item
is no longer last. Prompt preparation snapshots the selected inputs; later
items do not modify an already prepared request.
The duplex adapter retains its existing budgets: at most four audio payloads
(including the current input), an 8 MiB encoded-audio history budget (keeping
at least the newest payload), and at most eight prompt images. The recent
16-message window is rounded outwards to a complete turn. If any audio in an
older turn is no longer retained, the entire turn's model context, including
its images/text and assistant replies, is omitted. Images from that turn are
therefore **not** permanent session-wide model context. These limits differ
from the demo's one image-bearing turn and five audio-bearing turns; the
shared rule is grouping before pruning, rather than copying its UI budgets.
Grouping and pruning affect only the model-input view. They do not merge or
delete the source conversation items exposed to clients, and do not change
playback acknowledgement or interruption handling.
## Quick Start
### Start the Server
@@ -32,33 +61,19 @@ vllm-omni serve openbmb/MiniCPM-o-4_5 \
mounts `ws://<host>:8099/v1/realtime?duplex=1` (this page; `ws://<host>:8099/v1/duplex`
is an alias of the same route), `POST /v1/chat/completions`, `/v1/models` and
`/health`, and nothing else. The remaining turn-based HTTP routes (speech,
batch, embeddings, video, ...) are not served by a duplex model.
batch, embeddings, video, ...) are not served in duplex mode.
`/v1/chat/completions` is not a turn-based path on a duplex server: each
request opens a short-lived duplex session and reads the answer off it. Audio
content becomes an ordinary committed turn; text is seeded as the session's
opening turn, because a model-native model takes a turn only when it hears
speech. A model that cannot be seeded that way declares so, and a text request
to it is refused with 400. Two consequences are worth sizing for:
`/v1/chat/completions` uses the ordinary chat service on the same engine
when the plugin declares `supports_chat_completions`. HTTP chat requests do
not create duplex sessions or consume session admission slots. Their options
follow the [Chat Completions API](chat_completions_api.md) and the model's
capabilities and endpoint restrictions.
- **Every request holds an admission slot** for its lifetime, so
`duplex_session.max_sessions` caps HTTP concurrency as well as websocket
sessions. Beyond it a request is refused with HTTP 503.
- **Latency follows the model's own clock.** A model-native session generates
per audio unit, so even a short answer costs what that many units cost --
noticeably more than the same model would take turn-based.
`n > 1`, `logprobs`, `tools`, and image or video content parts have no
representation in one duplex turn and are refused with HTTP 400 -- a Realtime
conversation item carries text and audio only. `temperature`,
`max_completion_tokens`, `modalities` and
`chat_template_kwargs.use_tts_template` apply; the other
`chat_template_kwargs` keys are logged as ignored. `extra_body` is passed to
the session unchanged, exactly as a websocket client's session `extra_body`
is, which is how a request asking for `modalities: ["text", "audio"]` supplies
the `ref_audio` MiniCPM-o requires for audio output. A model that should not
serve the route lists it in the deploy config's `endpoint_restrictions`,
exactly as a turn-based model does.
For a turn-based online deployment, use `minicpmo_4_5_turn.yaml` or explicitly
set `session_mode: turn` in a deploy configuration derived from the model's
profile. This selects `AsyncOmni` and the ordinary API initialization instead
of the duplex handler. See [Full Duplex](full_duplex_api.md#enable-full-duplex)
for the command and configuration requirements.
### Run the Example Client
@@ -782,7 +797,10 @@ events with `server_event_seq > 40` are replayed in order)
```
with camera frames (omni video; rides the append that closes a 1 s model
unit; each entry is a **bare** base64 JPEG/PNG — no `data:` URL prefix)
unit; each entry is a **bare** base64 JPEG/PNG — no `data:` URL prefix).
Only a model whose Stage 0 interleaves a frame track at unit boundaries takes
this field; a turn model has no such boundary and rejects it, expecting
`conversation.item.create` with `input_image` content instead.
```json
{
@@ -959,6 +977,10 @@ deferred commit during an active response (`event.response_create_deferred`)
{"type": "conversation.item.truncated", "item_id": "item_resp_01", "content_index": 0, "audio_end_ms": 1850, "event": {"…": "…"}}
```
After this acknowledgement, `conversation.item.retrieve` returns the truncated
transcript. Without text/audio alignment marks, its prefix is estimated from
the requested position and audio duration; this is not exact word alignment.
#### Response lifecycle
`C→S response.create`
@@ -1345,8 +1367,9 @@ them out into typed events before they reach a client.
- Session capacity is bounded by `duplex_session.max_sessions` in the deploy
configuration; admission beyond it fails with `resource_exhausted`
(`rate_limit_error`, retryable).
- A duplex deployment serves the duplex session surfaces only: the websocket
route and `/v1/chat/completions`, the latter on a session per request (see
[Start the Server](#start-the-server)). The other turn-based HTTP routes
report "not available" on a MiniCPM-o 4.5 server; turn-based use of the
model stays available offline through `Omni` / `AsyncOmni`.
- A duplex deployment serves the websocket route and, when supported by the
model, ordinary `/v1/chat/completions` requests on the same engine without a
session per request (see [Start the Server](#start-the-server)). Other
turn-based HTTP routes report "not available" in duplex mode. To use the
ordinary online serving stack, select `session_mode: turn` as described in
[Full Duplex](full_duplex_api.md#enable-full-duplex).
docs/serving/speech_api.md
+9
View File
@@ -521,6 +521,15 @@ are cached in-process with a shared LRU so repeated requests with the same
all TTS model types; deleting a voice invalidates every model-type slot at
once.
Decoded reference waveforms use a separate LRU cache of owned, contiguous
float32 arrays at the source sampling rate. Its byte budget counts numeric
buffers (four bytes per mono sample), excluding cache metadata. Numeric storage
avoids retaining a Python float object for every sample during garbage
collection. MOSS reference encoding consumes these arrays directly; other
list-based interfaces, including MOSS Nano, receive temporary lists that are
not retained in the resolve cache. The MOSS reference encoder also releases
completed batch inputs before waiting for more work.
### Precomputed Custom Voices
Qwen3-TTS Base and VoxCPM2 can load offline-precomputed voices at startup.
docs/serving/streaming_video_output_api.md
+32 -3
View File
@@ -41,7 +41,7 @@ bytes.
| Server to client | `video.start` | Confirms the request ID, format, and accepted configuration |
| Server to client | `video.chunk_metadata` | Describes the next binary media frame |
| Server to client | Binary frame | Carries one fragmented MP4 chunk |
| Client to server | `session.interaction` | Queues an optional prompt update during generation |
| Client to server | `session.interaction` | Queues an optional interaction (e.g., prompt. camera action) during generation |
| Server to client | `session.interaction.queued` | Confirms that an interaction was queued |
| Client to server | `session.ping` | Refreshes the stall clock |
| Server to client | `session.pong` | Acknowledges a ping |
@@ -83,10 +83,39 @@ Models that implement interaction updates can change the active prompt:
}
```
Models that register the camera modality (for example LingBot-World) also
accept structural SE3 payloads on `event.multi_modal_data.camera`. Coordinates
follow Unity semantics (`+X` right, `+Y` up, `+Z` forward; same as LingBot's
native camera frame). WASD key tokens are not admitted by the engine:
```json
{
"type": "session.interaction",
"interaction": {
"event_id": "forward-1",
"event": {
"multi_modal_data": {
"camera": {
"mode": "velocity",
"data": {
"translation": [0.0, 0.0, 0.05],
"rotation": [0.0, 0.0, 0.0, 1.0]
}
}
}
}
}
}
```
`mode` is `target` (lerp to an absolute pose over `transition_chunks`) or
`velocity` (hold a per-frame SE3 delta until replaced). Prompt and camera may
be combined in one interaction event.
This feature is pipeline-dependent. An enabled endpoint does not imply that
the loaded model supports midway prompt updates.
the loaded model supports midway prompt or camera updates.
The default start timeout is 10 seconds and the server reports a stall after
about 60 seconds without generation progress or a `session.ping`. See the
[streaming video generation example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/streaming_video_generation)
for prompt-update scheduling and browser playback.
for prompt/camera-update scheduling and browser playback.
docs/serving/videos_api.md
+65 -4
View File
@@ -73,6 +73,10 @@ curl -L "http://localhost:8091/v1/videos/${video_id}/content" -o output.mp4
| `image_reference` | string | null | JSON-encoded reference image payload; do not combine with `input_reference` or `video_reference` |
| `video_reference` | string | null | JSON-encoded reference video payload; do not combine with `input_reference` or `image_reference` |
| `audio_reference` | string | null | JSON-encoded audio reference for speech-to-video: `{"audio_url": "..."}` — supports HTTP(s) URLs or base64 data URLs |
| `source_video` | file | null | MiniMax H3 latent-edit source video (`.mp4` or `.mov`, up to 512 MiB) |
| `source_audio` | file | null | Optional MiniMax H3 latent-edit source audio (`.wav` or `.mp3`, up to 512 MiB) |
| `video_noise_mask` | file | null | UTF-8 JSON MiniMax H3 video mask; `0` preserves and `1` regenerates a token |
| `audio_noise_mask` | file | null | UTF-8 JSON MiniMax H3 audio mask; `0` preserves and `1` regenerates a token |
| `width` | integer | model default | Output video width |
| `height` | integer | model default | Output video height |
| `num_frames` | integer | 1 | Number of generated frames |
@@ -214,6 +218,49 @@ upstream vLLM's media URL allowlist protection. Deployments should therefore
treat remote media URLs as untrusted and choose this setting as part of their
URL access policy.
### MiniMax H3 Latent-Mask Editing
MiniMax H3 accepts request-scoped source media and video/audio noise masks.
At least one mask is required. A nontrivial `video_noise_mask` requires
`source_video`, while a nontrivial `audio_noise_mask` requires either
`source_audio` or a `source_video` with an audio stream. Mask values are in
`[0, 1]`: `0` preserves the source, `1` regenerates it, and fractional values
blend the two behaviors. Exact all-one masks are no-ops and do not require a
source. Source uploads without a mask are rejected. Masks may be a JSON scalar
or arrays matching the H3 latent/token grid; pixel-resolution masks must be
resized or pooled by the client before upload.
For an aligned output of `F` frames at `W x H`, the video latent grid is
`[Tv, H/16, W/16]`, where `Tv = 2 + 5 * ((F - 5) / 17)`. The video mask may be
a scalar, a flat token vector, `[Tv, H/32, W/32]`, or the full latent grid. For
the model input, timestep, and velocity, a full-grid mask is max-pooled over
each 2x2 spatial token and fractional values are rounded upward to 1/256
levels. The final x0 restore uses the original, unquantized mask, so full-grid
masks retain cell-level preservation inside a model token. The audio length is
`Ta = round(F * 40 / 24)`, and its mask may be a scalar, `[Ta]`, `[2, Ta]`, or
a flat `2 * Ta` vector. If source audio is short, its missing latent tail is
forced to mask value `1` so H3 generates that portion. Each mask must be sent
as a UTF-8 JSON file part and is limited to 8 MiB. A file may contain either a
JSON scalar or an array.
```bash
curl -s http://localhost:8091/v1/videos/sync \
-F "prompt=Partially restyle the complete clip and soundtrack" \
-F 'extra_params={"task":"t2va","duration":4.0,"aspect_ratio":"16:9"}' \
-F "source_video=@source.mp4;type=video/mp4" \
-F "source_audio=@source.wav;type=audio/wav" \
-F "video_noise_mask=@video-mask.json;type=application/json" \
-F "audio_noise_mask=@audio-mask.json;type=application/json" \
-o edited.mp4
```
For example, each mask file in the request above may contain the JSON scalar
`0.5`.
The server streams source files to temporary request-scoped storage and removes
them after synchronous or asynchronous generation finishes. Models that do not
declare latent-mask editing support reject these fields.
### Speech-to-Video
For models that support audio-driven generation (e.g., Wan2.2-S2V), pass both
@@ -251,6 +298,7 @@ These `extra_params` control how the server turns decoded frames into MP4 bytes.
| Field | Type | Default | Description |
| --- | --- | --- | --- |
| `preencode_mp4` | boolean | false | Encode the MP4 on the worker while the VAE is still decoding, instead of after the full video is materialized |
| `preencode_batch_frames` | positive integer | 17 (H3, Wan T2V/I2V); 1 (Wan S2V) | Minimum accumulated frames per worker transfer/encoding batch; used only with `preencode_mp4=true` |
| `video_codec_options` | object | null | Encoder options passed through to the H.264 encoder, such as `{"preset": "ultrafast", "threads": "0"}` |
With `preencode_mp4` enabled, each committed VAE chunk leaves the accelerator and
@@ -261,16 +309,29 @@ unchanged: the same complete MP4, byte-for-byte equivalent frames.
```bash
curl -X POST http://localhost:8091/v1/videos/sync \
-F "prompt=A small robot walking through a neon city" \
-F 'extra_params={"preencode_mp4": true, "video_codec_options": {"preset": "ultrafast"}}' \
-F 'extra_params={"preencode_mp4": true, "preencode_batch_frames": 33, "video_codec_options": {"preset": "ultrafast"}}' \
-o output.mp4
```
Set `preencode_batch_frames` in `extra_params` (or `extra_args` for offline
sampling) to tune batching. The worker accumulates complete VAE chunks until
it has at least this many frames, then transfers and encodes them together.
It always flushes the final partial batch. This is a threshold, not an exact
chunk length: a value of 1 submits every native chunk immediately, and a value
smaller than a native chunk does not split it. Larger values reduce transfers
but retain more frames on the accelerator and delay encoding. The VAE decode
window and output frame count stay unchanged. Wan S2V keeps its existing
per-clip behavior by default. Zero, negative, fractional, boolean, string, and
null values are rejected when pre-encoding is enabled.
`preencode_mp4` applies to the complete-MP4 response paths only. The
`/v1/realtime/video` WebSocket endpoint rejects it, because that path already
overlaps encoding through its own incremental fragmented-MP4 encoder.
overlaps encoding through its own incremental fragmented-MP4 encoder. Wan also
rejects it together with `enable_frame_interpolation`, which needs the decoded
frames the pre-encoded path no longer materializes.
Support is per model: MiniMax-H3 implements it, and other models ignore the flag
and take the full-decode path.
Support is per model: MiniMax-H3 and Wan 2.2 (T2V, I2V, and S2V) implement it,
and other models ignore the flag and take the full-decode path.
## Storage
docs/user_guide/diffusion/attention_backends/rainfusion.md
+32
View File
@@ -78,3 +78,35 @@ Protected video tails require a compatible MindIE-SD release.
For common configuration and selector behavior, see the
[attention backend overview](../attention_backends.md) and the
[backend selection design](../../../design/feature/attention_backend_selection.md).
## Wan2.2 T2V quantization on Ascend
`RAINFUSION_ATTN` accepts `diffusion_kv_cache_dtype: fp8` or `mxfp4` for
Wan2.2 T2V A14B.
MindIE-SD must expose `sparse_attention` with `rf_v3` and an explicit `precision`
argument. Omni passes the selected precision directly to this API; the installed
native operators must support that precision. Unsupported inputs or an
incompatible Python API raise an actionable error; Omni does not change BSA
precision automatically. Update `diffusion_kv_cache_dtype` to a precision
supported by the installed MindIE-SD, or unset it. Native execution errors also
propagate without retry.
BSA FP8 and MXFP4 retain MindIE-SD's default rotation policy.
`diffusion_kv_cache_skip_layers`/`diffusion_kv_cache_skip_steps` explicitly
select floating-point BSA and keep attention sparse. The existing `block_sparse`
warmup/layer exclusions and short-sequence threshold instead select Dense
attention, using the configured Dense precision. Leave `block_sparse.precision`
at its default when using `diffusion_kv_cache_dtype` as the shared Dense/BSA
precision setting.
Configure BSA quantization in the model's existing deployment YAML with
`backend: RAINFUSION_ATTN` and stage-level `diffusion_kv_cache_dtype: fp8` or
`mxfp4`. The precision and skip selectors are siblings of
`diffusion_attention_config`; they are not per-role `quant` fields.
For 40 denoising steps, `diffusion_kv_cache_skip_steps: "0,1,38,39"` and
`diffusion_kv_cache_skip_layers: "0,39"` are recommended starting points. These
fixed step indices do not automatically follow a changed step count. Selected
forwards remain floating-point BSA; unsupported BSA precision is an error rather
than an automatic fallback. See
[quantized attention](../../quantization/quantized_kvcache.md#wan22-t2v-quantized-attention-on-ascend)
for the shared configuration structure and index semantics.
docs/user_guide/diffusion/mot_config.md
+2 -2
View File
@@ -28,7 +28,7 @@ and their optimal Triton tile configurations:
Config loading order (3-tier, see ops/mot_gemm.py):
1. $VLLM_TUNED_CONFIG_FOLDER/<filename> (env override)
2. This directory: vllm_omni/diffusion/layers/mot/configs/<filename>
2. This directory: vllm_omni/diffusion/models/bagel/mot/configs/<filename>
3. Conservative default config (compiles everywhere, sub-optimal perf)
If no config file matches the current device, a warning is printed with
@@ -39,7 +39,7 @@ To generate configs for your hardware:
python benchmarks/kernels/mot_linear_benchmarks.py \
--model ByteDance-Seed/BAGEL-7B-MoT \
--tp-size 1 --dtype w16a16 --tune \
--save-dir vllm_omni/diffusion/layers/mot/configs/
--save-dir vllm_omni/diffusion/models/bagel/mot/configs/
For multi-GPU tuning (uses Ray for parallel search):
docs/user_guide/diffusion/offloader/distributed_layerwise_offload.md
+8 -3
View File
@@ -353,9 +353,14 @@ modes can resolve the same `None` value to different schedules.
Cache-DiT on any selected component. Prompt-embedding cache is also
incompatible when the text encoder uses `allgather`. Rank-local cache hits
could otherwise make ranks enter different weight collectives.
- Per-tensor online FP8 linears use the ordinary loader and can run with either
DiT transfer path. With DiT `allgather`, every rank temporarily materializes
the complete FP8 model in host memory before DLO retains only its shard.
- Per-tensor online FP8, INT8, and MXFP8 linears use the ordinary loader and
can run with either DiT transfer path. With DiT `allgather`, every rank
temporarily materializes the complete quantized model in host memory before
DLO retains only its shard. Layers too wide for the NPU quantization kernel
(`npu_quant_matmul` rejects an output dimension past 65535) stay
unquantized and load straight into host memory under offload-after-quant —
their runtime layout is a plain contiguous bf16 weight, the same layout DLO
shards on the ordinary path, so they are allowed under `allgather` too.
Other online quantization methods require rank-local transfer for the
affected component until their runtime layouts are validated.
- Resident leading layers require DiT `rank-local` transfer and a model
docs/user_guide/examples/online_serving/bagel.md
+14 -8
View File
@@ -247,12 +247,18 @@ curl http://localhost:8091/v1/chat/completions \
-d @payload.json
```
BAGEL img2img derives its generated image size from the input image. It
preserves the input aspect ratio, aligns dimensions to the latent stride, and
applies the checkpoint size limit; explicit `height` and `width` values do not
override this model-specific resize policy. In step mode, the effective size is
resolved before scheduler admission so requests with different resulting
shapes are not combined.
When no output size is requested (`size=auto` on `/v1/images/edits`, or no
`height`/`width` in a chat request), BAGEL img2img derives the canvas from the
input image: it preserves the input aspect ratio, aligns dimensions to the
latent stride (16 px), and applies the checkpoint size limit. An explicit
`size`, `height`, or `width` is honored instead, in both the single-stage and
the two-stage deployment; the input image is still resized to a stride-aligned
size for the VAE/ViT prefill. The latent grid needs multiples of 16, so a
requested side that is not one is floored to the next multiple (1000x700
generates 992x688) and the server logs a warning; the response reports the
generated size. A side above the checkpoint limit is rejected with HTTP 400.
In step mode, the effective size is resolved before scheduler admission so
requests with different resulting shapes are not combined.
### Image to Text (img2text)
@@ -318,8 +324,8 @@ curl http://localhost:8091/v1/chat/completions \
| `--server` / `-s` | `http://localhost:8091` | Server URL |
| `--image-url` / `-i` | `None` | Input image URL or local path (img2img/img2text) |
| `--modality` / `-m` | `text2img` | `text2img`, `img2img`, `img2text`, `text2text` |
| `--height` | `512` | Text-to-image output height; BAGEL img2img derives its size from the input image |
| `--width` | `512` | Text-to-image output width; BAGEL img2img derives its size from the input image |
| `--height` | `512` | Output height, a multiple of 16; omit both `--height` and `--width` to let img2img derive the size from the input image |
| `--width` | `512` | Output width, a multiple of 16; omit both `--height` and `--width` to let img2img derive the size from the input image |
| `--steps` | `25` | Number of inference steps |
| `--seed` | `42` | Random seed |
| `--negative` | `None` | Negative prompt for CFG |
docs/user_guide/examples/online_serving/qwen3_omni.md
+33 -11
View File
@@ -2,7 +2,6 @@
Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/qwen3_omni>.
## 🛠️ Installation
Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/README.md)
@@ -15,6 +14,19 @@ Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
```
For a local deployment with multiple API frontend processes sharing one set
of stage engines, add `--api-server-count`:
```bash
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
--api-server-count 2 --port 8091
```
This mode supports local EngineCore stages with one local process group per
replica, including stages configured with multiple `num_replicas`. Headless,
remote, diffusion, intra-replica data-parallel, and Ray stage deployments
remain single-frontend.
The default deployment configuration situated at `vllm_omni/deploy/qwen3_omni_moe.yaml` is resolved and loaded
automatically via the model registry, obviating the necessity for the `--deploy-config` flag in standard deployment topologies.
The bundled Qwen3-Omni setup explicitly sets `moe_backend: triton` for the Thinker and Talker stages. This keeps
@@ -27,6 +39,7 @@ on the Thinker stage. Keep Talker and Code2Wav at `pipeline_parallel_size: 1`. T
default asynchronous scheduling and audio chunk streaming can remain enabled.
To explicitly utilize a custom deployment YAML, specify the configuration path:
```bash
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
--deploy-config /path/to/deploy_config_file
@@ -50,7 +63,7 @@ Adopt the stage-based CLI architecture to independently instantiate execution pr
The example below pins Stage 0 to GPU 0 and Stage 1/2 to GPU 1 via
`CUDA_VISIBLE_DEVICES`.
**1. Stage 0 (Thinker + API server)**
#### 1. Stage 0 (Thinker + API server)
```bash
CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
@@ -60,7 +73,7 @@ CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
--omni-master-port 26000
```
**2. Stage 1 (Talker)**
#### 2. Stage 1 (Talker)
```bash
CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
@@ -70,7 +83,7 @@ CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
--omni-master-port 26000
```
**3. Stage 2 (Code2Wav)**
#### 3. Stage 2 (Code2Wav)
```bash
CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
@@ -95,6 +108,9 @@ To experiment with another supported MoE backend, set `moe_backend` in the
deployment YAML for both the Thinker and Talker stages. For the stage-based
CLI, pass `--moe-backend` directly on each stage command.
For other stage settings, pass the option on that stage command. For example,
to change its GPU memory utilization:
```bash
CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
--stage-id 1 \
@@ -107,11 +123,12 @@ CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
### Send Multi-modal Request
Get into the example folder
```bash
cd examples/online_serving/qwen3_omni
```
#### Send request via python
#### Send request via python
```bash
python openai_chat_completion_client_for_multimodal_generation.py --query-type use_image --port 8091 --host "localhost"
@@ -126,7 +143,6 @@ The Python client supports the following command-line arguments:
- `--audio-path` (or `-a`): Path to local audio file or URL. If not provided and query-type is `use_audio`, uses default audio URL. Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs and common audio formats: MP3, WAV, OGG, FLAC, M4A. Example: `--audio-path /path/to/audio.wav` or `--audio-path https://example.com/audio.mp3`
- `--prompt` (or `-p`): Custom text prompt/question. If not provided, uses default prompt for the selected query type. Example: `--prompt "What are the main activities shown in this video?"`
For example, to use a local video file with custom prompt:
```bash
@@ -136,19 +152,20 @@ python openai_chat_completion_client_for_multimodal_generation.py \
--prompt "What are the main activities shown in this video?"
```
#### Send request via curl
#### Send request via curl
```bash
bash run_curl_multimodal_generation.sh use_image
```
## Modality control
You can control output modalities to specify which types of output the model should generate. This is useful when you only need text output and want to skip audio generation stages for better performance.
### Supported modalities
| Modalities | Output |
|------------|--------|
| ------------ | -------- |
| `["text"]` | Text only |
| `["audio"]` | Audio only |
| `["text", "audio"]` | Text + Audio |
@@ -226,7 +243,9 @@ print(response.choices[1].message.audio) # Audio response
```
## Streaming Output
If you want to enable streaming output, please set the argument as below. The final output will be obtained just after generated by corresponding stage. We support both text streaming output and audio streaming output. Other modalities can output normally.
```bash
python openai_chat_completion_client_for_multimodal_generation.py \
--query-type use_image \
@@ -259,12 +278,14 @@ The convenience script launches both the vLLM server and Gradio demo together:
```
This script will:
1. Start the vLLM server in the background
2. Wait for the server to be ready
3. Launch the Gradio demo
4. Handle cleanup when you press Ctrl+C
The script supports the following arguments:
- `--model`: Model name/path (default: Qwen/Qwen3-Omni-30B-A3B-Instruct)
- `--server-port`: Port for vLLM server (default: 8091)
- `--gradio-port`: Port for Gradio demo (default: 7861)
@@ -275,18 +296,19 @@ The script supports the following arguments:
#### Option 2: Manual Launch (Two-Step Process)
**Step 1: Launch the vLLM API server**
##### Step 1: Launch the vLLM API server
```bash
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
```
If you have a custom deploy config file:
```bash
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --deploy-config /path/to/deploy_config_file
```
**Step 2: Run the Gradio demo**
##### Step 2: Run the Gradio demo
In a separate terminal:
@@ -299,7 +321,7 @@ Then open `http://localhost:7861/` on your local browser to interact with the we
The gradio script supports the following arguments:
- `--model`: Model name/path (should match the server model)
- `--api-base`: Base URL for the vLLM API server (default: http://localhost:8091/v1)
- `--api-base`: Base URL for the vLLM API server (default: <http://localhost:8091/v1>)
- `--ip`: Host/IP for Gradio server (default: 127.0.0.1)
- `--port`: Port for Gradio server (default: 7861)
- `--share`: Share the Gradio demo publicly (creates a public link)
docs/user_guide/examples/online_serving/text_to_speech.md
+20
View File
@@ -413,6 +413,22 @@ vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni --port 8091
./qwen3_tts/run_server.sh Base
```
For a local deployment with multiple API frontend processes sharing one set
of TTS stage engines, add `--api-server-count`:
```bash
vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
--omni --api-server-count 2 --port 8091
```
This mode supports local EngineCore stages with one local process group per
replica, including stages configured with multiple `num_replicas`. Headless,
remote, intra-replica data-parallel, and Ray stage deployments are not
supported. Runtime voice upload and deletion are disabled with multiple
API frontends because their registries are process-local. Built-in voices,
inline `ref_audio`, and voices restored from `custom_voice_dir` at startup are
supported.
### Executor backend
Single-GPU serves now default to the uniproc executor (lower IPC overhead, the Base cloning use case from [#2603](https://github.com/vllm-project/vllm-omni/issues/2603) / [#2604](https://github.com/vllm-project/vllm-omni/pull/2604)). `vllm_omni/deploy/qwen3_tts.yaml` is the only Qwen3-TTS deploy config; pass `--deploy-config <path>` to override.
@@ -478,6 +494,10 @@ checkpoint. When a request names an uploaded voice, the server infers
`task_type="Base"`. Built-in presets such as `vivian` and `ryan` remain
CustomVoice speakers and require a CustomVoice checkpoint.
The runtime upload and delete routes require a single API frontend; with
`--api-server-count > 1`, use inline `ref_audio` or restore precomputed voices
from `custom_voice_dir` at startup.
### Precomputed custom voices
For reused Base voice-cloning speakers, precompute the reference artifacts once
docs/user_guide/quantization/mxfp4.md
+310 -104
View File
@@ -6,22 +6,25 @@ W4A4 MXFP4 (Microscaling FP4) quantizes both weights and activations to FP4
(`float4_e2m1fn_x2`, packed 2 values per byte) using the OCP MX format: groups
of 32 K-dimension elements share a single `float8_e8m0fnu` exponent scale.
On Ascend, single-scale MXFP4 also supports Universal Optimal Scaling
(UOS, `Qmax=7.25`), described in the single-scale scaling section below.
vLLM-Omni provides two quantization methods with different scale structures:
| Method | Scale structure | Mode | Use case |
|--------|----------------|------|----------|
| `mxfp4` | Single-scale (per-32 fine only) | Online only | Quick accuracy baseline; no checkpoint prep needed |
| `mxfp4_dualscale` | Dual-scale (fine per-32 + coarse per-512 + per-channel `mul_scale`) | Online + Offline | Production; better accuracy; offline recommended |
| -------- | ---------------- | ------ | ---------- |
| `mxfp4` | Single-scale (per-32 fine only) | Online + pre-quantized single-scale weights | Online baseline, or an existing single-scale checkpoint in the layout below |
| `mxfp4_dualscale` | Dual-scale (fine per-32 + coarse per-512 + per-channel `mul_scale`) | Online + Offline | W4A4 with dual-level scales; offline mode loads calibrated Smooth tensors |
!!! tip "Recommended: `mxfp4_dualscale` offline"
For production deployments, use the `mxfp4_dualscale` offline mode with a
!!! tip "Existing W4A4-only DualScale deployments"
For W4A4-only deployments, the existing `mxfp4_dualscale` offline mode uses a
pre-quantized checkpoint produced by msModelSlim. Offline checkpoints load
calibrated `mul_scale` tensors from disk, providing measurably better accuracy
than any online method. The one-time preprocessing cost amortises across all
subsequent inference runs.
calibrated `mul_scale` tensors from disk. Quality depends on the model,
calibration recipe and BF16 fallback policy; evaluate the checkpoint against
your quality target before deployment.
Use `mxfp4` online only for quick experimentation where preprocessing time
is not acceptable and accuracy loss is tolerable.
W4A8 layer/step fallback requires single-scale `mxfp4`, online or offline.
DualScale supports W4A4 and BF16 routing only.
!!! warning "Online single-scale ≠ Offline dual-scale"
`mxfp4_dualscale` offline mode uses `NPUMxfp4DualScaleLinearMethod`:
@@ -34,61 +37,244 @@ vLLM-Omni provides two quantization methods with different scale structures:
## Hardware Support
| Device | Support |
|--------|---------|
| NVIDIA Blackwell GPU (SM 100+) | ⭕ |
| NVIDIA Ada/Hopper GPU (SM 89+) | ⭕ |
| NVIDIA Ampere GPU (SM 80+) | ⭕ |
| AMD ROCm (gfx950 / MI355X) | ✅ |
| Intel XPU | ⭕ |
| Ascend NPU (Atlas 950 A5) | ✅ |
The following matrix describes these vLLM-Omni implementations, rather than
hardware support for the MXFP4 format in general.
Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this guide.
| Device | Single-scale online OCP | Single-scale offline | UOS / W4A8 fallback | DualScale |
| -------- | -------- | -------- | -------- | -------- |
| Ascend NPU (Atlas 950 A5) | ✅ | ✅ | ✅ | ✅ |
| AMD ROCm (gfx950 / MI355X) | ✅ | ❌ | ❌ | ❌ |
| Other AMD architectures | ❌ | ❌ | ❌ | ❌ |
| NVIDIA GPU | ❌ | ❌ | ❌ | ❌ |
| Intel XPU | ❌ | ❌ | ❌ | ❌ |
Legend: `✅` supported by the implementation, `❌` unsupported.
ROCm uses the existing AITER online path and requires `mxfp4_scale_alg=0`.
## Model Type Support
### Diffusion Model (Wan2.2)
| Model | Online | Offline | Notes |
|-------|--------|---------|-------|
| Wan2.2-T2V-A14B | `mxfp4` / `mxfp4_dualscale` | `mxfp4_dualscale` | MoE cascade (`transformer` + `transformer_2`); both transformers quantized with the same config |
| Wan2.2-I2V-A14B | `mxfp4` / `mxfp4_dualscale` | `mxfp4_dualscale` | MoE cascade; same scheme as T2V-A14B |
| Wan2.2-TI2V-5B | ❌ | ❌ | Parameter count too small; W4A4 causes unacceptable accuracy loss |
| ------- | -------- | --------- | ------- |
| Wan2.2-T2V-A14B | `mxfp4` / `mxfp4_dualscale` | `mxfp4` / `mxfp4_dualscale` | Cascade experts resolve checkpoint metadata independently; explicit global runtime policy applies to both |
| Wan2.2-I2V-A14B | `mxfp4` / `mxfp4_dualscale` | `mxfp4` / `mxfp4_dualscale` | Per-expert configuration uses the same interface as T2V; validate I2V checkpoints separately |
| Wan2.2-TI2V-5B | Requires validation | Requires validation | This guide does not establish checkpoint compatibility or generation quality for TI2V-5B |
The choice between `mxfp4` and `mxfp4_dualscale` in **online mode** is about
quantization quality, not model compatibility — both work on cascade (A14B) and
single-transformer models alike, the same as `mxfp8` online:
Both methods can be integrated with single-transformer and cascade models.
This alone does not establish support or quality for every model variant:
- `mxfp4`: single-scale, lower overhead, simpler compute, online only
- `mxfp4_dualscale`: dual-scale + optional BF16 fallback, better accuracy, online **and** offline
- `mxfp4`: single-scale; online and pre-quantized weights, with optional W4A8 fallback on NPU
- `mxfp4_dualscale`: dual-scale; online and offline, with BF16 fallback
**Offline** checkpoints for A14B are always in `mxfp4_dualscale` format (produced
by the merge script); there is no offline `mxfp4` single-scale format.
The existing `merge_mxfp4_dualscale_checkpoint.py` produces **offline
`mxfp4_dualscale`** checkpoints. Single-scale weights must satisfy the separate
layout described below; changing a DualScale checkpoint's method name to
`mxfp4` does not convert its weights. Prepare single-scale checkpoints with
external tooling. Real exported checkpoint compatibility and model quality
require validation of that checkpoint; the synthetic tests do not establish
UOS export or video acceptance.
!!! note "Per-layer BF16 fallback in offline cascade models"
The A14B offline checkpoint uses `quant_method: mxfp4_dualscale`. Most
An A14B offline DualScale checkpoint uses `quant_method: mxfp4_dualscale`. Most
linear layers are stored as W4A4 MXFP4 DualScale; precision-sensitive layers
retain their original BF16 weights and are listed in `ignored_layers` inside
each transformer's `config.json`. The two transformers may have different
`ignored_layers` sets — the pipeline reads each transformer's own `config.json`
and rebuilds the config locally when they differ, so routing is always
per-transformer-accurate.
!!! warning "TI2V-5B not supported"
Wan2.2-TI2V-5B is excluded from W4A4 quantization. Its smaller parameter
count makes it significantly more sensitive to 4-bit quantization noise,
resulting in unacceptable accuracy loss. Use [MXFP8](mxfp8.md) for TI2V-5B.
and resolves the storage format and BF16 layer policy independently.
## Configuration
### W4A8 fallback at selected layers and denoising steps (Ascend NPU)
For single-scale `mxfp4`, set `w4a8_fallback_steps` to a list of zero-based
request denoising loop indices (not scheduler timestep values).
Set `w4a8_fallback_layers` to exact runtime Linear paths, for example
`["blocks.10.attn1.to_qkv", "blocks.12.ffn.net_2"]`. A selected layer uses W4A8
at every step; at a selected step, all other quantized layers also use W4A8.
The rules are combined with **OR**. W4A8 uses MXFP8 activations with MXFP4
weights. Omitting both lists or passing `[]` keeps the existing W4A4 behavior. Existing
`ignored_layers` take priority: **BF16 > (layer OR step) W4A8 > W4A4**.
DualScale's existing leading BF16 block rule is unchanged; it does not enable
W4A8.
```python
from vllm_omni import Omni
omni = Omni(
model="<Wan2.2-T2V-A14B-Diffusers>",
quantization_config={
"method": "mxfp4",
"w4a8_fallback_layers": ["blocks.10.attn1.to_qkv"],
"w4a8_fallback_steps": [0, 1, 38, 39],
},
)
```
Both policies also work with offline single-scale `mxfp4`. Any nonempty
W4A8 list with `mxfp4_dualscale`, online or offline, raises an explicit error
at configuration construction. Empty lists preserve its existing W4A4 path.
Each expert's checkpoint still determines its serialized
format and BF16 layer routing. A serialized checkpoint config is the complete
storage contract: `ignored_layers`, or its `modules_to_not_convert` alias,
selects the BF16 layers, and omitting both means no explicit BF16 layers. A
method-only checkpoint value such as `"mxfp4"` or
`{"quant_method": "mxfp4"}` carries no storage policy and leaves a caller's
online `ignored_layers` unchanged. An explicitly supplied quantization config
retains its W4A8 layer and step lists when the loader rebuilds either expert's
config from disk, including empty lists that disable checkpoint policies.
Within an explicit runtime config, an omitted W4A8 list defaults to `[]`;
individual omitted fields do not inherit saved policies. Without an explicit
runtime config, auto-detection uses the saved checkpoint policies.
Layer names are relative to each expert (`blocks.10...`, without a
`transformer.` or `transformer_2.` prefix). A global config applies the same
list to both experts; per-component `transformer`/`transformer_2` configs can
select different lists. Wan selects these expert names exactly: a `transformer`
entry does not match `transformer_2`; an omitted expert uses `default` if set.
A component mapped to `null` (or an unmatched component
without a default) stays unquantized and requires a BF16 checkpoint; loading a
quantized checkpoint for that disabled component raises an error. A top-level
`quantization_config=None` retains checkpoint auto-detection.
Names are exact, with no regex, wildcard, block expansion,
or checkpoint-name remapping. Wan validates them against its runtime Linear
modules and logs the selected paths/count and BF16 overrides per PP rank.
Use `attn1.to_qkv` for fused self-attention; `attn1.to_q`, `to_k`, or `to_v`
checkpoint names raise an error. Cross-attention `attn2.to_q/to_k/to_v` remains
separate. Layer-only fallback does not require a denoise ForwardContext.
The text-to-video example accepts the JSON directly:
```bash
ASCEND_RT_VISIBLE_DEVICES=0,1,2,3 \
python examples/offline_inference/text_to_video/text_to_video.py \
--model /path/to/Wan2.2-T2V-A14B-Diffusers \
--quantization-config '{"method":"mxfp4","w4a8_fallback_steps":[0,2]}' \
--tensor-parallel-size 1 --ulysses-degree 2 --cfg-parallel-size 2 \
--enforce-eager --height 480 --width 832 --num-frames 81 \
--num-inference-steps 40 --seed 42 --output wan22-w4a8.mp4
```
Use either `--quantization` or `--quantization-config` in this example.
Serving accepts the same JSON through `--diffusion-quantization-config`.
The list uses one request's loop index across both high- and low-noise experts,
and both CFG branches see the same index. A new request starts at step zero.
Indices outside the request's step range are never selected. A pipeline must
publish `ForwardContext.denoise_step_idx`; requesting step fallback without
that context raises an error instead of silently ignoring the list.
Single-scale W4A4 and W4A8 reuse the same prepared `weight` and `weight_scale`.
Online loading quantizes the original BF16/FP16 weight once; offline loading
packs the numeric FP4 values and reshapes the saved E8M0 scale without
requantizing the weight. Only activation precision changes during forward.
There is no DualScale decoding, second W4 representation or `w4a8_weight` /
`w4a8_weight_scale` cache. No weight quantization runs inside the denoising loop.
The mixed FP8/FP4 GEMM uses BF16 output and a matching BF16 row bias for
compatibility with the target A5 runtime. An FP16 caller receives that result
converted back to FP16; its output shape and dtype are preserved.
The single-scale fallback validation target is eager Wan2.2 T2V A14B on A5.
Quality scores, optimal fallback lists, real UOS exports and additional
compile/offload combinations require separate evaluation. Existing W4A4-only
DualScale still needs complete 512-channel input groups; use TP=1 with
sequence/CFG parallelism for its initial multi-card run.
### `mxfp4` — Pre-Quantized Single-Scale Weights (Ascend NPU)
Set the following fields in each transformer's `config.json` when a
single-scale checkpoint uses calibrated Smooth scaling:
```json
{
"quantization_config": {
"quant_method": "mxfp4",
"is_checkpoint_mxfp4_serialized": true,
"require_smooth_scale": true,
"ignored_layers": []
}
}
```
Each quantized Linear must provide tensors under its model parameter name:
| Tensor | Checkpoint layout | Meaning |
| -------- | ------------------- | --------- |
| `weight` | `(N, K)` numeric FP4 values | Packed to FP4 after loading |
| `weight_scale` | `(N, ceil(K / 32))`, `uint8` | Raw E8M0 exponent bytes, required |
| `mul_scale` | `(K,)`, FP32 | Smooth pre-scale, required when declared above |
Both W4A4 and W4A8 quantize `x * mul_scale`. The Smooth tensor is converted
to the layer's activation dtype after loading. Row-parallel layers load the
corresponding input-channel shard. Pre-fused QKV weights must use one shared
Smooth tensor for the fused projection.
For backward compatibility, `require_smooth_scale` defaults to `false`:
an absent tensor uses identity scaling, while a present tensor is still loaded.
Without a declaration, an absent Smooth tensor cannot be distinguished from a
legacy unsmoothed checkpoint. Calibrated checkpoints should declare
`require_smooth_scale: true` so missing or misnamed Smooth tensors fail loading;
incorrect tensor shapes, non-floating tensors, NaN/Inf and nonpositive values
also fail. Values that overflow or underflow to zero in the activation dtype
are rejected during weight processing. The flag requires serialized single-scale
weights and is rejected for online quantization. It does not make DualScale
scales optional or change missing-parameter policies for other backends.
Either the checkpoint or an explicit caller can require Smooth: a `true`
declaration is retained when per-expert metadata rebuilds the configuration,
including when the other source omits the flag or sets it to `false`.
Prepare the canonical numeric FP4 checkpoint before inference using external
tooling. Runtime loading does not parse producer-specific export formats.
Retain the producer recipe to identify the weight-generation algorithm and
Smooth calibration; tensor storage alone does not establish UOS provenance.
### `mxfp4` — Single-Scale Block Scaling (OCP / UOS)
`mxfp4_scale_alg` selects the MXFP4 block-scaling method on NPU.
UOS is a scaling rule, not a separate activation dtype or an activation-only
algorithm. Both choices use FP4 E2M1 values with shared E8M0 scales.
| Mode | Weight preparation | A4 activation quantization | W4A8 fallback |
| --- | --- | --- | --- |
| Online | Selected scaling method, once at loading | Selected scaling method | Same W4; OCP MXFP8 A8 |
| Offline | Load existing W4 and scales unchanged | Selected scaling method | Same W4; OCP MXFP8 A8 |
The available settings are:
| Setting | MXFP4 scaling method | NPU parameters |
| --- | --- | --- |
| `0` (default) | OCP MX | `scale_alg=0` |
| `2` | UOS (`Qmax=7.25`) | `scale_alg=2, dst_type_max=7.25` |
Universal Optimal Scaling (UOS, `Qmax=7.25`) is derived in
[MXAttention: Data-Free Optimal Scaling and Pre-Normalization Quantization for MXFP4 Attention](https://arxiv.org/abs/2607.24377),
Section 4.1. The value `7.25` is the boundary used to select the shared
power-of-two scale; the maximum finite E2M1 value remains `6`.
This option applies UOS to Linear quantization. It does not implement the
paper's full MXAttention pipeline or Pre-Normalization Quantization (PNQ).
Both use FP4 E2M1 with E8M0 scales, `axis=-1`, `block_size=32`,
`round_mode="rint"`. W4A8 always quantizes the activation to
`float8_e4m3fn` with `scale_alg=0` and the same grouping/rounding, without
UOS parameters. A4 and A8 share the prepared W4 and scale. UOS is currently
NPU-only; ROCm retains its existing AITER online algorithm.
An explicit UOS online configuration is
`{"method": "mxfp4", "mxfp4_scale_alg": 2}`. For offline checkpoints the
setting affects **only A4 activations**. Existing weights are never requantized:
a UOS checkpoint can run UOS W4 + UOS A4 or UOS W4 + OCP A4, and both fall back to
UOS W4 + OCP A8. Record the weight-generation algorithm separately from the
runtime activation setting. Smooth remains active with either setting. The
UOS configuration explicitly selects `2`; attention computation is unchanged.
### `mxfp4` — Single-Scale Online Mode
Online mode requires no pre-processing. vLLM-Omni quantizes BF16 weights to
MXFP4 at load time using `npu_dynamic_mx_quant`. A single block scale
(`float8_e8m0fnu`, one per 32 K elements) is computed on the fly; no
calibration `mul_scale` is available. Applies equally to single-transformer
and cascade (A14B) models — both transformers in a cascade receive the same
quantization config automatically.
Online mode quantizes BF16/FP16 weights once at loading and computes activation
scales during forward. Each block of 32 K elements shares one E8M0 scale;
no calibrated `mul_scale` is available. An explicit global quantization config
applies to both cascade experts. Per-component configs can select different
policies; checkpoint auto-detection resolves each expert independently.
```python
from vllm_omni import Omni
@@ -103,7 +289,7 @@ outputs = omni.generate(
```bash
# Single-transformer or cascade model — same command
python text_to_video.py --model <your-model> --quantization mxfp4
python examples/offline_inference/text_to_video/text_to_video.py --model <your-model> --quantization mxfp4
# Online serving
vllm serve <your-model> --omni --quantization mxfp4
@@ -113,19 +299,19 @@ vllm serve <your-model> --omni --quantization mxfp4
Online DualScale mode computes both fine and coarse scales on the fly from BF16
weights using `npu_dynamic_dual_level_mx_quant`. Applies equally to
single-transformer and cascade (A14B) models. Compared to `mxfp4` online,
DualScale provides better quantization accuracy at higher compute cost.
single-transformer and cascade (A14B) models with compatible Linear layers.
Compare quality and performance using the intended model and workload.
The default configuration keeps the leading 5 transformer blocks in BF16
(`num_bf16_fallback_layers=5`). Accuracy evaluation on Wan2.2-A14B shows this
is sufficient to meet quality requirements and is the recommended setting.
(`num_bf16_fallback_layers=5`). This is an implementation default; select and
validate the BF16 policy against your own quality target.
```python
omni = Omni(model="<your-model>", quantization="mxfp4_dualscale")
```
```bash
python text_to_video.py --model <your-model> --quantization mxfp4_dualscale
python examples/offline_inference/text_to_video/text_to_video.py --model <your-model> --quantization mxfp4_dualscale
```
If accuracy debugging identifies additional precision-sensitive layers, they
@@ -136,21 +322,22 @@ omni = Omni(
model="<your-model>",
quantization_config={
"method": "mxfp4_dualscale",
"ignored_layers": ["blocks.10.attn1.to_q"], # explicit per-layer override
"ignored_layers": ["blocks.10.attn1.to_qkv"], # explicit per-layer override
},
)
```
BF16 fallback routing in online mode applies two rules in priority order:
1. **`ignored_layers`** (explicit per-layer override): any layer whose prefix
matches is kept in BF16 regardless of block index.
1. **`ignored_layers`** (explicit per-layer override): a listed runtime Linear
module is kept in BF16 regardless of block index. Use exact module paths,
not arbitrary block prefixes or names ending in `.weight`.
2. **`num_bf16_fallback_layers`** (coarse leading-block rule): the first N
transformer blocks (`blocks.0` … `blocks.N-1`) fall back to BF16. Defaults
to `5` (recommended). Layers outside `blocks.N.*`
(e.g. `condition_embedder`) are always quantized.
to `5`. Layers outside `blocks.N.*` are quantized unless explicitly listed
in `ignored_layers`.
### `mxfp4_dualscale` — DualScale Offline Mode (Recommended)
### `mxfp4_dualscale` — DualScale Offline Mode
Offline mode loads a pre-quantized DualScale checkpoint from msModelSlim. A
preprocessing step converts the raw quantized output to the diffusers format
@@ -169,8 +356,8 @@ inside `config.json`. At runtime, each layer's prefix is matched against
Each quantized linear layer stores four tensors:
| Tensor | Shape | dtype | Description |
|--------|-------|-------|-------------|
| `weight` | `(N, K)` | float8_e4m3fn | FP4 packed (2 values per byte) |
| -------- | ------- | ------- | ------------- |
| `weight` | `(N, K)` | float8_e4m3fn | Numeric FP4 values; packed into two values per byte after loading |
| `weight_scale` | `(N, K//32)` | uint8 | Fine block scale (`float8_e8m0fnu` bit pattern) |
| `weight_dual_scale` | `(N, K//512, 1)` | float32 | Coarse block scale |
| `mul_scale` | `(K,)` | float32 | Per-input-channel smooth pre-scale (from calibration) |
@@ -223,7 +410,7 @@ python vllm_omni/quantization/tools/merge_mxfp4_dualscale_checkpoint.py \
```
| Argument | Description |
|----------|-------------|
| ---------- | ------------- |
| `--model-type` | Model variant: `Wan2.2-T2V-A14B` or `Wan2.2-I2V-A14B` |
| `--original-model` | Root directory of the original BF16 diffusers model |
| `--quant-path` | Root directory of the msModelSlim quantized output |
@@ -251,14 +438,14 @@ The `quantization_config` injected into `config.json` for each transformer:
```
`ignored_layers` lists every linear layer that retains its original BF16 weight,
using vllm-omni model parameter names (QKV-fused, FFN underscored, `to_out`
using vllm-omni runtime module paths (QKV-fused, FFN underscored, `to_out`
unindexed). The exact entries are determined by the quantization tool (msModelSlim)
and may differ between `transformer` and `transformer_2` in a cascade model.
#### Step 3 — Serve
```bash
python text_to_video.py --model /path/to/Wan2.2-T2V-A14B-MXFP4-DualScale
python examples/offline_inference/text_to_video/text_to_video.py --model /path/to/Wan2.2-T2V-A14B-MXFP4-DualScale
# Online serving
vllm serve /path/to/Wan2.2-T2V-A14B-MXFP4-DualScale --omni
@@ -275,25 +462,32 @@ omni = Omni(model="/path/to/Wan2.2-T2V-A14B-MXFP4-DualScale")
## Parameters
### `mxfp4` (single-scale, online only)
### `mxfp4` (single-scale, online + offline)
| Parameter | Type | Default | Description |
|-----------|------|---------|-------------|
| ----------- | ------ | --------- | ------------- |
| `method` | str | — | `"mxfp4"` |
| `ignored_layers` | list[str] | `[]` | Layer prefixes to keep in BF16 |
| `mxfp4_scale_alg` | int | `0` | MXFP4 block scaling: `0` OCP MX, `2` UOS (`Qmax=7.25`, NPU only). Online: W4 and A4; offline: A4 only. A8 remains OCP MXFP8 |
| `is_checkpoint_mxfp4_serialized` | bool | `false` | Load pre-quantized single-scale weights and scales; declared in checkpoint `config.json` |
| `require_smooth_scale` | bool | `false` | Require calibrated Smooth tensors for offline single-scale weights |
| `ignored_layers` | list[str] | `[]` | Exact runtime Linear module paths to keep in BF16 |
| `w4a8_fallback_layers` | list[str] | `[]` | Exact runtime Linear paths that always use A8; NPU only |
| `w4a8_fallback_steps` | list[int] | `[]` | Zero-based denoise steps that use A8; NPU only |
### `mxfp4_dualscale` (dual-scale, online + offline)
| Parameter | Type | Default | Description |
|-----------|------|---------|-------------|
| ----------- | ------ | --------- | ------------- |
| `method` | str | — | `"mxfp4_dualscale"` |
| `is_checkpoint_serialized` | bool | `False` | `True` for offline DualScale checkpoints; auto-set from `config.json` when using the preprocessing script |
| `ignored_layers` | list[str] | `[]` | Layer prefixes to keep in BF16. **Works in both modes**: offline — populated by the merge script for interleaved sensitive layers; online — user-supplied for explicit per-layer precision override |
| `num_bf16_fallback_layers` | int | `5` | **Online mode only**: leading N transformer blocks (`blocks.0` … `blocks.N-1`) kept in BF16. Applied after `ignored_layers`; ignored in offline mode. Default of `5` is the evaluated recommended value for Wan2.2-A14B |
| `ignored_layers` | list[str] | `[]` | Exact runtime Linear module paths to keep in BF16. **Works in both modes**: offline — populated by the merge script for interleaved sensitive layers; online — user-supplied for explicit per-layer precision override |
| `num_bf16_fallback_layers` | int | `5` | **Online mode only**: leading N transformer blocks (`blocks.0` … `blocks.N-1`) kept in BF16. Applied after `ignored_layers`; ignored in offline mode. Validate the default of `5` against the intended model and quality target |
| `w4a8_fallback_layers` | list[str] | `[]` | Must remain empty; DualScale does not support W4A8 fallback |
| `w4a8_fallback_steps` | list[int] | `[]` | Must remain empty; DualScale does not support W4A8 fallback |
#### BF16 fallback priority (online mode)
```
```text
for each linear layer:
if prefix in ignored_layers → BF16 (explicit override, highest priority)
elif block_idx < num_bf16_fallback_layers → BF16 (coarse leading-block rule)
@@ -307,8 +501,9 @@ unless they appear in `ignored_layers`.
1. **Online single-scale (`mxfp4`)** quantizes BF16 weights at load time using
`npu_dynamic_mx_quant` (single-scale). No calibration `mul_scale` is
available — all output partitions receive an identity pre-scale. No offline
checkpoint format exists for this method.
available — all output partitions receive an identity pre-scale. Offline
single-scale checkpoints use the canonical numeric FP4 layout
described above.
2. **Online dual-scale (`mxfp4_dualscale`, `is_checkpoint_serialized=False`)**
quantizes BF16 weights using `npu_dynamic_dual_level_mx_quant` (fine + coarse
@@ -316,7 +511,7 @@ unless they appear in `ignored_layers`.
explicit `ignored_layers` stay in BF16 for accuracy.
3. **Offline dual-scale (`mxfp4_dualscale`, `is_checkpoint_serialized=True`)** —
**recommended for production** — loads four tensors per quantized layer: FP4
loads four tensors per quantized layer: FP4
weight, fine scale (`uint8` reinterpreted as `float8_e8m0fnu`), coarse scale
(`float32`), and per-input-channel `mul_scale` (`float32`). BF16 fallback
layers have no quantization tensors and are routed via `ignored_layers`.
@@ -325,21 +520,21 @@ unless they appear in `ignored_layers`.
layout as `float8_e8m0fnu`) and reinterpreted at load time without a lossy
float32 round-trip.
5. **Cascade model config propagation**: in a cascade model (transformer +
transformer_2), vLLM-Omni reads each transformer's own `config.json` and
rebuilds the quant config locally when `ignored_layers` differs between
transformers, ensuring per-layer routing is accurate for each. The first
transformer's config is propagated to `od_config` so the second transformer
can reuse it as a starting point.
5. **Cascade model configuration**: checkpoint auto-detection resolves each
expert's own `config.json` independently; the first expert's detected config
is not reused as the second expert's policy. An explicit global runtime
config applies to both experts, while per-component configs select policies
by the exact `transformer` / `transformer_2` names. Serialized checkpoint
metadata still determines each expert's storage format and BF16 layers.
6. **Self-attention QKV fusion**: Q, K, V projection weights are fused into a
single `QKVParallelLinear` layer at runtime. `ignored_layers` entries use the
fused name (`attn1.to_qkv`), written automatically by the merge script.
7. W4A4 carries higher quantization noise than W8A8 (16 vs 256 levels). The
DualScale offline method mitigates this with calibrated `mul_scale` smooth
quantization. Use `ignored_layers` and `num_bf16_fallback_layers` to trade
off compression vs. accuracy for precision-sensitive layers.
7. FP4 has fewer representable values than FP8, so evaluate quantization
error and generation quality for the chosen recipe. Offline Smooth scaling
and BF16 fallback can help with sensitive layers.
`num_bf16_fallback_layers` applies only to online DualScale.
## Adapting MXFP4 for a New Model
@@ -350,27 +545,26 @@ a merge script for offline checkpoints.
### Step 1 — Discover runtime layer names
`ignored_layers` entries must match the **runtime parameter names** used inside
vllm-omni, which may differ from the names stored in the diffusers checkpoint.
The canonical source of truth is the model's own `named_parameters()`.
`ignored_layers` entries use exact **runtime Linear module paths**, without
`.weight`. Inspect the actual transformer instance in model initialization or
a developer harness; `Omni` does not expose an `omni.pipeline.transformer` API.
For a transformer instance named `transformer`:
```python
from vllm_omni import Omni
from vllm.model_executor.layers.linear import LinearBase
# Load the model without quantization to inspect parameter names.
omni = Omni(model="/path/to/your-model") # no --quantization flag
for name, _ in omni.pipeline.transformer.named_parameters():
if "weight" in name and "scale" not in name:
for name, module in transformer.named_modules():
if isinstance(module, LinearBase):
print(name)
```
Compare the printed names against the diffusers checkpoint keys
Compare these module paths against the corresponding diffusers checkpoint keys
(`safetensors.safe_open` or `torch.load`) to identify any renames your model
applies. Common patterns that differ in Wan2.2 (and may appear in other
models):
| Diffusers checkpoint name | vllm-omni runtime name | Reason |
|---------------------------|------------------------|--------|
| --------------------------- | ------------------------ | -------- |
| `attn1.to_q`, `attn1.to_k`, `attn1.to_v` | `attn1.to_qkv` | Self-attention Q/K/V fused into `QKVParallelLinear` |
| `ffn.net.0.proj` | `ffn.net_0.proj` | Dots in sub-module names replaced with underscores |
| `ffn.net.2` | `ffn.net_2` | Same underscore rule |
@@ -382,8 +576,9 @@ fused runtime parameters.
!!! warning "Partial QKV fallback is not allowed"
If your model fuses Q, K, V into a single layer, `ignored_layers` must
include **all three or none**. A partial fallback (e.g. `to_q` in BF16 but
`to_k`, `to_v` quantized) cannot be expressed at runtime because they share
name the fused runtime module (for Wan, `attn1.to_qkv`). The source
checkpoint must keep **all three or none** in BF16. A partial fallback
(e.g. `to_q` in BF16 but `to_k`, `to_v` quantized) cannot be expressed at runtime because they share
one `QKVParallelLinear`. The merge script enforces this and raises an error
if only some of the trio appear as non-quantized.
@@ -392,12 +587,13 @@ fused runtime parameters.
#### Online mode
Pass `ignored_layers` directly in the quantization config using the **runtime
names** discovered in Step 1. No code changes to the model are required.
names** discovered in Step 1. This assumes the model already passes its
quantization config and module prefixes to compatible vLLM Linear layers.
```python
omni = Omni(
model="/path/to/your-model",
quantization={
quantization_config={
"method": "mxfp4_dualscale",
"ignored_layers": [
"blocks.0.attn1.to_qkv", # runtime name, not diffusers name
@@ -409,9 +605,14 @@ omni = Omni(
```
```bash
# CLI does not support list-typed ignored_layers directly.
# Use the Python API or set ignored_layers in config.json (offline).
python your_script.py --model /path/to/your-model --quantization mxfp4_dualscale
# The text-to-video example accepts list-valued fields through JSON.
python examples/offline_inference/text_to_video/text_to_video.py \
--model /path/to/your-model \
--quantization-config '{"method":"mxfp4_dualscale","ignored_layers":["blocks.0.attn1.to_qkv"]}'
# Serving accepts the same configuration.
vllm serve /path/to/your-model --omni \
--diffusion-quantization-config '{"method":"mxfp4_dualscale","ignored_layers":["blocks.0.attn1.to_qkv"]}'
```
The `num_bf16_fallback_layers` coarse rule is an alternative to listing layers
@@ -436,9 +637,13 @@ the merge script is correct. The injected block:
}
```
To add a layer manually (e.g. to pin an additional layer to BF16 without
re-running the merge script), edit `config.json` inside the transformer
subfolder. Use runtime names, not diffusers checkpoint names.
Adding an offline BF16 fallback layer requires both the original BF16 weight
from the base model and a matching `ignored_layers` entry. Regenerate the
checkpoint with the revised storage policy, restoring the original weight
(and bias, if present) and removing that layer's quantization tensors.
Editing `config.json` alone does not reconstruct BF16 weights from stored FP4
values. For fused QKV, restore all three projections together. Use runtime
module paths in `ignored_layers`.
### Step 3 — Write a merge script for offline mode
@@ -474,6 +679,7 @@ renaming rules and collect the results.
!!! tip "Validate before serving"
After producing the offline checkpoint, load it without a `--quantization`
flag and verify that vLLM-Omni auto-detects the correct method. Check that
the layer count reported in the startup log matches expectations: quantized
layer count + `ignored_layers` count should equal total linear layer count.
Any mismatch indicates a name-mapping bug in the merge script.
each intended runtime Linear module resolves to the expected quantized or
BF16 method and that every BF16 entry has original BF16 weights. Account
for fused modules and, in online DualScale, the leading-block fallback rule;
the raw length of `ignored_layers` alone is not a reliable layer count.
docs/user_guide/quantization/overview.md
+1 -1
View File
@@ -13,7 +13,7 @@ For the internal architecture and backend extension points, see the
| Mode | Guide | Description | Methods |
| ------ | ------- | ------------- | --------- |
| Online quantization | [Online Quantization](online.md) | vLLM-Omni computes quantized weights and scales while loading the model. | FP8 W8A8, Int8 W8A8, BitsAndBytes W4, MXFP8 W8A8, MXFP4 W4A4 |
| Runtime attention quantization | [Quantized KV Cache](quantized_kvcache.md) | vLLM-Omni dynamically quantizes eligible diffusion Flash Attention tensors during inference. | FP8 FA |
| Runtime attention quantization | [Quantized KV Cache](quantized_kvcache.md) | vLLM-Omni dynamically quantizes eligible diffusion attention tensors during inference. | FP8, MXFP8, MXFP4 |
| Pre-quantized checkpoints | Method-specific guides | The checkpoint or an offline quantizer provides quantized weights and scales before serving. | ModelOpt, AutoRound, TorchAO, msModelSlim, serialized Int8, offline MXFP8, offline MXFP4 DualScale |
## Hardware Support
docs/user_guide/quantization/quantized_kvcache.md
+78 -25
View File
@@ -2,41 +2,36 @@
## Overview
In DiT-based image and video generation, Flash Attention can take a large share
of denoising time, especially for high-resolution or long-frame workloads.
vLLM-Omni supports online FP8 quantization for eligible diffusion Flash
Attention (FA) to reduce FA latency while keeping model weights in their
original dtype.
In DiT-based image and video generation, attention can take a large share of
denoising time, especially for high-resolution or long-frame workloads.
vLLM-Omni supports online Q/K/V quantization for eligible diffusion attention
paths while keeping model weights in their original dtype.
This feature is configured through `diffusion_kv_cache_dtype` on
`OmniDiffusionConfig` (CLI: `--diffusion-kv-cache-dtype`). It is intentionally
**not** the same as vLLM's `--kv-cache-dtype`, which controls autoregressive
language-model KV cache storage and defaults to `"auto"`. Diffusion FA
quantization uses the dedicated diffusion flags so omni serve does not inherit
that default.
Runtime attention precision is configured through `diffusion_kv_cache_dtype`
on `OmniDiffusionConfig` (CLI: `--diffusion-kv-cache-dtype`). This setting is
separate from vLLM's `--kv-cache-dtype`, which controls autoregressive
language-model KV cache storage.
In vLLM-Omni diffusion pipelines, this is a runtime FA path: Q/K/V tensors are
dynamically quantized before the attention operator. It does not quantize model
weights and is separate from [FP8 W8A8](fp8.md), [Int8 W8A8](int8.md), or
pre-quantized checkpoint formats.
If `diffusion_kv_cache_dtype` is not set, behavior is unchanged and attention
runs in the native dtype.
If `diffusion_kv_cache_dtype` is not set, attention runs in the native dtype.
## Hardware Support
| Device | FP8 FA |
|--------|--------|
| Ascend NPU | ✅ |
| NVIDIA GPU | ❌ |
| AMD ROCm | ❌ |
| Intel XPU | ❌ |
| Device | Dense FP8 | Dense MXFP8/MXFP4 | BSA FP8/MXFP4 |
|--------|------------|-------------------|----------------|
| Ascend NPU | ✅ | ✅ | ✅ |
| NVIDIA GPU | ❌ | ❌ | ❌ |
| AMD ROCm | ❌ | ❌ | ❌ |
| Intel XPU | ❌ | ❌ | ❌ |
Legend: `✅` supported, `❌` unsupported.
FP8 FA is currently implemented only for the NPU Flash Attention backend. Other
backends do not support `diffusion_kv_cache_dtype="fp8"` for diffusion attention
and fall back to native dtype execution.
These runtime methods are implemented only by the NPU Flash Attention and BSA
backends. Other backends reject an incompatible explicit configuration.
## Model Type Support
@@ -44,7 +39,7 @@ and fall back to native dtype execution.
| Model | Scope | Status | Notes |
|-------|-------|--------|-------|
| Wan2.2 | Eligible DiT full-attention FA on Ascend NPU | Tested | Compare quality and latency against a BF16 baseline before production use |
| Wan2.2 | Self-attention on Ascend NPU | NPU interface and T2V smoke tested | Compare quality and latency against a BF16 baseline before production use |
| Other diffusion models | Eligible DiT full-attention FA on Ascend NPU | Not tested | You can try `diffusion_kv_cache_dtype="fp8"`; tune `diffusion_kv_cache_skip_steps` and `diffusion_kv_cache_skip_layers` when higher precision is needed |
### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
@@ -105,12 +100,13 @@ The legacy keyword aliases `kv_cache_dtype`, `kv_cache_skip_steps`, and
| Parameter | Type | Default | Description |
|-----------|------|---------|-------------|
| `diffusion_kv_cache_dtype` | str \| None | `None` | Set to `"fp8"` to enable dynamic FP8 FA on supported attention backends |
| `diffusion_kv_cache_dtype` | str \| None | `None` | Runtime attention method. Dense FA supports `fp8`, `mxfp8`, and `mxfp4`; BSA supports `fp8` and `mxfp4` |
| `diffusion_kv_cache_skip_steps` | str \| None | `None` | Denoising step selector to keep in native dtype, for example `"0,1,4-6"` |
| `diffusion_kv_cache_skip_layers` | str \| None | `None` | Transformer layer selector to keep in native dtype, for example `"0-2,10"` |
Selectors use comma-separated integers and inclusive ranges. Listed steps or
layers skip FP8 FA; all other eligible full-attention forwards use the FP8 path.
layers run floating-point attention through the selected Dense or BSA backend;
all other eligible forwards use `diffusion_kv_cache_dtype`.
## Validation and Notes
@@ -123,3 +119,60 @@ layers skip FP8 FA; all other eligible full-attention forwards use the FP8 path.
4. Report both latency and quality results when enabling this option for a new
model. For image or video models, include visual comparison and quantitative
metrics when available, such as PSNR or SSIM.
## Wan2.2 T2V quantized attention on Ascend
Wan2.2 T2V A14B supports quantized self-attention through MindIE-SD. Model
weights are unchanged, and model-level opt-out keeps cross-attention in its
native dtype.
| Attention path | self-attention `backend` | Supported `diffusion_kv_cache_dtype` |
| --- | --- | --- |
| Dense FA | `FLASH_ATTN` | `fp8`, `mxfp8`, `mxfp4` |
| BSA | `RAINFUSION_ATTN` | `fp8`, `mxfp4` |
Configure the method and fallback selectors once at the diffusion stage. The
per-role attention config only selects the self-attention backend:
```yaml
stages:
- stage_id: 0
diffusion_kv_cache_dtype: "mxfp8"
diffusion_kv_cache_skip_layers: "0,39"
diffusion_kv_cache_skip_steps: "0,1,38,39"
diffusion_attention_config:
per_role:
self:
backend: FLASH_ATTN # Use RAINFUSION_ATTN with fp8 or mxfp4 for BSA.
```
For 40 denoising steps, `diffusion_kv_cache_skip_steps: "0,1,38,39"` and
`diffusion_kv_cache_skip_layers: "0,39"` are recommended starting points. Steps
are zero-based across the complete request and do not reset when Wan switches
transformers; layer indices are zero-based and local to each transformer.
Selectors accept comma-separated indices and inclusive ranges such as
`"0,3-5"`. A selected forward uses floating-point attention while preserving
the Dense or BSA path.
Unsupported precision/input combinations raise an error and must be corrected
in the configuration; operator errors are not retried with another precision.
For BSA, change the backend to `RAINFUSION_ATTN` and set
`diffusion_kv_cache_dtype` to `fp8` or `mxfp4`.
See [RainFusion attention](../diffusion/attention_backends/rainfusion.md) for
sparse-path configuration and behavior.
### Requirement and interface test
Until these APIs are available in a MindIE-SD release, use revision
[`8637b5333b0225381b215390fd09a8732e671cc4`](https://gitcode.com/zqxu/MindIE-SD/commit/8637b5333b0225381b215390fd09a8732e671cc4).
Build the Python package, PyTorch plugin, and custom operators from that same
checkout. With CANN sourced, run the NPU interface tests from the Omni checkout:
```bash
python -m pytest tests/platforms/npu/quant/test_kv_quant_npu.py \
-k real_npu -vv -s -o addopts=''
```
A skipped NPU test is not a pass. Validate generated video quality against a
floating-point run with the same prompt, seed, resolution, and step count.
examples/model_display_names.yml
-1
View File
@@ -2,7 +2,6 @@
# The docs generator combines these names with the serving mode for each page H1.
aura_omni: AURA Omni
covo_audio: Covo-Audio-Chat
dynin_omni: Dynin-Omni
lance: Lance
mimo_audio: MiMo-Audio
ming_flash_omni: Ming-flash-omni 2.0
examples/offline_inference/dynin_omni/README.md
-110
View File
@@ -1,110 +0,0 @@
# Dynin-Omni: Offline inference
This folder contains a unified offline inference entrypoint:
- `end2end.py`
## 1. Environment Setup
Run from repository root:
```bash
cd <REPO_ROOT>
```
If needed, install this repo in editable mode:
```bash
pip install -e .
```
## 2. Extra Dependencies (EMOVA)
Install the following packages for EMOVA-related components:
```bash
pip install \
"phonemizer==3.3.0" \
"Unidecode==1.4.0" \
"hydra-core==1.3.2" \
"pytorch-lightning==1.1.0" \
"wget==3.2" \
"wrapt==2.1.1" \
"onnx==1.20.1" \
"frozendict==2.4.7" \
"inflect==7.5.0" \
"braceexpand==0.1.7" \
"webdataset==1.0.2" \
"torch-stft==0.1.4" \
"editdistance==0.8.1"
```
## 3. Hardware and VRAM Requirements
This example uses a 3-stage pipeline on one GPU by default
([`dynin_omni.yaml`](../../../vllm_omni/deploy/dynin_omni.yaml)):
- Stage-0 (`token2text`): `gpu_memory_utilization: 0.5`
- Stage-1 (`token2image`): `gpu_memory_utilization: 0.1`
- Stage-2 (`token2audio`): `gpu_memory_utilization: 0.1`
### Requested GPU Memory Budget from `gpu_memory_utilization`
| Stage | Utilization | A100 80GB | H200 141GB |
| :-- | :-- | :-- | :-- |
| Stage-0 (token2text) | 0.5 | ~40.0 GB | ~70.5 GB |
| Stage-1 (token2image) | 0.1 | ~8.0 GB | ~14.1 GB |
| Stage-2 (token2audio) | 0.1 | ~8.0 GB | ~14.1 GB |
| Total requested budget | 0.7 | ~56.0 GB | ~98.7 GB |
### Observed Runtime Signal (from your log)
- Stage-0 reported: `Model loading took 15.12 GiB memory` (weights footprint signal).
- Stages 1/2 can still add runtime memory depending on task path and backend allocations.
- Keep extra headroom for CUDA/PyTorch overhead and temporary allocations.
### GPU Compatibility
- Confirmed target GPUs for this setup: **NVIDIA H200**, **NVIDIA A100**.
- CI/e2e coverage in this repo also includes CUDA **L4** markers for Dynin tests.
## 4. End2End Run Examples
```bash
# t2t
python <REPO_ROOT>/examples/offline_inference/dynin_omni/end2end.py \
--task t2t --model snu-aidas/Dynin-Omni --text <INSTRUCTION_TEXT>
# i2t
python <REPO_ROOT>/examples/offline_inference/dynin_omni/end2end.py \
--task i2t --model snu-aidas/Dynin-Omni --image <IMAGE_PATH> --text "Please describe this image in detail."
# s2t
python <REPO_ROOT>/examples/offline_inference/dynin_omni/end2end.py \
--task s2t --model snu-aidas/Dynin-Omni --audio <AUDIO_PATH> --text "Transcribe the given audio."
# t2i
python <REPO_ROOT>/examples/offline_inference/dynin_omni/end2end.py \
--task t2i --model snu-aidas/Dynin-Omni --text <INSTRUCTION_TEXT>
# v2t
python <REPO_ROOT>/examples/offline_inference/dynin_omni/end2end.py \
--task v2t --model snu-aidas/Dynin-Omni --video <VIDEO_PATH> --text "Describe this video in detail."
# i2i
python <REPO_ROOT>/examples/offline_inference/dynin_omni/end2end.py \
--task i2i --model snu-aidas/Dynin-Omni --image <IMAGE_PATH> --text <INSTRUCTION_TEXT>
# t2s
python <REPO_ROOT>/examples/offline_inference/dynin_omni/end2end.py \
--task t2s --model snu-aidas/Dynin-Omni --text <INSTRUCTION_TEXT>
```
## 5. Notes
- Outputs are saved under task-specific directories in `/tmp` by default.
- You can override output path with `--output-dir`.
- If you want to force local config resolution, pass `--dynin-config-path <PATH_TO_DYNIN_OMNI_YAML>`.
- If you see the warning
`max_num_batched_tokens (32768) exceeds max_num_seqs * max_model_len (4096)`,
reduce `max_num_batched_tokens` in the deploy config (for example, `4096` in the CI config).
examples/offline_inference/dynin_omni/end2end.py
-1448
View File
File diff suppressed because it is too large. Load diff
examples/offline_inference/text_to_image/README.md
+30 -10
View File
@@ -19,23 +19,23 @@ This folder provides several entrypoints for experimenting with text-to-image di
### Supported Models
| Model | Image Shape | Peak VRAM (GiB) * | Model Weights (GiB) |
| Model | Image Shape | Peak VRAM (GiB) * | Model Weights (GiB) |
| ----- | ----------- | ----------- | ----------------- |
| `Qwen/Qwen-Image` | 1024 x 1024 | 60.0 | 53.7 |
| `Qwen/Qwen-Image-2512` |1024 x 1024 | 60.0 | 53.7 |
| `Qwen/Qwen-Image-2512` | 1024 x 1024 | 60.0 | 53.7 |
| `Tongyi-MAI/Z-Image-Turbo` | 1024 x 1024 | 24.8 | 19.2 |
| `stepfun-ai/NextStep-1.1` | 512 x 512 | 71.8 | 28.1 |
| `meituan-longcat/LongCat-Image` | 1024 x 1024 | 71.2 | 27.3 |
| `AIDC-AI/Ovis-Image-7B` | 1024 x 1024 | 71.8 | 17.1 |
| `OmniGen2/OmniGen2` | 1024 x 1024 | 20.1 | 14.7 |
| `OmniGen2/OmniGen2` | 1024 x 1024 | 20.1 | 14.7 |
| `stabilityai/stable-diffusion-3.5-medium` | 1024 x 1024 | 20.1 | 15.6 |
| `black-forest-labs/FLUX.1-dev` | 1024 x 1024 | 33.9 | 31.4 |
| `black-forest-labs/FLUX.1-schnell` | 1024 x 1024 | 33.9 | 31.4 |
| `black-forest-labs/FLUX.2-klein-4B` | 1024 x 1024 | 72.7 | 14.9 |
| `black-forest-labs/FLUX.2-klein-9B` | 1024 x 1024 | 37.1 | 32.3 |
| `black-forest-labs/FLUX.2-dev` | 1024 x 1024 | 65.7 | >80 (CPU offload required) |
| `HunyuanImage-3.0` | 1024 x 1024 | 80.0 (TP≥3) | 160 |
| `HiDream-I1-Full` | 1024 x 1024 | 63.7 | 57.7 |
| `HunyuanImage-3.0` | 1024 x 1024 | 80.0 (TP≥3) | 160 |
| `HiDream-I1-Full` | 1024 x 1024 | 63.7 | 57.7 |
| `krea/Krea-2-Raw`, `krea/Krea-2-Turbo` | 1024 (Raw) / 2048 (Turbo) | — | ~30 |
!!! info
@@ -112,9 +112,9 @@ python text_to_image.py \
| `--enable-cpu-offload` | flag | off | Enable CPU offloading for diffusion models |
| `--lora-path` | str | — | Path to PEFT LoRA adapter folder or checkpoint file |
| `--lora-scale` | float | `1.0` | Scale factor for LoRA weights |
| `--lora-backend` | str |`"peft"`| LoRA backend for loading LoRA adapters. Default: peft. Available options: peft, distill |
| `--use-system-prompt` | str | `None` | System prompt preset: `en_unified`, `en_vanilla`, `en_recaption`, `en_think_recaption`, `dynamic`, `None`, or custom text. Recommended: `en_unified`. Only for HunyuanImage-3.0.|
| `--system-prompt` | str | `None` | Custom system prompt text. Only used when `--use-system-prompt` is set to `custom`. Only for HunyuanImage-3.0.|
| `--lora-backend` | str | `"peft"` | LoRA backend for loading LoRA adapters. Default: peft. Available options: peft, distill |
| `--use-system-prompt` | str | `None` | System prompt preset: `en_unified`, `en_vanilla`, `en_recaption`, `en_think_recaption`, `dynamic`, `None`, or custom text. Recommended: `en_unified`. Only for HunyuanImage-3.0. |
| `--system-prompt` | str | `None` | Custom system prompt text. Only used when `--use-system-prompt` is set to `custom`. Only for HunyuanImage-3.0. |
| `--auxiliary-text-encoder` | str | `None` | Supplementary auxiliary text encoder parameters model name or path (especially for Hidream-l1-full). |
**NextStep-1.1 specific arguments:**
@@ -127,7 +127,7 @@ python text_to_image.py \
| `--use-norm` | flag | off | Apply layer normalization to sampled tokens |
> If you encounter OOM errors, try using `--vae-use-slicing` and `--vae-use-tiling` to reduce memory usage.
>
> Qwen-Image currently publishes best-effort presets at `1328x1328`, `1664x928`, `928x1664`, `1472x1140`, `1140x1472`, `1584x1056`, and `1056x1584`. Adjust `--height/--width` accordingly for the most reliable outcomes.
## More CLI Examples
@@ -216,6 +216,26 @@ python examples/offline_inference/text_to_image/text_to_image.py \
--output /output.png
```
### Anima Single-File Checkpoints
Anima uses the official model dimensions by default. A checkpoint may provide a JSON file with the same stem (for example, `anima.json` beside `anima.safetensors`) containing `transformer` and `text_conditioner` configuration objects. These override the default dimensions and must match the checkpoint weights. The shared tiny-model tests use this to load small random-weight models. Components in `text_encoder/`, `vae/`, `tokenizer/`, `t5_tokenizer/`, and `scheduler/` beside the checkpoint are discovered automatically.
To load Anima, point `--model` to the single-file checkpoint path, pass the native pipeline class name using `--model-class-name`, and supply the converted components directory using `--custom-pipeline-args`:
```bash
python examples/offline_inference/text_to_image/text_to_image.py \
--model /path/to/models/anima-official/split_files/diffusion_models/anima-base-v1.0.safetensors \
--model-class-name AnimaPipeline \
--custom-pipeline-args '{"components_path": "/path/to/models/anima-components"}' \
--prompt "A cinematic close-up of a glass teapot on a wooden table." \
--seed 42 \
--guidance-scale 4.0 \
--num-inference-steps 50 \
--height 1024 \
--width 1024 \
--output anima_output.png
```
### Krea 2
Krea 2 is published as diffusers-format checkpoints: `krea/Krea-2-Turbo` (few-step distilled) and `krea/Krea-2-Raw`.
@@ -336,7 +356,7 @@ python text_to_image.py \
LoRA adapters must be in PEFT format. A typical adapter directory structure:
```
```text
lora_adapter/
├── adapter_config.json
└── adapter_model.safetensors
examples/offline_inference/text_to_image/text_to_image.py
+27 -4
View File
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import argparse
import functools
@@ -58,6 +58,7 @@ def parse_json_object(value: str, flag_name: str = "argument") -> dict[str, Any]
parse_profiler_config = functools.partial(parse_json_object, flag_name="--profiler-config")
parse_custom_pipeline_args = functools.partial(parse_json_object, flag_name="--custom-pipeline-args")
def build_text_to_image_prompt(prompt: str, negative_prompt: str | None) -> dict[str, Any]:
@@ -379,6 +380,18 @@ def parse_args() -> argparse.Namespace:
default=None,
help="Supplementary auxiliary text encoder parameters model name or path (especially for Hidream-l1-full).",
)
parser.add_argument(
"--model-class-name",
type=str,
default=None,
help="Override the diffusion pipeline class name (e.g. AnimaPipeline).",
)
parser.add_argument(
"--custom-pipeline-args",
type=parse_custom_pipeline_args,
default=None,
help='JSON object passed to native/custom pipelines (e.g. \'{"components_path": "/path"}\').',
)
parser.add_argument(
"--trust-remote-code",
action="store_true",
@@ -558,7 +571,6 @@ def main():
"vae_patch_parallel_size": args.vae_patch_parallel_size,
"enable_expert_parallel": args.enable_expert_parallel,
"enable_cpu_offload": args.enable_cpu_offload,
"mode": "text-to-image",
"log_stats": args.log_stats,
"enable_diffusion_pipeline_profiler": args.enable_diffusion_pipeline_profiler,
"profiler_config": args.profiler_config,
@@ -576,9 +588,13 @@ def main():
omni_kwargs["trust_remote_code"] = True
if args.deploy_config:
omni_kwargs["deploy_config"] = args.deploy_config
if use_nextstep:
if args.model_class_name:
omni_kwargs["model_class_name"] = args.model_class_name
elif use_nextstep:
# NextStep-1.1 requires explicit pipeline class
omni_kwargs["model_class_name"] = "NextStep11Pipeline"
if args.custom_pipeline_args is not None:
omni_kwargs["custom_pipeline_args"] = args.custom_pipeline_args
# Cosmos3 loads its (gated) guardrail models at build time, so the guardrails
# gate is an engine-level config (offline analog of the server's --no-guardrails).
if args.extra_body and "guardrails" in args.extra_body:
@@ -614,6 +630,10 @@ def main():
print(f" LoRA: scale={args.lora_scale}")
if args.deploy_config:
print(f" deploy-config: {args.deploy_config}")
if args.model_class_name:
print(f" Model class name: {args.model_class_name}")
if args.custom_pipeline_args is not None:
print(f" Custom pipeline args: {args.custom_pipeline_args}")
print(f"{'=' * 60}\n")
# Build LoRA request when --lora-path is set
@@ -723,7 +743,10 @@ def main():
# stop-token-ids declaratively from the plain prompt + extra_body, so this
# example stays model-agnostic. Models without one are untouched.
ar_input_builder = get_ar_input_builder(model_class_name)
if ar_input_builder is not None:
# A model can also be deployed with only its diffusion stage. Keep those
# requests on the string-prompt path, as in the single-stage images API.
has_ar_stage = any(not isinstance(params, OmniDiffusionSamplingParams) for params in sampling_params_list)
if ar_input_builder is not None and has_ar_stage:
_apply_ar_stage_inputs(
ar_input_builder,
model=args.model,
examples/offline_inference/text_to_speech/README.md
+73 -27
View File
@@ -13,7 +13,7 @@ list of supported architectures across all modalities, see
## Supported Models
| Model | HuggingFace repo | Stages | Voice cloning | Streaming | Special modes | Sample rate |
|---|---|---|---|---|---|---|
| --- | --- | --- | --- | --- | --- | --- |
| AuK | `tencent/AuK`, `tencent/AuK-Flash` (assembled with `tools/prepare_auk_checkpoint.py`) | 2 (thinker encoder + DiT) | ✓ | — | instruction-driven editing, enhancement, separation; no example script (use `vllm_omni.model_extras.auk`) | 24 kHz |
| Audio8 TTS Preview | `Audio8/Audio8-TTS-Preview-0.6b` | dual-AR | ✓ | ✓ | 11 languages | 44.1 kHz |
| Breeze-TTS-2 | `BreezeBlue/Breeze-TTS-2` | 2 (talker + codec) | ✓ | ✓ (async chunk) | voice design / voice direction (`--instruction`) | 24 kHz |
@@ -27,7 +27,6 @@ list of supported architectures across all modalities, see
| OmniVoice | `k2-fsa/OmniVoice` | 2 (gen + dec) | ✓ | — | voice design, language hint | 24 kHz |
| Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | 2 (talker + code2wav) | ✓ (Base) | ✓ | 3 task variants | 24 kHz |
| VoxCPM2 | `openbmb/VoxCPM2` | single (native AR) | ✓ | ✓ (online) | continuation | 48 kHz |
| dots.tts | `dots-studio/dots.tts-soar` | single (native AR) | — (text-only) | ✓ (online) | no-reference synthesis | 48 kHz |
| IndexTTS-2 | `IndexTeam/IndexTTS-2` | 2 (AR talker + S2Mel DiT + BigVGAN) | ✓ (required) | — | emotion control (`--emo-audio`, `--emo-text`, `--emo-vector`) | 22.05 kHz |
| IndexTTS-2.5 | native `checkpoints/` bundle | 2 (AR talker + EnhancedCodec + S2Mel DiT + BigVGAN) | ✓ (required) | — | multilingual (`--lang`) + emotion control | 22.05 kHz |
| Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | varies | ✓ | ✓ | voice presets | 24 kHz |
@@ -54,32 +53,38 @@ codec: a slow AR transformer predicts one semantic token per codec frame, and a
4-layer fast AR predicts that frame's remaining 9 codebooks.
### Prerequisites
No extra packages: the codec architecture ships in tree and its weights
(`codec.pth`) come with the checkpoint.
### Quick start
```bash
python examples/offline_inference/text_to_speech/audio8_tts/end2end.py \
--text "Welcome to Audio8 TTS, a compact multilingual text to speech model."
```
### Voice cloning
```bash
python examples/offline_inference/text_to_speech/audio8_tts/end2end.py \
--text "Welcome to Audio8 TTS." \
--ref-audio /path/to/reference.wav \
--ref-text "The exact transcript of the reference recording."
```
The reference transcript must match what is actually spoken in the clip;
a mismatch degrades both stability and speaker similarity.
### Streaming
```bash
python examples/offline_inference/text_to_speech/audio8_tts/end2end.py \
--text "Welcome to Audio8 TTS." --streaming
```
### Notes
- Output: 44.1 kHz mono WAV; the codec emits ~21.5 frames/s (2048 samples per frame).
- Context is 2048 packed text+audio positions, so `max_model_len` is 2048.
- Greedy decoding (`temperature: 0`) is degenerate for this checkpoint — it
@@ -92,6 +97,7 @@ python examples/offline_inference/text_to_speech/audio8_tts/end2end.py \
- `--num-prompts 4` smoke-tests concurrency (stage 0 runs `max_num_seqs: 4`).
### Measured behaviour (1x H20, shared GPU)
`--streaming` on a 58-character English sentence: TTFA ~0.74 s (4-frame first
chunk), 7 chunks, 1.55 s wall for 3.16 s of audio (RTF ~0.49). Treat these as
smoke-test magnitudes, not a benchmark.
@@ -103,12 +109,14 @@ smoke-test magnitudes, not a benchmark.
2-stage TTS pipeline (`talker` + `code2wav`) at 24 kHz.
### Prerequisites
```bash
uv pip install -e .
# Includes soundfile, onnxruntime, x-transformers, einops via requirements.
```
Download the model snapshot:
```python
from huggingface_hub import snapshot_download
snapshot_download('FunAudioLLM/Fun-CosyVoice3-0.5B-2512',
@@ -116,15 +124,18 @@ snapshot_download('FunAudioLLM/Fun-CosyVoice3-0.5B-2512',
```
If your downloaded checkpoint lacks `config.json`, add it:
```json
{
"model_type": "cosyvoice3",
"architectures": ["CosyVoice3Model"]
}
```
This is required because `AutoConfig.register("cosyvoice3", CosyVoice3Config)` only registers the class mapping; the loader still reads `model_type` from `config.json` to select the class.
### Quick start
```bash
python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
--model pretrained_models/Fun-CosyVoice3-0.5B \
@@ -132,9 +143,11 @@ python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
```
### Voice cloning
If `--ref-audio` is omitted, the script downloads the upstream
[`zero_shot_prompt.wav`](https://github.com/FunAudioLLM/CosyVoice/blob/main/asset/zero_shot_prompt.wav) from the CosyVoice repo into the current directory.
To use your own clip, pass `--ref-audio /path/to/reference.wav`, and modify `--prompt-text` correspondingly.
```bash
python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
--model pretrained_models/Fun-CosyVoice3-0.5B \
@@ -144,9 +157,11 @@ python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
```
### Streaming
Streaming is enabled by default via `async_chunk: true` in `vllm_omni/deploy/cosyvoice3.yaml`. Pass `--no-async-chunk` on `vllm serve` to switch to the legacy synchronous path.
### Notes
- Stage 0 (`talker`) emits speech tokens; stage 1 (`code2wav`) runs flow matching + HiFiGAN to synthesize waveform.
- Deploy config auto-loads from `vllm_omni/deploy/cosyvoice3.yaml` based on HF `model_type`. Pass `--deploy-config <path>` to override.
@@ -157,6 +172,7 @@ Streaming is enabled by default via `async_chunk: true` in `vllm_omni/deploy/cos
2-stage TTS pipeline (AR + DiT flow-matching) at 24 kHz. Every request requires reference audio and its transcript for zero-shot voice cloning.
### Quick start
```bash
python examples/offline_inference/text_to_speech/glm_tts/end2end.py \
--model zai-org/GLM-TTS \
@@ -167,12 +183,14 @@ python examples/offline_inference/text_to_speech/glm_tts/end2end.py \
```
### Architecture
```text
Text → [Stage 0: AR] → Speech Tokens → [Stage 1: DiT + HiFT] → Audio (24 kHz)
(Llama-based) (32k vocab) (Flow Matching)
```
### Notes
- `--ref-audio` and `--ref-text` are **required** together; GLM-TTS does not support text-only synthesis.
- Reference audio should be 3-10 seconds.
- First run may be slow due to lazy loading of WhisperVQ tokenizer and CampPlus ONNX speaker embedder.
@@ -187,12 +205,14 @@ Text → [Stage 0: AR] → Speech Tokens → [Stage 1: DiT + HiFT] → Audio (24
No extra packages are required; the DAC codec is vendored in vLLM-Omni.
### Quick start
```bash
python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
--text "Hello, this is a test of the Fish Speech text to speech system."
```
### Voice cloning
```bash
python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
--text "Hello, this is a cloned voice." \
@@ -201,14 +221,17 @@ python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
```
### Streaming
```bash
python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
--text "Hello, this is a streaming test." \
--streaming
```
Streaming requires `async_chunk: true` in the deploy config.
### Notes
- Output: 44.1 kHz mono WAV.
- DAC codec weights (`codec.pth`) are loaded lazily from the model directory.
@@ -219,6 +242,7 @@ Streaming requires `async_chunk: true` in the deploy config.
Dense 0.5B two-stage TTS pipeline (`AR + flow` + audio VAE) at 44.1 kHz. The example covers style, IP voice, music-only generation, text-to-audio events, emotion, dialect, zero-shot cloning, podcast, speech+BGM, and speech+environment-sound cases.
### Quick start
```bash
python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
--case style \
@@ -227,6 +251,7 @@ python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
```
### Voice cloning
```bash
python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
--case zero_shot \
@@ -237,6 +262,7 @@ python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
```
### Streaming
```bash
python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
--case basic \
@@ -247,6 +273,7 @@ python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
```
### Notes
- `style`, `ip`, `bgm`, and `tta` do not require reference audio.
- Reference-audio cases use `--ref-audio`; `zero_shot` also requires `--ref-text`.
- `podcast` uses multiple references via `--ref-audio-paths`.
@@ -259,14 +286,17 @@ python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
Standalone talker-only deployment of Ming-flash-omni-2.0 at 44.1 kHz. Voice is controlled through caption fields (`风格` / `IP` / `语速`/`基频`/`音量`) rather than reference audio.
### Prerequisites
The example calls into `vllm_omni.model_executor.models.ming_flash_omni.prompt_utils` for the default prompt and instruction builder; no extra pip install on top of the base vLLM-Omni install.
### Quick start
```bash
python examples/offline_inference/text_to_speech/ming_flash_omni_tts/end2end.py --case style
```
### Cases
```bash
# ASMR-style whisper (caption-driven)
python examples/offline_inference/text_to_speech/ming_flash_omni_tts/end2end.py --case style
@@ -281,6 +311,7 @@ python examples/offline_inference/text_to_speech/ming_flash_omni_tts/end2end.py
Override the default text per case with `--text`, write to a custom path with `--output`.
### Notes
- Talker-only deployment — for the multimodal Ming-flash-omni example, see [`examples/offline_inference/ming_flash_omni/`](../../ming_flash_omni/).
- Deploy config: `vllm_omni/deploy/ming_flash_omni_tts.yaml` (single GPU, `enforce_eager`, `max_num_seqs: 1`).
- Decode defaults from the Ming cookbook: `max_decode_steps=200`, `cfg=2.0`, `sigma=0.25`, `temperature=0.0`, `use_zero_spk_emb=True`.
@@ -294,6 +325,7 @@ Single-stage 0.1B AR LM + MOSS-Audio-Tokenizer-Nano codec at 48 kHz mono (mixed
> **No built-in speaker presets.** `--ref-audio` is required on every call. Default `--mode voice_clone` matches upstream's recommended workflow; `--mode continuation` is exposed for completeness but upstream's continuation-with-prompt path emits very short / near-silent output, so it is rarely useful in practice. Sample reference clips ship in the upstream repo under [`assets/audio/`](https://github.com/OpenMOSS/MOSS-TTS-Nano/tree/main/assets/audio) (e.g. `zh_1.wav`, `en_2.wav`, `jp_2.wav`).
### Quick start
```bash
# Fetch a sample reference clip (one-off, user-scoped cache).
REF_DIR="${XDG_CACHE_HOME:-$HOME/.cache}/moss-tts-nano"
@@ -305,9 +337,11 @@ python examples/offline_inference/text_to_speech/moss_tts_nano/end2end.py \
--text "你好，这是MOSS-TTS-Nano的语音合成演示。" \
--ref-audio "$REF_DIR/zh_1.wav"
```
The first run downloads `OpenMOSS-Team/MOSS-TTS-Nano` and `OpenMOSS-Team/MOSS-Audio-Tokenizer-Nano` from Hugging Face.
### Reproducible runs
```bash
python examples/offline_inference/text_to_speech/moss_tts_nano/end2end.py \
--text "Deterministic test." \
@@ -316,6 +350,7 @@ python examples/offline_inference/text_to_speech/moss_tts_nano/end2end.py \
```
### Notes
- Output: 48 kHz mono WAV (the tokenizer is internally stereo at 48 kHz; the wrapper averages to mono before reaching the engine).
- Deploy config: `vllm_omni/deploy/moss_tts_nano.yaml` (auto-loaded; override with `--deploy-config`).
- Default `--max-new-frames 375` ≈ 14 s of audio; raise for longer outputs.
@@ -329,12 +364,15 @@ python examples/offline_inference/text_to_speech/moss_tts_nano/end2end.py \
Zero-shot multilingual TTS supporting 600+ languages, with three modes (auto / clone / design).
### Prerequisites
```bash
huggingface-cli download k2-fsa/OmniVoice
```
Voice cloning requires `transformers>=5.3.0`. Auto and design modes work with `transformers>=4.57.0`.
### Quick start (auto voice)
```bash
python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
--model k2-fsa/OmniVoice \
@@ -342,6 +380,7 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
```
### Voice cloning
```bash
python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
--model k2-fsa/OmniVoice \
@@ -351,6 +390,7 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
```
### Voice design
```bash
python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
--model k2-fsa/OmniVoice \
@@ -359,6 +399,7 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
```
### Language hint
```bash
python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
--model k2-fsa/OmniVoice \
@@ -367,6 +408,7 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
```
### Seed for Reproducibility
```bash
python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
--model k2-fsa/OmniVoice \
@@ -375,6 +417,7 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
```
### Notes
- Stage 0 (Generator): Qwen3-0.6B with 32-step iterative unmasking.
- Stage 1 (Decoder): HiggsAudioV2 RVQ + DAC at 24 kHz.
@@ -385,13 +428,16 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
3-task-variant TTS with 24 kHz output. Has its own argparse surface (this script does not follow the common `--text` / `--ref-audio` shape).
### Prerequisites
For ROCm builds, replace `onnxruntime` with `onnxruntime-rocm`:
```bash
pip uninstall onnxruntime
pip install onnxruntime-rocm
```
### Task variants
- `CustomVoice`: predefined speaker (speaker ID) with optional style instruction.
- `VoiceDesign`: text + descriptive instruction designs a new voice.
- `Base`: voice cloning from reference audio + transcript.
@@ -415,22 +461,27 @@ python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-ty
```
### Streaming
```bash
python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
--query-type CustomVoice \
--streaming \
--output-dir /tmp/out_stream
```
Streaming requires `async_chunk: true` in the deploy config.
### Word Timestamps
Word-level timestamps are currently a serving-path feature: launch
`vllm-omni serve` with `--forced-aligner` and request `word_timestamps`
(see `examples/online_serving/text_to_speech/README.md`). An offline
example is not provided in this release.
### Batched decoding
The Code2Wav stage supports batched decoding through the SpeechTokenizer. Pass multiple prompts via `--txt-prompts` and set `--batch-size` accordingly. To raise `max_num_seqs` on either stage, create an overlay based on `vllm_omni/deploy/qwen3_tts.yaml` and pass it with `--deploy-config`:
```bash
python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
--query-type CustomVoice \
@@ -438,9 +489,11 @@ python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
--batch-size 4 \
--deploy-config /path/to/qwen3_tts_batched.yaml
```
`--batch-size` must match a CUDA-graph capture size (1, 2, 4, 8, 16…).
### Notes
- Run `--help` for the full argument surface.
- See `qwen3_tts/end2end.py` for the prompt-length-estimation logic the Talker uses.
@@ -451,6 +504,7 @@ python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
Single-stage native AR TTS at 22.05 kHz. Pipeline: `Qwen3.5 backbone → 32 FSQ codebook heads (one frame/step) → NeMo NanoCodec`. Backbone runs under vLLM paged attention; the 32-head sampling + learned embedding feedback ride the native-AR runner hooks. `enforce_eager` (CUDA graph is a perf follow-up).
### Prerequisites
The NanoCodec decoder needs NeMo (NVIDIA Open Model License), installed separately:
```bash
@@ -480,15 +534,18 @@ failure rather than an environment one. The PyTorch sampler it falls back to cos
here.
### Quick start (zero-shot, default voice)
```bash
python examples/offline_inference/text_to_speech/gepard/end2end.py \
--text "Hello, this is a Gepard demo."
```
### Voice cloning
Not yet — PR1 is zero-shot only (the learned `null_prefix` default voice). Reference-audio cloning (`ref_compressor` speaker prefix) is a follow-up PR.
### Notes
- Output: 22.05 kHz mono WAV.
- First run downloads two checkpoints, not one: the model itself, and the NanoCodec decoder it names in `codec_id` (`nvidia/nemo-nano-codec-22khz-1.89kbps-21.5fps` by default). Both are fetched automatically; expect the first startup to be correspondingly slower.
- Offline and online: `examples/offline_inference/text_to_speech/gepard/end2end.py` and `/v1/audio/speech` (see the [online Gepard section](../../online_serving/text_to_speech/README.md#gepard-10)).
@@ -502,6 +559,7 @@ Not yet — PR1 is zero-shot only (the learned `null_prefix` default voice). Ref
Single-stage native AR TTS at 48 kHz. Pipeline: `feat_encoder → MiniCPM4 → FSQ → residual_lm → LocDiT → AudioVAE`.
### Prerequisites
```bash
pip install voxcpm
# or, for a local source checkout:
@@ -509,6 +567,7 @@ export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/voxcpm
```
### Quick start
```bash
python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
--model openbmb/VoxCPM2 \
@@ -516,7 +575,9 @@ python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
```
### Voice cloning
Pass a reference audio for isolated cloning, or both `--ref-audio` + `--ref-text` for prompt continuation:
```bash
python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
--text "Hello, this is a voice clone demo." \
@@ -525,39 +586,16 @@ python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
```
### Streaming
Streaming is exposed through the online OpenAI Speech API (`stream=true`). See [`examples/online_serving/text_to_speech/voxcpm2/gradio_demo.py`](../../online_serving/text_to_speech/voxcpm2/gradio_demo.py) for an AudioWorklet-based gapless streaming player; the offline `end2end.py` script does not expose a streaming path.
### Notes
- Output: 48 kHz mono WAV.
- Deploy config: `vllm_omni/deploy/voxcpm2.yaml` (auto-loaded by HF `model_type`).
---
## dots.tts
Single-stage native AR TTS at 48 kHz (rednote-hilab). Pipeline: `Qwen2.5-1.5B base LM → DiT (10-step Euler flow matching) → patch_encoder AR loopback → AudioVAE (streaming decode)`. Same "vLLM-native base LM + side-path computation" pattern as VoxCPM2, with a plain Qwen2 backbone instead of MiniCPM4 and no FSQ / residual LM stage.
### Quick start
```bash
python examples/offline_inference/text_to_speech/dots_tts/end2end.py \
--model dots-studio/dots.tts-soar \
--text "Hello, this is a test of dots TTS running on vLLM Omni."
```
### Voice cloning
Not wired in this release. Generation is text-only and does not consume reference audio, a named voice, or a precomputed speaker embedding. The CAM++ x-vector speaker encoder weights load, but `end2end.py` has no `--ref-audio`/`--ref-text` flags yet.
### Streaming
The AudioVAE decoder emits incremental audio through its internal streaming path (`init_stream_state` / `stream_step` / `stream_flush`). Online streaming is exposed through the OpenAI-compatible `/v1/audio/speech` endpoint with `stream=true`; see the [online serving guide](../../online_serving/text_to_speech/README.md#dotstts). The offline `end2end.py` script returns the consolidated waveform.
### Notes
- Output: 48 kHz mono WAV.
- Deploy config: `vllm_omni/deploy/dots_tts.yaml` (auto-loaded by HF `model_type`).
- Checkpoints: `dots-studio/dots.tts-soar` is the validated default. `dots.tts-base` shares the same architecture but is unvalidated in this repo. `dots.tts-mf` (MeanFlow, 2-4 step) is not supported yet.
- Known limitation: no CUDA graph capture and no batched side-path computation yet, so concurrent requests do not currently scale (each request's DiT Euler steps run serially). See `recipes/rednote-hilab/dots.tts.md` for details and the roadmap.
---
## IndexTTS-2 and IndexTTS-2.5
Both versions use a 2-stage TTS pipeline and produce 22.05 kHz mono audio. Stage 0 is a GPT AR talker. IndexTTS-2 Stage 1 uses RepCodec semantic embeddings, GPT latent, S2Mel CFM/DiT, and BigVGAN. IndexTTS-2.5 replaces RepCodec with EnhancedCodec and uses the official code-only path (`use_gpt_latent=false`). Every request requires reference audio for zero-shot voice cloning.
@@ -592,6 +630,7 @@ python examples/offline_inference/text_to_speech/indextts2/end2end.py \
```
### Emotion control
```bash
# Emotion from reference audio
python examples/offline_inference/text_to_speech/indextts2/end2end.py \
@@ -616,6 +655,7 @@ python examples/offline_inference/text_to_speech/indextts2/end2end.py \
```
### Notes
- `--ref-audio` is **required** — neither version provides text-only synthesis.
- IndexTTS-2.5 Stage 0 (AR Talker) generates mel codes from text + reference audio using plain vLLM sampling. It does not reproduce the official default `num_beams=3` beam search. For parity comparisons, run upstream with `num_beams=1`; output quality can differ from the official beam-search result.
- IndexTTS-2.5 accepts language codes such as `zh`, `en`, `zhen` (mixed Chinese/English), `ja`, and `yue`. `Mandarin` is a vLLM-Omni convenience alias for `zh`. Japanese (`ja`) uses `fugashi` tokenization and produces audio, but it does not automatically expand numbers, dates, or percentages; write those inputs as readable Japanese text before calling the model. Use `--no-text-normalization` only when the input is already normalized.
@@ -631,12 +671,15 @@ python examples/offline_inference/text_to_speech/indextts2/end2end.py \
Voxtral-4B-TTS (Mistral). Has its own argparse surface; uses voice presets and the `mistral_common` `SpeechRequest` protocol.
### Prerequisites
Latest `mistral_common` with `SpeechRequest` support:
```bash
pip install -e /path/to/mistral-common # or upgrade from PyPI when available
```
### Quick start (voice preset)
```bash
python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
--write-audio --voice cheerful_female \
@@ -645,6 +688,7 @@ python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
```
### Voice cloning (capability gated upstream)
```bash
python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
--write-audio \
@@ -654,12 +698,14 @@ python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
```
### Streaming + concurrency
```bash
python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
--num-prompts 32 --concurrency 8 --streaming --write-audio --voice neutral_female \
--model mistralai/Voxtral-4B-TTS-2603 \
--text "..."
```
Available voice presets are listed on the HF model card (`mistralai/Voxtral-4B-TTS-2603`).
### Notes
examples/offline_inference/text_to_speech/dots_tts/end2end.py
-125
View File
@@ -1,125 +0,0 @@
"""Offline dots.tts inference example (single-stage native AR pipeline).
Uses the single-stage native AR config (vllm_omni/deploy/dots_tts.yaml).
Reference checkpoint: dots-studio/dots.tts-soar.
"""
from __future__ import annotations
import os
import time
from pathlib import Path
import soundfile as sf
import torch
from vllm_omni import Omni
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
REPO_ROOT = Path(__file__).resolve().parents[4]
SAMPLE_RATE = 48_000
def parse_args():
parser = TrackingArgumentParser(description="Offline dots.tts native AR inference")
parser.add_argument(
"--model",
type=str,
default="dots-studio/dots.tts-soar",
help="dots.tts model path or HuggingFace repo ID.",
)
parser.add_argument(
"--text",
type=str,
default="Hello, this is a test of dots TTS running on vLLM Omni.",
help="Text to synthesize.",
)
parser.add_argument(
"--output-dir",
type=str,
default="output_audio",
help="Directory for output WAV files.",
)
parser.add_argument(
"--deploy-config",
type=str,
default=None,
help="Override the deploy config path. If unset, auto-loads "
"vllm_omni/deploy/dots_tts.yaml based on the HF model_type.",
)
return parser.parse_args()
def extract_audio(multimodal_output: dict) -> torch.Tensor:
"""Extract the final complete audio tensor from multimodal output.
The output processor concatenates per-step delta tensors under
``model_outputs``. Falls back to ``audio`` for backwards compat.
"""
audio = multimodal_output.get("model_outputs")
if audio is None:
audio = multimodal_output.get("audio")
if audio is None:
raise ValueError(f"No audio key in multimodal_output: {list(multimodal_output.keys())}")
if isinstance(audio, list):
valid = [torch.as_tensor(a).float().cpu().reshape(-1) for a in audio if a is not None]
if not valid:
raise ValueError("Audio list is empty or all elements are None.")
return torch.cat(valid, dim=0) if len(valid) > 1 else valid[0]
return torch.as_tensor(audio).float().cpu().reshape(-1)
def main():
args = parse_args()
output_dir = Path(args.output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
engine = Omni(
model=args.model,
deploy_config=args.deploy_config,
)
from transformers import AutoTokenizer
from vllm_omni.model_executor.models.dots_tts.dots_tts_prompt import (
build_dots_tts_prompt,
)
tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
prompt = build_dots_tts_prompt(
tokenizer=tokenizer,
text=args.text,
)
print(f"Model : {args.model}")
print(f"Text : {args.text}")
print(f"Output dir : {output_dir}")
print(f"Prompt len : {len(prompt['prompt_token_ids'])} tokens")
t_start = time.perf_counter()
outputs = engine.generate([prompt])
elapsed = time.perf_counter() - t_start
request_output = outputs[0]
mm = request_output.outputs[0].multimodal_output
audio = extract_audio(mm)
duration = audio.numel() / SAMPLE_RATE
rtf = elapsed / duration if duration > 0 else float("inf")
output_path = output_dir / "output.wav"
sf.write(str(output_path), audio.numpy(), SAMPLE_RATE, format="WAV")
print(f"Saved : {output_path}")
print(f"Duration : {duration:.2f}s")
print(f"Inference : {elapsed:.2f}s")
print(f"RTF : {rtf:.3f}")
if __name__ == "__main__":
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
main()
examples/offline_inference/text_to_video/text_to_video.py
+10 -1
View File
@@ -384,13 +384,20 @@ def parse_args() -> argparse.Namespace:
default=None,
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
)
parser.add_argument(
quantization_group = parser.add_mutually_exclusive_group()
quantization_group.add_argument(
"--quantization",
type=str,
default=None,
choices=["fp8", "mxfp8", "mxfp4", "mxfp4_dualscale", "int8"],
help="Quantization method for the transformer. mxfp8: W8A8 MXFP8 (NPU). mxfp4: W4A4 MXFP4 (NPU). mxfp4_dualscale: W4A4 MXFP4 dual-scale + BF16 fallback mixed (NPU). fp8: online FP8 (GPU).",
)
quantization_group.add_argument(
"--quantization-config",
type=json.loads,
default=None,
help='Quantization JSON, e.g. \'{"method":"mxfp4","w4a8_fallback_steps":[0,2]}\'.',
)
# Distributed and parallel execution
parser.add_argument(
@@ -572,6 +579,8 @@ def main():
omni_kwargs["flow_shift"] = args.flow_shift
if args.quantization is not None:
omni_kwargs["quantization"] = args.quantization
if args.quantization_config is not None:
omni_kwargs["quantization_config"] = args.quantization_config
if args.cache_backend is not None:
omni_kwargs["cache_backend"] = args.cache_backend
omni_kwargs["cache_config"] = cache_config
examples/online_serving/dynin_omni/README.md
-97
View File
@@ -1,97 +0,0 @@
# Dynin-Omni: Online serving
## Installation
Please refer to [README.md](../../../README.md).
## Launch the Server
First, find the `transformers_modules` path:
```bash
python - <<'PY'
from transformers.utils.hub import HF_MODULES_CACHE
print(HF_MODULES_CACHE)
PY
```
Then export it for both `PYTHONPATH` and `HF_MODULES_CACHE`:
```bash
export PYTHONPATH=<transformers_modules_path>:$PYTHONPATH
export HF_MODULES_CACHE=<transformers_modules_path>
```
Run from repository root. The 3-stage pipeline is resolved automatically from
the model type (`vllm_omni/deploy/dynin_omni.yaml`), so no config path is
required. Pass `--deploy-config` only to override the defaults:
```bash
vllm-omni serve snu-aidas/Dynin-Omni \
--omni \
--port 8091
```
If `vllm-omni` is not in PATH, run:
```bash
PYTHONPATH="$(pwd)" python -m vllm_omni.entrypoints.cli.main serve snu-aidas/Dynin-Omni \
--omni \
--port 8091
```
Wait until the server logs show both `All stages initialized successfully` and
`Application startup complete.` before sending requests.
## Send Requests via Python Client
Move to the example directory:
```bash
cd examples/online_serving/dynin_omni
```
### Text -> Image
```bash
python openai_chat_completion_client_for_multimodal_generation.py \
--query-type t2i \
--prompt "A realistic indoor living room with natural daylight."
```
### Image -> Image
```bash
python openai_chat_completion_client_for_multimodal_generation.py \
--query-type i2i \
--image-path ../../offline_inference/dynin_omni/data/image/sofa_under_water.jpg \
--prompt "Transform this surreal underwater setting into a realistic indoor living room while preserving the sofa layout."
```
### Text -> Speech
```bash
python openai_chat_completion_client_for_multimodal_generation.py \
--query-type t2s \
--prompt "Hello. This is Dynin-omni."
```
## CLI Arguments
- `--query-type` (`t2i|t2s|i2i`)
- `--model` (default: `snu-aidas/Dynin-Omni`)
- `--host` / `--port` (OpenAI-compatible vLLM endpoint)
- `--prompt` (custom text)
- `--image-path` (required for `i2i`)
- `--modalities` (optional output modalities override)
- `--output-dir` (default: `/tmp/dynin_online_outputs`)
## Notes
- This client currently supports only `t2i`, `t2s`, and `i2i`.
- `t2t` is intentionally not exposed in this online example.
- This example intentionally uses the OpenAI-compatible chat completion endpoint.
- Task routing for non-text outputs relies on Dynin task trigger tokens (`<|t2i|>`, `<|i2i|>`, `<|t2s|>`) injected by the client.
- Outputs are saved under `/tmp/dynin_online_outputs` by default.
- Dynin stage-0 warmup can take a while on first startup; do not send requests before startup completes.
- Dynin itself can execute text-returning tasks such as `t2t`, `s2t`, `i2t`, and `v2t`, but this online serving example currently runs stage-0 in `generation` mode. In that path, the generation worker does not surface the final text as `output.text`, so OpenAI chat responses for those text-output tasks may complete internally but still return empty text.
examples/online_serving/dynin_omni/openai_chat_completion_client_for_multimodal_generation.py
-342
View File
@@ -1,342 +0,0 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
import argparse
import base64
import json
import mimetypes
import os
import time
from pathlib import Path
from typing import Any
DEFAULT_MODEL = "snu-aidas/Dynin-Omni"
DEFAULT_OUTPUT_DIR = "/tmp/dynin_online_outputs"
QUERY_CHOICES = ("t2i", "t2s", "i2i")
DEFAULT_PROMPT_BY_QUERY = {
"t2i": "A high quality detailed living room interior photo.",
"t2s": "Please read this sentence naturally: Hello from Dynin-Omni online serving.",
"i2i": "Transform this image into a realistic indoor living room while preserving layout.",
}
DEFAULT_MODALITIES_BY_QUERY = {
"t2i": ["image"],
"t2s": ["audio"],
"i2i": ["image"],
}
OFFLINE_PARITY_STAGE_COUNT = 3
OFFLINE_PARITY_STAGE_SAMPLING = {
"max_tokens": 1,
"temperature": 0.0,
"top_p": 1.0,
"detokenize": False,
}
def _infer_mime_type(path: Path) -> str:
mime_type, _ = mimetypes.guess_type(str(path))
return mime_type or "application/octet-stream"
def _encode_file_as_data_url(path: Path) -> str:
mime_type = _infer_mime_type(path)
raw = path.read_bytes()
encoded = base64.b64encode(raw).decode("utf-8")
return f"data:{mime_type};base64,{encoded}"
def _to_image_url(path_or_url: str) -> str:
value = str(path_or_url)
if value.startswith(("http://", "https://", "data:image/")):
return value
path = Path(value).expanduser().resolve()
if not path.exists():
raise FileNotFoundError(f"Image file not found: {path}")
return _encode_file_as_data_url(path)
def _build_user_content(query_type: str, prompt: str, image_path: str | None) -> list[dict[str, Any]]:
if query_type == "t2i":
return [{"type": "text", "text": f"<|t2i|> {prompt}"}]
if query_type == "t2s":
return [{"type": "text", "text": f"<|t2s|> {prompt}"}]
if query_type == "i2i":
if not image_path:
raise ValueError("--image-path is required for query type i2i")
return [
{"type": "text", "text": f"<|i2i|> {prompt}"},
{"type": "image_url", "image_url": {"url": _to_image_url(image_path)}},
]
raise ValueError(f"Unsupported query_type: {query_type}")
def _collect_text_from_content(content: Any) -> list[str]:
texts: list[str] = []
if isinstance(content, str):
stripped = content.strip()
if stripped:
texts.append(stripped)
return texts
if isinstance(content, dict):
for key in ("text", "content", "value", "output_text"):
text_value = content.get(key)
if isinstance(text_value, str) and text_value.strip():
texts.append(text_value.strip())
return texts
if isinstance(content, list):
for item in content:
texts.extend(_collect_text_from_content(item))
return texts
content_text = getattr(content, "text", None)
if isinstance(content_text, str) and content_text.strip():
texts.append(content_text.strip())
content_value = getattr(content, "content", None)
if isinstance(content_value, str) and content_value.strip():
texts.append(content_value.strip())
output_text = getattr(content, "output_text", None)
if isinstance(output_text, str) and output_text.strip():
texts.append(output_text.strip())
return texts
def _extract_text_outputs(chat_completion: Any) -> list[str]:
texts: list[str] = []
for choice in getattr(chat_completion, "choices", []) or []:
message = getattr(choice, "message", None)
if message is None:
continue
content = getattr(message, "content", None)
texts.extend(_collect_text_from_content(content))
reasoning_content = getattr(message, "reasoning_content", None)
if isinstance(reasoning_content, str) and reasoning_content.strip():
texts.append(reasoning_content.strip())
choice_text = getattr(choice, "text", None)
if isinstance(choice_text, str) and choice_text.strip():
texts.append(choice_text.strip())
top_level_output_text = getattr(chat_completion, "output_text", None)
if isinstance(top_level_output_text, str) and top_level_output_text.strip():
texts.append(top_level_output_text.strip())
return texts
def _extract_image_data_urls(chat_completion: Any) -> list[str]:
urls: list[str] = []
for choice in getattr(chat_completion, "choices", []) or []:
message = getattr(choice, "message", None)
if message is None:
continue
content = getattr(message, "content", None)
if not isinstance(content, list):
continue
for item in content:
if not isinstance(item, dict):
continue
if item.get("type") != "image_url":
continue
image_url = (item.get("image_url") or {}).get("url")
if isinstance(image_url, str) and image_url.startswith("data:image"):
urls.append(image_url)
return urls
def _extract_audio_payloads(chat_completion: Any) -> list[bytes]:
payloads: list[bytes] = []
for choice in getattr(chat_completion, "choices", []) or []:
message = getattr(choice, "message", None)
if message is None:
continue
message_audio = getattr(message, "audio", None)
if message_audio is None:
continue
data_b64 = getattr(message_audio, "data", None)
if isinstance(data_b64, str) and data_b64:
try:
payloads.append(base64.b64decode(data_b64))
except Exception:
continue
return payloads
def _decode_data_url(data_url: str) -> tuple[bytes, str]:
header, data = data_url.split(",", 1)
mime_type = "image/png"
if ";" in header and ":" in header:
mime_type = header.split(":", 1)[1].split(";", 1)[0]
return base64.b64decode(data), mime_type
def _image_extension_from_mime(mime_type: str) -> str:
if mime_type == "image/jpeg":
return ".jpg"
if mime_type == "image/webp":
return ".webp"
if mime_type == "image/gif":
return ".gif"
return ".png"
def _save_outputs(
*,
query_type: str,
chat_completion: Any,
output_dir: Path,
) -> None:
output_dir.mkdir(parents=True, exist_ok=True)
stamp = time.strftime("%Y%m%d_%H%M%S")
text_outputs = _extract_text_outputs(chat_completion)
image_data_urls = _extract_image_data_urls(chat_completion)
audio_payloads = _extract_audio_payloads(chat_completion)
if text_outputs:
text_path = output_dir / f"{query_type}_{stamp}.txt"
text_path.write_text("\n\n".join(text_outputs) + "\n", encoding="utf-8")
print(f"[dynin-online] text saved: {text_path}")
print(text_outputs[0])
for idx, image_url in enumerate(image_data_urls):
image_bytes, mime_type = _decode_data_url(image_url)
ext = _image_extension_from_mime(mime_type)
image_path = output_dir / f"{query_type}_{stamp}_{idx}{ext}"
image_path.write_bytes(image_bytes)
print(f"[dynin-online] image saved: {image_path}")
for idx, audio_bytes in enumerate(audio_payloads):
audio_path = output_dir / f"{query_type}_{stamp}_{idx}.wav"
audio_path.write_bytes(audio_bytes)
print(f"[dynin-online] audio saved: {audio_path}")
if not text_outputs and not image_data_urls and not audio_payloads:
print("[dynin-online] no output extracted from response")
raw_path = output_dir / f"{query_type}_{stamp}_raw_response.json"
try:
if hasattr(chat_completion, "model_dump_json"):
serialized = chat_completion.model_dump_json(indent=2)
else:
if hasattr(chat_completion, "model_dump"):
raw_payload: Any = chat_completion.model_dump(mode="json")
else:
raw_payload = chat_completion
try:
serialized = json.dumps(raw_payload, ensure_ascii=False, indent=2)
except Exception:
serialized = json.dumps({"repr": repr(raw_payload)}, ensure_ascii=False, indent=2)
raw_path.write_text(serialized + "\n", encoding="utf-8")
print(f"[dynin-online] raw response saved: {raw_path}")
except Exception:
pass
def _build_offline_parity_sampling_params_list() -> list[dict[str, Any]]:
return [dict(OFFLINE_PARITY_STAGE_SAMPLING) for _ in range(OFFLINE_PARITY_STAGE_COUNT)]
def run_request(args: argparse.Namespace) -> None:
from openai import OpenAI
client = OpenAI(
api_key="EMPTY",
base_url=f"http://{args.host}:{args.port}/v1",
)
prompt = args.prompt.strip() if args.prompt else DEFAULT_PROMPT_BY_QUERY[args.query_type]
user_content = _build_user_content(
query_type=args.query_type,
prompt=prompt,
image_path=args.image_path,
)
if args.modalities:
modalities = [item.strip() for item in args.modalities.split(",") if item.strip()]
else:
modalities = DEFAULT_MODALITIES_BY_QUERY[args.query_type]
extra_body = {
"sampling_params_list": _build_offline_parity_sampling_params_list(),
}
chat_completion = client.chat.completions.create(
model=args.model,
messages=[{"role": "user", "content": user_content}],
modalities=modalities,
extra_body=extra_body,
)
_save_outputs(
query_type=args.query_type,
chat_completion=chat_completion,
output_dir=Path(args.output_dir).expanduser(),
)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Dynin-Omni online chat completion client")
parser.add_argument(
"--query-type",
"-q",
type=str,
default="t2i",
choices=QUERY_CHOICES,
help="Dynin query type",
)
parser.add_argument(
"--model",
"-m",
type=str,
default=DEFAULT_MODEL,
help="Model name/path",
)
parser.add_argument(
"--host",
type=str,
default="localhost",
help="Host/IP of the vLLM Omni API server",
)
parser.add_argument(
"--port",
type=int,
default=8091,
help="Port of the vLLM Omni API server",
)
parser.add_argument(
"--prompt",
"-p",
type=str,
default="",
help="Custom prompt text",
)
parser.add_argument(
"--image-path",
"-i",
type=str,
default=None,
help="Image path/URL for i2i",
)
parser.add_argument(
"--modalities",
type=str,
default="",
help="Comma-separated output modalities override (e.g., text,image,audio)",
)
parser.add_argument(
"--output-dir",
"-o",
type=str,
default=DEFAULT_OUTPUT_DIR,
help="Directory to save outputs",
)
return parser.parse_args()
def main() -> None:
args = parse_args()
os.environ.setdefault("HF_HUB_DISABLE_XET", "1")
run_request(args)
if __name__ == "__main__":
main()
examples/online_serving/minicpmo/README.md
+9
View File
@@ -316,3 +316,12 @@ them unless `--allow-invalid-clock` is explicit.
[`examples/offline_inference/minicpmo/`](../../offline_inference/minicpmo/)
- Recipe:
[`recipes/OpenBMB/MiniCPM-o-4_5.md`](../../../recipes/OpenBMB/MiniCPM-o-4_5.md)
### Shared realtime UI implementation
The `python -m examples.online_serving.minicpmo.realtime_web` command remains
available with the same MiniCPM defaults and required `--ref-audio`. Its assets
now live in [the shared realtime UI](../realtime_web/README.md), with a dedicated
`minicpm-native` profile preserving native duplex input, playback ACKs and camera
frames. Qwen3 uses a separate profile in the same shell, supporting manual STT
turns or Server VAD turns with speech interruption.
examples/online_serving/minicpmo/realtime_web/server.py
+14 -176
View File
@@ -1,185 +1,23 @@
"""Static host and same-origin Realtime proxy for the full-duplex demo."""
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
"""Backward-compatible MiniCPM entry point for the shared realtime UI."""
import argparse
import asyncio
import base64
import contextlib
import hashlib
import json
import logging
from pathlib import Path
from urllib.parse import urlencode
from examples.online_serving.realtime_web.server import (
APP_DIR,
STATIC_DIR,
_join_ws_url,
build_app,
)
from examples.online_serving.realtime_web.server import (
main as _main,
)
import uvicorn
import websockets
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from fastapi.responses import HTMLResponse, Response
from fastapi.staticfiles import StaticFiles
logger = logging.getLogger(__name__)
APP_DIR = Path(__file__).parent / "app"
STATIC_DIR = APP_DIR / "static"
def _join_ws_url(base: str, path: str, query: str) -> str:
return base.rstrip("/") + path + (("?" + query) if query else "")
async def _pump_client_to_backend(client: WebSocket, backend) -> None:
try:
while True:
message = await client.receive()
if message["type"] == "websocket.disconnect":
await backend.close()
return
if message.get("text") is not None:
await backend.send(message["text"])
elif message.get("bytes") is not None:
await backend.send(message["bytes"])
except (WebSocketDisconnect, websockets.ConnectionClosed):
with contextlib.suppress(Exception):
await backend.close()
async def _pump_backend_to_client(client: WebSocket, backend) -> None:
try:
async for message in backend:
if isinstance(message, bytes):
await client.send_bytes(message)
else:
await client.send_text(message)
except (WebSocketDisconnect, websockets.ConnectionClosed):
return
except RuntimeError as exc:
if "websocket.send" in str(exc) and "websocket.close" in str(exc):
return
raise
def _expected_proxy_close(exc: BaseException) -> bool:
if isinstance(exc, (WebSocketDisconnect, websockets.ConnectionClosed, asyncio.CancelledError)):
return True
return isinstance(exc, RuntimeError) and "websocket.send" in str(exc) and "websocket.close" in str(exc)
def build_app(
*,
ws_backend: str = "ws://127.0.0.1:8099",
model: str = "openbmb/MiniCPM-o-4_5",
public_realtime_url: str | None = None,
ref_audio: str | None = None,
) -> FastAPI:
app = FastAPI(title="Experimental Full-Duplex Web Demo")
index_path = APP_DIR / "index.html"
app_version_hash = hashlib.sha256()
for asset_path in (
STATIC_DIR / "app.js",
STATIC_DIR / "pcm_worklet.js",
STATIC_DIR / "playback_worklet.js",
):
app_version_hash.update(asset_path.read_bytes())
app_version = app_version_hash.hexdigest()[:12]
ref_audio_uri: str | None = None
if ref_audio:
ref_path = Path(ref_audio)
if not ref_path.is_file():
raise SystemExit(f"--ref-audio not found: {ref_audio}")
encoded = base64.b64encode(ref_path.read_bytes()).decode("ascii")
ref_audio_uri = f"data:audio/wav;base64,{encoded}"
@app.get("/", response_class=HTMLResponse)
def index() -> HTMLResponse:
config = json.dumps(
{
"model": model,
"realtimePath": public_realtime_url or "v1/realtime",
"refAudio": ref_audio_uri,
"appVersion": app_version,
},
ensure_ascii=True,
)
html = (
index_path.read_text(encoding="utf-8")
.replace("__FULL_DUPLEX_CONFIG__", config)
.replace("__FULL_DUPLEX_APP_VERSION__", app_version)
)
return HTMLResponse(html, headers={"Cache-Control": "no-store"})
@app.get("/healthz")
def healthz() -> Response:
return Response(content="ok", media_type="text/plain")
@app.websocket("/v1/realtime")
async def realtime_proxy(websocket: WebSocket) -> None:
await websocket.accept()
query = urlencode(websocket.query_params.multi_items())
backend_url = _join_ws_url(ws_backend, "/v1/realtime", query)
logger.info("Proxying Realtime WebSocket to %s", backend_url)
try:
async with websockets.connect(
backend_url,
max_size=64 * 1024 * 1024,
) as backend:
tasks = {
asyncio.create_task(_pump_client_to_backend(websocket, backend)),
asyncio.create_task(_pump_backend_to_client(websocket, backend)),
}
done, pending = await asyncio.wait(
tasks,
return_when=asyncio.FIRST_COMPLETED,
)
for task in pending:
task.cancel()
await asyncio.gather(*pending, return_exceptions=True)
for result in await asyncio.gather(*done, return_exceptions=True):
if isinstance(result, BaseException) and not _expected_proxy_close(result):
raise result
except (WebSocketDisconnect, websockets.ConnectionClosed):
return
except Exception:
logger.exception("Realtime WebSocket proxy failed")
with contextlib.suppress(Exception):
await websocket.close(code=1011)
app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
return app
__all__ = ["APP_DIR", "STATIC_DIR", "_join_ws_url", "build_app", "main"]
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--host", default="0.0.0.0")
parser.add_argument("--port", type=int, default=7862)
parser.add_argument("--ws-backend", default="ws://127.0.0.1:8099")
parser.add_argument(
"--public-realtime-url",
help="Browser-visible ws:// or wss:// Realtime URL; defaults to the same-origin proxy.",
)
parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5")
parser.add_argument(
"--ref-audio",
required=True,
help=(
"Reference voice wav for TTS voice cloning, e.g. the "
"official MiniCPM-o-Demo assets/ref_audio/ref_minicpm_signature.wav"
),
)
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
uvicorn.run(
build_app(
ws_backend=args.ws_backend,
model=args.model,
public_realtime_url=args.public_realtime_url,
ref_audio=args.ref_audio,
),
host=args.host,
port=args.port,
log_level="info",
)
_main(default_profile="minicpm-native")
if __name__ == "__main__":
examples/online_serving/qwen3_omni/README.md
+69 -15
View File
@@ -14,6 +14,17 @@ Please refer to [README.md](../../../README.md)
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
```
For a local deployment with multiple API frontend processes sharing one set
of stage engines, add `--api-server-count`:
```bash
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --api-server-count 2 --port 8091
```
This mode currently supports local EngineCore stages. It is not available
with headless or remote stage deployments, and pure diffusion stages remain
single-frontend.
The default deployment configuration, situated at `vllm_omni/deploy/qwen3_omni_moe.yaml`, is resolved and loaded
automatically via the model registry, obviating the `--deploy-config` flag in standard deployment topologies.
Asynchronous chunk streaming operates as **enabled by default** within this bundled configuration.
@@ -156,7 +167,7 @@ parser defaults). If you don't pass a flag, the YAML value wins.
> chunked vs end-to-end modes (e.g. qwen3_tts code2wav) dispatch
> automatically based on that bool — no extra flag or variant yaml is
> needed.
>
> ⚠️ **For multi-stage models that share GPUs (qwen3_omni_moe by default
> shares cuda:1 between stages 1 and 2), avoid using global memory flags.**
> A global `--gpu-memory-utilization 0.85` would apply to every stage and
@@ -309,24 +320,17 @@ python openai_realtime_client.py \
Server VAD uses the pinned Silero v6.2 ONNX artifact from `istupakov/silero-vad-onnx` and never downloads model files while processing a session. Make the artifact available in the Hugging Face cache before startup, or configure a local artifact in the deployment YAML:
```yaml
base_config: /path/to/vllm-omni/vllm_omni/deploy/qwen3_omni_duplex.yaml
duplex_session:
server_vad_model_path: /models/silero_vad.onnx
```
Start from the bundled Qwen3-Omni deployment YAML, add the fields above at the top level, and serve that complete configuration:
```bash
cp vllm_omni/deploy/qwen3_omni_moe.yaml /path/to/qwen3_omni_server_vad.yaml
# Edit /path/to/qwen3_omni_server_vad.yaml and add duplex_session.
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct \
--omni \
--port 8091 \
--deploy-config /path/to/qwen3_omni_server_vad.yaml
```
Keep Qwen's default `session_mode: turn`; `duplex_session` enables the Realtime handler without changing scheduler
semantics. Bare `/v1/realtime` selects that handler. The client uses `?duplex=0` only for the existing non-Server-VAD
wire flow; `?duplex=1` remains a supported compatibility alias.
For the engine-owned plugin, use an overlay based on
`vllm_omni/deploy/qwen3_omni_duplex.yaml`, with `duplex_session.server_vad_model_path`
pointing to your local artifact. See the [complete duplex setup](../realtime_web/README.md#qwen3-with-server-vad-automatic-turns).
The default `qwen3_omni_moe.yaml` remains a turn deployment. Duplex mode serves
`/v1/realtime?duplex=1` and `/v1/chat/completions`; the legacy STT handler requires
a turn deployment.
The Python client supports the following command-line arguments:
@@ -573,3 +577,53 @@ The gradio script supports the following arguments:
- `--ip`: Host/IP for Gradio server (default: 127.0.0.1)
- `--port`: Port for Gradio server (default: 7861)
- `--share`: Share the Gradio demo publicly (creates a public link)
## Browser voice call (shared realtime UI)
Run these commands from the repository root with the vLLM-Omni environment activated.
### Without VAD: manually submit each turn
Start the backend:
```bash
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
```
In another terminal, check backend readiness and start the UI:
```bash
curl --fail http://127.0.0.1:8091/health
python -m examples.online_serving.qwen3_omni.realtime_web \
--backend ws://127.0.0.1:8091 --stt --port 7863
```
Open `http://localhost:7863`, start a session, speak, and press **Send turn**.
`--stt` is the default. Each turn uses a separate connection without shared model
history; the conversation shown in the browser is a local log.
### With VAD: automatically submit after silence
Start the backend with a Server VAD deployment overlay and a compatible Silero
ONNX artifact, following the
[complete VAD setup](../realtime_web/README.md#qwen3-with-server-vad-automatic-turns).
After its health check succeeds, start the UI:
```bash
python -m examples.online_serving.qwen3_omni.realtime_web \
--backend ws://127.0.0.1:8091 --vad --port 7863
```
Speak and pause for 500 ms to submit automatically. `--vad` configures the client;
it does not load a VAD model or change the backend deployment by itself.
VAD mode keeps uploading audio during replies and supports speech interruption.
The engine-owned Qwen duplex plugin maintains conversation history and tracks
played replies through playback acknowledgements. Each committed utterance starts
a new generation request; the model does not use native streaming-input KV decoding.
STT mode pauses microphone upload while the reply is generated and played.
In VAD mode, click **Camera** to upload sampled frames with your spoken question;
see [camera setup and limits](../realtime_web/README.md#qwen-vad-camera-input).
STT remains audio-only. To switch back to manual turns, use the default turn
deployment, restart the UI with `--stt`, refresh the page, and reconnect. Stop the existing UI before reusing port 7863.
See the [shared UI guide](../realtime_web/README.md) for HTTPS access and testing.
examples/online_serving/qwen3_omni/openai_realtime_client.py
+2 -1
View File
@@ -130,7 +130,7 @@ async def run_client(
"prefix_padding_ms": 300,
"silence_duration_ms": 500,
"create_response": True,
"interrupt_response": False,
"interrupt_response": True,
},
}
},
@@ -233,6 +233,7 @@ async def run_client(
continue
if event_type == "response.done" and server_vad:
await ws.send(json.dumps({"type": "session.close"}))
break
if event_type == "error":
examples/online_serving/qwen3_omni/realtime_web/__init__.py
+4
View File
@@ -0,0 +1,4 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Qwen3-Omni turn-based browser call UI."""
examples/online_serving/qwen3_omni/realtime_web/__main__.py
+7
View File
@@ -0,0 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from examples.online_serving.realtime_web.server import main
if __name__ == "__main__":
main(default_profile="qwen3-turn")
examples/online_serving/realtime_web/README.md
+221
View File
@@ -0,0 +1,221 @@
# Shared Realtime Web UI
One browser shell provides microphone capture, speaker playback, a level meter,
conversation text, and an event log. Model profiles own the wire protocol.
This implements [RFC #7222](https://github.com/vllm-project/vllm-omni/issues/7222).
| Profile | Turn control | Camera | Playback acknowledgement |
| --- | --- | --- | --- |
| `minicpm-native` | Model-controlled listen/speak, continuous audio input | Frames accompany audio | Yes |
| `qwen3-turn --stt` (default) | User presses **Send turn** | No | No |
| `qwen3-turn --vad` | Server detects trailing silence; speech interrupts replies | Sampled frames with each spoken turn | Yes |
Qwen3 VAD uses an engine-owned duplex plugin: microphone upload continues
while replies stream, and speech can interrupt generation and playback. The
checkpoint still generates committed turns; this is not native streaming-input
KV decoding. STT remains explicit-turn and audio-only.
## MiniCPM: existing command stays valid
Start your MiniCPM backend using the [existing deployment instructions](../minicpmo/README.md), then:
```bash
python -m examples.online_serving.minicpmo.realtime_web \
--ws-backend ws://127.0.0.1:8099 \
--ref-audio /path/to/ref_minicpm_signature.wav
```
The compatibility wrapper serves the shared assets with `minicpm-native`.
The current native query, `extra_body`, reference voice, continuous microphone
upload, camera frames and playback acknowledgements are retained.
## Qwen3: explicit-turn STT adapter
Start the backend, then the UI in a second terminal:
```bash
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
python -m examples.online_serving.qwen3_omni.realtime_web \
--backend ws://127.0.0.1:8091 --stt --port 7863
```
Open `http://localhost:7863`, start a session, speak, and press **Send turn**.
Wait for the answer to finish playing before speaking again. The UI opens a
fresh STT connection for each turn; the displayed conversation is a local log,
and **previous turns are not provided as model history**. The STT endpoint
accepts only model selection, so system-prompt controls are hidden in this mode.
The adapter follows the shipped `qwen3_omni/openai_realtime_client.py` and
`vllm_omni/entrypoints/openai/realtime_connection.py`:
1. Explicit `duplex=0` selects the legacy STT handler on a turn deployment.
It does not enable STT on a duplex deployment.
2. Send `{type: "session.update", model: ...}` and `commit(final=false)`.
3. Stream mono PCM16 at 16 kHz; **Send turn** flushes buffered audio, then sends `commit(final=true)`.
4. `response.output_audio.delta.audio` contains PCM; `response.output_audio.done`
terminates this adapter, including a response with no audio.
5. The local implementation emits **model output** on `transcription.*`, so the
UI displays it as assistant text. This differs from the early RFC's suggested
user-transcript mapping. The VAD adapter also supports `response.output_text.*`
and audio-transcript events without duplicating the answer.
## Qwen3 with Server VAD (automatic turns)
Run commands from the repository root with the vLLM-Omni environment activated.
Prepare the pinned Silero v6.2 ONNX artifact as described in the
[Qwen3 Server VAD instructions](../qwen3_omni/README.md#realtime-websocket-client-openai_realtime_clientpy).
The backend requires ONNX Runtime and a compatible local artifact; setting
`--vad` on the UI alone does not enable VAD on the backend.
Create an overlay named `qwen3_vad.yaml`, replacing both paths with absolute paths
on the backend host:
```yaml
base_config: /path/to/vllm-omni/vllm_omni/deploy/qwen3_omni_duplex.yaml
duplex_session:
server_vad_model_path: /path/to/silero_vad.onnx
```
The base configuration enables duplex mode and sets the per-prompt audio and image limits.
Start the backend:
```bash
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
--deploy-config qwen3_vad.yaml
```
In a second terminal, wait until the health check succeeds, then start the UI:
```bash
curl --fail http://127.0.0.1:8091/health
python -m examples.online_serving.qwen3_omni.realtime_web \
--backend ws://127.0.0.1:8091 --vad --port 7863
```
Open `http://localhost:7863` and start a session. Speak and pause: the server
commits the turn automatically after 500 ms of silence. No **Send turn** action
is needed. Defaults are a speech threshold of 0.5 and 300 ms of prefix padding.
Speak again to interrupt an answer. The engine owns input buffering, VAD,
request cancellation, playback acknowledgements, and conversation history.
Each committed utterance starts a normal Thinker → Talker → Code2Wav request;
it does not append input to a live Qwen KV cache. The capability response reports
`implementation_level: turn_based_duplex` and `supports_core_resumable_request: false`.
The plugin retains up to four audio inputs (including the current turn), pruning
older audio at an 8 MiB base64 budget; one current utterance is bounded by the
engine's pending-input limit. The UI reports playback progress both on normal
drain and on interruption. Qwen does not provide text/audio alignment, so assistant
text enters history only after the complete audio response has been generated and
acknowledged as played. A partially played answer contributes no guessed text
prefix. Image conversation items remain available until deleted. Model context limits still apply.
The bundled duplex deploy allows four audio inputs and eight images per prompt.
Use headphones to avoid speaker audio triggering VAD interruption. Reference
voices and tool calls are not supported by this plugin.
### Switching between VAD and manual turns
| Mode | Backend | UI flag | Submit a turn |
| --- | --- | --- | --- |
| Without VAD | Default Qwen turn deployment | `--stt` (default) | Press **Send turn** |
| With VAD | Server VAD configuration and Silero artifact | `--vad` | Pause after speaking |
Stop the existing UI process before starting another on the same port, then
refresh the browser and reconnect. To use STT, start a backend with the default
turn deployment and restart the UI with `--stt`. The duplex backend exposes
Realtime duplex and ChatCompletion; it does not expose the legacy STT handler.
To enable VAD on a default backend, first restart that backend with the overlay.
Make `--backend` match the actual backend port; the UI and backend use separate
ports. `Connection refused` means the target backend is unavailable: check its
startup log and `/health` before connecting.
VAD connects with `duplex=1` and sends nested `session.audio.input` configuration
with `create_response: true` and `interrupt_response: true`. It sends playback
ACKs when speaker audio drains. `response.audio.done` / `response.output_audio.done`
only drain playback; `response.done` terminates a turn. Backpressure displays a
notice without pausing the microphone. Failed handshakes time out after 15 seconds;
Silero initialization failures display a specific error.
After updating the backend Python code, restart the backend, refresh the UI, and
start a new session so the plugin and browser assets use the updated code. A
successful `/health` check confirms readiness, not which deployment mode is active.
## Qwen VAD camera input
After starting a session with `--vad`, click **Camera** and allow access. The UI
sends one JPEG per second, resized to a longest side of 448 pixels, independently
of microphone uploads (including while muted). Each frame is a user conversation
item using the [OpenAI Realtime image input](https://developers.openai.com/api/docs/guides/realtime-conversations#image-inputs) shape:
```json
{"type":"conversation.item.create","item":{"id":"camera_1","type":"message","role":"user","content":[{"type":"input_image","image_url":"data:image/jpeg;base64,..."}]}}
```
Images become conversation context immediately; they do not start a response.
Speak and let VAD commit your audio, or send `response.create` after adding an
image/text item. A manual `input_audio_buffer.commit` only commits audio; follow
it with `response.create` to generate. Images persist across audio commits,
interruptions, and audio-buffer clears. Remove them with `conversation.item.delete`.
The browser retains eight camera items and deletes the oldest before adding a
ninth. Camera-off stops capture but leaves those items in the conversation.
This deployment accepts JPEG/PNG Base64 data URLs with a total image-context
budget of eight images and 4 MiB; external image URLs are not supported. Clients
must delete old image items when reaching the limit. There is no precise
video/audio timestamp alignment or native streaming-video KV update.
### Protocol compatibility boundary
Qwen VAD uses `session.update`, 24 kHz PCM16 `input_audio_buffer.append`,
`conversation.item.create` with `input_image`/`input_text`, and `response.create`.
The engine resamples input to Qwen's 16 kHz processing rate. Qwen rejects the
custom `video_frames` field on an append: it answers whole turns, so it has no
audio unit for a frame track to align to. MiniCPM keeps that field for its
native camera protocol, where Stage 0 interleaves frames at unit boundaries.
This is a supported subset, not full OpenAI Realtime API compatibility.
`duplex=1`, `overlap_policy`, `playback.ack`, and `session.close` remain vLLM-Omni
extensions. The legacy `--stt` mode is unchanged and is not the standard Realtime
conversation API. This demo does not add WebRTC, semantic VAD, or tool calling.
## Shared host options
The shared entry point can select either profile explicitly:
```bash
python -m examples.online_serving.realtime_web --profile qwen3-turn \
--backend ws://127.0.0.1:8091 --stt --port 7863
```
- `--backend` / `--ws-backend`: backend WebSocket origin (not the full `/v1/realtime` URL).
- `--model`: override the wrapper's model name.
- `--public-realtime-url`: optional browser-visible WebSocket URL; otherwise the
static host proxies `/v1/realtime` on the same origin.
- `--host`, `--port`: UI bind address and port (default port 7862).
- `--ref-audio`: required for MiniCPM; rejected for Qwen3.
Microphone and camera access require `localhost` or HTTPS. For a remote backend,
use an SSH tunnel to the UI host or an HTTPS reverse proxy with WebSocket support.
An explicit `wss://` public URL retains its scheme.
## Validation
```bash
node --test tests/examples/test_realtime_web_profiles.cjs
pytest tests/examples/test_minicpmo_realtime_web_server.py \
tests/examples/test_minicpmo_realtime_web_static.py \
tests/examples/test_realtime_web_profiles.py
pytest -m cpu tests/model_executor/models/qwen3_omni/test_duplex_plugin.py \
tests/engine/duplex/test_session_runner.py \
tests/engine/test_duplex_orchestrator.py
```
These CPU tests cover profile messages, two STT turns, VAD terminal/drain ordering,
backpressure, native capture during playback, server config injection, and shared
worklets. Backend regressions cover current audio inclusion in history-based
prompts, delayed playback ACK ordering, and no extra generation during silence
after a completed VAD turn. Hardware acceptance still requires a MiniCPM call
with camera/barge-in and two Qwen turns against each enabled backend, including
speech interruption in VAD mode; simulated tests do not
establish model quality or hardware latency.
examples/online_serving/realtime_web/__init__.py
+4
View File
@@ -0,0 +1,4 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Shared browser audio UI and model-specific realtime profiles."""
examples/online_serving/realtime_web/__main__.py
+7
View File
@@ -0,0 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from .server import main
if __name__ == "__main__":
main()
examples/online_serving/minicpmo/realtime_web/app/index.html → examples/online_serving/realtime_web/app/index.html
+12 -11
View File
@@ -4,37 +4,35 @@
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="color-scheme" content="light">
<title>MiniCPM-o 4.5 Voice</title>
<title>Omni Realtime Voice</title>
<link rel="stylesheet" href="static/styles.css">
</head>
<body>
<header class="app-header">
<div>
<p class="eyebrow">Experimental full-duplex runtime</p>
<h1>MiniCPM-o 4.5 Voice</h1>
<p id="pageEyebrow" class="eyebrow"></p>
<h1 id="pageTitle">Omni Realtime Voice</h1>
</div>
<span id="connectionState" class="status status-offline">Offline</span>
</header>
<main>
<p id="profileDescription" role="note"></p>
<section class="call-bar" aria-label="Session controls">
<div class="call-actions">
<button id="callButton" class="primary" type="button">Start session</button>
<button id="sendTurnButton" class="secondary" type="button" hidden disabled>Send turn</button>
<button id="muteButton" class="secondary" type="button" disabled>Mute</button>
<button id="cameraButton" class="secondary" type="button" disabled>Camera</button>
<video id="cameraPreview" muted playsinline style="display:none;width:96px;height:72px;border-radius:8px;object-fit:cover;background:#000"></video>
</div>
<div style="display:flex;gap:8px;align-items:flex-start;margin-top:8px;width:100%">
<div id="promptControls" style="display:flex;gap:8px;align-items:flex-start;margin-top:8px;width:100%">
<label style="display:inline-flex;flex-direction:column;gap:4px;font-size:13px;opacity:.85;white-space:nowrap">
Prompt preset
<select id="promptPreset" style="font-size:13px;padding:4px 6px;border-radius:6px">
<option value="omni" selected>Omni (official omni preset)</option>
<option value="chinese_call">中文通话 (official audio preset)</option>
<option value="english_call">English Call (official audio preset)</option>
<option value="custom">Custom</option>
</select>
</label>
<textarea id="systemPrompt" rows="2" placeholder="System prompt" style="flex:1;font-size:13px;padding:6px 8px;border-radius:6px;resize:vertical">Streaming Omni Conversation.</textarea>
<textarea id="systemPrompt" rows="2" placeholder="System prompt" style="flex:1;font-size:13px;padding:6px 8px;border-radius:6px;resize:vertical"></textarea>
</div>
<div class="meter" aria-label="Microphone level">
<span>Mic</span>
@@ -53,7 +51,7 @@
<p class="eyebrow">Live transcript</p>
<h2 id="conversationTitle">Conversation</h2>
</div>
<span class="policy-label">Model-controlled listen / speak</span>
<span id="policyLabel" class="policy-label"></span>
</div>
<div id="conversation" class="conversation" aria-live="polite">
<p id="emptyConversation" class="empty-state">Start a session and speak naturally.</p>
@@ -73,7 +71,10 @@
</details>
</main>
<script>window.FULL_DUPLEX_CONFIG = __FULL_DUPLEX_CONFIG__;</script>
<script>window.OMNI_REALTIME_CONFIG = __FULL_DUPLEX_CONFIG__; window.FULL_DUPLEX_CONFIG = window.OMNI_REALTIME_CONFIG;</script>
<script src="static/profiles/common.js?v=__FULL_DUPLEX_APP_VERSION__"></script>
<script src="static/profiles/minicpm_native.js?v=__FULL_DUPLEX_APP_VERSION__"></script>
<script src="static/profiles/qwen3_turn.js?v=__FULL_DUPLEX_APP_VERSION__"></script>
<script src="static/app.js?v=__FULL_DUPLEX_APP_VERSION__"></script>
</body>
</html>
examples/online_serving/minicpmo/realtime_web/app/static/app.js → examples/online_serving/realtime_web/app/static/app.js
+335 -140
View File
@@ -1,8 +1,10 @@
(() => {
'use strict';
const config = window.FULL_DUPLEX_CONFIG || {};
const config = window.OMNI_REALTIME_CONFIG || window.FULL_DUPLEX_CONFIG || {};
const profile = window.OmniRealtimeProfiles[config.profile || 'minicpm-native'](config);
const callButton = document.getElementById('callButton');
const sendTurnButton = document.getElementById('sendTurnButton');
const muteButton = document.getElementById('muteButton');
const cameraButton = document.getElementById('cameraButton');
const cameraPreview = document.getElementById('cameraPreview');
@@ -27,20 +29,26 @@
const INITIAL_PLAYBACK_BUFFER_MS = 400;
const SESSION_CLOSE_TIMEOUT_MS = 1000;
// Default prompts mirroring the official MiniCPM-o-Demo presets
// (assets/presets/{omni,audio_duplex}/*.yaml).
const PROMPT_PRESETS = {
omni: 'Streaming Omni Conversation.',
chinese_call: '扮演一个具有以上声音特征的助手。请认真、高质量地回复用户的问题。'
+ '请用高自然度的方式和用户聊天。你处于双工模式，可以一边听、一边说。'
+ '你是由面壁智能开发的人工智能助手：面壁小钢炮。',
english_call: 'Replicate the tone and style from the input audio. Your task is to be '
+ 'a helpful assistant using this voice pattern. Please answer the user\'s questions '
+ 'seriously and in a high quality. Please chat with the user in a high naturalness '
+ 'style. You are in duplex mode, where you can listen and speak at the same time.',
};
const PROMPT_PRESETS = profile.presets;
document.title = profile.title;
document.getElementById('pageTitle').textContent = profile.title;
document.getElementById('pageEyebrow').textContent = profile.eyebrow;
document.getElementById('profileDescription').textContent = profile.description;
document.getElementById('policyLabel').textContent = profile.policy;
cameraButton.hidden = !profile.camera;
sendTurnButton.hidden = !profile.clientCommit;
promptPreset.replaceChildren();
for (const name of [...Object.keys(PROMPT_PRESETS), 'custom']) {
const option = document.createElement('option');
option.value = name;
option.textContent = name.replaceAll('_', ' ');
promptPreset.appendChild(option);
}
systemPromptInput.value = Object.values(PROMPT_PRESETS)[0];
document.getElementById('promptControls').hidden = !profile.instructions;
let socket = null;
let closingSocket = null;
let mediaStream = null;
let captureContext = null;
let captureNode = null;
@@ -65,6 +73,70 @@
let liveUserTurn = null;
let liveAssistantTurn = null;
let sessionCloseResolver = null;
let responseComplete = false;
let playbackComplete = true;
let turnSubmitted = false;
let echoTimer = null;
let sessionGeneration = 0;
let audioChain = Promise.resolve();
// Retain interrupted IDs until startSession() resets the session. Queued
// events and pending audio decodes can outlive response.done, so removing
// IDs there could resume interrupted playback. This set grows with the
// interruptions in a call; sessionGeneration rejects old-session decodes
// after the set is cleared for a new session.
const interruptedResponses = new Set();
const pendingEvents = new Set();
let assistantTextChannel = null;
let connectionReady = false;
let turnCounter = 0;
let stopping = null;
let turnTimeout = null;
function markBusy() {
assistantActive = true;
if (profile.halfDuplex) pendingCapture = [];
}
function armTurnTimeout() {
clearTimeout(turnTimeout);
turnTimeout = window.setTimeout(() => {
failSession('Timed out waiting for the model response. Check the backend event log and restart the session.');
}, 120000);
}
async function failSession(message) {
appendLog(message, true);
await stopSession({ terminal: false });
setConnection('Error', 'error');
runtimeDetail.textContent = message;
}
function finishResponseIfReady() {
if (!responseComplete || !playbackComplete || echoTimer !== null) return;
const generation = sessionGeneration;
echoTimer = window.setTimeout(async () => {
echoTimer = null;
if (!running || generation !== sessionGeneration) return;
if (profile.reconnectEachTurn) {
connectionReady = false;
const previous = socket;
socket = null;
if (previous) { previous.onclose = null; previous.onmessage = null; previous.close(); }
try { await openSocket(); }
catch (error) { await failSession(error.message); return; }
if (!running || generation !== sessionGeneration) return;
}
assistantActive = false;
responseComplete = false;
turnSubmitted = false;
currentResponseId = null;
responseHasAudio = false;
assistantTextChannel = null;
if (profile.halfDuplex) pendingCapture = [];
sendTurnButton.disabled = !profile.clientCommit;
setModel(profile.waiting);
}, ECHO_GUARD_MS);
}
function staticAssetUrl(path) {
const version = String(config.appVersion || '').trim();
@@ -82,12 +154,7 @@
}
function realtimeUrl() {
const url = new URL(config.realtimePath, window.location.href);
url.protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
url.searchParams.set('duplex', '1');
url.searchParams.set('model', config.model || 'openbmb/MiniCPM-o-4_5');
url.searchParams.set('autostart', '0');
return url.toString();
return profile.url(config, window.location.href);
}
function setConnection(label, kind) {
@@ -250,7 +317,7 @@
}
function microphoneUploadEnabled() {
return running && !muted;
return running && connectionReady && !muted && (!profile.halfDuplex || !assistantActive);
}
function flushCapture() {
@@ -267,19 +334,9 @@
offset += chunk.length;
}
pendingCapture = [];
const pcm = resampleInt16(merged, captureRate, INPUT_RATE);
const appendEvent = {
type: 'input_audio_buffer.append',
audio: int16ToBase64(pcm),
format: 'pcm16',
sample_rate_hz: INPUT_RATE,
};
// Omni duplex: ~1 fps camera frame rides the audio append (official
// MiniCPM-o-Demo contract: one base64 JPEG per ~1 s chunk).
if (cameraPendingFrame) {
appendEvent.video_frames = [cameraPendingFrame];
cameraPendingFrame = null;
}
const pcm = resampleInt16(merged, captureRate, profile.inputSampleRate || INPUT_RATE);
const appendEvent = profile.append(int16ToBase64(pcm), cameraPendingFrame);
cameraPendingFrame = null;
socket.send(JSON.stringify(appendEvent));
}
@@ -287,13 +344,17 @@
currentResponseId = responseId || currentResponseId;
responseHasAudio = false;
assistantActive = true;
setModel('Speaking');
responseComplete = false;
playbackComplete = true;
if (profile.halfDuplex) pendingCapture = [];
setModel(profile.halfDuplex ? 'Thinking' : 'Speaking');
}
function feedPlayback(decoded, responseId) {
if (!decoded || !decoded.pcm || decoded.pcm.length === 0 || !playbackNode) return;
const pcm = resampleInt16(decoded.pcm, decoded.sourceRate, playbackRate);
responseHasAudio = true;
playbackComplete = false;
assistantActive = true;
setPlayback('Buffering');
playbackNode.port.postMessage({
@@ -305,6 +366,7 @@
}
function requestPlaybackDrain(responseId) {
if (!responseHasAudio) { playbackComplete = true; finishResponseIfReady(); return; }
if (!playbackNode) return;
playbackNode.port.postMessage({ type: 'drain', responseId: responseId || currentResponseId });
}
@@ -314,90 +376,133 @@
if (!responseId && playedMs > 0) appendLog('playback ack skipped: missing response id', true);
return;
}
socket.send(JSON.stringify({
type: 'playback.ack',
response_id: responseId,
item_id: `item_${responseId}`,
played_ms: playedMs,
committed_ms: playedMs,
}));
const ack = profile.ack(responseId, playedMs);
if (ack) socket.send(JSON.stringify(ack));
}
function playbackDrained(message) {
const responseId = message.responseId || currentResponseId;
sendPlaybackAck(responseId, Number(message.playedMs) || 0);
if (currentResponseId && responseId !== currentResponseId) return;
if (profile.playbackAck) sendPlaybackAck(responseId, Number(message.playedMs) || 0);
setPlayback('Idle');
if (message.underrunMs > 0) {
appendLog(`playback underrun ${message.underrunMs} ms`);
}
window.setTimeout(() => {
assistantActive = false;
currentResponseId = null;
responseHasAudio = false;
if (running) setModel('Listening');
}, ECHO_GUARD_MS);
playbackComplete = true;
if (message.underrunMs > 0) appendLog(`playback underrun ${message.underrunMs} ms`);
if (!profile.halfDuplex && !profile.waitForResponseDone) responseComplete = true;
finishResponseIfReady();
}
function handleEvent(event) {
async function handleAudioEvent(action) {
markBusy();
currentResponseId = action.responseId || currentResponseId || `turn-${turnCounter}`;
setModel('Speaking');
const generation = sessionGeneration;
const responseId = currentResponseId;
const decoded = await decodeAudioDelta(action.event);
if (generation === sessionGeneration && !interruptedResponses.has(responseId)) feedPlayback(decoded, responseId);
}
function handleTranscriptEvent(action) {
if (action.role === 'assistant') {
// Qwen may emit both text and audio-transcript representations.
// Display one channel per response instead of duplicating the answer.
if (profile.deduplicateTranscript && assistantTextChannel && action.channel !== assistantTextChannel) return;
if (action.text) assistantTextChannel = action.channel || assistantTextChannel;
}
if (action.kind === 'text') addTranscript(action.role, action.text);
else if (profile.deduplicateTranscript && action.role === 'assistant') {
if (action.text) {
const turn = ensureTurn('assistant');
turn.value = action.text;
turn.text.textContent = action.text;
}
} else finishTranscript(action.role, action.text);
}
async function handleEvent(event, action = profile.mapEvent(event)) {
appendEventLog(event);
const responseId = responseIdOf(event);
switch (event.type) {
case 'session.created':
case 'session.updated':
const responseId = action.responseId || responseIdOf(event);
if (responseId && interruptedResponses.has(responseId) && action.kind !== 'interrupt') return;
switch (action.kind) {
case 'connected':
setConnection('Connected', 'online');
setModel('Listening');
if (!assistantActive) setModel(profile.waiting);
break;
case 'response.listen':
case 'interrupt': {
if (responseId) interruptedResponses.add(responseId);
// A late cancellation belongs to its own response, not the newer one
// currently playing. Speech-start events interrupt the local playback.
if (responseId && responseId !== currentResponseId) break;
if (currentResponseId) interruptedResponses.add(currentResponseId);
sessionGeneration += 1;
audioChain = Promise.resolve();
// Move queued events off the interrupted decode, preserving any newer
// response already received. Its chunks must not be lost with the old one.
const queued = [...pendingEvents];
pendingEvents.clear();
for (const pending of queued) enqueueEvent(pending.event, pending.action, pending.source);
clearTimeout(echoTimer);
echoTimer = null;
// Local playback completes immediately on interruption, including
// when the server already completed generation.
clearTimeout(turnTimeout);
if (playbackNode) playbackNode.port.postMessage({ type: 'clear' });
responseComplete = true;
playbackComplete = true;
responseHasAudio = false;
assistantActive = false;
setModel('Listening');
currentResponseId = null;
assistantTextChannel = null;
finishTranscript('assistant');
setPlayback('Idle');
setModel(profile.waiting);
break;
case 'response.created':
case 'response.speak':
}
case 'listen':
assistantActive = false;
setModel(profile.waiting);
break;
case 'begin':
if (profile.halfDuplex) armTurnTimeout();
if (echoTimer !== null) { clearTimeout(echoTimer); echoTimer = null; }
beginAssistant(responseId);
break;
case 'response.output_audio.delta':
currentResponseId = responseId || currentResponseId;
assistantActive = true;
setModel('Speaking');
decodeAudioDelta(event)
.then((decoded) => feedPlayback(decoded, responseId))
.catch((error) => appendLog(`audio decode failed: ${error.message || error}`, true));
case 'audio':
await handleAudioEvent(action);
break;
case 'response.output_audio.done':
case 'drain':
requestPlaybackDrain(responseId);
break;
case 'response.output_audio_transcript.delta':
addTranscript('assistant', event.delta || '');
case 'text': case 'text-final':
handleTranscriptEvent(action);
break;
case 'response.output_audio_transcript.done':
finishTranscript('assistant', event.transcript || '');
break;
case 'conversation.item.input_audio_transcription.delta':
addTranscript('user', event.delta || '');
break;
case 'conversation.item.input_audio_transcription.completed':
finishTranscript('user', event.transcript || '');
break;
case 'response.done':
case 'done':
clearTimeout(turnTimeout);
responseComplete = true;
finishTranscript('assistant');
if (!responseHasAudio) requestPlaybackDrain(responseId);
requestPlaybackDrain(responseId);
finishResponseIfReady();
break;
case 'playback.acknowledged':
{
const acknowledgement = event.event || event;
runtimeDetail.textContent = `Playback committed ${acknowledgement.committed_ms || 0} ms`;
}
case 'backpressure':
armTurnTimeout();
markBusy();
setModel('Thinking / Speaking');
runtimeDetail.textContent = action.message;
break;
case 'session.closed':
case 'ack':
runtimeDetail.textContent = `Playback committed ${action.committedMs} ms`;
break;
case 'closed':
if (sessionCloseResolver) sessionCloseResolver();
sessionCloseResolver = null;
break;
case 'error':
setConnection('Error', 'error');
runtimeDetail.textContent = String(event.error || event.code || 'Server error');
break;
default:
if (action.fatal) { await failSession(action.message); break; }
// A rejected frame or a refused update costs one operation, not the
// call: report it and keep listening.
appendLog(action.message, true);
runtimeDetail.textContent = action.message;
break;
default: break;
}
}
@@ -406,8 +511,15 @@
playbackRate = playbackContext.sampleRate;
await playbackContext.audioWorklet.addModule(staticAssetUrl('static/playback_worklet.js'));
playbackNode = new AudioWorkletNode(playbackContext, 'fullduplex-pcm-playback');
const currentPlaybackNode = playbackNode;
playbackNode.port.onmessage = (message) => {
if (playbackNode !== currentPlaybackNode) return;
if (message.data.type === 'playback-started') setPlayback('Playing');
else if (message.data.type === 'playback-stopped') {
// The old response may report after the next one has started. ACK its
// own cursor without changing the current response's UI state.
if (profile.playbackAck) sendPlaybackAck(message.data.responseId, Number(message.data.playedMs) || 0);
}
else if (message.data.type === 'playback-drained') playbackDrained(message.data);
else if (message.data.type === 'playback-underrun') {
runtimeDetail.textContent = `Playback underrun ${message.data.underrunMs || 0} ms`;
@@ -448,51 +560,86 @@
await captureContext.resume();
}
function enqueueEvent(event, action, source) {
const pending = { event, action, source };
pendingEvents.add(pending);
const generation = sessionGeneration;
audioChain = audioChain.then(() => {
if (!pendingEvents.delete(pending)) return;
if (socket === source || action.kind === 'closed') return handleEvent(event, action);
}).catch((error) => {
if (generation === sessionGeneration) return failSession(`Server event failed: ${error.message}`);
});
}
function openSocket() {
return new Promise((resolve, reject) => {
const url = realtimeUrl();
socket = new WebSocket(url);
const current = new WebSocket(url);
socket = current;
connectionReady = false;
turnCounter += 1;
let settled = false;
socket.onopen = () => {
const timer = window.setTimeout(() => {
if (settled) return;
settled = true;
const extraBody = {
auto_response: true,
};
const session = {
modalities: ['audio', 'text'],
voice: 'default',
extra_body: extraBody,
};
// Reference voice for TTS cloning, provided by the server via
// --ref-audio (mirrors the official demo's default ref audio).
if (config.refAudio) session.ref_audio = config.refAudio;
const instructions = systemPromptInput ? systemPromptInput.value.trim() : '';
if (instructions) session.instructions = instructions;
socket.send(JSON.stringify({ type: 'session.update', session }));
runtimeDetail.textContent = `${captureRate} Hz capture / ${playbackRate} Hz playback`;
current.onclose = null;
current.close();
reject(new Error(`Session handshake timed out. ${profile.connectionHint}`));
}, 15000);
const rejectOnce = (message) => {
clearTimeout(timer);
if (!settled) { settled = true; reject(new Error(message)); }
};
current.onopen = () => {
const instructions = systemPromptInput.value.trim();
for (const event of profile.initialMessages(config, instructions)) current.send(JSON.stringify(event));
appendLog(`websocket open ${url}`);
resolve();
};
socket.onmessage = (message) => {
if (typeof message.data !== 'string') return;
try {
handleEvent(JSON.parse(message.data));
} catch (error) {
appendLog(`invalid server event: ${error.message || error}`, true);
current.onmessage = (message) => {
if (typeof message.data !== 'string' || (socket !== current && closingSocket !== current)) return;
let event;
try { event = JSON.parse(message.data); }
catch (error) {
if (socket !== current) return;
const detail = `Invalid server event: ${error.message}`;
if (!settled) rejectOnce(detail);
else void failSession(detail);
return;
}
};
socket.onerror = () => {
if (!settled) {
// Shutdown has detached the active socket. Only its close acknowledgement
// may pass, directly: the event queue can itself be awaiting cleanup.
if (closingSocket === current) {
if (event?.type === 'session.closed' && sessionCloseResolver) sessionCloseResolver();
return;
}
const action = profile.mapEvent(event);
if (action.kind === 'error' && !settled) {
rejectOnce(action.message);
return;
}
if (event.type === profile.readyEvent && !settled) {
clearTimeout(timer);
settled = true;
reject(new Error(`WebSocket connection failed: ${url}`));
connectionReady = true;
runtimeDetail.textContent = `${captureRate} Hz capture / ${playbackRate} Hz playback`;
resolve();
}
// Stop the speaker immediately, even while an old WAV is decoding.
// Ordinary completion still waits for its preceding audio chunks.
if (action.kind === 'interrupt') {
void handleEvent(event, action);
return;
}
enqueueEvent(event, action, current);
};
socket.onclose = (event) => {
current.onerror = () => rejectOnce(`WebSocket connection failed. ${profile.connectionHint}`);
current.onclose = (event) => {
clearTimeout(timer);
connectionReady = false;
appendLog(`websocket closed code=${event.code}`);
if (running) {
setConnection('Disconnected', 'error');
stopSession();
}
if (!settled) rejectOnce(`Connection closed before session ready. ${profile.connectionHint}`);
else if (running && socket === current) failSession(`Backend disconnected (${event.code}). ${profile.connectionHint}`);
};
});
}
@@ -515,46 +662,61 @@
setConnection('Connecting', 'connecting');
runtimeDetail.textContent = 'Requesting microphone access';
try {
sessionGeneration += 1;
audioChain = Promise.resolve();
interruptedResponses.clear();
pendingEvents.clear();
await openPlayback();
await openCapture();
await openSocket();
running = true;
muted = false;
assistantActive = false;
sendTimer = window.setInterval(flushCapture, SEND_INTERVAL_MS);
sendTimer = window.setInterval(flushCapture, profile.sendIntervalMs || SEND_INTERVAL_MS);
startClock();
callButton.textContent = 'End session';
callButton.classList.add('is-active');
muteButton.disabled = false;
cameraButton.disabled = false;
cameraButton.disabled = !profile.camera;
sendTurnButton.disabled = !profile.clientCommit;
setConnection('Connected', 'online');
setModel('Listening');
setModel(profile.waiting);
appendLog('session started');
} catch (error) {
appendLog(`start failed: ${error.message || error}`, true);
await stopSession({ terminal: false });
setConnection('Error', 'error');
runtimeDetail.textContent = String(error.message || error);
await stopSession();
} finally {
callButton.disabled = false;
}
}
async function startCamera() {
if (cameraStream) return;
if (!profile.camera || cameraStream) return;
cameraStream = await navigator.mediaDevices.getUserMedia({ video: true, audio: false });
cameraPreview.srcObject = cameraStream;
cameraPreview.style.display = '';
await cameraPreview.play().catch(() => {});
// Official omni-duplex cadence: one JPEG (quality 0.7) per ~1 s chunk,
// no client-side resize (the server normalizes at scale_resolution=448).
cameraTimer = window.setInterval(() => {
const captureCameraFrame = () => {
if (!cameraStream || cameraPreview.videoWidth === 0) return;
cameraCanvas.width = cameraPreview.videoWidth;
cameraCanvas.height = cameraPreview.videoHeight;
cameraCanvas.getContext('2d').drawImage(cameraPreview, 0, 0);
cameraPendingFrame = cameraCanvas.toDataURL('image/jpeg', 0.7).split(',')[1];
}, 1000);
const scale = profile.cameraMaxDimension
? Math.min(1, profile.cameraMaxDimension / Math.max(cameraPreview.videoWidth, cameraPreview.videoHeight)) : 1;
cameraCanvas.width = Math.max(1, Math.round(cameraPreview.videoWidth * scale));
cameraCanvas.height = Math.max(1, Math.round(cameraPreview.videoHeight * scale));
cameraCanvas.getContext('2d').drawImage(cameraPreview, 0, 0, cameraCanvas.width, cameraCanvas.height);
const frame = cameraCanvas.toDataURL('image/jpeg', 0.7).split(',')[1];
if (profile.imageMessages) {
if (connectionReady && socket?.readyState === WebSocket.OPEN) {
for (const event of profile.imageMessages(frame)) socket.send(JSON.stringify(event));
}
} else cameraPendingFrame = frame;
};
// Do not make the first spoken turn race a one-second timer.
captureCameraFrame();
cameraTimer = window.setInterval(captureCameraFrame, 1000);
cameraButton.textContent = 'Camera off';
cameraButton.classList.add('is-active');
appendLog('camera on (1 fps omni frames)');
@@ -590,15 +752,33 @@
const finish = () => {
if (done) return;
done = true;
clearTimeout(timer);
if (sessionCloseResolver === finish) sessionCloseResolver = null;
resolve();
};
sessionCloseResolver = finish;
window.setTimeout(finish, timeoutMs);
const timer = window.setTimeout(finish, timeoutMs);
});
}
async function stopSession({ terminal = true } = {}) {
function stopSession(options = {}) {
if (stopping) return stopping;
stopping = cleanupSession(options).finally(() => { stopping = null; });
return stopping;
}
async function cleanupSession({ terminal = true } = {}) {
sessionGeneration += 1;
pendingEvents.clear();
connectionReady = false;
clearTimeout(echoTimer);
clearTimeout(turnTimeout);
echoTimer = null;
responseComplete = false;
playbackComplete = true;
turnSubmitted = false;
assistantTextChannel = null;
sendTurnButton.disabled = true;
running = false;
assistantActive = false;
pendingCapture = [];
@@ -607,15 +787,16 @@
sendTimer = null;
clockTimer = null;
if (socket) {
const closingSocket = socket;
closingSocket = socket;
socket = null;
closingSocket.onclose = null;
if (terminal && closingSocket.readyState === WebSocket.OPEN) {
if (terminal && profile.closeSession && closingSocket.readyState === WebSocket.OPEN) {
const closed = waitForSessionClosed(closingSocket, SESSION_CLOSE_TIMEOUT_MS);
closingSocket.send(JSON.stringify({ type: 'session.close' }));
await closed;
}
closingSocket.close(1000, 'client stop');
closingSocket = null;
}
if (playbackNode) playbackNode.port.postMessage({ type: 'clear' });
if (mediaStream) {
@@ -632,6 +813,8 @@
playbackNode = null;
currentResponseId = null;
responseHasAudio = false;
finishTranscript('user');
finishTranscript('assistant');
meterFill.style.width = '0%';
sessionTimer.textContent = '00:00';
callButton.textContent = 'Start session';
@@ -659,6 +842,18 @@
if (running) stopSession();
else startSession();
});
sendTurnButton.addEventListener('click', () => {
if (!running || !connectionReady || !profile.clientCommit || turnSubmitted) return;
flushCapture();
for (const event of profile.commitMessages()) socket.send(JSON.stringify(event));
turnSubmitted = true;
responseComplete = false;
markBusy();
armTurnTimeout();
sendTurnButton.disabled = true;
setModel('Thinking');
appendLog('turn submitted');
});
muteButton.addEventListener('click', toggleMute);
clearLogButton.addEventListener('click', () => {
eventLog.textContent = '';
examples/online_serving/minicpmo/realtime_web/app/static/pcm_worklet.js → examples/online_serving/realtime_web/app/static/pcm_worklet.js
View File
File renamed without changes.
examples/online_serving/minicpmo/realtime_web/app/static/playback_worklet.js → examples/online_serving/realtime_web/app/static/playback_worklet.js
+7
View File
@@ -39,6 +39,13 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor {
}
this.notifyIfDrained();
} else if (message.type === 'clear') {
if (this.activeResponseId) {
this.port.postMessage({
type: 'playback-stopped',
responseId: this.activeResponseId,
playedMs: Math.round((this.playedFrames * 1000) / sampleRate),
});
}
this.queue = [];
this.offset = 0;
this.playedFrames = 0;
examples/online_serving/realtime_web/app/static/profiles/common.js
+56
View File
@@ -0,0 +1,56 @@
/* Shared wire helpers. Profiles translate server events into shell actions. */
(function (root) {
'use strict';
const profiles = root.OmniRealtimeProfiles = root.OmniRealtimeProfiles || {};
profiles.url = (config, location) => {
const url = new URL(config.realtimePath || 'v1/realtime', location);
if (url.protocol === 'http:') url.protocol = 'ws:';
if (url.protocol === 'https:') url.protocol = 'wss:';
return url;
};
/* Errors that leave the session unusable, so the shell has to tear it down.
Everything else costs one operation --- a rejected image, a refused
session.update, a late playback ack --- and the call carries on.
'runtime_append_failed' is deliberately absent: the engine reports it and
keeps the session, unlike 'runtime_append_task_failed' which kills it. */
profiles.fatalErrorCodes = new Set([
'unknown_session',
'session_closed',
'resource_exhausted',
'unsupported',
'unsupported_turn_detection',
'server_vad_unavailable',
'runtime_append_task_failed',
'internal_error',
]);
profiles.event = (event) => {
const responseId = event.response_id || event.response?.id || null;
const action = (kind, extra = {}) => ({ kind, responseId, ...extra });
switch (event.type) {
case 'session.created': case 'session.updated': return action('connected');
case 'response.created': return action('begin');
case 'response.audio.delta': case 'response.output_audio.delta':
return action('audio', { event: { ...event, delta: event.delta || event.audio || event.response?.audio } });
case 'response.audio.done': case 'response.output_audio.done': return action('drain');
case 'response.audio_transcript.delta': case 'response.output_audio_transcript.delta':
return action('text', { role: 'assistant', channel: 'audio', text: event.delta || '' });
case 'response.audio_transcript.done': case 'response.output_audio_transcript.done':
return action('text-final', { role: 'assistant', channel: 'audio', text: event.transcript || '' });
case 'conversation.item.input_audio_transcription.delta':
return action('text', { role: 'user', text: event.delta || '' });
case 'conversation.item.input_audio_transcription.completed':
return action('text-final', { role: 'user', text: event.transcript || '' });
case 'response.done': return action('done');
case 'session.closed': return action('closed');
case 'error': {
const code = event.code || event.error?.code;
return action('error', {
code,
fatal: profiles.fatalErrorCodes.has(code),
message: typeof event.error === 'string' ? event.error : event.error?.message || event.message || code || 'Server error',
});
}
default: return action('ignore');
}
};
})(globalThis);
examples/online_serving/realtime_web/app/static/profiles/minicpm_native.js
+55
View File
@@ -0,0 +1,55 @@
(function (root) {
'use strict';
const profiles = root.OmniRealtimeProfiles;
profiles['minicpm-native'] = () => ({
title: 'MiniCPM-o 4.5 Voice', eyebrow: 'Experimental full-duplex runtime',
policy: 'Model-controlled listen / speak',
description: 'Speak naturally. The model can listen and speak at the same time. Camera frames are sent with your audio.',
waiting: 'Listening', camera: true, playbackAck: true, clientCommit: false,
halfDuplex: false, closeSession: true, reconnectEachTurn: false,
readyEvent: 'session.updated', instructions: true, sendIntervalMs: 200,
presets: {
omni: 'Streaming Omni Conversation.',
chinese_call: '扮演一个具有以上声音特征的助手。请认真、高质量地回复用户的问题。'
+ '请用高自然度的方式和用户聊天。你处于双工模式，可以一边听、一边说。'
+ '你是由面壁智能开发的人工智能助手：面壁小钢炮。',
english_call: 'Replicate the tone and style from the input audio. Your task is to be '
+ 'a helpful assistant using this voice pattern. Please answer the user\'s questions '
+ 'seriously and in a high quality. Please chat with the user in a high naturalness '
+ 'style. You are in duplex mode, where you can listen and speak at the same time.',
},
url(config, location) {
const url = profiles.url(config, location);
url.searchParams.set('duplex', '1');
url.searchParams.set('model', config.model || 'openbmb/MiniCPM-o-4_5');
url.searchParams.set('autostart', '0');
return url.toString();
},
initialMessages(config, instructions) {
const session = { modalities: ['audio', 'text'], voice: 'default',
extra_body: { auto_response: true } };
if (config.refAudio) session.ref_audio = config.refAudio;
if (instructions) session.instructions = instructions;
return [{ type: 'session.update', session }];
},
append(audio, frame) {
const event = { type: 'input_audio_buffer.append', audio, format: 'pcm16', sample_rate_hz: 16000 };
if (frame) event.video_frames = [frame];
return event;
},
commitMessages: () => [],
ack(responseId, playedMs) {
return { type: 'playback.ack', response_id: responseId, item_id: `item_${responseId}`,
played_ms: playedMs, committed_ms: playedMs };
},
mapEvent(event) {
if (event.type === 'response.listen') return { kind: 'listen' };
if (event.type === 'response.speak') return { kind: 'begin', responseId: event.response_id || event.response?.id };
if (event.type === 'playback.acknowledged') {
return { kind: 'ack', committedMs: (event.event || event).committed_ms || 0 };
}
return profiles.event(event);
},
connectionHint: 'Check the MiniCPM duplex deployment and reference voice configuration.',
});
})(globalThis);
examples/online_serving/realtime_web/app/static/profiles/qwen3_turn.js
+116
View File
@@ -0,0 +1,116 @@
(function (root) {
'use strict';
const profiles = root.OmniRealtimeProfiles;
profiles['qwen3-turn'] = (config) => {
const vad = config.adapter === 'vad';
let interruptOnSpeech = vad;
let imageSeq = 0;
let imageItems = [];
// The engine refuses a ninth image (or 4 MiB of them) for the whole
// session, so the camera track retires its own oldest frame instead.
const MAX_IMAGE_ITEMS = 8;
const hint = 'This Qwen deploy needs session_mode: duplex and Server VAD support; use --stt or enable Server VAD.';
return {
title: 'Qwen3-Omni Voice', eyebrow: vad ? 'Engine-owned duplex call' : 'Turn-based realtime call',
policy: vad ? 'Server VAD • interruptible responses' : 'Send turn • one response at a time',
description: vad
? 'Pause to send your turn; speak again to interrupt. Microphone upload continues during playback. Requires the Qwen duplex plugin and Silero. Optional camera frames accompany each spoken turn.'
: 'Speak, then press Send turn. Microphone upload pauses while the model answers. Each turn uses a fresh connection; conversation history is not carried between turns. No barge-in or camera input.',
waiting: 'Waiting for you', camera: vad, cameraMaxDimension: 448, playbackAck: vad, clientCommit: !vad,
waitForResponseDone: true, halfDuplex: !vad, deduplicateTranscript: true, closeSession: vad, reconnectEachTurn: !vad,
inputSampleRate: vad ? 24000 : 16000,
readyEvent: vad ? 'session.updated' : 'session.created', instructions: vad, sendIntervalMs: 200,
presets: { assistant: 'You are a helpful assistant. Answer clearly and concisely.' },
url(config, location) {
const url = profiles.url(config, location);
for (const key of ['native_duplex', 'minicpmo45_native_duplex', 'autostart']) url.searchParams.delete(key);
// Explicit opt-out matches the local file client, including servers
// that auto-select the duplex endpoint when the query is omitted.
url.searchParams.set('duplex', vad ? '1' : '0');
url.searchParams.set('model', config.model || 'Qwen/Qwen3-Omni-30B-A3B-Instruct');
return url.toString();
},
initialMessages(config, instructions) {
imageItems = [];
interruptOnSpeech = vad;
if (!vad) return [
{ type: 'session.update', model: config.model },
{ type: 'input_audio_buffer.commit', final: false },
];
const session = { model: config.model, overlap_policy: 'barge_in_on_speech', audio: { input: {
format: { type: 'audio/pcm', rate: 24000 },
turn_detection: { type: 'server_vad', threshold: 0.5, prefix_padding_ms: 300,
silence_duration_ms: 500, create_response: true, interrupt_response: true },
} } };
if (instructions) session.instructions = instructions;
return [{ type: 'session.update', session }];
},
append(audio) {
return vad ? { type: 'input_audio_buffer.append', audio }
: { type: 'input_audio_buffer.append', audio, format: 'pcm16', sample_rate_hz: 16000 };
},
// Qwen is a turn model: a frame is a picture added to the conversation,
// not a track interleaved into the audio. That is exactly the OpenAI
// Realtime image interface, so the camera sends conversation items
// rather than an extension field on the append.
imageMessages(frame) {
if (!vad) return [];
const messages = [];
if (imageItems.length >= MAX_IMAGE_ITEMS) {
// Retire before creating: the engine counts what is already stored.
messages.push({ type: 'conversation.item.delete', item_id: imageItems.shift() });
}
const id = `camera_${++imageSeq}`;
imageItems.push(id);
messages.push({ type: 'conversation.item.create', item: {
id, type: 'message', role: 'user',
content: [{ type: 'input_image', image_url: `data:image/jpeg;base64,${frame}` }],
} });
return messages;
},
commitMessages: () => vad ? [] : [{ type: 'input_audio_buffer.commit', final: true }],
ack: (responseId, playedMs) => vad ? { type: 'playback.ack', response_id: responseId,
item_id: `item_${responseId}`, played_ms: playedMs, committed_ms: playedMs } : null,
mapEvent(event) {
if (event.type === 'session.created' || event.type === 'session.updated') {
const input = event.session?.audio?.input;
const turnDetection = input && 'turn_detection' in input
? input.turn_detection : event.session?.turn_detection;
if (turnDetection !== undefined) {
interruptOnSpeech = vad && turnDetection?.type === 'server_vad'
&& turnDetection.interrupt_response !== false;
}
}
// Generation can finish long before the speaker drains its queue. In
// that case the server has no active response left to cancel for us.
if (interruptOnSpeech && event.type === 'input_audio_buffer.speech_started') {
return { kind: 'interrupt' };
}
if (event.type === 'playback.acknowledged') return { kind: 'ack', committedMs: (event.event || event).committed_ms || 0 };
if (event.type === 'response.output_text.delta' || event.type === 'transcription.delta') {
// On the shipped STT route transcription.* is model-generated text,
// not a separate ASR transcript of the user. See realtime_connection.py.
return { kind: 'text', role: 'assistant', channel: 'text', text: event.delta || '' };
}
if (event.type === 'response.output_text.done' || event.type === 'transcription.done') {
return { kind: 'text-final', role: 'assistant', channel: 'text', text: event.text || '' };
}
if (vad && (event.type === 'output_audio_buffer.cleared' ||
(event.type === 'response.done' && event.response?.status === 'cancelled'))) {
return { kind: 'interrupt', responseId: event.response_id || event.response?.id || null };
}
if (event.type === 'input_audio_buffer.committed') return { kind: 'begin' };
if (event.type === 'input_audio_buffer.cleared') return { kind: 'backpressure', message: 'Input cleared. Model is answering; please wait.' };
if (!vad && event.type === 'response.output_audio.done') return { kind: 'done' };
const action = profiles.event(event);
if (action.kind === 'error') {
if (action.code === 'input_backpressure') return { ...action, kind: 'backpressure', message: 'Input buffer is full; please repeat dropped speech after the response.' };
if (action.code === 'server_vad_initialization_failed') action.message = 'Server VAD could not load Silero. Configure the server artifact or restart this UI with --stt.';
else if (vad && action.code === 'unsupported') action.message += ` ${hint}`;
}
return action;
},
connectionHint: vad ? hint : 'Check the Qwen /v1/realtime STT endpoint and model name.',
};
};
})(globalThis);
examples/online_serving/minicpmo/realtime_web/app/static/styles.css → examples/online_serving/realtime_web/app/static/styles.css
+3
View File
@@ -125,3 +125,6 @@ button:disabled { opacity: .46; cursor: default; }
@media (prefers-reduced-motion: reduce) {
*, *::before, *::after { animation-duration: .01ms !important; animation-iteration-count: 1 !important; }
}
/* Profile-specific controls may have inline flex styles. */
[hidden] { display: none !important; }
examples/online_serving/realtime_web/server.py
+229
View File
@@ -0,0 +1,229 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Static host and same-origin proxy for the shared realtime browser UI."""
from __future__ import annotations
import argparse
import asyncio
import base64
import contextlib
import hashlib
import json
import logging
from pathlib import Path
from urllib.parse import urlencode
import uvicorn
import websockets
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from fastapi.responses import HTMLResponse, Response
from fastapi.staticfiles import StaticFiles
logger = logging.getLogger(__name__)
APP_DIR = Path(__file__).parent / "app"
STATIC_DIR = APP_DIR / "static"
def _join_ws_url(base: str, path: str, query: str) -> str:
return base.rstrip("/") + path + (("?" + query) if query else "")
async def _pump_client_to_backend(client: WebSocket, backend) -> None:
try:
while True:
message = await client.receive()
if message["type"] == "websocket.disconnect":
await backend.close()
return
if message.get("text") is not None:
await backend.send(message["text"])
elif message.get("bytes") is not None:
await backend.send(message["bytes"])
except (WebSocketDisconnect, websockets.ConnectionClosed):
with contextlib.suppress(Exception):
await backend.close()
async def _pump_backend_to_client(client: WebSocket, backend) -> None:
try:
async for message in backend:
if isinstance(message, bytes):
await client.send_bytes(message)
else:
await client.send_text(message)
except (WebSocketDisconnect, websockets.ConnectionClosed):
return
except RuntimeError as exc:
if "websocket.send" in str(exc) and "websocket.close" in str(exc):
return
raise
def _expected_proxy_close(exc: BaseException) -> bool:
if isinstance(exc, (WebSocketDisconnect, websockets.ConnectionClosed, asyncio.CancelledError)):
return True
return isinstance(exc, RuntimeError) and "websocket.send" in str(exc) and "websocket.close" in str(exc)
def build_app(
*,
ws_backend: str | None = None,
model: str | None = None,
public_realtime_url: str | None = None,
ref_audio: str | None = None,
profile: str = "minicpm-native",
adapter: str = "stt",
) -> FastAPI:
if profile not in {"minicpm-native", "qwen3-turn"}:
raise ValueError(f"Unknown profile: {profile}")
if adapter not in {"stt", "vad"}:
raise ValueError(f"Unknown adapter: {adapter}")
if profile == "qwen3-turn" and ref_audio:
raise ValueError("Qwen3 profile does not accept --ref-audio")
native = profile == "minicpm-native"
ws_backend = ws_backend or ("ws://127.0.0.1:8099" if native else "ws://127.0.0.1:8091")
model = model or ("openbmb/MiniCPM-o-4_5" if native else "Qwen/Qwen3-Omni-30B-A3B-Instruct")
app = FastAPI(title="Omni Realtime Web UI")
index_path = APP_DIR / "index.html"
app_version_hash = hashlib.sha256()
for asset_path in (
STATIC_DIR / "app.js",
STATIC_DIR / "pcm_worklet.js",
STATIC_DIR / "playback_worklet.js",
*sorted((STATIC_DIR / "profiles").glob("*.js")),
):
app_version_hash.update(asset_path.read_bytes())
app_version = app_version_hash.hexdigest()[:12]
ref_audio_uri: str | None = None
if ref_audio:
ref_path = Path(ref_audio)
if not ref_path.is_file():
raise SystemExit(f"--ref-audio not found: {ref_audio}")
encoded = base64.b64encode(ref_path.read_bytes()).decode("ascii")
ref_audio_uri = f"data:audio/wav;base64,{encoded}"
@app.get("/", response_class=HTMLResponse)
def index() -> HTMLResponse:
config = json.dumps(
{
"model": model,
"profile": profile,
"adapter": adapter,
"realtimePath": public_realtime_url or "v1/realtime",
"refAudio": ref_audio_uri,
"appVersion": app_version,
},
ensure_ascii=True,
).replace("<", "\\u003c")
html = (
index_path.read_text(encoding="utf-8")
.replace("__FULL_DUPLEX_CONFIG__", config)
.replace("__FULL_DUPLEX_APP_VERSION__", app_version)
)
return HTMLResponse(html, headers={"Cache-Control": "no-store"})
@app.get("/healthz")
def healthz() -> Response:
return Response(content="ok", media_type="text/plain")
@app.websocket("/v1/realtime")
async def realtime_proxy(websocket: WebSocket) -> None:
await websocket.accept()
query = urlencode(websocket.query_params.multi_items())
backend_url = _join_ws_url(ws_backend, "/v1/realtime", query)
logger.info("Proxying Realtime WebSocket to %s", backend_url)
try:
async with websockets.connect(
backend_url,
max_size=64 * 1024 * 1024,
) as backend:
tasks = {
asyncio.create_task(_pump_client_to_backend(websocket, backend)),
asyncio.create_task(_pump_backend_to_client(websocket, backend)),
}
done, pending = await asyncio.wait(
tasks,
return_when=asyncio.FIRST_COMPLETED,
)
for task in pending:
task.cancel()
await asyncio.gather(*pending, return_exceptions=True)
for result in await asyncio.gather(*done, return_exceptions=True):
if isinstance(result, BaseException) and not _expected_proxy_close(result):
raise result
except (WebSocketDisconnect, websockets.ConnectionClosed):
return
except (OSError, TimeoutError) as exc:
message = (
f"Cannot connect to realtime backend {ws_backend}. "
"Check that vllm serve is running and --backend matches its --port. "
"For containers or remote hosts, use the backend address reachable from the UI server."
)
logger.warning("%s (%s)", message, exc)
with contextlib.suppress(Exception):
await websocket.send_json({"type": "error", "code": "backend_unavailable", "error": message})
await websocket.close(code=1011)
except Exception:
logger.exception("Realtime WebSocket proxy failed")
with contextlib.suppress(Exception):
await websocket.close(code=1011)
finally:
with contextlib.suppress(Exception):
await websocket.close()
app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
return app
def main(default_profile: str = "minicpm-native") -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--host", default="0.0.0.0")
parser.add_argument("--port", type=int, default=7862)
parser.add_argument("--profile", choices=("minicpm-native", "qwen3-turn"), default=default_profile)
parser.add_argument("--ws-backend", "--backend", dest="ws_backend")
mode = parser.add_mutually_exclusive_group()
mode.add_argument("--stt", dest="adapter", action="store_const", const="stt")
mode.add_argument("--vad", dest="adapter", action="store_const", const="vad")
parser.set_defaults(adapter="stt")
parser.add_argument(
"--public-realtime-url",
help="Browser-visible ws:// or wss:// Realtime URL; defaults to the same-origin proxy.",
)
parser.add_argument("--model")
parser.add_argument(
"--ref-audio",
help=(
"Reference voice wav for TTS voice cloning, e.g. the "
"official MiniCPM-o-Demo assets/ref_audio/ref_minicpm_signature.wav"
),
)
args = parser.parse_args()
native = args.profile == "minicpm-native"
if native and not args.ref_audio:
parser.error("--ref-audio is required for the minicpm-native profile")
if not native and args.ref_audio:
parser.error("--ref-audio is only supported by minicpm-native")
args.model = args.model or ("openbmb/MiniCPM-o-4_5" if native else "Qwen/Qwen3-Omni-30B-A3B-Instruct")
args.ws_backend = args.ws_backend or ("ws://127.0.0.1:8099" if native else "ws://127.0.0.1:8091")
logging.basicConfig(level=logging.INFO)
uvicorn.run(
build_app(
ws_backend=args.ws_backend,
model=args.model,
public_realtime_url=args.public_realtime_url,
ref_audio=args.ref_audio,
profile=args.profile,
adapter=args.adapter,
),
host=args.host,
port=args.port,
log_level="info",
)
if __name__ == "__main__":
main()
examples/online_serving/streaming_video_generation/README.md
+22 -6
View File
@@ -21,7 +21,7 @@ The `--diffusion-streaming-output` CLI flag is forwarded as `streaming_output=Tr
| Direction | Message | Format | Description |
| --- | --- | --- | --- |
| Client to server | `session.start` | JSON text: `{"type":"session.start","model":"...","prompt":"...","format":"m4s"}` | Starts generation. `format` is optional and accepts `m4s` (default). Sampling fields such as `width`, `height`, `fps`, `num_frames`, and `extra_params` may be included. Image-conditioned models take a first frame as `image_reference`: `{"image_url":"https://..."}` or a `data:` URL. |
| Client to server | `session.interaction` | JSON text: `{"type":"session.interaction","interaction":{"event_id":"xxx","event":{"prompt":"..."},"transition_chunks":3}}` | Updates the active prompt midway through generation. `event_id` is optional and `transition_chunks` defaults to the model setting. |
| Client to server | `session.interaction` | JSON text: `{"type":"session.interaction","interaction":{"event_id":"xxx","event":{"prompt":"..."},"transition_chunks":3}}` | Updates the active prompt and/or camera midway through generation. Camera at `event.multi_modal_data.camera` is structural SE3 (`mode` + `translation`/`rotation`). `event_id` is optional and `transition_chunks` defaults to the model setting. |
| Server to client | `video.start` | JSON text: `{"type":"video.start","request_id":"...","format":"m4s","config":{...}}` | Confirms the session and mirrors the accepted `format`. |
| Server to client | `video.chunk_metadata` | JSON text: `{"type":"video.chunk_metadata","request_id":"...","kind":"media","transport_chunk_index":0,"generation_chunk_index":0,"num_frames":9,"byte_length":1234,"started_event_ids":[],"active_event_ids":[],"completed_event_ids":[]}` | Precedes each binary frame and describes the immediately following payload. |
| Server to client | Video chunk | Binary WebSocket frame | Fragmented MP4 (`m4s`) video bytes. |
@@ -59,10 +59,10 @@ python streaming_video_client.py \
--output helios_stream.mp4
```
The client sends one `session.start` message, prints each received binary video chunk with its byte size and elapsed time, and saves the received bytes to `--output` after `session.done`.
The example client sends one `session.start` message, prints each received binary video chunk with its byte size and elapsed time, and saves the received bytes to `--output` after `session.done`.
The client remuxes the gathered stream to a regular progressive MP4 file so that local playback knows the video duration.
Schedule midway prompt updates with `--prompt-updates`. None are sent unless you ask for them, and only pipelines that implement midway prompt updates accept them (LingBot-World, for one, does not, and rejects the whole session). Each entry uses `"at"` as seconds on the client clock after the server sends `video.start`:
Schedule midway prompt updates with `--prompt-updates`. None are sent unless you ask for them, and only pipelines that implement midway prompt updates accept them. Each entry uses `"at"` as seconds on the client clock after the server sends `video.start`:
```bash
python streaming_video_client.py \
@@ -73,6 +73,21 @@ python streaming_video_client.py \
]'
```
Schedule midway camera updates with `--camera-updates` when the served model
registers the camera modality (e.g., LingBot World 2).
It accepts `["camera"]["data"]["translation"/"rotation"]` in the endpoint's required SE3 format.
It also accepts `["actions"]` at the update level and converts keystrokes to the SE3 format.
```bash
python streaming_video_client.py \
--model robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
--image-reference /path/to/first_frame.png \
--camera-updates '[
{"at": 1.0, "actions": ["w"]},
{"at": 3.0, "camera": {"mode": "target", "data": {"translation": [0.0, 0.0, 1.0], "rotation": [0.0, 0.0, 0.0, 1.0]}}, "transition_chunks": 2}
]'
```
The Helios demo continues the default prompt into an underwater storm:
```bash
@@ -144,6 +159,7 @@ python streaming_video_client.py \
--output lingbot_world_v2_stream.mp4
```
`camera_action_script` carries one three-latent-frame WASD action list per generated
chunk. Mid-session `session.interaction` only updates the prompt, so a LingBot rollout
follows the camera script it started with.
`camera_action_script` is an optional request-scoped WASD script (one
three-latent-frame action list per generated chunk). Prefer mid-session
`session.interaction` camera control via `--camera-updates` when you need
live motion; the engine wire format is structural SE3 only.
examples/online_serving/streaming_video_generation/camera_utils.py
+70
View File
@@ -0,0 +1,70 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Client-side helpers that map WASD/IJKL keys to structural SE3 camera payloads.
Convenience for demos.
"""
from __future__ import annotations
import math
from typing import Any
# Unity-frame (+X right, +Y up, +Z forward) step sizes used by the optional
# WASD helper. Magnitudes mirror LingBot's script integrator; the engine never
# sees key tokens — only structural SE3 payloads.
_WASD_STEP = 0.05
_WASD_PITCH_DEG = 4.0
_WASD_YAW_DEG = 6.0
# Match LingBot's per-frame pitch clamp; without absolute pose this only limits
# a single helper call, not cumulative held velocity.
_WASD_PITCH_LIMIT_DEG = 85.0
def _quat_from_axis_angle(axis: tuple[float, float, float], degrees: float) -> list[float]:
"""Return a unit quaternion ``[x, y, z, w]`` for a right-handed axis-angle."""
radians = math.radians(degrees)
half = 0.5 * radians
sine = math.sin(half)
return [axis[0] * sine, axis[1] * sine, axis[2] * sine, math.cos(half)]
def wasd_to_camera_payload(actions: list[str], *, mode: str = "velocity") -> dict[str, Any]:
"""Map WASD/IJKL key tokens to a structural camera interaction payload.
This is a client convenience only. Servers reject ``data.actions``.
The delta is chosen so that at the identity pose, ``T_base @ T_delta``
matches LingBot's script integrator rotation
``R_y(yaw) @ R_x(pitch)``. Without the current absolute pose this remains
an approximation once the camera has already yawed/pitched.
"""
keys = {str(action).lower() for action in actions}
dx = _WASD_STEP * (("d" in keys) - ("a" in keys))
dy = 0.0
dz = _WASD_STEP * (("w" in keys) - ("s" in keys))
pitch = _WASD_PITCH_DEG * (("i" in keys) - ("k" in keys))
yaw = _WASD_YAW_DEG * (("l" in keys) - ("j" in keys))
pitch = max(-_WASD_PITCH_LIMIT_DEG, min(_WASD_PITCH_LIMIT_DEG, pitch))
# R_delta = R_y(yaw) @ R_x(pitch): apply pitch first, then yaw.
qx, qy, qz, qw = 0.0, 0.0, 0.0, 1.0
if abs(pitch) > 0.0:
px, py, pz, pw = _quat_from_axis_angle((1.0, 0.0, 0.0), pitch)
qx, qy, qz, qw = px, py, pz, pw
if abs(yaw) > 0.0:
yx, yy, yz, yw = _quat_from_axis_angle((0.0, 1.0, 0.0), yaw)
# q = yaw ⊗ pitch
qx, qy, qz, qw = (
yw * qx + yx * qw + yy * qz - yz * qy,
yw * qy - yx * qz + yy * qw + yz * qx,
yw * qz + yx * qy - yy * qx + yz * qw,
yw * qw - yx * qx - yy * qy - yz * qz,
)
return {
"mode": mode,
"data": {
"translation": [dx, dy, dz],
"rotation": [qx, qy, qz, qw],
},
}
examples/online_serving/streaming_video_generation/streaming_video_client.py
+123 -5
View File
@@ -26,6 +26,7 @@ from pathlib import Path
from typing import Any
import av
from camera_utils import wasd_to_camera_payload
try:
from websockets.asyncio.client import connect # pyright: ignore[reportMissingImports]
@@ -56,6 +57,15 @@ class ScheduledPromptUpdate:
transition_chunks: int
@dataclass(frozen=True)
class ScheduledCameraUpdate:
"""Client-side schedule entry for a midway camera ``session.interaction``."""
at: float
camera: dict[str, Any]
transition_chunks: int | None
def _image_reference(value: str) -> dict[str, Any]:
"""Accept an http(s) URL as-is, or inline a local image as a data URL."""
if value.startswith(("http://", "https://", "data:")):
@@ -119,6 +129,61 @@ def _parse_prompt_updates(value: str) -> list[ScheduledPromptUpdate]:
return sorted(updates, key=lambda update: update.at)
def _parse_camera_updates(value: str) -> list[ScheduledCameraUpdate]:
try:
parsed = json.loads(value)
except json.JSONDecodeError as exc:
raise argparse.ArgumentTypeError(f"--camera-updates must be valid JSON: {exc}") from exc
if not isinstance(parsed, list):
raise argparse.ArgumentTypeError("--camera-updates must be a JSON array")
updates: list[ScheduledCameraUpdate] = []
for index, item in enumerate(parsed):
if not isinstance(item, dict):
raise argparse.ArgumentTypeError(f"--camera-updates[{index}] must be a JSON object")
try:
at = float(item["at"])
except (KeyError, TypeError, ValueError) as exc:
raise argparse.ArgumentTypeError(f"--camera-updates[{index}] must have a valid 'at' field") from exc
if at < 0:
raise argparse.ArgumentTypeError(f"--camera-updates[{index}].at must be >= 0")
if "camera" in item:
camera = item["camera"]
if not isinstance(camera, dict):
raise argparse.ArgumentTypeError(f"--camera-updates[{index}].camera must be an object")
elif "actions" in item:
actions = item["actions"]
if not isinstance(actions, list):
raise argparse.ArgumentTypeError(f"--camera-updates[{index}].actions must be a list of WASD/IJKL keys")
mode = item.get("mode", "velocity")
if mode not in ("target", "velocity"):
raise argparse.ArgumentTypeError(f"--camera-updates[{index}].mode must be target or velocity")
camera = wasd_to_camera_payload(actions, mode=mode)
else:
raise argparse.ArgumentTypeError(
f"--camera-updates[{index}] must provide 'camera' (structural SE3) "
"or 'actions' (client-side WASD helper)"
)
transition_chunks: int | None
if "transition_chunks" not in item:
transition_chunks = None if camera.get("mode") == "velocity" else DEFAULT_TRANSITION_CHUNKS
else:
try:
transition_chunks = int(item["transition_chunks"])
except (TypeError, ValueError) as exc:
raise argparse.ArgumentTypeError(
f"--camera-updates[{index}].transition_chunks must be an integer"
) from exc
if transition_chunks < 0:
raise argparse.ArgumentTypeError(f"--camera-updates[{index}].transition_chunks must be >= 0")
updates.append(ScheduledCameraUpdate(at=at, camera=camera, transition_chunks=transition_chunks))
return sorted(updates, key=lambda update: update.at)
def _maybe_set(payload: dict[str, Any], key: str, value: Any) -> None:
if value is not None:
payload[key] = value
@@ -173,7 +238,7 @@ async def _run_prompt_update_scheduler(
payload = {
"type": "session.interaction",
"interaction": {
"event_id": f"scheduled-{update.at:.3f}",
"event_id": f"scheduled-prompt-{update.at:.3f}",
"event": {"prompt": update.prompt},
"transition_chunks": update.transition_chunks,
},
@@ -183,6 +248,30 @@ async def _run_prompt_update_scheduler(
print(f"Sent session.interaction at t={update.at:.2f}s: {json.dumps(payload, ensure_ascii=False)}")
async def _run_camera_update_scheduler(
websocket: Any,
updates: list[ScheduledCameraUpdate],
*,
video_started_at: float,
send_lock: asyncio.Lock,
) -> None:
for update in updates:
delay = (video_started_at + update.at) - time.perf_counter()
if delay > 0:
await asyncio.sleep(delay)
interaction: dict[str, Any] = {
"event_id": f"scheduled-camera-{update.at:.3f}",
"event": {"multi_modal_data": {"camera": update.camera}},
}
if update.transition_chunks is not None:
interaction["transition_chunks"] = update.transition_chunks
payload = {"type": "session.interaction", "interaction": interaction}
async with send_lock:
await websocket.send(json.dumps(payload, ensure_ascii=False))
print(f"Sent session.interaction at t={update.at:.2f}s: {json.dumps(payload, ensure_ascii=False)}")
def _count_decoded_frames(video_bytes: bytes, *, stream_format: str) -> int | None:
"""Return the number of decodable video frames in a streamed byte sequence, if known."""
if not video_bytes:
@@ -241,6 +330,7 @@ async def stream_video(args: argparse.Namespace) -> None:
ws = None
send_lock = asyncio.Lock()
prompt_update_task: asyncio.Task[None] | None = None
camera_update_task: asyncio.Task[None] | None = None
pending_chunk_metadata: dict[str, Any] | None = None
try:
@@ -295,8 +385,8 @@ async def stream_video(args: argparse.Namespace) -> None:
if msg_type == "video.start":
stream_format = msg.get("format") or stream_format
print(f"Video session started: request_id={msg.get('request_id')} format={msg.get('format')}")
video_started_at = time.perf_counter()
if args.prompt_updates and prompt_update_task is None:
video_started_at = time.perf_counter()
prompt_update_task = asyncio.create_task(
_run_prompt_update_scheduler(
websocket,
@@ -305,6 +395,15 @@ async def stream_video(args: argparse.Namespace) -> None:
send_lock=send_lock,
)
)
if args.camera_updates and camera_update_task is None:
camera_update_task = asyncio.create_task(
_run_camera_update_scheduler(
websocket,
args.camera_updates,
video_started_at=video_started_at,
send_lock=send_lock,
)
)
elif msg_type == "video.chunk_metadata":
# Save the metadata for the incoming video chunk
pending_chunk_metadata = msg
@@ -315,6 +414,8 @@ async def stream_video(args: argparse.Namespace) -> None:
done = True
if prompt_update_task is not None:
prompt_update_task.cancel()
if camera_update_task is not None:
camera_update_task.cancel()
break
elif msg_type == "error":
print(f"ERROR: {msg.get('message', msg)}")
@@ -335,6 +436,10 @@ async def stream_video(args: argparse.Namespace) -> None:
prompt_update_task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await prompt_update_task
if camera_update_task is not None:
camera_update_task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await camera_update_task
print(
"Saving video... (May take a while. Remuxing concatenated chunks into one progressive MP4 file with total duration metadata in file header)"
)
@@ -405,9 +510,9 @@ def parse_args() -> argparse.Namespace:
default=None,
help="JSON object merged into extra_params; overrides preset keys on conflict.",
)
# Midway prompt updates are pipeline-specific: a model that does not implement
# them (LingBot-World, for one) rejects the whole session on the first one, so
# nothing is scheduled unless the caller asks for it.
# Midway interactions are pipeline-specific: unsupported modalities reject
# the update (and may leave the session running). Nothing is scheduled unless
# the caller asks for it.
parser.add_argument(
"--prompt-updates",
type=_parse_prompt_updates,
@@ -419,6 +524,19 @@ def parse_args() -> argparse.Namespace:
f"'transition_chunks' defaults to {DEFAULT_TRANSITION_CHUNKS}."
),
)
parser.add_argument(
"--camera-updates",
type=_parse_camera_updates,
default=[],
help=(
"JSON array of scheduled camera updates for models that register the "
"camera modality (e.g. LingBot-World). Each object needs 'at' and either "
"'camera' (structural SE3: mode + translation/rotation) or "
"'actions' (client-side WASD/IJKL helper converted before send). "
"Optional 'transition_chunks' defaults to None for velocity and "
f"{DEFAULT_TRANSITION_CHUNKS} for target."
),
)
return parser.parse_args()
examples/online_serving/text_to_speech/README.md
+19 -38
View File
@@ -13,7 +13,7 @@ For the full list of supported architectures across all modalities, see
## Supported Models
| Model | HuggingFace repo | Voice cloning | Streaming | Voice presets / upload | Gradio demo |
|---|---|---|---|---|---|
| --- | --- | --- | --- | --- | --- |
| Breeze-TTS-2 | `BreezeBlue/Breeze-TTS-2` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | speaker tags (`S0`..`S9`, default `S0`) | — |
| Audio8 TTS Preview | `Audio8/Audio8-TTS-Preview-0.6b` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | uploaded audio voice only; no presets | ✓ |
| Fish Speech S2 Pro | `fishaudio/s2-pro` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | ✓ |
@@ -27,7 +27,6 @@ For the full list of supported architectures across all modalities, see
| OmniVoice | `k2-fsa/OmniVoice` | ✓ | — | — | — |
| Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | ✓ (Base) | ✓ (PCM + WebSocket) | ✓ (presets + `/v1/audio/voices` upload) | ✓ (standard + FastRTC) |
| VoxCPM2 | `openbmb/VoxCPM2` | ✓ | ✓ (AudioWorklet via gradio) | — | ✓ |
| dots.tts | `dots-studio/dots.tts-soar` | — (text-only) | ✓ (PCM stream) | — (default placeholder only) | — |
| Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✓ (gated upstream) | ✓ | ✓ (presets) | ✓ |
CosyVoice3 is intentionally absent: no online example exists for it yet. See its [offline section](../../offline_inference/text_to_speech/README.md#cosyvoice3) instead.
@@ -143,42 +142,6 @@ python examples/online_serving/text_to_speech/gepard/speech_client.py \
- Concurrent requests at `max_num_seqs: 4` are supported. Native-AR recompute preemption is a known limitation of this architecture (a request that is preempted mid-generation can resume incorrectly); keep concurrency at or below `max_num_seqs` and treat preemption as out of scope until the platform fix lands.
- Optional comparison against the upstream Gepard reference server needs Blackwell/Hopper + CUDA 13 + Postgres and is not part of CI.
## dots.tts
Single-stage, text-only TTS at 48 kHz. The current integration supports
no-reference synthesis; voice cloning, reference audio, named voices, and
precomputed speaker embeddings are not supported yet.
### Launch
```bash
vllm serve dots-studio/dots.tts-soar --omni --trust-remote-code --port 8091
```
### Sending requests
```bash
# Non-streaming WAV
curl -X POST http://localhost:8091/v1/audio/speech \
-H "Content-Type: application/json" \
-d '{
"input": "Hello, this is a dots TTS online-serving test.",
"voice": "default",
"response_format": "wav"
}' --output output.wav
# Streaming 48 kHz mono PCM
curl -X POST http://localhost:8091/v1/audio/speech \
-H "Content-Type: application/json" \
-d '{
"input": "Hello, this is a streaming dots TTS test.",
"voice": "default",
"stream": true,
"stream_format": "audio",
"response_format": "pcm"
}' --no-buffer | play -t raw -r 48000 -e signed -b 16 -c 1 -
```
---
## GLM-TTS
@@ -781,6 +744,20 @@ vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni --port 8091
./qwen3_tts/run_server.sh Base
```
For a local deployment with multiple API frontend processes sharing one set
of TTS stage engines, add `--api-server-count`:
```bash
vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
--omni --api-server-count 2 --port 8091
```
This mode supports local EngineCore stages only; headless or remote stage
deployments are not supported. Runtime voice upload and deletion are disabled
with multiple API frontends because their registries are process-local.
Built-in voices, inline `ref_audio`, and voices restored from `custom_voice_dir`
at startup are supported.
### Executor backend
Single-GPU serves now default to the uniproc executor (lower IPC overhead, the Base cloning use case from [#2603](https://github.com/vllm-project/vllm-omni/issues/2603) / [#2604](https://github.com/vllm-project/vllm-omni/pull/2604)). `vllm_omni/deploy/qwen3_tts.yaml` is the only Qwen3-TTS deploy config; pass `--deploy-config <path>` to override.
@@ -835,6 +812,10 @@ checkpoint. When a request names an uploaded voice, the server infers
`task_type="Base"`. Built-in presets such as `vivian` and `ryan` remain
CustomVoice speakers and require a CustomVoice checkpoint.
The runtime upload and delete routes require a single API frontend; with
`--api-server-count > 1`, use inline `ref_audio` or restore precomputed voices
from `custom_voice_dir` at startup.
### Precomputed custom voices
For reused Base voice-cloning speakers, precompute the reference artifacts once
pyproject.toml
-2
View File
@@ -66,14 +66,12 @@ dev = [
"jiwer==4.0.0",
"zhon==2.1.1",
"zhconv==1.4.3",
"scipy==1.18.0",
"funasr==1.3.29",
"lpips==0.1.4", # for quantization fp8
"orjson==3.11.9", # to make v1/embeddings API fast
"voxcpm==2.0.3", # for VoxCPM2 path-locating in tests
"torchdiffeq==0.2.5", # for covo_audio Test
"FlagEmbedding==1.4.0", # for hunyuan_image3 tests
"s3tokenizer==0.3.0", # for tts test
# Provides top-level cosyvoice2; do not co-install stepaudio2-minicpmo,
# whose top-level stepaudio2 package conflicts with this distribution.
"step-audio2==1.0.0", # Step-Audio2 / MiniCPM-o 4.5 Token2Wav (flashcosyvoice / hyperpyyaml)
recipes/Bagel/BAGEL-7B-MoT.md
+13 -6
View File
@@ -120,12 +120,19 @@ python examples/offline_inference/image_to_image/image_edit.py \
The `--extra-args` JSON forwards BAGEL-specific parameters (e.g. `cfg_text_scale`,
`cfg_img_scale`, `cfg_interval`, `cfg_renorm_type`) into
`OmniDiffusionSamplingParams.extra_args` via the model-extras registry.
For BAGEL img2img, the pipeline derives the generated image size from the input
image: it preserves the input aspect ratio, aligns dimensions to the latent
stride, and applies the checkpoint size limit. Explicit `--height` and
`--width` values do not override that BAGEL-specific resize policy. Step-mode
admission resolves this effective size before batching so differently shaped
requests are scheduled separately.
For BAGEL img2img, an explicit `--height`/`--width` (or `size` on
`/v1/images/edits`) sets the output canvas, in both the single-stage and the
two-stage deployment; the input image is still resized to a stride-aligned size
for the VAE/ViT prefill. The latent grid needs multiples of 16, so a requested
side that is not one is floored to the next multiple (1000x700 generates
992x688) with a server-side warning, and the response reports the generated
size. A side above the checkpoint limit is rejected with HTTP 400. When no size
is requested
(`size=auto`, or no `height`/`width`), the pipeline derives the canvas from the
input image: it preserves the input aspect ratio, aligns dimensions to the
latent stride, and applies the checkpoint size limit. Step-mode admission
resolves this effective size before batching so differently shaped requests are
scheduled separately.
Run text-to-text with the shared understanding example. BAGEL's default
`bagel.yaml` deploy config is discovered from the checkpoint, so no model-specific
recipes/Boogu/Boogu-Image.md
+75 -41
View File
@@ -132,18 +132,19 @@ with `guidance_scale=1.0` remain valid on the same server: every rank evaluates
only the positive branch, no negative embeddings are built, and no guidance is
applied.
### 1 x RTX 4090 48 GB (FP8)
### 1 x RTX 4090 24 GB / 48 GB (FP8)
Boogu-Image supports two ways to quantize or load the DiT transformer:
Boogu-Image supports the following FP8 loading paths:
| Path | Base model | Edit model | Method | Scheme |
| ------------------ | -------------------------------- | -------------------------------- | ---------------------------- | ----------------------- |
| Serialized TorchAO | `Boogu/Boogu-Image-0.1-Base-fp8` | `Boogu/Boogu-Image-0.1-Edit-fp8` | `torchao_float8_weight_only` | FP8 weight-only (W8A16) |
| Native online FP8 | `Boogu/Boogu-Image-0.1-Base` | `Boogu/Boogu-Image-0.1-Edit` | `fp8` | Dynamic W8A8 |
| Path | Checkpoints | MLLM | DiT |
| --- | --- | --- | --- |
| Pre-quantized FP8 | `Boogu/Boogu-Image-0.1-{Base,Edit}-fp8` | HF checkpoint FP8 configuration | TorchAO FP8 weight-only (W8A16) |
| Online FP8 | `Boogu/Boogu-Image-0.1-{Base,Edit}` | Optional 128x128 block FP8: Triton W8A8 (vLLM 0.28.0); Marlin W8A16 (vLLM 0.29.0) | Native dynamic FP8 (CUTLASS W8A8) |
#### Environment
The following configuration was validated:
The following isolated environments were used for Base T2I and Edit I2I
measurements at 512x512:
- OS: Linux
- Python: 3.12
@@ -152,7 +153,9 @@ The following configuration was validated:
- vLLM-Omni version or commit: Use a commit that contains TorchAO FP8
checkpoint loading for diffusion models
- TorchAO version: 0.17.0
- `kernels` / `kernels-data`: 0.16.1 / matching kernels-data
- `kernels` / `kernels-data`: 0.16.1 / 0.16.1
- Transformers: 5.17.0
- Tokenizers: 0.23.1
#### Command
@@ -167,19 +170,61 @@ vllm serve Boogu/Boogu-Image-0.1-Base-fp8 \
'{"transformer":{"method":"torchao_float8_weight_only"}}'
```
Native online FP8 (currently applied only to eligible vLLM-quantizable linear
layers in the DiT transformer):
Native online FP8:
```bash
# DiT only
vllm serve Boogu/Boogu-Image-0.1-Base \
--omni \
--port 8091 \
--diffusion-quantization-config \
'{"transformer":{"method":"fp8"}}'
'{"mllm":null,"transformer":"fp8","vae":null}'
# MLLM + DiT
vllm serve Boogu/Boogu-Image-0.1-Base \
--omni \
--port 8091 \
--quantization fp8
# MLLM + DiT (using --diffusion-quantization-config)
vllm serve Boogu/Boogu-Image-0.1-Base \
--omni \
--port 8091 \
--diffusion-quantization-config \
'{"mllm":"fp8","transformer":"fp8","vae":null}'
```
For image editing, use the matching Edit model ID from the table and keep the
corresponding quantization configuration.
For image editing, replace `Base` with `Edit` and keep the corresponding
quantization configuration.
#### Notes
- **Memory usage:**
- The serialized TorchAO FP8 path uses approximately 21.1 GiB at 512x512
and 24.2-24.6 GiB at 1024x1024.
- The DiT-only online FP8 path uses approximately 26.9 GiB at 512x512
and 29.9-30.9 GiB at 1024x1024.
- The MLLM + DiT online FP8 path uses approximately 20.5-20.6 GiB at
512x512 and 23.6-24.5 GiB at 1024x1024.
- **Quantization scope:** For original Base/Edit checkpoints, `--quantization fp8`
quantizes the MLLM's language-model linear layers through per-block FP8
and the DiT's linear layers through native online FP8. The vision encoder,
embeddings, normalization layers and VAE remain unquantized.
- **Performance:** In the 48 GB Base/Edit benchmarks at 512x512, both DiT-only
and MLLM + DiT online FP8 reduced E2E latency by over 10 seconds compared with
official FP8 and vLLM-Omni pre-quantized FP8. The main runtime difference lies in the DiT quantization implementation, which switches from TorchAO weight-only FP8 (W8A16) to native CUTLASS
W8A8 kernels. This reduces additional dequantization overhead and enables
FP8 Tensor Core acceleration.
- **Known Limitation:**
- **Output consistency:** With MLLM + DiT online FP8, same-seed outputs may visibly differ from BF16 for some Base prompts, but these differences do not mean lower image quality.
- **Import compatibility conflict between TorchAO and Diffusers::** Loading pre-quantized checkpoints requires `torchao>=0.17.0`, which has a compatibility conflict with `diffusers==0.40.0`. This produces a warning when Diffusers is imported, but currently does not affect Boogu-Image weight loading.
- **Conflict between the default kernels and Transformers versions:** The default Transformers version range (`>=5.10.1,<5.15`) conflicts with `kernels==0.16.1` when loading pre-quantized FP8 MLLM weights through Hugging Face `FP8Linear`. When using this loading path with `kernels==0.16.1`, please upgrade Transformers to `5.17.0`.
#### Verification
@@ -190,19 +235,6 @@ ID passed to `vllm serve`. For example, use
corresponding Edit model.
Base text-to-image:
## Fast text-to-image (Boogu-Image-0.1-Turbo)
Turbo is the distilled text-to-image checkpoint. Its `model_index.json`
declares `BooguImageTurboPipeline`, which runs the four-step DMD path instead
of the scheduler-driven path used by Base.
### Command
```bash
vllm serve Boogu/Boogu-Image-0.1-Turbo --omni --port 8091
```
### Verification
```bash
curl -X POST http://localhost:8091/v1/images/generations \
@@ -231,22 +263,24 @@ curl -X POST http://localhost:8091/v1/images/edits \
| jq -r '.data[0].b64_json' | base64 -d > edited_fp8.png
```
#### Notes
## Fast text-to-image (Boogu-Image-0.1-Turbo)
- **Memory usage:**
- The serialized TorchAO FP8 path uses approximately 21.1 GiB at 512x512
and 24.2-24.6 GiB at 1024x1024.
- The native online FP8 path uses approximately 31.5 GiB for Base and
30.7 GiB for Edit at 1024x1024. It uses more memory because only eligible
DiT linear layers are quantized online, while the MLLM from the regular
Base/Edit checkpoint remains in BF16. Most of the observed difference
comes from the MLLM precision, although differences in the two DiT FP8
representations and kernels may also contribute.
- **Quantization scope:** The transformer-scoped configuration applies to the
DiT. With the serialized `-fp8` checkpoints, the MLLM follows its own
checkpoint-declared FP8 configuration. With the regular checkpoints used for
online FP8, the MLLM remains in BF16. The VAE and scheduler are outside this
quantization routing.
Turbo is the distilled text-to-image checkpoint. Its `model_index.json`
declares `BooguImageTurboPipeline`, which runs the four-step DMD path instead
of the scheduler-driven path used by Base.
### Command
```bash
vllm serve Boogu/Boogu-Image-0.1-Turbo --omni --port 8091
```
### Verification
```bash
curl -X POST http://localhost:8091/v1/images/generations \
-H "Content-Type: application/json" \
-d '{
"model": "Boogu/Boogu-Image-0.1-Turbo",
"prompt": "A mountain lake at sunset, photorealistic, cinematic lighting",
"size": "1024x1024",
recipes/BreezeBlue/Breeze-TTS-2.md
+25 -8
View File
@@ -30,10 +30,20 @@ Four prompt modes are selected automatically from the request fields:
| `ref_clone_tata` (clone) | `ref_audio` + `ref_text` + `input` | reference clip and its exact transcript |
| `ref_edit_tata` (voice direction) | reference trio + `instructions` | keep the reference timbre, direct the delivery |
Current scope: greedy sampling with `cfg_scale=1.0`; CFG ≠ 1.0 and
Current scope: greedy sampling with `cfg_scale=1.0` and a default
`repetition_penalty=1.1` on generated codebook-0 tokens; CFG ≠ 1.0 and
`negative_prompt` are rejected with explicit client errors until companion
request support lands.
The [official inference example](https://github.com/breezeblue-ai/breeze-tts/blob/main/infer.py)
also uses repetition penalty `1.1`, but its
[runtime defaults](https://github.com/breezeblue-ai/breeze-tts/blob/main/breeze_infer/runtime.py)
sample both the backbone and depth decoder with temperature `0.9` and top-k
`50`. The [official voice-design examples](https://huggingface.co/BreezeBlue/Breeze-TTS-2#-voice-design)
use CFG `4`. Those sampling and guidance settings are outside the current
port's scope, so the examples below do not establish parity with the official
voice-design outputs.
## Supported model contract
| Item | Value |
@@ -42,9 +52,9 @@ request support lands.
| Modes | plain, voice design (`instructions`), voice clone (`ref_audio` + `ref_text`), voice direction (reference + `instructions`) |
| Languages | English and Chinese (model card) |
| Reference audio | exactly one clip per request with its exact transcript; encoded by the bundled Qwen3-TTS tokenizer to 16 codebooks at 12.5 Hz |
| Output | 24 kHz mono; `wav` or `pcm`; streaming SSE `speech.audio.delta` |
| Output | 24 kHz mono; `wav` or `pcm`; raw audio streaming (`stream_format="audio"`) or SSE `speech.audio.delta` (`stream_format="sse"`) |
| Length | prompt bounded by stage-0 `max_model_len=4096`; synthesis bounded by `max_new_tokens` codec frames (default 2048, about 164 s) |
| Sampling | greedy (`temperature=0`) with `cfg_scale=1.0`; other guidance values and `negative_prompt` are rejected |
| Sampling | greedy (`temperature=0`) with `cfg_scale=1.0` and `repetition_penalty=1.1`; other guidance values and `negative_prompt` are rejected |
| Deployment profile | `vllm_omni/deploy/breeze_tts_2.yaml`: two stages on one GPU, async-chunk streaming with 8-frame codec chunks, stage 0 `gpu_memory_utilization=0.80`, stage 1 `0.15` |
## References
@@ -106,7 +116,7 @@ Voice design (instruction only, no reference audio):
curl -X POST http://localhost:8091/v1/audio/speech \
-H "Content-Type: application/json" \
-d '{
"input": "[laughs] Welcome to tonight's story time.",
"input": "(laugh) Welcome to our story time.",
"instructions": "A warm young woman, clear voice, lively delivery.",
"response_format": "wav"
}' --output breeze_design.wav
@@ -139,7 +149,7 @@ curl -X POST http://localhost:8091/v1/audio/speech \
}' --output breeze_direction.wav
```
Streaming PCM (SSE `speech.audio.delta` events):
Streaming raw PCM (`stream_format="audio"`):
```bash
curl -N -X POST http://localhost:8091/v1/audio/speech \
@@ -153,8 +163,9 @@ curl -N -X POST http://localhost:8091/v1/audio/speech \
}' --output breeze_stream.pcm
```
Expected results: finite, non-silent 24 kHz mono WAV files; streaming requests
return incremental PCM deltas. On 1x L20 (greedy, single request after
Expected results: finite, non-silent 24 kHz mono WAV files; the streaming example
returns raw PCM bytes. Use `stream_format="sse"` for `speech.audio.delta`
events containing base64-encoded audio. On 1x L20 (greedy, single request after
warm-up) a few seconds of speech completes in roughly 3.4–3.8 s wall time.
## Notes
@@ -166,6 +177,12 @@ warm-up) a few seconds of speech completes in roughly 3.4–3.8 s wall time.
supported yet.
- **CFG**: `guidance_scale`/`cfg_scale` must be `1.0`. Non-1.0 values and
`negative_prompt` return a client error.
- **Repetition penalty**: the default `1.1` matches upstream's penalty on
generated codebook-0 tokens and discourages repeated silence. Override it
with `extra_params: {"repetition_penalty": 1.0}` to disable it, or another
finite positive value. Text and reference audio are excluded from its history.
The legacy `tools/breeze_tts_2_upstream_golden.py` uses penalty `1.0`; set
the same serving override when comparing against those golden frames.
- **Length budget**: prompt length is bounded by stage 0 `max_model_len=4096`;
synthesis length by `max_new_tokens` (default 2048 frames ≈ 164 s at the
12.5 Hz codec frame rate) — set `max_new_tokens` for shorter caps.
@@ -181,7 +198,7 @@ warm-up) a few seconds of speech completes in roughly 3.4–3.8 s wall time.
| Feature | Status | Notes |
| ------- | ------ | ----- |
| Streaming output | ✓ | `async_chunk: true`, 8-frame inter-stage chunks, SSE `speech.audio.delta` PCM |
| Streaming output | ✓ | `async_chunk: true`, 8-frame inter-stage chunks; raw audio or SSE `speech.audio.delta` |
| Voice cloning | ✓ | one reference clip plus transcript per request |
| Voice design / voice direction | ✓ | `instructions` field, with or without a reference |
| Classifier-free guidance | ✗ | `cfg_scale` must be `1.0`; `negative_prompt` rejected (follow-up) |
recipes/MammothModa2/MammothModa2.md
+79 -28
View File
@@ -18,11 +18,18 @@ The generic example formats the AR prompt, drives the AR → DiT stage pipeline,
and forwards MammothModa2-specific generation parameters through the
pipeline-declared `extra_body` contract.
MammothModa2's DiT stage consumes its inputs through the multi-stage kwargs
interface (not `OmniDiffusionRequest`), so its generation knobs
(`text_guidance_scale`, `cfg_range`, `num_inference_steps`) are passed via
`--extra-body` rather than the standard `--num-inference-steps` / `--cfg-scale`
flags. Image size uses the standard `--height` / `--width` flags.
MammothModa2's DiT stage runs in the shared diffusion runtime in request mode.
The first integration intentionally supports one request and one image per
forward only (`max_num_seqs: 1`, `num_outputs_per_prompt: 1`). Request-level
batching, step execution, continuous batching, cache acceleration,
compilation, quantization, parallelism, and offload are not enabled by this
recipe.
Image size, seed, guidance, and denoising steps use the standard diffusion
request fields. `cfg_range` remains a MammothModa2-specific `extra_body`
parameter. For compatibility, the runtime also accepts the former
`text_guidance_scale` and `num_inference_steps` keys in `extra_body`; when
present and non-null, those keys take precedence over the standard fields.
## References
@@ -41,11 +48,11 @@ flags. Image size uses the standard `--height` / `--width` flags.
## Hardware Support
The default deploy config runs both the AR and DiT stages on a single GPU
(`devices: "0"`). The committed `gpu_memory_utilization` split (stage-0 AR `0.5`,
stage-1 DiT `0.3`) is sized for an ~80 GB GPU. The model also fits on a 48 GB GPU
after rebalancing the split so the AR weights (~23 GB) leave room for the KV
cache — see the note under *1x L40S 48GB*.
The default deploy config places both the AR and DiT stages on one GPU
(`devices: "0"`). Its committed `gpu_memory_utilization` split is 0.5 for
stage 0 and 0.3 for stage 1. The A800 validation section below also shows a
two-GPU placement with one stage per GPU for attributable timing and memory;
the measured results are summarized below.
## GPU
@@ -102,12 +109,13 @@ not implied.
> set the stage-0 (AR) value to `0.8` and the stage-1 (DiT) value to `0.16`
> before running. (On an ~80 GB GPU, leave the defaults unchanged.)
### 1x NVIDIA A800 80GB
#### Environment
- OS: Linux
- Python: Match the repository requirements for your checkout
- Driver / runtime: NVIDIA CUDA environment with one L40S 48 GB (verified) or an
~80 GB GPU for the default config
- Driver / runtime: NVIDIA CUDA environment with one A800 80 GB
- vLLM version: Match the repository requirements for your checkout
- vLLM-Omni version or commit: Use the commit you are deploying from
@@ -120,8 +128,7 @@ hf download bytedance-research/MammothModa2-Preview --local-dir ./MammothModa2-P
```
Run text-to-image with the shared offline example from the repository root. The
deploy config sets `trust_remote_code`, so no extra flag is needed. Forward the
MammothModa2 generation parameters as a JSON object through `--extra-body`:
deploy config sets `trust_remote_code`, so no extra flag is needed:
```bash
python examples/offline_inference/text_to_image/text_to_image.py \
@@ -130,23 +137,25 @@ python examples/offline_inference/text_to_image/text_to_image.py \
--prompt "A stylish woman riding a motorcycle in NYC, movie poster style" \
--height 1024 \
--width 1024 \
--extra-body '{"text_guidance_scale": 4.0, "cfg_range": [0.0, 1.0], "num_inference_steps": 50}' \
--seed 42 \
--guidance-scale 4.0 \
--num-inference-steps 50 \
--extra-body '{"cfg_range": [0.0, 1.0]}' \
--output mammoth_t2i.png
```
The `--extra-body` JSON forwards MammothModa2-specific parameters into
`OmniDiffusionSamplingParams.extra_args`. Keys are filtered against the model's
declared `extra_body_params` (see
The standard diffusion request fields are `height`, `width`, `seed`,
`guidance_scale`, and `num_inference_steps`; use their corresponding CLI flags
shown above. `--height` and `--width` must be multiples of 16.
`cfg_range` is the only recommended MammothModa2 field in `--extra-body`; it
sets the relative step range `[start, end]` over which CFG is applied (default
`[0.0, 1.0]`). For compatibility, `text_guidance_scale` and
`num_inference_steps` remain accepted `extra_body` aliases and, when non-null,
take precedence over the standard request fields. Model extras are filtered
against the declared `extra_body_params` (see
[`vllm_omni/model_extras/mammothmodal2_preview.py`](../../vllm_omni/model_extras/mammothmodal2_preview.py)),
so unknown keys for MammothModa2 are silently dropped:
- `text_guidance_scale` — classifier-free guidance scale for the DiT stage
(default `9.0`; CFG is active only when `> 1.0`).
- `cfg_range` — relative step range `[start, end]` over which CFG is applied
(default `[0.0, 1.0]`).
- `num_inference_steps` — number of DiT denoising steps (default `50`).
`--height` and `--width` must be multiples of 16.
so unknown MammothModa2 extras may be dropped.
Run text-to-text through the shared understanding example. It recognizes the
MammothModa2 checkpoint and automatically selects `mammoth_moda2_ar.yaml`:
@@ -177,7 +186,49 @@ ls -lh mammoth_t2i.png
python -c "from PIL import Image; print(Image.open('mammoth_t2i.png').size)"
```
### 1x AMD MI300X, MammothModa2 Preview
### 2x NVIDIA A800 80GB validation
Use one A800 per stage so AR and DiT memory and timing are attributable. The
per-stage override changes placement only; both stages remain single-rank.
```bash
VLLM_LOGGING_LEVEL=DEBUG vllm serve ./MammothModa2-Preview --omni \
--deploy-config vllm_omni/deploy/mammoth_moda2.yaml \
--stage-overrides '{"0":{"devices":"0"},"1":{"devices":"1"}}' \
--port 8099 \
--log-stats
```
Startup logs should identify stage 1 as `StageDiffusionClient` and resolve it
to `MammothModa2DiTPipeline`. `DiffusionEngine` step timing is a DEBUG-level,
per-request message, so it appears only after sending a text-to-image request
with `VLLM_LOGGING_LEVEL=DEBUG`; it is not a startup marker. Seeing the legacy
generation model runner for stage 1 is a failed migration.
#### Migration benchmark
The request-mode migration was checked on 2x NVIDIA A800 80GB PCIe with AR on
GPU 0 and DiT on GPU 1. Each revision ran one warmup followed by 10 serial
measured requests in the same initialized process. Both used BF16 eager mode,
1024x1024 output, 50 denoising steps, guidance scale 4.0, seed 42, and no
diffusion cache. The baseline was the pre-migration revision `caed3061`; the
candidate was `19de562a`. Lower latency is better.
| Metric | Baseline p50 | Baseline p95 | Candidate p50 | Candidate p95 |
| --- | ---: | ---: | ---: | ---: |
| End-to-end latency | 105.36 s | 106.01 s | 104.94 s | 105.79 s |
| AR stage latency | 86.97 s | 87.61 s | 86.26 s | 87.11 s |
| DiT stage latency | 18.27 s | 18.41 s | 18.60 s | 18.62 s |
Peak sampled device memory was 39,209 MiB on the AR GPU for both revisions.
The DiT GPU used 11,089 MiB for the baseline and 10,967 MiB for the candidate.
The candidate's shared runtime reported 372.02 ms p50 per denoising step and a
5.89 ms p50 AR-to-diffusion adapter time. All measured requests completed and
both revisions produced valid, prompt-aligned 1024x1024 RGB images. The small
latency differences are regression evidence, not a statistically significant
speedup claim.
### 1x AMD MI300X, MammothModa2 Preview (pre-migration baseline)
#### Environment
recipes/MiniMaxAI/MiniMax-H3-NPU-950PR.md
+183 -154
View File
@@ -1,195 +1,224 @@
# MiniMax-H3 on a single NPU 950PR
# MiniMax H3 on Ascend 950PR / 950DT
This recipe runs MiniMax-H3 with online INT8 quantization on one NPU 950PR
NPU (128 GB HiBL 1.0 HBM). It covers the single-card T2VA configuration at
1024x576. For the eight-card Atlas 800I A3 BF16 route at 768P, see
[MiniMax-H3-NPU.md](MiniMax-H3-NPU.md).
> Joint video and audio generation — Ascend NPU deployment guide for the
> four-card Ascend 950PR / 950DT route
## Capacity requirements
## Summary
| Resource | Requirement |
| --- | ---: |
| NPU | 1x NPU 950PR |
| NPU HBM | 128 GiB (131,072 MiB reported by `npu-smi`) |
| Observed HBM high-water mark | 118,442 MiB (90.4% of capacity) |
| Checkpoint storage | 135 GiB per partition, local disk strongly preferred |
| Container shared memory | 8 GiB, or a writable local filesystem for spill |
The observed high-water mark leaves about 12.3 GiB of headroom, so this card is
sized for exactly one resident task partition at 1024x576. Do not expect a
larger output shape or concurrency greater than one to fit.
Because a single 950PR holds the whole partition in HBM, this route does not
need layerwise offload, and the 200 GiB system-RAM floor carried by the 72 GiB
GPU recipes does not apply. The validated run completed inside a container
limited to 32 GiB of RAM. Passing `--enable-layerwise-offload` on this card is
actively harmful: it stages weights in non-reclaimable host memory and the
container's OOM killer terminates the server with exit code -9.
- Vendor: MiniMaxAI
- Model: [`MiniMaxAI/MiniMax-H3`](https://huggingface.co/MiniMaxAI/MiniMax-H3)
- Tasks: T2VA, FL2VA, and Ref2VA
- Mode: OpenAI-compatible `/v1/videos` HTTP serving
- Hardware: Ascend 950PR / Ascend 950DT, 4x NPU (128 GB HBM per device)
- Maintainer: Community
This recipe covers the four-card 950PR / 950DT configuration at 1344x768.
For the eight-card Atlas 800I A2 / A3 route, see
[MiniMax-H3-NPU.md](MiniMax-H3-NPU.md); checkpoint layout, container
preparation, MindIE-SD installation, and the ffmpeg/decord dependencies are
described there and apply unchanged.
## Environment
- Host architecture: x86_64
- driver: 25.7.rc1.6 (hal 7.35.23)
- firmware: 9.0.0.105.229
- CANN toolkit: 9.1.0 (`/usr/local//cann-9.1.0`)
- npu-smi: 25.7.rc1.6
- Python: 3.12.13
- PyTorch: 2.10.0+cpu
- torch_npu: 2.10.0.post2
- vLLM: 0.26.0
- vLLM-Omni: 0.26.1.dev103+g584d78c67.npu (commit `584d78c6`)
Install vLLM-Omni from a checkout with MiniMax-H3 support:
Use the A5 variant of the official vLLM-Omni NPU images (see
[NPU installation](../../docs/getting_started/installation/npu.md)):
```bash
uv venv
source .venv/bin/activate
uv pip install -e .
export IMAGE=quay.io/ascend/vllm-omni:v0.29.0-a5
```
Check the [tag list](https://quay.io/repository/ascend/vllm-omni?tab=tags)
for newer releases. Inside the container, install MindIE-SD and the optional
Ref2VA media dependencies as described in
[MiniMax-H3-NPU.md § Environment](MiniMax-H3-NPU.md#environment).
## Start a server
Task selection follows the eight-card recipe: pass `--task-type fl2va` or
`--task-type ref2va` to pick the task partition. The configurations below use
4-card USP, 4-card text-encoder TP, and VAE `tile` parallelism.
The two recommended configurations differ in how they fit memory: the
lossless configuration enables distributed layerwise offload (DLO) to avoid
activation OOM at longer durations, while the lossy configuration fits the
maximum supported MiniMax-H3 workload shape without OOM, so DLO is omitted
for performance.
### Recommended lossless configuration (Ascend 950PR / 950DT)
```bash
export MODEL=/path/to/MiniMax-H3/FL2VA
export PORT=8000
export PORT=9098
export MODEL=/path/to/MiniMax-H3
export VLLM_WORKER_MULTIPROC_METHOD=spawn
export VLLM_OMNI_VIDEO_SYNC_TIMEOUT=14400
export VLLM_OMNI_VIDEO_SYNC_TIMEOUT=4000
export PYTHONDONTWRITEBYTECODE=1
export HF_HUB_OFFLINE=1
export TRANSFORMERS_OFFLINE=1
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export HCCL_NPU_SOCKET_PORT_RANGE="auto"
vllm serve "${MODEL}" \
--omni \
--host 0.0.0.0 \
--port "${PORT}" \
--trust-remote-code \
--num-gpus 1 \
--tensor-parallel-size 1 \
--usp 1 \
--task-type fl2va \
--num-gpus 4 \
--usp 4 \
--ring 1 \
--text-encoder-tp-size 1 \
--vae-patch-parallel-size 1 \
--text-encoder-tp-size 4 \
--vae-parallel-mode tile \
--vae-use-tiling \
--init-timeout 14400 \
--stage-init-timeout 14400 \
--quantization int8
--vae-patch-parallel-size 4 \
--enable-diffusion-pipeline-profiler \
--enable-distributed-layerwise-offload \
--diffusion-attention-backend FLASH_ATTN
```
Do not pass `--diffusion-attention-backend CUDNN_ATTN`. cuDNN attention has no
NPU implementation and fails at the first denoise step. Leave the backend
unset: it resolves to `TORCH_SDPA`, or to `FLASH_ATTN` when MindIE-SD is
installed (see [MiniMax-H3-NPU.md § Environment](MiniMax-H3-NPU.md#environment)).
H3 is CFG-distilled, so `--cfg-parallel-size` must remain 1. The first
request includes regional compilation; warm the server once before measuring
steady-state latency.
H3 is CFG-distilled, so `--cfg-parallel-size` must remain 1.
### Recommended lossy configuration (Ascend 950PR / 950DT)
## Validated evidence
Measured on one NPU 950PR with the server command above plus
`--enable-diffusion-pipeline-profiler`, generating a 5 s 1024x576 T2VA clip from
the request in [§ T2VA request example](#t2va-request-example). MiniMax-H3
requested 60 denoise steps and executed 59 denoise updates, so per-step latency
is `denoise / 59`. `ffprobe` confirms every returned MP4 is 1024x576, 24 fps,
124 video frames, 5.175 s, with a 32 kHz audio track of 165,600 samples.
Five requests were issued back to back with no idle time between them. The card
is measurably faster on the first request than in steady state, so both regimes
are reported.
| Stage | First request, cold card | Thermal steady state |
| --- | ---: | ---: |
| End-to-end request | 462.13 s | 506.38 s |
| Text encode | 0.55 s | 0.05 s |
| Denoise (59 updates) | 429.29 s | 471.74 s |
| Per denoise update | 7,276 ms | 7,996 ms |
| VAE decode | 8.43 s | 8.14 s |
| Worker-to-server handoff | 24 s | 27 s |
| MP4 muxing | 0.57 s | 0.50 s |
| Server-reported `denoise_step_latency_ms` | 7,702 ms | 8,440 ms |
| Peak HBM (`npu-smi`) | 118,442 MiB | 118,442 MiB |
| Average NPU power | 566.9 W | 556.9 W |
| Peak NPU temperature | 104 C | 104 C |
Sample counts differ per row. The cold column is a single request. In the steady
column, text encode, denoise and VAE decode are means over the four subsequent
requests — denoise spanned 470.37 to 473.21 s, a 0.60% spread — while
end-to-end, handoff and muxing are means over the two of those four that
survived the handoff timeout described above.
**Report the steady-state column.** The cold card completes denoise 9.9% faster
while drawing only 1.8% more average power, and both regimes reach the same
104 C ceiling, so the first request is buying a short window of higher clocks
before the die saturates. Any benchmark that issues a single request against an
idle card will overstate throughput by about 10%.
**Do not read `denoise_step_latency_ms` as a per-step time.** The server divides
the entire stage wall time by the 60 *requested* steps, so it absorbs text
encoding, VAE decode and the 837 MiB handoff. All three successful requests
satisfy `denoise_step_latency_ms == e2e_stage_wall_time_ms / 60` exactly. The
per-update figure in the table is `diffuse / 59` from the pipeline profiler.
Denoise accounts for 93% of the steady-state request and the handoff accounts
for 5.3%, so keeping the transfer in shared memory should bring end-to-end down
to roughly 480 s.
Peak HBM is sampled externally with `npu-smi` at a 3 s interval. It is a
reserved high-water mark rather than live allocation: it reads 111,272 MiB after
weight loading and before the first request, and does not fall back after
requests complete. Re-measure for longer outputs, a different output shape, or
concurrency greater than one.
### Startup
Weight loading took **31 min 50 s** on the validated host (first log line
08:04:59, `Application startup complete` 08:36:49), with the checkpoint on a
shared GlusterFS network mount. This is why the server command sets
`--init-timeout 14400` and `--stage-init-timeout 14400`; the stock 600 s and
1800 s timeouts both abort mid-load with `TimeoutError` and exit code 143.
Reduce both timeouts when the partition is staged on local disk.
## T2VA request example
Adds EQBSA sparse attention (block-sparse with Q/K INT8 + V FP8 mixed
precision) and MXFP8 online quantization, and drops DLO:
```bash
export API_URL="http://127.0.0.1:${PORT}/v1/videos/sync"
export PORT=9098
export MODEL=/path/to/MiniMax-H3
export VLLM_WORKER_MULTIPROC_METHOD=spawn
export VLLM_OMNI_VIDEO_SYNC_TIMEOUT=4000
export PYTHONDONTWRITEBYTECODE=1
export HF_HUB_OFFLINE=1
export TRANSFORMERS_OFFLINE=1
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export HCCL_NPU_SOCKET_PORT_RANGE="auto"
curl -sS --max-time 1800 -X POST "${API_URL}" \
-F 'prompt=At night, three cats march into a bedroom playing tiny brass instruments, then abruptly file out, with synchronized room ambience.' \
-F 'width=1024' \
-F 'height=576' \
-F 'aspect_ratio=16:9' \
-F 'fps=24' \
-F 'num_inference_steps=60' \
-F 'flow_shift=12' \
-F 'seed=1101' \
-F 'extra_params={"task":"t2va","duration":5,"audio_flow_shift":3.0}' \
-o t2va.mp4
vllm serve "${MODEL}" \
--omni \
--host 0.0.0.0 \
--port "${PORT}" \
--trust-remote-code \
--task-type fl2va \
--num-gpus 4 \
--usp 4 \
--ring 1 \
--text-encoder-tp-size 4 \
--vae-parallel-mode tile \
--vae-use-tiling \
--vae-patch-parallel-size 4 \
--enable-diffusion-pipeline-profiler \
--diffusion-attention-config '{"default":{"backend":"RAINFUSION_ATTN","block_sparse":{"sparsity":0.8,"precision":"mix","start_step":8,"end_step":12}}}' \
--diffusion-quantization-config '{"transformer":{"method":"mxfp8"}}'
```
The recommended lossy configuration uses a **start=8 / end=12** dense-fallback
window; see below for more aggressive settings.
### Optional optimizations
The following sections list only the **increment or replacement** relative to
the recommended lossless configuration; environment variables and the
remaining flags stay unchanged.
#### Tuning the EQBSA dense-fallback window
`start_step` and `end_step` are the numbers of leading/trailing steps that
fall back to dense attention — not step indices. `precision` defaults to
`bf16`; EQBSA requires setting it to `mix` explicitly. For 1344x768, 50-step
T2VA:
| Configuration | `start_step` | `end_step` | Notes |
| --- | ---: | ---: | --- |
| Quality first (recommended) | 8 | 13 | General and complex-motion scenes |
| Balanced | 0 | 13 | Balanced |
| Speed first | 0 | 0 | Simple, low-motion scenes |
When frames look discontinuous or details unstable, increase `end_step`
first, then `start_step`; do not compensate by lowering `sparsity` alone.
These pairings were validated at 1344x768 / 50 steps only; re-evaluate for
other resolutions or step counts.
#### Cache-DiT
Append to enable DiT block caching with TaylorSeer extrapolation — see the
[Cache-DiT guide](../../docs/user_guide/diffusion/cache_acceleration/cache_dit.md):
```bash
--cache-backend cache_dit \
--enable-cache-dit-summary \
--cache-config '{"Fn_compute_blocks":2,"Bn_compute_blocks":1,"max_warmup_steps":4,"residual_diff_threshold":0.4,"max_continuous_cached_steps":4,"enable_taylorseer":true,"taylorseer_order":2}'
```
#### MXFP4 online quantization
Append to switch the DiT to MXFP4 online quantization:
```bash
--diffusion-quantization-config '{"transformer":{"method":"mxfp4"}}'
```
and remove `--enable-distributed-layerwise-offload`.
## Request examples
Identical to the eight-card recipe; see
[MiniMax-H3-NPU.md § HTTP API examples](MiniMax-H3-NPU.md#http-api-examples)
and the GPU recipe's full parameter table.
## Benchmarks (Ascend 950PR)
Measured with `--enable-diffusion-pipeline-profiler` on vLLM-Omni 0.28.0:
lossless rows use the recommended lossless configuration (with DLO), lossy
rows the recommended lossy configuration (EQBSA + MXFP8, without DLO). Memory
figures are per device.
| Task | Resolution | Frames | Duration (s) | Config | E2E (s) | DiT total (s) | DiT per-step (s) | DiT steps | Text encode (s) | Ref video encode (s) | Ref audio encode (s) | VAE decode (s) | Ref preprocess (s) | Post-process (s) | CPU MP4 (s) | Resident weights (GB) | Peak memory (GB) |
| ---- | ---- | ---- | ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
| t2va | 1344x768 | 124 | 5 | lossless | 203.95 | 198.10 | 4.04 | 49 | 0.57 | NA | NA | 5.05 | NA | 0.001 | 0.23 | 3.97 | 22.2 |
| t2va | 1344x768 | 362 | 15 | lossless | 1489.17 | 1474.35 | 30.08 | 49 | 0.69 | NA | NA | 13.45 | NA | 0.001 | 0.82 | 3.97 | 32.14 |
| t2va | 1344x768 | 124 | 5 | lossy | 113.86 | 109.81 | 2.24 | 49 | 0.03 | NA | NA | 3.82 | NA | 0.001 | 0.21 | 56.99 | 64.38 |
| t2va | 1344x768 | 362 | 15 | lossy | 762.17 | 749.40 | 15.29 | 49 | 0.03 | NA | NA | 12.19 | NA | 0.001 | 0.61 | 56.99 | 74.26 |
| ref2va | 1344x768 | 124 | 5 | lossless | 745.43 | 730.22 | 14.9 | 49 | 1.16 | 8.15 | 0.13 | 5.12 | 0.41 | 0.001 | 0.28 | 3.97 | 23.82 |
| ref2va | 1344x768 | 362 | 15 | lossless | 5887.24 | 5847.57 | 119.33 | 49 | 2.54 | 22.07 | 0.23 | 13.34 | 0.81 | 0.001 | 0.70 | 3.97 | 32.53 |
| ref2va | 1344x768 | 124 | 5 | lossy | 400.40 | 387.82 | 7.91 | 49 | 0.95 | 7.05 | 0.09 | 3.91 | 0.41 | 0.001 | 0.21 | 56.6 | 66.03 |
| ref2va | 1344x768 | 362 | 15 | lossy | 3037.84 | 3001.31 | 61.24 | 49 | 2.28 | 20.64 | 0.15 | 12.1 | 0.81 | 0.001 | 0.70 | 56.6 | 74.76 |
The resident-weight figures reflect the DLO strategy difference: with DLO
(lossless rows) DiT weights live in host memory and only the active layer
stays on the device, while without DLO (lossy rows) the quantized weights are
resident in HBM. Peak memory stays well inside the 128 GB per-device capacity
in both cases. These numbers describe the validated shapes rather than a
general throughput guarantee.
## Known limitations
- INT8 online quantization is validated for T2VA on this card. Use BF16 for
FL2VA and Ref2VA, or re-measure before relying on it.
- Single-card serving loads one task partition at a time. For Ref2VA, stop the
server and restart it against the `Ref2VA` directory.
- Sustained load drives the die to 104 C and costs about 10% of denoise
throughput relative to a cold card. Warm the server with at least one full
request before measuring, and treat steady state as the reportable number.
- Requests can fail during result handoff on hosts where the IPC spill path is
slow. See [§ Result handoff and shared
memory](#result-handoff-and-shared-memory).
- Loading the checkpoint from a network mount takes over 30 minutes and forces
very large init timeouts. Stage the partition on local disk when possible.
- The configuration measured here is 1024x576 at 60 steps, which differs from
the 1344x768 at 50 steps used by the GPU recipes in this directory. Denoise
cost scales superlinearly with token count, so the numbers are not directly
comparable across recipes.
- The image ships a `triton` package whose backend is not built
(`No module named 'triton._C.libtriton.'`). vLLM logs this as an error
at startup and disables Triton; the diffusion path does not need it and the
run completes normally.
- Task serving is partitioned by `--task-type`: T2VA/FL2VA and Ref2VA load
different DiTs, so switching between them requires a restart with the other
partition.
- H3 currently executes one generation request per diffusion batch.
- The first regional-compile request is a warmup and should not be included
in steady-state performance measurements.
- VAE patch parallelism requires size 1 or the full DiT group size and
supports the H3 native `tile` mode only.
- The lossy configuration (EQBSA + MXFP8) is validated for T2VA and Ref2VA;
use the lossless configuration for FL2VA or re-validate first.
- The lossless configuration trades throughput for activation headroom via
DLO; if your workload never approaches the maximum duration, dropping
`--enable-distributed-layerwise-offload` may improve latency at the risk of
OOM on longer durations.
## Additional resources
- [MiniMax-H3.md](MiniMax-H3.md) — full GPU guide
- [MiniMax-H3-NPU.md](MiniMax-H3-NPU.md) — eight-card Atlas 800I A3 BF16 guide
- [Int8 quantization](../../docs/user_guide/quantization/int8.md)
- [MiniMax-H3-NPU.md](MiniMax-H3-NPU.md) — eight-card Atlas 800I A2 / A3
guide (checkpoint, container, and dependency details)
- [NPU installation](../../docs/getting_started/installation/npu.md)
- [RainFusion attention](../../docs/user_guide/diffusion/attention_backends/rainfusion.md)
— block-sparse knobs and tuning
- [Online quantization](../../docs/user_guide/quantization/online.md)
— INT8 / MXFP8 / MXFP4
- [Cache-DiT guide](../../docs/user_guide/diffusion/cache_acceleration/cache_dit.md)
- [Supported models](../../docs/models/supported_models.md)
- [Video API](../../docs/serving/videos_api.md)
recipes/MiniMaxAI/MiniMax-H3-NPU.md
+294 -162
View File
@@ -1,7 +1,8 @@
# MiniMax H3 on Ascend NPU
> Joint video and audio generation with text, first/last keyframes, and mixed
> image/video/audio references — Ascend NPU deployment guide
> image/video/audio references — Ascend NPU deployment guide for Atlas 800I A2
> and Atlas 800I A3
## Summary
@@ -9,7 +10,7 @@
- Model: [`MiniMaxAI/MiniMax-H3`](https://huggingface.co/MiniMaxAI/MiniMax-H3)
- Tasks: T2VA, FL2VA, and Ref2VA
- Mode: OpenAI-compatible `/v1/videos` HTTP serving
- Hardware: Atlas 800I A3
- Hardware: Atlas 800I A2 / Atlas 800I A3, 8x NPU
- Maintainer: Community
This recipe adapts [MiniMax-H3.md](MiniMax-H3.md) for Ascend NPU
@@ -22,103 +23,288 @@ environments. Differences from the GPU path:
automatically falls back to soundfile / ffmpeg for wav/mp3/m4a/mp4 audio
inputs.
For the four-card Ascend 950PR / 950DT route, see
[MiniMax-H3-NPU-950PR.md](MiniMax-H3-NPU-950PR.md).
## Prerequisites
### Checkpoint
Same as the GPU recipe — Hugging Face access approval is required. Authenticate
once; `vllm serve` downloads the required nested components automatically:
Download the weights once; `vllm serve` accepts the repository ID or a local
path and resolves the nested components automatically:
```bash
hf auth login
export MODEL=MiniMaxAI/MiniMax-H3
export MODEL=/path/to/MiniMax-H3
hf download MiniMaxAI/MiniMax-H3 --local-dir "${MODEL}"
```
The checkpoint directory layout is:
```text
MiniMax-H3/
├── FL2VA/ # shared by T2VA/FL2VA tasks
├── Ref2VA/ # Ref2VA task weights
└── model_index.json
```
### Environment
- Ascend driver & firmware: 25.5.1
- CANN toolkit: 9.0.1
- Python: 3.12
- PyTorch: 2.10.0+cpu
- torch_npu: 2.10.0.post2
- Install vLLM-Omni from a checkout with MiniMax H3 support:
The recommended deployment uses the official vLLM-Omni NPU images, which ship
the matching vLLM / vLLM-Ascend / vLLM-Omni stack (see
[NPU installation](../../docs/getting_started/installation/npu.md)). Pick the
image matching the hardware — `quay.io/ascend/vllm-omni:v0.29.0` for
Atlas 800I A2, `quay.io/ascend/vllm-omni:v0.29.0-a3` for Atlas 800I A3 — and
check the
[tag list](https://quay.io/repository/ascend/vllm-omni?tab=tags) for newer
releases:
```bash
uv venv
source .venv/bin/activate
uv pip install -e .
export IMAGE=quay.io/ascend/vllm-omni:v0.29.0-a3
export CONTAINER_NAME=h3
docker run -it -u root --name ${CONTAINER_NAME} \
--privileged=true \
--shm-size=2000g \
--net=host \
--device /dev/davinci0 \
--device /dev/davinci1 \
--device /dev/davinci2 \
--device /dev/davinci3 \
--device /dev/davinci4 \
--device /dev/davinci5 \
--device /dev/davinci6 \
--device /dev/davinci7 \
--device /dev/davinci_manager \
--device /dev/devmm_svm \
--device /dev/hisi_hdc \
-v /home:/home \
-v /usr/local/dcmi:/usr/local/dcmi \
-v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
-v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
-v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
-v /etc/ascend_install.info:/etc/ascend_install.info \
-v /root/.cache:/root/.cache \
${IMAGE} \
/bin/bash
```
Install the **mindie-sd** operator library to enable Ascend-optimized fused
operators (`adalayernorm`, etc.) and the RainFusion `rf_v2` block-sparse
attention kernel:
Inside the container:
- **MindIE-SD** provides the Ascend-optimized fused operators
(`adalayernorm`, etc.), the LaserAttention kernel, and the RainFusion
block-sparse attention kernel. If the image does not ship it, build it from
source:
```bash
git clone https://gitcode.com/Ascend/MindIE-SD.git && cd MindIE-SD
# Comment out the tik_ops build step (not needed for this use case)
sed -i 's|^\(\s*\)source ${current_script_dir}/build_tik_ops.sh|\1# source ${current_script_dir}/build_tik_ops.sh|' build/build_ops.sh
git clone https://gitcode.com/Ascend/MindIE-SD.git
cd MindIE-SD
python setup.py bdist_wheel
cd dist
pip install mindiesd-*.whl
pip install dist/mindiesd-*.whl
```
- `ffmpeg` and `ffprobe` must be available on `PATH`. They are used for
reference-video preparation and MP4 output.
- Reference-video decoding uses `decord` when available and falls back to
PyAV otherwise.
- Audio inputs do not require TorchCodec: wav/mp3/m4a/mp4 files are loaded
at native sample rate through the soundfile / ffmpeg fallback.
- To track the latest vLLM-Omni, install it from a checkout:
```bash
git clone https://github.com/vllm-project/vllm-omni.git
cd vllm-omni
VLLM_OMNI_TARGET_DEVICE=npu pip install -e . --no-build-isolation
```
- `ffmpeg` and `ffprobe` must be available on `PATH` (reference-video
preparation and MP4 output). Reference-video decoding uses `decord` when
available and falls back to PyAV otherwise; `decord` is required only for
the Ref2VA workflow and can be built with:
```bash
apt update
apt install -y ffmpeg pkg-config libavcodec-dev libavformat-dev \
libavutil-dev libswscale-dev libavfilter-dev libavdevice-dev
git clone --depth 1 --recursive https://github.com/dmlc/decord.git
cd decord
mkdir -p build && cd build
cmake .. -DUSE_CUDA=0 -DDECODE_FFMPEG=1 -DCMAKE_BUILD_TYPE=Release
make
cd ../python
pip3 install . --no-build-isolation
```
## Start a server
Pass the repository ID directly. The pipeline loads the two nested DiTs while
sharing the tokenizer, processor, text encoder, and VAEs from `FL2VA`.
T2VA and FL2VA share one set of weights; the Ref2VA weights differ. Select
the task partition at startup with **`--task-type`** (`fl2va` or `ref2va`).
When unset, only the T2VA/FL2VA weights are loaded. All remaining flags are
task-agnostic.
### Multi-NPU: 768P combined-service configuration
### Recommended lossless configuration (Atlas 800I A2 / A3)
Use Ulysses sequence parallelism degree 8, text-encoder tensor parallelism
degree 8, native tiled VAE patch parallelism degree 8, and distributed
layerwise offload:
Lossless means fused operators plus the high-performance FLASH_ATTN backend
only. The configuration uses 8-card USP, 8-card text-encoder TP, distributed
layerwise offload (DLO), and VAE `tile` parallelism:
```bash
export ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export PORT=9098
export MODEL=MiniMaxAI/MiniMax-H3
export MODEL=/path/to/MiniMax-H3
export VLLM_WORKER_MULTIPROC_METHOD=spawn
export VLLM_OMNI_VIDEO_SYNC_TIMEOUT=1800
export VLLM_OMNI_VIDEO_SYNC_TIMEOUT=4000
export PYTHONDONTWRITEBYTECODE=1
export HF_HUB_OFFLINE=1
export TRANSFORMERS_OFFLINE=1
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export MINDIE_SD_FA_TYPE="ascend_laser_attention"
export HCCL_NPU_SOCKET_PORT_RANGE="auto"
vllm serve "${MODEL}" \
--omni \
--host 0.0.0.0 \
--port "${PORT}" \
--trust-remote-code \
--task-type fl2va \
--num-gpus 8 \
--usp 8 \
--ring 1 \
--text-encoder-tp-size 8 \
--enable-distributed-layerwise-offload \
--vae-parallel-mode tile \
--vae-use-tiling \
--vae-patch-parallel-size 8 \
--enable-distributed-layerwise-offload \
--enable-diffusion-pipeline-profiler \
--diffusion-attention-backend FLASH_ATTN
```
Do not add `--enforce-eager`. The first request includes regional
compilation; warm the server once before measuring steady-state latency.
H3 is CFG-distilled, so `--cfg-parallel-size` must remain 1.
Notes:
The same endpoint accepts `task=t2va`, `task=fl2va`, and `task=ref2va`; no
partition restart is required. Layerwise offload applies to both DiTs.
- Do not add `--enforce-eager`. The first request includes regional
compilation; warm the server once before measuring steady-state latency.
- H3 is CFG-distilled, so `--cfg-parallel-size` must remain 1.
- Keep `--ring 1` when switching to RainFusion attention (below): the
block-sparse kernel ranks key blocks over the whole sequence, so ring
parallelism would split away the keys it needs. Scale with `--usp` instead.
- DLO offloads DiT layers with parameters gathered across the parallel group
instead of replicated per rank, which is what fits the combined 768P
service into 64 GB HBM per device (Atlas 800I A3). Host (CPU) memory usage
is high with this option.
On Atlas 800I A3 (64 GB HBM per device) the combined service does not fit at
768P without offloading or sharding: use distributed layerwise offload (as
above) or HSDP — see
[§ Memory and attention optimizations](#memory-and-attention-optimizations-a3).
### Recommended lossy configuration (Atlas 800I A2 / A3)
### CPU MP4 response encoding (Atlas A2)
Adds RainFusion block-sparse attention and INT8 online quantization on top of
the lossless configuration. Validated for T2VA and Ref2VA; see
[Benchmarks](#benchmarks-atlas-800i-a2--a3):
```bash
vllm serve "${MODEL}" \
--omni \
--host 0.0.0.0 \
--port "${PORT}" \
--trust-remote-code \
--task-type fl2va \
--num-gpus 8 \
--usp 8 \
--ring 1 \
--text-encoder-tp-size 8 \
--vae-parallel-mode tile \
--vae-use-tiling \
--vae-patch-parallel-size 8 \
--enable-distributed-layerwise-offload \
--enable-diffusion-pipeline-profiler \
--diffusion-quantization-config '{"transformer":{"method":"int8"}}' \
--diffusion-attention-config '{"default": {"backend": "RAINFUSION_ATTN",
"block_sparse": {"sparsity": 0.8, "start_step": 12}}}'
```
Online INT8 quantization can be combined with distributed layerwise offload
while AllGather is enabled. Keep AllGather enabled for SP>1; use
`--dlo-no-use-allgather` only when collective communication is unavailable.
### Optional optimizations
The following sections list only the **increment or replacement** relative to
the recommended lossless configuration; environment variables and the
remaining flags stay unchanged.
#### Cache-DiT
Append to enable DiT block caching with TaylorSeer extrapolation — see the
[Cache-DiT guide](../../docs/user_guide/diffusion/cache_acceleration/cache_dit.md):
```bash
--cache-backend cache_dit \
--enable-cache-dit-summary \
--cache-config '{"Fn_compute_blocks":2,"Bn_compute_blocks":1,"max_warmup_steps":4,"residual_diff_threshold":0.4,"max_continuous_cached_steps":4,"enable_taylorseer":true,"taylorseer_order":2}'
```
#### EQBSA sparse attention
EQBSA applies Q/K INT8 with V FP8 mixed quantization on top of block-sparse
attention. Replace `--diffusion-attention-backend FLASH_ATTN` with:
```bash
--diffusion-attention-config '{"default":{"backend":"RAINFUSION_ATTN","block_sparse":{"sparsity":0.8,"start_step":8,"end_step":13,"precision":"mix"}}}'
```
`precision` defaults to `bf16`; enabling EQBSA requires setting it to `mix`
explicitly. `start_step` and `end_step` are the numbers of leading/trailing
steps that fall back to dense attention — not step indices. For 1344x768,
50-step T2VA:
| Configuration | `start_step` | `end_step` | Notes |
| --- | ---: | ---: | --- |
| Quality first (recommended) | 8 | 13 | General and complex-motion scenes |
| Balanced | 0 | 13 | Balanced |
| Speed first | 0 | 0 | Simple, low-motion scenes |
When frames look discontinuous or details unstable, increase `end_step`
first, then `start_step`; do not compensate by lowering `sparsity` alone.
These pairings were validated at 1344x768 / 50 steps only; re-evaluate for
other resolutions or step counts.
## HTTP API examples
The request contract is identical to the GPU recipe; see
[MiniMax-H3.md § HTTP API examples](MiniMax-H3.md#http-api-examples). Use the
validated 768P shapes (e.g. `width=1344 height=768`) on NPU. Quick start
against the synchronous endpoint:
```bash
export API_URL="http://127.0.0.1:${PORT}/v1/videos/sync"
curl -sS -X POST "${API_URL}" \
-F 'prompt=In a snowy blue-purple forest, Ori carefully walks past a sleeping giant; footsteps crunch in the snow while the creature breathes and softly snorts.' \
-F 'width=1344' \
-F 'height=768' \
-F 'aspect_ratio=16:9' \
-F 'fps=24' \
-F 'num_inference_steps=50' \
-F 'flow_shift=12.0' \
-F 'audio_flow_shift=3.0' \
-F 'seed=1101' \
-F 'extra_params={"task":"t2va","duration":5.0}' \
-o t2va.mp4
```
Switch tasks via `extra_params.task` (`t2va` / `fl2va` / `ref2va`); FL2VA and
Ref2VA additionally pass `input_references` (first-frame image, reference
video/audio). The full parameter table lives in the GPU recipe.
## FlashGen 4-step online LoRA (T2VA)
The FlashGen 4-step weights are a native-layout LoRA single file
(`key_format=minimax-h3-native`) loaded at runtime — no merge into the base
checkpoint. Download them from
[FlashGen/Minimax-H3-4step-lora-flashgen](https://modelscope.cn/models/FlashGen/Minimax-H3-4step-lora-flashgen),
then on top of the recommended configuration above:
- append `--lora-backend peft --lora-path "${FLASHGEN_LORA}"`,
- use `--diffusion-attention-backend FLASH_ATTN` (dense; drop the RainFusion
attention config),
- issue T2VA requests with `num_inference_steps=4` and the `lora` request
field (`name` / `path` / `scale`) — see
[MiniMax-H3.md § FlashGen native LoRA](MiniMax-H3.md#flashgen-native-lora).
T2VA only; FL2VA and Ref2VA are not supported, and neither
`--enable-cpu-offload` nor `--enable-layerwise-offload` may be combined with
it.
## CPU MP4 response encoding (Atlas A2)
Non-streaming MP4 responses use one public automatic encoder. It checks the
runtime frame shape, common dtype, and RGB channel-plane contiguity for every
@@ -127,121 +313,65 @@ inputs fall back to the legacy muxer before opening the PyAV container. No CLI
flag, model declaration, or user configuration is required. Streaming fMP4
output remains on its existing incremental path.
Current performance and correctness validation is limited to one Atlas A2 host
with 8x Ascend 910B4-1 NPUs, the `FL2VA`/`t2va` partition, one request at a
time, 1344x768 at 24 fps, and 5, 8.7, and 15 second requests. This optimization
only changes CPU MP4 response encoding; it does not change DiT execution or
stage 0.
## Ascend-optimized components
### Optional optimizations
The optimizations stacked by the recommended configurations above:
Two independent optimizations may be enabled on top of the configuration
above. Both are validated for T2VA only.
1. Ascend-friendly attention: the MindIE-SD `ascend_laser_attention` fused
kernel selected through `MINDIE_SD_FA_TYPE`, carried by the FLASH_ATTN
backend with a mask-free packed varlen path and K/V prefix slicing driven
by the packed `cu_seqlens` metadata.
2. Ascend-friendly fused operators: RMSNorm, AddRMSNorm, GQA, SwiGLU, and
rotary position embedding from MindIE-SD.
3. RainFusion block-sparse attention, optionally with EQBSA mixed INT8/FP8
precision — see
[RainFusion](../../docs/user_guide/diffusion/attention_backends/rainfusion.md).
4. Online quantization: INT8, MXFP8, and MXFP4 — see
[online quantization](../../docs/user_guide/quantization/online.md).
5. Parallelism: USP, text-encoder TP, and native VAE `tile` patch
parallelism, plus distributed layerwise offload.
6. Post-training: few-step distillation via the FlashGen online LoRA.
**RainFusion block-sparse attention** — switch the attention backend, keeping
every other flag unchanged:
## Benchmarks (Atlas 800I A2 / A3)
```bash
--diffusion-attention-backend RAINFUSION_ATTN
```
Measured with `--enable-diffusion-pipeline-profiler` on vLLM-Omni 0.28.0:
lossless rows use the recommended lossless configuration, lossy rows the
recommended lossy configuration. Memory figures are per device.
**INT8 online quantization** — add one flag to the server command above:
### Atlas 800I A2
```bash
--quantization int8
```
| Task | Resolution | Frames | Duration (s) | Config | E2E (s) | DiT total (s) | DiT per-step (s) | DiT steps | Text encode (s) | Ref video encode (s) | Ref audio encode (s) | VAE decode (s) | Ref preprocess (s) | Post-process (s) | CPU MP4 (s) | Resident weights (GB) | Peak memory (GB) |
| ---- | ---- | ---- | ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
| t2va | 1344x768 | 124 | 5 | lossless | 150.06 | 139.9 | 2.85 | 49 | 0.83 | NA | NA | 6.76 | NA | 0.462 | 1.18 | 2.97 | 24.7 |
| t2va | 1344x768 | 362 | 15 | lossless | 786.24 | 761.36 | 15.53 | 49 | 1.20 | NA | NA | 15.87 | NA | 1.98 | 2.95 | 2.97 | 36.21 |
| t2va | 1344x768 | 124 | 5 | lossy | 108.1 | 95.85 | 1.95 | 49 | 0.9 | NA | NA | 6.73 | NA | 0.745 | 1.04 | 2.55 | 22.47 |
| t2va | 1344x768 | 362 | 15 | lossy | 552.51 | 526.04 | 10.73 | 49 | 1.28 | NA | NA | 16.07 | NA | 1.90 | 3.76 | 2.55 | 35.11 |
| ref2va | 1344x768 | 124 | 5 | lossless | 474.89 | 421.68 | 8.6 | 49 | 2.42 | 13.31 | 0.38 | 6.47 | 3.94 | 0.489 | 1.16 | 3.4 | 23.45 |
| ref2va | 1344x768 | 362 | 15 | lossless | 3414.18 | 3336.98 | 68.08 | 49 | 6.48 | 30.85 | 0.59 | 15.8 | 12.59 | 1.99 | 3.08 | 3.4 | 36.28 |
| ref2va | 1344x768 | 124 | 5 | lossy | 380.92 | 326.50 | 6.66 | 49 | 2.47 | 13.48 | 0.38 | 6.36 | 4.68 | 0.591 | 1.22 | 2.55 | 22.57 |
| ref2va | 1344x768 | 362 | 15 | lossy | 2332.04 | 2249.63 | 45.89 | 49 | 6.33 | 35.1 | 0.59 | 15.72 | 11.07 | 1.35 | 2.3 | 2.55 | 35.58 |
Keep `--ring 1` when using RainFusion: the `rf_v2` kernel ranks key blocks
over the whole sequence, so ring parallelism would split away the keys it
needs. Scale with `--usp` instead.
### Atlas 800I A3
## Memory and attention optimizations (A3)
| Task | Resolution | Frames | Duration (s) | Config | E2E (s) | DiT total (s) | DiT per-step (s) | DiT steps | Text encode (s) | Ref video encode (s) | Ref audio encode (s) | VAE decode (s) | Ref preprocess (s) | Post-process (s) | CPU MP4 (s) | Resident weights (GB) | Peak memory (GB) |
| ---- | ---- | ---- | ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
| t2va | 1344x768 | 124 | 5 | lossless | 114.19 | 105.87 | 2.16 | 49 | 0.48 | NA | NA | 6.56 | NA | 0.001 | 0.46 | 3.49 | 24.7 |
| t2va | 1344x768 | 362 | 15 | lossless | 599.29 | 580.38 | 11.84 | 49 | 1.83 | NA | NA | 14.95 | NA | 0.001 | 1.02 | 3.49 | 36.21 |
| t2va | 1344x768 | 124 | 5 | lossy | 84.82 | 77.47 | 1.58 | 49 | 0.57 | NA | NA | 6.23 | NA | 0.001 | 0.61 | 2.63 | 22.47 |
| t2va | 1344x768 | 362 | 15 | lossy | 401.43 | 383.56 | 7.82 | 49 | 1.72 | NA | NA | 14.83 | NA | 0.001 | 1.14 | 2.63 | 35.11 |
| ref2va | 1344x768 | 124 | 5 | lossless | 355.53 | 311.86 | 6.36 | 49 | 1.89 | 12.37 | 0.24 | 5.55 | 2.49 | 0.001 | 0.43 | 3.49 | 23.45 |
| ref2va | 1344x768 | 362 | 15 | lossless | 2339.56 | 2277.61 | 46.47 | 49 | 5.61 | 34.46 | 0.36 | 13.61 | 6.83 | 0.001 | 1.11 | 3.49 | 36.28 |
| ref2va | 1344x768 | 124 | 5 | lossy | 306.82 | 247.72 | 5.05 | 49 | 1.88 | 12.71 | 0.24 | 5.24 | 2.45 | 0.001 | 0.54 | 2.63 | 22.57 |
| ref2va | 1344x768 | 362 | 15 | lossy | 1695.26 | 1634.58 | 33.35 | 49 | 5.57 | 33.27 | 0.36 | 13.51 | 6.96 | 0.001 | 1.22 | 2.63 | 35.58 |
### Fitting 768P into 64 GB HBM: distributed layerwise offload or HSDP
Each Atlas 800I A3 NPU has 64 GB of HBM, which is not enough for the
combined MiniMax-H3 service at 768P. Enable **one** of the following at
server startup:
**Distributed layerwise offload** — DiT layers are offloaded with parameters
gathered across the parallel group instead of replicated per rank:
```bash
--enable-distributed-layerwise-offload
```
Measured peak NPU memory is about 45 GB per device for Ref2VA with a 13.88 s
input video generating a 15 s 768P (1344x768) output video. Host (CPU) memory
usage is high with this option.
**HSDP** — hybrid sharded data parallelism for the DiT parameters. The
multi-stream memory-reuse knob is mandatory with this flag:
```bash
export MULTI_STREAM_MEMORY_REUSE=2
```
```bash
--use-hsdp
```
Host memory usage is small, but large shapes may still OOM; HBM usage
optimizations for this path are ongoing.
### FLASH_ATTN backend with MindIE-SD
Keep `--diffusion-attention-backend FLASH_ATTN` and install MindIE-SD (see
[§ Environment](#environment)). On NPU this backend carries most of the
memory-reduction and performance work: a mask-free packed varlen path that
never materializes the quadratic `full_qk` padding mask, and K/V prefix
slicing, both driven by the packed `cu_seqlens` metadata emitted by the H3
transformer.
### Optional: LaserAttention fused kernel
For an additional attention speedup, select the Ascend LaserAttention fused
kernel before starting the server:
```bash
export MINDIE_SD_FA_TYPE="ascend_laser_attention"
```
This requires the FLASH_ATTN backend and MindIE-SD. H3 automatically applies
exact power-of-two input pre-scaling (`laser_input_scale=256`) so the
kernel's fp16 workspace cannot overflow on outlier activations. Measured
speedup numbers will be added here.
## HTTP API examples
The request format is identical to the GPU recipe; see
[MiniMax-H3.md § HTTP API examples](MiniMax-H3.md#http-api-examples).
Use the validated 768P shapes (e.g. `width=1344 height=768`) on NPU.
## Key parameters
Same as the GPU recipe; see
[MiniMax-H3.md § Key parameters](MiniMax-H3.md#key-parameters).
The validated resolution on NPU is 768P (e.g. 1344x768).
## Validated NPU evidence
Measured on an Atlas 800I A3 server (8x NPU) with CANN 9.0.1,
PyTorch 2.10.0+cpu, and torch_npu 2.10.0.post2, using the multi-NPU
configuration above:
| Workload | Configuration |
|----------|---------------|
| T2VA, 209 frames, 1344x768 | TE TP8, distributed layerwise offload, Ulysses 8, VPP8 tile, regional compile |
| Ref2VA (prompt + video), 124 frames, 1344x768 | TE TP8, distributed layerwise offload, Ulysses 8, VPP8 tile, regional compile |
These measurements describe the validated shapes rather than a general
throughput guarantee.
These numbers describe the validated shapes rather than a general throughput
guarantee.
## Known limitations
- Combined serving requires sibling `FL2VA` and `Ref2VA` directories, loads
both task-specific DiTs, and loads one copy of every shared component.
- Task serving is partitioned by `--task-type`: T2VA/FL2VA and Ref2VA load
different DiTs, so switching between them requires a restart with the other
partition.
- H3 currently executes one generation request per diffusion batch.
- The first regional-compile request is a warmup and should not be included
in steady-state performance measurements.
@@ -250,17 +380,19 @@ throughput guarantee.
the same HTTP request contract.
- VAE patch parallelism requires size 1 or the full DiT group size and
supports the H3 native `tile` mode only.
- RainFusion block-sparse attention and INT8 quantization are validated for
T2VA only; use the BF16 dense configuration for FL2VA and Ref2VA.
- Online INT8 quantization can be combined with distributed layerwise offload
while AllGather is enabled. Keep AllGather enabled for SP>1; use
`--dlo-no-use-allgather` only when collective communication is unavailable.
- The lossy configuration (RainFusion + INT8) is validated for T2VA and
Ref2VA; use the lossless configuration for FL2VA or re-validate first.
## Additional resources
- [MiniMax-H3.md](MiniMax-H3.md) — full GPU guide
- [Attention backends § RAINFUSION_ATTN](../../docs/user_guide/diffusion/attention_backends.md#rainfusion_attn-backend-and-block-sparse-video-attention)
— RainFusion knobs and tuning
- [Int8 quantization](../../docs/user_guide/quantization/int8.md)
- [MiniMax-H3-NPU-950PR.md](MiniMax-H3-NPU-950PR.md) — four-card
Ascend 950PR / 950DT guide
- [NPU installation](../../docs/getting_started/installation/npu.md)
- [RainFusion attention](../../docs/user_guide/diffusion/attention_backends/rainfusion.md)
— block-sparse knobs and tuning
- [Online quantization](../../docs/user_guide/quantization/online.md)
— INT8 / MXFP8 / MXFP4
- [Cache-DiT guide](../../docs/user_guide/diffusion/cache_acceleration/cache_dit.md)
- [Supported models](../../docs/models/supported_models.md)
- [Video API](../../docs/serving/videos_api.md)
recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md
+15 -5
View File
@@ -16,6 +16,12 @@ Validated on:
50 steps, text-encoder TP2, and DiT USP2
- Common sampling settings: `flow_shift=12`, `seed=1101`
The commands and measurements below describe this pinned revision. Newer
versions have changed the scope of `--quantization fp8`; see
[Online FP8 quantization](MiniMax-H3.md#online-fp8-quantization) for the current
component selection and offload support. Re-qualify memory, output, and latency
on GB10 before comparing results across revisions.
## Capacity requirements
| Resource | DGX Spark (GB10) |
@@ -40,8 +46,10 @@ math relative to the RTX and datacenter recipes:
- **Do not use `--enable-cpu-offload`.** Moving weights from "VRAM" to "host RAM"
frees nothing here.
- **`--quantization fp8` is mandatory.** A BF16 partition is 135 GiB and does not
fit in 121 GiB. Online FP8 quantizes the DiT only (62 GiB to ~31 GiB); the
Qwen3-VL text encoder and both VAEs stay BF16.
fit in 121 GiB. At the validated revision, online FP8 quantizes the DiT only
(62 GiB to ~31 GiB); the Qwen3-VL text encoder and both VAEs stay BF16.
Current versions also quantize eligible text-decoder linears by default, so
the same flag does not imply the same quantization scope across revisions.
With ~97.7 GiB of the pool held by the allocator at peak, T2VA leaves roughly
23 GiB of headroom for the OS, page cache, and any other process on the box.
@@ -295,6 +303,8 @@ shorter than a 50-step Ref2VA run.
- Ref2VA requires a vLLM-Omni build newer than `v0.26.0`. See the version note
under **Validated on**: FP8 weight loading and image-only Ref2VA are both
broken on `release/v0.26.0`. The suggested version is `v0.26.1`.
- Online FP8 is incompatible with layerwise offload — the offload path produces a
weight stride the Cutlass FP8 kernel rejects. This is not a practical
restriction here since offload is unusable on GB10 anyway.
- At the validated revision, online FP8 with layerwise offload produced a weight
stride the Cutlass FP8 kernel rejected. Current H3 supports online FP8 with
layerwise offload and both DLO transfer paths; see the linked FP8 section
above. This does not change the GB10 capacity guidance: offload uses the same
unified-memory pool and is not enabled in this recipe.
recipes/MiniMaxAI/MiniMax-H3.md
+209 -1
View File
@@ -825,7 +825,7 @@ at most 15 seconds combined.
| FL2VA | first image, last image, or ordered first+last images | at most 2 images; `frame_indices` is `[0]`, `[-1]`, or `[0,-1]` |
| Ref2VA | image-only, image+image, image+video, video+audio, and mixed image/video/audio | images ≤9, videos ≤3, audios ≤3, total references ≤12; audio requires a visual reference |
The H3 output contract is 4–15 seconds at 24 FPS, stereo 32 kHz audio, and a
The default H3 output contract is 4–15 seconds at 24 FPS, stereo 32 kHz audio, and a
32-pixel canvas multiple. T2VA requires one named output ratio from `21:9`,
`16:9`, `4:3`, `1:1`, `3:4`, or `9:16`. FL2VA always follows the first input
image's ratio and ignores a generic `aspect_ratio` override. Ref2VA defaults to
@@ -836,6 +836,152 @@ default. `short_edge` controls the 768-pixel canvas and must be `768`.
outputs; the synchronous raw-MP4 endpoint returns the first output when more
than one is requested.
## Long video and driving audio
Set `extra_params.long_video=true` to opt into durations above 15 seconds.
Ref2VA long-video requests now default to latent continuation with global
temporal RoPE positions (A+B), described below. Set `long_video_mode=full`
explicitly to sample the entire target latent at once for comparison. Both modes
retain the native `17n+5` video-frame grid, `5n+2` video-latent grid, and 40 Hz
audio latents. A request for 75 seconds becomes 1,807 frames (75.292 seconds at
24 FPS). In full mode, attention cost and activation memory grow with the full
sequence length. Reference-video and reference-audio limits remain unchanged.
Other task types retain full-mode behavior. Requests are limited to 30 seconds
in full mode and 300 seconds in continuation mode; without `long_video=true`,
the limit remains 15 seconds. The limits cover duration aliases and explicit
`num_frames`, and externally encoded requests are checked again before diffusion.
Native frame-grid alignment can round the output slightly above the requested
limit. These are request sanity bounds, not a guarantee that a given resolution
fits GPU memory. Full mode still allocates the whole sequence, and continuation
retains cumulative latents and reference inputs.
Use `audio_mode=lock_source` with one `audio_reference` to drive generation
with a soundtrack, rather than treating it as a short reference. The encoder
places that waveform in the target audio latent, crops or zero-pads its two
channels independently to the output duration, and keeps those rows clean
(timestep 1) throughout sampling. This mode does not insert an `<Audio 1>`
reference tag. It supports request and step execution; the default `native`
mode continues generating audio normally. Returned audio is the audio VAE's
reconstruction, not a byte-for-byte copy of the input recording.
Against a Ref2VA server, using the same asset-server setup as above:
```bash
curl --fail-with-body -sS -X POST "${API_URL}" \
-F 'prompt=<prompt.txt' \
-F 'width=960' -F 'height=544' -F 'fps=24' \
-F 'num_inference_steps=50' -F 'flow_shift=12' -F 'seed=19960422' \
-F 'extra_params={"task":"ref2va","duration":75,"long_video":true,"audio_mode":"lock_source","audio_flow_shift":3,"preencode_mp4":true,"preencode_batch_frames":17}' \
-F "input_reference=@${REF_IMAGE};type=image/png" \
-F "audio_reference={\"audio_url\":\"${AUDIO_URL}\"}" \
-o long-video.mp4
```
`preencode_mp4` decodes and muxes temporal chunks on the worker, avoiding a
full decoded long video in the API process. It does not reduce the denoiser's
sequence length. For Turbo adapters, use the matching adapter's step count
and flow shift from the Turbo table below.
The [RunningHub workflow](https://www.runninghub.cn/post/2100530217365364737/)
uses T8's `MiniMaxH3AudioConditioningT8`, a full-length latent, and
`MiniMaxH3DualClockSamplerT8`; its `remix_source` strength is zero, equivalent
to source locking. This differs from rolling-overlap continuation nodes.
See [T8 conditioning](https://github.com/T8mars/comfyui-minimax-h3-audio-T8)
and [ComfyUI H3 masking](https://github.com/Comfy-Org/ComfyUI/blob/master/comfy/ldm/minimax/model.py).
The workflow's custom Dasiwa weights, Sol attention, and T8-specific LoRA are
not supplied by this feature.
### Latent-tail continuation with global temporal positions (A+B)
`long_video=true` selects this mode by default for Ref2VA. It can also be selected
explicitly with `long_video_mode=continuation`. This experimental mode supports
Ref2VA request execution with uncached denoising (`quality=lossless`); step
execution is rejected. Use `long_video_mode=full` explicitly for whole-target
sampling, including requests up to 30 seconds that require step execution.
Continuation currently rejects latent-mask editing (`video_noise_mask` or
`audio_noise_mask` with retained source regions). Full mode supports editing;
`audio_mode=lock_source` cannot be combined with audio latent-mask editing.
```json
{
"task": "ref2va",
"duration": 75,
"long_video": true,
"long_video_mode": "continuation",
"continuation_window_frames": 277,
"continuation_overlap_frames": 22,
"audio_mode": "lock_source",
"audio_flow_shift": 3,
"preencode_mp4": true,
"preencode_batch_frames": 17
}
```
These defaults sample seven windows for a 1,807-frame output. Each continuation
uses the preceding AV latent tail as fixed condition rows at the new target's
time origin, samples a fresh unmasked target, discards its hidden overlap, and
appends only the new suffix. The reference image and full prompt remain present
in every window. Both window and overlap use the `17n+5` frame grid; the window
must be 107..345 frames and larger than the overlap. A final window may be shorter.
Audio boundaries are rounded on the cumulative 24 FPS / 40 Hz timeline to avoid
drift; a locked driving track is sliced separately for each stereo channel.
Before RoPE evaluation, every window adds `start_frame * 40 / 24` to the temporal
position IDs of its target audio/video, reference audio/video, and latent-tail
guides. The start includes the overlap (not just the newly appended frames).
Text, static image references, spatial coordinates, and padding remain unchanged.
Offsets retain fractional precision independently of rounded audio slice indices.
For the default windows starting at frames 0, 255, 510, the added positions are
0, 425, 850. This follows the media-only global-offset convention in
[LongMedia temporal positioning](https://github.com/vizart-vj/ComfyUI-MiniMax-H3-LongMedia/blob/main/temporal_positioning.py).
The shifted layout is constructed once per window and then used by all denoising
steps and sequence-parallel ranks; no RoPE-frequency scaling is applied.
For a narrative with different scenes, pass `continuation_prompts` as a JSON
list containing exactly one non-empty prompt string per planned window (seven
strings for the default 75-second request). These prompts are independently
encoded with the same reference assets and selected in window order. Shot times
inside each prompt describe that local window, including its hidden overlap.
This option requires a local text encoder; external-encoder stage execution is
not supported. Media positions use one shared origin after the longest encoded
text prefix, so different prompt lengths do not shift the global AV clock.
Without this list, the request's single prompt is reused as before.
The cumulative latent is decoded once after all windows. Denoising memory is
bounded by the window, while cumulative latent storage still grows with duration.
This follows the [ComfyUI latent-tail continuation algorithm](https://github.com/ttulttul/ComfyUI-Minimax-H3-Continuation).
Global positions can still exceed the model's trained temporal range. This
mode does not guarantee seamless cuts or adherence to absolute shot timestamps in
a repeated prompt. For comparison, keep the model, seed, prompt, source media,
steps, shifts, and output size fixed, changing only `long_video_mode`.
For a reproducible approximately 20-second comparison, use a request-mode
Ref2VA server with caching disabled and the same `prompt.txt` and `REF_IMAGE`
for both requests. Use a single prompt without `continuation_prompts` so that
both modes receive identical text. The commands below generate native audio:
```bash
for mode in full continuation; do
curl --fail-with-body -sS --max-time 14400 "${API_URL}" \
-F 'prompt=<prompt.txt' \
-F "input_reference=@${REF_IMAGE};type=image/png" \
-F 'width=960' -F 'height=544' -F 'fps=24' \
-F 'num_inference_steps=50' -F 'flow_shift=12' -F 'seed=19960422' \
-F 'quality=lossless' \
-F "extra_params={\"task\":\"ref2va\",\"duration\":20,\"long_video\":true,\"long_video_mode\":\"${mode}\",\"audio_flow_shift\":3,\"preencode_mp4\":true}" \
-o "h3-20s-${mode}.mp4" || break
done
```
Both outputs should contain 481 frames (20.042 seconds at 24 FPS).
Continuation uses windows `[0,277)` and `[255,481)`; inspect frames 276/277
(about 11.54 seconds) for visual and audio discontinuities. Compare the complete
clips for motion, identity, prompt adherence, and sound, not just frame similarity:
the trajectories differ by design. Report the tested commit, checkpoint, hardware,
settings, and both videos with any comparison; the command alone is not quality
evidence. A matching seed does not imply matching noise over differently sized
full and windowed latents.
## Request-scoped quality
Add one of these fields to any HTTP request above. No Cache-DiT startup option
@@ -1106,6 +1252,68 @@ generations move the end-to-end figure toward the denoising one. Fusing the
adapter does not measurably change startup: weight loading took 77.3 s with it
against 85.8 s without.
### FastH3 8-Step V2 full checkpoint
[FastH3 8-Step V2](https://huggingface.co/FastVideo/FastVideo-FastH3-8-Step-V2)
ships a full Diffusers-format transformer, including learned VSA compression
gates. Serve the Hugging Face model ID or a downloaded snapshot directly:
```bash
export FASTH3_V2_MODEL=FastVideo/FastVideo-FastH3-8-Step-V2
```
Use a Hugging Face account with access to the release, and pin `--revision`
when reproducing a result. The existing component loader reads its safetensors
shards; the H3 weight loader maps names and loads Q/K/V and MLP projections into
native parameters in memory. No conversion command or second transformer
checkpoint is needed.
The transformer and text components come from the V2 release. To retain Omni's
native tiled and parallel VAE execution, the loader fetches only the video/audio
VAEs from the `MiniMaxAI/MiniMax-H3` revision pinned in V2's `provenance.json`.
Those components use the normal Hugging Face cache. No separate base-model
argument or full base-transformer download is required. Download time is a
preparation cost, separate from warmup and request latency.
Install the optional `fastvideo-kernel` build required by the
`FASTVIDEO_VSA` backend, including its H3 block-map entry point. A four-GPU
configuration is:
```bash
CUDA_VISIBLE_DEVICES=0,1,2,3 vllm serve "${FASTH3_V2_MODEL}" --omni \
--host 127.0.0.1 --port 8095 --trust-remote-code --task-type fl2va \
--served-model-name FastH3-V2 \
--num-gpus 4 --usp 4 --ring 1 \
--vae-patch-parallel-size 4 --vae-parallel-mode tile --vae-use-tiling \
--diffusion-attention-backend FASTVIDEO_VSA
curl --fail-with-body -sS http://127.0.0.1:8095/v1/videos/sync \
-F 'model=FastH3-V2' \
-F 'prompt=A red fox runs through fresh snow at dawn, with fast pawsteps, winter wind and distant birds.' \
-F 'width=1344' -F 'height=768' -F 'aspect_ratio=16:9' -F 'fps=24' \
-F 'num_inference_steps=8' -F 'guidance_scale=1' -F 'flow_shift=10' \
-F 'seed=1101' \
-F 'extra_params={"task":"t2va","duration":4.4,"audio_flow_shift":3.0}' \
-o fasth3_v2.mp4
```
V2 pins video/audio shifts **10/3**, guidance **1**, and the unshifted ladder
`[0.999, 0.874, 0.749, 0.624, 0.5, 0.375, 0.25, 0.125, 0.0]`.
Use **`num_inference_steps=8`** in Omni: these nine nodes bound eight model
evaluations. FastVideo's `--steps 9` counts nodes instead. The existing H3 ODE
update is reused without stochastic re-noising.
The trained attention policy is **80% sparsity with 64-token video tiles**.
Omni derives the number of selected video tiles from each request's geometry;
do not supply a fixed `--fastvideo-vsa-topk`. Text/condition/audio prefix keys
remain available to every query, and prefix queries remain dense. Missing VSA
geometry or a failed H3 kernel raises an error instead of serving the student
with dense attention.
This release supports T2VA only, with local or pure Ulysses attention. Additional
LoRA adapters, Ref2VA and FL2VA conditioning are unsupported. The legacy
four-step adapter keeps its original sampling and fixed-top-k behavior.
## Key parameters
| Parameter | Recommended value | Notes |
recipes/OpenBMB/VoxCPM2.md
+63 -7
View File
@@ -43,9 +43,65 @@ sufficient — no tensor parallelism is required.
## Hardware Support
This recipe documents one tested 24 GB consumer-GPU configuration.
Larger-VRAM (H20 / H100 / A100) and other vendor sections (ROCm, NPU) are
welcome as community validation lands.
This recipe documents tested single-card configurations for a 24 GB NVIDIA
GPU and an Ascend Atlas A2.
## Ascend NPU
### 1 x Ascend 910B3 64GB (Atlas A2)
VoxCPM2 runs on one Atlas A2 card with the standard deploy config. The model's
LocDiT path uses eager PyTorch on NPU; keep `enforce_eager: true`. The default
config enables fused QKV, fused gate/up projection, and the zero-delta-time
cache for this path.
#### Environment
- Hardware: 1 x Ascend 910B3 64GB (Atlas A2)
- Container: `quay.nju.edu.cn/ascend/vllm-omni:v0.28.0`
- vLLM: 0.28.0
- vLLM-Omni: current `main`
- Model: `openbmb/VoxCPM2`, bfloat16
#### Command
Expose one physical NPU to the container, then start the server normally. The
visible card is addressed as logical device `0` by `voxcpm2.yaml`.
```bash
export ASCEND_RT_VISIBLE_DEVICES=5
export VLLM_WORKER_MULTIPROC_METHOD=spawn
vllm serve /path/to/VoxCPM2 --omni \
--host 0.0.0.0 --port 8000
```
#### Performance validation
The component timer was measured after model warmup with a single Chinese TTS
request. Each decode step executes nine LocDiT estimator calls, so per-step
measurements compare the same amount of model work even when generated audio
duration differs.
| path | CFM estimator / call | decode / step | forward / step |
| ---- | -------------------- | ------------- | -------------- |
| Before (optimization flags skipped on NPU) | 18.84 ms | 193.86 ms | 265.35 ms |
| Fused QKV + fused MLP + zero-dt cache | 17.50 ms | 182.22 ms | 254.07 ms |
| Above + experimental reduced-precision RoPE | 14.93 ms | 157.36 ms | 227.33 ms |
The default fused path reduces CFM estimator latency by about **7.1%** and
complete decode-step latency by **6.0%**. Its generated WAV was byte-identical
to the pre-optimization baseline. Setting
`enable_loc_dit_fast_rope: true` can increase the reductions to about **20.8%**
and **18.8%**, respectively, but changes the recurrent acoustic state and is
therefore an opt-in quality/performance tradeoff. Its test audio was valid 48
kHz mono PCM with finite samples and no clipping; evaluate voice quality on a
representative corpus before enabling it in production.
Do not set `enforce_eager: false` for VoxCPM2 on Ascend yet. ACL Graph captures
the language-model runner, but the request-owned VoxCPM2 state has dynamic
prefill/decode shapes; replay can therefore feed an empty captured row into a
non-empty residual state and fail at `aclnnCat`.
## GPU
@@ -88,8 +144,8 @@ Pass `--deploy-config <path>` to override.
#### Verification
**Server cold-start**: ~60 s from `vllm serve` to `Application startup
complete` (subprocess fork + vLLM 0.21 init + model load + flashinfer JIT
+ torch.compile of LocDiT / feat_encoder / AudioVAE + CUDA-Graph warmup).
complete` (subprocess fork, vLLM 0.21 init, model load, flashinfer JIT,
torch.compile of LocDiT / feat_encoder / AudioVAE, and CUDA-Graph warmup).
The first request after startup pays a small additional cost; steady-state
requests are much faster.
@@ -199,7 +255,7 @@ after a one-off ~28 s engine init:
| ---- | -------- | --------- | ----- | ------------------------------------------------ |
| #1 | 6.72 s | 11.97 s | 1.782 | cold: torch.compile + CUDA-Graph capture |
| #2 | 6.24 s | 11.43 s | 1.831 | still runtime warmup |
| #3 | 6.88 s | 0.82 s | 0.120 | ⚡ steady-state |
| #3 | 6.88 s | 0.82 s | 0.120 | steady-state |
| #4 | 6.56 s | 0.78 s | 0.119 | steady-state |
| #5 | 5.76 s | 0.70 s | 0.121 | steady-state |
@@ -231,7 +287,7 @@ Same 5-call methodology as the zero-shot table, this time with a
| ---- | -------- | --------- | ----- | ------------------------------------------------ |
| #1 | 5.44 s | 12.38 s | 2.276 | cold: compile + CUDA-Graph capture + ref encode |
| #2 | 5.12 s | 2.47 s | 0.482 | most warmup done |
| #3 | 5.44 s | 0.76 s | 0.139 | ⚡ steady-state |
| #3 | 5.44 s | 0.76 s | 0.139 | steady-state |
| #4 | 4.96 s | 0.68 s | 0.137 | steady-state |
| #5 | 5.28 s | 0.71 s | 0.134 | steady-state |
recipes/README.md
+5 -1
View File
@@ -29,6 +29,8 @@ Example layout:
```text
recipes/
lerobot/
Pi05.md
Qwen/
Qwen3-Omni-H100.md
Qwen3-TTS-A100.md
@@ -47,12 +49,14 @@ recipes/
| [`BosonAI/Higgs-Audio-V3-TTS.md`](./BosonAI/Higgs-Audio-V3-TTS.md) | Online + offline multilingual TTS with voice cloning | 1x H100 80GB |
| [`BreezeBlue/Breeze-TTS-2.md`](./BreezeBlue/Breeze-TTS-2.md) | Online + offline TTS (plain / voice design / voice clone / voice direction, streaming PCM) | 1x NVIDIA L20 48GB |
| [`ByteDance/Lance.md`](./ByteDance/Lance.md) | Unified AR+diffusion: text/img/video gen + understanding (Lance 3B) | 1x B300 / A100 80GB |
| [`circlestone-labs/Anima.md`](./circlestone-labs/Anima.md) | Native single-file text-to-image serving | 1x AMD MI300X |
| [`fishaudio/Fish-Speech-S2-Pro.md`](./fishaudio/Fish-Speech-S2-Pro.md) | Online serving for TTS | 1x A800 80GB |
| [`Helios/Helios.md`](./Helios/Helios.md) | Text-to-video, image-to-video, and video-to-video generation | 1x NVIDIA H20 |
| [`HiDream-ai/HiDream-O1-Image.md`](./HiDream-ai/HiDream-O1-Image.md) | Text-to-image with shared offline example | 1x or 2x H100 80GB |
| [`inclusionAI/Ming-flash-omni-2.0.md`](./inclusionAI/Ming-flash-omni-2.0.md) | Online serving for multimodal chat + standalone TTS | 4x H100 / 1x H100 80GB |
| [`inclusionAI/Ming-omni-tts.md`](./inclusionAI/Ming-omni-tts.md) | Offline + online dense Ming TTS/audio generation | 1x H100 80GB / 1x AMD MI300X (ROCm 7.2) |
| [`k2-fsa/OmniVoice.md`](./k2-fsa/OmniVoice.md) | Offline and online multilingual text-to-speech | 1x AMD MI300X (ROCm 7.2) |
| [`lerobot/Pi05.md`](./lerobot/Pi05.md) | Online OpenPI serving for the π0.5 vision-language-action policy | 1x H100 80GB |
| [`IndexTeam/IndexTTS-2.md`](./IndexTeam/IndexTTS-2.md) | Online serving for voice-cloned TTS with optional emotion control | 1x L4 24GB or larger CUDA GPU |
| [`IndexTeam/IndexTTS-2_5.md`](./IndexTeam/IndexTTS-2_5.md) | Online + offline multilingual voice-cloned TTS with native speed and emotion control | 1x NVIDIA H20 96GB |
| [`MiniMaxAI/MiniMax-H3.md`](./MiniMaxAI/MiniMax-H3.md) | T2VA, FL2VA, image+audio Ref2VA, and multi-video Ref2VA serving | 1x GPU with CPU offload / 4x B300 |
@@ -72,7 +76,7 @@ recipes/
| [`cosmos3/Cosmos3-Edge.md`](./cosmos3/Cosmos3-Edge.md) | T2I / T2V / I2V + action (Physical-AI) generation on the Nemotron-based Edge transformer | 1x GPU (~8 GiB) |
| [`cosmos3/Cosmos3-Super.md`](./cosmos3/Cosmos3-Super.md) | 64B T2I / T2V / I2V / V2V generation (+ optional audio) / Action policy | 8x H200/H100/A100 / 2x H200 / B300 |
| [`OpenBMB/MiniCPM-o-4_5.md`](./OpenBMB/MiniCPM-o-4_5.md) | Online full-duplex serving and offline inference for omni multimodal chat (text / image / audio / video → text + 24 kHz speech) | 2x A100/H100 80GB / 3x mid-tier GPU / 8x RTX 4090 24GB |
| [`OpenBMB/VoxCPM2.md`](./OpenBMB/VoxCPM2.md) | Online + offline TTS with native AR pipeline (48 kHz, 30+ languages) | 1x RTX 4090 24GB |
| [`OpenBMB/VoxCPM2.md`](./OpenBMB/VoxCPM2.md) | Online + offline TTS with native AR pipeline (48 kHz, 30+ languages) | 1x RTX 4090 24GB / 1x Ascend 910B3 64GB |
| [`OpenMOSS/MOSS-TTS.md`](./OpenMOSS/MOSS-TTS.md) | Online + offline multilingual TTS (MOSS-TTS family, 8B) | 1x H100 80GB |
| [`OpenMOSS/MOSS-TTS-Nano-NPU.md`](./OpenMOSS/MOSS-TTS-Nano-NPU.md) | Online multilingual voice-cloned TTS | 1x Ascend 910B3 64GB |
| [`NVIDIA/Nemotron-Labs-Audex.md`](./NVIDIA/Nemotron-Labs-Audex.md) | TTS / text-to-audio / audio understanding / cascaded S2S (Audex 2B + 30B-A3B) | 1x H100 80GB |
Loaded 100 of 769 files, more files were not shown because too many files have changed in this diff. Show more