Compare commits
8
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
18252b9d8d | ||
|
|
a326cd0d57 | ||
|
|
c15f51a2f6 | ||
|
|
7c934b6188 | ||
|
|
74050cdacd | ||
|
|
f0cec3ecd4 | ||
|
|
7b885f9d13 | ||
|
|
69bcc6416d |
No files matched your search
@@ -10,7 +10,7 @@ steps:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
# Buildkite will automatically replace this with the actual commit hash
|
||||
VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
|
||||
VLLM_VERSION: "v0.22.0"
|
||||
VLLM_VERSION: "v0.21.0"
|
||||
priority: 100
|
||||
timeout_in_minutes: 60
|
||||
soft_fail: false
|
||||
@@ -1,28 +1,15 @@
|
||||
steps:
|
||||
|
||||
- label: "Simple · Diffusion & Model Executor Test"
|
||||
- label: "Simple Unit Test"
|
||||
agent_pool: mi325_1
|
||||
depends_on: amd-build
|
||||
timeout_in_minutes: 50
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_ROCM_USE_AITER=0
|
||||
# ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
|
||||
# that is not supported on CPU
|
||||
- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
|
||||
|
||||
- label: "Simple · Other Test"
|
||||
agent_pool: mi325_1
|
||||
depends_on: amd-build
|
||||
timeout_in_minutes: 50
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_ROCM_USE_AITER=0
|
||||
# ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
|
||||
# that is not supported on CPU
|
||||
- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
|
||||
- "timeout 60m pytest -v -s -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
|
||||
|
||||
- label: "Diffusion Images API LoRA E2E"
|
||||
agent_pool: mi325_1
|
||||
@@ -62,6 +49,16 @@ steps:
|
||||
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model
|
||||
|
||||
- label: "Diffusion Cache Backend Test"
|
||||
agent_pool: mi325_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- timeout 15m pytest -s -v -m "core_model and cache and diffusion and not distributed_cuda and L4"
|
||||
|
||||
- label: "Diffusion Sequence Parallelism Test (Need 4 GPUs)"
|
||||
agent_pool: mi325_4
|
||||
depends_on: amd-build
|
||||
@@ -126,7 +123,7 @@ steps:
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
- pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/online_serving/test_mimo_audio.py -m "advanced_model" --run-level "advanced_model"
|
||||
|
||||
- label: "Qwen3-TTS CustomVoice E2E Test"
|
||||
agent_pool: mi325_1
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
steps:
|
||||
|
||||
- label: "Simple · Diffusion & Model Executor Test"
|
||||
- label: "Simple Unit Test"
|
||||
agent_pool: mi325_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
@@ -9,18 +9,7 @@ steps:
|
||||
- export VLLM_ROCM_USE_AITER=0
|
||||
# ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
|
||||
# that is not supported on CPU
|
||||
- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
|
||||
|
||||
- label: "Simple · Other Test"
|
||||
agent_pool: mi325_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_ROCM_USE_AITER=0
|
||||
# ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
|
||||
# that is not supported on CPU
|
||||
- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
|
||||
- "timeout 60m pytest -v -s -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
|
||||
|
||||
- label: "Voxtral TTS CUDA Unit Test"
|
||||
agent_pool: mi325_1
|
||||
@@ -30,13 +19,13 @@ steps:
|
||||
commands:
|
||||
- timeout 10m pytest -s -v tests/model_executor/models/voxtral_tts/test_cuda_graph_acoustic_transformer.py
|
||||
|
||||
- label: "Diffusion · Batch Test"
|
||||
- label: "Diffusion Batching Test"
|
||||
agent_pool: mi325_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- timeout 20m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model"
|
||||
- timeout 20m pytest -s -v tests/diffusion/batching/test_diffusion_batching.py -m "core_model and diffusion" --run-level "core_model"
|
||||
|
||||
- label: "Custom Pipeline Test"
|
||||
agent_pool: mi325_1
|
||||
|
||||
@@ -107,7 +107,7 @@ steps:
|
||||
depends_on: upload-merge-pipeline
|
||||
steps:
|
||||
- label: "Entrypoint Test with H100"
|
||||
timeout_in_minutes: 60
|
||||
timeout_in_minutes: 35
|
||||
commands:
|
||||
- pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model"
|
||||
agents:
|
||||
|
||||
@@ -183,7 +183,7 @@ steps:
|
||||
depends_on: upload-ready-pipeline
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
timeout 15m bash -c "
|
||||
pytest -sv tests/distributed/ -m 'core_model and cuda' --run-level 'core_model'
|
||||
"
|
||||
agents:
|
||||
|
||||
@@ -53,9 +53,6 @@
|
||||
{% endif %}
|
||||
{{ indented_cmd | safe }}
|
||||
priority: 100
|
||||
{% if step.timeout_in_minutes %}
|
||||
timeout_in_minutes: {{ step.timeout_in_minutes }}
|
||||
{% endif %}
|
||||
{% if step.grade and step.grade == "Blocking" %}
|
||||
soft_fail: false
|
||||
{% else %}
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
se001|ocean waves crashing on a rocky beach|10.0
|
||||
se002|heavy rain on a metal roof at night|8.0
|
||||
se003|a dog barking in the distance|6.0
|
||||
se004|a crackling campfire in a quiet forest|10.0
|
||||
se005|footsteps on dry leaves|6.0
|
||||
se006|thunder rolling across a wide valley|8.0
|
||||
se007|a busy coffee shop with chatter and espresso machines|10.0
|
||||
se008|wind howling through tall pine trees|8.0
|
||||
se009|a small stream flowing over pebbles|10.0
|
||||
se010|a creaking wooden door slowly opening|6.0
|
||||
@@ -1,10 +0,0 @@
|
||||
ttsd001|prompt-wavs/spk1.wav|prompt-wavs/spk2.wav|[S1] Hello and welcome to the show today. [S2] Thanks for having me, it's a pleasure to be here. [S1] Let's dive right in.
|
||||
ttsd002|prompt-wavs/spk1.wav|prompt-wavs/spk2.wav|[S1] Did you finish the report? [S2] Almost — I'm just polishing the conclusion. [S1] Great, send it over when you're done.
|
||||
ttsd003|prompt-wavs/spk1.wav|prompt-wavs/spk2.wav|[S1] Have you tried that new restaurant downtown? [S2] Yes, the pasta is amazing. [S1] I'll book a table for Friday.
|
||||
ttsd004|prompt-wavs/spk1.wav|prompt-wavs/spk2.wav|[S1] What did you think of yesterday's meeting? [S2] Productive, but a bit long. [S1] Agreed, we should set an hour cap next time.
|
||||
ttsd005|prompt-wavs/spk1.wav|prompt-wavs/spk2.wav|[S1] The weather is beautiful today. [S2] Perfect for a hike. [S1] Want to drive out to the trails this afternoon?
|
||||
ttsd006|prompt-wavs/spk1.wav|prompt-wavs/spk2.wav|[S1] How did the demo go? [S2] Better than expected — the client signed off on the prototype. [S1] Fantastic news!
|
||||
ttsd007|prompt-wavs/spk1.wav|prompt-wavs/spk2.wav|[S1] Coffee or tea this morning? [S2] Coffee please, with a splash of milk. [S1] Coming right up.
|
||||
ttsd008|prompt-wavs/spk1.wav|prompt-wavs/spk2.wav|[S1] Did you read the email about the deadline change? [S2] I did, we now have until next Friday. [S1] That should give us enough buffer.
|
||||
ttsd009|prompt-wavs/spk1.wav|prompt-wavs/spk2.wav|[S1] Are you joining the team dinner tonight? [S2] I'll be there around seven. [S1] See you then.
|
||||
ttsd010|prompt-wavs/spk1.wav|prompt-wavs/spk2.wav|[S1] Have we backed up the database? [S2] Last backup ran two hours ago. [S1] Good, I'll trigger a fresh snapshot before the migration.
|
||||
Whitespace-only changes.
@@ -53,8 +53,6 @@ _TASK_TO_DATASET: dict[str, str] = {
|
||||
"voice_clone": "seed-tts",
|
||||
"default_voice": "seed-tts-text",
|
||||
"voice_design": "seed-tts-design",
|
||||
"dialogue": "ttsd",
|
||||
"sound_effect": "sound-effect",
|
||||
}
|
||||
|
||||
# Default design dataset path (bundled with the repo)
|
||||
@@ -83,7 +81,6 @@ def build_bench_args(
|
||||
output_dir: str | None,
|
||||
result_filename: str | None,
|
||||
extra_cli_args: list[str],
|
||||
output_len: int | None = None,
|
||||
) -> list[str]:
|
||||
"""Build the ``vllm bench serve --omni`` command for one (task, concurrency) run."""
|
||||
dataset_name = _TASK_TO_DATASET[task]
|
||||
@@ -142,9 +139,6 @@ def build_bench_args(
|
||||
if wer_eval:
|
||||
cmd.append("--seed-tts-wer-eval")
|
||||
|
||||
if output_len is not None:
|
||||
cmd += ["--hf-output-len", str(output_len)]
|
||||
|
||||
if output_dir or result_filename:
|
||||
out_dir = output_dir or "."
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
@@ -244,15 +238,6 @@ def main() -> None:
|
||||
parser.add_argument("--host", default="localhost")
|
||||
parser.add_argument("--port", type=int, default=8000)
|
||||
parser.add_argument("--model-configs", default=str(_DEFAULT_MODEL_CONFIGS), help="Path to model_configs.yaml")
|
||||
parser.add_argument(
|
||||
"--output-len",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Override per-request max_new_tokens forwarded to the server "
|
||||
"(seed-tts datasets default to 2048; the MOSS-TTS full talker takes "
|
||||
"this as 'max_new_frames' so smaller values cap audio length per "
|
||||
"request and keep bench runtime tractable).",
|
||||
)
|
||||
parser.add_argument("extra", nargs=argparse.REMAINDER, help="Extra args passed directly to vllm bench serve")
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -307,7 +292,6 @@ def main() -> None:
|
||||
output_dir=args.output_dir,
|
||||
result_filename=result_filename,
|
||||
extra_cli_args=args.extra or [],
|
||||
output_len=args.output_len,
|
||||
)
|
||||
result = run_one_benchmark(cmd)
|
||||
if result is not None:
|
||||
|
||||
@@ -14,7 +14,6 @@ models:
|
||||
supported_tasks: [default_voice, voice_design]
|
||||
backend: openai-audio-speech
|
||||
endpoint: /v1/audio/speech
|
||||
trust_remote_code: true
|
||||
task_extra_body:
|
||||
default_voice:
|
||||
voice: Vivian
|
||||
@@ -28,7 +27,6 @@ models:
|
||||
supported_tasks: [voice_clone]
|
||||
backend: openai-audio-speech
|
||||
endpoint: /v1/audio/speech
|
||||
trust_remote_code: true
|
||||
task_extra_body:
|
||||
voice_clone:
|
||||
task_type: Base
|
||||
@@ -38,66 +36,7 @@ models:
|
||||
trust_remote_code: true
|
||||
backend: openai-audio-speech
|
||||
endpoint: /v1/audio/speech
|
||||
trust_remote_code: true
|
||||
task_extra_body:
|
||||
voice_clone: {}
|
||||
default_voice:
|
||||
voice: default
|
||||
|
||||
# MOSS-TTS family. After the serving_speech.py variant-detection refactor
|
||||
# all five variants now go through OpenAI /v1/audio/speech with the
|
||||
# variant-specific request fields (ambient_sound, ref_audio_2, instructions).
|
||||
OpenMOSS-Team/MOSS-TTS:
|
||||
supported_tasks: [voice_clone]
|
||||
backend: openai-audio-speech
|
||||
endpoint: /v1/audio/speech
|
||||
task_extra_body:
|
||||
voice_clone: {}
|
||||
|
||||
# MOSS-TTS-v1.5: continued-training upgrade of MOSS-TTS 1.0 — same
|
||||
# MossTTSDelay-8B architecture and API, so identical wiring. Adds 31-language
|
||||
# synthesis via the `language` field and inline `[pause Xs]` markers.
|
||||
OpenMOSS-Team/MOSS-TTS-v1.5:
|
||||
supported_tasks: [voice_clone]
|
||||
backend: openai-audio-speech
|
||||
endpoint: /v1/audio/speech
|
||||
task_extra_body:
|
||||
voice_clone: {}
|
||||
|
||||
OpenMOSS-Team/MOSS-TTS-Realtime:
|
||||
supported_tasks: [voice_clone]
|
||||
backend: openai-audio-speech
|
||||
endpoint: /v1/audio/speech
|
||||
task_extra_body:
|
||||
voice_clone: {}
|
||||
|
||||
# MOSS-TTSD synthesizes [S1]/[S2] dialogue. The seed-tts voice_clone dataset
|
||||
# has single-speaker rows; bench piggybacks on it (both speakers share the
|
||||
# same ref_audio). For real dialogue evaluation use the ttsd dataset.
|
||||
OpenMOSS-Team/MOSS-TTSD-v1.0:
|
||||
supported_tasks: [voice_clone, dialogue]
|
||||
backend: openai-audio-speech
|
||||
endpoint: /v1/audio/speech
|
||||
task_extra_body:
|
||||
voice_clone: {}
|
||||
dialogue: {}
|
||||
|
||||
# MOSS-SoundEffect ignores text/ref_audio; expects ambient_sound +
|
||||
# duration_seconds. The sound_effect dataset module emits those.
|
||||
OpenMOSS-Team/MOSS-SoundEffect:
|
||||
supported_tasks: [sound_effect]
|
||||
backend: openai-audio-speech
|
||||
endpoint: /v1/audio/speech
|
||||
task_extra_body:
|
||||
sound_effect:
|
||||
duration_seconds: 10
|
||||
|
||||
# MOSS-VoiceGenerator: instructions-driven (no ref_audio). seed-tts-design
|
||||
# dataset already emits {instructions, target_text}, so voice_design works
|
||||
# out of the box once the serving validator accepts instructions.
|
||||
OpenMOSS-Team/MOSS-VoiceGenerator:
|
||||
supported_tasks: [voice_design]
|
||||
backend: openai-audio-speech
|
||||
endpoint: /v1/audio/speech
|
||||
task_extra_body:
|
||||
voice_design: {}
|
||||
@@ -1,8 +1,5 @@
|
||||
# Target vllm version: v0.21.0 (releases/v0.21.0 branch).
|
||||
# The pinned VLLM_PRECOMPILED_WHEEL_COMMIT and VLLM_BASE_TAG below must
|
||||
# be updated on every rebase to match the new upstream vllm commit/tag.
|
||||
ARG VLLM_BASE_IMAGE=vllm/vllm-openai
|
||||
ARG VLLM_BASE_TAG=v0.22.0
|
||||
ARG VLLM_BASE_TAG=v0.21.0
|
||||
FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG}
|
||||
ARG APP_DIR=/workspace/vllm-omni
|
||||
WORKDIR ${APP_DIR}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
ARG BASE_IMAGE=vllm/vllm-openai:v0.22.0
|
||||
ARG BASE_IMAGE=vllm/vllm-openai:v0.21.0
|
||||
FROM ${BASE_IMAGE}
|
||||
|
||||
ARG COMMON_WORKDIR=/app
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.22.0
|
||||
ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.21.0
|
||||
FROM ${BASE_IMAGE} AS base
|
||||
|
||||
# Declare a variable to know if we want to use the nightly build or the stable build.
|
||||
|
||||
@@ -88,7 +88,7 @@ ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
|
||||
ENV UV_INDEX_STRATEGY="unsafe-best-match"
|
||||
ENV UV_LINK_MODE="copy"
|
||||
|
||||
ARG VLLM_VERSION=v0.22.0
|
||||
ARG VLLM_VERSION=v0.21.0
|
||||
RUN git clone -b ${VLLM_VERSION} https://github.com/vllm-project/vllm
|
||||
WORKDIR /workspace/vllm
|
||||
|
||||
|
||||
@@ -13,10 +13,6 @@ This section contains design documents and architecture specifications for vLLM-
|
||||
- [Adding Step Execution Support for Diffusion Pipelines](feature/diffusion_step_execution.md)
|
||||
- [Continuous Batching for Step-Wise Diffusion](feature/diffusion_continuous_batching.md)
|
||||
|
||||
## Infrastructure Design Documents
|
||||
|
||||
- [Prometheus Metrics](metrics.md)
|
||||
|
||||
## Module Design Documents
|
||||
|
||||
- [AR Module](module/ar_module.md)
|
||||
|
||||
@@ -1,208 +0,0 @@
|
||||
# Prometheus Metrics Design
|
||||
|
||||
This document describes how vLLM-Omni exposes Prometheus metrics for multi-stage pipelines, the constraints that shaped the design, and how the pipeline-level metrics coexist with upstream vLLM per-engine metrics. This iteration covers the text and audio surface; diffusion / image / video families land in a follow-up.
|
||||
|
||||
## Objectives
|
||||
|
||||
- Expose pipeline-level request and latency metrics that span the full multi-stage execution (orchestrator scope).
|
||||
- Preserve all upstream vLLM per-engine metrics (`vllm:*`) for stages backed by an AR LLM engine, and reshape their `engine` label into `stage` + `replica` so multi-replica deployments gain per-replica visibility automatically.
|
||||
- Expose per-modality SLO metrics that the upstream `vllm:*` families do not capture — audio TTFP / RTF / duration / frames / streaming continuity / silent-loss.
|
||||
- Expose per-replica-edge cross-stage transfer metrics so the slack between E2E latency and the sum of per-stage `gen_time` (queueing, serialization, network) becomes attributable.
|
||||
- Keep the metrics collection overhead low enough that it does not regress TTFA or throughput.
|
||||
|
||||
## Background
|
||||
|
||||
### Upstream vLLM Metrics
|
||||
|
||||
Upstream vLLM defines ~37 Prometheus metric families under the `vllm:` prefix. These are registered by `PrometheusStatLogger` and cover engine-level state: KV cache usage, running/waiting request counts, token throughput, TTFT, inter-token latency, e2e latency, and so on. They are served via the `/metrics` HTTP endpoint provided by `prometheus_fastapi_instrumentator` and the default `prometheus_client` WSGI handler.
|
||||
|
||||
vLLM's `unregister_vllm_metrics()` function strips every `prometheus_client` collector whose `_name` attribute contains the substring `"vllm"`. This runs during engine initialization to clean up stale collectors from prior instantiations within the same process.
|
||||
|
||||
### The Problem
|
||||
|
||||
vLLM-Omni runs multiple engine instances (stages × replicas) within a single process, coordinated by an Orchestrator. The pipeline needs its own metrics — aggregate request counts, end-to-end latency across all stages, per-modality SLO signals, and cross-stage transfer attribution — that do not exist in upstream vLLM. All pipeline-level metrics use the `vllm:omni_` prefix to distinguish them from upstream per-engine metrics. At import time (see `vllm_omni/patch.py`) we replace `unregister_vllm_metrics()` with a scoped version that still strips upstream `vllm:*` collectors before each new `PrometheusStatLogger` registers (so multi-engine processes don't crash on duplicate registration), but preserves anything prefixed `vllm:omni_` / `vllm_omni`.
|
||||
|
||||
Upstream per-engine metrics retain the `vllm:` prefix but are now registered by `OmniPrometheusStatLogger`, a thin subclass of upstream's `PrometheusStatLogger` that reshapes the single `engine` label into a `stage` + `replica` pair (see "OmniPrometheusStatLogger wrap" below).
|
||||
|
||||
## Architecture
|
||||
|
||||
### Component Overview
|
||||
|
||||
```text
|
||||
+------------------------+
|
||||
| API Server (FastAPI) |
|
||||
| GET /metrics |
|
||||
+-----------+------------+
|
||||
|
|
||||
prometheus_client default registry
|
||||
|
|
||||
+--------+--------+--------+--------+--------+
|
||||
| |
|
||||
vllm:omni_* vllm:*
|
||||
collectors collectors
|
||||
| |
|
||||
+----+--------+ +-----------+ +----------+ +-----------+
|
||||
| OmniPromet- | | OmniMod- | | OmniTra- | | OmniProm- |
|
||||
| heusMetrics | | alityMet- | | nsferMe- | | etheusSt- |
|
||||
| | | rics | | trics | | atLogger |
|
||||
+----+--------+ +-----+-----+ +----+-----+ +----+------+
|
||||
| | | |
|
||||
OmniBase OmniBase Orchestrator Orchestrator
|
||||
(request life- (finalize + (record_trans- (per-(stage,
|
||||
cycle, success/ streaming fer_tx/rx replica)
|
||||
fail counter) hooks via hooks via scheduler/
|
||||
observe_* emit hook in iteration
|
||||
APIs) OrchestratorAg- stats)
|
||||
gregator)
|
||||
```
|
||||
|
||||
### Data Flow
|
||||
|
||||
There are four independent paths for metric collection.
|
||||
|
||||
**Path 1: Pipeline-level metrics (`vllm:omni_*`)**
|
||||
|
||||
`OmniPrometheusMetrics` registers the Gauge / Counter / Histogram collectors at import time. It is instantiated once per entrypoint, labeled with the model name. The entrypoint calls its methods as requests progress:
|
||||
|
||||
- `set_running(n)` / `set_waiting(n)` — updated after each request completes. The running count comes from `OmniRequestCounter`, a simple counter incremented/decremented by the Orchestrator as it tracks requests. Waiting is derived as `total - running`.
|
||||
- `request_succeeded(e2e_seconds, finished_reason="stop")` — recorded when a request finishes at the final stage. `finished_reason` is extracted from `engine_outputs.outputs[0].finish_reason` (vLLM `CompletionOutput` convention) and increments `vllm:omni_requests_success_total{finished_reason}`.
|
||||
- `request_failed()` — recorded by the cleanup path when a request exits without natural completion. Internally maps to `finished_reason="abort"` so a single Counter family covers both natural and aborted completion.
|
||||
|
||||
**Path 2: Audio modality metrics (`vllm:omni_audio_*`)**
|
||||
|
||||
`OmniModalityMetrics` registers seven audio families with `{model_name, stage, replica}` (plus an extra `threshold_ms` / `reason` label on the two extra-cardinality Counters). Three observation sites:
|
||||
|
||||
- `observe_modality_at_finalize(...)` — called from `omni_base._process_single_result` inside the existing `e2e_done` finalize guard. For `output_type == "audio"` it emits `audio_frames_total`, `audio_duration_s`, `audio_rtf` (or `audio_skipped_requests_total{reason="no_audio_data"}` when no audio was produced). Sample rate is resolved from `engine_outputs.multimodal_output` via `definitions.resolve_audio_sample_rate(...)` (fallback chain mirrors `serving_chat.py`'s audio response path).
|
||||
- `observe_audio_first_packet(...)` — called from the OpenAI SSE audio branch in `serving_chat.py` on the first audio packet for a request. The once-per-request guard is held by `ClientRequestState.first_audio_ts`. The `request_arrival_ts` anchor is stored in `ClientRequestState` by `async_omni.generate()`, computed at request entry.
|
||||
- `observe_audio_streaming_finalize(...)` — called from `serving_chat.py` after the streaming chunk loop exhausts. It runs the per-chunk player simulation from `vllm_omni/benchmarks/audio_continuity.py` to compute the worst-case underrun and emits `audio_underrun_s` plus (when the request stayed below the threshold) `audio_continuity_ok_total{threshold_ms}`. Per-chunk PCM byte counts and arrival timestamps are recorded by the same audio branch that updates `first_audio_ts`.
|
||||
|
||||
**Path 3: Cross-stage transfer metrics (`vllm:omni_transfer_*`)**
|
||||
|
||||
`OmniTransferMetrics` registers four Histogram families with `{model_name, from_stage, from_replica, to_stage, to_replica}` labels. Each observation corresponds to one physical transfer hop (one chunk between adjacent stages), not the per-request accumulated total — so the histograms track per-transfer distribution.
|
||||
|
||||
The hook lives in `OrchestratorAggregator.record_transfer_tx` and `record_transfer_rx`. After the existing `TransferEdgeStats` accumulation, the aggregator calls `_emit_transfer_tx` / `_emit_transfer_rx`. Those:
|
||||
|
||||
1. Resolve `from_replica` / `to_replica` via a `replica_resolver` callback supplied by `async_omni.py`. The resolver delegates to `stage_pool.get_bound_replica_id(request_id)` — i.e. the orchestrator's existing sticky-routing binding is the source of truth.
|
||||
2. Convert the underlying `_ms` accumulators to seconds and call the `_s`-suffixed observe methods on `OmniTransferMetrics`.
|
||||
|
||||
Defensive fail-safe: if `transfer_emitter` or `replica_resolver` is missing, or the resolver returns `None` for either side, the emit is skipped silently (the underlying `TransferEdgeStats` accumulation is unaffected).
|
||||
|
||||
**Path 4: Per-engine metrics (`vllm:*`, stage/replica wrap)**
|
||||
|
||||
The Orchestrator instantiates `OmniPrometheusStatLogger` (a thin subclass of upstream `vllm.v1.metrics.loggers.PrometheusStatLogger`) and feeds it scheduler stats and iteration stats after processing each batch of engine outputs. This populates the standard ~37 vLLM metric families (TTFT, ITL, TPOT, KV cache usage, etc.) using the same upstream code path — but with the `engine` label reshaped into `stage` + `replica` so multi-replica deployments produce distinct series per replica. See the next section for the wrap mechanics.
|
||||
|
||||
### Shared State Between Threads
|
||||
|
||||
The Orchestrator runs in a background thread. The API server (OmniBase) runs in the asyncio event loop thread. `OmniRequestCounter` bridges them — a plain Python object with an `int` field. The Orchestrator increments/decrements it; the entrypoint reads it for gauge updates. No lock is needed because the counter is advisory (a stale read by one Prometheus scrape interval is acceptable). It is created by `AsyncOmniEngine.__init__()` and passed to the Orchestrator at construction time.
|
||||
|
||||
### Metric Registration and Lifecycle
|
||||
|
||||
All `vllm:omni_*` collectors are registered once when their owning class (`OmniPrometheusMetrics` / `OmniModalityMetrics` / `OmniTransferMetrics`) is imported. Per-`(stage, replica)` labels are bound lazily on first observation to avoid registering label sets for combinations that never produce data.
|
||||
|
||||
The `prometheus_client` default registry holds all collectors. FastAPI's `/metrics` endpoint serves the default registry, so `vllm:omni_*` and the wrapped `vllm:*` metrics appear in the same scrape response alongside `http_*` and `process_*` metrics from the instrumentator and the Python client runtime.
|
||||
|
||||
## OmniPrometheusStatLogger Wrap
|
||||
|
||||
Upstream `PrometheusStatLogger.__init__` hard-codes `labelnames = ["model_name", "engine"]` as a local variable, references it across ~37 metric-family construction sites, and uses the `engine` label value in five different `.labels()` call shapes (kwarg with int engine, kwarg with str engine, positional with str engine in the middle, plus a `metrics_info["engine"] = str(...)` dict pattern). To reshape `engine` into `stage` + `replica` without forking the entire upstream `__init__`, the wrap uses three coordinated mechanisms:
|
||||
|
||||
1. **Class-level metric class slot overrides.** `OmniPrometheusStatLogger` overrides `_gauge_cls`, `_counter_cls`, `_histogram_cls` (which upstream calls via `self._gauge_cls(...)` etc.) with `_RelabelGauge` / `_RelabelCounter` / `_RelabelHistogram` wrapper classes. These intercept the `labelnames` kwarg at metric family creation time and replace `engine` with `("stage", "replica")`.
|
||||
2. **Property descriptor for `per_engine_labelvalues`.** Upstream builds `self.per_engine_labelvalues = {idx: [model_name, str(idx)]}` inside `__init__` and then captures it into a local variable for `create_metric_per_engine` calls. By making `per_engine_labelvalues` a Python property on the subclass, the setter intercepts upstream's assignment and rewrites each 2-tuple into a 3-tuple `[model_name, stage, replica]` using the `stage_replica_map` supplied at construction time. The captured local then sees the rewritten dict.
|
||||
3. **Override of `.labels()` on the wrapper classes.** For the five call sites that pass `engine` directly (kwarg or positional, int or str), `_RelabelMixin.labels()` translates the engine value back to `(stage, replica)` via a process-level `_ENGINE_INDEX_MAP` populated by `OmniPrometheusStatLogger.__init__`. This handles `gauge_engine_sleep_state.labels(engine=idx, ...)`, `counter_request_success_base.labels(model_name, str(idx), str(reason))`, `info_gauge.labels(**metrics_info)`, etc.
|
||||
|
||||
The three sub-helpers that upstream `PrometheusStatLogger.__init__` constructs (`spec_decoding_prom` / `kv_connector_prom` / `perf_metrics_prom`) use their own `_counter_cls` / `_gauge_cls` / `_histogram_cls` slots and would otherwise build families with the raw 2-element labelnames. `_OmniPerfMetricsProm` / `_OmniSpecDecodingProm` / `_OmniKVConnectorProm` subclass each helper to route the same relabel mixin through their internal family construction.
|
||||
|
||||
The `Orchestrator` constructs `stage_replica_map` from the static `stage_pools` configuration at startup:
|
||||
|
||||
```python
|
||||
stage_replica_map = {
|
||||
flat_idx: (str(stage_id), str(replica_id))
|
||||
for flat_idx, (stage_id, replica_id) in enumerate(
|
||||
(s, r)
|
||||
for s, pool in enumerate(stage_pools)
|
||||
for r in range(pool.num_replicas)
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
A reverse map `(stage_id, replica_id) -> flat_idx` is maintained on the Orchestrator so the per-replica `record(engine_idx=...)` call site can look up the right flat index.
|
||||
|
||||
> Dynamic add/remove of replicas at runtime is intentionally out of scope — the upstream `PrometheusStatLogger` materializes per-engine_idx child metrics at init time, and supporting hot-add would require non-trivial intervention into upstream's per-family child dictionaries.
|
||||
|
||||
## Gating: `--log-stats`
|
||||
|
||||
All metrics — both the 15 `vllm:omni_*` families and the ~65 upstream `vllm:*` wrap families — are gated by the user's `--log-stats` CLI flag (default off). The flag is plumbed from `OmniBase.__init__(log_stats=...)` through `AsyncOmniEngine` to the stage-spawn helpers and to the three Prometheus metric classes (`OmniPrometheusMetrics` / `OmniModalityMetrics` / `OmniTransferMetrics`), and is forwarded to `Orchestrator._init_metrics_state(...)`.
|
||||
|
||||
Behavior with `--log-stats=off` (default):
|
||||
|
||||
- The three `Omni*Metrics` classes register their module-level Gauge / Counter / Histogram families at import time (prometheus_client requires up-front registration), but each `observe / inc / set` method early-returns. The per-label child series for `vllm:omni_*` stay materialized but never have data written to them.
|
||||
- `OmniPrometheusStatLogger` is not constructed in `_init_metrics_state`, so the ~65 upstream `vllm:*` wrap families are not registered in the default registry at all.
|
||||
- The engine core's `Scheduler.make_stats()` also short-circuits inside upstream (`if not self.log_stats: return None`), so no `SchedulerStats` is produced per step — the per-iteration cost is bounded by the existing upstream gate.
|
||||
|
||||
Behavior with `--log-stats=on`: all metric paths fire normally; the orchestrator's per-replica recording is bounded only by `OmniSchedulerMixin.make_stats()`'s per-scheduler 1 Hz throttle (see next section).
|
||||
|
||||
The overhead with the flag on is small enough that an A/B benchmark on Qwen3-Omni-30B single replica (30 sequential audio requests) showed a mean latency delta of +0.6% (Welch's t = 0.318, n=30, not statistically significant at α=0.05); the `/metrics` line count drops from 1358 to 124 lines when the flag is off.
|
||||
|
||||
## Throttling: `make_stats()` Override
|
||||
|
||||
Upstream vLLM's `Scheduler.make_stats()` runs on every AR generation step, returning a SchedulerStats object for the orchestrator. Under vLLM's architecture, this is fine. But since vLLM-Omni requires that the object be serialized and transferred over ZMQ, receiving a SchedulerStats object on every step can introduce unacceptable overhead to the system.
|
||||
|
||||
`OmniSchedulerMixin.make_stats()` (in `vllm_omni/core/sched/omni_scheduler_mixin.py`) throttles stats emission to at most once per second **per scheduler** — i.e. per `(stage, replica)` since each replica owns its own scheduler instance. Between intervals it returns `None`, which the engine core skips serializing. This keeps gauges fresh enough for Prometheus scrapes (typically 15-30s intervals) while eliminating the per-step overhead.
|
||||
|
||||
The orchestrator side does not add its own throttle on top: the per-replica recording loop gates only on `raw_outputs.scheduler_stats is not None` (i.e. this replica's scheduler passed its own 1 Hz gate). A previous global `_last_stats_ts` on the orchestrator was removed because it starved every replica other than the first to emit within each second.
|
||||
|
||||
## Metric Definitions
|
||||
|
||||
### Pipeline (4)
|
||||
|
||||
| Metric | Type | Labels | Description |
|
||||
|--------|------|--------|-------------|
|
||||
| `vllm:omni_num_requests_running` | Gauge | `model_name` | Requests currently executing across all stages |
|
||||
| `vllm:omni_num_requests_waiting` | Gauge | `model_name` | Requests queued but not yet scheduled |
|
||||
| `vllm:omni_requests_success_total` | Counter | `model_name`, `finished_reason` | Total requests by completion reason ({stop, length, abort, ...}); aborts cover client-disconnect / cancellation paths in addition to upstream `FinishReason.ABORT` |
|
||||
| `vllm:omni_e2e_request_latency_s` | Histogram | `model_name` | Pipeline-global end-to-end latency in seconds |
|
||||
|
||||
### Audio (7)
|
||||
|
||||
Labels: `{model_name, stage, replica}` plus the listed extra label.
|
||||
|
||||
| Metric | Type | Extra label | Description |
|
||||
|--------|------|-------------|-------------|
|
||||
| `vllm:omni_audio_ttfp_s` | Histogram | — | Time from request arrival to first audio packet/frame |
|
||||
| `vllm:omni_audio_duration_s` | Histogram | — | Audio content duration (`audio_frames / sample_rate`) |
|
||||
| `vllm:omni_audio_rtf` | Histogram | — | Real-time factor `stage_gen_time_s / audio_duration_s` (SLO `< 1`); uses `RTF_BUCKETS` |
|
||||
| `vllm:omni_audio_frames_total` | Counter | — | Cumulative audio frames generated |
|
||||
| `vllm:omni_audio_underrun_s` | Histogram | — | Per-request worst-case player deficit; `> 0` indicates listener heard silent gaps |
|
||||
| `vllm:omni_audio_continuity_ok_total` | Counter | `threshold_ms` | Incremented when the request's worst underrun stayed below `threshold_ms` |
|
||||
| `vllm:omni_audio_skipped_requests_total` | Counter | `reason` | Silent-loss counter — code2wav rejected malformed codec input and returned `200 OK` with empty audio |
|
||||
|
||||
### Cross-stage transfer (4)
|
||||
|
||||
Labels: `{model_name, from_stage, from_replica, to_stage, to_replica}`.
|
||||
|
||||
| Metric | Type | Description |
|
||||
|--------|------|-------------|
|
||||
| `vllm:omni_transfer_size_bytes` | Histogram | Per-transfer payload size in bytes |
|
||||
| `vllm:omni_transfer_tx_s` | Histogram | Sender-side time (serialize + submit to connector) |
|
||||
| `vllm:omni_transfer_rx_s` | Histogram | Receiver-side time (recv + deserialize) |
|
||||
| `vllm:omni_transfer_in_flight_s` | Histogram | Network in-flight time (TX done → RX recv start) |
|
||||
|
||||
### LLM stage-level (wrapped `vllm:*`)
|
||||
|
||||
After the wrap, every upstream `vllm:*` family — TTFT, ITL, TPOT, e2e latency, KV cache usage, scheduler running/waiting, request success counts, etc. — carries `{model_name, stage, replica}` labels. For the full upstream catalog see [the vLLM docs](https://github.com/vllm-project/vllm/blob/main/docs/usage/metrics.md); note that metrics depending on features unsupported in vLLM-Omni (e.g. speculative decoding, LoRA) will not be available.
|
||||
|
||||
## Naming Convention
|
||||
|
||||
- All time-bearing metrics use the `_s` suffix (values in seconds). Two bucket families are used:
|
||||
- `SECONDS_BUCKETS` (0.05 s – 300 s) for e2e / generation / TTFP style values.
|
||||
- `SECONDS_FAST_BUCKETS` (0.001 s – 60 s) for fine-grained cross-stage transfer and audio-underrun values that need millisecond-level resolution.
|
||||
- Counters use the `_total` suffix (auto-appended by `prometheus_client`).
|
||||
- Sizes use the `_bytes` suffix.
|
||||
- All omni-specific families are prefixed `vllm:omni_`. The upstream `unregister_vllm_metrics()` function is monkey-patched to a scoped version that still strips upstream `vllm:*` collectors (so multi-engine init within one process does not crash on duplicate registration) but preserves anything prefixed `vllm:omni_` / `vllm_omni`.
|
||||
|
||||
## Logging vs. Prometheus
|
||||
|
||||
`OrchestratorAggregator` (in `vllm_omni/metrics/stats.py`) is the logging-oriented metrics path. It collects detailed per-request, per-stage, and per-transfer statistics and prints formatted tables to the `INFO` log. This is designed for development and debugging — individual request traces, transfer bandwidth, inter-stage timing.
|
||||
|
||||
`OmniPrometheusMetrics` / `OmniModalityMetrics` / `OmniTransferMetrics` form the Prometheus-oriented path. They record aggregate counters, gauges, and histograms suitable for time-series monitoring and alerting. Both paths share the same source data (`StageRequestStats`, `TransferEdgeStats`) — `OrchestratorAggregator.record_transfer_tx/rx` in particular calls both the existing accumulator code and the Prometheus emit hook in the same method body. The two consumption models can run simultaneously without coupling.
|
||||
|
||||
The separation follows upstream vLLM's pattern of `LoggingStatLogger` vs. `PrometheusStatLogger` — same underlying data, different consumption models.
|
||||
@@ -411,8 +411,6 @@ Notes:
|
||||
- `runtime.max_batch_size` controls stage-level batching.
|
||||
- Thinker/Talker commonly use `enforce_eager: false` for CUDA Graph paths.
|
||||
- Code2Wav often remains eager (`enforce_eager: true`) depending on runtime behavior.
|
||||
- Qwen3-Omni defaults `VLLM_USE_FLASHINFER_MOE_FP16=0`. The Triton has been more stable & faster
|
||||
than the FlashInfer CUTLASS unquantized MoE backend on recent vLLM rebases.
|
||||
|
||||
#### 2) Enable async chunk
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ vLLM-Omni is a Python library that supports the following GPU variants. The libr
|
||||
|
||||
### Pre-built wheels
|
||||
|
||||
Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, 0.21.0, and 0.22.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
|
||||
Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, and 0.21.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
|
||||
|
||||
=== "NVIDIA CUDA"
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ Therefore, it is recommended to install vLLM and vLLM-Omni with a **fresh new**
|
||||
|
||||
vLLM-Omni is built based on vLLM. Please install it with command below.
|
||||
```bash
|
||||
uv pip install vllm==0.22.0 --torch-backend=auto
|
||||
uv pip install vllm==0.21.0 --torch-backend=auto
|
||||
```
|
||||
|
||||
#### Installation of vLLM-Omni
|
||||
@@ -39,13 +39,13 @@ uv pip install 'vllm-omni[demo]'
|
||||
# --8<-- [start:build-wheel-from-source]
|
||||
|
||||
#### Installation of vLLM
|
||||
If you do not need to modify source code of vLLM, you can directly install the stable 0.22.0 release version of the library
|
||||
If you do not need to modify source code of vLLM, you can directly install the stable 0.21.0 release version of the library
|
||||
|
||||
```bash
|
||||
uv pip install vllm==0.22.0 --torch-backend=auto
|
||||
uv pip install vllm==0.21.0 --torch-backend=auto
|
||||
```
|
||||
|
||||
The 0.22.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
||||
The 0.21.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
||||
|
||||
#### Installation of vLLM-Omni
|
||||
Since vllm-omni is rapidly evolving, it's recommended to install it from source
|
||||
@@ -66,12 +66,12 @@ If you want to check, modify or debug with source code of vLLM, install the libr
|
||||
```bash
|
||||
git clone https://github.com/vllm-project/vllm.git
|
||||
cd vllm
|
||||
git checkout v0.22.0
|
||||
git checkout v0.21.0
|
||||
```
|
||||
Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set `VLLM_PRECOMPILED_WHEEL_LOCATION` as your local absolute path of whl file.
|
||||
```bash
|
||||
#For CUDA 13.0 (the default for v0.22.0; the wheel filename has no `+cu130` suffix)
|
||||
export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.22.0/vllm-0.22.0-cp38-abi3-manylinux_2_35_x86_64.whl
|
||||
#For CUDA 13.0 (the default for v0.21.0; the wheel filename has no `+cu130` suffix)
|
||||
export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.21.0/vllm-0.21.0-cp38-abi3-manylinux_2_35_x86_64.whl
|
||||
```
|
||||
Install vllm with command below (If you have no existing PyTorch).
|
||||
```bash
|
||||
@@ -102,7 +102,7 @@ docker run --runtime nvidia --gpus 2 \
|
||||
--env "HF_TOKEN=$HF_TOKEN" \
|
||||
-p 8091:8091 \
|
||||
--ipc=host \
|
||||
vllm/vllm-omni:v0.22.0 \
|
||||
vllm/vllm-omni:v0.21.0 \
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||
```
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ vLLM-Omni current recommends the steps in under setup through Docker Images.
|
||||
|
||||
vLLM-Omni is built based on vLLM. Please install it with command below.
|
||||
```bash
|
||||
uv pip install vllm==0.22.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.22.0/rocm722
|
||||
uv pip install vllm==0.21.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.21.0/rocm722
|
||||
```
|
||||
|
||||
#### Installation of vLLM-Omni
|
||||
@@ -34,13 +34,13 @@ uv pip install onnxruntime-rocm
|
||||
# --8<-- [start:build-wheel-from-source]
|
||||
|
||||
#### Installation of vLLM
|
||||
If you do not need to modify source code of vLLM, you can directly install the stable 0.22.0 release version of the library
|
||||
If you do not need to modify source code of vLLM, you can directly install the stable 0.21.0 release version of the library
|
||||
|
||||
```bash
|
||||
uv pip install vllm==0.22.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.22.0/rocm722
|
||||
uv pip install vllm==0.21.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.21.0/rocm722
|
||||
```
|
||||
|
||||
The pre-built 0.22.0 vLLM wheel targets ROCm 7.2.2. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
||||
The pre-built 0.21.0 vLLM wheel targets ROCm 7.2.2. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
||||
|
||||
#### Installation of vLLM-Omni
|
||||
Since vllm-omni is rapidly evolving, it's recommended to install it from source
|
||||
@@ -58,7 +58,7 @@ If you want to check, modify or debug with source code of vLLM, install the libr
|
||||
```bash
|
||||
git clone https://github.com/vllm-project/vllm.git
|
||||
cd vllm
|
||||
git checkout v0.22.0
|
||||
git checkout v0.21.0
|
||||
python3 -m pip install -r requirements/rocm.txt
|
||||
python3 setup.py develop
|
||||
```
|
||||
@@ -130,7 +130,7 @@ docker run --rm \
|
||||
-v ~/.cache/huggingface:/root/.cache/huggingface \
|
||||
--env "HF_TOKEN=$HF_TOKEN" \
|
||||
-p 8091:8091 \
|
||||
vllm/vllm-omni-rocm:v0.22.0 \
|
||||
vllm/vllm-omni-rocm:v0.21.0 \
|
||||
--model Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||
```
|
||||
|
||||
@@ -149,7 +149,7 @@ docker run --rm -it \
|
||||
-v ~/.cache/huggingface:/root/.cache/huggingface \
|
||||
--env "HF_TOKEN=$HF_TOKEN" \
|
||||
--entrypoint bash \
|
||||
vllm/vllm-omni-rocm:v0.22.0
|
||||
vllm/vllm-omni-rocm:v0.21.0
|
||||
```
|
||||
|
||||
# --8<-- [end:pre-built-images]
|
||||
@@ -19,10 +19,10 @@ uv venv --python 3.12 --seed
|
||||
source .venv/bin/activate
|
||||
|
||||
# On CUDA
|
||||
uv pip install vllm==0.22.0 --torch-backend=auto
|
||||
uv pip install vllm==0.21.0 --torch-backend=auto
|
||||
|
||||
# On ROCm
|
||||
uv pip install vllm==0.22.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.22.0/rocm721
|
||||
uv pip install vllm==0.21.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.21.0/rocm721
|
||||
|
||||
git clone https://github.com/vllm-project/vllm-omni.git
|
||||
cd vllm-omni
|
||||
@@ -35,7 +35,7 @@ For additional installation methods — please see the [installation guide](inst
|
||||
!!! note
|
||||
It is important to install the same major & minor version of vLLM and vLLM Omni, otherwise things may not work as expected. If the versions are misaligned, you will see a warning when you import vLLM Omni.
|
||||
|
||||
If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.22.0` and vLLM Omni `0.22.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.
|
||||
If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.21.0` and vLLM Omni `0.21.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.
|
||||
|
||||
## Offline Inference
|
||||
|
||||
|
||||
@@ -119,10 +119,9 @@ def extract_omni_serve_subparser_init():
|
||||
exec_globals = {
|
||||
"_FlexibleArgumentParser": _FlexibleArgumentParser,
|
||||
"FlexibleArgumentParser": _FlexibleArgumentParser,
|
||||
"TrackingArgumentParser": _FlexibleArgumentParser, # only needs add_argument, not tracking
|
||||
"TrackingNamespace": None, # type annotation only
|
||||
"make_arg_parser": lambda parser: parser, # no-op for doc
|
||||
"_ensure_vllm_platform": lambda: None, # no-op for doc
|
||||
"nullify_stage_engine_defaults": lambda parser: None, # no-op for doc
|
||||
"VLLM_SUBCMD_PARSER_EPILOG": "",
|
||||
"logger": logger,
|
||||
"DummySubparsers": DummySubparsers,
|
||||
|
||||
@@ -33,7 +33,6 @@ th {
|
||||
| `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `Cosmos3OmniDiffusersPipeline` | Cosmos3 T2I, T2V, I2V | `nvidia/Cosmos3-Nano` | ✅︎ | | | |
|
||||
| `WanSpeechToVideoPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `LTX2Pipeline` | LTX-2-T2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | |
|
||||
|
||||
@@ -381,44 +381,6 @@ are cached in-process with a shared LRU so repeated requests with the same
|
||||
all TTS model types; deleting a voice invalidates every model-type slot at
|
||||
once.
|
||||
|
||||
### Precomputed Custom Voices
|
||||
|
||||
Qwen3-TTS Base and VoxCPM2 can load offline-precomputed voices at startup.
|
||||
Generate a directory containing `custom_voice_manifest.json` plus one
|
||||
`.safetensors` file per voice, then set the pipeline-wide deploy config field:
|
||||
|
||||
```yaml
|
||||
custom_voice_dir: /path/to/custom_voices
|
||||
```
|
||||
|
||||
Qwen3-TTS profiles are created with:
|
||||
|
||||
```bash
|
||||
python examples/online_serving/text_to_speech/qwen3_tts/precompute_custom_voice.py \
|
||||
--model Qwen/Qwen3-TTS-12Hz-1.7B-Base \
|
||||
--voice-name alice \
|
||||
--ref-audio /path/to/reference.wav \
|
||||
--ref-text "Original transcript of the reference audio" \
|
||||
--mode icl \
|
||||
--output-dir /path/to/custom_voices
|
||||
```
|
||||
|
||||
VoxCPM2 profiles are created with:
|
||||
|
||||
```bash
|
||||
python examples/online_serving/text_to_speech/voxcpm2/precompute_custom_voice.py \
|
||||
--model openbmb/VoxCPM2 \
|
||||
--voice-name alice \
|
||||
--ref-audio /path/to/reference.wav \
|
||||
--mode ref_continuation \
|
||||
--prompt-text "Original transcript of the reference audio" \
|
||||
--output-dir /path/to/custom_voices
|
||||
```
|
||||
|
||||
Only profiles whose safetensors payload can be loaded and validated are exposed
|
||||
by `GET /v1/audio/voices`. Valid precomputed voices can be used in
|
||||
`POST /v1/audio/speech` by passing `voice="alice"` without `ref_audio`.
|
||||
|
||||
**Configuration (environment variables):**
|
||||
|
||||
| Variable | Default | Description |
|
||||
|
||||
@@ -1,98 +0,0 @@
|
||||
# Production Metrics
|
||||
|
||||
vLLM-Omni exposes Prometheus metrics via the `/metrics` endpoint on the OpenAI-compatible API server. This page covers the text and audio surface; diffusion / image / video metrics are tracked in a follow-up PR.
|
||||
|
||||
```bash
|
||||
vllm-omni serve Qwen/Qwen3-Omni-30B-A3B-Instruct --port 8000 --log-stats
|
||||
curl http://localhost:8000/metrics
|
||||
```
|
||||
|
||||
**`--log-stats` is required to populate metric data.** Without the flag, the endpoint still returns `200 OK` but the upstream `vllm:*` wrap is not registered at all, and the 15 `vllm:omni_*` families are registered as placeholders with no data written to them. This keeps the runtime cost essentially zero for deployments that don't need monitoring. With the flag, all ~80 families populate.
|
||||
|
||||
## Metric Namespaces
|
||||
|
||||
| Prefix | Source | Present when |
|
||||
|--------|--------|--------------|
|
||||
| `vllm:omni_` | vLLM-Omni orchestrator / audio modality / cross-stage transfer | Pipeline-dependent |
|
||||
| `vllm:` | Upstream vLLM engine, wrapped by `OmniPrometheusStatLogger` to expose `{stage, replica}` | Pipeline includes an LLM (AR) stage |
|
||||
| `http_` / `process_` | Uvicorn / Python runtime | Always |
|
||||
|
||||
## Pipeline-Level Metrics (`vllm:omni_`)
|
||||
|
||||
Defined in `vllm_omni/metrics/prometheus.py`. Track request lifecycle across the full multi-stage pipeline.
|
||||
|
||||
### Request counts
|
||||
|
||||
| Metric | Type | Labels | Description |
|
||||
|--------|------|--------|-------------|
|
||||
| `vllm:omni_num_requests_running` | Gauge | `model_name` | Pipeline-global in-flight requests (dispatched to engine, not yet finalized) |
|
||||
| `vllm:omni_num_requests_waiting` | Gauge | `model_name` | Requests waiting in the Orchestrator queue |
|
||||
| `vllm:omni_requests_success_total` | Counter | `model_name`, `finished_reason` | Total requests by completion reason. `finished_reason` ∈ {`stop`, `length`, `abort`, ...} mirroring upstream `vllm:request_success_total`; aborts cover client disconnect / cancellation paths in addition to upstream `FinishReason.ABORT` |
|
||||
|
||||
### Latency
|
||||
|
||||
| Metric | Type | Labels | Description |
|
||||
|--------|------|--------|-------------|
|
||||
| `vllm:omni_e2e_request_latency_s` | Histogram | `model_name` | Pipeline-global end-to-end request latency in seconds |
|
||||
|
||||
## Audio Modality Metrics (`vllm:omni_`)
|
||||
|
||||
Emitted at request finalize, except for `audio_ttfp_s` (streaming-hook at the first audio packet) and `audio_underrun_s` / `audio_continuity_ok_total` (streaming finalize, after the chunk stream is exhausted). All carry `{model_name, stage, replica}` plus the listed extra label.
|
||||
|
||||
| Metric | Type | Extra label | Description |
|
||||
|--------|------|-------------|-------------|
|
||||
| `vllm:omni_audio_ttfp_s` | Histogram | — | Time from request arrival to first audio packet/frame |
|
||||
| `vllm:omni_audio_duration_s` | Histogram | — | Audio content duration (`audio_frames / sample_rate`) |
|
||||
| `vllm:omni_audio_rtf` | Histogram | — | Real-time factor (`stage_gen_time_s / audio_duration_s`); streaming TTS SLO red line `< 1`; uses `RTF_BUCKETS` |
|
||||
| `vllm:omni_audio_frames_total` | Counter | — | Cumulative audio frame count; throughput via `rate()` |
|
||||
| `vllm:omni_audio_underrun_s` | Histogram | — | Per-request worst-case player deficit; `> 0` indicates listener heard silent gaps |
|
||||
| `vllm:omni_audio_continuity_ok_total` | Counter | `threshold_ms` | Incremented when the request's worst underrun stayed below `threshold_ms` |
|
||||
| `vllm:omni_audio_skipped_requests_total` | Counter | `reason` | Silent-loss counter — code2wav rejected malformed codec input and returned `200 OK` with empty audio |
|
||||
|
||||
The continuity math comes from `vllm_omni/benchmarks/audio_continuity.py::compute_continuity_stats` so the server-side observation aligns with the bench-side definition.
|
||||
|
||||
## Cross-Stage Transfer Metrics (`vllm:omni_`)
|
||||
|
||||
Per-physical-transfer histograms tracking the data hop between adjacent stages. Labels `{model_name, from_stage, from_replica, to_stage, to_replica}` let dashboards attribute latency to specific replica edges. `from_replica` / `to_replica` are resolved from the orchestrator's sticky-routing binding (`stage_pool.get_bound_replica_id(request_id)`), so no extra plumbing through `TransferEdgeStats` is needed.
|
||||
|
||||
| Metric | Type | Description |
|
||||
|--------|------|-------------|
|
||||
| `vllm:omni_transfer_size_bytes` | Histogram | Per-transfer payload size in bytes |
|
||||
| `vllm:omni_transfer_tx_s` | Histogram | Sender-side time (serialize + submit to connector) |
|
||||
| `vllm:omni_transfer_rx_s` | Histogram | Receiver-side time (recv + deserialize) |
|
||||
| `vllm:omni_transfer_in_flight_s` | Histogram | Network in-flight time (TX done → RX recv start) |
|
||||
|
||||
## vLLM Engine Metrics (`vllm:`)
|
||||
|
||||
When the pipeline includes an LLM stage, the upstream vLLM engine exposes its full set of ~37 metric families under the `vllm:` prefix.
|
||||
|
||||
vLLM-Omni wraps the upstream `vllm.v1.metrics.loggers.PrometheusStatLogger` with `OmniPrometheusStatLogger` so that the original `engine` single label is reshaped into `stage` + `replica`. Every `vllm:*` family — TTFT, ITL, TPOT, e2e latency, KV cache usage, scheduler running/waiting, request success counts, etc. — therefore gains per-`(stage, replica)` visibility automatically. No omni-side duplicate is needed for the text path.
|
||||
|
||||
```text
|
||||
# Before wrap:
|
||||
vllm:num_requests_running{model_name="...", engine="1"} 3.0
|
||||
|
||||
# After wrap:
|
||||
vllm:num_requests_running{model_name="...", stage="1", replica="0"} 2.0
|
||||
vllm:num_requests_running{model_name="...", stage="1", replica="1"} 1.0
|
||||
```
|
||||
|
||||
For the full list of upstream metrics, see [the vLLM docs](https://github.com/vllm-project/vllm/blob/main/docs/usage/metrics.md).
|
||||
|
||||
## Metric Availability by Pipeline Type
|
||||
|
||||
| Metric group | Multi-stage LLM (Qwen3-Omni) |
|
||||
|---|---|
|
||||
| `vllm:omni_` request tracking + latency | With `--log-stats` |
|
||||
| `vllm:omni_` audio modality | With `--log-stats`, if pipeline has a talker stage |
|
||||
| `vllm:omni_` transfer | With `--log-stats`, if pipeline has ≥ 2 stages |
|
||||
| `vllm:` engine metrics (per `(stage, replica)`) | With `--log-stats` |
|
||||
| `vllm:` MFU metrics | With `--log-stats --enable-mfu-metrics` |
|
||||
|
||||
## Naming Convention
|
||||
|
||||
- All time-bearing metrics use the `_s` suffix (values in seconds). Buckets are `SECONDS_BUCKETS` for e2e / generation-style values and `SECONDS_FAST_BUCKETS` (1 ms → 60 s) for the fine-grained transfer and audio-underrun values.
|
||||
- Counters use the `_total` suffix (auto-appended by `prometheus_client`).
|
||||
- Sizes use the `_bytes` suffix.
|
||||
- All omni-specific families are prefixed `vllm:omni_`. The upstream `unregister_vllm_metrics()` function is monkey-patched (see `vllm_omni/patch.py`) to a scoped version that still strips upstream `vllm:*` collectors so multi-engine init within one process does not crash on duplicate registration, but preserves anything prefixed `vllm:omni_` / `vllm_omni`.
|
||||
- Text and audio first-output use distinct families (`vllm:time_to_first_token_seconds` reused from upstream for text; `vllm:omni_audio_ttfp_s` for audio) rather than a single metric with a `modality` label.
|
||||
@@ -133,12 +133,10 @@ The following tables show which models support each feature:
|
||||
| **Stable-Diffusion3.5** | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ (decode) | ❌ | ❌ |
|
||||
| **Z-Image** | ✅ | ✅ | ✅ | ❓ | ✅ (TP=2 only) | ❌ | ✅ | ❌ | ✅ (decode) | ✅ | ❌ |
|
||||
| **ERNIE-Image** | ❌ | ✅ | ✅ | ❓ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| **Cosmos3** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (decode) | ✅ | ❌ |
|
||||
|
||||
> Notes:
|
||||
> 1. Nextstep_1(T2I) does not support cache acceleration methods such as TeaCache or Cache-DiT.
|
||||
> 2. `Tongyi-MAI/Z-Image-Turbo` and `SII-GAIR/daVinci-MagiHuman-Base-1080p` are distilled models with minimal NFEs; CFG-Parallel is not necessary.
|
||||
> 3. Cosmos3 T2I uses `Cosmos3OmniDiffusersPipeline` with `modalities=["image"]`. Model-level CPU offload is not supported; use layerwise offload.
|
||||
|
||||
### VideoGen
|
||||
|
||||
@@ -151,8 +149,6 @@ The following tables show which models support each feature:
|
||||
| **Helios** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| **HunyuanVideo-1.5 T2V I2V** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (encode/decode) | ✅ | ❌ |
|
||||
| **DreamID-Omni** | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| **Cosmos3** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (encode/decode) | ✅ | ❌ |
|
||||
|
||||
|
||||
**Frame Interpolation Support**
|
||||
|
||||
|
||||
@@ -17,9 +17,6 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||
|
||||
The default deployment configuration situated at `vllm_omni/deploy/qwen3_omni_moe.yaml` is resolved and loaded
|
||||
automatically via the model registry, obviating the necessity for the `--deploy-config` flag in standard deployment topologies.
|
||||
The bundled Qwen3-Omni setup defaults `VLLM_USE_FLASHINFER_MOE_FP16=0`. This keeps the Thinker & Talker on vLLM's
|
||||
Triton unquantized MoE path and avoids the performance regression observed with the FlashInfer CUTLASS unquantized MoE
|
||||
backend.
|
||||
Asynchronous chunk streaming is **enabled by default** within the bundled configuration.
|
||||
|
||||
To explicitly utilize a custom deployment YAML, specify the configuration path:
|
||||
@@ -75,12 +72,6 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
|
||||
--stage-overrides '{"1": {"gpu_memory_utilization": 0.5}}'
|
||||
```
|
||||
|
||||
To experiment with the FlashInfer FP16 MoE path, set `VLLM_USE_FLASHINFER_MOE_FP16=1` before launching the server:
|
||||
```bash
|
||||
VLLM_USE_FLASHINFER_MOE_FP16=1 \
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||
```
|
||||
|
||||
For the stage-based CLI, you usually do **not** need `--stage-overrides` for
|
||||
that kind of change. Since each command launches one stage, just pass the knob
|
||||
directly on that stage command:
|
||||
|
||||
@@ -380,31 +380,6 @@ curl -X POST http://localhost:8091/v1/audio/voices \
|
||||
```
|
||||
Uploaded voices are then usable as `voice="custom_voice_1"` on subsequent requests.
|
||||
|
||||
### Precomputed custom voices
|
||||
For reused Base voice-cloning speakers, precompute the reference artifacts once and load them at server startup:
|
||||
```bash
|
||||
python qwen3_tts/precompute_custom_voice.py \
|
||||
--model Qwen/Qwen3-TTS-12Hz-1.7B-Base \
|
||||
--voice-name alice \
|
||||
--ref-audio /path/to/reference.wav \
|
||||
--ref-text "Original transcript of the reference audio" \
|
||||
--mode icl \
|
||||
--output-dir /path/to/custom_voices
|
||||
```
|
||||
`--mode icl` stores both `speaker_embedding` and `ref_code`; `--mode xvec` stores only the speaker embedding. Add the output directory to a deploy config:
|
||||
```yaml
|
||||
custom_voice_dir: /path/to/custom_voices
|
||||
```
|
||||
Then start the server with that config and call the Speech API with only the voice name:
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base --omni --deploy-config /path/to/qwen3_tts_custom_voice.yaml
|
||||
|
||||
curl -X POST http://localhost:8091/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"input":"Hello from a precomputed voice.","voice":"alice","task_type":"Base"}' \
|
||||
--output alice.wav
|
||||
```
|
||||
|
||||
### Streaming PCM
|
||||
```bash
|
||||
curl -X POST http://localhost:8091/v1/audio/speech \
|
||||
@@ -444,10 +419,8 @@ python qwen3_tts/gradio_fastrtc_demo.py --api-base http://localhost:8000
|
||||
`qwen3_tts/batch_speech_client.py` issues many concurrent requests for throughput measurement.
|
||||
|
||||
### Notes
|
||||
- Base voice cloning has uniproc-vs-mp tradeoffs depending on per-request reference audio cost; see the executor-backend section above.
|
||||
- With async chunking, Qwen3-TTS Base voice cloning sends the full reference context in the first Code2Wav packet, then caches that prefix on the Code2Wav stage for follow-up chunks in the same request.
|
||||
- Base voice cloning has per-request reference-audio cost; the uniproc default keeps IPC overhead off the critical path. See the executor-backend section above for background.
|
||||
- `vllm_omni/deploy/qwen3_tts.yaml` is the default deploy config (loaded by HF `model_type`); per-stage runtime overrides are available via `--stage-N-<field> <value>`.
|
||||
- Under vocoder-bound overload (single-stream `rtf_p99 ≥ 1` at the target concurrency), set `active_stream_window: 2` at the top of the deploy yaml to cap simultaneously active Stage 1 streams. Off by default; trades TTFP for streaming continuity. See [#3592](https://github.com/vllm-project/vllm-omni/pull/3592) for the mechanism and tradeoff numbers.
|
||||
|
||||
---
|
||||
|
||||
@@ -475,23 +448,6 @@ python voxcpm2/openai_speech_client.py \
|
||||
```
|
||||
The `ref_audio` field accepts local file paths (auto-base64), HTTP URLs, or `data:audio/wav;base64,...` data URIs.
|
||||
|
||||
### Precomputed custom voices
|
||||
For repeated VoxCPM2 speakers, precompute the prompt cache and load it through `custom_voice_dir`:
|
||||
```bash
|
||||
python voxcpm2/precompute_custom_voice.py \
|
||||
--model openbmb/VoxCPM2 \
|
||||
--voice-name alice \
|
||||
--ref-audio /path/to/reference.wav \
|
||||
--mode ref_continuation \
|
||||
--prompt-text "Original transcript of the reference audio" \
|
||||
--output-dir /path/to/custom_voices
|
||||
```
|
||||
Add the output directory to the deploy config:
|
||||
```yaml
|
||||
custom_voice_dir: /path/to/custom_voices
|
||||
```
|
||||
After startup, `/v1/audio/voices` lists `alice`, and `/v1/audio/speech` can use `voice="alice"` without sending `ref_audio`.
|
||||
|
||||
### Gradio demo (gapless streaming via AudioWorklet)
|
||||
```bash
|
||||
python voxcpm2/gradio_demo.py
|
||||
|
||||
@@ -74,7 +74,7 @@ Online:
|
||||
vllm serve Qwen/Qwen-Image \
|
||||
--omni \
|
||||
--port 8000 \
|
||||
--diffusion-quantization-config '{"method":"gguf","gguf_model":"QuantStack/Qwen-Image-GGUF/Qwen_Image-Q4_K_M.gguf"}'
|
||||
--quantization-config '{"method":"gguf","gguf_model":"QuantStack/Qwen-Image-GGUF/Qwen_Image-Q4_K_M.gguf"}'
|
||||
```
|
||||
|
||||
## Parameters
|
||||
|
||||
@@ -195,101 +195,6 @@ stages:
|
||||
devices: "0,1"
|
||||
```
|
||||
|
||||
### VAE Patch Parallelism
|
||||
|
||||
[VAE Patch Parallelism](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/parallelism/vae_patch_parallel.html) splits Bagel VAE **decode/encode** tiles across multiple GPUs on the **DiT stage**, reducing **per-GPU peak memory during VAE decode**. Use it when high-resolution `text2img` or `img2img` hits VAE OOM or large decode spikes.
|
||||
|
||||
**Bagel-specific notes:**
|
||||
|
||||
- Implemented in `BagelPipeline` via `DistributedAutoEncoder` (DiT stage only).
|
||||
- **Single-stage** is the simplest path: one DiT process with TP + VAE patch parallel.
|
||||
- **Two-stage**: enable on **stage 1 (DiT)** only; stage 0 (Thinker) keeps encoder-only `VAEEncoder` and does not use VAE patch parallel.
|
||||
- You need a DiT `world_size` ≥ `vae_patch_parallel_size` (typically `tensor_parallel_size=2` on that stage). VAE PP reuses the DiT process group; it is not a standalone second-GPU VAE worker.
|
||||
|
||||
**Single-stage via deploy YAML** (recommended for `end2end.py`):
|
||||
|
||||
```yaml
|
||||
pipeline: bagel_single_stage
|
||||
async_chunk: false
|
||||
|
||||
stages:
|
||||
- stage_id: 0
|
||||
max_num_batched_tokens: 32768
|
||||
max_num_seqs: 1
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
devices: "0,1"
|
||||
vae_use_tiling: true
|
||||
parallel_config:
|
||||
tensor_parallel_size: 2
|
||||
vae_patch_parallel_size: 2
|
||||
default_sampling_params:
|
||||
seed: 52
|
||||
```
|
||||
|
||||
```bash
|
||||
cd examples/offline_inference/bagel
|
||||
|
||||
CUDA_VISIBLE_DEVICES=0,1 python end2end.py \
|
||||
--model /path/to/BAGEL-7B-MoT \
|
||||
--deploy-config /path/to/bagel_single_stage_vae_pp.yaml \
|
||||
--modality text2img \
|
||||
--prompts "A cute cat" \
|
||||
--steps 10 \
|
||||
--output ./out_vae_pp
|
||||
```
|
||||
|
||||
**Single-stage via `Omni` kwargs** (same flags as online serving):
|
||||
|
||||
```python
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
|
||||
omni = Omni(
|
||||
model="ByteDance-Seed/BAGEL-7B-MoT",
|
||||
deploy_config="vllm_omni/deploy/bagel_single_stage.yaml",
|
||||
tensor_parallel_size=2,
|
||||
vae_patch_parallel_size=2,
|
||||
vae_use_tiling=True,
|
||||
)
|
||||
# Then call omni.generate(...) as in end2end.py
|
||||
```
|
||||
|
||||
**Two-stage (VAE PP on DiT only):**
|
||||
|
||||
```yaml
|
||||
stages:
|
||||
- stage_id: 0
|
||||
devices: "0"
|
||||
# AR Thinker — no vae_patch_parallel here
|
||||
|
||||
- stage_id: 1
|
||||
devices: "0,1"
|
||||
vae_use_tiling: true
|
||||
parallel_config:
|
||||
tensor_parallel_size: 2
|
||||
vae_patch_parallel_size: 2
|
||||
```
|
||||
|
||||
```bash
|
||||
python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
|
||||
--deploy-config /path/to/bagel_vae_pp.yaml \
|
||||
--modality text2img \
|
||||
--prompts "A cute cat"
|
||||
```
|
||||
|
||||
**Startup log checks:**
|
||||
|
||||
```text
|
||||
INFO ... vae_patch_parallel_size=2 requires vae_use_tiling; automatically enabling it.
|
||||
```
|
||||
|
||||
| Setting | Role |
|
||||
| :------ | :--- |
|
||||
| `parallel_config.tensor_parallel_size` | DiT world size / TP (must be ≥ `vae_patch_parallel_size`) |
|
||||
| `parallel_config.vae_patch_parallel_size` | Number of ranks for distributed VAE tiles (`1` = off) |
|
||||
| `vae_use_tiling` | Enable spatial tiling (auto-enabled when `vae_patch_parallel_size > 1`) |
|
||||
|
||||
#### Hybrid Sharded Data Parallel (HSDP)
|
||||
|
||||
For larger Bagel deployments on multiple GPUs, you can enable HSDP (Hybrid Sharded Data Parallel) by modifying the stage configuration (for example, [`bagel.yaml`](../../../vllm_omni/deploy/bagel.yaml)). HSDP shards transformer weights across GPUs to reduce per-GPU memory usage.
|
||||
|
||||
@@ -121,6 +121,9 @@ def parse_args():
|
||||
help="Temperature for text generation sampling (default: 0.3).",
|
||||
)
|
||||
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
|
||||
nullify_stage_engine_defaults(parser)
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
@@ -151,7 +154,6 @@ def main():
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
|
||||
omni_kwargs = vars(args).copy()
|
||||
omni_kwargs.pop("model", None)
|
||||
deploy_config = args.deploy_config
|
||||
if args.think and deploy_config is None:
|
||||
deploy_config = "vllm_omni/deploy/bagel_think.yaml"
|
||||
@@ -162,7 +164,9 @@ def main():
|
||||
if args.quantization:
|
||||
omni_kwargs["quantization_config"] = args.quantization
|
||||
|
||||
omni = Omni(model=model_name, **omni_kwargs)
|
||||
# Override CLI --model with the derived model_name.
|
||||
omni_kwargs["model"] = model_name
|
||||
omni = Omni(**omni_kwargs)
|
||||
|
||||
formatted_prompts = []
|
||||
for p in prompts:
|
||||
|
||||
@@ -18,6 +18,8 @@ import numpy as np
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
|
||||
TASK_CHOICES = ("t2t", "t2i", "t2s", "i2i", "i2t", "s2t", "v2t")
|
||||
|
||||
TASK_DEFAULT_RUNTIME = {
|
||||
@@ -970,6 +972,8 @@ def parse_args(repo_root: Path) -> argparse.Namespace:
|
||||
parser.add_argument("--vq-model-audio-local-files-only", action=argparse.BooleanOptionalAction, default=None)
|
||||
|
||||
parser.add_argument("--disable-hf-xet", action=argparse.BooleanOptionalAction, default=True)
|
||||
|
||||
nullify_stage_engine_defaults(parser)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@@ -107,6 +107,9 @@ def parse_args():
|
||||
),
|
||||
)
|
||||
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
|
||||
nullify_stage_engine_defaults(parser)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@@ -210,12 +210,6 @@ def main():
|
||||
args = parse_args()
|
||||
os.makedirs(args.output, exist_ok=True)
|
||||
|
||||
# Lance aligns more naturally with upstream when using FLASH_ATTN
|
||||
# (upstream uses flash_attn_varlen_func; SDPA accumulates ~5x more
|
||||
# numerical drift through the 36-layer Qwen2 stack on B300). Default
|
||||
# to flash_attn when the user hasn't explicitly set a backend.
|
||||
os.environ.setdefault("DIFFUSION_ATTENTION_BACKEND", "FLASH_ATTN")
|
||||
|
||||
if args.txt_prompts:
|
||||
with open(args.txt_prompts, encoding="utf-8") as f:
|
||||
prompts = [ln.strip() for ln in f if ln.strip()]
|
||||
|
||||
@@ -288,19 +288,11 @@ def _init_models(
|
||||
f" replicas={replicas_per_omni}...",
|
||||
flush=True,
|
||||
)
|
||||
# Lance single-stage engine defaults — mirrors the offline ``end2end.py``
|
||||
# flat-kwargs path so this demo does not depend on ``deploy/lance.yaml``.
|
||||
# ``pipeline="lance"`` is the explicit selector (Lance HF config has no
|
||||
# ``model_type`` field, so auto-detect from the model dir alone fails).
|
||||
lance_engine_defaults = dict(
|
||||
pipeline="lance",
|
||||
max_num_batched_tokens=32768,
|
||||
max_num_seqs=1,
|
||||
enforce_eager=True,
|
||||
trust_remote_code=True,
|
||||
enable_prefix_caching=False,
|
||||
async_chunk=False,
|
||||
)
|
||||
# Lance needs ``vllm_omni/deploy/lance.yaml`` as the pipeline-selector
|
||||
# (Lance HF config has no ``model_type`` field for auto-detect). Pass
|
||||
# via ``deploy_config`` — flat kwargs alone don't trigger pipeline
|
||||
# resolution.
|
||||
lance_engine_defaults = dict(deploy_config="vllm_omni/deploy/lance.yaml")
|
||||
|
||||
img_kwargs: dict = {"model": img_ckpt, **lance_engine_defaults}
|
||||
if img_devices != "0":
|
||||
@@ -1245,11 +1237,6 @@ def build_ui(
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
# Lance aligns better with upstream when using FLASH_ATTN
|
||||
# (upstream uses flash_attn_varlen_func; SDPA accumulates ~5x more
|
||||
# numerical drift through the 36-layer Qwen2 stack). Default to
|
||||
# flash_attn unless the user explicitly set a backend.
|
||||
os.environ.setdefault("DIFFUSION_ATTENTION_BACKEND", "FLASH_ATTN")
|
||||
print("=" * 60, flush=True)
|
||||
print("[unified] Lance × vllm-omni demo", flush=True)
|
||||
print(f" snapshot: {args.model}", flush=True)
|
||||
|
||||
@@ -19,6 +19,7 @@ from vllm import SamplingParams
|
||||
from vllm.multimodal.image import convert_image_mode
|
||||
|
||||
from vllm_omni import Omni
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
|
||||
DEFAULT_SYSTEM = "You are a helpful assistant."
|
||||
DEFAULT_QUESTION = "Please summarize the content of this image."
|
||||
@@ -48,6 +49,7 @@ def parse_args() -> argparse.Namespace:
|
||||
action="store_true",
|
||||
help="Enable diffusion pipeline profiler to display stage durations.",
|
||||
)
|
||||
nullify_stage_engine_defaults(parser)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ from PIL import Image
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
from vllm_omni import Omni
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -117,6 +118,7 @@ def parse_args() -> argparse.Namespace:
|
||||
)
|
||||
p.add_argument("--out", type=str, default="output.png", help="Path to save the generated image.")
|
||||
p.add_argument("--trust-remote-code", action="store_true", help="Trust remote code when loading the model.")
|
||||
nullify_stage_engine_defaults(p)
|
||||
args = p.parse_args()
|
||||
if not args.prompt:
|
||||
args.prompt = ["A stylish woman with sunglasses riding a motorcycle in NYC."]
|
||||
|
||||
@@ -21,10 +21,11 @@ from message_convert import (
|
||||
to_prompt,
|
||||
)
|
||||
from vllm import SamplingParams
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniTokensPrompt
|
||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
|
||||
|
||||
SEED = 42
|
||||
|
||||
@@ -342,7 +343,7 @@ def main(args):
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = TrackingArgumentParser(description="Demo on using vLLM for offline inference with audio language models")
|
||||
parser = FlexibleArgumentParser(description="Demo on using vLLM for offline inference with audio language models")
|
||||
parser.add_argument(
|
||||
"--model-name",
|
||||
"-m",
|
||||
@@ -438,6 +439,7 @@ def parse_args():
|
||||
"vllm_omni/deploy/mimo_audio.yaml based on the HF model_type.",
|
||||
)
|
||||
|
||||
nullify_stage_engine_defaults(parser)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@@ -53,7 +53,7 @@ python examples/offline_inference/ming_flash_omni/end2end.py \
|
||||
--image-path ./3_0.png
|
||||
```
|
||||
|
||||
### Omni-Speech (Thinker + Talker) {#omni-speech-thinker--talker}
|
||||
### Omni-Speech (Thinker + Talker)
|
||||
|
||||
The default deploy YAML already runs thinker+talker, so spoken output only requires requesting `audio` (or `text,audio`) modalities.
|
||||
The thinker processes your multimodal input, generates text, then the talker synthesises the response as speech.
|
||||
|
||||
@@ -17,13 +17,14 @@ from vllm.assets.image import ImageAsset
|
||||
from vllm.assets.video import VideoAsset, video_to_ndarrays
|
||||
from vllm.multimodal.image import convert_image_mode
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
import vllm_omni
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
|
||||
# Imports the processor also registers itself
|
||||
from vllm_omni.transformers_utils.processors.ming import MingFlashOmniProcessor # noqa: F401
|
||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
|
||||
|
||||
SEED = 42
|
||||
MODEL_NAME = "Jonathan1909/Ming-flash-omni-2.0"
|
||||
@@ -396,7 +397,7 @@ def main(args):
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = TrackingArgumentParser(description="Ming-flash-omni 2.0 offline inference example")
|
||||
parser = FlexibleArgumentParser(description="Ming-flash-omni 2.0 offline inference example")
|
||||
parser.add_argument(
|
||||
"--query-type",
|
||||
"-q",
|
||||
@@ -495,6 +496,7 @@ def parse_args():
|
||||
help="Output directory for results.",
|
||||
)
|
||||
|
||||
nullify_stage_engine_defaults(parser)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@@ -19,9 +19,10 @@ from vllm.assets.video import VideoAsset, video_to_ndarrays
|
||||
from vllm.multimodal.image import convert_image_mode
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
|
||||
|
||||
SEED = 42
|
||||
|
||||
@@ -431,7 +432,7 @@ def main(args):
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = TrackingArgumentParser(description="Demo on using vLLM for offline inference with audio language models")
|
||||
parser = FlexibleArgumentParser(description="Demo on using vLLM for offline inference with audio language models")
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
type=str,
|
||||
@@ -553,6 +554,7 @@ def parse_args():
|
||||
default=False,
|
||||
help="Use py_generator mode. The returned type of Omni.generate() is a Python Generator object.",
|
||||
)
|
||||
nullify_stage_engine_defaults(parser)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@@ -19,9 +19,10 @@ from vllm.assets.image import ImageAsset
|
||||
from vllm.assets.video import VideoAsset, video_to_ndarrays
|
||||
from vllm.multimodal.image import convert_image_mode
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
|
||||
|
||||
SEED = 42
|
||||
|
||||
@@ -421,7 +422,7 @@ def main(args):
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = TrackingArgumentParser(description="Demo on using vLLM for offline inference with audio language models")
|
||||
parser = FlexibleArgumentParser(description="Demo on using vLLM for offline inference with audio language models")
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
type=str,
|
||||
@@ -560,6 +561,7 @@ def parse_args():
|
||||
help="Model dtype (auto, half, float16, bfloat16, float, float32).",
|
||||
)
|
||||
|
||||
nullify_stage_engine_defaults(parser)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@@ -39,9 +39,10 @@ from vllm.assets.image import ImageAsset
|
||||
from vllm.assets.video import VideoAsset, video_to_ndarrays
|
||||
from vllm.multimodal.image import convert_image_mode
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -382,9 +383,7 @@ async def run_all(args):
|
||||
print(f"[Info] Creating AsyncOmni with deploy_config={args.deploy_config}")
|
||||
async_omni = None
|
||||
try:
|
||||
# ``from_cli_args`` forwards only explicitly-passed CLI args so
|
||||
# argparse defaults do not silently override deploy YAML values.
|
||||
async_omni = AsyncOmni.from_cli_args(args, model=args.model)
|
||||
async_omni = AsyncOmni(**vars(args))
|
||||
|
||||
# Use default sampling params from stage config (they are pre-configured
|
||||
# in the YAML for each stage).
|
||||
@@ -454,7 +453,7 @@ async def run_all(args):
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = TrackingArgumentParser(
|
||||
parser = FlexibleArgumentParser(
|
||||
description=(
|
||||
"Offline inference with async_chunk enabled via AsyncOmni. "
|
||||
"Downstream stages start before upstream stages finish, "
|
||||
@@ -590,6 +589,7 @@ def parse_args():
|
||||
default=16000,
|
||||
help="Sampling rate for audio loading.",
|
||||
)
|
||||
nullify_stage_engine_defaults(parser)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ import os
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
@@ -167,6 +168,7 @@ def parse_args():
|
||||
help="Enable cache-dit summary logging after diffusion forward passes.",
|
||||
)
|
||||
|
||||
nullify_stage_engine_defaults(parser)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@@ -307,6 +307,9 @@ def parse_args() -> argparse.Namespace:
|
||||
help="Supplementary auxiliary text encoder parameters model name or path (especially for Hidream-l1-full).",
|
||||
)
|
||||
current_omni_platform.pre_register_and_update(parser)
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
|
||||
nullify_stage_engine_defaults(parser)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import soundfile as sf
|
||||
from vllm import SamplingParams
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokenizer
|
||||
from vllm_omni.model_executor.models.cosyvoice3.utils import extract_text_token
|
||||
@@ -64,6 +65,7 @@ def run_e2e():
|
||||
required=True,
|
||||
help="Path to tokenizer directory (e.g., <model_path>/CosyVoice-BlankEN).",
|
||||
)
|
||||
nullify_stage_engine_defaults(parser)
|
||||
args = parser.parse_args()
|
||||
# Ensure tokenizer directory exists
|
||||
if not os.path.exists(args.tokenizer):
|
||||
|
||||
@@ -26,14 +26,16 @@ import torch
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
from vllm_omni import AsyncOmni, Omni
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
from vllm_omni.model_executor.models.fish_speech.dac_utils import DAC_HOP_LENGTH, DAC_SAMPLE_RATE
|
||||
from vllm_omni.model_executor.models.fish_speech.prompt_utils import (
|
||||
build_fish_text_only_prompt_ids,
|
||||
estimate_fish_voice_clone_prompt_len_from_normalized,
|
||||
normalize_fish_voice_clone_texts,
|
||||
)
|
||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -209,7 +211,7 @@ async def main_streaming(args):
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = TrackingArgumentParser(
|
||||
parser = FlexibleArgumentParser(
|
||||
description="Fish Speech S2 Pro offline inference with vLLM Omni",
|
||||
)
|
||||
parser.add_argument(
|
||||
@@ -265,6 +267,7 @@ def parse_args():
|
||||
default=False,
|
||||
help="Stream audio chunks as they arrive via AsyncOmni.",
|
||||
)
|
||||
nullify_stage_engine_defaults(parser)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@@ -30,9 +30,10 @@ from pathlib import Path
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
import torch
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
from vllm_omni import Omni
|
||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
|
||||
SAMPLE_RATE = 24_000
|
||||
DEFAULT_TEXTS = (
|
||||
@@ -42,7 +43,7 @@ DEFAULT_TEXTS = (
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = TrackingArgumentParser(description="Offline higgs-audio v2 inference")
|
||||
parser = FlexibleArgumentParser(description="Offline higgs-audio v2 inference")
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
type=str,
|
||||
@@ -87,6 +88,7 @@ def parse_args():
|
||||
default=None,
|
||||
help="Transcript of the reference clip. Required when --ref-audio is set.",
|
||||
)
|
||||
nullify_stage_engine_defaults(parser)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@@ -5,11 +5,11 @@ from typing import Any
|
||||
|
||||
import soundfile as sf
|
||||
import torch
|
||||
|
||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniTokensPrompt
|
||||
from vllm_omni.model_executor.models.ming_flash_omni.prompt_utils import (
|
||||
@@ -68,7 +68,7 @@ def save_audio(mm: dict[str, Any], output_path: str) -> None:
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = TrackingArgumentParser(description="Ming-flash-omni standalone talker offline e2e example")
|
||||
parser = FlexibleArgumentParser(description="Ming-flash-omni standalone talker offline e2e example")
|
||||
parser.add_argument("--model", type=str, default=MODEL_NAME, help="Model name or local path.")
|
||||
parser.add_argument(
|
||||
"--deploy-config",
|
||||
@@ -89,6 +89,7 @@ def parse_args():
|
||||
parser.add_argument("--init-timeout", type=int, default=600, help="Engine init timeout in seconds.")
|
||||
parser.add_argument("--stage-init-timeout", type=int, default=300, help="Single stage init timeout in seconds.")
|
||||
|
||||
nullify_stage_engine_defaults(parser)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@@ -1,377 +0,0 @@
|
||||
"""Offline inference example for MOSS-TTS variants.
|
||||
|
||||
Covers all five models:
|
||||
- OpenMOSS-Team/MOSS-TTS (8B, general TTS)
|
||||
- OpenMOSS-Team/MOSS-TTSD-v1.0 (8B, dialogue TTS)
|
||||
- OpenMOSS-Team/MOSS-SoundEffect (8B, sound effect synthesis)
|
||||
- OpenMOSS-Team/MOSS-TTS-Realtime (1.7B, low-latency streaming)
|
||||
- OpenMOSS-Team/MOSS-VoiceGenerator(1.7B, voice design)
|
||||
|
||||
Usage
|
||||
-----
|
||||
# Standard TTS with voice cloning:
|
||||
python end2end.py \\
|
||||
--model OpenMOSS-Team/MOSS-TTS \\
|
||||
--text "Hello, this is a MOSS-TTS test." \\
|
||||
--ref-audio path/to/reference.wav \\
|
||||
--output output.wav
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gc
|
||||
import sys
|
||||
|
||||
import soundfile as sf
|
||||
import torch
|
||||
|
||||
|
||||
def _load_ref_audio(path: str, target_sr: int = 24000) -> torch.Tensor:
|
||||
"""Load and resample reference audio to (1, T) at target_sr."""
|
||||
data, sr = sf.read(path, always_2d=True) # (T, C)
|
||||
wav = torch.from_numpy(data.T.astype("float32")) # (C, T)
|
||||
if wav.shape[0] > 1:
|
||||
wav = wav.mean(dim=0, keepdim=True)
|
||||
if sr != target_sr:
|
||||
import torchaudio
|
||||
|
||||
wav = torchaudio.functional.resample(wav, sr, target_sr)
|
||||
return wav # (1, T)
|
||||
|
||||
|
||||
def _build_unified_codes(
|
||||
model_id: str,
|
||||
text: str,
|
||||
ref_audio: torch.Tensor | None,
|
||||
*,
|
||||
instruction: str | None = None,
|
||||
tokens: int | None = None,
|
||||
quality: str | None = None,
|
||||
sound_event: str | None = None,
|
||||
ambient_sound: str | None = None,
|
||||
language: str | None = None,
|
||||
) -> tuple[list[int], torch.Tensor, int]:
|
||||
"""Build the upstream MossTTSDelay unified-codes prompt.
|
||||
|
||||
Returns (text_token_ids, audio_codes_LxNQ, n_vq) where
|
||||
``text_token_ids`` is the prefill prompt for the talker and
|
||||
``audio_codes_LxNQ`` is the matching delay-pattern audio code grid
|
||||
(``audio_pad_code`` everywhere except inside the reference-audio block).
|
||||
"""
|
||||
from transformers import AutoProcessor
|
||||
|
||||
proc = AutoProcessor.from_pretrained(model_id, trust_remote_code=True)
|
||||
|
||||
reference: list[torch.Tensor] | None = None
|
||||
if ref_audio is not None:
|
||||
# Pre-encode so the processor doesn't need torchaudio.load (which
|
||||
# now requires torchcodec on recent torchaudio releases).
|
||||
# encode_audios_from_wav expects (C, T) per item.
|
||||
wav_2d = ref_audio if ref_audio.dim() == 2 else ref_audio.unsqueeze(0)
|
||||
codes_list = proc.encode_audios_from_wav([wav_2d], sampling_rate=24000, n_vq=proc.model_config.n_vq)
|
||||
reference = [codes_list[0]]
|
||||
|
||||
user_msg_kwargs: dict = {"text": text}
|
||||
if reference is not None:
|
||||
user_msg_kwargs["reference"] = reference
|
||||
if instruction is not None:
|
||||
user_msg_kwargs["instruction"] = instruction
|
||||
if tokens is not None:
|
||||
user_msg_kwargs["tokens"] = tokens
|
||||
if quality is not None:
|
||||
user_msg_kwargs["quality"] = quality
|
||||
if sound_event is not None:
|
||||
user_msg_kwargs["sound_event"] = sound_event
|
||||
if ambient_sound is not None:
|
||||
user_msg_kwargs["ambient_sound"] = ambient_sound
|
||||
if language is not None:
|
||||
user_msg_kwargs["language"] = language
|
||||
|
||||
user_msg = proc.build_user_message(**user_msg_kwargs)
|
||||
batch = proc(conversations=[[user_msg]], mode="generation")
|
||||
unified = batch["input_ids"][0] # (L, 1+n_vq)
|
||||
n_vq = int(unified.shape[-1] - 1)
|
||||
text_ids = unified[:, 0].tolist()
|
||||
audio_codes = unified[:, 1:].contiguous().to(torch.int64) # (L, n_vq)
|
||||
|
||||
# Free the processor (incl. ~1.6B-param audio_tokenizer on CPU) before
|
||||
# we instantiate Omni — we only needed it for the one-shot encode.
|
||||
del proc
|
||||
gc.collect()
|
||||
|
||||
return text_ids, audio_codes, n_vq
|
||||
|
||||
|
||||
def _build_realtime_prompt(
|
||||
model_id: str,
|
||||
text: str,
|
||||
ref_audio: torch.Tensor | None,
|
||||
) -> tuple[list[int], torch.Tensor, int]:
|
||||
"""Build the prefill prompt for MOSS-TTS-Realtime.
|
||||
|
||||
Realtime uses a different prompt format than the delay variants: a system
|
||||
prompt + (optional) voice-clone context with the encoded reference audio
|
||||
inlined, followed by the user text and an assistant header. The format is
|
||||
a ``(L, channels+1=17)`` int grid where col 0 is text/special tokens and
|
||||
cols 1..16 are RVQ codebook entries (``audio_channel_pad`` outside the
|
||||
ref-audio block).
|
||||
|
||||
The upstream ``MossTTSRealtimeProcessor`` is shipped via
|
||||
``trust_remote_code`` but is not auto-discovered by ``AutoProcessor``
|
||||
(no ``processor_config.json``) — we import its module directly from the
|
||||
snapshot directory and use its ``make_ensemble``/``make_user_prompt``.
|
||||
"""
|
||||
import importlib.util
|
||||
import sys as _sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from huggingface_hub import snapshot_download
|
||||
from transformers import AutoModel, AutoTokenizer
|
||||
|
||||
snap_dir = Path(snapshot_download(repo_id=model_id))
|
||||
proc_module_path = snap_dir / "processing_mossttsrealtime.py"
|
||||
if not proc_module_path.exists():
|
||||
raise FileNotFoundError(f"realtime processor module missing at {proc_module_path}")
|
||||
|
||||
# Load the processor module without polluting sys.path globally.
|
||||
spec = importlib.util.spec_from_file_location("_moss_tts_realtime_proc", proc_module_path)
|
||||
mod = importlib.util.module_from_spec(spec) # type: ignore[arg-type]
|
||||
_sys.modules[spec.name] = mod # type: ignore[union-attr]
|
||||
spec.loader.exec_module(mod) # type: ignore[union-attr]
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
|
||||
realtime_processor = mod.MossTTSRealtimeProcessor(tokenizer=tokenizer)
|
||||
|
||||
# Encode reference audio via the standalone MOSS Audio Tokenizer (CPU is
|
||||
# fine — it runs once per request).
|
||||
audio_tokens = None
|
||||
if ref_audio is not None:
|
||||
codec = AutoModel.from_pretrained("OpenMOSS-Team/MOSS-Audio-Tokenizer", trust_remote_code=True).eval()
|
||||
wav_2d = ref_audio if ref_audio.dim() == 2 else ref_audio.unsqueeze(0)
|
||||
with torch.no_grad():
|
||||
enc = codec.batch_encode([wav_2d.squeeze(0)], num_quantizers=16)
|
||||
# enc.audio_codes: (NQ, B, T); we want (T, NQ) numpy.
|
||||
codes = enc.audio_codes[:, 0, : int(enc.audio_codes_lengths[0].item())]
|
||||
audio_tokens = codes.transpose(0, 1).contiguous().cpu().numpy() # (T, 16)
|
||||
del codec
|
||||
gc.collect()
|
||||
|
||||
# Build the inference-style prefill (mirrors upstream
|
||||
# ``MossTTSRealtimeInference._build_prefill_batch``):
|
||||
# system_prompt [+ voice_clone_block]
|
||||
# <|im_start|>assistant\n
|
||||
# <text_tokens...> (col 0 = text, cols 1..16 = audio_channel_pad)
|
||||
# <last text token> (col 1 = audio_bos at the last text position)
|
||||
# Generation continues from here; the local transformer emits real audio
|
||||
# codes for codebook 0 directly (no BOS frame at gen_step=0).
|
||||
system_grid = realtime_processor.make_ensemble(prompt_audio_tokens=audio_tokens) # (L_sys, 17)
|
||||
|
||||
# Append "<|im_start|>assistant\n" header (the snapshot's make_ensemble
|
||||
# omits it; inferencer.py's version includes it. We add it here.)
|
||||
assistant_header_ids = tokenizer.encode("<|im_start|>assistant\n", add_special_tokens=False)
|
||||
assistant_grid = np.full((len(assistant_header_ids), 17), 1024, dtype=np.int64)
|
||||
assistant_grid[:, 0] = assistant_header_ids
|
||||
|
||||
# Append the FIRST 12 text tokens with audio_bos (1025) at the last
|
||||
# included text position (audio column 0 == grid col 1). The remaining
|
||||
# text tokens are fed one-per-step during generation via the talker's
|
||||
# compute_logits → vLLM sampler pipeline (see ``ids.all`` in the
|
||||
# additional_information). This matches upstream
|
||||
# ``MossTTSRealtimeInference._build_prefill_batch`` (prefill_max_text=12).
|
||||
text_ids_only = tokenizer.encode(text, add_special_tokens=False)
|
||||
if not text_ids_only:
|
||||
raise ValueError("Empty text after tokenisation")
|
||||
PREFILL_MAX_TEXT = 12
|
||||
cur_len = min(len(text_ids_only), PREFILL_MAX_TEXT)
|
||||
text_grid = np.full((cur_len, 17), 1024, dtype=np.int64)
|
||||
text_grid[:, 0] = text_ids_only[:cur_len]
|
||||
text_grid[-1, 1] = 1025 # audio_bos_token at last *prefilled* text token
|
||||
|
||||
grid = np.concatenate([system_grid, assistant_grid, text_grid], axis=0)
|
||||
text_ids = grid[:, 0].tolist()
|
||||
audio_codes = torch.from_numpy(grid[:, 1:].astype(np.int64).copy()) # (L, 16)
|
||||
|
||||
# ``remaining_text_ids`` will be fed one-per-step by the talker.
|
||||
remaining_text_ids = list(text_ids_only[cur_len:])
|
||||
return text_ids, audio_codes, 16, remaining_text_ids
|
||||
|
||||
|
||||
def run_tts(args: argparse.Namespace) -> None:
|
||||
from vllm import SamplingParams
|
||||
|
||||
from vllm_omni import Omni
|
||||
|
||||
deploy_map = {
|
||||
"OpenMOSS-Team/MOSS-TTS": "moss_tts",
|
||||
"OpenMOSS-Team/MOSS-TTSD-v1.0": "moss_ttsd",
|
||||
"OpenMOSS-Team/MOSS-SoundEffect": "moss_sound_effect",
|
||||
"OpenMOSS-Team/MOSS-TTS-Realtime": "moss_tts_realtime",
|
||||
"OpenMOSS-Team/MOSS-VoiceGenerator": "moss_voice_generator",
|
||||
}
|
||||
deploy_config = deploy_map.get(args.model, "moss_tts")
|
||||
|
||||
is_sound_effect = "SoundEffect" in args.model
|
||||
is_voice_gen = "VoiceGenerator" in args.model
|
||||
is_realtime = "Realtime" in args.model
|
||||
|
||||
ref_audio = None
|
||||
builder_kwargs: dict = {}
|
||||
|
||||
remaining_text_ids: list[int] | None = None
|
||||
if is_realtime:
|
||||
# MOSS-TTS-Realtime uses a different prompt format and has its own
|
||||
# per-step depth transformer; the delay-style ``MossTTSDelayProcessor``
|
||||
# path doesn't apply.
|
||||
if not args.ref_audio:
|
||||
sys.exit("MOSS-TTS-Realtime currently requires --ref-audio (voice clone).")
|
||||
ref_audio = _load_ref_audio(args.ref_audio)
|
||||
text = args.text or ""
|
||||
text_ids, audio_codes, n_vq, remaining_text_ids = _build_realtime_prompt(args.model, text, ref_audio)
|
||||
print(
|
||||
f"Prefill prompt: {len(text_ids)} tokens, {n_vq}-quantizer audio block (realtime); "
|
||||
f"{len(remaining_text_ids)} text tokens to stream"
|
||||
)
|
||||
elif is_sound_effect:
|
||||
if not args.ambient_sound:
|
||||
sys.exit("MOSS-SoundEffect requires --ambient-sound")
|
||||
builder_kwargs["ambient_sound"] = args.ambient_sound
|
||||
# MOSS-SoundEffect emits ~12.5 codec frames per second; ``tokens``
|
||||
# tells the AR loop the target duration.
|
||||
if args.duration_tokens:
|
||||
builder_kwargs["tokens"] = args.duration_tokens
|
||||
elif args.duration_seconds:
|
||||
builder_kwargs["tokens"] = max(1, int(float(args.duration_seconds) * 12.5))
|
||||
text = args.text or ""
|
||||
elif is_voice_gen:
|
||||
# MOSS-VoiceGenerator needs an instruction (voice description) plus
|
||||
# the text to synthesise; no reference audio.
|
||||
if not args.instruction:
|
||||
sys.exit("MOSS-VoiceGenerator requires --instruction (voice description)")
|
||||
builder_kwargs["instruction"] = args.instruction
|
||||
text = args.text or ""
|
||||
else:
|
||||
if not args.ref_audio:
|
||||
sys.exit("Voice cloning requires --ref-audio")
|
||||
ref_audio = _load_ref_audio(args.ref_audio)
|
||||
text = args.text or ""
|
||||
|
||||
if not is_realtime:
|
||||
text_ids, audio_codes, n_vq = _build_unified_codes(args.model, text, ref_audio, **builder_kwargs)
|
||||
print(f"Prefill prompt: {len(text_ids)} tokens, {n_vq}-quantizer audio block")
|
||||
|
||||
omni = Omni(model=args.model, deploy_config=deploy_config, stage_init_timeout=600)
|
||||
|
||||
talker_sp = SamplingParams(
|
||||
temperature=1.7,
|
||||
top_p=0.8,
|
||||
top_k=25,
|
||||
max_tokens=2048,
|
||||
seed=42,
|
||||
detokenize=False,
|
||||
)
|
||||
codec_sp = SamplingParams(
|
||||
temperature=0.0,
|
||||
top_p=1.0,
|
||||
top_k=-1,
|
||||
max_tokens=65536,
|
||||
seed=42,
|
||||
detokenize=True,
|
||||
)
|
||||
|
||||
additional_info: dict = {
|
||||
# Reference-audio codes consumed by the talker prefill (additive
|
||||
# embedding); shape is (L, n_vq) where pad_code marks non-audio
|
||||
# positions. The OmniPayload schema reserves ``codes.ref`` for
|
||||
# exactly this kind of conditioning input.
|
||||
"codes": {"ref": audio_codes},
|
||||
}
|
||||
if remaining_text_ids:
|
||||
# Realtime variant: feed the remaining text tokens (everything past
|
||||
# the first ``PREFILL_MAX_TEXT``) one-per-step during decode via the
|
||||
# talker's compute_logits → vLLM sampler one-hot trick.
|
||||
additional_info["ids"] = {"all": remaining_text_ids}
|
||||
|
||||
request = {
|
||||
"prompt_token_ids": text_ids,
|
||||
"additional_information": additional_info,
|
||||
}
|
||||
|
||||
import time as _time
|
||||
|
||||
chunks: list[torch.Tensor] = []
|
||||
t_gen_start = _time.monotonic()
|
||||
t_first_audio: float | None = None
|
||||
for stage_outputs in omni.generate(request, [talker_sp, codec_sp]):
|
||||
# Each yielded ``stage_outputs`` is an OmniRequestOutput; the codec
|
||||
# stage attaches its waveform under ``multimodal_output["model_outputs"]``.
|
||||
# Avoid ``a or b`` when ``a`` may be a tensor (truthiness raises).
|
||||
mm = stage_outputs.multimodal_output or {}
|
||||
wav = mm.get("audio")
|
||||
if wav is None:
|
||||
wav = mm.get("model_outputs")
|
||||
if wav is None:
|
||||
continue
|
||||
if isinstance(wav, list):
|
||||
non_empty = [w.reshape(-1) for w in wav if isinstance(w, torch.Tensor) and w.numel() > 0]
|
||||
if non_empty and t_first_audio is None:
|
||||
t_first_audio = _time.monotonic()
|
||||
chunks.extend(non_empty)
|
||||
elif isinstance(wav, torch.Tensor) and wav.numel() > 0:
|
||||
if t_first_audio is None:
|
||||
t_first_audio = _time.monotonic()
|
||||
chunks.append(wav.reshape(-1))
|
||||
|
||||
t_gen_end = _time.monotonic()
|
||||
if not chunks:
|
||||
print("No audio in output — check model loading logs.")
|
||||
return
|
||||
|
||||
wav_t = torch.cat(chunks).float().cpu()
|
||||
sf.write(args.output, wav_t.numpy(), 24000)
|
||||
print(f"Saved {len(wav_t) / 24000:.2f}s of audio to {args.output}")
|
||||
ttfa_ms = (t_first_audio - t_gen_start) * 1000.0 if t_first_audio is not None else float("nan")
|
||||
gen_total_ms = (t_gen_end - t_gen_start) * 1000.0
|
||||
print(f"TTFA_MS={ttfa_ms:.1f} GEN_TOTAL_MS={gen_total_ms:.1f} AUDIO_S={len(wav_t) / 24000:.2f}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="MOSS-TTS offline inference example")
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
default="OpenMOSS-Team/MOSS-TTS",
|
||||
help="HuggingFace model ID",
|
||||
)
|
||||
parser.add_argument("--text", default="", help="Text to synthesise (TTS variants)")
|
||||
parser.add_argument("--ref-audio", default=None, help="Reference audio for voice cloning")
|
||||
parser.add_argument(
|
||||
"--ambient-sound",
|
||||
default=None,
|
||||
help="Sound description for MOSS-SoundEffect",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--duration-tokens",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Target duration in tokens (1 s ≈ 12.5 tokens) for MOSS-SoundEffect",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--duration-seconds",
|
||||
type=float,
|
||||
default=None,
|
||||
help="Target duration in seconds (converted via ~12.5 tokens/s) for MOSS-SoundEffect",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--instruction",
|
||||
default=None,
|
||||
help="Voice description for MOSS-VoiceGenerator (e.g. 'a young woman with a clear voice')",
|
||||
)
|
||||
parser.add_argument("--output", default="output.wav", help="Output WAV path")
|
||||
args = parser.parse_args()
|
||||
run_tts(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -38,8 +38,7 @@ from pathlib import Path
|
||||
import soundfile as sf
|
||||
import torch
|
||||
from vllm import SamplingParams
|
||||
|
||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
# Prevent multiprocessing from re-importing CUDA in the wrong context.
|
||||
os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
@@ -156,7 +155,7 @@ def main(args) -> None:
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = TrackingArgumentParser(description="MOSS-TTS-Nano offline inference")
|
||||
parser = FlexibleArgumentParser(description="MOSS-TTS-Nano offline inference")
|
||||
parser.add_argument("--text", default="Hello, this is MOSS-TTS-Nano speaking.", help="Text to synthesize.")
|
||||
parser.add_argument(
|
||||
"--ref-audio",
|
||||
|
||||
@@ -21,6 +21,7 @@ import os
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
@@ -85,6 +86,7 @@ def run_e2e():
|
||||
default=None,
|
||||
help="Random seed for generation",
|
||||
)
|
||||
nullify_stage_engine_defaults(parser)
|
||||
args = parser.parse_args()
|
||||
|
||||
if not os.path.exists(args.stage_config):
|
||||
|
||||
@@ -15,8 +15,10 @@ import torch
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
from vllm_omni import AsyncOmni, Omni
|
||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -433,7 +435,7 @@ async def main_streaming(args):
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = TrackingArgumentParser(description="Demo on using vLLM for offline inference with audio language models")
|
||||
parser = FlexibleArgumentParser(description="Demo on using vLLM for offline inference with audio language models")
|
||||
parser.add_argument(
|
||||
"--query-type",
|
||||
"-q",
|
||||
@@ -546,6 +548,7 @@ def parse_args():
|
||||
help="Number of prompts per batch (default: 1, sequential).",
|
||||
)
|
||||
|
||||
nullify_stage_engine_defaults(parser)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@@ -12,16 +12,17 @@ from pathlib import Path
|
||||
|
||||
import soundfile as sf
|
||||
import torch
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
from vllm_omni import Omni
|
||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
SAMPLE_RATE = 48_000
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = TrackingArgumentParser(description="Offline VoxCPM2 native AR inference")
|
||||
parser = FlexibleArgumentParser(description="Offline VoxCPM2 native AR inference")
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
type=str,
|
||||
@@ -59,6 +60,7 @@ def parse_args():
|
||||
default=None,
|
||||
help="Optional transcript of --ref-audio (enables continuation mode).",
|
||||
)
|
||||
nullify_stage_engine_defaults(parser)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@@ -27,10 +27,11 @@ except ImportError:
|
||||
"pip install -e ."
|
||||
)
|
||||
from vllm import SamplingParams
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
from vllm_omni import AsyncOmni
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -229,7 +230,7 @@ def run_non_streaming(inputs, sampling_params_list, model_name, args, output_dir
|
||||
|
||||
|
||||
def parse_args() -> Namespace:
|
||||
parser = TrackingArgumentParser(description="Demo on using vLLM for offline inference with Voxtral TTS")
|
||||
parser = FlexibleArgumentParser(description="Demo on using vLLM for offline inference with Voxtral TTS")
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
type=str,
|
||||
@@ -312,6 +313,7 @@ def parse_args() -> Namespace:
|
||||
default=None,
|
||||
help="Quantization method (e.g. 'fp8'). Applied to the language model only.",
|
||||
)
|
||||
nullify_stage_engine_defaults(parser)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@@ -68,67 +68,6 @@ vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 --tensor-parallel-size
|
||||
|
||||
Or set `tensor_parallel_size` per stage in a custom deploy YAML.
|
||||
|
||||
### VAE Patch Parallelism
|
||||
|
||||
[VAE Patch Parallelism](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/parallelism/vae_patch_parallel.html) distributes Bagel VAE **decode/encode** across multiple GPUs by splitting latent tiles. It lowers **per-GPU peak memory during VAE decode**, which helps high-resolution `text2img` / `img2img` when VAE becomes a bottleneck.
|
||||
|
||||
**Scope for Bagel:**
|
||||
|
||||
| Topology | VAE patch parallel |
|
||||
| :------- | :----------------- |
|
||||
| **Single-stage** (DiT only) | Supported on stage 0 (`BagelPipeline` + `DistributedAutoEncoder`) |
|
||||
| **Two-stage** | Supported on **stage 1 (DiT)** only; stage 0 (Thinker) uses encoder-only VAE and is unrelated |
|
||||
|
||||
**Requirements:**
|
||||
|
||||
- `vae_patch_parallel_size > 1` and a distributed VAE (`DistributedAutoEncoder` on the DiT pipeline).
|
||||
- The DiT process group must have at least `vae_patch_parallel_size` ranks. In practice this means the diffusion stage `world_size` must be ≥ 2 (commonly `tensor_parallel_size=2` on that stage).
|
||||
- `vae_use_tiling` must be enabled. If you set `vae_patch_parallel_size > 1` and omit tiling, the registry auto-enables `vae_use_tiling` at startup.
|
||||
|
||||
VAE patch parallel **reuses the DiT process group** (`dit_group`); it does not create a separate VAE-only worker pool. It is not a substitute for single-GPU VAE tiling (`vae_pp=1`).
|
||||
|
||||
**Online serving (single-stage, 2 GPUs):**
|
||||
|
||||
```bash
|
||||
CUDA_VISIBLE_DEVICES=0,1 vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 \
|
||||
--deploy-config vllm_omni/deploy/bagel_single_stage.yaml \
|
||||
--tensor-parallel-size 2 \
|
||||
--vae-patch-parallel-size 2 \
|
||||
--vae-use-tiling
|
||||
```
|
||||
|
||||
**Online serving (two-stage, VAE PP on DiT stage 1):** use a custom deploy YAML, for example:
|
||||
|
||||
```yaml
|
||||
stages:
|
||||
- stage_id: 0
|
||||
devices: "0"
|
||||
# Thinker (AR) — no VAE patch parallel here
|
||||
|
||||
- stage_id: 1
|
||||
devices: "0,1"
|
||||
vae_use_tiling: true
|
||||
parallel_config:
|
||||
tensor_parallel_size: 2
|
||||
vae_patch_parallel_size: 2
|
||||
```
|
||||
|
||||
```bash
|
||||
vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 \
|
||||
--deploy-config /path/to/bagel_vae_pp.yaml
|
||||
```
|
||||
|
||||
**Verify it is active** (check server logs at startup):
|
||||
|
||||
```text
|
||||
INFO ... vae_patch_parallel_size=2 requires vae_use_tiling; automatically enabling it.
|
||||
```
|
||||
|
||||
| CLI flag | Default | Description |
|
||||
| :------- | :------ | :---------- |
|
||||
| `--vae-patch-parallel-size` | `1` | Number of DiT ranks used for VAE tile parallelism. Set to `2` or higher to enable. Should be ≤ DiT process group size (typically match `--tensor-parallel-size` on the diffusion stage). |
|
||||
| `--vae-use-tiling` | off | Enable VAE spatial tiling. Required for VAE patch parallel (auto-enabled when `vae_patch_parallel_size > 1`). |
|
||||
|
||||
#### Hybrid Sharded Data Parallel (HSDP)
|
||||
|
||||
For larger Bagel deployments on multiple GPUs, you can enable HSDP (Hybrid Sharded Data Parallel) by modifying the stage configuration (for example, [`bagel.yaml`](../../../vllm_omni/deploy/bagel.yaml)). HSDP shards transformer weights across GPUs to reduce per-GPU memory usage.
|
||||
@@ -384,8 +323,6 @@ python openai_chat_client.py \
|
||||
| `stages[].gpu_memory_utilization` | per-stage | Fraction of GPU memory to use |
|
||||
| `stages[].enforce_eager` | per-stage | Disable CUDA graphs |
|
||||
| `stages[].tensor_parallel_size` | per-stage | TP degree for this stage |
|
||||
| `stages[].parallel_config.vae_patch_parallel_size` | per-stage (DiT) | VAE tile parallelism degree (DiT stage only) |
|
||||
| `stages[].vae_use_tiling` | per-stage (DiT) | Enable VAE tiling (required for VAE patch parallel) |
|
||||
| `connectors` | top-level | Define available connector instances (SHM, Mooncake) |
|
||||
| `platforms` | top-level | Platform-specific overrides (e.g. `xpu`) |
|
||||
|
||||
|
||||
+2
-3
@@ -12,8 +12,7 @@ from typing import Any
|
||||
import requests
|
||||
from openai import OpenAI
|
||||
from vllm.assets.audio import AudioAsset
|
||||
|
||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
# Modify OpenAI's API key and API base to use vLLM's API server.
|
||||
openai_api_key = "EMPTY"
|
||||
@@ -494,7 +493,7 @@ def run_multimodal_generation(args) -> None:
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = TrackingArgumentParser(description="Demo on using vLLM for offline inference with audio language models")
|
||||
parser = FlexibleArgumentParser(description="Demo on using vLLM for offline inference with audio language models")
|
||||
parser.add_argument(
|
||||
"--query-type",
|
||||
"-q",
|
||||
|
||||
@@ -1,65 +0,0 @@
|
||||
# vLLM-Omni · MiniCPM-o 4.5 Online Demo
|
||||
|
||||
Gradio-based web UI for **MiniCPM-o 4.5** served via `vllm-omni`'s
|
||||
OpenAI-compatible endpoints.
|
||||
|
||||
The UI supports:
|
||||
|
||||
- **Inputs**: text prompt + optional image, audio (file or mic), video.
|
||||
- **Outputs**: text + speech (WAV player).
|
||||
|
||||
## 1. Start the backend server
|
||||
|
||||
The deploy config auto-loads via `--omni`; the default
|
||||
`vllm_omni/deploy/minicpmo_4_5.yaml` targets a 2-GPU layout (thinker on
|
||||
GPU 0, talker + t2w sharing GPU 1). For other hardware layouts pick
|
||||
one of the deploy variants below.
|
||||
|
||||
| deploy config | GPUs | Notes |
|
||||
|---|---|---|
|
||||
| `minicpmo_4_5.yaml` (default) | 2 | Thinker on GPU0, talker+t2w on GPU1. |
|
||||
| `minicpmo_4_5_3gpu.yaml` | 3 | Thinker 2-way TP on GPU0/1, talker+t2w share GPU2. |
|
||||
| `minicpmo_4_5_8x4090.yaml` | 8 | Full 8x4090 layout. |
|
||||
|
||||
Default (2-GPU):
|
||||
|
||||
```bash
|
||||
vllm serve openbmb/MiniCPM-o-4_5 --omni \
|
||||
--trust-remote-code \
|
||||
--host 0.0.0.0 --port 8099
|
||||
```
|
||||
|
||||
Other layouts via `--deploy-config`:
|
||||
|
||||
```bash
|
||||
vllm serve openbmb/MiniCPM-o-4_5 --omni \
|
||||
--deploy-config vllm_omni/deploy/minicpmo_4_5_8x4090.yaml \
|
||||
--trust-remote-code \
|
||||
--host 0.0.0.0 --port 8099
|
||||
```
|
||||
|
||||
## 2. Launch the Gradio demo
|
||||
|
||||
```bash
|
||||
bash examples/online_serving/minicpmo/run_gradio_demo.sh
|
||||
|
||||
# Or run the python entry point directly:
|
||||
python examples/online_serving/minicpmo/gradio_demo.py \
|
||||
--minicpmo45-api-base http://localhost:8099/v1 \
|
||||
--minicpmo45-model openbmb/MiniCPM-o-4_5 \
|
||||
--port 7862
|
||||
```
|
||||
|
||||
Open `http://<host>:7862` in a browser.
|
||||
|
||||
## Notes
|
||||
|
||||
- **TTS trigger**: the demo sets
|
||||
`extra_body.chat_template_kwargs.use_tts_template=True`, which appends
|
||||
`<|tts_bos|>` to the assistant prefix.
|
||||
- Uncheck **"Generate speech output (TTS)"** to get text-only responses
|
||||
(faster).
|
||||
- The audio output is the raw WAV returned by the stage-1 talker +
|
||||
Token2Wav; sample rate is 24 kHz.
|
||||
- Video input is forwarded as a base64 `video_url` entry; the server needs
|
||||
decord/torchvision to decode it.
|
||||
@@ -1,580 +0,0 @@
|
||||
"""Gradio demo for MiniCPM-o 4.5 online serving through vllm-omni.
|
||||
|
||||
Supports:
|
||||
- Text / Image / Audio / Video inputs (OpenAI chat-completions multimodal format).
|
||||
- Text + audio outputs, returned side-by-side in the UI.
|
||||
|
||||
Usage:
|
||||
# run a 4.5 server on :8099
|
||||
python gradio_demo.py \
|
||||
--minicpmo45-api-base http://localhost:8099/v1 \
|
||||
--minicpmo45-model openbmb/MiniCPM-o-4_5 \
|
||||
--port 7862
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import io
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import gradio as gr
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
from openai import OpenAI
|
||||
from PIL import Image
|
||||
|
||||
MINICPMO45 = "MiniCPM-o-4.5"
|
||||
|
||||
# Default reference audio per model, relative to the model path. Used to build
|
||||
# the audio_assistant system prompt so that the LLM adopts the "speak in this
|
||||
# voice" persona (matches the official MiniCPM-o demo).
|
||||
_DEFAULT_REF_AUDIO = {
|
||||
MINICPMO45: "assets/HT_ref_audio.wav",
|
||||
}
|
||||
|
||||
# audio_assistant prompt text, mirrors MiniCPMO.get_sys_prompt("audio_assistant")
|
||||
# in the MiniCPM-o-Demo reference implementation.
|
||||
_AUDIO_ASSISTANT_PROMPT = {
|
||||
"zh": {
|
||||
"prefix": "模仿音频样本的音色并生成新的内容。",
|
||||
"suffix": (
|
||||
"你的任务是用这种声音模式来当一个助手。请认真、高质量地回复用户的问题。"
|
||||
"请用高自然度的方式和用户聊天。你是由面壁智能开发的人工智能助手:面壁小钢炮。"
|
||||
),
|
||||
},
|
||||
"en": {
|
||||
"prefix": "Use the voice in the audio prompt to synthesize new content.",
|
||||
"suffix": "You are a helpful assistant with the above voice style.",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Media helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def image_to_base64_data_url(image: Image.Image) -> str:
|
||||
buf = io.BytesIO()
|
||||
if image.mode != "RGB":
|
||||
image = image.convert("RGB")
|
||||
image.save(buf, format="JPEG", quality=92)
|
||||
b64 = base64.b64encode(buf.getvalue()).decode("utf-8")
|
||||
return f"data:image/jpeg;base64,{b64}"
|
||||
|
||||
|
||||
def audio_to_base64_data_url(audio_np: np.ndarray, sample_rate: int) -> str:
|
||||
if audio_np.dtype != np.int16:
|
||||
if audio_np.dtype in (np.float32, np.float64):
|
||||
audio_np = np.clip(audio_np, -1.0, 1.0)
|
||||
audio_np = (audio_np * 32767).astype(np.int16)
|
||||
else:
|
||||
audio_np = audio_np.astype(np.int16)
|
||||
buf = io.BytesIO()
|
||||
sf.write(buf, audio_np, sample_rate, format="WAV")
|
||||
b64 = base64.b64encode(buf.getvalue()).decode("utf-8")
|
||||
return f"data:audio/wav;base64,{b64}"
|
||||
|
||||
|
||||
def ref_audio_to_data_url(path: str, target_sr: int = 16000, max_s: float | None = 8.0) -> str:
|
||||
"""Load a file, downmix to mono, resample to 16kHz, optionally truncate, and
|
||||
return a data-URL usable as audio_url content in the chat API.
|
||||
|
||||
MiniCPM-o expects 16kHz mono audio input; long reference audios can waste
|
||||
tokens so we cap at max_s seconds by default (3-10s is the recommended range).
|
||||
"""
|
||||
data, sr = sf.read(path)
|
||||
if data.ndim > 1:
|
||||
data = data[:, 0]
|
||||
data = data.astype(np.float32)
|
||||
if sr != target_sr:
|
||||
try:
|
||||
# Demo client only; librosa here resamples audio before sending to
|
||||
# the server. The fallback below covers environments without it.
|
||||
import librosa # noqa: TID251
|
||||
|
||||
data = librosa.resample(data, orig_sr=sr, target_sr=target_sr)
|
||||
except ImportError:
|
||||
from math import gcd
|
||||
|
||||
g = gcd(sr, target_sr)
|
||||
up, down = target_sr // g, sr // g
|
||||
idx = (np.arange(len(data) * up) / up).astype(np.int64)
|
||||
data = data[idx][::down]
|
||||
sr = target_sr
|
||||
if max_s is not None and len(data) > int(max_s * sr):
|
||||
data = data[: int(max_s * sr)]
|
||||
return audio_to_base64_data_url(data, sr)
|
||||
|
||||
|
||||
def video_to_base64_data_url(path: str) -> str:
|
||||
p = Path(path)
|
||||
if not p.exists():
|
||||
raise FileNotFoundError(f"Video not found: {path}")
|
||||
ext = p.suffix.lower()
|
||||
mime = {
|
||||
".mp4": "video/mp4",
|
||||
".webm": "video/webm",
|
||||
".mov": "video/quicktime",
|
||||
".avi": "video/x-msvideo",
|
||||
".mkv": "video/x-matroska",
|
||||
}.get(ext, "video/mp4")
|
||||
b64 = base64.b64encode(p.read_bytes()).decode("utf-8")
|
||||
return f"data:{mime};base64,{b64}"
|
||||
|
||||
|
||||
def process_audio_input(audio_input: Any | None) -> tuple[np.ndarray, int] | None:
|
||||
"""Normalize Gradio audio input to (np.ndarray mono float32, sample_rate).
|
||||
|
||||
Gradio's `gr.Audio(type="numpy")` yields (sr, ndarray); we also accept
|
||||
filesystem paths and a few nested tuple variants that appear in different
|
||||
Gradio versions.
|
||||
"""
|
||||
if audio_input is None:
|
||||
return None
|
||||
|
||||
def _from_path(p: str) -> tuple[np.ndarray, int] | None:
|
||||
if not p:
|
||||
return None
|
||||
fp = Path(p)
|
||||
if not fp.exists():
|
||||
return None
|
||||
data, sr = sf.read(fp)
|
||||
if data.ndim > 1:
|
||||
data = data[:, 0]
|
||||
return data.astype(np.float32), int(sr)
|
||||
|
||||
audio_np: np.ndarray | None = None
|
||||
sr: int | None = None
|
||||
|
||||
if isinstance(audio_input, tuple) and len(audio_input) == 2:
|
||||
a, b = audio_input
|
||||
if isinstance(a, int | float) and isinstance(b, np.ndarray):
|
||||
sr, audio_np = int(a), b
|
||||
elif isinstance(a, str):
|
||||
loaded = _from_path(a)
|
||||
if loaded is not None:
|
||||
audio_np, sr = loaded
|
||||
elif isinstance(audio_input, str):
|
||||
loaded = _from_path(audio_input)
|
||||
if loaded is not None:
|
||||
audio_np, sr = loaded
|
||||
|
||||
if audio_np is None or sr is None:
|
||||
return None
|
||||
if audio_np.ndim > 1:
|
||||
audio_np = audio_np[:, 0]
|
||||
return audio_np.astype(np.float32), sr
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-model inference config
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ModelEndpoint:
|
||||
"""Endpoint + TTS-trigger convention for one MiniCPM-o variant."""
|
||||
|
||||
def __init__(self, name: str, api_base: str, model_path: str):
|
||||
self.name = name
|
||||
self.api_base = api_base
|
||||
self.model_path = model_path
|
||||
self.client = OpenAI(api_key="EMPTY", base_url=api_base)
|
||||
self._ref_audio_data_url: str | None = None
|
||||
|
||||
@property
|
||||
def ref_audio_path(self) -> str | None:
|
||||
rel = _DEFAULT_REF_AUDIO.get(self.name)
|
||||
if not rel:
|
||||
return None
|
||||
full = os.path.join(self.model_path, rel)
|
||||
return full if os.path.exists(full) else None
|
||||
|
||||
def get_ref_audio_data_url(self) -> str | None:
|
||||
"""Lazily load and cache the default reference audio as a 16kHz mono
|
||||
data URL. Returns None if the reference audio file is missing."""
|
||||
if self._ref_audio_data_url is not None:
|
||||
return self._ref_audio_data_url
|
||||
p = self.ref_audio_path
|
||||
if not p:
|
||||
print(f"[{self.name}] no default reference audio found; falling back to text-only system prompt")
|
||||
return None
|
||||
try:
|
||||
self._ref_audio_data_url = ref_audio_to_data_url(p)
|
||||
print(f"[{self.name}] loaded reference audio: {p}")
|
||||
except Exception as e:
|
||||
print(f"[{self.name}] failed to load reference audio {p}: {e}")
|
||||
return None
|
||||
return self._ref_audio_data_url
|
||||
|
||||
def build_extras(self) -> dict[str, Any]:
|
||||
"""Extra kwargs passed to chat.completions.create for this model.
|
||||
|
||||
4.5: use_tts_template=True chat-template flag appends <|tts_bos|>.
|
||||
"""
|
||||
if self.name == MINICPMO45:
|
||||
return {
|
||||
"extra_body": {
|
||||
"chat_template_kwargs": {"use_tts_template": True},
|
||||
"modalities": ["text", "audio"],
|
||||
},
|
||||
}
|
||||
return {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# System prompt (matches MiniCPMO.get_sys_prompt("audio_assistant") in the
|
||||
# official MiniCPM-o-Demo). When TTS is disabled we fall back to a plain
|
||||
# text-only system prompt.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def build_audio_assistant_system(
|
||||
ref_audio_data_url: str | None,
|
||||
language: str = "zh",
|
||||
) -> dict[str, Any]:
|
||||
"""Return an OpenAI-style system message carrying the reference audio."""
|
||||
txt = _AUDIO_ASSISTANT_PROMPT.get(language, _AUDIO_ASSISTANT_PROMPT["zh"])
|
||||
if ref_audio_data_url is None:
|
||||
return {
|
||||
"role": "system",
|
||||
"content": (
|
||||
"You are MiniCPM-o, a helpful multimodal assistant that can "
|
||||
"understand images, audio and video, and respond in text and speech."
|
||||
),
|
||||
}
|
||||
return {
|
||||
"role": "system",
|
||||
"content": [
|
||||
{"type": "text", "text": txt["prefix"]},
|
||||
{"type": "audio_url", "audio_url": {"url": ref_audio_data_url}},
|
||||
{"type": "text", "text": txt["suffix"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Core inference function
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def run_inference(
|
||||
endpoint: ModelEndpoint,
|
||||
user_prompt: str,
|
||||
audio_tuple: tuple[np.ndarray, int] | None,
|
||||
image_pil: Image.Image | None,
|
||||
video_path: str | None,
|
||||
enable_tts: bool,
|
||||
max_tokens: int,
|
||||
temperature: float,
|
||||
) -> Iterable[tuple[str, tuple[int, np.ndarray] | None]]:
|
||||
user_prompt = user_prompt or ""
|
||||
if not user_prompt.strip() and not audio_tuple and not image_pil and not video_path:
|
||||
yield "Please provide a text prompt or at least one multimodal input.", None
|
||||
return
|
||||
|
||||
try:
|
||||
content: list[dict[str, Any]] = []
|
||||
|
||||
if audio_tuple is not None:
|
||||
audio_np, sr = audio_tuple
|
||||
content.append(
|
||||
{"type": "audio_url", "audio_url": {"url": audio_to_base64_data_url(audio_np, sr)}},
|
||||
)
|
||||
|
||||
if image_pil is not None:
|
||||
img = image_pil if image_pil.mode == "RGB" else image_pil.convert("RGB")
|
||||
content.append(
|
||||
{"type": "image_url", "image_url": {"url": image_to_base64_data_url(img)}},
|
||||
)
|
||||
|
||||
if video_path:
|
||||
content.append(
|
||||
{"type": "video_url", "video_url": {"url": video_to_base64_data_url(video_path)}},
|
||||
)
|
||||
|
||||
if user_prompt.strip():
|
||||
content.append({"type": "text", "text": user_prompt})
|
||||
|
||||
if enable_tts:
|
||||
ref_url = endpoint.get_ref_audio_data_url()
|
||||
system_msg = build_audio_assistant_system(ref_url, language="zh")
|
||||
else:
|
||||
system_msg = {
|
||||
"role": "system",
|
||||
"content": (
|
||||
"You are MiniCPM-o, a helpful multimodal assistant that can "
|
||||
"understand images, audio and video, and respond in text and speech."
|
||||
),
|
||||
}
|
||||
|
||||
messages = [
|
||||
system_msg,
|
||||
{"role": "user", "content": content},
|
||||
]
|
||||
|
||||
kwargs: dict[str, Any] = {
|
||||
"model": endpoint.model_path,
|
||||
"messages": messages,
|
||||
"max_tokens": int(max_tokens),
|
||||
"temperature": float(temperature),
|
||||
}
|
||||
if enable_tts:
|
||||
extras = endpoint.build_extras()
|
||||
# Merge extra_body dicts cleanly.
|
||||
extra_body = dict(extras.get("extra_body", {}))
|
||||
kwargs["extra_body"] = extra_body
|
||||
else:
|
||||
# Text-only: suppress audio output.
|
||||
kwargs["extra_body"] = {"modalities": ["text"]}
|
||||
|
||||
completion = endpoint.client.chat.completions.create(**kwargs)
|
||||
|
||||
text_parts: list[str] = []
|
||||
audio_out: tuple[int, np.ndarray] | None = None
|
||||
|
||||
for choice in completion.choices:
|
||||
msg = choice.message
|
||||
if getattr(msg, "content", None):
|
||||
text_parts.append(msg.content)
|
||||
audio_obj = getattr(msg, "audio", None)
|
||||
if audio_obj and getattr(audio_obj, "data", None):
|
||||
raw = base64.b64decode(audio_obj.data)
|
||||
if len(raw) > 80:
|
||||
data, sr = sf.read(io.BytesIO(raw))
|
||||
if data.ndim > 1:
|
||||
data = data[:, 0]
|
||||
audio_out = (int(sr), data.astype(np.float32))
|
||||
|
||||
text_response = "\n\n".join(t for t in text_parts if t) or "(no text returned)"
|
||||
yield text_response, audio_out
|
||||
except Exception as exc: # noqa: BLE001
|
||||
yield f"Inference failed: {type(exc).__name__}: {exc}", None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# UI
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
DEFAULT_PROMPTS = [
|
||||
"你好,请用一句话介绍北京。",
|
||||
"Describe what you see in the image.",
|
||||
"Transcribe the audio and translate it into English.",
|
||||
"What is happening in this video?",
|
||||
]
|
||||
|
||||
|
||||
def build_interface(endpoints: dict[str, ModelEndpoint], default_model: str) -> gr.Blocks:
|
||||
model_choices = list(endpoints.keys())
|
||||
|
||||
css = """
|
||||
#generate-btn button { width: 100%; font-size: 1.05rem; }
|
||||
.media-row > div { min-height: 260px; }
|
||||
.model-banner { color: #555; font-size: 0.9rem; }
|
||||
"""
|
||||
|
||||
with gr.Blocks(css=css, title="vLLM-Omni · MiniCPM-o Demo") as demo:
|
||||
gr.Markdown(
|
||||
"# vLLM-Omni · MiniCPM-o Online Demo\n"
|
||||
"Multimodal chat (text / image / audio / video in; text + speech out) for "
|
||||
"MiniCPM-o 4.5 served by vllm-omni."
|
||||
)
|
||||
|
||||
with gr.Row():
|
||||
model_dd = gr.Dropdown(
|
||||
label="Model",
|
||||
choices=model_choices,
|
||||
value=default_model,
|
||||
scale=2,
|
||||
)
|
||||
endpoint_info = gr.Markdown(
|
||||
elem_classes="model-banner",
|
||||
value=f"`{endpoints[default_model].api_base}` → `{endpoints[default_model].model_path}`",
|
||||
)
|
||||
|
||||
with gr.Row():
|
||||
prompt_box = gr.Textbox(
|
||||
label="Text Prompt",
|
||||
placeholder="Ask the model anything. Combine with image / audio / video below.",
|
||||
lines=3,
|
||||
value=DEFAULT_PROMPTS[0],
|
||||
scale=3,
|
||||
)
|
||||
with gr.Column(scale=1):
|
||||
tts_checkbox = gr.Checkbox(label="Generate speech output (TTS)", value=True)
|
||||
max_tokens_slider = gr.Slider(label="Max tokens", minimum=32, maximum=4096, value=1024, step=32)
|
||||
temperature_slider = gr.Slider(label="Temperature", minimum=0.0, maximum=1.5, value=0.7, step=0.05)
|
||||
|
||||
with gr.Row(elem_classes="media-row"):
|
||||
image_input = gr.Image(label="Image (optional)", type="pil", sources=["upload"])
|
||||
audio_input = gr.Audio(
|
||||
label="Audio (optional)",
|
||||
type="numpy",
|
||||
sources=["upload", "microphone"],
|
||||
)
|
||||
video_input = gr.Video(label="Video (optional)", sources=["upload"])
|
||||
|
||||
with gr.Row():
|
||||
generate_btn = gr.Button("Generate", variant="primary", elem_id="generate-btn")
|
||||
clear_btn = gr.Button("Clear")
|
||||
|
||||
with gr.Row():
|
||||
text_output = gr.Textbox(label="Text Output", lines=10, scale=2)
|
||||
audio_output = gr.Audio(label="Audio Output", interactive=False, scale=1)
|
||||
|
||||
gr.Examples(
|
||||
examples=[
|
||||
["你好,请用一句话介绍北京。"],
|
||||
["Describe what you see in detail."],
|
||||
["请先识别这段语音的内容,再用中文回复说话人。"],
|
||||
["What is happening in this video?"],
|
||||
],
|
||||
inputs=[prompt_box],
|
||||
label="Quick prompts",
|
||||
)
|
||||
|
||||
def _on_model_change(name: str) -> str:
|
||||
ep = endpoints[name]
|
||||
return f"`{ep.api_base}` → `{ep.model_path}`"
|
||||
|
||||
model_dd.change(_on_model_change, inputs=[model_dd], outputs=[endpoint_info])
|
||||
|
||||
def _run(
|
||||
model_name: str,
|
||||
prompt: str,
|
||||
image_pil: Image.Image | None,
|
||||
audio_in: Any,
|
||||
video_path: str | None,
|
||||
enable_tts: bool,
|
||||
max_tokens: int,
|
||||
temperature: float,
|
||||
):
|
||||
ep = endpoints[model_name]
|
||||
audio_tuple = process_audio_input(audio_in)
|
||||
yield from run_inference(
|
||||
ep,
|
||||
prompt,
|
||||
audio_tuple,
|
||||
image_pil,
|
||||
video_path,
|
||||
enable_tts,
|
||||
max_tokens,
|
||||
temperature,
|
||||
)
|
||||
|
||||
generate_btn.click(
|
||||
_run,
|
||||
inputs=[
|
||||
model_dd,
|
||||
prompt_box,
|
||||
image_input,
|
||||
audio_input,
|
||||
video_input,
|
||||
tts_checkbox,
|
||||
max_tokens_slider,
|
||||
temperature_slider,
|
||||
],
|
||||
outputs=[text_output, audio_output],
|
||||
)
|
||||
|
||||
def _clear():
|
||||
return "", None, None, None, "", None
|
||||
|
||||
clear_btn.click(
|
||||
_clear,
|
||||
outputs=[
|
||||
prompt_box,
|
||||
image_input,
|
||||
audio_input,
|
||||
video_input,
|
||||
text_output,
|
||||
audio_output,
|
||||
],
|
||||
)
|
||||
|
||||
demo.queue()
|
||||
return demo
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--minicpmo45-api-base", default=os.environ.get("MINICPMO45_API_BASE", "http://localhost:8099/v1"))
|
||||
p.add_argument(
|
||||
"--minicpmo45-model",
|
||||
default=os.environ.get("MINICPMO45_MODEL", "openbmb/MiniCPM-o-4_5"),
|
||||
)
|
||||
p.add_argument("--host", default="0.0.0.0")
|
||||
p.add_argument("--port", type=int, default=7862)
|
||||
p.add_argument("--share", action="store_true")
|
||||
p.add_argument(
|
||||
"--ssl-certfile",
|
||||
default=os.environ.get("GRADIO_SSL_CERTFILE", ""),
|
||||
help="Path to TLS certificate (PEM). Browsers require HTTPS to access the microphone.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--ssl-keyfile",
|
||||
default=os.environ.get("GRADIO_SSL_KEYFILE", ""),
|
||||
help="Path to TLS private key (PEM).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--ssl-verify",
|
||||
action="store_true",
|
||||
default=os.environ.get("GRADIO_SSL_VERIFY", "0") == "1",
|
||||
help="Require browser to verify the certificate (off by default; self-signed certs should stay off).",
|
||||
)
|
||||
return p.parse_args()
|
||||
|
||||
|
||||
def _ping(api_base: str, timeout: float = 3.0) -> bool:
|
||||
import urllib.request
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(api_base.rstrip("/").replace("/v1", "/health"), timeout=timeout):
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
endpoints: dict[str, ModelEndpoint] = {}
|
||||
|
||||
if args.minicpmo45_api_base and args.minicpmo45_model:
|
||||
ok = _ping(args.minicpmo45_api_base)
|
||||
status = "reachable" if ok else "not reachable (will error on generate)"
|
||||
print(f"[4.5] {args.minicpmo45_api_base} ({status})")
|
||||
endpoints[MINICPMO45] = ModelEndpoint(MINICPMO45, args.minicpmo45_api_base, args.minicpmo45_model)
|
||||
|
||||
if not endpoints:
|
||||
raise SystemExit("No endpoints configured. Pass --minicpmo45-api-base/--minicpmo45-model.")
|
||||
|
||||
default_model = next(iter(endpoints.keys()))
|
||||
demo = build_interface(endpoints, default_model)
|
||||
|
||||
launch_kwargs: dict = {
|
||||
"server_name": args.host,
|
||||
"server_port": args.port,
|
||||
"share": args.share,
|
||||
}
|
||||
if args.ssl_certfile and args.ssl_keyfile:
|
||||
launch_kwargs.update(
|
||||
ssl_certfile=args.ssl_certfile,
|
||||
ssl_keyfile=args.ssl_keyfile,
|
||||
ssl_verify=args.ssl_verify,
|
||||
)
|
||||
print(f"[tls] cert={args.ssl_certfile} key={args.ssl_keyfile} verify={args.ssl_verify}")
|
||||
demo.launch(**launch_kwargs)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,36 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Launch the MiniCPM-o 4.5 gradio demo.
|
||||
#
|
||||
# Prereq:
|
||||
# Start a vllm-omni OpenAI server for MiniCPM-o 4.5 on :8099 (see the
|
||||
# 8x4090 stage config under vllm_omni/model_executor/stage_configs).
|
||||
set -e
|
||||
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
|
||||
: "${MINICPMO45_API_BASE:=http://localhost:8099/v1}"
|
||||
: "${MINICPMO45_MODEL:=openbmb/MiniCPM-o-4_5}"
|
||||
: "${GRADIO_HOST:=0.0.0.0}"
|
||||
: "${GRADIO_PORT:=7862}"
|
||||
# HTTPS (browsers require a secure context for microphone access).
|
||||
# Set GRADIO_SSL_CERTFILE / GRADIO_SSL_KEYFILE to enable TLS.
|
||||
: "${GRADIO_SSL_CERTFILE:=}"
|
||||
: "${GRADIO_SSL_KEYFILE:=}"
|
||||
|
||||
export MINICPMO45_API_BASE MINICPMO45_MODEL
|
||||
|
||||
SSL_ARGS=()
|
||||
if [ -n "$GRADIO_SSL_CERTFILE" ] && [ -n "$GRADIO_SSL_KEYFILE" ] \
|
||||
&& [ -f "$GRADIO_SSL_CERTFILE" ] && [ -f "$GRADIO_SSL_KEYFILE" ]; then
|
||||
SSL_ARGS=(--ssl-certfile "$GRADIO_SSL_CERTFILE" --ssl-keyfile "$GRADIO_SSL_KEYFILE")
|
||||
echo "HTTPS enabled: cert=$GRADIO_SSL_CERTFILE key=$GRADIO_SSL_KEYFILE"
|
||||
else
|
||||
echo "HTTPS disabled (cert/key not found). Microphone won't work on remote browsers."
|
||||
fi
|
||||
|
||||
exec python "$HERE/gradio_demo.py" \
|
||||
--minicpmo45-api-base "$MINICPMO45_API_BASE" \
|
||||
--minicpmo45-model "$MINICPMO45_MODEL" \
|
||||
--host "$GRADIO_HOST" \
|
||||
--port "$GRADIO_PORT" \
|
||||
"${SSL_ARGS[@]}"
|
||||
@@ -6,8 +6,7 @@ from typing import NamedTuple
|
||||
import requests
|
||||
from openai import OpenAI
|
||||
from vllm.assets.audio import AudioAsset
|
||||
|
||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
SEED = 42
|
||||
|
||||
@@ -479,7 +478,7 @@ def run_multimodal_generation(args, client: OpenAI) -> None:
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = TrackingArgumentParser(description="Demo on using vLLM for offline inference with audio language models")
|
||||
parser = FlexibleArgumentParser(description="Demo on using vLLM for offline inference with audio language models")
|
||||
parser.add_argument(
|
||||
"--query-type",
|
||||
"-q",
|
||||
|
||||
@@ -412,31 +412,6 @@ curl -X POST http://localhost:8091/v1/audio/voices \
|
||||
```
|
||||
Uploaded voices are then usable as `voice="custom_voice_1"` on subsequent requests.
|
||||
|
||||
### Precomputed custom voices
|
||||
For reused Base voice-cloning speakers, precompute the reference artifacts once and load them at server startup:
|
||||
```bash
|
||||
python qwen3_tts/precompute_custom_voice.py \
|
||||
--model Qwen/Qwen3-TTS-12Hz-1.7B-Base \
|
||||
--voice-name alice \
|
||||
--ref-audio /path/to/reference.wav \
|
||||
--ref-text "Original transcript of the reference audio" \
|
||||
--mode icl \
|
||||
--output-dir /path/to/custom_voices
|
||||
```
|
||||
`--mode icl` stores both `speaker_embedding` and `ref_code`; `--mode xvec` stores only the speaker embedding. Add the output directory to a deploy config:
|
||||
```yaml
|
||||
custom_voice_dir: /path/to/custom_voices
|
||||
```
|
||||
Then start the server with that config and call the Speech API with only the voice name:
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base --omni --deploy-config /path/to/qwen3_tts_custom_voice.yaml
|
||||
|
||||
curl -X POST http://localhost:8091/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"input":"Hello from a precomputed voice.","voice":"alice","task_type":"Base"}' \
|
||||
--output alice.wav
|
||||
```
|
||||
|
||||
### Streaming PCM
|
||||
```bash
|
||||
curl -X POST http://localhost:8091/v1/audio/speech \
|
||||
@@ -472,8 +447,7 @@ python qwen3_tts/streaming_speech_client.py --text "..." --simulate-stt --stt-de
|
||||
`qwen3_tts/batch_speech_client.py` issues many concurrent requests for throughput measurement.
|
||||
|
||||
### Notes
|
||||
- Base voice cloning has uniproc-vs-mp tradeoffs depending on per-request reference audio cost; see the executor-backend section above.
|
||||
- With async chunking, Qwen3-TTS Base voice cloning sends the full reference context in the first Code2Wav packet, then caches that prefix on the Code2Wav stage for follow-up chunks in the same request.
|
||||
- Base voice cloning has per-request reference-audio cost; the uniproc default keeps IPC overhead off the critical path. See the executor-backend section above for background.
|
||||
- `vllm_omni/deploy/qwen3_tts.yaml` is the default deploy config (loaded by HF `model_type`); per-stage runtime overrides are available via `--stage-N-<field> <value>`.
|
||||
|
||||
---
|
||||
@@ -500,23 +474,6 @@ python voxcpm2/openai_speech_client.py \
|
||||
```
|
||||
The `ref_audio` field accepts local file paths (auto-base64), HTTP URLs, or `data:audio/wav;base64,...` data URIs.
|
||||
|
||||
### Precomputed custom voices
|
||||
For repeated VoxCPM2 speakers, precompute the prompt cache and load it through `custom_voice_dir`:
|
||||
```bash
|
||||
python voxcpm2/precompute_custom_voice.py \
|
||||
--model openbmb/VoxCPM2 \
|
||||
--voice-name alice \
|
||||
--ref-audio /path/to/reference.wav \
|
||||
--mode ref_continuation \
|
||||
--prompt-text "Original transcript of the reference audio" \
|
||||
--output-dir /path/to/custom_voices
|
||||
```
|
||||
Add the output directory to the deploy config:
|
||||
```yaml
|
||||
custom_voice_dir: /path/to/custom_voices
|
||||
```
|
||||
After startup, `/v1/audio/voices` lists `alice`, and `/v1/audio/speech` can use `voice="alice"` without sending `ref_audio`.
|
||||
|
||||
### Gradio demo (gapless streaming via AudioWorklet)
|
||||
```bash
|
||||
python voxcpm2/gradio_demo.py
|
||||
|
||||
@@ -1,243 +0,0 @@
|
||||
"""Pre-compute Qwen3-TTS custom voice profiles.
|
||||
|
||||
The generated directory can be passed to the server via
|
||||
``custom_voice_dir`` in ``vllm_omni/deploy/qwen3_tts.yaml``. Requests can then
|
||||
use ``/v1/audio/speech`` with ``voice="<name>"`` and no per-request ref_audio.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from safetensors import safe_open
|
||||
from safetensors.torch import save_file
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from vllm_omni.utils.custom_voice_io import safe_voice_stem # noqa: E402
|
||||
|
||||
MANIFEST_NAME = "custom_voice_manifest.json"
|
||||
_SPEAKER_ENCODER_SAMPLE_RATE = 24000
|
||||
_SPEAKER_ENCODER_NUM_MELS = 128
|
||||
_SPEAKER_ENCODER_N_FFT = 1024
|
||||
_SPEAKER_ENCODER_HOP_SIZE = 256
|
||||
_SPEAKER_ENCODER_WIN_SIZE = 1024
|
||||
_SPEAKER_ENCODER_FMIN = 0
|
||||
_SPEAKER_ENCODER_FMAX = 12000
|
||||
|
||||
|
||||
def _resolve_model_dir(model: str) -> str:
|
||||
if os.path.isdir(model):
|
||||
return model
|
||||
from transformers.utils.hub import cached_file
|
||||
|
||||
config_path = cached_file(model, "config.json")
|
||||
if config_path is None:
|
||||
raise FileNotFoundError(f"Could not resolve config.json for {model}")
|
||||
return os.path.dirname(config_path)
|
||||
|
||||
|
||||
def _read_audio_mono(path: str) -> tuple[np.ndarray, int]:
|
||||
import soundfile as sf
|
||||
|
||||
wav, sr = sf.read(path, dtype="float32")
|
||||
if wav.ndim > 1:
|
||||
wav = wav.mean(axis=-1)
|
||||
return np.asarray(wav, dtype=np.float32), int(sr)
|
||||
|
||||
|
||||
def _resample(wav: np.ndarray, sr: int, target_sr: int) -> np.ndarray:
|
||||
if sr == target_sr:
|
||||
return wav.astype(np.float32)
|
||||
from vllm.multimodal.audio import AudioResampler
|
||||
|
||||
return AudioResampler(target_sr=target_sr).resample(wav.astype(np.float32), orig_sr=int(sr))
|
||||
|
||||
|
||||
def _load_speaker_encoder(model_dir: str, device: torch.device):
|
||||
from vllm_omni.model_executor.models.qwen3_tts.configuration_qwen3_tts import Qwen3TTSConfig
|
||||
from vllm_omni.model_executor.models.qwen3_tts.qwen3_tts_talker import Qwen3TTSSpeakerEncoder
|
||||
|
||||
config = Qwen3TTSConfig.from_pretrained(model_dir)
|
||||
encoder = Qwen3TTSSpeakerEncoder(config.speaker_encoder_config)
|
||||
index_path = Path(model_dir) / "model.safetensors.index.json"
|
||||
state: dict[str, torch.Tensor] = {}
|
||||
if index_path.exists():
|
||||
index = json.loads(index_path.read_text(encoding="utf-8"))
|
||||
weight_map = index.get("weight_map", {})
|
||||
shard_names = sorted({v for k, v in weight_map.items() if k.startswith("speaker_encoder.")})
|
||||
shard_paths = [Path(model_dir) / name for name in shard_names]
|
||||
else:
|
||||
shard_paths = sorted(Path(model_dir).glob("model*.safetensors"))
|
||||
for shard in shard_paths:
|
||||
with safe_open(str(shard), framework="pt", device="cpu") as f:
|
||||
for key in f.keys():
|
||||
if key.startswith("speaker_encoder."):
|
||||
state[key.removeprefix("speaker_encoder.")] = f.get_tensor(key)
|
||||
if not state:
|
||||
raise RuntimeError(
|
||||
f"No speaker_encoder.* tensors found under {model_dir}. "
|
||||
"Use a Qwen3-TTS Base checkpoint for pre-computing custom voice profiles."
|
||||
)
|
||||
encoder.load_state_dict(state)
|
||||
encoder.to(device=device, dtype=torch.bfloat16).eval()
|
||||
return config, encoder
|
||||
|
||||
|
||||
def _speaker_embedding(
|
||||
encoder: torch.nn.Module,
|
||||
speaker_encoder_config: object,
|
||||
wav: np.ndarray,
|
||||
sr: int,
|
||||
device: torch.device,
|
||||
) -> torch.Tensor:
|
||||
from vllm_omni.model_executor.models.qwen3_tts.prompt_embeds_builder import mel_spectrogram
|
||||
|
||||
target_sr = int(getattr(speaker_encoder_config, "sample_rate", _SPEAKER_ENCODER_SAMPLE_RATE))
|
||||
mel_dim = int(getattr(speaker_encoder_config, "mel_dim", _SPEAKER_ENCODER_NUM_MELS))
|
||||
if target_sr != _SPEAKER_ENCODER_SAMPLE_RATE or mel_dim != _SPEAKER_ENCODER_NUM_MELS:
|
||||
raise ValueError(
|
||||
"Unsupported Qwen3-TTS speaker encoder mel config for precompute: "
|
||||
f"sample_rate={target_sr}, mel_dim={mel_dim}. "
|
||||
f"Expected sample_rate={_SPEAKER_ENCODER_SAMPLE_RATE}, mel_dim={_SPEAKER_ENCODER_NUM_MELS} "
|
||||
"to match the serving-time speaker embedding path."
|
||||
)
|
||||
wav = _resample(wav, sr, target_sr)
|
||||
wav_t = torch.from_numpy(wav).to(device=device, dtype=torch.float32)
|
||||
mels = mel_spectrogram(
|
||||
wav_t.unsqueeze(0),
|
||||
n_fft=_SPEAKER_ENCODER_N_FFT,
|
||||
num_mels=_SPEAKER_ENCODER_NUM_MELS,
|
||||
sampling_rate=_SPEAKER_ENCODER_SAMPLE_RATE,
|
||||
hop_size=_SPEAKER_ENCODER_HOP_SIZE,
|
||||
win_size=_SPEAKER_ENCODER_WIN_SIZE,
|
||||
fmin=_SPEAKER_ENCODER_FMIN,
|
||||
fmax=_SPEAKER_ENCODER_FMAX,
|
||||
).transpose(1, 2)
|
||||
with torch.inference_mode():
|
||||
return encoder(mels.to(device=device, dtype=torch.bfloat16))[0].float().cpu().contiguous()
|
||||
|
||||
|
||||
def _ref_code(model_dir: str, wav: np.ndarray, sr: int, device: torch.device) -> torch.Tensor:
|
||||
from vllm_omni.model_executor.models.qwen3_tts.qwen3_tts_tokenizer import Qwen3TTSTokenizer
|
||||
|
||||
tok = Qwen3TTSTokenizer.from_pretrained(
|
||||
str(Path(model_dir) / "speech_tokenizer"),
|
||||
torch_dtype=torch.bfloat16,
|
||||
)
|
||||
try:
|
||||
del tok.model.decoder
|
||||
tok.model.decoder = None
|
||||
tok.model.encoder.to(device)
|
||||
tok.device = device
|
||||
except Exception:
|
||||
tok.device = device
|
||||
with torch.inference_mode():
|
||||
enc = tok.encode(wav, sr=int(sr), return_dict=True)
|
||||
codes = getattr(enc, "audio_codes", None)
|
||||
if isinstance(codes, list):
|
||||
codes = codes[0] if codes else None
|
||||
if not isinstance(codes, torch.Tensor):
|
||||
raise RuntimeError("Speech tokenizer did not return audio_codes")
|
||||
if codes.ndim == 3:
|
||||
codes = codes[0]
|
||||
return codes.to(dtype=torch.int32, device="cpu").contiguous()
|
||||
|
||||
|
||||
def _load_manifest(output_dir: Path, model: str, hidden_size: int) -> dict[str, Any]:
|
||||
path = output_dir / MANIFEST_NAME
|
||||
if path.exists():
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"model_type": "qwen3_tts",
|
||||
"model": model,
|
||||
"hidden_size": hidden_size,
|
||||
"voices": {},
|
||||
}
|
||||
|
||||
|
||||
def _write_voice(
|
||||
*,
|
||||
model: str,
|
||||
output_dir: Path,
|
||||
voice_name: str,
|
||||
ref_audio: str,
|
||||
ref_text: str | None,
|
||||
mode: str,
|
||||
speaker_description: str | None,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
model_dir = _resolve_model_dir(model)
|
||||
config, encoder = _load_speaker_encoder(model_dir, device)
|
||||
wav, sr = _read_audio_mono(ref_audio)
|
||||
if wav.size < 1024:
|
||||
raise ValueError(f"Reference audio too short: {wav.size} samples")
|
||||
|
||||
tensors: dict[str, torch.Tensor] = {
|
||||
"speaker_embedding": _speaker_embedding(encoder, config.speaker_encoder_config, wav, sr, device),
|
||||
}
|
||||
if mode == "icl":
|
||||
if not ref_text or not ref_text.strip():
|
||||
raise ValueError("--ref-text is required for --mode icl")
|
||||
tensors["ref_code"] = _ref_code(model_dir, wav, sr, device)
|
||||
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
filename = f"{safe_voice_stem(voice_name)}.safetensors"
|
||||
save_file(tensors, str(output_dir / filename))
|
||||
|
||||
hidden_size = int(getattr(config.talker_config, "hidden_size", tensors["speaker_embedding"].numel()))
|
||||
manifest = _load_manifest(output_dir, model, hidden_size)
|
||||
entry: dict[str, Any] = {
|
||||
"name": voice_name,
|
||||
"file": filename,
|
||||
"mode": mode,
|
||||
"embedding_dim": int(tensors["speaker_embedding"].numel()),
|
||||
}
|
||||
if "ref_code" in tensors:
|
||||
entry["ref_code_length"] = int(tensors["ref_code"].shape[0])
|
||||
if ref_text:
|
||||
entry["ref_text"] = ref_text
|
||||
if speaker_description:
|
||||
entry["speaker_description"] = speaker_description
|
||||
manifest.setdefault("voices", {})[voice_name] = entry
|
||||
(output_dir / MANIFEST_NAME).write_text(json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
print(f"Wrote {output_dir / filename}")
|
||||
print(f"Updated {output_dir / MANIFEST_NAME}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Pre-compute Qwen3-TTS custom voice profile")
|
||||
parser.add_argument("--model", required=True, help="Qwen3-TTS Base model path or Hugging Face ID")
|
||||
parser.add_argument("--voice-name", required=True)
|
||||
parser.add_argument("--ref-audio", required=True)
|
||||
parser.add_argument("--ref-text", default=None)
|
||||
parser.add_argument("--mode", choices=["xvec", "icl"], default="xvec")
|
||||
parser.add_argument("--speaker-description", default=None)
|
||||
parser.add_argument("--output-dir", required=True)
|
||||
parser.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
|
||||
args = parser.parse_args()
|
||||
|
||||
_write_voice(
|
||||
model=args.model,
|
||||
output_dir=Path(args.output_dir),
|
||||
voice_name=args.voice_name,
|
||||
ref_audio=args.ref_audio,
|
||||
ref_text=args.ref_text,
|
||||
mode=args.mode,
|
||||
speaker_description=args.speaker_description,
|
||||
device=torch.device(args.device),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,128 +0,0 @@
|
||||
"""Pre-compute VoxCPM2 custom voice profiles.
|
||||
|
||||
The generated directory can be passed to the server via
|
||||
``custom_voice_dir`` in ``vllm_omni/deploy/voxcpm2.yaml``. Requests can then
|
||||
use ``/v1/audio/speech`` with ``voice="<name>"`` and no per-request ref_audio.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from safetensors.torch import save_file
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from vllm_omni.utils.custom_voice_io import safe_voice_stem # noqa: E402
|
||||
|
||||
MANIFEST_NAME = "custom_voice_manifest.json"
|
||||
|
||||
|
||||
def _load_tts(model: str, device: torch.device):
|
||||
from vllm_omni.model_executor.models.voxcpm2.voxcpm2_import_utils import import_voxcpm2_core
|
||||
|
||||
VoxCPM = import_voxcpm2_core()
|
||||
native = VoxCPM.from_pretrained(model, load_denoiser=False, optimize=False)
|
||||
return native.tts_model.to(device).eval()
|
||||
|
||||
|
||||
def _load_manifest(output_dir: Path, model: str) -> dict[str, Any]:
|
||||
path = output_dir / MANIFEST_NAME
|
||||
if path.exists():
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"model_type": "voxcpm2",
|
||||
"model": model,
|
||||
"voices": {},
|
||||
}
|
||||
|
||||
|
||||
def _write_voice(
|
||||
*,
|
||||
model: str,
|
||||
output_dir: Path,
|
||||
voice_name: str,
|
||||
ref_audio: str,
|
||||
prompt_text: str | None,
|
||||
mode: str,
|
||||
speaker_description: str | None,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
if mode in ("continuation", "ref_continuation") and not prompt_text:
|
||||
raise ValueError("--prompt-text is required for continuation/ref_continuation modes")
|
||||
|
||||
tts = _load_tts(model, device)
|
||||
tensors: dict[str, torch.Tensor] = {}
|
||||
with torch.inference_mode():
|
||||
if mode in ("reference", "ref_continuation"):
|
||||
tensors["ref_audio_feat"] = tts._encode_wav(ref_audio, padding_mode="right").float().cpu().contiguous()
|
||||
if mode in ("continuation", "ref_continuation"):
|
||||
tensors["audio_feat"] = tts._encode_wav(ref_audio, padding_mode="left").float().cpu().contiguous()
|
||||
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
filename = f"{safe_voice_stem(voice_name)}.safetensors"
|
||||
save_file(tensors, str(output_dir / filename))
|
||||
|
||||
manifest = _load_manifest(output_dir, model)
|
||||
entry: dict[str, Any] = {
|
||||
"name": voice_name,
|
||||
"file": filename,
|
||||
"mode": mode,
|
||||
}
|
||||
if "ref_audio_feat" in tensors:
|
||||
entry["ref_audio_feat_len"] = int(tensors["ref_audio_feat"].shape[0])
|
||||
if "audio_feat" in tensors:
|
||||
entry["audio_feat_len"] = int(tensors["audio_feat"].shape[0])
|
||||
if prompt_text:
|
||||
entry["prompt_text"] = prompt_text
|
||||
if speaker_description:
|
||||
entry["speaker_description"] = speaker_description
|
||||
|
||||
manifest.setdefault("voices", {})[voice_name] = entry
|
||||
(output_dir / MANIFEST_NAME).write_text(json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
print(f"Wrote {output_dir / filename}")
|
||||
print(f"Updated {output_dir / MANIFEST_NAME}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Pre-compute VoxCPM2 custom voice profile")
|
||||
parser.add_argument("--model", default="openbmb/VoxCPM2", help="VoxCPM2 model path or Hugging Face ID")
|
||||
parser.add_argument("--voice-name", required=True)
|
||||
parser.add_argument("--ref-audio", required=True)
|
||||
parser.add_argument(
|
||||
"--prompt-text",
|
||||
default=None,
|
||||
help="Transcript of ref audio for continuation/ref_continuation modes",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--mode",
|
||||
choices=["reference", "continuation", "ref_continuation"],
|
||||
default="reference",
|
||||
)
|
||||
parser.add_argument("--speaker-description", default=None)
|
||||
parser.add_argument("--output-dir", required=True)
|
||||
parser.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
|
||||
args = parser.parse_args()
|
||||
|
||||
_write_voice(
|
||||
model=args.model,
|
||||
output_dir=Path(args.output_dir),
|
||||
voice_name=args.voice_name,
|
||||
ref_audio=args.ref_audio,
|
||||
prompt_text=args.prompt_text,
|
||||
mode=args.mode,
|
||||
speaker_description=args.speaker_description,
|
||||
device=torch.device(args.device),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -76,18 +76,6 @@ local = [
|
||||
"FlagEmbedding",
|
||||
]
|
||||
|
||||
# Optional runtime dependencies for the MiniCPM-o 4.5 talker pipeline.
|
||||
# The talker loads MiniCPMTTS via `trust_remote_code` from the model repo,
|
||||
# and that module performs `from stepaudio2 import Token2wav` against the
|
||||
# MiniCPM-o-flavored Token2wav vocoder (PyPI: `stepaudio2-minicpmo`, NOT
|
||||
# upstream `stepfun-ai/Step-Audio2` — the upstream signature rejects
|
||||
# `n_timesteps` and crashes at construction). A clean `vllm-omni` install
|
||||
# does not pull this in by default to avoid forcing the dep onto users
|
||||
# who only need other models; opt in with `pip install vllm-omni[minicpmo]`.
|
||||
minicpmo = [
|
||||
"stepaudio2-minicpmo",
|
||||
]
|
||||
|
||||
docs = [
|
||||
"mkdocs>=1.5.0",
|
||||
"mkdocs-api-autonav",
|
||||
@@ -113,13 +101,6 @@ Documentation = "https://vllm-omni.readthedocs.io"
|
||||
[project.scripts]
|
||||
vllm-omni = "vllm_omni.entrypoints.cli.main:main"
|
||||
|
||||
# Register vllm-omni's model registrations on every vllm subprocess. vllm
|
||||
# auto-loads anything advertised under `vllm.general_plugins` in its own
|
||||
# `load_general_plugins()`, so worker subprocesses spawned by vllm (which
|
||||
# only import vllm, not vllm_omni) will still pick up the omni archs.
|
||||
[project.entry-points."vllm.general_plugins"]
|
||||
vllm_omni_register_models = "vllm_omni.engine.arg_utils:register_omni_models_to_vllm"
|
||||
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["."]
|
||||
@@ -168,10 +149,6 @@ ignore = [
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"examples/**" = ["E501"] # Allow long lines in examples
|
||||
"tests/**" = ["E501"] # Allow long lines in tests
|
||||
# MiniCPM-o reference modeling code vendored from upstream HF/transformers/MiniCPM-o.
|
||||
# These files keep upstream conventions (star imports, long lines, math-symbol
|
||||
# argument names) so they stay easy to diff against upstream releases.
|
||||
"vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py" = ["E402", "E501", "F403", "F405", "N803"]
|
||||
|
||||
[tool.ruff.lint.flake8-tidy-imports.banned-api]
|
||||
"librosa".msg = "The librosa module is banned, use vllm.multimodal helpers instead"
|
||||
|
||||
@@ -1,229 +0,0 @@
|
||||
# MiniCPM-o 4.5
|
||||
|
||||
> Online serving for omni multimodal chat (text / image / audio / video → text + 24 kHz speech)
|
||||
|
||||
## Summary
|
||||
|
||||
- Vendor: OpenBMB
|
||||
- Model: [`openbmb/MiniCPM-o-4_5`](https://huggingface.co/openbmb/MiniCPM-o-4_5)
|
||||
- Task: Omni multimodal chat — accepts text / image / audio / video input;
|
||||
emits text and 24 kHz mono speech in the same response
|
||||
- Mode: Online serving via the OpenAI-compatible `/v1/chat/completions`
|
||||
API, plus a bundled Gradio demo (text + speech UI)
|
||||
- Maintainer: [`@tc-mb`](https://github.com/tc-mb) (MiniCPM-V / MiniCPM-o team)
|
||||
|
||||
## When to use this recipe
|
||||
|
||||
Use this recipe as a known-good starting point for serving
|
||||
`openbmb/MiniCPM-o-4_5` on vLLM-Omni. MiniCPM-o 4.5 is the omni member
|
||||
of the MiniCPM-o family — it pairs a multimodal-understanding thinker
|
||||
LLM with a streaming `MiniCPMTTS + Token2Wav` talker so a single
|
||||
`/v1/chat/completions` call can return text and 24 kHz speech in one
|
||||
shot. The recipe covers three shipped GPU layouts (2 / 3 / 8 GPUs)
|
||||
selected via `--deploy-config`.
|
||||
|
||||
## References
|
||||
|
||||
- Default deploy configs (auto-loaded by HF `model_type=minicpmo` +
|
||||
`hf_config.version="4.5"`):
|
||||
- 2-GPU layout (default):
|
||||
[`vllm_omni/deploy/minicpmo_4_5.yaml`](../../vllm_omni/deploy/minicpmo_4_5.yaml)
|
||||
- 3-GPU layout (thinker TP=2):
|
||||
[`vllm_omni/deploy/minicpmo_4_5_3gpu.yaml`](../../vllm_omni/deploy/minicpmo_4_5_3gpu.yaml)
|
||||
- 8x RTX 4090 layout:
|
||||
[`vllm_omni/deploy/minicpmo_4_5_8x4090.yaml`](../../vllm_omni/deploy/minicpmo_4_5_8x4090.yaml)
|
||||
- Online example + Gradio demo:
|
||||
[`examples/online_serving/minicpmo/`](../../examples/online_serving/minicpmo/)
|
||||
- Pipeline / talker source:
|
||||
[`vllm_omni/model_executor/models/minicpmo_4_5/`](../../vllm_omni/model_executor/models/minicpmo_4_5/)
|
||||
- Stage-input processor (thinker → talker bridge):
|
||||
[`vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py`](../../vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py)
|
||||
- Upstream model card:
|
||||
[`openbmb/MiniCPM-o-4_5`](https://huggingface.co/openbmb/MiniCPM-o-4_5)
|
||||
- Integration PR:
|
||||
[vllm-project/vllm-omni#3642](https://github.com/vllm-project/vllm-omni/pull/3642)
|
||||
|
||||
## Hardware Support
|
||||
|
||||
Three GPU layouts ship with default deploy configs. Pick the layout that
|
||||
matches your hardware and pass it via `--deploy-config`; the talker
|
||||
(`MiniCPMTTS + Token2Wav`) always lives on its own GPU because of the
|
||||
in-process vocoder, and the thinker is the part that scales out via TP.
|
||||
|
||||
| Layout | Thinker | Talker + Token2Wav | Typical hardware |
|
||||
| --- | --- | --- | --- |
|
||||
| 2-GPU (default) | GPU 0 | GPU 1 | 2x A100/H100/H200 80GB |
|
||||
| 3-GPU (thinker TP=2) | GPU 0,1 (TP=2) | GPU 2 | 3x mid-tier GPUs |
|
||||
| 8x RTX 4090 24GB | GPU 0–3 (TP=4) | GPU 4 | 8x RTX 4090 consumer |
|
||||
|
||||
## GPU
|
||||
|
||||
### 2 x GPU (default — single command)
|
||||
|
||||
The default
|
||||
[`vllm_omni/deploy/minicpmo_4_5.yaml`](../../vllm_omni/deploy/minicpmo_4_5.yaml)
|
||||
puts the thinker on GPU 0 (`~70 %` memory, `enforce_eager: true`,
|
||||
`max_num_seqs: 1`) and the talker + Token2Wav vocoder on GPU 1
|
||||
(`~75 %` memory). This is the recommended starting layout — works on
|
||||
any pair of 80GB-class GPUs (A100, H100, H200) and on most 40GB+
|
||||
pairs as long as the thinker model weights fit.
|
||||
|
||||
#### Environment
|
||||
|
||||
- OS: Linux
|
||||
- Python: 3.10+
|
||||
- vLLM / vLLM-Omni: >= 0.21.0 (or current `main`)
|
||||
- Optional Talker dep: `stepaudio2-minicpmo` (see Notes for why this is
|
||||
required and how to install it)
|
||||
|
||||
#### Command
|
||||
|
||||
```bash
|
||||
vllm serve openbmb/MiniCPM-o-4_5 --omni \
|
||||
--trust-remote-code \
|
||||
--host 0.0.0.0 --port 8099
|
||||
```
|
||||
|
||||
The deploy config is auto-loaded by the model registry — no
|
||||
`--deploy-config` flag needed for this default 2-GPU layout.
|
||||
|
||||
#### Verification
|
||||
|
||||
**Quick smoke test (text-only output)**:
|
||||
|
||||
```bash
|
||||
curl http://localhost:8099/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"messages": [{"role": "user", "content": "Briefly introduce yourself."}],
|
||||
"modalities": ["text"]
|
||||
}'
|
||||
```
|
||||
|
||||
**Text + speech in one response** (the headline 4.5 feature). The TTS
|
||||
path is gated by a Jinja flag on the chat template, passed through
|
||||
`extra_body.chat_template_kwargs.use_tts_template=true`:
|
||||
|
||||
```bash
|
||||
curl http://localhost:8099/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"messages": [{"role": "user", "content": "Say hello, then introduce vLLM in one sentence."}],
|
||||
"modalities": ["text", "audio"],
|
||||
"extra_body": {"chat_template_kwargs": {"use_tts_template": true}}
|
||||
}'
|
||||
```
|
||||
|
||||
Response carries text in `choices[0].message.content` and base64 WAV
|
||||
in `choices[0].message.audio.data` (24 kHz mono, see Notes).
|
||||
|
||||
**Gradio demo (text + image + audio + video UI)**:
|
||||
|
||||
```bash
|
||||
bash examples/online_serving/minicpmo/run_gradio_demo.sh
|
||||
# or run the python entry point directly:
|
||||
python examples/online_serving/minicpmo/gradio_demo.py \
|
||||
--minicpmo45-api-base http://localhost:8099/v1 \
|
||||
--minicpmo45-model openbmb/MiniCPM-o-4_5 \
|
||||
--port 7862
|
||||
```
|
||||
|
||||
Open `http://<host>:7862` and try a text prompt with the **"Generate
|
||||
speech output (TTS)"** checkbox on / off.
|
||||
|
||||
#### Notes
|
||||
|
||||
- Memory budget: thinker weights occupy GPU 0 at `gpu_memory_utilization:
|
||||
0.7`; talker + Token2Wav vocoder share GPU 1 at `0.75`.
|
||||
- `--trust-remote-code` is required — the HF repo ships a custom
|
||||
`MiniCPMO` config / model class.
|
||||
- Pin: `enforce_eager: true` on both stages (CUDA graph capture is off
|
||||
by design for the talker's Token2Wav path).
|
||||
- Stage 1 (talker) is hard-capped to `max_num_seqs: 1`: the talker
|
||||
only consumes `runtime_additional_information[0]`, so any value > 1
|
||||
makes concurrent requests share request-0's audio. This is the same
|
||||
cap baked into the deploy config.
|
||||
|
||||
### 3 x GPU (thinker TP=2)
|
||||
|
||||
Use
|
||||
[`vllm_omni/deploy/minicpmo_4_5_3gpu.yaml`](../../vllm_omni/deploy/minicpmo_4_5_3gpu.yaml)
|
||||
when you have a third GPU available and want the thinker on 2-way
|
||||
tensor parallel for higher throughput; the talker stays on its own
|
||||
GPU (talker has its own in-process Token2Wav vocoder, so co-locating
|
||||
it with the thinker risks OOM under load).
|
||||
|
||||
#### Command
|
||||
|
||||
```bash
|
||||
vllm serve openbmb/MiniCPM-o-4_5 --omni \
|
||||
--deploy-config vllm_omni/deploy/minicpmo_4_5_3gpu.yaml \
|
||||
--trust-remote-code \
|
||||
--host 0.0.0.0 --port 8099
|
||||
```
|
||||
|
||||
Verification and Notes mirror the 2-GPU section; thinker latency
|
||||
roughly halves under load thanks to TP=2.
|
||||
|
||||
### 8 x RTX 4090 24GB (consumer-GPU layout)
|
||||
|
||||
Use
|
||||
[`vllm_omni/deploy/minicpmo_4_5_8x4090.yaml`](../../vllm_omni/deploy/minicpmo_4_5_8x4090.yaml)
|
||||
on an 8x RTX 4090 host. Thinker uses 4-way TP across GPUs 0–3
|
||||
(`~85 %` mem each ≈ 20.4 GiB/card), talker + Token2Wav lives on GPU 4
|
||||
(`~90 %` mem). GPUs 5–7 are left free.
|
||||
|
||||
#### Command
|
||||
|
||||
```bash
|
||||
vllm serve openbmb/MiniCPM-o-4_5 --omni \
|
||||
--deploy-config vllm_omni/deploy/minicpmo_4_5_8x4090.yaml \
|
||||
--trust-remote-code \
|
||||
--host 0.0.0.0 --port 8099
|
||||
```
|
||||
|
||||
#### Notes
|
||||
|
||||
- `max_model_len` is capped at 4096 in this layout — 8192 still OOMs on
|
||||
4090s. Raise it if your cards have more headroom (e.g. 4090 D /
|
||||
custom 32 GB SKUs), but verify with a long-prompt run before
|
||||
promoting.
|
||||
- All other knobs match the 2-GPU section; the only difference is the
|
||||
per-card memory pressure on the thinker shards.
|
||||
|
||||
## Notes (applies to all layouts)
|
||||
|
||||
- **Talker dependency**: the `MiniCPM-o 4.5` talker calls
|
||||
`from stepaudio2 import Token2wav` against the MiniCPM-o-flavored
|
||||
vocoder (PyPI package `stepaudio2-minicpmo` — NOT the upstream
|
||||
`stepfun-ai/Step-Audio2`, whose `Token2wav.__init__` signature
|
||||
rejects `n_timesteps`). Install via the published extra:
|
||||
|
||||
```bash
|
||||
pip install 'vllm-omni[minicpmo]'
|
||||
```
|
||||
|
||||
Equivalent direct install: `pip install stepaudio2-minicpmo`. A
|
||||
missing dep raises `ImportError` at first request with the same
|
||||
install hint instead of silently emitting empty audio.
|
||||
|
||||
- **TTS trigger**: speech output is only emitted when the client passes
|
||||
`extra_body.chat_template_kwargs.use_tts_template=true`. Without it,
|
||||
the response is text-only (which is also faster).
|
||||
|
||||
- **Output audio**: 24 kHz mono WAV inside the OpenAI-style
|
||||
`message.audio.data` (base64). The Gradio demo's WAV player decodes
|
||||
this automatically.
|
||||
|
||||
- **Routing**: MiniCPM-o 4.5 and 2.6 both ship `architectures=
|
||||
["MiniCPMO"]` in HF config; routing is disambiguated by
|
||||
`hf_config.version == "4.5"` via the
|
||||
`hf_config_predicate` on the 4.5 pipeline. A 2.6 checkpoint loaded
|
||||
with this recipe's `--deploy-config` will be rejected at startup
|
||||
rather than silently misrouted.
|
||||
|
||||
- **Async chunking**: disabled in all three deploy configs
|
||||
(`async_chunk: false`) — the talker batches a single full thinker
|
||||
output, not chunks.
|
||||
@@ -1,249 +0,0 @@
|
||||
# VoxCPM2
|
||||
|
||||
> Online + offline TTS with native AR pipeline (48 kHz, 30+ languages)
|
||||
|
||||
## Summary
|
||||
|
||||
- Vendor: OpenBMB
|
||||
- Model: `openbmb/VoxCPM2`
|
||||
- Task: Text-to-speech (zero-shot synthesis and reference-audio voice cloning)
|
||||
- Mode: Online serving via the OpenAI-compatible `/v1/audio/speech` API, plus
|
||||
an offline end-to-end example
|
||||
- Maintainer: Community
|
||||
|
||||
## When to use this recipe
|
||||
|
||||
Use this recipe as a known-good starting point for serving `openbmb/VoxCPM2`
|
||||
on vLLM-Omni on consumer-class GPUs. VoxCPM2 is a 2B-parameter native AR
|
||||
TTS model that emits 48 kHz mono audio and supports 30+ languages out of
|
||||
the box. It runs as a single-stage pipeline (`MiniCPM4 base LM → FSQ →
|
||||
MiniCPM4 residual LM → LocDiT CFM solver → AudioVAE`) and supports both
|
||||
zero-shot synthesis and reference-audio voice cloning, with optional
|
||||
gapless streaming via the bundled Gradio AudioWorklet demo.
|
||||
|
||||
Because the model is only 2B params, a single 24 GB consumer GPU is
|
||||
sufficient — no tensor parallelism is required.
|
||||
|
||||
## References
|
||||
|
||||
- Example guide (online):
|
||||
[`examples/online_serving/text_to_speech/README.md`](../../examples/online_serving/text_to_speech/README.md#voxcpm2)
|
||||
- Online client / Gradio demo:
|
||||
[`examples/online_serving/text_to_speech/voxcpm2/`](../../examples/online_serving/text_to_speech/voxcpm2/)
|
||||
- Offline end-to-end script:
|
||||
[`examples/offline_inference/text_to_speech/voxcpm2/end2end.py`](../../examples/offline_inference/text_to_speech/voxcpm2/end2end.py)
|
||||
- Default deploy config:
|
||||
[`vllm_omni/deploy/voxcpm2.yaml`](../../vllm_omni/deploy/voxcpm2.yaml)
|
||||
- Talker / pipeline source:
|
||||
[`vllm_omni/model_executor/models/voxcpm2/`](../../vllm_omni/model_executor/models/voxcpm2/)
|
||||
- Speech API reference:
|
||||
[`docs/serving/speech_api.md`](../../docs/serving/speech_api.md#voxcpm2)
|
||||
- Related issue or discussion:
|
||||
[RFC: add recipes folder](https://github.com/vllm-project/vllm-omni/issues/2645)
|
||||
|
||||
## Hardware Support
|
||||
|
||||
This recipe documents one tested 24 GB consumer-GPU configuration.
|
||||
Larger-VRAM (H20 / H100 / A100) and other vendor sections (ROCm, NPU) are
|
||||
welcome as community validation lands.
|
||||
|
||||
## GPU
|
||||
|
||||
### 1 x RTX 4090 24GB (Single GPU, Minimum Recommended)
|
||||
|
||||
VoxCPM2 (2B params in bfloat16) fits comfortably on a single 24 GB GPU.
|
||||
The bundled default config at
|
||||
[`vllm_omni/deploy/voxcpm2.yaml`](../../vllm_omni/deploy/voxcpm2.yaml)
|
||||
(`gpu_memory_utilization: 0.9`, `max_num_seqs: 4`, `enforce_eager: true`)
|
||||
loads cleanly on a 4090 with ~4.9 GiB for model weights, ~15.2 GiB for
|
||||
KV cache, and ~2 GiB for the talker's CUDA-Graph / CFM / VAE buffers,
|
||||
for a total resident footprint of about **22 GiB / 24 GiB** in steady
|
||||
state.
|
||||
|
||||
#### Environment
|
||||
|
||||
- OS: Linux (Ubuntu 22.04)
|
||||
- Python: 3.12
|
||||
- Driver / runtime: NVIDIA driver 575.57.08, CUDA 12.9 runtime
|
||||
- torch: 2.11.0+cu128
|
||||
- vLLM: 0.21.0
|
||||
- vLLM-Omni: 0.20.1.dev (current `main`)
|
||||
- Native VoxCPM2 source: `voxcpm` >= 2.0
|
||||
|
||||
#### Command
|
||||
|
||||
Start the server from the repository root. On a clean 4090 with no other
|
||||
tenant on the GPU, the defaults from `voxcpm2.yaml` work as-is:
|
||||
|
||||
```bash
|
||||
vllm serve openbmb/VoxCPM2 --omni \
|
||||
--host 0.0.0.0 --port 8000
|
||||
```
|
||||
|
||||
The deploy config at
|
||||
[`vllm_omni/deploy/voxcpm2.yaml`](../../vllm_omni/deploy/voxcpm2.yaml)
|
||||
is loaded automatically by the model registry (HF `model_type=voxcpm2`).
|
||||
Pass `--deploy-config <path>` to override.
|
||||
|
||||
#### Verification
|
||||
|
||||
**Server cold-start**: ~60 s from `vllm serve` to `Application startup
|
||||
complete` (subprocess fork + vLLM 0.21 init + model load + flashinfer JIT
|
||||
+ torch.compile of LocDiT / feat_encoder / AudioVAE + CUDA-Graph warmup).
|
||||
The first request after startup pays a small additional cost; steady-state
|
||||
requests are much faster.
|
||||
|
||||
**T1 — zero-shot synthesis (curl, WAV)**:
|
||||
|
||||
```bash
|
||||
curl -X POST http://127.0.0.1:8000/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "voxcpm2",
|
||||
"input": "Hello, this is VoxCPM2 served by vLLM-Omni on a single RTX 4090.",
|
||||
"voice": "default",
|
||||
"response_format": "wav"
|
||||
}' --output output.wav
|
||||
```
|
||||
|
||||
Observed: `output.wav` 491 KB, 5.12 s @ 48 kHz mono, peak amplitude 0.996.
|
||||
Server-to-client wall time: 2.6 s (server-side RTF ~0.5 after warmup).
|
||||
|
||||
**T2 — bundled Python client (zero-shot)**:
|
||||
|
||||
```bash
|
||||
python examples/online_serving/text_to_speech/voxcpm2/openai_speech_client.py \
|
||||
--text "This audio was generated by the bundled openai_speech_client.py script against a vLLM-Omni server." \
|
||||
--api-base http://127.0.0.1:8000 \
|
||||
--output output.wav
|
||||
```
|
||||
|
||||
Observed: `output.wav` 968 KB, 10.08 s @ 48 kHz mono, peak amplitude 0.949.
|
||||
|
||||
**T3 — voice cloning from a reference WAV**:
|
||||
|
||||
```bash
|
||||
python examples/online_serving/text_to_speech/voxcpm2/openai_speech_client.py \
|
||||
--text "This should sound like the reference speaker, but saying a different sentence on a single RTX 4090." \
|
||||
--ref-audio /path/to/reference.wav \
|
||||
--api-base http://127.0.0.1:8000 \
|
||||
--output cloned.wav
|
||||
```
|
||||
|
||||
Observed (using the T2 output `output.wav` as `--ref-audio`):
|
||||
`cloned.wav` 568 KB, 5.92 s @ 48 kHz mono, peak amplitude 0.990;
|
||||
end-to-end wall time 3.16 s.
|
||||
|
||||
`--ref-audio` accepts local file paths (auto-base64), HTTP/HTTPS URLs, or
|
||||
`data:audio/wav;base64,...` data URIs. The `voice` field is required by
|
||||
the OpenAI schema but ignored by VoxCPM2 unless it refers to an
|
||||
uploaded voice — voice cloning is driven entirely by `ref_audio`.
|
||||
|
||||
**T4 — streaming PCM (raw, 48 kHz s16le)**:
|
||||
|
||||
```bash
|
||||
curl -X POST http://127.0.0.1:8000/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "voxcpm2",
|
||||
"input": "Streaming PCM test on a single RTX 4090 with VoxCPM2.",
|
||||
"voice": "default",
|
||||
"stream": true,
|
||||
"response_format": "pcm"
|
||||
}' --no-buffer | play -t raw -r 48000 -e signed -b 16 -c 1 -
|
||||
```
|
||||
|
||||
Observed: 230400 int16 samples = 4.80 s of PCM at 48 kHz, peak 32525 /
|
||||
32767 (~99 % full scale). The player sample rate is **48 kHz**, not
|
||||
24 kHz.
|
||||
|
||||
**T5 — gapless streaming (Gradio AudioWorklet demo)**:
|
||||
|
||||
```bash
|
||||
python examples/online_serving/text_to_speech/voxcpm2/gradio_demo.py \
|
||||
--api-base http://127.0.0.1:8000
|
||||
```
|
||||
|
||||
Observed: the demo binds to `http://0.0.0.0:7860` in ~4 s after launch.
|
||||
The UI ("VoxCPM2 Streaming Demo") exposes three modes — **Voice Design**
|
||||
(control instruction only), **Controllable Cloning** (ref audio +
|
||||
optional style control), and **Ultimate Cloning** (ref audio +
|
||||
transcript for audio continuation) — plus a `Stream (gapless)`
|
||||
checkbox (default on) that drives the AudioWorklet streaming path. The
|
||||
right pane includes runnable example prompts and a `Ready` status
|
||||
indicator. End-to-end browser playback through `Generate Speech` was
|
||||
verified working on the same vLLM-Omni server backing T1–T4 above.
|
||||
|
||||
**T6 — offline inference (no server, single process per request)**:
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
|
||||
--text "Hello, this is VoxCPM2 native AR synthesis running on vLLM-Omni and a single RTX 4090." \
|
||||
--output-dir output_audio
|
||||
```
|
||||
|
||||
`end2end.py` is a one-shot script (init → one `generate()` → exit), so
|
||||
its reported RTF includes per-process compile + CUDA-Graph capture +
|
||||
first-request runtime warmup. Note that the default `voxcpm2.yaml` sets
|
||||
`enforce_eager: true`, so vLLM's engine-level CUDA graphs are off by
|
||||
design; the "CUDA-Graph capture" referenced throughout this recipe is the
|
||||
talker's own `_CapturedGraph` over the CFM solver / AudioVAE, not the LM
|
||||
decode loop. Flipping `enforce_eager=false` does **not** remove this cost.
|
||||
To separate the warmup cost from the
|
||||
talker's true per-step throughput, the table below reports **5
|
||||
consecutive `engine.generate([prompt])` calls in the same process**
|
||||
after a one-off ~28 s engine init:
|
||||
|
||||
| call | duration | inference | RTF | notes |
|
||||
| ---- | -------- | --------- | ----- | ------------------------------------------------ |
|
||||
| #1 | 6.72 s | 11.97 s | 1.782 | cold: torch.compile + CUDA-Graph capture |
|
||||
| #2 | 6.24 s | 11.43 s | 1.831 | still runtime warmup |
|
||||
| #3 | 6.88 s | 0.82 s | 0.120 | ⚡ steady-state |
|
||||
| #4 | 6.56 s | 0.78 s | 0.119 | steady-state |
|
||||
| #5 | 5.76 s | 0.70 s | 0.121 | steady-state |
|
||||
|
||||
**Steady-state RTF (mean of calls #3–#5): 0.120**, i.e. ~8× real-time on
|
||||
a single 4090. The high call-#1 / call-#2 numbers are one-time warmup
|
||||
costs that **amortise to zero after the second request** — they are not
|
||||
the model's per-step throughput.
|
||||
|
||||
A literal one-shot `python end2end.py` invocation in a fresh process
|
||||
only sees call #1 (~RTF 1.8). For repeated workloads, either reuse the
|
||||
same engine across calls (as in the table above) or use the online
|
||||
serving path (T1–T5); a long-lived server amortises the warmup across
|
||||
all subsequent requests (T1 wall-time RTF was 0.51 with a warm server).
|
||||
|
||||
Voice cloning + continuation mode (`ref_audio` + transcript):
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
|
||||
--text "Hello, this is the offline voice cloning test on a single RTX 4090." \
|
||||
--ref-audio /path/to/reference.wav \
|
||||
--ref-text "Original transcript of the reference audio."
|
||||
```
|
||||
|
||||
Same 5-call methodology as the zero-shot table, this time with a
|
||||
10.08 s reference WAV supplied via `--ref-audio` (and matching
|
||||
`--ref-text`):
|
||||
|
||||
| call | duration | inference | RTF | notes |
|
||||
| ---- | -------- | --------- | ----- | ------------------------------------------------ |
|
||||
| #1 | 5.44 s | 12.38 s | 2.276 | cold: compile + CUDA-Graph capture + ref encode |
|
||||
| #2 | 5.12 s | 2.47 s | 0.482 | most warmup done |
|
||||
| #3 | 5.44 s | 0.76 s | 0.139 | ⚡ steady-state |
|
||||
| #4 | 4.96 s | 0.68 s | 0.137 | steady-state |
|
||||
| #5 | 5.28 s | 0.71 s | 0.134 | steady-state |
|
||||
|
||||
**Steady-state cloning RTF (mean of calls #3–#5): 0.137**, i.e. only
|
||||
**+0.017 RTF** above the zero-shot steady-state of 0.120. That delta is
|
||||
the per-request `AudioVAE.encode` over the reference clip (~85 ms for a
|
||||
10 s ref WAV) plus the ref-continuation prompt build; the AR decode
|
||||
steps themselves run at the same per-step rate as zero-shot.
|
||||
|
||||
#### Notes
|
||||
|
||||
- Prerequisites: `pip install voxcpm soundfile httpx ninja`. `ninja` must be on `PATH` because `flashinfer` JIT-compiles a top-k sampling kernel during the first profile pass (without it, startup dies with `FileNotFoundError: 'ninja'`).
|
||||
- Memory usage: ~4.9 GiB model weights + ~15.2 GiB KV cache ≈ 22 GiB / 24 GiB resident with the default `voxcpm2.yaml`. On a shared GPU, pass `--gpu-memory-utilization 0.75` (or lower) if startup fails the free-memory check.
|
||||
- Key flags: `--omni` is required. `--trust-remote-code` is not needed — the HF config registers under `model_type=voxcpm2`.
|
||||
- Output: 48 kHz mono. When streaming PCM to a player, use `-r 48000`.
|
||||
- Cold start: ~60 s server boot + ~25 s first-inference overhead (torch.compile + flashinfer JIT + the talker's own `_CapturedGraph` capture over CFM / AudioVAE — not vLLM engine CUDA graphs, which `enforce_eager: true` disables). Steady-state inference RTF is **~0.12** (~8× real-time on a single 4090; see T6 table for the per-call breakdown); online server requests run at ~RTF 0.5 wall-time (includes HTTP round-trip). Keep the server warm for interactive use.
|
||||
@@ -1,145 +0,0 @@
|
||||
# MOSS-TTS
|
||||
|
||||
## Summary
|
||||
|
||||
- Vendor: OpenMOSS
|
||||
- Models: `OpenMOSS-Team/MOSS-TTS` (8B), `OpenMOSS-Team/MOSS-TTS-v1.5` (8B),
|
||||
`OpenMOSS-Team/MOSS-TTS-Realtime` (1.7B), `OpenMOSS-Team/MOSS-TTSD-v1.0` (8B),
|
||||
`OpenMOSS-Team/MOSS-SoundEffect` (8B), `OpenMOSS-Team/MOSS-VoiceGenerator` (1.7B)
|
||||
- Task: Text-to-speech synthesis, sound effect generation, zero-shot voice design
|
||||
- Mode: Online serving via the OpenAI-compatible `/v1/audio/speech` API; offline inference
|
||||
- Maintainer: Community
|
||||
|
||||
## When to use this recipe
|
||||
|
||||
Use this recipe for 24 kHz multilingual TTS with voice cloning (20 languages including
|
||||
Chinese and English). Choose a variant based on your latency and quality requirements:
|
||||
|
||||
| Model | Params | Use case |
|
||||
|---|---|---|
|
||||
| MOSS-TTS | 8B | General TTS, highest quality |
|
||||
| MOSS-TTS-v1.5 | 8B | General TTS upgrade of 1.0: 31 languages, steadier cloning, `[pause Xs]` markers (set `language` for best results); same `MossTTSDelay` API |
|
||||
| MOSS-TTS-Realtime | 1.7B | Lowest latency (TTFB ~180 ms), streaming-first |
|
||||
| MOSS-TTSD-v1.0 | 8B | Multi-turn dialogue TTS |
|
||||
| MOSS-SoundEffect | 8B | Sound effect synthesis from text description |
|
||||
| MOSS-VoiceGenerator | 1.7B | Zero-shot voice design |
|
||||
|
||||
All variants share the same codec (`OpenMOSS-Team/MOSS-Audio-Tokenizer`, ~7 GB) and
|
||||
output 24 kHz mono audio.
|
||||
|
||||
## References
|
||||
|
||||
- Offline inference example: [`examples/offline_inference/text_to_speech/moss_tts/`](../../examples/offline_inference/text_to_speech/moss_tts/)
|
||||
- Deploy configs: [`vllm_omni/deploy/moss_tts.yaml`](../../vllm_omni/deploy/moss_tts.yaml) and variants
|
||||
- HuggingFace org: <https://huggingface.co/OpenMOSS-Team>
|
||||
|
||||
## Hardware Support
|
||||
|
||||
### GPU
|
||||
|
||||
#### 1x H100 80GB — MOSS-TTS (8B)
|
||||
|
||||
##### Environment
|
||||
|
||||
- OS: Linux
|
||||
- Python: 3.11+
|
||||
- CUDA 12.8
|
||||
- vLLM-Omni version: see `vllm_omni/__version__.py`
|
||||
|
||||
##### Command
|
||||
|
||||
```bash
|
||||
# The codec is loaded automatically from OpenMOSS-Team/MOSS-Audio-Tokenizer.
|
||||
# Override the path with MOSS_TTS_CODEC_PATH if you have a local copy.
|
||||
vllm serve OpenMOSS-Team/MOSS-TTS --omni --port 8091
|
||||
```
|
||||
|
||||
##### Verification
|
||||
|
||||
Voice cloning (provide a reference audio clip):
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8091/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "OpenMOSS-Team/MOSS-TTS",
|
||||
"input": "Hello, this is a voice cloning test.",
|
||||
"voice": "default",
|
||||
"ref_audio": "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS/main/assets/audio/zh_1.wav",
|
||||
"response_format": "wav"
|
||||
}' --output output.wav
|
||||
```
|
||||
|
||||
##### Notes
|
||||
|
||||
- Peak GPU memory: ~18 GB for the talker (8B) + ~8 GB for the codec decoder on the same device.
|
||||
Use `gpu_memory_utilization: 0.85` in `moss_tts.yaml` (default).
|
||||
- Output: 24 kHz mono WAV.
|
||||
- The `MOSS_TTS_CODEC_PATH` environment variable overrides the codec checkpoint location.
|
||||
|
||||
---
|
||||
|
||||
#### 1x A10G 24GB — MOSS-TTS-Realtime (1.7B)
|
||||
|
||||
##### Environment
|
||||
|
||||
- OS: Linux
|
||||
- Python: 3.11+
|
||||
- CUDA 12.8
|
||||
|
||||
##### Command
|
||||
|
||||
```bash
|
||||
vllm serve OpenMOSS-Team/MOSS-TTS-Realtime --omni --port 8091
|
||||
```
|
||||
|
||||
##### Verification
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8091/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "OpenMOSS-Team/MOSS-TTS-Realtime",
|
||||
"input": "This is a low-latency streaming TTS test.",
|
||||
"voice": "default",
|
||||
"ref_audio": "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS/main/assets/audio/zh_1.wav",
|
||||
"response_format": "wav",
|
||||
"stream": true
|
||||
}' --output output.wav
|
||||
```
|
||||
|
||||
##### Notes
|
||||
|
||||
- Peak GPU memory: ~6 GB for the talker (1.7B) + ~8 GB for the codec decoder.
|
||||
- First-audio latency (TTFB): ~180 ms on A10G.
|
||||
- `codec_chunk_frames: 15` in `moss_tts_realtime.yaml` for lower TTFA than the 8B variant.
|
||||
|
||||
---
|
||||
|
||||
#### 1x A10G 24GB — MOSS-SoundEffect (8B, sound effect synthesis)
|
||||
|
||||
##### Command
|
||||
|
||||
```bash
|
||||
vllm serve OpenMOSS-Team/MOSS-SoundEffect --omni --port 8091
|
||||
```
|
||||
|
||||
##### Verification
|
||||
|
||||
Sound effect synthesis takes a text description instead of reference audio:
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8091/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "OpenMOSS-Team/MOSS-SoundEffect",
|
||||
"input": "Thunder rumbling, rain pattering on a tin roof.",
|
||||
"response_format": "wav"
|
||||
}' --output thunder.wav
|
||||
```
|
||||
|
||||
##### Notes
|
||||
|
||||
- No `ref_audio` required or accepted for MOSS-SoundEffect.
|
||||
- Input field maps to the `ambient_sound` parameter in the upstream processor.
|
||||
- Rate: ~12.5 tokens per second; longer descriptions produce longer audio.
|
||||
@@ -36,9 +36,6 @@ recipes/
|
||||
| [`LTX/LTX-2.md`](./LTX/LTX-2.md) | Text-to-video and image-to-video serving | 1x H200 141GB |
|
||||
| [`LTX/LTX-2.3.md`](./LTX/LTX-2.3.md) | Text-to-video with audio generation (22B) | 1x GPU (96GB VRAM) |
|
||||
| [`mistralai/Voxtral-TTS.md`](./mistralai/Voxtral-TTS.md) | Online serving for TTS | 1x RTX 4090 24GB |
|
||||
| [`nvidia/Cosmos3-Nano.md`](./nvidia/Cosmos3-Nano.md) | Text-to-image, text-to-video, and image-to-video generation | 1x H200 141GB / B300 |
|
||||
| [`OpenBMB/MiniCPM-o-4_5.md`](./OpenBMB/MiniCPM-o-4_5.md) | Online serving for omni multimodal chat (text / image / audio / video → text + 24 kHz speech) | 2x A100/H100 80GB / 3x mid-tier GPU / 8x RTX 4090 24GB |
|
||||
| [`OpenBMB/VoxCPM2.md`](./OpenBMB/VoxCPM2.md) | Online + offline TTS with native AR pipeline (48 kHz, 30+ languages) | 1x RTX 4090 24GB |
|
||||
| [`Qwen/Qwen-Image.md`](./Qwen/Qwen-Image.md) | Text-to-image serving with step-wise continuous batching replay and ModelOpt mixed FP8/NVFP4 | 1x A100 80GB / 2x B200 |
|
||||
| [`Qwen/Qwen-Image.md`](./Qwen/Qwen-Image.md) | Text-to-image serving with step-wise continuous batching replay | 1x A100 80GB |
|
||||
| [`Qwen/Qwen-Image-Edit.md`](./Qwen/Qwen-Image-Edit.md) | Text-guided single-image editing | 1x or 2x H200 141GB |
|
||||
@@ -51,7 +48,6 @@ recipes/
|
||||
| [`Wan-AI/Wan2.2-S2V.md`](./Wan-AI/Wan2.2-S2V.md) | Speech-to-video serving (Wan2.2 14B) | 2x A100/H100 80GB |
|
||||
| [`zai-org/GLM-TTS.md`](./zai-org/GLM-TTS.md) | Online serving for Chinese/English zero-shot voice-cloned TTS | 1x A40 48GB |
|
||||
| [`GLM/GLM-Image.md`](./GLM/GLM-Image.md) | Online serving for image generation | 1x A800 80GB / 2x A800 80GB |
|
||||
|
|
||||
|
||||
Within a single recipe file, include different hardware support sections such
|
||||
as `GPU`, `ROCm`, and `NPU`, and add concrete tested configurations like
|
||||
|
||||
@@ -1,215 +0,0 @@
|
||||
# Cosmos3-Nano
|
||||
|
||||
> Text-to-image, text-to-video, and image-to-video serving
|
||||
|
||||
## Summary
|
||||
|
||||
- Vendor: NVIDIA
|
||||
- Model: `nvidia/Cosmos3-Nano`
|
||||
- Task: Text-to-image (T2I), text-to-video (T2V), and image-to-video (I2V) generation
|
||||
- Mode: Online serving with the OpenAI-compatible image/video APIs, plus offline generation via the `Omni` API
|
||||
- Maintainer: Community
|
||||
|
||||
## When to use this recipe
|
||||
|
||||
Use this recipe to deploy `nvidia/Cosmos3-Nano` for image and video generation.
|
||||
A single pipeline class (`Cosmos3OmniDiffusersPipeline`) serves all three modes;
|
||||
the mode is selected per request:
|
||||
|
||||
- **T2I** — `POST /v1/images/generations` (or a prompt carrying `modalities=["image"]`).
|
||||
- **T2V** — `POST /v1/videos/sync` with `num_frames > 1` and no reference image.
|
||||
- **I2V** — `POST /v1/videos/sync` with a reference image (`input_reference` file
|
||||
upload, or `image_reference` JSON).
|
||||
|
||||
## References
|
||||
|
||||
- Model card (authoritative usage + example assets): <https://huggingface.co/nvidia/Cosmos3-Nano>
|
||||
- Example inputs/outputs live in the repo's `assets/` (`example_t2v_prompt.json`,
|
||||
`example_i2v_prompt.json`, `example_i2v_input.jpg`, `negative_prompt.json`).
|
||||
- Prompt upsampling (recommended for quality): the model expects JSON-upsampled
|
||||
structured prompts; see NVIDIA's `cosmos-framework` prompt-upsampling docs.
|
||||
- Pipeline: [`vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py`](../../vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py)
|
||||
- Smoke tests (canonical request formats): [`tests/e2e/accuracy/test_cosmos3_similarity.py`](../../tests/e2e/accuracy/test_cosmos3_similarity.py)
|
||||
|
||||
## Hardware Support
|
||||
|
||||
## GPU
|
||||
|
||||
### 1x H200 141GB / B300 (Online serving)
|
||||
|
||||
#### Environment
|
||||
|
||||
- OS: Ubuntu 22.04+
|
||||
- Python: 3.12+
|
||||
- Driver / runtime: NVIDIA CUDA environment
|
||||
- vLLM version: match the repository requirements from your current checkout
|
||||
- vLLM-Omni version or commit: use the commit you are deploying from
|
||||
|
||||
#### Command
|
||||
|
||||
Safety guardrails are **on by default** (NVIDIA Open Model License). They load
|
||||
the **gated** `nvidia/Cosmos-1.0-Guardrail` model, so to keep them on you must:
|
||||
|
||||
1. `pip install cosmos-guardrail`
|
||||
2. Accept the license at <https://huggingface.co/nvidia/Cosmos-1.0-Guardrail>
|
||||
3. Export a token with access: `export HF_TOKEN=hf_...`
|
||||
|
||||
Then launch the recommended server:
|
||||
|
||||
```bash
|
||||
vllm serve nvidia/Cosmos3-Nano \
|
||||
--omni \
|
||||
--host 0.0.0.0 --port 8000 \
|
||||
--init-timeout 1800
|
||||
```
|
||||
|
||||
To run **without** guardrails (you are responsible for license compliance),
|
||||
add `--no-guardrails` (no token/`cosmos-guardrail` needed). For extra GPUs use
|
||||
`--ulysses-degree N` (context parallel) or `--tensor-parallel-size N`;
|
||||
`--enable-layerwise-offload` reduces VRAM on smaller GPUs. The pipeline
|
||||
auto-resolves from `model_index.json`; pass
|
||||
`--model-class-name Cosmos3OmniDiffusersPipeline` to force it explicitly.
|
||||
|
||||
#### Verification
|
||||
|
||||
Best quality uses the JSON-upsampled prompts from `assets/` (download with
|
||||
`hf download nvidia/Cosmos3-Nano assets/ --local-dir Cosmos3-Nano`). Minimal
|
||||
self-contained examples:
|
||||
|
||||
```bash
|
||||
curl http://localhost:8000/v1/models
|
||||
|
||||
# Text-to-image -> /v1/images/generations (1024x1024, 50 steps; base64 PNG)
|
||||
curl -sS -X POST http://localhost:8000/v1/images/generations \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "nvidia/Cosmos3-Nano",
|
||||
"prompt": "A photorealistic red sports car on a city street at golden hour, cinematic lighting.",
|
||||
"negative_prompt": "blurry, distorted, low quality",
|
||||
"size": "1024x1024", "n": 1, "response_format": "b64_json",
|
||||
"num_inference_steps": 50, "guidance_scale": 7.0, "seed": 42
|
||||
}' | python -c "import sys,json,base64; open('cosmos3_t2i.png','wb').write(base64.b64decode(json.load(sys.stdin)['data'][0]['b64_json']))"
|
||||
|
||||
# Text-to-video -> /v1/videos/sync (720p, 189 frames @ 24fps; official params)
|
||||
curl -sS -X POST http://localhost:8000/v1/videos/sync \
|
||||
-H "Accept: video/mp4" \
|
||||
-F "model=nvidia/Cosmos3-Nano" \
|
||||
-F "prompt=A robot arm is cleaning a plate in the kitchen" \
|
||||
-F "negative_prompt=blurry, distorted, low quality, jittery, deformed" \
|
||||
-F "size=1280x720" -F "num_frames=189" -F "fps=24" \
|
||||
-F "num_inference_steps=35" -F "guidance_scale=6.0" \
|
||||
-F "max_sequence_length=4096" -F "flow_shift=10.0" \
|
||||
-F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":true}' \
|
||||
-F "seed=123" \
|
||||
-o cosmos3_t2v.mp4
|
||||
|
||||
# Image-to-video -> /v1/videos/sync with an uploaded reference image
|
||||
curl -sS -X POST http://localhost:8000/v1/videos/sync \
|
||||
-H "Accept: video/mp4" \
|
||||
-F "model=nvidia/Cosmos3-Nano" \
|
||||
-F "prompt=The scene comes to life with smooth, natural motion." \
|
||||
-F "negative_prompt=blurry, distorted, low quality" \
|
||||
-F "size=1280x720" -F "num_frames=189" -F "fps=24" \
|
||||
-F "num_inference_steps=35" -F "guidance_scale=6.0" \
|
||||
-F "max_sequence_length=4096" -F "flow_shift=10.0" \
|
||||
-F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":true}' \
|
||||
-F "seed=1111" \
|
||||
-F "input_reference=@/path/to/reference.jpg;type=image/jpeg" \
|
||||
-o cosmos3_i2v.mp4
|
||||
```
|
||||
|
||||
#### Notes
|
||||
|
||||
- **Measured latency (1x B300, bf16, guardrails off):**
|
||||
- T2I 1024² — 10 / 25 / 50 steps → ~0.4 / 0.7 / **1.3 s**
|
||||
- T2V 1280×720 @ 35 steps — 25 / 49 / 93 / **189** frames → ~7 / 15 / 33 / **~93 s**
|
||||
- I2V 1280×720, 189 frames @ 35 steps → ~**99 s**
|
||||
- Guardrails-on overhead: ~8% on T2I, negligible on video.
|
||||
- **Memory:** transformer ~17 GiB (bf16); peak ~46 GiB for 720p video on 1 GPU;
|
||||
full repo (transformer + Wan VAE + Qwen3-VL vision encoder + audio tokenizer)
|
||||
~33 GB on disk.
|
||||
- **Determinism:** identical seed reproduces identical output on the same
|
||||
hardware; outputs are not bit-identical across different GPU types.
|
||||
- **Supported sizes (per model card):** 256p / 480p / 720p at 16:9, 4:3, 1:1,
|
||||
3:4, 9:16. Defaults: T2I 1024², 50 steps, guidance 7.0; T2V/I2V 1280×720,
|
||||
189 frames, 35 steps, guidance 6.0, `flow_shift=10.0`.
|
||||
- **Key flags / params:** `--no-guardrails` (server) or
|
||||
`extra_params={"guardrails":false}` (per request) toggles safety;
|
||||
`use_resolution_template` / `use_duration_template` are off by default and only
|
||||
needed when not using upsampled prompts that already encode resolution/duration.
|
||||
- **Known limitations:**
|
||||
- Guardrails-on requires `cosmos-guardrail` **and** access to the gated
|
||||
`nvidia/Cosmos-1.0-Guardrail` repo (accept license + `HF_TOKEN`); otherwise
|
||||
the server fails at pipeline build with a gated-repo / safety-checker error.
|
||||
- A guardrail-blocked prompt currently returns HTTP 500
|
||||
(`"Guardrail blocked prompt"`).
|
||||
- Video + audio, and action (policy / forward- / inverse-dynamics) modalities
|
||||
are not part of this integration yet.
|
||||
|
||||
### 1x GPU (Offline generation)
|
||||
|
||||
#### Environment
|
||||
|
||||
- OS: Ubuntu 22.04+
|
||||
- Python: 3.12+
|
||||
- Driver / runtime: NVIDIA CUDA environment
|
||||
- vLLM-Omni version or commit: use the commit you are deploying from
|
||||
|
||||
#### Command
|
||||
|
||||
```python
|
||||
# cosmos3_offline.py — run with: python cosmos3_offline.py
|
||||
import torch
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
|
||||
def main():
|
||||
omni = Omni(
|
||||
model="nvidia/Cosmos3-Nano",
|
||||
model_class_name="Cosmos3OmniDiffusersPipeline",
|
||||
trust_remote_code=True,
|
||||
enforce_eager=True,
|
||||
# Keep guardrails on by installing cosmos-guardrail + gated-repo access;
|
||||
# this disables them for a quick local run.
|
||||
model_config={"guardrails": False},
|
||||
)
|
||||
gen = torch.Generator(device="cpu").manual_seed(42)
|
||||
|
||||
# Text-to-image (modalities=["image"]). For T2V use modalities=["video"]
|
||||
# plus num_frames/fps; for I2V add multi_modal_data={"image": <PIL.Image>}.
|
||||
outputs = omni.generate(
|
||||
{
|
||||
"prompt": "A photorealistic red sports car at golden hour, cinematic lighting.",
|
||||
"negative_prompt": "blurry, distorted, low quality",
|
||||
"modalities": ["image"],
|
||||
},
|
||||
OmniDiffusionSamplingParams(
|
||||
height=1024, width=1024, generator=gen,
|
||||
guidance_scale=7.0, num_inference_steps=50, num_outputs_per_prompt=1,
|
||||
),
|
||||
)
|
||||
outputs[0].request_output.images[0].save("cosmos3_t2i.png")
|
||||
omni.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
```
|
||||
|
||||
#### Verification
|
||||
|
||||
```bash
|
||||
python cosmos3_offline.py
|
||||
python -c "from PIL import Image; im=Image.open('cosmos3_t2i.png'); print('image', im.size, im.mode)"
|
||||
```
|
||||
|
||||
#### Notes
|
||||
|
||||
- Same `Cosmos3OmniDiffusersPipeline` as online; mode is chosen by
|
||||
`prompt["modalities"]` (`["image"]` → T2I, `["video"]` → T2V) plus
|
||||
`num_frames`/`fps`, and `multi_modal_data={"image": ...}` for I2V. For video,
|
||||
frames are returned in `outputs[0].request_output.images` as an
|
||||
`(B, F, H, W, 3)` array.
|
||||
- The offline entry must be guarded by `if __name__ == "__main__":` — the engine
|
||||
spawns workers with the `spawn` start method.
|
||||
@@ -1,10 +1,4 @@
|
||||
# Common dependencies for all platforms
|
||||
# transformers: keep aligned with upstream vLLM constraint.
|
||||
# vLLM pins ">= 4.56.0, != 5.0.*, != 5.1.*, != 5.2.*, != 5.3.*, != 5.4.*, != 5.5.0".
|
||||
# 5.9.0 breaks models that define _keys_to_ignore_on_load_unexpected as a list
|
||||
# (e.g. OpenMOSS-Team/MOSS-TTS-Nano) because it changed
|
||||
# _adjust_missing_and_unexpected_keys from list+list to set|set.
|
||||
transformers >= 4.56.0, != 5.0.*, != 5.1.*, != 5.2.*, != 5.3.*, != 5.4.*, != 5.5.0, < 5.9.0
|
||||
av>=14.0.0
|
||||
omegaconf>=2.3.0
|
||||
diffusers==0.38.0
|
||||
|
||||
@@ -30,12 +30,12 @@ from PIL import Image
|
||||
from pytest_mock import MockerFixture
|
||||
from vllm import SamplingParams
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni as RealAsyncOmni
|
||||
from vllm_omni.entrypoints.cli.serve import OmniServeCommand
|
||||
from vllm_omni.inputs.data import OmniSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
@@ -444,7 +444,7 @@ def mock_async_omni(
|
||||
@pytest.fixture
|
||||
def api_server(unused_tcp_port_factory, server_case: ServerCase, mock_async_omni):
|
||||
"""Set up a API server in background process from command line with parametrized model name and mocked AsyncOmni."""
|
||||
parser = TrackingArgumentParser()
|
||||
parser = FlexibleArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest="command")
|
||||
cmd = OmniServeCommand()
|
||||
cmd.subparser_init(subparsers)
|
||||
|
||||
@@ -157,75 +157,6 @@ def test_update_no_multimodal():
|
||||
assert torch.all(slot_states == new_states)
|
||||
|
||||
|
||||
def test_update_uses_precomputed_hidden_states_cpu():
|
||||
"""Precomputed CPU staging should be the only hidden-state cache source."""
|
||||
cache = get_omni_pcache()
|
||||
|
||||
num_tokens_unpadded = 8
|
||||
slot_offset = 8
|
||||
slot_mapping = torch.arange(slot_offset, slot_offset + num_tokens_unpadded)
|
||||
hidden_states = torch.zeros((num_tokens_unpadded, HIDDEN_SIZE), dtype=DTYPE)
|
||||
hidden_states_cpu = torch.rand((num_tokens_unpadded, HIDDEN_SIZE), dtype=DTYPE)
|
||||
|
||||
cache.update_omni_tensor_prefix_cache(
|
||||
hidden_states=hidden_states,
|
||||
hidden_states_cpu=hidden_states_cpu,
|
||||
multimodal_outputs=None,
|
||||
num_tokens_unpadded=num_tokens_unpadded,
|
||||
slot_mapping=slot_mapping,
|
||||
)
|
||||
|
||||
hs_rows = cache.hidden_states_cache.view(NUM_BLOCKS * BLOCK_SIZE, HIDDEN_SIZE)
|
||||
for slot_idx, new_states in zip(slot_mapping, hidden_states_cpu):
|
||||
slot_states = hs_rows[slot_idx]
|
||||
assert torch.all(slot_states == new_states)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"hidden_states_cpu",
|
||||
[
|
||||
torch.rand((4, HIDDEN_SIZE), dtype=DTYPE),
|
||||
torch.rand((8, HIDDEN_SIZE), dtype=OTHER_DTYPE),
|
||||
torch.rand((8, HIDDEN_SIZE + 1), dtype=DTYPE),
|
||||
torch.rand((HIDDEN_SIZE, 8), dtype=DTYPE).t(),
|
||||
],
|
||||
)
|
||||
def test_update_rejects_invalid_hidden_states_cpu(hidden_states_cpu):
|
||||
cache = get_omni_pcache()
|
||||
|
||||
num_tokens_unpadded = 8
|
||||
slot_mapping = torch.arange(num_tokens_unpadded)
|
||||
hidden_states = torch.rand((num_tokens_unpadded, HIDDEN_SIZE), dtype=DTYPE)
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
cache.update_omni_tensor_prefix_cache(
|
||||
hidden_states=hidden_states,
|
||||
hidden_states_cpu=hidden_states_cpu,
|
||||
multimodal_outputs=None,
|
||||
num_tokens_unpadded=num_tokens_unpadded,
|
||||
slot_mapping=slot_mapping,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_update_rejects_gpu_hidden_states_cpu():
|
||||
cache = get_omni_pcache()
|
||||
|
||||
num_tokens_unpadded = 8
|
||||
slot_mapping = torch.arange(num_tokens_unpadded)
|
||||
hidden_states = torch.rand((num_tokens_unpadded, HIDDEN_SIZE), dtype=DTYPE)
|
||||
hidden_states_cpu = torch.rand((num_tokens_unpadded, HIDDEN_SIZE), dtype=DTYPE, device="cuda")
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
cache.update_omni_tensor_prefix_cache(
|
||||
hidden_states=hidden_states,
|
||||
hidden_states_cpu=hidden_states_cpu,
|
||||
multimodal_outputs=None,
|
||||
num_tokens_unpadded=num_tokens_unpadded,
|
||||
slot_mapping=slot_mapping,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"feat_dims",
|
||||
[
|
||||
@@ -428,76 +359,6 @@ def test_get_merged_hidden_states(num_tokens_padded, mocker):
|
||||
assert torch.all(req2_merged_states == req2_new_states)
|
||||
|
||||
|
||||
def test_get_merged_hidden_states_uses_precomputed_hidden_states_cpu(mocker):
|
||||
cache = get_omni_pcache()
|
||||
|
||||
orig_num_tokens_unpadded = 8
|
||||
slot_offset = 8
|
||||
orig_slot_mapping = torch.arange(slot_offset, slot_offset + orig_num_tokens_unpadded)
|
||||
orig_hidden_states = torch.rand((orig_num_tokens_unpadded, HIDDEN_SIZE), dtype=DTYPE)
|
||||
|
||||
cache.update_omni_tensor_prefix_cache(
|
||||
hidden_states=orig_hidden_states,
|
||||
multimodal_outputs=None,
|
||||
num_tokens_unpadded=orig_num_tokens_unpadded,
|
||||
slot_mapping=orig_slot_mapping,
|
||||
)
|
||||
|
||||
num_new_toks_req1 = 3
|
||||
num_new_toks_req2 = 2
|
||||
cache.add_prefix_cached_new_req_id("req1")
|
||||
num_scheduled_tokens = {
|
||||
"req1": num_new_toks_req1,
|
||||
"req2": num_new_toks_req2,
|
||||
}
|
||||
new_hidden_states = torch.zeros(
|
||||
(num_new_toks_req1 + num_new_toks_req2, HIDDEN_SIZE),
|
||||
dtype=DTYPE,
|
||||
)
|
||||
hidden_states_cpu = torch.rand_like(new_hidden_states)
|
||||
req1_new_states = hidden_states_cpu[:num_new_toks_req1]
|
||||
req2_new_states = hidden_states_cpu[-num_new_toks_req2:]
|
||||
input_batch = MockInputBatch(num_computed_tokens_cpu=torch.Tensor([orig_num_tokens_unpadded, 0]))
|
||||
|
||||
mocker.patch(
|
||||
"vllm_omni.core.prefix_cache.OmniTensorPrefixCache._get_cached_block_ids",
|
||||
new=fake_get_cached_block_ids,
|
||||
)
|
||||
merged_states = cache.get_merged_hidden_states(
|
||||
query_start_loc=[0, num_new_toks_req1],
|
||||
input_batch=input_batch,
|
||||
hidden_states=new_hidden_states,
|
||||
hidden_states_cpu=hidden_states_cpu,
|
||||
num_scheduled_tokens=num_scheduled_tokens,
|
||||
)
|
||||
|
||||
assert torch.all(merged_states["req1"][:orig_num_tokens_unpadded] == orig_hidden_states)
|
||||
assert torch.all(merged_states["req1"][-num_new_toks_req1:] == req1_new_states)
|
||||
assert torch.all(merged_states["req2"] == req2_new_states)
|
||||
|
||||
|
||||
def test_get_merged_hidden_states_rejects_short_hidden_states_cpu(mocker):
|
||||
cache = get_omni_pcache()
|
||||
num_scheduled_tokens = {
|
||||
"req1": 3,
|
||||
"req2": 2,
|
||||
}
|
||||
input_batch = MockInputBatch(num_computed_tokens_cpu=torch.Tensor([0, 0]))
|
||||
|
||||
mocker.patch(
|
||||
"vllm_omni.core.prefix_cache.OmniTensorPrefixCache._get_cached_block_ids",
|
||||
new=fake_get_cached_block_ids,
|
||||
)
|
||||
with pytest.raises(RuntimeError):
|
||||
cache.get_merged_hidden_states(
|
||||
query_start_loc=[0, 3],
|
||||
input_batch=input_batch,
|
||||
hidden_states=torch.rand((5, HIDDEN_SIZE), dtype=DTYPE),
|
||||
hidden_states_cpu=torch.rand((4, HIDDEN_SIZE), dtype=DTYPE),
|
||||
num_scheduled_tokens=num_scheduled_tokens,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens_padded", [None, 16])
|
||||
@pytest.mark.parametrize(
|
||||
"feat_dims",
|
||||
|
||||
@@ -223,12 +223,8 @@ def _get_open_port() -> int:
|
||||
return s.getsockname()[1]
|
||||
|
||||
|
||||
def _wait_for_port(host: str, port: int, timeout: int = 1200, proc: subprocess.Popen | None = None) -> None:
|
||||
"""Block until the given host:port accepts connections or timeout expires.
|
||||
|
||||
If *proc* is provided, also monitors the process; raises RuntimeError
|
||||
immediately if the server process exits before the port becomes available.
|
||||
"""
|
||||
def _wait_for_port(host: str, port: int, timeout: int = 1200) -> None:
|
||||
"""Block until the given host:port accepts connections or timeout expires."""
|
||||
start = time.time()
|
||||
while time.time() - start < timeout:
|
||||
try:
|
||||
@@ -238,10 +234,6 @@ def _wait_for_port(host: str, port: int, timeout: int = 1200, proc: subprocess.P
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
if proc is not None:
|
||||
ret = proc.poll()
|
||||
if ret is not None:
|
||||
raise RuntimeError(f"Server process exited with code {ret} before port {host}:{port} became ready")
|
||||
time.sleep(2)
|
||||
raise RuntimeError(f"Server did not start on {host}:{port} within {timeout}s")
|
||||
|
||||
@@ -339,7 +331,7 @@ class DiffusionServer:
|
||||
env=env,
|
||||
cwd=str(Path(__file__).parent.parent.parent.parent),
|
||||
)
|
||||
_wait_for_port(self.host, self.port, proc=self.proc)
|
||||
_wait_for_port(self.host, self.port)
|
||||
print(f"DiffusionServer ready on {self.host}:{self.port}")
|
||||
|
||||
def __enter__(self):
|
||||
|
||||
@@ -5,8 +5,7 @@
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"use_omni": false,
|
||||
"extra_cli_args": [
|
||||
"--no-enable-prefix-caching",
|
||||
"--skip-mm-profiling"
|
||||
"--no-enable-prefix-caching"
|
||||
]
|
||||
},
|
||||
"benchmark_params": [
|
||||
|
||||
-19
@@ -25,7 +25,6 @@ SEPARATE_CFG_ENABLERS = [
|
||||
cd_backend.enable_cache_for_helios,
|
||||
cd_backend.enable_cache_for_wan22,
|
||||
cd_backend.enable_cache_for_longcat_image,
|
||||
cd_backend.enable_cache_for_cosmos3,
|
||||
]
|
||||
|
||||
SAMPLE_CACHE_CONFIG = DiffusionCacheConfig()
|
||||
@@ -48,24 +47,6 @@ def test_separate_cfg(mock_cache_dit, mock_block_adapter, enabler):
|
||||
assert adapter_kwargs["has_separate_cfg"] is True
|
||||
|
||||
|
||||
@patch("vllm_omni.diffusion.cache.cache_dit_backend.BlockAdapter")
|
||||
@patch("vllm_omni.diffusion.cache.cache_dit_backend.cache_dit")
|
||||
def test_cosmos3_cache_dit_wraps_gen_layers(mock_cache_dit, mock_block_adapter):
|
||||
"""Cosmos3 should cache only the repeated GEN pathway blocks."""
|
||||
mock_pipeline = Mock()
|
||||
gen_layers = object()
|
||||
mock_pipeline.transformer.gen_layers = gen_layers
|
||||
|
||||
cd_backend.enable_cache_for_cosmos3(mock_pipeline, SAMPLE_CACHE_CONFIG)
|
||||
|
||||
mock_cache_dit.enable_cache.assert_called_once()
|
||||
adapter_kwargs = mock_block_adapter.call_args.kwargs
|
||||
assert adapter_kwargs["transformer"] is mock_pipeline.transformer
|
||||
assert adapter_kwargs["blocks"] == [gen_layers]
|
||||
assert adapter_kwargs["has_separate_cfg"] is True
|
||||
assert adapter_kwargs["check_forward_pattern"] is False
|
||||
|
||||
|
||||
# This test is skipped on ROCm since rocm_unquantized_gemm doesn't support CPU backend
|
||||
@pytest.mark.skipif(
|
||||
current_omni_platform.is_rocm(),
|
||||
|
||||
@@ -19,9 +19,8 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
NUM_TOKENS = 8
|
||||
HIDDEN_DIM = 16
|
||||
NUM_TIMESTEPS = 5
|
||||
# generate_image samples (num_timesteps + 1) points then drops the last, matching
|
||||
# upstream lance: see Bagel.generate_image in bagel_transformer.py.
|
||||
EXPECTED_STEPS = NUM_TIMESTEPS
|
||||
# generate_image uses timesteps[:-1], so actual steps = NUM_TIMESTEPS - 1
|
||||
EXPECTED_STEPS = NUM_TIMESTEPS - 1
|
||||
|
||||
|
||||
def _make_mock_bagel(mocker: MockerFixture):
|
||||
@@ -179,7 +178,7 @@ class TestTrajectoryRecording:
|
||||
bagel, args = bagel_and_args
|
||||
|
||||
# Recompute the expected timestep schedule (mirrors generate_image logic)
|
||||
ts = torch.linspace(1, 0, args["num_timesteps"] + 1)
|
||||
ts = torch.linspace(1, 0, args["num_timesteps"])
|
||||
shift = args.get("timestep_shift", 1.0)
|
||||
ts = shift * ts / (1 + (shift - 1) * ts)
|
||||
expected_timesteps = ts[:-1] # last element is dropped
|
||||
@@ -197,7 +196,7 @@ class TestTrajectoryRecording:
|
||||
bagel, args = bagel_and_args
|
||||
|
||||
# Compute the number of denoising steps the same way generate_image does
|
||||
ts = torch.linspace(1, 0, args["num_timesteps"] + 1)
|
||||
ts = torch.linspace(1, 0, args["num_timesteps"])
|
||||
shift = args.get("timestep_shift", 1.0)
|
||||
ts = shift * ts / (1 + (shift - 1) * ts)
|
||||
num_steps = len(ts) - 1 # timesteps = ts[:-1]
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,482 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import types
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from torch import nn
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
|
||||
class StubScheduler:
|
||||
def __init__(self, timesteps: list[int] | None = None, *, flow_shift: float = 1.0) -> None:
|
||||
self.timesteps = torch.tensor(timesteps or [9, 3], dtype=torch.int64)
|
||||
self.config = SimpleNamespace(num_train_timesteps=1000, flow_shift=flow_shift)
|
||||
self.set_timesteps_calls: list[tuple[int, torch.device]] = []
|
||||
self.step_calls: list[tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = []
|
||||
|
||||
def set_timesteps(self, num_steps: int, device: torch.device) -> None:
|
||||
self.set_timesteps_calls.append((num_steps, device))
|
||||
self.timesteps = torch.arange(num_steps, 0, -1, dtype=torch.int64, device=device)
|
||||
|
||||
def step(self, noise_pred: torch.Tensor, timestep: torch.Tensor, latents: torch.Tensor, **kwargs):
|
||||
del kwargs
|
||||
self.step_calls.append((noise_pred.clone(), timestep.clone(), latents.clone()))
|
||||
return (latents + noise_pred,)
|
||||
|
||||
|
||||
class _ModeLatentDist:
|
||||
def __init__(self, latents: torch.Tensor) -> None:
|
||||
self._latents = latents
|
||||
|
||||
def mode(self) -> torch.Tensor:
|
||||
return self._latents
|
||||
|
||||
|
||||
class StubCosmos3VAE:
|
||||
dtype = torch.float32
|
||||
|
||||
def __init__(self, z_dim: int = 2, *, temporal: int = 4, spatial: int = 8) -> None:
|
||||
self.config = SimpleNamespace(
|
||||
z_dim=z_dim,
|
||||
scale_factor_temporal=temporal,
|
||||
scale_factor_spatial=spatial,
|
||||
latents_mean=[0.0] * z_dim,
|
||||
latents_std=[1.0] * z_dim,
|
||||
)
|
||||
|
||||
def encode(self, video: torch.Tensor):
|
||||
latent_frames = (video.shape[2] - 1) // self.config.scale_factor_temporal + 1
|
||||
latent_height = video.shape[-2] // self.config.scale_factor_spatial
|
||||
latent_width = video.shape[-1] // self.config.scale_factor_spatial
|
||||
latents = torch.ones(
|
||||
video.shape[0],
|
||||
self.config.z_dim,
|
||||
latent_frames,
|
||||
latent_height,
|
||||
latent_width,
|
||||
dtype=video.dtype,
|
||||
device=video.device,
|
||||
)
|
||||
return SimpleNamespace(latent_dist=_ModeLatentDist(latents))
|
||||
|
||||
def decode(self, latents: torch.Tensor, return_dict: bool = False):
|
||||
del return_dict
|
||||
return (latents,)
|
||||
|
||||
|
||||
class StubCosmos3Transformer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
latent_channel_size: int = 2,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.latent_channel_size = latent_channel_size
|
||||
self.cached_kv: Any | None = None
|
||||
self.cached_freqs_gen: Any | None = None
|
||||
self.calls: list[dict[str, Any]] = []
|
||||
self.reset_calls = 0
|
||||
|
||||
def reset_cache(self) -> None:
|
||||
self.reset_calls += 1
|
||||
self.cached_kv = None
|
||||
self.cached_freqs_gen = None
|
||||
|
||||
def forward(
|
||||
self,
|
||||
*,
|
||||
hidden_states: torch.Tensor,
|
||||
timestep: torch.Tensor,
|
||||
text_ids: torch.Tensor,
|
||||
text_mask: torch.Tensor,
|
||||
**kwargs: Any,
|
||||
) -> torch.Tensor:
|
||||
token = int(text_ids.reshape(-1)[0].item()) if text_ids.numel() else 0
|
||||
self.calls.append(
|
||||
{
|
||||
"token": token,
|
||||
"timestep": timestep.clone(),
|
||||
"text_mask": text_mask.clone(),
|
||||
"cache_before": self.cached_kv,
|
||||
"kwargs": dict(kwargs),
|
||||
}
|
||||
)
|
||||
if self.cached_kv is None:
|
||||
marker = torch.tensor([token], dtype=torch.float32)
|
||||
self.cached_kv = [(marker, marker + 100)]
|
||||
self.cached_freqs_gen = (marker + 200, marker + 300)
|
||||
return torch.full_like(hidden_states, float(token))
|
||||
|
||||
|
||||
def passthrough_progress_bar(iterable):
|
||||
return iterable
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def fake_cosmos3_guardrails(monkeypatch: pytest.MonkeyPatch):
|
||||
module = types.ModuleType("vllm_omni.diffusion.models.cosmos3.guardrails")
|
||||
module.is_guardrails_enabled = lambda od_config, sampling_params=None: False
|
||||
module.ensure_initialized = lambda od_config: None
|
||||
module.check_text_safety = lambda text: None
|
||||
module.check_video_safety = lambda video: video
|
||||
monkeypatch.setitem(sys.modules, module.__name__, module)
|
||||
return module
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def make_cosmos3_pipeline():
|
||||
def _make():
|
||||
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import (
|
||||
Cosmos3OmniDiffusersPipeline,
|
||||
)
|
||||
|
||||
pipeline = object.__new__(Cosmos3OmniDiffusersPipeline)
|
||||
nn.Module.__init__(pipeline)
|
||||
pipeline.od_config = SimpleNamespace()
|
||||
pipeline.device = torch.device("cpu")
|
||||
pipeline.dtype = torch.float32
|
||||
pipeline.transformer = StubCosmos3Transformer(latent_channel_size=2)
|
||||
pipeline.vae = StubCosmos3VAE(z_dim=2)
|
||||
pipeline.vae_scale_factor_temporal = 4
|
||||
pipeline.vae_scale_factor_spatial = 8
|
||||
pipeline.scheduler = StubScheduler([9, 3], flow_shift=1.0)
|
||||
pipeline._base_scheduler_config = pipeline.scheduler.config
|
||||
pipeline._engine_init_flow_shift = 1.0
|
||||
pipeline._current_flow_shift = 1.0
|
||||
pipeline._guidance_scale = None
|
||||
pipeline._num_timesteps = None
|
||||
pipeline._cache_dit_requires_paired_cfg = False
|
||||
pipeline.progress_bar = passthrough_progress_bar
|
||||
return pipeline
|
||||
|
||||
return _make
|
||||
|
||||
|
||||
def make_sampling_params(**overrides: Any) -> SimpleNamespace:
|
||||
values = {
|
||||
"height": None,
|
||||
"width": None,
|
||||
"num_frames": None,
|
||||
"num_inference_steps": None,
|
||||
"guidance_scale": None,
|
||||
"generator": None,
|
||||
"seed": 123,
|
||||
"num_outputs_per_prompt": 1,
|
||||
"frame_rate": None,
|
||||
"resolved_frame_rate": None,
|
||||
"max_sequence_length": None,
|
||||
"extra_args": {},
|
||||
}
|
||||
values.update(overrides)
|
||||
return SimpleNamespace(**values)
|
||||
|
||||
|
||||
def _ids(value: int) -> torch.Tensor:
|
||||
return torch.tensor([[value]], dtype=torch.long)
|
||||
|
||||
|
||||
def _mask() -> torch.Tensor:
|
||||
return torch.ones(1, 1, dtype=torch.long)
|
||||
|
||||
|
||||
def test_pipeline_registered_and_exported() -> None:
|
||||
from vllm_omni.diffusion.cache.cache_dit_backend import CUSTOM_DIT_ENABLERS
|
||||
from vllm_omni.diffusion.models import cosmos3
|
||||
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import Cosmos3OmniDiffusersPipeline
|
||||
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
|
||||
from vllm_omni.diffusion.registry import (
|
||||
_DIFFUSION_MODELS,
|
||||
_DIFFUSION_POST_PROCESS_FUNCS,
|
||||
_DIFFUSION_PRE_PROCESS_FUNCS,
|
||||
)
|
||||
|
||||
assert issubclass(Cosmos3OmniDiffusersPipeline, nn.Module)
|
||||
assert issubclass(Cosmos3OmniDiffusersPipeline, ProgressBarMixin)
|
||||
assert Cosmos3OmniDiffusersPipeline.support_image_input is True
|
||||
assert _DIFFUSION_MODELS["Cosmos3OmniDiffusersPipeline"] == (
|
||||
"cosmos3",
|
||||
"pipeline_cosmos3",
|
||||
"Cosmos3OmniDiffusersPipeline",
|
||||
)
|
||||
assert _DIFFUSION_PRE_PROCESS_FUNCS["Cosmos3OmniDiffusersPipeline"] == "get_cosmos3_pre_process_func"
|
||||
assert _DIFFUSION_POST_PROCESS_FUNCS["Cosmos3OmniDiffusersPipeline"] == "get_cosmos3_post_process_func"
|
||||
assert "Cosmos3OmniDiffusersPipeline" in CUSTOM_DIT_ENABLERS
|
||||
assert "Cosmos3OmniDiffusersPipeline" in cosmos3.__all__
|
||||
|
||||
|
||||
def test_preprocess_i2v_image_input() -> None:
|
||||
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import get_cosmos3_pre_process_func
|
||||
|
||||
preprocess = get_cosmos3_pre_process_func(SimpleNamespace())
|
||||
i2v = SimpleNamespace(
|
||||
prompts=[{"prompt": "A slow camera push.", "multi_modal_data": {"image": Image.new("RGB", (320, 160))}}],
|
||||
sampling_params=SimpleNamespace(height=None, width=None, extra_args={}),
|
||||
)
|
||||
|
||||
result = preprocess(i2v)
|
||||
assert (result.sampling_params.height, result.sampling_params.width) == (672, 1344)
|
||||
assert tuple(result.prompts[0]["additional_information"]["preprocessed_image"].shape[-2:]) == (672, 1344)
|
||||
|
||||
|
||||
def test_postprocess_handles_image_video_and_validation() -> None:
|
||||
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import get_cosmos3_post_process_func
|
||||
|
||||
func = get_cosmos3_post_process_func(SimpleNamespace())
|
||||
video = torch.zeros(1, 3, 1, 4, 4)
|
||||
|
||||
assert func(video, output_type="latent") is video
|
||||
assert func({"image": video})[0].size == (4, 4)
|
||||
|
||||
with pytest.raises(ValueError, match="text-to-image postprocess expects"):
|
||||
func({"image": torch.zeros(1, 3, 2, 4, 4)})
|
||||
with pytest.raises(ValueError, match="both image and video"):
|
||||
func({"image": video, "video": video})
|
||||
|
||||
|
||||
def test_prompt_formatting_and_checkpoint_key_remap(make_cosmos3_pipeline) -> None:
|
||||
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import Cosmos3OmniDiffusersPipeline
|
||||
|
||||
pipeline = make_cosmos3_pipeline()
|
||||
captured: list[str] = []
|
||||
pipeline._tokenize_prompt = lambda text, *args, **kwargs: (captured.append(text) or _ids(len(captured)), _mask())
|
||||
|
||||
pipeline._format_and_tokenize_prompts(
|
||||
"A robot",
|
||||
"bad",
|
||||
num_frames=48,
|
||||
frame_rate=24,
|
||||
height=720,
|
||||
width=1280,
|
||||
max_sequence_length=32,
|
||||
sp=SimpleNamespace(
|
||||
extra_args={
|
||||
"negative_metadata_mode": "inverse",
|
||||
# Duration/resolution metadata templates are off by default
|
||||
# (commit "Change resolution and duration templates to off by
|
||||
# default"); enable them explicitly to exercise the formatting.
|
||||
"use_duration_template": True,
|
||||
"use_resolution_template": True,
|
||||
}
|
||||
),
|
||||
use_system_prompt=True,
|
||||
is_t2i=False,
|
||||
)
|
||||
assert "The video is 2.0 seconds long" in captured[0]
|
||||
assert "The video is not 2.0 seconds long" in captured[1]
|
||||
|
||||
remaps = {
|
||||
"embed_tokens.weight": "transformer.language_model.embed_tokens.weight",
|
||||
"model.embed_tokens.weight": "transformer.language_model.embed_tokens.weight",
|
||||
"norm.weight": "transformer.language_model.norm.weight",
|
||||
"norm_moe_gen.weight": "transformer.norm_moe_gen.weight",
|
||||
"proj_in.weight": "transformer.proj_in.weight",
|
||||
"proj_out.bias": "transformer.proj_out.bias",
|
||||
"layers.3.self_attn.to_q.weight": "transformer.language_model.layers.3.self_attn.to_q.weight",
|
||||
"layers.3.self_attn.to_out.weight": "transformer.language_model.layers.3.self_attn.to_out.weight",
|
||||
"layers.3.self_attn.norm_q.weight": "transformer.language_model.layers.3.self_attn.norm_q.weight",
|
||||
"layers.3.self_attn.add_q_proj.weight": "transformer.gen_layers.3.cross_attention.to_q.weight",
|
||||
"layers.3.self_attn.to_add_out.weight": "transformer.gen_layers.3.cross_attention.to_out.weight",
|
||||
"layers.3.self_attn.norm_added_q.weight": "transformer.gen_layers.3.cross_attention.norm_q.weight",
|
||||
"transformer.model.layers.3.self_attn.add_k_proj.weight": (
|
||||
"transformer.gen_layers.3.cross_attention.to_k.weight"
|
||||
),
|
||||
}
|
||||
assert {key: Cosmos3OmniDiffusersPipeline._remap_ckpt_key(key) for key in remaps} == remaps
|
||||
|
||||
|
||||
def test_prepare_latents_for_video_and_image(make_cosmos3_pipeline) -> None:
|
||||
pipeline = make_cosmos3_pipeline()
|
||||
latents = pipeline._prepare_latents(16, 24, 5, torch.Generator(device="cpu").manual_seed(0))
|
||||
assert latents.shape == (1, 2, 2, 2, 3)
|
||||
|
||||
pipeline._encode_conditioning_video = lambda *args, **kwargs: torch.full((1, 2, 2, 2, 3), 5.0)
|
||||
i2v_latents, velocity_mask, image_latent = pipeline._prepare_latents_i2v(
|
||||
torch.zeros(1, 3, 16, 24), 16, 24, 5, torch.Generator(device="cpu").manual_seed(0)
|
||||
)
|
||||
torch.testing.assert_close(i2v_latents[:, :, 0], torch.full((1, 2, 2, 3), 5.0))
|
||||
assert velocity_mask.tolist() == [[[[[0.0]], [[1.0]]]]]
|
||||
assert image_latent.shape == (1, 2, 1, 2, 3)
|
||||
|
||||
|
||||
def test_diffuse_covers_cfg_and_i2v_steps(make_cosmos3_pipeline) -> None:
|
||||
pipeline = make_cosmos3_pipeline()
|
||||
latents = torch.zeros(1, 2, 1, 1, 1)
|
||||
|
||||
result = pipeline.diffuse(
|
||||
latents=latents,
|
||||
timesteps=torch.tensor([900, 100]),
|
||||
cond_ids=_ids(2),
|
||||
cond_mask=_mask(),
|
||||
uncond_ids=_ids(1),
|
||||
uncond_mask=_mask(),
|
||||
guidance_scale=3.0,
|
||||
shared_kwargs={"video_shape": (1, 1, 1), "fps": 24.0},
|
||||
guidance_interval=(500.0, 1000.0),
|
||||
)
|
||||
assert [call["token"] for call in pipeline.transformer.calls] == [2, 1, 2]
|
||||
torch.testing.assert_close(result, torch.full_like(latents, 6.0))
|
||||
|
||||
i2v = pipeline.diffuse(
|
||||
latents=torch.zeros(1, 2, 2, 1, 1),
|
||||
timesteps=torch.tensor([7]),
|
||||
cond_ids=_ids(2),
|
||||
cond_mask=_mask(),
|
||||
uncond_ids=_ids(1),
|
||||
uncond_mask=_mask(),
|
||||
guidance_scale=1.0,
|
||||
shared_kwargs={"video_shape": (2, 1, 1), "fps": 24.0},
|
||||
velocity_mask=torch.tensor([[[[[0.0]], [[1.0]]]]]),
|
||||
image_latent=torch.full((1, 2, 1, 1, 1), 7.0),
|
||||
)
|
||||
torch.testing.assert_close(i2v[:, :, 0:1], torch.full((1, 2, 1, 1, 1), 7.0))
|
||||
|
||||
|
||||
def test_diffuse_keeps_paired_cfg_when_cache_dit_active(make_cosmos3_pipeline) -> None:
|
||||
"""With cache-dit active the uncond pass is kept even outside the guidance
|
||||
interval (so cache-dit's has_separate_cfg parity stays in phase), and the
|
||||
output is numerically identical to the skip path.
|
||||
|
||||
Contrast with ``test_diffuse_covers_cfg_and_i2v_steps`` (no marker), where
|
||||
the same inputs skip the out-of-interval uncond pass: calls == [2, 1, 2].
|
||||
"""
|
||||
pipeline = make_cosmos3_pipeline()
|
||||
# Marker normally set by ``enable_cache_for_cosmos3`` when cache-dit is on.
|
||||
pipeline._cache_dit_requires_paired_cfg = True
|
||||
latents = torch.zeros(1, 2, 1, 1, 1)
|
||||
|
||||
result = pipeline.diffuse(
|
||||
latents=latents,
|
||||
timesteps=torch.tensor([900, 100]),
|
||||
cond_ids=_ids(2),
|
||||
cond_mask=_mask(),
|
||||
uncond_ids=_ids(1),
|
||||
uncond_mask=_mask(),
|
||||
guidance_scale=3.0,
|
||||
shared_kwargs={"video_shape": (1, 1, 1), "fps": 24.0},
|
||||
guidance_interval=(500.0, 1000.0),
|
||||
)
|
||||
|
||||
# t=900 is inside the interval (cond+uncond); t=100 is outside but the
|
||||
# uncond pass is still issued -> paired cond/uncond at every step.
|
||||
assert [call["token"] for call in pipeline.transformer.calls] == [2, 1, 2, 1]
|
||||
# Identical result to the skip path: out-of-interval combine uses scale=1.0,
|
||||
# so combine_cfg_noise(cond=2, uncond=1, 1.0) == 2 == the skipped cond value.
|
||||
torch.testing.assert_close(result, torch.full_like(latents, 6.0))
|
||||
|
||||
|
||||
class TestForwardRouting:
|
||||
def _install_forward_stubs(self, pipeline):
|
||||
captured: dict[str, object] = {"diffuse_calls": [], "prepare_calls": []}
|
||||
|
||||
def fake_format(prompt, negative_prompt, num_frames, frame_rate, height, width, *args, **kwargs):
|
||||
captured["format"] = {
|
||||
"prompt": prompt,
|
||||
"negative_prompt": negative_prompt,
|
||||
"num_frames": num_frames,
|
||||
"frame_rate": frame_rate,
|
||||
"height": height,
|
||||
"width": width,
|
||||
"is_t2i": kwargs["is_t2i"],
|
||||
}
|
||||
return _ids(2), _mask(), _ids(1), _mask()
|
||||
|
||||
def fake_prepare(height, width, num_frames, generator):
|
||||
captured["prepare_calls"].append((height, width, num_frames, generator.initial_seed()))
|
||||
return torch.zeros(1, 2, 1, 1, 1)
|
||||
|
||||
def fake_diffuse(**kwargs):
|
||||
captured["diffuse_calls"].append(kwargs)
|
||||
return kwargs["latents"] + len(captured["diffuse_calls"])
|
||||
|
||||
pipeline._format_and_tokenize_prompts = fake_format
|
||||
pipeline._prepare_latents = fake_prepare
|
||||
pipeline._set_flow_shift = lambda target: captured.setdefault("flow_shifts", []).append(target)
|
||||
pipeline.diffuse = fake_diffuse
|
||||
pipeline._decode_latents = lambda latents: latents
|
||||
return captured
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("prompt", "sampling_params", "expected"),
|
||||
[
|
||||
(
|
||||
{"prompt": "A painted robot", "modalities": ["image"]},
|
||||
make_sampling_params(num_outputs_per_prompt=2),
|
||||
{"key": "image", "is_t2i": True, "flow": [3.0], "steps": [50, 50], "frames": 1},
|
||||
),
|
||||
(
|
||||
"A warehouse robot",
|
||||
make_sampling_params(),
|
||||
{"key": "video", "is_t2i": False, "flow": [1.0], "steps": [35], "frames": 189},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_forward_defaults_and_mode_selection(
|
||||
self,
|
||||
make_cosmos3_pipeline,
|
||||
prompt,
|
||||
sampling_params,
|
||||
expected,
|
||||
) -> None:
|
||||
pipeline = make_cosmos3_pipeline()
|
||||
captured = self._install_forward_stubs(pipeline)
|
||||
|
||||
output = pipeline.forward(SimpleNamespace(prompts=[prompt], sampling_params=sampling_params))
|
||||
|
||||
assert expected["key"] in output.output
|
||||
assert captured["format"]["is_t2i"] is expected["is_t2i"]
|
||||
assert captured["format"]["num_frames"] == expected["frames"]
|
||||
assert captured["flow_shifts"] == expected["flow"]
|
||||
assert [call[0] for call in pipeline.scheduler.set_timesteps_calls] == expected["steps"]
|
||||
|
||||
def test_forward_i2v_route(self, make_cosmos3_pipeline) -> None:
|
||||
pipeline = make_cosmos3_pipeline()
|
||||
captured = self._install_forward_stubs(pipeline)
|
||||
image_tensor = torch.zeros(1, 3, 16, 16)
|
||||
velocity_mask = torch.ones(1, 1, 1, 1, 1)
|
||||
|
||||
pipeline._prepare_latents_i2v = lambda *args, **kwargs: (
|
||||
torch.zeros(1, 2, 1, 1, 1),
|
||||
velocity_mask,
|
||||
torch.zeros(1, 2, 1, 1, 1),
|
||||
)
|
||||
pipeline.forward(
|
||||
SimpleNamespace(
|
||||
prompts=[
|
||||
{
|
||||
"prompt": "move",
|
||||
"modalities": ["video"],
|
||||
"additional_information": {"preprocessed_image": image_tensor},
|
||||
}
|
||||
],
|
||||
sampling_params=make_sampling_params(height=16, width=16, num_frames=5),
|
||||
)
|
||||
)
|
||||
assert captured["diffuse_calls"][-1]["shared_kwargs"]["noisy_frame_mask"] is velocity_mask
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("prompt", "sampling_params", "message"),
|
||||
[
|
||||
(["one", "two"], make_sampling_params(), "single prompt"),
|
||||
([{"prompt": "one", "modalities": ["image", "video"]}], make_sampling_params(), "both image and video"),
|
||||
],
|
||||
)
|
||||
def test_forward_rejects_invalid_public_requests(
|
||||
self,
|
||||
make_cosmos3_pipeline,
|
||||
prompt,
|
||||
sampling_params,
|
||||
message,
|
||||
) -> None:
|
||||
pipeline = make_cosmos3_pipeline()
|
||||
|
||||
with pytest.raises(ValueError, match=message):
|
||||
pipeline.forward(SimpleNamespace(prompts=prompt, sampling_params=sampling_params))
|
||||
@@ -1,158 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
|
||||
def _tiny_cosmos3_config(**overrides):
|
||||
config = {
|
||||
"hidden_size": 8,
|
||||
"num_hidden_layers": 0,
|
||||
"num_attention_heads": 2,
|
||||
"num_key_value_heads": 2,
|
||||
"head_dim": 4,
|
||||
"intermediate_size": 16,
|
||||
"vocab_size": 32,
|
||||
"latent_patch_size": 1,
|
||||
"latent_channel": 2,
|
||||
"rope_scaling": {"mrope_section": [1, 1, 0]},
|
||||
}
|
||||
config.update(overrides)
|
||||
return config
|
||||
|
||||
|
||||
def test_mrope_position_ids_cover_text_and_video() -> None:
|
||||
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import (
|
||||
compute_mrope_position_ids_text,
|
||||
compute_mrope_position_ids_vision,
|
||||
)
|
||||
|
||||
text_ids, text_offset = compute_mrope_position_ids_text(num_tokens=3, temporal_offset=5)
|
||||
assert text_ids.tolist() == [[5, 6, 7], [5, 6, 7], [5, 6, 7]]
|
||||
assert text_offset == 8
|
||||
|
||||
vision_ids, vision_offset = compute_mrope_position_ids_vision(2, 2, 3, temporal_offset=10, fps=None)
|
||||
assert vision_ids.shape == (3, 12)
|
||||
assert vision_ids[0].tolist() == [10] * 6 + [11] * 6
|
||||
assert vision_offset == 12
|
||||
|
||||
modulated_ids, modulated_offset = compute_mrope_position_ids_vision(
|
||||
2,
|
||||
1,
|
||||
1,
|
||||
temporal_offset=10,
|
||||
fps=12.0,
|
||||
base_fps=24.0,
|
||||
temporal_compression_factor=4,
|
||||
)
|
||||
torch.testing.assert_close(modulated_ids[0], torch.tensor([10.0, 12.0]))
|
||||
assert modulated_offset == 13
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("key", "value"),
|
||||
[
|
||||
("qk_norm_for_diffusion", False),
|
||||
("qk_norm_for_text", False),
|
||||
("position_embedding_type", "rotary"),
|
||||
("unified_3d_mrope_reset_spatial_ids", False),
|
||||
("joint_attn_implementation", "one_way"),
|
||||
],
|
||||
)
|
||||
def test_validate_supported_config_rejects_unsupported_flags(key: str, value) -> None:
|
||||
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3VFMTransformer
|
||||
|
||||
with pytest.raises(ValueError, match=f"{key}="):
|
||||
Cosmos3VFMTransformer._validate_supported_config({key: value})
|
||||
Cosmos3VFMTransformer._validate_supported_config({})
|
||||
Cosmos3VFMTransformer._validate_supported_config(None)
|
||||
|
||||
|
||||
def test_transformer_sharding_offload_and_patch_round_trip_contracts() -> None:
|
||||
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3VFMTransformer
|
||||
|
||||
model = object.__new__(Cosmos3VFMTransformer)
|
||||
nn.Module.__init__(model)
|
||||
model.language_model = nn.Module()
|
||||
model.language_model.layers = nn.ModuleList([nn.Linear(2, 2) for _ in range(2)])
|
||||
model.gen_layers = nn.ModuleList([nn.Linear(2, 2)])
|
||||
model.norm_moe_gen = nn.LayerNorm(2)
|
||||
|
||||
matched = [
|
||||
name
|
||||
for name, module in model.named_modules()
|
||||
if any(condition(name, module) for condition in model._hsdp_shard_conditions)
|
||||
]
|
||||
assert matched == ["language_model.layers.0", "language_model.layers.1", "gen_layers.0"]
|
||||
assert Cosmos3VFMTransformer._layerwise_offload_blocks_attrs == ["gen_layers"]
|
||||
assert Cosmos3VFMTransformer._repeated_blocks == ["Cosmos3GenDecoderLayer"]
|
||||
|
||||
model.latent_patch_size = 2
|
||||
model.latent_channel_size = 3
|
||||
latents = torch.arange(1 * 3 * 1 * 3 * 5, dtype=torch.float32).reshape(1, 3, 1, 3, 5)
|
||||
torch.testing.assert_close(model.unpatchify(model.patchify(latents, t=1, h=3, w=5), t=1, h=3, w=5), latents)
|
||||
|
||||
|
||||
def test_forward_returns_video_prediction(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from vllm_omni.diffusion.models.cosmos3 import transformer_cosmos3
|
||||
|
||||
monkeypatch.setattr(transformer_cosmos3, "_get_ulysses_state", lambda: (1, 0, None))
|
||||
|
||||
output = transformer_cosmos3.Cosmos3VFMTransformer(
|
||||
SimpleNamespace(tf_model_config=_tiny_cosmos3_config(), dtype=torch.float32)
|
||||
)(
|
||||
hidden_states=torch.zeros(1, 2, 1, 2, 2),
|
||||
timestep=torch.tensor([1.0]),
|
||||
text_ids=torch.tensor([[1, 2]], dtype=torch.long),
|
||||
text_mask=torch.ones(1, 2, dtype=torch.long),
|
||||
video_shape=(1, 2, 2),
|
||||
fps=24.0,
|
||||
)
|
||||
|
||||
assert tuple(output.shape) == (1, 2, 1, 2, 2)
|
||||
|
||||
|
||||
def test_compute_rope_freqs_places_text_and_video_positions() -> None:
|
||||
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3VFMTransformer
|
||||
|
||||
class FakeRotary:
|
||||
def __init__(self) -> None:
|
||||
self.position_ids: list[torch.Tensor] = []
|
||||
|
||||
def __call__(self, x, position_ids):
|
||||
del x
|
||||
self.position_ids.append(position_ids.detach().cpu())
|
||||
batch, seq = position_ids.shape[1], position_ids.shape[2]
|
||||
return torch.zeros(batch, seq, 4), torch.ones(batch, seq, 4)
|
||||
|
||||
rotary = FakeRotary()
|
||||
model = object.__new__(Cosmos3VFMTransformer)
|
||||
nn.Module.__init__(model)
|
||||
model.language_model = SimpleNamespace(rotary_emb=rotary)
|
||||
model.temporal_modality_margin = 100
|
||||
model.base_fps = 24.0
|
||||
model.temporal_compression_factor = 4
|
||||
model.enable_fps_modulation = False
|
||||
|
||||
freqs_und, freqs_gen = model._compute_rope_freqs(
|
||||
text_mask=torch.tensor([[1, 1, 0], [1, 0, 0]], dtype=torch.long),
|
||||
t=2,
|
||||
hp=1,
|
||||
wp=1,
|
||||
fps=24.0,
|
||||
device=torch.device("cpu"),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
text_pos, vision_pos = rotary.position_ids
|
||||
assert text_pos[:, 0, :].tolist() == [[0, 1, 0], [0, 1, 0], [0, 1, 0]]
|
||||
assert vision_pos[0, 0].tolist() == [102, 103]
|
||||
assert freqs_und[0].shape == (2, 3, 1, 4)
|
||||
assert freqs_gen[0].shape == (2, 2, 1, 4)
|
||||
@@ -17,7 +17,7 @@ AUDIO_MODEL = {
|
||||
}
|
||||
|
||||
IMAGE_VIDEO_MODELS = {
|
||||
"riverclouds/qwen_image_random": {"cuda": 2200, "rocm": 2100},
|
||||
"riverclouds/qwen_image_random": {"cuda": 2500, "rocm": 2100},
|
||||
}
|
||||
|
||||
MODELS = {**AUDIO_MODEL, **IMAGE_VIDEO_MODELS}
|
||||
|
||||
@@ -10,9 +10,6 @@ from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.diffusion_engine import _move_tensor_tree_to_cpu
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -66,80 +63,6 @@ class MockScheduler:
|
||||
return [req.request_id for req in sched_output.scheduled_new_reqs]
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.cpu
|
||||
def test_move_tensor_tree_keeps_cpu_tensor_identity() -> None:
|
||||
tensor = torch.arange(8, dtype=torch.float32)
|
||||
|
||||
moved = _move_tensor_tree_to_cpu(tensor)
|
||||
|
||||
assert moved is tensor
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.cpu
|
||||
def test_move_tensor_tree_preserves_nested_structure_without_mutating_input() -> None:
|
||||
tensor = torch.arange(4, dtype=torch.float32)
|
||||
nested_tensor = torch.arange(6, dtype=torch.float32).reshape(2, 3)
|
||||
sentinel = object()
|
||||
payload = {
|
||||
"tensor": tensor,
|
||||
"list": [nested_tensor, sentinel],
|
||||
"tuple": ({"inner": tensor}, "metadata"),
|
||||
"scalar": 3,
|
||||
}
|
||||
|
||||
moved = _move_tensor_tree_to_cpu(payload)
|
||||
|
||||
assert moved is not payload
|
||||
assert set(moved) == {"tensor", "list", "tuple", "scalar"}
|
||||
assert moved["list"] is not payload["list"]
|
||||
assert moved["tuple"] is not payload["tuple"]
|
||||
assert moved["tuple"][0] is not payload["tuple"][0]
|
||||
assert moved["tensor"] is tensor
|
||||
assert moved["list"][0] is nested_tensor
|
||||
assert moved["list"][1] is sentinel
|
||||
assert moved["tuple"][0]["inner"] is tensor
|
||||
assert moved["tuple"][1] == "metadata"
|
||||
assert moved["scalar"] == 3
|
||||
assert payload["list"][0] is nested_tensor
|
||||
assert payload["list"][1] is sentinel
|
||||
assert payload["tuple"][0]["inner"] is tensor
|
||||
assert payload["tuple"][1] == "metadata"
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.cpu
|
||||
def test_move_tensor_tree_returns_non_tensor_values_unchanged() -> None:
|
||||
value = object()
|
||||
|
||||
moved = _move_tensor_tree_to_cpu(value)
|
||||
|
||||
assert moved is value
|
||||
|
||||
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.cuda
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
def test_move_tensor_tree_moves_nested_cuda_tensors_to_cpu() -> None:
|
||||
tensor = torch.arange(8, dtype=torch.float32, device="cuda")
|
||||
other = torch.arange(4, dtype=torch.int64, device="cuda")
|
||||
payload = {"tensor": tensor, "items": [other, ("keep", tensor)]}
|
||||
|
||||
moved = _move_tensor_tree_to_cpu(payload)
|
||||
|
||||
assert moved["tensor"].device.type == "cpu"
|
||||
assert moved["items"][0].device.type == "cpu"
|
||||
assert moved["items"][1][1].device.type == "cpu"
|
||||
torch.testing.assert_close(moved["tensor"], tensor.cpu())
|
||||
torch.testing.assert_close(moved["items"][0], other.cpu())
|
||||
torch.testing.assert_close(moved["items"][1][1], tensor.cpu())
|
||||
assert moved["items"][1][0] == "keep"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_add_req_and_wait_for_response():
|
||||
from vllm_omni.diffusion.diffusion_engine import DiffusionEngine
|
||||
|
||||
@@ -1,183 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import contextlib
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput
|
||||
from vllm_omni.diffusion.ipc import (
|
||||
_SHM_TENSOR_THRESHOLD,
|
||||
_pack_value_if_large,
|
||||
_unpack_if_shm_handle,
|
||||
pack_diffusion_output_shm,
|
||||
unpack_diffusion_output_shm,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _large_numel(dtype: torch.dtype) -> int:
|
||||
return (_SHM_TENSOR_THRESHOLD // torch.empty((), dtype=dtype).element_size()) + 1
|
||||
|
||||
|
||||
def _cleanup_shm_handle(value: object) -> None:
|
||||
if isinstance(value, dict) and value.get("__tensor_shm__"):
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
_unpack_if_shm_handle(value)
|
||||
|
||||
|
||||
def test_diffusion_output_dict_tensors_round_trip_through_shm() -> None:
|
||||
image = torch.arange(300_000, dtype=torch.float32)
|
||||
video = torch.arange(300_000, dtype=torch.float32) * 2
|
||||
output = DiffusionOutput(output={"image": image, "video": video, "metadata": {"keep": "inline"}})
|
||||
|
||||
pack_diffusion_output_shm(output)
|
||||
|
||||
assert output.output["image"]["__tensor_shm__"] is True
|
||||
assert output.output["video"]["__tensor_shm__"] is True
|
||||
assert output.output["metadata"] == {"keep": "inline"}
|
||||
|
||||
unpack_diffusion_output_shm(output)
|
||||
|
||||
torch.testing.assert_close(output.output["image"], image)
|
||||
torch.testing.assert_close(output.output["video"], video)
|
||||
assert output.output["metadata"] == {"keep": "inline"}
|
||||
|
||||
|
||||
def test_diffusion_output_tuple_tensors_round_trip_through_shm() -> None:
|
||||
# LTX2 / DreamID return (video, audio) tuples as DiffusionOutput.output.
|
||||
video = torch.arange(300_000, dtype=torch.float32)
|
||||
audio = torch.arange(300_000, dtype=torch.float32) * 3
|
||||
output = DiffusionOutput(output=(video, audio))
|
||||
|
||||
pack_diffusion_output_shm(output)
|
||||
|
||||
assert isinstance(output.output, tuple)
|
||||
assert output.output[0]["__tensor_shm__"] is True
|
||||
assert output.output[1]["__tensor_shm__"] is True
|
||||
|
||||
unpack_diffusion_output_shm(output)
|
||||
|
||||
assert isinstance(output.output, tuple)
|
||||
torch.testing.assert_close(output.output[0], video)
|
||||
torch.testing.assert_close(output.output[1], audio)
|
||||
|
||||
|
||||
def test_diffusion_output_list_tensors_round_trip_through_shm() -> None:
|
||||
frames = [torch.arange(300_000, dtype=torch.float32), torch.arange(300_000, dtype=torch.float32) + 1]
|
||||
output = DiffusionOutput(output=list(frames))
|
||||
|
||||
pack_diffusion_output_shm(output)
|
||||
|
||||
assert isinstance(output.output, list)
|
||||
assert all(isinstance(item, dict) and item["__tensor_shm__"] is True for item in output.output)
|
||||
|
||||
unpack_diffusion_output_shm(output)
|
||||
|
||||
assert isinstance(output.output, list)
|
||||
torch.testing.assert_close(output.output[0], frames[0])
|
||||
torch.testing.assert_close(output.output[1], frames[1])
|
||||
|
||||
|
||||
def test_pack_value_keeps_tensor_at_threshold_inline() -> None:
|
||||
tensor = torch.arange(
|
||||
_SHM_TENSOR_THRESHOLD // torch.empty((), dtype=torch.float32).element_size(),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
|
||||
packed = _pack_value_if_large(tensor)
|
||||
|
||||
assert packed is tensor
|
||||
|
||||
|
||||
def test_pack_value_packs_large_tensor_and_round_trips() -> None:
|
||||
tensor = torch.arange(_large_numel(torch.float32), dtype=torch.float32)
|
||||
packed = _pack_value_if_large(tensor)
|
||||
|
||||
try:
|
||||
assert isinstance(packed, dict)
|
||||
assert packed["__tensor_shm__"] is True
|
||||
assert packed["shape"] == [tensor.numel()]
|
||||
assert packed["torch_dtype"] == "torch.float32"
|
||||
|
||||
unpacked = _unpack_if_shm_handle(packed)
|
||||
assert isinstance(unpacked, torch.Tensor)
|
||||
torch.testing.assert_close(unpacked, tensor)
|
||||
finally:
|
||||
_cleanup_shm_handle(packed)
|
||||
|
||||
|
||||
def test_pack_value_recurses_nested_dicts_and_lists_without_mutating_inline_values() -> None:
|
||||
large = torch.arange(_large_numel(torch.float32), dtype=torch.float32)
|
||||
small = torch.arange(8, dtype=torch.float32)
|
||||
list_tensor = torch.arange(_large_numel(torch.float32), dtype=torch.float32)
|
||||
payload = {
|
||||
"media": {
|
||||
"large": large,
|
||||
"small": small,
|
||||
},
|
||||
"list_value": [list_tensor],
|
||||
"metadata": {"prompt": "keep inline"},
|
||||
}
|
||||
|
||||
packed = _pack_value_if_large(payload)
|
||||
|
||||
try:
|
||||
assert packed is not payload
|
||||
assert packed["media"] is not payload["media"]
|
||||
assert packed["media"]["large"]["__tensor_shm__"] is True
|
||||
assert packed["media"]["small"] is small
|
||||
# Lists are recursed too: the large tensor inside is packed and a new
|
||||
# list is returned, while the input payload is left untouched.
|
||||
assert packed["list_value"] is not payload["list_value"]
|
||||
assert packed["list_value"][0]["__tensor_shm__"] is True
|
||||
assert payload["list_value"][0] is list_tensor
|
||||
assert packed["metadata"] == {"prompt": "keep inline"}
|
||||
|
||||
torch.testing.assert_close(_unpack_if_shm_handle(packed["media"]["large"]), large)
|
||||
torch.testing.assert_close(_unpack_if_shm_handle(packed["list_value"][0]), list_tensor)
|
||||
finally:
|
||||
if isinstance(packed, dict):
|
||||
_cleanup_shm_handle(packed.get("media", {}).get("large"))
|
||||
list_value = packed.get("list_value")
|
||||
if isinstance(list_value, list) and list_value:
|
||||
_cleanup_shm_handle(list_value[0])
|
||||
|
||||
|
||||
def test_pack_value_preserves_dtype_shape_and_values_for_bfloat16() -> None:
|
||||
tensor = torch.arange(_large_numel(torch.bfloat16), dtype=torch.float32).to(torch.bfloat16).reshape(1, -1)
|
||||
packed = _pack_value_if_large(tensor)
|
||||
|
||||
try:
|
||||
assert isinstance(packed, dict)
|
||||
assert packed["__tensor_shm__"] is True
|
||||
assert packed["shape"] == list(tensor.shape)
|
||||
assert packed["torch_dtype"] == "torch.bfloat16"
|
||||
assert packed["numpy_dtype"] == "float32"
|
||||
|
||||
unpacked = _unpack_if_shm_handle(packed)
|
||||
assert isinstance(unpacked, torch.Tensor)
|
||||
assert unpacked.dtype == torch.bfloat16
|
||||
torch.testing.assert_close(unpacked, tensor)
|
||||
finally:
|
||||
_cleanup_shm_handle(packed)
|
||||
|
||||
|
||||
def test_pack_value_packs_non_contiguous_large_tensor_values() -> None:
|
||||
tensor = torch.arange(_large_numel(torch.float32) * 2, dtype=torch.float32).reshape(-1, 2)[:, 0]
|
||||
assert not tensor.is_contiguous()
|
||||
|
||||
packed = _pack_value_if_large(tensor)
|
||||
|
||||
try:
|
||||
assert isinstance(packed, dict)
|
||||
assert packed["__tensor_shm__"] is True
|
||||
assert packed["shape"] == list(tensor.shape)
|
||||
|
||||
unpacked = _unpack_if_shm_handle(packed)
|
||||
assert isinstance(unpacked, torch.Tensor)
|
||||
torch.testing.assert_close(unpacked, tensor)
|
||||
finally:
|
||||
_cleanup_shm_handle(packed)
|
||||
@@ -301,4 +301,8 @@ def test_vllm_set_forward_context_implementation(monkeypatch):
|
||||
"vllm_config",
|
||||
),
|
||||
),
|
||||
("ir_op_priority", None),
|
||||
("enable_torch_wrap", vllm_config.compilation_config.ir_enable_torch_wrap),
|
||||
("enable_torch_wrap_exit", vllm_config.compilation_config.ir_enable_torch_wrap),
|
||||
("ir_op_priority_exit", None),
|
||||
], ERROR_MESSAGE
|
||||
@@ -11,54 +11,26 @@ from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _get_available_port() -> int:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.bind(("", 0))
|
||||
return sock.getsockname()[1]
|
||||
|
||||
|
||||
def _bind_available_port_below(max_port: int, port_inc: int) -> socket.socket:
|
||||
for _ in range(100):
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
try:
|
||||
sock.bind(("", 0))
|
||||
port = sock.getsockname()[1]
|
||||
if port >= max_port - port_inc:
|
||||
sock.close()
|
||||
continue
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as next_sock:
|
||||
next_sock.bind(("", port + port_inc))
|
||||
return sock
|
||||
except OSError:
|
||||
sock.close()
|
||||
raise RuntimeError(f"Failed to reserve an available port below {max_port - port_inc}")
|
||||
|
||||
|
||||
class TestOmniDiffusionConfigMasterPort:
|
||||
def test_honors_master_port_env(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
port = _get_available_port()
|
||||
monkeypatch.setenv("MASTER_PORT", str(port))
|
||||
monkeypatch.setenv("MASTER_PORT", "43210")
|
||||
config = OmniDiffusionConfig(model="test")
|
||||
assert config.master_port == port
|
||||
assert config.master_port == 43210
|
||||
|
||||
def test_honors_explicit_master_port(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("MASTER_PORT", raising=False)
|
||||
port = _get_available_port()
|
||||
config = OmniDiffusionConfig(model="test", master_port=port)
|
||||
assert config.master_port == port
|
||||
config = OmniDiffusionConfig(model="test", master_port=40000)
|
||||
assert config.master_port == 40000
|
||||
|
||||
def test_master_port_env_takes_precedence_over_kwarg(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
env_port = _get_available_port()
|
||||
kwarg_port = _get_available_port()
|
||||
monkeypatch.setenv("MASTER_PORT", str(env_port))
|
||||
config = OmniDiffusionConfig(model="test", master_port=kwarg_port)
|
||||
assert config.master_port == env_port
|
||||
monkeypatch.setenv("MASTER_PORT", "43210")
|
||||
config = OmniDiffusionConfig(model="test", master_port=40000)
|
||||
assert config.master_port == 43210
|
||||
|
||||
def test_explicit_master_port_is_stable_without_jitter(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("MASTER_PORT", raising=False)
|
||||
port = _get_available_port()
|
||||
ports = {OmniDiffusionConfig(model="test", master_port=port).master_port for _ in range(5)}
|
||||
assert ports == {port}
|
||||
ports = {OmniDiffusionConfig(model="test", master_port=40123).master_port for _ in range(5)}
|
||||
assert ports == {40123}
|
||||
|
||||
def test_default_master_port_is_available(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("MASTER_PORT", raising=False)
|
||||
@@ -70,7 +42,8 @@ class TestOmniDiffusionConfigMasterPort:
|
||||
|
||||
def test_settle_port_steps_when_port_is_busy(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("MASTER_PORT", raising=False)
|
||||
with _bind_available_port_below(max_port=60000, port_inc=37) as sock:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.bind(("", 0))
|
||||
busy_port = sock.getsockname()[1]
|
||||
config = OmniDiffusionConfig(model="test", master_port=busy_port)
|
||||
assert config.master_port == busy_port + 37
|
||||
@@ -85,7 +85,7 @@ class MockOmniRequestOutput:
|
||||
BASE_HEIGHT = 512
|
||||
BASE_WIDTH = 512
|
||||
BASE_INFER_STEPS = 10
|
||||
DELAY_BASE = 0.01
|
||||
DELAY_BASE = 2
|
||||
|
||||
|
||||
class MockDiffusionEngine:
|
||||
@@ -93,7 +93,7 @@ class MockDiffusionEngine:
|
||||
def simulate_step_delay(height, width, num_inference_steps) -> float:
|
||||
return (height / BASE_HEIGHT) * (width / BASE_WIDTH) * (num_inference_steps / BASE_INFER_STEPS)
|
||||
|
||||
DELAY_BASE = 0.01
|
||||
DELAY_BASE = 2
|
||||
delay_scale = simulate_step_delay(
|
||||
request.sampling_params.height, request.sampling_params.width, request.sampling_params.num_inference_steps
|
||||
)
|
||||
@@ -139,7 +139,7 @@ async def test_proc_process_request_with_batching_async_output():
|
||||
assert len(results) == len(test_requests)
|
||||
base_time = DELAY_BASE
|
||||
time_gap_std = DELAY_BASE * 2 * 2 * 1 # height/width/steps infer time scale
|
||||
eps = 0.1
|
||||
eps = 0.5
|
||||
for i, (res, elapsed_time) in enumerate(results):
|
||||
assert res.request_id == test_requests[i]["request_id"]
|
||||
assert isinstance(res, MockOmniRequestOutput)
|
||||
|
||||
@@ -50,7 +50,6 @@ def build_adapter(monkeypatch, mocker: MockerFixture):
|
||||
stage_id: int = 1,
|
||||
model_mode: str = "ar",
|
||||
max_num_seqs: int = 2,
|
||||
active_stream_window: int = 0,
|
||||
connector_extra: dict | None = None,
|
||||
):
|
||||
connector = mocker.MagicMock()
|
||||
@@ -77,11 +76,7 @@ def build_adapter(monkeypatch, mocker: MockerFixture):
|
||||
classmethod(lambda cls, _model_config: connector),
|
||||
)
|
||||
|
||||
model_config = SimpleNamespace(
|
||||
worker_type=model_mode,
|
||||
max_num_seqs=max_num_seqs,
|
||||
active_stream_window=active_stream_window,
|
||||
)
|
||||
model_config = SimpleNamespace(worker_type=model_mode)
|
||||
scheduler_config = SimpleNamespace(max_num_seqs=max_num_seqs)
|
||||
adapter = OmniChunkTransferAdapter(
|
||||
SimpleNamespace(model_config=model_config, scheduler_config=scheduler_config)
|
||||
@@ -401,75 +396,6 @@ def test_process_and_restore_queues(build_adapter):
|
||||
assert adapter.waiting_for_chunk_running_requests == deque()
|
||||
|
||||
|
||||
def test_fifo_promotion(build_adapter):
|
||||
adapter, _ = build_adapter(stage_id=1, max_num_seqs=2, active_stream_window=2)
|
||||
reqs = [_req(f"req-{idx}", RequestStatus.WAITING) for idx in range(1, 5)]
|
||||
waiting_queue = DummyWaitingQueue(reqs)
|
||||
running_queue = []
|
||||
|
||||
adapter.process_pending_chunks(waiting_queue, running_queue)
|
||||
|
||||
assert list(adapter._active_streams) == ["req-1", "req-2"]
|
||||
assert waiting_queue == reqs[2:]
|
||||
assert reqs[0].status == RequestStatus.WAITING_FOR_CHUNK
|
||||
assert reqs[1].status == RequestStatus.WAITING_FOR_CHUNK
|
||||
assert reqs[2].status == RequestStatus.WAITING
|
||||
|
||||
adapter.finished_requests.add("req-1")
|
||||
# Eviction is deferred to postprocess_scheduler_output in the runtime path
|
||||
# (commit c4d95fd9 — otherwise the terminal chunk deadlocks at c=8 K=2).
|
||||
# Simulate it here so promotion can pick up the freed slot.
|
||||
adapter._evict_finished_active_streams({"req-1"})
|
||||
adapter.process_pending_chunks(waiting_queue, running_queue)
|
||||
|
||||
assert list(adapter._active_streams) == ["req-2", "req-3"]
|
||||
# Promotion + chunk processing happen in the same call, so req-3 is
|
||||
# already WAITING_FOR_CHUNK by the time we check.
|
||||
assert reqs[2].status == RequestStatus.WAITING_FOR_CHUNK
|
||||
|
||||
|
||||
def test_legacy_k0(build_adapter):
|
||||
adapter, _ = build_adapter(stage_id=1, max_num_seqs=1, active_stream_window=0)
|
||||
waiting_req = _req("waiting", RequestStatus.WAITING)
|
||||
running_req_1 = _req("running-1", RequestStatus.RUNNING)
|
||||
running_req_2 = _req("running-2", RequestStatus.RUNNING)
|
||||
waiting_queue = DummyWaitingQueue([waiting_req])
|
||||
running_queue = [running_req_1, running_req_2]
|
||||
|
||||
adapter.requests_with_ready_chunks.update({"running-1", "running-2"})
|
||||
adapter.process_pending_chunks(waiting_queue, running_queue)
|
||||
|
||||
assert waiting_req.status == RequestStatus.WAITING_FOR_CHUNK
|
||||
assert adapter.waiting_for_chunk_waiting_requests == deque([waiting_req])
|
||||
assert running_queue == [running_req_1]
|
||||
assert waiting_queue == [running_req_2]
|
||||
assert running_req_2.status == RequestStatus.PREEMPTED
|
||||
assert adapter._active_streams == {}
|
||||
|
||||
|
||||
def test_finished_releases_slot(build_adapter):
|
||||
adapter, _ = build_adapter(stage_id=1, max_num_seqs=1, active_stream_window=1)
|
||||
req_1 = _req("req-1", RequestStatus.WAITING)
|
||||
req_2 = _req("req-2", RequestStatus.WAITING)
|
||||
waiting_queue = DummyWaitingQueue([req_1, req_2])
|
||||
running_queue = []
|
||||
|
||||
adapter.process_pending_chunks(waiting_queue, running_queue)
|
||||
assert list(adapter._active_streams) == ["req-1"]
|
||||
assert waiting_queue == [req_2]
|
||||
|
||||
adapter.finished_requests.add("req-1")
|
||||
# Eviction is deferred to postprocess_scheduler_output in the runtime path
|
||||
# (commit c4d95fd9). Simulate it so promotion can pick up the freed slot.
|
||||
adapter._evict_finished_active_streams({"req-1"})
|
||||
adapter.process_pending_chunks(waiting_queue, running_queue)
|
||||
|
||||
assert list(adapter._active_streams) == ["req-2"]
|
||||
# Promotion + chunk processing happen in the same call, so req-2 is
|
||||
# already WAITING_FOR_CHUNK by the time we check.
|
||||
assert req_2.status == RequestStatus.WAITING_FOR_CHUNK
|
||||
|
||||
|
||||
def test_postprocess_scheduler_output(build_adapter):
|
||||
adapter, _ = build_adapter()
|
||||
adapter.requests_with_ready_chunks = {"new-ready", "cached-ready", "leftover"}
|
||||
@@ -496,7 +422,6 @@ def test_postprocess_scheduler_output(build_adapter):
|
||||
def _populate_adapter_state(adapter, req_id="req-1", ext_id="ext-1"):
|
||||
"""Fill every per-request structure so cleanup can be verified."""
|
||||
adapter.finished_requests.add(req_id)
|
||||
adapter._active_streams[req_id] = SimpleNamespace(request_id=req_id)
|
||||
adapter.get_req_chunk[req_id] = 3
|
||||
adapter.requests_with_ready_chunks.add(req_id)
|
||||
adapter.request_ids_mapping[req_id] = ext_id
|
||||
@@ -517,7 +442,6 @@ def test_cleanup_clears_all_state(build_adapter):
|
||||
adapter.cleanup(req_id, ext_id)
|
||||
|
||||
assert req_id not in adapter.finished_requests
|
||||
assert req_id not in adapter._active_streams
|
||||
assert req_id not in adapter.get_req_chunk
|
||||
assert req_id not in adapter.requests_with_ready_chunks
|
||||
assert req_id not in adapter.request_ids_mapping
|
||||
@@ -779,7 +703,6 @@ def test_generation_scheduler_calls_cleanup_on_finished(monkeypatch, mocker: Moc
|
||||
kv_connector_output=None,
|
||||
cudagraph_stats=None,
|
||||
req_id_to_index={"req-s1": 0},
|
||||
routed_experts=None,
|
||||
routed_experts_dict=None,
|
||||
)
|
||||
|
||||
@@ -866,7 +789,6 @@ def test_ar_scheduler_defers_cleanup_and_queues_save_on_finished(mocker: MockerF
|
||||
cudagraph_stats=None,
|
||||
req_id_to_index={"req-ar": 0},
|
||||
kv_extracted_req_ids=None,
|
||||
routed_experts=None,
|
||||
routed_experts_dict=None,
|
||||
)
|
||||
|
||||
|
||||
@@ -68,13 +68,13 @@ def test_omni_coordinator_pub_coalescing_on_rapid_queue_updates():
|
||||
|
||||
sub_ctx = zmq.Context.instance()
|
||||
sub = sub_ctx.socket(zmq.SUB)
|
||||
sub.connect(coordinator.pub_zmq_addr)
|
||||
sub.connect(pub_addr)
|
||||
sub.setsockopt(zmq.SUBSCRIBE, b"")
|
||||
|
||||
time.sleep(0.3) # PUB/SUB slow-joiner
|
||||
|
||||
client = OmniCoordClientForStage(
|
||||
coordinator.router_zmq_addr,
|
||||
router_addr,
|
||||
"tcp://stage:coalesce",
|
||||
"tcp://stage:coalesce-out",
|
||||
0,
|
||||
@@ -131,7 +131,7 @@ def test_omni_coordinator_registration_broadcast():
|
||||
|
||||
sub_ctx = zmq.Context.instance()
|
||||
sub = sub_ctx.socket(zmq.SUB)
|
||||
sub.connect(coordinator.pub_zmq_addr)
|
||||
sub.connect(pub_addr)
|
||||
sub.setsockopt(zmq.SUBSCRIBE, b"")
|
||||
|
||||
# ZMQ PUB/SUB slow-joiner: allow SUB to connect before clients register.
|
||||
@@ -139,9 +139,9 @@ def test_omni_coordinator_registration_broadcast():
|
||||
|
||||
# Create 3 stage clients; each auto-registers on init.
|
||||
clients = [
|
||||
OmniCoordClientForStage(coordinator.router_zmq_addr, "tcp://stage:10001", "tcp://stage:10001-out", 0),
|
||||
OmniCoordClientForStage(coordinator.router_zmq_addr, "tcp://stage:10002", "tcp://stage:10002-out", 0),
|
||||
OmniCoordClientForStage(coordinator.router_zmq_addr, "tcp://stage:10003", "tcp://stage:10003-out", 1),
|
||||
OmniCoordClientForStage(router_addr, "tcp://stage:10001", "tcp://stage:10001-out", 0),
|
||||
OmniCoordClientForStage(router_addr, "tcp://stage:10002", "tcp://stage:10002-out", 0),
|
||||
OmniCoordClientForStage(router_addr, "tcp://stage:10003", "tcp://stage:10003-out", 1),
|
||||
]
|
||||
|
||||
msg = _wait_for_replica_list(sub, expected_count=3)
|
||||
@@ -183,19 +183,19 @@ def test_omni_coordinator_heartbeat_timeout_handling():
|
||||
|
||||
sub_ctx = zmq.Context.instance()
|
||||
sub = sub_ctx.socket(zmq.SUB)
|
||||
sub.connect(coordinator.pub_zmq_addr)
|
||||
sub.connect(pub_addr)
|
||||
sub.setsockopt(zmq.SUBSCRIBE, b"")
|
||||
|
||||
time.sleep(0.3)
|
||||
|
||||
# A and B: real clients that send heartbeats every 5s.
|
||||
client_a = OmniCoordClientForStage(coordinator.router_zmq_addr, "tcp://stage:a", "tcp://stage:a-out", 0)
|
||||
client_b = OmniCoordClientForStage(coordinator.router_zmq_addr, "tcp://stage:b", "tcp://stage:b-out", 0)
|
||||
client_a = OmniCoordClientForStage(router_addr, "tcp://stage:a", "tcp://stage:a-out", 0)
|
||||
client_b = OmniCoordClientForStage(router_addr, "tcp://stage:b", "tcp://stage:b-out", 0)
|
||||
|
||||
# C: raw DEALER that sends only registration, no heartbeat.
|
||||
dealer_ctx = zmq.Context.instance()
|
||||
dealer_c = dealer_ctx.socket(zmq.DEALER)
|
||||
dealer_c.connect(coordinator.router_zmq_addr)
|
||||
dealer_c.connect(router_addr)
|
||||
reg_event = {
|
||||
"input_addr": "tcp://stage:c",
|
||||
"output_addr": "tcp://stage:c-out",
|
||||
@@ -254,12 +254,12 @@ def test_omni_coordinator_replica_shutdown_handling():
|
||||
|
||||
sub_ctx = zmq.Context.instance()
|
||||
sub = sub_ctx.socket(zmq.SUB)
|
||||
sub.connect(coordinator.pub_zmq_addr)
|
||||
sub.connect(pub_addr)
|
||||
sub.setsockopt(zmq.SUBSCRIBE, b"")
|
||||
|
||||
time.sleep(0.3) # PUB/SUB slow-joiner
|
||||
|
||||
client = OmniCoordClientForStage(coordinator.router_zmq_addr, "tcp://stage:shutdown", "tcp://stage:shutdown-out", 0)
|
||||
client = OmniCoordClientForStage(router_addr, "tcp://stage:shutdown", "tcp://stage:shutdown-out", 0)
|
||||
|
||||
msg = _wait_for_replica_list(sub, expected_count=1)
|
||||
assert msg is not None
|
||||
|
||||
@@ -1,132 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from tests.e2e.accuracy.helpers import model_output_dir
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniServer
|
||||
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]
|
||||
|
||||
MODEL_ENV_VAR = "VLLM_TEST_COSMOS3_MODEL"
|
||||
MODEL_ID = "cosmos3"
|
||||
PROMPT = "A small warehouse robot moves a blue box across a clean floor."
|
||||
NEGATIVE_PROMPT = "blurry, distorted, low quality"
|
||||
SEED = 42
|
||||
WIDTH = HEIGHT = 256
|
||||
NUM_INFERENCE_STEPS = 2
|
||||
|
||||
|
||||
def _model_name() -> str:
|
||||
model = os.environ.get(MODEL_ENV_VAR)
|
||||
if not model:
|
||||
pytest.skip(f"Set {MODEL_ENV_VAR} to run Cosmos3 full-model smoke tests.")
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("Cosmos3 full-model smoke tests require CUDA.")
|
||||
return model
|
||||
|
||||
|
||||
def _server_args() -> list[str]:
|
||||
return [
|
||||
"--num-gpus",
|
||||
"1",
|
||||
"--model-class-name",
|
||||
"Cosmos3OmniDiffusersPipeline",
|
||||
"--stage-init-timeout",
|
||||
"900",
|
||||
"--init-timeout",
|
||||
"1200",
|
||||
]
|
||||
|
||||
|
||||
def _image_data_url(image: Image.Image) -> str:
|
||||
buf = io.BytesIO()
|
||||
image.save(buf, format="PNG")
|
||||
return f"data:image/png;base64,{base64.b64encode(buf.getvalue()).decode('ascii')}"
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_cosmos3_t2i_serving_smoke(accuracy_artifact_root: Path) -> None:
|
||||
output_dir = model_output_dir(accuracy_artifact_root, MODEL_ID)
|
||||
with OmniServer(_model_name(), _server_args(), use_omni=True) as server:
|
||||
response = requests.post(
|
||||
f"http://{server.host}:{server.port}/v1/images/generations",
|
||||
json={
|
||||
"model": server.model,
|
||||
"prompt": PROMPT,
|
||||
"negative_prompt": NEGATIVE_PROMPT,
|
||||
"size": f"{WIDTH}x{HEIGHT}",
|
||||
"n": 1,
|
||||
"response_format": "b64_json",
|
||||
"num_inference_steps": NUM_INFERENCE_STEPS,
|
||||
"guidance_scale": 1.0,
|
||||
"seed": SEED,
|
||||
},
|
||||
timeout=1800,
|
||||
)
|
||||
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
assert len(payload["data"]) == 1
|
||||
image = Image.open(io.BytesIO(base64.b64decode(payload["data"][0]["b64_json"]))).convert("RGB")
|
||||
image.save(output_dir / "cosmos3_t2i.png")
|
||||
assert image.size == (WIDTH, HEIGHT)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("name", "prompt", "num_frames", "image_reference"),
|
||||
[
|
||||
("t2v", PROMPT, "1", None),
|
||||
(
|
||||
"i2v",
|
||||
"The blue rectangle moves slowly forward.",
|
||||
"5",
|
||||
Image.new("RGB", (96, 64), color=(40, 80, 160)),
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.benchmark
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_cosmos3_video_serving_smoke(
|
||||
accuracy_artifact_root: Path,
|
||||
name: str,
|
||||
prompt: str,
|
||||
num_frames: str,
|
||||
image_reference: Image.Image | None,
|
||||
) -> None:
|
||||
output_dir = model_output_dir(accuracy_artifact_root, MODEL_ID)
|
||||
data = {
|
||||
"model": "",
|
||||
"prompt": prompt,
|
||||
"negative_prompt": NEGATIVE_PROMPT,
|
||||
"size": f"{WIDTH}x{HEIGHT}",
|
||||
"num_frames": num_frames,
|
||||
"fps": "1",
|
||||
"num_inference_steps": str(NUM_INFERENCE_STEPS),
|
||||
"guidance_scale": "1.0",
|
||||
"seed": str(SEED),
|
||||
}
|
||||
if image_reference is not None:
|
||||
data["image_reference"] = json.dumps({"image_url": _image_data_url(image_reference)})
|
||||
|
||||
with OmniServer(_model_name(), _server_args(), use_omni=True) as server:
|
||||
data["model"] = server.model
|
||||
response = requests.post(f"http://{server.host}:{server.port}/v1/videos/sync", data=data, timeout=1800)
|
||||
|
||||
response.raise_for_status()
|
||||
assert response.headers["content-type"].startswith("video/mp4")
|
||||
assert response.content
|
||||
(output_dir / f"cosmos3_{name}.mp4").write_bytes(response.content)
|
||||
@@ -26,6 +26,10 @@ from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
pytestmark = pytest.mark.skip(
|
||||
reason="Current Hugging Face account is not yet authorized to access this model.",
|
||||
)
|
||||
|
||||
QUANTIZED_MODEL = "vllm-project-org/FLUX.1-dev-AutoRound-w4a16"
|
||||
BASELINE_MODEL = "black-forest-labs/FLUX.1-dev"
|
||||
|
||||
@@ -59,27 +63,6 @@ def _first_request_images(outputs) -> list:
|
||||
return req_out.images
|
||||
|
||||
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_flux_autoround_w4a16_load():
|
||||
"""Load the W4A16 quantized FLUX model and run a minimal generation.
|
||||
|
||||
Verifies that the quantized checkpoint loads end-to-end (quant_config
|
||||
propagated to all transformer blocks) and produces a valid image.
|
||||
"""
|
||||
with OmniRunner(QUANTIZED_MODEL, enforce_eager=True) as runner:
|
||||
outputs = runner.omni.generate(
|
||||
"a cup of coffee on a table",
|
||||
_sampling_params(),
|
||||
)
|
||||
images = _first_request_images(outputs)
|
||||
assert len(images) >= 1, "Expected at least one generated image"
|
||||
img = images[0]
|
||||
assert img.width == WIDTH and img.height == HEIGHT
|
||||
arr = np.array(img)
|
||||
assert arr.std() > 1.0, "Generated image appears blank (std ≈ 0)"
|
||||
|
||||
|
||||
def _generate_image(model_name: str, **extra_kwargs) -> tuple[list, float]:
|
||||
"""Load a FLUX model, generate one image, return (images, peak_memory_mb)."""
|
||||
gc.collect()
|
||||
@@ -104,7 +87,6 @@ def _generate_image(model_name: str, **extra_kwargs) -> tuple[list, float]:
|
||||
return images, peak
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="BF16 baseline model cannot be loaded correctly yet.")
|
||||
@pytest.mark.full_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
@@ -123,7 +105,6 @@ def test_flux_autoround_w4a16_generates_image(omni_runner: OmniRunner):
|
||||
assert arr.std() > 1.0, "Generated image appears blank (std ≈ 0)"
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="BF16 baseline model cannot be loaded correctly yet.")
|
||||
@pytest.mark.full_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
|
||||
@@ -1,230 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""E2E offline inference tests for MOSS-TTS two-stage pipeline.
|
||||
|
||||
Tests the MossTTSDelayModel path using MOSS-VoiceGenerator (1.7B) — the
|
||||
smallest MossTTSDelayModel variant — so the test runs on a single A10G or L4
|
||||
without requiring an 80 GB GPU.
|
||||
|
||||
A second test class covers MOSS-TTS-Realtime (1.7B, MossTTSRealtime) to
|
||||
exercise the local-transformer generation path.
|
||||
|
||||
Both classes use a module-scoped engine so the model is loaded once per test
|
||||
module execution. Do NOT add a second engine fixture in this file — that
|
||||
triggers mid-module teardown races (see skill invariant I4).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import urllib.request
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from vllm import SamplingParams
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from vllm_omni import Omni
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Selected by the nightly "TTS · Function Test" (-m "full_model and L4 and tts").
|
||||
# Without the tts + run-level marks these tests are collected but never run.
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
||||
|
||||
SAMPLE_RATE = 24_000 # All MOSS-TTS full variants output 24 kHz
|
||||
|
||||
REF_AUDIO_URL = "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS/main/assets/audio/reference_zh_1.wav"
|
||||
|
||||
_DEFAULT_SAMPLING = SamplingParams(
|
||||
temperature=1.7,
|
||||
top_p=0.8,
|
||||
top_k=25,
|
||||
max_tokens=512, # short for tests (~20 s at 24 kHz, ~12.5 tokens/s)
|
||||
seed=42,
|
||||
detokenize=False,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Session fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def ref_audio_path(tmp_path_factory) -> str:
|
||||
"""Download the upstream reference clip once per session.
|
||||
|
||||
Set ``MOSS_TTS_SKIP_ON_NET_FAIL=1`` to skip in air-gapped environments.
|
||||
"""
|
||||
cache_dir = tmp_path_factory.mktemp("moss_tts_ref")
|
||||
target = cache_dir / "zh_1.wav"
|
||||
try:
|
||||
with urllib.request.urlopen(REF_AUDIO_URL, timeout=30) as resp:
|
||||
target.write_bytes(resp.read())
|
||||
except Exception as exc:
|
||||
msg = f"Cannot fetch reference clip {REF_AUDIO_URL}: {exc}"
|
||||
if os.environ.get("MOSS_TTS_SKIP_ON_NET_FAIL"):
|
||||
pytest.skip(msg)
|
||||
pytest.fail(msg)
|
||||
if not target.exists() or target.stat().st_size == 0:
|
||||
pytest.fail(f"Reference clip empty after download: {target}")
|
||||
return str(target)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _build_request(text: str, ref_audio_path: str, seed: int = 42) -> dict:
|
||||
return {
|
||||
"prompt": "<|im_start|>assistant\n",
|
||||
"additional_information": {
|
||||
"task_type": ["voice_clone"],
|
||||
"text": [text],
|
||||
"mode": ["voice_clone"],
|
||||
"prompt_audio_path": [ref_audio_path],
|
||||
"seed": [seed],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _collect_audio(omni: Omni, request: dict) -> tuple[torch.Tensor, int]:
|
||||
"""Run one request; return (waveform_cpu, sample_rate)."""
|
||||
for stage_outputs in omni.generate(request, _DEFAULT_SAMPLING):
|
||||
for req_output in stage_outputs.request_output:
|
||||
mm = req_output.outputs[0].multimodal_output
|
||||
assert mm is not None, "Expected non-None multimodal_output"
|
||||
# Never use ``a or b`` on tensor values — a multi-element tensor
|
||||
# raises on truthiness. Check ``is None`` explicitly and handle the
|
||||
# pre-consolidation list form.
|
||||
audio = mm.get("audio")
|
||||
if audio is None:
|
||||
audio = mm.get("model_outputs")
|
||||
sr = mm.get("sr")
|
||||
assert audio is not None, "Expected 'audio' or 'model_outputs' in multimodal_output"
|
||||
if isinstance(audio, list):
|
||||
audio = torch.cat(
|
||||
[t.reshape(-1) for t in audio if isinstance(t, torch.Tensor) and t.numel() > 0],
|
||||
dim=0,
|
||||
)
|
||||
assert isinstance(audio, torch.Tensor), f"Expected Tensor, got {type(audio)}"
|
||||
return audio.reshape(-1).cpu(), int(sr.item()) if sr is not None else SAMPLE_RATE
|
||||
raise AssertionError("No stage outputs received")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MossTTSDelayModel — MOSS-VoiceGenerator 1.7B
|
||||
# (tests the delay-pattern generation path on a small GPU)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_DELAY_MODEL = "OpenMOSS-Team/MOSS-VoiceGenerator"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def delay_engine():
|
||||
return Omni(model=_DELAY_MODEL, stage_init_timeout=300)
|
||||
|
||||
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_moss_tts_delay_english(delay_engine, ref_audio_path):
|
||||
"""MossTTSDelayModel: English voice_clone produces non-empty 24 kHz audio."""
|
||||
req = _build_request("Hello, this is a MOSS-TTS voice cloning test.", ref_audio_path)
|
||||
audio, sr = _collect_audio(delay_engine, req)
|
||||
|
||||
assert sr == SAMPLE_RATE, f"Expected {SAMPLE_RATE} Hz, got {sr}"
|
||||
assert audio.numel() > 0, "Audio tensor is empty"
|
||||
assert not torch.all(audio == 0), "Audio is silence"
|
||||
|
||||
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_moss_tts_delay_chinese(delay_engine, ref_audio_path):
|
||||
"""MossTTSDelayModel: Chinese input produces non-empty audio."""
|
||||
req = _build_request("你好,这是语音合成测试。", ref_audio_path)
|
||||
audio, sr = _collect_audio(delay_engine, req)
|
||||
|
||||
assert sr == SAMPLE_RATE
|
||||
assert audio.numel() > 0
|
||||
assert not torch.all(audio == 0)
|
||||
|
||||
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_moss_tts_delay_deterministic(delay_engine, ref_audio_path):
|
||||
"""MossTTSDelayModel: same seed yields identical waveforms."""
|
||||
req = _build_request("Reproducibility check.", ref_audio_path, seed=99)
|
||||
audio1, _ = _collect_audio(delay_engine, req)
|
||||
audio2, _ = _collect_audio(delay_engine, req)
|
||||
|
||||
assert audio1.shape == audio2.shape, "Shapes differ across identical seeds"
|
||||
assert torch.allclose(audio1, audio2, atol=1e-4), "Waveforms differ across identical seeds"
|
||||
|
||||
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_moss_tts_delay_batch(delay_engine, ref_audio_path):
|
||||
"""MossTTSDelayModel: batch of two requests each returns non-empty audio."""
|
||||
requests = [
|
||||
_build_request("First sentence.", ref_audio_path),
|
||||
_build_request("Second sentence.", ref_audio_path),
|
||||
]
|
||||
results: list[torch.Tensor] = []
|
||||
for stage_outputs in delay_engine.generate(requests, [_DEFAULT_SAMPLING] * 2):
|
||||
for req_output in stage_outputs.request_output:
|
||||
mm = req_output.outputs[0].multimodal_output
|
||||
assert mm is not None
|
||||
audio = mm.get("audio")
|
||||
if audio is None:
|
||||
audio = mm.get("model_outputs")
|
||||
assert audio is not None
|
||||
if isinstance(audio, list):
|
||||
audio = torch.cat(
|
||||
[t.reshape(-1) for t in audio if isinstance(t, torch.Tensor) and t.numel() > 0],
|
||||
dim=0,
|
||||
)
|
||||
results.append(audio.reshape(-1).cpu())
|
||||
|
||||
assert len(results) == 2, f"Expected 2 outputs, got {len(results)}"
|
||||
for i, audio in enumerate(results):
|
||||
assert audio.numel() > 0, f"Audio[{i}] is empty"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MossTTSRealtime — MOSS-TTS-Realtime 1.7B
|
||||
# (tests the local-transformer generation path)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_REALTIME_MODEL = "OpenMOSS-Team/MOSS-TTS-Realtime"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def realtime_engine():
|
||||
return Omni(model=_REALTIME_MODEL, stage_init_timeout=300)
|
||||
|
||||
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_moss_tts_realtime_english(realtime_engine, ref_audio_path):
|
||||
"""MossTTSRealtime: English voice_clone produces non-empty 24 kHz audio."""
|
||||
req = _build_request("This is a real-time TTS streaming test.", ref_audio_path)
|
||||
audio, sr = _collect_audio(realtime_engine, req)
|
||||
|
||||
assert sr == SAMPLE_RATE, f"Expected {SAMPLE_RATE} Hz, got {sr}"
|
||||
assert audio.numel() > 0, "Audio tensor is empty"
|
||||
assert not torch.all(audio == 0), "Audio is silence"
|
||||
|
||||
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_moss_tts_realtime_deterministic(realtime_engine, ref_audio_path):
|
||||
"""MossTTSRealtime: same seed yields identical waveforms."""
|
||||
req = _build_request("Determinism check for realtime model.", ref_audio_path, seed=7)
|
||||
audio1, _ = _collect_audio(realtime_engine, req)
|
||||
audio2, _ = _collect_audio(realtime_engine, req)
|
||||
|
||||
assert audio1.shape == audio2.shape
|
||||
assert torch.allclose(audio1, audio2, atol=1e-4)
|
||||
@@ -1,113 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""E2E offline inference test for MOSS-TTS-v1.5 (MossTTSDelay-8B).
|
||||
|
||||
MOSS-TTS-v1.5 is a continued-training upgrade of MOSS-TTS 1.0 with the SAME
|
||||
``MossTTSDelay`` architecture and API, so it reuses the existing talker/codec
|
||||
path unchanged — this test just pins that the 8B v1.5 checkpoint loads and
|
||||
produces audio.
|
||||
|
||||
The model is 8B, so this lives in its own file (one module-scoped engine per
|
||||
file, per skill invariant I4) and is gated behind an 80 GB GPU; the small-GPU
|
||||
MossTTSDelay coverage stays in ``test_moss_tts.py`` (1.7B VoiceGenerator).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import urllib.request
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from vllm import SamplingParams
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from vllm_omni import Omni
|
||||
|
||||
# H100-gated (8B). Picked up by the nightly H100 omni job
|
||||
# (-m "full_model and H100 and omni"); the tts mark keeps it consistent with
|
||||
# the other MOSS-TTS tests and any tts-scoped sweeps.
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
||||
|
||||
SAMPLE_RATE = 24_000
|
||||
REF_AUDIO_URL = "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS/main/assets/audio/reference_zh_1.wav"
|
||||
_MODEL = "OpenMOSS-Team/MOSS-TTS-v1.5"
|
||||
|
||||
_DEFAULT_SAMPLING = SamplingParams(
|
||||
temperature=1.7,
|
||||
top_p=0.8,
|
||||
top_k=25,
|
||||
max_tokens=512, # short for tests (~20 s at 24 kHz, ~12.5 tokens/s)
|
||||
seed=42,
|
||||
detokenize=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def ref_audio_path(tmp_path_factory) -> str:
|
||||
"""Download the upstream reference clip once per session.
|
||||
|
||||
Set ``MOSS_TTS_SKIP_ON_NET_FAIL=1`` to skip in air-gapped environments.
|
||||
"""
|
||||
target = tmp_path_factory.mktemp("moss_tts_v15_ref") / "zh_1.wav"
|
||||
try:
|
||||
with urllib.request.urlopen(REF_AUDIO_URL, timeout=30) as resp:
|
||||
target.write_bytes(resp.read())
|
||||
except Exception as exc: # noqa: BLE001
|
||||
msg = f"Cannot fetch reference clip {REF_AUDIO_URL}: {exc}"
|
||||
if os.environ.get("MOSS_TTS_SKIP_ON_NET_FAIL"):
|
||||
pytest.skip(msg)
|
||||
pytest.fail(msg)
|
||||
if not target.exists() or target.stat().st_size == 0:
|
||||
pytest.fail(f"Reference clip empty after download: {target}")
|
||||
return str(target)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def v15_engine():
|
||||
return Omni(model=_MODEL, stage_init_timeout=600)
|
||||
|
||||
|
||||
def _build_request(text: str, ref_audio_path: str) -> dict:
|
||||
return {
|
||||
"prompt": "<|im_start|>assistant\n",
|
||||
"additional_information": {
|
||||
"task_type": ["voice_clone"],
|
||||
"text": [text],
|
||||
"mode": ["voice_clone"],
|
||||
"prompt_audio_path": [ref_audio_path],
|
||||
"seed": [42],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _collect_audio(omni: Omni, request: dict) -> tuple[torch.Tensor, int]:
|
||||
for stage_outputs in omni.generate(request, _DEFAULT_SAMPLING):
|
||||
for req_output in stage_outputs.request_output:
|
||||
mm = req_output.outputs[0].multimodal_output
|
||||
assert mm is not None, "Expected non-None multimodal_output"
|
||||
audio = mm.get("audio")
|
||||
if audio is None:
|
||||
audio = mm.get("model_outputs")
|
||||
sr = mm.get("sr")
|
||||
assert audio is not None, "Expected 'audio' or 'model_outputs' in multimodal_output"
|
||||
if isinstance(audio, list):
|
||||
audio = torch.cat(
|
||||
[t.reshape(-1) for t in audio if isinstance(t, torch.Tensor) and t.numel() > 0],
|
||||
dim=0,
|
||||
)
|
||||
assert isinstance(audio, torch.Tensor), f"Expected Tensor, got {type(audio)}"
|
||||
return audio.reshape(-1).cpu(), int(sr.item()) if sr is not None else SAMPLE_RATE
|
||||
raise AssertionError("No stage outputs received")
|
||||
|
||||
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"})
|
||||
def test_moss_tts_v15_voice_clone(v15_engine, ref_audio_path):
|
||||
"""MOSS-TTS-v1.5 (8B): voice_clone produces non-empty 24 kHz audio."""
|
||||
req = _build_request("Hello, this is a MOSS-TTS v1.5 voice cloning test.", ref_audio_path)
|
||||
audio, sr = _collect_audio(v15_engine, req)
|
||||
|
||||
assert sr == SAMPLE_RATE, f"Expected {SAMPLE_RATE} Hz, got {sr}"
|
||||
assert audio.numel() > 0, "Audio tensor is empty"
|
||||
assert not torch.all(audio == 0), "Audio is silence"
|
||||
@@ -39,8 +39,7 @@ else:
|
||||
stage_config = get_cuda_graph_config()
|
||||
|
||||
# Create parameter combinations for model and stage config
|
||||
# Qwen2.5-Omni with TP=3 needs longer init timeout
|
||||
test_params = [(model, stage_config, {"stage_init_timeout": 1200, "init_timeout": 1800}) for model in models]
|
||||
test_params = [(model, stage_config) for model in models]
|
||||
|
||||
|
||||
def get_question(prompt_type="mix"):
|
||||
|
||||
@@ -9,8 +9,6 @@ Coverage:
|
||||
- Ulysses-SP
|
||||
- Ring-Attention
|
||||
- Layerwise Offloading
|
||||
- Hybrid Sharded Data Parallel
|
||||
- Tensor Parallelism + VAE Patch Parallelism
|
||||
|
||||
assert_diffusion_response validates successful generation and the expected
|
||||
512x512 resolution.
|
||||
@@ -40,28 +38,12 @@ BAGEL_HSDP_2_DEPLOY = modify_stage_config(
|
||||
BAGEL_CI_DEPLOY,
|
||||
updates={"stages": {0: {"devices": "0"}, 1: {"devices": "0,1"}}},
|
||||
)
|
||||
BAGEL_TP_VAE_PP_2_DEPLOY = modify_stage_config(
|
||||
BAGEL_CI_DEPLOY,
|
||||
updates={
|
||||
"stages": {
|
||||
0: {"devices": "0"},
|
||||
1: {
|
||||
"devices": "0,1",
|
||||
"parallel_config": {
|
||||
"tensor_parallel_size": 2,
|
||||
"vae_patch_parallel_size": 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _get_diffusion_feature_cases(model: str):
|
||||
"""Return L4 diffusion feature cases for Bagel.
|
||||
TeaCache, Cache-DiT, CFG-Parallel,
|
||||
Ulysses-SP, Ring-Attention, Layerwise Offloading,
|
||||
Hybrid Sharded Data Parallel, Tensor Parallelism, VAE Patch Parallelism.
|
||||
Ulysses-SP, Ring-Attention, Layerwise Offloading.
|
||||
"""
|
||||
|
||||
return [
|
||||
@@ -153,15 +135,6 @@ def _get_diffusion_feature_cases(model: str):
|
||||
id="parallel_hsdp_2",
|
||||
marks=HSDP_2_FEATURE_MARKS,
|
||||
),
|
||||
# Tensor Parallelism (TP) + VAE Patch Parallelism (size=2)
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
stage_config_path=BAGEL_TP_VAE_PP_2_DEPLOY,
|
||||
),
|
||||
id="tp_vae_patch_parallel_2",
|
||||
marks=PARALLEL_2_FEATURE_MARKS,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@@ -184,7 +157,6 @@ def test_bagel(
|
||||
- Ring-Attention (degree=2)
|
||||
- Layerwise Offloading
|
||||
- Hybrid Sharded Data Parallel (size=2)
|
||||
- Tensor Parallelism (TP) + VAE Patch Parallelism (size=2)
|
||||
|
||||
Validation is delegated to assert_diffusion_response in tests/helpers/assertions.py,
|
||||
which checks output dimensions and basic correctness.
|
||||
|
||||
@@ -1,155 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
E2E online tests for the full MOSS-TTS family via /v1/audio/speech.
|
||||
|
||||
The offline suite (``tests/e2e/offline_inference/test_moss_tts.py`` and
|
||||
``test_moss_tts_v1_5.py``) covers the engine path; this file covers the
|
||||
*serving* path (``serving_speech.py`` → ``_detect_moss_variant`` → the
|
||||
delay/realtime prompt builders), which the Nano online test does not exercise.
|
||||
|
||||
We use MOSS-TTS-Realtime (1.7B, ``MossTTSRealtime``) because it is the
|
||||
smallest full-family variant and fits a single L4 — the 8B delay variants
|
||||
(MOSS-TTS / v1.5) are H100-gated and stay in the offline suite. Realtime uses
|
||||
the local-transformer generation path and, like the ``tts`` delay variant,
|
||||
requires ``ref_audio`` for voice cloning.
|
||||
|
||||
The reference clip is fetched from the upstream repo
|
||||
(``assets/audio/reference_zh_1.wav``, ~1.5 MB) and passed as a base64 data URL.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import os
|
||||
import urllib.request
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
||||
|
||||
MODEL = "OpenMOSS-Team/MOSS-TTS-Realtime"
|
||||
REF_AUDIO_URL = "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS/HEAD/assets/audio/reference_zh_1.wav"
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def ref_audio_data_url() -> str:
|
||||
"""Fetch the upstream sample clip and return it as a base64 data URL.
|
||||
|
||||
The fetch failure is escalated to a hard failure (not pytest.skip) so a
|
||||
broken network path does not silently mask regressions in
|
||||
/v1/audio/speech. Set ``MOSS_TTS_SKIP_ON_NET_FAIL=1`` to opt into skipping
|
||||
in air-gapped environments.
|
||||
"""
|
||||
try:
|
||||
with urllib.request.urlopen(REF_AUDIO_URL, timeout=30) as resp:
|
||||
data = resp.read()
|
||||
except Exception as e: # noqa: BLE001
|
||||
msg = f"Cannot fetch upstream reference clip {REF_AUDIO_URL}: {e}"
|
||||
if os.environ.get("MOSS_TTS_SKIP_ON_NET_FAIL"):
|
||||
pytest.skip(msg)
|
||||
pytest.fail(msg)
|
||||
if not data:
|
||||
pytest.fail(f"Reference clip empty: {REF_AUDIO_URL}")
|
||||
return f"data:audio/wav;base64,{base64.b64encode(data).decode('ascii')}"
|
||||
|
||||
|
||||
def get_prompt(prompt_type: str = "text") -> str:
|
||||
"""Plain natural-sounding sentences for text-to-audio tests.
|
||||
|
||||
Avoid the model's own name in the input — the codec mispronounces it,
|
||||
which can trip transcript-similarity checks without a real regression.
|
||||
"""
|
||||
prompts = {
|
||||
"text": "Hello, this is a real-time voice cloning demo for testing.",
|
||||
"chinese": "你好,这是一段实时语音合成测试。",
|
||||
}
|
||||
return prompts.get(prompt_type, prompts["text"])
|
||||
|
||||
|
||||
tts_server_params = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
stage_config_path=get_deploy_config_path("moss_tts_realtime.yaml"),
|
||||
server_args=["--disable-log-stats"],
|
||||
),
|
||||
id="moss_tts_realtime",
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
||||
def test_text_to_audio_001(omni_server, openai_client, ref_audio_data_url) -> None:
|
||||
"""
|
||||
Realtime voice_clone via /v1/audio/speech, non-streaming.
|
||||
Deploy Setting: moss_tts_realtime.yaml
|
||||
Input Modal: text + reference audio (voice clone)
|
||||
Output Modal: audio (24 kHz, WAV)
|
||||
Input Setting: stream=False
|
||||
Datasets: single request
|
||||
"""
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"input": get_prompt(),
|
||||
"stream": False,
|
||||
"response_format": "wav",
|
||||
"ref_audio": ref_audio_data_url,
|
||||
}
|
||||
|
||||
openai_client.send_audio_speech_request(request_config)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
||||
def test_text_to_audio_002_streaming(omni_server, openai_client, ref_audio_data_url) -> None:
|
||||
"""
|
||||
Realtime voice_clone via /v1/audio/speech, streaming PCM.
|
||||
Deploy Setting: moss_tts_realtime.yaml
|
||||
Input Modal: text + reference audio (voice clone)
|
||||
Output Modal: audio (24 kHz, PCM stream)
|
||||
Input Setting: stream=True
|
||||
Datasets: single request
|
||||
|
||||
NOTE: ``min_hnr_db=-5.0`` mirrors the Nano streaming case — MOSS
|
||||
voice_clone output is intrinsically noisy by the HNR metric even when it
|
||||
sounds correct; the relaxed threshold still catches catastrophic decoder
|
||||
failures (HNR far below 0) without flaking on normal output.
|
||||
"""
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"input": get_prompt(),
|
||||
"stream": True,
|
||||
"response_format": "pcm",
|
||||
"ref_audio": ref_audio_data_url,
|
||||
"min_hnr_db": -5.0,
|
||||
}
|
||||
|
||||
openai_client.send_audio_speech_request(request_config)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
||||
def test_text_to_audio_003_chinese(omni_server, openai_client, ref_audio_data_url) -> None:
|
||||
"""
|
||||
Realtime voice_clone via /v1/audio/speech, Chinese input.
|
||||
Deploy Setting: moss_tts_realtime.yaml
|
||||
Input Modal: text (Chinese) + reference audio (voice clone)
|
||||
Output Modal: audio (24 kHz, WAV)
|
||||
Input Setting: stream=False
|
||||
Datasets: single request
|
||||
"""
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"input": get_prompt("chinese"),
|
||||
"stream": False,
|
||||
"response_format": "wav",
|
||||
"ref_audio": ref_audio_data_url,
|
||||
}
|
||||
|
||||
openai_client.send_audio_speech_request(request_config)
|
||||
@@ -97,7 +97,7 @@ def _get_diffusion_feature_cases(model: str):
|
||||
"--vae-patch-parallel-size",
|
||||
"2",
|
||||
"--vae-use-tiling",
|
||||
"--diffusion-quantization-config",
|
||||
"--quantization-config",
|
||||
'{"method":"fp8"}',
|
||||
],
|
||||
),
|
||||
|
||||
@@ -312,7 +312,7 @@ def test_initialize_stages_exposes_logical_stage_views_and_builds_top_level_inpu
|
||||
monkeypatch.setattr(
|
||||
engine_mod,
|
||||
"build_llm_stage_output_processor",
|
||||
lambda plan, _cfg, **_kw: stage0_output_processor if plan.stage_idx == 0 else stage1_output_processor,
|
||||
lambda plan, _cfg: stage0_output_processor if plan.stage_idx == 0 else stage1_output_processor,
|
||||
)
|
||||
monkeypatch.setattr(engine_mod, "build_stage0_input_processor", lambda _cfg: top_level_input_processor)
|
||||
|
||||
@@ -640,6 +640,38 @@ def test_initialize_llm_replica_passes_stage_init_timeout_to_complete_stage_hand
|
||||
assert captured_timeout == 302
|
||||
|
||||
|
||||
def test_async_omni_engine_reads_tokenizer_from_engine_args(monkeypatch):
|
||||
import vllm_omni.engine.async_omni_engine as engine_mod
|
||||
|
||||
class DummyThread:
|
||||
def __init__(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
def start(self):
|
||||
pass
|
||||
|
||||
def is_alive(self):
|
||||
return False
|
||||
|
||||
fake_engine_args = types.SimpleNamespace(
|
||||
_explicit_fields=frozenset({"tokenizer"}),
|
||||
explicit_kwargs=lambda: {"tokenizer": "/tokenizer/from-engine-args"},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(engine_mod.threading, "Thread", DummyThread)
|
||||
monkeypatch.setattr(
|
||||
AsyncOmniEngine,
|
||||
"_resolve_stage_configs",
|
||||
lambda self, model, kwargs: ("dummy-config", [types.SimpleNamespace(engine_args={})]),
|
||||
)
|
||||
monkeypatch.setattr(AsyncOmniEngine, "_wait_for_orchestrator_init", lambda *_, **__: None)
|
||||
|
||||
engine = AsyncOmniEngine("dummy-model", engine_args=fake_engine_args)
|
||||
|
||||
assert engine.tokenizer == "/tokenizer/from-engine-args"
|
||||
engine.shutdown()
|
||||
|
||||
|
||||
def test_build_engine_args_cli_tokenizer_overrides_inferred_base_tokenizer(tmp_path):
|
||||
from vllm_omni.engine.stage_init_utils import build_engine_args_dict
|
||||
|
||||
|
||||
@@ -1157,37 +1157,3 @@ async def test_multi_replica_cfg_companion_inherits_parent_affinity(orchestrator
|
||||
assert stage0_r1.add_request_calls[1][0].request_id == "parent-neg"
|
||||
finally:
|
||||
await _shutdown_orchestrator(orchestrator_fixture)
|
||||
|
||||
|
||||
def test_orchestrator_does_not_re_introduce_global_stats_throttle() -> None:
|
||||
"""Regression: each (stage, replica) must independently publish its wrapped
|
||||
vllm:* stats when its scheduler emits non-None scheduler_stats.
|
||||
|
||||
A previous version of Orchestrator carried a global self._last_stats_ts /
|
||||
_stats_interval_s gate around _stat_logger.record(). Because
|
||||
OmniSchedulerMixin.make_stats() already throttles at 1 Hz per scheduler
|
||||
(one per (stage, replica)), the extra global gate starved every replica
|
||||
other than the first to emit within each second — their {stage, replica}
|
||||
gauges/counters went stale.
|
||||
|
||||
The fix removed the global gate entirely; the only signal needed is
|
||||
'this replica's scheduler emitted non-None scheduler_stats'. This test
|
||||
fails loudly if someone reintroduces the global throttle.
|
||||
"""
|
||||
import inspect
|
||||
|
||||
from vllm_omni.engine.orchestrator import Orchestrator
|
||||
|
||||
source = inspect.getsource(Orchestrator)
|
||||
assert "_last_stats_ts" not in source, (
|
||||
"Orchestrator must not gate stat recording on a global timestamp. "
|
||||
"OmniSchedulerMixin.make_stats() already throttles per scheduler "
|
||||
"(per (stage, replica)); an outer global gate starves all but the "
|
||||
"first replica to emit within each 1s window."
|
||||
)
|
||||
assert "_stats_interval_s" not in source
|
||||
assert "raw_outputs.scheduler_stats is not None" in source, (
|
||||
"Orchestrator must gate stat recording solely on "
|
||||
"raw_outputs.scheduler_stats being non-None — the per-scheduler 1Hz "
|
||||
"throttle in OmniSchedulerMixin.make_stats() is the only gate needed."
|
||||
)
|
||||
@@ -403,6 +403,39 @@ class TestSingleStageModeDetection:
|
||||
assert engine.single_stage_mode is True
|
||||
assert engine._single_stage_id_filter is None
|
||||
|
||||
def test_engine_args_create_only_forwards_explicit_fields(self, mocker: MockerFixture):
|
||||
from vllm_omni.engine.arg_utils import OmniEngineArgs
|
||||
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
def fake_resolve(self, model: str, kwargs: dict[str, Any]):
|
||||
captured.update(kwargs)
|
||||
return "/fake/path", [_make_stage_cfg(0)]
|
||||
|
||||
mocker.patch.object(AsyncOmniEngine, "_resolve_stage_configs", fake_resolve)
|
||||
mocker.patch.object(AsyncOmniEngine, "_bootstrap_orchestrator")
|
||||
mock_thread_cls = mocker.patch("threading.Thread")
|
||||
mock_future_cls = mocker.patch("concurrent.futures.Future")
|
||||
mock_future = mocker.Mock()
|
||||
mock_future.result.return_value = mocker.Mock()
|
||||
mock_future_cls.return_value = mock_future
|
||||
mock_thread = mocker.Mock()
|
||||
mock_thread.is_alive.return_value = False
|
||||
mock_thread_cls.return_value = mock_thread
|
||||
|
||||
ea = OmniEngineArgs.create(model="ignored", gpu_memory_utilization=0.5)
|
||||
AsyncOmniEngine(model="fake-model", engine_args=ea)
|
||||
|
||||
assert captured["gpu_memory_utilization"] == 0.5
|
||||
assert "model" not in captured
|
||||
assert "max_num_seqs" not in captured
|
||||
|
||||
def test_bare_engine_args_rejected(self, mocker: MockerFixture):
|
||||
from vllm_omni.engine.arg_utils import OmniEngineArgs
|
||||
|
||||
with pytest.raises(TypeError, match="OmniEngineArgs.create"):
|
||||
self._make_engine_no_thread(mocker, engine_args=OmniEngineArgs(model="fake-model"))
|
||||
|
||||
def test_master_address_and_port_stored(self, mocker: MockerFixture):
|
||||
engine = self._make_engine_no_thread(
|
||||
mocker,
|
||||
|
||||
Loaded 100 of 272 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user