Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d98f94fa29 | ||
|
|
908e7d32d9 | ||
|
|
d6f1d3d61c | ||
|
|
c39ac9fffd | ||
|
|
580a0efe40 | ||
|
|
329c98ff9d | ||
|
|
36a5e9cfd2 | ||
|
|
0b04f2a0d6 | ||
|
|
5db89b41c4 | ||
|
|
e500c00545 | ||
|
|
8ddcb3621f | ||
|
|
2e6c83c89c | ||
|
|
d9063d7141 | ||
|
|
8c33793ffc | ||
|
|
4024a8ccd6 | ||
|
|
fb563fbd19 | ||
|
|
0fd2620d56 | ||
|
|
0d0f4e669e | ||
|
|
d94108693a | ||
|
|
d612233bef | ||
|
|
f6e8b77642 | ||
|
|
d8edf0c9d5 | ||
|
|
67bd02f767 | ||
|
|
0814d845dc | ||
|
|
bb973cb10c | ||
|
|
fb3abb38fb | ||
|
|
eb81914c64 | ||
|
|
1951dca786 | ||
|
|
d71cacc74c | ||
|
|
465f1464d2 | ||
|
|
f6894f4308 | ||
|
|
a2d1dafc65 | ||
|
|
6dc4638a7e | ||
|
|
d35cdd2dab | ||
|
|
0958e62ecc | ||
|
|
3901ad8165 | ||
|
|
b84c99c122 | ||
|
|
733f4f6a36 | ||
|
|
67d14b5ac4 | ||
|
|
64efa66869 | ||
|
|
6646a66d41 | ||
|
|
378359fc11 | ||
|
|
9c260b3774 | ||
|
|
9177b37e2f | ||
|
|
50bc51ffdf | ||
|
|
044240d8eb | ||
|
|
f4d922d8f6 | ||
|
|
eaa16e1941 | ||
|
|
8a6786731e |
No files matched your search
+14
-4
@@ -26,23 +26,33 @@ steps:
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
# L2 Test
|
||||
# L2 Test — PR ready label (diff-aware) or main+NIGHTLY=1 (--e2e, full E2E group)
|
||||
- label: "Upload Ready Pipeline"
|
||||
depends_on: image-build
|
||||
key: upload-ready-pipeline
|
||||
if: __UPLOAD_READY_IF__
|
||||
commands:
|
||||
- python3 .buildkite/scripts/upload_pipeline.py --upload .buildkite/test-ready.yml
|
||||
- |
|
||||
EXTRA=""
|
||||
if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then
|
||||
EXTRA="--e2e"
|
||||
fi
|
||||
python3 .buildkite/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/test-ready.yml
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
# L3 Test
|
||||
# L3 Test — PR merge-test label (diff-aware) or main+NIGHTLY=1 (--e2e, full E2E group)
|
||||
- label: "Upload Merge Pipeline"
|
||||
depends_on: image-build
|
||||
key: upload-merge-pipeline
|
||||
if: __UPLOAD_MERGE_IF__
|
||||
commands:
|
||||
- python3 .buildkite/scripts/upload_pipeline.py --upload .buildkite/test-merge.yml
|
||||
- |
|
||||
EXTRA=""
|
||||
if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then
|
||||
EXTRA="--e2e"
|
||||
fi
|
||||
python3 .buildkite/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/test-merge.yml
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ steps:
|
||||
depends_on: image-build
|
||||
key: upload-ready-pipeline
|
||||
commands:
|
||||
- buildkite-agent pipeline upload .buildkite/test-ready.yml
|
||||
- python3 .buildkite/scripts/upload_pipeline.py --upload --all .buildkite/test-ready.yml
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
@@ -23,7 +23,7 @@ steps:
|
||||
depends_on: image-build
|
||||
key: upload-merge-pipeline
|
||||
commands:
|
||||
- buildkite-agent pipeline upload .buildkite/test-merge.yml
|
||||
- python3 .buildkite/scripts/upload_pipeline.py --upload --all .buildkite/test-merge.yml
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
@@ -33,6 +33,6 @@ steps:
|
||||
key: upload-nightly-pipeline
|
||||
if: build.env("NIGHTLY") == "1"
|
||||
commands:
|
||||
- buildkite-agent pipeline upload .buildkite/test-nightly.yml
|
||||
- python3 .buildkite/scripts/upload_pipeline.py --upload --all .buildkite/test-nightly.yml
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
@@ -13,13 +13,19 @@ Test pipeline mode (e.g. test-merge.yml):
|
||||
recognize this uploader-only key).
|
||||
|
||||
Usage:
|
||||
python3 upload_pipeline.py [--upload] <pipeline.yml>
|
||||
python3 upload_pipeline.py [--upload] [--all | --e2e] <pipeline.yml>
|
||||
|
||||
# Bootstrap (replaces upload_pipeline_with_skip_ci.sh):
|
||||
python3 upload_pipeline.py --upload .buildkite/pipeline.yml
|
||||
|
||||
# Test pipeline (replaces upload_test_pipeline_with_diff_skip.py):
|
||||
python3 upload_pipeline.py --upload .buildkite/test-merge.yml
|
||||
|
||||
# Full suite (rebase pipeline): keep all steps, still strip the uploader-only key:
|
||||
python3 upload_pipeline.py --upload --all .buildkite/test-merge.yml
|
||||
|
||||
# Nightly L2 E2E only: keep the E2E Test group, ignore source_file_dependencies:
|
||||
python3 upload_pipeline.py --upload --e2e .buildkite/test-ready.yml
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -44,6 +50,7 @@ LOG = "upload_pipeline"
|
||||
ROOT = Path(__file__).resolve().parent.parent.parent
|
||||
DOC_SEP = "\n---\n"
|
||||
BOOTSTRAP_MARKER = "__IMAGE_BUILD_IF__"
|
||||
E2E_GROUP_MARKER = "E2E Test"
|
||||
|
||||
|
||||
def _log(message: str) -> None:
|
||||
@@ -319,18 +326,18 @@ def render_bootstrap_pipeline(text: str, *, skip_ci: bool) -> str:
|
||||
nightly_only = (
|
||||
'(build.pull_request.labels includes "nightly-test") || (build.branch == "main" && build.env("NIGHTLY") == "1")'
|
||||
)
|
||||
nightly_main = 'build.branch == "main" && build.env("NIGHTLY") == "1"'
|
||||
ready_pr = 'build.branch != "main" && build.pull_request.labels includes "ready"'
|
||||
merge_main = 'build.branch == "main" && build.env("NIGHTLY") != "1" && build.env("WEEKLY") != "1"'
|
||||
merge_pr = 'build.branch != "main" && build.pull_request.labels includes "merge-test"'
|
||||
if skip_ci:
|
||||
image_if = f"'{nightly_only}'"
|
||||
ready_if = "'false'"
|
||||
merge_if = "'false'"
|
||||
ready_if = f"'({nightly_main})'"
|
||||
merge_if = f"'({nightly_main})'"
|
||||
else:
|
||||
image_if = "'true'"
|
||||
ready_if = '\'build.branch != "main" && build.pull_request.labels includes "ready"\''
|
||||
merge_if = (
|
||||
'\'(build.branch == "main" && build.env("NIGHTLY") != "1" '
|
||||
'&& build.env("WEEKLY") != "1") || '
|
||||
'(build.branch != "main" && build.pull_request.labels includes "merge-test")\''
|
||||
)
|
||||
ready_if = f"'(({nightly_main}) || ({ready_pr}))'"
|
||||
merge_if = f"'(({nightly_main}) || (({merge_main}) || ({merge_pr})))'"
|
||||
|
||||
return (
|
||||
continuation.replace("__IMAGE_BUILD_IF__", image_if)
|
||||
@@ -393,6 +400,20 @@ def _filter_steps(steps: list[Any], changed_files: list[str]) -> list[Any]:
|
||||
return filtered
|
||||
|
||||
|
||||
def _is_e2e_group(step: dict[str, Any]) -> bool:
|
||||
group = step.get("group")
|
||||
return isinstance(group, str) and E2E_GROUP_MARKER in group
|
||||
|
||||
|
||||
def _select_e2e_group_steps(steps: list[Any]) -> list[Any]:
|
||||
selected = [step for step in steps if isinstance(step, dict) and _is_e2e_group(step)]
|
||||
if not selected:
|
||||
_log(f"no group matching {E2E_GROUP_MARKER!r} found")
|
||||
else:
|
||||
_log(f"keep {len(selected)} group(s) matching {E2E_GROUP_MARKER!r}")
|
||||
return selected
|
||||
|
||||
|
||||
def _strip_uploader_metadata_from_steps(steps: list[Any]) -> list[Any]:
|
||||
"""Remove uploader-only keys while keeping all steps (no diff filtering)."""
|
||||
stripped: list[Any] = []
|
||||
@@ -419,11 +440,16 @@ def _strip_uploader_metadata_from_steps(steps: list[Any]) -> list[Any]:
|
||||
def render_test_pipeline(
|
||||
doc: dict[str, Any],
|
||||
changed_files: list[str] | None,
|
||||
*,
|
||||
e2e_only: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
steps = doc.get("steps")
|
||||
if not isinstance(steps, list):
|
||||
return doc
|
||||
if changed_files is not None:
|
||||
if e2e_only:
|
||||
steps = _select_e2e_group_steps(steps)
|
||||
steps = _strip_uploader_metadata_from_steps(steps)
|
||||
elif changed_files is not None:
|
||||
steps = _filter_steps(steps, changed_files)
|
||||
else:
|
||||
steps = _strip_uploader_metadata_from_steps(steps)
|
||||
@@ -437,11 +463,22 @@ def resolve_pipeline_path(arg: str) -> Path:
|
||||
return ROOT / path
|
||||
|
||||
|
||||
def render_pipeline(path: Path) -> str:
|
||||
def render_pipeline(
|
||||
path: Path,
|
||||
*,
|
||||
force_all: bool = False,
|
||||
e2e_only: bool = False,
|
||||
) -> str:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
diff_range = resolve_diff_range()
|
||||
changed_files = resolve_changed_files()
|
||||
|
||||
# ``--all`` forces the keep-all-steps path (no diff-aware skipping) while still
|
||||
# stripping ``source_file_dependencies``. Used by the rebase pipeline so main builds
|
||||
# run the full e2e suite (see .buildkite/rebase-pipeline.yaml).
|
||||
if force_all or e2e_only:
|
||||
changed_files = None
|
||||
|
||||
if BOOTSTRAP_MARKER in text:
|
||||
rendered = render_bootstrap_pipeline(
|
||||
text,
|
||||
@@ -453,7 +490,7 @@ def render_pipeline(path: Path) -> str:
|
||||
if not isinstance(doc, dict):
|
||||
raise ValueError(f"invalid pipeline YAML: {path}")
|
||||
|
||||
doc = render_test_pipeline(doc, changed_files)
|
||||
doc = render_test_pipeline(doc, changed_files, e2e_only=e2e_only)
|
||||
|
||||
return yaml.safe_dump(doc, sort_keys=False)
|
||||
|
||||
@@ -480,6 +517,17 @@ def main() -> int:
|
||||
action="store_true",
|
||||
help="Pipe rendered YAML to buildkite-agent pipeline upload",
|
||||
)
|
||||
mode = parser.add_mutually_exclusive_group()
|
||||
mode.add_argument(
|
||||
"--all",
|
||||
action="store_true",
|
||||
help="Keep all steps (disable diff-aware skipping); still strips source_file_dependencies",
|
||||
)
|
||||
mode.add_argument(
|
||||
"--e2e",
|
||||
action="store_true",
|
||||
help="Keep only the E2E Test group; disable diff-aware skipping for those steps",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
path = resolve_pipeline_path(args.pipeline)
|
||||
@@ -487,7 +535,7 @@ def main() -> int:
|
||||
_log(f"missing pipeline file: {path}")
|
||||
return 1
|
||||
|
||||
rendered = render_pipeline(path)
|
||||
rendered = render_pipeline(path, force_all=args.all, e2e_only=args.e2e)
|
||||
if args.upload:
|
||||
upload_to_buildkite(rendered)
|
||||
else:
|
||||
|
||||
@@ -363,6 +363,42 @@ steps:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "TTS · IndexTTS2 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_indextts2.py
|
||||
- tests/e2e/offline_inference/test_indextts2.py
|
||||
- tests/model_executor/models/indextts2/
|
||||
- tests/model_executor/stage_input_processors/test_indextts2.py
|
||||
- tests/assets/indextts2/
|
||||
- vllm_omni/model_executor/models/indextts2/
|
||||
- vllm_omni/model_executor/stage_input_processors/indextts2.py
|
||||
- vllm_omni/deploy/indextts2.yaml
|
||||
- vllm_omni/deploy/indextts2_low_latency.yaml
|
||||
- vllm_omni/entrypoints/openai/serving_speech.py
|
||||
- vllm_omni/entrypoints/openai/tts_adapters/indextts2.py
|
||||
- vllm_omni/config/pipeline_registry.py
|
||||
- vllm_omni/model_executor/models/registry.py
|
||||
timeout_in_minutes: 20
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
pytest -s -v tests/e2e/online_serving/test_indextts2.py tests/e2e/offline_inference/test_indextts2.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
"
|
||||
agents:
|
||||
queue: "gpu_4_queue"
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
- "HF_TOKEN"
|
||||
gpus: all
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "Omni · Qwen3-Omni Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_qwen3_omni.py
|
||||
@@ -509,6 +545,34 @@ steps:
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "TTS · SoulX-Singer Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_soulxsinger.py
|
||||
- vllm_omni/diffusion/models/soulx_singer/
|
||||
- vllm_omni/deploy/soulxsinger_svc.yaml
|
||||
- vllm_omni/deploy/soulxsinger_svs.yaml
|
||||
timeout_in_minutes: 30
|
||||
commands:
|
||||
- |
|
||||
timeout 30m bash -c "
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
pytest -s -v tests/e2e/offline_inference/test_soulxsinger.py -k 'svc or precomputed' -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
"
|
||||
agents:
|
||||
queue: "gpu_1_queue"
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
shm-size: "8gb"
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
- "HF_TOKEN"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "Diffusion · Qwen Image Edit Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen_image_edit.py
|
||||
|
||||
@@ -1279,7 +1279,7 @@ steps:
|
||||
- label: ":full_moon: Diffusion X2V · Function Test"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_hunyuan_video_15_expansion.py tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and cuda" --run-level "full_model"
|
||||
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_hunyuan_video_15_expansion.py tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py tests/e2e/online_serving/test_ltx2_expansion.py -m "full_model and cuda" --run-level "full_model"
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
@@ -1322,6 +1322,7 @@ steps:
|
||||
- pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model
|
||||
- pytest -s -v tests/e2e/accuracy/test_ltx2_3_video_similarity.py -m full_model --run-level full_model
|
||||
- pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m full_model --run-level full_model
|
||||
- pytest -s -v tests/e2e/accuracy/test_video_streaming_output_similarity.py -m full_model --run-level full_model
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
|
||||
@@ -85,6 +85,66 @@ steps:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "Reliability Test - hunyuan-image"
|
||||
timeout_in_minutes: 180
|
||||
depends_on: upload-weekly-pipeline
|
||||
if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
|
||||
commands:
|
||||
- pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow"
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 4
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "Reliability Test - voxcpm2"
|
||||
timeout_in_minutes: 180
|
||||
depends_on: upload-weekly-pipeline
|
||||
if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
|
||||
commands:
|
||||
- pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m "slow"
|
||||
agents:
|
||||
queue: "gpu_1_queue"
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
shm-size: "8gb"
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
- "HF_TOKEN"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
# Invalid HTTP / WS payloads vs live Omni servers (Issue #3649 tracks known skips).
|
||||
- group: "Reliability Test - Invalid parameters Test"
|
||||
depends_on: upload-weekly-pipeline
|
||||
|
||||
@@ -63,6 +63,8 @@ If multiple rows apply (e.g., a diffusion model is also a new model), union the
|
||||
|
||||
Ask: "Quick mode or full mode?" Then walk the checklist for the detected PR type from [references/checklists.md](references/checklists.md). Each item produces ✓, ✗, or ⚠.
|
||||
|
||||
**Also run the Code-Quality sweep on every PR**, regardless of type or mode: the five diff-scoped checks in [references/code-quality.md](references/code-quality.md) — kwargs fragility, broad-except swallow, `Any`/wrong type hints, hot-path `.clone()`/`deepcopy`, and event-loop blocking — plus the advisory conventions (log level, structured logging, synchronization, cleanup, dependencies, naming) in [checklists.md](references/checklists.md). These count only lines the PR *adds* — the pre-existing backlog across the repo is out of scope.
|
||||
|
||||
### Step 5: Print Report
|
||||
|
||||
```
|
||||
@@ -74,6 +76,7 @@ Pre-check report for <branch>
|
||||
Dimension Result
|
||||
───────────────── ──────
|
||||
PR title format ✓
|
||||
Code quality ⚠ 1 broad except, 2 Any hints
|
||||
PR desc integrity ✓
|
||||
Registry/config ✓
|
||||
Dead code ⚠ 2 warnings
|
||||
|
||||
@@ -35,6 +35,31 @@ PR title must follow the project convention documented in `docs/contributing/REA
|
||||
|
||||
---
|
||||
|
||||
## All PRs: Code Quality (run on every PR, quick & full)
|
||||
|
||||
Diff-scoped sweep for five fragility patterns that are pervasive in the existing codebase — **only lines the PR adds count**, not the pre-existing backlog. Detection commands, codebase examples, and full severity rules live in [code-quality.md](code-quality.md).
|
||||
|
||||
- [ ] **No new `**kwargs` string-lookup plumbing:** the diff adds no `**kwargs` + `kwargs["..."]` / `kwargs.get("...")` / `"..." in kwargs`. ⚠ any new instance; ✗ if a new string key is duplicated across ≥2 files without a shared constant, or unknown keys are silently dropped on a fail-fast path.
|
||||
- [ ] **No new broad exception swallow:** the diff adds no `except Exception:` / bare `except:`. ⚠ any new instance; ✗ for `except: pass` / `except Exception: return None` / `continue` on a fail-fast path (init, config validation, weight loading, request handling).
|
||||
- [ ] **No new `Any` / wrong type hints:** the diff adds no `: Any` / `-> Any` / untyped production signature, and no new `SimpleNamespace` in tests faking an object that already has a real typed stub. ⚠ any new instance; ✗ for a *wrong* (actively misleading) annotation or a `SimpleNamespace` test fake of a typed object.
|
||||
- [ ] **No new hot-path copy:** the diff adds no `.clone()` / `.copy_()` / `copy.deepcopy(...)` inside a per-step loop or per-request path. ⚠ any new instance; ✗ for `.clone()` of a full latent/activations tensor inside an AR or diffusion step loop without a comment, or `deepcopy` of a scheduler / sampling-config on the request path.
|
||||
- [ ] **No new event-loop blocking:** the diff adds no `time.sleep` / blocking HTTP (`requests`, `urllib`) / `await` inside a held lock in `async def` code. ⚠ any new instance; ✗ for blocking sleep or HTTP on the serving path (`engine/`, `entrypoints/`, async `connectors/`), or a lock held across an `await`. Blocking inside a dedicated worker thread is fine.
|
||||
|
||||
Roll the ⚠/✗ counts into the report as a single **Code quality** dimension row.
|
||||
|
||||
### Conventions (eyeball — no grep)
|
||||
|
||||
Judgment calls that don't grep cleanly — apply them while reading the diff, not as a mechanical check:
|
||||
|
||||
- **Right log level** — `info` is for end-user-visible events; routine internal events are `debug`; hot paths are `trace` or removed. Logging is not free.
|
||||
- **Structured fields over f-strings** — pass dynamic values as structured fields (`logger.info("msg", extra={...})`), not interpolated strings, so they stay greppable and parseable.
|
||||
- **No reflexive synchronization** — don't add `threading.Lock` / `asyncio.Lock` / `deepcopy` / re-wrap shared state unless there is real concurrent mutation; the GIL covers most Python code. Owners decide their own synchronization — don't pre-lock in a constructor.
|
||||
- **Context managers for cleanup** — use `with` / `contextlib` for files, locks, CUDA streams, and profilers; not raw `open()` + manual `.close()` or ad-hoc `finally`.
|
||||
- **No new dependencies without justification** — prefer the stdlib and existing vllm-omni deps; a new top-level dep needs a reason (check `pyproject.toml` / `requirements*.txt`).
|
||||
- **Naming / interface discipline** — don't rename or change interfaces just for taste in this PR, but do fix names that actively mislead (e.g. `serve` implying a long-running server, `Instance` where multiple instance kinds coexist).
|
||||
|
||||
---
|
||||
|
||||
## Bug Fix PRs
|
||||
|
||||
### Quick
|
||||
@@ -49,7 +74,7 @@ PR title must follow the project convention documented in `docs/contributing/REA
|
||||
- [ ] **Regression test exists:** `git diff --name-only` includes at least one `tests/` file
|
||||
- [ ] **Regression test reproduces the original error:** the test would fail on main, pass on this branch
|
||||
- [ ] **Fix matches root cause exactly:** no "fix the symptom + something else" — if the root cause is dtype, the fix is dtype, not dtype + reformatting
|
||||
- [ ] **No silent failure risk:** no bare `except: pass`, no `try: ... except: return None`, no empty fallback added
|
||||
- [ ] **No silent failure risk:** no bare `except: pass`, no `try: ... except: return None`, no empty fallback added. See the broad-except pattern in [code-quality.md](code-quality.md#2-broad-exception-swallow) for the full severity rule (⚠ any new broad catch; ✗ for a swallow on a fail-fast path).
|
||||
- [ ] **Upstream pattern match:** the fix follows the same pattern as existing code for similar cases (grep for analogous `from_pretrained` calls, etc.)
|
||||
- [ ] **Environment documented:** torch/transformers/vllm versions listed if the bug is version-dependent
|
||||
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
# Code-Quality Patterns
|
||||
|
||||
Five fragility patterns to sweep on **every** PR, in both quick and full mode. These are pervasive in the existing codebase (hundreds of pre-existing sites) — so the checks are **diff-scoped**: only lines the PR *adds* count. Whole-repo greps light up every PR with the backlog and are useless.
|
||||
|
||||
All commands assume:
|
||||
```bash
|
||||
BASE="$(git merge-base HEAD origin/main)"
|
||||
```
|
||||
|
||||
**Severity:** ⚠ for any *new* instance introduced by the diff; ✗ for the acute sub-cases defined per pattern. Acute ✗ findings belong in the report's blocking count.
|
||||
|
||||
---
|
||||
|
||||
## 1. `**kwargs` string-lookup passthrough
|
||||
|
||||
### Anti-pattern
|
||||
```python
|
||||
def forward(self, **kwargs):
|
||||
# BLOCKER-prone: raw dict + string keys duplicated across files
|
||||
if "runtime_additional_information" in kwargs and \
|
||||
"model_intermediate_buffer" not in kwargs:
|
||||
...
|
||||
```
|
||||
Real instances of this exact guard are copy-pasted across `cosyvoice3.py`, `indextts2_talker.py` (×2), `indextts2_s2mel_decoder.py`, `qwen3_tts_talker.py`, `qwen3_omni.py`. A typo in any string key silently no-ops; there is no type check, no IDE completion, and unknown keys are silently dropped.
|
||||
|
||||
### Why dangerous
|
||||
Raw `dict` + `**kwargs` + lists of param-name strings is **fragile by construction**: the compiler can't see a missing or misspelled key, so failures are silent. Each duplicated copy is a new place for the strings to drift.
|
||||
|
||||
### Severity
|
||||
- ⚠ any *new* `**kwargs` signature whose body does `kwargs["..."]` / `kwargs.get("...")` / `"..." in kwargs`.
|
||||
- ✗ when the diff introduces a **new** string key that is duplicated across ≥2 files without a shared module constant, or when `**kwargs` plumbing **silently drops unknown keys** on a fail-fast path (init, config validation, weight loading).
|
||||
|
||||
### Detect (diff-scoped)
|
||||
```bash
|
||||
git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" \
|
||||
| grep -E '\*\*kwargs|kwargs(\.get|\[)|"[a-z_]+"\s+in\s+kwargs'
|
||||
```
|
||||
|
||||
### Fix
|
||||
Prefer explicit typed params. The repo already has the right primitive: `msgspec.Struct` with `forbid_unknown_fields=True` rejects unknown keys instead of silently dropping them — see `_StructBase` in `vllm_omni/data_entry_keys.py:105` and `MoriPullRequest` in `vllm_omni/distributed/omni_connectors/connectors/mori_transfer_engine_connector.py:81`. A `@dataclass` or `TypedDict` works too.
|
||||
|
||||
If `**kwargs` is unavoidable (vLLM base-class compat), centralize the key strings as module-level constants behind **one** typed accessor, instead of repeating the literal across files.
|
||||
|
||||
---
|
||||
|
||||
## 2. Broad exception swallow
|
||||
|
||||
### Anti-pattern
|
||||
```python
|
||||
# BLOCKER-prone: catches everything, returns None on any failure
|
||||
try:
|
||||
evt = build_event(...)
|
||||
return evt
|
||||
except Exception:
|
||||
return None
|
||||
```
|
||||
Real clusters: `vllm_omni/metrics/stats.py:240,274,367,450` and `vllm_omni/metrics/{modality,utils}.py` — all `except Exception: return None`.
|
||||
|
||||
### Why dangerous
|
||||
A generic `except Exception:` is **too flexible**. If we catch an exception type we don't expect, it is almost always an unhandled edge case that needs to be *fixed* — not swallowed into `None`/`pass`/`continue`. Broad catches turn fixable bugs into silent wrong behavior and make debugging impossible.
|
||||
|
||||
The repo backs this with ruff `BLE001` (broad-exception catch); it is currently suppressed via `# noqa: BLE001` in `vllm_omni/patch.py:413`. Prefer fixing over suppressing.
|
||||
|
||||
### Severity
|
||||
- ⚠ any *new* `except Exception:` / `except BaseException:` / bare `except:`.
|
||||
- ✗ for `except:` / `except Exception: pass|return None|continue` on a **fail-fast path** (init, config validation, weight loading, request handling, connector setup). This extends the Bug-Fix "No silent failure risk" ✗ in [checklists.md](checklists.md).
|
||||
|
||||
### Detect (diff-scoped)
|
||||
```bash
|
||||
# Any new broad catch (⚠ baseline):
|
||||
git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" \
|
||||
| grep -E 'except\s*(Exception|BaseException)?\s*:'
|
||||
# Note: this matches only the immediate-colon swallow forms (`except :`,
|
||||
# `except Exception:`) — it deliberately excludes specific catches like
|
||||
# `except ValueError:` and the `except Exception as e:` variant. Eyeball
|
||||
# the latter; `as e` usually means it is logged, but still verify.
|
||||
```
|
||||
|
||||
### Fix
|
||||
Catch the **specific** types you actually expect (`ValueError`, `KeyError`, `AttributeError`, `OSError`, `torch.cuda.OutOfMemoryError`, `msgspec.ValidationError`, …). At a genuine top-level / best-effort boundary (metrics, signal handlers), at minimum **log** the exception; never swallow into a bare `pass`/`return None` on a path that should fail loudly.
|
||||
|
||||
---
|
||||
|
||||
## 3. `Any` / wrong type annotations
|
||||
|
||||
### Anti-pattern
|
||||
```python
|
||||
# ⚠ Any param + Any return — no contract, no narrowing
|
||||
def _extract_mm_output(engine_outputs: Any) -> dict[str, Any]:
|
||||
...
|
||||
```
|
||||
|
||||
### Why dangerous (root cause)
|
||||
`Any` disables the type checker for that position, so typos and wrong shapes pass silently. The repo has 636 `: Any` params, 244 `-> Any` returns, and **1,173 `SimpleNamespace` in `tests/`**. The `Any` leak is largely test-driven: vibe-coded unit tests fake objects with `SimpleNamespace`, which forces the production code consuming them to be typed `Any`, and the `Any` then propagates outward into public signatures.
|
||||
|
||||
### Severity
|
||||
- ⚠ any *new* `: Any` / `-> Any` / fully-untyped signature in production code.
|
||||
- ✗ for a **wrong** annotation (actively misleading — e.g. `-> bool` that returns `Optional[bool]`), or a *new* `SimpleNamespace` in a test that mimics an object which already has a real typed stub / `@dataclass` / `TypedDict` / `Protocol`.
|
||||
|
||||
### Detect (diff-scoped)
|
||||
```bash
|
||||
# Any leaking into new signatures:
|
||||
git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" \
|
||||
| grep -E ':\s*Any\b|->\s*Any\b'
|
||||
# SimpleNamespace newly added in tests (the Any-leak source):
|
||||
git diff ${BASE}...HEAD -- "tests/" | grep "^+[^+]" | grep "SimpleNamespace"
|
||||
```
|
||||
|
||||
### Fix
|
||||
Replace `Any` with the concrete type, a `Protocol`, or a `Union`. If a type is genuinely dynamic, prefer `object` + `isinstance` narrowing over `Any` — `object` keeps the checker engaged. In tests, replace `SimpleNamespace` with the real class, a small `@dataclass`/`TypedDict`, or a `Protocol` + fake, so the system under test stays fully typed.
|
||||
|
||||
---
|
||||
|
||||
## 4. Unnecessary copy — `.clone()` / `copy.deepcopy` in hot paths
|
||||
|
||||
### Anti-pattern
|
||||
```python
|
||||
# Per-step in a diffusion/AR loop: clones the full latent every iteration
|
||||
for t in timesteps:
|
||||
trajectory_latents.append(x_t.clone()) # O(N) GPU alloc + memcpy each step
|
||||
```
|
||||
Real clusters: `vllm_omni/diffusion/models/bagel/bagel_transformer.py` calls `x_t.clone()` ~9× across a trajectory loop (lines 1820–2186); the cache hooks `vllm_omni/diffusion/cache/teacache/hook.py:155` and `magcache/hook.py:182` clone `hidden_states` per step. Separately, `copy.deepcopy(self.scheduler)` / `copy.deepcopy(req.sampling_params)` is copy-pasted across diffusion pipelines — `pipeline_qwen_image.py:786`, `pipeline_hunyuan_image3.py:1904`, `pipeline_ltx2*.py` (×3), `pipeline_dreamzero.py:1085`, `diffusion_model_runner.py:398` — a full object-graph deepcopy on the request path. (Contrast `dreamzero/causal_wan_model.py:203`, which carries a `# No .clone() needed.` comment — the aware pattern.)
|
||||
|
||||
### Why dangerous
|
||||
A `.clone()` of a GPU tensor or a `copy.deepcopy` of a config object inside a per-step / per-request path is a hidden allocator: it reads as one Python line but triggers a GPU memcpy or a deep object-graph walk every iteration. Across 50 diffusion steps or hundreds of AR steps it dominates latency and VRAM bandwidth. `deepcopy` additionally ignores any custom device / `__copy__` logic and can drag tensors through CPU.
|
||||
|
||||
### Severity
|
||||
- ⚠ any *new* `.clone()` / `.copy_()` / `copy.deepcopy(...)` inside a per-step loop or per-request path.
|
||||
- ✗ for `.clone()` of a full latent/activations tensor inside an AR or diffusion step loop without an explanatory comment, or `copy.deepcopy` of a scheduler / sampling-config on the request hot path.
|
||||
|
||||
### Detect (diff-scoped)
|
||||
```bash
|
||||
git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" \
|
||||
| grep -E '\.clone\(\)|\.copy_\(|copy\.deepcopy|deepcopy\('
|
||||
```
|
||||
|
||||
### Fix
|
||||
Tensors: prefer a view (`x[..., :k]`, `torch.narrow`, `as_strided`) or write into a pre-allocated buffer with `dst.copy_(src)` instead of `.clone()`; if the caller no longer needs the original, `move`/`del` it rather than copy. Schedulers / configs: build a fresh lightweight instance per request, or split out the mutable per-request bits so the shared immutable part isn't deepcopied. If a copy is genuinely required (e.g. the source is mutated in place later), keep it and add a one-line comment saying why.
|
||||
|
||||
---
|
||||
|
||||
## 5. Blocking the asyncio event loop / lock held across `await`
|
||||
|
||||
### Anti-pattern
|
||||
```python
|
||||
# BLOCKER-prone: blocking sleep / blocking HTTP / lock held across an await,
|
||||
# all inside an async (event-loop) function
|
||||
async def handle(self, req):
|
||||
with self._lock:
|
||||
resp = await self._client.call(req) # lock held across await -> serializes all requests
|
||||
time.sleep(0.1) # stalls the single loop for every concurrent request
|
||||
```
|
||||
|
||||
### Why dangerous
|
||||
vLLM-Omni serves over a single asyncio event loop. Any synchronous blocking work on an `async def` path — `time.sleep`, blocking HTTP (`requests`/`urllib`), heavy CPU work, or a lock held across an `await` — stalls the loop for *every* concurrent request, not just the blocking one. A lock held across `await` silently serializes the whole pipeline; blocking I/O tanks throughput and inflates tail latency.
|
||||
|
||||
### Severity
|
||||
- ⚠ any *new* `time.sleep(`, blocking HTTP call (`requests.get/post`, `urllib.request`, `urlopen`), or `await` inside a `with Lock` / `async with` critical section, in `async def` code.
|
||||
- ✗ for `time.sleep` or blocking HTTP on the serving path (`engine/`, `entrypoints/`, async `connectors/`), or a lock acquired and held across an `await` on the request path.
|
||||
|
||||
### Detect (diff-scoped)
|
||||
```bash
|
||||
# New blocking primitives introduced by the diff — then eyeball whether each
|
||||
# sits in an async function on the loop vs a dedicated worker thread:
|
||||
git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" \
|
||||
| grep -E 'time\.sleep|requests\.(get|post|put|delete)|urllib\.request|urlopen'
|
||||
# Lock acquired with an await potentially inside the critical section (manual eyeball):
|
||||
git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" | grep -E 'async with|\.acquire\(\)'
|
||||
```
|
||||
Note: blocking inside a **dedicated worker thread is correct** — the repo already does this (`diffusion/model_loader/hub_prefetch.py`, `distributed/omni_connectors/kv_transfer_manager.py`, the `mooncake`/`mori` connectors poll with `time.sleep` on their own threads). The check targets the *event-loop* path only.
|
||||
|
||||
### Fix
|
||||
Move blocking work off the loop with `asyncio.to_thread(...)` / `run_in_executor`, or use a native async client (`httpx.AsyncClient` / `aiohttp`). Never `await` while holding a lock — drop the lock before the `await` and re-acquire after, or restructure so the critical section contains no `await`.
|
||||
|
||||
---
|
||||
|
||||
## Running the sweep
|
||||
|
||||
Run all five detections against `${BASE}...HEAD`, then for each hit decide ⚠ vs ✗ using the severity rules above. Roll the results into the report as a single **Code quality** dimension row (count of ⚠ and ✗), alongside the type-specific checklist from [checklists.md](checklists.md).
|
||||
@@ -19,6 +19,7 @@ body:
|
||||
|
||||
Consider redacting or replacing sensitive values with placeholders like `<YOUR_TOKEN_HERE>` when sharing configuration or code examples.
|
||||
- type: textarea
|
||||
id: current-environment
|
||||
attributes:
|
||||
label: Your current environment
|
||||
description: |
|
||||
@@ -42,6 +43,7 @@ body:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: code-version
|
||||
attributes:
|
||||
label: Your code version
|
||||
description: |
|
||||
@@ -66,6 +68,7 @@ body:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: bug-description
|
||||
attributes:
|
||||
label: 🐛 Describe the bug
|
||||
description: |
|
||||
|
||||
@@ -241,6 +241,7 @@ datasets/
|
||||
*.json
|
||||
!tests/dfx/perf/tests/*.json
|
||||
!apps/ComfyUI-vLLM-Omni/example_workflows/*.json
|
||||
!tests/assets/soulxsinger/*.json
|
||||
*.jsonl
|
||||
*.parquet
|
||||
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
# LVSA Long-Video Showcase
|
||||
|
||||
**Training-free block-sparse attention for long-video diffusion**, running on
|
||||
vLLM-Omni via the public plugin entry points (no core changes). LVSA speeds up
|
||||
extended-horizon generation and prevents the freeze/loop failure mode dense
|
||||
attention exhibits beyond a model's training length — with no fine-tuning and no
|
||||
weight changes.
|
||||
|
||||
Upstream: **[LongVideoSparseAttention](https://github.com/JiusiServe/LongVideoSparseAttention)**
|
||||
(algorithm, all model adapters, full docs).
|
||||
|
||||
## Results
|
||||
|
||||
Wan 2.1 1.3B, single A100 80 GB, mean over 5 prompts (Dense vs LVSA-FlashInfer):
|
||||
|
||||
| Horizon | 1× | 2× | 3× | 4× | 5× | 6× |
|
||||
|---|---|---|---|---|---|---|
|
||||
| **Speedup** | 0.9× | 1.4× | 1.9× | 2.5× | 3.0× | **3.5×** |
|
||||
| **VQeval composite** (Dense→LVSA) | 63→63 | 59→63 | 61→63 | 61→64 | 59→64 | 59→63 |
|
||||
|
||||
The speedup **grows with length** (LVSA is the dense regime at 1× by design, so
|
||||
the win starts at 2×), and quality *improves* at extension — LVSA's rotating
|
||||
keyframes prevent the looping/static output dense produces beyond its horizon
|
||||
(VQeval loop-quality +27 to +34 at ≥3×). At long horizons dense **OOMs on 80 GB**
|
||||
(HunyuanVideo ≥2×, Wan2.1-14B and Cosmos at 6×) where LVSA still generates.
|
||||
|
||||
→ Full sweep (5 models × 6 horizons, VQeval + VBench-Long) and the
|
||||
quality-metric methodology:
|
||||
**[benchmarks/](https://github.com/JiusiServe/LongVideoSparseAttention/tree/main/benchmarks)**.
|
||||
|
||||
## Quickstart
|
||||
|
||||
```bash
|
||||
# 1. vLLM-Omni 0.22.0 (stable) + matching vLLM
|
||||
pip install "vllm==0.22.0"
|
||||
pip install --no-build-isolation \
|
||||
"vllm-omni @ git+https://github.com/vllm-project/vllm-omni.git@v0.22.0"
|
||||
|
||||
# 2. LVSA core + plugin
|
||||
git clone https://github.com/JiusiServe/LongVideoSparseAttention.git
|
||||
( cd LongVideoSparseAttention && pip install -e . && pip install -e lvsa-vllm-omni/ )
|
||||
|
||||
# 3. this showcase's client dep
|
||||
pip install requests
|
||||
```
|
||||
|
||||
FlashInfer (used by the default `flashinfer` backend) **ships with vLLM** — step 1
|
||||
already pulls the pinned `flashinfer-python` + `flashinfer-cubin`, so the fused
|
||||
backend works with no extra install. Two optional extras:
|
||||
|
||||
```bash
|
||||
# (optional) faster startup — prebuilt JIT cache, matched to your CUDA + flashinfer
|
||||
CU=cu128 # match your CUDA: cu121 / cu124 / cu128 / …
|
||||
FIVER=$(python -c "import importlib.metadata as m; print(m.version('flashinfer-python'))")
|
||||
pip install --extra-index-url "https://flashinfer.ai/whl/${CU}" \
|
||||
"flashinfer-jit-cache==${FIVER}+${CU}"
|
||||
|
||||
# (fallback) no FlashInfer in your environment? run the SDPA backend instead:
|
||||
export LVSA_BACKEND=sdpa
|
||||
```
|
||||
|
||||
```bash
|
||||
# Serve an LVSA-enabled endpoint (Wan 2.1 1.3B at 2× horizon):
|
||||
MODEL=/path/to/Wan2.1-T2V-1.3B-Diffusers MODEL_FAMILY=wan FRAMES=161 \
|
||||
bash serve_lvsa.sh
|
||||
|
||||
# In another shell — generate:
|
||||
python generate.py --frames 161 --height 480 --width 832 \
|
||||
--prompt "A dog running through a sunlit forest." --out wan_lvsa.mp4
|
||||
```
|
||||
|
||||
`serve_lvsa.sh` sets the LVSA env vars for the chosen `MODEL_FAMILY`
|
||||
(`wan` / `hunyuan`). Wan and HunyuanVideo run through the LVSA **attention
|
||||
backend** by default — it stays sparse even under sequence-parallel, unlike the
|
||||
monkey-patch hooks.
|
||||
|
||||
## Configuration
|
||||
|
||||
The one knob you must get right is the model's training horizon
|
||||
(`LVSA_REFERENCE_LATENT_FRAMES`: Wan = 21, HunyuanVideo = 33); `serve_lvsa.sh`
|
||||
sets it per family. For the full environment-variable reference, the multi-GPU
|
||||
support matrix, and tuning (`sparsity_scale`, rotating keyframes, non-standard
|
||||
resolutions), see the upstream docs:
|
||||
|
||||
- **[lvsa-vllm-omni plugin reference](https://github.com/JiusiServe/LongVideoSparseAttention/blob/main/lvsa-vllm-omni/README.md)** — every `LVSA_*` env var
|
||||
- **[docs/parallelism.md](https://github.com/JiusiServe/LongVideoSparseAttention/blob/main/docs/parallelism.md)** — TP / Ulysses / HSDP support per model
|
||||
- **[docs/tuning.md](https://github.com/JiusiServe/LongVideoSparseAttention/blob/main/docs/tuning.md)** — picking knobs for your model and horizon
|
||||
@@ -0,0 +1,93 @@
|
||||
#!/usr/bin/env python
|
||||
"""Generate a long video through an LVSA-enabled vLLM-Omni server.
|
||||
|
||||
A thin client over the Omni video API — start the server first with
|
||||
``serve_lvsa.sh``. The sparse attention runs server-side; this just submits a
|
||||
job and saves the result.
|
||||
|
||||
Uses the **asynchronous** endpoint (``POST /v1/videos`` → poll
|
||||
``GET /v1/videos/{id}`` → download ``GET /v1/videos/{id}/content``) rather than
|
||||
``POST /v1/videos/sync``. The sync endpoint is capped by a fixed server-side
|
||||
timeout (``VIDEO_SYNC_TIMEOUT_S``, 600 s) and is documented as
|
||||
"for benchmark and testing scenarios" — long videos (LVSA's use case) routinely
|
||||
exceed it. The async job has no such cap.
|
||||
|
||||
python generate.py --frames 481 --height 480 --width 832 \
|
||||
--prompt "A golden retriever running through a sunlit forest." \
|
||||
--out dog_forest_lvsa.mp4
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import time
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--server", default=os.environ.get("OMNI_SERVER", "http://localhost:8000"))
|
||||
ap.add_argument("--prompt", default="A dog running through a sunlit forest.")
|
||||
ap.add_argument("--frames", type=int, default=161) # Wan 2x horizon
|
||||
ap.add_argument("--height", type=int, default=480)
|
||||
ap.add_argument("--width", type=int, default=832)
|
||||
ap.add_argument("--fps", type=int, default=16)
|
||||
ap.add_argument("--steps", type=int, default=50)
|
||||
ap.add_argument("--guidance", type=float, default=5.0)
|
||||
ap.add_argument("--out", default="lvsa_long_video.mp4")
|
||||
ap.add_argument("--poll-interval", type=float, default=5.0, help="Seconds between status polls.")
|
||||
ap.add_argument(
|
||||
"--max-wait", type=float, default=7200.0, help="Give up if the job hasn't finished after this many seconds."
|
||||
)
|
||||
args = ap.parse_args()
|
||||
|
||||
base = args.server.rstrip("/")
|
||||
form = {
|
||||
"prompt": args.prompt,
|
||||
"size": f"{args.width}x{args.height}",
|
||||
"num_frames": str(args.frames),
|
||||
"fps": str(args.fps),
|
||||
"num_inference_steps": str(args.steps),
|
||||
"guidance_scale": str(args.guidance),
|
||||
}
|
||||
|
||||
# ── 1. submit the async job ──────────────────────────────────────────────
|
||||
print(f"[lvsa-app] POST {base}/v1/videos {args.width}x{args.height} {args.frames}f steps={args.steps}")
|
||||
resp = requests.post(f"{base}/v1/videos", data=form, timeout=120)
|
||||
resp.raise_for_status()
|
||||
job = resp.json()
|
||||
vid = job["id"]
|
||||
print(f"[lvsa-app] job {vid} ({job.get('status', '?')})")
|
||||
|
||||
# ── 2. poll until completed / failed ─────────────────────────────────────
|
||||
deadline = time.time() + args.max_wait
|
||||
last = None
|
||||
while True:
|
||||
if time.time() > deadline:
|
||||
raise SystemExit(f"[lvsa-app] timed out after {args.max_wait:.0f}s (job {vid} still running)")
|
||||
time.sleep(args.poll_interval)
|
||||
r = requests.get(f"{base}/v1/videos/{vid}", timeout=60)
|
||||
if r.status_code != 200:
|
||||
# retrieve_video returns a non-200 JSON when the job has FAILED
|
||||
raise SystemExit(f"[lvsa-app] job failed: {r.text}")
|
||||
st = r.json()
|
||||
status = st.get("status")
|
||||
progress = st.get("progress", 0)
|
||||
if (status, progress) != last:
|
||||
print(f"[lvsa-app] {status} {progress}%")
|
||||
last = (status, progress)
|
||||
if status == "completed":
|
||||
break
|
||||
if status == "failed":
|
||||
raise SystemExit(f"[lvsa-app] job failed: {st.get('error')}")
|
||||
|
||||
# ── 3. download the rendered mp4 ─────────────────────────────────────────
|
||||
c = requests.get(f"{base}/v1/videos/{vid}/content", timeout=600)
|
||||
c.raise_for_status()
|
||||
with open(args.out, "wb") as f:
|
||||
f.write(c.content)
|
||||
print(f"[lvsa-app] saved {args.out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env bash
|
||||
# Start an LVSA-enabled vLLM-Omni server for Wan or HunyuanVideo.
|
||||
#
|
||||
# LVSA is selected as the attention backend via --diffusion-attention-config (see
|
||||
# below); the `LVSA_*` env vars only tune it. The `lvsa-vllm-omni` plugin registers
|
||||
# itself through vLLM-Omni's `general_plugins` entry points — nothing in vLLM-Omni
|
||||
# core is modified. Watch the log for "[LVSA] Geometry detected" to confirm sparse
|
||||
# attention is active for your request geometry.
|
||||
#
|
||||
# MODEL=/path/to/Wan2.1-T2V-1.3B-Diffusers MODEL_FAMILY=wan FRAMES=161 bash serve_lvsa.sh
|
||||
# MODEL=/path/to/HunyuanVideo-1.5-... MODEL_FAMILY=hunyuan FRAMES=193 bash serve_lvsa.sh
|
||||
set -euo pipefail
|
||||
|
||||
MODEL=${MODEL:?set MODEL=/path/to/Wan-or-HunyuanVideo-Diffusers}
|
||||
MODEL_FAMILY=${MODEL_FAMILY:-wan} # wan | hunyuan
|
||||
PORT=${PORT:-8000}
|
||||
FRAMES=${FRAMES:-161} # target horizon in frames
|
||||
VAE_T=${VAE_T:-4} # VAE temporal compression (Wan/Hunyuan = 4)
|
||||
BACKEND=${LVSA_BACKEND:-flashinfer} # flashinfer (fused, fastest) | sdpa
|
||||
|
||||
T_LAT=$(( (FRAMES - 1) / VAE_T + 1 ))
|
||||
|
||||
# Per-family training reference horizon (Wan train ref = 21, Hunyuan = 33).
|
||||
case "$MODEL_FAMILY" in
|
||||
wan) REFERENCE=${REFERENCE:-21} ;;
|
||||
hunyuan) REFERENCE=${REFERENCE:-33} ;;
|
||||
*) echo "MODEL_FAMILY must be 'wan' or 'hunyuan'"; exit 1 ;;
|
||||
esac
|
||||
export LVSA_BACKEND="$BACKEND"
|
||||
export LVSA_TOTAL_LATENT_FRAMES="$T_LAT"
|
||||
export LVSA_REFERENCE_LATENT_FRAMES="$REFERENCE"
|
||||
export LVSA_ROTATE_KEYFRAMES=${LVSA_ROTATE_KEYFRAMES:-1}
|
||||
|
||||
echo "[lvsa-app] serving $MODEL ($MODEL_FAMILY) on :$PORT"
|
||||
echo "[lvsa-app] LVSA backend=$BACKEND T_lat=$T_LAT (frames=$FRAMES) reference=$REFERENCE"
|
||||
echo "[lvsa-app] confirm activation by grep '\\[LVSA\\] Geometry detected' in this log"
|
||||
|
||||
# vLLM-Omni 0.22 selects the LVSA attention backend per-role via
|
||||
# --diffusion-attention-config. The backend is what drives sparse attention
|
||||
# here; it engages under tensor-parallel too, unlike the LVSA_*_HOOK monkey-patch
|
||||
# path (which falls back to dense under sequence-parallel), so we deliberately do
|
||||
# NOT set a hook.
|
||||
exec vllm serve "$MODEL" --omni --host 0.0.0.0 --port "$PORT" \
|
||||
--diffusion-attention-config '{"per_role": {"self": {"backend": "LVSA"}}}' \
|
||||
--init-timeout 1800 ${EXTRA_ARGS:-}
|
||||
@@ -80,6 +80,17 @@ models:
|
||||
task_extra_body:
|
||||
voice_clone: {}
|
||||
|
||||
# MOSS-TTS-Local-Transformer-v1.5: Qwen3 backbone + 1-layer GPT2-style local
|
||||
# depth transformer (n_vq=12), paired with the 48 kHz stereo
|
||||
# MOSS-Audio-Tokenizer-v2 codec. Same voice-clone request shape as the
|
||||
# other MOSS-TTS variants.
|
||||
OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5:
|
||||
supported_tasks: [voice_clone]
|
||||
backend: openai-audio-speech
|
||||
endpoint: /v1/audio/speech
|
||||
task_extra_body:
|
||||
voice_clone: {}
|
||||
|
||||
# MOSS-TTSD synthesizes [S1]/[S2] dialogue. The seed-tts voice_clone dataset
|
||||
# has single-speaker rows; bench piggybacks on it (both speakers share the
|
||||
# same ref_audio). For real dialogue evaluation use the ttsd dataset.
|
||||
|
||||
@@ -101,7 +101,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force && \
|
||||
export CMAKE_PREFIX_PATH="$(python3 -c 'import site; print(site.getsitepackages()[0])'):${CMAKE_PREFIX_PATH}"
|
||||
|
||||
ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib/"
|
||||
ENV LD_LIBRARY_PATH="/opt/intel/oneapi/ccl/2021.15/lib:$LD_LIBRARY_PATH:/usr/local/lib/"
|
||||
|
||||
ENV VLLM_TARGET_DEVICE=xpu
|
||||
ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
|
||||
@@ -155,7 +155,7 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
|
||||
/tests/dfx/stability/tests/test_wan22.json<br>
|
||||
<strong>Reliability:</strong><br>
|
||||
tests/dfx/reliability/test_reliability_{model_key}.py<br>
|
||||
(e.g. <code>test_reliability_qwen3_omni.py</code>, <code>test_reliability_wan22.py</code>)
|
||||
(e.g. <code>test_reliability_qwen3_omni.py</code>, <code>test_reliability_wan22.py</code>, <code>test_reliability_hunyuan_image.py</code>, <code>test_reliability_voxcpm2.py</code>)
|
||||
</td>
|
||||
<td>
|
||||
<a href="#chapter-4-l5-level-testing-stability-and-reliability-testing">Chapter 4</a><br>
|
||||
@@ -741,14 +741,16 @@ L5 level testing focuses on the performance of model services under ***long-runn
|
||||
### 4.2 Testing Content and Scope
|
||||
|
||||
- ***Long-term Stability (Stability) Testing***: Uses JSON under `tests/dfx/stability/tests/` (for example `test_qwen3_omni.json` and `test_wan22.json`) to run the service under moderate load for an extended period (e.g., over 12 hours), monitoring whether metrics like memory/VRAM usage, response time, and throughput degrade over time, and whether the service process remains stable.
|
||||
- ***Reliability Testing***: Uses pytest suites under `tests/dfx/reliability/` to inject controlled faults against a **live** `vllm_omni serve` instance (same **`omni_server` / `omni_server_function`** fixture style as E2E). Current suites emphasize **GPU memory pressure** (CUDA sidecar “memory hog”), **worker / runtime process kill** (`SIGKILL` on `VLLM::Worker` for Qwen3-Omni or `multiprocessing.spawn` for Wan2.2 video workers), **large multimodal chat** or **`/v1/videos`** jobs under OOM, **`/health` → 503** and **fast-fail / non-hanging concurrent** requests after kill, and **OpenAI-style 5xx error contracts** (e.g. text vs text+audio under OOM). **Post-fault recovery** checks exist where enabled (some cases may be `skip` while issues are tracked). See the Reliability `<details>` block in Section 4.4 for file-level responsibilities and CI markers (`slow`, `hardware_test`, POSIX-only kill).
|
||||
- ***Reliability Testing***: Uses pytest suites under `tests/dfx/reliability/` to inject controlled faults against a **live** `vllm_omni serve` instance (same **`omni_server` / `omni_server_function`** fixture style as E2E). Current suites emphasize **GPU memory pressure** (CUDA sidecar “memory hog”), **worker / runtime process kill** (`SIGKILL` on `VLLM::Worker` for Qwen3-Omni, `multiprocessing.spawn` for Wan2.2 video workers, or `vLLM-Omni::` for HunyuanImage DiT workers), **large multimodal chat**, **`/v1/videos`**, **`/v1/images/generations`**, or **`/v1/audio/speech`** jobs under OOM, **`/health` → 503** and **fast-fail / non-hanging concurrent** requests after kill, and **OpenAI-style 5xx error contracts** (e.g. text vs text+audio under OOM). **Post-fault recovery** checks exist where enabled (some cases may be `skip` while issues are tracked). See the Reliability `<details>` block in Section 4.4 for file-level responsibilities and CI markers (`slow`, `hardware_test`, POSIX-only kill).
|
||||
|
||||
### 4.3 Test Directory and Execution Files
|
||||
|
||||
- ***Stability Test Configuration***: `tests/dfx/stability/tests/test_qwen3_omni.json`, `tests/dfx/stability/tests/test_wan22.json` (one JSON per model / runner family)
|
||||
- ***Reliability Test Suite*** (`tests/dfx/reliability/`):
|
||||
- `test_reliability_qwen3_omni.py` — Qwen3-Omni chat / multimodal reliability (GPU OOM, process kill, recovery, error contract under `--async-chunk` vs default).
|
||||
- `test_reliability_wan22.py` — Wan2.2 T2V video API reliability (`/v1/videos` under OOM and process kill, recovery).
|
||||
- `test_reliability_wan22.py` — Wan2.2 I2V video API reliability (`/v1/videos` under OOM and process kill, recovery).
|
||||
- `test_reliability_hunyuan_image.py` — HunyuanImage-3.0-Instruct DiT-only reliability (`/v1/images/generations` under OOM and process kill; deploy `hunyuan_image3_dit.yaml`, H100 × 4).
|
||||
- `test_reliability_voxcpm2.py` — VoxCPM2 TTS reliability (`/v1/audio/speech` under OOM and process kill; deploy `voxcpm2.yaml`, L4 × 1).
|
||||
- `helpers.py` — Shared primitives used by current suites: raw HTTP probes for `/v1/chat/completions` and `/health`, OpenAI-style error parsing, GPU OOM sidecar (`inject_gpu_oom` / `stop_gpu_oom_hogs`), and `pgrep`-based process-kill injector construction (`make_process_kill_fault_injector`).
|
||||
- `conftest.py` — `fault_injector` and `omni_server_after_fault` / `omni_server_after_fault_function` fixtures to run a callable **after** the server is ready.
|
||||
- `README.md` — Short local run commands for this directory.
|
||||
@@ -846,22 +848,33 @@ Reliability tests are **short fault-injection** integration runs (L5 **(b)** in
|
||||
| `helpers.py` | Shared helpers used by current reliability suites: raw `POST`/`GET` probes (`/v1/chat/completions`, `/health`), OpenAI error parsing (`extract_openai_error_contract_from_bytes`), GPU OOM sidecar lifecycle (`inject_gpu_oom`, `stop_gpu_oom_hogs`), and process-kill injector builder (`make_process_kill_fault_injector`). |
|
||||
| `conftest.py` | Pytest fixtures: indirect `fault_injector`, `omni_server_after_fault` / `omni_server_after_fault_function` (run injector after server is ready, then yield server). |
|
||||
| `test_reliability_qwen3_omni.py` | Qwen3-Omni: OOM vs **text vs text+audio** error contract, large multimodal chat under OOM, concurrent pressure, **SIGKILL** on `VLLM::Worker`, `/health` → 503 + fast-fail + concurrent chat; optional OOM recovery scenario (may be skipped while tracked in issues). |
|
||||
| `test_reliability_wan22.py` | Wan2.2 T2V: large `/v1/videos` under OOM, **SIGKILL** on `multiprocessing.spawn` chain, health / fast-fail / concurrent video requests; optional recovery test (may be skipped). |
|
||||
| `test_reliability_wan22.py` | Wan2.2 I2V: large `/v1/videos` under OOM, **SIGKILL** on `multiprocessing.spawn` chain, health / fast-fail / concurrent video requests; optional recovery test (may be skipped). |
|
||||
| `test_reliability_hunyuan_image.py` | HunyuanImage DiT-only: large `/v1/images/generations` under OOM, **SIGKILL** on `vLLM-Omni::` workers and serve/tree targets, health / fast-fail / concurrent image requests; some OOM/recovery cases may be skipped while tracked in issues. |
|
||||
| `test_reliability_voxcpm2.py` | VoxCPM2: `/v1/audio/speech` under OOM (error contract), **SIGKILL** on `VLLM::` workers and serve/tree targets, health / fast-fail / concurrent speech requests; some OOM cases may be skipped while tracked in issues. |
|
||||
| `README.md` | Minimal run / collect examples. |
|
||||
|
||||
#### Parametrization and markers
|
||||
|
||||
- Each test module defines a **`RELIABILITY_SCENARIOS`** list (`test_name`, `server_params`: model, `stage_config_name` or diffusion `server_args`, etc.). **`create_reliability_omni_server_params()`** in `tests/dfx/conftest.py` resolves stage paths (including XPU substitutions where applicable) and builds **`OmniServerParams`** lists consumed by **`@pytest.mark.parametrize(..., indirect=True)`** on `omni_server` or `omni_server_function`.
|
||||
- Cases are tagged **`@pytest.mark.slow`** for weekly / selective CI. GPU-heavy suites use **`@hardware_test(res={"cuda": "H100"}, num_cards=...)`** (Qwen3-Omni paths often require **2** cards; Wan2.2 video paths **1** card).
|
||||
- Cases are tagged **`@pytest.mark.slow`** for weekly / selective CI. GPU-heavy suites use **`@hardware_test(res={"cuda": "H100"}, num_cards=...)`** or **`@hardware_test(res={"cuda": "L4"}, num_cards=1)`** (Qwen3-Omni **2**× H100; Wan2.2 **1**× H100; HunyuanImage DiT **4**× H100; VoxCPM2 **1**× L4).
|
||||
- **Process-kill** tests use **`@pytest.mark.skipif(os.name == "nt", ...)`** because injection uses POSIX **`pgrep` / `kill`**.
|
||||
|
||||
#### CI trigger
|
||||
|
||||
Weekly Buildkite (`.buildkite/test-weekly.yml`) runs, for example:
|
||||
Weekly Buildkite (`.buildkite/test-weekly.yml`) runs one step per model suite (trigger: `WEEKLY=1` or PR label `weekly-test`), for example:
|
||||
|
||||
| Buildkite step | Test file | CI hardware |
|
||||
| -------------- | --------- | ----------- |
|
||||
| Reliability Test - qwen3-omni | `test_reliability_qwen3_omni.py` | H100 × 2 (`mithril-h100-pool`) |
|
||||
| Reliability Test - wan22 | `test_reliability_wan22.py` | H100 × 2 (`mithril-h100-pool`) |
|
||||
| Reliability Test - hunyuan-image | `test_reliability_hunyuan_image.py` | H100 × 4 (`mithril-h100-pool`) |
|
||||
| Reliability Test - voxcpm2 | `test_reliability_voxcpm2.py` | L4 × 1 (`gpu_1_queue`) |
|
||||
|
||||
```bash
|
||||
pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m "slow"
|
||||
pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m "slow"
|
||||
pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow"
|
||||
pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m "slow"
|
||||
```
|
||||
|
||||
#### Local commands
|
||||
@@ -870,6 +883,8 @@ pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m "slow"
|
||||
pytest --collect-only tests/dfx/reliability
|
||||
pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m slow
|
||||
pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m slow
|
||||
pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m slow
|
||||
pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m slow
|
||||
```
|
||||
|
||||
#### Adding a new model suite
|
||||
@@ -882,7 +897,7 @@ pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m slow
|
||||
</details>
|
||||
|
||||
- - ***Stability***: `pytest -s -v tests/dfx/stability/scripts/test_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/test_stability_wan22.py` (or add `test_stability_<model>.py` alongside a matching JSON config)
|
||||
- ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m slow` and/or `pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m slow` (add `test_reliability_<suite>.py` for new models)
|
||||
- ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_<model>.py -m slow` (current suites: `qwen3_omni`, `wan22`, `hunyuan_image`, `voxcpm2`; add `test_reliability_<suite>.py` and a matching step in `.buildkite/test-weekly.yml` for new models)
|
||||
|
||||
## Summary
|
||||
|
||||
|
||||
@@ -96,15 +96,21 @@ Our test scripts use the pytest framework. First, please use `git clone https://
|
||||
# Stability: Wan2.2 (v1/videos diffusion benchmark loop)
|
||||
pytest -s -v dfx/stability/scripts/test_stability_wan22.py -m slow
|
||||
|
||||
# Reliability: Qwen3-Omni
|
||||
# Reliability: Qwen3-Omni (H100 × 2)
|
||||
pytest -s -v dfx/reliability/test_reliability_qwen3_omni.py -m slow
|
||||
|
||||
# Reliability: Wan2.2
|
||||
# Reliability: Wan2.2 (H100)
|
||||
pytest -s -v dfx/reliability/test_reliability_wan22.py -m slow
|
||||
|
||||
# Reliability: HunyuanImage DiT (H100 × 4)
|
||||
pytest -s -v dfx/reliability/test_reliability_hunyuan_image.py -m slow
|
||||
|
||||
# Reliability: VoxCPM2 (L4)
|
||||
pytest -s -v dfx/reliability/test_reliability_voxcpm2.py -m slow
|
||||
|
||||
```
|
||||
|
||||
The latest L5 commands for CI can be found in the [pipeline](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-ready.yml).
|
||||
The latest L5 CI jobs (reliability + invalid-parameter weekly steps) are in [test-weekly.yml](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-weekly.yml).
|
||||
|
||||
You can find more information about markers in the documentation: [marker doc](./tests_markers.md)
|
||||
|
||||
|
||||
@@ -37,8 +37,9 @@ th {
|
||||
| `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `Cosmos3OmniDiffusersPipeline` | Cosmos3 T2I, T2V, I2V, V2V, T2V with sound, action policy | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `WanSpeechToVideoPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `Wan22S2VPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `Wan22VACEPipeline` | Wan2.2-VACE | `Pyros13/Wan2.2-VACE-Fun-A14B-Diffusers` | ✅︎ | | | |
|
||||
| `LTX2Pipeline` | LTX-2-T2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | |
|
||||
| `LTX2ImageToVideoPipeline` | LTX-2-I2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | |
|
||||
| `LTX2TwoStagesPipeline` | LTX-2-T2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | |
|
||||
@@ -63,6 +64,8 @@ th {
|
||||
| `FluxPipeline` | FLUX.1-schnell | `black-forest-labs/FLUX.1-schnell` | ✅︎ | ✅︎ | | ✅︎ |
|
||||
| `OmniGen2Pipeline` | OmniGen2 | `OmniGen2/OmniGen2` | ✅︎ | ✅︎ | | ✅︎ |
|
||||
| `StableAudioPipeline` | Stable-Audio-Open | `stabilityai/stable-audio-open-1.0` | ✅︎ | ✅︎ | | ✅︎ |
|
||||
| `SoulXSingerPipeline` | SoulX-Singer (SVS) | `Soul-AILab/SoulX-Singer` | ✅︎ | | | |
|
||||
| `SoulXSingerSVCPipeline` | SoulX-Singer-SVC | `Soul-AILab/SoulX-Singer` (`model-svc.pt`) | ✅︎ | | | |
|
||||
| `AudioXPipeline` | AudioX | `zhangj1an/AudioX` | ✅︎ | ✅︎ | | |
|
||||
| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
@@ -74,6 +77,7 @@ th {
|
||||
| `MossTTSRealtime` | MOSS-TTS-Realtime | `OpenMOSS-Team/MOSS-TTS-Realtime` | ✅︎ | | | |
|
||||
| `HiggsAudioV2ForConditionalGeneration` | Higgs-Audio v2 | `bosonai/higgs-audio-v2-generation-3B-base` | ✅︎ | | | |
|
||||
| `HiggsMultimodalQwen3ForConditionalGeneration` | Higgs-Audio v3 (TTS) | `bosonai/higgs-audio-v3-tts-4b` | ✅︎ | | | |
|
||||
| `IndexTTS2TalkerForConditionalGeneration` | IndexTTS-2 | `IndexTeam/IndexTTS-2` | ✅︎ | | | |
|
||||
| `NextStep11Pipeline` | NextStep-1.1 | `stepfun-ai/NextStep-1.1` | ✅︎ | ✅︎ | | ✅︎ |
|
||||
| `MiMoAudioModel` | MiMo-Audio-7B-Instruct | `XiaomiMiMo/MiMo-Audio-7B-Instruct` | ✅︎ | ✅︎ | | |
|
||||
| `MiMoV2ASRForCausalLM` | MiMo-V2.5-ASR | `XiaomiMiMo/MiMo-V2.5-ASR` | ✅︎ | ✅︎ | | |
|
||||
|
||||
@@ -32,6 +32,20 @@ curl -X POST http://localhost:8000/v1/images/generations \
|
||||
}' | jq -r '.data[0].b64_json' | base64 -d > dragon.png
|
||||
```
|
||||
|
||||
**Using curl save to file:**
|
||||
|
||||
```bash
|
||||
curl -o dragon.png -X POST http://localhost:8000/v1/images/generations \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"prompt": "a dragon laying over the spine of the Green Mountains of Vermont",
|
||||
"size": "1024x1024",
|
||||
"seed": 42,
|
||||
"response_format":"file"
|
||||
}'
|
||||
```
|
||||
|
||||
|
||||
**Using Python:**
|
||||
|
||||
```python
|
||||
@@ -57,6 +71,36 @@ img = Image.open(io.BytesIO(img_bytes))
|
||||
img.save("cat.png")
|
||||
```
|
||||
|
||||
**Using Python save to file:**
|
||||
|
||||
```python
|
||||
import requests
|
||||
import base64
|
||||
from PIL import Image
|
||||
import io
|
||||
import re
|
||||
|
||||
response = requests.post(
|
||||
"http://localhost:8000/v1/images/generations",
|
||||
json={
|
||||
"prompt": "a black and white cat wearing a princess tiara",
|
||||
"size": "1024x1024",
|
||||
"num_inference_steps": 50,
|
||||
"seed": 42,
|
||||
"response_format":"file"
|
||||
}
|
||||
)
|
||||
|
||||
# save to file
|
||||
content_disposition = response.headers.get("Content-Disposition", "")
|
||||
match = re.search(r'filename="?(.+)"?', content_disposition)
|
||||
filename = match.group(1) if match else "save.png"
|
||||
with open(filename, "wb") as f:
|
||||
for chunk in response.iter_content(8192):
|
||||
f.write(chunk)
|
||||
print("saved:", filename)
|
||||
```
|
||||
|
||||
**Using OpenAI SDK:**
|
||||
|
||||
```python
|
||||
|
||||
@@ -233,9 +233,9 @@ Server -> Client:
|
||||
|
||||
| Message | Description |
|
||||
|---------|-------------|
|
||||
| `{"type": "audio.start", "sentence_index": 0, "sentence_text": "...", "format": "pcm", "sample_rate": 24000}` | Audio generation starting for a sentence |
|
||||
| `{"type": "audio.start", "sentence_index": 0, "sentence_text": "...", "format": "pcm", "sample_rate": 24000}` | Audio generation starting for the buffered input |
|
||||
| Binary frame | Raw audio bytes (one or more PCM chunks when `stream_audio=true`) |
|
||||
| `{"type": "audio.done", "sentence_index": 0, "total_bytes": 96000, "error": false}` | Audio complete for a sentence |
|
||||
| `{"type": "audio.done", "sentence_index": 0, "total_bytes": 96000, "error": false}` | Audio complete for the buffered input |
|
||||
| `{"type": "session.done", "total_sentences": N}` | Session complete |
|
||||
| `{"type": "error", "message": "..."}` | Non-fatal error |
|
||||
|
||||
@@ -245,8 +245,7 @@ All REST API parameters are supported, plus:
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `stream_audio` | bool | false | Stream one or more PCM chunks per sentence over WebSocket |
|
||||
| `split_granularity` | string | "sentence" | Text splitting granularity |
|
||||
| `stream_audio` | bool | false | Stream one or more PCM chunks for the buffered input over WebSocket |
|
||||
|
||||
|
||||
```bash
|
||||
|
||||
@@ -197,6 +197,7 @@ Factory function `get_offload_backend()` selects the appropriate backend based o
|
||||
| StableDiffusion3Pipeline | `stabilityai/stable-diffusion-3.5-medium` | `SD3Transformer2DModel` | - | ✓ | `"transformer_blocks"` |
|
||||
| Wan22I2VPipeline | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | `"blocks"` |
|
||||
| Wan22Pipeline | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | `"blocks"` |
|
||||
| SoulXSingerPipeline / SoulXSingerSVCPipeline | `Soul-AILab/SoulX-Singer` | `DiffLlama` (`cfm_decoder.model.diff_estimator`) | ✓ | ✓ | `"layers"` |
|
||||
| BagelPipeline | `ByteDance-Seed/BAGEL-7B-MoT` | `Qwen2MoTModel` | - | ✓ | `"layers"`, `"customized modules"` |
|
||||
|
||||
**Notes:**
|
||||
|
||||
@@ -146,7 +146,7 @@ The following tables show which models support each feature:
|
||||
| Model | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | Pipeline-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
|
||||
|------------------------------|:---------:|:----------:|:---------------------:|:--------------:|:-----------------:|:-----------------:|:------:|:-------------------------:|:--------------------:|:--------------:|:----------------:|
|
||||
| **Wan2.2** | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (encode/decode) | ❌ | ❌ |
|
||||
| **Wan2.2-S2V** | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ (encode/decode) | ❌ | ❌ |
|
||||
| **Wan2.2-S2V** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ (encode/decode) | ❌ | ❌ |
|
||||
| **Wan2.1-VACE** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
|
||||
| **LTX-2** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| **LTX-2.3** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ (decode) | ❌ | ❌ |
|
||||
|
||||
@@ -65,7 +65,7 @@ More failure modes (for example, network jitter and network interruption) will b
|
||||
|
||||
## Source of Conclusions
|
||||
|
||||
The behaviors and conclusions above are summarized from current fault injection validation results on `Qwen3-Omni` and `Wan2.2`.
|
||||
The behaviors and conclusions above are summarized from current fault injection validation results on **`Qwen3-Omni`**, **`Wan2.2`**, **`HunyuanImage-3.0-Instruct`** (DiT-only, `/v1/images/generations`), and **`VoxCPM2`** (`/v1/audio/speech`). Automated coverage lives under `tests/dfx/reliability/` (`test_reliability_qwen3_omni.py`, `test_reliability_wan22.py`, `test_reliability_hunyuan_image.py`, `test_reliability_voxcpm2.py`) and runs weekly via `.buildkite/test-weekly.yml`.
|
||||
|
||||
### Example: `SIGTERM` fault injection log (Qwen3-Omni)
|
||||
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"prompt": "A robotic arm, primarily white with black joints and cables, is shown in a clean, modern indoor setting with a white tabletop. The arm, equipped with a gripper holding a small, light green pitcher, is positioned above a clear glass containing a reddish-brown liquid and a spoon. The robotic arm is in the process of pouring a transparent liquid into the glass. To the left of the pitcher, there is an opened jar with a similar reddish-brown substance visible through its transparent body. In the background, a vase with white flowers and a brown couch are partially visible, adding to the contemporary ambiance. The lighting is bright, casting soft shadows on the table. The robotic arm's movements are smooth and controlled, demonstrating precision in its task. As the video progresses, the robotic arm completes the pour, leaving the glass half-filled with the reddish-brown liquid. The jar remains untouched throughout the sequence, and the spoon inside the glass remains stationary. The other robotic arm on the right side also stays stationary throughout the video. The final frame captures the robotic arm with the pitcher finishing the pour, with the glass now filled to a higher level, while the pitcher is slightly tilted but still held securely by the gripper.",
|
||||
"negative_prompt": "",
|
||||
"vision_path": "https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/vision/robot_pouring.mp4",
|
||||
"height": 720,
|
||||
"width": 1280,
|
||||
"num_frames": 189,
|
||||
"num_inference_steps": 35,
|
||||
"guidance_scale": 6.0,
|
||||
"flow_shift": 10.0,
|
||||
"fps": 24,
|
||||
"condition_frame_indexes_vision": [0, 1],
|
||||
"condition_video_keep": "first"
|
||||
}
|
||||
@@ -74,6 +74,12 @@ def parse_args():
|
||||
parser.add_argument("--sys-type", type=str, default=None, help="Override system prompt type.")
|
||||
parser.add_argument("--deploy-config", type=str, default=None, help="Custom deploy YAML path.")
|
||||
parser.add_argument("--stage-configs-path", type=str, default=None, help="Custom legacy stage config YAML path.")
|
||||
parser.add_argument(
|
||||
"--stream",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Enable streaming CoT display; print AR text token-by-token in real time.",
|
||||
)
|
||||
parser.add_argument("--log-stats", action="store_true", default=False)
|
||||
parser.add_argument("--init-timeout", type=int, default=300, help="Initialization timeout in seconds.")
|
||||
parser.add_argument("--enforce-eager", action="store_true", help="Disable torch.compile.")
|
||||
@@ -222,6 +228,8 @@ def main():
|
||||
prompt_dict["modalities"] = ["text"]
|
||||
formatted_prompts.append(prompt_dict)
|
||||
|
||||
from vllm.sampling_params import RequestOutputKind
|
||||
|
||||
params_list = list(omni.default_sampling_params_list)
|
||||
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
@@ -253,6 +261,10 @@ def main():
|
||||
sp.width = args.width
|
||||
elif hasattr(sp, "stop_token_ids"):
|
||||
sp.stop_token_ids = ar_stop_token_ids
|
||||
# When --stream is set, request DELTA output from the AR stage
|
||||
# so we can display CoT text token-by-token in real time.
|
||||
if args.stream and hasattr(sp, "output_kind"):
|
||||
sp.output_kind = RequestOutputKind.DELTA
|
||||
|
||||
print(f"\n{'=' * 60}")
|
||||
print("HunyuanImage-3.0 Generation Configuration:")
|
||||
@@ -281,22 +293,32 @@ def main():
|
||||
print(f" Prompts: {prompts}")
|
||||
print(f"{'=' * 60}\n")
|
||||
|
||||
omni_outputs = list(omni.generate(prompts=formatted_prompts, sampling_params_list=params_list))
|
||||
# When --stream is set, print AR CoT text token-by-token in real time.
|
||||
# Otherwise, collect and print the full AR text once when stage 0 finishes.
|
||||
omni_outputs = omni.generate(
|
||||
prompts=formatted_prompts,
|
||||
sampling_params_list=params_list,
|
||||
py_generator=True,
|
||||
use_tqdm=False,
|
||||
)
|
||||
img_idx = 0
|
||||
for req_output in omni_outputs:
|
||||
ro = getattr(req_output, "request_output", None)
|
||||
txt = ""
|
||||
if ro and getattr(ro, "outputs", None):
|
||||
txt = "".join(getattr(o, "text", "") or "" for o in ro.outputs)
|
||||
if not txt:
|
||||
ar_text = getattr(req_output, "custom_output", {}).get("ar_generated_text")
|
||||
if isinstance(ar_text, list):
|
||||
txt = "\n".join(text for text in ar_text if text)
|
||||
else:
|
||||
txt = ar_text or ""
|
||||
if txt:
|
||||
print(f"[Output] Text:\n{txt}")
|
||||
stage_id = getattr(req_output, "stage_id", None)
|
||||
|
||||
# AR stage text — each CompletionOutput.text is already a delta when
|
||||
# output_kind=DELTA, so we can print it directly (matching the pattern
|
||||
# in serving_chat.py).
|
||||
if stage_id == 0 and ro and getattr(ro, "outputs", None):
|
||||
for o in ro.outputs:
|
||||
text = getattr(o, "text", "") or ""
|
||||
if text:
|
||||
print(text, end="", flush=True)
|
||||
# Non-streaming: one shot with full text — emit a trailing newline.
|
||||
if not args.stream:
|
||||
print(flush=True)
|
||||
|
||||
# Collect images from diffusion stage
|
||||
images = getattr(req_output, "images", None)
|
||||
if not images and ro and hasattr(ro, "images"):
|
||||
images = ro.images
|
||||
@@ -304,7 +326,7 @@ def main():
|
||||
for j, img in enumerate(images):
|
||||
save_path = os.path.join(args.output, f"output_{img_idx}_{j}.png")
|
||||
img.save(save_path)
|
||||
print(f"[Output] Saved image to {save_path}")
|
||||
print(f"\n[Output] Saved image to {save_path}")
|
||||
img_idx += 1
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# Image-To-Video
|
||||
|
||||
This example demonstrates how to generate videos from images using Wan2.2 Image-to-Video models with vLLM-Omni's offline inference API.
|
||||
This shared example generates videos from images with VACE, Wan2.2, LTX2,
|
||||
HunyuanVideo-1.5, Cosmos3, and other compatible pipelines.
|
||||
|
||||
## Local CLI Usage
|
||||
|
||||
@@ -48,19 +49,38 @@ python image_to_video.py \
|
||||
--output i2v_output.mp4
|
||||
```
|
||||
|
||||
### Cosmos3
|
||||
|
||||
```bash
|
||||
# Cosmos3 bundles example frames under assets/ (any RGB image works too):
|
||||
python image_to_video.py \
|
||||
--model nvidia/Cosmos3-Nano \
|
||||
--image /path/to/Cosmos3-Nano/assets/example_i2v_input.jpg \
|
||||
--prompt "The scene comes to life with smooth, natural motion." \
|
||||
--negative-prompt "blurry, distorted, low quality" \
|
||||
--height 720 --width 1280 --num-frames 189 --fps 24 \
|
||||
--num-inference-steps 35 --guidance-scale 6.0 \
|
||||
--extra-body '{"flow_shift": 10.0, "max_sequence_length": 4096, "guardrails": false}' \
|
||||
--output cosmos3_i2v.mp4
|
||||
```
|
||||
|
||||
Key arguments:
|
||||
|
||||
- `--model`: Model ID (I2V-A14B for MoE, TI2V-5B for unified T2V+I2V).
|
||||
- `--image`: Path to input image (required).
|
||||
- `--image`: Path to the first-frame or source image.
|
||||
- `--last-image`: Optional last-frame condition for models such as VACE.
|
||||
- `--mask-image`: Optional inpainting mask. White pixels are regenerated and black pixels are preserved.
|
||||
- `--reference-image`: Optional reference image; repeat it to provide multiple references.
|
||||
- `--extra-body`: JSON object of model-specific generation params, filtered against the model's declared `extra_body_params` (see [`vllm_omni/model_extras`](../../../vllm_omni/model_extras)). Used by Cosmos3.
|
||||
- `--prompt`: Text description of desired motion/animation.
|
||||
- `--height/--width`: Output resolution (auto-calculated from image if not set). Dimensions should be multiples of 16.
|
||||
- `--num-frames`: Number of frames (default 81).
|
||||
- `--num-frames`: Number of frames (default: model-specific — Wan 81, LTX2 121, Cosmos3 189).
|
||||
- `--guidance-scale` and `--guidance-scale-high`: CFG scale (applied to low/high-noise stages for MoE).
|
||||
- `--negative-prompt`: Optional list of artifacts to suppress.
|
||||
- `--boundary-ratio`: Boundary split ratio for two-stage MoE models.
|
||||
- `--flow-shift`: Scheduler flow shift (5.0 for 720p, 12.0 for 480p).
|
||||
- `--flow-shift`: Scheduler flow shift (default: model-specific — Wan/LTX2 5.0, Cosmos3 10.0).
|
||||
- `--sample-solver`: Wan2.2 sampling solver. Use `unipc` for the default multistep solver, or `euler` for Lightning/Distill checkpoints.
|
||||
- `--num-inference-steps`: Number of denoising steps (default 50).
|
||||
- `--num-inference-steps`: Number of denoising steps (default: model-specific — Wan 50, LTX2 40, Cosmos3 35).
|
||||
- `--fps`: Frames per second for the saved MP4 (requires `diffusers` export_to_video).
|
||||
- `--output`: Path to save the generated video.
|
||||
- `--vae-use-slicing`: Enable VAE slicing for memory optimization.
|
||||
@@ -76,5 +96,98 @@ Key arguments:
|
||||
|
||||
> ℹ️ If you encounter OOM errors, try using `--vae-use-slicing` and `--vae-use-tiling` to reduce memory usage.
|
||||
|
||||
## Wan2.1 VACE Conditional Tasks
|
||||
|
||||
The shared script selects the VACE conditioning structure from the media inputs.
|
||||
No explicit mode parameter is required: the script constructs the source video,
|
||||
mask, or reference images consumed by the VACE pipeline.
|
||||
Download the same Hugging Face assets used by the original VACE example:
|
||||
|
||||
```bash
|
||||
wget -O astronaut.jpg https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/astronaut.jpg
|
||||
wget -O vace_first_frame.png https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/flf2v_input_first_frame.png
|
||||
wget -O vace_last_frame.png https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/flf2v_input_last_frame.png
|
||||
```
|
||||
|
||||
### Image-to-Video (I2V)
|
||||
|
||||
```bash
|
||||
python image_to_video.py \
|
||||
--model Wan-AI/Wan2.1-VACE-1.3B-diffusers \
|
||||
--image astronaut.jpg \
|
||||
--prompt "An astronaut emerging from a cracked, otherworldly egg on the surface of the moon" \
|
||||
--seed 42 --height 480 --width 832 --num-frames 81 \
|
||||
--num-inference-steps 30 --guidance-scale 5.0 --flow-shift 5.0 \
|
||||
--vae-use-tiling --output vace_i2v_output.mp4
|
||||
```
|
||||
|
||||
### Video-to-Last-Frame (V2LF)
|
||||
|
||||
```bash
|
||||
python image_to_video.py \
|
||||
--model Wan-AI/Wan2.1-VACE-1.3B-diffusers \
|
||||
--last-image astronaut.jpg \
|
||||
--prompt "An astronaut emerging from a cracked, otherworldly egg on the surface of the moon" \
|
||||
--seed 42 --height 480 --width 832 --num-frames 81 \
|
||||
--num-inference-steps 30 --guidance-scale 5.0 --flow-shift 5.0 \
|
||||
--vae-use-tiling --output vace_v2lf_output.mp4
|
||||
```
|
||||
|
||||
### First-Last-Frame-to-Video (FLF2V)
|
||||
|
||||
```bash
|
||||
python image_to_video.py \
|
||||
--model Wan-AI/Wan2.1-VACE-1.3B-diffusers \
|
||||
--image vace_first_frame.png \
|
||||
--last-image vace_last_frame.png \
|
||||
--prompt "CG animation style, a small blue bird takes off from a branch and lands on another branch" \
|
||||
--seed 42 --height 512 --width 512 --num-frames 81 \
|
||||
--num-inference-steps 30 --guidance-scale 5.0 --flow-shift 5.0 \
|
||||
--vae-use-tiling --output vace_flf2v_output.mp4
|
||||
```
|
||||
|
||||
### Inpainting
|
||||
|
||||
Create a mask matching the original VACE example: a 160-pixel-wide white
|
||||
vertical stripe marks the region to regenerate.
|
||||
|
||||
```bash
|
||||
python - <<'PY'
|
||||
from PIL import Image
|
||||
|
||||
mask = Image.new("L", (832, 480), 0)
|
||||
mask.paste(255, (336, 0, 496, 480))
|
||||
mask.save("vace_center_mask.png")
|
||||
PY
|
||||
```
|
||||
|
||||
```bash
|
||||
python image_to_video.py \
|
||||
--model Wan-AI/Wan2.1-VACE-1.3B-diffusers \
|
||||
--image astronaut.jpg \
|
||||
--mask-image vace_center_mask.png \
|
||||
--prompt "Shrek, the ogre, walks out of a building in a happy mood" \
|
||||
--seed 42 --height 480 --width 832 --num-frames 81 \
|
||||
--num-inference-steps 30 --guidance-scale 5.0 --flow-shift 5.0 \
|
||||
--vae-use-tiling --output vace_inpaint_output.mp4
|
||||
```
|
||||
|
||||
### Reference-to-Video (R2V)
|
||||
|
||||
Repeat `--reference-image` to provide more than one reference image.
|
||||
|
||||
```bash
|
||||
python image_to_video.py \
|
||||
--model Wan-AI/Wan2.1-VACE-1.3B-diffusers \
|
||||
--reference-image astronaut.jpg \
|
||||
--prompt "Camera slowly zooms out from the character walking in a garden" \
|
||||
--seed 42 --height 480 --width 832 --num-frames 81 \
|
||||
--num-inference-steps 30 --guidance-scale 5.0 --flow-shift 5.0 \
|
||||
--vae-use-tiling --output vace_r2v_output.mp4
|
||||
```
|
||||
|
||||
The VACE T2V command is documented in the shared
|
||||
[`text_to_video.py`](../text_to_video/text_to_video.md#wan21-vace-t2v) example.
|
||||
|
||||
For Wan2.2 LightX2V-converted local Diffusers directories and related LoRA
|
||||
assets, see the [LoRA guide](../../../docs/user_guide/diffusion/lora.md#wan22-lightx2v-offline-assembly).
|
||||
@@ -2,13 +2,14 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Image-to-Video generation example using Wan2.2 I2V/TI2V models, LTX2, or HunyuanVideo-1.5.
|
||||
Image-to-Video generation example using Wan2.2 I2V/TI2V models, LTX2, HunyuanVideo-1.5, or Wan2.1 VACE.
|
||||
|
||||
Supports:
|
||||
- Wan2.2-I2V-A14B-Diffusers: MoE model with CLIP image encoder
|
||||
- Wan2.2-TI2V-5B-Diffusers: Unified T2V+I2V model (dense 5B)
|
||||
- LTX2 image-to-video pipeline
|
||||
- HunyuanVideo-1.5 I2V: SigLIP + VAE dual image conditioning
|
||||
- Wan2.1 VACE: first/last-frame, inpainting, and reference conditioning
|
||||
|
||||
Usage:
|
||||
# Wan I2V-A14B (MoE)
|
||||
@@ -30,9 +31,16 @@ Usage:
|
||||
python image_to_video.py --model hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v \
|
||||
--image input.jpg --prompt "A cat playing with yarn" \
|
||||
--flow-shift 5.0 --guidance-scale 6.0
|
||||
|
||||
# Cosmos3 I2V (image conditioning)
|
||||
python image_to_video.py --model nvidia/Cosmos3-Nano \
|
||||
--image input.jpg --prompt "The scene comes to life with smooth, natural motion." \
|
||||
--num-frames 189 --num-inference-steps 35 --guidance-scale 6.0 --fps 24 \
|
||||
--extra-body '{"flow_shift": 10.0, "max_sequence_length": 4096, "guardrails": false}'
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import functools
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
@@ -43,39 +51,59 @@ import PIL.Image
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.model_extras import (
|
||||
build_image_to_video_prompt,
|
||||
get_extra_body_params,
|
||||
get_model_class_name,
|
||||
)
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
|
||||
def parse_profiler_config(value: str) -> dict[str, Any]:
|
||||
def parse_json_object(value: str, flag_name: str = "argument") -> dict[str, Any]:
|
||||
"""Parse a CLI value as a JSON object, attributing errors to ``flag_name``."""
|
||||
try:
|
||||
config = json.loads(value)
|
||||
except json.JSONDecodeError as e:
|
||||
raise argparse.ArgumentTypeError(f"--profiler-config must be valid JSON: {e}") from e
|
||||
raise argparse.ArgumentTypeError(f"{flag_name} must be valid JSON: {e}") from e
|
||||
if not isinstance(config, dict):
|
||||
raise argparse.ArgumentTypeError("--profiler-config must be a JSON object")
|
||||
raise argparse.ArgumentTypeError(f"{flag_name} must be a JSON object")
|
||||
return config
|
||||
|
||||
|
||||
parse_profiler_config = functools.partial(parse_json_object, flag_name="--profiler-config")
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Generate a video from an image (Wan2.2, LTX2, HunyuanVideo-1.5).")
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Generate a video from one or more images (Wan2.2, LTX2, HunyuanVideo-1.5, or Wan2.1 VACE)."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
default="Wan-AI/Wan2.2-I2V-A14B-Diffusers",
|
||||
help="Diffusers I2V model ID or local path (Wan2.2 or HunyuanVideo-1.5).",
|
||||
help="Diffusers I2V model ID or local path (Wan2.2, HunyuanVideo-1.5, or Wan2.1 VACE).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model-class-name",
|
||||
default=None,
|
||||
help="Override model class name (e.g., LTX2ImageToVideoPipeline).",
|
||||
)
|
||||
parser.add_argument("--image", required=True, help="Path to input image.")
|
||||
parser.add_argument("--image", help="Path to the first-frame or source image.")
|
||||
parser.add_argument("--last-image", help="Path to a last-frame condition (used by models such as VACE).")
|
||||
parser.add_argument("--mask-image", help="Path to an inpainting mask (used by models such as VACE).")
|
||||
parser.add_argument(
|
||||
"--reference-image",
|
||||
action="append",
|
||||
default=None,
|
||||
help="Path to a reference image. Repeat to provide multiple references.",
|
||||
)
|
||||
parser.add_argument("--prompt", default="", help="Text prompt describing the desired motion.")
|
||||
parser.add_argument("--negative-prompt", default="", help="Negative prompt.")
|
||||
parser.add_argument("--seed", type=int, default=42, help="Random seed.")
|
||||
parser.add_argument("--guidance-scale", type=float, default=5.0, help="CFG scale.")
|
||||
parser.add_argument("--guidance-scale", type=float, default=None, help="CFG scale. Default: model-specific.")
|
||||
parser.add_argument(
|
||||
"--guidance-scale-high", type=float, default=None, help="Optional separate CFG for high-noise (MoE only)."
|
||||
)
|
||||
@@ -83,8 +111,10 @@ def parse_args() -> argparse.Namespace:
|
||||
"--height", type=int, default=None, help="Video height (auto-calculated from image if not set)."
|
||||
)
|
||||
parser.add_argument("--width", type=int, default=None, help="Video width (auto-calculated from image if not set).")
|
||||
parser.add_argument("--num-frames", type=int, default=81, help="Number of frames.")
|
||||
parser.add_argument("--num-inference-steps", type=int, default=50, help="Sampling steps.")
|
||||
parser.add_argument("--num-frames", type=int, default=None, help="Number of frames. Default: model-specific.")
|
||||
parser.add_argument(
|
||||
"--num-inference-steps", type=int, default=None, help="Sampling steps. Default: model-specific."
|
||||
)
|
||||
parser.add_argument("--boundary-ratio", type=float, default=0.875, help="Boundary split ratio for MoE models.")
|
||||
parser.add_argument(
|
||||
"--frame-rate",
|
||||
@@ -93,7 +123,10 @@ def parse_args() -> argparse.Namespace:
|
||||
help="Optional generation frame rate (used by models like LTX2). Defaults to --fps.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--flow-shift", type=float, default=5.0, help="Scheduler flow_shift (5.0 for 720p, 12.0 for 480p)."
|
||||
"--flow-shift",
|
||||
type=float,
|
||||
default=None,
|
||||
help="Scheduler flow_shift. Default: model-specific (Wan 5.0, Cosmos3 10.0).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--sample-solver",
|
||||
@@ -244,6 +277,19 @@ def parse_args() -> argparse.Namespace:
|
||||
default=None,
|
||||
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
|
||||
)
|
||||
parser.add_argument(
|
||||
"--extra-body",
|
||||
type=functools.partial(parse_json_object, flag_name="--extra-body"),
|
||||
default=None,
|
||||
help=(
|
||||
"Model-specific generation params as a JSON object. Keys are filtered "
|
||||
"against the model's declared extra_body_params (see vllm_omni/model_extras), "
|
||||
"so unknown keys for the chosen model are silently dropped. "
|
||||
'Cosmos3 V2V example: \'{"condition_frame_indexes_vision": [0, 1], '
|
||||
'"condition_video_keep": "first", "flow_shift": 10.0, '
|
||||
'"max_sequence_length": 4096, "guardrails": false}\'.'
|
||||
),
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
@@ -267,31 +313,63 @@ def main():
|
||||
model_name = str(args.model).lower() if args.model is not None else ""
|
||||
model_class_name = args.model_class_name
|
||||
is_ltx2 = "ltx2" in model_name or (model_class_name and "ltx2" in model_class_name.lower())
|
||||
is_cosmos = "cosmos" in model_name or (model_class_name is not None and "cosmos" in model_class_name.lower())
|
||||
if model_class_name is None and is_ltx2:
|
||||
model_class_name = "LTX2ImageToVideoPipeline"
|
||||
|
||||
# Load input image
|
||||
image = PIL.Image.open(args.image).convert("RGB")
|
||||
image = PIL.Image.open(args.image).convert("RGB") if args.image else None
|
||||
last_image = PIL.Image.open(args.last_image).convert("RGB") if args.last_image else None
|
||||
mask_image = PIL.Image.open(args.mask_image).convert("L") if args.mask_image else None
|
||||
reference_images = (
|
||||
[PIL.Image.open(path).convert("RGB") for path in args.reference_image] if args.reference_image else None
|
||||
)
|
||||
dimension_image = image or last_image or (reference_images[0] if reference_images else None)
|
||||
if dimension_image is None:
|
||||
raise ValueError("Provide --image, --last-image, or at least one --reference-image.")
|
||||
|
||||
fps = args.fps if args.fps is not None else (24 if is_ltx2 else 16)
|
||||
# Per-model generation defaults, applied only when the matching flag is omitted.
|
||||
# Cosmos3 would otherwise silently inherit the Wan2.2 defaults (wrong size/steps/shift).
|
||||
if is_cosmos:
|
||||
d_fps, d_guidance, d_num_frames, d_steps, d_flow_shift, d_max_area, d_mod = (
|
||||
24,
|
||||
6.0,
|
||||
189,
|
||||
35,
|
||||
10.0,
|
||||
1280 * 720,
|
||||
16,
|
||||
)
|
||||
elif is_ltx2:
|
||||
d_fps, d_guidance, d_num_frames, d_steps, d_flow_shift, d_max_area, d_mod = 24, 4.0, 121, 40, 5.0, 512 * 768, 32
|
||||
else: # Wan2.2 / HunyuanVideo-1.5
|
||||
d_fps, d_guidance, d_num_frames, d_steps, d_flow_shift, d_max_area, d_mod = 16, 5.0, 81, 50, 5.0, 480 * 832, 16
|
||||
|
||||
fps = args.fps if args.fps is not None else d_fps
|
||||
frame_rate = args.frame_rate if args.frame_rate is not None else float(fps)
|
||||
guidance_scale = args.guidance_scale if args.guidance_scale is not None else (4.0 if is_ltx2 else 5.0)
|
||||
num_frames = args.num_frames if args.num_frames is not None else (121 if is_ltx2 else 81)
|
||||
num_inference_steps = args.num_inference_steps if args.num_inference_steps is not None else (40 if is_ltx2 else 50)
|
||||
guidance_scale = args.guidance_scale if args.guidance_scale is not None else d_guidance
|
||||
num_frames = args.num_frames if args.num_frames is not None else d_num_frames
|
||||
num_inference_steps = args.num_inference_steps if args.num_inference_steps is not None else d_steps
|
||||
flow_shift = args.flow_shift if args.flow_shift is not None else d_flow_shift
|
||||
|
||||
# Calculate dimensions if not provided
|
||||
# Calculate dimensions if not provided (model-aware max area).
|
||||
height = args.height
|
||||
width = args.width
|
||||
if height is None or width is None:
|
||||
# Default to 480P area for Wan2.2 I2V, 512x768 area for LTX2
|
||||
max_area = 512 * 768 if is_ltx2 else 480 * 832
|
||||
mod_value = 32 if is_ltx2 else 16
|
||||
calc_height, calc_width = calculate_dimensions(image, max_area=max_area, mod_value=mod_value)
|
||||
calc_height, calc_width = calculate_dimensions(dimension_image, max_area=d_max_area, mod_value=d_mod)
|
||||
height = height or calc_height
|
||||
width = width or calc_width
|
||||
|
||||
# Resize image to target dimensions
|
||||
image = image.resize((width, height), PIL.Image.Resampling.LANCZOS)
|
||||
media_inputs: dict[str, Any] = {}
|
||||
if image is not None:
|
||||
media_inputs["image"] = image.resize((width, height), PIL.Image.Resampling.LANCZOS)
|
||||
if last_image is not None:
|
||||
media_inputs["last_image"] = last_image.resize((width, height), PIL.Image.Resampling.LANCZOS)
|
||||
if mask_image is not None:
|
||||
media_inputs["mask"] = mask_image.resize((width, height), PIL.Image.Resampling.NEAREST)
|
||||
if reference_images is not None:
|
||||
media_inputs["reference_images"] = [
|
||||
reference.resize((width, height), PIL.Image.Resampling.LANCZOS) for reference in reference_images
|
||||
]
|
||||
|
||||
# Configure cache based on backend type
|
||||
cache_config = None
|
||||
@@ -342,7 +420,7 @@ def main():
|
||||
vae_use_slicing=args.vae_use_slicing,
|
||||
vae_use_tiling=args.vae_use_tiling,
|
||||
boundary_ratio=args.boundary_ratio,
|
||||
flow_shift=args.flow_shift,
|
||||
flow_shift=flow_shift,
|
||||
diffusion_kv_cache_dtype=args.diffusion_kv_cache_dtype,
|
||||
diffusion_kv_cache_skip_steps=args.diffusion_kv_cache_skip_steps,
|
||||
diffusion_kv_cache_skip_layers=args.diffusion_kv_cache_skip_layers,
|
||||
@@ -357,7 +435,13 @@ def main():
|
||||
)
|
||||
if args.quantization is not None:
|
||||
omni_kwargs["quantization"] = args.quantization
|
||||
# Cosmos3 loads its (gated) guardrail models at build time, so the guardrails
|
||||
# gate is an engine-level config (offline analog of the server's --no-guardrails).
|
||||
if args.extra_body and "guardrails" in args.extra_body:
|
||||
omni_kwargs["model_config"] = {"guardrails": bool(args.extra_body["guardrails"])}
|
||||
omni = Omni(**omni_kwargs)
|
||||
model_class_name = get_model_class_name(omni) or model_class_name
|
||||
declared_extra_body_params = get_extra_body_params(model_class_name)
|
||||
|
||||
if profiler_enabled:
|
||||
print("[Profiler] Starting profiling...")
|
||||
@@ -367,8 +451,8 @@ def main():
|
||||
print(f"\n{'=' * 60}")
|
||||
print("Generation Configuration:")
|
||||
print(f" Model: {args.model}")
|
||||
print(f" Inference steps: {args.num_inference_steps}")
|
||||
print(f" Frames: {args.num_frames}")
|
||||
print(f" Inference steps: {num_inference_steps}")
|
||||
print(f" Frames: {num_frames}")
|
||||
print(f" Solver: {args.sample_solver}")
|
||||
print(f" diffusion_kv_cache_dtype(config): {args.diffusion_kv_cache_dtype}")
|
||||
print(f" diffusion_kv_cache_skip_steps(config): {args.diffusion_kv_cache_skip_steps}")
|
||||
@@ -381,30 +465,43 @@ def main():
|
||||
print(f" Video size: {args.width}x{args.height}")
|
||||
print(f"{'=' * 60}\n")
|
||||
|
||||
prompt_dict = build_image_to_video_prompt(
|
||||
model_class_name=model_class_name,
|
||||
prompt=args.prompt,
|
||||
negative_prompt=args.negative_prompt,
|
||||
media_inputs=media_inputs,
|
||||
height=height,
|
||||
width=width,
|
||||
num_frames=num_frames,
|
||||
)
|
||||
sampling_params = OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
generator=generator,
|
||||
guidance_scale=guidance_scale,
|
||||
guidance_scale_2=args.guidance_scale_high,
|
||||
boundary_ratio=args.boundary_ratio,
|
||||
num_inference_steps=num_inference_steps,
|
||||
num_frames=num_frames,
|
||||
frame_rate=frame_rate,
|
||||
extra_args={
|
||||
"sample_solver": args.sample_solver,
|
||||
"flow_shift": flow_shift,
|
||||
},
|
||||
)
|
||||
|
||||
# Route model-specific knobs through extra_body, filtered against the model's
|
||||
# declared extra_body_params. Models without a declaration only forward explicit
|
||||
# --extra-body JSON (e.g. Cosmos3 V2V's condition_frame_indexes_vision).
|
||||
extra_body = dict(args.extra_body or {})
|
||||
if declared_extra_body_params:
|
||||
apply_declared_extra_args(sampling_params, declared_extra_body_params, extra_body)
|
||||
elif extra_body:
|
||||
sampling_params.extra_args.update({k: v for k, v in extra_body.items() if v is not None})
|
||||
|
||||
generation_start = time.perf_counter()
|
||||
# omni.generate() returns Generator[OmniRequestOutput, None, None]
|
||||
frames = omni.generate(
|
||||
{
|
||||
"prompt": args.prompt,
|
||||
"negative_prompt": args.negative_prompt,
|
||||
"multi_modal_data": {"image": image},
|
||||
},
|
||||
OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
generator=generator,
|
||||
guidance_scale=guidance_scale,
|
||||
guidance_scale_2=args.guidance_scale_high,
|
||||
boundary_ratio=args.boundary_ratio,
|
||||
num_inference_steps=num_inference_steps,
|
||||
num_frames=num_frames,
|
||||
frame_rate=frame_rate,
|
||||
extra_args={
|
||||
"sample_solver": args.sample_solver,
|
||||
"flow_shift": args.flow_shift,
|
||||
},
|
||||
),
|
||||
)
|
||||
frames = omni.generate(prompt_dict, sampling_params)
|
||||
generation_end = time.perf_counter()
|
||||
generation_time = generation_end - generation_start
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import argparse
|
||||
import functools
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
@@ -38,16 +39,20 @@ def is_nextstep_model(model_name: str) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def parse_profiler_config(value: str) -> dict[str, Any]:
|
||||
def parse_json_object(value: str, flag_name: str = "argument") -> dict[str, Any]:
|
||||
"""Parse a CLI value as a JSON object, attributing errors to ``flag_name``."""
|
||||
try:
|
||||
config = json.loads(value)
|
||||
except json.JSONDecodeError as e:
|
||||
raise argparse.ArgumentTypeError(f"--profiler-config must be valid JSON: {e}") from e
|
||||
raise argparse.ArgumentTypeError(f"{flag_name} must be valid JSON: {e}") from e
|
||||
if not isinstance(config, dict):
|
||||
raise argparse.ArgumentTypeError("--profiler-config must be a JSON object")
|
||||
raise argparse.ArgumentTypeError(f"{flag_name} must be a JSON object")
|
||||
return config
|
||||
|
||||
|
||||
parse_profiler_config = functools.partial(parse_json_object, flag_name="--profiler-config")
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Generate an image with supported diffusion models.")
|
||||
parser.add_argument(
|
||||
@@ -269,7 +274,7 @@ def parse_args() -> argparse.Namespace:
|
||||
)
|
||||
parser.add_argument(
|
||||
"--extra-body",
|
||||
type=parse_profiler_config,
|
||||
type=functools.partial(parse_json_object, flag_name="--extra-body"),
|
||||
default=None,
|
||||
help=(
|
||||
"Model-specific generation params as a JSON object, e.g. "
|
||||
@@ -425,6 +430,10 @@ def main():
|
||||
if use_nextstep:
|
||||
# NextStep-1.1 requires explicit pipeline class
|
||||
omni_kwargs["model_class_name"] = "NextStep11Pipeline"
|
||||
# Cosmos3 loads its (gated) guardrail models at build time, so the guardrails
|
||||
# gate is an engine-level config (offline analog of the server's --no-guardrails).
|
||||
if args.extra_body and "guardrails" in args.extra_body:
|
||||
omni_kwargs["model_config"] = {"guardrails": bool(args.extra_body["guardrails"])}
|
||||
omni = Omni(**omni_kwargs)
|
||||
model_class_name = get_model_class_name(omni)
|
||||
declared_extra_body_params = get_extra_body_params(model_class_name)
|
||||
|
||||
@@ -23,6 +23,7 @@ list of supported architectures across all modalities, see
|
||||
| OmniVoice | `k2-fsa/OmniVoice` | 2 (gen + dec) | ✓ | — | voice design, language hint | 24 kHz |
|
||||
| Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | 2 (talker + code2wav) | ✓ (Base) | ✓ | 3 task variants | 24 kHz |
|
||||
| VoxCPM2 | `openbmb/VoxCPM2` | single (native AR) | ✓ | ✓ (online) | continuation | 48 kHz |
|
||||
| IndexTTS-2 | `IndexTeam/IndexTTS-2` | 2 (AR talker + S2Mel DiT + BigVGAN) | ✓ (required) | — | emotion control (`--emo-audio`, `--emo-text`, `--emo-vector`) | 22.05 kHz |
|
||||
| Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | varies | ✓ | ✓ | voice presets | 24 kHz |
|
||||
|
||||
## Common Quick Start
|
||||
@@ -436,6 +437,50 @@ Streaming is exposed through the online OpenAI Speech API (`stream=true`). See [
|
||||
|
||||
---
|
||||
|
||||
## IndexTTS-2
|
||||
|
||||
2-stage TTS pipeline (GPT AR talker + S2Mel CFM DiT + BigVGAN vocoder) at 22.05 kHz. Every request requires reference audio for zero-shot voice cloning. Supports emotion conditioning via audio, text, or 8-dim vector.
|
||||
|
||||
### Quick start
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/indextts2/end2end.py \
|
||||
--model IndexTeam/IndexTTS-2 \
|
||||
--text "你好,这是一个语音合成测试。" \
|
||||
--ref-audio /path/to/reference.wav
|
||||
```
|
||||
|
||||
### Emotion control
|
||||
```bash
|
||||
# Emotion from reference audio
|
||||
python examples/offline_inference/text_to_speech/indextts2/end2end.py \
|
||||
--model IndexTeam/IndexTTS-2 \
|
||||
--text "今天天气真好!" \
|
||||
--ref-audio /path/to/ref.wav \
|
||||
--emo-audio /path/to/happy.wav
|
||||
|
||||
# Emotion from 8-dim vector (happy angry sad afraid disgusted melancholy surprised calm)
|
||||
python examples/offline_inference/text_to_speech/indextts2/end2end.py \
|
||||
--model IndexTeam/IndexTTS-2 \
|
||||
--text "今天天气真好!" \
|
||||
--ref-audio /path/to/ref.wav \
|
||||
--emo-vector 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
|
||||
|
||||
# Emotion from text description
|
||||
python examples/offline_inference/text_to_speech/indextts2/end2end.py \
|
||||
--model IndexTeam/IndexTTS-2 \
|
||||
--text "今天天气真好!" \
|
||||
--ref-audio /path/to/ref.wav \
|
||||
--emo-text "happy and excited"
|
||||
```
|
||||
|
||||
### Notes
|
||||
- `--ref-audio` is **required** — IndexTTS-2 does not support text-only synthesis.
|
||||
- Stage 0 (AR Talker): GPT-2 generates mel codes from text + reference audio.
|
||||
- Stage 1 (S2Mel + BigVGAN): CFM DiT converts mel codes to waveform at 22.05 kHz.
|
||||
- Deploy config: `vllm_omni/deploy/indextts2.yaml`. Stage 1 runs with `enforce_eager: true` (DiT has dynamic shapes).
|
||||
|
||||
---
|
||||
|
||||
## Voxtral TTS
|
||||
|
||||
Voxtral-4B-TTS (Mistral). Has its own argparse surface; uses voice presets and the `mistral_common` `SpeechRequest` protocol.
|
||||
@@ -476,3 +521,100 @@ Available voice presets are listed on the HF model card (`mistralai/Voxtral-4B-T
|
||||
- `--num-prompts N` replicates the prompt for performance measurement.
|
||||
- `--concurrency M` requires `--streaming` and must evenly divide `--num-prompts`.
|
||||
- Run `--help` for the full argument surface.
|
||||
|
||||
---
|
||||
|
||||
## SoulX-Singer
|
||||
|
||||
Singing voice synthesis (SVS) and conversion (SVC) at 24 kHz. Script: `soulxsinger/end2end.py`. Deploy: `vllm_omni/deploy/soulxsinger_svs.yaml` or `soulxsinger_svc.yaml`.
|
||||
|
||||
### Prerequisites
|
||||
|
||||
Download DiT and preprocess weights, then set up separate SVS / SVC view directories. Copy `soulxsinger/utils/phoneme/phone_set.json` from upstream [SoulX-Singer](https://github.com/Soul-AILab/SoulX-Singer) into the model weights dir as `phoneme/phone_set.json` — HuggingFace does not ship it.
|
||||
|
||||
```bash
|
||||
# 1. DiT weights
|
||||
export BASE=path/to/SoulX-Singer
|
||||
export PREPROCESS=path/to/SoulX-Singer-Preprocess
|
||||
export SVC_DIR=path/to/SoulX-Singer-svc
|
||||
|
||||
huggingface-cli download Soul-AILab/SoulX-Singer --local-dir "$BASE"
|
||||
|
||||
# 2. Preprocess weights (required)
|
||||
huggingface-cli download Soul-AILab/SoulX-Singer-Preprocess --local-dir "$PREPROCESS"
|
||||
export SOULX_PREPROCESS_WEIGHTS_DIR="$PREPROCESS"
|
||||
|
||||
# 3. SVS / SVC view directories
|
||||
mkdir -p "$SVC_DIR"
|
||||
cp $BASE/{config.yaml,README.md,assets} $SVC_DIR
|
||||
mv $BASE/model-svc.pt $SVC_DIR/model-svc.pt
|
||||
|
||||
cat > "$BASE/config.json" <<'EOF'
|
||||
{
|
||||
"model_type": "soulxsinger",
|
||||
"architectures": ["SoulXSingerPipeline"],
|
||||
"max_num_seqs": 1
|
||||
}
|
||||
EOF
|
||||
|
||||
cat > "$SVC_DIR/config.json" <<'EOF'
|
||||
{
|
||||
"model_type": "soulxsinger",
|
||||
"architectures": ["SoulXSingerSVCPipeline"],
|
||||
"max_num_seqs": 1
|
||||
}
|
||||
EOF
|
||||
```
|
||||
|
||||
`config.yaml` hyper-parameters live under `$BASE`; each view's `config.json` `architectures` field is the single source of truth for SVS vs SVC. Point `--model` at the matching directory (`$BASE` for SVS, `$SVC_DIR` for SVC). Deploy YAML is chosen automatically from `config.json`; optional `--svs` / `--svc` only assert the mode matches.
|
||||
|
||||
**Online preprocess** is the default: pass `--prompt-audio` and `--target-audio`, and the worker runs vocal separation, F0, and (for SVS) lyrics/MIDI before DiT. Install only what your run needs:
|
||||
|
||||
```bash
|
||||
pip install "BS-RoFormer" # vocal sep + F0 on GPU — SVS and SVC
|
||||
```
|
||||
|
||||
Mandarin SVS also needs FunASR and Chinese G2P; `ffmpeg` must be on `PATH`:
|
||||
|
||||
```bash
|
||||
# install optional dependencies:
|
||||
pip install -e ".[soulx-svs]"
|
||||
```
|
||||
|
||||
English SVS adds NeMo ASR and NLTK data; pass `--language English`:
|
||||
|
||||
```bash
|
||||
pip install "nemo_toolkit[asr]==2.6.1" lhotse==1.32.2
|
||||
python -c "import nltk; nltk.download('cmudict'); nltk.download('averaged_perceptron_tagger_eng')"
|
||||
```
|
||||
|
||||
**Precomputed metadata** is the alternative: pass both `--prompt-metadata-path` and `--target-metadata-path` and skip online ASR/ROSVOT — none of the packages above are required. JSON can be produced by integrated preprocess on a prior run, or by upstream [SoulX-Singer](https://github.com/Soul-AILab/SoulX-Singer) `preprocess/` scripts if you prefer to run that outside vLLM-Omni.
|
||||
|
||||
### Quick start
|
||||
|
||||
```bash
|
||||
# SVS — default demo audio: tests/assets/soulxsinger/zh_prompt.mp3 + music.mp3
|
||||
python examples/offline_inference/text_to_speech/soulxsinger/end2end.py \
|
||||
--model "$BASE" \
|
||||
--preprocess-weights-dir "$PREPROCESS" \
|
||||
--control score \
|
||||
--num-inference-steps 32 \
|
||||
-o output.wav
|
||||
|
||||
python examples/offline_inference/text_to_speech/soulxsinger/end2end.py \
|
||||
--model "$SVC_DIR" \
|
||||
--preprocess-weights-dir "$PREPROCESS" \
|
||||
--svc \
|
||||
--num-inference-steps 32 \
|
||||
-o output_svc.wav
|
||||
```
|
||||
|
||||
`SOULX_PREPROCESS_WEIGHTS_DIR` makes `--preprocess-weights-dir` optional. Long SVS targets are handled in one request. See `end2end.py --help` for `--pitch-shift`, `--vocal-sep`, `--auto-shift`, and language/control options.
|
||||
|
||||
### Notes
|
||||
|
||||
- Output: 24 kHz mono WAV; batch only.
|
||||
- Defaults match upstream: `--guidance-scale 3.0`, `--seed 42`, `--auto-shift` on.
|
||||
- SVS `--control`: `score` or `melody`. MIDI / lyric QC: upstream `midi_editor` only.
|
||||
|
||||
---
|
||||
@@ -0,0 +1,187 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Offline inference example for IndexTTS2 via vLLM-Omni.
|
||||
|
||||
Two-stage pipeline: GPT AR (Stage 0) → S2Mel + BigVGAN (Stage 1).
|
||||
Output is 22050 Hz mono WAV.
|
||||
|
||||
Usage:
|
||||
python end2end.py \
|
||||
--model /path/to/IndexTeam/IndexTTS-2 \
|
||||
--text "你好,这是一个语音合成测试。" \
|
||||
--ref-audio /path/to/ref.wav
|
||||
|
||||
# With emotion audio:
|
||||
python end2end.py \
|
||||
--model /path/to/IndexTeam/IndexTTS-2 \
|
||||
--text "今天天气真好!" \
|
||||
--ref-audio /path/to/ref.wav \
|
||||
--emo-audio /path/to/happy.wav
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import soundfile as sf
|
||||
import torch
|
||||
from vllm import SamplingParams
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
|
||||
from vllm_omni import Omni # noqa: E402
|
||||
from vllm_omni.model_executor.models.indextts2.prompt_utils import ( # noqa: E402
|
||||
build_indextts2_prefill_prompt_ids,
|
||||
)
|
||||
|
||||
|
||||
def build_request(
|
||||
model: str,
|
||||
text: str,
|
||||
ref_audio_path: str | None = None,
|
||||
emo_audio_path: str | None = None,
|
||||
emo_text: str | None = None,
|
||||
emo_vector: list[float] | None = None,
|
||||
emo_alpha: float | None = None,
|
||||
use_emo_text: bool = False,
|
||||
use_random: bool = False,
|
||||
) -> dict:
|
||||
additional: dict = {"text": [text]}
|
||||
if ref_audio_path:
|
||||
additional["voice"] = [str(ref_audio_path)]
|
||||
if emo_audio_path:
|
||||
additional["emo_audio"] = [str(emo_audio_path)]
|
||||
if emo_text:
|
||||
additional["emo_text"] = [emo_text]
|
||||
if emo_vector is not None:
|
||||
additional["emo_vector"] = [emo_vector]
|
||||
if emo_alpha is not None:
|
||||
additional["emo_alpha"] = [emo_alpha]
|
||||
if use_emo_text:
|
||||
additional["use_emo_text"] = [True]
|
||||
if use_random:
|
||||
additional["use_random"] = [True]
|
||||
return {
|
||||
"prompt_token_ids": build_indextts2_prefill_prompt_ids(model, text),
|
||||
"additional_information": additional,
|
||||
}
|
||||
|
||||
|
||||
def save_audio(waveform: torch.Tensor, path: str, sample_rate: int = 22050) -> None:
|
||||
audio_np = waveform.float().numpy()
|
||||
sf.write(path, audio_np, sample_rate)
|
||||
print(f" Saved {path} ({audio_np.shape}, {sample_rate} Hz)")
|
||||
|
||||
|
||||
def extract_audio(mm: dict) -> tuple[torch.Tensor | None, int]:
|
||||
audio = mm.get("audio")
|
||||
if audio is None:
|
||||
audio = mm.get("model_outputs")
|
||||
if isinstance(audio, list):
|
||||
chunks = [chunk.reshape(-1) for chunk in audio if isinstance(chunk, torch.Tensor) and chunk.numel() > 0]
|
||||
audio = torch.cat(chunks, dim=0) if chunks else None
|
||||
|
||||
sr_val = mm.get("sr")
|
||||
if isinstance(sr_val, list):
|
||||
sr_val = sr_val[-1] if sr_val else None
|
||||
if hasattr(sr_val, "item"):
|
||||
sample_rate = int(sr_val.item())
|
||||
else:
|
||||
sample_rate = int(sr_val) if sr_val is not None else 22050
|
||||
|
||||
return audio if isinstance(audio, torch.Tensor) else None, sample_rate
|
||||
|
||||
|
||||
def main(args) -> None:
|
||||
omni = Omni(
|
||||
model=args.model,
|
||||
deploy_config=args.deploy_config,
|
||||
stage_init_timeout=args.stage_init_timeout,
|
||||
)
|
||||
|
||||
# Stage 0: GPT AR (text → mel codes)
|
||||
gpt_sampling = SamplingParams(
|
||||
temperature=0.8,
|
||||
top_p=0.8,
|
||||
top_k=30,
|
||||
max_tokens=1500,
|
||||
repetition_penalty=10.0,
|
||||
stop_token_ids=[8193],
|
||||
seed=args.seed if args.seed is not None else 42,
|
||||
detokenize=False,
|
||||
)
|
||||
# Stage 1: S2Mel + BigVGAN (non-AR, params mostly ignored)
|
||||
s2mel_sampling = SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=65536,
|
||||
detokenize=True,
|
||||
)
|
||||
sampling_params = [gpt_sampling, s2mel_sampling]
|
||||
|
||||
output_dir = Path(args.output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
print(f"Synthesizing: {args.text!r}")
|
||||
if args.ref_audio:
|
||||
print(f" ref_audio: {args.ref_audio}")
|
||||
inputs = build_request(
|
||||
model=args.model,
|
||||
text=args.text,
|
||||
ref_audio_path=args.ref_audio,
|
||||
emo_audio_path=args.emo_audio,
|
||||
emo_text=args.emo_text,
|
||||
emo_vector=args.emo_vector,
|
||||
emo_alpha=args.emo_alpha,
|
||||
use_emo_text=args.use_emo_text,
|
||||
use_random=args.use_random,
|
||||
)
|
||||
|
||||
for i, omni_out in enumerate(omni.generate(inputs, sampling_params_list=sampling_params)):
|
||||
mm = omni_out.multimodal_output
|
||||
if not mm:
|
||||
print(f" [req {i}] No multimodal output.")
|
||||
continue
|
||||
audio, sr = extract_audio(mm)
|
||||
if audio is None:
|
||||
print(f" [req {i}] No waveform in multimodal_output.")
|
||||
continue
|
||||
out_path = str(output_dir / f"output_{i}.wav")
|
||||
save_audio(audio.cpu(), out_path, sr)
|
||||
|
||||
print("Done.")
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = FlexibleArgumentParser(description="IndexTTS2 offline inference")
|
||||
parser.add_argument("--model", required=True, help="HF model path for IndexTTS2.")
|
||||
parser.add_argument("--text", default="你好,这是IndexTTS2语音合成测试。")
|
||||
parser.add_argument("--ref-audio", required=True, help="Reference audio for voice cloning.")
|
||||
parser.add_argument("--emo-audio", default=None, help="Emotion reference audio.")
|
||||
parser.add_argument("--emo-text", default=None, help="Emotion description text.")
|
||||
parser.add_argument(
|
||||
"--emo-vector",
|
||||
type=float,
|
||||
nargs=8,
|
||||
default=None,
|
||||
help="8-dim emotion vector: happy angry sad afraid disgusted melancholic surprised calm.",
|
||||
)
|
||||
parser.add_argument("--emo-alpha", type=float, default=None, help="Emotion weight in [0, 1].")
|
||||
parser.add_argument("--use-emo-text", action="store_true", help="Infer emotion vector from emo-text or text.")
|
||||
parser.add_argument("--use-random", action="store_true", help="Use random emotion prototypes.")
|
||||
parser.add_argument("--seed", type=int, default=None)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
default=os.path.join(
|
||||
os.environ.get("XDG_CACHE_HOME", os.path.join(os.path.expanduser("~"), ".cache")),
|
||||
"indextts2_output",
|
||||
),
|
||||
)
|
||||
parser.add_argument("--deploy-config", default=None)
|
||||
parser.add_argument("--stage-init-timeout", type=int, default=600)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(parse_args())
|
||||
@@ -1,11 +1,12 @@
|
||||
"""Offline inference example for MOSS-TTS variants.
|
||||
|
||||
Covers all five models:
|
||||
- OpenMOSS-Team/MOSS-TTS (8B, general TTS)
|
||||
- OpenMOSS-Team/MOSS-TTSD-v1.0 (8B, dialogue TTS)
|
||||
- OpenMOSS-Team/MOSS-SoundEffect (8B, sound effect synthesis)
|
||||
- OpenMOSS-Team/MOSS-TTS-Realtime (1.7B, low-latency streaming)
|
||||
- OpenMOSS-Team/MOSS-VoiceGenerator(1.7B, voice design)
|
||||
Covers six models:
|
||||
- OpenMOSS-Team/MOSS-TTS (8B, general TTS)
|
||||
- OpenMOSS-Team/MOSS-TTSD-v1.0 (8B, dialogue TTS)
|
||||
- OpenMOSS-Team/MOSS-SoundEffect (8B, sound effect synthesis)
|
||||
- OpenMOSS-Team/MOSS-TTS-Realtime (1.7B, low-latency streaming)
|
||||
- OpenMOSS-Team/MOSS-VoiceGenerator (1.7B, voice design)
|
||||
- OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5 (n_vq=12, 48 kHz stereo)
|
||||
|
||||
Usage
|
||||
-----
|
||||
@@ -210,12 +211,18 @@ def run_tts(args: argparse.Namespace) -> None:
|
||||
"OpenMOSS-Team/MOSS-SoundEffect": "moss_sound_effect",
|
||||
"OpenMOSS-Team/MOSS-TTS-Realtime": "moss_tts_realtime",
|
||||
"OpenMOSS-Team/MOSS-VoiceGenerator": "moss_voice_generator",
|
||||
"OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5": "moss_tts_local",
|
||||
}
|
||||
deploy_config = deploy_map.get(args.model, "moss_tts")
|
||||
|
||||
is_sound_effect = "SoundEffect" in args.model
|
||||
is_voice_gen = "VoiceGenerator" in args.model
|
||||
is_realtime = "Realtime" in args.model
|
||||
# Local-v1.5 ships its own AutoProcessor (processor_config.json +
|
||||
# processing_moss_tts.py), so it reuses the same _build_unified_codes
|
||||
# path as the Delay variants below — just a different n_vq/codec.
|
||||
is_local = "Local" in args.model
|
||||
output_sr = 48000 if is_local else 24000
|
||||
|
||||
ref_audio = None
|
||||
builder_kwargs: dict = {}
|
||||
@@ -314,27 +321,33 @@ def run_tts(args: argparse.Namespace) -> None:
|
||||
wav = mm.get("model_outputs")
|
||||
if wav is None:
|
||||
continue
|
||||
# Local-v1.5's codec emits stereo chunks shaped (C, T); keep the
|
||||
# channel axis intact (don't flatten) so concatenation below stays
|
||||
# along the time axis. Mono variants stay (T,) as before.
|
||||
if isinstance(wav, list):
|
||||
non_empty = [w.reshape(-1) for w in wav if isinstance(w, torch.Tensor) and w.numel() > 0]
|
||||
non_empty = [w for w in wav if isinstance(w, torch.Tensor) and w.numel() > 0]
|
||||
if non_empty and t_first_audio is None:
|
||||
t_first_audio = _time.monotonic()
|
||||
chunks.extend(non_empty)
|
||||
elif isinstance(wav, torch.Tensor) and wav.numel() > 0:
|
||||
if t_first_audio is None:
|
||||
t_first_audio = _time.monotonic()
|
||||
chunks.append(wav.reshape(-1))
|
||||
chunks.append(wav)
|
||||
|
||||
t_gen_end = _time.monotonic()
|
||||
if not chunks:
|
||||
print("No audio in output — check model loading logs.")
|
||||
return
|
||||
|
||||
wav_t = torch.cat(chunks).float().cpu()
|
||||
sf.write(args.output, wav_t.numpy(), 24000)
|
||||
print(f"Saved {len(wav_t) / 24000:.2f}s of audio to {args.output}")
|
||||
cat_dim = -1 if chunks[0].dim() == 2 else 0
|
||||
wav_t = torch.cat(chunks, dim=cat_dim).float().cpu()
|
||||
n_samples = wav_t.shape[-1]
|
||||
# soundfile wants (T,) mono or (T, C) multi-channel.
|
||||
sf.write(args.output, (wav_t.transpose(0, 1) if wav_t.dim() == 2 else wav_t).numpy(), output_sr)
|
||||
print(f"Saved {n_samples / output_sr:.2f}s of audio to {args.output}")
|
||||
ttfa_ms = (t_first_audio - t_gen_start) * 1000.0 if t_first_audio is not None else float("nan")
|
||||
gen_total_ms = (t_gen_end - t_gen_start) * 1000.0
|
||||
print(f"TTFA_MS={ttfa_ms:.1f} GEN_TOTAL_MS={gen_total_ms:.1f} AUDIO_S={len(wav_t) / 24000:.2f}")
|
||||
print(f"TTFA_MS={ttfa_ms:.1f} GEN_TOTAL_MS={gen_total_ms:.1f} AUDIO_S={n_samples / output_sr:.2f}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
"""Repeatable SoulX-Singer offline benchmark (preprocess + DiT, RTF + stage timings).
|
||||
|
||||
Usage:
|
||||
python benchmark.py --model /path/to/SoulX-Singer --svs \\
|
||||
--prompt-audio ../SoulX-Singer/example/audio/zh_prompt.mp3 \\
|
||||
--target-audio ../SoulX-Singer/example/audio/music.mp3 \\
|
||||
--preprocess-weights-dir /path/to/SoulX-Singer-Preprocess \\
|
||||
-o benchmark.wav
|
||||
|
||||
python benchmark.py ... --enable-diffusion-pipeline-profiler
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import statistics
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import soundfile as sf
|
||||
from end2end import (
|
||||
SVC_DEPLOY_CONFIG,
|
||||
SVS_DEPLOY_CONFIG,
|
||||
add_inference_args,
|
||||
build_sampling,
|
||||
extract_audio,
|
||||
resolve_preprocess_weights_dir,
|
||||
)
|
||||
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
|
||||
|
||||
def _audio_duration_sec(outputs) -> float:
|
||||
audio_np, sr = extract_audio(outputs)
|
||||
return audio_np.size / sr
|
||||
|
||||
|
||||
def _stage_durations_from_output(outputs) -> dict[str, float]:
|
||||
omni_out = outputs[0]
|
||||
durations = getattr(omni_out, "stage_durations", None) or {}
|
||||
if durations:
|
||||
return dict(durations)
|
||||
ro = getattr(omni_out, "request_output", None)
|
||||
if ro is not None:
|
||||
return dict(getattr(ro, "stage_durations", None) or {})
|
||||
return {}
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="SoulX-Singer offline benchmark")
|
||||
add_inference_args(parser)
|
||||
parser.add_argument("--warmup", type=int, default=1, help="Warmup runs (excluded from stats)")
|
||||
parser.add_argument("--runs", type=int, default=3, help="Measured runs")
|
||||
parser.add_argument(
|
||||
"--enable-diffusion-pipeline-profiler",
|
||||
action="store_true",
|
||||
help="Enable low-overhead stage timing in DiffusionOutput.stage_durations",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--enforce-eager",
|
||||
action="store_true",
|
||||
help="Disable torch.compile on diff_estimator (eager baseline for A/B)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dtype",
|
||||
type=str,
|
||||
default=None,
|
||||
choices=["bfloat16", "bf16", "float16", "fp16", "half"],
|
||||
help="DiT trunk dtype (default from deploy YAML). Use float16 to match upstream --fp16.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
"-o",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Save WAV from the last measured run (after warmup)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--save-each-run",
|
||||
action="store_true",
|
||||
help="With --output, also write measured runs as <stem>_run1.wav, _run2.wav, ...",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
preprocess_dir = resolve_preprocess_weights_dir(args.preprocess_weights_dir)
|
||||
deploy_config = args.deploy_config or str(SVS_DEPLOY_CONFIG if args.svs else SVC_DEPLOY_CONFIG)
|
||||
mode = "SVS" if args.svs else "SVC"
|
||||
|
||||
omni_kwargs: dict = {
|
||||
"model": args.model,
|
||||
"deploy_config": deploy_config,
|
||||
"async_chunk": False, # SoulX-Singer currently supports only batch mode (pseudo-streaming was stashed)
|
||||
}
|
||||
if args.enable_diffusion_pipeline_profiler:
|
||||
omni_kwargs["enable_diffusion_pipeline_profiler"] = True
|
||||
if args.enforce_eager:
|
||||
omni_kwargs["enforce_eager"] = True
|
||||
if args.dtype is not None:
|
||||
omni_kwargs["dtype"] = args.dtype
|
||||
|
||||
compile_mode = "eager" if args.enforce_eager else "torch.compile"
|
||||
dtype_label = args.dtype or "from deploy YAML"
|
||||
print(f"Loading SoulX-Singer {mode} from {args.model} [{compile_mode}, dtype={dtype_label}]")
|
||||
|
||||
omni = Omni(**omni_kwargs)
|
||||
kind = "svs" if args.svs else "svc"
|
||||
sampling = build_sampling(args, preprocess_weights_dir=preprocess_dir, kind=kind)
|
||||
prompt = {"prompt_token_ids": [0]}
|
||||
|
||||
latencies_ms: list[float] = []
|
||||
rtfs: list[float] = []
|
||||
last_stages: dict[str, float] = {}
|
||||
last_measured_outputs = None
|
||||
|
||||
total_runs = args.warmup + args.runs
|
||||
for run_idx in range(total_runs):
|
||||
is_warmup = run_idx < args.warmup
|
||||
label = "warmup" if is_warmup else f"run {run_idx - args.warmup + 1}/{args.runs}"
|
||||
t0 = time.perf_counter()
|
||||
outputs = list(omni.generate([prompt], sampling))
|
||||
elapsed_ms = (time.perf_counter() - t0) * 1000.0
|
||||
audio_sec = _audio_duration_sec(outputs)
|
||||
rtf = (elapsed_ms / 1000.0) / audio_sec if audio_sec > 0 else float("inf")
|
||||
print(f"[{label}] client={elapsed_ms:.1f} ms, audio={audio_sec:.2f}s, RTF={rtf:.3f}")
|
||||
if not is_warmup:
|
||||
latencies_ms.append(elapsed_ms)
|
||||
rtfs.append(rtf)
|
||||
last_stages = _stage_durations_from_output(outputs)
|
||||
last_measured_outputs = outputs
|
||||
if args.output and args.save_each_run:
|
||||
measured_idx = run_idx - args.warmup + 1
|
||||
out_path = Path(args.output)
|
||||
run_path = out_path.with_name(f"{out_path.stem}_run{measured_idx}{out_path.suffix}")
|
||||
audio_np, sr = extract_audio(outputs)
|
||||
sf.write(str(run_path), audio_np, sr)
|
||||
print(f" saved {run_path} ({sr} Hz, {audio_np.size / sr:.2f}s)")
|
||||
|
||||
omni.close()
|
||||
|
||||
if args.output and last_measured_outputs is not None:
|
||||
audio_np, sr = extract_audio(last_measured_outputs)
|
||||
out_path = Path(args.output)
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
sf.write(str(out_path), audio_np, sr)
|
||||
print(f"\nSaved last measured run → {out_path} ({sr} Hz, {audio_np.size / sr:.2f}s)")
|
||||
|
||||
if latencies_ms:
|
||||
print("\n=== Summary (measured runs) ===")
|
||||
print(f"client_ms: mean={statistics.mean(latencies_ms):.1f}, stdev={statistics.pstdev(latencies_ms):.1f}")
|
||||
print(f"RTF: mean={statistics.mean(rtfs):.3f}, stdev={statistics.pstdev(rtfs):.3f}")
|
||||
if last_stages:
|
||||
print("\nStage durations (last measured run):")
|
||||
for name, value in sorted(last_stages.items()):
|
||||
if name.endswith("_ms"):
|
||||
print(f" {name}: {value:.1f} ms")
|
||||
else:
|
||||
print(f" {name}: {value * 1000.0:.1f} ms")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,285 @@
|
||||
"""Offline SoulX-Singer SVS / SVC: single-stage DiT (preprocess inline).
|
||||
|
||||
Pass raw prompt/target audio; vLLM-Omni runs integrated preprocess then inference.
|
||||
|
||||
Usage:
|
||||
# SVS (default)
|
||||
python end2end.py --model /path/to/SoulX-Singer \\
|
||||
--prompt-audio path/to/prompt/audio.mp3 \\
|
||||
--target-audio path/to/target/audio.mp3 \\
|
||||
--preprocess-weights-dir /path/to/SoulX-Singer-Preprocess \\
|
||||
-o output.wav
|
||||
|
||||
# SVC (config.json must declare SoulXSingerSVCPipeline; --model points at SVC view)
|
||||
python end2end.py --model /path/to/SoulX-Singer-svc \\
|
||||
--prompt-audio path/to/prompt/audio.mp3 \\
|
||||
--target-audio path/to/target/audio.mp3 \\
|
||||
--preprocess-weights-dir /path/to/SoulX-Singer-Preprocess \\
|
||||
-o output_svc.wav
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
|
||||
from vllm_omni.diffusion.models.soulx_singer.utils import resolve_soulx_kind, validate_soulx_extra_args
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
SVS_DEPLOY_CONFIG = REPO_ROOT / "vllm_omni" / "deploy" / "soulxsinger_svs.yaml"
|
||||
SVC_DEPLOY_CONFIG = REPO_ROOT / "vllm_omni" / "deploy" / "soulxsinger_svc.yaml"
|
||||
DEFAULT_PROMPT_AUDIO = REPO_ROOT / "tests" / "assets" / "soulxsinger" / "zh_prompt.mp3"
|
||||
DEFAULT_TARGET_AUDIO = REPO_ROOT / "tests" / "assets" / "soulxsinger" / "music.mp3"
|
||||
_SAMPLE_RATE = 24000
|
||||
|
||||
|
||||
def _require_paths(paths: dict[str, str | None]) -> None:
|
||||
missing = [name for name, p in paths.items() if not p or not os.path.isfile(p)]
|
||||
if missing:
|
||||
raise FileNotFoundError("Missing file(s): " + ", ".join(f"{k}={paths[k]!r}" for k in missing))
|
||||
|
||||
|
||||
def resolve_preprocess_weights_dir(explicit: str | None) -> str:
|
||||
if explicit:
|
||||
path = Path(explicit).expanduser().resolve()
|
||||
if not (path / "rmvpe" / "rmvpe.pt").is_file():
|
||||
raise FileNotFoundError(f"--preprocess-weights-dir must contain rmvpe/rmvpe.pt: {path}")
|
||||
return str(path)
|
||||
env = os.environ.get("SOULX_PREPROCESS_WEIGHTS_DIR")
|
||||
if env:
|
||||
path = Path(env).expanduser().resolve()
|
||||
if (path / "rmvpe" / "rmvpe.pt").is_file():
|
||||
return str(path)
|
||||
raise FileNotFoundError(
|
||||
"SoulX preprocess weights not found. Pass --preprocess-weights-dir or set "
|
||||
"SOULX_PREPROCESS_WEIGHTS_DIR to a directory containing rmvpe/rmvpe.pt "
|
||||
"(huggingface-cli download Soul-AILab/SoulX-Singer-Preprocess)."
|
||||
)
|
||||
|
||||
|
||||
def build_sampling(
|
||||
args: argparse.Namespace,
|
||||
*,
|
||||
preprocess_weights_dir: str,
|
||||
kind: str,
|
||||
) -> OmniDiffusionSamplingParams:
|
||||
extra: dict = {
|
||||
"prompt_audio": os.path.abspath(args.prompt_audio),
|
||||
"target_audio": os.path.abspath(args.target_audio),
|
||||
"preprocess_weights_dir": preprocess_weights_dir,
|
||||
"vocal_sep": args.vocal_sep,
|
||||
"auto_shift": args.auto_shift,
|
||||
"pitch_shift": args.pitch_shift,
|
||||
}
|
||||
if kind == "svs":
|
||||
extra["language"] = args.language
|
||||
extra["control"] = args.control
|
||||
if args.prompt_metadata_path:
|
||||
extra["prompt_metadata_path"] = os.path.abspath(args.prompt_metadata_path)
|
||||
if args.target_metadata_path:
|
||||
extra["target_metadata_path"] = os.path.abspath(args.target_metadata_path)
|
||||
if args.prompt_metadata_path or args.target_metadata_path:
|
||||
extra["audio_path"] = os.path.abspath(args.prompt_wav_path or args.prompt_audio)
|
||||
|
||||
extra = validate_soulx_extra_args(kind, extra)
|
||||
|
||||
return OmniDiffusionSamplingParams(
|
||||
num_inference_steps=args.num_inference_steps,
|
||||
guidance_scale=args.guidance_scale,
|
||||
seed=args.seed,
|
||||
extra_args=extra,
|
||||
)
|
||||
|
||||
|
||||
def extract_audio(outputs) -> tuple[np.ndarray, int]:
|
||||
out = outputs[0]
|
||||
mm = getattr(out, "multimodal_output", None)
|
||||
if mm is None:
|
||||
ro = getattr(out, "request_output", None)
|
||||
if ro is not None:
|
||||
mm = getattr(ro, "multimodal_output", None)
|
||||
if mm is None and ro.outputs:
|
||||
mm = getattr(ro.outputs[0], "multimodal_output", None)
|
||||
if not mm or "audio" not in mm:
|
||||
raise RuntimeError("No audio in multimodal_output")
|
||||
|
||||
audio_val = mm["audio"]
|
||||
if isinstance(audio_val, list):
|
||||
chunks = [
|
||||
np.asarray(chunk.detach().cpu().float().numpy() if hasattr(chunk, "detach") else chunk).reshape(-1)
|
||||
for chunk in audio_val
|
||||
if chunk is not None
|
||||
]
|
||||
audio_np = np.concatenate(chunks, axis=0) if chunks else np.array([], dtype=np.float32)
|
||||
elif hasattr(audio_val, "cpu"):
|
||||
audio_np = audio_val.detach().cpu().numpy().squeeze()
|
||||
else:
|
||||
audio_np = np.asarray(audio_val, dtype=np.float32).squeeze()
|
||||
|
||||
sr_val = mm.get("sr", mm.get("audio_sample_rate", _SAMPLE_RATE))
|
||||
if isinstance(sr_val, list) and sr_val:
|
||||
sr_val = sr_val[-1]
|
||||
if hasattr(sr_val, "item"):
|
||||
sr_val = sr_val.item()
|
||||
return audio_np.astype(np.float32, copy=False), int(sr_val)
|
||||
|
||||
|
||||
def add_inference_args(parser: argparse.ArgumentParser) -> None:
|
||||
mode = parser.add_mutually_exclusive_group()
|
||||
mode.add_argument(
|
||||
"--svc",
|
||||
action="store_true",
|
||||
help="Assert SVC mode (must match config.json architectures)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
type=str,
|
||||
required=True,
|
||||
help="SoulX-Singer weight directory ($BASE for SVS, $SVC_DIR for SVC)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--deploy-config",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Override deploy YAML (default: soulxsinger_svs.yaml or soulxsinger_svc.yaml)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prompt-audio",
|
||||
type=str,
|
||||
default=str(DEFAULT_PROMPT_AUDIO),
|
||||
help="Prompt / reference audio (mp3 or wav)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--target-audio",
|
||||
type=str,
|
||||
default=str(DEFAULT_TARGET_AUDIO),
|
||||
help="Target content audio (mp3 or wav)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prompt-metadata-path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="SVS precomputed prompt metadata.json (skips online ASR/ROSVOT for prompt)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--target-metadata-path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="SVS precomputed target metadata.json (skips online ASR/ROSVOT for target)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prompt-wav-path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="SVS prompt vocal wav for metadata processor when using --prompt-metadata-path (default: --prompt-audio)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--preprocess-weights-dir",
|
||||
type=str,
|
||||
default=None,
|
||||
help="SoulX-Singer-Preprocess weights (or set SOULX_PREPROCESS_WEIGHTS_DIR)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--language",
|
||||
type=str,
|
||||
default="Mandarin",
|
||||
help="SVS lyrics language (Mandarin, Cantonese, English)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--control",
|
||||
type=str,
|
||||
default="melody",
|
||||
choices=["score", "melody"],
|
||||
help="SVS control mode",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--vocal-sep",
|
||||
action=argparse.BooleanOptionalAction,
|
||||
default=False,
|
||||
help="Run vocal separation during preprocess",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--auto-shift",
|
||||
action=argparse.BooleanOptionalAction,
|
||||
default=True,
|
||||
help="Auto pitch shift to align target pitch with prompt",
|
||||
)
|
||||
parser.add_argument("--pitch-shift", type=int, default=0)
|
||||
parser.add_argument("--num-inference-steps", type=int, default=32)
|
||||
parser.add_argument("--guidance-scale", type=float, default=3.0)
|
||||
parser.add_argument(
|
||||
"--seed",
|
||||
type=int,
|
||||
default=42,
|
||||
help="Optional CFM RNG seed. Omit for non-deterministic sampling.",
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="SoulX-Singer offline SVS / SVC (preprocess + DiT)")
|
||||
add_inference_args(parser)
|
||||
parser.add_argument("--output", "-o", type=str, default="output.wav")
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
kind = resolve_soulx_kind(args.model)
|
||||
|
||||
if not (args.svc ^ (kind == "svs")):
|
||||
raise ValueError(f"mode mismatch: {args.model}/config.json (declares {kind!r}), --svc={args.svc}")
|
||||
|
||||
required = {
|
||||
"prompt_audio": args.prompt_audio,
|
||||
"target_audio": args.target_audio,
|
||||
}
|
||||
if kind == "svs" and bool(args.prompt_metadata_path) ^ bool(args.target_metadata_path):
|
||||
raise ValueError(
|
||||
"SVS precomputed metadata requires both --prompt-metadata-path and "
|
||||
"--target-metadata-path (or neither, for online preprocess)."
|
||||
)
|
||||
if args.prompt_metadata_path:
|
||||
required["prompt_metadata_path"] = args.prompt_metadata_path
|
||||
required["target_metadata_path"] = args.target_metadata_path
|
||||
if args.prompt_wav_path:
|
||||
required["prompt_wav_path"] = args.prompt_wav_path
|
||||
_require_paths(required)
|
||||
preprocess_dir = resolve_preprocess_weights_dir(args.preprocess_weights_dir)
|
||||
deploy_config = args.deploy_config or str(SVC_DEPLOY_CONFIG if kind == "svc" else SVS_DEPLOY_CONFIG)
|
||||
mode = kind.upper()
|
||||
|
||||
print(f"Loading SoulX-Singer {mode} from {args.model}")
|
||||
print(f" deploy: {deploy_config}")
|
||||
print(f" prompt: {args.prompt_audio}")
|
||||
print(f" target: {args.target_audio}")
|
||||
|
||||
omni = Omni(model=args.model, deploy_config=deploy_config, async_chunk=False) # SoulX-Singer is batch-only
|
||||
sampling = build_sampling(
|
||||
args,
|
||||
preprocess_weights_dir=preprocess_dir,
|
||||
kind=kind,
|
||||
)
|
||||
prompt = {"prompt_token_ids": [0], "multi_modal_data": {}}
|
||||
|
||||
print("Running generate (inline preprocess + DiT)...")
|
||||
outputs = list(omni.generate([prompt], sampling))
|
||||
if not outputs:
|
||||
raise RuntimeError("No output from omni.generate")
|
||||
|
||||
err = getattr(outputs[0], "error", None)
|
||||
if err is not None:
|
||||
raise RuntimeError(f"Pipeline error: {err}")
|
||||
|
||||
audio_np, sr = extract_audio(outputs)
|
||||
sf.write(args.output, audio_np, sr)
|
||||
print(f"Saved {args.output} ({sr} Hz, {len(audio_np) / sr:.2f}s)")
|
||||
omni.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -6,9 +6,11 @@ A unified script for text-to-video generation. Supports multiple models with mod
|
||||
|
||||
| Model | Default Resolution | Default Frames | Default Steps | Guidance | VRAM (BF16) |
|
||||
|---|---|---|---|---|---|
|
||||
| `Wan-AI/Wan2.1-VACE-1.3B-diffusers` | 480x832 | 81 | 30 | 5.0 | ~20 GiB (RTX 5090, VAE tiling) |
|
||||
| `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | 720x1280 | 81 | 40 | 4.0 | ~60 GiB |
|
||||
| `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v` | 480x832 | 121 | 50 | 6.0 | 1×A100 80GB |
|
||||
| `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | 720x1280 | 121 | 50 | 6.0 | FP8 + VAE tiling required |
|
||||
| `nvidia/Cosmos3-Nano` | 720x1280 | 189 | 35 | 6.0 | ~46 GiB (peak, 720p) |
|
||||
| `BestWishYsh/Helios-Base` / `Helios-Mid` / `Helios-Distilled` | 384x640 | 99 | 50 | 5.0 / 5.0 / 1.0 | — |
|
||||
|
||||
## Local CLI Usage
|
||||
@@ -30,6 +32,28 @@ python text_to_video.py \
|
||||
--output t2v_out.mp4
|
||||
```
|
||||
|
||||
### Wan2.1 VACE (T2V)
|
||||
|
||||
VACE text-to-video uses this shared entrypoint. Conditional VACE tasks use
|
||||
the shared [`image_to_video.py`](../image_to_video/README.md#wan21-vace-conditional-tasks)
|
||||
entrypoint, which constructs the pipeline-native conditioning data from the
|
||||
provided media inputs. No explicit mode parameter is required.
|
||||
|
||||
```bash
|
||||
python text_to_video.py \
|
||||
--model Wan-AI/Wan2.1-VACE-1.3B-diffusers \
|
||||
--prompt "A sleek, humanoid robot stands in a vast warehouse filled with neatly stacked cardboard boxes on industrial shelves." \
|
||||
--seed 0 \
|
||||
--height 480 \
|
||||
--width 832 \
|
||||
--num-frames 81 \
|
||||
--num-inference-steps 30 \
|
||||
--guidance-scale 5.0 \
|
||||
--flow-shift 5.0 \
|
||||
--vae-use-tiling \
|
||||
--output vace_t2v_output.mp4
|
||||
```
|
||||
|
||||
LTX2 example:
|
||||
|
||||
```bash
|
||||
@@ -101,6 +125,20 @@ python text_to_video.py \
|
||||
--output quick_test.mp4
|
||||
```
|
||||
|
||||
### Cosmos3
|
||||
|
||||
```bash
|
||||
python text_to_video.py \
|
||||
--model nvidia/Cosmos3-Nano \
|
||||
--prompt "A robot arm is cleaning a plate in the kitchen." \
|
||||
--negative-prompt "blurry, distorted, low quality, jittery, deformed" \
|
||||
--height 720 --width 1280 --num-frames 189 --fps 24 \
|
||||
--num-inference-steps 35 --guidance-scale 6.0 \
|
||||
--extra-body '{"flow_shift": 10.0, "max_sequence_length": 4096, "guardrails": false,
|
||||
"use_resolution_template": false, "use_duration_template": false}' \
|
||||
--output cosmos3_t2v.mp4
|
||||
```
|
||||
|
||||
### Helios (T2V)
|
||||
|
||||
Helios ships three variants. Model-specific knobs (declared in
|
||||
@@ -167,6 +205,7 @@ python text_to_video.py \
|
||||
- `--audio-sample-rate`: audio sample rate for embedded audio (when the pipeline returns audio).
|
||||
- `--quantization`: quantization method (`fp8` for FP8, `gguf` for GGUF).
|
||||
- `--flow-shift`: scheduler flow_shift parameter.
|
||||
- `--extra-body`: JSON object of model-specific generation params, filtered against the model's declared `extra_body_params` (see [`vllm_omni/model_extras`](../../../vllm_omni/model_extras)). Used by Cosmos3 (see above).
|
||||
|
||||
### Wan2.2-specific
|
||||
|
||||
|
||||
@@ -11,12 +11,23 @@ import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.model_extras import get_extra_body_params, get_model_class_name
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
_MODEL_PRESETS = {
|
||||
"vace": {
|
||||
"height": 480,
|
||||
"width": 832,
|
||||
"num_frames": 81,
|
||||
"num_inference_steps": 30,
|
||||
"guidance_scale": 5.0,
|
||||
"fps": 16,
|
||||
"output": "vace_t2v_output.mp4",
|
||||
},
|
||||
"wan": {
|
||||
"height": 720,
|
||||
"width": 1280,
|
||||
@@ -35,6 +46,16 @@ _MODEL_PRESETS = {
|
||||
"fps": 24,
|
||||
"output": "hunyuan_video_15_output.mp4",
|
||||
},
|
||||
"cosmos": {
|
||||
"height": 720,
|
||||
"width": 1280,
|
||||
"num_frames": 189,
|
||||
"num_inference_steps": 35,
|
||||
"guidance_scale": 6.0,
|
||||
"fps": 24,
|
||||
"flow_shift": 10.0,
|
||||
"output": "cosmos3_t2v_output.mp4",
|
||||
},
|
||||
"helios": {
|
||||
"height": 384,
|
||||
"width": 640,
|
||||
@@ -49,6 +70,10 @@ _MODEL_PRESETS = {
|
||||
|
||||
def _detect_preset(model: str) -> dict:
|
||||
model_lower = model.lower()
|
||||
if "vace" in model_lower:
|
||||
return _MODEL_PRESETS["vace"]
|
||||
if "cosmos" in model_lower:
|
||||
return _MODEL_PRESETS["cosmos"]
|
||||
if "hunyuan" in model_lower:
|
||||
return _MODEL_PRESETS["hunyuan"]
|
||||
if "helios" in model_lower:
|
||||
@@ -106,8 +131,12 @@ def parse_args() -> argparse.Namespace:
|
||||
type=parse_extra_body,
|
||||
default=None,
|
||||
help="JSON dict of model-specific extra_body params (declared in vllm_omni/model_extras/), "
|
||||
"merged into sampling extra_args. Example (Helios-Distilled): "
|
||||
'\'{"is_enable_stage2": true, "pyramid_num_inference_steps_list": [2, 2, 2], "is_amplify_first_chunk": true}\'.',
|
||||
"merged into sampling extra_args. Unknown keys for the chosen model are dropped. "
|
||||
'Cosmos3 example: \'{"flow_shift": 10.0, "max_sequence_length": 4096, "guardrails": false, '
|
||||
'"use_resolution_template": false, "use_duration_template": false}\' '
|
||||
'(add "generate_sound": true and optional "sound_duration" for synchronized audio). '
|
||||
'Helios-Distilled example: \'{"is_enable_stage2": true, '
|
||||
'"pyramid_num_inference_steps_list": [2, 2, 2], "is_amplify_first_chunk": true}\'.',
|
||||
)
|
||||
parser.add_argument("--seed", type=int, default=42, help="Random seed.")
|
||||
parser.add_argument("--guidance-scale", type=float, default=None, help="CFG scale. Default: model-specific.")
|
||||
@@ -357,7 +386,14 @@ def main():
|
||||
omni_kwargs["cache_config"] = cache_config
|
||||
omni_kwargs["enable_cache_dit_summary"] = args.enable_cache_dit_summary
|
||||
|
||||
# Cosmos3 loads its (gated) guardrail models at build time, so the guardrails
|
||||
# gate is an engine-level config (offline analog of the server's --no-guardrails).
|
||||
if args.extra_body and "guardrails" in args.extra_body:
|
||||
omni_kwargs["model_config"] = {"guardrails": bool(args.extra_body["guardrails"])}
|
||||
|
||||
omni = Omni(**omni_kwargs)
|
||||
model_class_name = get_model_class_name(omni) or model_class_name
|
||||
declared_extra_body_params = get_extra_body_params(model_class_name)
|
||||
|
||||
if profiler_enabled:
|
||||
print("[Profiler] Starting profiling...")
|
||||
@@ -392,14 +428,21 @@ def main():
|
||||
)
|
||||
if args.guidance_scale_high is not None:
|
||||
sampling_kwargs["guidance_scale_2"] = args.guidance_scale_high
|
||||
if args.extra_body:
|
||||
# Model-specific knobs (declared in vllm_omni/model_extras/) routed via extra_args.
|
||||
sampling_kwargs["extra_args"] = dict(args.extra_body)
|
||||
|
||||
sampling_params = OmniDiffusionSamplingParams(**sampling_kwargs)
|
||||
# Route model-specific knobs through extra_body, filtered against the model's
|
||||
# declared extra_body_params. Models without a declaration only forward explicit
|
||||
# --extra-body JSON (preserving the generic flags' legacy behavior).
|
||||
extra_body = dict(args.extra_body or {})
|
||||
if declared_extra_body_params:
|
||||
apply_declared_extra_args(sampling_params, declared_extra_body_params, extra_body)
|
||||
elif extra_body:
|
||||
sampling_params.extra_args.update({k: v for k, v in extra_body.items() if v is not None})
|
||||
|
||||
generation_start = time.perf_counter()
|
||||
frames = omni.generate(
|
||||
prompt_dict,
|
||||
OmniDiffusionSamplingParams(**sampling_kwargs),
|
||||
sampling_params,
|
||||
)
|
||||
generation_end = time.perf_counter()
|
||||
generation_time = generation_end - generation_start
|
||||
|
||||
@@ -1,148 +0,0 @@
|
||||
# VACE Video Generation
|
||||
|
||||
Generate videos from text prompts, images, or video conditions using vLLM-Omni's VACE diffusion pipeline.
|
||||
|
||||
- `vace_video_generation.py`: command-line script for multi-mode video generation with advanced options.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [Overview](#overview)
|
||||
- [Quick Start](#quick-start)
|
||||
- [Key Arguments](#key-arguments)
|
||||
- [More CLI Examples](#more-cli-examples)
|
||||
|
||||
## Overview
|
||||
|
||||
[VACE](https://github.com/ali-vilab/VACE) (Video All-in-one Creation Engine) supports multiple video tasks through a single unified model, including text-to-video, image-to-video, first-last-frame interpolation, inpainting, and reference image-guided generation.
|
||||
|
||||
### Supported Models
|
||||
|
||||
| Model | Architecture | Peak VRAM (GiB) * | Model Weights (GiB) | HuggingFace |
|
||||
|-------|-------------|-------------------|---------------------|-------------|
|
||||
| Wan2.1-VACE (1.3B) | Wan2.1 | TBD | ~10 | [Wan-AI/Wan2.1-VACE-1.3B-diffusers](https://huggingface.co/Wan-AI/Wan2.1-VACE-1.3B-diffusers) |
|
||||
| Wan2.1-VACE (14B) | Wan2.1 | TBD | ~38 | [Wan-AI/Wan2.1-VACE-14B-diffusers](https://huggingface.co/Wan-AI/Wan2.1-VACE-14B-diffusers) |
|
||||
|
||||
!!! info
|
||||
*Peak VRAM: based on basic single-card usage, 480x832 resolution, 81 frames, 30 inference steps, without any acceleration/optimization features.
|
||||
|
||||
Default model: `Wan-AI/Wan2.1-VACE-14B-diffusers`
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Python API
|
||||
|
||||
Text-to-video generation:
|
||||
|
||||
```python
|
||||
import torch
|
||||
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
if __name__ == "__main__":
|
||||
omni = Omni(model="Wan-AI/Wan2.1-VACE-1.3B-diffusers")
|
||||
prompt = "A sleek robot stands in a vast warehouse filled with boxes"
|
||||
outputs = omni.generate(
|
||||
prompt,
|
||||
OmniDiffusionSamplingParams(
|
||||
height=480,
|
||||
width=832,
|
||||
num_frames=81,
|
||||
num_inference_steps=30,
|
||||
guidance_scale=5.0,
|
||||
),
|
||||
)
|
||||
video = outputs[0].images
|
||||
|
||||
from diffusers.utils import export_to_video
|
||||
export_to_video(list(video[0]), "t2v_output.mp4", fps=16)
|
||||
omni.close()
|
||||
```
|
||||
|
||||
### Local CLI Usage
|
||||
|
||||
```bash
|
||||
python vace_video_generation.py \
|
||||
--mode t2v \
|
||||
--prompt "A sleek robot stands in a vast warehouse filled with boxes" \
|
||||
--height 480 --width 832 --num-frames 81 \
|
||||
--output t2v_output.mp4
|
||||
```
|
||||
|
||||
## Key Arguments
|
||||
|
||||
| Argument | Type | Default | Description |
|
||||
| -------- | ---- | ------- | ----------- |
|
||||
| `--mode` | str | `"t2v"` | VACE task mode: `t2v`, `i2v`, `v2lf`, `flf2v`, `inpaint`, `r2v` |
|
||||
| `--model` | str | `"Wan-AI/Wan2.1-VACE-14B-diffusers"` | Model ID or local path |
|
||||
| `--prompt` | str | `"A cat walking in a garden"` | Text description of desired video |
|
||||
| `--negative-prompt` | str | `""` | Negative prompt for classifier-free guidance |
|
||||
| `--image` | str | `None` | Input image path (for I2V, R2V, FLF2V, inpaint modes) |
|
||||
| `--last-image` | str | `None` | Last frame image path (for FLF2V mode) |
|
||||
| `--height` | int | `480` | Output video height in pixels (should be a multiple of 16) |
|
||||
| `--width` | int | `832` | Output video width in pixels (should be a multiple of 16) |
|
||||
| `--num-frames` | int | `81` | Number of video frames to generate |
|
||||
| `--num-inference-steps` | int | `30` | Number of denoising steps (more steps = higher quality, slower) |
|
||||
| `--guidance-scale` | float | `5.0` | Classifier-free guidance scale |
|
||||
| `--flow-shift` | float | `5.0` | Scheduler flow shift parameter |
|
||||
| `--seed` | int | `42` | Random seed for deterministic sampling |
|
||||
| `--fps` | int | `16` | Frames per second for the saved MP4 |
|
||||
| `--output` | str | `"vace_output.mp4"` | Path to save the generated video |
|
||||
| `--vae-use-tiling` | flag | on | Enable VAE tiling for memory optimization |
|
||||
| `--cfg-parallel-size` | int | `1` | Set to `2` to enable CFG Parallel |
|
||||
| `--ulysses-degree` | int | `1` | Ulysses sequence parallel degree for multi-GPU inference |
|
||||
| `--ring-degree` | int | `1` | Ring sequence parallel degree for hybrid Ulysses + Ring inference |
|
||||
| `--tensor-parallel-size` | int | — | Tensor parallel size |
|
||||
| `--enforce-eager` | flag | off | Disable torch.compile |
|
||||
|
||||
> If you encounter OOM errors, try `--vae-use-tiling` or multi-GPU parallelism options (`--ulysses-degree`, `--cfg-parallel-size`).
|
||||
|
||||
## More CLI Examples
|
||||
|
||||
### Image-to-Video (I2V)
|
||||
|
||||
First frame is kept, remaining frames are generated:
|
||||
|
||||
```bash
|
||||
python vace_video_generation.py \
|
||||
--mode i2v \
|
||||
--image astronaut.jpg \
|
||||
--prompt "An astronaut emerging from a cracked egg on the moon" \
|
||||
--height 480 --width 832 --num-frames 81 \
|
||||
--output i2v_output.mp4
|
||||
```
|
||||
|
||||
### First-Last-Frame Interpolation (FLF2V)
|
||||
|
||||
```bash
|
||||
python vace_video_generation.py \
|
||||
--mode flf2v \
|
||||
--image first_frame.jpg --last-image last_frame.jpg \
|
||||
--prompt "A bird takes off from a branch and lands on another" \
|
||||
--height 512 --width 512 --num-frames 81 \
|
||||
--output flf2v_output.mp4
|
||||
```
|
||||
|
||||
### Inpainting
|
||||
|
||||
Center vertical stripe is masked and regenerated:
|
||||
|
||||
```bash
|
||||
python vace_video_generation.py \
|
||||
--mode inpaint \
|
||||
--image scene.jpg \
|
||||
--prompt "Shrek walks out of a building" \
|
||||
--height 480 --width 832 --num-frames 81 \
|
||||
--output inpaint_output.mp4
|
||||
```
|
||||
|
||||
### Reference Image-guided (R2V)
|
||||
|
||||
```bash
|
||||
python vace_video_generation.py \
|
||||
--mode r2v \
|
||||
--image reference.jpg \
|
||||
--prompt "Camera slowly zooms out from the character" \
|
||||
--height 480 --width 832 --num-frames 81 \
|
||||
--output r2v_output.mp4
|
||||
```
|
||||
@@ -1,209 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""VACE video generation example.
|
||||
|
||||
VACE (Video All-in-one Creation Engine) supports multiple video tasks:
|
||||
- T2V: Text-to-Video
|
||||
- I2V: Image-to-Video (first frame conditioning)
|
||||
- V2LF: Video-to-Last-Frame
|
||||
- FLF2V: First-Last-Frame interpolation
|
||||
- Inpainting: Masked region generation
|
||||
- R2V: Reference image-guided generation
|
||||
|
||||
Usage examples:
|
||||
# T2V (text-to-video)
|
||||
python vace_video_generation.py --mode t2v --prompt "A robot in a warehouse"
|
||||
|
||||
# I2V (image-to-video, first frame kept)
|
||||
python vace_video_generation.py --mode i2v --image input.jpg --prompt "..."
|
||||
|
||||
# FLF2V (first-last frame interpolation)
|
||||
python vace_video_generation.py --mode flf2v --image first.jpg --last-image last.jpg
|
||||
|
||||
# R2V (reference image guided)
|
||||
python vace_video_generation.py --mode r2v --image ref.jpg --prompt "..."
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import PIL.Image
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="VACE video generation.")
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
default="Wan-AI/Wan2.1-VACE-14B-diffusers",
|
||||
help="VACE model ID or local path.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--mode",
|
||||
default="t2v",
|
||||
choices=["t2v", "i2v", "v2lf", "flf2v", "inpaint", "r2v"],
|
||||
help="Generation mode.",
|
||||
)
|
||||
parser.add_argument("--prompt", default="A cat walking in a garden", help="Text prompt.")
|
||||
parser.add_argument("--negative-prompt", default="", help="Negative prompt.")
|
||||
parser.add_argument("--image", type=str, default=None, help="Input image path (for I2V, R2V, FLF2V, inpaint).")
|
||||
parser.add_argument("--last-image", type=str, default=None, help="Last frame image path (for FLF2V).")
|
||||
parser.add_argument("--video-dir", type=str, default=None, help="Directory of video frames (for inpaint).")
|
||||
parser.add_argument("--seed", type=int, default=42, help="Random seed.")
|
||||
parser.add_argument("--guidance-scale", type=float, default=5.0, help="CFG guidance scale.")
|
||||
parser.add_argument("--height", type=int, default=480, help="Video height.")
|
||||
parser.add_argument("--width", type=int, default=832, help="Video width.")
|
||||
parser.add_argument("--num-frames", type=int, default=81, help="Number of frames.")
|
||||
parser.add_argument("--num-inference-steps", type=int, default=30, help="Sampling steps.")
|
||||
parser.add_argument("--flow-shift", type=float, default=5.0, help="Scheduler flow_shift.")
|
||||
parser.add_argument("--output", type=str, default="vace_output.mp4", help="Output video path.")
|
||||
parser.add_argument("--fps", type=int, default=16, help="Output video FPS.")
|
||||
parser.add_argument("--vae-use-tiling", action="store_true", default=True, help="Enable VAE tiling.")
|
||||
parser.add_argument("--enforce-eager", action="store_true", help="Disable torch.compile.")
|
||||
parser.add_argument("--ulysses-degree", type=int, default=1, help="Ulysses SP degree.")
|
||||
parser.add_argument("--ring-degree", type=int, default=1, help="Ring attention degree.")
|
||||
parser.add_argument("--cfg-parallel-size", type=int, default=1, choices=[1, 2], help="CFG parallel size.")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def build_prompts(args):
|
||||
"""Build prompt dict with multi_modal_data based on mode."""
|
||||
h, w, nf = args.height, args.width, args.num_frames
|
||||
|
||||
gray = PIL.Image.new("RGB", (w, h), (128, 128, 128))
|
||||
mask_black = PIL.Image.new("L", (w, h), 0)
|
||||
mask_white = PIL.Image.new("L", (w, h), 255)
|
||||
|
||||
prompt_data = {
|
||||
"prompt": args.prompt,
|
||||
"negative_prompt": args.negative_prompt,
|
||||
}
|
||||
|
||||
if args.mode == "t2v":
|
||||
return prompt_data
|
||||
|
||||
if args.mode == "r2v":
|
||||
assert args.image, "--image required for R2V mode"
|
||||
ref_img = PIL.Image.open(args.image).convert("RGB").resize((w, h))
|
||||
prompt_data["multi_modal_data"] = {"reference_images": [ref_img]}
|
||||
return prompt_data
|
||||
|
||||
if args.mode == "i2v":
|
||||
assert args.image, "--image required for I2V mode"
|
||||
img = PIL.Image.open(args.image).convert("RGB").resize((w, h))
|
||||
prompt_data["multi_modal_data"] = {
|
||||
"video": [img] + [gray] * (nf - 1),
|
||||
"mask": [mask_black] + [mask_white] * (nf - 1),
|
||||
}
|
||||
return prompt_data
|
||||
|
||||
if args.mode == "v2lf":
|
||||
assert args.image, "--image required for V2LF mode"
|
||||
img = PIL.Image.open(args.image).convert("RGB").resize((w, h))
|
||||
prompt_data["multi_modal_data"] = {
|
||||
"video": [gray] * (nf - 1) + [img],
|
||||
"mask": [mask_white] * (nf - 1) + [mask_black],
|
||||
}
|
||||
return prompt_data
|
||||
|
||||
if args.mode == "flf2v":
|
||||
assert args.image and args.last_image, "--image and --last-image required for FLF2V"
|
||||
first = PIL.Image.open(args.image).convert("RGB").resize((w, h))
|
||||
last = PIL.Image.open(args.last_image).convert("RGB").resize((w, h))
|
||||
prompt_data["multi_modal_data"] = {
|
||||
"video": [first] + [gray] * (nf - 2) + [last],
|
||||
"mask": [mask_black] + [mask_white] * (nf - 2) + [mask_black],
|
||||
}
|
||||
return prompt_data
|
||||
|
||||
if args.mode == "inpaint":
|
||||
assert args.image, "--image required for inpaint mode"
|
||||
img = PIL.Image.open(args.image).convert("RGB").resize((w, h))
|
||||
d = 80
|
||||
frames, masks = [], []
|
||||
for _ in range(nf):
|
||||
base = np.array(img).copy()
|
||||
mask = PIL.Image.new("L", (w, h), 0)
|
||||
stripe = PIL.Image.new("L", (2 * d, h), 255)
|
||||
mask.paste(stripe, (w // 2 - d, 0))
|
||||
base[np.array(mask) > 128] = 128
|
||||
frames.append(PIL.Image.fromarray(base))
|
||||
masks.append(mask)
|
||||
prompt_data["multi_modal_data"] = {"video": frames, "mask": masks}
|
||||
return prompt_data
|
||||
|
||||
raise ValueError(f"Unknown mode: {args.mode}")
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
|
||||
|
||||
parallel_config = DiffusionParallelConfig(
|
||||
ulysses_degree=args.ulysses_degree,
|
||||
ring_degree=args.ring_degree,
|
||||
cfg_parallel_size=args.cfg_parallel_size,
|
||||
)
|
||||
|
||||
omni = Omni(
|
||||
model=args.model,
|
||||
vae_use_tiling=args.vae_use_tiling,
|
||||
flow_shift=args.flow_shift,
|
||||
enforce_eager=args.enforce_eager,
|
||||
parallel_config=parallel_config,
|
||||
)
|
||||
|
||||
prompt_data = build_prompts(args)
|
||||
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"VACE {args.mode.upper()} Generation")
|
||||
print(f" Model: {args.model}")
|
||||
print(f" Size: {args.width}x{args.height}, {args.num_frames} frames, {args.num_inference_steps} steps")
|
||||
print(f"{'=' * 60}\n")
|
||||
|
||||
start = time.perf_counter()
|
||||
outputs = omni.generate(
|
||||
prompt_data,
|
||||
OmniDiffusionSamplingParams(
|
||||
height=args.height,
|
||||
width=args.width,
|
||||
num_frames=args.num_frames,
|
||||
num_inference_steps=args.num_inference_steps,
|
||||
guidance_scale=args.guidance_scale,
|
||||
generator=generator,
|
||||
),
|
||||
)
|
||||
elapsed = time.perf_counter() - start
|
||||
|
||||
video = outputs[0].images
|
||||
if isinstance(video, list):
|
||||
video = video[0]
|
||||
if isinstance(video, torch.Tensor):
|
||||
video = video.cpu().numpy()
|
||||
if video.ndim == 5:
|
||||
video = video[0]
|
||||
print(f"Output shape: {video.shape}, Time: {elapsed:.1f}s")
|
||||
|
||||
output_path = Path(args.output)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
from diffusers.utils import export_to_video
|
||||
|
||||
if np.issubdtype(video.dtype, np.integer):
|
||||
video = video.astype(np.float32) / 255.0
|
||||
export_to_video(list(video), str(output_path), fps=args.fps)
|
||||
print(f"Saved to {output_path}")
|
||||
|
||||
omni.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -88,7 +88,7 @@ curl -s http://127.0.0.1:8091/v1/chat/completions \
|
||||
|
||||
### Optional knobs
|
||||
|
||||
Pass image-gen overrides via the diffusion-stage `sampling_params_list[1].extra_args.image_gen`:
|
||||
Pass image-gen overrides as flat keys on the diffusion-stage `sampling_params_list[1].extra_args`:
|
||||
|
||||
| Key | Default | Description |
|
||||
| --- | --- | --- |
|
||||
@@ -97,25 +97,48 @@ Pass image-gen overrides via the diffusion-stage `sampling_params_list[1].extra_
|
||||
| `cfg` | 2.0 | Classifier-free guidance scale. |
|
||||
| `seed` | 42 | Per-request RNG seed (deterministic when ByT5 is also seed-stable). |
|
||||
| `byte5_text` | (auto) | Override the glyph text for ByT5 enhancement; raw strings are auto-wrapped to Ming's `Text "...". ` format. |
|
||||
| `negative_prompt` | empty | Real CFG negative conditioning (must be set on **stage-0 thinker** `sampling_params` so `expand_cfg_prompts` spawns the companion). |
|
||||
| `negative_prompt` | empty | Real CFG negative conditioning (set on **stage-0 thinker** `extra_args` so `expand_cfg_prompts` spawns the companion). |
|
||||
|
||||
Example with all the knobs:
|
||||
|
||||
```bash
|
||||
curl http://127.0.0.1:8091/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "Jonathan1909/Ming-flash-omni-2.0",
|
||||
"modalities": ["image"],
|
||||
"sampling_params_list": [
|
||||
{"temperature":0.4,"top_p":0.9,"top_k":1,"max_tokens":1,"seed":42,
|
||||
"extra_args":{"image_gen":{"negative_prompt":"ugly, blurry, distorted"}}},
|
||||
{"seed":42,"extra_args":{"image_gen":{
|
||||
"steps":6,"cfg":1.5,"height":512,"width":512,"seed":123,
|
||||
"byte5_text":["理解与生成统一"]}}}
|
||||
],
|
||||
"messages": [{"role": "user", "content": "Draw a poster."}]
|
||||
}' | jq -r '.choices[0].message.content[0].image_url.url | split(",")[1]' | base64 -d > ming_imagegen_knobs.png
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "Jonathan1909/Ming-flash-omni-2.0",
|
||||
"modalities": ["image"],
|
||||
"sampling_params_list": [
|
||||
{
|
||||
"temperature": 0.4,
|
||||
"top_p": 0.9,
|
||||
"top_k": 1,
|
||||
"max_tokens": 1,
|
||||
"seed": 42,
|
||||
"extra_args": {
|
||||
"negative_prompt": "ugly, blurry, distorted"
|
||||
}
|
||||
},
|
||||
{
|
||||
"seed": 42,
|
||||
"extra_args": {
|
||||
"steps": 6,
|
||||
"cfg": 1.5,
|
||||
"height": 512,
|
||||
"width": 512,
|
||||
"seed": 123,
|
||||
"byte5_text": ["理解与生成统一"]
|
||||
}
|
||||
}
|
||||
],
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Draw a poster."
|
||||
}
|
||||
]
|
||||
}' \
|
||||
| jq -r '.choices[0].message.content[0].image_url.url | split(",")[1]' \
|
||||
| base64 -d > ming_imagegen_knobs.png
|
||||
```
|
||||
|
||||
### img2img (reference image)
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
# Streaming Video Generation
|
||||
|
||||
This example uses the custom WebSocket endpoint `WS /v1/realtime/video` to receive a video byte stream as chunks are produced.
|
||||
It covers text-only video generation. Image/reference input is intentionally not included for now.
|
||||
|
||||
## Start The Server
|
||||
|
||||
Start a diffusion video model with streaming output enabled:
|
||||
|
||||
```bash
|
||||
vllm serve BestWishYsh/Helios-Distilled \
|
||||
--omni \
|
||||
--diffusion-streaming-output \
|
||||
--port 8000
|
||||
```
|
||||
|
||||
The `--diffusion-streaming-output` CLI flag is forwarded as `streaming_output=True` in the default diffusion stage `engine_args`, then loaded by `OmniDiffusionConfig.from_kwargs()`.
|
||||
|
||||
## WebSocket Protocol
|
||||
|
||||
| Direction | Message | Format | Description |
|
||||
| --- | --- | --- | --- |
|
||||
| Client to server | `session.start` | JSON text: `{"type":"session.start","model":"...","prompt":"...","format":"m4s"}` | Starts generation. `format` is optional and accepts `m4s` (default). Sampling fields such as `width`, `height`, `fps`, `num_frames`, and `extra_params` may be included. |
|
||||
| Server to client | `video.start` | JSON text: `{"type":"video.start","request_id":"...","format":"m4s","config":{...}}` | Confirms the session and mirrors the accepted `format`. |
|
||||
| Server to client | Video chunk | Binary WebSocket frame | Fragmented MP4 (`m4s`) video bytes. |
|
||||
| Client to server | `session.stop` | JSON text: `{"type":"session.stop"}` | Requests cancellation of the active session. |
|
||||
| Client to server | `session.ping` | JSON text: `{"type":"session.ping"}` | Optional keepalive; refreshes the server stall clock. |
|
||||
| Server to client | `session.done` | JSON text: `{"type":"session.done","request_id":"...","chunks":3,"stopped":false}` | Ends a completed or stopped session. |
|
||||
| Server to client | `session.pong` | JSON text: `{"type":"session.pong"}` | Reply to `session.ping`. |
|
||||
| Server to client | `error` | JSON text: `{"type":"error","message":"..."}` | Reports invalid input, unsupported formats, generation failures, control-message errors, or stall timeout. |
|
||||
|
||||
During generation the client normally sends only `session.start` and then receives binary chunks; silence on the client socket is expected. The server closes the session with a stall error only when there is no engine progress and no `session.ping` for about 60 seconds.
|
||||
|
||||
## Install Client Dependency
|
||||
|
||||
```bash
|
||||
pip install websockets
|
||||
# For the Gradio demo:
|
||||
pip install 'vllm-omni[demo]' websockets
|
||||
```
|
||||
|
||||
## Run The Client
|
||||
|
||||
```bash
|
||||
python streaming_video_client.py \
|
||||
--host 127.0.0.1 \
|
||||
--port 8000 \
|
||||
--model BestWishYsh/Helios-Distilled \
|
||||
--prompt "A serene lakeside sunrise with mist over the water." \
|
||||
--width 640 \
|
||||
--height 384 \
|
||||
--fps 16 \
|
||||
--num-frames 99 \
|
||||
--guidance-scale 1.0 \
|
||||
--seed 42 \
|
||||
--output helios_stream.mp4
|
||||
```
|
||||
|
||||
The client sends one `session.start` message, prints each received binary video chunk with its byte size and elapsed time, and saves the received bytes to `--output` after `session.done`.
|
||||
The client remuxes the gathered stream to a regular progressive MP4 file so that local playback knows the video duration.
|
||||
|
||||
## Run The Gradio Demo
|
||||
|
||||
```bash
|
||||
python gradio_demo.py \
|
||||
--host 127.0.0.1 \
|
||||
--port 7860
|
||||
```
|
||||
|
||||
The Gradio demo requests fMP4 (`m4s`) chunks and appends them directly in the browser with a Media Source Extensions player.
|
||||
|
||||
## Model Choice
|
||||
|
||||
### Helios
|
||||
|
||||
The example uses `BestWishYsh/Helios-Distilled` model by default.
|
||||
|
||||
To ensure streaming-level generation speed, `pyramid_num_inference_steps_list` is suggested to be as low as `[1, 1, 1]`. Both example clients uses the following Helios-Distilled preset by default:
|
||||
|
||||
```json
|
||||
{
|
||||
"is_enable_stage2": true,
|
||||
"pyramid_num_stages": 3,
|
||||
"pyramid_num_inference_steps_list": [1, 1, 1],
|
||||
"is_amplify_first_chunk": true
|
||||
}
|
||||
```
|
||||
|
||||
Disable it in the CLI example with `--no-helios-distilled-preset`, or override/extend it with `--extra-params`:
|
||||
|
||||
```bash
|
||||
python streaming_video_client.py \
|
||||
--extra-params '{"pyramid_num_inference_steps_list":[2, 2, 2]}'
|
||||
```
|
||||
@@ -0,0 +1,301 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Gradio demo for `/v1/realtime/video`.
|
||||
|
||||
Usage:
|
||||
python gradio_demo.py --host 127.0.0.1 --port 7860
|
||||
|
||||
Start a vLLM-Omni server with streaming output enabled first, for example:
|
||||
vllm serve BestWishYsh/Helios-Distilled --omni --diffusion-streaming-output --port 8000
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
import gradio as gr
|
||||
except ImportError:
|
||||
raise ImportError("gradio is required to run this demo. Install it with: pip install 'vllm-omni[demo]'") from None
|
||||
|
||||
DEFAULT_PROMPT = "A vibrant tropical fish swimming gracefully among colorful coral reefs in a clear, turquoise ocean. The fish has bright blue and yellow scales with a small, distinctive orange spot on its side, its fins moving fluidly. The coral reefs are alive with a variety of marine life, including small schools of colorful fish and sea turtles gliding by. The water is crystal clear, allowing for a view of the sandy ocean floor below. The reef itself is adorned with a mix of hard and soft corals in shades of red, orange, and green. The photo captures the fish from a slightly elevated angle, emphasizing its lively movements and the vivid colors of its surroundings. A close-up shot with dynamic movement."
|
||||
VIDEO_STREAM_VIEW_HTML = Path(__file__).with_name("video-stream-view.html")
|
||||
VIDEO_STREAM_VIEW_JS = Path(__file__).with_name("video-stream-view.js")
|
||||
|
||||
|
||||
def _maybe_set(payload: dict[str, Any], key: str, value: Any) -> None:
|
||||
if value is None:
|
||||
return
|
||||
if isinstance(value, str) and not value.strip():
|
||||
return
|
||||
payload[key] = value
|
||||
|
||||
|
||||
def _optional_int(value: int | float | None) -> int | None:
|
||||
return None if value is None else int(value)
|
||||
|
||||
|
||||
def _optional_float(value: int | float | None) -> float | None:
|
||||
return None if value is None else float(value)
|
||||
|
||||
|
||||
def _build_session_start(
|
||||
*,
|
||||
model: str,
|
||||
prompt: str,
|
||||
negative_prompt: str,
|
||||
width: int | float | None,
|
||||
height: int | float | None,
|
||||
fps: int | float | None,
|
||||
num_frames: int | float | None,
|
||||
num_inference_steps: int | float | None,
|
||||
guidance_scale: int | float | None,
|
||||
guidance_scale_2: int | float | None,
|
||||
boundary_ratio: int | float | None,
|
||||
flow_shift: int | float | None,
|
||||
true_cfg_scale: int | float | None,
|
||||
seed: int | float | None,
|
||||
) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {
|
||||
"type": "session.start",
|
||||
"model": model,
|
||||
"prompt": prompt,
|
||||
"format": "m4s",
|
||||
}
|
||||
|
||||
_maybe_set(payload, "negative_prompt", negative_prompt)
|
||||
_maybe_set(payload, "width", _optional_int(width))
|
||||
_maybe_set(payload, "height", _optional_int(height))
|
||||
_maybe_set(payload, "fps", _optional_int(fps))
|
||||
_maybe_set(payload, "num_frames", _optional_int(num_frames))
|
||||
_maybe_set(payload, "num_inference_steps", _optional_int(num_inference_steps))
|
||||
_maybe_set(payload, "guidance_scale", _optional_float(guidance_scale))
|
||||
_maybe_set(payload, "guidance_scale_2", _optional_float(guidance_scale_2))
|
||||
_maybe_set(payload, "boundary_ratio", _optional_float(boundary_ratio))
|
||||
_maybe_set(payload, "flow_shift", _optional_float(flow_shift))
|
||||
_maybe_set(payload, "true_cfg_scale", _optional_float(true_cfg_scale))
|
||||
_maybe_set(payload, "seed", _optional_int(seed))
|
||||
|
||||
if "Helios" in model:
|
||||
payload["extra_params"] = {
|
||||
"is_enable_stage2": True,
|
||||
"pyramid_num_stages": 3,
|
||||
"pyramid_num_inference_steps_list": [1, 1, 1],
|
||||
"is_amplify_first_chunk": True,
|
||||
}
|
||||
|
||||
return payload
|
||||
|
||||
|
||||
def _set_optional_field_enabled(enabled: bool) -> Any:
|
||||
return gr.update(interactive=enabled)
|
||||
|
||||
|
||||
def build_browser_stream_config(
|
||||
host: str,
|
||||
port: int | float,
|
||||
model: str,
|
||||
prompt: str,
|
||||
negative_prompt: str,
|
||||
width: int | float | None,
|
||||
height: int | float | None,
|
||||
fps: int | float,
|
||||
num_frames: int | float | None,
|
||||
num_inference_steps: int | float | None,
|
||||
include_guidance_scale: bool,
|
||||
guidance_scale: int | float | None,
|
||||
include_guidance_scale_2: bool,
|
||||
guidance_scale_2: int | float | None,
|
||||
include_seed: bool,
|
||||
seed: int | float | None,
|
||||
include_boundary_ratio: bool,
|
||||
boundary_ratio: int | float | None,
|
||||
include_flow_shift: bool,
|
||||
flow_shift: int | float | None,
|
||||
include_true_cfg_scale: bool,
|
||||
true_cfg_scale: int | float | None,
|
||||
) -> tuple[str, Any]:
|
||||
"""Build the browser-side WebSocket request config for the HTML/MSE player."""
|
||||
if not model.strip():
|
||||
raise gr.Error("Model is required.")
|
||||
if not prompt.strip():
|
||||
raise gr.Error("Prompt is required.")
|
||||
|
||||
payload = _build_session_start(
|
||||
model=model.strip(),
|
||||
prompt=prompt.strip(),
|
||||
negative_prompt=negative_prompt,
|
||||
width=width,
|
||||
height=height,
|
||||
fps=fps,
|
||||
num_frames=num_frames,
|
||||
num_inference_steps=num_inference_steps,
|
||||
guidance_scale=guidance_scale if include_guidance_scale else None,
|
||||
guidance_scale_2=guidance_scale_2 if include_guidance_scale_2 else None,
|
||||
boundary_ratio=boundary_ratio if include_boundary_ratio else None,
|
||||
flow_shift=flow_shift if include_flow_shift else None,
|
||||
true_cfg_scale=true_cfg_scale if include_true_cfg_scale else None,
|
||||
seed=seed if include_seed else None,
|
||||
)
|
||||
config = {
|
||||
"url": f"ws://{host}:{int(port)}/v1/realtime/video",
|
||||
"payload": payload,
|
||||
}
|
||||
return json.dumps(config, ensure_ascii=False), gr.update(value="Streaming...", interactive=False)
|
||||
|
||||
|
||||
def create_demo() -> gr.Blocks:
|
||||
with gr.Blocks(title="Streaming Video Generation") as demo:
|
||||
gr.Markdown("# Streaming Video Generation")
|
||||
gr.Markdown(
|
||||
"Connects to `WS /v1/realtime/video` in the browser and appends fMP4 chunks "
|
||||
"directly to a Media Source Extensions video player."
|
||||
)
|
||||
|
||||
with gr.Row():
|
||||
with gr.Column(scale=1):
|
||||
host = gr.Textbox(label="Server Host", value="127.0.0.1")
|
||||
port = gr.Number(label="Server Port", value=8000, precision=0)
|
||||
model = gr.Textbox(label="Model", value="BestWishYsh/Helios-Distilled")
|
||||
prompt = gr.Textbox(label="Prompt", value=DEFAULT_PROMPT, lines=3)
|
||||
negative_prompt = gr.Textbox(label="Negative Prompt", value="", lines=2)
|
||||
|
||||
with gr.Row():
|
||||
width = gr.Number(label="Width", value=640, precision=0)
|
||||
height = gr.Number(label="Height", value=384, precision=0)
|
||||
|
||||
with gr.Row():
|
||||
fps = gr.Number(label="FPS", value=16, precision=0)
|
||||
num_frames = gr.Number(label="Num Frames", value=132, precision=0)
|
||||
num_inference_steps = gr.Number(label="Num Inference Steps", value=50, precision=0)
|
||||
|
||||
with gr.Row():
|
||||
with gr.Column():
|
||||
guidance_scale = gr.Number(
|
||||
label="Guidance Scale",
|
||||
value=1.0,
|
||||
interactive=True,
|
||||
scale=4,
|
||||
)
|
||||
include_guidance_scale = gr.Checkbox(label="Send", value=True, scale=1, min_width=70)
|
||||
with gr.Column():
|
||||
guidance_scale_2 = gr.Number(
|
||||
label="Guidance Scale 2",
|
||||
value=None,
|
||||
interactive=False,
|
||||
scale=4,
|
||||
)
|
||||
include_guidance_scale_2 = gr.Checkbox(label="Send", value=False, scale=1, min_width=70)
|
||||
with gr.Column():
|
||||
true_cfg_scale = gr.Number(
|
||||
label="True CFG Scale",
|
||||
value=None,
|
||||
interactive=False,
|
||||
scale=4,
|
||||
)
|
||||
include_true_cfg_scale = gr.Checkbox(label="Send", value=False, scale=1, min_width=70)
|
||||
# I want it to wrap after 3, but it auto wrap after 2. I'll rather fully delegate to auto wrapping.
|
||||
with gr.Column():
|
||||
boundary_ratio = gr.Number(
|
||||
label="Boundary Ratio",
|
||||
value=None,
|
||||
interactive=False,
|
||||
scale=4,
|
||||
)
|
||||
include_boundary_ratio = gr.Checkbox(label="Send", value=False, scale=1, min_width=70)
|
||||
with gr.Column():
|
||||
flow_shift = gr.Number(label="Flow Shift", value=None, interactive=False, scale=4)
|
||||
include_flow_shift = gr.Checkbox(label="Send", value=False, scale=1, min_width=70)
|
||||
with gr.Column():
|
||||
seed = gr.Number(label="Seed", value=42, precision=0, interactive=True, scale=4)
|
||||
include_seed = gr.Checkbox(label="Send", value=True, scale=1, min_width=70)
|
||||
|
||||
with gr.Row():
|
||||
start_button = gr.Button("Start", variant="primary", elem_id="streaming-video-start")
|
||||
hidden_stream_config = gr.Textbox(value="", visible=False)
|
||||
|
||||
with gr.Column(scale=1):
|
||||
gr.HTML(
|
||||
VIDEO_STREAM_VIEW_HTML.read_text(encoding="utf-8"),
|
||||
label="Streaming Preview",
|
||||
js_on_load=VIDEO_STREAM_VIEW_JS.read_text(encoding="utf-8"),
|
||||
)
|
||||
|
||||
inputs = [
|
||||
host,
|
||||
port,
|
||||
model,
|
||||
prompt,
|
||||
negative_prompt,
|
||||
width,
|
||||
height,
|
||||
fps,
|
||||
num_frames,
|
||||
num_inference_steps,
|
||||
include_guidance_scale,
|
||||
guidance_scale,
|
||||
include_guidance_scale_2,
|
||||
guidance_scale_2,
|
||||
include_seed,
|
||||
seed,
|
||||
include_boundary_ratio,
|
||||
boundary_ratio,
|
||||
include_flow_shift,
|
||||
flow_shift,
|
||||
include_true_cfg_scale,
|
||||
true_cfg_scale,
|
||||
]
|
||||
for toggle, field in [
|
||||
(include_guidance_scale, guidance_scale),
|
||||
(include_guidance_scale_2, guidance_scale_2),
|
||||
(include_seed, seed),
|
||||
(include_boundary_ratio, boundary_ratio),
|
||||
(include_flow_shift, flow_shift),
|
||||
(include_true_cfg_scale, true_cfg_scale),
|
||||
]:
|
||||
toggle.change(
|
||||
fn=_set_optional_field_enabled,
|
||||
inputs=toggle,
|
||||
outputs=field,
|
||||
)
|
||||
start_button.click(
|
||||
fn=build_browser_stream_config,
|
||||
inputs=inputs,
|
||||
outputs=[hidden_stream_config, start_button],
|
||||
).then(
|
||||
fn=lambda config: config,
|
||||
inputs=hidden_stream_config,
|
||||
outputs=hidden_stream_config,
|
||||
js="""
|
||||
(config) => {
|
||||
if (window.vllmStreamingVideoStart) {
|
||||
window.vllmStreamingVideoStart(config);
|
||||
}
|
||||
return config;
|
||||
}
|
||||
""",
|
||||
)
|
||||
|
||||
return demo
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Streaming video generation Gradio demo")
|
||||
parser.add_argument("--host", default="127.0.0.1", help="Host/IP for gradio launch")
|
||||
parser.add_argument("--port", type=int, default=7860, help="Port for gradio launch")
|
||||
parser.add_argument("--share", action="store_true", help="Create a public Gradio link")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
demo = create_demo()
|
||||
demo.queue().launch(server_name=args.host, server_port=args.port, share=args.share)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,279 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Text-only WebSocket client for `/v1/realtime/video`.
|
||||
|
||||
The client prints a line for every binary video chunk it receives and writes
|
||||
the received bytes to disk when the session finishes or is interrupted.
|
||||
Image/reference input is intentionally not supported in this example yet.
|
||||
|
||||
Requirements:
|
||||
pip install websockets
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import av
|
||||
|
||||
try:
|
||||
from websockets.asyncio.client import connect # pyright: ignore[reportMissingImports]
|
||||
except ImportError:
|
||||
try:
|
||||
from websockets import connect # pyright: ignore[reportMissingImports]
|
||||
except ImportError:
|
||||
print("Please install websockets: pip install websockets")
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
HELIOS_DISTILLED_EXTRA_PARAMS = {
|
||||
"is_enable_stage2": True,
|
||||
"pyramid_num_stages": 3,
|
||||
"pyramid_num_inference_steps_list": [1, 1, 1],
|
||||
"is_amplify_first_chunk": True,
|
||||
}
|
||||
|
||||
|
||||
def _json_object(value: str, *, argument_name: str) -> dict[str, Any]:
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise argparse.ArgumentTypeError(f"{argument_name} must be valid JSON: {exc}") from exc
|
||||
if not isinstance(parsed, dict):
|
||||
raise argparse.ArgumentTypeError(f"{argument_name} must be a JSON object")
|
||||
return parsed
|
||||
|
||||
|
||||
def _maybe_set(payload: dict[str, Any], key: str, value: Any) -> None:
|
||||
if value is not None:
|
||||
payload[key] = value
|
||||
|
||||
|
||||
def build_session_start(args: argparse.Namespace) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {
|
||||
"type": "session.start",
|
||||
"model": args.model,
|
||||
"prompt": args.prompt,
|
||||
}
|
||||
|
||||
_maybe_set(payload, "negative_prompt", args.negative_prompt)
|
||||
_maybe_set(payload, "width", args.width)
|
||||
_maybe_set(payload, "height", args.height)
|
||||
_maybe_set(payload, "size", args.size)
|
||||
_maybe_set(payload, "seconds", args.seconds)
|
||||
_maybe_set(payload, "fps", args.fps)
|
||||
_maybe_set(payload, "num_frames", args.num_frames)
|
||||
_maybe_set(payload, "num_inference_steps", args.num_inference_steps)
|
||||
_maybe_set(payload, "guidance_scale", args.guidance_scale)
|
||||
_maybe_set(payload, "guidance_scale_2", args.guidance_scale_2)
|
||||
_maybe_set(payload, "boundary_ratio", args.boundary_ratio)
|
||||
_maybe_set(payload, "flow_shift", args.flow_shift)
|
||||
_maybe_set(payload, "true_cfg_scale", args.true_cfg_scale)
|
||||
_maybe_set(payload, "seed", args.seed)
|
||||
|
||||
extra_params: dict[str, Any] = {}
|
||||
if "Helios-Distilled" in args.model and args.helios_distilled_preset:
|
||||
extra_params.update(HELIOS_DISTILLED_EXTRA_PARAMS)
|
||||
if args.extra_params:
|
||||
extra_params.update(args.extra_params)
|
||||
if extra_params:
|
||||
payload["extra_params"] = extra_params
|
||||
|
||||
return payload
|
||||
|
||||
|
||||
def _count_decoded_frames(video_bytes: bytes, *, stream_format: str) -> int | None:
|
||||
"""Return the number of decodable video frames in a streamed byte sequence, if known."""
|
||||
if not video_bytes:
|
||||
return 0
|
||||
|
||||
av_format = "mp4" if stream_format == "m4s" else None
|
||||
try:
|
||||
with av.open(io.BytesIO(video_bytes), format=av_format, mode="r") as container:
|
||||
stream = container.streams.video[0]
|
||||
return sum(1 for _ in container.decode(stream))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def save_video(
|
||||
chunks: list[bytes],
|
||||
output_path: Path,
|
||||
*,
|
||||
fps: float,
|
||||
stream_format: str,
|
||||
) -> None:
|
||||
if not chunks:
|
||||
print("No video chunks received; nothing to save.")
|
||||
return
|
||||
|
||||
from vllm_omni.diffusion.utils.media_utils import finalize_streaming_video_bytes
|
||||
|
||||
streamed_bytes = b"".join(chunks)
|
||||
playback_bytes = finalize_streaming_video_bytes(
|
||||
streamed_bytes,
|
||||
input_format="m4s",
|
||||
fps=fps,
|
||||
)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp_path = output_path.with_suffix(output_path.suffix + ".tmp")
|
||||
with tmp_path.open("wb") as f:
|
||||
f.write(playback_bytes)
|
||||
os.replace(tmp_path, output_path)
|
||||
print(
|
||||
f"Saved {len(chunks)} streamed chunk(s), {len(streamed_bytes)} streamed bytes "
|
||||
f"-> {len(playback_bytes)} playback bytes at {output_path}"
|
||||
)
|
||||
|
||||
|
||||
async def stream_video(args: argparse.Namespace) -> None:
|
||||
payload = build_session_start(args)
|
||||
output_path = Path(args.output)
|
||||
url = f"ws://{args.host}:{args.port}/v1/realtime/video"
|
||||
chunks: list[bytes] = []
|
||||
total_bytes = 0
|
||||
total_frames = 0
|
||||
accumulated_stream = bytearray()
|
||||
stream_format = "m4s"
|
||||
done = False
|
||||
ws = None
|
||||
|
||||
try:
|
||||
async with connect(url, max_size=None) as websocket:
|
||||
ws = websocket
|
||||
await websocket.send(json.dumps(payload))
|
||||
print(f"Connected: {url}")
|
||||
print(f"Sent session.start: {json.dumps(payload, ensure_ascii=False)}")
|
||||
started_at = time.perf_counter()
|
||||
last_chunk_at = started_at
|
||||
|
||||
while True:
|
||||
message = await websocket.recv()
|
||||
now = time.perf_counter()
|
||||
|
||||
if isinstance(message, bytes):
|
||||
chunks.append(message)
|
||||
chunk_bytes = len(message)
|
||||
chunk_elapsed = now - last_chunk_at
|
||||
last_chunk_at = now
|
||||
|
||||
total_bytes += chunk_bytes
|
||||
accumulated_stream.extend(message)
|
||||
decoded_total = _count_decoded_frames(bytes(accumulated_stream), stream_format=stream_format)
|
||||
if decoded_total is not None:
|
||||
chunk_frames = max(decoded_total - total_frames, 0)
|
||||
total_frames = decoded_total
|
||||
else:
|
||||
chunk_frames = 0
|
||||
|
||||
total_elapsed = now - started_at
|
||||
print(
|
||||
f"[chunk {len(chunks):03d}] "
|
||||
f"bytes={chunk_bytes} "
|
||||
f"frames={chunk_frames} "
|
||||
f"elapsed={chunk_elapsed:.2f}s "
|
||||
f"total_bytes={total_bytes} "
|
||||
f"total_frames={total_frames} "
|
||||
f"total_elapsed={total_elapsed:.2f}s"
|
||||
)
|
||||
continue
|
||||
|
||||
msg = json.loads(message)
|
||||
msg_type = msg.get("type")
|
||||
if msg_type == "video.start":
|
||||
stream_format = msg.get("format") or stream_format
|
||||
print(f"Video session started: request_id={msg.get('request_id')} format={msg.get('format')}")
|
||||
elif msg_type == "session.done":
|
||||
print(f"Session complete: {json.dumps(msg, ensure_ascii=False)}")
|
||||
done = True
|
||||
break
|
||||
elif msg_type == "error":
|
||||
print(f"ERROR: {msg.get('message', msg)}")
|
||||
else:
|
||||
print(f"Received control message: {msg}")
|
||||
except (KeyboardInterrupt, asyncio.CancelledError):
|
||||
print("\nInterrupted. Asking server to stop the active session...")
|
||||
if ws is not None:
|
||||
try:
|
||||
await ws.send(json.dumps({"type": "session.stop"}))
|
||||
await ws.close(code=1000, reason="client interrupted")
|
||||
except Exception:
|
||||
pass
|
||||
raise
|
||||
finally:
|
||||
print(
|
||||
"Saving video... (May take a while. Remuxing concatenated chunks into one progressive MP4 file with total duration metadata in file header)"
|
||||
)
|
||||
save_video(
|
||||
chunks,
|
||||
output_path,
|
||||
fps=float(payload.get("fps") or args.fps),
|
||||
stream_format=stream_format,
|
||||
)
|
||||
if not done:
|
||||
print("Saved partial output because the stream did not finish normally.")
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Streaming video generation WebSocket client")
|
||||
parser.add_argument("--host", default="127.0.0.1", help="API server host")
|
||||
parser.add_argument("--port", type=int, default=8000, help="API server port")
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
default="BestWishYsh/Helios-Distilled",
|
||||
help="Model name served by the API server",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prompt",
|
||||
default="A vibrant tropical fish swimming gracefully among colorful coral reefs in a clear, turquoise ocean. The fish has bright blue and yellow scales with a small, distinctive orange spot on its side, its fins moving fluidly. The coral reefs are alive with a variety of marine life, including small schools of colorful fish and sea turtles gliding by. The water is crystal clear, allowing for a view of the sandy ocean floor below. The reef itself is adorned with a mix of hard and soft corals in shades of red, orange, and green. The photo captures the fish from a slightly elevated angle, emphasizing its lively movements and the vivid colors of its surroundings. A close-up shot with dynamic movement.",
|
||||
help="Text prompt for video generation",
|
||||
)
|
||||
parser.add_argument("--output", default="streaming_video_output.mp4", help="Path for the received video bytes")
|
||||
parser.add_argument("--width", type=int, default=640, help="Video width")
|
||||
parser.add_argument("--height", type=int, default=384, help="Video height")
|
||||
parser.add_argument("--size", default=None, help="Video size as WIDTHxHEIGHT; alternative to width/height")
|
||||
parser.add_argument("--seconds", default=None, help="Clip duration in seconds")
|
||||
parser.add_argument("--fps", type=int, default=16, help="Frames per second")
|
||||
parser.add_argument("--num-frames", type=int, default=99, help="Number of generated frames")
|
||||
|
||||
parser.add_argument("--negative-prompt", default=None, help="Negative prompt")
|
||||
parser.add_argument("--num-inference-steps", type=int, default=None, help="Number of diffusion steps")
|
||||
parser.add_argument("--guidance-scale", type=float, default=1.0, help="Classifier-free guidance scale")
|
||||
parser.add_argument("--guidance-scale-2", type=float, default=None, help="High-noise CFG scale for video models")
|
||||
parser.add_argument("--boundary-ratio", type=float, default=None, help="Boundary split ratio for video models")
|
||||
parser.add_argument("--flow-shift", type=float, default=None, help="Scheduler flow_shift for video models")
|
||||
parser.add_argument("--true-cfg-scale", type=float, default=None, help="True CFG scale for models that support it")
|
||||
parser.add_argument("--seed", type=int, default=42, help="Random seed")
|
||||
|
||||
parser.add_argument(
|
||||
"--helios-distilled-preset",
|
||||
action=argparse.BooleanOptionalAction,
|
||||
default=True,
|
||||
help="Send the Helios-Distilled extra_params used by the bundled example.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--extra-params",
|
||||
type=lambda value: _json_object(value, argument_name="--extra-params"),
|
||||
default=None,
|
||||
help="JSON object merged into extra_params; overrides preset keys on conflict.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
try:
|
||||
asyncio.run(stream_video(args))
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,11 @@
|
||||
<!-- This file and the corresponding JS file are needed to display a streamable video view in the Gradio demo.
|
||||
Gradio's built-in video player does not support modern M4S streaming format. -->
|
||||
<div id="vllm-streaming-video-view" style="display:flex; flex-direction:column; gap:10px;">
|
||||
<video id="vllm-streaming-video-player" autoplay muted controls playsinline
|
||||
style="width:100%; min-height:320px; background:#111; border-radius:8px;"></video>
|
||||
<div style="display:flex; align-items:center; gap:10px;">
|
||||
<span id="vllm-streaming-video-status" style="font-weight:600;">Ready</span>
|
||||
</div>
|
||||
<pre id="vllm-streaming-video-log" style="height:220px; overflow:auto; margin:0; padding:10px; border-radius:8px;
|
||||
background:#0f172a; color:#dbeafe; font-size:12px; white-space:pre-wrap;">Ready.</pre>
|
||||
</div>
|
||||
@@ -0,0 +1,198 @@
|
||||
/**
|
||||
* This file and the corresponding HTML file are needed to display a streamable video view in the Gradio demo.
|
||||
* Gradio's built-in video player does not support modern M4S streaming format.
|
||||
*/
|
||||
(function () {
|
||||
const playerId = "vllm-streaming-video-player";
|
||||
const statusId = "vllm-streaming-video-status";
|
||||
const logId = "vllm-streaming-video-log";
|
||||
let ws = null;
|
||||
let mediaSource = null;
|
||||
let sourceBuffer = null;
|
||||
let queue = [];
|
||||
let chunkCount = 0;
|
||||
let totalBytes = 0;
|
||||
let done = false;
|
||||
let objectUrl = null;
|
||||
|
||||
function el(id) {
|
||||
return document.getElementById(id);
|
||||
}
|
||||
|
||||
function setStatus(text) {
|
||||
const node = el(statusId);
|
||||
if (node) node.textContent = text;
|
||||
}
|
||||
|
||||
function log(line) {
|
||||
const node = el(logId);
|
||||
if (!node) return;
|
||||
node.textContent += "\n" + line;
|
||||
node.scrollTop = node.scrollHeight;
|
||||
}
|
||||
|
||||
function setStartButton(disabled, label) {
|
||||
const btn = document.querySelector("#streaming-video-start button") || document.getElementById("streaming-video-start");
|
||||
if (!btn) return;
|
||||
btn.disabled = disabled;
|
||||
btn.textContent = label;
|
||||
}
|
||||
|
||||
function chooseMime() {
|
||||
const candidates = [
|
||||
'video/mp4; codecs="avc1.42E01E"',
|
||||
'video/mp4; codecs="avc1.4D401F"',
|
||||
'video/mp4; codecs="avc1.64001F"',
|
||||
"video/mp4"
|
||||
];
|
||||
for (const mime of candidates) {
|
||||
if (window.MediaSource && MediaSource.isTypeSupported(mime)) return mime;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function pump() {
|
||||
if (!sourceBuffer || sourceBuffer.updating || queue.length === 0) return;
|
||||
try {
|
||||
sourceBuffer.appendBuffer(queue.shift());
|
||||
} catch (err) {
|
||||
done = true;
|
||||
setStatus("SourceBuffer error");
|
||||
setStartButton(false, "Restart");
|
||||
log("SourceBuffer append error: " + err.message);
|
||||
finishStream();
|
||||
}
|
||||
}
|
||||
|
||||
function finishStream() {
|
||||
if (!mediaSource || mediaSource.readyState !== "open") return;
|
||||
if (sourceBuffer && sourceBuffer.updating) return;
|
||||
if (queue.length > 0) return;
|
||||
try {
|
||||
mediaSource.endOfStream();
|
||||
} catch (_) { }
|
||||
}
|
||||
|
||||
function stopCurrent() {
|
||||
if (ws && ws.readyState === WebSocket.OPEN) {
|
||||
try { ws.send(JSON.stringify({ type: "session.stop" })); } catch (_) { }
|
||||
try { ws.close(); } catch (_) { }
|
||||
}
|
||||
ws = null;
|
||||
sourceBuffer = null;
|
||||
mediaSource = null;
|
||||
queue = [];
|
||||
if (objectUrl) URL.revokeObjectURL(objectUrl);
|
||||
objectUrl = null;
|
||||
}
|
||||
|
||||
window.vllmStreamingVideoStart = function (configJson) {
|
||||
stopCurrent();
|
||||
chunkCount = 0;
|
||||
totalBytes = 0;
|
||||
done = false;
|
||||
const logNode = el(logId);
|
||||
if (logNode) logNode.textContent = "Starting...";
|
||||
setStatus("Starting...");
|
||||
|
||||
let config;
|
||||
try {
|
||||
config = JSON.parse(configJson);
|
||||
} catch (err) {
|
||||
setStatus("Invalid request");
|
||||
setStartButton(false, "Restart");
|
||||
log("Invalid request config: " + err.message);
|
||||
return configJson;
|
||||
}
|
||||
|
||||
const mime = chooseMime();
|
||||
if (!mime) {
|
||||
setStatus("MSE unsupported");
|
||||
setStartButton(false, "Restart");
|
||||
log("This browser does not support MP4 Media Source Extensions playback.");
|
||||
return configJson;
|
||||
}
|
||||
|
||||
const video = el(playerId);
|
||||
if (!video) return configJson;
|
||||
mediaSource = new MediaSource();
|
||||
objectUrl = URL.createObjectURL(mediaSource);
|
||||
video.src = objectUrl;
|
||||
|
||||
mediaSource.addEventListener("sourceopen", () => {
|
||||
try {
|
||||
sourceBuffer = mediaSource.addSourceBuffer(mime);
|
||||
sourceBuffer.mode = "segments";
|
||||
sourceBuffer.addEventListener("updateend", () => {
|
||||
pump();
|
||||
if (done) finishStream();
|
||||
});
|
||||
} catch (err) {
|
||||
setStatus("SourceBuffer error");
|
||||
setStartButton(false, "Restart");
|
||||
log("SourceBuffer error: " + err.message);
|
||||
return;
|
||||
}
|
||||
|
||||
ws = new WebSocket(config.url);
|
||||
ws.binaryType = "arraybuffer";
|
||||
ws.onopen = () => {
|
||||
setStatus("Streaming...");
|
||||
setStartButton(true, "Streaming...");
|
||||
log("Connected: " + config.url);
|
||||
ws.send(JSON.stringify(config.payload));
|
||||
log("Sent session.start: " + JSON.stringify(config.payload));
|
||||
};
|
||||
ws.onmessage = (event) => {
|
||||
if (typeof event.data === "string") {
|
||||
const msg = JSON.parse(event.data);
|
||||
if (msg.type === "video.start") {
|
||||
log("Video session started: request_id=" + (msg.request_id || "") + " format=" + (msg.format || ""));
|
||||
} else if (msg.type === "session.done") {
|
||||
done = true;
|
||||
setStatus("Done");
|
||||
setStartButton(false, "Restart");
|
||||
log("Session complete: " + JSON.stringify(msg));
|
||||
finishStream();
|
||||
} else if (msg.type === "error") {
|
||||
done = true;
|
||||
setStatus("Error");
|
||||
setStartButton(false, "Restart");
|
||||
log("ERROR: " + (msg.message || JSON.stringify(msg)));
|
||||
finishStream();
|
||||
} else {
|
||||
log("Control message: " + JSON.stringify(msg));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const data = event.data;
|
||||
queue.push(data);
|
||||
chunkCount += 1;
|
||||
totalBytes += data.byteLength;
|
||||
log(`[chunk ${String(chunkCount).padStart(3, "0")}] bytes=${data.byteLength} total_bytes=${totalBytes}`);
|
||||
pump();
|
||||
};
|
||||
ws.onerror = () => {
|
||||
done = true;
|
||||
setStatus("Error");
|
||||
setStartButton(false, "Restart");
|
||||
log("ERROR: WebSocket error");
|
||||
finishStream();
|
||||
};
|
||||
ws.onclose = () => {
|
||||
if (!done) {
|
||||
done = true;
|
||||
setStatus("Closed");
|
||||
setStartButton(false, "Restart");
|
||||
log("WebSocket closed before session.done");
|
||||
finishStream();
|
||||
}
|
||||
};
|
||||
}, { once: true });
|
||||
|
||||
return configJson;
|
||||
};
|
||||
|
||||
console.log("video-stream-view.html loaded");
|
||||
})();
|
||||
@@ -16,6 +16,7 @@ For the full list of supported architectures across all modalities, see
|
||||
|---|---|---|---|---|---|
|
||||
| Fish Speech S2 Pro | `fishaudio/s2-pro` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | ✓ |
|
||||
| GLM-TTS | `zai-org/GLM-TTS` | ✓ (`ref_audio`+`ref_text`, required) | ✓ (PCM stream) | — | ✓ |
|
||||
| IndexTTS-2 | `IndexTeam/IndexTTS-2` | ✓ (`ref_audio` or uploaded `voice`) | compat only, non-chunk | uploaded audio voice only; no presets | — |
|
||||
| Ming-omni-tts | `inclusionAI/Ming-omni-tts-0.5B` | ✓ (`ref_audio` / `speaker_embedding`) | ✓ (PCM stream) | IP labels + structured `instructions` | — |
|
||||
| Ming-flash-omni-TTS | `Jonathan1909/Ming-flash-omni-2.0` | — (caption-controlled) | — | caption fields (`instructions`) | — |
|
||||
| MOSS-TTS-Nano | `OpenMOSS-Team/MOSS-TTS-Nano` | ✓ (`ref_audio` required) | ✓ (PCM stream) | — | ✓ |
|
||||
@@ -23,6 +24,7 @@ For the full list of supported architectures across all modalities, see
|
||||
| Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | ✓ (Base) | ✓ (PCM + WebSocket) | ✓ (presets + `/v1/audio/voices` upload) | ✓ (standard + FastRTC) |
|
||||
| VoxCPM2 | `openbmb/VoxCPM2` | ✓ | ✓ (AudioWorklet via gradio) | — | ✓ |
|
||||
| Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✓ (gated upstream) | ✓ | ✓ (presets) | ✓ |
|
||||
| SoulX-Singer | `Soul-AILab/SoulX-Singer` | ✓ (prompt audio) | — (batch only) | — (prompt + target audio) | — (chat client) |
|
||||
|
||||
CosyVoice3 is intentionally absent: no online example exists for it yet. See its [offline section](../../offline_inference/text_to_speech/README.md#cosyvoice3) instead.
|
||||
|
||||
@@ -34,7 +36,7 @@ Launch the server (defaults shown — adjust `--port`, `--gpu-memory-utilization
|
||||
vllm serve <hf-repo-or-local-path> --omni --port 8091
|
||||
```
|
||||
|
||||
Send a TTS request via curl:
|
||||
Send a TTS request via curl. These generic snippets assume a model with a preset/default voice; voice-cloning-only models such as IndexTTS-2 require `ref_audio` or an uploaded audio `voice` (see model-specific sections below).
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8091/v1/audio/speech \
|
||||
@@ -90,7 +92,7 @@ curl -X POST http://localhost:8091/v1/audio/speech \
|
||||
}' --no-buffer | play -t raw -r 24000 -e signed -b 16 -c 1 -
|
||||
```
|
||||
|
||||
Adjust the player's sample rate to match the model (44.1 kHz for Fish Speech, 48 kHz for VoxCPM2, 24 kHz for the others).
|
||||
Adjust the player's sample rate to match the model (44.1 kHz for Fish Speech, 48 kHz for VoxCPM2, 22.05 kHz for IndexTTS-2, and 24 kHz for many others).
|
||||
|
||||
For full request-shape documentation (all parameters, response formats, error codes), see the [Speech API reference](../../../docs/serving/speech_api.md).
|
||||
|
||||
@@ -135,6 +137,40 @@ bash examples/online_serving/text_to_speech/glm_tts/run_gradio_demo.sh
|
||||
|
||||
---
|
||||
|
||||
## IndexTTS-2
|
||||
|
||||
2-stage TTS (GPT AR + S2Mel CFM DiT + BigVGAN) at 22.05 kHz. Requests use `ref_audio` for voice cloning, or an uploaded audio `voice` from `/v1/audio/voices`. Supports emotion conditioning via `emo_audio`, `emo_text`, or `emo_vector` passed in `extra_params`.
|
||||
|
||||
### Launch
|
||||
```bash
|
||||
vllm serve IndexTeam/IndexTTS-2 --omni --trust-remote-code --port 8092
|
||||
# or, to pass the bundled deploy config explicitly:
|
||||
bash examples/online_serving/text_to_speech/indextts2/run_server.sh
|
||||
```
|
||||
|
||||
### Sending requests
|
||||
```bash
|
||||
# Voice cloning (ref_audio required)
|
||||
python examples/online_serving/text_to_speech/indextts2/speech_client.py \
|
||||
--text "你好,世界!" \
|
||||
--ref-audio /path/to/reference.wav
|
||||
|
||||
# With emotion audio
|
||||
python examples/online_serving/text_to_speech/indextts2/speech_client.py \
|
||||
--text "今天心情很好!" \
|
||||
--ref-audio /path/to/ref.wav \
|
||||
--emo-audio /path/to/happy.wav
|
||||
```
|
||||
|
||||
### Notes
|
||||
- Output: 22.05 kHz mono WAV.
|
||||
- Provide `ref_audio` on the documented raw request path, or pass `voice` only when it names an uploaded audio voice; IndexTTS-2 does not provide a built-in text-only preset voice.
|
||||
- Emotion params (`emo_audio`, `emo_text`, `emo_vector`, `emo_alpha`, `use_emo_text`, `use_random`) are passed via the `extra_params` field. Official precedence is `use_emo_text` > `emo_vector` > `emo_audio` > same emotion as the speaker reference.
|
||||
- `stream=true` is accepted as an OpenAI-compatible response path, but IndexTTS-2 is not async-chunk streaming; audio is produced after S2Mel receives the full mel-code sequence.
|
||||
- Deploy config: `vllm_omni/deploy/indextts2.yaml` (auto-loaded).
|
||||
|
||||
---
|
||||
|
||||
## Fish Speech S2 Pro
|
||||
|
||||
4B dual-AR TTS at 44.1 kHz. Server uses the DAC codec.
|
||||
@@ -626,3 +662,65 @@ The demo handles voice-preset selection and reference-audio upload. `voxtral_tts
|
||||
- Voice presets are listed on the HF model card (`mistralai/Voxtral-4B-TTS-2603`).
|
||||
- Voice cloning is gated upstream and may require a recent `mistral_common`.
|
||||
- A standalone CLI client is not yet shipped; the gradio demo is the canonical reference for now.
|
||||
|
||||
---
|
||||
|
||||
## SoulX-Singer
|
||||
|
||||
Singing voice synthesis (SVS) and conversion (SVC) at 24 kHz. Single-stage DiT with inline preprocess. Uses the `/v1/chat/completions` endpoint with multimodal input (`prompt_audio` + `target_audio`).
|
||||
|
||||
### Prerequisites
|
||||
|
||||
Download DiT and preprocess weights, then set up separate SVS / SVC view directories and install dependencies as described in the [offline README](../../offline_inference/text_to_speech/README.md#soulx-singer). `config.json` `architectures` field is the single source of truth for SVS vs SVC — point `MODEL` at the matching directory.
|
||||
|
||||
### Launch
|
||||
|
||||
```bash
|
||||
# SVS (default)
|
||||
export MODEL=/path/to/SoulX-Singer
|
||||
export PREPROCESS=/path/to/SoulX-Singer-Preprocess
|
||||
bash examples/online_serving/text_to_speech/soulxsinger/run_server.sh
|
||||
|
||||
# SVC
|
||||
export MODE=svc
|
||||
export MODEL=/path/to/SoulX-Singer-svc
|
||||
bash examples/online_serving/text_to_speech/soulxsinger/run_server.sh
|
||||
```
|
||||
|
||||
Or equivalently, set `SOULX_PREPROCESS_WEIGHTS_DIR` and launch directly:
|
||||
```bash
|
||||
export SOULX_PREPROCESS_WEIGHTS_DIR=$PREPROCESS
|
||||
vllm serve $MODEL --omni \
|
||||
--deploy-config vllm_omni/deploy/soulxsinger_${MODE}.yaml \
|
||||
--port 8192 --trust-remote-code --enforce-eager
|
||||
```
|
||||
|
||||
### Sending requests
|
||||
|
||||
Audio paths must be reachable from the server host (local filesystem or data URL). The client sends prompt vocal via `input_audio` and target accompaniment via `extra_args['target_audio']`.
|
||||
|
||||
```bash
|
||||
# Default demo audio: tests/assets/soulxsinger/zh_prompt.mp3 + music.mp3
|
||||
python examples/online_serving/text_to_speech/soulxsinger/openai_chat_client.py \
|
||||
--prompt-audio /path/on/server/zh_prompt.mp3 \
|
||||
--target-audio /path/on/server/music.mp3 \
|
||||
--preprocess-weights-dir /path/on/server/SoulX-Singer-Preprocess \
|
||||
-o output.wav
|
||||
```
|
||||
|
||||
Use precomputed metadata to skip online preprocess with following command:
|
||||
```bash
|
||||
python examples/online_serving/text_to_speech/soulxsinger/openai_chat_client.py \
|
||||
--prompt-metadata-path /path/on/server/zh_prompt.json \
|
||||
--target-metadata-path /path/on/server/music.json \
|
||||
--audio-path /path/on/server/zh_prompt.mp3 \
|
||||
-o output.wav
|
||||
```
|
||||
|
||||
`SOULX_PREPROCESS_WEIGHTS_DIR` makes `--preprocess-weights-dir` optional. See `openai_chat_client.py --help` for `--vocal-sep`, `--language`, `--num-inference-steps`, `--guidance-scale`, and `--seed`.
|
||||
|
||||
### Notes
|
||||
|
||||
- Output: 24 kHz mono WAV; batch only.
|
||||
- Defaults match upstream: `--guidance-scale 3.0`, `--seed 42`, `--auto-shift` on.
|
||||
- SVS `--control`: `score` or `melody`. MIDI / lyric QC: upstream `midi_editor` only.
|
||||
@@ -0,0 +1,25 @@
|
||||
#!/bin/bash
|
||||
# Launch vLLM-Omni server for IndexTTS2
|
||||
#
|
||||
# Usage from repository root:
|
||||
# examples/online_serving/text_to_speech/indextts2/run_server.sh
|
||||
# CUDA_VISIBLE_DEVICES=0 PORT=8092 MODEL=/path/to/IndexTeam/IndexTTS-2 examples/online_serving/text_to_speech/indextts2/run_server.sh
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT_DIR="$(cd -- "$SCRIPT_DIR/../../../.." && pwd)"
|
||||
|
||||
MODEL="${MODEL:-IndexTeam/IndexTTS-2}"
|
||||
PORT="${PORT:-8092}"
|
||||
DEPLOY_CONFIG="${DEPLOY_CONFIG:-$ROOT_DIR/vllm_omni/deploy/indextts2.yaml}"
|
||||
|
||||
echo "Starting IndexTTS2 server with model: $MODEL"
|
||||
|
||||
FLASHINFER_DISABLE_VERSION_CHECK=1 \
|
||||
vllm serve "$MODEL" \
|
||||
--host 0.0.0.0 \
|
||||
--port "$PORT" \
|
||||
--omni \
|
||||
--trust-remote-code \
|
||||
--deploy-config "$DEPLOY_CONFIG"
|
||||
@@ -0,0 +1,126 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""OpenAI-compatible client for IndexTTS2 TTS via /v1/audio/speech endpoint.
|
||||
|
||||
Examples:
|
||||
# With reference audio for voice cloning
|
||||
python speech_client.py --text "你好,世界!" \
|
||||
--ref-audio /path/to/reference.wav
|
||||
|
||||
# With emotion audio
|
||||
python speech_client.py --text "今天心情很好!" \
|
||||
--ref-audio /path/to/ref.wav \
|
||||
--emo-audio /path/to/happy.wav
|
||||
|
||||
Server setup:
|
||||
vllm serve IndexTeam/IndexTTS-2 --omni --host 0.0.0.0 --port 8092
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import os
|
||||
|
||||
import httpx
|
||||
|
||||
DEFAULT_API_BASE = "http://localhost:8092"
|
||||
DEFAULT_API_KEY = "sk-empty"
|
||||
|
||||
|
||||
def encode_audio_to_base64(audio_path: str) -> str:
|
||||
if not os.path.exists(audio_path):
|
||||
raise FileNotFoundError(f"Audio file not found: {audio_path}")
|
||||
ext = audio_path.lower().rsplit(".", 1)[-1]
|
||||
mime = {"wav": "audio/wav", "mp3": "audio/mpeg", "flac": "audio/flac"}.get(ext, "audio/wav")
|
||||
with open(audio_path, "rb") as f:
|
||||
b64 = base64.b64encode(f.read()).decode("utf-8")
|
||||
return f"data:{mime};base64,{b64}"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="IndexTTS2 OpenAI speech client")
|
||||
parser.add_argument("--text", type=str, required=True)
|
||||
parser.add_argument("--ref-audio", type=str, default=None, help="Reference audio for voice cloning")
|
||||
parser.add_argument("--emo-audio", type=str, default=None, help="Emotion reference audio")
|
||||
parser.add_argument("--emo-text", type=str, default=None, help="Emotion description text")
|
||||
parser.add_argument(
|
||||
"--emo-vector",
|
||||
type=float,
|
||||
nargs=8,
|
||||
default=None,
|
||||
help="8-dim emotion vector: happy angry sad afraid disgusted melancholic surprised calm",
|
||||
)
|
||||
parser.add_argument("--emo-alpha", type=float, default=None, help="Emotion weight in [0, 1]")
|
||||
parser.add_argument("--use-emo-text", action="store_true", help="Infer emotion vector from emo-text or text")
|
||||
parser.add_argument("--use-random", action="store_true", help="Use random emotion prototypes")
|
||||
parser.add_argument("--model", type=str, default="IndexTeam/IndexTTS-2")
|
||||
parser.add_argument("--voice", type=str, default=None, help="Uploaded voice name to use instead of --ref-audio")
|
||||
parser.add_argument("--output", type=str, default="output.wav")
|
||||
parser.add_argument("--api-base", type=str, default=DEFAULT_API_BASE)
|
||||
parser.add_argument("--api-key", type=str, default=DEFAULT_API_KEY)
|
||||
parser.add_argument("--response-format", type=str, default="wav")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.ref_audio and not args.voice:
|
||||
parser.error("IndexTTS2 requires --ref-audio or --voice for voice cloning")
|
||||
|
||||
payload: dict = {
|
||||
"model": args.model,
|
||||
"input": args.text,
|
||||
"response_format": args.response_format,
|
||||
}
|
||||
if args.voice:
|
||||
payload["voice"] = args.voice
|
||||
|
||||
if args.ref_audio:
|
||||
ref = args.ref_audio
|
||||
if ref.startswith(("http://", "https://", "data:")):
|
||||
payload["ref_audio"] = ref
|
||||
else:
|
||||
payload["ref_audio"] = encode_audio_to_base64(ref)
|
||||
|
||||
extra_params = {}
|
||||
if args.emo_audio:
|
||||
emo = args.emo_audio
|
||||
if emo.startswith(("http://", "https://", "data:")):
|
||||
extra_params["emo_audio"] = emo
|
||||
else:
|
||||
extra_params["emo_audio"] = encode_audio_to_base64(emo)
|
||||
if args.emo_text:
|
||||
extra_params["emo_text"] = args.emo_text
|
||||
if args.emo_vector is not None:
|
||||
extra_params["emo_vector"] = args.emo_vector
|
||||
if args.emo_alpha is not None:
|
||||
extra_params["emo_alpha"] = args.emo_alpha
|
||||
if args.use_emo_text:
|
||||
extra_params["use_emo_text"] = True
|
||||
if args.use_random:
|
||||
extra_params["use_random"] = True
|
||||
if extra_params:
|
||||
payload["extra_params"] = extra_params
|
||||
|
||||
url = f"{args.api_base}/v1/audio/speech"
|
||||
print(f"POST {url}")
|
||||
print(f" text: {args.text}")
|
||||
if args.ref_audio:
|
||||
print(f" ref_audio: {args.ref_audio[:80]}...")
|
||||
|
||||
with httpx.Client(timeout=300) as client:
|
||||
resp = client.post(
|
||||
url,
|
||||
json=payload,
|
||||
headers={"Authorization": f"Bearer {args.api_key}"},
|
||||
)
|
||||
|
||||
if resp.status_code != 200:
|
||||
print(f"Error {resp.status_code}: {resp.text[:500]}")
|
||||
return
|
||||
|
||||
with open(args.output, "wb") as f:
|
||||
f.write(resp.content)
|
||||
print(f"Saved: {args.output} ({len(resp.content):,} bytes)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,186 @@
|
||||
#!/usr/bin/env python3
|
||||
"""SoulX-Singer OpenAI-compatible chat client (SVS / SVC).
|
||||
|
||||
Sends prompt audio via ``input_audio`` and target accompaniment via
|
||||
``extra_args['target_audio']`` (server-local path). For integrated preprocess,
|
||||
also pass ``preprocess_weights_dir`` in ``extra_args``.
|
||||
|
||||
Usage:
|
||||
python openai_chat_client.py \\
|
||||
--prompt-audio /path/on/server/zh_prompt.mp3 \\
|
||||
--target-audio /path/on/server/music.mp3 \\
|
||||
--preprocess-weights-dir /path/on/server/SoulX-Singer-Preprocess \\
|
||||
-o output.wav
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import io
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
import soundfile
|
||||
import torch
|
||||
|
||||
|
||||
def _audio_to_data_url(path: Path) -> str:
|
||||
with path.open("rb") as handle:
|
||||
data = base64.b64encode(handle.read()).decode("ascii")
|
||||
return f"data:audio/mpeg;base64,{data}"
|
||||
|
||||
|
||||
def _save_wav(audio: torch.Tensor, path: Path, sample_rate: int) -> None:
|
||||
audio = audio.to(torch.float32)
|
||||
peak = audio.abs().max().clamp(min=1e-8)
|
||||
audio = audio / peak
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
soundfile.write(str(path), audio.clamp(-1.0, 1.0).cpu().T.numpy(), sample_rate, subtype="PCM_16")
|
||||
|
||||
|
||||
def _decode_audio_from_response(body: dict) -> tuple[torch.Tensor, int]:
|
||||
for choice in body.get("choices", []):
|
||||
audio_obj = choice.get("message", {}).get("audio")
|
||||
if isinstance(audio_obj, dict) and audio_obj.get("data"):
|
||||
data, sr = soundfile.read(
|
||||
io.BytesIO(base64.b64decode(audio_obj["data"])),
|
||||
dtype="float32",
|
||||
always_2d=True,
|
||||
)
|
||||
return torch.from_numpy(data).transpose(0, 1), sr
|
||||
brief = {k: v for k, v in body.items() if k != "choices"}
|
||||
raise RuntimeError(f"no audio in response message.audio: {brief}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
repo_root = Path(__file__).resolve().parents[4]
|
||||
default_assets = repo_root / "tests" / "assets" / "soulxsinger"
|
||||
|
||||
parser = argparse.ArgumentParser(description="SoulX-Singer online chat client")
|
||||
parser.add_argument("--port", type=int, default=8192)
|
||||
parser.add_argument("--model", default="Soul-AILab/SoulX-Singer")
|
||||
parser.add_argument(
|
||||
"--prompt-audio",
|
||||
default=str(default_assets / "zh_prompt.mp3"),
|
||||
help="Prompt vocal audio (path on server if using extra_args, or local for input_audio)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--target-audio",
|
||||
default=str(default_assets / "music.mp3"),
|
||||
help="Target accompaniment path on the server (extra_args['target_audio'])",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prompt-metadata-path",
|
||||
default=None,
|
||||
help="SVS precomputed prompt metadata.json",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--target-metadata-path",
|
||||
default=None,
|
||||
help="SVS precomputed target metadata.json",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--audio-path",
|
||||
default=None,
|
||||
help="SVS prompt vocal wav for precomputed metadata",
|
||||
)
|
||||
parser.add_argument("--preprocess-weights-dir", default=None)
|
||||
parser.add_argument("--output", "-o", default="soulxsinger_out.wav")
|
||||
parser.add_argument("--svc", action="store_true", help="Use SVC mode knobs")
|
||||
parser.add_argument("--language", default="Mandarin")
|
||||
parser.add_argument("--num-inference-steps", type=int, default=32)
|
||||
parser.add_argument("--guidance-scale", type=float, default=3.0)
|
||||
parser.add_argument(
|
||||
"--seed",
|
||||
type=int,
|
||||
default=42,
|
||||
help="Optional CFM seed. Omit for non-deterministic sampling.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--auto-shift",
|
||||
action=argparse.BooleanOptionalAction,
|
||||
default=True,
|
||||
help="Auto pitch shift (default: on, original upstream infer.sh)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--control",
|
||||
default="melody",
|
||||
choices=["melody", "score"],
|
||||
help="SVS control mode",
|
||||
)
|
||||
parser.add_argument("--vocal-sep", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
meta_paths = (args.prompt_metadata_path, args.target_metadata_path, args.audio_path)
|
||||
if any(meta_paths) and not all(meta_paths):
|
||||
print(
|
||||
"ERROR: precomputed metadata requires --prompt-metadata-path, "
|
||||
"--target-metadata-path, and --audio-path together.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
|
||||
extra_args: dict = {
|
||||
"vocal_sep": args.vocal_sep,
|
||||
"auto_shift": args.auto_shift,
|
||||
"pitch_shift": 0,
|
||||
}
|
||||
if all(meta_paths):
|
||||
extra_args.update(
|
||||
{
|
||||
"prompt_metadata_path": str(Path(args.prompt_metadata_path).expanduser().resolve()),
|
||||
"target_metadata_path": str(Path(args.target_metadata_path).expanduser().resolve()),
|
||||
"audio_path": str(Path(args.audio_path).expanduser().resolve()),
|
||||
}
|
||||
)
|
||||
content = [{"type": "text", "text": "soulx-singer"}]
|
||||
else:
|
||||
prompt_path = Path(args.prompt_audio).expanduser().resolve()
|
||||
if not prompt_path.is_file():
|
||||
print(f"ERROR: prompt audio not found: {prompt_path}", file=sys.stderr)
|
||||
return 2
|
||||
extra_args["prompt_audio"] = str(prompt_path)
|
||||
extra_args["target_audio"] = str(Path(args.target_audio).expanduser().resolve())
|
||||
if args.preprocess_weights_dir:
|
||||
extra_args["preprocess_weights_dir"] = str(Path(args.preprocess_weights_dir).expanduser().resolve())
|
||||
content = [
|
||||
{"type": "text", "text": "soulx-singer"},
|
||||
{
|
||||
"type": "input_audio",
|
||||
"input_audio": {"data": _audio_to_data_url(prompt_path), "format": "mp3"},
|
||||
},
|
||||
]
|
||||
if not args.svc:
|
||||
extra_args["language"] = args.language
|
||||
extra_args["control"] = args.control
|
||||
|
||||
payload = {
|
||||
"model": args.model,
|
||||
"modalities": ["audio"],
|
||||
"messages": [{"role": "user", "content": content}],
|
||||
"num_inference_steps": args.num_inference_steps,
|
||||
"guidance_scale": args.guidance_scale,
|
||||
"extra_args": extra_args,
|
||||
}
|
||||
if args.seed is not None:
|
||||
payload["seed"] = args.seed
|
||||
|
||||
print(f"POST http://localhost:{args.port}/v1/chat/completions")
|
||||
response = requests.post(
|
||||
f"http://localhost:{args.port}/v1/chat/completions",
|
||||
headers={"Content-Type": "application/json"},
|
||||
json=payload,
|
||||
timeout=1800,
|
||||
)
|
||||
response.raise_for_status()
|
||||
audio, sample_rate = _decode_audio_from_response(response.json())
|
||||
_save_wav(audio, Path(args.output), sample_rate)
|
||||
duration = audio.shape[-1] / sample_rate
|
||||
print(f"saved {args.output} sr={sample_rate}Hz duration={duration:.2f}s")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,39 @@
|
||||
#!/bin/bash
|
||||
# Launch vLLM-Omni server for SoulX-Singer (single-stage DiT, preprocess inline).
|
||||
#
|
||||
# Usage:
|
||||
# MODEL=/path/to/SoulX-Singer PREPROCESS=/path/to/Preprocess \
|
||||
#
|
||||
# Audio paths in client extra_args must be readable on the server host.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../../../.." && pwd)"
|
||||
|
||||
MODEL="${MODEL:-Soul-AILab/SoulX-Singer}"
|
||||
MODE="${MODE:-svs}"
|
||||
PORT="${PORT:-8192}"
|
||||
GPUS="${GPUS:-0}"
|
||||
|
||||
if [[ "$MODE" == "svc" ]]; then
|
||||
DEPLOY_CONFIG="${DEPLOY_CONFIG:-$REPO_ROOT/vllm_omni/deploy/soulxsinger_svc.yaml}"
|
||||
else
|
||||
DEPLOY_CONFIG="${DEPLOY_CONFIG:-$REPO_ROOT/vllm_omni/deploy/soulxsinger_svs.yaml}"
|
||||
fi
|
||||
|
||||
echo "Starting SoulX-Singer server"
|
||||
echo " MODEL=$MODEL"
|
||||
echo " MODE=$MODE"
|
||||
echo " PORT=$PORT"
|
||||
echo " DEPLOY_CONFIG=$DEPLOY_CONFIG"
|
||||
echo " CUDA_VISIBLE_DEVICES=$GPUS"
|
||||
|
||||
CUDA_VISIBLE_DEVICES="$GPUS" \
|
||||
vllm serve "$MODEL" \
|
||||
--omni \
|
||||
--deploy-config "$DEPLOY_CONFIG" \
|
||||
--host 0.0.0.0 \
|
||||
--port "$PORT" \
|
||||
--trust-remote-code \
|
||||
--enforce-eager
|
||||
@@ -40,6 +40,7 @@ dev = [
|
||||
"pytest-asyncio>=0.21.0",
|
||||
"pytest-cov>=4.0.0",
|
||||
"pytest-mock>=3.10.0",
|
||||
"websockets>=15.0.1",
|
||||
"datasets>=2.14.0",
|
||||
"mypy==1.11.1",
|
||||
"pre-commit==4.0.1",
|
||||
@@ -118,6 +119,12 @@ docs = [
|
||||
"pydantic",
|
||||
]
|
||||
|
||||
soulx-svs = [
|
||||
"funasr>=1.0.0",
|
||||
"g2pM>=0.1.2.5",
|
||||
"g2p-en>=2.1.0",
|
||||
"ToJyutping>=3.2.0",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://github.com/vllm-project/vllm-omni"
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
# IndexTTS-2 for voice-cloned TTS on 1x GPU
|
||||
|
||||
## Summary
|
||||
|
||||
- Vendor: IndexTeam
|
||||
- Model: `IndexTeam/IndexTTS-2`
|
||||
- Task: Text-to-speech with voice cloning and optional emotion control
|
||||
- Mode: Online serving with the OpenAI-compatible `/v1/audio/speech` API
|
||||
- Maintainer: Community
|
||||
|
||||
## When to use this recipe
|
||||
|
||||
Use this recipe to serve IndexTTS-2 as a two-stage TTS system for
|
||||
voice-cloned speech synthesis. Each request provides synthesis text plus either
|
||||
a reference audio clip or an uploaded audio voice; the model returns 22.05 kHz
|
||||
mono speech. IndexTTS-2 also supports optional emotion conditioning through
|
||||
model-specific `extra_params`.
|
||||
|
||||
IndexTTS-2 is **not an async-chunk streaming model** in the bundled deployment
|
||||
configuration: the S2Mel flow-matching stage consumes the full mel-code sequence
|
||||
from the AR talker before it produces audio. The OpenAI-compatible endpoint can
|
||||
still be called with `stream=true` as an HTTP compatibility path, but that should
|
||||
not be interpreted as incremental model streaming.
|
||||
|
||||
## References
|
||||
|
||||
- Upstream model:
|
||||
[IndexTeam/IndexTTS-2 on Hugging Face](https://huggingface.co/IndexTeam/IndexTTS-2)
|
||||
- Online serving example:
|
||||
[`examples/online_serving/text_to_speech/README.md#indextts-2`](../../examples/online_serving/text_to_speech/README.md#indextts-2)
|
||||
- Offline inference example:
|
||||
[`examples/offline_inference/text_to_speech/README.md#indextts-2`](../../examples/offline_inference/text_to_speech/README.md#indextts-2)
|
||||
- OpenAI-compatible client:
|
||||
[`examples/online_serving/text_to_speech/indextts2/speech_client.py`](../../examples/online_serving/text_to_speech/indextts2/speech_client.py)
|
||||
- Deploy config:
|
||||
[`vllm_omni/deploy/indextts2.yaml`](../../vllm_omni/deploy/indextts2.yaml)
|
||||
|
||||
## Hardware Support
|
||||
|
||||
### GPU
|
||||
|
||||
### 1x L4 24GB or larger CUDA GPU
|
||||
|
||||
#### Environment
|
||||
|
||||
- OS: Linux
|
||||
- Python: 3.10+
|
||||
- Driver / runtime: NVIDIA CUDA environment
|
||||
- vLLM version: Match the repository requirements for your checkout
|
||||
- vLLM-Omni version or commit: Use the commit you are deploying from
|
||||
|
||||
#### Command
|
||||
|
||||
Start the server from the repository root:
|
||||
|
||||
```bash
|
||||
vllm serve IndexTeam/IndexTTS-2 --omni --trust-remote-code --port 8092
|
||||
```
|
||||
|
||||
The bundled deployment config is selected automatically for the `indextts2`
|
||||
model type. To make the config choice explicit, pass it directly:
|
||||
|
||||
```bash
|
||||
vllm serve IndexTeam/IndexTTS-2 --omni --trust-remote-code --port 8092 \
|
||||
--deploy-config vllm_omni/deploy/indextts2.yaml
|
||||
```
|
||||
|
||||
#### Verification
|
||||
|
||||
Run the bundled client. Local audio paths passed to this client are converted to
|
||||
base64 data URLs before they are sent to the HTTP API:
|
||||
|
||||
```bash
|
||||
python examples/online_serving/text_to_speech/indextts2/speech_client.py \
|
||||
--api-base http://localhost:8092 \
|
||||
--text "你好,这是 IndexTTS-2 语音合成测试。" \
|
||||
--ref-audio /path/to/reference.wav \
|
||||
--output indextts2.wav
|
||||
```
|
||||
|
||||
For a raw HTTP request, pass `ref_audio` as an HTTP(S) URL, a base64 data URL,
|
||||
or a `file://` URI allowed by the server's `--allowed-local-media-path`. Do not
|
||||
pass an arbitrary local filesystem path directly in JSON unless your client
|
||||
converts it first:
|
||||
|
||||
```bash
|
||||
curl http://localhost:8092/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "IndexTeam/IndexTTS-2",
|
||||
"input": "你好,这是 IndexTTS-2 语音合成测试。",
|
||||
"response_format": "wav",
|
||||
"ref_audio": "data:audio/wav;base64,<BASE64_ENCODED_AUDIO>"
|
||||
}' \
|
||||
--output indextts2.wav
|
||||
```
|
||||
|
||||
Optional emotion conditioning is passed through `extra_params`. Use one emotion
|
||||
source mode at a time:
|
||||
|
||||
```bash
|
||||
curl http://localhost:8092/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "IndexTeam/IndexTTS-2",
|
||||
"input": "今天心情很好!",
|
||||
"response_format": "wav",
|
||||
"ref_audio": "data:audio/wav;base64,<BASE64_ENCODED_REFERENCE_AUDIO>",
|
||||
"extra_params": {
|
||||
"emo_vector": [1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
|
||||
"emo_alpha": 0.8
|
||||
}
|
||||
}' \
|
||||
--output indextts2_happy.wav
|
||||
```
|
||||
|
||||
#### Notes
|
||||
|
||||
- Hardware scope: the default config is a single-GPU CUDA deployment. Stage 0
|
||||
and Stage 1 share GPU 0 by default.
|
||||
- Audio output: 22.05 kHz mono WAV.
|
||||
- Voice cloning: `ref_audio` is required for the documented raw request path.
|
||||
Alternatively, pass `voice` only when it names an uploaded audio voice from
|
||||
`/v1/audio/voices`; IndexTTS-2 does not provide a built-in text-only preset
|
||||
voice in this serving recipe.
|
||||
- Reference audio format: use an HTTP(S) URL, a base64 data URL, or a `file://`
|
||||
URI allowed by the server's `--allowed-local-media-path` for raw API calls.
|
||||
The bundled Python client accepts local paths and converts them to base64 data
|
||||
URLs.
|
||||
- Emotion control: `extra_params.emo_audio`, `extra_params.emo_vector`, and
|
||||
`extra_params.use_emo_text` are alternative conditioning modes. Official
|
||||
precedence is `use_emo_text` > `emo_vector` > `emo_audio` > same emotion as
|
||||
the speaker reference. The `emo_vector` order is `[happy, angry, sad, afraid,
|
||||
disgusted, melancholic, surprised, calm]`.
|
||||
- Streaming limitation: `async_chunk` is disabled in
|
||||
`vllm_omni/deploy/indextts2.yaml`; S2Mel requires the full mel-code sequence
|
||||
before audio generation.
|
||||
- First request can be slower because reference encoders and auxiliary models
|
||||
are lazily loaded and cached.
|
||||
@@ -5,7 +5,7 @@
|
||||
## Summary
|
||||
|
||||
- Vendor: JD (Joy Future Academy)
|
||||
- Model: [`jdopensource/JoyAI-VL-Interaction-Preview`](https://huggingface.co/jdopensource/JoyAI-VL-Interaction-Preview) (8B, vanilla Qwen3-VL weights)
|
||||
- Model: [`jdopensource/JoyAI-VL-Interaction-Preview`](https://huggingface.co/jdopensource/JoyAI-VL-Interaction-Preview) (8B, Qwen3-VL architecture with weights retrained for streaming interaction)
|
||||
- Task: Per-tick proactive interaction over a live video stream — the model decides
|
||||
on its own each second to speak, stay silent, or delegate a hard question
|
||||
- Mode: Online serving — an OpenAI-compatible interaction orchestrator in front of
|
||||
@@ -25,7 +25,8 @@ framework internals and how to add another full-duplex model, see
|
||||
- OS: Linux
|
||||
- Python: 3.10+
|
||||
- Hardware: 1x GPU. The default `T_s=100` frame window + `--max-model-len 131072` wants
|
||||
≈48GB+; for ~24GB, lower `--chunk-frames`, `--max-model-len`, and the image limit together
|
||||
≈48GB+; for ~24GB, lower `--chunk-frames`, `--max-model-len`, and the image limit together,
|
||||
and/or load the weights in fp8 (see "Smaller GPU: fp8 weights" below — ~9.9 GiB vs 16.8 GiB)
|
||||
- vLLM / vLLM-Omni: versions from your current checkout
|
||||
|
||||
## Start server
|
||||
@@ -33,7 +34,7 @@ framework internals and how to add another full-duplex model, see
|
||||
From repository root:
|
||||
|
||||
```bash
|
||||
# 1. Serve the model (plain `vllm serve`, NOT --omni; it is vanilla Qwen3-VL).
|
||||
# 1. Serve the model (plain `vllm serve`, NOT --omni; it uses the Qwen3-VL architecture).
|
||||
# The image limit must cover the short-term frame window (chunk_frames, default 100 = T_s);
|
||||
# prefix caching keeps the accumulating window cheap. Lower both for smaller GPUs.
|
||||
vllm serve jdopensource/JoyAI-VL-Interaction-Preview \
|
||||
@@ -46,6 +47,26 @@ python -m vllm_omni.experimental.fullduplex.joyvl.serving.server --port 8070 \
|
||||
--main-backend-url http://127.0.0.1:8061/v1 --main-model JoyAI-VL-Interaction-Preview
|
||||
```
|
||||
|
||||
### Smaller GPU: fp8 weights
|
||||
|
||||
Add `--quantization fp8` to the step-1 `vllm serve` to load the weights in fp8 online
|
||||
from the bf16 checkpoint (no separate quantized checkpoint needed; uses vLLM's generic
|
||||
fp8 path, since the model is a standard Qwen3-VL VLM):
|
||||
|
||||
```bash
|
||||
vllm serve jdopensource/JoyAI-VL-Interaction-Preview \
|
||||
--served-model-name JoyAI-VL-Interaction-Preview --port 8061 \
|
||||
--quantization fp8 --max-model-len 131072 --enable-prefix-caching \
|
||||
--limit-mm-per-prompt '{"image":256,"video":1}'
|
||||
```
|
||||
|
||||
Measured here: model weights **9.9 GiB** vs **16.8 GiB** bf16 (**−41%**), which lets the 8B
|
||||
fit a much smaller card. This is a **memory** option, not a speed one: at one request per
|
||||
tick (the streaming regime) fp8 was **not** faster (≈19% slower in our test) because
|
||||
single-stream decode is memory-latency-bound and pays fp8's dynamic-scale overhead; its
|
||||
throughput gain needs batching. Decisions stay coherent, but fp8 can shift the
|
||||
speak/silence boundary, so for timing-critical alerting validate output against bf16 first.
|
||||
|
||||
Optional one-shot launch (model + orchestrator + JD webui + ASR/TTS/background,
|
||||
all env-configurable). The JD webui frontend is external — set `WEBUI_DIR` to point at it:
|
||||
|
||||
@@ -176,14 +197,22 @@ the audio-track caveat.
|
||||
|
||||
## Notes
|
||||
|
||||
- `--omni` is **not** used: the model is standard Qwen3-VL, so stock `vllm serve`
|
||||
runs the forward pass; this recipe only adds the interaction/serving layer.
|
||||
- `--omni` is **not** used: the model keeps the Qwen3-VL architecture (only the
|
||||
weights are retrained), so stock `vllm serve` runs the forward pass; this recipe
|
||||
only adds the interaction/serving layer.
|
||||
- On a host without `nvcc` / `ninja`, `vllm serve` of the 8B can crash engine-core in the
|
||||
FlashInfer sampler JIT (`FileNotFoundError: 'ninja'`) during `profile_run`. Set
|
||||
`VLLM_USE_FLASHINFER_SAMPLER=0` (or install `ninja`) to work around it.
|
||||
- `force_silence_before_query` is on by default — the model stays silent until an
|
||||
instruction arrives; give a standing task (e.g. "translate the on-screen text")
|
||||
to arm proactive output.
|
||||
- **Frame resolution is the main latency knob.** The system/memory prefix is prefix-cached,
|
||||
so per-tick *new* compute is dominated by the new frame's vision tokens. Measured on the
|
||||
8B: ~256×192 frames hit Qwen3-VL's min-pixel floor (~72 vision tokens, ~17 ms/tick) vs
|
||||
~302 tokens / ~38 ms at 640×480 — about 2× cheaper. Downsample frames to ~256×192 for the
|
||||
tightest latency / highest concurrency; one GPU then sustains ~150–180 concurrent 1 fps
|
||||
streams with p95 < 200 ms. Per-tick latency is already far inside the 1 fps budget, so
|
||||
serving is rarely the bottleneck — resolution is the lever if you need more headroom.
|
||||
- Speech is external and pluggable: point `ASR_URL` / `TTS_URL` at the bridges in
|
||||
`examples/online_serving/joyvl_interaction/bridges/` or any compatible service.
|
||||
- The decision prompts, sampling, and 3-tier summary memory (`T_s=100`, mid→long every
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
# Wan2.2 VACE-Fun Diffusers
|
||||
|
||||
> All-in-one video creation and editing with Wan2.2 VACE-Fun A14B in diffusers format.
|
||||
|
||||
## Summary
|
||||
|
||||
- Vendor: Pyros13
|
||||
- Model: `Pyros13/Wan2.2-VACE-Fun-A14B-Diffusers`
|
||||
- Base model: `alibaba-pai/Wan2.2-VACE-Fun-A14B`
|
||||
- Task: VACE video generation and editing (T2V / I2V / R2V / V2V / inpainting)
|
||||
- Mode: Offline inference and OpenAI-compatible online serving
|
||||
- Maintainer: Community
|
||||
|
||||
## When To Use This Recipe
|
||||
|
||||
Use this recipe to run Wan2.2-VACE-Fun-A14B through the standard diffusers-style
|
||||
`Wan22VACEPipeline` path in vLLM-Omni. The checkpoint is a diffusers conversion of
|
||||
`alibaba-pai/Wan2.2-VACE-Fun-A14B`, with `model_index.json`, `transformer/`,
|
||||
`transformer_2/`, `text_encoder/`, `tokenizer/`, `vae/`, and `scheduler/`
|
||||
subfolders. Because the model already uses the expected diffusers layout, vLLM-Omni
|
||||
does not need any original-format key remapping or `.pth` component loaders.
|
||||
|
||||
The model is a two-expert architecture. The high-noise `transformer` and low-noise
|
||||
`transformer_2` are switched by the Wan2.2 boundary during denoising.
|
||||
|
||||
## References
|
||||
|
||||
- Model card: <https://huggingface.co/Pyros13/Wan2.2-VACE-Fun-A14B-Diffusers>
|
||||
- Base model card: <https://huggingface.co/alibaba-pai/Wan2.2-VACE-Fun-A14B>
|
||||
- Related example: [`examples/offline_inference/vace/vace_video_generation.md`](../../examples/offline_inference/vace/vace_video_generation.md)
|
||||
- Related issue: [vllm-project/vllm-omni#4206](https://github.com/vllm-project/vllm-omni/issues/4206)
|
||||
- PR: [vllm-project/vllm-omni#4667](https://github.com/vllm-project/vllm-omni/pull/4667)
|
||||
|
||||
## Offline Inference
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/vace/vace_video_generation.py \
|
||||
--model Pyros13/Wan2.2-VACE-Fun-A14B-Diffusers \
|
||||
--mode i2v --image ./i2v_input.JPG \
|
||||
--prompt "Summer beach vacation style, a white cat wearing sunglasses sits on a surfboard" \
|
||||
--height 480 --width 832 --num-frames 81 --num-inference-steps 30 \
|
||||
--seed 42 --guidance-scale 5.0 \
|
||||
--vae-use-tiling --enforce-eager \
|
||||
--output vace_fun_i2v.mp4
|
||||
```
|
||||
|
||||
See [`examples/offline_inference/vace/vace_video_generation.md`](../../examples/offline_inference/vace/vace_video_generation.md)
|
||||
for other VACE modes and their input flags.
|
||||
|
||||
## Online Serving
|
||||
|
||||
### Server
|
||||
|
||||
```bash
|
||||
vllm serve Pyros13/Wan2.2-VACE-Fun-A14B-Diffusers --omni \
|
||||
--model-class-name Wan22VACEPipeline \
|
||||
--vae-use-tiling \
|
||||
--enforce-eager \
|
||||
--port 8091
|
||||
```
|
||||
|
||||
For multi-GPU sequence-parallel serving, add the same parallelism flags you would
|
||||
use offline, for example `--ulysses-degree 4`. The explicit `--model-class-name`
|
||||
is optional for this diffusers checkpoint, but keeping it in examples makes the
|
||||
selected pipeline obvious.
|
||||
|
||||
### Client
|
||||
|
||||
```bash
|
||||
no_proxy=127.0.0.1 \
|
||||
curl -X POST http://127.0.0.1:8091/v1/videos/sync \
|
||||
-F "prompt=Summer beach vacation style, a white cat wearing sunglasses sits on a surfboard" \
|
||||
-F "input_reference=@/absolute/path/to/i2v_input.JPG" \
|
||||
-F "width=832" \
|
||||
-F "height=480" \
|
||||
-F "num_frames=81" \
|
||||
-F "fps=16" \
|
||||
-F "num_inference_steps=30" \
|
||||
-F "guidance_scale=5.0" \
|
||||
--output vace_fun_i2v_serve.mp4
|
||||
```
|
||||
|
||||
For video-to-video or inpainting-style requests, upload the reference video with
|
||||
`input_reference=@/absolute/path/to/input.mp4` or use `video_reference` with a URL
|
||||
or JSON-safe data URI.
|
||||
|
||||
## Cache-DiT / TaylorSeer Notes
|
||||
|
||||
`Wan22VACEPipeline` uses the shared Wan2.2 Cache-DiT enabler for both the
|
||||
high-noise and low-noise transformers. The adapter intentionally wraps only the
|
||||
main denoising `blocks`. VACE `vace_blocks` form a conditioning branch: each step
|
||||
they combine the current latent with the VACE context to produce hints that are
|
||||
then injected into selected main blocks. Recomputing that branch preserves the
|
||||
control signal; caching the main backbone still gives the acceleration target.
|
||||
|
||||
## Validation
|
||||
|
||||
Validated on NVIDIA B300 with `mkt_part_000.mp4`, 736x1280, 61 frames, 30
|
||||
inference steps, `seed=1`, and `boundary_ratio=0.875`.
|
||||
|
||||
The Pyros13 diffusers checkpoint produced a byte-identical MP4 to the
|
||||
`alibaba-pai/Wan2.2-VACE-Fun-A14B` original-format checkpoint under the same
|
||||
vLLM-Omni pipeline and inputs:
|
||||
|
||||
- SHA256: `2e2db74a612a009b0d6ad6d739e039883f0de84ce5cdf2676324daf78fb05ef8`
|
||||
- Frame shape: 61 x 736 x 1280
|
||||
- MAE / RMSE / max pixel diff: 0 / 0 / 0
|
||||
- PSNR: infinite
|
||||
- Mean SSIM: 1.0
|
||||
@@ -43,6 +43,7 @@ recipes/
|
||||
| [`Helios/Helios.md`](./Helios/Helios.md) | Text-to-video, image-to-video, and video-to-video generation | 1x NVIDIA H20 |
|
||||
| [`inclusionAI/Ming-flash-omni-2.0.md`](./inclusionAI/Ming-flash-omni-2.0.md) | Online serving for multimodal chat + standalone TTS | 4x H100 / 1x H100 80GB |
|
||||
| [`inclusionAI/Ming-omni-tts.md`](./inclusionAI/Ming-omni-tts.md) | Offline + online dense Ming TTS/audio generation | 1x H100 80GB / 1x AMD MI300X (ROCm 7.2) |
|
||||
| [`IndexTeam/IndexTTS-2.md`](./IndexTeam/IndexTTS-2.md) | Online serving for voice-cloned TTS with optional emotion control | 1x L4 24GB or larger CUDA GPU |
|
||||
| [`LTX/LTX-2.md`](./LTX/LTX-2.md) | Text-to-video and image-to-video serving | 1x H200 141GB |
|
||||
| [`LTX/LTX-2.3.md`](./LTX/LTX-2.3.md) | Text-to-video with audio generation (22B) | 1x GPU (96GB VRAM) |
|
||||
| [`mistralai/Voxtral-TTS.md`](./mistralai/Voxtral-TTS.md) | Online serving for TTS | 1x RTX 4090 24GB |
|
||||
@@ -57,10 +58,12 @@ recipes/
|
||||
| [`Qwen/Qwen3-Omni.md`](./Qwen/Qwen3-Omni.md) | Online serving for multimodal chat | 1x A100 80GB |
|
||||
| [`Qwen/Qwen3-TTS.md`](./Qwen/Qwen3-TTS.md) | Text-to-speech serving (CustomVoice / VoiceDesign / Base) | 1x H100/A100 80GB |
|
||||
| [`SenseNova/SenseNova-U1.md`](./SenseNova/SenseNova-U1.md) | Unified image generation and understanding | 1x H200 (144GB) |
|
||||
| [`StabilityAI/Stable-Audio-Open.md`](./StabilityAI/Stable-Audio-Open.md) | Offline + online text-to-audio generation (Stable Audio Open) | 1x RTX 4090 24GB |
|
||||
| [`Tencent/Covo-Audio-Chat.md`](./Tencent/Covo-Audio-Chat.md) | Online serving for audio chat | 1x A100 80GB |
|
||||
| [`Tencent/HunyuanImage-3.0-Instruct.md`](./Tencent/HunyuanImage-3.0-Instruct.md) | DiT-only text-to-image serving and benchmark, including ModelOpt mixed FP8/NVFP4 | 4x H100/H800 80GB / 2x B200 |
|
||||
| [`Wan-AI/Wan2.2-I2V.md`](./Wan-AI/Wan2.2-I2V.md) | Image-to-video serving (Wan2.2 14B) | 8x Ascend NPU (A2/A3) |
|
||||
| [`Wan-AI/Wan2.2-S2V.md`](./Wan-AI/Wan2.2-S2V.md) | Speech-to-video serving (Wan2.2 14B) | 2x A100/H100 80GB |
|
||||
| [`Wan-AI/Wan2.1-VACE.md`](./Wan-AI/Wan2.1-VACE.md) | Unified T2V, I2V, V2LF, FLF2V, inpaint, and R2V | 1x RTX 5090 (1.3B) / 1x L40S 48GB with layerwise offload (14B) |
|
||||
| [`StabilityAI/Stable-Diffusion-3.5.md`](./StabilityAI/Stable-Diffusion-3.5.md) | Text-to-image serving (SD 3.5-medium and SD 3.5-large) | 1x RTX A6000 48GB |
|
||||
| [`zai-org/GLM-TTS.md`](./zai-org/GLM-TTS.md) | Online serving for Chinese/English zero-shot voice-cloned TTS | 1x A40 48GB |
|
||||
| [`GLM/GLM-Image.md`](./GLM/GLM-Image.md) | Online serving for image generation | 1x A800 80GB / 2x A800 80GB |
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
# Stable-Audio-Open Text-To-Audio Generation on 1x GPU
|
||||
|
||||
> Text-to-audio recipe for Stable Audio Open with offline inference and
|
||||
> OpenAI-compatible online serving on 1x RTX 4090 24GB.
|
||||
|
||||
## Summary
|
||||
|
||||
- Vendor: Stability AI
|
||||
- Model: `stabilityai/stable-audio-open-1.0`
|
||||
- Task: Text-to-audio generation
|
||||
- Mode: Offline inference and online serving
|
||||
- Maintainer: Community
|
||||
|
||||
## When to use this recipe
|
||||
|
||||
Use this recipe when you want to run Stable Audio Open on a single RTX 4090
|
||||
24GB GPU for music or sound-effect generation. The recipe covers a 10-second
|
||||
offline validation sample with TeaCache and online serving through the
|
||||
`/v1/audio/generate` endpoint.
|
||||
|
||||
## References
|
||||
|
||||
- Model: <https://huggingface.co/stabilityai/stable-audio-open-1.0>
|
||||
- Offline example:
|
||||
[`examples/offline_inference/text_to_audio`](../../examples/offline_inference/text_to_audio)
|
||||
- Online example:
|
||||
[`examples/online_serving/stable_audio`](../../examples/online_serving/stable_audio)
|
||||
- Audio generation API:
|
||||
[`docs/serving/audio_generate_api.md`](../../docs/serving/audio_generate_api.md)
|
||||
- Related issue:
|
||||
[#2645](https://github.com/vllm-project/vllm-omni/issues/2645)
|
||||
|
||||
## Hardware Support
|
||||
|
||||
### GPU
|
||||
|
||||
#### 1x RTX 4090 24GB
|
||||
|
||||
##### Environment
|
||||
|
||||
- OS: Ubuntu 22.04.5
|
||||
- Python: 3.12
|
||||
- GPU: NVIDIA GeForce RTX 4090, 24564 MiB VRAM
|
||||
- Driver / runtime: NVIDIA driver 595.80, CUDA-capable runtime matching the
|
||||
repository build
|
||||
- vLLM version: 0.22.0
|
||||
- vLLM-Omni version: source checkout
|
||||
- PyTorch: 2.11.0+cu130
|
||||
- Model path used in the commands below: `/path/to/stable-audio-open-1.0`
|
||||
|
||||
Stable Audio Open is a gated Hugging Face model. Accept the model license on
|
||||
the Hugging Face model card before downloading the checkpoint.
|
||||
|
||||
```bash
|
||||
hf auth login
|
||||
|
||||
hf download stabilityai/stable-audio-open-1.0 \
|
||||
--local-dir /path/to/stable-audio-open-1.0
|
||||
```
|
||||
|
||||
##### Commands
|
||||
|
||||
Run a 10-second offline validation sample from the repository root:
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_audio/text_to_audio.py \
|
||||
--model /path/to/stable-audio-open-1.0 \
|
||||
--prompt "A gentle piano melody with soft room ambience" \
|
||||
--negative-prompt "Low quality, distorted, noisy" \
|
||||
--seed 42 \
|
||||
--guidance-scale 7.0 \
|
||||
--audio-length 10.0 \
|
||||
--num-inference-steps 50 \
|
||||
--cache-backend tea_cache \
|
||||
--output examples/offline_inference/text_to_audio/stable_audio_10s.wav
|
||||
```
|
||||
|
||||
Start the online serving endpoint:
|
||||
|
||||
```bash
|
||||
vllm-omni serve /path/to/stable-audio-open-1.0 \
|
||||
--host 0.0.0.0 \
|
||||
--port 8091 \
|
||||
--gpu-memory-utilization 0.9 \
|
||||
--trust-remote-code \
|
||||
--enforce-eager \
|
||||
--omni
|
||||
```
|
||||
|
||||
Generate a 10-second WAV file from the repository root in another terminal:
|
||||
|
||||
```bash
|
||||
curl http://localhost:8091/health
|
||||
|
||||
curl -X POST http://localhost:8091/v1/audio/generate \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input": "A gentle piano melody with soft room ambience",
|
||||
"audio_length": 10.0,
|
||||
"num_inference_steps": 50,
|
||||
"guidance_scale": 7.0,
|
||||
"negative_prompt": "Low quality, distorted, noisy",
|
||||
"seed": 42,
|
||||
"response_format": "wav"
|
||||
}' \
|
||||
--output examples/online_serving/stable_audio/piano_10s.wav
|
||||
```
|
||||
|
||||
##### Verification
|
||||
|
||||
Check that:
|
||||
|
||||
- The offline command writes a valid WAV file.
|
||||
- The server responds on `http://localhost:8091/health`.
|
||||
- The online request writes a valid WAV file.
|
||||
- The generated audio sample rate is 44.1 kHz.
|
||||
- The generated duration is approximately 10 seconds.
|
||||
- Peak sampled GPU memory is within the 24GB RTX 4090 budget. In the validated
|
||||
run, offline and online generation each peaked at about 12.6 GiB.
|
||||
|
||||
Validate the offline outputs:
|
||||
|
||||
```bash
|
||||
ls -lh examples/offline_inference/text_to_audio/stable_audio_10s.wav
|
||||
|
||||
python - <<'PY'
|
||||
import soundfile as sf
|
||||
|
||||
path = "examples/offline_inference/text_to_audio/stable_audio_10s.wav"
|
||||
audio, sample_rate = sf.read(path)
|
||||
print("sample_rate:", sample_rate)
|
||||
print("shape:", audio.shape)
|
||||
print("duration:", len(audio) / sample_rate)
|
||||
PY
|
||||
```
|
||||
|
||||
Validate the online output:
|
||||
|
||||
```bash
|
||||
ls -lh examples/online_serving/stable_audio/piano_10s.wav
|
||||
|
||||
python - <<'PY'
|
||||
import soundfile as sf
|
||||
|
||||
path = "examples/online_serving/stable_audio/piano_10s.wav"
|
||||
audio, sample_rate = sf.read(path)
|
||||
print("sample_rate:", sample_rate)
|
||||
print("shape:", audio.shape)
|
||||
print("duration:", len(audio) / sample_rate)
|
||||
PY
|
||||
```
|
||||
|
||||
##### Notes
|
||||
|
||||
- `stable-audio-open-1.0` can generate up to about 47 seconds of 44.1 kHz
|
||||
stereo audio. This recipe validates 10-second WAV outputs.
|
||||
- `--cache-backend tea_cache` is supported and was used for the 10-second
|
||||
offline validation command.
|
||||
- The model is gated on Hugging Face and requires license acceptance before
|
||||
download.
|
||||
- If online serving fails while importing `torchaudio`, make sure the
|
||||
`torchaudio` wheel matches the installed PyTorch and CUDA build. The
|
||||
validated environment used `torch==2.11.0+cu130` and
|
||||
`torchaudio==2.11.0+cu130`.
|
||||
- The `NIXL is not available`, `GLOO_SOCKET_IFNAME`, and `torchsde` boundary
|
||||
warnings observed during validation did not prevent successful generation.
|
||||
- This recipe was validated on one RTX 4090 24GB GPU only. Other GPU counts,
|
||||
ROCm, XPU, and NPU setups are not covered here.
|
||||
- Long generations, higher inference-step counts, and non-WAV response formats
|
||||
were not benchmarked in this recipe.
|
||||
@@ -0,0 +1,147 @@
|
||||
# Wan2.1 VACE Unified Video Generation
|
||||
|
||||
## Summary
|
||||
|
||||
- Vendor: Wan-AI
|
||||
- Models: `Wan2.1-VACE-1.3B-diffusers`, `Wan2.1-VACE-14B-diffusers`
|
||||
- Tasks: T2V, I2V, V2LF, FLF2V, inpainting, and R2V
|
||||
- Mode: offline inference through the shared task examples
|
||||
|
||||
The shared examples construct VACE's pipeline-native conditioning data from the
|
||||
provided media inputs. They do not require a mode parameter or a model-specific
|
||||
example script.
|
||||
|
||||
## References
|
||||
|
||||
- [VACE project](https://github.com/ali-vilab/VACE)
|
||||
- [Hugging Face 14B checkpoint](https://huggingface.co/Wan-AI/Wan2.1-VACE-14B-diffusers)
|
||||
- [ModelScope 14B checkpoint](https://modelscope.cn/models/Wan-AI/Wan2.1-VACE-14B-diffusers)
|
||||
- [Shared text-to-video example](../../examples/offline_inference/text_to_video/text_to_video.md)
|
||||
- [Shared image-to-video example](../../examples/offline_inference/image_to_video/README.md)
|
||||
- [Migration RFC #4539](https://github.com/vllm-project/vllm-omni/issues/4539)
|
||||
|
||||
## Hardware Support
|
||||
|
||||
The 1.3B checkpoint was validated on an RTX 5090; the 81-frame T2V command below
|
||||
used approximately 20 GiB of peak reserved GPU memory with VAE tiling. The 14B
|
||||
checkpoint requires approximately 70 GiB of disk space and was validated on a
|
||||
single 48 GiB L40S with VAE tiling and layerwise offload. Start with the smoke
|
||||
configuration before attempting an 81-frame run.
|
||||
|
||||
## Setup
|
||||
|
||||
The public examples use the Hugging Face model ID by default:
|
||||
|
||||
```bash
|
||||
MODEL="${MODEL:-Wan-AI/Wan2.1-VACE-14B-diffusers}"
|
||||
ASSET_DIR="${ASSET_DIR:-$HOME/datasets/vace}"
|
||||
OUT="${OUT:-$PWD/vace-results/14b}"
|
||||
mkdir -p "$ASSET_DIR" "$OUT"
|
||||
```
|
||||
|
||||
For environments where Hugging Face downloads are unavailable, download the
|
||||
same checkpoint from ModelScope and point `MODEL` at the local directory:
|
||||
|
||||
```bash
|
||||
MODEL="$HOME/models/Wan2.1-VACE-14B-diffusers"
|
||||
modelscope download --model Wan-AI/Wan2.1-VACE-14B-diffusers --local_dir "$MODEL"
|
||||
```
|
||||
|
||||
Download the public conditioning images and create a mask matching the output
|
||||
size. White pixels are regenerated and black pixels are preserved.
|
||||
|
||||
```bash
|
||||
wget -O "$ASSET_DIR/astronaut.jpg" \
|
||||
https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/astronaut.jpg
|
||||
wget -O "$ASSET_DIR/vace_first_frame.png" \
|
||||
https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/flf2v_input_first_frame.png
|
||||
wget -O "$ASSET_DIR/vace_last_frame.png" \
|
||||
https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/flf2v_input_last_frame.png
|
||||
|
||||
python - "$ASSET_DIR/vace_center_mask.png" <<'PY'
|
||||
import sys
|
||||
from PIL import Image
|
||||
|
||||
mask = Image.new("L", (832, 480), 0)
|
||||
mask.paste(255, (336, 0, 496, 480))
|
||||
mask.save(sys.argv[1])
|
||||
PY
|
||||
|
||||
ASTRONAUT="$ASSET_DIR/astronaut.jpg"
|
||||
FIRST="$ASSET_DIR/vace_first_frame.png"
|
||||
LAST="$ASSET_DIR/vace_last_frame.png"
|
||||
MASK="$ASSET_DIR/vace_center_mask.png"
|
||||
```
|
||||
|
||||
## Commands
|
||||
|
||||
The commands below reproduce the original VACE examples with `81 frames / 30
|
||||
steps`. Most tasks use `480x832`; FLF2V uses `512x512`. For a smoke test,
|
||||
replace the frame and step counts with `5 frames / 2 steps`.
|
||||
|
||||
```bash
|
||||
# T2V
|
||||
python examples/offline_inference/text_to_video/text_to_video.py \
|
||||
--model "$MODEL" \
|
||||
--prompt "A sleek, humanoid robot stands in a vast warehouse filled with neatly stacked cardboard boxes on industrial shelves." \
|
||||
--seed 0 \
|
||||
--height 480 --width 832 --num-frames 81 --num-inference-steps 30 \
|
||||
--guidance-scale 5.0 --flow-shift 5.0 --vae-use-tiling --enable-layerwise-offload \
|
||||
--output "$OUT/t2v.mp4"
|
||||
|
||||
# I2V
|
||||
python examples/offline_inference/image_to_video/image_to_video.py \
|
||||
--model "$MODEL" --image "$ASTRONAUT" \
|
||||
--prompt "An astronaut emerging from a cracked, otherworldly egg on the surface of the moon" \
|
||||
--seed 42 \
|
||||
--height 480 --width 832 --num-frames 81 --num-inference-steps 30 \
|
||||
--guidance-scale 5.0 --flow-shift 5.0 --vae-use-tiling --enable-layerwise-offload \
|
||||
--output "$OUT/i2v.mp4"
|
||||
|
||||
# V2LF
|
||||
python examples/offline_inference/image_to_video/image_to_video.py \
|
||||
--model "$MODEL" --last-image "$ASTRONAUT" \
|
||||
--prompt "An astronaut emerging from a cracked, otherworldly egg on the surface of the moon" \
|
||||
--seed 42 \
|
||||
--height 480 --width 832 --num-frames 81 --num-inference-steps 30 \
|
||||
--guidance-scale 5.0 --flow-shift 5.0 --vae-use-tiling --enable-layerwise-offload \
|
||||
--output "$OUT/v2lf.mp4"
|
||||
|
||||
# FLF2V
|
||||
python examples/offline_inference/image_to_video/image_to_video.py \
|
||||
--model "$MODEL" --image "$FIRST" --last-image "$LAST" \
|
||||
--prompt "CG animation style, a small blue bird takes off from a branch and lands on another branch" \
|
||||
--seed 42 --height 512 --width 512 --num-frames 81 --num-inference-steps 30 \
|
||||
--guidance-scale 5.0 --flow-shift 5.0 --vae-use-tiling --enable-layerwise-offload \
|
||||
--output "$OUT/flf2v.mp4"
|
||||
|
||||
# Inpaint
|
||||
python examples/offline_inference/image_to_video/image_to_video.py \
|
||||
--model "$MODEL" --image "$ASTRONAUT" --mask-image "$MASK" \
|
||||
--prompt "Shrek, the ogre, walks out of a building in a happy mood" \
|
||||
--seed 42 \
|
||||
--height 480 --width 832 --num-frames 81 --num-inference-steps 30 \
|
||||
--guidance-scale 5.0 --flow-shift 5.0 --vae-use-tiling --enable-layerwise-offload \
|
||||
--output "$OUT/inpaint.mp4"
|
||||
|
||||
# R2V; repeat --reference-image to add references
|
||||
python examples/offline_inference/image_to_video/image_to_video.py \
|
||||
--model "$MODEL" --reference-image "$ASTRONAUT" \
|
||||
--prompt "Camera slowly zooms out from the character walking in a garden" \
|
||||
--seed 42 \
|
||||
--height 480 --width 832 --num-frames 81 --num-inference-steps 30 \
|
||||
--guidance-scale 5.0 --flow-shift 5.0 --vae-use-tiling --enable-layerwise-offload \
|
||||
--output "$OUT/r2v.mp4"
|
||||
```
|
||||
|
||||
## Verification
|
||||
|
||||
```bash
|
||||
for video in "$OUT"/*.mp4; do
|
||||
ffprobe -v error -select_streams v:0 \
|
||||
-show_entries stream=codec_name,width,height,nb_frames,r_frame_rate,duration \
|
||||
-of default=noprint_wrappers=1 "$video"
|
||||
done
|
||||
```
|
||||
|
||||
For the 1.3B checkpoint, `--enable-layerwise-offload` is generally unnecessary.
|
||||
@@ -367,60 +367,58 @@ vllm serve nvidia/Cosmos3-Nano-Policy-DROID \
|
||||
|
||||
#### Command
|
||||
|
||||
```python
|
||||
# cosmos3_offline.py — run with: python cosmos3_offline.py
|
||||
import torch
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
Cosmos3 runs through the standard task examples; pass model-specific knobs via
|
||||
`--extra-body`. Guardrails are on by default — pass `"guardrails": false` for a
|
||||
quick local run (install `cosmos-guardrail` + accept the gated repo to enable them).
|
||||
|
||||
```bash
|
||||
# Text-to-image -> examples/offline_inference/text_to_image
|
||||
python examples/offline_inference/text_to_image/text_to_image.py \
|
||||
--model nvidia/Cosmos3-Nano \
|
||||
--prompt "A photorealistic red sports car at golden hour, cinematic lighting." \
|
||||
--negative-prompt "blurry, distorted, low quality" \
|
||||
--height 1024 --width 1024 --num-inference-steps 50 --guidance-scale 7.0 \
|
||||
--extra-body '{"flow_shift": 3.0, "guardrails": false}' \
|
||||
--output cosmos3_t2i.png
|
||||
|
||||
def main():
|
||||
omni = Omni(
|
||||
model="nvidia/Cosmos3-Nano",
|
||||
model_class_name="Cosmos3OmniDiffusersPipeline",
|
||||
trust_remote_code=True,
|
||||
enforce_eager=True,
|
||||
# Guardrails are disabled here for a quick local run; install
|
||||
# cosmos-guardrail + gated-repo access and drop this to enable them.
|
||||
model_config={"guardrails": False},
|
||||
)
|
||||
gen = torch.Generator(device="cpu").manual_seed(42)
|
||||
# Text-to-video -> examples/offline_inference/text_to_video
|
||||
python examples/offline_inference/text_to_video/text_to_video.py \
|
||||
--model nvidia/Cosmos3-Nano \
|
||||
--prompt "A robot arm is cleaning a plate in the kitchen." \
|
||||
--negative-prompt "blurry, distorted, low quality, jittery, deformed" \
|
||||
--height 720 --width 1280 --num-frames 189 --fps 24 \
|
||||
--num-inference-steps 35 --guidance-scale 6.0 \
|
||||
--extra-body '{"flow_shift": 10.0, "max_sequence_length": 4096, "guardrails": false,
|
||||
"use_resolution_template": false, "use_duration_template": false}' \
|
||||
--output cosmos3_t2v.mp4
|
||||
|
||||
# Text-to-image (modalities=["image"]). For T2V use modalities=["video"]
|
||||
# plus num_frames/fps; for I2V add multi_modal_data={"image": <PIL.Image>}.
|
||||
outputs = omni.generate(
|
||||
{
|
||||
"prompt": "A photorealistic red sports car at golden hour, cinematic lighting.",
|
||||
"negative_prompt": "blurry, distorted, low quality",
|
||||
"modalities": ["image"],
|
||||
},
|
||||
OmniDiffusionSamplingParams(
|
||||
height=1024, width=1024, generator=gen,
|
||||
guidance_scale=7.0, num_inference_steps=50, num_outputs_per_prompt=1,
|
||||
),
|
||||
)
|
||||
outputs[0].request_output.images[0].save("cosmos3_t2i.png")
|
||||
omni.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
# Image-to-video -> examples/offline_inference/image_to_video
|
||||
# (Cosmos3 bundles example frames under assets/; any RGB image works too.)
|
||||
python examples/offline_inference/image_to_video/image_to_video.py \
|
||||
--model nvidia/Cosmos3-Nano \
|
||||
--image /path/to/Cosmos3-Nano/assets/example_i2v_input.jpg \
|
||||
--prompt "The scene comes to life with smooth, natural motion." \
|
||||
--height 720 --width 1280 --num-frames 189 --fps 24 \
|
||||
--num-inference-steps 35 --guidance-scale 6.0 \
|
||||
--extra-body '{"flow_shift": 10.0, "max_sequence_length": 4096, "guardrails": false}' \
|
||||
--output cosmos3_i2v.mp4
|
||||
```
|
||||
|
||||
#### Verification
|
||||
|
||||
```bash
|
||||
python cosmos3_offline.py
|
||||
python -c "from PIL import Image; im=Image.open('cosmos3_t2i.png'); print('image', im.size, im.mode)"
|
||||
ffprobe -v error -show_entries stream=codec_type,nb_frames,width,height cosmos3_t2v.mp4
|
||||
```
|
||||
|
||||
#### Notes
|
||||
|
||||
- Same `Cosmos3OmniDiffusersPipeline` as online; mode is chosen by
|
||||
`prompt["modalities"]` (`["image"]` → T2I, `["video"]` → T2V) plus
|
||||
`num_frames`/`fps`, `multi_modal_data={"image": ...}` for I2V, and
|
||||
`multi_modal_data={"video": [<PIL frames>]}` or a video tensor/array for V2V.
|
||||
For video, frames are returned in `outputs[0].request_output.images` as an
|
||||
`(B, F, H, W, 3)` array.
|
||||
- The offline entry must be guarded by `if __name__ == "__main__":` — the engine
|
||||
spawns workers with the `spawn` start method.
|
||||
- A single `Cosmos3OmniDiffusersPipeline` serves every mode; the standard examples
|
||||
select it automatically from `model_index.json`. T2I is chosen by the
|
||||
`text_to_image` prompt builder (which marks `modalities=["image"]`); `text_to_video`
|
||||
defaults to T2V; `image_to_video` adds `multi_modal_data={"image": ...}` (I2V).
|
||||
V2V is served online (`/v1/videos/sync`).
|
||||
- Model-specific knobs (`flow_shift`, `max_sequence_length`, `condition_*`,
|
||||
`generate_sound`/`sound_duration`, `guardrails`, `action_*`, ...) are declared
|
||||
once in `vllm_omni/model_extras/cosmos3.py` and forwarded through `--extra-body`;
|
||||
unknown keys for the model are dropped.
|
||||
Binary file not shown.
@@ -0,0 +1,16 @@
|
||||
[
|
||||
{
|
||||
"index": "vocal_240_51240",
|
||||
"language": "Mandarin",
|
||||
"time": [
|
||||
240,
|
||||
51240
|
||||
],
|
||||
"duration": "0.21 0.18 0.34 0.38 0.26 0.22 0.50 0.20 0.33 0.13 0.60 0.22 0.38 0.52 0.30 0.12 1.82 0.98 0.20 0.26 0.38 0.36 0.58 0.54 0.26 0.50 1.46 3.03 0.24 0.24 0.29 0.25 0.20 0.14 0.80 0.22 0.54 0.30 0.24 0.16 0.58 0.28 0.38 1.73 0.79 0.24 0.30 0.34 0.30 0.30 0.34 0.21 0.21 0.36 0.34 0.23 1.85 1.90 0.23 0.39 0.68 0.50 0.31 0.43 0.76 0.38 2.00 1.87 0.68 0.72 0.56 0.62 0.80 0.40 0.42 1.68 1.79 0.70 0.66 0.54 0.24 0.48 0.68 0.40 2.34 0.14",
|
||||
"text": "<SP> 只 是 因 为 为 在 人 群 群 中 多 看 了 你 一 眼 <SP> 再 也 没 能 忘 掉 你 容 颜 <SP> 梦 想 着 着 偶 偶 然 然 有 <SP> 一 一 天 再 相 见 <SP> 从 此 我 开 始 始 孤 孤 单 思 念 念 <SP> 想 想 你 时 你 你 在 天 边 <SP> 想 你 时 你 在 眼 前 前 <SP> 想 你 时 你 你 在 脑 海 <SP>",
|
||||
"phoneme": "<SP> zh_zhi3 zh_shi4 zh_yin1 zh_wei4 zh_wei2 zh_zai4 zh_ren2 zh_qun2 zh_qun2 zh_zhong1 zh_duo1 zh_kan4 zh_le5 zh_ni3 zh_yi1 zh_yan3 <SP> zh_zai4 zh_ye3 zh_mei2 zh_neng2 zh_wang4 zh_diao4 zh_ni3 zh_rong2 zh_yan2 <SP> zh_meng4 zh_xiang3 zh_zhe5 zh_zhe5 zh_ou3 zh_ou3 zh_ran2 zh_ran2 zh_you3 <SP> zh_yi1 zh_yi1 zh_tian1 zh_zai4 zh_xiang1 zh_jian4 <SP> zh_cong2 zh_ci3 zh_wo3 zh_kai1 zh_shi3 zh_shi3 zh_gu1 zh_gu1 zh_dan1 zh_si1 zh_nian4 zh_nian4 <SP> zh_xiang3 zh_xiang3 zh_ni3 zh_shi2 zh_ni3 zh_ni3 zh_zai4 zh_tian1 zh_bian1 <SP> zh_xiang3 zh_ni3 zh_shi2 zh_ni3 zh_zai4 zh_yan3 zh_qian2 zh_qian2 <SP> zh_xiang3 zh_ni3 zh_shi2 zh_ni3 zh_ni3 zh_zai4 zh_nao3 zh_hai3 <SP>",
|
||||
"note_pitch": "0 64 64 64 66 68 66 66 66 64 64 64 64 66 64 60 61 0 63 63 63 64 66 63 61 59 56 0 68 66 66 68 68 66 66 64 64 0 64 66 61 61 61 64 0 62 63 63 64 64 66 65 66 61 58 59 56 0 69 71 66 68 68 71 66 64 61 0 66 61 68 66 64 63 61 59 0 71 66 68 68 71 66 64 61 0",
|
||||
"note_type": "1 2 2 2 2 3 2 2 2 3 2 2 2 2 2 2 2 1 2 2 2 2 2 2 2 2 2 1 2 2 2 3 2 3 2 3 2 1 2 3 2 2 2 2 1 2 2 2 2 2 3 2 3 2 2 2 3 1 2 3 2 2 2 3 2 2 2 1 2 2 2 2 2 2 2 3 1 2 2 2 2 3 2 2 2 1",
|
||||
"f0": "0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 318.9 332.2 331.2 323.9 312.3 0.0 0.0 0.0 0.0 0.0 0.0 340.9 342.3 340.2 337.4 333.5 329.3 327.8 328.3 329.1 330.5 331.9 331.5 329.2 327.6 327.4 327.7 329.6 330.5 329.5 328.1 328.5 330.1 330.4 329.5 328.9 331.0 333.6 332.8 331.7 330.2 330.6 330.9 329.5 327.6 332.4 343.5 363.9 368.7 369.3 368.2 366.7 371.3 382.0 392.9 402.3 409.8 413.8 416.3 416.4 415.4 414.1 413.4 415.3 417.4 418.3 413.6 416.0 0.0 0.0 0.0 0.0 368.2 362.5 361.3 362.0 362.9 364.3 366.4 367.5 368.4 368.9 368.1 367.6 369.7 371.1 371.1 369.5 368.0 367.3 366.6 367.1 369.5 372.0 371.3 368.2 368.1 369.6 369.9 373.4 376.5 360.2 285.6 0.0 0.0 0.0 0.0 0.0 0.0 0.0 384.0 376.7 372.0 371.7 373.9 376.3 375.5 361.2 333.0 317.8 319.0 329.1 333.5 333.2 333.2 333.3 328.0 314.5 0.0 0.0 0.0 320.6 326.5 332.9 335.7 334.1 329.9 327.4 326.3 326.6 329.1 330.5 330.0 328.0 326.7 328.6 331.1 332.4 332.1 332.8 333.2 333.1 331.8 328.6 318.9 293.2 323.6 327.5 330.0 332.8 331.1 319.4 272.2 0.0 0.0 0.0 0.0 278.5 294.5 310.0 318.5 322.7 327.4 334.1 337.2 336.3 328.5 321.0 324.2 341.5 362.2 374.7 375.8 370.5 366.0 365.2 368.7 371.7 374.4 373.0 368.7 370.4 374.8 375.1 372.5 368.7 363.0 357.1 359.0 366.8 377.1 379.5 371.5 359.9 351.0 358.4 371.7 377.9 375.8 367.8 359.7 363.6 375.3 379.9 377.5 372.9 359.8 345.2 337.1 334.5 333.4 332.8 329.6 319.2 292.8 261.7 253.4 262.8 273.0 278.1 279.3 278.6 277.6 277.9 278.3 278.0 277.3 276.7 275.6 275.3 276.1 277.8 278.2 277.7 277.6 277.2 276.4 275.5 273.9 271.8 270.7 274.7 282.7 285.5 281.4 273.1 264.4 262.2 268.9 278.4 284.3 283.6 277.3 268.5 262.2 263.2 270.7 278.8 285.0 287.3 282.0 272.1 265.2 266.0 272.1 278.2 283.8 285.1 281.4 272.6 267.2 269.1 276.1 282.7 289.3 290.5 285.6 273.9 267.5 267.4 271.8 277.7 282.6 283.1 277.8 269.8 265.6 268.6 273.8 281.1 286.2 287.2 282.3 270.4 264.5 263.3 263.0 268.5 268.3 268.7 270.9 277.2 277.4 284.3 285.7 282.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 311.5 306.4 306.0 306.8 305.8 305.0 307.2 310.8 312.4 313.5 314.4 314.6 313.7 312.2 311.1 311.1 312.8 314.6 312.7 306.5 299.7 302.7 308.7 315.2 314.2 310.9 309.4 309.8 310.6 310.7 310.8 311.4 312.2 313.0 312.5 312.2 312.2 311.9 312.0 312.2 311.4 306.1 303.3 309.3 315.0 325.3 324.4 323.9 325.0 326.4 327.7 327.7 328.1 328.6 327.8 328.3 328.3 327.6 328.6 336.1 347.2 359.4 371.0 374.9 371.3 366.5 369.5 375.5 376.2 370.2 365.7 367.3 372.9 376.9 375.3 368.4 360.1 358.0 365.4 380.8 383.5 380.4 374.4 367.6 365.9 372.0 376.3 377.9 375.7 372.6 368.6 361.2 353.7 0.0 309.4 302.3 299.2 300.6 305.7 308.5 308.6 309.0 310.9 310.9 309.5 308.6 309.0 311.3 313.0 314.4 314.5 313.0 312.1 311.2 308.0 299.5 295.8 295.0 285.7 278.0 280.1 281.2 279.9 278.3 278.3 279.5 279.7 272.2 259.1 0.0 0.0 0.0 240.7 236.0 233.8 233.5 234.6 235.7 235.6 236.2 239.8 245.4 248.8 250.1 250.4 249.9 249.1 247.2 241.7 231.7 216.0 197.6 192.1 197.2 204.3 206.5 205.4 201.7 200.8 203.5 208.2 210.9 210.3 207.1 203.5 202.3 202.1 202.8 205.1 207.3 208.1 207.3 205.4 202.1 199.3 200.5 206.1 212.9 212.9 206.5 197.1 191.1 191.2 196.8 203.6 207.8 208.7 206.4 201.7 197.3 197.4 200.7 206.5 209.9 209.9 206.9 203.3 201.2 203.4 207.7 210.5 208.9 207.6 206.5 204.2 202.0 203.6 205.7 210.8 213.1 214.3 210.6 204.1 199.7 202.3 211.9 217.7 215.1 215.0 215.3 213.3 0.0 0.0 216.3 0.0 0.0 195.2 209.0 205.3 201.3 196.8 195.7 195.8 195.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 431.9 425.8 421.0 417.7 419.3 422.7 423.4 421.1 417.9 411.9 408.6 411.5 414.9 419.6 426.7 424.2 0.0 0.0 0.0 280.8 279.4 369.7 371.8 370.6 369.8 370.5 370.5 371.4 374.6 374.0 361.0 332.7 328.0 0.0 359.0 365.0 371.1 373.8 371.9 369.7 369.6 371.0 378.3 390.4 403.4 414.1 417.9 417.5 417.0 416.6 415.7 414.9 414.5 413.8 413.5 413.6 413.8 415.2 416.9 417.3 415.4 415.0 414.1 411.4 409.2 403.5 392.7 375.6 367.5 365.8 368.8 370.7 370.8 369.0 367.0 366.0 366.2 367.8 369.2 368.8 367.2 366.7 367.3 367.3 368.8 368.5 366.8 365.3 364.0 363.0 365.0 367.3 367.7 365.5 364.0 364.9 367.4 370.3 371.3 369.9 366.5 364.1 368.3 385.8 406.7 409.4 399.7 376.0 357.2 359.9 372.4 377.5 366.9 345.1 320.6 315.8 321.8 329.0 329.9 326.4 324.5 324.8 324.9 325.3 325.1 324.7 325.9 326.8 326.6 326.6 326.4 325.4 323.2 322.5 323.9 326.2 328.6 329.9 329.5 329.0 328.2 327.7 327.9 329.6 331.2 331.4 334.2 340.1 335.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.Line truncated
|
||||
}
|
||||
]
|
||||
Binary file not shown.
@@ -0,0 +1,16 @@
|
||||
[
|
||||
{
|
||||
"index": "vocal_320_10687",
|
||||
"language": "Mandarin",
|
||||
"time": [
|
||||
320,
|
||||
10687
|
||||
],
|
||||
"duration": "0.23 0.34 0.26 0.70 0.52 0.46 0.36 0.44 0.14 0.24 0.64 0.47 0.51 1.10 0.28 0.38 0.32 0.32 0.38 0.32 0.31 0.19 1.45",
|
||||
"text": "<SP> 除 了 想 你 你 <SP> 除 了 了 爱 你 你 <SP> 我 什 么 什 么 都 愿 愿 意",
|
||||
"phoneme": "<SP> zh_chu2 zh_le5 zh_xiang3 zh_ni3 zh_ni3 <SP> zh_chu2 zh_le5 zh_le5 zh_ai4 zh_ni3 zh_ni3 <SP> zh_wo3 zh_shen2 zh_me5 zh_shen2 zh_me5 zh_dou1 zh_yuan4 zh_yuan4 zh_yi4",
|
||||
"note_pitch": "0 62 65 67 67 69 0 67 69 67 65 67 69 67 67 66 64 64 60 60 65 67 0",
|
||||
"note_type": "1 2 2 2 2 3 1 2 2 3 2 2 3 1 2 2 2 2 2 2 2 3 2",
|
||||
"f0": "0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 294.1 288.2 290.6 294.6 295.7 292.8 291.5 294.4 295.4 294.7 293.5 292.2 294.4 295.8 293.2 297.7 320.1 338.1 348.3 348.7 344.4 342.6 346.2 354.8 356.9 353.1 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 401.1 403.8 405.0 402.1 398.4 395.6 393.3 392.0 391.4 390.3 390.2 390.3 390.3 391.3 392.8 393.6 391.7 390.5 391.4 391.6 391.5 393.5 393.8 390.8 387.4 387.8 389.3 390.8 392.2 391.6 390.2 389.8 389.1 388.4 390.0 395.5 397.2 396.7 395.5 395.1 394.6 394.9 395.6 395.2 394.6 395.4 395.9 394.0 391.7 390.7 391.7 392.6 391.6 395.7 405.8 441.7 462.5 463.8 450.2 430.9 414.5 415.2 426.7 439.8 454.4 462.9 447.8 422.5 400.6 403.2 423.5 451.3 482.3 492.8 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 435.9 414.3 406.2 402.8 398.2 396.7 396.7 395.2 398.3 416.9 441.6 446.9 441.5 437.3 435.3 432.9 431.6 432.3 434.8 436.7 433.8 422.8 406.7 390.9 382.0 381.7 384.7 382.7 368.1 357.4 355.6 355.1 352.4 348.1 346.4 348.7 351.6 355.0 354.4 351.7 349.9 349.2 348.1 346.0 345.4 344.2 344.4 345.5 346.6 349.0 349.7 349.1 349.5 349.6 349.7 349.4 349.5 352.2 354.5 355.7 355.6 356.9 359.4 361.5 363.8 360.4 354.3 357.2 363.9 372.4 382.6 399.1 402.9 400.6 395.5 390.5 388.9 390.2 391.1 391.9 391.5 390.4 390.2 391.3 391.6 391.0 388.6 386.2 389.1 403.8 430.2 441.8 449.5 448.1 443.2 438.3 432.9 430.7 434.6 442.0 447.6 446.0 440.3 434.7 431.1 435.7 442.3 445.9 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 423.8 402.1 398.5 397.6 393.4 390.5 391.4 402.5 427.6 442.5 442.5 435.1 430.1 430.8 439.9 447.4 442.2 426.1 412.3 399.8 391.8 389.5 388.5 387.8 386.4 384.7 384.5 387.9 391.2 391.8 392.9 393.8 392.0 392.0 395.4 398.1 398.2 396.3 393.1 391.1 388.9 386.6 383.1 381.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 357.8 367.6 370.9 365.4 359.3 355.7 358.1 372.7 396.8 404.0 398.4 392.6 389.2 388.8 383.6 362.3 341.1 325.1 326.3 327.9 331.3 333.3 326.0 319.4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 356.4 353.5 349.0 346.5 338.4 328.4 323.7 326.2 334.2 338.4 331.6 309.8 280.6 256.4 252.8 256.1 258.3 256.9 257.6 261.1 259.7 258.7 258.6 260.2 262.2 262.4 262.9 264.2 263.4 260.2 256.4 253.2 238.7 223.3 231.6 257.7 258.1 258.4 258.8 258.0 256.8 255.8 254.0 255.1 258.1 261.9 263.7 262.6 256.7 253.1 250.2 246.7 258.7 294.4 327.1 342.6 346.0 344.1 341.2 342.3 345.2 350.1 364.7 382.5 396.1 396.1 389.2 381.8 381.1 387.2 397.0 399.3 390.7 374.3 360.1 350.6 346.6 347.4 350.7 354.4 354.3 351.7 349.8 348.4 346.9 347.0 348.4 349.5 351.0 352.3 353.6 353.3 350.5 348.3 345.5 344.3 344.4 347.0 350.6 352.0 351.0 350.8 350.1 347.6 345.7 347.0 350.3 351.7 350.7 348.5 346.9 347.7 349.0 349.1 348.5 346.8 346.3 348.4 349.0 349.2 351.1 349.6 348.3 350.5 351.1 348.0 347.6 349.1 351.3 356.0 361.3 360.6 354.0 341.0 316.2 302.9 302.1 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0"
|
||||
}
|
||||
]
|
||||
Binary file not shown.
@@ -0,0 +1,752 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the additive structured Omni config."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import fields
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm_omni.config import omni_config as omni_config_module
|
||||
from vllm_omni.config.omni_config import (
|
||||
BaseVllmOmniStageConfig,
|
||||
OmniStageCacheConfig,
|
||||
OmniStageConnectorConfig,
|
||||
OmniStageDiffusionParallelConfig,
|
||||
OmniStageLoadConfig,
|
||||
OmniStageModelConfig,
|
||||
OmniStageParallelConfig,
|
||||
OmniStageRuntimeConfig,
|
||||
OmniStageSchedulerConfig,
|
||||
VllmOmniARStageConfig,
|
||||
VllmOmniConfig,
|
||||
VllmOmniDiffusionStageConfig,
|
||||
VllmOmniGenerationStageConfig,
|
||||
)
|
||||
from vllm_omni.config.pipeline_registry import OMNI_PIPELINES
|
||||
from vllm_omni.config.stage_config import (
|
||||
_STAGE_DEPLOY_FIELDS,
|
||||
PIPELINE_WIDE_ENGINE_FIELDS,
|
||||
DeployConfig,
|
||||
PipelineConfig,
|
||||
StageDeployConfig,
|
||||
StageExecutionType,
|
||||
load_deploy_config,
|
||||
merge_pipeline_deploy,
|
||||
)
|
||||
from vllm_omni.engine.stage_init_utils import build_engine_args_dict
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
_DEPLOY_DIR = Path(__file__).parents[2] / "vllm_omni" / "deploy"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _stable_test_platform(monkeypatch):
|
||||
from vllm_omni import platforms
|
||||
|
||||
platform = platforms.current_omni_platform
|
||||
monkeypatch.setattr(platform, "device_name", "cpu", raising=False)
|
||||
monkeypatch.setattr(platform, "device_type", "cpu", raising=False)
|
||||
|
||||
|
||||
def _load_default_deploy(model_type: str) -> DeployConfig:
|
||||
deploy_path = _DEPLOY_DIR / f"{model_type}.yaml"
|
||||
if deploy_path.exists():
|
||||
return load_deploy_config(deploy_path)
|
||||
return DeployConfig()
|
||||
|
||||
|
||||
def _resolve_pipeline_or_skip(model_type: str) -> PipelineConfig:
|
||||
registered = OMNI_PIPELINES[model_type]
|
||||
pipeline = registered(None) if callable(registered) else registered
|
||||
if pipeline is None:
|
||||
pytest.skip(f"Pipeline {model_type!r} requires an HF config to resolve")
|
||||
return pipeline
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_type", sorted(OMNI_PIPELINES))
|
||||
def test_vllm_omni_config_from_registry_matches_merge_pipeline_deploy(model_type: str):
|
||||
pipeline = _resolve_pipeline_or_skip(model_type)
|
||||
legacy_deploy = _load_default_deploy(model_type)
|
||||
|
||||
legacy_stages = merge_pipeline_deploy(pipeline, legacy_deploy)
|
||||
omni_config = VllmOmniConfig.from_registry(model_type)
|
||||
|
||||
assert omni_config.pipeline_config is pipeline
|
||||
assert len(omni_config.stage_configs) == len(legacy_stages)
|
||||
|
||||
for legacy_stage, omni_stage in zip(legacy_stages, omni_config.stage_configs, strict=True):
|
||||
assert omni_config.stage_by_id(legacy_stage.stage_id) is omni_stage
|
||||
|
||||
assert omni_stage.stage_pipeline_config is pipeline.get_stage(legacy_stage.stage_id)
|
||||
assert omni_stage.model_config.default_sampling_params == legacy_stage.yaml_extras.get(
|
||||
"default_sampling_params"
|
||||
)
|
||||
assert omni_stage.connector_config.output_connectors == legacy_stage.yaml_extras.get("output_connectors")
|
||||
assert omni_stage.connector_config.input_connectors == legacy_stage.yaml_extras.get("input_connectors")
|
||||
assert omni_stage.runtime_config.devices == legacy_stage.yaml_runtime.get("devices")
|
||||
assert omni_stage.runtime_config.num_replicas == legacy_stage.yaml_runtime.get("num_replicas", 1)
|
||||
|
||||
engine_args = legacy_stage.yaml_engine_args
|
||||
assert omni_stage.model_config.enforce_eager == engine_args.get("enforce_eager", False)
|
||||
assert omni_stage.load_config.load_format == engine_args.get("load_format", "auto")
|
||||
assert omni_stage.load_config.tokenizer_mode == engine_args.get("tokenizer_mode", "auto")
|
||||
assert omni_stage.cache_config.gpu_memory_utilization == engine_args.get("gpu_memory_utilization", 0.90)
|
||||
assert omni_stage.scheduler_config.max_num_seqs == engine_args.get("max_num_seqs", 128)
|
||||
assert omni_stage.scheduler_config.max_num_batched_tokens == engine_args.get("max_num_batched_tokens")
|
||||
assert omni_stage.scheduler_config.async_scheduling == engine_args.get("async_scheduling", True)
|
||||
legacy_parallel_config = engine_args.get("parallel_config") or {}
|
||||
assert omni_stage.parallel_config.tensor_parallel_size == legacy_parallel_config.get(
|
||||
"tensor_parallel_size",
|
||||
engine_args.get("tensor_parallel_size", 1),
|
||||
)
|
||||
|
||||
if omni_stage.stage_pipeline_config.execution_type == StageExecutionType.DIFFUSION:
|
||||
assert isinstance(omni_stage, VllmOmniDiffusionStageConfig)
|
||||
assert omni_stage.diffusion_config is not None
|
||||
assert omni_stage.diffusion_config.stage_id == legacy_stage.stage_id
|
||||
assert omni_stage.diffusion_config.model_arch == engine_args.get("model_arch")
|
||||
elif omni_stage.stage_pipeline_config.execution_type == StageExecutionType.LLM_AR:
|
||||
assert isinstance(omni_stage, VllmOmniARStageConfig)
|
||||
assert not hasattr(omni_stage, "diffusion_config")
|
||||
else:
|
||||
assert isinstance(omni_stage, VllmOmniGenerationStageConfig)
|
||||
assert not hasattr(omni_stage, "diffusion_config")
|
||||
|
||||
|
||||
def test_stage_by_id_raises_for_unknown_stage():
|
||||
omni_config = VllmOmniConfig.from_registry("qwen3_tts")
|
||||
|
||||
with pytest.raises(KeyError, match="no stage 99"):
|
||||
omni_config.stage_by_id(99)
|
||||
|
||||
|
||||
def test_resolve_execution_mode_rejects_unknown_execution_type():
|
||||
with pytest.raises(ValueError, match="Unsupported stage execution type"):
|
||||
omni_config_module._resolve_execution_mode("unknown_execution_type")
|
||||
|
||||
|
||||
def test_from_registry_preserves_current_pipeline_config_object():
|
||||
omni_config = VllmOmniConfig.from_registry("minicpmo_4_5")
|
||||
pipeline = _resolve_pipeline_or_skip("minicpmo_4_5")
|
||||
|
||||
assert omni_config.pipeline_config is pipeline
|
||||
assert not hasattr(omni_config, "pipeline")
|
||||
assert "hf_config_predicate" in {f.name for f in fields(PipelineConfig)}
|
||||
assert omni_config.pipeline_config.hf_config_predicate is pipeline.hf_config_predicate
|
||||
|
||||
|
||||
def test_from_registry_normalizes_stage_engine_extras_without_expanding_stage_deploy_config():
|
||||
assert not hasattr(StageDeployConfig, "model_config")
|
||||
assert not hasattr(StageDeployConfig, "parallel_config")
|
||||
|
||||
stage = VllmOmniConfig.from_registry("dreamzero", deploy_config_path="dreamzero_tp1_cfg2").stage_by_id(0)
|
||||
|
||||
assert isinstance(stage, VllmOmniDiffusionStageConfig)
|
||||
assert stage.parallel_config.tensor_parallel_size == 1
|
||||
assert stage.parallel_config.cfg_parallel_size == 2
|
||||
assert stage.diffusion_config.model_config["default_robot_embodiment"] == "roboarena"
|
||||
|
||||
|
||||
def test_from_registry_applies_cli_overrides_without_stage_config_runtime_bridge():
|
||||
omni_config = VllmOmniConfig.from_registry(
|
||||
"qwen3_tts",
|
||||
cli_overrides={
|
||||
"stage_0_max_num_seqs": 7,
|
||||
"stage_1_tensor_parallel_size": 2,
|
||||
},
|
||||
)
|
||||
|
||||
stage0 = omni_config.stage_by_id(0)
|
||||
stage1 = omni_config.stage_by_id(1)
|
||||
|
||||
assert stage0.scheduler_config.max_num_seqs == 7
|
||||
assert stage1.parallel_config.tensor_parallel_size == 2
|
||||
assert stage1.runtime_config.num_gpus == stage1.parallel_config.world_size
|
||||
|
||||
|
||||
def test_runtime_num_gpus_is_derived_from_parallel_world_size():
|
||||
omni_config = VllmOmniConfig.from_registry("hunyuan_image3_dit")
|
||||
stage = omni_config.stage_by_id(0)
|
||||
|
||||
assert stage.parallel_config.tensor_parallel_size == 4
|
||||
assert stage.parallel_config.world_size == 4
|
||||
assert stage.runtime_config.num_gpus == 4
|
||||
|
||||
|
||||
def test_runtime_num_gpus_ignores_stale_runtime_override():
|
||||
omni_config = VllmOmniConfig.from_registry(
|
||||
"hunyuan_image3_dit",
|
||||
cli_overrides={
|
||||
"stage_0_num_gpus": 1,
|
||||
},
|
||||
)
|
||||
stage = omni_config.stage_by_id(0)
|
||||
|
||||
assert stage.parallel_config.world_size == 4
|
||||
assert stage.runtime_config.num_gpus == 4
|
||||
|
||||
|
||||
def test_from_registry_does_not_route_server_cli_keys_to_diffusion_stage():
|
||||
omni_config = VllmOmniConfig.from_registry(
|
||||
"dreamzero",
|
||||
deploy_config_path="dreamzero_tp1_cfg2",
|
||||
cli_overrides={
|
||||
"host": "0.0.0.0",
|
||||
"port": 8000,
|
||||
"api_key": "secret",
|
||||
"stage_0_host": "127.0.0.1",
|
||||
"stage_0_port": 23456,
|
||||
},
|
||||
)
|
||||
|
||||
stage = omni_config.stage_by_id(0)
|
||||
|
||||
assert isinstance(stage, VllmOmniDiffusionStageConfig)
|
||||
assert stage.diffusion_config.host == "127.0.0.1"
|
||||
assert stage.diffusion_config.port == 23456
|
||||
assert not hasattr(stage.diffusion_config, "api_key")
|
||||
|
||||
|
||||
def test_pipeline_deploy_cli_fields_reuse_legacy_pipeline_wide_engine_fields():
|
||||
assert omni_config_module._PIPELINE_DEPLOY_CLI_FIELDS is PIPELINE_WIDE_ENGINE_FIELDS
|
||||
assert "active_stream_window" in omni_config_module._PIPELINE_DEPLOY_CLI_FIELDS
|
||||
assert "custom_voice_dir" in omni_config_module._PIPELINE_DEPLOY_CLI_FIELDS
|
||||
|
||||
|
||||
def test_pipeline_wide_model_fields_are_retained_on_structured_stage_configs(tmp_path):
|
||||
custom_voice_dir = tmp_path / "voices"
|
||||
omni_config = VllmOmniConfig.from_registry(
|
||||
"qwen3_tts",
|
||||
cli_overrides={
|
||||
"active_stream_window": 2,
|
||||
"custom_voice_dir": str(custom_voice_dir),
|
||||
},
|
||||
)
|
||||
|
||||
assert {stage.model_config.active_stream_window for stage in omni_config.stage_configs} == {2}
|
||||
assert {stage.model_config.custom_voice_dir for stage in omni_config.stage_configs} == {str(custom_voice_dir)}
|
||||
|
||||
|
||||
def test_stage_deploy_engine_fields_reuse_legacy_stage_deploy_fields():
|
||||
assert omni_config_module._STAGE_DEPLOY_ENGINE_FIELDS == tuple(_STAGE_DEPLOY_FIELDS)
|
||||
assert "tensor_parallel_size" in omni_config_module._STAGE_DEPLOY_ENGINE_FIELDS
|
||||
assert "stage_id" not in omni_config_module._STAGE_DEPLOY_ENGINE_FIELDS
|
||||
|
||||
|
||||
def test_public_config_exports_use_stage_specific_sub_config_names():
|
||||
import vllm_omni.config as config_pkg
|
||||
|
||||
generic_names = {
|
||||
"CacheConfig",
|
||||
"ConnectorConfig",
|
||||
"LoadConfig",
|
||||
"ModelConfig",
|
||||
"OrchestratorConfig",
|
||||
"ParallelConfig",
|
||||
"RuntimeConfig",
|
||||
"SchedulerConfig",
|
||||
}
|
||||
|
||||
assert generic_names.isdisjoint(config_pkg.__all__)
|
||||
assert {
|
||||
"OmniStageCacheConfig",
|
||||
"OmniStageConnectorConfig",
|
||||
"OmniStageDiffusionParallelConfig",
|
||||
"OmniStageLoadConfig",
|
||||
"OmniStageModelConfig",
|
||||
"VllmOmniOrchestratorConfig",
|
||||
"OmniStageParallelConfig",
|
||||
"OmniStageRuntimeConfig",
|
||||
"OmniStageSchedulerConfig",
|
||||
"StageConfigType",
|
||||
}.issubset(config_pkg.__all__)
|
||||
|
||||
|
||||
def test_from_registry_keeps_worker_backend_separate_from_distributed_executor_backend():
|
||||
omni_config = VllmOmniConfig.from_registry("dreamzero", deploy_config_path="dreamzero_tp1_cfg2")
|
||||
|
||||
stage = omni_config.stage_by_id(0)
|
||||
assert isinstance(stage, VllmOmniDiffusionStageConfig)
|
||||
assert stage.diffusion_config.distributed_executor_backend == "mp"
|
||||
assert omni_config.orchestrator_config.worker_backend == "multi_process"
|
||||
|
||||
|
||||
def test_from_registry_maps_orchestrator_cli_overrides():
|
||||
omni_config = VllmOmniConfig.from_registry(
|
||||
"qwen3_tts",
|
||||
cli_overrides={
|
||||
"stage_init_timeout": 1200,
|
||||
"init_timeout": 1800,
|
||||
"worker_backend": "ray",
|
||||
"ray_address": "ray://127.0.0.1:10001",
|
||||
"omni_master_address": "127.0.0.1",
|
||||
"omni_master_port": 12345,
|
||||
"omni_dp_size_local": 2,
|
||||
"omni_lb_policy": "round_robin",
|
||||
"omni_heartbeat_timeout": 9.5,
|
||||
"shm_threshold_bytes": 4096,
|
||||
"batch_timeout": 3,
|
||||
},
|
||||
)
|
||||
|
||||
orchestrator_config = omni_config.orchestrator_config
|
||||
assert orchestrator_config.stage_init_timeout == 1200
|
||||
assert orchestrator_config.init_timeout == 1800
|
||||
assert orchestrator_config.worker_backend == "ray"
|
||||
assert orchestrator_config.ray_address == "ray://127.0.0.1:10001"
|
||||
assert orchestrator_config.omni_master_address == "127.0.0.1"
|
||||
assert orchestrator_config.omni_master_port == 12345
|
||||
assert orchestrator_config.omni_dp_size_local == 2
|
||||
assert orchestrator_config.omni_lb_policy == "round_robin"
|
||||
assert orchestrator_config.omni_heartbeat_timeout == 9.5
|
||||
assert orchestrator_config.shm_threshold_bytes == 4096
|
||||
assert orchestrator_config.batch_timeout == 3
|
||||
|
||||
|
||||
def test_from_registry_records_loaded_deploy_path_on_orchestrator_config():
|
||||
omni_config = VllmOmniConfig.from_registry("dreamzero", deploy_config_path="dreamzero_tp1_cfg2")
|
||||
|
||||
assert omni_config.pipeline_config.model_type == "dreamzero"
|
||||
assert omni_config.orchestrator_config.deploy_config_path == str(_DEPLOY_DIR / "dreamzero_tp1_cfg2.yaml")
|
||||
|
||||
|
||||
def test_from_registry_dispatches_async_chunk_processors_without_mutating_topology():
|
||||
pipeline = _resolve_pipeline_or_skip("qwen3_tts")
|
||||
|
||||
async_config = VllmOmniConfig.from_registry("qwen3_tts")
|
||||
assert async_config.stage_by_id(0).custom_process_next_stage_input_func.endswith("talker2code2wav_async_chunk")
|
||||
assert async_config.stage_by_id(1).custom_process_input_func is None
|
||||
|
||||
sync_config = VllmOmniConfig.from_registry("qwen3_tts", cli_overrides={"async_chunk": False})
|
||||
assert sync_config.stage_by_id(0).custom_process_next_stage_input_func.endswith("talker2code2wav_full_payload")
|
||||
assert sync_config.stage_by_id(1).custom_process_input_func.endswith("talker2code2wav_token_only")
|
||||
|
||||
assert pipeline.get_stage(0).custom_process_next_stage_input_func.endswith("talker2code2wav_full_payload")
|
||||
assert pipeline.get_stage(1).custom_process_input_func is None
|
||||
|
||||
|
||||
def test_vllm_omni_stage_config_public_fields_use_typed_stage_realizations():
|
||||
assert not hasattr(BaseVllmOmniStageConfig, "from_stage_config")
|
||||
assert not hasattr(BaseVllmOmniStageConfig, "to_legacy_stage_config")
|
||||
|
||||
public_fields = {f.name for f in fields(BaseVllmOmniStageConfig)}
|
||||
|
||||
assert public_fields == {
|
||||
"stage_pipeline_config",
|
||||
"model_config",
|
||||
"load_config",
|
||||
"cache_config",
|
||||
"scheduler_config",
|
||||
"connector_config",
|
||||
"runtime_config",
|
||||
"parallel_config",
|
||||
"quantization_config",
|
||||
}
|
||||
assert "diffusion_config" not in public_fields
|
||||
assert {f.name for f in fields(VllmOmniDiffusionStageConfig)} == public_fields | {"diffusion_config"}
|
||||
assert {f.name for f in fields(VllmOmniARStageConfig)} == public_fields
|
||||
assert {f.name for f in fields(VllmOmniGenerationStageConfig)} == public_fields
|
||||
|
||||
|
||||
def test_runtime_config_fields_match_rfc_runtime_scope():
|
||||
assert {f.name for f in fields(OmniStageRuntimeConfig)} == {
|
||||
"devices",
|
||||
"num_replicas",
|
||||
"env",
|
||||
"num_gpus",
|
||||
"log_level",
|
||||
"log_stats",
|
||||
"profiler_config",
|
||||
}
|
||||
|
||||
|
||||
def test_sub_config_fields_match_rfc_scopes():
|
||||
assert {f.name for f in fields(OmniStageModelConfig)} == {
|
||||
"active_stream_window",
|
||||
"enable_sleep_mode",
|
||||
"default_sampling_params",
|
||||
"subtalker_sampling_params",
|
||||
"has_sampling_extra_args",
|
||||
"custom_voice_dir",
|
||||
"task_type",
|
||||
"codec_frame_rate_hz",
|
||||
"enforce_eager",
|
||||
"enable_flashinfer_autotune",
|
||||
"compilation_config",
|
||||
"enable_multithread_weight_load",
|
||||
"num_weight_load_threads",
|
||||
"disable_autocast",
|
||||
}
|
||||
assert {f.name for f in fields(OmniStageLoadConfig)} == {
|
||||
"load_format",
|
||||
"tokenizer_mode",
|
||||
"config_format",
|
||||
"skip_mm_profiling",
|
||||
}
|
||||
assert {f.name for f in fields(OmniStageCacheConfig)} == {
|
||||
"gpu_memory_utilization",
|
||||
"enable_prefix_caching",
|
||||
"disable_hybrid_kv_cache_manager",
|
||||
"mm_processor_cache_gb",
|
||||
}
|
||||
assert {f.name for f in fields(OmniStageSchedulerConfig)} == {
|
||||
"max_num_seqs",
|
||||
"max_num_batched_tokens",
|
||||
"max_model_len",
|
||||
"enable_chunked_prefill",
|
||||
"async_scheduling",
|
||||
}
|
||||
assert {f.name for f in fields(OmniStageConnectorConfig)} == {
|
||||
"stage_connector",
|
||||
"output_connectors",
|
||||
"input_connectors",
|
||||
}
|
||||
assert {f.name for f in fields(OmniStageParallelConfig)} == {
|
||||
"pipeline_parallel_size",
|
||||
"data_parallel_size",
|
||||
"tensor_parallel_size",
|
||||
"enable_expert_parallel",
|
||||
"world_size",
|
||||
}
|
||||
assert {f.name for f in fields(OmniStageDiffusionParallelConfig)} == {
|
||||
"pipeline_parallel_size",
|
||||
"data_parallel_size",
|
||||
"tensor_parallel_size",
|
||||
"sequence_parallel_size",
|
||||
"ulysses_degree",
|
||||
"ring_degree",
|
||||
"ulysses_mode",
|
||||
"cfg_parallel_size",
|
||||
"vae_patch_parallel_size",
|
||||
"use_hsdp",
|
||||
"mask_sp_padding",
|
||||
"hsdp_shard_size",
|
||||
"hsdp_replicate_size",
|
||||
"enable_expert_parallel",
|
||||
"world_size",
|
||||
}
|
||||
|
||||
|
||||
def test_diffusion_parallel_config_fields_cover_legacy_surface():
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
|
||||
legacy_fields = {f.name for f in fields(DiffusionParallelConfig)}
|
||||
structured_fields = {f.name for f in fields(OmniStageDiffusionParallelConfig)}
|
||||
expected_upstream_fields = {"mask_sp_padding"}
|
||||
|
||||
assert legacy_fields | expected_upstream_fields <= structured_fields
|
||||
assert structured_fields - legacy_fields - expected_upstream_fields == {"world_size"}
|
||||
|
||||
|
||||
def test_diffusion_parallel_config_keeps_current_diffusion_parallel_surface():
|
||||
cfg = OmniStageDiffusionParallelConfig(
|
||||
pipeline_parallel_size=2,
|
||||
data_parallel_size=3,
|
||||
tensor_parallel_size=4,
|
||||
cfg_parallel_size=3,
|
||||
mask_sp_padding=True,
|
||||
)
|
||||
|
||||
assert cfg.pipeline_parallel_size == 2
|
||||
assert cfg.data_parallel_size == 3
|
||||
assert cfg.cfg_parallel_size == 3
|
||||
assert cfg.mask_sp_padding is True
|
||||
assert cfg.world_size == 72
|
||||
|
||||
|
||||
def test_parallel_config_derived_fields_are_not_init_inputs():
|
||||
with pytest.raises(ValidationError):
|
||||
OmniStageParallelConfig(world_size=4)
|
||||
|
||||
with pytest.raises(ValidationError):
|
||||
OmniStageDiffusionParallelConfig(world_size=4)
|
||||
|
||||
with pytest.raises(ValidationError):
|
||||
OmniStageDiffusionParallelConfig(sequence_parallel_size=2)
|
||||
|
||||
|
||||
def test_diffusion_parallel_config_matches_diffusion_parallel_world_size_for_vae_patch_parallel():
|
||||
cfg = OmniStageDiffusionParallelConfig(
|
||||
tensor_parallel_size=2,
|
||||
cfg_parallel_size=2,
|
||||
vae_patch_parallel_size=4,
|
||||
)
|
||||
|
||||
assert cfg.vae_patch_parallel_size == 4
|
||||
assert cfg.world_size == 4
|
||||
|
||||
|
||||
def test_diffusion_parallel_config_supports_diffusion_hsdp_auto_sharding():
|
||||
cfg = OmniStageDiffusionParallelConfig(
|
||||
pipeline_parallel_size=2,
|
||||
ulysses_degree=2,
|
||||
use_hsdp=True,
|
||||
hsdp_shard_size=-1,
|
||||
hsdp_replicate_size=2,
|
||||
)
|
||||
|
||||
assert cfg.hsdp_shard_size == 2
|
||||
assert cfg.world_size == 4
|
||||
|
||||
|
||||
def test_diffusion_parallel_config_rejects_hsdp_with_tp_or_dp():
|
||||
with pytest.raises(ValueError, match="cannot be used with TP or DP"):
|
||||
OmniStageDiffusionParallelConfig(tensor_parallel_size=2, use_hsdp=True, hsdp_shard_size=2)
|
||||
|
||||
with pytest.raises(ValueError, match="cannot be used with TP or DP"):
|
||||
OmniStageDiffusionParallelConfig(data_parallel_size=2, use_hsdp=True, hsdp_shard_size=2)
|
||||
|
||||
|
||||
def test_from_registry_preserves_legacy_pp_dp_for_world_size():
|
||||
cfg = VllmOmniConfig.from_registry("hunyuan_image3_dit").stage_by_id(0).parallel_config
|
||||
|
||||
assert cfg.pipeline_parallel_size == 1
|
||||
assert cfg.data_parallel_size == 1
|
||||
assert cfg.tensor_parallel_size == 4
|
||||
assert cfg.world_size == 4
|
||||
|
||||
|
||||
def test_from_registry_derives_sequence_parallel_size_from_degrees(tmp_path):
|
||||
deploy_path = tmp_path / "dreamzero_derived_parallel.yaml"
|
||||
deploy_path.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"pipeline: dreamzero",
|
||||
"async_chunk: false",
|
||||
"stages:",
|
||||
" - stage_id: 0",
|
||||
" parallel_config:",
|
||||
" sequence_parallel_size: 99",
|
||||
" ulysses_degree: 2",
|
||||
" ring_degree: 3",
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
stage = VllmOmniConfig.from_registry("dreamzero", deploy_config_path=str(deploy_path)).stage_by_id(0)
|
||||
|
||||
assert isinstance(stage, VllmOmniDiffusionStageConfig)
|
||||
assert stage.parallel_config.sequence_parallel_size == 6
|
||||
assert stage.parallel_config.world_size == 6
|
||||
|
||||
|
||||
def test_diffusion_parallel_config_rejects_cfg_parallel_size_outside_current_bound():
|
||||
with pytest.raises(ValidationError):
|
||||
OmniStageDiffusionParallelConfig(cfg_parallel_size=4)
|
||||
|
||||
|
||||
def test_stage_realizations_use_stage_specific_parallel_config_types():
|
||||
qwen_config = VllmOmniConfig.from_registry("qwen3_tts")
|
||||
ar_stage = qwen_config.stage_by_id(0)
|
||||
generation_stage = qwen_config.stage_by_id(1)
|
||||
diffusion_stage = VllmOmniConfig.from_registry("hunyuan_image3_dit").stage_by_id(0)
|
||||
|
||||
assert isinstance(ar_stage, VllmOmniARStageConfig)
|
||||
assert type(ar_stage.parallel_config) is OmniStageParallelConfig
|
||||
assert not hasattr(ar_stage.parallel_config, "cfg_parallel_size")
|
||||
assert not hasattr(ar_stage.parallel_config, "sequence_parallel_size")
|
||||
assert not hasattr(ar_stage.parallel_config, "ulysses_degree")
|
||||
|
||||
assert isinstance(generation_stage, VllmOmniGenerationStageConfig)
|
||||
assert type(generation_stage.parallel_config) is OmniStageParallelConfig
|
||||
assert not hasattr(generation_stage.parallel_config, "cfg_parallel_size")
|
||||
assert not hasattr(generation_stage.parallel_config, "sequence_parallel_size")
|
||||
assert not hasattr(generation_stage.parallel_config, "ulysses_degree")
|
||||
|
||||
assert isinstance(diffusion_stage, VllmOmniDiffusionStageConfig)
|
||||
assert isinstance(diffusion_stage.parallel_config, OmniStageDiffusionParallelConfig)
|
||||
assert diffusion_stage.parallel_config.cfg_parallel_size == 1
|
||||
assert diffusion_stage.parallel_config.sequence_parallel_size == 1
|
||||
assert diffusion_stage.parallel_config.ulysses_degree == 1
|
||||
|
||||
|
||||
def test_from_registry_preserves_diffusion_parallel_mask_sp_padding(tmp_path):
|
||||
deploy_path = tmp_path / "dreamzero_mask_sp_padding.yaml"
|
||||
deploy_path.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"pipeline: dreamzero",
|
||||
"async_chunk: false",
|
||||
"stages:",
|
||||
" - stage_id: 0",
|
||||
" parallel_config:",
|
||||
" mask_sp_padding: true",
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
stage = VllmOmniConfig.from_registry("dreamzero", deploy_config_path=str(deploy_path)).stage_by_id(0)
|
||||
|
||||
assert isinstance(stage, VllmOmniDiffusionStageConfig)
|
||||
assert stage.parallel_config.mask_sp_padding is True
|
||||
|
||||
|
||||
def test_from_registry_matches_stage_config_to_omegaconf_behavior_for_representative_stage():
|
||||
pipeline = _resolve_pipeline_or_skip("qwen3_tts")
|
||||
legacy_stage = merge_pipeline_deploy(pipeline, _load_default_deploy("qwen3_tts"))[0]
|
||||
omega_stage = legacy_stage.to_omegaconf()
|
||||
omni_stage = VllmOmniConfig.from_registry("qwen3_tts").stage_by_id(legacy_stage.stage_id)
|
||||
|
||||
assert omega_stage.stage_id == omni_stage.stage_id
|
||||
assert omega_stage.stage_type == omni_stage.stage_type.value
|
||||
assert omega_stage.engine_input_source == omni_stage.input_sources
|
||||
assert omega_stage.final_output == omni_stage.final_output
|
||||
assert omega_stage.final_output_type == omni_stage.final_output_type
|
||||
assert omega_stage.is_comprehension == omni_stage.is_comprehension
|
||||
assert omega_stage.engine_args.model_stage == omni_stage.model_stage
|
||||
assert omega_stage.engine_args.worker_type == omni_stage.worker_type
|
||||
assert omega_stage.engine_args.scheduler_cls == omni_stage.scheduler_cls
|
||||
assert omega_stage.runtime.process is True
|
||||
assert omega_stage.runtime.requires_multimodal_data == omni_stage.requires_multimodal_data
|
||||
|
||||
|
||||
def test_from_registry_matches_to_omegaconf_diffusion_parallel_config():
|
||||
pipeline = _resolve_pipeline_or_skip("hunyuan_image3_dit")
|
||||
legacy_stage = merge_pipeline_deploy(pipeline, _load_default_deploy("hunyuan_image3_dit"))[0]
|
||||
omega_stage = legacy_stage.to_omegaconf()
|
||||
omni_stage = VllmOmniConfig.from_registry("hunyuan_image3_dit").stage_by_id(legacy_stage.stage_id)
|
||||
|
||||
assert (
|
||||
omega_stage.engine_args.parallel_config.pipeline_parallel_size
|
||||
== omni_stage.parallel_config.pipeline_parallel_size
|
||||
)
|
||||
assert omega_stage.engine_args.parallel_config.data_parallel_size == omni_stage.parallel_config.data_parallel_size
|
||||
assert (
|
||||
omega_stage.engine_args.parallel_config.tensor_parallel_size == omni_stage.parallel_config.tensor_parallel_size
|
||||
)
|
||||
assert (
|
||||
omega_stage.engine_args.parallel_config.sequence_parallel_size
|
||||
== omni_stage.parallel_config.sequence_parallel_size
|
||||
)
|
||||
assert omega_stage.engine_args.parallel_config.cfg_parallel_size == omni_stage.parallel_config.cfg_parallel_size
|
||||
assert (
|
||||
omega_stage.engine_args.parallel_config.vae_patch_parallel_size
|
||||
== omni_stage.parallel_config.vae_patch_parallel_size
|
||||
)
|
||||
|
||||
|
||||
def test_from_registry_matches_build_engine_args_dict_behavior_for_representative_stage(monkeypatch):
|
||||
from vllm_omni.engine import stage_init_utils
|
||||
|
||||
monkeypatch.setattr(stage_init_utils, "resolve_worker_cls", lambda engine_args: None)
|
||||
pipeline = _resolve_pipeline_or_skip("qwen3_tts")
|
||||
legacy_stage = merge_pipeline_deploy(pipeline, _load_default_deploy("qwen3_tts"))[0]
|
||||
omega_stage = legacy_stage.to_omegaconf()
|
||||
legacy_engine_args = build_engine_args_dict(
|
||||
omega_stage,
|
||||
model="/tmp/qwen3-tts",
|
||||
stage_connector_spec={"name": "SharedMemoryConnector", "extra": {}},
|
||||
)
|
||||
omni_stage = VllmOmniConfig.from_registry("qwen3_tts").stage_by_id(legacy_stage.stage_id)
|
||||
|
||||
assert legacy_engine_args["model"] == "/tmp/qwen3-tts"
|
||||
assert legacy_engine_args["stage_id"] == omni_stage.stage_id
|
||||
assert legacy_engine_args["model_stage"] == omni_stage.model_stage
|
||||
assert legacy_engine_args["worker_type"] == omni_stage.worker_type
|
||||
assert legacy_engine_args["scheduler_cls"] == omni_stage.scheduler_cls
|
||||
assert legacy_engine_args["stage_connector_spec"] == {"name": "SharedMemoryConnector", "extra": {}}
|
||||
assert legacy_engine_args["has_sampling_extra_args"] == bool(
|
||||
(omni_stage.model_config.default_sampling_params or {}).get("extra_args")
|
||||
)
|
||||
assert omni_stage.model_config.has_sampling_extra_args == legacy_engine_args["has_sampling_extra_args"]
|
||||
|
||||
|
||||
def test_from_registry_derives_has_sampling_extra_args_from_stage_defaults():
|
||||
stage = VllmOmniConfig.from_registry("voxtral_tts").stage_by_id(0)
|
||||
|
||||
assert (stage.model_config.default_sampling_params or {}).get("extra_args")
|
||||
assert stage.model_config.has_sampling_extra_args is True
|
||||
|
||||
|
||||
def test_diffusion_config_preserves_existing_coercion_hooks():
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.data import AttentionConfig, DiffusionCacheConfig
|
||||
|
||||
cfg = omni_config_module._DiffusionConfigProjection(
|
||||
dtype="float32",
|
||||
cache_config={"rel_l1_thresh": 0.3},
|
||||
diffusion_attention_config={"default": "flash_attn"},
|
||||
diffusion_kv_cache_skip_steps="0-2,4",
|
||||
diffusion_kv_cache_skip_layers=[1, 3],
|
||||
)
|
||||
|
||||
assert cfg.dtype is torch.float32
|
||||
assert isinstance(cfg.cache_config, DiffusionCacheConfig)
|
||||
assert isinstance(cfg.diffusion_attention_config, AttentionConfig)
|
||||
assert cfg.diffusion_attention_config.default.backend == "flash_attn"
|
||||
assert cfg.diffusion_kv_cache_skip_step_indices == {0, 1, 2, 4}
|
||||
assert cfg.diffusion_kv_cache_skip_layer_indices == {1, 3}
|
||||
assert cfg.max_cpu_loras == 1
|
||||
|
||||
|
||||
def test_diffusion_config_from_kwargs_reuses_legacy_normalization(monkeypatch):
|
||||
monkeypatch.setenv("DIFFUSION_CACHE_BACKEND", "TEA_CACHE")
|
||||
|
||||
cfg = omni_config_module._DiffusionConfigProjection.from_kwargs(
|
||||
diffusion_attention_backend="flash_attn",
|
||||
kv_cache_dtype="fp8",
|
||||
kv_cache_skip_steps="0-1",
|
||||
kv_cache_skip_layers=[2],
|
||||
static_lora_scale=0.25,
|
||||
diffusers_load_kwargs=None,
|
||||
diffusers_call_kwargs=None,
|
||||
)
|
||||
|
||||
assert cfg.diffusion_attention_config.default.backend == "flash_attn"
|
||||
assert cfg.diffusion_kv_cache_dtype == "fp8"
|
||||
assert cfg.diffusion_kv_cache_skip_step_indices == {0, 1}
|
||||
assert cfg.diffusion_kv_cache_skip_layer_indices == {2}
|
||||
assert cfg.lora_scale == 0.25
|
||||
assert cfg.cache_backend == "tea_cache"
|
||||
assert cfg.diffusers_load_kwargs == {}
|
||||
assert cfg.diffusers_call_kwargs == {}
|
||||
|
||||
|
||||
def test_from_registry_normalizes_diffusion_config_aliases_from_engine_args(tmp_path):
|
||||
deploy_path = tmp_path / "dreamzero_diffusion_aliases.yaml"
|
||||
deploy_path.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"pipeline: dreamzero",
|
||||
"async_chunk: false",
|
||||
"stages:",
|
||||
" - stage_id: 0",
|
||||
" diffusion_attention_backend: flash_attn",
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
stage = VllmOmniConfig.from_registry(
|
||||
"dreamzero",
|
||||
deploy_config_path=str(deploy_path),
|
||||
).stage_by_id(0)
|
||||
|
||||
assert isinstance(stage, VllmOmniDiffusionStageConfig)
|
||||
assert stage.diffusion_config.diffusion_attention_config.default.backend == "flash_attn"
|
||||
|
||||
|
||||
def test_diffusion_config_field_classification_covers_current_fields():
|
||||
classified_fields = (
|
||||
omni_config_module._DIFFUSION_SHARED_CONFIG_FIELDS
|
||||
| omni_config_module._DIFFUSION_RUNTIME_CONFIG_FIELDS
|
||||
| omni_config_module._DIFFUSION_ONLY_CONFIG_FIELDS
|
||||
)
|
||||
|
||||
assert classified_fields == {f.name for f in fields(omni_config_module._DiffusionConfigProjection)}
|
||||
assert {
|
||||
"enable_prompt_embed_cache",
|
||||
"prompt_embed_cache_size",
|
||||
"diffusion_kv_cache_dtype",
|
||||
} <= omni_config_module._DIFFUSION_ONLY_CONFIG_FIELDS
|
||||
assert {
|
||||
"revision",
|
||||
"trust_remote_code",
|
||||
"distributed_executor_backend",
|
||||
} <= omni_config_module._DIFFUSION_SHARED_CONFIG_FIELDS
|
||||
assert "prompt_file_path" in omni_config_module._DIFFUSION_RUNTIME_CONFIG_FIELDS
|
||||
@@ -1,146 +1,68 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the central pipeline registry (2.5/N)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
"""Tests for out of tree registration to OMNI_PIPELINES."""
|
||||
|
||||
import pytest
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm_omni.config.pipeline_registry import _OMNI_PIPELINES
|
||||
from vllm_omni.config.stage_config import (
|
||||
_PIPELINE_REGISTRY,
|
||||
PipelineConfig,
|
||||
StageExecutionType,
|
||||
StagePipelineConfig,
|
||||
register_pipeline,
|
||||
)
|
||||
from vllm_omni.config.pipeline_registry import OMNI_PIPELINES, register_pipeline
|
||||
from vllm_omni.config.stage_config import PipelineConfig, pipeline_cfg_resolver
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class TestCentralRegistryDeclarations:
|
||||
"""Every in-tree pipeline must be declared exactly once in the central registry."""
|
||||
@pytest.fixture
|
||||
def custom_resolver():
|
||||
"""Build a reusable custom resolver for PipelineConfigs."""
|
||||
|
||||
def test_omni_entries_visible_in_registry(self):
|
||||
for key in _OMNI_PIPELINES:
|
||||
assert key in _PIPELINE_REGISTRY
|
||||
class CustomConfigType(PretrainedConfig):
|
||||
pass
|
||||
|
||||
def test_expected_omni_pipelines_present(self):
|
||||
# Guard against accidental removal during future refactors.
|
||||
assert "qwen2_5_omni" in _OMNI_PIPELINES
|
||||
assert "qwen2_5_omni_thinker_only" in _OMNI_PIPELINES
|
||||
assert "qwen3_omni_moe" in _OMNI_PIPELINES
|
||||
assert "aura_omni" in _OMNI_PIPELINES
|
||||
assert "qwen3_tts" in _OMNI_PIPELINES
|
||||
@pipeline_cfg_resolver(config_type=CustomConfigType)
|
||||
def custom_resolver(
|
||||
hf_config: CustomConfigType,
|
||||
) -> PipelineConfig:
|
||||
return PipelineConfig(model_type="resolved_type")
|
||||
|
||||
return custom_resolver
|
||||
|
||||
|
||||
class TestLazyLoading:
|
||||
"""Pipelines are imported only on first access."""
|
||||
|
||||
def test_contains_without_import(self):
|
||||
# ``in`` hits the lazy map, not the loaded cache.
|
||||
assert "qwen3_omni_moe" in _PIPELINE_REGISTRY
|
||||
|
||||
def test_getitem_loads_correct_pipeline(self):
|
||||
pipeline = _PIPELINE_REGISTRY["qwen3_omni_moe"]
|
||||
assert pipeline.model_type == "qwen3_omni_moe"
|
||||
assert pipeline.model_arch == "Qwen3OmniMoeForConditionalGeneration"
|
||||
|
||||
def test_getitem_loads_aura_omni_pipeline(self):
|
||||
pipeline = _PIPELINE_REGISTRY["aura_omni"]
|
||||
assert pipeline.model_type == "aura_omni"
|
||||
assert [stage.model_stage for stage in pipeline.stages] == [
|
||||
"asr",
|
||||
"aura",
|
||||
"qwen3_tts",
|
||||
"code2wav",
|
||||
]
|
||||
assert pipeline.stages[0].model_arch == "Qwen3ASRForConditionalGeneration"
|
||||
assert pipeline.stages[1].model_arch == "AuraQwen3VLForConditionalGeneration"
|
||||
assert pipeline.stages[2].model_arch == "Qwen3TTSTalkerForConditionalGeneration"
|
||||
assert pipeline.stages[3].model_arch == "Qwen3TTSCode2Wav"
|
||||
|
||||
def test_unknown_model_type_returns_none_via_get(self):
|
||||
assert _PIPELINE_REGISTRY.get("not_a_real_pipeline") is None
|
||||
|
||||
def test_unknown_model_type_raises_keyerror_via_getitem(self):
|
||||
with pytest.raises(KeyError):
|
||||
_PIPELINE_REGISTRY["not_a_real_pipeline"]
|
||||
|
||||
def test_iteration_yields_registered_pipelines(self):
|
||||
keys = set(_PIPELINE_REGISTRY)
|
||||
assert "qwen2_5_omni" in keys
|
||||
assert "qwen3_omni_moe" in keys
|
||||
|
||||
def test_iteration_resilience(self, monkeypatch, caplog):
|
||||
# Synthetic failure to verify iteration robustness against load errors.
|
||||
target_logger = logging.getLogger("vllm_omni.config.stage_config")
|
||||
target_logger.addHandler(caplog.handler)
|
||||
prev_level = target_logger.level
|
||||
target_logger.setLevel(logging.ERROR)
|
||||
try:
|
||||
monkeypatch.setattr(
|
||||
_PIPELINE_REGISTRY,
|
||||
"_lazy_map",
|
||||
{
|
||||
"healthy": ("vllm_omni.model_executor.models.qwen2_5_omni.pipeline", "QWEN2_5_OMNI_PIPELINE"),
|
||||
"broken": ("vllm_omni.non_existent_module", "SomeConfig"),
|
||||
"attr_fail": ("vllm_omni.config.stage_config", "NON_EXISTENT_VAR"),
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(_PIPELINE_REGISTRY, "_loaded", {})
|
||||
|
||||
# Assert that list(_PIPELINE_REGISTRY.values()) and dict(_PIPELINE_REGISTRY.items())
|
||||
# both yield only healthy entries.
|
||||
healthy_instance = _PIPELINE_REGISTRY["healthy"]
|
||||
assert list(_PIPELINE_REGISTRY.values()) == [healthy_instance]
|
||||
assert dict(_PIPELINE_REGISTRY.items()) == {"healthy": healthy_instance}
|
||||
assert "Failed to import pipeline module" in caplog.text
|
||||
assert "Pipeline variable 'NON_EXISTENT_VAR' not found" in caplog.text
|
||||
|
||||
finally:
|
||||
target_logger.removeHandler(caplog.handler)
|
||||
target_logger.setLevel(prev_level)
|
||||
def test_register_pipeline_config(clean_pipeline_registry):
|
||||
"""Ensure that we can register a custom pipeline config to OMNI_PIPELINES."""
|
||||
new_model_type = "new_model_type"
|
||||
pipe_cfg = PipelineConfig(model_type=new_model_type)
|
||||
assert new_model_type not in OMNI_PIPELINES
|
||||
register_pipeline(pipe_cfg)
|
||||
assert new_model_type in OMNI_PIPELINES
|
||||
assert OMNI_PIPELINES[new_model_type] is pipe_cfg
|
||||
|
||||
|
||||
class TestDynamicRegistration:
|
||||
"""``register_pipeline()`` still works for plugins and tests."""
|
||||
def test_register_pipeline_config_with_model_type(clean_pipeline_registry):
|
||||
"""Ensure that we can register a custom pipeline config with an explicit model_type to OMNI_PIPELINES."""
|
||||
new_model_type = "new_model_type"
|
||||
unused_model_type = "foo"
|
||||
pipe_cfg = PipelineConfig(model_type=unused_model_type)
|
||||
assert new_model_type not in OMNI_PIPELINES
|
||||
assert unused_model_type not in OMNI_PIPELINES
|
||||
|
||||
def test_register_adds_to_registry(self):
|
||||
custom = PipelineConfig(
|
||||
model_type="_test_dynamic_registration",
|
||||
model_arch="DynamicTestModel",
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="test",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
final_output=True,
|
||||
),
|
||||
),
|
||||
)
|
||||
register_pipeline(custom)
|
||||
try:
|
||||
assert "_test_dynamic_registration" in _PIPELINE_REGISTRY
|
||||
assert _PIPELINE_REGISTRY["_test_dynamic_registration"] is custom
|
||||
finally:
|
||||
# Don't leak the test registration into other tests.
|
||||
if "_test_dynamic_registration" in _PIPELINE_REGISTRY:
|
||||
del _PIPELINE_REGISTRY["_test_dynamic_registration"]
|
||||
# Registering with an explicitly provided model_type uses
|
||||
# the passed value instead of the pipeline_cfg.model_type
|
||||
register_pipeline(pipe_cfg, new_model_type)
|
||||
assert new_model_type in OMNI_PIPELINES
|
||||
assert unused_model_type not in OMNI_PIPELINES
|
||||
assert OMNI_PIPELINES[new_model_type] is pipe_cfg
|
||||
|
||||
def test_dynamic_registration_overrides_lazy_entry(self):
|
||||
# Build a substitute for qwen3_omni_moe that we can distinguish.
|
||||
original = _PIPELINE_REGISTRY["qwen3_omni_moe"]
|
||||
override = PipelineConfig(
|
||||
model_type="qwen3_omni_moe",
|
||||
model_arch="OverriddenArch",
|
||||
stages=original.stages,
|
||||
)
|
||||
register_pipeline(override)
|
||||
try:
|
||||
assert _PIPELINE_REGISTRY["qwen3_omni_moe"].model_arch == "OverriddenArch"
|
||||
finally:
|
||||
# Remove the dynamic override so later tests see the original.
|
||||
if "qwen3_omni_moe" in _PIPELINE_REGISTRY._loaded:
|
||||
del _PIPELINE_REGISTRY["qwen3_omni_moe"]
|
||||
|
||||
def test_register_resolver(custom_resolver, clean_pipeline_registry):
|
||||
"""Ensure that we can register a custom resolver to OMNI_PIPELINES."""
|
||||
new_model_type = "new_model_type"
|
||||
assert new_model_type not in OMNI_PIPELINES
|
||||
register_pipeline(custom_resolver, new_model_type)
|
||||
assert new_model_type in OMNI_PIPELINES
|
||||
assert OMNI_PIPELINES[new_model_type] is custom_resolver
|
||||
|
||||
|
||||
def test_register_resolver_requires_model_type(custom_resolver, clean_pipeline_registry):
|
||||
"""Ensure that registering a custom resolver to OMNI_PIPELINES requires an explicit model_type."""
|
||||
with pytest.raises(ValueError):
|
||||
register_pipeline(custom_resolver)
|
||||
@@ -17,6 +17,7 @@ from tests.model_executor.helpers import bootstrap_vllm_layer_custom_op_modules
|
||||
bootstrap_vllm_layer_custom_op_modules()
|
||||
|
||||
pytest_plugins = (
|
||||
"tests.helpers.fixtures.config",
|
||||
"tests.helpers.fixtures.env",
|
||||
"tests.helpers.fixtures.log",
|
||||
"tests.helpers.fixtures.run_args",
|
||||
|
||||
@@ -115,6 +115,7 @@ class TestFullPayloadCoordinatorSelection(unittest.TestCase):
|
||||
("MiMoAudioModel", "code2wav"),
|
||||
("Qwen3TTSCode2Wav", "code2wav"),
|
||||
("CosyVoice3Model", "cosyvoice3_code2wav"),
|
||||
("IndexTTS2S2MelDecoder", "indextts2_s2mel_decoder"),
|
||||
("DyninOmniForConditionalGeneration", "token2image"),
|
||||
("DyninOmniForConditionalGeneration", "token2audio"),
|
||||
}
|
||||
|
||||
@@ -6,9 +6,13 @@ The detailed RFC document is maintained outside this repository (local/internal
|
||||
- Fault helpers: `conftest.py`, `helpers.py`
|
||||
- Qwen3 tests: `test_reliability_qwen3_omni.py`
|
||||
- Wan2.2 tests: `test_reliability_wan22.py`
|
||||
- HunyuanImage tests: `test_reliability_hunyuan_image.py`
|
||||
- VoxCPM2 tests: `test_reliability_voxcpm2.py`
|
||||
|
||||
```bash
|
||||
pytest --collect-only tests/dfx/reliability
|
||||
pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m slow
|
||||
pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m slow
|
||||
pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m slow
|
||||
pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m slow
|
||||
```
|
||||
@@ -621,7 +621,6 @@ def _voices_upload_multipart_files(kind: str | None) -> dict[str, Any] | None:
|
||||
{"consent": "c1", "name": "v1"},
|
||||
("audio_sample", "speaker_embedding", "provided"),
|
||||
id="neither_audio_nor_embedding",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"wav_ok",
|
||||
@@ -673,7 +672,6 @@ def _voices_upload_multipart_files(kind: str | None) -> dict[str, Any] | None:
|
||||
{"consent": "consent_ok", "name": "evil/name"},
|
||||
("voice name", "invalid voice name"),
|
||||
id="invalid_name_path_sep",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"wav_ok",
|
||||
@@ -698,21 +696,18 @@ def _voices_upload_multipart_files(kind: str | None) -> dict[str, Any] | None:
|
||||
{"consent": "consent_ok", "name": "v_bad_mime_{uuid}"},
|
||||
("mime", "unsupported", "audio/mp4"),
|
||||
id="audio_unsupported_mime",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"wav_short",
|
||||
{"consent": "consent_ok", "name": "v_short_{uuid}"},
|
||||
("too short", "reference audio"),
|
||||
id="audio_too_short",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"wav_bad_body",
|
||||
{"consent": "consent_ok", "name": "v_bad_wav_{uuid}"},
|
||||
("decode", "audio", "Format not recognised"),
|
||||
id="audio_decode_error",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
None,
|
||||
@@ -723,7 +718,6 @@ def _voices_upload_multipart_files(kind: str | None) -> dict[str, Any] | None:
|
||||
},
|
||||
("speaker_embedding", "valid JSON"),
|
||||
id="speaker_embedding_invalid_json",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
None,
|
||||
@@ -734,7 +728,6 @@ def _voices_upload_multipart_files(kind: str | None) -> dict[str, Any] | None:
|
||||
},
|
||||
("speaker_embedding", "non-empty"),
|
||||
id="speaker_embedding_empty",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
None,
|
||||
@@ -745,7 +738,6 @@ def _voices_upload_multipart_files(kind: str | None) -> dict[str, Any] | None:
|
||||
},
|
||||
("speaker_embedding", "finite"),
|
||||
id="speaker_embedding_nan",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
None,
|
||||
@@ -756,7 +748,6 @@ def _voices_upload_multipart_files(kind: str | None) -> dict[str, Any] | None:
|
||||
},
|
||||
("expected", "dimensions", "1024"),
|
||||
id="speaker_embedding_wrong_dim",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"wav_ok",
|
||||
@@ -767,7 +758,6 @@ def _voices_upload_multipart_files(kind: str | None) -> dict[str, Any] | None:
|
||||
},
|
||||
("mutually exclusive", "audio_sample", "speaker_embedding"),
|
||||
id="audio_and_embedding_mutually_exclusive",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
@@ -0,0 +1,478 @@
|
||||
"""
|
||||
HunyuanImage-3.0-Instruct reliability integration tests (DiT-only deploy).
|
||||
|
||||
Uses ``hunyuan_image3_dit.yaml`` (single diffusion stage, 4-GPU TP=4, no
|
||||
AR/Mooncake connector) for simpler local bring-up. Scenarios follow
|
||||
``docs/user_guide/fault_injection_reliability_matrix.md``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import concurrent.futures
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.dfx.conftest import (
|
||||
assert_fault_exception,
|
||||
create_reliability_omni_server_params,
|
||||
resolve_oom_device_spec,
|
||||
)
|
||||
from tests.dfx.reliability.helpers import (
|
||||
FaultInjector,
|
||||
assert_no_server_tree_process_residual_and_gpu_release,
|
||||
get_health_raw,
|
||||
inject_gpu_oom,
|
||||
make_process_kill_fault_injector,
|
||||
make_server_root_kill_fault_injector,
|
||||
make_server_tree_kill_fault_injector,
|
||||
post_json_raw,
|
||||
run_fault_injection_with_rate_load,
|
||||
stop_gpu_oom_hogs,
|
||||
worker_residual_timeout_after_kill_signal,
|
||||
)
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OpenAIClientHandler
|
||||
|
||||
RELIABILITY_SCENARIOS: list[dict[str, Any]] = [
|
||||
{
|
||||
"test_name": "hunyuan_image_dit_reliability_default",
|
||||
"server_params": {
|
||||
"model": "tencent/HunyuanImage-3.0-Instruct",
|
||||
"stage_config_name": "hunyuan_image3_dit.yaml",
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
DEPLOY_CONFIGS_DIR = Path(__file__).resolve().parent.parent.parent.parent / "vllm_omni" / "deploy"
|
||||
OOM_INJECTION_CONFIG = {
|
||||
"target_mem_ratio": 0.95,
|
||||
"hold_seconds": 0,
|
||||
"startup_timeout_sec": 20,
|
||||
"strict": True,
|
||||
}
|
||||
FAULT_ERROR_KEYWORDS = (
|
||||
"oom",
|
||||
"out of memory",
|
||||
"cuda",
|
||||
"job failed",
|
||||
"unknown error",
|
||||
"internal server error",
|
||||
"500 server error",
|
||||
)
|
||||
PROCESS_KILL_ERROR_KEYWORDS = (
|
||||
"timeout",
|
||||
"did not complete within",
|
||||
"connection",
|
||||
"engine",
|
||||
"orchestrator",
|
||||
"dead",
|
||||
"internal",
|
||||
"500",
|
||||
"503",
|
||||
)
|
||||
# Matrix: worker SIGTERM/SIGKILL; DiT-only HunyuanImage workers are vLLM-Omni::.
|
||||
WORKER_KILL_PATTERNS = ("vLLM-Omni::",)
|
||||
SERVE_SIGNAL_PARAMS = [
|
||||
pytest.param("SIGTERM", id="sigterm"),
|
||||
pytest.param("SIGINT", id="sigint"),
|
||||
pytest.param("SIGKILL", id="sigkill"),
|
||||
]
|
||||
TREE_SIGNAL_PARAMS = [
|
||||
pytest.param("SIGTERM", id="sigterm"),
|
||||
pytest.param("SIGKILL", id="sigkill"),
|
||||
]
|
||||
TREE_WITH_LOAD_SIGNAL_PARAMS = [
|
||||
pytest.param("SIGTERM", id="sigterm"),
|
||||
pytest.param("SIGKILL", id="sigkill"),
|
||||
]
|
||||
WORKER_SIGNAL_FAULT_PARAMS = [
|
||||
pytest.param(
|
||||
make_process_kill_fault_injector(
|
||||
grep_patterns=WORKER_KILL_PATTERNS,
|
||||
signal_name="SIGTERM",
|
||||
limit=1,
|
||||
post_kill_wait_seconds=2.0,
|
||||
),
|
||||
id="runtime_process_chain_sigterm",
|
||||
),
|
||||
pytest.param(
|
||||
make_process_kill_fault_injector(
|
||||
grep_patterns=WORKER_KILL_PATTERNS,
|
||||
signal_name="SIGKILL",
|
||||
limit=1,
|
||||
post_kill_wait_seconds=2.0,
|
||||
),
|
||||
id="runtime_process_chain_sigkill",
|
||||
),
|
||||
]
|
||||
|
||||
HUNYUAN_PARAMS = create_reliability_omni_server_params(RELIABILITY_SCENARIOS, DEPLOY_CONFIGS_DIR)
|
||||
INFLIGHT_INJECTION_REQUEST_RATE = 0.3
|
||||
INFLIGHT_INJECTION_REQUEST_COUNT = 10
|
||||
|
||||
|
||||
def _image_request_config(omni_server: Any) -> dict[str, Any]:
|
||||
return {
|
||||
"json": {
|
||||
"model": omni_server.model,
|
||||
"prompt": "A simple red apple on a white background.",
|
||||
"size": "512x512",
|
||||
"n": 1,
|
||||
"response_format": "b64_json",
|
||||
"num_inference_steps": 4,
|
||||
"guidance_scale": 7.5,
|
||||
"seed": 42,
|
||||
"bot_task": "none",
|
||||
"use_system_prompt": "en_unified",
|
||||
},
|
||||
"timeout": 600,
|
||||
}
|
||||
|
||||
|
||||
def _submit_image_generation(openai_client: OpenAIClientHandler, omni_server: Any) -> None:
|
||||
responses = openai_client.send_images_generations_http_request(_image_request_config(omni_server))
|
||||
resp = responses[0]
|
||||
if resp.status_code >= 400:
|
||||
raise RuntimeError(
|
||||
f"/v1/images/generations failed: status={resp.status_code}, body={getattr(resp, 'error_message', '')!r}"
|
||||
)
|
||||
|
||||
|
||||
def _assert_post_fault_image_fast_fail(host: str, port: int, *, model: str, scenario: str) -> None:
|
||||
payload = {
|
||||
"model": model,
|
||||
"prompt": "post-fault fast-fail check",
|
||||
"size": "512x512",
|
||||
"n": 1,
|
||||
"response_format": "b64_json",
|
||||
"num_inference_steps": 4,
|
||||
"bot_task": "none",
|
||||
"use_system_prompt": "en_unified",
|
||||
}
|
||||
start = time.monotonic()
|
||||
try:
|
||||
status, body = post_json_raw(host, port, "/v1/images/generations", payload, timeout_sec=20)
|
||||
elapsed = time.monotonic() - start
|
||||
assert elapsed < 15, f"[{scenario} fast_fail] /v1/images/generations did not fail fast: {elapsed:.2f}s"
|
||||
assert status >= 500, (
|
||||
f"[{scenario} fast_fail] expected server-side error after fault, got status={status}, body={body[:200]!r}"
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
elapsed = time.monotonic() - start
|
||||
assert elapsed < 15, f"[{scenario} fast_fail] exception was too slow after fault: {elapsed:.2f}s"
|
||||
|
||||
|
||||
def _assert_post_fault_health_terminal(host: str, port: int, *, scenario: str) -> None:
|
||||
deadline = time.monotonic() + 20.0
|
||||
last_observation = ""
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
status, body = get_health_raw(host, port, timeout_sec=5)
|
||||
last_observation = f"http={status}, body={body[:200]!r}"
|
||||
if status == 503:
|
||||
return
|
||||
except Exception as exc: # noqa: BLE001
|
||||
last_observation = f"exception={exc!r}"
|
||||
return
|
||||
time.sleep(0.5)
|
||||
pytest.fail(f"[{scenario} health] no terminal post-fault health observed: {last_observation}")
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.skip(reason="issue#4285")
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
||||
@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True)
|
||||
def test_reliability_fault_gpu_oom_image_large_request_failure(
|
||||
omni_server_function,
|
||||
openai_client_function,
|
||||
) -> None:
|
||||
stage_config_path = getattr(omni_server_function, "stage_config_path", None)
|
||||
device_spec = resolve_oom_device_spec(OOM_INJECTION_CONFIG, stage_config_path)
|
||||
handle = inject_gpu_oom(
|
||||
device=device_spec,
|
||||
target_mem_ratio=OOM_INJECTION_CONFIG["target_mem_ratio"],
|
||||
hold_seconds=OOM_INJECTION_CONFIG["hold_seconds"],
|
||||
startup_timeout_sec=OOM_INJECTION_CONFIG["startup_timeout_sec"],
|
||||
strict=OOM_INJECTION_CONFIG["strict"],
|
||||
)
|
||||
try:
|
||||
try:
|
||||
_submit_image_generation(openai_client_function, omni_server_function)
|
||||
except Exception as exc:
|
||||
assert_fault_exception(exc, FAULT_ERROR_KEYWORDS)
|
||||
else:
|
||||
pytest.fail("expected large /v1/images/generations request failure during GPU OOM injection")
|
||||
finally:
|
||||
stop_gpu_oom_hogs(handle)
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
|
||||
@pytest.mark.parametrize(
|
||||
"fault_injector",
|
||||
WORKER_SIGNAL_FAULT_PARAMS,
|
||||
indirect=True,
|
||||
)
|
||||
@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True)
|
||||
def test_reliability_fault_process_kill_request_failure(
|
||||
omni_server_after_fault_function,
|
||||
openai_client_function,
|
||||
) -> None:
|
||||
try:
|
||||
_submit_image_generation(openai_client_function, omni_server_after_fault_function)
|
||||
except Exception as exc:
|
||||
assert_fault_exception(exc, PROCESS_KILL_ERROR_KEYWORDS)
|
||||
else:
|
||||
pytest.fail("expected /v1/images/generations request failure after process-kill injection")
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.skip(reason="issue#4522")
|
||||
@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
|
||||
@pytest.mark.parametrize(
|
||||
"fault_injector",
|
||||
WORKER_SIGNAL_FAULT_PARAMS,
|
||||
indirect=True,
|
||||
)
|
||||
@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True)
|
||||
def test_reliability_fault_process_kill_health_fast_fail_and_concurrent(
|
||||
omni_server_after_fault_function,
|
||||
openai_client_function,
|
||||
) -> None:
|
||||
host = omni_server_after_fault_function.host
|
||||
port = omni_server_after_fault_function.port
|
||||
model = omni_server_after_fault_function.model
|
||||
|
||||
payload = {
|
||||
"model": model,
|
||||
"prompt": "fast-fail check",
|
||||
"size": "512x512",
|
||||
"n": 1,
|
||||
"response_format": "b64_json",
|
||||
"num_inference_steps": 4,
|
||||
"bot_task": "none",
|
||||
"use_system_prompt": "en_unified",
|
||||
}
|
||||
start = time.monotonic()
|
||||
try:
|
||||
status, body = post_json_raw(host, port, "/v1/images/generations", payload, timeout_sec=20)
|
||||
elapsed = time.monotonic() - start
|
||||
assert elapsed < 15, (
|
||||
f"[process_kill fast_fail] /v1/images/generations did not fail fast after fault: {elapsed:.2f}s"
|
||||
)
|
||||
assert status >= 500, (
|
||||
"[process_kill fast_fail] expected server-side error after fatal fault, "
|
||||
f"got status={status}, body={body[:200]!r}"
|
||||
)
|
||||
except Exception:
|
||||
elapsed = time.monotonic() - start
|
||||
assert elapsed < 15, (
|
||||
f"[process_kill fast_fail] /v1/images/generations exception was too slow after fault: {elapsed:.2f}s"
|
||||
)
|
||||
|
||||
start = time.monotonic()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor:
|
||||
futures = [
|
||||
executor.submit(_submit_image_generation, openai_client_function, omni_server_after_fault_function)
|
||||
for _ in range(3)
|
||||
]
|
||||
done, pending = concurrent.futures.wait(
|
||||
futures,
|
||||
timeout=40,
|
||||
return_when=concurrent.futures.ALL_COMPLETED,
|
||||
)
|
||||
|
||||
elapsed = time.monotonic() - start
|
||||
assert not pending, f"[process_kill concurrent] some fault-time image requests hung: pending={len(pending)}"
|
||||
assert elapsed < 40, f"[process_kill concurrent] fault-time image request convergence is too slow: {elapsed:.2f}s"
|
||||
|
||||
fault_observed = False
|
||||
for future in done:
|
||||
try:
|
||||
future.result()
|
||||
except Exception as exc:
|
||||
fault_observed = True
|
||||
assert_fault_exception(exc, PROCESS_KILL_ERROR_KEYWORDS)
|
||||
assert fault_observed, (
|
||||
"[process_kill concurrent] expected at least one /v1/images/generations request to fail after fault"
|
||||
)
|
||||
|
||||
deadline = time.monotonic() + 20.0
|
||||
last_observation = ""
|
||||
final_health_status: int | None = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
status, body = get_health_raw(host, port, timeout_sec=5)
|
||||
last_observation = f"http={status}, body={body[:200]!r}"
|
||||
final_health_status = status
|
||||
if status == 503:
|
||||
break
|
||||
except Exception as exc: # noqa: BLE001
|
||||
last_observation = f"exception={exc!r}"
|
||||
time.sleep(0.5)
|
||||
assert final_health_status == 503, (
|
||||
"[process_kill health] expected /health 503 after fatal fault, "
|
||||
f"got status={final_health_status}, last_observation={last_observation}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.skip(reason="issue#4526")
|
||||
@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
|
||||
@pytest.mark.parametrize(
|
||||
"fault_injector",
|
||||
WORKER_SIGNAL_FAULT_PARAMS,
|
||||
indirect=True,
|
||||
)
|
||||
@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True)
|
||||
def test_reliability_fault_process_kill_worker_with_load_request_failure(
|
||||
omni_server_function,
|
||||
openai_client_function,
|
||||
fault_injector: FaultInjector,
|
||||
) -> None:
|
||||
scenario = "kill_worker_with_load"
|
||||
load_result = run_fault_injection_with_rate_load(
|
||||
submit_request=lambda: _submit_image_generation(openai_client_function, omni_server_function),
|
||||
inject_fault=lambda: fault_injector(omni_server_function),
|
||||
num_requests=INFLIGHT_INJECTION_REQUEST_COUNT,
|
||||
request_rate=INFLIGHT_INJECTION_REQUEST_RATE,
|
||||
completion_timeout_sec=120.0,
|
||||
)
|
||||
assert load_result["failure_observed"], (
|
||||
f"[{scenario}] expected at least one load request failure after fault; load_result={load_result}"
|
||||
)
|
||||
host = omni_server_function.host
|
||||
port = omni_server_function.port
|
||||
_assert_post_fault_image_fast_fail(host, port, model=omni_server_function.model, scenario=scenario)
|
||||
_assert_post_fault_health_terminal(host, port, scenario=scenario)
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
||||
@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
|
||||
@pytest.mark.parametrize("signal_name", SERVE_SIGNAL_PARAMS)
|
||||
@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True)
|
||||
def test_reliability_fault_process_kill_serve_root_with_load_fast_fail_and_cleanup(
|
||||
omni_server_function,
|
||||
openai_client_function,
|
||||
signal_name: str,
|
||||
) -> None:
|
||||
scenario = f"kill_serve_root_with_load_{signal_name.lower()}"
|
||||
injector = make_server_root_kill_fault_injector(signal_name=signal_name, post_kill_wait_seconds=2.0)
|
||||
load_result = run_fault_injection_with_rate_load(
|
||||
submit_request=lambda: _submit_image_generation(openai_client_function, omni_server_function),
|
||||
inject_fault=lambda: injector(omni_server_function),
|
||||
num_requests=INFLIGHT_INJECTION_REQUEST_COUNT,
|
||||
request_rate=INFLIGHT_INJECTION_REQUEST_RATE,
|
||||
completion_timeout_sec=120.0,
|
||||
)
|
||||
assert load_result["failure_observed"], (
|
||||
f"[{scenario}] expected at least one load request failure after fault; load_result={load_result}"
|
||||
)
|
||||
host = omni_server_function.host
|
||||
port = omni_server_function.port
|
||||
_assert_post_fault_image_fast_fail(host, port, model=omni_server_function.model, scenario=scenario)
|
||||
_assert_post_fault_health_terminal(host, port, scenario=scenario)
|
||||
assert_no_server_tree_process_residual_and_gpu_release(
|
||||
omni_server_function,
|
||||
scenario=scenario,
|
||||
timeout_sec=worker_residual_timeout_after_kill_signal(signal_name),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
||||
@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
|
||||
@pytest.mark.parametrize("signal_name", SERVE_SIGNAL_PARAMS)
|
||||
@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True)
|
||||
def test_reliability_fault_process_kill_serve_root_no_load_fast_fail_and_cleanup(
|
||||
omni_server_function,
|
||||
signal_name: str,
|
||||
) -> None:
|
||||
scenario = f"kill_serve_root_no_load_{signal_name.lower()}"
|
||||
injector = make_server_root_kill_fault_injector(signal_name=signal_name, post_kill_wait_seconds=2.0)
|
||||
injector(omni_server_function)
|
||||
host = omni_server_function.host
|
||||
port = omni_server_function.port
|
||||
_assert_post_fault_image_fast_fail(host, port, model=omni_server_function.model, scenario=scenario)
|
||||
_assert_post_fault_health_terminal(host, port, scenario=scenario)
|
||||
assert_no_server_tree_process_residual_and_gpu_release(
|
||||
omni_server_function,
|
||||
scenario=scenario,
|
||||
timeout_sec=worker_residual_timeout_after_kill_signal(signal_name),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
|
||||
@pytest.mark.parametrize("signal_name", TREE_SIGNAL_PARAMS)
|
||||
@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True)
|
||||
def test_reliability_fault_process_kill_tree_no_load_fast_fail_and_cleanup(
|
||||
omni_server_function,
|
||||
signal_name: str,
|
||||
) -> None:
|
||||
scenario = f"kill_serve_tree_no_load_{signal_name.lower()}"
|
||||
injector = make_server_tree_kill_fault_injector(
|
||||
signal_name=signal_name,
|
||||
post_kill_wait_seconds=2.0,
|
||||
inter_kill_wait_seconds=0.1,
|
||||
)
|
||||
injector(omni_server_function)
|
||||
host = omni_server_function.host
|
||||
port = omni_server_function.port
|
||||
_assert_post_fault_image_fast_fail(host, port, model=omni_server_function.model, scenario=scenario)
|
||||
_assert_post_fault_health_terminal(host, port, scenario=scenario)
|
||||
assert_no_server_tree_process_residual_and_gpu_release(
|
||||
omni_server_function,
|
||||
scenario=scenario,
|
||||
timeout_sec=worker_residual_timeout_after_kill_signal(signal_name),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.skip(reason="issue#4526")
|
||||
@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
|
||||
@pytest.mark.parametrize("signal_name", TREE_WITH_LOAD_SIGNAL_PARAMS)
|
||||
@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True)
|
||||
def test_reliability_fault_process_kill_tree_with_load_fast_fail_and_cleanup(
|
||||
omni_server_function,
|
||||
openai_client_function,
|
||||
signal_name: str,
|
||||
) -> None:
|
||||
scenario = f"kill_serve_tree_with_load_{signal_name.lower()}"
|
||||
injector = make_server_tree_kill_fault_injector(
|
||||
signal_name=signal_name,
|
||||
post_kill_wait_seconds=2.0,
|
||||
inter_kill_wait_seconds=0.1,
|
||||
)
|
||||
load_result = run_fault_injection_with_rate_load(
|
||||
submit_request=lambda: _submit_image_generation(openai_client_function, omni_server_function),
|
||||
inject_fault=lambda: injector(omni_server_function),
|
||||
num_requests=INFLIGHT_INJECTION_REQUEST_COUNT,
|
||||
request_rate=INFLIGHT_INJECTION_REQUEST_RATE,
|
||||
completion_timeout_sec=120.0,
|
||||
)
|
||||
assert load_result["failure_observed"], (
|
||||
f"[{scenario}] expected at least one load request failure after fault; load_result={load_result}"
|
||||
)
|
||||
host = omni_server_function.host
|
||||
port = omni_server_function.port
|
||||
_assert_post_fault_image_fast_fail(host, port, model=omni_server_function.model, scenario=scenario)
|
||||
_assert_post_fault_health_terminal(host, port, scenario=scenario)
|
||||
assert_no_server_tree_process_residual_and_gpu_release(
|
||||
omni_server_function,
|
||||
scenario=scenario,
|
||||
timeout_sec=worker_residual_timeout_after_kill_signal(signal_name),
|
||||
)
|
||||
@@ -0,0 +1,505 @@
|
||||
"""
|
||||
VoxCPM2 reliability integration tests.
|
||||
|
||||
Scenarios follow ``docs/user_guide/fault_injection_reliability_matrix.md``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import concurrent.futures
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.dfx.conftest import (
|
||||
assert_fault_exception,
|
||||
create_reliability_omni_server_params,
|
||||
resolve_oom_device_spec,
|
||||
)
|
||||
from tests.dfx.reliability.helpers import (
|
||||
FaultInjector,
|
||||
assert_no_server_tree_process_residual_and_gpu_release,
|
||||
extract_openai_error_contract_from_bytes,
|
||||
get_health_raw,
|
||||
inject_gpu_oom,
|
||||
make_process_kill_fault_injector,
|
||||
make_server_root_kill_fault_injector,
|
||||
make_server_tree_kill_fault_injector,
|
||||
post_json_raw,
|
||||
run_fault_injection_with_rate_load,
|
||||
stop_gpu_oom_hogs,
|
||||
worker_residual_timeout_after_kill_signal,
|
||||
)
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.media import load_test_audio_data_url
|
||||
from tests.helpers.runtime import OpenAIClientHandler
|
||||
|
||||
RELIABILITY_SCENARIOS: list[dict[str, Any]] = [
|
||||
{
|
||||
"test_name": "voxcpm2_reliability_default",
|
||||
"server_params": {
|
||||
"model": "openbmb/VoxCPM2",
|
||||
"stage_config_name": "voxcpm2.yaml",
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
DEPLOY_CONFIGS_DIR = Path(__file__).resolve().parent.parent.parent.parent / "vllm_omni" / "deploy"
|
||||
OOM_INJECTION_CONFIG = {
|
||||
"target_mem_ratio": 0.96,
|
||||
"hold_seconds": 0,
|
||||
"startup_timeout_sec": 20,
|
||||
"strict": False,
|
||||
}
|
||||
FAULT_ERROR_KEYWORDS = (
|
||||
"the request failed",
|
||||
"oom",
|
||||
"out of memory",
|
||||
"cuda",
|
||||
"orchestrator",
|
||||
"timeout",
|
||||
"connection",
|
||||
"500",
|
||||
"503",
|
||||
)
|
||||
PROCESS_KILL_ERROR_KEYWORDS = (
|
||||
"timeout",
|
||||
"did not complete within",
|
||||
"connection",
|
||||
"engine",
|
||||
"orchestrator",
|
||||
"dead",
|
||||
"internal",
|
||||
"500",
|
||||
"503",
|
||||
)
|
||||
RUNTIME_WORKER_PATTERN = "VLLM::"
|
||||
SERVE_SIGNAL_PARAMS = [
|
||||
pytest.param("SIGTERM", id="sigterm"),
|
||||
pytest.param("SIGINT", id="sigint"),
|
||||
pytest.param("SIGKILL", id="sigkill"),
|
||||
]
|
||||
SERVE_WITH_LOAD_SIGNAL_PARAMS = [
|
||||
pytest.param("SIGTERM", id="sigterm"),
|
||||
pytest.param("SIGINT", id="sigint"),
|
||||
pytest.param("SIGKILL", id="sigkill"),
|
||||
]
|
||||
TREE_SIGNAL_PARAMS = [
|
||||
pytest.param("SIGTERM", id="sigterm"),
|
||||
pytest.param("SIGKILL", id="sigkill"),
|
||||
]
|
||||
TREE_WITH_LOAD_SIGNAL_PARAMS = [
|
||||
pytest.param("SIGTERM", id="sigterm"),
|
||||
pytest.param("SIGKILL", id="sigkill"),
|
||||
]
|
||||
WORKER_SIGNAL_FAULT_PARAMS = [
|
||||
pytest.param(
|
||||
make_process_kill_fault_injector(
|
||||
grep_patterns=RUNTIME_WORKER_PATTERN,
|
||||
signal_name="SIGTERM",
|
||||
limit=1,
|
||||
post_kill_wait_seconds=2.0,
|
||||
),
|
||||
id="runtime_process_chain_sigterm",
|
||||
),
|
||||
pytest.param(
|
||||
make_process_kill_fault_injector(
|
||||
grep_patterns=RUNTIME_WORKER_PATTERN,
|
||||
signal_name="SIGKILL",
|
||||
limit=1,
|
||||
post_kill_wait_seconds=2.0,
|
||||
),
|
||||
id="runtime_process_chain_sigkill",
|
||||
),
|
||||
]
|
||||
|
||||
VOXCPM2_PARAMS = create_reliability_omni_server_params(RELIABILITY_SCENARIOS, DEPLOY_CONFIGS_DIR)
|
||||
INFLIGHT_INJECTION_REQUEST_RATE = 0.3
|
||||
INFLIGHT_INJECTION_REQUEST_COUNT = 10
|
||||
_SPEECH_FF_HTTP_TIMEOUT_SEC = 30
|
||||
_SPEECH_FF_MAX_ELAPSED_SEC = 30
|
||||
|
||||
|
||||
def _speech_request_config(omni_server: Any) -> dict[str, Any]:
|
||||
return {
|
||||
"model": omni_server.model,
|
||||
"input": "The weather is nice today, perfect for a walk in the park.",
|
||||
"stream": False,
|
||||
"timeout": 120.0,
|
||||
"response_format": "wav",
|
||||
"voice": "default",
|
||||
}
|
||||
|
||||
|
||||
def _submit_speech_request(openai_client: OpenAIClientHandler, omni_server: Any) -> None:
|
||||
openai_client.send_audio_speech_request(_speech_request_config(omni_server), request_num=1)
|
||||
|
||||
|
||||
def _oom_speech_request_config(omni_server: Any) -> dict[str, Any]:
|
||||
base = "The weather is nice today, perfect for a walk in the park. "
|
||||
return {
|
||||
"model": omni_server.model,
|
||||
"input": base * 80,
|
||||
"max_new_tokens": 4096,
|
||||
"ref_audio": load_test_audio_data_url("qwen3_tts/clone_2.wav"),
|
||||
"stream": False,
|
||||
"timeout": 300.0,
|
||||
"response_format": "wav",
|
||||
"voice": "default",
|
||||
}
|
||||
|
||||
|
||||
def _submit_oom_speech_request(openai_client: OpenAIClientHandler, omni_server: Any) -> None:
|
||||
openai_client.send_audio_speech_request(_oom_speech_request_config(omni_server), request_num=1)
|
||||
|
||||
|
||||
def _assert_post_fault_speech_fast_fail(host: str, port: int, *, model: str, scenario: str) -> None:
|
||||
payload = {
|
||||
"model": model,
|
||||
"input": "post-fault fast-fail check",
|
||||
"voice": "default",
|
||||
}
|
||||
start = time.monotonic()
|
||||
try:
|
||||
status, body = post_json_raw(host, port, "/v1/audio/speech", payload, timeout_sec=20)
|
||||
elapsed = time.monotonic() - start
|
||||
assert elapsed < 15, f"[{scenario} fast_fail] /v1/audio/speech did not fail fast: {elapsed:.2f}s"
|
||||
assert status >= 500, (
|
||||
f"[{scenario} fast_fail] expected server-side error after fault, got status={status}, body={body[:200]!r}"
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
elapsed = time.monotonic() - start
|
||||
assert elapsed < 15, f"[{scenario} fast_fail] exception was too slow after fault: {elapsed:.2f}s"
|
||||
|
||||
|
||||
def _assert_post_fault_health_terminal(host: str, port: int, *, scenario: str) -> None:
|
||||
deadline = time.monotonic() + 20.0
|
||||
last_observation = ""
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
status, body = get_health_raw(host, port, timeout_sec=5)
|
||||
last_observation = f"http={status}, body={body[:200]!r}"
|
||||
if status == 503:
|
||||
return
|
||||
except Exception as exc: # noqa: BLE001
|
||||
last_observation = f"exception={exc!r}"
|
||||
return
|
||||
time.sleep(0.5)
|
||||
pytest.fail(f"[{scenario} health] no terminal post-fault health observed: {last_observation}")
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.tts
|
||||
@pytest.mark.skip(reason="issue#4285")
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True)
|
||||
def test_reliability_fault_gpu_oom_speech_request_failure(
|
||||
omni_server_function,
|
||||
openai_client_function,
|
||||
) -> None:
|
||||
stage_config_path = getattr(omni_server_function, "stage_config_path", None)
|
||||
device_spec = resolve_oom_device_spec(OOM_INJECTION_CONFIG, stage_config_path)
|
||||
handle = inject_gpu_oom(
|
||||
device=device_spec,
|
||||
target_mem_ratio=OOM_INJECTION_CONFIG["target_mem_ratio"],
|
||||
hold_seconds=OOM_INJECTION_CONFIG["hold_seconds"],
|
||||
startup_timeout_sec=OOM_INJECTION_CONFIG["startup_timeout_sec"],
|
||||
strict=OOM_INJECTION_CONFIG["strict"],
|
||||
)
|
||||
try:
|
||||
try:
|
||||
_submit_oom_speech_request(openai_client_function, omni_server_function)
|
||||
except Exception as exc:
|
||||
assert_fault_exception(exc, FAULT_ERROR_KEYWORDS)
|
||||
else:
|
||||
pytest.fail("expected /v1/audio/speech request failure during GPU OOM injection")
|
||||
finally:
|
||||
stop_gpu_oom_hogs(handle)
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.tts
|
||||
@pytest.mark.skip(reason="issue#4285")
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True)
|
||||
def test_reliability_fault_gpu_oom_speech_error_contract(
|
||||
omni_server_function,
|
||||
) -> None:
|
||||
stage_config_path = getattr(omni_server_function, "stage_config_path", None)
|
||||
device_spec = resolve_oom_device_spec(OOM_INJECTION_CONFIG, stage_config_path)
|
||||
handle = inject_gpu_oom(
|
||||
device=device_spec,
|
||||
target_mem_ratio=OOM_INJECTION_CONFIG["target_mem_ratio"],
|
||||
hold_seconds=OOM_INJECTION_CONFIG["hold_seconds"],
|
||||
startup_timeout_sec=OOM_INJECTION_CONFIG["startup_timeout_sec"],
|
||||
strict=OOM_INJECTION_CONFIG["strict"],
|
||||
)
|
||||
host = omni_server_function.host
|
||||
port = omni_server_function.port
|
||||
oom_config = _oom_speech_request_config(omni_server_function)
|
||||
payload = {
|
||||
"model": oom_config["model"],
|
||||
"input": oom_config["input"],
|
||||
"voice": oom_config["voice"],
|
||||
"max_new_tokens": oom_config["max_new_tokens"],
|
||||
"ref_audio": oom_config["ref_audio"],
|
||||
}
|
||||
try:
|
||||
status, body = post_json_raw(host, port, "/v1/audio/speech", payload, timeout_sec=300)
|
||||
finally:
|
||||
stop_gpu_oom_hogs(handle)
|
||||
|
||||
assert status >= 500, f"expected speech runtime error under OOM, got status={status}"
|
||||
speech_error = extract_openai_error_contract_from_bytes(body)
|
||||
assert speech_error is not None, f"speech error payload not OpenAI-compatible: {body[:300]!r}"
|
||||
assert "code" in speech_error, f"speech error lacks code field: {speech_error!r}"
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.tts
|
||||
@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
|
||||
@pytest.mark.parametrize(
|
||||
"fault_injector",
|
||||
WORKER_SIGNAL_FAULT_PARAMS,
|
||||
indirect=True,
|
||||
)
|
||||
@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True)
|
||||
def test_reliability_fault_process_kill_request_failure(
|
||||
omni_server_after_fault_function,
|
||||
openai_client_function,
|
||||
) -> None:
|
||||
try:
|
||||
_submit_speech_request(openai_client_function, omni_server_after_fault_function)
|
||||
except Exception as exc:
|
||||
assert_fault_exception(exc, PROCESS_KILL_ERROR_KEYWORDS)
|
||||
else:
|
||||
pytest.fail("expected /v1/audio/speech request failure after process-kill injection")
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.tts
|
||||
@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
|
||||
@pytest.mark.parametrize(
|
||||
"fault_injector",
|
||||
WORKER_SIGNAL_FAULT_PARAMS,
|
||||
indirect=True,
|
||||
)
|
||||
@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True)
|
||||
def test_reliability_fault_process_kill_fast_fail_and_concurrent(
|
||||
omni_server_after_fault_function,
|
||||
openai_client_function,
|
||||
) -> None:
|
||||
host = omni_server_after_fault_function.host
|
||||
port = omni_server_after_fault_function.port
|
||||
model = omni_server_after_fault_function.model
|
||||
payload = {
|
||||
"model": model,
|
||||
"input": "Say hello in one short sentence.",
|
||||
"voice": "default",
|
||||
}
|
||||
|
||||
start = time.monotonic()
|
||||
try:
|
||||
status, body = post_json_raw(
|
||||
host,
|
||||
port,
|
||||
"/v1/audio/speech",
|
||||
payload,
|
||||
timeout_sec=_SPEECH_FF_HTTP_TIMEOUT_SEC,
|
||||
)
|
||||
elapsed = time.monotonic() - start
|
||||
assert elapsed <= _SPEECH_FF_MAX_ELAPSED_SEC + 1.0, (
|
||||
f"[process_kill fast_fail] /v1/audio/speech did not fail fast after fault: {elapsed:.2f}s"
|
||||
)
|
||||
assert status >= 500, (
|
||||
"[process_kill fast_fail] expected server-side failure after fault, "
|
||||
f"got status={status}, body={body[:200]!r}"
|
||||
)
|
||||
except Exception:
|
||||
elapsed = time.monotonic() - start
|
||||
assert elapsed <= _SPEECH_FF_MAX_ELAPSED_SEC + 1.0, (
|
||||
f"[process_kill fast_fail] /v1/audio/speech exception was too slow after fault: {elapsed:.2f}s"
|
||||
)
|
||||
|
||||
start = time.monotonic()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor:
|
||||
futures = [
|
||||
executor.submit(_submit_speech_request, openai_client_function, omni_server_after_fault_function)
|
||||
for _ in range(3)
|
||||
]
|
||||
done, pending = concurrent.futures.wait(
|
||||
futures,
|
||||
timeout=30,
|
||||
return_when=concurrent.futures.ALL_COMPLETED,
|
||||
)
|
||||
|
||||
elapsed = time.monotonic() - start
|
||||
assert not pending, f"[process_kill concurrent] some fault-time speech requests hung: pending={len(pending)}"
|
||||
assert elapsed < 30, f"[process_kill concurrent] fault-time speech request convergence is too slow: {elapsed:.2f}s"
|
||||
|
||||
fault_observed = False
|
||||
for future in done:
|
||||
try:
|
||||
future.result()
|
||||
except Exception as exc:
|
||||
fault_observed = True
|
||||
assert_fault_exception(exc, PROCESS_KILL_ERROR_KEYWORDS)
|
||||
assert fault_observed, (
|
||||
"[process_kill concurrent] expected at least one /v1/audio/speech request to fail after fault"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.tts
|
||||
@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
|
||||
@pytest.mark.parametrize(
|
||||
"fault_injector",
|
||||
WORKER_SIGNAL_FAULT_PARAMS,
|
||||
indirect=True,
|
||||
)
|
||||
@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True)
|
||||
def test_reliability_fault_process_kill_worker_with_load_request_failure(
|
||||
omni_server_function,
|
||||
openai_client_function,
|
||||
fault_injector: FaultInjector,
|
||||
) -> None:
|
||||
scenario = "kill_worker_with_load"
|
||||
load_result = run_fault_injection_with_rate_load(
|
||||
submit_request=lambda: _submit_speech_request(openai_client_function, omni_server_function),
|
||||
inject_fault=lambda: fault_injector(omni_server_function),
|
||||
num_requests=INFLIGHT_INJECTION_REQUEST_COUNT,
|
||||
request_rate=INFLIGHT_INJECTION_REQUEST_RATE,
|
||||
completion_timeout_sec=120.0,
|
||||
)
|
||||
assert load_result["failure_observed"], (
|
||||
f"[{scenario}] expected at least one load request failure after fault; load_result={load_result}"
|
||||
)
|
||||
host = omni_server_function.host
|
||||
port = omni_server_function.port
|
||||
_assert_post_fault_speech_fast_fail(host, port, model=omni_server_function.model, scenario=scenario)
|
||||
_assert_post_fault_health_terminal(host, port, scenario=scenario)
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
|
||||
@pytest.mark.parametrize("signal_name", SERVE_SIGNAL_PARAMS)
|
||||
@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True)
|
||||
def test_reliability_fault_process_kill_serve_root_no_load_fast_fail_and_cleanup(
|
||||
omni_server_function,
|
||||
signal_name: str,
|
||||
) -> None:
|
||||
scenario = f"kill_serve_root_no_load_{signal_name.lower()}"
|
||||
injector = make_server_root_kill_fault_injector(signal_name=signal_name, post_kill_wait_seconds=2.0)
|
||||
injector(omni_server_function)
|
||||
host = omni_server_function.host
|
||||
port = omni_server_function.port
|
||||
_assert_post_fault_speech_fast_fail(host, port, model=omni_server_function.model, scenario=scenario)
|
||||
_assert_post_fault_health_terminal(host, port, scenario=scenario)
|
||||
assert_no_server_tree_process_residual_and_gpu_release(
|
||||
omni_server_function,
|
||||
scenario=scenario,
|
||||
timeout_sec=worker_residual_timeout_after_kill_signal(signal_name),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
|
||||
@pytest.mark.parametrize("signal_name", SERVE_WITH_LOAD_SIGNAL_PARAMS)
|
||||
@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True)
|
||||
def test_reliability_fault_process_kill_serve_root_with_load_fast_fail_and_cleanup(
|
||||
omni_server_function,
|
||||
openai_client_function,
|
||||
signal_name: str,
|
||||
) -> None:
|
||||
scenario = f"kill_serve_root_with_load_{signal_name.lower()}"
|
||||
injector = make_server_root_kill_fault_injector(signal_name=signal_name, post_kill_wait_seconds=2.0)
|
||||
load_result = run_fault_injection_with_rate_load(
|
||||
submit_request=lambda: _submit_speech_request(openai_client_function, omni_server_function),
|
||||
inject_fault=lambda: injector(omni_server_function),
|
||||
num_requests=INFLIGHT_INJECTION_REQUEST_COUNT,
|
||||
request_rate=INFLIGHT_INJECTION_REQUEST_RATE,
|
||||
completion_timeout_sec=300.0,
|
||||
)
|
||||
assert load_result["failure_observed"], (
|
||||
f"[{scenario}] expected at least one load request failure after fault; load_result={load_result}"
|
||||
)
|
||||
host = omni_server_function.host
|
||||
port = omni_server_function.port
|
||||
_assert_post_fault_speech_fast_fail(host, port, model=omni_server_function.model, scenario=scenario)
|
||||
_assert_post_fault_health_terminal(host, port, scenario=scenario)
|
||||
assert_no_server_tree_process_residual_and_gpu_release(
|
||||
omni_server_function,
|
||||
scenario=scenario,
|
||||
timeout_sec=worker_residual_timeout_after_kill_signal(signal_name),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.tts
|
||||
@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
|
||||
@pytest.mark.parametrize("signal_name", TREE_SIGNAL_PARAMS)
|
||||
@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True)
|
||||
def test_reliability_fault_process_kill_tree_no_load_fast_fail_and_cleanup(
|
||||
omni_server_function,
|
||||
signal_name: str,
|
||||
) -> None:
|
||||
scenario = f"kill_serve_tree_no_load_{signal_name.lower()}"
|
||||
injector = make_server_tree_kill_fault_injector(
|
||||
signal_name=signal_name,
|
||||
post_kill_wait_seconds=2.0,
|
||||
inter_kill_wait_seconds=0.1,
|
||||
)
|
||||
injector(omni_server_function)
|
||||
host = omni_server_function.host
|
||||
port = omni_server_function.port
|
||||
_assert_post_fault_speech_fast_fail(host, port, model=omni_server_function.model, scenario=scenario)
|
||||
_assert_post_fault_health_terminal(host, port, scenario=scenario)
|
||||
assert_no_server_tree_process_residual_and_gpu_release(
|
||||
omni_server_function,
|
||||
scenario=scenario,
|
||||
timeout_sec=worker_residual_timeout_after_kill_signal(signal_name),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.tts
|
||||
@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
|
||||
@pytest.mark.parametrize("signal_name", TREE_WITH_LOAD_SIGNAL_PARAMS)
|
||||
@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True)
|
||||
def test_reliability_fault_process_kill_tree_with_load_fast_fail_and_cleanup(
|
||||
omni_server_function,
|
||||
openai_client_function,
|
||||
signal_name: str,
|
||||
) -> None:
|
||||
scenario = f"kill_serve_tree_with_load_{signal_name.lower()}"
|
||||
injector = make_server_tree_kill_fault_injector(
|
||||
signal_name=signal_name,
|
||||
post_kill_wait_seconds=2.0,
|
||||
inter_kill_wait_seconds=0.1,
|
||||
)
|
||||
load_result = run_fault_injection_with_rate_load(
|
||||
submit_request=lambda: _submit_speech_request(openai_client_function, omni_server_function),
|
||||
inject_fault=lambda: injector(omni_server_function),
|
||||
num_requests=INFLIGHT_INJECTION_REQUEST_COUNT,
|
||||
request_rate=INFLIGHT_INJECTION_REQUEST_RATE,
|
||||
completion_timeout_sec=120.0,
|
||||
)
|
||||
assert load_result["failure_observed"], (
|
||||
f"[{scenario}] expected at least one load request failure after fault; load_result={load_result}"
|
||||
)
|
||||
host = omni_server_function.host
|
||||
port = omni_server_function.port
|
||||
_assert_post_fault_speech_fast_fail(host, port, model=omni_server_function.model, scenario=scenario)
|
||||
_assert_post_fault_health_terminal(host, port, scenario=scenario)
|
||||
assert_no_server_tree_process_residual_and_gpu_release(
|
||||
omni_server_function,
|
||||
scenario=scenario,
|
||||
timeout_sec=worker_residual_timeout_after_kill_signal(signal_name),
|
||||
)
|
||||
@@ -0,0 +1,69 @@
|
||||
"""
|
||||
VoxCPM2 stability: OmniServer + ``vllm bench serve --omni`` for a fixed duration.
|
||||
|
||||
Configuration: ``tests/dfx/stability/tests/test_voxcpm2.json``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.dfx.conftest import (
|
||||
create_benchmark_indices,
|
||||
create_test_parameter_mapping,
|
||||
create_unique_server_params,
|
||||
load_configs,
|
||||
)
|
||||
from tests.dfx.stability.helpers import _run_one_vllm_bench_batch, run_stability_benchmark_loop
|
||||
|
||||
STABILITY_DIR = Path(__file__).resolve().parent.parent
|
||||
DEPLOY_CONFIGS_DIR = STABILITY_DIR / "deploy"
|
||||
CONFIG_FILE_PATH = str(STABILITY_DIR / "tests" / "test_voxcpm2.json")
|
||||
DEFAULT_NUM_PROMPTS_PER_BATCH = 20
|
||||
STABILITY_SERVER_TIMEOUT_ARGS = ["--stage-init-timeout", "600"]
|
||||
|
||||
try:
|
||||
BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH)
|
||||
except FileNotFoundError:
|
||||
BENCHMARK_CONFIGS = []
|
||||
|
||||
test_params = create_unique_server_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else []
|
||||
server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {}
|
||||
benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping)
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.tts
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
@pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True)
|
||||
def test_stability_voxcpm2(omni_server, stability_benchmark_params):
|
||||
test_name = stability_benchmark_params["test_name"]
|
||||
params = stability_benchmark_params["params"]
|
||||
duration_sec = params.get("duration_sec", 300)
|
||||
num_prompts_per_batch = params.get("num_prompts_per_batch", DEFAULT_NUM_PROMPTS_PER_BATCH)
|
||||
request_rate = params.get("request_rate")
|
||||
max_concurrency = params.get("max_concurrency")
|
||||
|
||||
bench_params = {
|
||||
k: v
|
||||
for k, v in params.items()
|
||||
if k not in ("duration_sec", "request_rate", "max_concurrency", "num_prompts_per_batch")
|
||||
}
|
||||
|
||||
result = run_stability_benchmark_loop(
|
||||
host=omni_server.host,
|
||||
port=omni_server.port,
|
||||
model=omni_server.model,
|
||||
duration_sec=duration_sec,
|
||||
params=bench_params,
|
||||
request_rate=request_rate,
|
||||
max_concurrency=max_concurrency,
|
||||
result_dir=str(STABILITY_DIR),
|
||||
num_prompts_per_batch=num_prompts_per_batch,
|
||||
run_one_batch=_run_one_vllm_bench_batch,
|
||||
)
|
||||
|
||||
assert result.get("failed", 0) == 0, f"[{test_name}] Failed requests detected: {result.get('errors', [])}"
|
||||
assert result.get("completed", 0) > 0, f"[{test_name}] No requests completed"
|
||||
@@ -0,0 +1,35 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_voxcpm2_stability",
|
||||
"server_params": {
|
||||
"model": "openbmb/VoxCPM2",
|
||||
"trust_remote_code": true
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "random",
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"trust_remote_code": true,
|
||||
"duration_sec": 86400,
|
||||
"max_concurrency": 2,
|
||||
"num_prompts_per_batch": 50,
|
||||
"random_input_len": {
|
||||
"min": 0,
|
||||
"max": 1000
|
||||
},
|
||||
"random_output_len": {
|
||||
"min": 0,
|
||||
"max": 1000
|
||||
},
|
||||
"random_range_ratio": 0.0,
|
||||
"extra_body": {
|
||||
"voice": "default",
|
||||
"language": "English"
|
||||
},
|
||||
"ignore_eos": true,
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
+32
@@ -42,6 +42,10 @@ SEPARATE_CFG_TRANSFORMERS = [
|
||||
SAMPLE_CACHE_CONFIG = DiffusionCacheConfig()
|
||||
|
||||
|
||||
def test_wan22_vace_uses_wan22_custom_cache_dit_enabler():
|
||||
assert cd_backend.CUSTOM_DIT_ENABLERS["Wan22VACEPipeline"] is cd_backend.enable_cache_for_wan22
|
||||
|
||||
|
||||
@pytest.mark.parametrize("transformer_model", SEPARATE_CFG_TRANSFORMERS)
|
||||
def test_cache_dit_configs_have_separate_cfg(transformer_model):
|
||||
"""Check models with separate CFG set has_separate_cfg=True in their configs."""
|
||||
@@ -62,6 +66,34 @@ def test_separate_wan22_custom_enabler_has_separate_cfg(mock_cache_dit, mock_blo
|
||||
assert adapter_kwargs["has_separate_cfg"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("has_transformer_2", [False, True])
|
||||
@patch("vllm_omni.diffusion.cache.cache_dit_backend.BlockAdapter")
|
||||
@patch("vllm_omni.diffusion.cache.cache_dit_backend.cache_dit")
|
||||
def test_wan22_custom_enabler_passes_taylorseer_calibrator(
|
||||
mock_cache_dit,
|
||||
mock_block_adapter,
|
||||
has_transformer_2,
|
||||
):
|
||||
mock_pipeline = Mock()
|
||||
mock_pipeline.transformer.blocks = [Mock()]
|
||||
if has_transformer_2:
|
||||
mock_pipeline.transformer_2.blocks = [Mock()]
|
||||
else:
|
||||
mock_pipeline.transformer_2 = None
|
||||
cache_config = DiffusionCacheConfig(enable_taylorseer=True, taylorseer_order=1)
|
||||
|
||||
cd_backend.enable_cache_for_wan22(mock_pipeline, cache_config)
|
||||
|
||||
enable_cache_kwargs = mock_cache_dit.enable_cache.call_args.kwargs
|
||||
calibrator_config = enable_cache_kwargs["calibrator_config"]
|
||||
assert calibrator_config is not None
|
||||
assert calibrator_config.taylorseer_order == 1
|
||||
|
||||
adapter_kwargs = mock_block_adapter.call_args.kwargs
|
||||
for modifier in adapter_kwargs["params_modifiers"]:
|
||||
assert modifier._context_kwargs["calibrator_config"] is calibrator_config
|
||||
|
||||
|
||||
@patch("vllm_omni.diffusion.cache.cache_dit_backend.BlockAdapter")
|
||||
@patch("vllm_omni.diffusion.cache.cache_dit_backend.cache_dit")
|
||||
def test_cosmos3_cache_dit_wraps_gen_layers(mock_cache_dit, mock_block_adapter):
|
||||
|
||||
@@ -23,6 +23,7 @@ from PIL import Image
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
from vllm_omni.diffusion.distributed.utils import build_local_sp_padding_mask
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
@@ -412,3 +413,85 @@ def test_sp_correctness_advanced(model_name: str):
|
||||
f"{baseline_str:<12} {r['sp_ms']:.0f}ms{'':<7} {speedup_str:<10} {status}"
|
||||
)
|
||||
print("=" * 70)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (current_omni_platform.is_cuda() or current_omni_platform.is_xpu()),
|
||||
reason="Only tested on CUDA and XPU",
|
||||
)
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parallel
|
||||
@pytest.mark.core_model
|
||||
def test_local_sp_padding_mask(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A partially padded SP shard must receive a local-length mask."""
|
||||
mask = build_local_sp_padding_mask(
|
||||
batch_size=2,
|
||||
local_seq_len=4,
|
||||
sp_original_seq_len=5,
|
||||
sp_padding_size=3,
|
||||
sequence_parallel_rank=1,
|
||||
device=torch.device(current_omni_platform.device_type),
|
||||
)
|
||||
|
||||
expected = torch.tensor(
|
||||
[
|
||||
[True, False, False, False],
|
||||
[True, False, False, False],
|
||||
],
|
||||
dtype=torch.bool,
|
||||
device=mask.device,
|
||||
)
|
||||
assert mask is not None
|
||||
assert mask.shape == (2, 4)
|
||||
assert torch.equal(mask, expected)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (current_omni_platform.is_cuda() or current_omni_platform.is_xpu()),
|
||||
reason="Only tested on CUDA and XPU",
|
||||
)
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parallel
|
||||
@pytest.mark.core_model
|
||||
def test_local_sp_padding_mask_no_padding(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A rank whose local shard contains no padding should not get a mask."""
|
||||
mask = build_local_sp_padding_mask(
|
||||
batch_size=2,
|
||||
local_seq_len=4,
|
||||
sp_original_seq_len=5,
|
||||
sp_padding_size=3,
|
||||
sequence_parallel_rank=0,
|
||||
device=torch.device(current_omni_platform.device_type),
|
||||
)
|
||||
|
||||
assert mask is None
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (current_omni_platform.is_cuda() or current_omni_platform.is_xpu()),
|
||||
reason="Only tested on CUDA and XPU",
|
||||
)
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parallel
|
||||
@pytest.mark.core_model
|
||||
def test_wan_sp_plan() -> None:
|
||||
"""Wan2.2 must shard hidden states before CacheDiT-wrapped transformer blocks."""
|
||||
try:
|
||||
from vllm_omni.diffusion.distributed.sp_plan import validate_sp_plan
|
||||
from vllm_omni.diffusion.models.wan2_2.wan2_2_transformer import WanTransformer3DModel
|
||||
except ImportError as exc:
|
||||
pytest.skip(f"WanTransformer3DModel not available: {exc}")
|
||||
|
||||
plan = getattr(WanTransformer3DModel, "_sp_plan", None)
|
||||
|
||||
assert plan is not None
|
||||
assert "_sp_shard_point" in plan
|
||||
assert "blocks.0" not in plan
|
||||
|
||||
shard_plan = plan["_sp_shard_point"]
|
||||
assert 0 in shard_plan
|
||||
assert shard_plan[0].split_dim == 1
|
||||
assert shard_plan[0].expected_dims == 3
|
||||
assert shard_plan[0].split_output is True
|
||||
|
||||
validate_sp_plan(plan)
|
||||
@@ -19,9 +19,19 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
|
||||
class StubScheduler:
|
||||
def __init__(self, timesteps: list[int] | None = None, *, flow_shift: float = 1.0) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
timesteps: list[int] | None = None,
|
||||
*,
|
||||
flow_shift: float = 1.0,
|
||||
use_karras_sigmas: bool = True,
|
||||
) -> None:
|
||||
self.timesteps = torch.tensor(timesteps or [9, 3], dtype=torch.int64)
|
||||
self.config = SimpleNamespace(num_train_timesteps=1000, flow_shift=flow_shift)
|
||||
self.config = SimpleNamespace(
|
||||
num_train_timesteps=1000,
|
||||
flow_shift=flow_shift,
|
||||
use_karras_sigmas=use_karras_sigmas,
|
||||
)
|
||||
self.set_timesteps_calls: list[tuple[int, torch.device]] = []
|
||||
self.step_calls: list[tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = []
|
||||
|
||||
@@ -194,6 +204,8 @@ def make_cosmos3_pipeline():
|
||||
pipeline._base_scheduler_config = pipeline.scheduler.config
|
||||
pipeline._engine_init_flow_shift = 1.0
|
||||
pipeline._current_flow_shift = 1.0
|
||||
pipeline._base_scheduler_use_karras_sigmas = True
|
||||
pipeline._current_scheduler_use_karras_sigmas = True
|
||||
pipeline._guidance_scale = None
|
||||
pipeline._num_timesteps = None
|
||||
pipeline._cosmos3_branch_caches = None
|
||||
@@ -292,8 +304,8 @@ def stub_real_pipeline_init(monkeypatch: pytest.MonkeyPatch):
|
||||
class _StubDiffusersScheduler:
|
||||
from_config_calls: list[dict[str, Any]] = []
|
||||
|
||||
def __init__(self, *, flow_shift: float = 1.0) -> None:
|
||||
self.config = SimpleNamespace(flow_shift=flow_shift)
|
||||
def __init__(self, *, flow_shift: float = 1.0, use_karras_sigmas: bool = True) -> None:
|
||||
self.config = SimpleNamespace(flow_shift=flow_shift, use_karras_sigmas=use_karras_sigmas)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(cls, *args, **kwargs):
|
||||
@@ -302,7 +314,13 @@ def stub_real_pipeline_init(monkeypatch: pytest.MonkeyPatch):
|
||||
@classmethod
|
||||
def from_config(cls, config, **kwargs):
|
||||
cls.from_config_calls.append({"config": config, "kwargs": dict(kwargs)})
|
||||
return cls(flow_shift=kwargs.get("flow_shift", getattr(config, "flow_shift", 1.0)))
|
||||
return cls(
|
||||
flow_shift=kwargs.get("flow_shift", getattr(config, "flow_shift", 1.0)),
|
||||
use_karras_sigmas=kwargs.get(
|
||||
"use_karras_sigmas",
|
||||
getattr(config, "use_karras_sigmas", True),
|
||||
),
|
||||
)
|
||||
|
||||
class _StubVideoProcessor:
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
@@ -349,10 +367,13 @@ def test_pipeline_init_skips_tokenizer_when_sound_disabled(stub_real_pipeline_in
|
||||
|
||||
pipeline = Cosmos3OmniDiffusersPipeline(od_config=_make_od_config(sound_gen=False))
|
||||
|
||||
assert stub_real_pipeline_init.from_config_calls == []
|
||||
assert pipeline._sound_tokenizer is None
|
||||
assert pipeline.transformer.sound_gen is False
|
||||
assert not hasattr(pipeline.transformer, "audio_proj_in")
|
||||
assert not hasattr(pipeline.transformer, "audio_proj_out")
|
||||
assert pipeline._base_scheduler_use_karras_sigmas is True
|
||||
assert pipeline._current_scheduler_use_karras_sigmas is True
|
||||
|
||||
|
||||
def test_pipeline_init_rebuilds_scheduler_with_cosmos3_defaults(stub_real_pipeline_init) -> None:
|
||||
@@ -366,8 +387,52 @@ def test_pipeline_init_rebuilds_scheduler_with_cosmos3_defaults(stub_real_pipeli
|
||||
assert stub_real_pipeline_init.from_config_calls
|
||||
call = stub_real_pipeline_init.from_config_calls[-1]
|
||||
assert getattr(call["config"], "flow_shift") == 1.0
|
||||
assert call["kwargs"] == {"flow_shift": 2.5, "use_karras_sigmas": False}
|
||||
assert call["kwargs"] == {"flow_shift": 2.5}
|
||||
assert pipeline._engine_init_flow_shift == 2.5
|
||||
assert pipeline._base_scheduler_use_karras_sigmas is True
|
||||
|
||||
|
||||
def test_set_flow_shift_restores_checkpoint_scheduler_mode(
|
||||
make_cosmos3_pipeline,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from vllm_omni.diffusion.models.cosmos3 import pipeline_cosmos3
|
||||
|
||||
class RecordingScheduler:
|
||||
from_config_calls: list[dict[str, Any]] = []
|
||||
|
||||
def __init__(self, *, flow_shift: float, use_karras_sigmas: bool) -> None:
|
||||
self.config = SimpleNamespace(flow_shift=flow_shift, use_karras_sigmas=use_karras_sigmas)
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config, **kwargs):
|
||||
cls.from_config_calls.append({"config": config, "kwargs": dict(kwargs)})
|
||||
return cls(
|
||||
flow_shift=kwargs.get("flow_shift", getattr(config, "flow_shift", 1.0)),
|
||||
use_karras_sigmas=kwargs.get(
|
||||
"use_karras_sigmas",
|
||||
getattr(config, "use_karras_sigmas", True),
|
||||
),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(pipeline_cosmos3, "UniPCMultistepScheduler", RecordingScheduler)
|
||||
pipeline = make_cosmos3_pipeline()
|
||||
|
||||
pipeline._set_flow_shift(10.0, use_karras_sigmas=False)
|
||||
assert RecordingScheduler.from_config_calls[-1]["kwargs"] == {
|
||||
"flow_shift": 10.0,
|
||||
"use_karras_sigmas": False,
|
||||
}
|
||||
assert pipeline._current_flow_shift == 10.0
|
||||
assert pipeline._current_scheduler_use_karras_sigmas is False
|
||||
|
||||
pipeline._set_flow_shift(10.0)
|
||||
assert RecordingScheduler.from_config_calls[-1]["kwargs"] == {"flow_shift": 10.0}
|
||||
assert pipeline._current_flow_shift == 10.0
|
||||
assert pipeline._current_scheduler_use_karras_sigmas is True
|
||||
|
||||
pipeline._set_flow_shift(10.0)
|
||||
assert len(RecordingScheduler.from_config_calls) == 2
|
||||
|
||||
|
||||
def test_pipeline_init_passes_tokenizer_attrs_into_transformer(
|
||||
@@ -1022,7 +1087,12 @@ def test_forward_transfer_uses_source_fps_except_wsm(make_cosmos3_pipeline, hint
|
||||
pipeline._encode_video_tensor = fake_encode
|
||||
pipeline._prepare_transfer_latents = fake_prepare
|
||||
pipeline.diffuse_transfer = fake_diffuse_transfer
|
||||
pipeline._set_flow_shift = lambda target: captured.setdefault("flow_shifts", []).append(target)
|
||||
|
||||
def fake_set_flow_shift(target, *, use_karras_sigmas=None):
|
||||
captured.setdefault("flow_shifts", []).append(target)
|
||||
captured.setdefault("scheduler_use_karras_sigmas", []).append(use_karras_sigmas)
|
||||
|
||||
pipeline._set_flow_shift = fake_set_flow_shift
|
||||
pipeline._decode_latents = lambda latents: torch.zeros(1, 3, 5, 16, 16, device="meta")
|
||||
|
||||
control = torch.zeros(3, 5, 16, 16, dtype=torch.uint8)
|
||||
@@ -1055,6 +1125,8 @@ def test_forward_transfer_uses_source_fps_except_wsm(make_cosmos3_pipeline, hint
|
||||
assert captured["format_frame_rate"] == expected_fps
|
||||
assert captured["shared_kwargs"]["fps"] == expected_fps
|
||||
assert captured["flow_shifts"] == [10.0]
|
||||
# Transfer uses the V2V flow-sigma schedule (karras off) so flow_shift applies.
|
||||
assert captured["scheduler_use_karras_sigmas"] == [False]
|
||||
assert output.custom_output["fps"] == expected_fps
|
||||
assert output.output["video"].device.type == "meta"
|
||||
|
||||
@@ -1068,7 +1140,7 @@ def test_forward_transfer_runs_multichunk_overlap_path(
|
||||
|
||||
pipeline._transfer_bucket_size = lambda sp, source_hw: (16, 16)
|
||||
pipeline._format_and_tokenize_prompts = lambda *args, **kwargs: (_ids(2), _mask(), _ids(1), _mask())
|
||||
pipeline._set_flow_shift = lambda target: captured.setdefault("flow_shifts", []).append(target)
|
||||
pipeline._set_flow_shift = lambda target, **_kwargs: captured.setdefault("flow_shifts", []).append(target)
|
||||
|
||||
original_prepare = pipeline._prepare_transfer_latents
|
||||
|
||||
@@ -1213,7 +1285,12 @@ class TestForwardRouting:
|
||||
|
||||
pipeline._format_and_tokenize_prompts = fake_format
|
||||
pipeline._prepare_latents = fake_prepare
|
||||
pipeline._set_flow_shift = lambda target: captured.setdefault("flow_shifts", []).append(target)
|
||||
|
||||
def fake_set_flow_shift(target, *, use_karras_sigmas=None):
|
||||
captured.setdefault("flow_shifts", []).append(target)
|
||||
captured.setdefault("scheduler_use_karras_sigmas", []).append(use_karras_sigmas)
|
||||
|
||||
pipeline._set_flow_shift = fake_set_flow_shift
|
||||
pipeline.diffuse = fake_diffuse
|
||||
pipeline._decode_latents = lambda latents: latents
|
||||
return captured
|
||||
@@ -1227,7 +1304,6 @@ class TestForwardRouting:
|
||||
{
|
||||
"key": "image",
|
||||
"is_t2i": True,
|
||||
"use_system_prompt": True,
|
||||
"flow": [3.0],
|
||||
"steps": [50, 50],
|
||||
"frames": 1,
|
||||
@@ -1239,7 +1315,6 @@ class TestForwardRouting:
|
||||
{
|
||||
"key": "video",
|
||||
"is_t2i": False,
|
||||
"use_system_prompt": True,
|
||||
"flow": [1.0],
|
||||
"steps": [35],
|
||||
"frames": 189,
|
||||
@@ -1261,9 +1336,9 @@ class TestForwardRouting:
|
||||
|
||||
assert expected["key"] in output.output
|
||||
assert captured["format"]["is_t2i"] is expected["is_t2i"]
|
||||
assert captured["format"]["use_system_prompt"] is expected["use_system_prompt"]
|
||||
assert captured["format"]["num_frames"] == expected["frames"]
|
||||
assert captured["flow_shifts"] == expected["flow"]
|
||||
assert captured["scheduler_use_karras_sigmas"] == [None]
|
||||
assert [call[0] for call in pipeline.scheduler.set_timesteps_calls] == expected["steps"]
|
||||
|
||||
def test_forward_i2v_sound_and_action_routes(self, make_cosmos3_pipeline) -> None:
|
||||
@@ -1290,7 +1365,6 @@ class TestForwardRouting:
|
||||
)
|
||||
)
|
||||
assert captured["diffuse_calls"][-1]["shared_kwargs"]["noisy_frame_mask"] is velocity_mask
|
||||
assert captured["format"]["use_system_prompt"] is True
|
||||
|
||||
video_tensor = torch.zeros(1, 3, 5, 16, 16)
|
||||
v2v_condition = torch.full((1, 2, 2, 1, 1), 4.0)
|
||||
@@ -1316,8 +1390,8 @@ class TestForwardRouting:
|
||||
)
|
||||
)
|
||||
assert captured["flow_shifts"][-1] == 10.0
|
||||
assert captured["scheduler_use_karras_sigmas"][-1] is False
|
||||
assert captured["format"]["negative_prompt"] == ""
|
||||
assert captured["format"]["use_system_prompt"] is True
|
||||
assert captured["diffuse_calls"][-1]["shared_kwargs"]["noisy_frame_mask"] is v2v_mask
|
||||
assert captured["diffuse_calls"][-1]["condition_latents"] is v2v_condition
|
||||
|
||||
@@ -1333,7 +1407,6 @@ class TestForwardRouting:
|
||||
)
|
||||
)
|
||||
assert captured["diffuse_calls"][-1]["sound_latents"] is sound_latents
|
||||
assert captured["format"]["use_system_prompt"] is True
|
||||
assert output.output["audio_sample_rate"] == 10
|
||||
|
||||
pipeline.transformer = pipeline.transformer.__class__(latent_channel_size=2, action_gen=True, action_dim=4)
|
||||
@@ -1359,7 +1432,6 @@ class TestForwardRouting:
|
||||
)
|
||||
)
|
||||
assert captured["diffuse_calls"][-1]["shared_kwargs"]["action_domain_ids"].tolist() == [7]
|
||||
assert captured["format"]["use_system_prompt"] is False
|
||||
assert output.custom_output["action"].shape == (1, 2, 2)
|
||||
assert "action_only_output" not in output.custom_output
|
||||
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for HunyuanImage3 AR image RGB conversion."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from vllm_omni.model_executor.models.hunyuan_image3.hunyuan_image3 import (
|
||||
HunyuanImage3Processor,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _FixedResolutionGroup:
|
||||
def get_target_size(self, width: int, height: int):
|
||||
return width, height
|
||||
|
||||
def get_base_size_and_ratio_index(self, width: int, height: int):
|
||||
return 1024, 0
|
||||
|
||||
|
||||
def test_ar_processor_uses_pil_rgb_conversion_for_rgba_images():
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def fake_vit_processor(image: Image.Image):
|
||||
captured["mode"] = image.mode
|
||||
captured["pixel"] = image.getpixel((0, 0))
|
||||
return {
|
||||
"pixel_values": torch.zeros((1, 1, 3), dtype=torch.float32),
|
||||
"pixel_attention_mask": torch.ones((1, 1), dtype=torch.bool),
|
||||
"spatial_shapes": torch.tensor([[1, 1]], dtype=torch.long),
|
||||
}
|
||||
|
||||
def fake_vae_processor(_image: Image.Image):
|
||||
return torch.zeros((1, 3, 1, 1), dtype=torch.float32)
|
||||
|
||||
processor = object.__new__(HunyuanImage3Processor)
|
||||
processor.hf_config = SimpleNamespace(
|
||||
vit={"num_channels": 3},
|
||||
vae_downsample_factor=(1, 1),
|
||||
patch_size=1,
|
||||
)
|
||||
processor.reso_group = _FixedResolutionGroup()
|
||||
processor.vision_encoder_processor = fake_vit_processor
|
||||
processor.vae_processor = fake_vae_processor
|
||||
|
||||
processor.process_image(Image.new("RGBA", (1, 1), color=(255, 0, 0, 0)))
|
||||
|
||||
assert captured["mode"] == "RGB"
|
||||
assert captured["pixel"] == (255, 0, 0)
|
||||
@@ -20,6 +20,7 @@ RATIO_START = 200
|
||||
RATIO_END = 210
|
||||
RATIO_OTHER_START = 220
|
||||
RATIO_OTHER_END = 223
|
||||
TIMESTEP = 224
|
||||
|
||||
|
||||
class FakeSamplerModel:
|
||||
@@ -60,6 +61,67 @@ class FakeSamplerModel:
|
||||
_apply_ratio_restriction = _Real._apply_ratio_restriction
|
||||
|
||||
|
||||
class FakeTokenEmbedding:
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return torch.stack(
|
||||
[
|
||||
input_ids.to(dtype=torch.float32),
|
||||
input_ids.to(dtype=torch.float32) + 0.5,
|
||||
],
|
||||
dim=-1,
|
||||
)
|
||||
|
||||
|
||||
class FakeEmbeddingModel:
|
||||
def __init__(self):
|
||||
self.model = FakeTokenEmbedding()
|
||||
self._timestep_token_id = TIMESTEP
|
||||
|
||||
from vllm_omni.model_executor.models.hunyuan_image3.hunyuan_image3 import (
|
||||
HunyuanImage3ForConditionalGeneration as _Real,
|
||||
)
|
||||
|
||||
embed_input_ids = _Real.embed_input_ids
|
||||
|
||||
def _timestep_encode(self, timestep: torch.Tensor) -> torch.Tensor:
|
||||
return torch.full(
|
||||
(1, 2),
|
||||
-7.0,
|
||||
device=timestep.device,
|
||||
dtype=timestep.dtype,
|
||||
)
|
||||
|
||||
|
||||
class TestTimestepEmbedding:
|
||||
def test_timestep_embedding_avoids_scalar_item_sync(self, monkeypatch):
|
||||
model = FakeEmbeddingModel()
|
||||
input_ids = torch.tensor([1, TIMESTEP, 2, TIMESTEP], dtype=torch.long)
|
||||
|
||||
def fail_item(_self):
|
||||
raise AssertionError("embed_input_ids should not synchronize timestep count")
|
||||
|
||||
monkeypatch.setattr(torch.Tensor, "item", fail_item)
|
||||
|
||||
actual = model.embed_input_ids(input_ids)
|
||||
expected = model.model.embed_input_ids(input_ids)
|
||||
expected[input_ids == TIMESTEP] = -7.0
|
||||
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
def test_timestep_embedding_preserves_no_timestep_inputs(self, monkeypatch):
|
||||
model = FakeEmbeddingModel()
|
||||
input_ids = torch.tensor([1, 2, 3], dtype=torch.long)
|
||||
|
||||
def fail_item(_self):
|
||||
raise AssertionError("embed_input_ids should not synchronize timestep count")
|
||||
|
||||
monkeypatch.setattr(torch.Tensor, "item", fail_item)
|
||||
|
||||
actual = model.embed_input_ids(input_ids)
|
||||
|
||||
torch.testing.assert_close(actual, model.model.embed_input_ids(input_ids))
|
||||
|
||||
|
||||
class TestGetForcedToken:
|
||||
"""Tests for the stateless _get_forced_token method."""
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_vace import (
|
||||
create_vace_transformer_from_config,
|
||||
get_wan22_vace_pre_process_func,
|
||||
)
|
||||
from vllm_omni.diffusion.models.wan2_2.wan2_2_vace_transformer import WanVACETransformer3DModel
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
@@ -93,6 +94,10 @@ def test_create_vace_transformer_from_config_maps_vace_specific_keys(monkeypatch
|
||||
}
|
||||
|
||||
|
||||
def test_vace_transformer_layerwise_offloads_conditioning_blocks() -> None:
|
||||
assert WanVACETransformer3DModel._layerwise_offload_blocks_attrs == ["vace_blocks", "blocks"]
|
||||
|
||||
|
||||
def test_vace_prepare_masks_encodes_spatial_stride_and_reference_padding() -> None:
|
||||
pipeline = _make_vace_pipeline()
|
||||
mask = torch.ones(1, 3, 5, 16, 16)
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.forward_context import set_forward_context
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_attention_uses_compile_boundary_for_hsdp(monkeypatch):
|
||||
attention = object.__new__(Attention)
|
||||
calls = []
|
||||
|
||||
def _boundary(query, key, value, attn_metadata=None):
|
||||
calls.append("boundary")
|
||||
return query
|
||||
|
||||
def _impl(query, key, value, attn_metadata=None):
|
||||
calls.append("impl")
|
||||
return query
|
||||
|
||||
attention._forward_hsdp_compile_boundary = _boundary
|
||||
attention._forward_impl = _impl
|
||||
monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)
|
||||
|
||||
config = SimpleNamespace(parallel_config=SimpleNamespace(use_hsdp=True))
|
||||
query = torch.empty(1)
|
||||
with set_forward_context(omni_diffusion_config=config):
|
||||
assert Attention.forward(attention, query, query, query) is query
|
||||
|
||||
assert calls == ["boundary"]
|
||||
|
||||
|
||||
def test_attention_keeps_compiled_impl_without_hsdp(monkeypatch):
|
||||
attention = object.__new__(Attention)
|
||||
calls = []
|
||||
|
||||
def _boundary(query, key, value, attn_metadata=None):
|
||||
calls.append("boundary")
|
||||
return query
|
||||
|
||||
def _impl(query, key, value, attn_metadata=None):
|
||||
calls.append("impl")
|
||||
return query
|
||||
|
||||
attention._forward_hsdp_compile_boundary = _boundary
|
||||
attention._forward_impl = _impl
|
||||
monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)
|
||||
|
||||
config = SimpleNamespace(parallel_config=SimpleNamespace(use_hsdp=False))
|
||||
query = torch.empty(1)
|
||||
with set_forward_context(omni_diffusion_config=config):
|
||||
assert Attention.forward(attention, query, query, query) is query
|
||||
|
||||
assert calls == ["impl"]
|
||||
|
||||
|
||||
def test_attention_keeps_compiled_impl_without_diffusion_config(monkeypatch):
|
||||
attention = object.__new__(Attention)
|
||||
calls = []
|
||||
|
||||
def _boundary(query, key, value, attn_metadata=None):
|
||||
calls.append("boundary")
|
||||
return query
|
||||
|
||||
def _impl(query, key, value, attn_metadata=None):
|
||||
calls.append("impl")
|
||||
return query
|
||||
|
||||
attention._forward_hsdp_compile_boundary = _boundary
|
||||
attention._forward_impl = _impl
|
||||
monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)
|
||||
|
||||
query = torch.empty(1)
|
||||
with set_forward_context(vllm_config=None):
|
||||
assert Attention.forward(attention, query, query, query) is query
|
||||
|
||||
assert calls == ["impl"]
|
||||
@@ -1,20 +1,28 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch.nn as nn
|
||||
|
||||
import vllm_omni.diffusion.compile as compile_module
|
||||
from vllm_omni.diffusion.compile import regionally_compile
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _WrappedBlock(nn.Module):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.compile_called = False
|
||||
self.forward_compiled = False
|
||||
|
||||
def compile(self, *args, **kwargs):
|
||||
self.compile_called = True
|
||||
return self
|
||||
|
||||
def forward(self, x):
|
||||
return x
|
||||
|
||||
|
||||
class _ModelWithWrappedRepeatedBlocks(nn.Module):
|
||||
_repeated_blocks = ["OriginalBlock"]
|
||||
@@ -26,10 +34,23 @@ class _ModelWithWrappedRepeatedBlocks(nn.Module):
|
||||
self.other_blocks = nn.ModuleList([_WrappedBlock()])
|
||||
|
||||
|
||||
def test_regionally_compile_matches_wrapped_blocks_by_declared_container_attr():
|
||||
def test_regionally_compile_matches_wrapped_blocks_by_declared_container_attr(monkeypatch):
|
||||
model = _ModelWithWrappedRepeatedBlocks()
|
||||
compile_calls = []
|
||||
|
||||
regionally_compile(model)
|
||||
def _compile(fn, *args, **kwargs):
|
||||
compile_calls.append((fn, args, kwargs))
|
||||
|
||||
assert all(block.compile_called for block in model.transformer_blocks)
|
||||
def _compiled(*fn_args, **fn_kwargs):
|
||||
return f"compiled:{fn(*fn_args, **fn_kwargs)}"
|
||||
|
||||
return _compiled
|
||||
|
||||
monkeypatch.setattr(compile_module.torch, "compile", _compile)
|
||||
|
||||
regionally_compile(model, dynamic=True)
|
||||
|
||||
assert len(compile_calls) == 2
|
||||
assert all(not block.compile_called for block in model.transformer_blocks)
|
||||
assert not model.other_blocks[0].compile_called
|
||||
assert model.transformer_blocks[0].forward("ok") == "compiled:ok"
|
||||
@@ -9,6 +9,7 @@ from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput
|
||||
from vllm_omni.diffusion.diffusion_engine import DiffusionEngine
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.sched import DiffusionRequestStatus, RequestScheduler
|
||||
@@ -33,6 +34,7 @@ def _make_engine() -> DiffusionEngine:
|
||||
engine._rpc_lock = threading.RLock()
|
||||
engine._cv = threading.Condition(engine._rpc_lock)
|
||||
engine._out_queue = {}
|
||||
engine._out_queue_streaming = {}
|
||||
engine._closed = False
|
||||
engine._shutdown_complete = False
|
||||
engine.abort_queue = queue.Queue()
|
||||
@@ -57,6 +59,23 @@ def test_close_completes_pending_async_waiters() -> None:
|
||||
event_loop.close()
|
||||
|
||||
|
||||
def test_close_completes_pending_streaming_waiters() -> None:
|
||||
engine = _make_engine()
|
||||
event_loop = asyncio.new_event_loop()
|
||||
try:
|
||||
engine.main_loop = event_loop
|
||||
queue: asyncio.Queue[DiffusionOutput] = asyncio.Queue()
|
||||
engine._out_queue_streaming["pending-stream"] = queue
|
||||
|
||||
engine.close()
|
||||
|
||||
output = queue.get_nowait()
|
||||
assert output.error == "DiffusionEngine is closed."
|
||||
assert output.finished is True
|
||||
finally:
|
||||
event_loop.close()
|
||||
|
||||
|
||||
def test_handle_finished_requests_ignores_already_drained_waiter() -> None:
|
||||
class RacingOutQueue(dict):
|
||||
def __contains__(self, key):
|
||||
|
||||
@@ -9,6 +9,7 @@ import torch
|
||||
|
||||
import vllm_omni.diffusion.worker.diffusion_model_runner as model_runner_module
|
||||
from tests.helpers.mark import hardware_test
|
||||
from vllm_omni.diffusion.data import DiffusionOutput
|
||||
from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner
|
||||
|
||||
pytestmark = [pytest.mark.diffusion]
|
||||
@@ -31,6 +32,46 @@ class _DummyPipeline:
|
||||
return self._output
|
||||
|
||||
|
||||
class _ChunkStepPipeline:
|
||||
device = torch.device("cpu")
|
||||
supports_step_execution = True
|
||||
|
||||
def __init__(self, outputs):
|
||||
self._outputs = outputs
|
||||
self.prepare_calls = 0
|
||||
self.decode_calls = 0
|
||||
|
||||
def prepare_encode(self, state):
|
||||
self.prepare_calls += 1
|
||||
state.prompt_embeds = torch.zeros(1, 1, 1)
|
||||
state.latents = torch.zeros(1, 1)
|
||||
state.timesteps = torch.tensor([1.0, 0.0])
|
||||
state.step_index = 0
|
||||
state.step_in_chunk = 0
|
||||
state.chunk_num_steps = 2
|
||||
state.total_chunks = len(self._outputs)
|
||||
return state
|
||||
|
||||
def denoise_step(self, input_batch, states):
|
||||
del states
|
||||
return torch.ones_like(input_batch.latents)
|
||||
|
||||
def step_scheduler(self, state, noise_pred):
|
||||
state.latents = noise_pred
|
||||
state.step_index += 1
|
||||
state.step_in_chunk += 1
|
||||
|
||||
def post_decode(self, state):
|
||||
output = self._outputs[self.decode_calls]
|
||||
self.decode_calls += 1
|
||||
state.chunk_index += 1
|
||||
state.step_index = 0
|
||||
state.step_in_chunk = 0
|
||||
if not state.request_denoise_completed:
|
||||
state.latents = torch.zeros(1, 1)
|
||||
return output
|
||||
|
||||
|
||||
def _make_request(skip_cache_refresh: bool = True):
|
||||
sampling_params = SimpleNamespace(
|
||||
generator=None,
|
||||
@@ -39,6 +80,7 @@ def _make_request(skip_cache_refresh: bool = True):
|
||||
num_inference_steps=4,
|
||||
)
|
||||
return SimpleNamespace(
|
||||
request_id="req-test",
|
||||
prompts=["a prompt"],
|
||||
sampling_params=sampling_params,
|
||||
skip_cache_refresh=skip_cache_refresh,
|
||||
@@ -52,20 +94,97 @@ def _make_runner(cache_backend, cache_backend_name: str, enable_cache_dit_summar
|
||||
runner.pipeline = _DummyPipeline(output=SimpleNamespace(output="ok"))
|
||||
runner.cache_backend = cache_backend
|
||||
runner.offload_backend = None
|
||||
runner.state_cache = {}
|
||||
runner.prompt_embed_cache = None
|
||||
runner.od_config = SimpleNamespace(
|
||||
cache_backend=cache_backend_name,
|
||||
enable_cache_dit_summary=enable_cache_dit_summary,
|
||||
parallel_config=SimpleNamespace(use_hsdp=False),
|
||||
streaming_output=False,
|
||||
)
|
||||
runner.kv_transfer_manager = SimpleNamespace(
|
||||
receive_kv_cache=lambda req, target_device=None: None,
|
||||
receive_multi_kv_cache=lambda req, cfg_kv_collect_func=None, target_device=None: None,
|
||||
receive_multi_kv_cache_distributed=lambda req, cfg_kv_collect_func=None, target_device=None: None,
|
||||
receive_multi_kv_cache_distributed=lambda *a, **k: None,
|
||||
)
|
||||
runner._kv_prefetch_enabled = False
|
||||
return runner
|
||||
|
||||
|
||||
def _make_compile_runner(*, use_hsdp: bool):
|
||||
runner = object.__new__(DiffusionModelRunner)
|
||||
runner.pipeline = SimpleNamespace(transformer=SimpleNamespace())
|
||||
runner.od_config = SimpleNamespace(parallel_config=SimpleNamespace(use_hsdp=use_hsdp))
|
||||
return runner
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
@pytest.mark.parametrize("use_hsdp", [False, True])
|
||||
def test_compile_transformer_regionally_compiles_blocks(monkeypatch, use_hsdp):
|
||||
runner = _make_compile_runner(use_hsdp=use_hsdp)
|
||||
compile_calls = []
|
||||
|
||||
def _regionally_compile(model, *args, **kwargs):
|
||||
compile_calls.append((model, args, kwargs))
|
||||
return model
|
||||
|
||||
monkeypatch.setattr(model_runner_module, "regionally_compile", _regionally_compile)
|
||||
|
||||
DiffusionModelRunner._compile_transformer(runner, "transformer")
|
||||
|
||||
assert compile_calls == [
|
||||
(
|
||||
runner.pipeline.transformer,
|
||||
(),
|
||||
{"dynamic": True},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
def test_execute_stepwise_streaming_returns_chunks_at_boundaries(monkeypatch):
|
||||
"""Step streaming returns empty step results until a chunk decode boundary."""
|
||||
chunks = [
|
||||
DiffusionOutput(output="chunk-0", finished=False, chunk_index=0, total_chunks=2),
|
||||
DiffusionOutput(output="chunk-1", finished=True, chunk_index=1, total_chunks=2),
|
||||
]
|
||||
runner = _make_runner(cache_backend=None, cache_backend_name=None)
|
||||
runner.pipeline = _ChunkStepPipeline(chunks)
|
||||
runner.od_config.streaming_output = True
|
||||
runner.od_config.step_execution = True
|
||||
req = _make_request(skip_cache_refresh=True)
|
||||
req.request_id = "req"
|
||||
|
||||
monkeypatch.setattr(model_runner_module, "set_forward_context", _noop_forward_context)
|
||||
monkeypatch.setattr(model_runner_module.current_omni_platform, "reset_peak_memory_stats", lambda: None)
|
||||
monkeypatch.setattr(model_runner_module.current_omni_platform, "max_memory_reserved", lambda: 0)
|
||||
monkeypatch.setattr(model_runner_module.current_omni_platform, "max_memory_allocated", lambda: 0)
|
||||
scheduler_output = SimpleNamespace(
|
||||
finished_req_ids=set(),
|
||||
scheduled_new_reqs=[SimpleNamespace(request_id="req", req=req)],
|
||||
scheduled_cached_reqs=SimpleNamespace(request_ids=[]),
|
||||
)
|
||||
|
||||
first = DiffusionModelRunner.execute_stepwise(runner, scheduler_output)
|
||||
assert first.get_request_output("req").result is None
|
||||
|
||||
scheduler_output = SimpleNamespace(
|
||||
finished_req_ids=set(),
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=SimpleNamespace(request_ids=["req"]),
|
||||
)
|
||||
second = DiffusionModelRunner.execute_stepwise(runner, scheduler_output)
|
||||
assert second.get_request_output("req").result == chunks[0]
|
||||
assert second.get_request_output("req").finished is False
|
||||
|
||||
DiffusionModelRunner.execute_stepwise(runner, scheduler_output)
|
||||
fourth = DiffusionModelRunner.execute_stepwise(runner, scheduler_output)
|
||||
assert fourth.get_request_output("req").result == chunks[1]
|
||||
assert fourth.get_request_output("req").finished is True
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_execute_model_skips_cache_summary_without_active_cache_backend(monkeypatch):
|
||||
@@ -76,6 +195,9 @@ def test_execute_model_skips_cache_summary_without_active_cache_backend(monkeypa
|
||||
cache_summary_calls = []
|
||||
|
||||
monkeypatch.setattr(model_runner_module, "set_forward_context", _noop_forward_context)
|
||||
monkeypatch.setattr(model_runner_module.current_omni_platform, "reset_peak_memory_stats", lambda: None)
|
||||
monkeypatch.setattr(model_runner_module.current_omni_platform, "max_memory_reserved", lambda: 0)
|
||||
monkeypatch.setattr(model_runner_module.current_omni_platform, "max_memory_allocated", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
model_runner_module,
|
||||
"cache_summary",
|
||||
@@ -101,6 +223,9 @@ def test_execute_model_emits_cache_summary_with_active_cache_dit_backend(monkeyp
|
||||
cache_summary_calls = []
|
||||
|
||||
monkeypatch.setattr(model_runner_module, "set_forward_context", _noop_forward_context)
|
||||
monkeypatch.setattr(model_runner_module.current_omni_platform, "reset_peak_memory_stats", lambda: None)
|
||||
monkeypatch.setattr(model_runner_module.current_omni_platform, "max_memory_reserved", lambda: 0)
|
||||
monkeypatch.setattr(model_runner_module.current_omni_platform, "max_memory_allocated", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
model_runner_module,
|
||||
"cache_summary",
|
||||
@@ -157,6 +282,7 @@ def test_load_model_clears_cache_backend_for_unsupported_pipeline(monkeypatch):
|
||||
cache_config={},
|
||||
model_class_name="NextStep11Pipeline",
|
||||
enforce_eager=True,
|
||||
streaming_output=False,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(model_runner_module, "LoadConfig", lambda: object())
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import queue
|
||||
import threading
|
||||
from types import SimpleNamespace
|
||||
@@ -238,6 +239,35 @@ class TestRequestScheduler:
|
||||
assert finished == {req_id}
|
||||
assert self.scheduler.get_request_state(req_id).status == DiffusionRequestStatus.FINISHED_COMPLETED
|
||||
|
||||
def test_streaming_output_keeps_request_running_until_final_chunk(self) -> None:
|
||||
scheduler = StepScheduler()
|
||||
scheduler.initialize(SimpleNamespace())
|
||||
req_id = scheduler.add_request(_make_request("stream"))
|
||||
|
||||
sched_output = scheduler.schedule()
|
||||
chunk = RunnerOutput(
|
||||
request_id=req_id,
|
||||
step_index=1,
|
||||
finished=False,
|
||||
result=DiffusionOutput(output="chunk-0", finished=False, chunk_index=0, total_chunks=2),
|
||||
)
|
||||
finished = scheduler.update_from_output(sched_output, chunk)
|
||||
|
||||
assert finished == set()
|
||||
assert scheduler.get_request_state(req_id).status == DiffusionRequestStatus.RUNNING
|
||||
assert scheduler.has_requests() is True
|
||||
|
||||
final_chunk = RunnerOutput(
|
||||
request_id=req_id,
|
||||
step_index=2,
|
||||
finished=True,
|
||||
result=DiffusionOutput(output="chunk-1", finished=True, chunk_index=1, total_chunks=2),
|
||||
)
|
||||
finished = scheduler.update_from_output(sched_output, final_chunk)
|
||||
|
||||
assert finished == {req_id}
|
||||
assert scheduler.get_request_state(req_id).status == DiffusionRequestStatus.FINISHED_COMPLETED
|
||||
|
||||
def test_fifo_single_request_scheduling(self) -> None:
|
||||
req_id_a = self.scheduler.add_request(_make_request("a"))
|
||||
req_id_b = self.scheduler.add_request(_make_request("b"))
|
||||
@@ -369,6 +399,7 @@ class TestRequestScheduler:
|
||||
class TestDiffusionEngine:
|
||||
def test_add_req_and_wait_for_response_single_path(self, mocker: MockerFixture) -> None:
|
||||
engine = DiffusionEngine.__new__(DiffusionEngine)
|
||||
engine.od_config = SimpleNamespace(streaming_output=False)
|
||||
engine.scheduler = RequestScheduler()
|
||||
engine.scheduler.initialize(SimpleNamespace())
|
||||
engine._rpc_lock = threading.RLock()
|
||||
@@ -391,6 +422,7 @@ class TestDiffusionEngine:
|
||||
scheduler = _StubScheduler(request, runner_output)
|
||||
|
||||
engine = DiffusionEngine.__new__(DiffusionEngine)
|
||||
engine.od_config = SimpleNamespace(streaming_output=False)
|
||||
engine.scheduler = scheduler
|
||||
engine._rpc_lock = threading.RLock()
|
||||
engine._cv = threading.Condition(engine._rpc_lock)
|
||||
@@ -410,7 +442,7 @@ class TestDiffusionEngine:
|
||||
) -> None:
|
||||
request = _make_request("init")
|
||||
scheduler = _StubScheduler(request, DiffusionOutput(output=None))
|
||||
od_config = SimpleNamespace(model_class_name="mock_model")
|
||||
od_config = SimpleNamespace(model_class_name="mock_model", streaming_output=False)
|
||||
fake_executor_cls = mocker.Mock(return_value=mocker.Mock())
|
||||
|
||||
monkeypatch.setattr(
|
||||
@@ -472,7 +504,7 @@ class TestDiffusionEngine:
|
||||
|
||||
def test_finalize_finished_request_returns_aborted_output(self) -> None:
|
||||
engine = DiffusionEngine.__new__(DiffusionEngine)
|
||||
engine.scheduler = RequestScheduler()
|
||||
engine.scheduler = StepScheduler()
|
||||
engine.scheduler.initialize(SimpleNamespace())
|
||||
|
||||
req_id = engine.scheduler.add_request(_make_request("req-finalize"))
|
||||
@@ -483,12 +515,77 @@ class TestDiffusionEngine:
|
||||
assert output.aborted is True
|
||||
assert output.abort_message == "Request req-finalize aborted."
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_runner_output_notifies_each_chunk(self) -> None:
|
||||
engine = DiffusionEngine.__new__(DiffusionEngine)
|
||||
engine.scheduler = StepScheduler()
|
||||
engine.scheduler.initialize(SimpleNamespace())
|
||||
engine._rpc_lock = threading.RLock()
|
||||
engine._cv = threading.Condition(engine._rpc_lock)
|
||||
engine._out_queue_streaming = {}
|
||||
engine.main_loop = asyncio.get_running_loop()
|
||||
|
||||
req_id = engine.scheduler.add_request(_make_request("stream-engine"))
|
||||
queue: asyncio.Queue[DiffusionOutput] = asyncio.Queue()
|
||||
engine._out_queue_streaming[req_id] = queue
|
||||
sched_output = engine.scheduler.schedule()
|
||||
|
||||
chunk = RunnerOutput(
|
||||
request_id=req_id,
|
||||
step_index=1,
|
||||
finished=False,
|
||||
result=DiffusionOutput(output="chunk-0", finished=False, chunk_index=0, total_chunks=2),
|
||||
)
|
||||
finished_req_ids = engine.scheduler.update_from_output(sched_output, chunk)
|
||||
engine._handle_step_streaming_runner_output(finished_req_ids, sched_output.scheduled_request_ids, chunk)
|
||||
|
||||
notified_chunk = await asyncio.wait_for(queue.get(), timeout=1)
|
||||
assert notified_chunk.output == "chunk-0"
|
||||
assert notified_chunk.finished is False
|
||||
assert engine.scheduler.get_request_state(req_id).status == DiffusionRequestStatus.RUNNING
|
||||
|
||||
final_chunk = RunnerOutput(
|
||||
request_id=req_id,
|
||||
step_index=2,
|
||||
finished=True,
|
||||
result=DiffusionOutput(output="chunk-1", finished=True, chunk_index=1, total_chunks=2),
|
||||
)
|
||||
finished_req_ids = engine.scheduler.update_from_output(sched_output, final_chunk)
|
||||
engine._handle_step_streaming_runner_output(finished_req_ids, sched_output.scheduled_request_ids, final_chunk)
|
||||
|
||||
notified_final = await asyncio.wait_for(queue.get(), timeout=1)
|
||||
assert notified_final.output == "chunk-1"
|
||||
assert notified_final.finished is True
|
||||
assert engine.scheduler.get_request_state(req_id) is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_finished_streaming_request_without_runner_output_notifies_waiter(self) -> None:
|
||||
engine = DiffusionEngine.__new__(DiffusionEngine)
|
||||
engine.scheduler = RequestScheduler()
|
||||
engine.scheduler.initialize(SimpleNamespace())
|
||||
engine._rpc_lock = threading.RLock()
|
||||
engine._cv = threading.Condition(engine._rpc_lock)
|
||||
engine._out_queue_streaming = {}
|
||||
engine.main_loop = asyncio.get_running_loop()
|
||||
|
||||
req_id = engine.scheduler.add_request(_make_request("stream-abort"))
|
||||
queue: asyncio.Queue[DiffusionOutput] = asyncio.Queue()
|
||||
engine._out_queue_streaming[req_id] = queue
|
||||
engine.scheduler.finish_requests(req_id, DiffusionRequestStatus.FINISHED_ABORTED)
|
||||
|
||||
engine._handle_empty_streaming_requests({req_id})
|
||||
|
||||
output = await asyncio.wait_for(queue.get(), timeout=1)
|
||||
assert output.aborted is True
|
||||
assert output.finished is True
|
||||
assert engine.scheduler.get_request_state(req_id) is None
|
||||
|
||||
def test_initializes_step_scheduler_when_step_execution_enabled(
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
od_config = SimpleNamespace(model_class_name="mock_model")
|
||||
od_config = SimpleNamespace(model_class_name="mock_model", streaming_output=False)
|
||||
od_config.step_execution = True
|
||||
fake_executor = mocker.Mock()
|
||||
fake_executor_cls = mocker.Mock(return_value=fake_executor)
|
||||
|
||||
@@ -249,6 +249,7 @@ def _make_runner():
|
||||
runner.od_config = SimpleNamespace(
|
||||
cache_backend=None,
|
||||
parallel_config=SimpleNamespace(use_hsdp=False),
|
||||
streaming_output=False,
|
||||
)
|
||||
runner.device = torch.device("cpu")
|
||||
runner.pipeline = _StepPipeline()
|
||||
@@ -267,6 +268,7 @@ def _make_distributed_runner(mode: str, device: torch.device):
|
||||
runner.od_config = SimpleNamespace(
|
||||
cache_backend=None,
|
||||
parallel_config=SimpleNamespace(use_hsdp=False),
|
||||
streaming_output=False,
|
||||
)
|
||||
runner.device = device
|
||||
runner.pipeline = _DistributedStepPipeline(mode=mode, device=device)
|
||||
@@ -327,7 +329,7 @@ def _make_cached_scheduler_output(request_id="req-1", step_id=1, finished_req_id
|
||||
|
||||
def _make_engine(scheduler, execute_fn=None) -> DiffusionEngine:
|
||||
engine = object.__new__(DiffusionEngine)
|
||||
engine.od_config = SimpleNamespace(model_class_name="QwenImagePipeline")
|
||||
engine.od_config = SimpleNamespace(model_class_name="QwenImagePipeline", streaming_output=False)
|
||||
engine.pre_process_func = None
|
||||
engine.post_process_func = None
|
||||
engine.scheduler = scheduler
|
||||
@@ -563,6 +565,7 @@ class TestRunner:
|
||||
step_execution=True,
|
||||
model_class_name="RequestOnlyPipeline",
|
||||
parallel_config=SimpleNamespace(use_hsdp=False),
|
||||
streaming_output=False,
|
||||
)
|
||||
runner.device = torch.device("cpu")
|
||||
runner.pipeline = None
|
||||
@@ -710,6 +713,7 @@ class TestExecutor:
|
||||
|
||||
def test_execute_step_passes_through_runner_output(self, mocker: MockerFixture):
|
||||
executor = object.__new__(MultiprocDiffusionExecutor)
|
||||
executor.od_config = SimpleNamespace(streaming_output=False)
|
||||
executor._ensure_open = lambda: None
|
||||
expected = RunnerOutput(request_id="req-step", step_index=1, finished=False, result=None)
|
||||
executor.collective_rpc = mocker.Mock(return_value=expected)
|
||||
|
||||
@@ -0,0 +1,525 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Integration tests for diffusion pipeline streaming output mode.
|
||||
|
||||
Covers:
|
||||
- Mock pipeline -> StageDiffusionClient (ZMQ subprocess path)
|
||||
- Mock pipeline -> InlineStageDiffusionClient -> Orchestrator -> AsyncOmni
|
||||
- Mock pipeline -> InlineStageDiffusionClient -> Orchestrator -> AsyncOmni -> `/v1/realtime/video`
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import queue
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
import zmq
|
||||
from fastapi import FastAPI, WebSocket
|
||||
from pytest_mock import MockerFixture
|
||||
from starlette.testclient import TestClient
|
||||
from vllm.utils.network_utils import get_open_zmq_ipc_path
|
||||
|
||||
import vllm_omni.diffusion.worker.diffusion_model_runner as model_runner_module
|
||||
from tests.engine.test_orchestrator import OrchestratorFixture, _build_harness, _wait_for
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.inline_stage_diffusion_client import InlineStageDiffusionClient
|
||||
from vllm_omni.diffusion.stage_diffusion_client import StageDiffusionClient
|
||||
from vllm_omni.diffusion.stage_diffusion_proc import StageDiffusionProc
|
||||
from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner
|
||||
from vllm_omni.distributed.omni_connectors.utils.serialization import (
|
||||
OmniMsgpackDecoder,
|
||||
OmniMsgpackEncoder,
|
||||
)
|
||||
from vllm_omni.engine.async_omni_engine import StageRuntimeInfo
|
||||
from vllm_omni.engine.messages import ShutdownRequestMessage, StageSubmissionMessage
|
||||
from vllm_omni.engine.stage_init_utils import StageMetadata
|
||||
from vllm_omni.entrypoints.async_omni import AsyncEventResolver, AsyncOmni
|
||||
from vllm_omni.entrypoints.openai.serving_video_output_stream import OmniStreamingVideoOutputHandler
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _StepStreamingPipeline:
|
||||
supports_step_execution = True
|
||||
|
||||
def __init__(self, outputs: list[DiffusionOutput]) -> None:
|
||||
self.outputs = outputs
|
||||
self.requests: list[Any] = []
|
||||
|
||||
def step_outputs(self, request):
|
||||
self.requests.append(request)
|
||||
return list(self.outputs)
|
||||
|
||||
|
||||
class _FailingStreamingPipeline:
|
||||
supports_step_execution = True
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.requests: list[Any] = []
|
||||
|
||||
def step_outputs(self, request):
|
||||
self.requests.append(request)
|
||||
raise RuntimeError("stream failed midway")
|
||||
|
||||
|
||||
class _PipelineBackedEngine:
|
||||
"""DiffusionEngine stand-in that exposes step-streaming chunks from a mock pipeline."""
|
||||
|
||||
def __init__(self, pipeline: _StepStreamingPipeline | _FailingStreamingPipeline) -> None:
|
||||
self.pipeline = pipeline
|
||||
self.executor = SimpleNamespace(
|
||||
register_failure_callback=MagicMock(),
|
||||
check_health=MagicMock(),
|
||||
)
|
||||
|
||||
async def step_streaming(self, request):
|
||||
try:
|
||||
outputs = self.pipeline.step_outputs(request)
|
||||
except Exception as exc:
|
||||
outputs = [
|
||||
DiffusionOutput(custom_output={"chunk": 0}, finished=False),
|
||||
DiffusionOutput(error=str(exc), finished=True),
|
||||
]
|
||||
for output in outputs:
|
||||
if output.error is not None:
|
||||
yield [OmniRequestOutput.from_error(request_id=request.request_id, error_message=output.error)]
|
||||
continue
|
||||
custom_output = output.custom_output or {}
|
||||
yield [
|
||||
OmniRequestOutput.from_diffusion(
|
||||
request_id=request.request_id,
|
||||
images=list(custom_output.get("images") or []),
|
||||
custom_output=custom_output,
|
||||
finished=output.finished,
|
||||
)
|
||||
]
|
||||
|
||||
def abort(self, request_id: str) -> None:
|
||||
del request_id
|
||||
|
||||
|
||||
class TestPipelineStreamingOutputToStageDiffusionClient:
|
||||
"""Streaming pipeline output over ZMQ into ``StageDiffusionClient``."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_output_reaches_stage_client_from_mock_pipeline(self) -> None:
|
||||
"""Mock pipeline chunks over ZMQ reach StageDiffusionClient with correct finished flags."""
|
||||
pipeline = _StepStreamingPipeline(
|
||||
[
|
||||
DiffusionOutput(custom_output={"chunk": 0}, finished=False),
|
||||
DiffusionOutput(custom_output={"chunk": 1}, finished=True),
|
||||
]
|
||||
)
|
||||
|
||||
outputs = await self._run_streaming_client_proc(pipeline, request_id="req-stream")
|
||||
|
||||
assert [output.request_id for output in outputs] == ["req-stream", "req-stream"]
|
||||
assert [output.finished for output in outputs] == [False, True]
|
||||
assert [output.custom_output["chunk"] for output in outputs] == [0, 1]
|
||||
assert pipeline.requests[0].request_id == "req-stream"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_midway_error_reaches_stage_client_after_prior_chunk(self) -> None:
|
||||
"""A pipeline error after the first chunk still delivers that chunk then a terminal error output."""
|
||||
pipeline = _FailingStreamingPipeline()
|
||||
|
||||
outputs = await self._run_streaming_client_proc(pipeline, request_id="req-error")
|
||||
|
||||
assert len(outputs) == 2
|
||||
assert outputs[0].request_id == "req-error"
|
||||
assert outputs[0].finished is False
|
||||
assert outputs[0].custom_output["chunk"] == 0
|
||||
assert outputs[1].request_id == "req-error"
|
||||
assert outputs[1].finished is True
|
||||
assert outputs[1].error is not None
|
||||
assert "stream failed midway" in outputs[1].error
|
||||
|
||||
@classmethod
|
||||
async def _run_streaming_client_proc(
|
||||
cls,
|
||||
pipeline: _StepStreamingPipeline | _FailingStreamingPipeline,
|
||||
*,
|
||||
request_id: str,
|
||||
) -> list[OmniRequestOutput]:
|
||||
request_address = get_open_zmq_ipc_path()
|
||||
response_address = get_open_zmq_ipc_path()
|
||||
proc = object.__new__(StageDiffusionProc)
|
||||
proc._od_config = SimpleNamespace(streaming_output=True)
|
||||
proc._engine = _PipelineBackedEngine(pipeline)
|
||||
proc._closed = False
|
||||
client = cls._make_client(request_address, response_address)
|
||||
proc_task = asyncio.create_task(proc.run_loop(request_address, response_address))
|
||||
try:
|
||||
await asyncio.sleep(0.05)
|
||||
await client.add_request_async(
|
||||
request_id,
|
||||
"prompt",
|
||||
OmniDiffusionSamplingParams(),
|
||||
)
|
||||
deadline = asyncio.get_running_loop().time() + 2.0
|
||||
outputs: list[OmniRequestOutput] = []
|
||||
while (not outputs or not outputs[-1].finished) and asyncio.get_running_loop().time() < deadline:
|
||||
output = client.get_diffusion_output_nowait()
|
||||
if output is not None:
|
||||
outputs.append(output)
|
||||
else:
|
||||
await asyncio.sleep(0.01)
|
||||
return outputs
|
||||
finally:
|
||||
try:
|
||||
client._request_socket.send(client._encoder.encode({"type": "shutdown"}))
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
await asyncio.wait_for(proc_task, timeout=2)
|
||||
finally:
|
||||
cls._close_client(client)
|
||||
|
||||
@staticmethod
|
||||
def _make_client(request_address: str, response_address: str) -> StageDiffusionClient:
|
||||
client = object.__new__(StageDiffusionClient)
|
||||
client.stage_id = 0
|
||||
client.replica_id = 0
|
||||
client.final_output = True
|
||||
client.final_output_type = "image"
|
||||
client.default_sampling_params = {}
|
||||
client.requires_multimodal_data = False
|
||||
client.custom_process_input_func = None
|
||||
client.engine_input_source = []
|
||||
client._proc = None
|
||||
client._proc_manager = None
|
||||
client._owns_process = False
|
||||
client._zmq_ctx = zmq.Context()
|
||||
client._request_socket = client._zmq_ctx.socket(zmq.PUSH)
|
||||
client._request_socket.bind(request_address)
|
||||
client._response_socket = client._zmq_ctx.socket(zmq.PULL)
|
||||
client._response_socket.bind(response_address)
|
||||
client._encoder = OmniMsgpackEncoder()
|
||||
client._decoder = OmniMsgpackDecoder()
|
||||
client._output_queue: asyncio.Queue[OmniRequestOutput] = asyncio.Queue()
|
||||
client._rpc_results = {}
|
||||
client._pending_rpcs = set()
|
||||
client._tasks = {}
|
||||
client._shutting_down = False
|
||||
client._engine_dead = False
|
||||
return client
|
||||
|
||||
@staticmethod
|
||||
def _close_client(client: StageDiffusionClient) -> None:
|
||||
client._request_socket.close(linger=0)
|
||||
client._response_socket.close(linger=0)
|
||||
client._zmq_ctx.term()
|
||||
|
||||
|
||||
class TestPipelineStreamingOutputToEntrypoint:
|
||||
"""Streaming pipeline output through inline stage, orchestrator, and entrypoints."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_output_reaches_async_omni_from_mock_pipeline(self) -> None:
|
||||
"""Mock pipeline streaming chunks reach AsyncOmni.generate() via inline stage and orchestrator."""
|
||||
pipeline = _StepStreamingPipeline(
|
||||
[
|
||||
DiffusionOutput(custom_output={"chunk": 0}, finished=False),
|
||||
DiffusionOutput(custom_output={"chunk": 1}, finished=True),
|
||||
]
|
||||
)
|
||||
inline_client = self._make_inline_pipeline_client(pipeline)
|
||||
fixture = _build_harness([inline_client])
|
||||
omni = self._make_async_omni(self._OrchestratorBridgeEngine(fixture))
|
||||
|
||||
try:
|
||||
outputs: list[OmniRequestOutput] = []
|
||||
async for output in omni.generate(
|
||||
prompt={"prompt": "a cat"},
|
||||
request_id="req-omni",
|
||||
sampling_params_list=[OmniDiffusionSamplingParams()],
|
||||
output_modalities=["image"],
|
||||
):
|
||||
outputs.append(output)
|
||||
|
||||
await _wait_for(lambda: len(pipeline.requests) == 1)
|
||||
assert pipeline.requests[0].request_id.startswith("req-omni-")
|
||||
assert [output.custom_output["chunk"] for output in outputs] == [0, 1]
|
||||
assert [getattr(output.request_output, "finished", output.finished) for output in outputs] == [False, True]
|
||||
finally:
|
||||
await self._shutdown_pipeline_omni_harness(omni, fixture, inline_client)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_output_reaches_api_websocket_from_mock_pipeline(self, mocker: MockerFixture) -> None:
|
||||
"""Mock pipeline streaming chunks reach the video WebSocket handler through AsyncOmni.generate()."""
|
||||
frames = [np.full((4, 4, 3), fill_value=i, dtype=np.uint8) for i in range(2)]
|
||||
pipeline = _StepStreamingPipeline(
|
||||
[
|
||||
DiffusionOutput(custom_output={"chunk": 0, "images": [frames]}, finished=False),
|
||||
DiffusionOutput(custom_output={"chunk": 1, "images": [frames]}, finished=True),
|
||||
]
|
||||
)
|
||||
inline_client = self._make_inline_pipeline_client(pipeline)
|
||||
fixture = _build_harness([inline_client])
|
||||
bridge_engine = self._OrchestratorBridgeEngine(fixture)
|
||||
omni = self._make_async_omni(bridge_engine)
|
||||
|
||||
encoded = [b"pipeline-fmp4-0", b"pipeline-fmp4-1"]
|
||||
|
||||
class FakeStreamingVideoEncoder:
|
||||
encode_calls: list[int] = []
|
||||
|
||||
def encode(self, video):
|
||||
del video
|
||||
idx = len(self.encode_calls)
|
||||
self.encode_calls.append(idx)
|
||||
return encoded[idx]
|
||||
|
||||
def close(self):
|
||||
return b""
|
||||
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video_output_stream.create_streaming_video_encoder",
|
||||
lambda *, output_format, fps, video_codec_options=None: FakeStreamingVideoEncoder(),
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video_output_stream.get_stage_type",
|
||||
return_value="diffusion",
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video_output_stream.build_stage_sampling_params_list",
|
||||
return_value=[OmniDiffusionSamplingParams()],
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video_output_stream.get_default_sampling_params_list",
|
||||
return_value=[OmniDiffusionSamplingParams()],
|
||||
)
|
||||
|
||||
handler = OmniStreamingVideoOutputHandler(
|
||||
engine_client=omni,
|
||||
model_name="test-model",
|
||||
stage_configs=bridge_engine.stage_configs,
|
||||
stall_timeout=5.0,
|
||||
start_timeout=5.0,
|
||||
)
|
||||
app = FastAPI()
|
||||
|
||||
@app.websocket("/v1/realtime/video")
|
||||
async def ws_endpoint(websocket: WebSocket):
|
||||
await handler.handle_session(websocket)
|
||||
|
||||
try:
|
||||
with TestClient(app) as client:
|
||||
with client.websocket_connect("/v1/realtime/video") as ws:
|
||||
ws.send_json({"type": "session.start", "prompt": "integration test"})
|
||||
assert ws.receive_json()["type"] == "video.start"
|
||||
assert ws.receive_bytes() == b"pipeline-fmp4-0"
|
||||
assert ws.receive_bytes() == b"pipeline-fmp4-1"
|
||||
done = ws.receive_json()
|
||||
assert done["type"] == "session.done"
|
||||
assert done["chunks"] == 2
|
||||
|
||||
await _wait_for(lambda: len(pipeline.requests) == 1)
|
||||
assert pipeline.requests[0].request_id
|
||||
finally:
|
||||
await self._shutdown_pipeline_omni_harness(omni, fixture, inline_client)
|
||||
|
||||
class _OrchestratorBridgeEngine:
|
||||
"""Minimal AsyncOmni engine facade backed by a live Orchestrator harness."""
|
||||
|
||||
def __init__(self, fixture: OrchestratorFixture) -> None:
|
||||
self._fixture = fixture
|
||||
self.stage_metadata = [
|
||||
StageRuntimeInfo(
|
||||
stage_type="diffusion",
|
||||
final_output=True,
|
||||
final_output_type="image",
|
||||
)
|
||||
]
|
||||
self.stage_configs: list[Any] = [SimpleNamespace(stage_type="diffusion")]
|
||||
self.default_sampling_params_list = [OmniDiffusionSamplingParams()]
|
||||
self.num_stages = 1
|
||||
self.supported_tasks = ("generate",)
|
||||
self._alive = True
|
||||
|
||||
async def add_request_async(
|
||||
self,
|
||||
*,
|
||||
request_id: str,
|
||||
prompt: Any,
|
||||
sampling_params_list: list[Any],
|
||||
final_stage_id: int,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
self._fixture.request_sync_q.put_nowait(
|
||||
StageSubmissionMessage(
|
||||
type="add_request",
|
||||
request_id=request_id,
|
||||
prompt=prompt,
|
||||
original_prompt=prompt,
|
||||
output_prompt_text=None,
|
||||
sampling_params_list=sampling_params_list,
|
||||
final_stage_id=final_stage_id,
|
||||
final_output_stage_ids=kwargs.get("final_output_stage_ids"),
|
||||
preprocess_ms=0.0,
|
||||
request_timestamp=kwargs.get("arrival_time", time.time()),
|
||||
enqueue_ts=time.perf_counter(),
|
||||
)
|
||||
)
|
||||
|
||||
async def try_get_output_async(self) -> Any | None:
|
||||
try:
|
||||
return self._fixture.output_sync_q.get_nowait()
|
||||
except queue.Empty:
|
||||
return None
|
||||
|
||||
def get_stage_metadata(self, stage_id: int) -> StageRuntimeInfo:
|
||||
return self.stage_metadata[stage_id]
|
||||
|
||||
def is_alive(self) -> bool:
|
||||
return self._fixture.thread.is_alive()
|
||||
|
||||
async def abort_async(self, request_ids: list[str]) -> None:
|
||||
del request_ids
|
||||
|
||||
@classmethod
|
||||
def _make_inline_pipeline_client(cls, pipeline: _StepStreamingPipeline) -> InlineStageDiffusionClient:
|
||||
"""Build an inline diffusion stage client whose engine runs the given mock pipeline."""
|
||||
metadata = StageMetadata(
|
||||
stage_id=0,
|
||||
stage_type="diffusion",
|
||||
engine_output_type="image",
|
||||
is_comprehension=False,
|
||||
requires_multimodal_data=False,
|
||||
engine_input_source=[],
|
||||
final_output=True,
|
||||
final_output_type="image",
|
||||
default_sampling_params=OmniDiffusionSamplingParams(),
|
||||
custom_process_input_func=None,
|
||||
model_stage=None,
|
||||
runtime_cfg=None,
|
||||
)
|
||||
pipeline_engine = _PipelineBackedEngine(pipeline)
|
||||
with patch.object(InlineStageDiffusionClient, "_enrich_config"):
|
||||
with patch(
|
||||
"vllm_omni.diffusion.inline_stage_diffusion_client.DiffusionEngine.make_engine",
|
||||
return_value=pipeline_engine,
|
||||
):
|
||||
od_config = MagicMock(spec=OmniDiffusionConfig)
|
||||
od_config.streaming_output = True
|
||||
return InlineStageDiffusionClient(
|
||||
model="test_model",
|
||||
od_config=od_config,
|
||||
metadata=metadata,
|
||||
batch_size=1,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _make_async_omni(engine: _OrchestratorBridgeEngine) -> AsyncOmni:
|
||||
omni = object.__new__(AsyncOmni)
|
||||
omni.engine = engine
|
||||
omni.log_stats = False
|
||||
omni._pause_cond = asyncio.Condition()
|
||||
omni._paused = False
|
||||
omni.request_states = {}
|
||||
omni.final_output_task = None
|
||||
omni.event_resolver = AsyncEventResolver()
|
||||
omni._enable_ar_profiler = False
|
||||
omni._is_sleeping = False
|
||||
omni.prom_metrics = MagicMock()
|
||||
omni.mod_metrics = MagicMock()
|
||||
omni.resolve_sampling_params_list = lambda params, allow_delta_coercion: params
|
||||
omni._compute_final_stage_id = lambda output_modalities: 0
|
||||
omni._compute_final_output_stage_ids = lambda output_modalities: [0]
|
||||
omni.default_sampling_params_list = engine.default_sampling_params_list
|
||||
omni._log_summary_and_cleanup = lambda request_id: omni.request_states.pop(request_id, None)
|
||||
return omni
|
||||
|
||||
@staticmethod
|
||||
async def _shutdown_pipeline_omni_harness(
|
||||
omni: AsyncOmni,
|
||||
fixture: OrchestratorFixture,
|
||||
inline_client: InlineStageDiffusionClient,
|
||||
) -> None:
|
||||
if omni.final_output_task is not None:
|
||||
omni.final_output_task.cancel()
|
||||
await asyncio.gather(omni.final_output_task, return_exceptions=True)
|
||||
inline_client.shutdown()
|
||||
fixture.request_sync_q.put_nowait(ShutdownRequestMessage())
|
||||
await asyncio.to_thread(fixture.thread.join, 5)
|
||||
|
||||
|
||||
def _make_vllm_config():
|
||||
@contextlib.contextmanager
|
||||
def set_priority(*args, **kwargs):
|
||||
del args, kwargs
|
||||
yield
|
||||
|
||||
return SimpleNamespace(
|
||||
kernel_config=SimpleNamespace(ir_op_priority=SimpleNamespace(set_priority=set_priority)),
|
||||
compilation_config=SimpleNamespace(ir_enable_torch_wrap=True),
|
||||
)
|
||||
|
||||
|
||||
class TestSupportedPipelines:
|
||||
"""Streaming-output protocol checks for supported pipelines."""
|
||||
|
||||
def test_helios_supports_step_execution_for_streaming_output(self) -> None:
|
||||
from vllm_omni.diffusion.models.helios.pipeline_helios import HeliosPipeline
|
||||
from vllm_omni.diffusion.models.interface import SupportsStepExecution, supports_step_execution
|
||||
|
||||
# Avoid loading model weights; protocol membership depends on the class contract.
|
||||
pipeline = object.__new__(HeliosPipeline)
|
||||
|
||||
assert pipeline.supports_step_execution is True
|
||||
assert supports_step_execution(pipeline) is True
|
||||
assert isinstance(pipeline, SupportsStepExecution) is True
|
||||
|
||||
def test_load_model_rejects_streaming_output_without_step_execution(self, monkeypatch) -> None:
|
||||
class _NoStepPipeline:
|
||||
def forward(self): ...
|
||||
|
||||
class _FakeDiffusersPipelineLoader:
|
||||
def __init__(self, *args, **kwargs): ...
|
||||
|
||||
def load_model(self, **kwargs):
|
||||
return _NoStepPipeline()
|
||||
|
||||
class _FakeDeviceMemoryProfiler:
|
||||
consumed_memory = 0
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
runner = object.__new__(DiffusionModelRunner)
|
||||
runner.vllm_config = _make_vllm_config()
|
||||
runner.od_config = SimpleNamespace(
|
||||
enable_cpu_offload=False,
|
||||
enable_layerwise_offload=False,
|
||||
enforce_eager=True,
|
||||
cache_backend=None,
|
||||
cache_config=None,
|
||||
streaming_output=True,
|
||||
step_execution=False,
|
||||
model_class_name="NoStepPipeline",
|
||||
parallel_config=SimpleNamespace(use_hsdp=False),
|
||||
)
|
||||
runner.device = torch.device("cpu")
|
||||
runner.pipeline = None
|
||||
runner.cache_backend = None
|
||||
runner.offload_backend = None
|
||||
runner.kv_transfer_manager = SimpleNamespace()
|
||||
|
||||
monkeypatch.setattr(model_runner_module, "DiffusersPipelineLoader", _FakeDiffusersPipelineLoader)
|
||||
monkeypatch.setattr(model_runner_module, "DeviceMemoryProfiler", _FakeDeviceMemoryProfiler)
|
||||
monkeypatch.setattr(model_runner_module, "get_offload_backend", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(model_runner_module, "get_cache_backend", lambda *args, **kwargs: None)
|
||||
|
||||
with pytest.raises(ValueError, match="NoStepPipeline"):
|
||||
DiffusionModelRunner.load_model(runner)
|
||||
@@ -46,6 +46,7 @@ def client(mock_engine):
|
||||
)
|
||||
with patch.object(InlineStageDiffusionClient, "_enrich_config"):
|
||||
od_config = MagicMock(spec=OmniDiffusionConfig)
|
||||
od_config.streaming_output = False
|
||||
c = InlineStageDiffusionClient(model="test_model", od_config=od_config, metadata=metadata, batch_size=1)
|
||||
yield c
|
||||
c.shutdown()
|
||||
@@ -72,6 +73,35 @@ async def test_inline_dispatch_request_success(client, mock_engine):
|
||||
mock_engine.step.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_inline_dispatch_request_streaming_success(client, mock_engine):
|
||||
"""Inline StageDiffusionClient correctly receives streaming chunk output from Diffusion Engine"""
|
||||
chunks = [
|
||||
OmniRequestOutput.from_diffusion(request_id="req-stream", images=[], finished=False),
|
||||
OmniRequestOutput.from_diffusion(request_id="req-stream", images=[], finished=True),
|
||||
]
|
||||
|
||||
async def _step_streaming(_request):
|
||||
for chunk in chunks:
|
||||
yield [chunk]
|
||||
|
||||
client.od_config.streaming_output = True
|
||||
mock_engine.step_streaming = _step_streaming
|
||||
|
||||
await client.add_request_async("req-stream", "A test prompt", OmniDiffusionSamplingParams())
|
||||
|
||||
outputs = []
|
||||
for _ in range(20):
|
||||
output = client.get_diffusion_output_nowait()
|
||||
if output is not None:
|
||||
outputs.append(output)
|
||||
if output.finished:
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
assert outputs == chunks
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_inline_dispatch_request_error(client, mock_engine):
|
||||
# Setup mock engine step to raise an exception
|
||||
|
||||
@@ -21,6 +21,7 @@ from vllm_omni.diffusion.ipc import DIFFUSION_RPC_RESULT_ENVELOPE
|
||||
from vllm_omni.diffusion.sched import RequestScheduler
|
||||
from vllm_omni.diffusion.stage_diffusion_proc import StageDiffusionProc
|
||||
from vllm_omni.diffusion.worker.diffusion_worker import WorkerProc
|
||||
from vllm_omni.diffusion.worker.utils import RunnerOutput
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
@@ -49,7 +50,7 @@ def _make_executor(num_gpus: int = 1):
|
||||
|
||||
Returns ``(executor, request_queue, result_queue)``.
|
||||
"""
|
||||
od_cfg = SimpleNamespace(num_gpus=num_gpus)
|
||||
od_cfg = SimpleNamespace(num_gpus=num_gpus, streaming_output=False)
|
||||
monkeypatch = pytest.MonkeyPatch()
|
||||
monkeypatch.setattr(MultiprocDiffusionExecutor, "_init_executor", lambda self: None)
|
||||
executor = MultiprocDiffusionExecutor(od_cfg)
|
||||
@@ -75,6 +76,7 @@ def _make_engine(num_gpus: int = 1):
|
||||
"""Create a lightweight ``DiffusionEngine`` wired to mocked executor."""
|
||||
executor, req_q, res_q = _make_executor(num_gpus)
|
||||
engine = DiffusionEngine.__new__(DiffusionEngine)
|
||||
engine.od_config = SimpleNamespace(streaming_output=False)
|
||||
sched = RequestScheduler()
|
||||
sched.initialize(SimpleNamespace())
|
||||
engine.scheduler = sched
|
||||
@@ -581,6 +583,58 @@ class TestMultiprocExecutorRaisesEngineDeadError:
|
||||
)
|
||||
|
||||
|
||||
class TestMultiprocExecutorStepStreamingOutput:
|
||||
"""Streaming output uses step execution and one worker reply per step."""
|
||||
|
||||
def test_execute_step_allows_streaming_output_mode(self):
|
||||
executor, req_q, res_q = _make_executor()
|
||||
executor.od_config = SimpleNamespace(streaming_output=True) # pyright: ignore[reportAttributeAccessIssue]
|
||||
runner_outputs = [
|
||||
RunnerOutput(
|
||||
request_id="sched-stream",
|
||||
step_index=1,
|
||||
finished=False,
|
||||
result=DiffusionOutput(output={"chunk": 0}, finished=False, chunk_index=0, total_chunks=2),
|
||||
),
|
||||
RunnerOutput(
|
||||
request_id="sched-stream",
|
||||
step_index=2,
|
||||
finished=True,
|
||||
result=DiffusionOutput(output={"chunk": 1}, finished=True, chunk_index=1, total_chunks=2),
|
||||
),
|
||||
]
|
||||
scheduler_output = SimpleNamespace(
|
||||
scheduled_request_ids=["sched-stream"],
|
||||
)
|
||||
|
||||
def _worker():
|
||||
for runner_output in runner_outputs:
|
||||
req_q.get(timeout=10)
|
||||
res_q.put(runner_output)
|
||||
|
||||
thread = threading.Thread(target=_worker, daemon=True)
|
||||
thread.start()
|
||||
|
||||
first: RunnerOutput = MultiprocDiffusionExecutor.execute_step( # pyright: ignore[reportAssignmentType]
|
||||
executor,
|
||||
scheduler_output, # pyright: ignore[reportArgumentType]
|
||||
)
|
||||
second: RunnerOutput = MultiprocDiffusionExecutor.execute_step( # pyright: ignore[reportAssignmentType]
|
||||
executor,
|
||||
scheduler_output, # pyright: ignore[reportArgumentType]
|
||||
)
|
||||
|
||||
assert first is runner_outputs[0]
|
||||
assert first.result is not None
|
||||
assert first.result.output == {"chunk": 0}
|
||||
assert first.finished is False
|
||||
assert second is runner_outputs[1]
|
||||
assert second.result is not None
|
||||
assert second.result.output == {"chunk": 1}
|
||||
assert second.finished is True
|
||||
thread.join(timeout=2)
|
||||
|
||||
|
||||
class TestDiffusionEngineDeadErrorPassthrough:
|
||||
"""``DiffusionEngine.add_req_and_wait_for_response`` re-raises
|
||||
``EngineDeadError`` from executor and wraps other errors."""
|
||||
|
||||
@@ -11,6 +11,7 @@ import pytest
|
||||
|
||||
from vllm_omni.diffusion.stage_diffusion_proc import StageDiffusionProc
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
|
||||
|
||||
@@ -54,6 +55,7 @@ def test_process_batch_request_preserves_parent_request_id_and_kv_sender_info():
|
||||
|
||||
proc = object.__new__(StageDiffusionProc)
|
||||
proc._engine = SimpleNamespace(step=step)
|
||||
proc._od_config = SimpleNamespace(streaming_output=False)
|
||||
proc._executor = ThreadPoolExecutor(max_workers=1)
|
||||
|
||||
try:
|
||||
@@ -102,6 +104,40 @@ class MockDiffusionEngine:
|
||||
return [MockOmniRequestOutput(request_id=request.request_id)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_proc_streaming_request_yields_each_engine_chunk():
|
||||
"""Ensure that the streaming output chunks from DiffusionEngine reaches StageDiffusionProc"""
|
||||
captured = {}
|
||||
chunks = [
|
||||
OmniRequestOutput.from_diffusion(request_id="", images=[], finished=False),
|
||||
OmniRequestOutput.from_diffusion(request_id="", images=[], finished=True),
|
||||
]
|
||||
|
||||
class _StreamingEngine:
|
||||
async def step_streaming(self, request):
|
||||
captured["request"] = request
|
||||
for chunk in chunks:
|
||||
yield [chunk]
|
||||
|
||||
stage_proc = object.__new__(StageDiffusionProc)
|
||||
stage_proc._engine = _StreamingEngine()
|
||||
|
||||
outputs = [
|
||||
output
|
||||
async for output in stage_proc._process_streaming_request(
|
||||
request_id="req-stream",
|
||||
prompt="prompt",
|
||||
sampling_params_dict=asdict(OmniDiffusionSamplingParams()),
|
||||
kv_sender_info={0: {"host": "127.0.0.1"}},
|
||||
)
|
||||
]
|
||||
|
||||
assert outputs == chunks
|
||||
assert [output.request_id for output in outputs] == ["req-stream", "req-stream"]
|
||||
assert [output.finished for output in outputs] == [False, True]
|
||||
assert captured["request"].kv_sender_info == {0: {"host": "127.0.0.1"}}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_proc_process_request_with_batching_async_output():
|
||||
stage_proc = object.__new__(StageDiffusionProc)
|
||||
|
||||
@@ -184,7 +184,7 @@ class TestWorkerWrapperBaseDelegation:
|
||||
mock_reqs = [mocker.Mock()]
|
||||
result = wrapper.execute_model(mock_reqs, mock_od_config)
|
||||
|
||||
wrapper.worker.execute_model.assert_called_once_with(mock_reqs, mock_od_config)
|
||||
wrapper.worker.execute_model.assert_called_once_with(mock_reqs, mock_od_config, kv_prefetch_jobs=None)
|
||||
assert result == mock_output
|
||||
|
||||
def test_load_weights_delegation(self, mocker: MockerFixture, mock_od_config):
|
||||
|
||||
@@ -0,0 +1,356 @@
|
||||
"""Background KV prefetch — CPU-testable mechanism.
|
||||
|
||||
Covers opt-in gating, the start_prefetch / consume_prefetched_kv round trip over a
|
||||
mock connector, dedup, per-call sender_info isolation, role classification, and
|
||||
miss/abort paths. GPU H2D and D2D paths need CUDA (integration tests).
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.distributed.omni_connectors.kv_transfer_manager import (
|
||||
OmniKVCacheConfig,
|
||||
OmniKVTransferManager,
|
||||
ReceiveRole,
|
||||
)
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
# start_prefetch() skips stubs without a sender endpoint, so submissions carry it.
|
||||
_SENDER_INFO = {"host": "127.0.0.1", "zmq_port": 50051}
|
||||
|
||||
|
||||
class MockConnector:
|
||||
def __init__(self):
|
||||
self.store = {}
|
||||
self.get_calls = []
|
||||
|
||||
def put(self, from_stage, to_stage, put_key, data):
|
||||
self.store[f"{from_stage}->{to_stage}:{put_key}"] = data
|
||||
return True, len(str(data)), None
|
||||
|
||||
def get(self, from_stage, to_stage, get_key, metadata=None):
|
||||
self.get_calls.append({"get_key": get_key, "metadata": metadata})
|
||||
key = f"{from_stage}->{to_stage}:{get_key}"
|
||||
if key in self.store:
|
||||
return self.store[key], len(str(self.store[key]))
|
||||
return None
|
||||
|
||||
|
||||
def _make_sender_receiver(*, async_prefetch=True):
|
||||
"""Build sender+receiver managers sharing one mock connector, seed one KV."""
|
||||
connector = MockConnector()
|
||||
|
||||
sender = OmniKVTransferManager(
|
||||
OmniKVCacheConfig(
|
||||
connector_config={"type": "mock"}, from_stage="sender", to_stage="receiver", need_send_cache=True
|
||||
)
|
||||
)
|
||||
sender._connector = connector
|
||||
|
||||
receiver = OmniKVTransferManager(
|
||||
OmniKVCacheConfig(
|
||||
connector_config={"type": "mock"},
|
||||
from_stage="sender",
|
||||
stage_id="receiver",
|
||||
need_recv_cache=True,
|
||||
recv_timeout=1.0,
|
||||
),
|
||||
async_prefetch=async_prefetch,
|
||||
)
|
||||
receiver._connector = connector
|
||||
return sender, receiver, connector
|
||||
|
||||
|
||||
def _seed_payload(sender, req_id, *, num_layers=2, block_size=4, num_heads=4, head_dim=8):
|
||||
kv_caches = [torch.randn(2, 5, block_size, num_heads, head_dim) for _ in range(num_layers)]
|
||||
sender.handle_finished_requests_kv_transfer(
|
||||
{req_id: {"block_ids": [0, 1], "seq_len": 10}}, kv_caches, block_size, "float32"
|
||||
)
|
||||
|
||||
|
||||
def _req(rid):
|
||||
"""A minimal request carrying just ``request_id`` (consume_prefetched_kv only needs that)."""
|
||||
return SimpleNamespace(request_id=rid)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Opt-in gating
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_async_prefetch_defaults_false():
|
||||
mgr = OmniKVTransferManager(OmniKVCacheConfig())
|
||||
assert mgr._async_prefetch is False
|
||||
|
||||
|
||||
def test_from_od_config_enables_prefetch_flag():
|
||||
od = SimpleNamespace(omni_kv_config={"enable_kv_async_prefetch": True})
|
||||
assert OmniKVTransferManager.from_od_config(od)._async_prefetch is True
|
||||
od2 = SimpleNamespace(omni_kv_config=None)
|
||||
assert OmniKVTransferManager.from_od_config(od2)._async_prefetch is False
|
||||
od3 = SimpleNamespace(omni_kv_config={"enable_kv_async_prefetch": False})
|
||||
assert OmniKVTransferManager.from_od_config(od3)._async_prefetch is False
|
||||
|
||||
|
||||
def test_prefetch_preserved_with_connector_config():
|
||||
# The flag must survive merging with a populated connector config.
|
||||
od = SimpleNamespace(
|
||||
omni_kv_config={
|
||||
"connector_config": {"type": "mock"},
|
||||
"need_recv_cache": True,
|
||||
"enable_kv_async_prefetch": True,
|
||||
},
|
||||
)
|
||||
assert OmniKVTransferManager.from_od_config(od)._async_prefetch is True
|
||||
|
||||
|
||||
def test_prefetch_disabled_with_cfg_companion_collector():
|
||||
# A CFG companion collector forces the synchronous path (see _resolve_async_prefetch).
|
||||
od = SimpleNamespace(
|
||||
omni_kv_config={"enable_kv_async_prefetch": True},
|
||||
cfg_kv_collect_func=lambda *a, **k: {},
|
||||
)
|
||||
assert OmniKVTransferManager.from_od_config(od)._async_prefetch is False
|
||||
|
||||
|
||||
def test_from_vllm_config_ar_path_is_always_sync():
|
||||
# The AR entry point never enables prefetch.
|
||||
model_config = SimpleNamespace(omni_kv_config=None)
|
||||
vllm_config = SimpleNamespace(kv_transfer_config=None)
|
||||
assert OmniKVTransferManager.from_vllm_config(vllm_config, model_config)._async_prefetch is False
|
||||
|
||||
|
||||
def test_start_prefetch_noop_when_disabled():
|
||||
_, receiver, _ = _make_sender_receiver(async_prefetch=False)
|
||||
receiver.start_prefetch({"request_id": "r1", "kv_sender_info": None})
|
||||
assert receiver._prefetch_futures == {}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Round trip
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_prefetch_round_trip_returns_data():
|
||||
sender, receiver, _ = _make_sender_receiver()
|
||||
_seed_payload(sender, "rid-1")
|
||||
|
||||
receiver.start_prefetch({"request_id": "rid-1", "kv_sender_info": _SENDER_INFO})
|
||||
assert "rid-1" in receiver._prefetch_futures
|
||||
|
||||
data, size = receiver.consume_prefetched_kv(_req("rid-1"))
|
||||
assert data is not None
|
||||
assert "layer_blocks" in data
|
||||
assert data["metadata"]["seq_len"] == 10
|
||||
assert size > 0
|
||||
# Future is consumed (popped) on retrieval.
|
||||
assert "rid-1" not in receiver._prefetch_futures
|
||||
|
||||
|
||||
def test_consume_prefetched_kv_miss_without_prefetch_returns_none():
|
||||
_, receiver, _ = _make_sender_receiver(async_prefetch=False)
|
||||
assert receiver.consume_prefetched_kv(_req("never-prefetched")) == (None, 0)
|
||||
|
||||
|
||||
def test_start_prefetch_dedups_same_request():
|
||||
sender, receiver, _ = _make_sender_receiver()
|
||||
_seed_payload(sender, "rid-dup")
|
||||
receiver.start_prefetch({"request_id": "rid-dup", "kv_sender_info": _SENDER_INFO})
|
||||
fut1 = receiver._prefetch_futures["rid-dup"]
|
||||
receiver.start_prefetch({"request_id": "rid-dup", "kv_sender_info": _SENDER_INFO})
|
||||
assert receiver._prefetch_futures["rid-dup"] is fut1
|
||||
|
||||
|
||||
def test_start_prefetch_skips_without_sender_info():
|
||||
# No endpoint => no submission; the sync path handles it.
|
||||
sender, receiver, _ = _make_sender_receiver()
|
||||
_seed_payload(sender, "rid-nosi")
|
||||
receiver.start_prefetch({"request_id": "rid-nosi", "kv_sender_info": None})
|
||||
assert receiver._prefetch_futures == {}
|
||||
|
||||
|
||||
def test_consume_prefetched_kv_sweeps_stale_entries():
|
||||
# Consuming a slot drops every other tracked prefetch.
|
||||
sender, receiver, _ = _make_sender_receiver()
|
||||
_seed_payload(sender, "rid-stale")
|
||||
receiver.start_prefetch({"request_id": "rid-stale", "kv_sender_info": _SENDER_INFO})
|
||||
assert "rid-stale" in receiver._prefetch_futures
|
||||
# Miss on the current request still sweeps stale entries.
|
||||
receiver._async_prefetch = False
|
||||
assert receiver.consume_prefetched_kv(_req("rid-current")) == (None, 0)
|
||||
assert receiver._prefetch_futures == {}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# consume_prefetched_kv: miss returns None, stale sweep
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_start_prefetch_sweeps_orphan():
|
||||
# An orphan prefetch (e.g. aborted) is dropped when the next prefetch starts.
|
||||
sender, receiver, _ = _make_sender_receiver()
|
||||
_seed_payload(sender, "rid-orphan")
|
||||
_seed_payload(sender, "rid-next")
|
||||
receiver.start_prefetch({"request_id": "rid-orphan", "kv_sender_info": _SENDER_INFO})
|
||||
receiver.consume_prefetched_kv(_req("rid-orphan")) # complete the future
|
||||
receiver.start_prefetch({"request_id": "rid-orphan", "kv_sender_info": _SENDER_INFO}) # re-add as leftover
|
||||
receiver.start_prefetch({"request_id": "rid-next", "kv_sender_info": _SENDER_INFO})
|
||||
assert "rid-orphan" not in receiver._prefetch_futures
|
||||
assert "rid-next" in receiver._prefetch_futures
|
||||
|
||||
|
||||
def test_shutdown_prefetch_clears_state():
|
||||
sender, receiver, _ = _make_sender_receiver()
|
||||
_seed_payload(sender, "rid-sd")
|
||||
receiver.start_prefetch({"request_id": "rid-sd", "kv_sender_info": _SENDER_INFO})
|
||||
receiver.shutdown_prefetch()
|
||||
assert receiver._prefetch_futures == {}
|
||||
assert receiver._prefetch_executor is None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Correctness: per-call sender_info must not touch instance state
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_receive_with_sender_info_does_not_mutate_instance_state():
|
||||
sender, receiver, _ = _make_sender_receiver()
|
||||
_seed_payload(sender, "rid-si")
|
||||
assert receiver._sender_base_host is None
|
||||
assert receiver._sender_base_zmq_port is None
|
||||
|
||||
receiver.receive_kv_cache_for_request(
|
||||
"rid-si",
|
||||
target_device=None,
|
||||
sender_info={"host": "1.2.3.4", "zmq_port": 50051},
|
||||
)
|
||||
# The per-call endpoint must not leak into instance fields.
|
||||
assert receiver._sender_base_host is None
|
||||
assert receiver._sender_base_zmq_port is None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Receive-role classification (recv_role) and the follower prefetch guard
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
import vllm_omni.diffusion.distributed.parallel_state as ps # noqa: E402
|
||||
|
||||
|
||||
def _patch_topo(monkeypatch, *, world_size, world_rank=0, cfg_size=1, cfg_rank=0, sp_size=1, sp_rank=0):
|
||||
monkeypatch.setattr(ps, "get_world_group", lambda: SimpleNamespace(world_size=world_size, rank_in_group=world_rank))
|
||||
monkeypatch.setattr(ps, "get_classifier_free_guidance_world_size", lambda: cfg_size)
|
||||
monkeypatch.setattr(ps, "get_classifier_free_guidance_rank", lambda: cfg_rank)
|
||||
monkeypatch.setattr(ps, "get_cfg_group", lambda: None)
|
||||
monkeypatch.setattr(ps, "get_sequence_parallel_world_size", lambda: sp_size)
|
||||
monkeypatch.setattr(ps, "get_sequence_parallel_rank", lambda: sp_rank)
|
||||
monkeypatch.setattr(ps, "get_sp_group", lambda: None)
|
||||
|
||||
|
||||
def _set_tp(mgr, src, tgt):
|
||||
mgr._tp_topo = SimpleNamespace(source_tp_size=src, target_tp_size=tgt, local_rank=0)
|
||||
|
||||
|
||||
def _role(mgr):
|
||||
"""Recompute the receive role from the (patched) parallel state, bypassing the cache."""
|
||||
mgr._topo_config = None
|
||||
return mgr.topo_config.role
|
||||
|
||||
|
||||
def test_recv_role_world1_is_local(monkeypatch):
|
||||
mgr = OmniKVTransferManager(OmniKVCacheConfig())
|
||||
_set_tp(mgr, 1, 1)
|
||||
_patch_topo(monkeypatch, world_size=1)
|
||||
assert _role(mgr) is ReceiveRole.LOCAL
|
||||
|
||||
|
||||
def test_recv_role_pure_tp_is_local(monkeypatch):
|
||||
mgr = OmniKVTransferManager(OmniKVCacheConfig())
|
||||
_set_tp(mgr, 2, 2)
|
||||
_patch_topo(monkeypatch, world_size=2)
|
||||
assert _role(mgr) is ReceiveRole.LOCAL
|
||||
|
||||
|
||||
def test_recv_role_cfg_leader_and_follower(monkeypatch):
|
||||
mgr = OmniKVTransferManager(OmniKVCacheConfig())
|
||||
_set_tp(mgr, 2, 2)
|
||||
_patch_topo(monkeypatch, world_size=4, cfg_size=2, cfg_rank=0)
|
||||
assert _role(mgr) is ReceiveRole.LEADER
|
||||
_patch_topo(monkeypatch, world_size=4, cfg_size=2, cfg_rank=1)
|
||||
assert _role(mgr) is ReceiveRole.FOLLOWER
|
||||
|
||||
|
||||
def test_recv_role_sp_leader_and_follower(monkeypatch):
|
||||
mgr = OmniKVTransferManager(OmniKVCacheConfig())
|
||||
_set_tp(mgr, 2, 2)
|
||||
_patch_topo(monkeypatch, world_size=4, sp_size=2, sp_rank=0)
|
||||
assert _role(mgr) is ReceiveRole.LEADER
|
||||
_patch_topo(monkeypatch, world_size=4, sp_size=2, sp_rank=1)
|
||||
assert _role(mgr) is ReceiveRole.FOLLOWER
|
||||
|
||||
|
||||
def test_recv_role_tp_and_sp_leader_per_tp_rank(monkeypatch):
|
||||
# TP=2 + SP=2: sp_rank 0 is leader (pulls), sp_rank 1 is follower (gets broadcast).
|
||||
mgr = OmniKVTransferManager(OmniKVCacheConfig())
|
||||
_set_tp(mgr, 2, 2)
|
||||
_patch_topo(monkeypatch, world_size=4, sp_size=2, sp_rank=0)
|
||||
assert _role(mgr) is ReceiveRole.LEADER
|
||||
_patch_topo(monkeypatch, world_size=4, sp_size=2, sp_rank=1)
|
||||
assert _role(mgr) is ReceiveRole.FOLLOWER
|
||||
|
||||
|
||||
def test_recv_role_hsdp_world_broadcast(monkeypatch):
|
||||
# TP inactive, world > 1 (HSDP): rank-0 leader, others follower.
|
||||
mgr = OmniKVTransferManager(OmniKVCacheConfig())
|
||||
_set_tp(mgr, 1, 1)
|
||||
_patch_topo(monkeypatch, world_size=2, world_rank=0)
|
||||
assert _role(mgr) is ReceiveRole.LEADER
|
||||
_patch_topo(monkeypatch, world_size=2, world_rank=1)
|
||||
assert _role(mgr) is ReceiveRole.FOLLOWER
|
||||
|
||||
|
||||
def test_recv_role_uninitialized_defaults_local(monkeypatch):
|
||||
mgr = OmniKVTransferManager(OmniKVCacheConfig())
|
||||
|
||||
def _boom():
|
||||
raise AssertionError("world group is not initialized")
|
||||
|
||||
monkeypatch.setattr(ps, "get_world_group", _boom)
|
||||
assert _role(mgr) is ReceiveRole.LOCAL
|
||||
|
||||
|
||||
def test_start_prefetch_skips_follower(monkeypatch):
|
||||
# A follower never pulls, so start_prefetch must not submit.
|
||||
sender, receiver, _ = _make_sender_receiver()
|
||||
_seed_payload(sender, "rid-fol")
|
||||
_set_tp(receiver, 2, 2)
|
||||
_patch_topo(monkeypatch, world_size=4, cfg_size=2, cfg_rank=1)
|
||||
receiver.start_prefetch({"request_id": "rid-fol", "kv_sender_info": _SENDER_INFO})
|
||||
assert receiver._prefetch_futures == {}
|
||||
|
||||
|
||||
def test_start_prefetch_submits_for_owner(monkeypatch):
|
||||
sender, receiver, _ = _make_sender_receiver()
|
||||
_seed_payload(sender, "rid-own")
|
||||
_set_tp(receiver, 2, 2)
|
||||
_patch_topo(monkeypatch, world_size=4, cfg_size=2, cfg_rank=0)
|
||||
receiver.start_prefetch({"request_id": "rid-own", "kv_sender_info": _SENDER_INFO})
|
||||
assert "rid-own" in receiver._prefetch_futures
|
||||
|
||||
|
||||
def test_consume_then_apply_attaches_payload():
|
||||
sender, receiver, _ = _make_sender_receiver()
|
||||
_seed_payload(sender, "rid-apply")
|
||||
receiver.start_prefetch({"request_id": "rid-apply", "kv_sender_info": _SENDER_INFO})
|
||||
data, _ = receiver.consume_prefetched_kv(_req("rid-apply"))
|
||||
assert data is not None
|
||||
|
||||
req = OmniDiffusionRequest(prompts=["p"], sampling_params=OmniDiffusionSamplingParams(), request_id="rid-apply")
|
||||
# Mirror consume_and_distribute_kv_cache's LOCAL apply path (CPU: record_stream no-op).
|
||||
receiver._record_stream_for_prefetched(data)
|
||||
receiver.apply_kv_cache_to_request(req, data)
|
||||
assert req.past_key_values is not None
|
||||
@@ -1,7 +1,6 @@
|
||||
import json
|
||||
import struct
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -206,17 +205,6 @@ def test_update_sender_info_uses_configured_source_stage():
|
||||
assert manager.config.connector_config["sender_zmq_port"] == 50152
|
||||
|
||||
|
||||
def test_clone_received_payload_tensors_breaks_buffer_alias():
|
||||
payload, key_tensor = _make_serialized_payload()
|
||||
raw = np.frombuffer(bytearray(payload), dtype=np.uint8)
|
||||
data = KVCacheTransferData.from_bytes(memoryview(raw))
|
||||
|
||||
OmniKVTransferManager._clone_received_payload_tensors(data)
|
||||
raw[:] = 0
|
||||
|
||||
assert torch.equal(data["layer_blocks"]["key_cache"][0], key_tensor)
|
||||
|
||||
|
||||
def test_receive_kv_cache_uses_exponential_backoff(monkeypatch):
|
||||
config = OmniKVCacheConfig(
|
||||
connector_config={"type": "mock"},
|
||||
@@ -526,6 +514,57 @@ def test_integration_flow(common_constants):
|
||||
assert req.kv_metadata["seq_len"] == 10
|
||||
|
||||
|
||||
def test_deferred_release_isolates_data_from_pool_buffer():
|
||||
payload, key_tensor = _make_serialized_payload()
|
||||
|
||||
# ManagedBuffer-like object whose release() zeros storage (simulates reuse).
|
||||
raw_array = bytearray(payload)
|
||||
buf_tensor = torch.frombuffer(raw_array, dtype=torch.uint8)
|
||||
|
||||
class _CorruptingManagedBuffer:
|
||||
def __init__(self, storage, tensor):
|
||||
self._storage = storage
|
||||
self._tensor = tensor
|
||||
|
||||
@property
|
||||
def tensor(self):
|
||||
return self._tensor
|
||||
|
||||
def release(self):
|
||||
for i in range(len(self._storage)):
|
||||
self._storage[i] = 0
|
||||
|
||||
managed_buf = _CorruptingManagedBuffer(raw_array, buf_tensor)
|
||||
|
||||
class _ManagedBufConnector:
|
||||
def __init__(self, buf, from_stage, to_stage, req_id):
|
||||
self._buf = buf
|
||||
self._key = f"{from_stage}->{to_stage}:omni_{from_stage}_to_{to_stage}_kv_cache_{req_id}"
|
||||
|
||||
def get(self, from_stage, to_stage, get_key, metadata=None):
|
||||
if f"{from_stage}->{to_stage}:{get_key}" == self._key:
|
||||
return self._buf, len(self._buf._storage)
|
||||
return None
|
||||
|
||||
config = OmniKVCacheConfig(
|
||||
connector_config={"type": "mock"},
|
||||
from_stage="sender",
|
||||
stage_id="receiver",
|
||||
need_recv_cache=True,
|
||||
recv_timeout=1.0,
|
||||
)
|
||||
manager = OmniKVTransferManager(config)
|
||||
req_id = "req-payload"
|
||||
manager._connector = _ManagedBufConnector(managed_buf, "sender", "receiver", req_id)
|
||||
|
||||
data, size = manager.receive_kv_cache_for_request(req_id, target_device=torch.device("cpu"))
|
||||
|
||||
assert data is not None
|
||||
assert size > 0
|
||||
# Returned data must survive the pool buffer being zeroed by release().
|
||||
assert torch.equal(data["layer_blocks"]["key_cache"][0], key_tensor)
|
||||
|
||||
|
||||
def test_manager_extraction_no_connector(kv_config, common_constants):
|
||||
"""Test extraction when connector is unavailable (should still return IDs)."""
|
||||
block_size = common_constants["block_size"]
|
||||
|
||||
@@ -586,8 +586,17 @@ class TestDistributedReceive:
|
||||
return_value=0,
|
||||
),
|
||||
patch("vllm_omni.diffusion.distributed.parallel_state.get_cfg_group", return_value=cfg_group),
|
||||
patch(
|
||||
"vllm_omni.diffusion.distributed.parallel_state.get_sequence_parallel_world_size",
|
||||
return_value=1,
|
||||
),
|
||||
patch(
|
||||
"vllm_omni.diffusion.distributed.parallel_state.get_sequence_parallel_rank",
|
||||
return_value=0,
|
||||
),
|
||||
patch("vllm_omni.diffusion.distributed.parallel_state.get_sp_group", return_value=None),
|
||||
):
|
||||
assert mgr.receive_multi_kv_cache_distributed(req) is True
|
||||
assert mgr.receive_multi_kv_cache_distributed(req, target_device=torch.device("cpu")) is True
|
||||
|
||||
mgr.receive_multi_kv_cache.assert_called_once()
|
||||
assert mgr.receive_multi_kv_cache.call_args.args[2] == torch.device("cpu")
|
||||
@@ -642,6 +651,15 @@ class TestDistributedReceive:
|
||||
return_value=1,
|
||||
),
|
||||
patch("vllm_omni.diffusion.distributed.parallel_state.get_cfg_group", return_value=cfg_group),
|
||||
patch(
|
||||
"vllm_omni.diffusion.distributed.parallel_state.get_sequence_parallel_world_size",
|
||||
return_value=1,
|
||||
),
|
||||
patch(
|
||||
"vllm_omni.diffusion.distributed.parallel_state.get_sequence_parallel_rank",
|
||||
return_value=0,
|
||||
),
|
||||
patch("vllm_omni.diffusion.distributed.parallel_state.get_sp_group", return_value=None),
|
||||
):
|
||||
assert mgr.receive_multi_kv_cache_distributed(req) is True
|
||||
|
||||
@@ -666,7 +684,27 @@ class TestDistributedReceive:
|
||||
world_group = _MockBroadcastGroup(world_size=2, rank_in_group=1)
|
||||
mgr.receive_multi_kv_cache = MagicMock(return_value=True)
|
||||
|
||||
with patch("vllm_omni.diffusion.distributed.parallel_state.get_world_group", return_value=world_group):
|
||||
with (
|
||||
patch("vllm_omni.diffusion.distributed.parallel_state.get_world_group", return_value=world_group),
|
||||
patch(
|
||||
"vllm_omni.diffusion.distributed.parallel_state.get_classifier_free_guidance_world_size",
|
||||
return_value=1,
|
||||
),
|
||||
patch(
|
||||
"vllm_omni.diffusion.distributed.parallel_state.get_classifier_free_guidance_rank",
|
||||
return_value=0,
|
||||
),
|
||||
patch("vllm_omni.diffusion.distributed.parallel_state.get_cfg_group", return_value=None),
|
||||
patch(
|
||||
"vllm_omni.diffusion.distributed.parallel_state.get_sequence_parallel_world_size",
|
||||
return_value=1,
|
||||
),
|
||||
patch(
|
||||
"vllm_omni.diffusion.distributed.parallel_state.get_sequence_parallel_rank",
|
||||
return_value=0,
|
||||
),
|
||||
patch("vllm_omni.diffusion.distributed.parallel_state.get_sp_group", return_value=None),
|
||||
):
|
||||
assert mgr.receive_multi_kv_cache_distributed(req, target_device=torch.device("cpu")) is True
|
||||
|
||||
mgr.receive_multi_kv_cache.assert_called_once_with(req, None, torch.device("cpu"))
|
||||
@@ -722,7 +760,7 @@ class TestDistributedReceive:
|
||||
),
|
||||
patch("vllm_omni.diffusion.distributed.parallel_state.get_sp_group", return_value=sp_group),
|
||||
):
|
||||
assert mgr.receive_multi_kv_cache_distributed(req) is True
|
||||
assert mgr.receive_multi_kv_cache_distributed(req, target_device=torch.device("cpu")) is True
|
||||
|
||||
# Verify behavior based on role
|
||||
if is_owner:
|
||||
@@ -817,7 +855,7 @@ class TestDistributedReceive:
|
||||
),
|
||||
patch("vllm_omni.diffusion.distributed.parallel_state.get_sp_group", return_value=sp_group),
|
||||
):
|
||||
assert mgr.receive_multi_kv_cache_distributed(req) is True
|
||||
assert mgr.receive_multi_kv_cache_distributed(req, target_device=torch.device("cpu")) is True
|
||||
|
||||
# Verify behavior based on role
|
||||
if role == "owner":
|
||||
|
||||
@@ -305,7 +305,7 @@ def assert_video_similarity_metrics(
|
||||
offline_path: Path,
|
||||
ssim_threshold: float,
|
||||
psnr_threshold: float,
|
||||
) -> None:
|
||||
) -> tuple[float, float]:
|
||||
ssim_output = run_ffmpeg_similarity("ssim", online_path, offline_path)
|
||||
psnr_output = run_ffmpeg_similarity("psnr", online_path, offline_path)
|
||||
ssim_score = parse_ssim_score(ssim_output)
|
||||
@@ -339,6 +339,7 @@ def assert_video_similarity_metrics(
|
||||
f"PSNR below threshold: got {psnr_score:.6f}, expected >= {psnr_threshold:.6f}. "
|
||||
f"online={online_path} offline={offline_path}"
|
||||
)
|
||||
return ssim_score, psnr_score
|
||||
|
||||
|
||||
def assert_similarity(
|
||||
|
||||
@@ -33,7 +33,7 @@ Prerequisites
|
||||
|
||||
* **Seed-TTS** optional extras for WER/SIM/UTMOS::
|
||||
|
||||
pip install 'vllm-omni[seed-tts-eval]'
|
||||
pip install 'vllm-omni[dev]'
|
||||
|
||||
Examples
|
||||
--------
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
"""
|
||||
Online serving smoke test for streaming video output similarity.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.e2e.accuracy.helpers import assert_video_similarity_metrics, probe_video
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.runtime import DiffusionResponse, OmniServer, OpenAIClientHandler
|
||||
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
|
||||
|
||||
MODEL = "BestWishYsh/Helios-Distilled"
|
||||
PROMPT = "A serene lakeside sunrise with mist over the water."
|
||||
WIDTH = 640
|
||||
HEIGHT = 384
|
||||
FPS = 16
|
||||
NUM_FRAMES = 65
|
||||
NUM_INFERENCE_STEPS = 5
|
||||
SEED = 42
|
||||
MIN_PSNR_DB = 28.0
|
||||
MIN_SSIM = 0.95
|
||||
|
||||
SINGLE_CARD_MARKS = hardware_marks(res={"cuda": "H100"})
|
||||
|
||||
HELIOS_DISTILLED_EXTRA_PARAMS = {
|
||||
"is_enable_stage2": True,
|
||||
"pyramid_num_stages": 3,
|
||||
"pyramid_num_inference_steps_list": [1, 1, 1],
|
||||
"is_amplify_first_chunk": True,
|
||||
}
|
||||
|
||||
|
||||
def _server_args(*, streaming_output: bool) -> list[str]:
|
||||
args = [
|
||||
"--stage-init-timeout",
|
||||
"600",
|
||||
"--init-timeout",
|
||||
"900",
|
||||
"--log-stats",
|
||||
]
|
||||
if streaming_output:
|
||||
args.append("--diffusion-streaming-output")
|
||||
return args
|
||||
|
||||
|
||||
def _request_config(model: str, *, serialize_extra_params: bool) -> dict:
|
||||
extra_params = (
|
||||
json.dumps(HELIOS_DISTILLED_EXTRA_PARAMS) if serialize_extra_params else HELIOS_DISTILLED_EXTRA_PARAMS
|
||||
)
|
||||
return {
|
||||
"model": model,
|
||||
"form_data": {
|
||||
"prompt": PROMPT,
|
||||
"width": WIDTH,
|
||||
"height": HEIGHT,
|
||||
"fps": FPS,
|
||||
"num_frames": NUM_FRAMES,
|
||||
"num_inference_steps": NUM_INFERENCE_STEPS,
|
||||
"guidance_scale": 1.0,
|
||||
"seed": SEED,
|
||||
"extra_params": extra_params,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _client_for(server: OmniServer, *, run_level: str) -> OpenAIClientHandler:
|
||||
return OpenAIClientHandler(
|
||||
host=server.host,
|
||||
port=server.port,
|
||||
api_key="EMPTY",
|
||||
run_level=run_level,
|
||||
log_stats=server.log_stats,
|
||||
)
|
||||
|
||||
|
||||
def _write_video_artifact(response: DiffusionResponse, output_path: Path) -> Path:
|
||||
assert response.videos, "No video artifact returned"
|
||||
output_path.write_bytes(response.videos[0])
|
||||
return output_path
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
pytest.param(
|
||||
MODEL,
|
||||
id="helios_distilled",
|
||||
marks=SINGLE_CARD_MARKS,
|
||||
)
|
||||
],
|
||||
)
|
||||
def test_helios_streaming_output_matches_non_streaming(
|
||||
model: str,
|
||||
model_prefix: str,
|
||||
run_level: str,
|
||||
tmp_path: Path,
|
||||
record_property,
|
||||
):
|
||||
model = model_prefix + model
|
||||
|
||||
with OmniServer(model, _server_args(streaming_output=True), use_omni=True) as server:
|
||||
client = _client_for(server, run_level=run_level)
|
||||
streaming_response = client.send_streaming_video_diffusion_request(
|
||||
_request_config(server.model, serialize_extra_params=False)
|
||||
)[0]
|
||||
|
||||
streaming_path = _write_video_artifact(streaming_response, tmp_path / "helios_streaming.mp4")
|
||||
record_property("helios_streaming_video", str(streaming_path))
|
||||
record_property("helios_streaming_latency_s", streaming_response.e2e_latency)
|
||||
|
||||
with OmniServer(model, _server_args(streaming_output=False), use_omni=True) as server:
|
||||
client = _client_for(server, run_level=run_level)
|
||||
non_streaming_response = client.send_video_diffusion_request(
|
||||
_request_config(server.model, serialize_extra_params=True)
|
||||
)[0]
|
||||
|
||||
non_streaming_path = _write_video_artifact(non_streaming_response, tmp_path / "helios_non_streaming.mp4")
|
||||
record_property("helios_non_streaming_video", str(non_streaming_path))
|
||||
record_property("helios_non_streaming_latency_s", non_streaming_response.e2e_latency)
|
||||
|
||||
streaming_metadata = probe_video(streaming_path)
|
||||
non_streaming_metadata = probe_video(non_streaming_path)
|
||||
assert streaming_metadata == non_streaming_metadata, (
|
||||
f"Video metadata mismatch for helios_streaming_output:\n"
|
||||
f"streaming={streaming_metadata}\n"
|
||||
f"non_streaming={non_streaming_metadata}\n"
|
||||
f"streaming_path={streaming_path}\n"
|
||||
f"non_streaming_path={non_streaming_path}"
|
||||
)
|
||||
|
||||
ssim, psnr = assert_video_similarity_metrics(
|
||||
label="helios_streaming_output",
|
||||
online_path=streaming_path,
|
||||
offline_path=non_streaming_path,
|
||||
ssim_threshold=MIN_SSIM,
|
||||
psnr_threshold=MIN_PSNR_DB,
|
||||
)
|
||||
record_property("helios_streaming_vs_non_streaming_psnr_db", psnr)
|
||||
record_property("helios_streaming_vs_non_streaming_ssim_percent", ssim * 100.0)
|
||||
@@ -0,0 +1,96 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""E2E offline inference tests for IndexTTS2 two-stage pipeline.
|
||||
|
||||
Stage 0 (GPT AR) → Stage 1 (S2Mel + BigVGAN). Output is 22050 Hz mono WAV.
|
||||
Covers: basic TTS.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
from vllm_omni import Omni
|
||||
from vllm_omni.model_executor.models.indextts2.prompt_utils import (
|
||||
build_indextts2_prefill_prompt_ids,
|
||||
)
|
||||
|
||||
MODEL_NAME = "IndexTeam/IndexTTS-2"
|
||||
STAGE_CONFIG = get_deploy_config_path("indextts2.yaml")
|
||||
|
||||
_OMNI_RUNNER_PARAM = (
|
||||
MODEL_NAME,
|
||||
STAGE_CONFIG,
|
||||
)
|
||||
|
||||
ASSETS_DIR = Path(__file__).resolve().parents[2] / "assets" / "indextts2"
|
||||
REF_AUDIO = str(ASSETS_DIR / "ref_audio.wav")
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.advanced_model,
|
||||
pytest.mark.tts,
|
||||
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
|
||||
]
|
||||
|
||||
SAMPLE_RATE = 22050
|
||||
|
||||
|
||||
def _audio_from_mm(mm: dict) -> torch.Tensor | None:
|
||||
audio = mm.get("audio")
|
||||
if audio is None:
|
||||
audio = mm.get("model_outputs")
|
||||
if isinstance(audio, list):
|
||||
chunks = [chunk.reshape(-1) for chunk in audio if isinstance(chunk, torch.Tensor) and chunk.numel() > 0]
|
||||
return torch.cat(chunks, dim=0) if chunks else None
|
||||
return audio if isinstance(audio, torch.Tensor) else None
|
||||
|
||||
|
||||
def _sample_rate_from_mm(mm: dict) -> int:
|
||||
sr = mm.get("sr")
|
||||
if isinstance(sr, list):
|
||||
sr = sr[-1] if sr else None
|
||||
if hasattr(sr, "item"):
|
||||
return int(sr.item())
|
||||
return int(sr) if sr is not None else SAMPLE_RATE
|
||||
|
||||
|
||||
def _build_request(text: str, ref_audio: str, **extra) -> dict:
|
||||
info = {
|
||||
"text": [text],
|
||||
"voice": [ref_audio],
|
||||
}
|
||||
info.update(extra)
|
||||
return {
|
||||
"prompt_token_ids": build_indextts2_prefill_prompt_ids(MODEL_NAME, text),
|
||||
"additional_information": info,
|
||||
}
|
||||
|
||||
|
||||
def _collect_audio(omni: Omni, request: dict) -> tuple[torch.Tensor, int]:
|
||||
for omni_out in omni.generate(request):
|
||||
mm = omni_out.multimodal_output
|
||||
assert mm is not None, "Expected multimodal_output"
|
||||
audio = _audio_from_mm(mm)
|
||||
assert audio is not None, "Expected audio output"
|
||||
return audio.cpu(), _sample_rate_from_mm(mm)
|
||||
raise AssertionError("No outputs received")
|
||||
|
||||
|
||||
def _assert_valid_audio(audio: torch.Tensor, sr: int) -> None:
|
||||
assert sr == SAMPLE_RATE
|
||||
assert audio.numel() > 0
|
||||
assert torch.any(audio != 0).item()
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_basic_english(omni_runner: OmniRunner) -> None:
|
||||
"""English TTS: text + ref_audio, no emotion conditioning."""
|
||||
req = _build_request("Hello, this is a voice synthesis test.", REF_AUDIO)
|
||||
audio, sr = _collect_audio(omni_runner.omni, req)
|
||||
_assert_valid_audio(audio, sr)
|
||||
@@ -88,6 +88,7 @@ def _get_test_config() -> str:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/4700"),
|
||||
pytest.mark.full_model,
|
||||
pytest.mark.tts,
|
||||
pytest.mark.parametrize("omni_runner", [(MODEL, _get_test_config())], indirect=True),
|
||||
|
||||
@@ -95,6 +95,7 @@ def _get_test_config() -> str:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/4700"),
|
||||
pytest.mark.full_model,
|
||||
pytest.mark.tts,
|
||||
pytest.mark.parametrize("omni_runner", [(MODEL, _get_test_config())], indirect=True),
|
||||
|
||||
@@ -14,7 +14,7 @@ from tests.helpers.stage_config import get_deploy_config_path, modify_stage_conf
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
|
||||
|
||||
thinker_only_models = ["Qwen/Qwen3-Omni-30B-A3B-Captioner"]
|
||||
|
||||
# Single CI deploy YAML; rocm/xpu deltas are picked automatically via the
|
||||
# platforms: section. Only CUDA needs an extra enforce_eager tweak.
|
||||
@@ -40,6 +40,10 @@ else:
|
||||
|
||||
# Create parameter combinations for model and stage config
|
||||
test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
|
||||
# we can use the same config for a model that only has thinker (i.e., does not
|
||||
# enable audio output) because the resolver should figure out that it doesn't
|
||||
# need the full pipeline based on the HF config.
|
||||
thinker_test_params = [(model, stage_config) for model in thinker_only_models for stage_config in stage_configs]
|
||||
|
||||
|
||||
def get_question(prompt_type="video"):
|
||||
@@ -61,3 +65,15 @@ def test_video_to_audio(omni_runner, omni_runner_handler) -> None:
|
||||
|
||||
# Test single completion
|
||||
omni_runner_handler.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_runner", thinker_test_params, indirect=True)
|
||||
def test_thinker_only_model_request(omni_runner, omni_runner_handler) -> None:
|
||||
"""Test that we can load and run a request through a model that only has the thinker stage."""
|
||||
request_config = {"prompts": "what color is the sky?", "modalities": ["text"]}
|
||||
|
||||
# Test single completion
|
||||
omni_runner_handler.send_omni_request(request_config)
|
||||
@@ -0,0 +1,241 @@
|
||||
"""E2E offline inference tests for SoulX-Singer (single-stage, preprocess inline)."""
|
||||
|
||||
import functools
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.media import get_asset_path
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
PROMPT_AUDIO = get_asset_path("soulxsinger/zh_prompt.mp3")
|
||||
TARGET_AUDIO = get_asset_path("soulxsinger/music.mp3")
|
||||
PROMPT_METADATA = get_asset_path("soulxsinger/zh_prompt.json")
|
||||
TARGET_METADATA = get_asset_path("soulxsinger/music.json")
|
||||
SAMPLE_RATE = 24_000
|
||||
|
||||
# phone_set.json (phoneme vocab) is not shipped on HuggingFace; SVS loads it
|
||||
# from the model dir. Stage it from this pinned upstream commit so the SVS DiT
|
||||
# can run in CI; the SVS tests skip gracefully if it cannot be fetched.
|
||||
_PHONE_SET_URL = (
|
||||
"https://raw.githubusercontent.com/Soul-AILab/SoulX-Singer/"
|
||||
"81aeb3ae772c70093c3de74dc23c92d983801ae4/soulxsinger/utils/phoneme/phone_set.json"
|
||||
)
|
||||
|
||||
if not PROMPT_AUDIO.is_file() or not TARGET_AUDIO.is_file():
|
||||
pytest.skip(
|
||||
f"Missing SoulX-Singer audio assets: {PROMPT_AUDIO.name}, {TARGET_AUDIO.name}",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.advanced_model, pytest.mark.diffusion, pytest.mark.tts]
|
||||
|
||||
_CASES = (
|
||||
pytest.param(
|
||||
"SoulXSingerPipeline",
|
||||
"soulxsinger_svs.yaml",
|
||||
{
|
||||
"language": "Mandarin",
|
||||
"vocal_sep": False,
|
||||
"control": "score",
|
||||
"auto_shift": False,
|
||||
"pitch_shift": 0,
|
||||
},
|
||||
("g2pM", "g2p_en"),
|
||||
id="svs",
|
||||
),
|
||||
pytest.param(
|
||||
"SoulXSingerSVCPipeline",
|
||||
"soulxsinger_svc.yaml",
|
||||
{"vocal_sep": False, "auto_shift": False, "pitch_shift": 0},
|
||||
(),
|
||||
id="svc",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _resolve_weights() -> tuple[Path, Path, Path]:
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
base = Path(snapshot_download("Soul-AILab/SoulX-Singer", allow_patterns=["*"]))
|
||||
|
||||
# phone_set.json is not on HF; best-effort stage it from pinned upstream so
|
||||
# SVS can load. SVS tests skip (not fail) if both this and a manual copy are absent.
|
||||
phone_set = base / "phoneme" / "phone_set.json"
|
||||
if not phone_set.is_file() and not (base / "phone_set.json").is_file():
|
||||
try:
|
||||
from urllib.request import urlretrieve
|
||||
|
||||
phone_set.parent.mkdir(exist_ok=True)
|
||||
urlretrieve(_PHONE_SET_URL, phone_set)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# make a temporary svc directory
|
||||
svc_dir = base.parent / "SoulX-Singer-SVC"
|
||||
svc_dir.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(base / "config.yaml", svc_dir / "config.yaml")
|
||||
shutil.move(base / "model-svc.pt", svc_dir / "model-svc.pt")
|
||||
|
||||
if raw := os.environ.get("SOULX_PREPROCESS_WEIGHTS_DIR"):
|
||||
pre = Path(raw).expanduser().resolve()
|
||||
if (pre / "rmvpe" / "rmvpe.pt").is_file():
|
||||
return base, svc_dir, pre
|
||||
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
pre = Path(snapshot_download("Soul-AILab/SoulX-Singer-Preprocess", allow_patterns=["*"]))
|
||||
return base, svc_dir, pre
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def soulx_weights() -> tuple[Path, Path, Path]:
|
||||
try:
|
||||
return _resolve_weights()
|
||||
except Exception as exc:
|
||||
pytest.skip(f"Set SOULXSINGER_MODEL_DIR / SOULX_PREPROCESS_WEIGHTS_DIR. ({exc})")
|
||||
|
||||
|
||||
def _flatten_audio(audio_val) -> np.ndarray:
|
||||
import torch
|
||||
|
||||
if isinstance(audio_val, list):
|
||||
chunks = [c.detach().cpu().float().numpy().reshape(-1) for c in audio_val if c is not None]
|
||||
return np.concatenate(chunks) if chunks else np.array([], dtype=np.float32)
|
||||
if isinstance(audio_val, torch.Tensor):
|
||||
return audio_val.detach().cpu().float().numpy().reshape(-1)
|
||||
return np.asarray(audio_val, dtype=np.float32).reshape(-1)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.parametrize("architecture,deploy_yaml,extra_args,py_deps", _CASES)
|
||||
def test_soulxsinger_multistage_from_audio(
|
||||
soulx_weights: tuple[Path, Path, Path],
|
||||
architecture: str,
|
||||
deploy_yaml: str,
|
||||
extra_args: dict,
|
||||
py_deps: tuple[str, ...],
|
||||
) -> None:
|
||||
for mod in py_deps:
|
||||
try:
|
||||
importlib.import_module(mod)
|
||||
except ImportError as exc:
|
||||
pytest.fail(f"SoulX SVS requires {mod}: {exc}")
|
||||
|
||||
base_dir, svc_dir, preprocess_dir = soulx_weights
|
||||
|
||||
# SVS mode requires phone_set.json in the model directory
|
||||
if architecture == "SoulXSingerPipeline":
|
||||
json.dump(
|
||||
{"model_type": "soulxsinger", "architectures": ["SoulXSingerPipeline"], "max_num_seqs": 1},
|
||||
open(base_dir / "config.json", "w"),
|
||||
indent=4,
|
||||
)
|
||||
model = str(base_dir)
|
||||
if not (base_dir / "phoneme" / "phone_set.json").is_file() and not (base_dir / "phone_set.json").is_file():
|
||||
pytest.skip(
|
||||
"SoulX-Singer SVS test requires phoneme/phone_set.json. "
|
||||
"Copy it from github.com/Soul-AILab/SoulX-Singer into the model dir. "
|
||||
"See `examples/offline_inference/text_to_speech/README.md` for details."
|
||||
)
|
||||
if architecture == "SoulXSingerSVCPipeline":
|
||||
json.dump(
|
||||
{"model_type": "soulxsinger", "architectures": ["SoulXSingerSVCPipeline"], "max_num_seqs": 1},
|
||||
open(svc_dir / "config.json", "w"),
|
||||
indent=4,
|
||||
)
|
||||
model = str(svc_dir)
|
||||
|
||||
with OmniRunner(
|
||||
model,
|
||||
stage_configs_path=get_deploy_config_path(deploy_yaml),
|
||||
async_chunk=False,
|
||||
) as runner:
|
||||
sampling = OmniDiffusionSamplingParams(
|
||||
num_inference_steps=4,
|
||||
guidance_scale=3.0,
|
||||
seed=42,
|
||||
extra_args={
|
||||
"prompt_audio": str(PROMPT_AUDIO),
|
||||
"target_audio": str(TARGET_AUDIO),
|
||||
"preprocess_weights_dir": str(preprocess_dir),
|
||||
**extra_args,
|
||||
},
|
||||
)
|
||||
prompt = {"prompt_token_ids": [0]}
|
||||
outputs = runner.generate([prompt], sampling)
|
||||
|
||||
assert outputs and outputs[0].error is None, outputs[0].error if outputs else "no output"
|
||||
mm = outputs[0].multimodal_output
|
||||
assert isinstance(mm, dict) and "audio" in mm
|
||||
audio = _flatten_audio(mm["audio"])
|
||||
assert 12_000 <= audio.size
|
||||
assert np.isfinite(audio).all() and float(np.max(np.abs(audio))) > 1e-4
|
||||
duration_s = audio.size / SAMPLE_RATE
|
||||
assert 50.0 <= duration_s <= 52.0, f"duration={duration_s:.1f}s"
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_soulxsinger_svs_precomputed(soulx_weights: tuple[Path, Path, Path]) -> None:
|
||||
"""SVS from precomputed note/lyric metadata.
|
||||
|
||||
Exercises the SVS DiT + vocoder serving path while skipping the heavy
|
||||
upstream preprocess (funasr ASR + ROSVOT note transcription), so it runs in
|
||||
the merge gate without those non-pip dependencies.
|
||||
"""
|
||||
base_dir, _svc_dir, preprocess_dir = soulx_weights
|
||||
|
||||
if not PROMPT_METADATA.is_file() or not TARGET_METADATA.is_file():
|
||||
pytest.skip(f"Missing SoulX-Singer metadata fixtures: {PROMPT_METADATA.name}, {TARGET_METADATA.name}")
|
||||
if not (base_dir / "phoneme" / "phone_set.json").is_file() and not (base_dir / "phone_set.json").is_file():
|
||||
pytest.skip(
|
||||
"SoulX-Singer SVS requires phoneme/phone_set.json (upstream fetch unavailable). "
|
||||
"See `examples/offline_inference/text_to_speech/README.md`."
|
||||
)
|
||||
|
||||
json.dump(
|
||||
{"model_type": "soulxsinger", "architectures": ["SoulXSingerPipeline"], "max_num_seqs": 1},
|
||||
open(base_dir / "config.json", "w"),
|
||||
indent=4,
|
||||
)
|
||||
|
||||
with OmniRunner(
|
||||
str(base_dir),
|
||||
stage_configs_path=get_deploy_config_path("soulxsinger_svs.yaml"),
|
||||
async_chunk=False,
|
||||
) as runner:
|
||||
sampling = OmniDiffusionSamplingParams(
|
||||
num_inference_steps=4,
|
||||
guidance_scale=3.0,
|
||||
seed=42,
|
||||
extra_args={
|
||||
"language": "Mandarin",
|
||||
"control": "score",
|
||||
"prompt_metadata_path": str(PROMPT_METADATA),
|
||||
"target_metadata_path": str(TARGET_METADATA),
|
||||
"audio_path": str(PROMPT_AUDIO),
|
||||
"preprocess_weights_dir": str(preprocess_dir),
|
||||
},
|
||||
)
|
||||
prompt = {"prompt_token_ids": [0]}
|
||||
outputs = runner.generate([prompt], sampling)
|
||||
|
||||
assert outputs and outputs[0].error is None, outputs[0].error if outputs else "no output"
|
||||
mm = outputs[0].multimodal_output
|
||||
assert isinstance(mm, dict) and "audio" in mm
|
||||
audio = _flatten_audio(mm["audio"])
|
||||
assert 12_000 <= audio.size
|
||||
assert np.isfinite(audio).all() and float(np.max(np.abs(audio))) > 1e-4
|
||||
duration_s = audio.size / SAMPLE_RATE
|
||||
assert 50.0 <= duration_s <= 52.0, f"duration={duration_s:.1f}s"
|
||||
Loaded 100 of 360 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user