Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
69fd466fba | ||
|
|
cb08af976b | ||
|
|
617c6ca55f |
No files matched your search
@@ -389,29 +389,6 @@ steps:
|
|||||||
volumes:
|
volumes:
|
||||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||||
|
|
||||||
- label: "TTS · VoxCPM2 Online Test"
|
|
||||||
timeout_in_minutes: 30
|
|
||||||
commands:
|
|
||||||
- |
|
|
||||||
timeout 30m bash -c "
|
|
||||||
export VLLM_LOGGING_LEVEL=DEBUG
|
|
||||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
||||||
pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
|
||||||
"
|
|
||||||
agents:
|
|
||||||
queue: "gpu_1_queue"
|
|
||||||
plugins:
|
|
||||||
- docker#v5.2.0:
|
|
||||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
|
||||||
always-pull: true
|
|
||||||
propagate-environment: true
|
|
||||||
shm-size: "8gb"
|
|
||||||
environment:
|
|
||||||
- "HF_HOME=/fsx/hf_cache"
|
|
||||||
- "HF_TOKEN"
|
|
||||||
volumes:
|
|
||||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
|
||||||
|
|
||||||
- label: "Omni · Higgs-Audio V2 Test"
|
- label: "Omni · Higgs-Audio V2 Test"
|
||||||
timeout_in_minutes: 30
|
timeout_in_minutes: 30
|
||||||
commands:
|
commands:
|
||||||
|
|||||||
+4
-188
@@ -394,11 +394,11 @@ steps:
|
|||||||
depends_on: upload-nightly-pipeline
|
depends_on: upload-nightly-pipeline
|
||||||
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
|
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
|
||||||
steps:
|
steps:
|
||||||
- label: ":full_moon: TTS · Function Test with L4"
|
- label: ":full_moon: TTS · Function Test"
|
||||||
timeout_in_minutes: 90
|
timeout_in_minutes: 90
|
||||||
commands:
|
commands:
|
||||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||||
- pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_voxcpm2_tts_expansion.py
|
- pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy
|
||||||
agents:
|
agents:
|
||||||
queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
|
queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
|
||||||
plugins:
|
plugins:
|
||||||
@@ -413,69 +413,6 @@ steps:
|
|||||||
volumes:
|
volumes:
|
||||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||||
|
|
||||||
# - label: ":full_moon: TTS · Function Test with H100"
|
|
||||||
# timeout_in_minutes: 90
|
|
||||||
# commands:
|
|
||||||
# - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
|
||||||
# - pytest -s -v tests/e2e/ -m "full_model and H100 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy
|
|
||||||
# agents:
|
|
||||||
# queue: "mithril-h100-pool"
|
|
||||||
# plugins:
|
|
||||||
# - kubernetes:
|
|
||||||
# podSpec:
|
|
||||||
# containers:
|
|
||||||
# - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
|
||||||
# resources:
|
|
||||||
# limits:
|
|
||||||
# nvidia.com/gpu: 1
|
|
||||||
# volumeMounts:
|
|
||||||
# - name: devshm
|
|
||||||
# mountPath: /dev/shm
|
|
||||||
# - name: hf-cache
|
|
||||||
# mountPath: /root/.cache/huggingface
|
|
||||||
# env:
|
|
||||||
# - name: HF_HOME
|
|
||||||
# value: /root/.cache/huggingface
|
|
||||||
# - name: HF_TOKEN
|
|
||||||
# valueFrom:
|
|
||||||
# secretKeyRef:
|
|
||||||
# name: hf-token-secret
|
|
||||||
# key: token
|
|
||||||
# nodeSelector:
|
|
||||||
# node.kubernetes.io/instance-type: gpu-h100-sxm
|
|
||||||
# volumes:
|
|
||||||
# - name: devshm
|
|
||||||
# emptyDir:
|
|
||||||
# medium: Memory
|
|
||||||
# - name: hf-cache
|
|
||||||
# hostPath:
|
|
||||||
# path: /mnt/hf-cache
|
|
||||||
# type: DirectoryOrCreate
|
|
||||||
|
|
||||||
- label: ":full_moon: TTS · Accuracy Test · VoxCPM2"
|
|
||||||
timeout_in_minutes: 30
|
|
||||||
commands:
|
|
||||||
- |
|
|
||||||
timeout 30m bash -c "
|
|
||||||
export VLLM_LOGGING_LEVEL=DEBUG
|
|
||||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
||||||
export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
|
||||||
pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts_expansion.py -m 'full_model and cuda and tts' --run-level 'full_model'
|
|
||||||
"
|
|
||||||
agents:
|
|
||||||
queue: "gpu_1_queue"
|
|
||||||
plugins:
|
|
||||||
- docker#v5.2.0:
|
|
||||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
|
||||||
always-pull: true
|
|
||||||
propagate-environment: true
|
|
||||||
shm-size: "8gb"
|
|
||||||
environment:
|
|
||||||
- "HF_HOME=/fsx/hf_cache"
|
|
||||||
- "HF_TOKEN"
|
|
||||||
volumes:
|
|
||||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
|
||||||
|
|
||||||
- label: ":full_moon: TTS · Perf Test"
|
- label: ":full_moon: TTS · Perf Test"
|
||||||
key: nightly-tts-performance
|
key: nightly-tts-performance
|
||||||
timeout_in_minutes: 180
|
timeout_in_minutes: 180
|
||||||
@@ -522,8 +459,8 @@ steps:
|
|||||||
path: /mnt/hf-cache
|
path: /mnt/hf-cache
|
||||||
type: DirectoryOrCreate
|
type: DirectoryOrCreate
|
||||||
|
|
||||||
- label: ":full_moon: TTS · VoxCPM2 · Perf Test"
|
- label: ":full_moon: VoxCPM2 · Perf Test"
|
||||||
key: nightly-tts-performance-voxcpm2
|
key: nightly-voxcpm2-performance
|
||||||
timeout_in_minutes: 180
|
timeout_in_minutes: 180
|
||||||
commands:
|
commands:
|
||||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||||
@@ -758,87 +695,6 @@ steps:
|
|||||||
path: /mnt/hf-cache
|
path: /mnt/hf-cache
|
||||||
type: DirectoryOrCreate
|
type: DirectoryOrCreate
|
||||||
|
|
||||||
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU DreamZero"
|
|
||||||
timeout_in_minutes: 120
|
|
||||||
commands:
|
|
||||||
- >-
|
|
||||||
pytest -sv
|
|
||||||
tests/e2e/online_serving/test_dreamzero_expansion.py
|
|
||||||
-m "full_model and diffusion and H100 and distributed_cuda"
|
|
||||||
--run-level "full_model"
|
|
||||||
agents:
|
|
||||||
queue: "mithril-h100-pool"
|
|
||||||
plugins:
|
|
||||||
- kubernetes:
|
|
||||||
podSpec:
|
|
||||||
containers:
|
|
||||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
|
||||||
resources:
|
|
||||||
limits:
|
|
||||||
nvidia.com/gpu: 2
|
|
||||||
volumeMounts:
|
|
||||||
- name: devshm
|
|
||||||
mountPath: /dev/shm
|
|
||||||
- name: hf-cache
|
|
||||||
mountPath: /root/.cache/huggingface
|
|
||||||
env:
|
|
||||||
- name: HF_HOME
|
|
||||||
value: /root/.cache/huggingface
|
|
||||||
- name: HF_TOKEN
|
|
||||||
valueFrom:
|
|
||||||
secretKeyRef:
|
|
||||||
name: hf-token-secret
|
|
||||||
key: token
|
|
||||||
nodeSelector:
|
|
||||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
|
||||||
volumes:
|
|
||||||
- name: devshm
|
|
||||||
emptyDir:
|
|
||||||
medium: Memory
|
|
||||||
- name: hf-cache
|
|
||||||
hostPath:
|
|
||||||
path: /mnt/hf-cache
|
|
||||||
type: DirectoryOrCreate
|
|
||||||
|
|
||||||
|
|
||||||
- label: ":full_moon: Diffusion X2I(&A&T) · GLM-Image Function Test with H100"
|
|
||||||
timeout_in_minutes: 120
|
|
||||||
commands:
|
|
||||||
- pytest -sv tests/e2e/online_serving/test_glm_image_expansion.py -m "full_model and diffusion and H100" --run-level "full_model"
|
|
||||||
agents:
|
|
||||||
queue: "mithril-h100-pool"
|
|
||||||
plugins:
|
|
||||||
- kubernetes:
|
|
||||||
podSpec:
|
|
||||||
containers:
|
|
||||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
|
||||||
resources:
|
|
||||||
limits:
|
|
||||||
nvidia.com/gpu: 2
|
|
||||||
volumeMounts:
|
|
||||||
- name: devshm
|
|
||||||
mountPath: /dev/shm
|
|
||||||
- name: hf-cache
|
|
||||||
mountPath: /root/.cache/huggingface
|
|
||||||
env:
|
|
||||||
- name: HF_HOME
|
|
||||||
value: /root/.cache/huggingface
|
|
||||||
- name: HF_TOKEN
|
|
||||||
valueFrom:
|
|
||||||
secretKeyRef:
|
|
||||||
name: hf-token-secret
|
|
||||||
key: token
|
|
||||||
nodeSelector:
|
|
||||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
|
||||||
volumes:
|
|
||||||
- name: devshm
|
|
||||||
emptyDir:
|
|
||||||
medium: Memory
|
|
||||||
- name: hf-cache
|
|
||||||
hostPath:
|
|
||||||
path: /mnt/hf-cache
|
|
||||||
type: DirectoryOrCreate
|
|
||||||
|
|
||||||
|
|
||||||
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4"
|
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4"
|
||||||
timeout_in_minutes: 60
|
timeout_in_minutes: 60
|
||||||
@@ -937,46 +793,6 @@ steps:
|
|||||||
path: /mnt/hf-cache
|
path: /mnt/hf-cache
|
||||||
type: DirectoryOrCreate
|
type: DirectoryOrCreate
|
||||||
|
|
||||||
- label: ":full_moon: HunyuanImage3-DIT · Accuracy Test"
|
|
||||||
timeout_in_minutes: 180
|
|
||||||
commands:
|
|
||||||
- export HUNYUAN_IMAGE3_MODEL="tencent/HunyuanImage-3.0-Instruct"
|
|
||||||
- export HUNYUAN_IMAGE3_DEVICES="0,1,2,3"
|
|
||||||
- pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m full_model --run-level full_model
|
|
||||||
agents:
|
|
||||||
queue: "mithril-h100-pool"
|
|
||||||
plugins:
|
|
||||||
- kubernetes:
|
|
||||||
podSpec:
|
|
||||||
containers:
|
|
||||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
|
||||||
resources:
|
|
||||||
limits:
|
|
||||||
nvidia.com/gpu: 4
|
|
||||||
volumeMounts:
|
|
||||||
- name: devshm
|
|
||||||
mountPath: /dev/shm
|
|
||||||
- name: hf-cache
|
|
||||||
mountPath: /root/.cache/huggingface
|
|
||||||
env:
|
|
||||||
- name: HF_HOME
|
|
||||||
value: /root/.cache/huggingface
|
|
||||||
- name: HF_TOKEN
|
|
||||||
valueFrom:
|
|
||||||
secretKeyRef:
|
|
||||||
name: hf-token-secret
|
|
||||||
key: token
|
|
||||||
nodeSelector:
|
|
||||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
|
||||||
volumes:
|
|
||||||
- name: devshm
|
|
||||||
emptyDir:
|
|
||||||
medium: Memory
|
|
||||||
- name: hf-cache
|
|
||||||
hostPath:
|
|
||||||
path: /mnt/hf-cache
|
|
||||||
type: DirectoryOrCreate
|
|
||||||
|
|
||||||
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image"
|
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image"
|
||||||
key: nightly-diffusion-x2iat-performance-qwen-image
|
key: nightly-diffusion-x2iat-performance-qwen-image
|
||||||
timeout_in_minutes: 180
|
timeout_in_minutes: 180
|
||||||
|
|||||||
@@ -273,42 +273,6 @@ steps:
|
|||||||
path: /mnt/hf-cache
|
path: /mnt/hf-cache
|
||||||
type: DirectoryOrCreate
|
type: DirectoryOrCreate
|
||||||
|
|
||||||
- label: "Omni · Qwen3-Omni Realtime WebSocket"
|
|
||||||
commands:
|
|
||||||
- |
|
|
||||||
timeout 30m bash -c "
|
|
||||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
||||||
pytest -s -v tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py::TestQwen3OmniRealtimeWebSocket::test_streaming_audio_input_pcm_output_async_chunk_smoke -m 'core_model' --run-level 'core_model'
|
|
||||||
"
|
|
||||||
agents:
|
|
||||||
queue: "mithril-h100-pool"
|
|
||||||
plugins:
|
|
||||||
- kubernetes:
|
|
||||||
podSpec:
|
|
||||||
containers:
|
|
||||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
|
||||||
resources:
|
|
||||||
limits:
|
|
||||||
nvidia.com/gpu: 2
|
|
||||||
volumeMounts:
|
|
||||||
- name: devshm
|
|
||||||
mountPath: /dev/shm
|
|
||||||
- name: hf-cache
|
|
||||||
mountPath: /root/.cache/huggingface
|
|
||||||
env:
|
|
||||||
- name: HF_HOME
|
|
||||||
value: /root/.cache/huggingface
|
|
||||||
nodeSelector:
|
|
||||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
|
||||||
volumes:
|
|
||||||
- name: devshm
|
|
||||||
emptyDir:
|
|
||||||
medium: Memory
|
|
||||||
- name: hf-cache
|
|
||||||
hostPath:
|
|
||||||
path: /mnt/hf-cache
|
|
||||||
type: DirectoryOrCreate
|
|
||||||
|
|
||||||
- label: "Omni · VoxCPM2 Test"
|
- label: "Omni · VoxCPM2 Test"
|
||||||
timeout_in_minutes: 20
|
timeout_in_minutes: 20
|
||||||
commands:
|
commands:
|
||||||
@@ -332,29 +296,6 @@ steps:
|
|||||||
volumes:
|
volumes:
|
||||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||||
|
|
||||||
- label: "TTS · VoxCPM2 Online Test"
|
|
||||||
timeout_in_minutes: 30
|
|
||||||
commands:
|
|
||||||
- |
|
|
||||||
timeout 30m bash -c "
|
|
||||||
export VLLM_LOGGING_LEVEL=DEBUG
|
|
||||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
||||||
pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts.py -m 'core_model and tts' --run-level 'core_model'
|
|
||||||
"
|
|
||||||
agents:
|
|
||||||
queue: "gpu_1_queue"
|
|
||||||
plugins:
|
|
||||||
- docker#v5.2.0:
|
|
||||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
|
||||||
always-pull: true
|
|
||||||
propagate-environment: true
|
|
||||||
shm-size: "8gb"
|
|
||||||
environment:
|
|
||||||
- "HF_HOME=/fsx/hf_cache"
|
|
||||||
- "HF_TOKEN"
|
|
||||||
volumes:
|
|
||||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
|
||||||
|
|
||||||
- label: "TTS · Qwen3-TTS CustomVoice Test"
|
- label: "TTS · Qwen3-TTS CustomVoice Test"
|
||||||
commands:
|
commands:
|
||||||
- |
|
- |
|
||||||
@@ -399,29 +340,6 @@ steps:
|
|||||||
volumes:
|
volumes:
|
||||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||||
|
|
||||||
- label: "TTS · Higgs-Audio-V3 Test"
|
|
||||||
commands:
|
|
||||||
- |
|
|
||||||
timeout 20m bash -c "
|
|
||||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
||||||
export VLLM_USE_DEEP_GEMM=0
|
|
||||||
export VLLM_MOE_USE_DEEP_GEMM=0
|
|
||||||
pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'core_model' --run-level 'core_model'
|
|
||||||
"
|
|
||||||
agents:
|
|
||||||
queue: "gpu_4_queue"
|
|
||||||
plugins:
|
|
||||||
- docker#v5.2.0:
|
|
||||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
|
||||||
always-pull: true
|
|
||||||
propagate-environment: true
|
|
||||||
shm-size: "8gb"
|
|
||||||
environment:
|
|
||||||
- "HF_HOME=/fsx/hf_cache"
|
|
||||||
- "HF_TOKEN"
|
|
||||||
volumes:
|
|
||||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
|
||||||
|
|
||||||
- label: "Diffusion · Qwen Image Edit Test"
|
- label: "Diffusion · Qwen Image Edit Test"
|
||||||
commands:
|
commands:
|
||||||
- |
|
- |
|
||||||
|
|||||||
@@ -1,97 +0,0 @@
|
|||||||
---
|
|
||||||
name: precheck-pr
|
|
||||||
description: Self-check your branch before creating a PR — catch dead code, verify accuracy/perf claims, validate PR title format, and confirm merge readiness. Use when the user says "precheck", "self review", "pre-submit check", or "check my PR before I open it." Never posts to GitHub.
|
|
||||||
---
|
|
||||||
|
|
||||||
# PR Pre-Check
|
|
||||||
|
|
||||||
Self-review your branch before creating a PR against `vllm-project/vllm-omni`. Two modes: **quick** catches showstoppers, **full** does a thorough maintainer-grade review. Never posts to GitHub; the report is for the contributor's terminal only.
|
|
||||||
|
|
||||||
## Mode Selection
|
|
||||||
|
|
||||||
| Mode | When | Time |
|
|
||||||
|------|------|------|
|
|
||||||
| **Quick** | About to push, final sanity check | ~3 min |
|
|
||||||
| **Full** | Ready for review, want maintainer-level scan | ~10 min |
|
|
||||||
|
|
||||||
Default to quick if unsure. Run full before marking a PR "ready for review."
|
|
||||||
|
|
||||||
## Workflow
|
|
||||||
|
|
||||||
### Step 1: Detect Base Branch
|
|
||||||
|
|
||||||
```bash
|
|
||||||
BASE_SHA=$(git merge-base HEAD origin/main 2>/dev/null \
|
|
||||||
|| git merge-base HEAD main 2>/dev/null \
|
|
||||||
|| echo origin/main)
|
|
||||||
echo "diffing against ${BASE_SHA}"
|
|
||||||
git diff --name-only ${BASE_SHA}...HEAD
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 2: Validate PR Title
|
|
||||||
|
|
||||||
Check the most recent commit message (or branch name if no commit yet) against the project convention. Valid prefixes:
|
|
||||||
|
|
||||||
| Prefix | Applies to |
|
|
||||||
|--------|-----------|
|
|
||||||
| `[Bugfix]` | Bug fixes |
|
|
||||||
| `[CI/Build]` | Build or CI improvements |
|
|
||||||
| `[Doc]` | Documentation changes |
|
|
||||||
| `[Model]` | New/improved models (include model name) |
|
|
||||||
| `[Frontend]` | Frontend changes (API server, OmniLLM class, etc.) |
|
|
||||||
| `[Kernel]` | CUDA/kernel changes |
|
|
||||||
| `[Core]` | Core logic changes (OmniProcessor, OmniARScheduler, etc.) |
|
|
||||||
| `[Hardware][Vendor]` | Hardware-specific (e.g., `[Hardware][Ascend]`) |
|
|
||||||
| `[Misc]` | Other changes (use sparingly) |
|
|
||||||
|
|
||||||
✗ if: missing prefix, wrong case (`[bugfix]`), or WIP/Draft in title.
|
|
||||||
⚠ if: `[Model]` prefix without the model identifier (e.g., `[Model] Add new model` — should be `[Model] Add <ModelName> ...`).
|
|
||||||
|
|
||||||
### Step 3: Categorize the PR
|
|
||||||
|
|
||||||
| Diff contains | PR type |
|
|
||||||
|---------------|---------|
|
|
||||||
| New files under `vllm_omni/model_executor/models/<name>/` | **New Model** |
|
|
||||||
| Changes to `vllm_omni/diffusion/` | **Diffusion Model** |
|
|
||||||
| `[Bugfix]` prefix or single-file fix | **Bug Fix** |
|
|
||||||
| Perf/benchmark/throughput claims in commit msg or diff | **Performance** |
|
|
||||||
| Everything else | **General** |
|
|
||||||
|
|
||||||
If multiple rows apply (e.g., a diffusion model is also a new model), union the checklists.
|
|
||||||
|
|
||||||
### Step 4: Run Checklist
|
|
||||||
|
|
||||||
Ask: "Quick mode or full mode?" Then walk the checklist for the detected PR type from [references/checklists.md](references/checklists.md). Each item produces ✓, ✗, or ⚠.
|
|
||||||
|
|
||||||
### Step 5: Print Report
|
|
||||||
|
|
||||||
```
|
|
||||||
Pre-check report for <branch>
|
|
||||||
|
|
||||||
Mode: quick | full
|
|
||||||
Type: <new-model | diffusion-model | bug-fix | perf | general>
|
|
||||||
|
|
||||||
Dimension Result
|
|
||||||
───────────────── ──────
|
|
||||||
PR title format ✓
|
|
||||||
PR desc integrity ✓
|
|
||||||
Registry/config ✓
|
|
||||||
Dead code ⚠ 2 warnings
|
|
||||||
Accuracy ✓
|
|
||||||
Benchmark ✗ missing software versions
|
|
||||||
|
|
||||||
Verdict: 1 blocking | 2 warnings | recommend fixing ✗ before PR
|
|
||||||
```
|
|
||||||
|
|
||||||
**Severity:**
|
|
||||||
|
|
||||||
| Mark | Meaning |
|
|
||||||
|------|---------|
|
|
||||||
| ✗ | Blocking — fix before opening PR |
|
|
||||||
| ⚠ | Warning — consider fixing |
|
|
||||||
| ✓ | Pass |
|
|
||||||
| — | Skipped (not applicable) |
|
|
||||||
|
|
||||||
### Stop Here
|
|
||||||
|
|
||||||
Do not post comments, open PRs, or modify files. The report is for the contributor's terminal only.
|
|
||||||
@@ -1,135 +0,0 @@
|
|||||||
# Pre-Check Checklists
|
|
||||||
|
|
||||||
Each item is a concrete check to run against the diff. ✗ = fix before PR. ⚠ = consider fixing.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## All PRs: Title Format (check first)
|
|
||||||
|
|
||||||
PR title must follow the project convention documented in `docs/contributing/README.md`:
|
|
||||||
|
|
||||||
| Prefix | Applies to |
|
|
||||||
|--------|-----------|
|
|
||||||
| `[Bugfix]` | Bug fixes |
|
|
||||||
| `[CI/Build]` | Build or CI improvements |
|
|
||||||
| `[Doc]` | Documentation changes |
|
|
||||||
| `[Model]` | New/improved models (include model name) |
|
|
||||||
| `[Frontend]` | Frontend changes (API server, OmniLLM class, etc.) |
|
|
||||||
| `[Kernel]` | CUDA/kernel changes |
|
|
||||||
| `[Core]` | Core logic changes (OmniProcessor, OmniARScheduler, etc.) |
|
|
||||||
| `[Hardware][Vendor]` | Hardware-specific (e.g., `[Hardware][Ascend]`) |
|
|
||||||
| `[Misc]` | Other changes (use sparingly) |
|
|
||||||
|
|
||||||
**Examples:**
|
|
||||||
```
|
|
||||||
✓ [Model] Add MiniCPM-o 4.5 support
|
|
||||||
✓ [Bugfix] Fix Ovis image text encoder dtype
|
|
||||||
✓ [Core] Qwen3-Omni performance optimization
|
|
||||||
✓ [Hardware][Ascend] Fix VoxCPM2 on Ascend
|
|
||||||
✗ Fix Ovis image text encoder dtype ← missing prefix
|
|
||||||
✗ [bugfix] Fix Ovis image text encoder ← wrong case
|
|
||||||
✗ [WIP] Add model ← WIP in title
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Title has valid prefix** — ✗ if missing prefix, wrong case, or WIP/Draft in title. For `[Model]` PRs, the title MUST also include the model identifier (e.g., `[Model] Add <ModelName> ...`). ⚠ if missing the model name.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Bug Fix PRs
|
|
||||||
|
|
||||||
### Quick
|
|
||||||
|
|
||||||
- [ ] **PR body has 4 sections:** what broke (error message), repro steps, root cause, fix description
|
|
||||||
- [ ] **Fix is minimal:** diff touches only files directly related to the bug. No unrelated refactors or style changes.
|
|
||||||
- [ ] **Repro is runnable:** the PR body includes a copy-paste command that reproduces the bug
|
|
||||||
- [ ] **Branch is rebased:** `git merge-base HEAD origin/main` is recent, no merge conflicts
|
|
||||||
|
|
||||||
### Full (adds)
|
|
||||||
|
|
||||||
- [ ] **Regression test exists:** `git diff --name-only` includes at least one `tests/` file
|
|
||||||
- [ ] **Regression test reproduces the original error:** the test would fail on main, pass on this branch
|
|
||||||
- [ ] **Fix matches root cause exactly:** no "fix the symptom + something else" — if the root cause is dtype, the fix is dtype, not dtype + reformatting
|
|
||||||
- [ ] **No silent failure risk:** no bare `except: pass`, no `try: ... except: return None`, no empty fallback added
|
|
||||||
- [ ] **Upstream pattern match:** the fix follows the same pattern as existing code for similar cases (grep for analogous `from_pretrained` calls, etc.)
|
|
||||||
- [ ] **Environment documented:** torch/transformers/vllm versions listed if the bug is version-dependent
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Performance PRs
|
|
||||||
|
|
||||||
### Quick
|
|
||||||
|
|
||||||
- [ ] **Before/after numbers in PR body:** at minimum, one metric (RTF, RPS, latency, VRAM) with main vs PR values
|
|
||||||
- [ ] **Hardware specified:** GPU model, count, VRAM. "L4" or "H20" is sufficient. "tested on GPU" is not.
|
|
||||||
- [ ] **Concurrency scaling:** if the optimization claims better throughput, numbers at 2+ concurrency levels
|
|
||||||
- [ ] **No unexplained regressions:** if any metric goes down at any concurrency level, explain why
|
|
||||||
- [ ] **Branch is rebased:** `git merge-base HEAD origin/main` is recent, no merge conflicts
|
|
||||||
|
|
||||||
### Full (adds)
|
|
||||||
|
|
||||||
- [ ] **Software versions:** torch, CUDA, vllm, vllm-omni versions in PR body
|
|
||||||
- [ ] **Warmup stated:** "1 warmup + 3 measured" or equivalent
|
|
||||||
- [ ] **Stddev or range:** not just a single best run. "average of 2 rounds" is minimum.
|
|
||||||
- [ ] **VRAM measured:** peak GPU memory before and after
|
|
||||||
- [ ] **Benchmark script checked in:** under `tests/` or `examples/`, runnable by reviewer
|
|
||||||
- [ ] **Exact command line in PR:** not pseudocode, not "run the benchmark"
|
|
||||||
- [ ] **Pytest summary included:** `36 passed, 3 skipped, 17 warnings in 2897.66s` — proves the whole suite passes
|
|
||||||
- [ ] **Config changes documented:** if `max_num_seqs`, `gpu_memory_utilization`, or deploy YAML changed, old and new values stated with reasoning
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## New Model PRs
|
|
||||||
|
|
||||||
### Quick
|
|
||||||
|
|
||||||
- [ ] **PR body file list matches `git diff --name-only`:** every file path in the description exists in the diff
|
|
||||||
- [ ] **Registry entries match claimed count:** count `_OMNI_MODELS` or `_DIFFUSION_MODELS` entries added vs claimed in body
|
|
||||||
- [ ] **Pipeline config `custom_process_input_func` paths resolve:** each referenced module.function exists
|
|
||||||
- [ ] **`__init__.py` exports match registry:** each registered class appears in `__init__.py` and `__all__`
|
|
||||||
- [ ] **Shell script modes match README:** `run_curl.sh` supports every mode the README documents
|
|
||||||
- [ ] **Branch is rebased:** `git merge-base HEAD origin/main` is recent, no merge conflicts
|
|
||||||
|
|
||||||
### Full (adds)
|
|
||||||
|
|
||||||
- [ ] **Dead code scan (5 patterns):**
|
|
||||||
1. Dead `forward()` — defined but never called through the pipeline path
|
|
||||||
2. Dead factory/builder functions — instantiated nowhere
|
|
||||||
3. Dead wrappers — thin wrappers with no callers
|
|
||||||
4. Dead branch guards — `if version > X` where X is always satisfied
|
|
||||||
5. Unused parameters — `__init__` args never accessed
|
|
||||||
- [ ] **Copy-paste detection:** no string constant defined in 3+ files, no cross-module validation duplication, no near-identical shape coercion fns
|
|
||||||
- [ ] **Import hygiene:** no re-export-only imports, no module-level side effects without docs, no `import os` in function body when already at top. Function-body imports are acceptable when the import is only reachable from a closure or factory return value (i.e., `import` would otherwise execute at module load even though the user may never trigger that path). Either lift to top-of-file or annotate with `# noqa: PLC0415`.
|
|
||||||
- [ ] **Accuracy:** at least one test compares output against reference; audio = valid WAV at correct sample rate; determinism verified if seed claimed. ⚠ if missing and not explicitly deferred to a follow-up PR or RFC section; — if explicitly deferred (the deferral target SHOULD be linked in the PR body).
|
|
||||||
- [ ] **Performance:** hardware, software versions, warmup, and metrics (RTF, VRAM, RPS, latency) stated. ⚠ if missing and not explicitly deferred; — if explicitly deferred.
|
|
||||||
- [ ] **Benchmark settings:** model config (TP, PP, max_model_len, enforce_eager, quant), runtime config (batch, input/output spec), environment versions
|
|
||||||
- [ ] **Benchmark script checked in + exact command line + pytest summary**
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Diffusion Model PRs
|
|
||||||
|
|
||||||
Diffusion models live under `vllm_omni/diffusion/`. In addition to the New Model checklist above, verify:
|
|
||||||
|
|
||||||
- [ ] **Transformer adapter follows contract:** inherits from `nn.Module`, implements `load_weights()`, uses `vllm_omni.diffusion.attention.layer.Attention`
|
|
||||||
- [ ] **Pipeline has both offline and online paths:** `forward()` for batch generation, streaming path for online serving
|
|
||||||
- [ ] **Cache-DiT integration:** at least one acceleration (TeaCache, FBCache, etc.) is wired up. ✗ for new **top-level** pipelines. For **subclass** pipelines that inherit `__call__` from a parent where Cache-DiT is already absent, downgrade to ⚠ and recommend filing a follow-up issue against the parent pipeline family.
|
|
||||||
- [ ] **Parallelism config:** TP/SP/USP/CFG-Parallel options exposed in pipeline config
|
|
||||||
- [ ] **Docs table updated:** model added to `docs/models/supported_models.md` with correct metadata
|
|
||||||
- [ ] **E2E test exists:** at least one test exercises the full generate path
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## General PRs
|
|
||||||
|
|
||||||
### Quick
|
|
||||||
|
|
||||||
- [ ] **PR body matches diff:** description doesn't claim changes not in the diff
|
|
||||||
- [ ] **Branch is rebased:** `git merge-base HEAD origin/main` is recent, no merge conflicts
|
|
||||||
- [ ] **CI gates passing:** DCO, pre-commit, build — check `gh pr view --json statusCheckRollup` if a PR already exists
|
|
||||||
|
|
||||||
### Full (adds)
|
|
||||||
|
|
||||||
- [ ] **No new dead code in changed files:** scan for unreachable paths, unused imports
|
|
||||||
- [ ] **Test coverage:** if core code changed (`engine/`, `stages/`, `connectors/`), tests are added or existing tests cover the path
|
|
||||||
- [ ] **Import hygiene:** no redundant imports added
|
|
||||||
- [ ] **No unrelated changes:** diff scope matches PR description
|
|
||||||
@@ -25,12 +25,7 @@ include:
|
|||||||
- `add-tts-model`: covers integration of new TTS models and related serving
|
- `add-tts-model`: covers integration of new TTS models and related serving
|
||||||
workflows
|
workflows
|
||||||
- `generate-release-note`: helps prepare release notes for repository changes
|
- `generate-release-note`: helps prepare release notes for repository changes
|
||||||
- `precheck-pr`: self-check a branch before creating a PR — validates PR title
|
|
||||||
format, catches dead code, verifies accuracy/perf claims, and confirms merge
|
|
||||||
readiness
|
|
||||||
- `review-pr`: provides a structured workflow for reviewing pull requests
|
- `review-pr`: provides a structured workflow for reviewing pull requests
|
||||||
- `vllm-omni-npu-model-runner-upgrade`: upgrades NPU model runners to align with the
|
|
||||||
latest vllm-ascend NPUModelRunner
|
|
||||||
|
|
||||||
## Maintenance Guidelines
|
## Maintenance Guidelines
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,4 @@
|
|||||||
<!-- markdownlint-disable -->
|
<!-- markdownlint-disable -->
|
||||||
**Before submitting:** run the precheck-pr skill with code agent for a self-check against project conventions — catches dead code, missing benchmarks, and title format issues.
|
|
||||||
|
|
||||||
PLEASE FILL IN THE PR DESCRIPTION HERE ENSURING ALL CHECKLIST ITEMS (AT THE BOTTOM) HAVE BEEN CONSIDERED.
|
PLEASE FILL IN THE PR DESCRIPTION HERE ENSURING ALL CHECKLIST ITEMS (AT THE BOTTOM) HAVE BEEN CONSIDERED.
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
@@ -20,5 +18,4 @@ PLEASE FILL IN THE PR DESCRIPTION HERE ENSURING ALL CHECKLIST ITEMS (AT THE BOTT
|
|||||||
- [ ] (Optional) Release notes update. If your change is user-facing, please update the release notes draft.
|
- [ ] (Optional) Release notes update. If your change is user-facing, please update the release notes draft.
|
||||||
</details>
|
</details>
|
||||||
|
|
||||||
**BEFORE SUBMITTING,** please read <https://github.com/vllm-project/vllm-omni/blob/main/CONTRIBUTING.md>
|
**BEFORE SUBMITTING, PLEASE READ <https://github.com/vllm-project/vllm-omni/blob/main/CONTRIBUTING.md>** (anything written below this line will be removed by GitHub Actions)
|
||||||
(anything written below this line will be removed by GitHub Actions)
|
|
||||||
@@ -16,7 +16,6 @@ Easy, fast, and cheap omni-modality model serving for everyone
|
|||||||
---
|
---
|
||||||
|
|
||||||
*Latest News* 🔥
|
*Latest News* 🔥
|
||||||
- [2026/06] We released [0.22.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.22.0) - an **omnimodal world-model** release aligned with vLLM 0.22, featuring [Nvidia Cosmos3](recipes/cosmos3/Cosmos3-Nano.md)/DreamZero world model support, expanded quantization coverage across Blackwell/NPU/XPU, TTS production improvements, new models including MiniCPM-o 4.5, MOSS-TTS, and Lance, plus RL integration with [VeRL-Omni](https://github.com/verl-project/verl-omni).
|
|
||||||
- [2026/05] We released [0.20.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.20.0) - refreshes the serving/runtime stack for large-scale omni workloads, and improves diffusion model performance, quantization, and hardware readiness across CUDA, ROCm, MUSA, NPU, and XPU backends.
|
- [2026/05] We released [0.20.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.20.0) - refreshes the serving/runtime stack for large-scale omni workloads, and improves diffusion model performance, quantization, and hardware readiness across CUDA, ROCm, MUSA, NPU, and XPU backends.
|
||||||
- [2026/03] We released [0.18.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.18.0) - strengthens the core runtime through a large entrypoint refactor and scheduler/runtime cleanups, expands unified quantization and diffusion execution, broadens multimodal model coverage, and improves production readiness across audio, omni, image, video, RL, and multi-platform deployments.
|
- [2026/03] We released [0.18.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.18.0) - strengthens the core runtime through a large entrypoint refactor and scheduler/runtime cleanups, expands unified quantization and diffusion execution, broadens multimodal model coverage, and improves production readiness across audio, omni, image, video, RL, and multi-platform deployments.
|
||||||
- [2026/03] Check out our first public [project deepdive](https://youtu.be/sgwNfsNnR9I) at the vLLM Hong Kong Meetup!
|
- [2026/03] Check out our first public [project deepdive](https://youtu.be/sgwNfsNnR9I) at the vLLM Hong Kong Meetup!
|
||||||
|
|||||||
@@ -151,9 +151,6 @@ class VllmOmniImageClient:
|
|||||||
seed: int | None = None,
|
seed: int | None = None,
|
||||||
negative_prompt: str | None = None,
|
negative_prompt: str | None = None,
|
||||||
output_compression: int | None = None,
|
output_compression: int | None = None,
|
||||||
bot_task: str | None = None,
|
|
||||||
sys_type: str | None = None,
|
|
||||||
system_prompt: str | None = None,
|
|
||||||
) -> Image.Image:
|
) -> Image.Image:
|
||||||
if not isinstance(images, list):
|
if not isinstance(images, list):
|
||||||
images = [images]
|
images = [images]
|
||||||
@@ -173,12 +170,6 @@ class VllmOmniImageClient:
|
|||||||
data["negative_prompt"] = negative_prompt
|
data["negative_prompt"] = negative_prompt
|
||||||
if output_compression is not None:
|
if output_compression is not None:
|
||||||
data["output_compression"] = str(output_compression)
|
data["output_compression"] = str(output_compression)
|
||||||
if bot_task is not None:
|
|
||||||
data["bot_task"] = bot_task
|
|
||||||
if sys_type is not None:
|
|
||||||
data["sys_type"] = sys_type
|
|
||||||
if system_prompt is not None:
|
|
||||||
data["system_prompt"] = system_prompt
|
|
||||||
|
|
||||||
files = [
|
files = [
|
||||||
(
|
(
|
||||||
|
|||||||
@@ -64,47 +64,6 @@ def resolve_model_name(*, model_name: str | None, model: str | None = None, outp
|
|||||||
raise ValueError("model-name is required when it cannot be inferred from model or output-root")
|
raise ValueError("model-name is required when it cannot be inferred from model or output-root")
|
||||||
|
|
||||||
|
|
||||||
def _coerce_score_int(value: Any) -> int:
|
|
||||||
if isinstance(value, bool):
|
|
||||||
raise TypeError(f"Expected numeric score, got bool: {value!r}")
|
|
||||||
if isinstance(value, (int, float)):
|
|
||||||
return int(value)
|
|
||||||
if isinstance(value, str):
|
|
||||||
stripped = value.strip()
|
|
||||||
if stripped.lstrip("-").isdigit():
|
|
||||||
return int(stripped)
|
|
||||||
raise TypeError(f"Expected numeric score, got {type(value).__name__}: {value!r}")
|
|
||||||
|
|
||||||
|
|
||||||
def _expand_score_entry(entry: Any) -> list[int]:
|
|
||||||
if isinstance(entry, dict):
|
|
||||||
if "score" in entry:
|
|
||||||
return [_coerce_score_int(entry["score"])]
|
|
||||||
if "value" in entry:
|
|
||||||
return [_coerce_score_int(entry["value"])]
|
|
||||||
numeric_values = [_coerce_score_int(value) for value in entry.values()]
|
|
||||||
if numeric_values:
|
|
||||||
return numeric_values
|
|
||||||
raise TypeError(f"Score dict has no numeric values: {entry!r}")
|
|
||||||
return [_coerce_score_int(entry)]
|
|
||||||
|
|
||||||
|
|
||||||
def _normalize_score_list(score: Any) -> list[int]:
|
|
||||||
if isinstance(score, dict):
|
|
||||||
entries: list[Any] = [score]
|
|
||||||
elif isinstance(score, list):
|
|
||||||
entries = score
|
|
||||||
else:
|
|
||||||
entries = [score]
|
|
||||||
|
|
||||||
normalized: list[int] = []
|
|
||||||
for entry in entries:
|
|
||||||
normalized.extend(_expand_score_entry(entry))
|
|
||||||
if not normalized:
|
|
||||||
raise ValueError(f"Empty score list after normalization: {score!r}")
|
|
||||||
return normalized
|
|
||||||
|
|
||||||
|
|
||||||
def parse_score_payload(raw_text: str) -> dict[str, Any]:
|
def parse_score_payload(raw_text: str) -> dict[str, Any]:
|
||||||
try:
|
try:
|
||||||
parsed = extract_json_object(raw_text)
|
parsed = extract_json_object(raw_text)
|
||||||
@@ -114,7 +73,10 @@ def parse_score_payload(raw_text: str) -> dict[str, Any]:
|
|||||||
parsed = {"score": json.loads(stripped), "reasoning": ""}
|
parsed = {"score": json.loads(stripped), "reasoning": ""}
|
||||||
else:
|
else:
|
||||||
raise
|
raise
|
||||||
parsed["score"] = _normalize_score_list(parsed.get("score", []))
|
score = parsed.get("score", [])
|
||||||
|
if not isinstance(score, list):
|
||||||
|
score = [score]
|
||||||
|
parsed["score"] = [int(value) for value in score]
|
||||||
return parsed
|
return parsed
|
||||||
|
|
||||||
|
|
||||||
@@ -429,9 +391,6 @@ class GEditBenchRunner:
|
|||||||
num_inference_steps: int = 20,
|
num_inference_steps: int = 20,
|
||||||
guidance_scale: float | None = None,
|
guidance_scale: float | None = None,
|
||||||
seed: int | None = 42,
|
seed: int | None = 42,
|
||||||
bot_task: str | None = None,
|
|
||||||
sys_type: str | None = None,
|
|
||||||
system_prompt: str | None = None,
|
|
||||||
):
|
):
|
||||||
self.dataset_ref = dataset_ref
|
self.dataset_ref = dataset_ref
|
||||||
self.output_root = output_root
|
self.output_root = output_root
|
||||||
@@ -441,9 +400,6 @@ class GEditBenchRunner:
|
|||||||
self.num_inference_steps = num_inference_steps
|
self.num_inference_steps = num_inference_steps
|
||||||
self.guidance_scale = guidance_scale
|
self.guidance_scale = guidance_scale
|
||||||
self.seed = seed
|
self.seed = seed
|
||||||
self.bot_task = bot_task
|
|
||||||
self.sys_type = sys_type
|
|
||||||
self.system_prompt = system_prompt
|
|
||||||
self.client = VllmOmniImageClient(base_url=base_url, api_key=api_key)
|
self.client = VllmOmniImageClient(base_url=base_url, api_key=api_key)
|
||||||
|
|
||||||
def generate(
|
def generate(
|
||||||
@@ -522,9 +478,6 @@ class GEditBenchRunner:
|
|||||||
num_inference_steps=self.num_inference_steps,
|
num_inference_steps=self.num_inference_steps,
|
||||||
guidance_scale=self.guidance_scale,
|
guidance_scale=self.guidance_scale,
|
||||||
seed=self.seed,
|
seed=self.seed,
|
||||||
bot_task=self.bot_task,
|
|
||||||
sys_type=self.sys_type,
|
|
||||||
system_prompt=self.system_prompt,
|
|
||||||
)
|
)
|
||||||
edited_image.save(output_path)
|
edited_image.save(output_path)
|
||||||
return {
|
return {
|
||||||
@@ -680,25 +633,6 @@ def build_parser() -> argparse.ArgumentParser:
|
|||||||
generate.add_argument("--num-inference-steps", type=int, default=20)
|
generate.add_argument("--num-inference-steps", type=int, default=20)
|
||||||
generate.add_argument("--guidance-scale", type=float, default=None)
|
generate.add_argument("--guidance-scale", type=float, default=None)
|
||||||
generate.add_argument("--seed", type=int, default=42)
|
generate.add_argument("--seed", type=int, default=42)
|
||||||
generate.add_argument(
|
|
||||||
"--bot-task",
|
|
||||||
type=str,
|
|
||||||
default=None,
|
|
||||||
choices=["think", "recaption", "think_recaption", "vanilla"],
|
|
||||||
help="HunyuanImage-3 prompting mode for /v1/images/edits (optional).",
|
|
||||||
)
|
|
||||||
generate.add_argument(
|
|
||||||
"--sys-type",
|
|
||||||
type=str,
|
|
||||||
default=None,
|
|
||||||
help="Hunyuan system-prompt type (e.g. en_unified). Defaults from --bot-task when omitted.",
|
|
||||||
)
|
|
||||||
generate.add_argument(
|
|
||||||
"--system-prompt",
|
|
||||||
type=str,
|
|
||||||
default=None,
|
|
||||||
help="Custom Hunyuan system prompt body (sys_type=custom).",
|
|
||||||
)
|
|
||||||
generate.add_argument("--workers", type=int, default=1)
|
generate.add_argument("--workers", type=int, default=1)
|
||||||
generate.add_argument("--max-samples", type=int, default=None)
|
generate.add_argument("--max-samples", type=int, default=None)
|
||||||
generate.add_argument("--samples-per-group", type=int, default=None)
|
generate.add_argument("--samples-per-group", type=int, default=None)
|
||||||
@@ -741,9 +675,6 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
num_inference_steps=args.num_inference_steps,
|
num_inference_steps=args.num_inference_steps,
|
||||||
guidance_scale=args.guidance_scale,
|
guidance_scale=args.guidance_scale,
|
||||||
seed=args.seed,
|
seed=args.seed,
|
||||||
bot_task=args.bot_task,
|
|
||||||
sys_type=args.sys_type,
|
|
||||||
system_prompt=args.system_prompt,
|
|
||||||
)
|
)
|
||||||
records = runner.generate(
|
records = runner.generate(
|
||||||
model_name=model_name,
|
model_name=model_name,
|
||||||
|
|||||||
@@ -92,6 +92,7 @@ Core scheduling and caching components.
|
|||||||
Configuration classes.
|
Configuration classes.
|
||||||
|
|
||||||
- [vllm_omni.config.model.OmniModelConfig][]
|
- [vllm_omni.config.model.OmniModelConfig][]
|
||||||
|
- [vllm_omni.config.stage_config.ModelPipeline][]
|
||||||
- [vllm_omni.config.stage_config.StageConfig][]
|
- [vllm_omni.config.stage_config.StageConfig][]
|
||||||
- [vllm_omni.config.stage_config.StageConfigFactory][]
|
- [vllm_omni.config.stage_config.StageConfigFactory][]
|
||||||
- [vllm_omni.config.stage_config.StageType][]
|
- [vllm_omni.config.stage_config.StageType][]
|
||||||
|
|||||||
Binary file not shown.
|
Before Width: | Height: | Size: 403 KiB After Width: | Height: | Size: 408 KiB |
+3
-111
@@ -2,7 +2,7 @@
|
|||||||
The vllm bench command launches the vLLM-Omni benchmark to evaluate the performance of multimodal models.
|
The vllm bench command launches the vLLM-Omni benchmark to evaluate the performance of multimodal models.
|
||||||
|
|
||||||
## Notes
|
## Notes
|
||||||
We currently only support using the "openai-chat-omni" and "openai-image-edits-omni" backend.
|
We currently only support using the "openai-chat-omni" backend.
|
||||||
|
|
||||||
## Basic Parameter Description
|
## Basic Parameter Description
|
||||||
You can use `vllm bench serve --omni --help=all` to get descriptions of all parameters. The commonly used parameters are described below:
|
You can use `vllm bench serve --omni --help=all` to get descriptions of all parameters. The commonly used parameters are described below:
|
||||||
@@ -52,11 +52,7 @@ You can use `vllm bench serve --omni --help=all` to get descriptions of all para
|
|||||||
- `--percentile-metrics`
|
- `--percentile-metrics`
|
||||||
"Comma-separated list of selected metrics to report percentiles."
|
"Comma-separated list of selected metrics to report percentiles."
|
||||||
"This argument specifies the metrics to report percentiles."
|
"This argument specifies the metrics to report percentiles."
|
||||||
'Allowed metric names are "ttft", "tpot", "itl", "ttfc", "tpoc", "icl", '
|
'Allowed metric names are "ttft", "tpot", "itl", "e2el", "audio_ttfp", "audio_rtf", "audio_duration". '
|
||||||
'"tpop", "e2el", "audio_ttfp", "audio_rtf", "audio_duration". '
|
|
||||||
|
|
||||||
- `-print-stage`
|
|
||||||
Print per-stage benchmark metrics for --omni serving when stage metrics are returned by the server. Disabled by default.
|
|
||||||
|
|
||||||
- `--save-result`
|
- `--save-result`
|
||||||
Specify to save benchmark results to a json file
|
Specify to save benchmark results to a json file
|
||||||
@@ -280,9 +276,7 @@ Generate synthetic image、video、audio inputs alongside random text prompts to
|
|||||||
|
|
||||||
Notes:
|
Notes:
|
||||||
|
|
||||||
- Works only with online benchmark via:
|
- Works only with online benchmark via the OpenAI backend (`--backend openai-chat-omni`) and endpoint `/v1/chat/completions`.
|
||||||
- the OpenAI chat backend (`--backend openai-chat-omni`) and endpoint `/v1/chat/completions`.
|
|
||||||
- the OpenAI edit image backend (`--backend openai-image-edits-omni`) and endpoint `/v1/images/edits`.
|
|
||||||
|
|
||||||
Start the server (example):
|
Start the server (example):
|
||||||
|
|
||||||
@@ -363,105 +357,3 @@ How sampling works:
|
|||||||
This should be seen as an edge case, and if this behavior can be avoided by setting `--random-mm-limit-mm-per-prompt` to a large number. Note that this might result in errors due to engine config `--limit-mm-per-prompt`.
|
This should be seen as an edge case, and if this behavior can be avoided by setting `--random-mm-limit-mm-per-prompt` to a large number. Note that this might result in errors due to engine config `--limit-mm-per-prompt`.
|
||||||
- The resulting request contains synthetic image data in `multi_modal_data` (OpenAI Chat format). When `random-mm` is used with the OpenAI Chat backend, prompts remain text and MM content is attached via `multi_modal_data`.
|
- The resulting request contains synthetic image data in `multi_modal_data` (OpenAI Chat format). When `random-mm` is used with the OpenAI Chat backend, prompts remain text and MM content is attached via `multi_modal_data`.
|
||||||
</details>
|
</details>
|
||||||
|
|
||||||
### Multi-Stage Benchmark
|
|
||||||
|
|
||||||
<details class="admonition abstract" markdown="1">
|
|
||||||
<summary>Show more</summary>
|
|
||||||
|
|
||||||
Start the server (change the port and model):
|
|
||||||
|
|
||||||
```bash
|
|
||||||
vllm serve --omni --port 29999 --model ~/Qwen3-Omni-30B-A3B-Instruct
|
|
||||||
```
|
|
||||||
|
|
||||||
Then run the benchmarking script (remember to add the flag `--print-stage`):
|
|
||||||
```bash
|
|
||||||
vllm bench serve --omni \
|
|
||||||
--dataset-name random \
|
|
||||||
--port 29999 \
|
|
||||||
--max-concurrency 1 \
|
|
||||||
--num-warmups 2 \
|
|
||||||
--model ~/Qwen3-Omni-30B-A3B-Instruct \
|
|
||||||
--endpoint /v1/chat/completions \
|
|
||||||
--backend openai-chat-omni \
|
|
||||||
--num-prompts 3 \
|
|
||||||
--random-input-len 2500 \
|
|
||||||
--ignore-eos \
|
|
||||||
--percentile-metrics ttft,tpot,itl,e2el,audio_ttfp,audio_rtf,ttfc,tpoc,icl \
|
|
||||||
--random-output-len 900 \
|
|
||||||
--extra_body '{"modalities": ["text","audio"]}' \
|
|
||||||
--print-stage
|
|
||||||
```
|
|
||||||
|
|
||||||
Besides the "Serving Benchmark Result", there will be "Stage Benchmark Result" below:
|
|
||||||
```text
|
|
||||||
============= Stage Benchmark Result =============
|
|
||||||
=============== Stage 0 (thinker) ================
|
|
||||||
-------------------Stage Timing-------------------
|
|
||||||
Mean stage_gen_time (ms): 18243.16
|
|
||||||
Median stage_gen_time (ms): 18220.83
|
|
||||||
P99 stage_gen_time (ms): 18296.75
|
|
||||||
================== Text Result ===================
|
|
||||||
Stage generated tokens: 2700
|
|
||||||
-----------Serving Time to First Token------------
|
|
||||||
Mean Serving TTFT (ms): 816.31
|
|
||||||
Median Serving TTFT (ms): 818.94
|
|
||||||
P99 Serving TTFT (ms): 829.76
|
|
||||||
-----Time per Output Token (excl. 1st token)------
|
|
||||||
Mean TPOP (ms): 19.41
|
|
||||||
Median TPOP (ms): 19.40
|
|
||||||
P99 TPOP (ms): 19.47
|
|
||||||
---------------Inter-token Latency----------------
|
|
||||||
Mean ITL (ms): 19.41
|
|
||||||
Median ITL (ms): 19.12
|
|
||||||
P99 ITL (ms): 23.58
|
|
||||||
================ Stage 1 (talker) ================
|
|
||||||
-------------------Stage Timing-------------------
|
|
||||||
Mean stage_gen_time (ms): 18849.64
|
|
||||||
Median stage_gen_time (ms): 13641.15
|
|
||||||
P99 stage_gen_time (ms): 33655.58
|
|
||||||
============= Internal Stream Result =============
|
|
||||||
-----------Serving Time to First Chunk------------
|
|
||||||
Mean Serving TTFC (ms): 1029.14
|
|
||||||
Median Serving TTFC (ms): 1027.98
|
|
||||||
P99 Serving TTFC (ms): 1032.59
|
|
||||||
-----Time per Output Chunk (excl. 1st chunk)------
|
|
||||||
Mean TPOP (ms): 17.53
|
|
||||||
Median TPOP (ms): 18.70
|
|
||||||
P99 TPOP (ms): 18.88
|
|
||||||
---------------Inter-chunk Latency----------------
|
|
||||||
Mean ICL (ms): 16.27
|
|
||||||
Median ICL (ms): 18.19
|
|
||||||
P99 ICL (ms): 25.86
|
|
||||||
=============== Stage 2 (code2wav) ===============
|
|
||||||
-------------------Stage Timing-------------------
|
|
||||||
Mean stage_gen_time (ms): 18860.74
|
|
||||||
Median stage_gen_time (ms): 13655.84
|
|
||||||
P99 stage_gen_time (ms): 33659.79
|
|
||||||
================== Audio Result ==================
|
|
||||||
Stage audio duration generated(s): 260.11
|
|
||||||
Stage audio frames generated: 6242730
|
|
||||||
-----------Serving Time to First Packet-----------
|
|
||||||
Mean Serving AUDIO_TTFP (ms): 1366.16
|
|
||||||
Median Serving AUDIO_TTFP (ms): 1367.20
|
|
||||||
P99 Serving AUDIO_TTFP (ms): 1368.26
|
|
||||||
-----------------Real Time Factor-----------------
|
|
||||||
Mean AUDIO_RTF: 0.24
|
|
||||||
Median AUDIO_RTF: 0.26
|
|
||||||
P99 AUDIO_RTF: 0.27
|
|
||||||
```
|
|
||||||
Explanation:
|
|
||||||
- stage_gen_time: Time from submitting a request to a specific stage to that stage finishing generation.
|
|
||||||
|
|
||||||
- Serving TTFC (Time to First Chunk): Time from the HTTP request being accepted by the serving frontend to
|
|
||||||
the stage producing its first non-empty output chunk. Keep the name TTFT in text stage, TTFP in audio stage
|
|
||||||
for easier mapping.
|
|
||||||
|
|
||||||
- TPOP (Time per Output Chunk): Average time from the first output chunk to stage completion, divided by the
|
|
||||||
number of remaining output chunks. The TPOP abbreviation follows Qwen3.5-Omni Technical Report.
|
|
||||||
|
|
||||||
- ICL (Inter-Chunk Latency): Time between two consecutive output chunks produced by the same stage. Keep the
|
|
||||||
name ITL in text stage for easier mapping.
|
|
||||||
|
|
||||||
</details>
|
|
||||||
@@ -116,20 +116,6 @@ Only specific types of PRs will be reviewed. The PR title is prefixed appropriat
|
|||||||
!!! note
|
!!! note
|
||||||
If the PR spans more than one category, please include all relevant prefixes.
|
If the PR spans more than one category, please include all relevant prefixes.
|
||||||
|
|
||||||
### Pre-Check Before Submitting
|
|
||||||
|
|
||||||
Before submitting a PR, run the precheck-pr skill with code agent for a self-review against project conventions:
|
|
||||||
|
|
||||||
```
|
|
||||||
/precheck-pr
|
|
||||||
```
|
|
||||||
|
|
||||||
The skill offers two modes:
|
|
||||||
- **Quick (~3 min):** catches showstoppers — PR title format, missing benchmark claims, rebase status
|
|
||||||
- **Full (~10 min):** thorough maintainer-grade review — dead code scan, copy-paste detection, import hygiene
|
|
||||||
|
|
||||||
The precheck covers five PR types: Bug Fix, Performance, New Model, Diffusion Model, and General. Each type has a tailored checklist that validates evidence quality (repro steps, A/B benchmarks, registry entries, etc.). See the skill in `.claude/skills/precheck-pr/` for the full checklist.
|
|
||||||
|
|
||||||
### Local Test
|
### Local Test
|
||||||
Please run the L1 and L2 test cases locally first and attach the results before contacting us to add the "ready" label. Please refer to the [test instructions](./ci/test_guide.md) for running the test cases.
|
Please run the L1 and L2 test cases locally first and attach the results before contacting us to add the "ready" label. Please refer to the [test instructions](./ci/test_guide.md) for running the test cases.
|
||||||
|
|
||||||
|
|||||||
@@ -305,8 +305,8 @@ class GPUWorker:
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Load model
|
# Load model
|
||||||
model_loader = DiffusersPipelineLoader(load_config, self.od_config)
|
model_loader = DiffusersPipelineLoader(load_config)
|
||||||
self.pipeline = model_loader.load_model(load_device=f"cuda:{rank}")
|
self.pipeline = model_loader.load_model(od_config, load_device=f"cuda:{rank}")
|
||||||
|
|
||||||
# Setup cache backend
|
# Setup cache backend
|
||||||
from vllm_omni.diffusion.cache.selector import get_cache_backend
|
from vllm_omni.diffusion.cache.selector import get_cache_backend
|
||||||
|
|||||||
@@ -45,7 +45,7 @@ If you do not need to modify source code of vLLM, you can directly install the s
|
|||||||
uv pip install vllm==0.22.0 --torch-backend=auto
|
uv pip install vllm==0.22.0 --torch-backend=auto
|
||||||
```
|
```
|
||||||
|
|
||||||
The 0.22.0 release of vLLM ships CUDA 12.9-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
The 0.22.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
||||||
|
|
||||||
#### Installation of vLLM-Omni
|
#### Installation of vLLM-Omni
|
||||||
Since vllm-omni is rapidly evolving, it's recommended to install it from source
|
Since vllm-omni is rapidly evolving, it's recommended to install it from source
|
||||||
@@ -70,8 +70,8 @@ git checkout v0.22.0
|
|||||||
```
|
```
|
||||||
Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set `VLLM_PRECOMPILED_WHEEL_LOCATION` as your local absolute path of whl file.
|
Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set `VLLM_PRECOMPILED_WHEEL_LOCATION` as your local absolute path of whl file.
|
||||||
```bash
|
```bash
|
||||||
#For CUDA 12.9 (the default for v0.22.0; the wheel filename has no `+cu129` suffix)
|
#For CUDA 13.0 (the default for v0.22.0; the wheel filename has no `+cu130` suffix)
|
||||||
export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.22.0/vllm-0.22.0-cp38-abi3-manylinux_2_28_x86_64.whl
|
export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.22.0/vllm-0.22.0-cp38-abi3-manylinux_2_35_x86_64.whl
|
||||||
```
|
```
|
||||||
Install vllm with command below (If you have no existing PyTorch).
|
Install vllm with command below (If you have no existing PyTorch).
|
||||||
```bash
|
```bash
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ source .venv/bin/activate
|
|||||||
uv pip install vllm==0.22.0 --torch-backend=auto
|
uv pip install vllm==0.22.0 --torch-backend=auto
|
||||||
|
|
||||||
# On ROCm
|
# On ROCm
|
||||||
uv pip install vllm==0.22.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.22.0/rocm722
|
uv pip install vllm==0.22.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.22.0/rocm721
|
||||||
|
|
||||||
git clone https://github.com/vllm-project/vllm-omni.git
|
git clone https://github.com/vllm-project/vllm-omni.git
|
||||||
cd vllm-omni
|
cd vllm-omni
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ th {
|
|||||||
| `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
| `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
| `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
| `Cosmos3OmniDiffusersPipeline` | Cosmos3 T2I, T2V, I2V, V2V, T2V with sound, action policy | `nvidia/Cosmos3-Nano` | ✅︎ | | | |
|
| `Cosmos3OmniDiffusersPipeline` | Cosmos3 T2I, T2V, I2V | `nvidia/Cosmos3-Nano` | ✅︎ | | | |
|
||||||
| `WanSpeechToVideoPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `WanSpeechToVideoPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
| `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
| `LTX2Pipeline` | LTX-2-T2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | |
|
| `LTX2Pipeline` | LTX-2-T2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | |
|
||||||
@@ -42,7 +42,6 @@ th {
|
|||||||
| `LTX2ImageToVideoTwoStagesPipeline` | LTX-2-I2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | |
|
| `LTX2ImageToVideoTwoStagesPipeline` | LTX-2-I2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | |
|
||||||
| `LTX23Pipeline` | LTX-2.3-T2V | `dg845/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | |
|
| `LTX23Pipeline` | LTX-2.3-T2V | `dg845/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | |
|
||||||
| `LTX23ImageToVideoPipeline` | LTX-2.3-I2V | `dg845/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | |
|
| `LTX23ImageToVideoPipeline` | LTX-2.3-I2V | `dg845/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | |
|
||||||
| `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | | | |
|
|
||||||
| `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | |
|
| `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | |
|
||||||
| `MagiHumanPipeline` | MagiHuman | `SII-GAIR/daVinci-MagiHuman-Base-1080p` | ✅︎ | ✅︎ | | |
|
| `MagiHumanPipeline` | MagiHuman | `SII-GAIR/daVinci-MagiHuman-Base-1080p` | ✅︎ | ✅︎ | | |
|
||||||
| `OvisImagePipeline` | Ovis-Image | `OvisAI/Ovis-Image` | ✅︎ | ✅︎ | | ✅︎ |
|
| `OvisImagePipeline` | Ovis-Image | `OvisAI/Ovis-Image` | ✅︎ | ✅︎ | | ✅︎ |
|
||||||
@@ -50,8 +49,6 @@ th {
|
|||||||
| `LongCatImageEditPipeline` | LongCat-Image-Edit | `meituan-longcat/LongCat-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `LongCatImageEditPipeline` | LongCat-Image-Edit | `meituan-longcat/LongCat-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
| `StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` | ✅︎ | ✅︎ | | ✅︎ |
|
| `StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` | ✅︎ | ✅︎ | | ✅︎ |
|
||||||
| `CosyVoice3Model` | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✅︎ | ✅︎ | | ✅︎ |
|
| `CosyVoice3Model` | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✅︎ | ✅︎ | | ✅︎ |
|
||||||
| `OmniVoiceModel` | OmniVoice | `k2-fsa/OmniVoice` | ✅︎ | | | |
|
|
||||||
| `VoxCPM2TalkerForConditionalGeneration` | VoxCPM2 | `openbmb/VoxCPM2` | ✅︎ | | | |
|
|
||||||
| `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | |
|
| `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | |
|
||||||
| `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
| `FluxKontextPipeline` | FLUX.1-Kontext-dev | `black-forest-labs/FLUX.1-Kontext-dev` | ✅︎ | ✅︎ | | |
|
| `FluxKontextPipeline` | FLUX.1-Kontext-dev | `black-forest-labs/FLUX.1-Kontext-dev` | ✅︎ | ✅︎ | | |
|
||||||
@@ -64,10 +61,6 @@ th {
|
|||||||
| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-0.6B-Base` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-0.6B-Base` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
| `GLMTTSForConditionalGeneration` | GLM-TTS | `zai-org/GLM-TTS` | ✅︎ | | | |
|
| `GLMTTSForConditionalGeneration` | GLM-TTS | `zai-org/GLM-TTS` | ✅︎ | | | |
|
||||||
| `MossTTSNanoForCausalLM` | MOSS-TTS-Nano | `OpenMOSS-Team/MOSS-TTS-Nano` | ✅︎ | | | |
|
|
||||||
| `MossTTSDelayModel` | MOSS-TTS, MOSS-TTSD, MOSS-SoundEffect, MOSS-VoiceGenerator | `OpenMOSS-Team/MOSS-VoiceGenerator` | ✅︎ | | | |
|
|
||||||
| `MossTTSRealtime` | MOSS-TTS-Realtime | `OpenMOSS-Team/MOSS-TTS-Realtime` | ✅︎ | | | |
|
|
||||||
| `HiggsAudioV2ForConditionalGeneration` | Higgs-Audio v2 | `bosonai/higgs-audio-v2-generation-3B-base` | ✅︎ | | | |
|
|
||||||
| `NextStep11Pipeline` | NextStep-1.1 | `stepfun-ai/NextStep-1.1` | ✅︎ | ✅︎ | | ✅︎ |
|
| `NextStep11Pipeline` | NextStep-1.1 | `stepfun-ai/NextStep-1.1` | ✅︎ | ✅︎ | | ✅︎ |
|
||||||
| `MiMoAudioModel` | MiMo-Audio-7B-Instruct | `XiaomiMiMo/MiMo-Audio-7B-Instruct` | ✅︎ | ✅︎ | | |
|
| `MiMoAudioModel` | MiMo-Audio-7B-Instruct | `XiaomiMiMo/MiMo-Audio-7B-Instruct` | ✅︎ | ✅︎ | | |
|
||||||
| `MiMoV2ASRForCausalLM` | MiMo-V2.5-ASR | `XiaomiMiMo/MiMo-V2.5-ASR` | ✅︎ | ✅︎ | | |
|
| `MiMoV2ASRForCausalLM` | MiMo-V2.5-ASR | `XiaomiMiMo/MiMo-V2.5-ASR` | ✅︎ | ✅︎ | | |
|
||||||
@@ -75,13 +68,11 @@ th {
|
|||||||
| `FishSpeechSlowARForConditionalGeneration` | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✅︎ | ✅︎ | | |
|
| `FishSpeechSlowARForConditionalGeneration` | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✅︎ | ✅︎ | | |
|
||||||
| `DreamIDOmniPipeline` | DreamID-Omni | `XuGuo699/DreamID-Omni` | ✅︎ | ✅︎ | | |
|
| `DreamIDOmniPipeline` | DreamID-Omni | `XuGuo699/DreamID-Omni` | ✅︎ | ✅︎ | | |
|
||||||
| `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only) | `SenseNova/SenseNova-U1-8B-MoT` | ✅︎ | | | |
|
| `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only) | `SenseNova/SenseNova-U1-8B-MoT` | ✅︎ | | | |
|
||||||
| `LancePipeline` | Lance | `bytedance-research/Lance` | ✅︎ | | | |
|
|
||||||
| `HunyuanVideo15Pipeline` | HunyuanVideo-1.5-T2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | ✅︎ | ✅︎ | | |
|
| `HunyuanVideo15Pipeline` | HunyuanVideo-1.5-T2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | ✅︎ | ✅︎ | | |
|
||||||
| `HunyuanVideo15ImageToVideoPipeline` | HunyuanVideo-1.5-I2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v` | ✅︎ | ✅︎ | | |
|
| `HunyuanVideo15ImageToVideoPipeline` | HunyuanVideo-1.5-I2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v` | ✅︎ | ✅︎ | | |
|
||||||
| `VoxtralTTSForConditionalGeneration` | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✅︎ | ✅︎ | | |
|
| `VoxtralTTSForConditionalGeneration` | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✅︎ | ✅︎ | | |
|
||||||
| `CovoAudioForConditionalGeneration` | Covo-Audio-Chat | `tencent/Covo-Audio-Chat` | ✅︎ | | | |
|
| `CovoAudioForConditionalGeneration` | Covo-Audio-Chat | `tencent/Covo-Audio-Chat` | ✅︎ | | | |
|
||||||
|`DyninOmniForConditionalGeneration` | Dynin-Omni | `snu-aidas/Dynin-Omni` | ✅︎ | | | |
|
|`DyninOmniForConditionalGeneration` | Dynin-Omni | `snu-aidas/Dynin-Omni` | ✅︎ | | | |
|
||||||
| `MiniCPMO45OmniForConditionalGeneration` | MiniCPM-o 4.5 | `openbmb/MiniCPM-o-4_5` | ✅︎ | | ✅︎ | |
|
|
||||||
| `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
|`HiDreamImagePipeline` | HiDream-I1-Full | `HiDream-ai/HiDream-I1-Full` | ✅︎ | ✅︎ | | |
|
|`HiDreamImagePipeline` | HiDream-I1-Full | `HiDream-ai/HiDream-I1-Full` | ✅︎ | ✅︎ | | |
|
||||||
|
|
||||||
|
|||||||
@@ -116,7 +116,7 @@ Content-Type: application/json
|
|||||||
| `initial_codec_chunk_frames` | integer | null | Per-request initial chunk size override for TTFA tuning. When null, IC is computed dynamically based on server load. |
|
| `initial_codec_chunk_frames` | integer | null | Per-request initial chunk size override for TTFA tuning. When null, IC is computed dynamically based on server load. |
|
||||||
| `stream` | bool | false | Stream raw PCM chunks as they are decoded (requires `response_format="pcm"`) |
|
| `stream` | bool | false | Stream raw PCM chunks as they are decoded (requires `response_format="pcm"`) |
|
||||||
|
|
||||||
**Supported languages:** Only applicable to Qwen3-TTS. Derived from the model configuration (`talker_config.codec_language_id` in the checkpoint's `config.json`), plus `Auto`, which is always accepted. Official Qwen3-TTS checkpoints support: Auto, Chinese, English, Japanese, Korean, German, French, Russian, Portuguese, Spanish, Italian.
|
**Supported languages:** Auto, Chinese, English, Japanese, Korean, German, French, Russian, Portuguese, Spanish, Italian
|
||||||
|
|
||||||
#### Voice Clone Parameters (Base task)
|
#### Voice Clone Parameters (Base task)
|
||||||
|
|
||||||
|
|||||||
@@ -67,9 +67,8 @@ curl -L "http://localhost:8091/v1/videos/${video_id}/content" -o output.mp4
|
|||||||
|
|
||||||
| Parameter | Type | Default | Description |
|
| Parameter | Type | Default | Description |
|
||||||
|-----------|------|---------|-------------|
|
|-----------|------|---------|-------------|
|
||||||
| `input_reference` | file | null | Uploaded reference image or video for image-to-video/video-to-video requests |
|
| `input_reference` | file | null | Uploaded reference image for image-to-video requests |
|
||||||
| `image_reference` | string | null | JSON-encoded reference image payload; do not combine with `input_reference` or `video_reference` |
|
| `image_reference` | string | null | JSON-encoded reference image payload; do not combine with `input_reference` |
|
||||||
| `video_reference` | string | null | JSON-encoded reference video payload; do not combine with `input_reference` or `image_reference` |
|
|
||||||
| `width` | integer | model default | Output video width |
|
| `width` | integer | model default | Output video width |
|
||||||
| `height` | integer | model default | Output video height |
|
| `height` | integer | model default | Output video height |
|
||||||
| `num_frames` | integer | model default | Number of generated frames |
|
| `num_frames` | integer | model default | Number of generated frames |
|
||||||
@@ -81,8 +80,6 @@ curl -L "http://localhost:8091/v1/videos/${video_id}/content" -o output.mp4
|
|||||||
| `flow_shift` | number | null | Scheduler flow-shift value |
|
| `flow_shift` | number | null | Scheduler flow-shift value |
|
||||||
| `true_cfg_scale` | number | null | True CFG scale when supported by the model |
|
| `true_cfg_scale` | number | null | True CFG scale when supported by the model |
|
||||||
| `seed` | integer | null | Random seed for reproducibility |
|
| `seed` | integer | null | Random seed for reproducibility |
|
||||||
| `generate_sound` | boolean | false | Request model-generated audio for video models that support sound generation |
|
|
||||||
| `sound_duration` | number | null | Duration in seconds for generated audio; defaults to generated video duration |
|
|
||||||
| `negative_prompt` | string | null | Text describing what to avoid in the generated video |
|
| `negative_prompt` | string | null | Text describing what to avoid in the generated video |
|
||||||
| `enable_frame_interpolation` | boolean | null | Enable post-generation frame interpolation |
|
| `enable_frame_interpolation` | boolean | null | Enable post-generation frame interpolation |
|
||||||
| `frame_interpolation_exp` | integer | null | Interpolation exponent; `1=2x`, `2=4x`, and so on |
|
| `frame_interpolation_exp` | integer | null | Interpolation exponent; `1=2x`, `2=4x`, and so on |
|
||||||
@@ -126,41 +123,6 @@ curl -s http://localhost:8091/v1/videos \
|
|||||||
-F "fps=16"
|
-F "fps=16"
|
||||||
```
|
```
|
||||||
|
|
||||||
### Video-to-Video
|
|
||||||
|
|
||||||
For models that support video conditioning, upload the reference video with
|
|
||||||
`input_reference`:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl -s http://localhost:8091/v1/videos \
|
|
||||||
-F "prompt=continue this motion with consistent subjects and lighting" \
|
|
||||||
-F "input_reference=@input.mp4;type=video/mp4" \
|
|
||||||
-F "width=1280" \
|
|
||||||
-F "height=720" \
|
|
||||||
-F "num_frames=80" \
|
|
||||||
-F "fps=16"
|
|
||||||
```
|
|
||||||
|
|
||||||
You can also pass a JSON-safe video URL or `data:video/...;base64,...` payload
|
|
||||||
through `video_reference`. Do not send `video_reference` together with
|
|
||||||
`input_reference` or `image_reference`.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl -s http://localhost:8091/v1/videos \
|
|
||||||
-F "prompt=continue this motion with consistent subjects and lighting" \
|
|
||||||
-F 'video_reference={"video_url":"https://example.com/input.mp4"}' \
|
|
||||||
-F "width=1280" \
|
|
||||||
-F "height=720" \
|
|
||||||
-F "num_frames=80" \
|
|
||||||
-F "fps=16"
|
|
||||||
```
|
|
||||||
|
|
||||||
JSON references currently support `image_url`/`video_url`; `file_id` references
|
|
||||||
are not implemented yet. Models may expose additional V2V controls through
|
|
||||||
`extra_params`. For example, Cosmos3 supports
|
|
||||||
`condition_frame_indexes_vision` and `condition_video_keep` to select which
|
|
||||||
decoded reference frames are used as clean conditioning.
|
|
||||||
|
|
||||||
### Synchronous Generation
|
### Synchronous Generation
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -184,10 +146,7 @@ export VLLM_OMNI_STORAGE_PATH=/var/tmp/vllm-omni-videos
|
|||||||
|
|
||||||
## Model-Specific Examples
|
## Model-Specific Examples
|
||||||
|
|
||||||
For complete text-to-video, image-to-video, and model-specific video-to-video
|
For complete text-to-video and image-to-video walkthroughs, see:
|
||||||
walkthroughs, see:
|
|
||||||
|
|
||||||
- [Text-to-Video](../user_guide/examples/online_serving/text_to_video.md)
|
- [Text-to-Video](../user_guide/examples/online_serving/text_to_video.md)
|
||||||
- [Image-to-Video](../user_guide/examples/online_serving/image_to_video.md)
|
- [Image-to-Video](../user_guide/examples/online_serving/image_to_video.md)
|
||||||
- [Cosmos3 recipes](https://github.com/vllm-project/vllm-omni/blob/main/recipes/cosmos3/Cosmos3-Nano.md)
|
|
||||||
for model-specific video-to-video examples and conditioning controls
|
|
||||||
@@ -145,10 +145,9 @@ The following tables show which models support each feature:
|
|||||||
| Model | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | Pipeline-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
|
| Model | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | Pipeline-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
|
||||||
|------------------------------|:---------:|:----------:|:---------------------:|:--------------:|:-----------------:|:-----------------:|:------:|:-------------------------:|:--------------------:|:--------------:|:----------------:|
|
|------------------------------|:---------:|:----------:|:---------------------:|:--------------:|:-----------------:|:-----------------:|:------:|:-------------------------:|:--------------------:|:--------------:|:----------------:|
|
||||||
| **Wan2.2** | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (encode/decode) | ❌ | ❌ |
|
| **Wan2.2** | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (encode/decode) | ❌ | ❌ |
|
||||||
| **Wan2.2-S2V** | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ (encode/decode) | ❌ | ❌ |
|
|
||||||
| **Wan2.1-VACE** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
|
| **Wan2.1-VACE** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
|
||||||
| **LTX-2** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
| **LTX-2** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||||
| **LTX-2.3** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
|
| **LTX-2.3** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||||
| **Helios** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
| **Helios** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||||
| **HunyuanVideo-1.5 T2V I2V** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (encode/decode) | ✅ | ❌ |
|
| **HunyuanVideo-1.5 T2V I2V** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (encode/decode) | ✅ | ❌ |
|
||||||
| **DreamID-Omni** | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
| **DreamID-Omni** | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||||
|
|||||||
@@ -28,50 +28,6 @@ in deep DiT blocks.
|
|||||||
Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this
|
Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this
|
||||||
guide. FP8 on Ampere may use a weight-only path where available.
|
guide. FP8 on Ampere may use a weight-only path where available.
|
||||||
|
|
||||||
### Faster FP8 GEMM on Blackwell (quack)
|
|
||||||
|
|
||||||
On Blackwell (SM 100+), vLLM runs FP8 linears through the FlashInfer kernel, which
|
|
||||||
applies the bias as a separate kernel after the GEMM. On the small GEMMs in video
|
|
||||||
DiTs this bias add is a significant overhead. Installing the optional `quack` kernel
|
|
||||||
lets vLLM-Omni fuse `alpha * (A @ B) + bias` into a single CuteDSL GEMM, recovering
|
|
||||||
that overhead (e.g. HunyuanVideo-1.5 FP8 goes from slower-than-BF16 to faster).
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# CUDA 12.9
|
|
||||||
pip install vllm-omni[quack]
|
|
||||||
|
|
||||||
# CUDA 13.x
|
|
||||||
pip install 'quack-kernels[cu13]' --extra-index-url https://download.pytorch.org/whl/cu130
|
|
||||||
```
|
|
||||||
|
|
||||||
It is enabled automatically once installed (no flag needed) and is **Blackwell-only**:
|
|
||||||
on Hopper/Ada the CUTLASS FP8 kernel already fuses bias, so quack is not used there.
|
|
||||||
Set `VLLM_OMNI_USE_QUACK_FP8=0` to force the FlashInfer path. If `quack-kernels` is
|
|
||||||
not installed, FP8 still works — it just keeps the unfused FlashInfer path.
|
|
||||||
|
|
||||||
#### Compile cache and warmup
|
|
||||||
|
|
||||||
quack JIT-compiles its kernel once per distinct GEMM shape (tens of seconds, longer
|
|
||||||
the first time across all autotuned configs). The compiled `.o` files are cached on
|
|
||||||
disk and reused on later runs, so this is a one-time cost — **not** per request.
|
|
||||||
|
|
||||||
vLLM-Omni points that cache at `~/.cache/vllm_omni/quack` (override with
|
|
||||||
`QUACK_CACHE_DIR`) instead of quack's default under `/tmp`, so it survives restarts.
|
|
||||||
In containers, set `QUACK_CACHE_DIR` to a mounted/persistent path — or bake it into
|
|
||||||
the image — so the first cold start does not recompile. The engine's startup dummy
|
|
||||||
run already exercises the kernels, so with a warm cache the first real request is fast.
|
|
||||||
|
|
||||||
To pre-warm specific shapes (e.g. at image build time):
|
|
||||||
|
|
||||||
```python
|
|
||||||
from vllm_omni.quantization.quack_fp8 import warmup_quack_fp8
|
|
||||||
# (M, K, N) per linear; M = number of tokens for your resolution/frame count
|
|
||||||
warmup_quack_fp8([(14040, 2048, 6144), (14040, 2048, 2048)])
|
|
||||||
```
|
|
||||||
|
|
||||||
> The PyPI package is `quack-kernels` (imported as `quack`); plain `pip install
|
|
||||||
> quack` is an unrelated statistics library. Requires CUDA 12.9+ and Python 3.12.
|
|
||||||
|
|
||||||
## Model Type Support
|
## Model Type Support
|
||||||
|
|
||||||
### Diffusion Model (Qwen-Image, Wan2.2)
|
### Diffusion Model (Qwen-Image, Wan2.2)
|
||||||
|
|||||||
@@ -1,14 +0,0 @@
|
|||||||
{
|
|
||||||
"prompt": "A robotic arm, primarily white with black joints and cables, is shown in a clean, modern indoor setting with a white tabletop. The arm, equipped with a gripper holding a small, light green pitcher, is positioned above a clear glass containing a reddish-brown liquid and a spoon. The robotic arm is in the process of pouring a transparent liquid into the glass. To the left of the pitcher, there is an opened jar with a similar reddish-brown substance visible through its transparent body. In the background, a vase with white flowers and a brown couch are partially visible, adding to the contemporary ambiance. The lighting is bright, casting soft shadows on the table. The robotic arm's movements are smooth and controlled, demonstrating precision in its task. As the video progresses, the robotic arm completes the pour, leaving the glass half-filled with the reddish-brown liquid. The jar remains untouched throughout the sequence, and the spoon inside the glass remains stationary. The other robotic arm on the right side also stays stationary throughout the video. The final frame captures the robotic arm with the pitcher finishing the pour, with the glass now filled to a higher level, while the pitcher is slightly tilted but still held securely by the gripper.",
|
|
||||||
"negative_prompt": "",
|
|
||||||
"vision_path": "https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/vision/robot_pouring.mp4",
|
|
||||||
"height": 720,
|
|
||||||
"width": 1280,
|
|
||||||
"num_frames": 189,
|
|
||||||
"num_inference_steps": 35,
|
|
||||||
"guidance_scale": 6.0,
|
|
||||||
"flow_shift": 10.0,
|
|
||||||
"fps": 24,
|
|
||||||
"condition_frame_indexes_vision": [0, 1],
|
|
||||||
"condition_video_keep": "first"
|
|
||||||
}
|
|
||||||
@@ -1,304 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import uuid
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import cv2
|
|
||||||
import numpy as np
|
|
||||||
import torch
|
|
||||||
from PIL import Image
|
|
||||||
|
|
||||||
from vllm_omni import Omni
|
|
||||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
|
||||||
from vllm_omni.outputs import OmniRequestOutput
|
|
||||||
|
|
||||||
WORKER_EXTENSION = "vllm_omni.diffusion.models.dreamzero.video_export_worker.DreamZeroVideoExportWorkerExtension"
|
|
||||||
ASSET_REPO_ID = "YangshenDeng/vllm-omni-dreamzero-assets"
|
|
||||||
DEFAULT_SESSION_PREFIX = "dreamzero-export"
|
|
||||||
RELATIVE_OFFSETS = [-23, -16, -8, 0]
|
|
||||||
ACTION_HORIZON = 24
|
|
||||||
CAMERA_FILES = {
|
|
||||||
"observation/exterior_image_0_left": "exterior_image_1_left.mp4",
|
|
||||||
"observation/exterior_image_1_left": "exterior_image_2_left.mp4",
|
|
||||||
"observation/wrist_image_left": "wrist_image_left.mp4",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _parse_args() -> argparse.Namespace:
|
|
||||||
repo_root = Path(__file__).resolve().parents[3]
|
|
||||||
parser = argparse.ArgumentParser(description="Export DreamZero prediction video from downloaded example inputs.")
|
|
||||||
parser.add_argument("--model", default="GEAR-Dreams/DreamZero-DROID")
|
|
||||||
parser.add_argument("--deploy-config", type=Path, required=True)
|
|
||||||
parser.add_argument(
|
|
||||||
"--video-dir",
|
|
||||||
type=Path,
|
|
||||||
default=repo_root / "outputs" / "dreamzero" / "assets",
|
|
||||||
help="Directory containing the three camera MP4 files.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--output-dir", type=Path, default=repo_root / "outputs" / "dreamzero" / "generated_predictions"
|
|
||||||
)
|
|
||||||
parser.add_argument("--output-stem", default="dreamzero_prediction")
|
|
||||||
parser.add_argument(
|
|
||||||
"--prompt",
|
|
||||||
default="Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan",
|
|
||||||
)
|
|
||||||
parser.add_argument("--session-id", default=None)
|
|
||||||
parser.add_argument("--save-input-video", action="store_true")
|
|
||||||
parser.add_argument("--save-gif", action="store_true")
|
|
||||||
parser.add_argument("--save-actions", action="store_true")
|
|
||||||
parser.add_argument("--fps", type=int, default=5)
|
|
||||||
return parser.parse_args()
|
|
||||||
|
|
||||||
|
|
||||||
def _load_all_frames(video_path: Path) -> np.ndarray:
|
|
||||||
cap = cv2.VideoCapture(str(video_path))
|
|
||||||
frames = []
|
|
||||||
while True:
|
|
||||||
ok, frame = cap.read()
|
|
||||||
if not ok:
|
|
||||||
break
|
|
||||||
frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
|
|
||||||
cap.release()
|
|
||||||
if not frames:
|
|
||||||
raise RuntimeError(f"No frames loaded from {video_path}")
|
|
||||||
return np.stack(frames, axis=0)
|
|
||||||
|
|
||||||
|
|
||||||
def _load_camera_frames(video_dir: Path) -> dict[str, np.ndarray]:
|
|
||||||
camera_frames: dict[str, np.ndarray] = {}
|
|
||||||
for camera_key, file_name in CAMERA_FILES.items():
|
|
||||||
video_path = video_dir / file_name
|
|
||||||
if not video_path.exists():
|
|
||||||
raise FileNotFoundError(
|
|
||||||
f"Missing DreamZero example asset: {video_path}. "
|
|
||||||
"Download the example videos with: "
|
|
||||||
f"`hf download {ASSET_REPO_ID} --repo-type dataset --local-dir {video_dir}`"
|
|
||||||
)
|
|
||||||
camera_frames[camera_key] = _load_all_frames(video_path)
|
|
||||||
return camera_frames
|
|
||||||
|
|
||||||
|
|
||||||
def _build_frame_schedule(total_frames: int, num_chunks: int) -> list[list[int]]:
|
|
||||||
chunks: list[list[int]] = []
|
|
||||||
current_frame = 23
|
|
||||||
for _ in range(num_chunks):
|
|
||||||
indices = [max(current_frame + offset, 0) for offset in RELATIVE_OFFSETS]
|
|
||||||
if indices[-1] >= total_frames:
|
|
||||||
break
|
|
||||||
chunks.append(indices)
|
|
||||||
current_frame += ACTION_HORIZON
|
|
||||||
return chunks
|
|
||||||
|
|
||||||
|
|
||||||
def _make_obs_from_video(
|
|
||||||
camera_frames: dict[str, np.ndarray],
|
|
||||||
frame_indices: list[int],
|
|
||||||
*,
|
|
||||||
prompt: str,
|
|
||||||
session_id: str,
|
|
||||||
) -> dict:
|
|
||||||
obs: dict = {}
|
|
||||||
for camera_key, all_frames in camera_frames.items():
|
|
||||||
selected = all_frames[frame_indices]
|
|
||||||
obs[camera_key] = selected[0] if len(frame_indices) == 1 else selected
|
|
||||||
|
|
||||||
obs["observation/joint_position"] = np.zeros(7, dtype=np.float32)
|
|
||||||
obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32)
|
|
||||||
obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32)
|
|
||||||
obs["prompt"] = prompt
|
|
||||||
obs["session_id"] = session_id
|
|
||||||
return obs
|
|
||||||
|
|
||||||
|
|
||||||
def _build_observations(video_dir: Path, prompt: str, session_id: str) -> tuple[dict[str, np.ndarray], list[dict]]:
|
|
||||||
camera_frames = _load_camera_frames(video_dir)
|
|
||||||
total_frames = min(frames.shape[0] for frames in camera_frames.values())
|
|
||||||
chunks = _build_frame_schedule(total_frames, 1)
|
|
||||||
observations = [
|
|
||||||
_make_obs_from_video(camera_frames, [0], prompt=prompt, session_id=session_id),
|
|
||||||
]
|
|
||||||
if chunks:
|
|
||||||
observations.append(
|
|
||||||
_make_obs_from_video(
|
|
||||||
camera_frames,
|
|
||||||
chunks[0],
|
|
||||||
prompt=prompt,
|
|
||||||
session_id=session_id,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
if len(observations) < 2:
|
|
||||||
raise RuntimeError("Need at least two DreamZero example observations to export a prediction video.")
|
|
||||||
return camera_frames, observations[:2]
|
|
||||||
|
|
||||||
|
|
||||||
def _extract_latents(output: OmniRequestOutput) -> torch.Tensor:
|
|
||||||
if not isinstance(output, OmniRequestOutput):
|
|
||||||
raise TypeError(f"Expected OmniRequestOutput, got {type(output)!r}")
|
|
||||||
if not output.images:
|
|
||||||
raise RuntimeError("DreamZero output does not contain video latents in `images`.")
|
|
||||||
|
|
||||||
latents = output.images[0]
|
|
||||||
if not isinstance(latents, torch.Tensor):
|
|
||||||
raise TypeError(f"Expected tensor latents, got {type(latents)!r}")
|
|
||||||
|
|
||||||
latents = latents.detach().cpu()
|
|
||||||
if latents.dim() == 4:
|
|
||||||
latents = latents.unsqueeze(0)
|
|
||||||
if latents.dim() != 5:
|
|
||||||
raise ValueError(f"Unexpected latent shape: {tuple(latents.shape)}")
|
|
||||||
|
|
||||||
if latents.shape[1] < latents.shape[2]:
|
|
||||||
latents = latents.transpose(1, 2).contiguous()
|
|
||||||
return latents
|
|
||||||
|
|
||||||
|
|
||||||
def _write_mp4(path: Path, frames: np.ndarray, fps: int) -> None:
|
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
height, width = frames.shape[1:3]
|
|
||||||
writer = cv2.VideoWriter(
|
|
||||||
str(path),
|
|
||||||
cv2.VideoWriter_fourcc(*"mp4v"),
|
|
||||||
float(fps),
|
|
||||||
(width, height),
|
|
||||||
)
|
|
||||||
if not writer.isOpened():
|
|
||||||
raise RuntimeError(f"Failed to open video writer for {path}")
|
|
||||||
try:
|
|
||||||
for frame in frames:
|
|
||||||
writer.write(cv2.cvtColor(frame, cv2.COLOR_RGB2BGR))
|
|
||||||
finally:
|
|
||||||
writer.release()
|
|
||||||
|
|
||||||
|
|
||||||
def _write_gif(path: Path, frames: np.ndarray, fps: int) -> None:
|
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
images = [Image.fromarray(frame) for frame in frames]
|
|
||||||
duration_ms = max(int(round(1000 / max(fps, 1))), 1)
|
|
||||||
images[0].save(
|
|
||||||
path,
|
|
||||||
save_all=True,
|
|
||||||
append_images=images[1:],
|
|
||||||
duration=duration_ms,
|
|
||||||
loop=0,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _stitch_input_frames(camera_frames: dict[str, np.ndarray]) -> np.ndarray:
|
|
||||||
total_frames = min(frames.shape[0] for frames in camera_frames.values())
|
|
||||||
stitched = []
|
|
||||||
for frame_index in range(total_frames):
|
|
||||||
left = camera_frames["observation/exterior_image_0_left"][frame_index]
|
|
||||||
right = camera_frames["observation/exterior_image_1_left"][frame_index]
|
|
||||||
wrist = camera_frames["observation/wrist_image_left"][frame_index]
|
|
||||||
pad = np.zeros((left.shape[0], left.shape[1], 3), dtype=np.uint8)
|
|
||||||
canvas = np.concatenate([left, right], axis=1)
|
|
||||||
bottom = np.concatenate([wrist, pad], axis=1)
|
|
||||||
stitched.append(np.concatenate([canvas, bottom], axis=0))
|
|
||||||
return np.stack(stitched, axis=0)
|
|
||||||
|
|
||||||
|
|
||||||
def _run_generation(
|
|
||||||
model: str, deploy_config_path: Path, observations: list[dict]
|
|
||||||
) -> tuple[Omni, list[OmniRequestOutput]]:
|
|
||||||
omni = Omni(
|
|
||||||
model=model,
|
|
||||||
deploy_config=str(deploy_config_path),
|
|
||||||
enforce_eager=True,
|
|
||||||
worker_extension_cls=WORKER_EXTENSION,
|
|
||||||
)
|
|
||||||
|
|
||||||
outputs: list[OmniRequestOutput] = []
|
|
||||||
for index, obs in enumerate(observations):
|
|
||||||
sampling_params = OmniDiffusionSamplingParams(
|
|
||||||
extra_args={
|
|
||||||
"reset": index == 0,
|
|
||||||
"session_id": obs["session_id"],
|
|
||||||
"robot_obs": obs,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
result = omni.generate(obs["prompt"], sampling_params_list=[sampling_params])
|
|
||||||
if not result:
|
|
||||||
raise RuntimeError(f"No output returned for DreamZero request {index}")
|
|
||||||
outputs.append(result[0])
|
|
||||||
return omni, outputs
|
|
||||||
|
|
||||||
|
|
||||||
def _decode_with_worker(omni: Omni, full_latents: torch.Tensor) -> np.ndarray:
|
|
||||||
stage_client = omni.engine.stage_clients[0]
|
|
||||||
engine = getattr(stage_client, "_engine", None)
|
|
||||||
if engine is None:
|
|
||||||
raise RuntimeError("DreamZero export requires inline diffusion stage access.")
|
|
||||||
|
|
||||||
decoded = engine.executor.collective_rpc(
|
|
||||||
"decode_video_latents_to_uint8",
|
|
||||||
args=(full_latents,),
|
|
||||||
unique_reply_rank=0,
|
|
||||||
exec_all_ranks=True,
|
|
||||||
)
|
|
||||||
if isinstance(decoded, torch.Tensor):
|
|
||||||
decoded = decoded.numpy()
|
|
||||||
if not isinstance(decoded, np.ndarray):
|
|
||||||
raise TypeError(f"Unexpected decoded output type: {type(decoded)!r}")
|
|
||||||
return decoded
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
args = _parse_args()
|
|
||||||
session_id = args.session_id or f"{DEFAULT_SESSION_PREFIX}-{uuid.uuid4()}"
|
|
||||||
|
|
||||||
camera_frames, observations = _build_observations(
|
|
||||||
video_dir=args.video_dir,
|
|
||||||
prompt=args.prompt,
|
|
||||||
session_id=session_id,
|
|
||||||
)
|
|
||||||
|
|
||||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
if args.save_input_video:
|
|
||||||
input_frames = _stitch_input_frames(camera_frames)
|
|
||||||
_write_mp4(args.output_dir / f"{args.output_stem}_input.mp4", input_frames, fps=15)
|
|
||||||
if args.save_gif:
|
|
||||||
_write_gif(args.output_dir / f"{args.output_stem}_input.gif", input_frames[::3], fps=5)
|
|
||||||
|
|
||||||
omni = None
|
|
||||||
try:
|
|
||||||
omni, outputs = _run_generation(
|
|
||||||
model=args.model,
|
|
||||||
deploy_config_path=args.deploy_config,
|
|
||||||
observations=observations,
|
|
||||||
)
|
|
||||||
latent_steps = [_extract_latents(output) for output in outputs]
|
|
||||||
full_latents = torch.cat(latent_steps, dim=2)
|
|
||||||
frames = _decode_with_worker(omni, full_latents)
|
|
||||||
finally:
|
|
||||||
if omni is not None:
|
|
||||||
omni.close()
|
|
||||||
|
|
||||||
mp4_path = args.output_dir / f"{args.output_stem}.mp4"
|
|
||||||
|
|
||||||
_write_mp4(mp4_path, frames, fps=args.fps)
|
|
||||||
print(f"SAVED_MP4={mp4_path}")
|
|
||||||
|
|
||||||
if args.save_gif:
|
|
||||||
gif_path = args.output_dir / f"{args.output_stem}.gif"
|
|
||||||
_write_gif(gif_path, frames, fps=args.fps)
|
|
||||||
print(f"SAVED_GIF={gif_path}")
|
|
||||||
|
|
||||||
if args.save_actions:
|
|
||||||
npz_path = args.output_dir / f"{args.output_stem}_actions.npz"
|
|
||||||
np.savez(
|
|
||||||
npz_path,
|
|
||||||
step0=np.asarray(outputs[0].multimodal_output.get("actions")),
|
|
||||||
step1=np.asarray(outputs[1].multimodal_output.get("actions")),
|
|
||||||
)
|
|
||||||
print(f"SAVED_ACTIONS={npz_path}")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -42,7 +42,7 @@ pip install \
|
|||||||
## 3. Hardware and VRAM Requirements
|
## 3. Hardware and VRAM Requirements
|
||||||
|
|
||||||
This example uses a 3-stage pipeline on one GPU by default
|
This example uses a 3-stage pipeline on one GPU by default
|
||||||
([`dynin_omni.yaml`](../../../vllm_omni/deploy/dynin_omni.yaml)):
|
([`dynin_omni.yaml`](../../../vllm_omni/model_executor/stage_configs/dynin_omni.yaml)):
|
||||||
|
|
||||||
- Stage-0 (`token2text`): `gpu_memory_utilization: 0.5`
|
- Stage-0 (`token2text`): `gpu_memory_utilization: 0.5`
|
||||||
- Stage-1 (`token2image`): `gpu_memory_utilization: 0.1`
|
- Stage-1 (`token2image`): `gpu_memory_utilization: 0.1`
|
||||||
|
|||||||
@@ -878,8 +878,8 @@ def parse_args(repo_root: Path) -> argparse.Namespace:
|
|||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--stage-config-path",
|
"--stage-config-path",
|
||||||
type=str,
|
type=str,
|
||||||
default=str(repo_root / "vllm_omni/deploy/dynin_omni.yaml"),
|
default=str(repo_root / "vllm_omni/model_executor/stage_configs/dynin_omni.yaml"),
|
||||||
help="Path to deploy config yaml.",
|
help="Path to stage config yaml.",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--dynin-config-path",
|
"--dynin-config-path",
|
||||||
|
|||||||
@@ -16,10 +16,9 @@ Usage:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import json
|
import os
|
||||||
import time
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
@@ -30,16 +29,6 @@ from vllm_omni.outputs import OmniRequestOutput
|
|||||||
from vllm_omni.platforms import current_omni_platform
|
from vllm_omni.platforms import current_omni_platform
|
||||||
|
|
||||||
|
|
||||||
def parse_profiler_config(value: str) -> dict[str, Any]:
|
|
||||||
try:
|
|
||||||
config = json.loads(value)
|
|
||||||
except json.JSONDecodeError as e:
|
|
||||||
raise argparse.ArgumentTypeError(f"--profiler-config must be valid JSON: {e}") from e
|
|
||||||
if not isinstance(config, dict):
|
|
||||||
raise argparse.ArgumentTypeError("--profiler-config must be a JSON object")
|
|
||||||
return config
|
|
||||||
|
|
||||||
|
|
||||||
def parse_args() -> argparse.Namespace:
|
def parse_args() -> argparse.Namespace:
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
description="Generate a talking-head video from a reference image and audio (Wan2.2 S2V)."
|
description="Generate a talking-head video from a reference image and audio (Wan2.2 S2V)."
|
||||||
@@ -98,12 +87,6 @@ def parse_args() -> argparse.Namespace:
|
|||||||
default=3.0,
|
default=3.0,
|
||||||
help="Scheduler flow shift (default: 3.0).",
|
help="Scheduler flow shift (default: 3.0).",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
|
||||||
"--boundary-ratio",
|
|
||||||
type=float,
|
|
||||||
default=None,
|
|
||||||
help="Boundary split ratio for low/high DiT stages (S2V). Default varies by model.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--output",
|
"--output",
|
||||||
type=str,
|
type=str,
|
||||||
@@ -195,10 +178,30 @@ def parse_args() -> argparse.Namespace:
|
|||||||
help="Enable cache-dit summary logging after diffusion forward passes.",
|
help="Enable cache-dit summary logging after diffusion forward passes.",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--profiler-config",
|
"--profile-dir",
|
||||||
type=parse_profiler_config,
|
type=str,
|
||||||
default=None,
|
default=None,
|
||||||
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
|
help="Enable torch profiler and save traces to this directory.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--profile-record-shapes",
|
||||||
|
action="store_true",
|
||||||
|
help="Record tensor shapes in profiler (increases trace size).",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--profile-with-stack",
|
||||||
|
action="store_true",
|
||||||
|
help="Record stack traces in profiler (increases overhead).",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--profile-with-memory",
|
||||||
|
action="store_true",
|
||||||
|
help="Profile memory usage.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--profile-with-flops",
|
||||||
|
action="store_true",
|
||||||
|
help="Estimate FLOPs for operations.",
|
||||||
)
|
)
|
||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
@@ -236,9 +239,23 @@ def main():
|
|||||||
ulysses_degree=args.ulysses_degree,
|
ulysses_degree=args.ulysses_degree,
|
||||||
)
|
)
|
||||||
|
|
||||||
profiler_enabled = args.profiler_config is not None
|
# Check if profiling is requested via CLI args or environment variable
|
||||||
|
profile_dir = args.profile_dir or os.getenv("VLLM_TORCH_PROFILER_DIR")
|
||||||
|
profiler_enabled = bool(profile_dir)
|
||||||
|
profiler_config = None
|
||||||
|
if profiler_enabled:
|
||||||
|
from vllm.config import ProfilerConfig
|
||||||
|
|
||||||
omni_kwargs = dict(
|
profiler_config = ProfilerConfig(
|
||||||
|
profiler="torch",
|
||||||
|
torch_profiler_dir=profile_dir,
|
||||||
|
torch_profiler_record_shapes=args.profile_record_shapes,
|
||||||
|
torch_profiler_with_stack=args.profile_with_stack,
|
||||||
|
torch_profiler_with_memory=args.profile_with_memory,
|
||||||
|
torch_profiler_with_flops=args.profile_with_flops,
|
||||||
|
)
|
||||||
|
|
||||||
|
omni = Omni(
|
||||||
model=args.model,
|
model=args.model,
|
||||||
model_class_name="WanS2VPipeline",
|
model_class_name="WanS2VPipeline",
|
||||||
flow_shift=args.flow_shift,
|
flow_shift=args.flow_shift,
|
||||||
@@ -252,12 +269,8 @@ def main():
|
|||||||
cache_backend=args.cache_backend,
|
cache_backend=args.cache_backend,
|
||||||
cache_config=cache_config,
|
cache_config=cache_config,
|
||||||
enable_cache_dit_summary=args.enable_cache_dit_summary,
|
enable_cache_dit_summary=args.enable_cache_dit_summary,
|
||||||
profiler_config=args.profiler_config,
|
profiler_config=profiler_config,
|
||||||
)
|
)
|
||||||
if args.boundary_ratio is not None:
|
|
||||||
omni_kwargs["boundary_ratio"] = args.boundary_ratio
|
|
||||||
|
|
||||||
omni = Omni(**omni_kwargs)
|
|
||||||
|
|
||||||
# Print generation configuration
|
# Print generation configuration
|
||||||
print(f"\n{'=' * 60}")
|
print(f"\n{'=' * 60}")
|
||||||
@@ -312,16 +325,13 @@ def main():
|
|||||||
if profiler_enabled:
|
if profiler_enabled:
|
||||||
print("\n[Profiler] Stopping profiler and collecting results...")
|
print("\n[Profiler] Stopping profiler and collecting results...")
|
||||||
profile_results = omni.stop_profile()
|
profile_results = omni.stop_profile()
|
||||||
if profile_results and isinstance(profile_results, dict):
|
if profile_results and isinstance(profile_results, list):
|
||||||
traces = profile_results.get("traces", [])
|
|
||||||
print("\n" + "=" * 60)
|
print("\n" + "=" * 60)
|
||||||
print("PROFILING RESULTS:")
|
print("PROFILING RESULTS:")
|
||||||
for rank, trace in enumerate(traces):
|
for i, result in enumerate(profile_results):
|
||||||
print(f"\nRank {rank}:")
|
print(f"\nStage {i}:")
|
||||||
if trace:
|
if result:
|
||||||
print(f" • Trace: {trace}")
|
print(f" {result}")
|
||||||
if not traces:
|
|
||||||
print(" No traces collected.")
|
|
||||||
print("=" * 60)
|
print("=" * 60)
|
||||||
else:
|
else:
|
||||||
print("[Profiler] No valid profiling data returned.")
|
print("[Profiler] No valid profiling data returned.")
|
||||||
|
|||||||
@@ -1,204 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
"""Offline higgs-audio v3 inference example.
|
|
||||||
|
|
||||||
Runs Stage 0 (Qwen3 talker) + Stage 1 (HiggsAudio codec) end-to-end
|
|
||||||
through the vLLM-Omni engine without going through the HTTP server, and
|
|
||||||
saves a 24 kHz mono WAV per prompt.
|
|
||||||
|
|
||||||
Example (plain TTS):
|
|
||||||
|
|
||||||
python examples/offline_inference/text_to_speech/higgs_audio_v3/end2end.py \\
|
|
||||||
--texts "Hello world." \\
|
|
||||||
--output-dir results/higgs_v3_wavs
|
|
||||||
|
|
||||||
Example (voice clone):
|
|
||||||
|
|
||||||
python examples/offline_inference/text_to_speech/higgs_audio_v3/end2end.py \\
|
|
||||||
--texts "Hello world." \\
|
|
||||||
--ref-audio path/to/reference.wav \\
|
|
||||||
--ref-text "Transcript of the reference clip." \\
|
|
||||||
--output-dir results/higgs_v3_wavs
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import os
|
|
||||||
|
|
||||||
os.environ.setdefault("VLLM_USE_DEEP_GEMM", "0")
|
|
||||||
os.environ.setdefault("VLLM_MOE_USE_DEEP_GEMM", "0")
|
|
||||||
|
|
||||||
import re
|
|
||||||
import time
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
import soundfile as sf
|
|
||||||
import torch
|
|
||||||
|
|
||||||
from vllm_omni import Omni
|
|
||||||
|
|
||||||
SAMPLE_RATE = 24_000
|
|
||||||
DEFAULT_TEXTS = (
|
|
||||||
"Hello world.",
|
|
||||||
"The quick brown fox jumps over the lazy dog.",
|
|
||||||
"Today is a beautiful day for a walk in the park.",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def parse_args():
|
|
||||||
import argparse
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(description="Offline higgs-audio v3 inference")
|
|
||||||
parser.add_argument(
|
|
||||||
"--model",
|
|
||||||
type=str,
|
|
||||||
default="bosonai/higgs-audio-v3-tts-4b",
|
|
||||||
help="Stage-0 talker model id or path.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--texts",
|
|
||||||
type=str,
|
|
||||||
nargs="+",
|
|
||||||
default=list(DEFAULT_TEXTS),
|
|
||||||
help="One or more plain-text prompts to synthesize.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--output-dir",
|
|
||||||
type=str,
|
|
||||||
default="results/higgs_v3_wavs",
|
|
||||||
help="Directory to write per-prompt WAV files.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--deploy-config",
|
|
||||||
type=str,
|
|
||||||
default=None,
|
|
||||||
help="Override the deploy config path.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--max-new-tokens",
|
|
||||||
type=int,
|
|
||||||
default=2048,
|
|
||||||
help="Cap on Stage-0 codec frames per request.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--ref-audio",
|
|
||||||
type=str,
|
|
||||||
default=None,
|
|
||||||
help="Reference audio for voice clone (WAV/FLAC/MP3 path).",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--ref-text",
|
|
||||||
type=str,
|
|
||||||
default=None,
|
|
||||||
help="Transcript of the reference audio. Optional but improves fidelity.",
|
|
||||||
)
|
|
||||||
return parser.parse_args()
|
|
||||||
|
|
||||||
|
|
||||||
def _slugify(text: str) -> str:
|
|
||||||
slug = re.sub(r"\s+", "_", text.strip().lower())
|
|
||||||
slug = re.sub(r"[^a-z0-9_]+", "", slug)
|
|
||||||
return slug[:48] or "prompt"
|
|
||||||
|
|
||||||
|
|
||||||
def _extract_pcm(multimodal_output: dict) -> torch.Tensor:
|
|
||||||
audio = multimodal_output.get("model_outputs")
|
|
||||||
if audio is None:
|
|
||||||
audio = multimodal_output.get("audio")
|
|
||||||
if audio is None:
|
|
||||||
raise ValueError(f"no audio key in multimodal_output: {list(multimodal_output.keys())}")
|
|
||||||
if isinstance(audio, list):
|
|
||||||
valid = [torch.as_tensor(a).float().cpu().reshape(-1) for a in audio if a is not None]
|
|
||||||
if not valid:
|
|
||||||
raise ValueError("audio list is empty")
|
|
||||||
return torch.cat(valid, dim=0) if len(valid) > 1 else valid[0]
|
|
||||||
return torch.as_tensor(audio).float().cpu().reshape(-1)
|
|
||||||
|
|
||||||
|
|
||||||
def _pcm_to_int16(pcm: torch.Tensor) -> np.ndarray:
|
|
||||||
arr = pcm.numpy()
|
|
||||||
if arr.dtype.kind == "f":
|
|
||||||
arr = np.clip(arr, -1.0, 1.0)
|
|
||||||
arr = (arr * 32767.0).astype(np.int16)
|
|
||||||
else:
|
|
||||||
arr = arr.astype(np.int16)
|
|
||||||
return arr
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
args = parse_args()
|
|
||||||
output_dir = Path(args.output_dir)
|
|
||||||
output_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
engine = Omni(model=args.model, deploy_config=args.deploy_config, trust_remote_code=True)
|
|
||||||
|
|
||||||
from transformers import AutoTokenizer
|
|
||||||
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_tokenizer import (
|
|
||||||
HiggsAudioV3TokenizerAdapter,
|
|
||||||
apply_delay_pattern,
|
|
||||||
encode_reference_audio,
|
|
||||||
)
|
|
||||||
|
|
||||||
tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
|
|
||||||
adapter = HiggsAudioV3TokenizerAdapter(tokenizer)
|
|
||||||
|
|
||||||
# Load and encode reference audio once (if voice cloning)
|
|
||||||
ref_codes_delayed: torch.Tensor | None = None
|
|
||||||
if args.ref_audio is not None:
|
|
||||||
ref_wav, ref_sr = sf.read(args.ref_audio, always_2d=False)
|
|
||||||
if ref_wav.ndim == 2:
|
|
||||||
ref_wav = ref_wav.mean(axis=1)
|
|
||||||
ref_codes_raw = encode_reference_audio(ref_wav, int(ref_sr))
|
|
||||||
ref_codes_delayed = apply_delay_pattern(ref_codes_raw)
|
|
||||||
print(f"Reference : {args.ref_audio}")
|
|
||||||
print(f"Ref codes : {ref_codes_raw.shape[0]} frames -> {ref_codes_delayed.shape[0]} delayed")
|
|
||||||
if args.ref_text:
|
|
||||||
print(f"Ref text : {args.ref_text}")
|
|
||||||
|
|
||||||
print(f"Model : {args.model}")
|
|
||||||
print(f"Prompts : {len(args.texts)}")
|
|
||||||
print(f"Output dir : {output_dir}")
|
|
||||||
print(f"Voice clone : {'yes' if ref_codes_delayed is not None else 'no'}")
|
|
||||||
|
|
||||||
total_elapsed = 0.0
|
|
||||||
total_dur = 0.0
|
|
||||||
for text in args.texts:
|
|
||||||
if ref_codes_delayed is not None:
|
|
||||||
prompt_ids = adapter.build_prompt(
|
|
||||||
text,
|
|
||||||
num_ref_tokens=int(ref_codes_delayed.shape[0]),
|
|
||||||
reference_text=args.ref_text,
|
|
||||||
)
|
|
||||||
prompt = {
|
|
||||||
"prompt_token_ids": prompt_ids,
|
|
||||||
"additional_information": {
|
|
||||||
"audio_input_ids": ref_codes_delayed.to(torch.long),
|
|
||||||
"audio_input_ids_mask": torch.ones(ref_codes_delayed.shape[0], dtype=torch.bool),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
else:
|
|
||||||
prompt_ids = adapter.build_prompt(text)
|
|
||||||
prompt = {"prompt_token_ids": prompt_ids}
|
|
||||||
|
|
||||||
t_start = time.perf_counter()
|
|
||||||
outputs = engine.generate([prompt])
|
|
||||||
elapsed = time.perf_counter() - t_start
|
|
||||||
total_elapsed += elapsed
|
|
||||||
|
|
||||||
mm = outputs[0].outputs[0].multimodal_output
|
|
||||||
pcm = _extract_pcm(mm)
|
|
||||||
slug = _slugify(text)
|
|
||||||
out_path = output_dir / f"{slug}.wav"
|
|
||||||
sf.write(str(out_path), _pcm_to_int16(pcm), SAMPLE_RATE, format="WAV", subtype="PCM_16")
|
|
||||||
dur = pcm.numel() / SAMPLE_RATE
|
|
||||||
total_dur += dur
|
|
||||||
print(f" {slug:<50} dur={dur:5.2f}s -> {out_path}")
|
|
||||||
|
|
||||||
rtf = total_elapsed / total_dur if total_dur > 0 else float("inf")
|
|
||||||
print(f"Total infer : {total_elapsed:.2f}s total audio: {total_dur:.2f}s RTF: {rtf:.3f}")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -1,132 +0,0 @@
|
|||||||
# DreamZero
|
|
||||||
|
|
||||||
DreamZero is the robot-policy serving example for the OpenPI-compatible `/v1/realtime/robot/openpi` endpoint.
|
|
||||||
|
|
||||||
## Supported checkpoints
|
|
||||||
|
|
||||||
| Model | HuggingFace repo |
|
|
||||||
|---|---|
|
|
||||||
| DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` |
|
|
||||||
|
|
||||||
## Quick start
|
|
||||||
|
|
||||||
### Start the server
|
|
||||||
|
|
||||||
```bash
|
|
||||||
bash examples/online_serving/dreamzero/run_server.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
By default this launches:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
vllm serve GEAR-Dreams/DreamZero-DROID --omni --port 8000 \
|
|
||||||
--served-model-name dreamzero-droid \
|
|
||||||
--deploy-config vllm_omni/deploy/dreamzero_tp1_cfg2.yaml \
|
|
||||||
--enforce-eager --disable-log-stats
|
|
||||||
```
|
|
||||||
|
|
||||||
Override `MODEL`, `PORT`, `HOST`, `DEPLOY_CONFIG`, or `SERVED_MODEL_NAME` through the script environment if needed.
|
|
||||||
|
|
||||||
### Install example dependencies
|
|
||||||
|
|
||||||
The core `pip install -e .` setup does not include the extra packages used by the DreamZero example scripts.
|
|
||||||
|
|
||||||
- `openpi_client.py`:
|
|
||||||
`openpi-client`, `websockets`, `opencv-python`
|
|
||||||
- `droid_sim_eval_client.py`:
|
|
||||||
`mediapy`, `websockets`, `openpi-client`
|
|
||||||
|
|
||||||
The DROID sim-eval script also needs an Isaac Lab environment that provides `isaaclab`, `isaaclab_tasks`, `sim_evals`, and `gymnasium`.
|
|
||||||
|
|
||||||
If you run the DROID client on Python < 3.12, also install `typing-extensions`.
|
|
||||||
|
|
||||||
### Configure TP and CFG parallelism
|
|
||||||
|
|
||||||
The bundled DreamZero configs intentionally keep only:
|
|
||||||
|
|
||||||
| Config | Purpose |
|
|
||||||
|---|---|
|
|
||||||
| `vllm_omni/deploy/dreamzero.yaml` | Default TP=1, CFG parallel disabled |
|
|
||||||
| `vllm_omni/deploy/dreamzero_tp1_cfg2.yaml` | TP=1, CFG parallel size=2 |
|
|
||||||
|
|
||||||
For other topologies, use CLI parallelism flags and update stage 0 `devices` with `--stage-overrides`. The number of listed devices must match `tensor_parallel_size * cfg_parallel_size`.
|
|
||||||
|
|
||||||
TP=2 with CFG parallel disabled:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
vllm serve GEAR-Dreams/DreamZero-DROID --omni --port 8000 \
|
|
||||||
--served-model-name dreamzero-droid \
|
|
||||||
--deploy-config vllm_omni/deploy/dreamzero.yaml \
|
|
||||||
--tensor-parallel-size 2 \
|
|
||||||
--stage-overrides '{"0": {"devices": "0,1"}}' \
|
|
||||||
--enforce-eager --disable-log-stats
|
|
||||||
```
|
|
||||||
|
|
||||||
TP=2 with CFG parallel size=2:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
vllm serve GEAR-Dreams/DreamZero-DROID --omni --port 8000 \
|
|
||||||
--served-model-name dreamzero-droid \
|
|
||||||
--deploy-config vllm_omni/deploy/dreamzero.yaml \
|
|
||||||
--tensor-parallel-size 2 \
|
|
||||||
--cfg-parallel-size 2 \
|
|
||||||
--stage-overrides '{"0": {"devices": "0,1,2,3"}}' \
|
|
||||||
--enforce-eager --disable-log-stats
|
|
||||||
```
|
|
||||||
|
|
||||||
### Download example assets
|
|
||||||
|
|
||||||
The OpenPI client and DROID sim-eval example expect the three camera MP4 files in `outputs/dreamzero/assets`.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
hf download YangshenDeng/vllm-omni-dreamzero-assets --repo-type dataset --local-dir outputs/dreamzero/assets
|
|
||||||
```
|
|
||||||
|
|
||||||
### Run the OpenPI client
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python examples/online_serving/dreamzero/openpi_client.py \
|
|
||||||
--host 127.0.0.1 \
|
|
||||||
--port 8000 \
|
|
||||||
--video-dir outputs/dreamzero/assets
|
|
||||||
```
|
|
||||||
|
|
||||||
This client uses downloaded example videos and talks to the OpenPI websocket server.
|
|
||||||
|
|
||||||
### Run DROID sim eval
|
|
||||||
|
|
||||||
```bash
|
|
||||||
${ISAACLAB_LAUNCHER} -p examples/online_serving/dreamzero/droid_sim_eval_client.py \
|
|
||||||
--host 127.0.0.1 \
|
|
||||||
--port 8000 \
|
|
||||||
--scene 1 \
|
|
||||||
--episodes 1 \
|
|
||||||
--headless \
|
|
||||||
--device cuda:0
|
|
||||||
```
|
|
||||||
|
|
||||||
Set `ISAACLAB_LAUNCHER=path/to/isaaclab.sh` from the vLLM-Omni repository root before running the command.
|
|
||||||
This launches Isaac Lab / sim-evals and runs the DROID benchmark loop against the same websocket endpoint.
|
|
||||||
|
|
||||||
### Export comparison videos offline
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python examples/offline_inference/dreamzero/export_prediction_video.py \
|
|
||||||
--deploy-config vllm_omni/deploy/dreamzero_tp1_cfg2.yaml \
|
|
||||||
--save-input-video \
|
|
||||||
--save-gif
|
|
||||||
```
|
|
||||||
|
|
||||||
The export script uses local `Omni` inference, not the websocket server. It writes the input rollout video and the predicted output video artifacts for side-by-side inspection.
|
|
||||||
|
|
||||||
### MolmoSpace demo
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python examples/online_serving/dreamzero/molmospace_dreamzero_eval_demo.py \
|
|
||||||
--host 127.0.0.1 \
|
|
||||||
--port 8000 \
|
|
||||||
--benchmark_dir /path/to/benchmark \
|
|
||||||
--output_dir /path/to/output
|
|
||||||
```
|
|
||||||
|
|
||||||
This demo adapts DreamZero to the MolmoSpace-style remote policy eval loop.
|
|
||||||
@@ -1,819 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
"""Run one or more DROID sim-eval rollouts against the vLLM DreamZero server.
|
|
||||||
|
|
||||||
This script is the vLLM/OpenPI adaptation of the upstream DreamZero
|
|
||||||
`eval_utils/run_sim_eval.py` sim-eval client.
|
|
||||||
|
|
||||||
Behavior intentionally kept close to upstream:
|
|
||||||
- same DROID observation extraction (`external_cam`, `external_cam_2`,
|
|
||||||
`wrist_cam`, joint position, gripper position)
|
|
||||||
- same resize-with-pad preprocessing to `(180, 320)`
|
|
||||||
- same `open_loop_horizon=8`
|
|
||||||
- same gripper binarization rule (`> 0.5 -> 1`, else `0`)
|
|
||||||
- same per-scene language prompts
|
|
||||||
|
|
||||||
Unlike upstream DreamZero, vLLM serves the compatible websocket policy endpoint
|
|
||||||
at `/v1/realtime/robot/openpi`, so this script includes the path suffix in the
|
|
||||||
client URI.
|
|
||||||
|
|
||||||
Run this script through Isaac Lab's launcher from the vLLM-Omni repository
|
|
||||||
root, for example:
|
|
||||||
|
|
||||||
"${ISAACLAB_LAUNCHER}" -p \
|
|
||||||
examples/online_serving/dreamzero/droid_sim_eval_client.py \
|
|
||||||
--host 127.0.0.1 \
|
|
||||||
--port 8000 \
|
|
||||||
--scene 1 \
|
|
||||||
--episodes 1 \
|
|
||||||
--headless \
|
|
||||||
--device cuda:1
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import json
|
|
||||||
import logging
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
import uuid
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from datetime import UTC, datetime
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
import torch
|
|
||||||
|
|
||||||
try:
|
|
||||||
import mediapy
|
|
||||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
|
||||||
raise ImportError("Optional DROID sim-eval client requires `mediapy`.") from exc
|
|
||||||
|
|
||||||
try:
|
|
||||||
from typing import override
|
|
||||||
except ImportError:
|
|
||||||
try:
|
|
||||||
from typing_extensions import override
|
|
||||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
|
||||||
raise ImportError("Optional DROID sim-eval client requires `typing-extensions` on Python < 3.12.") from exc
|
|
||||||
|
|
||||||
try:
|
|
||||||
import websockets.sync.client
|
|
||||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
|
||||||
raise ImportError("Optional DROID sim-eval client requires `websockets`.") from exc
|
|
||||||
|
|
||||||
# NOTE:
|
|
||||||
# This directory already contains a local file named `openpi_client.py`.
|
|
||||||
# However, what we want here is the *installed* `openpi_client` package from
|
|
||||||
# upstream OpenPI, not the sibling example file. When a script is executed
|
|
||||||
# directly, Python often puts the script directory at `sys.path[0]`, which
|
|
||||||
# would cause `import openpi_client` to resolve to the local example file and
|
|
||||||
# create a circular import.
|
|
||||||
#
|
|
||||||
# To avoid that ambiguity, temporarily remove the current example directory
|
|
||||||
# from the front of `sys.path`, import the real package, and then restore the
|
|
||||||
# path afterwards.
|
|
||||||
example_dir = str(Path(__file__).resolve().parent)
|
|
||||||
removed_path = False
|
|
||||||
if sys.path and sys.path[0] == example_dir:
|
|
||||||
sys.path.pop(0)
|
|
||||||
removed_path = True
|
|
||||||
try:
|
|
||||||
from openpi_client import image_tools, msgpack_numpy
|
|
||||||
from openpi_client.base_policy import BasePolicy
|
|
||||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
|
||||||
raise ImportError("Optional DROID sim-eval client requires the `openpi-client` package.") from exc
|
|
||||||
finally:
|
|
||||||
if removed_path:
|
|
||||||
sys.path.insert(0, example_dir)
|
|
||||||
|
|
||||||
# -----------------------------------------------------------------------------
|
|
||||||
# Constants
|
|
||||||
# -----------------------------------------------------------------------------
|
|
||||||
#
|
|
||||||
# These values intentionally mirror the upstream DreamZero sim-eval client
|
|
||||||
# where possible. The important distinction is:
|
|
||||||
#
|
|
||||||
# - ACTION_HORIZON = 24
|
|
||||||
# The model returns 24 future actions per inference call.
|
|
||||||
# - DEFAULT_OPEN_LOOP_HORIZON = 8
|
|
||||||
# The sim client only executes the first 8 actions locally before asking
|
|
||||||
# the server to replan from a fresh observation.
|
|
||||||
#
|
|
||||||
# So a single server call predicts 24x8 actions, but the rollout consumes only
|
|
||||||
# 8 of them before replanning.
|
|
||||||
PING_INTERVAL_SECS = 60
|
|
||||||
PING_TIMEOUT_SECS = 600
|
|
||||||
DEFAULT_PATH = "/v1/realtime/robot/openpi"
|
|
||||||
DEFAULT_OPEN_LOOP_HORIZON = 8
|
|
||||||
ACTION_HORIZON = 24
|
|
||||||
ACTION_DIM = 8
|
|
||||||
DEFAULT_OUTPUT_ROOT = Path("runs") / "dreamzero_sim_eval"
|
|
||||||
SCENE_PROMPTS = {
|
|
||||||
1: "put the cube in the bowl",
|
|
||||||
2: "pick up the can and put it in the mug",
|
|
||||||
3: "put the banana in the bin",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _decode_action_response(response: bytes | str) -> np.ndarray:
|
|
||||||
if isinstance(response, str):
|
|
||||||
raise RuntimeError(f"Error in inference server:\n{response}")
|
|
||||||
decoded = msgpack_numpy.unpackb(response)
|
|
||||||
if isinstance(decoded, dict) and decoded.get("type") == "error":
|
|
||||||
message = decoded.get("message", decoded)
|
|
||||||
raise RuntimeError(f"Error in inference server:\n{message}")
|
|
||||||
return np.asarray(decoded, dtype=np.float32)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class StepRecord:
|
|
||||||
"""One fully materialized rollout step for later JSON export.
|
|
||||||
|
|
||||||
The `episode_00.json` artifact is intended to be human-readable and
|
|
||||||
post-process-friendly. Instead of keeping raw tensors around, each step is
|
|
||||||
flattened into plain Python types so it can be serialized directly.
|
|
||||||
"""
|
|
||||||
|
|
||||||
# Index of this control step within the episode.
|
|
||||||
step_index: int
|
|
||||||
# Whether this step triggered a fresh model call (as opposed to reusing the
|
|
||||||
# cached open-loop chunk from the previous server response).
|
|
||||||
used_server_call: bool
|
|
||||||
# End-to-end latency of the server call that produced the current chunk.
|
|
||||||
# This is `None` on steps that only reuse cached actions.
|
|
||||||
chunk_latency_s: float | None
|
|
||||||
# The concrete 8-D action sent into the simulator at this step.
|
|
||||||
action: list[float]
|
|
||||||
# Observed 7-DoF arm joint positions before the next environment step.
|
|
||||||
joint_position: list[float]
|
|
||||||
# Observed gripper scalar before the next environment step.
|
|
||||||
gripper_position: list[float]
|
|
||||||
# Reward and termination signals directly returned by the simulator.
|
|
||||||
reward: float
|
|
||||||
terminated: bool
|
|
||||||
truncated: bool
|
|
||||||
# Optional scene object positions for downstream debugging / success
|
|
||||||
# heuristics. This may be empty if the environment does not expose them.
|
|
||||||
scene_objects: dict[str, list[float]]
|
|
||||||
|
|
||||||
|
|
||||||
class OpenPIWebsocketClientPolicy(BasePolicy):
|
|
||||||
"""Minimal websocket client for the DreamZero/OpenPI policy protocol.
|
|
||||||
|
|
||||||
Protocol shape:
|
|
||||||
- connect -> server immediately sends a metadata dict
|
|
||||||
- infer -> send msgpack observation, receive action chunk
|
|
||||||
- reset -> send msgpack reset command, receive confirmation string
|
|
||||||
|
|
||||||
This class intentionally stays very small because the more interesting
|
|
||||||
DreamZero-specific behavior lives one layer above, in
|
|
||||||
`DreamZeroJointPosClient`.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
host: str = "127.0.0.1",
|
|
||||||
port: int = 8000,
|
|
||||||
path: str = DEFAULT_PATH,
|
|
||||||
) -> None:
|
|
||||||
# vLLM serves the robot endpoint under `/v1/realtime/robot/openpi`.
|
|
||||||
self._uri = f"ws://{host}:{port}{path}"
|
|
||||||
# Upstream protocol uses msgpack with numpy support, not JSON.
|
|
||||||
self._packer = msgpack_numpy.Packer()
|
|
||||||
# Connect immediately and cache the server handshake metadata.
|
|
||||||
self._ws, self._server_metadata = self._wait_for_server()
|
|
||||||
|
|
||||||
def get_server_metadata(self) -> dict[str, Any]:
|
|
||||||
"""Return a copy of the server handshake metadata."""
|
|
||||||
|
|
||||||
return dict(self._server_metadata)
|
|
||||||
|
|
||||||
def _wait_for_server(self):
|
|
||||||
"""Connect to the websocket server and read the initial metadata frame."""
|
|
||||||
|
|
||||||
logging.info("Connecting to %s", self._uri)
|
|
||||||
conn = websockets.sync.client.connect(
|
|
||||||
self._uri,
|
|
||||||
compression=None,
|
|
||||||
max_size=None,
|
|
||||||
ping_interval=PING_INTERVAL_SECS,
|
|
||||||
ping_timeout=PING_TIMEOUT_SECS,
|
|
||||||
)
|
|
||||||
metadata = msgpack_numpy.unpackb(conn.recv())
|
|
||||||
if not isinstance(metadata, dict):
|
|
||||||
raise TypeError(f"Expected dict metadata from server, got {type(metadata)!r}")
|
|
||||||
return conn, metadata
|
|
||||||
|
|
||||||
@override
|
|
||||||
def infer(self, obs: dict[str, Any]) -> np.ndarray:
|
|
||||||
"""Send an inference request and return the decoded action chunk."""
|
|
||||||
|
|
||||||
# Keep the upstream DreamZero/OpenPI convention that the request itself
|
|
||||||
# tells the server which logical endpoint is being called.
|
|
||||||
payload = dict(obs)
|
|
||||||
payload["endpoint"] = "infer"
|
|
||||||
self._ws.send(self._packer.pack(payload))
|
|
||||||
response = self._ws.recv()
|
|
||||||
return _decode_action_response(response)
|
|
||||||
|
|
||||||
@override
|
|
||||||
def reset(self, reset_info: dict[str, Any] | None = None) -> str:
|
|
||||||
"""Tell the server to reset its session-side state."""
|
|
||||||
|
|
||||||
payload = dict(reset_info or {})
|
|
||||||
payload["endpoint"] = "reset"
|
|
||||||
self._ws.send(self._packer.pack(payload))
|
|
||||||
response = self._ws.recv()
|
|
||||||
if isinstance(response, str):
|
|
||||||
return response
|
|
||||||
decoded = msgpack_numpy.unpackb(response)
|
|
||||||
if not isinstance(decoded, dict) or decoded.get("status") != "reset successful":
|
|
||||||
raise RuntimeError(f"Unexpected reset response: {decoded!r}")
|
|
||||||
return str(decoded["status"])
|
|
||||||
|
|
||||||
def close(self) -> None:
|
|
||||||
"""Close the websocket connection explicitly."""
|
|
||||||
|
|
||||||
self._ws.close()
|
|
||||||
|
|
||||||
|
|
||||||
class DreamZeroJointPosClient:
|
|
||||||
"""DROID sim-eval client that talks to the vLLM OpenPI websocket server.
|
|
||||||
|
|
||||||
This is the main compatibility layer between:
|
|
||||||
- Isaac Lab DROID observations (`obs["policy"][...]`)
|
|
||||||
- DreamZero/OpenPI websocket payloads
|
|
||||||
- local open-loop action reuse across several simulator steps
|
|
||||||
|
|
||||||
In other words:
|
|
||||||
simulator obs -> websocket request -> action chunk -> one action per step
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
remote_host: str = "127.0.0.1",
|
|
||||||
remote_port: int = 8000,
|
|
||||||
path: str = DEFAULT_PATH,
|
|
||||||
open_loop_horizon: int = DEFAULT_OPEN_LOOP_HORIZON,
|
|
||||||
) -> None:
|
|
||||||
# Low-level transport client.
|
|
||||||
self.client = OpenPIWebsocketClientPolicy(remote_host, remote_port, path=path)
|
|
||||||
# Number of actions to execute locally before replanning.
|
|
||||||
self.open_loop_horizon = open_loop_horizon
|
|
||||||
# Cursor into the currently cached action chunk.
|
|
||||||
self.actions_from_chunk_completed = 0
|
|
||||||
# Most recent `(ACTION_HORIZON, ACTION_DIM)` server response.
|
|
||||||
self.pred_action_chunk: np.ndarray | None = None
|
|
||||||
# Session id is part of the DreamZero serving contract. Changing it
|
|
||||||
# causes the server side to treat the rollout as a fresh episode.
|
|
||||||
self.session_id = str(uuid.uuid4())
|
|
||||||
# Simple runtime stats for reporting.
|
|
||||||
self.server_calls = 0
|
|
||||||
self.last_chunk_latency_s: float | None = None
|
|
||||||
self.last_used_server_call = False
|
|
||||||
|
|
||||||
def metadata(self) -> dict[str, Any]:
|
|
||||||
"""Expose the server metadata to callers / logs."""
|
|
||||||
|
|
||||||
return self.client.get_server_metadata()
|
|
||||||
|
|
||||||
def reset(self) -> str:
|
|
||||||
"""Reset local chunk state and remote session state.
|
|
||||||
|
|
||||||
Local reset:
|
|
||||||
- drop cached action chunk
|
|
||||||
- rewind chunk cursor
|
|
||||||
- allocate a fresh session id
|
|
||||||
|
|
||||||
Remote reset:
|
|
||||||
- send a websocket `reset` message so the server can clear any
|
|
||||||
request/session-side state it associates with this client
|
|
||||||
"""
|
|
||||||
|
|
||||||
self.actions_from_chunk_completed = 0
|
|
||||||
self.pred_action_chunk = None
|
|
||||||
self.session_id = str(uuid.uuid4())
|
|
||||||
self.last_chunk_latency_s = None
|
|
||||||
self.last_used_server_call = False
|
|
||||||
return self.client.reset({})
|
|
||||||
|
|
||||||
def infer(self, obs: dict[str, Any], instruction: str) -> dict[str, Any]:
|
|
||||||
"""Turn one simulator observation into one executable 8-D action.
|
|
||||||
|
|
||||||
Key behavior:
|
|
||||||
- call the server only when the local chunk cache is empty/exhausted
|
|
||||||
- otherwise, keep consuming the cached chunk open-loop
|
|
||||||
- always return exactly one 8-D action for the current simulator step
|
|
||||||
"""
|
|
||||||
|
|
||||||
# Convert Isaac Lab observation structure into a plain numpy-friendly
|
|
||||||
# record that is easier to serialize and visualize.
|
|
||||||
curr_obs = self._extract_observation(obs)
|
|
||||||
self.last_used_server_call = False
|
|
||||||
|
|
||||||
# Replan if:
|
|
||||||
# 1. this is the first step of a rollout / chunk
|
|
||||||
# 2. we already consumed `open_loop_horizon` actions from the current chunk
|
|
||||||
# 3. no cached chunk is currently available
|
|
||||||
if (
|
|
||||||
self.actions_from_chunk_completed == 0
|
|
||||||
or self.actions_from_chunk_completed >= self.open_loop_horizon
|
|
||||||
or self.pred_action_chunk is None
|
|
||||||
):
|
|
||||||
self.actions_from_chunk_completed = 0
|
|
||||||
# Build the exact DreamZero/OpenPI payload expected by the server.
|
|
||||||
#
|
|
||||||
# Notes:
|
|
||||||
# - images are resized/padded to the serving contract's 180x320
|
|
||||||
# - proprio is cast to float64 to match upstream client behavior
|
|
||||||
# - cartesian_position is currently unused by DreamZero DROID, so
|
|
||||||
# a dummy zero vector is sent for protocol completeness
|
|
||||||
request_data = {
|
|
||||||
"observation/exterior_image_0_left": image_tools.resize_with_pad(curr_obs["right_image"], 180, 320),
|
|
||||||
"observation/exterior_image_1_left": image_tools.resize_with_pad(curr_obs["left_image"], 180, 320),
|
|
||||||
"observation/wrist_image_left": image_tools.resize_with_pad(curr_obs["wrist_image"], 180, 320),
|
|
||||||
"observation/joint_position": curr_obs["joint_position"].astype(np.float64),
|
|
||||||
"observation/cartesian_position": np.zeros((6,), dtype=np.float64),
|
|
||||||
"observation/gripper_position": curr_obs["gripper_position"].astype(np.float64),
|
|
||||||
"prompt": instruction,
|
|
||||||
"session_id": self.session_id,
|
|
||||||
}
|
|
||||||
|
|
||||||
# Measure end-to-end server latency for this chunk request.
|
|
||||||
start = time.perf_counter()
|
|
||||||
actions = self.client.infer(request_data)
|
|
||||||
self.last_chunk_latency_s = time.perf_counter() - start
|
|
||||||
self.last_used_server_call = True
|
|
||||||
self.server_calls += 1
|
|
||||||
|
|
||||||
# DreamZero DROID serving is expected to return an action chunk with
|
|
||||||
# 24 future actions, each action being 8-D.
|
|
||||||
if actions.ndim != 2:
|
|
||||||
raise AssertionError(f"Expected 2D action array, got shape {actions.shape}")
|
|
||||||
if actions.shape != (ACTION_HORIZON, ACTION_DIM):
|
|
||||||
raise AssertionError(f"Expected action shape {(ACTION_HORIZON, ACTION_DIM)}, got {actions.shape}")
|
|
||||||
self.pred_action_chunk = actions
|
|
||||||
|
|
||||||
# Consume exactly one action row from the cached chunk for this
|
|
||||||
# simulator step.
|
|
||||||
action = np.array(self.pred_action_chunk[self.actions_from_chunk_completed], copy=True)
|
|
||||||
self.actions_from_chunk_completed += 1
|
|
||||||
|
|
||||||
# Upstream DreamZero sim-eval binarizes the gripper command.
|
|
||||||
action[-1] = 1.0 if action[-1].item() > 0.5 else 0.0
|
|
||||||
|
|
||||||
# Produce a human-friendly visualization strip for videos:
|
|
||||||
# right external | wrist | left external
|
|
||||||
img1 = image_tools.resize_with_pad(curr_obs["right_image"], 224, 224)
|
|
||||||
img2 = image_tools.resize_with_pad(curr_obs["wrist_image"], 224, 224)
|
|
||||||
img3 = image_tools.resize_with_pad(curr_obs["left_image"], 224, 224)
|
|
||||||
viz = np.concatenate([img1, img2, img3], axis=1)
|
|
||||||
|
|
||||||
# Return both the executable action and auxiliary debug info.
|
|
||||||
return {
|
|
||||||
"action": action,
|
|
||||||
"viz": viz,
|
|
||||||
"joint_position": curr_obs["joint_position"],
|
|
||||||
"gripper_position": curr_obs["gripper_position"],
|
|
||||||
"used_server_call": self.last_used_server_call,
|
|
||||||
"chunk_latency_s": self.last_chunk_latency_s if self.last_used_server_call else None,
|
|
||||||
}
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _extract_observation(obs_dict: dict[str, Any]) -> dict[str, np.ndarray]:
|
|
||||||
"""Extract the pieces DreamZero cares about from Isaac Lab observations.
|
|
||||||
|
|
||||||
`sim-evals` exposes camera frames and robot state inside the
|
|
||||||
`obs["policy"]` group. This helper converts those tensors into numpy
|
|
||||||
arrays so they can be fed into image preprocessing / websocket packing.
|
|
||||||
"""
|
|
||||||
|
|
||||||
policy = obs_dict["policy"]
|
|
||||||
# Isaac Lab stores camera observations as batched tensors; use env 0.
|
|
||||||
right_image = policy["external_cam"][0].clone().detach().cpu().numpy()
|
|
||||||
left_image = policy["external_cam_2"][0].clone().detach().cpu().numpy()
|
|
||||||
wrist_image = policy["wrist_cam"][0].clone().detach().cpu().numpy()
|
|
||||||
# Robot proprioception.
|
|
||||||
joint_position = policy["arm_joint_pos"].clone().detach().cpu().numpy()
|
|
||||||
gripper_position = policy["gripper_pos"].clone().detach().cpu().numpy()
|
|
||||||
|
|
||||||
return {
|
|
||||||
"right_image": right_image,
|
|
||||||
"left_image": left_image,
|
|
||||||
"wrist_image": wrist_image,
|
|
||||||
"joint_position": joint_position,
|
|
||||||
"gripper_position": gripper_position,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _scene_instruction(scene: int) -> str:
|
|
||||||
"""Map a numeric scene id onto the fixed language prompt used for rollout."""
|
|
||||||
|
|
||||||
try:
|
|
||||||
return SCENE_PROMPTS[scene]
|
|
||||||
except KeyError as exc:
|
|
||||||
raise ValueError(f"Unsupported scene {scene}. Available scenes: {sorted(SCENE_PROMPTS)}") from exc
|
|
||||||
|
|
||||||
|
|
||||||
def _capture_scene_objects(env: Any) -> dict[str, list[float]]:
|
|
||||||
"""Best-effort extraction of scene object root positions.
|
|
||||||
|
|
||||||
This is only for debugging / reporting. The rollout logic itself does not
|
|
||||||
depend on these positions.
|
|
||||||
"""
|
|
||||||
|
|
||||||
objects: dict[str, list[float]] = {}
|
|
||||||
scene = getattr(env, "scene", None)
|
|
||||||
if scene is None:
|
|
||||||
return objects
|
|
||||||
|
|
||||||
# Skip non-task entities such as cameras, the robot, and lighting.
|
|
||||||
for name in scene.keys():
|
|
||||||
if name in {"robot", "external_cam", "external_cam_2", "wrist_cam", "sphere_light", "scene"}:
|
|
||||||
continue
|
|
||||||
entity = scene[name]
|
|
||||||
data = getattr(entity, "data", None)
|
|
||||||
root_pos_w = getattr(data, "root_pos_w", None)
|
|
||||||
if root_pos_w is None:
|
|
||||||
continue
|
|
||||||
# Convert tensors to plain lists for JSON serialization.
|
|
||||||
value = root_pos_w[0].detach().cpu().to(torch.float32).tolist()
|
|
||||||
objects[str(name)] = [float(x) for x in value]
|
|
||||||
return objects
|
|
||||||
|
|
||||||
|
|
||||||
def _maybe_infer_success(scene: int, final_objects: dict[str, list[float]]) -> dict[str, Any]:
|
|
||||||
"""Best-effort geometric heuristic.
|
|
||||||
|
|
||||||
The simulator itself does not expose a built-in success term; this function
|
|
||||||
provides a transparent fallback for human-readable reporting only.
|
|
||||||
"""
|
|
||||||
|
|
||||||
task_pairs = {
|
|
||||||
1: ("cube", "bowl"),
|
|
||||||
2: ("can", "mug"),
|
|
||||||
3: ("banana", "bin"),
|
|
||||||
}
|
|
||||||
source_name, target_name = task_pairs.get(scene, (None, None))
|
|
||||||
if source_name not in final_objects or target_name not in final_objects:
|
|
||||||
return {
|
|
||||||
"has_builtin_success": False,
|
|
||||||
"heuristic_success": None,
|
|
||||||
"reason": "scene object names unavailable for heuristic",
|
|
||||||
}
|
|
||||||
|
|
||||||
source = np.asarray(final_objects[source_name], dtype=np.float32)
|
|
||||||
target = np.asarray(final_objects[target_name], dtype=np.float32)
|
|
||||||
xy_distance = float(np.linalg.norm(source[:2] - target[:2]))
|
|
||||||
z_delta = float(source[2] - target[2])
|
|
||||||
heuristic_success = bool(xy_distance < 0.12 and z_delta > -0.08)
|
|
||||||
return {
|
|
||||||
"has_builtin_success": False,
|
|
||||||
"heuristic_success": heuristic_success,
|
|
||||||
"xy_distance": xy_distance,
|
|
||||||
"z_delta": z_delta,
|
|
||||||
"source_object": source_name,
|
|
||||||
"target_object": target_name,
|
|
||||||
"reason": "no built-in env success flag; using final object pose heuristic",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _dump_json(path: Path, payload: dict[str, Any]) -> None:
|
|
||||||
"""Write a JSON file with stable UTF-8 formatting."""
|
|
||||||
|
|
||||||
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
||||||
|
|
||||||
|
|
||||||
def _scalar_from_env_value(value: Any) -> float:
|
|
||||||
"""Normalize simulator scalar outputs into a plain Python float.
|
|
||||||
|
|
||||||
Isaac Lab / Gym values may come back as tensors, numpy arrays, tuples, or
|
|
||||||
direct Python scalars depending on the wrapper stack. Centralizing the
|
|
||||||
conversion here makes the rollout loop cleaner and more robust.
|
|
||||||
"""
|
|
||||||
|
|
||||||
if isinstance(value, torch.Tensor):
|
|
||||||
return float(value.reshape(-1)[0].detach().cpu().item())
|
|
||||||
if isinstance(value, np.ndarray):
|
|
||||||
return float(value.reshape(-1)[0])
|
|
||||||
if isinstance(value, (list, tuple)):
|
|
||||||
return float(np.asarray(value).reshape(-1)[0])
|
|
||||||
return float(value)
|
|
||||||
|
|
||||||
|
|
||||||
def _bool_from_env_value(value: Any) -> bool:
|
|
||||||
"""Normalize simulator boolean-like outputs into a plain Python bool."""
|
|
||||||
|
|
||||||
if isinstance(value, torch.Tensor):
|
|
||||||
return bool(value.reshape(-1)[0].detach().cpu().item())
|
|
||||||
if isinstance(value, np.ndarray):
|
|
||||||
return bool(value.reshape(-1)[0])
|
|
||||||
if isinstance(value, (list, tuple)):
|
|
||||||
return bool(np.asarray(value).reshape(-1)[0])
|
|
||||||
return bool(value)
|
|
||||||
|
|
||||||
|
|
||||||
def _make_output_dir(output_root: Path, scene: int) -> Path:
|
|
||||||
"""Create a timestamped output directory for one sim-eval run."""
|
|
||||||
|
|
||||||
timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
|
||||||
output_dir = output_root / f"scene{scene}_{timestamp}"
|
|
||||||
output_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
return output_dir
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
"""Entry point for one or more DROID simulation rollouts.
|
|
||||||
|
|
||||||
High-level flow:
|
|
||||||
1. parse command-line flags
|
|
||||||
2. bootstrap Isaac Lab / sim-evals imports
|
|
||||||
3. create the DROID environment
|
|
||||||
4. connect the DreamZero websocket client
|
|
||||||
5. run `episodes` rollouts
|
|
||||||
6. export videos + JSON summaries
|
|
||||||
"""
|
|
||||||
|
|
||||||
# Script-level arguments.
|
|
||||||
parser = argparse.ArgumentParser(description=__doc__)
|
|
||||||
parser.add_argument("--episodes", type=int, default=1, help="Number of episodes to run.")
|
|
||||||
parser.add_argument("--scene", type=int, default=1, help="DROID scene id (1/2/3).")
|
|
||||||
parser.add_argument("--host", type=str, default="127.0.0.1", help="vLLM DreamZero server host.")
|
|
||||||
parser.add_argument("--port", type=int, default=8000, help="vLLM DreamZero server port.")
|
|
||||||
parser.add_argument("--path", type=str, default=DEFAULT_PATH, help="Websocket path suffix.")
|
|
||||||
parser.add_argument(
|
|
||||||
"--open-loop-horizon",
|
|
||||||
type=int,
|
|
||||||
default=DEFAULT_OPEN_LOOP_HORIZON,
|
|
||||||
help="How many actions to consume locally before requesting the next chunk.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--output-root",
|
|
||||||
type=Path,
|
|
||||||
default=DEFAULT_OUTPUT_ROOT,
|
|
||||||
help="Directory where videos and trajectory logs are stored.",
|
|
||||||
)
|
|
||||||
|
|
||||||
try:
|
|
||||||
from isaaclab.app import AppLauncher
|
|
||||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
|
||||||
raise ImportError(
|
|
||||||
"Optional DROID sim-eval client requires Isaac Lab (`isaaclab`). "
|
|
||||||
"Launch it from an Isaac Lab environment, e.g. via `isaaclab.sh -p`."
|
|
||||||
) from exc
|
|
||||||
|
|
||||||
# Let Isaac Lab inject its own runtime flags (e.g. headless, device).
|
|
||||||
AppLauncher.add_app_launcher_args(parser)
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
# DreamZero sim-eval always needs camera observations enabled.
|
|
||||||
args.enable_cameras = True
|
|
||||||
# Boot Isaac Sim / Isaac Lab.
|
|
||||||
app_launcher = AppLauncher(args)
|
|
||||||
simulation_app = app_launcher.app
|
|
||||||
# Set defaults so the `finally` block can clean up safely even if an
|
|
||||||
# earlier step fails.
|
|
||||||
env = None
|
|
||||||
client = None
|
|
||||||
|
|
||||||
# Import simulator modules only *after* the app is launched. This matches
|
|
||||||
# Isaac Lab's required import ordering.
|
|
||||||
try:
|
|
||||||
import gymnasium as gym
|
|
||||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
|
||||||
raise ImportError("Optional DROID sim-eval client requires `gymnasium`.") from exc
|
|
||||||
|
|
||||||
try:
|
|
||||||
import sim_evals.environments # noqa: F401
|
|
||||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
|
||||||
raise ImportError(
|
|
||||||
"Optional DROID sim-eval client requires the external `sim-evals` package or checkout to be importable."
|
|
||||||
) from exc
|
|
||||||
|
|
||||||
try:
|
|
||||||
from isaaclab_tasks.utils import parse_env_cfg
|
|
||||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
|
||||||
raise ImportError("Optional DROID sim-eval client requires `isaaclab_tasks`.") from exc
|
|
||||||
|
|
||||||
# Resolve output location and scene prompt.
|
|
||||||
output_dir = _make_output_dir(args.output_root.expanduser().resolve(), args.scene)
|
|
||||||
instruction = _scene_instruction(args.scene)
|
|
||||||
|
|
||||||
# Build the DROID environment configuration from `sim-evals`.
|
|
||||||
env_cfg = parse_env_cfg(
|
|
||||||
"DROID",
|
|
||||||
device=args.device,
|
|
||||||
num_envs=1,
|
|
||||||
use_fabric=True,
|
|
||||||
)
|
|
||||||
# Select one of the pre-authored scenes/tasks.
|
|
||||||
env_cfg.set_scene(args.scene)
|
|
||||||
env = gym.make("DROID", cfg=env_cfg)
|
|
||||||
|
|
||||||
# Upstream sim-evals resets twice so materials / cameras are fully ready.
|
|
||||||
obs, _ = env.reset()
|
|
||||||
obs, _ = env.reset()
|
|
||||||
|
|
||||||
# Connect the websocket policy client.
|
|
||||||
client = DreamZeroJointPosClient(
|
|
||||||
remote_host=args.host,
|
|
||||||
remote_port=args.port,
|
|
||||||
path=args.path,
|
|
||||||
open_loop_horizon=args.open_loop_horizon,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Aggregated per-run results.
|
|
||||||
all_episode_summaries: list[dict[str, Any]] = []
|
|
||||||
max_steps = int(env.env.max_episode_length)
|
|
||||||
logging.info("DreamZero metadata: %s", client.metadata())
|
|
||||||
logging.info("Scene %s prompt: %s", args.scene, instruction)
|
|
||||||
logging.info("Writing outputs to %s", output_dir)
|
|
||||||
|
|
||||||
try:
|
|
||||||
# No gradients are needed in inference-only rollout mode.
|
|
||||||
with torch.no_grad():
|
|
||||||
for episode_index in range(args.episodes):
|
|
||||||
# Per-episode collectors.
|
|
||||||
frames: list[np.ndarray] = []
|
|
||||||
step_records: list[StepRecord] = []
|
|
||||||
episode_start = time.perf_counter()
|
|
||||||
server_time_s = 0.0
|
|
||||||
final_reward = 0.0
|
|
||||||
terminated = False
|
|
||||||
truncated = False
|
|
||||||
|
|
||||||
for step_index in range(max_steps):
|
|
||||||
# Ask the policy for the next action. Internally this may or
|
|
||||||
# may not trigger a real server request depending on whether
|
|
||||||
# the local chunk cache has been exhausted.
|
|
||||||
logging.debug("Episode %d step %d: requesting action", episode_index, step_index)
|
|
||||||
result = client.infer(obs, instruction)
|
|
||||||
logging.debug(
|
|
||||||
"Episode %d step %d: got action (server_call=%s latency=%s)",
|
|
||||||
episode_index,
|
|
||||||
step_index,
|
|
||||||
result["used_server_call"],
|
|
||||||
result["chunk_latency_s"],
|
|
||||||
)
|
|
||||||
# Save one visualization frame per simulator step.
|
|
||||||
frames.append(result["viz"])
|
|
||||||
|
|
||||||
# Isaac Lab expects batched actions, hence `[None]`.
|
|
||||||
action_tensor = torch.tensor(result["action"], dtype=torch.float32)[None]
|
|
||||||
logging.debug("Episode %d step %d: stepping env", episode_index, step_index)
|
|
||||||
obs, reward, term, trunc, info = env.step(action_tensor)
|
|
||||||
logging.debug("Episode %d step %d: env.step returned", episode_index, step_index)
|
|
||||||
logging.debug("Episode %d step %d: parsing reward/flags", episode_index, step_index)
|
|
||||||
logging.debug(
|
|
||||||
"Episode %d step %d: raw types reward=%s term=%s trunc=%s",
|
|
||||||
episode_index,
|
|
||||||
step_index,
|
|
||||||
type(reward).__name__,
|
|
||||||
type(term).__name__,
|
|
||||||
type(trunc).__name__,
|
|
||||||
)
|
|
||||||
# Normalize environment outputs into plain Python scalars so
|
|
||||||
# the rest of the code does not depend on wrapper-specific types.
|
|
||||||
reward_value = _scalar_from_env_value(reward)
|
|
||||||
term_value = _bool_from_env_value(term)
|
|
||||||
trunc_value = _bool_from_env_value(trunc)
|
|
||||||
logging.debug(
|
|
||||||
"Episode %d step %d: parsed reward=%s term=%s trunc=%s",
|
|
||||||
episode_index,
|
|
||||||
step_index,
|
|
||||||
reward_value,
|
|
||||||
term_value,
|
|
||||||
trunc_value,
|
|
||||||
)
|
|
||||||
# Keep scene-object capture optional. It is useful for
|
|
||||||
# debugging / success heuristics, but the rollout should not
|
|
||||||
# fail if the environment does not expose object roots.
|
|
||||||
scene_objects = _capture_scene_objects(env)
|
|
||||||
|
|
||||||
# Accumulate total server-side time only on steps that
|
|
||||||
# triggered a fresh chunk inference.
|
|
||||||
if result["chunk_latency_s"] is not None:
|
|
||||||
server_time_s += float(result["chunk_latency_s"])
|
|
||||||
|
|
||||||
# Materialize one JSON-serializable trajectory record.
|
|
||||||
logging.debug("Episode %d step %d: appending trajectory", episode_index, step_index)
|
|
||||||
step_records.append(
|
|
||||||
StepRecord(
|
|
||||||
step_index=step_index,
|
|
||||||
used_server_call=bool(result["used_server_call"]),
|
|
||||||
chunk_latency_s=(
|
|
||||||
float(result["chunk_latency_s"]) if result["chunk_latency_s"] is not None else None
|
|
||||||
),
|
|
||||||
action=[float(x) for x in np.asarray(result["action"], dtype=np.float32).tolist()],
|
|
||||||
joint_position=[
|
|
||||||
float(x) for x in np.asarray(result["joint_position"], dtype=np.float32).tolist()
|
|
||||||
],
|
|
||||||
gripper_position=[
|
|
||||||
float(x) for x in np.asarray(result["gripper_position"], dtype=np.float32).tolist()
|
|
||||||
],
|
|
||||||
reward=reward_value,
|
|
||||||
terminated=term_value,
|
|
||||||
truncated=trunc_value,
|
|
||||||
scene_objects=scene_objects,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
logging.debug("Episode %d step %d: trajectory appended", episode_index, step_index)
|
|
||||||
|
|
||||||
# Track final status for summary export.
|
|
||||||
final_reward = reward_value
|
|
||||||
terminated = term_value
|
|
||||||
truncated = trunc_value
|
|
||||||
if term_value or trunc_value:
|
|
||||||
# End the rollout early if the environment terminates.
|
|
||||||
break
|
|
||||||
|
|
||||||
# Episode-level timing and video export.
|
|
||||||
episode_wall_time_s = time.perf_counter() - episode_start
|
|
||||||
video_path = output_dir / f"episode_{episode_index:02d}.mp4"
|
|
||||||
logging.info("Episode %d: writing video to %s", episode_index, video_path)
|
|
||||||
mediapy.write_video(video_path, frames, fps=15)
|
|
||||||
|
|
||||||
# Reset the policy server between episodes.
|
|
||||||
logging.info("Episode %d: sending reset", episode_index)
|
|
||||||
reset_response = client.reset()
|
|
||||||
final_objects = step_records[-1].scene_objects if step_records else {}
|
|
||||||
success_report = _maybe_infer_success(args.scene, final_objects)
|
|
||||||
|
|
||||||
# Assemble the per-episode summary that is written to
|
|
||||||
# `episode_XX.json`.
|
|
||||||
episode_summary = {
|
|
||||||
"episode_index": episode_index,
|
|
||||||
"prompt": instruction,
|
|
||||||
"video_path": str(video_path),
|
|
||||||
"steps_executed": len(step_records),
|
|
||||||
"max_steps": max_steps,
|
|
||||||
"terminated": terminated,
|
|
||||||
"truncated": truncated,
|
|
||||||
"final_reward": final_reward,
|
|
||||||
"server_calls": client.server_calls,
|
|
||||||
"server_time_s": server_time_s,
|
|
||||||
"episode_wall_time_s": episode_wall_time_s,
|
|
||||||
"avg_server_time_per_call_s": (
|
|
||||||
server_time_s / client.server_calls if client.server_calls else None
|
|
||||||
),
|
|
||||||
"reset_response": reset_response,
|
|
||||||
"success_report": success_report,
|
|
||||||
"server_metadata": client.metadata(),
|
|
||||||
"trajectory": [record.__dict__ for record in step_records],
|
|
||||||
}
|
|
||||||
_dump_json(output_dir / f"episode_{episode_index:02d}.json", episode_summary)
|
|
||||||
all_episode_summaries.append(episode_summary)
|
|
||||||
|
|
||||||
logging.info(
|
|
||||||
"Episode %d done: steps=%d wall=%.2fs server_calls=%d heuristic_success=%s",
|
|
||||||
episode_index,
|
|
||||||
len(step_records),
|
|
||||||
episode_wall_time_s,
|
|
||||||
client.server_calls,
|
|
||||||
success_report.get("heuristic_success"),
|
|
||||||
)
|
|
||||||
|
|
||||||
# Reset per-episode counters while keeping the client alive.
|
|
||||||
client.server_calls = 0
|
|
||||||
|
|
||||||
# Top-level run summary across all episodes.
|
|
||||||
summary = {
|
|
||||||
"scene": args.scene,
|
|
||||||
"prompt": instruction,
|
|
||||||
"episodes": args.episodes,
|
|
||||||
"host": args.host,
|
|
||||||
"port": args.port,
|
|
||||||
"path": args.path,
|
|
||||||
"device": args.device,
|
|
||||||
"output_dir": str(output_dir),
|
|
||||||
"server_metadata": client.metadata(),
|
|
||||||
"episodes_summary": all_episode_summaries,
|
|
||||||
}
|
|
||||||
_dump_json(output_dir / "summary.json", summary)
|
|
||||||
# Also print the summary to stdout so the caller can capture it in logs.
|
|
||||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
|
||||||
finally:
|
|
||||||
# Best-effort cleanup. Avoid masking the main error if cleanup fails.
|
|
||||||
try:
|
|
||||||
if client is not None:
|
|
||||||
client.client.close()
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
if env is not None:
|
|
||||||
env.close()
|
|
||||||
simulation_app.close()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
# Keep the script-level logs readable. Per-step rollout details are still
|
|
||||||
# available via `DEBUG` if needed, but websocket / asyncio internals are
|
|
||||||
# usually too noisy for normal usage.
|
|
||||||
logging.basicConfig(level=logging.INFO)
|
|
||||||
logging.getLogger("websockets").setLevel(logging.WARNING)
|
|
||||||
logging.getLogger("asyncio").setLevel(logging.WARNING)
|
|
||||||
main()
|
|
||||||
@@ -1,86 +0,0 @@
|
|||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
os.environ.setdefault("MUJOCO_GL", "egl")
|
|
||||||
os.environ.setdefault("PYOPENGL_PLATFORM", "egl")
|
|
||||||
|
|
||||||
# Import base configs at module top level so the subclasses below are pickle-
|
|
||||||
# resolvable (worker processes import this module fresh via __main__).
|
|
||||||
from molmo_spaces.configs.policy_configs_baselines import ( # noqa: E402
|
|
||||||
DreamZeroPolicyConfig,
|
|
||||||
)
|
|
||||||
from molmo_spaces.evaluation.configs.evaluation_configs import ( # noqa: E402
|
|
||||||
DreamZeroPolicyEvalConfig,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
# We only need to change the backend host and port to the vllm-host!
|
|
||||||
class DreamZeroVllmOmniPolicyConfig(DreamZeroPolicyConfig):
|
|
||||||
remote_config: dict = dict(host="127.0.0.1", port=8000)
|
|
||||||
|
|
||||||
|
|
||||||
class DreamZeroVllmOmniEvalConfig(DreamZeroPolicyEvalConfig):
|
|
||||||
policy_config: DreamZeroVllmOmniPolicyConfig = DreamZeroVllmOmniPolicyConfig()
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
parser = argparse.ArgumentParser()
|
|
||||||
parser.add_argument("--host", default="127.0.0.1")
|
|
||||||
parser.add_argument("--port", type=int, default=8000)
|
|
||||||
parser.add_argument(
|
|
||||||
"--benchmark_dir",
|
|
||||||
required=True,
|
|
||||||
help=(
|
|
||||||
"Path to a MolmoSpaces benchmark directory, for example "
|
|
||||||
"$MOLMOSPACES_BENCHMARK_DIR/20260327/ithor/FrankaCloseHardBench/"
|
|
||||||
"FrankaCloseHardBench_20260206_json_benchmark"
|
|
||||||
),
|
|
||||||
)
|
|
||||||
parser.add_argument("--max_episodes", type=int, default=1)
|
|
||||||
parser.add_argument("--task_horizon_steps", type=int, default=80)
|
|
||||||
parser.add_argument(
|
|
||||||
"--output_dir",
|
|
||||||
required=True,
|
|
||||||
help="Directory to write evaluation outputs (created if missing).",
|
|
||||||
)
|
|
||||||
parser.add_argument("--episode_idx", type=int, default=None)
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
policy_config = DreamZeroVllmOmniPolicyConfig(remote_config=dict(host=args.host, port=args.port))
|
|
||||||
DreamZeroVllmOmniPolicyConfig.model_fields["remote_config"].default = policy_config.remote_config
|
|
||||||
DreamZeroVllmOmniEvalConfig.model_fields["policy_config"].default = policy_config
|
|
||||||
|
|
||||||
# Import after env vars are set so MuJoCo picks EGL.
|
|
||||||
from molmo_spaces.evaluation import run_evaluation
|
|
||||||
|
|
||||||
cfg_cls = DreamZeroVllmOmniEvalConfig
|
|
||||||
|
|
||||||
output_dir = args.output_dir
|
|
||||||
Path(output_dir).mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
print(f"[eval] benchmark_dir={args.benchmark_dir}")
|
|
||||||
print(f"[eval] max_episodes={args.max_episodes} task_horizon_steps={args.task_horizon_steps}")
|
|
||||||
print(f"[eval] remote policy: ws://{args.host}:{args.port}/v1/realtime/robot/openpi")
|
|
||||||
|
|
||||||
results = run_evaluation(
|
|
||||||
eval_config_cls=cfg_cls,
|
|
||||||
benchmark_dir=Path(args.benchmark_dir),
|
|
||||||
max_episodes=args.max_episodes,
|
|
||||||
task_horizon_steps=args.task_horizon_steps,
|
|
||||||
num_workers=1,
|
|
||||||
use_wandb=False,
|
|
||||||
output_dir=output_dir,
|
|
||||||
episode_idx=args.episode_idx,
|
|
||||||
)
|
|
||||||
|
|
||||||
print(f"[eval] success={results.success_count}/{results.total_count} ({results.success_rate:.1%})")
|
|
||||||
print(f"[eval] output_dir={results.output_dir}")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
@@ -1,311 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import json
|
|
||||||
import logging
|
|
||||||
import sys
|
|
||||||
import uuid
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
try:
|
|
||||||
import cv2
|
|
||||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
|
||||||
raise ImportError("DreamZero OpenPI example requires `opencv-python`.") from exc
|
|
||||||
|
|
||||||
try:
|
|
||||||
example_dir = str(Path(__file__).resolve().parent)
|
|
||||||
removed_path = False
|
|
||||||
if sys.path and sys.path[0] == example_dir:
|
|
||||||
sys.path.pop(0)
|
|
||||||
removed_path = True
|
|
||||||
try:
|
|
||||||
from openpi_client.websocket_client_policy import WebsocketClientPolicy
|
|
||||||
finally:
|
|
||||||
if removed_path:
|
|
||||||
sys.path.insert(0, example_dir)
|
|
||||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
|
||||||
raise ImportError("DreamZero OpenPI example requires `openpi-client`.") from exc
|
|
||||||
|
|
||||||
DEFAULT_HOST = "127.0.0.1"
|
|
||||||
DEFAULT_PORT = 8000
|
|
||||||
DEFAULT_PATH = "/v1/realtime/robot/openpi"
|
|
||||||
DEFAULT_PROMPT = "Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan"
|
|
||||||
ACTION_HORIZON = 24
|
|
||||||
DEFAULT_ACTION_DIM = 8
|
|
||||||
RELATIVE_OFFSETS = [-23, -16, -8, 0]
|
|
||||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
|
||||||
ASSET_REPO_ID = "YangshenDeng/vllm-omni-dreamzero-assets"
|
|
||||||
DEFAULT_VIDEO_DIR = REPO_ROOT / "outputs" / "dreamzero" / "assets"
|
|
||||||
CAMERA_FILES = {
|
|
||||||
"observation/exterior_image_0_left": "exterior_image_1_left.mp4",
|
|
||||||
"observation/exterior_image_1_left": "exterior_image_2_left.mp4",
|
|
||||||
"observation/wrist_image_left": "wrist_image_left.mp4",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _as_action_array(response: Any) -> np.ndarray:
|
|
||||||
if isinstance(response, dict) and response.get("type") == "error":
|
|
||||||
message = response.get("message", response)
|
|
||||||
raise RuntimeError(f"Inference failed: {message}")
|
|
||||||
return np.asarray(response, dtype=np.float32)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class DreamZeroServerMetadata:
|
|
||||||
image_resolution: tuple[int, int]
|
|
||||||
n_external_cameras: int
|
|
||||||
needs_wrist_camera: bool
|
|
||||||
needs_stereo_camera: bool
|
|
||||||
needs_session_id: bool
|
|
||||||
action_space: str
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_dict(cls, payload: dict[str, Any]) -> DreamZeroServerMetadata:
|
|
||||||
required_keys = (
|
|
||||||
"image_resolution",
|
|
||||||
"n_external_cameras",
|
|
||||||
"needs_wrist_camera",
|
|
||||||
"needs_stereo_camera",
|
|
||||||
"needs_session_id",
|
|
||||||
"action_space",
|
|
||||||
)
|
|
||||||
missing_keys = [key for key in required_keys if key not in payload]
|
|
||||||
if missing_keys:
|
|
||||||
raise ValueError(f"Missing DreamZero metadata keys: {missing_keys}")
|
|
||||||
|
|
||||||
image_resolution = payload["image_resolution"]
|
|
||||||
if not isinstance(image_resolution, (list, tuple)) or len(image_resolution) != 2:
|
|
||||||
raise ValueError(f"Invalid image_resolution: {image_resolution!r}")
|
|
||||||
|
|
||||||
return cls(
|
|
||||||
image_resolution=(int(image_resolution[0]), int(image_resolution[1])),
|
|
||||||
n_external_cameras=int(payload["n_external_cameras"]),
|
|
||||||
needs_wrist_camera=bool(payload["needs_wrist_camera"]),
|
|
||||||
needs_stereo_camera=bool(payload["needs_stereo_camera"]),
|
|
||||||
needs_session_id=bool(payload["needs_session_id"]),
|
|
||||||
action_space=str(payload["action_space"]),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _openpi_uri(host: str, port: int, path: str) -> str:
|
|
||||||
if host.startswith(("ws://", "wss://")):
|
|
||||||
return host
|
|
||||||
return f"ws://{host}:{port}{path}"
|
|
||||||
|
|
||||||
|
|
||||||
def _close_policy_client(client: WebsocketClientPolicy) -> None:
|
|
||||||
ws = getattr(client, "_ws", None)
|
|
||||||
if ws is not None:
|
|
||||||
ws.close()
|
|
||||||
|
|
||||||
|
|
||||||
def load_all_frames(video_path: Path) -> np.ndarray:
|
|
||||||
cap = cv2.VideoCapture(str(video_path))
|
|
||||||
frames = []
|
|
||||||
while True:
|
|
||||||
ok, frame = cap.read()
|
|
||||||
if not ok:
|
|
||||||
break
|
|
||||||
frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
|
|
||||||
cap.release()
|
|
||||||
if not frames:
|
|
||||||
raise RuntimeError(f"No frames loaded from {video_path}")
|
|
||||||
return np.stack(frames, axis=0)
|
|
||||||
|
|
||||||
|
|
||||||
def load_camera_frames(video_dir: Path) -> dict[str, np.ndarray]:
|
|
||||||
camera_frames: dict[str, np.ndarray] = {}
|
|
||||||
for camera_key, file_name in CAMERA_FILES.items():
|
|
||||||
video_path = video_dir / file_name
|
|
||||||
if not video_path.exists():
|
|
||||||
raise FileNotFoundError(
|
|
||||||
f"Missing DreamZero example asset: {video_path}. "
|
|
||||||
"Download the example videos with: "
|
|
||||||
f"`hf download {ASSET_REPO_ID} --repo-type dataset --local-dir {video_dir}`"
|
|
||||||
)
|
|
||||||
camera_frames[camera_key] = load_all_frames(video_path)
|
|
||||||
return camera_frames
|
|
||||||
|
|
||||||
|
|
||||||
def build_frame_schedule(total_frames: int, num_chunks: int) -> list[list[int]]:
|
|
||||||
chunks: list[list[int]] = []
|
|
||||||
current_frame = 23
|
|
||||||
for _ in range(num_chunks):
|
|
||||||
indices = [max(current_frame + offset, 0) for offset in RELATIVE_OFFSETS]
|
|
||||||
if indices[-1] >= total_frames:
|
|
||||||
break
|
|
||||||
chunks.append(indices)
|
|
||||||
current_frame += ACTION_HORIZON
|
|
||||||
return chunks
|
|
||||||
|
|
||||||
|
|
||||||
def make_obs_from_video(
|
|
||||||
camera_frames: dict[str, np.ndarray],
|
|
||||||
frame_indices: list[int],
|
|
||||||
*,
|
|
||||||
prompt: str,
|
|
||||||
session_id: str,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
obs: dict[str, Any] = {}
|
|
||||||
for camera_key, all_frames in camera_frames.items():
|
|
||||||
selected = all_frames[frame_indices]
|
|
||||||
obs[camera_key] = selected[0] if len(frame_indices) == 1 else selected
|
|
||||||
|
|
||||||
obs["observation/joint_position"] = np.zeros(7, dtype=np.float32)
|
|
||||||
obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32)
|
|
||||||
obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32)
|
|
||||||
obs["prompt"] = prompt
|
|
||||||
obs["session_id"] = session_id
|
|
||||||
return obs
|
|
||||||
|
|
||||||
|
|
||||||
def build_demo_observations(
|
|
||||||
camera_frames: dict[str, np.ndarray],
|
|
||||||
*,
|
|
||||||
prompt: str,
|
|
||||||
session_id: str,
|
|
||||||
num_chunks: int = 2,
|
|
||||||
) -> list[dict[str, Any]]:
|
|
||||||
if num_chunks < 1:
|
|
||||||
raise ValueError("num_chunks must be at least 1")
|
|
||||||
|
|
||||||
total_frames = min(frames.shape[0] for frames in camera_frames.values())
|
|
||||||
observations = [
|
|
||||||
make_obs_from_video(
|
|
||||||
camera_frames,
|
|
||||||
[0],
|
|
||||||
prompt=prompt,
|
|
||||||
session_id=session_id,
|
|
||||||
)
|
|
||||||
]
|
|
||||||
for indices in build_frame_schedule(total_frames, num_chunks - 1):
|
|
||||||
observations.append(
|
|
||||||
make_obs_from_video(
|
|
||||||
camera_frames,
|
|
||||||
indices,
|
|
||||||
prompt=prompt,
|
|
||||||
session_id=session_id,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return observations
|
|
||||||
|
|
||||||
|
|
||||||
def validate_session_result(
|
|
||||||
result: dict[str, Any],
|
|
||||||
*,
|
|
||||||
expected_action_horizon: int = ACTION_HORIZON,
|
|
||||||
expected_action_dim: int = DEFAULT_ACTION_DIM,
|
|
||||||
expected_action_count: int | None = None,
|
|
||||||
) -> None:
|
|
||||||
metadata = DreamZeroServerMetadata.from_dict(result["metadata"])
|
|
||||||
if metadata.image_resolution != (180, 320):
|
|
||||||
raise AssertionError(f"Unexpected image_resolution: {metadata.image_resolution}")
|
|
||||||
if metadata.n_external_cameras != 2:
|
|
||||||
raise AssertionError(f"Unexpected n_external_cameras: {metadata.n_external_cameras}")
|
|
||||||
if not metadata.needs_wrist_camera:
|
|
||||||
raise AssertionError("DreamZero example expects wrist camera metadata")
|
|
||||||
if metadata.action_space != "joint_position":
|
|
||||||
raise AssertionError(f"Unexpected action_space: {metadata.action_space}")
|
|
||||||
|
|
||||||
actions = result["actions"]
|
|
||||||
if expected_action_count is not None and len(actions) != expected_action_count:
|
|
||||||
raise AssertionError(f"Expected {expected_action_count} action tensors, got {len(actions)}")
|
|
||||||
if not actions:
|
|
||||||
raise AssertionError("Expected at least one action tensor")
|
|
||||||
for index, action in enumerate(actions):
|
|
||||||
if action.shape != (expected_action_horizon, expected_action_dim):
|
|
||||||
raise AssertionError(
|
|
||||||
f"Action {index} shape mismatch: expected "
|
|
||||||
f"{(expected_action_horizon, expected_action_dim)}, got {action.shape}"
|
|
||||||
)
|
|
||||||
if not np.isfinite(action).all():
|
|
||||||
raise AssertionError(f"Action {index} contains non-finite values")
|
|
||||||
|
|
||||||
|
|
||||||
def run_policy_session(
|
|
||||||
*,
|
|
||||||
host: str = DEFAULT_HOST,
|
|
||||||
port: int = DEFAULT_PORT,
|
|
||||||
path: str = DEFAULT_PATH,
|
|
||||||
video_dir: Path = DEFAULT_VIDEO_DIR,
|
|
||||||
prompt: str = DEFAULT_PROMPT,
|
|
||||||
session_id: str | None = None,
|
|
||||||
num_chunks: int = 2,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
session_id = session_id or str(uuid.uuid4())
|
|
||||||
camera_frames = load_camera_frames(video_dir)
|
|
||||||
observations = build_demo_observations(
|
|
||||||
camera_frames,
|
|
||||||
prompt=prompt,
|
|
||||||
session_id=session_id,
|
|
||||||
num_chunks=num_chunks,
|
|
||||||
)
|
|
||||||
|
|
||||||
uri = _openpi_uri(host, port, path)
|
|
||||||
logging.info("Connecting to %s", uri)
|
|
||||||
client = WebsocketClientPolicy(host=uri)
|
|
||||||
try:
|
|
||||||
metadata = dict(client.get_server_metadata())
|
|
||||||
actions = [_as_action_array(client.infer(obs)) for obs in observations]
|
|
||||||
return {
|
|
||||||
"metadata": metadata,
|
|
||||||
"actions": actions,
|
|
||||||
"session_id": session_id,
|
|
||||||
}
|
|
||||||
finally:
|
|
||||||
_close_policy_client(client)
|
|
||||||
|
|
||||||
|
|
||||||
def format_action_summary(index: int, action: np.ndarray) -> str:
|
|
||||||
return (
|
|
||||||
f"Action {index}: shape={tuple(action.shape)} dtype={action.dtype} "
|
|
||||||
f"min={action.min():.6f} max={action.max():.6f}"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def parse_args() -> argparse.Namespace:
|
|
||||||
parser = argparse.ArgumentParser(description="DreamZero OpenPI client example with downloaded real videos.")
|
|
||||||
parser.add_argument("--host", default=DEFAULT_HOST)
|
|
||||||
parser.add_argument("--port", type=int, default=DEFAULT_PORT)
|
|
||||||
parser.add_argument("--path", default=DEFAULT_PATH)
|
|
||||||
parser.add_argument(
|
|
||||||
"--video-dir", type=Path, default=DEFAULT_VIDEO_DIR, help="Directory containing the three camera MP4 files."
|
|
||||||
)
|
|
||||||
parser.add_argument("--prompt", default=DEFAULT_PROMPT)
|
|
||||||
parser.add_argument("--session-id", default=None)
|
|
||||||
parser.add_argument("--num-chunks", type=int, default=2)
|
|
||||||
return parser.parse_args()
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
args = parse_args()
|
|
||||||
logging.basicConfig(level=logging.INFO)
|
|
||||||
|
|
||||||
result = run_policy_session(
|
|
||||||
host=args.host,
|
|
||||||
port=args.port,
|
|
||||||
path=args.path,
|
|
||||||
video_dir=args.video_dir,
|
|
||||||
prompt=args.prompt,
|
|
||||||
session_id=args.session_id,
|
|
||||||
num_chunks=args.num_chunks,
|
|
||||||
)
|
|
||||||
validate_session_result(result, expected_action_count=args.num_chunks)
|
|
||||||
|
|
||||||
print("Server metadata:", json.dumps(result["metadata"], sort_keys=True))
|
|
||||||
for index, action in enumerate(result["actions"]):
|
|
||||||
print(format_action_summary(index, action))
|
|
||||||
print("Session ID:", result["session_id"])
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
MODEL="${MODEL:-GEAR-Dreams/DreamZero-DROID}"
|
|
||||||
HOST="${HOST:-127.0.0.1}"
|
|
||||||
PORT="${PORT:-8000}"
|
|
||||||
DEPLOY_CONFIG="${DEPLOY_CONFIG:-vllm_omni/deploy/dreamzero_tp1_cfg2.yaml}"
|
|
||||||
SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-dreamzero-droid}"
|
|
||||||
|
|
||||||
args=(
|
|
||||||
serve
|
|
||||||
"$MODEL"
|
|
||||||
--omni
|
|
||||||
--host "$HOST"
|
|
||||||
--port "$PORT"
|
|
||||||
--served-model-name "$SERVED_MODEL_NAME"
|
|
||||||
--deploy-config "$DEPLOY_CONFIG"
|
|
||||||
--enforce-eager
|
|
||||||
--disable-log-stats
|
|
||||||
)
|
|
||||||
|
|
||||||
ATTENTION_BACKEND="${ATTENTION_BACKEND:-torch}" \
|
|
||||||
DIFFUSION_ATTENTION_BACKEND="${DIFFUSION_ATTENTION_BACKEND:-TORCH_SDPA}" \
|
|
||||||
vllm "${args[@]}"
|
|
||||||
@@ -22,14 +22,13 @@ export PYTHONPATH=<transformers_modules_path>:$PYTHONPATH
|
|||||||
export HF_MODULES_CACHE=<transformers_modules_path>
|
export HF_MODULES_CACHE=<transformers_modules_path>
|
||||||
```
|
```
|
||||||
|
|
||||||
Run from repository root. The 3-stage pipeline is resolved automatically from
|
Run from repository root:
|
||||||
the model type (`vllm_omni/deploy/dynin_omni.yaml`), so no config path is
|
|
||||||
required. Pass `--deploy-config` only to override the defaults:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
vllm-omni serve snu-aidas/Dynin-Omni \
|
vllm-omni serve snu-aidas/Dynin-Omni \
|
||||||
--omni \
|
--omni \
|
||||||
--port 8091
|
--port 8091 \
|
||||||
|
--stage-configs-path "$(pwd)/vllm_omni/model_executor/stage_configs/dynin_omni.yaml"
|
||||||
```
|
```
|
||||||
|
|
||||||
If `vllm-omni` is not in PATH, run:
|
If `vllm-omni` is not in PATH, run:
|
||||||
@@ -37,7 +36,8 @@ If `vllm-omni` is not in PATH, run:
|
|||||||
```bash
|
```bash
|
||||||
PYTHONPATH="$(pwd)" python -m vllm_omni.entrypoints.cli.main serve snu-aidas/Dynin-Omni \
|
PYTHONPATH="$(pwd)" python -m vllm_omni.entrypoints.cli.main serve snu-aidas/Dynin-Omni \
|
||||||
--omni \
|
--omni \
|
||||||
--port 8091
|
--port 8091 \
|
||||||
|
--stage-configs-path "$(pwd)/vllm_omni/model_executor/stage_configs/dynin_omni.yaml"
|
||||||
```
|
```
|
||||||
|
|
||||||
Wait until the server logs show both `All stages initialized successfully` and
|
Wait until the server logs show both `All stages initialized successfully` and
|
||||||
|
|||||||
@@ -198,14 +198,7 @@ def _build_prompt_for_query_type(
|
|||||||
return query_func(custom_prompt=custom_prompt)
|
return query_func(custom_prompt=custom_prompt)
|
||||||
if query_type == "use_audio_in_video":
|
if query_type == "use_audio_in_video":
|
||||||
return query_func(video_path=video_path, custom_prompt=custom_prompt)
|
return query_func(video_path=video_path, custom_prompt=custom_prompt)
|
||||||
if query_type == "use_mixed_modalities":
|
# use_mixed_modalities / use_multi_audios
|
||||||
return query_func(
|
|
||||||
video_path=video_path,
|
|
||||||
image_path=image_path,
|
|
||||||
audio_path=audio_path,
|
|
||||||
custom_prompt=custom_prompt,
|
|
||||||
)
|
|
||||||
# use_multi_audios
|
|
||||||
return query_func(custom_prompt=custom_prompt)
|
return query_func(custom_prompt=custom_prompt)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -51,18 +51,13 @@ def build_payload(
|
|||||||
response_format: str = "wav",
|
response_format: str = "wav",
|
||||||
stream: bool = False,
|
stream: bool = False,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""Build the /v1/audio/speech request payload.
|
"""Build the /v1/audio/speech request payload."""
|
||||||
|
|
||||||
Fish Speech has no built-in speakers; omit ``voice`` so the model
|
|
||||||
generates with a random timbre (controllable via text tags like
|
|
||||||
``[cool]``, ``[warm]``, etc.). When ``ref_audio`` is supplied we
|
|
||||||
enter voice-cloning mode instead.
|
|
||||||
"""
|
|
||||||
if not text or not text.strip():
|
if not text or not text.strip():
|
||||||
raise gr.Error("Please enter text to synthesize.")
|
raise gr.Error("Please enter text to synthesize.")
|
||||||
|
|
||||||
payload: dict = {
|
payload: dict = {
|
||||||
"input": text.strip(),
|
"input": text.strip(),
|
||||||
|
"voice": "default",
|
||||||
"response_format": "pcm" if stream else response_format,
|
"response_format": "pcm" if stream else response_format,
|
||||||
"stream": stream,
|
"stream": stream,
|
||||||
}
|
}
|
||||||
@@ -198,8 +193,7 @@ def build_interface(api_base: str, stream_chunk_seconds: float = 0.5):
|
|||||||
|
|
||||||
with gr.Accordion("Voice Cloning (optional)", open=False):
|
with gr.Accordion("Voice Cloning (optional)", open=False):
|
||||||
gr.Markdown(
|
gr.Markdown(
|
||||||
"Upload or link a short reference audio (10-30s) and provide its transcript to clone the voice. "
|
"Upload or link a short reference audio (10-30s) and provide its transcript to clone the voice."
|
||||||
"Leave empty for random timbre; use text tags like `[cool]`, `[warm]`, `[happy]` to steer the output."
|
|
||||||
)
|
)
|
||||||
ref_audio = gr.Audio(
|
ref_audio = gr.Audio(
|
||||||
label="Reference Audio",
|
label="Reference Audio",
|
||||||
|
|||||||
@@ -39,6 +39,7 @@ def run_tts(args) -> None:
|
|||||||
payload = {
|
payload = {
|
||||||
"model": args.model,
|
"model": args.model,
|
||||||
"input": args.text,
|
"input": args.text,
|
||||||
|
"voice": "default",
|
||||||
"response_format": args.response_format,
|
"response_format": args.response_format,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,40 +0,0 @@
|
|||||||
# Higgs-Audio V3 Online Serving
|
|
||||||
|
|
||||||
## Start the server
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Default: GPU 0, port 8095
|
|
||||||
./examples/online_serving/text_to_speech/higgs_audio_v3/run_server.sh
|
|
||||||
|
|
||||||
# Custom GPU / port
|
|
||||||
PORT=8096 GPUS=0,1 ./examples/online_serving/text_to_speech/higgs_audio_v3/run_server.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
## Plain text TTS
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python examples/online_serving/text_to_speech/higgs_audio_v3/batch_speech_client.py \
|
|
||||||
--base-url http://localhost:8095 \
|
|
||||||
--output-dir /tmp/higgs_v3_batch \
|
|
||||||
--prompts "Hello world." "The quick brown fox jumps over the lazy dog."
|
|
||||||
```
|
|
||||||
|
|
||||||
## Voice clone
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python examples/online_serving/text_to_speech/higgs_audio_v3/batch_speech_client.py \
|
|
||||||
--base-url http://localhost:8095 \
|
|
||||||
--output-dir /tmp/higgs_v3_clone \
|
|
||||||
--ref-audio path/to/reference.wav \
|
|
||||||
--ref-text "transcript of the reference clip" \
|
|
||||||
--prompts "Text to synthesize in the cloned voice."
|
|
||||||
```
|
|
||||||
|
|
||||||
## curl example
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl -X POST http://localhost:8095/v1/audio/speech \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{"model": "higgs_audio_v3", "input": "Hello world."}' \
|
|
||||||
--output hello.wav
|
|
||||||
```
|
|
||||||
@@ -1,115 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
"""Batch client for the higgs-audio v3 online server.
|
|
||||||
|
|
||||||
Sends prompts to ``/v1/audio/speech`` and saves the returned WAV files.
|
|
||||||
|
|
||||||
Usage (plain text -> speech):
|
|
||||||
|
|
||||||
python examples/online_serving/text_to_speech/higgs_audio_v3/batch_speech_client.py \
|
|
||||||
--base-url http://localhost:8095 \
|
|
||||||
--output-dir /tmp/higgs_v3_batch \
|
|
||||||
--prompts "Hello world." "The quick brown fox jumps over the lazy dog."
|
|
||||||
|
|
||||||
Usage (voice clone):
|
|
||||||
|
|
||||||
python examples/online_serving/text_to_speech/higgs_audio_v3/batch_speech_client.py \
|
|
||||||
--base-url http://localhost:8095 \
|
|
||||||
--output-dir /tmp/higgs_v3_clone \
|
|
||||||
--ref-audio path/to/reference.wav \
|
|
||||||
--ref-text "the transcript of the reference clip" \
|
|
||||||
--prompts "Hello world."
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import base64
|
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
DEFAULT_PROMPTS = (
|
|
||||||
"Hello world.",
|
|
||||||
"The quick brown fox jumps over the lazy dog.",
|
|
||||||
"Today is a beautiful day for a walk in the park.",
|
|
||||||
"Innovation distinguishes between a leader and a follower.",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _slug(text: str) -> str:
|
|
||||||
import re
|
|
||||||
|
|
||||||
s = re.sub(r"\s+", "_", text.strip().lower())
|
|
||||||
return re.sub(r"[^a-z0-9_]+", "", s)[:32] or "prompt"
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
||||||
parser.add_argument("--base-url", default="http://localhost:8095")
|
|
||||||
parser.add_argument("--model", default="higgs_audio_v3")
|
|
||||||
parser.add_argument("--prompts", nargs="+", default=list(DEFAULT_PROMPTS))
|
|
||||||
parser.add_argument("--output-dir", type=Path, default=Path("/tmp/higgs_v3_batch"))
|
|
||||||
parser.add_argument("--format", choices=("wav", "pcm"), default="wav")
|
|
||||||
parser.add_argument("--max-new-tokens", type=int, default=2048)
|
|
||||||
parser.add_argument("--seed", type=int, default=42)
|
|
||||||
parser.add_argument("--timeout-s", type=float, default=120.0)
|
|
||||||
parser.add_argument(
|
|
||||||
"--ref-audio",
|
|
||||||
type=Path,
|
|
||||||
default=None,
|
|
||||||
help="Reference clip for voice clone (WAV/FLAC/MP3 path). Pair with --ref-text.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--ref-text",
|
|
||||||
type=str,
|
|
||||||
default=None,
|
|
||||||
help="Transcript of the reference clip. Optional but improves fidelity.",
|
|
||||||
)
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
ref_audio_data_url: str | None = None
|
|
||||||
if args.ref_audio is not None:
|
|
||||||
if not args.ref_audio.exists():
|
|
||||||
print(f"ref-audio file not found: {args.ref_audio}", file=sys.stderr)
|
|
||||||
return 2
|
|
||||||
mime = "audio/wav" if args.ref_audio.suffix.lower() == ".wav" else "audio/mpeg"
|
|
||||||
ref_b64 = base64.b64encode(args.ref_audio.read_bytes()).decode("ascii")
|
|
||||||
ref_audio_data_url = f"data:{mime};base64,{ref_b64}"
|
|
||||||
|
|
||||||
try:
|
|
||||||
import httpx
|
|
||||||
except ImportError:
|
|
||||||
print("this client needs `httpx`. Install with `pip install httpx`.", file=sys.stderr)
|
|
||||||
return 2
|
|
||||||
|
|
||||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
url = args.base_url.rstrip("/") + "/v1/audio/speech"
|
|
||||||
failures = 0
|
|
||||||
with httpx.Client(timeout=args.timeout_s) as client:
|
|
||||||
for prompt in args.prompts:
|
|
||||||
payload = {
|
|
||||||
"model": args.model,
|
|
||||||
"input": prompt,
|
|
||||||
"response_format": args.format,
|
|
||||||
"max_new_tokens": args.max_new_tokens,
|
|
||||||
"seed": args.seed,
|
|
||||||
}
|
|
||||||
if ref_audio_data_url is not None:
|
|
||||||
payload["ref_audio"] = ref_audio_data_url
|
|
||||||
if args.ref_text:
|
|
||||||
payload["ref_text"] = args.ref_text
|
|
||||||
resp = client.post(url, json=payload)
|
|
||||||
if resp.status_code != 200:
|
|
||||||
print(f"[FAIL] {prompt!r} -> {resp.status_code}: {resp.text[:200]}", file=sys.stderr)
|
|
||||||
failures += 1
|
|
||||||
continue
|
|
||||||
suffix = ".wav" if args.format == "wav" else ".pcm"
|
|
||||||
out = args.output_dir / f"{_slug(prompt)}{suffix}"
|
|
||||||
out.write_bytes(resp.content)
|
|
||||||
print(f"[ ok ] {prompt!r} -> {out} ({len(resp.content)} bytes)")
|
|
||||||
|
|
||||||
return 1 if failures else 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
@@ -1,32 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
# Launch vLLM-Omni server for higgs-audio v3.
|
|
||||||
#
|
|
||||||
# Supports plain text TTS and voice cloning via /v1/audio/speech.
|
|
||||||
#
|
|
||||||
# Usage:
|
|
||||||
# ./run_server.sh # default port 8095, GPU 0
|
|
||||||
# PORT=8096 GPUS=0,1 ./run_server.sh
|
|
||||||
# MODEL=/path/to/local/checkpoint ./run_server.sh
|
|
||||||
|
|
||||||
set -e
|
|
||||||
|
|
||||||
MODEL="${MODEL:-bosonai/higgs-audio-v3-tts-4b}"
|
|
||||||
PORT="${PORT:-8095}"
|
|
||||||
GPUS="${GPUS:-0}"
|
|
||||||
GPU_UTIL="${GPU_UTIL:-0.6}"
|
|
||||||
|
|
||||||
echo "Starting higgs-audio v3 server"
|
|
||||||
echo " MODEL=$MODEL"
|
|
||||||
echo " PORT=$PORT"
|
|
||||||
echo " CUDA_VISIBLE_DEVICES=$GPUS"
|
|
||||||
|
|
||||||
CUDA_VISIBLE_DEVICES="$GPUS" \
|
|
||||||
VLLM_USE_DEEP_GEMM=0 \
|
|
||||||
VLLM_MOE_USE_DEEP_GEMM=0 \
|
|
||||||
vllm-omni serve "$MODEL" \
|
|
||||||
--deploy-config vllm_omni/deploy/higgs_multimodal_qwen3.yaml \
|
|
||||||
--host 0.0.0.0 \
|
|
||||||
--port "$PORT" \
|
|
||||||
--gpu-memory-utilization "$GPU_UTIL" \
|
|
||||||
--trust-remote-code \
|
|
||||||
--omni
|
|
||||||
@@ -1,247 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
"""Verify a ModelOpt FP8 diffusers checkpoint exported by
|
|
||||||
quantize_wan2_2_modelopt_fp8.py / quantize_hunyuanvideo_15_modelopt_fp8.py.
|
|
||||||
|
|
||||||
Three checks:
|
|
||||||
A. transformer/config.json has a sane quantization_config block.
|
|
||||||
B. transformer/*.safetensors contains FP8 (float8_e4m3fn) quantized tensors.
|
|
||||||
C. transformer disk size is materially smaller than a BF16 baseline.
|
|
||||||
|
|
||||||
Example:
|
|
||||||
python examples/quantization/check_modelopt_fp8_export.py \\
|
|
||||||
--output ./hv15-480p-modelopt-fp8
|
|
||||||
|
|
||||||
# Optional: compare disk size against a local or HF BF16 baseline.
|
|
||||||
python examples/quantization/check_modelopt_fp8_export.py \\
|
|
||||||
--output ./hv15-480p-modelopt-fp8 \\
|
|
||||||
--baseline hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import json
|
|
||||||
import sys
|
|
||||||
from collections import Counter
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
SUPPORTED_ALGOS = ("FP8",)
|
|
||||||
|
|
||||||
|
|
||||||
def _check_config(transformer_dir: Path) -> tuple[int, str | None]:
|
|
||||||
"""Returns (status, quant_algo). status: 0 pass, 1 fail, 2 warn."""
|
|
||||||
cfg_path = transformer_dir / "config.json"
|
|
||||||
if not cfg_path.exists():
|
|
||||||
print(f"[FAIL] {cfg_path} missing.")
|
|
||||||
return 1, None
|
|
||||||
|
|
||||||
with cfg_path.open(encoding="utf-8") as f:
|
|
||||||
cfg = json.load(f)
|
|
||||||
|
|
||||||
qc = cfg.get("quantization_config")
|
|
||||||
if not isinstance(qc, dict):
|
|
||||||
print(f"[FAIL] No `quantization_config` block in {cfg_path}.")
|
|
||||||
return 1, None
|
|
||||||
|
|
||||||
print(f"[A] quantization_config from {cfg_path}:")
|
|
||||||
print(json.dumps(qc, indent=2))
|
|
||||||
|
|
||||||
quant_algo = qc.get("quant_algo")
|
|
||||||
issues = []
|
|
||||||
if qc.get("quant_method") != "modelopt":
|
|
||||||
issues.append(f"quant_method={qc.get('quant_method')!r} (expected 'modelopt')")
|
|
||||||
if quant_algo not in SUPPORTED_ALGOS:
|
|
||||||
issues.append(
|
|
||||||
f"quant_algo={quant_algo!r} (expected one of {SUPPORTED_ALGOS} — vllm-omni adapter may not auto-detect)"
|
|
||||||
)
|
|
||||||
|
|
||||||
if issues:
|
|
||||||
print("[A] WARN — config looks incomplete:")
|
|
||||||
for issue in issues:
|
|
||||||
print(f" - {issue}")
|
|
||||||
return 2, quant_algo
|
|
||||||
print(f"[A] PASS — config looks correct (quant_algo={quant_algo}).")
|
|
||||||
return 0, quant_algo
|
|
||||||
|
|
||||||
|
|
||||||
def _read_safetensors_header(path: Path) -> dict:
|
|
||||||
"""Read the JSON header of a safetensors file. Bypass-safe — doesn't materialize tensors.
|
|
||||||
|
|
||||||
Returns {tensor_name: {'dtype': 'F8_E4M3', 'shape': [...], 'data_offsets': [...]}}.
|
|
||||||
Header dtype strings: F8_E4M3, F8_E5M2, BF16, F16, F32, F64, I8, I16, I32, I64, BOOL, U8, ...
|
|
||||||
"""
|
|
||||||
import struct
|
|
||||||
|
|
||||||
with open(path, "rb") as f:
|
|
||||||
header_len = struct.unpack("<Q", f.read(8))[0]
|
|
||||||
header = json.loads(f.read(header_len))
|
|
||||||
header.pop("__metadata__", None)
|
|
||||||
return header
|
|
||||||
|
|
||||||
|
|
||||||
def _classify_weight_scale_granularity(weight_scale_shapes: list[list[int]]) -> str:
|
|
||||||
"""Infer per-tensor vs per-channel vs per-block from sample weight_scale shapes.
|
|
||||||
|
|
||||||
ModelOpt block-wise produces shapes like `[16, 1, 16, 1]` (broadcasting dims of 1
|
|
||||||
interleaved with block-count dims). We count "meaningful" dims — ones with size > 1 —
|
|
||||||
and classify: 0 meaningful dims = per-tensor (scalar), 1 = per-channel, 2+ = per-block.
|
|
||||||
"""
|
|
||||||
if not weight_scale_shapes:
|
|
||||||
return "no weight_scale tensors found"
|
|
||||||
|
|
||||||
def meaningful_dims(shape: list[int]) -> int:
|
|
||||||
return sum(1 for d in shape if d > 1)
|
|
||||||
|
|
||||||
per_tensor = sum(1 for s in weight_scale_shapes if meaningful_dims(s) == 0)
|
|
||||||
per_channel = sum(1 for s in weight_scale_shapes if meaningful_dims(s) == 1)
|
|
||||||
per_block = sum(1 for s in weight_scale_shapes if meaningful_dims(s) >= 2)
|
|
||||||
total = len(weight_scale_shapes)
|
|
||||||
if per_tensor == total:
|
|
||||||
return "per-tensor (all scalar scales)"
|
|
||||||
if per_channel == total:
|
|
||||||
return "per-channel (1 meaningful dim)"
|
|
||||||
if per_block == total:
|
|
||||||
return "per-block (2+ meaningful dims — e.g. [M//bm, 1, N//bn, 1] for tiles)"
|
|
||||||
return f"mixed: per-tensor={per_tensor}, per-channel={per_channel}, per-block={per_block} of {total}"
|
|
||||||
|
|
||||||
|
|
||||||
def _check_safetensors(transformer_dir: Path) -> int:
|
|
||||||
"""Returns 0 on pass, 1 on fail. Reads on-disk dtype from the safetensors header."""
|
|
||||||
files = sorted(transformer_dir.glob("*.safetensors"))
|
|
||||||
if not files:
|
|
||||||
print(f"[FAIL] No *.safetensors in {transformer_dir}.")
|
|
||||||
return 1
|
|
||||||
|
|
||||||
header_dtype_counts: Counter[str] = Counter()
|
|
||||||
sample_quant_weight_keys: list[str] = []
|
|
||||||
sample_scale_keys: list[str] = []
|
|
||||||
weight_scale_shapes: list[list[int]] = []
|
|
||||||
sample_weight_scale_entries: list[tuple[str, list[int]]] = []
|
|
||||||
|
|
||||||
for f in files:
|
|
||||||
try:
|
|
||||||
header = _read_safetensors_header(f)
|
|
||||||
except Exception as exc:
|
|
||||||
print(f"[B] WARN — could not parse header of {f}: {exc}")
|
|
||||||
continue
|
|
||||||
for k, info in header.items():
|
|
||||||
dtype = info.get("dtype", "?")
|
|
||||||
header_dtype_counts[dtype] += 1
|
|
||||||
# FP8 stores weights as F8_E4M3 directly.
|
|
||||||
if dtype.startswith("F8") and k.endswith(".weight") and len(sample_quant_weight_keys) < 5:
|
|
||||||
sample_quant_weight_keys.append(k)
|
|
||||||
if k.endswith(("_scale", ".weight_scale", ".input_scale", "_scale_inv")) and len(sample_scale_keys) < 5:
|
|
||||||
sample_scale_keys.append(k)
|
|
||||||
if k.endswith(".weight_scale"):
|
|
||||||
weight_scale_shapes.append(info.get("shape", []))
|
|
||||||
if len(sample_weight_scale_entries) < 5:
|
|
||||||
sample_weight_scale_entries.append((k, info.get("shape", [])))
|
|
||||||
|
|
||||||
print(f"\n[B] On-disk dtype counts across {len(files)} safetensors file(s) (from header, not get_tensor):")
|
|
||||||
for dtype, count in sorted(header_dtype_counts.items(), key=lambda kv: -kv[1]):
|
|
||||||
marker = " <-- FP8" if dtype.startswith("F8") else ""
|
|
||||||
print(f" {dtype:10s} {count:>6d}{marker}")
|
|
||||||
|
|
||||||
quant_count = sum(c for d, c in header_dtype_counts.items() if d.startswith("F8"))
|
|
||||||
if quant_count == 0:
|
|
||||||
print("[B] FAIL — no FP8 tensors on disk. Calibration likely did not actually quantize the weights.")
|
|
||||||
return 1
|
|
||||||
|
|
||||||
print(f"[B] PASS — {quant_count} FP8 tensors stored on disk.")
|
|
||||||
if sample_quant_weight_keys:
|
|
||||||
print(f" sample quantized weight tensors: {sample_quant_weight_keys[:3]}")
|
|
||||||
if sample_scale_keys:
|
|
||||||
print(f" sample scale tensors: {sample_scale_keys[:3]}")
|
|
||||||
print(" (Note: torch's get_tensor() may return these as bf16 views on some versions —")
|
|
||||||
print(" irrelevant; vLLM's loader uses native FP8 ops.)")
|
|
||||||
|
|
||||||
# Weight-scale granularity — per-tensor (scalar) vs per-channel (1-D) vs per-block (N-D).
|
|
||||||
print(f"\n weight_scale granularity: {_classify_weight_scale_granularity(weight_scale_shapes)}")
|
|
||||||
for key, shape in sample_weight_scale_entries[:3]:
|
|
||||||
print(f" {key}: shape {shape}")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
def _disk_size_gib(p: Path) -> float:
|
|
||||||
return sum(f.stat().st_size for f in p.rglob("*") if f.is_file()) / (1024**3)
|
|
||||||
|
|
||||||
|
|
||||||
def _check_size_vs_baseline(transformer_dir: Path, baseline: str | None) -> int:
|
|
||||||
"""Returns 0 always (informational only)."""
|
|
||||||
quant_size = _disk_size_gib(transformer_dir)
|
|
||||||
print(f"\n[C] FP8 transformer disk size: {quant_size:.2f} GiB")
|
|
||||||
|
|
||||||
if baseline is None:
|
|
||||||
print("[C] SKIP — pass --baseline <path or HF id> to compare against BF16.")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
baseline_path = Path(baseline)
|
|
||||||
if not baseline_path.exists():
|
|
||||||
# Try HF download.
|
|
||||||
try:
|
|
||||||
from huggingface_hub import snapshot_download
|
|
||||||
except ImportError:
|
|
||||||
print("[C] SKIP — huggingface_hub not installed and baseline not a local path.")
|
|
||||||
return 0
|
|
||||||
print(f" Downloading baseline transformer from HF: {baseline}")
|
|
||||||
baseline_path = Path(snapshot_download(baseline, allow_patterns=["transformer/*"]))
|
|
||||||
|
|
||||||
bf16_dir = baseline_path / "transformer" if (baseline_path / "transformer").exists() else baseline_path
|
|
||||||
bf16_size = _disk_size_gib(bf16_dir)
|
|
||||||
if bf16_size == 0:
|
|
||||||
print(f"[C] WARN — baseline transformer dir empty: {bf16_dir}")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
# Expected reduction for FP8: ~50%.
|
|
||||||
min_reduction = 30
|
|
||||||
reduction = (1 - quant_size / bf16_size) * 100
|
|
||||||
print(f"[C] BF16 baseline transformer disk size: {bf16_size:.2f} GiB ({bf16_dir})")
|
|
||||||
print(f"[C] Disk reduction: {reduction:.1f}% (FP8 transformer is {quant_size / bf16_size:.0%} of BF16)")
|
|
||||||
if reduction < min_reduction:
|
|
||||||
print(
|
|
||||||
f"[C] WARN — FP8 should typically reduce disk by ~40-50%; <{min_reduction}% suggests partial quantization."
|
|
||||||
)
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
||||||
p.add_argument("--output", required=True, help="Path to the exported ModelOpt FP8 checkpoint root.")
|
|
||||||
p.add_argument(
|
|
||||||
"--baseline",
|
|
||||||
default=None,
|
|
||||||
help="Optional BF16 baseline (local diffusers dir or HF id) for disk-size comparison.",
|
|
||||||
)
|
|
||||||
args = p.parse_args()
|
|
||||||
|
|
||||||
out_root = Path(args.output).expanduser().resolve()
|
|
||||||
transformer_dir = out_root / "transformer"
|
|
||||||
if not transformer_dir.exists():
|
|
||||||
print(f"[FAIL] {transformer_dir} does not exist.")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
print(f"Checking: {out_root}\n")
|
|
||||||
|
|
||||||
fail = 0
|
|
||||||
config_status, _quant_algo = _check_config(transformer_dir)
|
|
||||||
fail |= config_status
|
|
||||||
fail |= _check_safetensors(transformer_dir)
|
|
||||||
_check_size_vs_baseline(transformer_dir, args.baseline)
|
|
||||||
|
|
||||||
print()
|
|
||||||
if fail == 0:
|
|
||||||
print("=" * 60)
|
|
||||||
print("ALL CHECKS PASSED — checkpoint looks ready for vllm-omni serving.")
|
|
||||||
elif fail == 1:
|
|
||||||
print("=" * 60)
|
|
||||||
print("FAILURES detected — calibration may need to be re-run.")
|
|
||||||
sys.exit(1)
|
|
||||||
else:
|
|
||||||
print("=" * 60)
|
|
||||||
print("WARNINGS only — checkpoint may serve but with caveats. See [A] above.")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -1,448 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
"""Quantize HunyuanVideo-1.5 (480p T2V) to a ModelOpt FP8 Hugging Face checkpoint.
|
|
||||||
|
|
||||||
Calibrates the DiT transformer using a small video prompt set and exports a
|
|
||||||
diffusers-style directory whose transformer carries ModelOpt FP8 metadata.
|
|
||||||
The exported checkpoint is consumable by vllm-omni's ModelOpt FP8 adapter
|
|
||||||
(see vllm_omni/diffusion/model_loader/checkpoint_adapters/modelopt.py).
|
|
||||||
|
|
||||||
Layers kept full precision match the #2728 / #2795 pattern: modulation,
|
|
||||||
AdaLayerNorm, entry/exit projections, embeddings, the token refiner path,
|
|
||||||
and final proj_out. MHA quantizers are off by default; HV-1.5 self-attention
|
|
||||||
empirically degrades under FP8 (see #2920 ablation).
|
|
||||||
|
|
||||||
Example:
|
|
||||||
python examples/quantization/quantize_hunyuanvideo_15_modelopt_fp8.py \\
|
|
||||||
--model hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v \\
|
|
||||||
--output ./hv15-480p-modelopt-fp8 \\
|
|
||||||
--overwrite
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import copy
|
|
||||||
import json
|
|
||||||
import re
|
|
||||||
import shutil
|
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import torch
|
|
||||||
from diffusers import DiffusionPipeline
|
|
||||||
|
|
||||||
DEFAULT_PROMPTS = [
|
|
||||||
"A dog running across a field of golden wheat.",
|
|
||||||
"An astronaut riding a horse across the surface of Mars, red dust swirling, cinematic wide shot.",
|
|
||||||
"A hummingbird hovering in front of a vibrant red flower, slow motion, macro shot.",
|
|
||||||
"A crackling campfire at night under a starry sky, sparks rising into the dark.",
|
|
||||||
"An underwater shot of a coral reef with tropical fish swimming by, sun rays piercing the water.",
|
|
||||||
"A close-up of a blooming rose covered in morning dew, soft natural light.",
|
|
||||||
"A peaceful mountain village at dawn, mist rolling over the rooftops, cinematic establishing shot.",
|
|
||||||
"A skateboarder doing a kickflip in an urban plaza, slow motion, golden hour lighting.",
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def _build_parser() -> argparse.ArgumentParser:
|
|
||||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
||||||
p.add_argument("--model", required=True, help="Input HV-1.5 diffusers directory or HF id.")
|
|
||||||
p.add_argument("--output", required=True, help="Output directory for the ModelOpt FP8 checkpoint.")
|
|
||||||
p.add_argument("--dtype", choices=("bfloat16", "float16"), default="bfloat16")
|
|
||||||
p.add_argument("--height", type=int, default=480)
|
|
||||||
p.add_argument("--width", type=int, default=832)
|
|
||||||
p.add_argument(
|
|
||||||
"--num-frames",
|
|
||||||
type=int,
|
|
||||||
default=33,
|
|
||||||
help="Frames per calibration sample. 33 matches the typical short benchmark.",
|
|
||||||
)
|
|
||||||
p.add_argument("--guidance-scale", type=float, default=6.0)
|
|
||||||
p.add_argument(
|
|
||||||
"--calib-steps",
|
|
||||||
type=int,
|
|
||||||
default=10,
|
|
||||||
help="Denoising steps per calibration prompt (10 is enough for amax statistics).",
|
|
||||||
)
|
|
||||||
p.add_argument("--calib-size", type=int, default=8, help="How many prompts to use for calibration.")
|
|
||||||
p.add_argument("--seed", type=int, default=42)
|
|
||||||
p.add_argument(
|
|
||||||
"--prompt",
|
|
||||||
action="append",
|
|
||||||
default=[],
|
|
||||||
help="Custom calibration prompt. Repeat to provide multiple.",
|
|
||||||
)
|
|
||||||
p.add_argument(
|
|
||||||
"--quantize-mha",
|
|
||||||
action="store_true",
|
|
||||||
help="Enable FP8 attention K/V/softmax quantizers. Off by default — empirically degrades HV-1.5 video output.",
|
|
||||||
)
|
|
||||||
p.add_argument(
|
|
||||||
"--weight-block-size",
|
|
||||||
type=str,
|
|
||||||
default=None,
|
|
||||||
help="Per-block weight quantization as 'M,N' (e.g. '128,128' for 128x128 tiles). "
|
|
||||||
"Default: per-tensor (one scale per linear). Block-wise typically gives tighter quality at "
|
|
||||||
"negligible memory cost. Static FP8 is exempt from upstream vLLM's online block-wise gate.",
|
|
||||||
)
|
|
||||||
p.add_argument("--overwrite", action="store_true", help="Replace an existing output directory.")
|
|
||||||
return p
|
|
||||||
|
|
||||||
|
|
||||||
def _parse_block_size(spec: str | None) -> list[int] | None:
|
|
||||||
if spec is None:
|
|
||||||
return None
|
|
||||||
parts = [int(x) for x in spec.split(",") if x.strip()]
|
|
||||||
if len(parts) != 2:
|
|
||||||
raise SystemExit(f"--weight-block-size must be 'M,N' (2 ints), got {spec!r}")
|
|
||||||
return parts
|
|
||||||
|
|
||||||
|
|
||||||
def _require_modelopt() -> Any:
|
|
||||||
try:
|
|
||||||
import modelopt.torch.quantization as mtq
|
|
||||||
except ModuleNotFoundError as exc:
|
|
||||||
raise SystemExit(
|
|
||||||
"NVIDIA ModelOpt is not installed. Install with:\n"
|
|
||||||
" pip install 'nvidia-modelopt[all]'\n"
|
|
||||||
f"Original error: {exc}"
|
|
||||||
) from exc
|
|
||||||
return mtq
|
|
||||||
|
|
||||||
|
|
||||||
def _ensure_paths(args: argparse.Namespace) -> tuple[str, Path]:
|
|
||||||
model_path = args.model
|
|
||||||
output_dir = Path(args.output).expanduser().resolve()
|
|
||||||
if output_dir.exists():
|
|
||||||
if not args.overwrite:
|
|
||||||
raise SystemExit(f"Output directory already exists: {output_dir}\nPass --overwrite to replace it.")
|
|
||||||
shutil.rmtree(output_dir)
|
|
||||||
return model_path, output_dir
|
|
||||||
|
|
||||||
|
|
||||||
def _select_dtype(name: str) -> torch.dtype:
|
|
||||||
return {"bfloat16": torch.bfloat16, "float16": torch.float16}[name]
|
|
||||||
|
|
||||||
|
|
||||||
def _build_prompts(args: argparse.Namespace) -> list[str]:
|
|
||||||
prompts = args.prompt or DEFAULT_PROMPTS
|
|
||||||
if args.calib_size <= 0:
|
|
||||||
raise SystemExit("--calib-size must be positive.")
|
|
||||||
if len(prompts) < args.calib_size:
|
|
||||||
repeats = (args.calib_size + len(prompts) - 1) // len(prompts)
|
|
||||||
prompts = (prompts * repeats)[: args.calib_size]
|
|
||||||
return prompts[: args.calib_size]
|
|
||||||
|
|
||||||
|
|
||||||
# Layers to KEEP at full precision (mirror of the #2920 wiring + #2728/#2795 skip pattern).
|
|
||||||
# - x_embedder, image_embedder, context_embedder*, time_embed*, cond_type_embed: entry/embedding
|
|
||||||
# - norm_out, norm1*.linear, norm1_context*.linear, norm2*, norm2_context*: AdaLayerNorm modulation
|
|
||||||
# - proj_out: final output projection
|
|
||||||
# - token_refiner*: text-encoder refinement uses diffusers raw nn.Linear
|
|
||||||
def _filter_func_hv15(name: str) -> bool:
|
|
||||||
pattern = re.compile(
|
|
||||||
r"(proj_out.*|"
|
|
||||||
r".*(x_embedder|image_embedder|context_embedder|context_embedder_2|"
|
|
||||||
r"time_embed|cond_type_embed|"
|
|
||||||
r"norm_out|norm1\.linear|norm1_context\.linear|norm2|norm2_context|"
|
|
||||||
r"token_refiner).*)"
|
|
||||||
)
|
|
||||||
return pattern.match(name) is not None
|
|
||||||
|
|
||||||
|
|
||||||
def _mha_filter_func(name: str) -> bool:
|
|
||||||
pattern = re.compile(
|
|
||||||
r".*(q_bmm_quantizer|k_bmm_quantizer|v_bmm_quantizer|softmax_quantizer|bmm2_output_quantizer).*"
|
|
||||||
)
|
|
||||||
return pattern.match(name) is not None
|
|
||||||
|
|
||||||
|
|
||||||
def _disable_known_problematic_quantizers(mtq: Any, backbone: torch.nn.Module, *, quantize_mha: bool) -> None:
|
|
||||||
if not hasattr(mtq, "disable_quantizer"):
|
|
||||||
return
|
|
||||||
mtq.disable_quantizer(backbone, _filter_func_hv15)
|
|
||||||
if not quantize_mha:
|
|
||||||
mtq.disable_quantizer(backbone, _mha_filter_func)
|
|
||||||
|
|
||||||
|
|
||||||
def _load_pipeline(model_path: str, dtype: torch.dtype) -> DiffusionPipeline:
|
|
||||||
pipe = DiffusionPipeline.from_pretrained(model_path, torch_dtype=dtype)
|
|
||||||
if hasattr(pipe, "set_progress_bar_config"):
|
|
||||||
pipe.set_progress_bar_config(disable=True)
|
|
||||||
pipe.to("cuda")
|
|
||||||
return pipe
|
|
||||||
|
|
||||||
|
|
||||||
def _build_forward_loop(pipe: DiffusionPipeline, args: argparse.Namespace, prompts: list[str]):
|
|
||||||
generator = torch.Generator(device="cuda")
|
|
||||||
|
|
||||||
# Try to set guidance on the pipeline's guider object up front (modern
|
|
||||||
# diffusers HV-1.5 uses a Guider abstraction, not a per-call kwarg). Falls
|
|
||||||
# back silently — calibration uses whatever default the pipeline ships with.
|
|
||||||
guider = getattr(pipe, "guider", None)
|
|
||||||
if guider is not None and hasattr(guider, "guidance_scale"):
|
|
||||||
try:
|
|
||||||
guider.guidance_scale = args.guidance_scale
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
base_kwargs = dict(
|
|
||||||
height=args.height,
|
|
||||||
width=args.width,
|
|
||||||
num_frames=args.num_frames,
|
|
||||||
num_inference_steps=args.calib_steps,
|
|
||||||
output_type="latent",
|
|
||||||
)
|
|
||||||
|
|
||||||
def forward_loop(*_unused_args, **_unused_kwargs) -> None:
|
|
||||||
with torch.inference_mode():
|
|
||||||
for idx, prompt in enumerate(prompts):
|
|
||||||
generator.manual_seed(args.seed + idx)
|
|
||||||
# Try with guidance_scale first; fall back without on TypeError
|
|
||||||
# for pipelines (like HV-1.5) that take CFG via guider config.
|
|
||||||
try:
|
|
||||||
pipe(prompt=prompt, generator=generator, guidance_scale=args.guidance_scale, **base_kwargs)
|
|
||||||
except TypeError as exc:
|
|
||||||
if "guidance_scale" not in str(exc):
|
|
||||||
raise
|
|
||||||
pipe(prompt=prompt, generator=generator, **base_kwargs)
|
|
||||||
|
|
||||||
return forward_loop
|
|
||||||
|
|
||||||
|
|
||||||
def _summarize_export(output_dir: Path) -> None:
|
|
||||||
cfg_path = output_dir / "transformer" / "config.json"
|
|
||||||
if not cfg_path.exists():
|
|
||||||
print(f"[warn] {cfg_path} missing.", file=sys.stderr)
|
|
||||||
return
|
|
||||||
with cfg_path.open(encoding="utf-8") as f:
|
|
||||||
cfg = json.load(f)
|
|
||||||
qc = cfg.get("quantization_config")
|
|
||||||
if not isinstance(qc, dict):
|
|
||||||
print("[warn] No quantization_config in transformer/config.json.", file=sys.stderr)
|
|
||||||
return
|
|
||||||
print("Export summary:")
|
|
||||||
print(f" quant_method: {qc.get('quant_method')}")
|
|
||||||
print(f" quant_algo: {qc.get('quant_algo')}")
|
|
||||||
producer = qc.get("producer")
|
|
||||||
if isinstance(producer, dict):
|
|
||||||
print(f" producer: {producer.get('name')} {producer.get('version')}")
|
|
||||||
print(f" config path: {cfg_path}")
|
|
||||||
|
|
||||||
|
|
||||||
def _force_export_quantized_weights(backbone: torch.nn.Module, dtype: torch.dtype) -> int:
|
|
||||||
"""Convert in-memory weights of quantized modules to actual FP8 storage.
|
|
||||||
|
|
||||||
`export_hf_checkpoint` skips this step for unknown model types (HV-1.5 isn't
|
|
||||||
in ModelOpt's recognized-model registry), so we must call the per-weight
|
|
||||||
export helper ourselves. Same workaround as the HunyuanImage-3 calibration
|
|
||||||
helper.
|
|
||||||
"""
|
|
||||||
from modelopt.torch.export.quant_utils import (
|
|
||||||
QUANTIZATION_NONE,
|
|
||||||
get_quantization_format,
|
|
||||||
quantizer_attr_names,
|
|
||||||
weight_attr_names,
|
|
||||||
)
|
|
||||||
from modelopt.torch.export.unified_export_hf import _export_quantized_weight
|
|
||||||
|
|
||||||
exported = 0
|
|
||||||
for name, module in backbone.named_modules():
|
|
||||||
try:
|
|
||||||
quantization_format = get_quantization_format(module)
|
|
||||||
except Exception as exc:
|
|
||||||
print(f"[warn] Could not inspect quantization format for {name}: {exc}", file=sys.stderr)
|
|
||||||
continue
|
|
||||||
if quantization_format == QUANTIZATION_NONE:
|
|
||||||
continue
|
|
||||||
for weight_name in weight_attr_names(module):
|
|
||||||
quantizer_attrs = quantizer_attr_names(weight_name)
|
|
||||||
weight_quantizer = getattr(module, quantizer_attrs.weight_quantizer, None)
|
|
||||||
if weight_quantizer is None or not getattr(weight_quantizer, "is_enabled", False):
|
|
||||||
continue
|
|
||||||
_export_quantized_weight(module, dtype, weight_name)
|
|
||||||
exported += 1
|
|
||||||
return exported
|
|
||||||
|
|
||||||
|
|
||||||
def _hv15_quant_config_block(weight_block_size: list[int] | None = None) -> dict:
|
|
||||||
"""Mirror ModelOpt FP8 metadata expected by vllm-omni's adapter (#2913).
|
|
||||||
|
|
||||||
Same shape as the HunyuanImage-3 author's _hunyuan_quant_config(). When
|
|
||||||
`weight_block_size` is given, advertise block-wise weight quantization in
|
|
||||||
the saved metadata (so consumers know to expect multi-element scale tensors).
|
|
||||||
"""
|
|
||||||
weights_cfg: dict = {"dynamic": False, "num_bits": 8, "type": "float"}
|
|
||||||
if weight_block_size is not None:
|
|
||||||
weights_cfg["strategy"] = "block"
|
|
||||||
weights_cfg["block_structure"] = f"{weight_block_size[0]}x{weight_block_size[1]}"
|
|
||||||
return {
|
|
||||||
"config_groups": {
|
|
||||||
"group_0": {
|
|
||||||
"input_activations": {"dynamic": False, "num_bits": 8, "type": "float"},
|
|
||||||
"weights": weights_cfg,
|
|
||||||
"targets": ["Linear"],
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"ignore": [
|
|
||||||
"context_embedder*",
|
|
||||||
"context_embedder_2*",
|
|
||||||
"cond_type_embed*",
|
|
||||||
"image_embedder*",
|
|
||||||
"norm1.linear*",
|
|
||||||
"norm1_context.linear*",
|
|
||||||
"norm2*",
|
|
||||||
"norm2_context*",
|
|
||||||
"norm_out*",
|
|
||||||
"proj_out*",
|
|
||||||
"time_embed*",
|
|
||||||
"token_refiner*",
|
|
||||||
"x_embedder*",
|
|
||||||
],
|
|
||||||
"producer": {"name": "modelopt"},
|
|
||||||
"quant_algo": "FP8",
|
|
||||||
"quant_method": "modelopt",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _patch_quant_config(output_dir: Path, weight_block_size: list[int] | None = None) -> None:
|
|
||||||
"""Inject quant_algo: FP8 + config_groups into transformer/config.json so
|
|
||||||
vllm-omni's adapter (#2913) recognises the checkpoint as ModelOpt FP8."""
|
|
||||||
cfg_path = output_dir / "transformer" / "config.json"
|
|
||||||
with cfg_path.open(encoding="utf-8") as f:
|
|
||||||
cfg = json.load(f)
|
|
||||||
|
|
||||||
new_qc = _hv15_quant_config_block(weight_block_size=weight_block_size)
|
|
||||||
existing = cfg.get("quantization_config")
|
|
||||||
if isinstance(existing, dict):
|
|
||||||
producer = existing.get("producer")
|
|
||||||
if isinstance(producer, dict):
|
|
||||||
new_qc["producer"] = producer
|
|
||||||
|
|
||||||
cfg["quantization_config"] = new_qc
|
|
||||||
with cfg_path.open("w", encoding="utf-8") as f:
|
|
||||||
json.dump(cfg, f, indent=2)
|
|
||||||
|
|
||||||
|
|
||||||
def _save_pipeline_with_fp8_transformer(
|
|
||||||
pipe: DiffusionPipeline,
|
|
||||||
model_path: str,
|
|
||||||
output_dir: Path,
|
|
||||||
max_shard_size: str = "5GB",
|
|
||||||
) -> None:
|
|
||||||
"""Save the pipeline with the (now FP8) transformer.
|
|
||||||
|
|
||||||
Copies the source directory verbatim except for `transformer/`, then
|
|
||||||
saves the transformer with quantizers hidden so the state dict contains
|
|
||||||
only the FP8 weights + scale tensors.
|
|
||||||
"""
|
|
||||||
from modelopt.torch.export.diffusers_utils import hide_quantizers_from_state_dict
|
|
||||||
|
|
||||||
src = Path(model_path)
|
|
||||||
if not src.exists():
|
|
||||||
from huggingface_hub import snapshot_download
|
|
||||||
|
|
||||||
src = Path(snapshot_download(model_path))
|
|
||||||
|
|
||||||
if output_dir.exists():
|
|
||||||
shutil.rmtree(output_dir)
|
|
||||||
shutil.copytree(src, output_dir, ignore=shutil.ignore_patterns("transformer"))
|
|
||||||
|
|
||||||
transformer_out = output_dir / "transformer"
|
|
||||||
# `hide_quantizers_from_state_dict` walks named_modules(); pass the actual
|
|
||||||
# nn.Module (transformer), not the diffusers Pipeline wrapper.
|
|
||||||
with hide_quantizers_from_state_dict(pipe.transformer):
|
|
||||||
pipe.transformer.save_pretrained(
|
|
||||||
str(transformer_out),
|
|
||||||
safe_serialization=True,
|
|
||||||
max_shard_size=max_shard_size,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
args = _build_parser().parse_args()
|
|
||||||
if not torch.cuda.is_available():
|
|
||||||
raise SystemExit("CUDA is required for ModelOpt FP8 quantization.")
|
|
||||||
|
|
||||||
mtq = _require_modelopt()
|
|
||||||
model_path, output_dir = _ensure_paths(args)
|
|
||||||
dtype = _select_dtype(args.dtype)
|
|
||||||
prompts = _build_prompts(args)
|
|
||||||
|
|
||||||
print("Quantization plan:")
|
|
||||||
weight_block_size = _parse_block_size(args.weight_block_size)
|
|
||||||
|
|
||||||
print(f" input: {args.model}")
|
|
||||||
print(f" output: {output_dir}")
|
|
||||||
print(f" dtype: {dtype}")
|
|
||||||
print(f" height/width: {args.height}x{args.width}")
|
|
||||||
print(f" num_frames: {args.num_frames}")
|
|
||||||
print(f" calib_size: {len(prompts)}")
|
|
||||||
print(f" calib_steps: {args.calib_steps}")
|
|
||||||
print(f" quantize_mha: {args.quantize_mha}")
|
|
||||||
print(
|
|
||||||
f" weight strategy: {'block-wise ' + str(weight_block_size) if weight_block_size else 'per-tensor (default)'}"
|
|
||||||
)
|
|
||||||
|
|
||||||
pipe = _load_pipeline(model_path, dtype)
|
|
||||||
backbone = pipe.transformer
|
|
||||||
|
|
||||||
quant_config = copy.deepcopy(mtq.FP8_DEFAULT_CFG)
|
|
||||||
if weight_block_size is not None:
|
|
||||||
# Switch from per-tensor (default) to block-wise weight quantization.
|
|
||||||
# ModelOpt's wildcard "*weight_quantizer" matches every linear's weight quantizer.
|
|
||||||
quant_config["quant_cfg"]["*weight_quantizer"] = {
|
|
||||||
"num_bits": (4, 3), # E4M3 (FP8 weights, same as default)
|
|
||||||
"block_sizes": {-1: weight_block_size[1], -2: weight_block_size[0]},
|
|
||||||
}
|
|
||||||
print(
|
|
||||||
f" -> overriding weight quantizer with block_sizes={weight_block_size} "
|
|
||||||
f"({weight_block_size[0]}x{weight_block_size[1]} tiles)"
|
|
||||||
)
|
|
||||||
|
|
||||||
forward_loop = _build_forward_loop(pipe, args, prompts)
|
|
||||||
quantized = mtq.quantize(backbone, quant_config, forward_loop)
|
|
||||||
if quantized is not None:
|
|
||||||
pipe.transformer = quantized
|
|
||||||
backbone = quantized
|
|
||||||
|
|
||||||
_disable_known_problematic_quantizers(mtq, backbone, quantize_mha=args.quantize_mha)
|
|
||||||
|
|
||||||
print("\nForcing FP8 weight serialization (HV-1.5 isn't in ModelOpt's recognized-model registry,")
|
|
||||||
print("so we have to call the per-weight export helper ourselves)...")
|
|
||||||
exported = _force_export_quantized_weights(backbone, dtype)
|
|
||||||
print(f" -> {exported} weights converted to FP8 in memory")
|
|
||||||
if exported == 0:
|
|
||||||
raise SystemExit(
|
|
||||||
"No quantized weights were exported. Calibration may have skipped every layer "
|
|
||||||
"(check the disable_quantizer regex) or `mtq.quantize` did not actually wrap any "
|
|
||||||
"weight quantizers."
|
|
||||||
)
|
|
||||||
|
|
||||||
print("\nSaving pipeline with FP8 transformer...")
|
|
||||||
_save_pipeline_with_fp8_transformer(pipe, model_path, output_dir)
|
|
||||||
_patch_quant_config(output_dir, weight_block_size=weight_block_size)
|
|
||||||
print(f"Saved to: {output_dir}")
|
|
||||||
_summarize_export(output_dir)
|
|
||||||
|
|
||||||
print("\nNext: validate the checkpoint with vllm-omni:")
|
|
||||||
print(
|
|
||||||
" python examples/offline_inference/text_to_video/text_to_video.py \\\n"
|
|
||||||
f" --model {output_dir} \\\n"
|
|
||||||
" --quantization fp8 \\\n"
|
|
||||||
" --prompt 'A dog running across a field of golden wheat.' \\\n"
|
|
||||||
f" --height {args.height} --width {args.width} --num-frames {args.num_frames} \\\n"
|
|
||||||
" --num-inference-steps 30 --guidance-scale 6.0 --seed 42 \\\n"
|
|
||||||
" --output outputs/hv15_modelopt_fp8.mp4 \\\n"
|
|
||||||
" --enforce-eager"
|
|
||||||
)
|
|
||||||
print(
|
|
||||||
"\n (--quantization fp8 is auto-upgraded to ModelOpt FP8 at runtime because the "
|
|
||||||
"checkpoint's config.json has modelopt metadata.)"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -1,471 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
"""Quantize Wan2.2 (TI2V-5B, 704x1280 T2V) to a ModelOpt FP8 Hugging Face checkpoint.
|
|
||||||
|
|
||||||
Calibrates the DiT transformer using a small video prompt set and exports a
|
|
||||||
diffusers-style directory whose transformer carries ModelOpt FP8 metadata.
|
|
||||||
The exported checkpoint is consumable by vllm-omni's ModelOpt FP8 adapter
|
|
||||||
(see vllm_omni/diffusion/model_loader/checkpoint_adapters/modelopt.py).
|
|
||||||
|
|
||||||
Layers kept full precision match the #2728 / #2795 pattern: condition embedder
|
|
||||||
(time/text/image), patch embedding, modulation (scale_shift_table), final
|
|
||||||
norm + proj_out, and sequence-parallel helpers. All attention + FFN linears
|
|
||||||
are quantized — static calibration handles the numerics that online FP8
|
|
||||||
couldn't (see #2920 ablation).
|
|
||||||
|
|
||||||
Default target is `Wan-AI/Wan2.2-TI2V-5B-Diffusers`, the dense 5B variant that
|
|
||||||
fits 80GB BF16. The A14B MoE variants need 2+ GPUs and are out of scope here.
|
|
||||||
|
|
||||||
Example:
|
|
||||||
python examples/quantization/quantize_wan2_2_modelopt_fp8.py \\
|
|
||||||
--model Wan-AI/Wan2.2-TI2V-5B-Diffusers \\
|
|
||||||
--output ./wan22-ti2v-modelopt-fp8 \\
|
|
||||||
--overwrite
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import copy
|
|
||||||
import json
|
|
||||||
import re
|
|
||||||
import shutil
|
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import torch
|
|
||||||
from diffusers import DiffusionPipeline
|
|
||||||
|
|
||||||
DEFAULT_PROMPTS = [
|
|
||||||
"A dog running across a field of golden wheat.",
|
|
||||||
"An astronaut riding a horse across the surface of Mars, red dust swirling, cinematic wide shot.",
|
|
||||||
"A hummingbird hovering in front of a vibrant red flower, slow motion, macro shot.",
|
|
||||||
"A crackling campfire at night under a starry sky, sparks rising into the dark.",
|
|
||||||
"An underwater shot of a coral reef with tropical fish swimming by, sun rays piercing the water.",
|
|
||||||
"A close-up of a blooming rose covered in morning dew, soft natural light.",
|
|
||||||
"A peaceful mountain village at dawn, mist rolling over the rooftops, cinematic establishing shot.",
|
|
||||||
"A skateboarder doing a kickflip in an urban plaza, slow motion, golden hour lighting.",
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def _build_parser() -> argparse.ArgumentParser:
|
|
||||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
||||||
p.add_argument("--model", required=True, help="Input Wan2.2 diffusers directory or HF id.")
|
|
||||||
p.add_argument("--output", required=True, help="Output directory for the ModelOpt FP8 checkpoint.")
|
|
||||||
p.add_argument("--dtype", choices=("bfloat16", "float16"), default="bfloat16")
|
|
||||||
p.add_argument("--height", type=int, default=704, help="Calibration video height (Wan2.2 TI2V-5B native: 704).")
|
|
||||||
p.add_argument("--width", type=int, default=1280, help="Calibration video width (Wan2.2 TI2V-5B native: 1280).")
|
|
||||||
p.add_argument(
|
|
||||||
"--num-frames",
|
|
||||||
type=int,
|
|
||||||
default=49,
|
|
||||||
help="Frames per calibration sample. 49 matches the typical short benchmark; "
|
|
||||||
"use 17 to reduce memory pressure during calibration.",
|
|
||||||
)
|
|
||||||
p.add_argument("--guidance-scale", type=float, default=5.0)
|
|
||||||
p.add_argument(
|
|
||||||
"--calib-steps",
|
|
||||||
type=int,
|
|
||||||
default=10,
|
|
||||||
help="Denoising steps per calibration prompt (10 is enough for amax statistics).",
|
|
||||||
)
|
|
||||||
p.add_argument("--calib-size", type=int, default=8, help="How many prompts to use for calibration.")
|
|
||||||
p.add_argument("--seed", type=int, default=42)
|
|
||||||
p.add_argument(
|
|
||||||
"--prompt",
|
|
||||||
action="append",
|
|
||||||
default=[],
|
|
||||||
help="Custom calibration prompt. Repeat to provide multiple.",
|
|
||||||
)
|
|
||||||
p.add_argument(
|
|
||||||
"--quantize-mha",
|
|
||||||
action="store_true",
|
|
||||||
help="Enable FP8 attention K/V/softmax quantizers. Off by default — Wan2.2's long attention "
|
|
||||||
"sequences amplified FP8 drift in the online ablation (see #2920).",
|
|
||||||
)
|
|
||||||
p.add_argument(
|
|
||||||
"--weight-block-size",
|
|
||||||
type=str,
|
|
||||||
default=None,
|
|
||||||
help="Per-block weight quantization as 'M,N' (e.g. '128,128'). Default per-tensor. "
|
|
||||||
"Note: vllm-omni's ModelOpt adapter may not yet dispatch block-wise scales — check #2924 "
|
|
||||||
"for the HV-1.5 investigation status before relying on this.",
|
|
||||||
)
|
|
||||||
p.add_argument("--overwrite", action="store_true", help="Replace an existing output directory.")
|
|
||||||
return p
|
|
||||||
|
|
||||||
|
|
||||||
def _parse_block_size(spec: str | None) -> list[int] | None:
|
|
||||||
if spec is None:
|
|
||||||
return None
|
|
||||||
parts = [int(x) for x in spec.split(",") if x.strip()]
|
|
||||||
if len(parts) != 2:
|
|
||||||
raise SystemExit(f"--weight-block-size must be 'M,N' (2 ints), got {spec!r}")
|
|
||||||
return parts
|
|
||||||
|
|
||||||
|
|
||||||
def _require_modelopt() -> Any:
|
|
||||||
try:
|
|
||||||
import modelopt.torch.quantization as mtq
|
|
||||||
except ModuleNotFoundError as exc:
|
|
||||||
raise SystemExit(
|
|
||||||
"NVIDIA ModelOpt is not installed. Install with:\n"
|
|
||||||
" pip install 'nvidia-modelopt[all]'\n"
|
|
||||||
f"Original error: {exc}"
|
|
||||||
) from exc
|
|
||||||
return mtq
|
|
||||||
|
|
||||||
|
|
||||||
def _ensure_paths(args: argparse.Namespace) -> tuple[str, Path]:
|
|
||||||
model_path = args.model
|
|
||||||
output_dir = Path(args.output).expanduser().resolve()
|
|
||||||
if output_dir.exists():
|
|
||||||
if not args.overwrite:
|
|
||||||
raise SystemExit(f"Output directory already exists: {output_dir}\nPass --overwrite to replace it.")
|
|
||||||
shutil.rmtree(output_dir)
|
|
||||||
return model_path, output_dir
|
|
||||||
|
|
||||||
|
|
||||||
def _select_dtype(name: str) -> torch.dtype:
|
|
||||||
return {"bfloat16": torch.bfloat16, "float16": torch.float16}[name]
|
|
||||||
|
|
||||||
|
|
||||||
def _build_prompts(args: argparse.Namespace) -> list[str]:
|
|
||||||
prompts = args.prompt or DEFAULT_PROMPTS
|
|
||||||
if args.calib_size <= 0:
|
|
||||||
raise SystemExit("--calib-size must be positive.")
|
|
||||||
if len(prompts) < args.calib_size:
|
|
||||||
repeats = (args.calib_size + len(prompts) - 1) // len(prompts)
|
|
||||||
prompts = (prompts * repeats)[: args.calib_size]
|
|
||||||
return prompts[: args.calib_size]
|
|
||||||
|
|
||||||
|
|
||||||
# Layers to KEEP at full precision. Wan2.2's module naming:
|
|
||||||
# - condition_embedder: time_embedder, time_proj, text_embedder, image_embedder (I2V)
|
|
||||||
# - patch_embedding: Conv3dLayer (already not Linear, belt-and-suspenders skip)
|
|
||||||
# - scale_shift_table: nn.Parameter modulation (not Linear, but pattern guard)
|
|
||||||
# - norm_out: AdaLayerNorm final
|
|
||||||
# - proj_out: final nn.Linear
|
|
||||||
# - timestep_proj_prepare / output_scale_shift_prepare: SP helpers
|
|
||||||
def _filter_func_wan22(name: str) -> bool:
|
|
||||||
pattern = re.compile(
|
|
||||||
r"(proj_out.*|"
|
|
||||||
r".*(condition_embedder|patch_embedding|"
|
|
||||||
r"norm_out|scale_shift_table|"
|
|
||||||
r"timestep_proj_prepare|output_scale_shift_prepare).*)"
|
|
||||||
)
|
|
||||||
return pattern.match(name) is not None
|
|
||||||
|
|
||||||
|
|
||||||
def _mha_filter_func(name: str) -> bool:
|
|
||||||
pattern = re.compile(
|
|
||||||
r".*(q_bmm_quantizer|k_bmm_quantizer|v_bmm_quantizer|softmax_quantizer|bmm2_output_quantizer).*"
|
|
||||||
)
|
|
||||||
return pattern.match(name) is not None
|
|
||||||
|
|
||||||
|
|
||||||
def _disable_known_problematic_quantizers(mtq: Any, backbone: torch.nn.Module, *, quantize_mha: bool) -> None:
|
|
||||||
if not hasattr(mtq, "disable_quantizer"):
|
|
||||||
return
|
|
||||||
mtq.disable_quantizer(backbone, _filter_func_wan22)
|
|
||||||
if not quantize_mha:
|
|
||||||
mtq.disable_quantizer(backbone, _mha_filter_func)
|
|
||||||
|
|
||||||
|
|
||||||
def _load_pipeline(model_path: str, dtype: torch.dtype) -> DiffusionPipeline:
|
|
||||||
pipe = DiffusionPipeline.from_pretrained(model_path, torch_dtype=dtype)
|
|
||||||
if hasattr(pipe, "set_progress_bar_config"):
|
|
||||||
pipe.set_progress_bar_config(disable=True)
|
|
||||||
pipe.to("cuda")
|
|
||||||
return pipe
|
|
||||||
|
|
||||||
|
|
||||||
def _build_forward_loop(pipe: DiffusionPipeline, args: argparse.Namespace, prompts: list[str]):
|
|
||||||
generator = torch.Generator(device="cuda")
|
|
||||||
|
|
||||||
# Try setting guidance on the pipeline's guider if present (newer diffusers APIs).
|
|
||||||
guider = getattr(pipe, "guider", None)
|
|
||||||
if guider is not None and hasattr(guider, "guidance_scale"):
|
|
||||||
try:
|
|
||||||
guider.guidance_scale = args.guidance_scale
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
base_kwargs = dict(
|
|
||||||
height=args.height,
|
|
||||||
width=args.width,
|
|
||||||
num_frames=args.num_frames,
|
|
||||||
num_inference_steps=args.calib_steps,
|
|
||||||
output_type="latent",
|
|
||||||
)
|
|
||||||
|
|
||||||
def forward_loop(*_unused_args, **_unused_kwargs) -> None:
|
|
||||||
with torch.inference_mode():
|
|
||||||
for idx, prompt in enumerate(prompts):
|
|
||||||
generator.manual_seed(args.seed + idx)
|
|
||||||
# Try with guidance_scale first; fall back without on TypeError
|
|
||||||
# for pipelines that take CFG via guider config only.
|
|
||||||
try:
|
|
||||||
pipe(prompt=prompt, generator=generator, guidance_scale=args.guidance_scale, **base_kwargs)
|
|
||||||
except TypeError as exc:
|
|
||||||
if "guidance_scale" not in str(exc):
|
|
||||||
raise
|
|
||||||
pipe(prompt=prompt, generator=generator, **base_kwargs)
|
|
||||||
|
|
||||||
return forward_loop
|
|
||||||
|
|
||||||
|
|
||||||
def _summarize_export(output_dir: Path) -> None:
|
|
||||||
cfg_path = output_dir / "transformer" / "config.json"
|
|
||||||
if not cfg_path.exists():
|
|
||||||
print(f"[warn] {cfg_path} missing.", file=sys.stderr)
|
|
||||||
return
|
|
||||||
with cfg_path.open(encoding="utf-8") as f:
|
|
||||||
cfg = json.load(f)
|
|
||||||
qc = cfg.get("quantization_config")
|
|
||||||
if not isinstance(qc, dict):
|
|
||||||
print("[warn] No quantization_config in transformer/config.json.", file=sys.stderr)
|
|
||||||
return
|
|
||||||
print("Export summary:")
|
|
||||||
print(f" quant_method: {qc.get('quant_method')}")
|
|
||||||
print(f" quant_algo: {qc.get('quant_algo')}")
|
|
||||||
producer = qc.get("producer")
|
|
||||||
if isinstance(producer, dict):
|
|
||||||
print(f" producer: {producer.get('name')} {producer.get('version')}")
|
|
||||||
print(f" config path: {cfg_path}")
|
|
||||||
|
|
||||||
|
|
||||||
def _force_export_quantized_weights(backbone: torch.nn.Module, dtype: torch.dtype) -> int:
|
|
||||||
"""Convert in-memory weights of quantized modules to actual FP8 storage.
|
|
||||||
|
|
||||||
`export_hf_checkpoint` skips this step for unknown model types (Wan2.2 isn't
|
|
||||||
in ModelOpt's recognized-model registry), so we must call the per-weight
|
|
||||||
export helper ourselves. Same workaround as the HunyuanVideo-1.5 / HunyuanImage-3
|
|
||||||
calibration helpers.
|
|
||||||
"""
|
|
||||||
from modelopt.torch.export.quant_utils import (
|
|
||||||
QUANTIZATION_NONE,
|
|
||||||
get_quantization_format,
|
|
||||||
quantizer_attr_names,
|
|
||||||
weight_attr_names,
|
|
||||||
)
|
|
||||||
from modelopt.torch.export.unified_export_hf import _export_quantized_weight
|
|
||||||
|
|
||||||
exported = 0
|
|
||||||
for name, module in backbone.named_modules():
|
|
||||||
try:
|
|
||||||
quantization_format = get_quantization_format(module)
|
|
||||||
except Exception as exc:
|
|
||||||
print(f"[warn] Could not inspect quantization format for {name}: {exc}", file=sys.stderr)
|
|
||||||
continue
|
|
||||||
if quantization_format == QUANTIZATION_NONE:
|
|
||||||
continue
|
|
||||||
for weight_name in weight_attr_names(module):
|
|
||||||
quantizer_attrs = quantizer_attr_names(weight_name)
|
|
||||||
weight_quantizer = getattr(module, quantizer_attrs.weight_quantizer, None)
|
|
||||||
if weight_quantizer is None or not getattr(weight_quantizer, "is_enabled", False):
|
|
||||||
continue
|
|
||||||
_export_quantized_weight(module, dtype, weight_name)
|
|
||||||
exported += 1
|
|
||||||
return exported
|
|
||||||
|
|
||||||
|
|
||||||
def _wan22_quant_config_block(weight_block_size: list[int] | None = None) -> dict:
|
|
||||||
"""Mirror ModelOpt FP8 metadata expected by vllm-omni's adapter (#2913)."""
|
|
||||||
weights_cfg: dict = {"dynamic": False, "num_bits": 8, "type": "float"}
|
|
||||||
if weight_block_size is not None:
|
|
||||||
weights_cfg["strategy"] = "block"
|
|
||||||
weights_cfg["block_structure"] = f"{weight_block_size[0]}x{weight_block_size[1]}"
|
|
||||||
return {
|
|
||||||
"config_groups": {
|
|
||||||
"group_0": {
|
|
||||||
"input_activations": {"dynamic": False, "num_bits": 8, "type": "float"},
|
|
||||||
"weights": weights_cfg,
|
|
||||||
"targets": ["Linear"],
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"ignore": [
|
|
||||||
"condition_embedder*",
|
|
||||||
"norm_out*",
|
|
||||||
"output_scale_shift_prepare*",
|
|
||||||
"patch_embedding*",
|
|
||||||
"proj_out*",
|
|
||||||
"scale_shift_table*",
|
|
||||||
"timestep_proj_prepare*",
|
|
||||||
],
|
|
||||||
"producer": {"name": "modelopt"},
|
|
||||||
"quant_algo": "FP8",
|
|
||||||
"quant_method": "modelopt",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _patch_quant_config(output_dir: Path, weight_block_size: list[int] | None = None) -> None:
|
|
||||||
"""Inject quant_algo: FP8 + config_groups into each transformer's config.json
|
|
||||||
so vllm-omni's adapter (#2913) recognises the checkpoint as ModelOpt FP8.
|
|
||||||
|
|
||||||
Patches both ``transformer`` and ``transformer_2`` (A14B) when present."""
|
|
||||||
for sub in ("transformer", "transformer_2"):
|
|
||||||
cfg_path = output_dir / sub / "config.json"
|
|
||||||
if not cfg_path.exists():
|
|
||||||
continue
|
|
||||||
with cfg_path.open(encoding="utf-8") as f:
|
|
||||||
cfg = json.load(f)
|
|
||||||
|
|
||||||
new_qc = _wan22_quant_config_block(weight_block_size=weight_block_size)
|
|
||||||
existing = cfg.get("quantization_config")
|
|
||||||
if isinstance(existing, dict):
|
|
||||||
producer = existing.get("producer")
|
|
||||||
if isinstance(producer, dict):
|
|
||||||
new_qc["producer"] = producer
|
|
||||||
|
|
||||||
cfg["quantization_config"] = new_qc
|
|
||||||
with cfg_path.open("w", encoding="utf-8") as f:
|
|
||||||
json.dump(cfg, f, indent=2)
|
|
||||||
|
|
||||||
|
|
||||||
def _list_transformers(pipe: DiffusionPipeline) -> list[tuple[str, torch.nn.Module]]:
|
|
||||||
"""Return [(attr_name, module)] for every DiT on the pipeline.
|
|
||||||
|
|
||||||
Wan2.2 A14B has both ``transformer`` (low-noise) and ``transformer_2``
|
|
||||||
(high-noise). TI2V-5B has only ``transformer``.
|
|
||||||
"""
|
|
||||||
out = []
|
|
||||||
for attr in ("transformer", "transformer_2"):
|
|
||||||
mod = getattr(pipe, attr, None)
|
|
||||||
if mod is not None:
|
|
||||||
out.append((attr, mod))
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
def _save_pipeline_with_fp8_transformer(
|
|
||||||
pipe: DiffusionPipeline,
|
|
||||||
model_path: str,
|
|
||||||
output_dir: Path,
|
|
||||||
max_shard_size: str = "5GB",
|
|
||||||
) -> None:
|
|
||||||
"""Copy source dir verbatim minus transformer dirs, then save each quantized DiT.
|
|
||||||
|
|
||||||
Handles A14B's dual transformer (``transformer`` + ``transformer_2``) as well
|
|
||||||
as the single-DiT TI2V-5B.
|
|
||||||
"""
|
|
||||||
from modelopt.torch.export.diffusers_utils import hide_quantizers_from_state_dict
|
|
||||||
|
|
||||||
src = Path(model_path)
|
|
||||||
if not src.exists():
|
|
||||||
from huggingface_hub import snapshot_download
|
|
||||||
|
|
||||||
src = Path(snapshot_download(model_path))
|
|
||||||
|
|
||||||
if output_dir.exists():
|
|
||||||
shutil.rmtree(output_dir)
|
|
||||||
shutil.copytree(src, output_dir, ignore=shutil.ignore_patterns("transformer", "transformer_2"))
|
|
||||||
|
|
||||||
for attr, backbone in _list_transformers(pipe):
|
|
||||||
transformer_out = output_dir / attr
|
|
||||||
# Pass the nn.Module (transformer), not the Pipeline wrapper.
|
|
||||||
with hide_quantizers_from_state_dict(backbone):
|
|
||||||
backbone.save_pretrained(
|
|
||||||
str(transformer_out),
|
|
||||||
safe_serialization=True,
|
|
||||||
max_shard_size=max_shard_size,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
args = _build_parser().parse_args()
|
|
||||||
if not torch.cuda.is_available():
|
|
||||||
raise SystemExit("CUDA is required for ModelOpt FP8 quantization.")
|
|
||||||
|
|
||||||
mtq = _require_modelopt()
|
|
||||||
model_path, output_dir = _ensure_paths(args)
|
|
||||||
dtype = _select_dtype(args.dtype)
|
|
||||||
prompts = _build_prompts(args)
|
|
||||||
weight_block_size = _parse_block_size(args.weight_block_size)
|
|
||||||
|
|
||||||
print("Quantization plan:")
|
|
||||||
print(f" input: {args.model}")
|
|
||||||
print(f" output: {output_dir}")
|
|
||||||
print(f" dtype: {dtype}")
|
|
||||||
print(f" height/width: {args.height}x{args.width}")
|
|
||||||
print(f" num_frames: {args.num_frames}")
|
|
||||||
print(f" calib_size: {len(prompts)}")
|
|
||||||
print(f" calib_steps: {args.calib_steps}")
|
|
||||||
print(f" quantize_mha: {args.quantize_mha}")
|
|
||||||
print(
|
|
||||||
f" weight strategy: {'block-wise ' + str(weight_block_size) if weight_block_size else 'per-tensor (default)'}"
|
|
||||||
)
|
|
||||||
|
|
||||||
pipe = _load_pipeline(model_path, dtype)
|
|
||||||
transformers = _list_transformers(pipe)
|
|
||||||
if not transformers:
|
|
||||||
raise SystemExit("Pipeline has no transformer or transformer_2 attribute.")
|
|
||||||
print(f" found {len(transformers)} transformer(s): {', '.join(a for a, _ in transformers)}")
|
|
||||||
|
|
||||||
quant_config = copy.deepcopy(mtq.FP8_DEFAULT_CFG)
|
|
||||||
if weight_block_size is not None:
|
|
||||||
quant_config["quant_cfg"]["*weight_quantizer"] = {
|
|
||||||
"num_bits": (4, 3),
|
|
||||||
"block_sizes": {-1: weight_block_size[1], -2: weight_block_size[0]},
|
|
||||||
}
|
|
||||||
print(
|
|
||||||
f" -> overriding weight quantizer with block_sizes={weight_block_size} "
|
|
||||||
f"({weight_block_size[0]}x{weight_block_size[1]} tiles)"
|
|
||||||
)
|
|
||||||
|
|
||||||
forward_loop = _build_forward_loop(pipe, args, prompts)
|
|
||||||
|
|
||||||
# Quantize each DiT. The forward_loop runs the full pipeline, so quantizers
|
|
||||||
# on every attached transformer (A14B: transformer + transformer_2) get
|
|
||||||
# calibrated during the same passes.
|
|
||||||
for attr, backbone in transformers:
|
|
||||||
print(f"\nQuantizing {attr}...")
|
|
||||||
quantized = mtq.quantize(backbone, quant_config, forward_loop)
|
|
||||||
if quantized is not None:
|
|
||||||
setattr(pipe, attr, quantized)
|
|
||||||
_disable_known_problematic_quantizers(mtq, getattr(pipe, attr), quantize_mha=args.quantize_mha)
|
|
||||||
|
|
||||||
# Export weights to FP8 only AFTER all transformers are calibrated. Exporting
|
|
||||||
# inside the loop would convert transformer's weights to real FP8, and the
|
|
||||||
# next transformer's forward_loop (full pipeline) would then fake-quantize an
|
|
||||||
# already-FP8 weight -> "fake_e4m3fy not implemented for Float8_e4m3fn".
|
|
||||||
print("\nForcing FP8 weight serialization (Wan2.2 isn't in ModelOpt's recognized-model registry,")
|
|
||||||
print("so we have to call the per-weight export helper ourselves)...")
|
|
||||||
total_exported = 0
|
|
||||||
for attr, backbone in _list_transformers(pipe):
|
|
||||||
exported = _force_export_quantized_weights(backbone, dtype)
|
|
||||||
print(f" -> {exported} weights converted to FP8 in {attr}")
|
|
||||||
total_exported += exported
|
|
||||||
|
|
||||||
if total_exported == 0:
|
|
||||||
raise SystemExit(
|
|
||||||
"No quantized weights were exported. Calibration may have skipped every layer "
|
|
||||||
"(check the disable_quantizer regex) or `mtq.quantize` did not actually wrap any "
|
|
||||||
"weight quantizers."
|
|
||||||
)
|
|
||||||
|
|
||||||
print("\nSaving pipeline with FP8 transformer(s)...")
|
|
||||||
_save_pipeline_with_fp8_transformer(pipe, model_path, output_dir)
|
|
||||||
_patch_quant_config(output_dir, weight_block_size=weight_block_size)
|
|
||||||
print(f"Saved to: {output_dir}")
|
|
||||||
_summarize_export(output_dir)
|
|
||||||
|
|
||||||
print("\nNext: validate the checkpoint with vllm-omni:")
|
|
||||||
print(
|
|
||||||
" python examples/offline_inference/text_to_video/text_to_video.py \\\n"
|
|
||||||
f" --model {output_dir} \\\n"
|
|
||||||
" --quantization fp8 \\\n"
|
|
||||||
" --prompt 'A dog running across a field of golden wheat.' \\\n"
|
|
||||||
f" --height {args.height} --width {args.width} --num-frames {args.num_frames} \\\n"
|
|
||||||
" --num-inference-steps 30 --guidance-scale 5.0 --seed 42 \\\n"
|
|
||||||
" --output outputs/wan22_modelopt_fp8.mp4"
|
|
||||||
)
|
|
||||||
print(
|
|
||||||
"\n (--quantization fp8 is auto-upgraded to ModelOpt FP8 at runtime because the "
|
|
||||||
"checkpoint's config.json has modelopt metadata.)"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
+1
-9
@@ -64,9 +64,7 @@ dev = [
|
|||||||
"lpips>=0.1.4", # for quantization fp8
|
"lpips>=0.1.4", # for quantization fp8
|
||||||
"orjson>=3.11.7", # to make v1/embeddings API fast
|
"orjson>=3.11.7", # to make v1/embeddings API fast
|
||||||
"voxcpm>=2.0.2", # for VoxCPM2 path-locating in tests
|
"voxcpm>=2.0.2", # for VoxCPM2 path-locating in tests
|
||||||
"torchdiffeq>=0.2.5", # for covo_audio Test
|
"torchdiffeq>=0.2.5" # for covo_audio Test
|
||||||
"FlagEmbedding", # for hunyuan_image3 tests
|
|
||||||
"kernels<0.15.0" # for accuracy tests comparison with diffusers output (0.15.0 incompatible with diffusers 0.38.0)
|
|
||||||
]
|
]
|
||||||
|
|
||||||
demo = [
|
demo = [
|
||||||
@@ -90,12 +88,6 @@ minicpmo = [
|
|||||||
"stepaudio2-minicpmo",
|
"stepaudio2-minicpmo",
|
||||||
]
|
]
|
||||||
|
|
||||||
# Optional Blackwell-only fused-bias FP8 GEMM; package is `quack-kernels` (imported
|
|
||||||
# as `quack`), not the unrelated `quack`. See docs/user_guide/quantization/fp8.md.
|
|
||||||
quack = [
|
|
||||||
"quack-kernels>=0.3.11",
|
|
||||||
]
|
|
||||||
|
|
||||||
docs = [
|
docs = [
|
||||||
"mkdocs>=1.5.0",
|
"mkdocs>=1.5.0",
|
||||||
"mkdocs-api-autonav",
|
"mkdocs-api-autonav",
|
||||||
|
|||||||
@@ -1,110 +0,0 @@
|
|||||||
# Higgs-Audio V3 TTS
|
|
||||||
|
|
||||||
> Multilingual text-to-speech with voice cloning on 1×H100
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
|
|
||||||
- Vendor: Boson AI
|
|
||||||
- Model: `bosonai/higgs-audio-v3-tts-4b`
|
|
||||||
- Task: Text-to-speech synthesis with optional voice cloning (100+ languages)
|
|
||||||
- Mode: Online serving with the OpenAI-compatible `/v1/audio/speech` API; offline batch inference
|
|
||||||
- Maintainer: @yuekaiz
|
|
||||||
|
|
||||||
## When to use this recipe
|
|
||||||
|
|
||||||
Use this recipe to serve `bosonai/higgs-audio-v3-tts-4b` for high-quality
|
|
||||||
multilingual TTS. The model generates 24 kHz speech, supports zero-shot voice
|
|
||||||
cloning from a reference clip, and handles 100+ languages with inline control
|
|
||||||
tokens for emotion, style, and prosody. The architecture is a ~4B Qwen3
|
|
||||||
backbone with fused multi-codebook embedding/head (8 codebooks × 1026 vocab,
|
|
||||||
MusicGen-style delay pattern).
|
|
||||||
|
|
||||||
## References
|
|
||||||
|
|
||||||
- Model card: [bosonai/higgs-audio-v3-tts-4b](https://huggingface.co/bosonai/higgs-audio-v3-tts-4b)
|
|
||||||
- Offline example: [`examples/offline_inference/text_to_speech/higgs_audio_v3/end2end.py`](../../examples/offline_inference/text_to_speech/higgs_audio_v3/end2end.py)
|
|
||||||
- Online example: [`examples/online_serving/text_to_speech/higgs_audio_v3/`](../../examples/online_serving/text_to_speech/higgs_audio_v3/)
|
|
||||||
- Benchmark results: see Performance section below
|
|
||||||
|
|
||||||
## Hardware Support
|
|
||||||
|
|
||||||
## GPU
|
|
||||||
|
|
||||||
### 1×H100 80GB
|
|
||||||
|
|
||||||
#### Environment
|
|
||||||
|
|
||||||
- OS: Linux
|
|
||||||
- Python: 3.12+
|
|
||||||
- CUDA: 12.x
|
|
||||||
- vLLM version: 0.22.0
|
|
||||||
- vLLM-Omni version or commit: `36e048fd` (branch `higgs-v3`)
|
|
||||||
|
|
||||||
#### Command
|
|
||||||
|
|
||||||
**Online serving:**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
vllm-omni serve bosonai/higgs-audio-v3-tts-4b \
|
|
||||||
--host 0.0.0.0 --port 8095 \
|
|
||||||
--trust-remote-code --omni
|
|
||||||
```
|
|
||||||
|
|
||||||
The default deploy config `vllm_omni/deploy/higgs_multimodal_qwen3.yaml` is
|
|
||||||
loaded automatically by model registry (HF `model_type=higgs_multimodal_qwen3`).
|
|
||||||
|
|
||||||
**Offline batch inference:**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python examples/offline_inference/text_to_speech/higgs_audio_v3/end2end.py \
|
|
||||||
--texts "Hello world." "The quick brown fox jumps over the lazy dog." \
|
|
||||||
--output-dir results/higgs_v3_wavs
|
|
||||||
```
|
|
||||||
|
|
||||||
**Offline voice clone:**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python examples/offline_inference/text_to_speech/higgs_audio_v3/end2end.py \
|
|
||||||
--texts "Text to synthesize in the cloned voice." \
|
|
||||||
--ref-audio path/to/reference.wav \
|
|
||||||
--ref-text "Transcript of the reference clip." \
|
|
||||||
--output-dir results/higgs_v3_clone
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Verification
|
|
||||||
|
|
||||||
Basic TTS via curl:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl -X POST http://localhost:8095/v1/audio/speech \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{
|
|
||||||
"model": "bosonai/higgs-audio-v3-tts-4b",
|
|
||||||
"input": "Hello, how are you?"
|
|
||||||
}' --output hello.wav
|
|
||||||
```
|
|
||||||
|
|
||||||
Voice clone via Python client:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python examples/online_serving/text_to_speech/higgs_audio_v3/batch_speech_client.py \
|
|
||||||
--base-url http://localhost:8095 \
|
|
||||||
--model bosonai/higgs-audio-v3-tts-4b \
|
|
||||||
--ref-audio path/to/reference.wav \
|
|
||||||
--ref-text "Transcript of the reference." \
|
|
||||||
--prompts "Text to clone."
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Notes
|
|
||||||
|
|
||||||
- Memory usage: Stage 0 (talker, ~4B) uses ~60% GPU memory; Stage 1 (codec decoder) uses ~25%.
|
|
||||||
- Key flags: `--trust-remote-code` and `--omni` are required.
|
|
||||||
- Output: 24 kHz mono WAV.
|
|
||||||
- Voice cloning: `ref_audio` accepts WAV/FLAC/MP3; `ref_text` is optional but improves fidelity.
|
|
||||||
- Deploy config: `vllm_omni/deploy/higgs_multimodal_qwen3.yaml` (auto-discovered from `model_type`).
|
|
||||||
- `max_num_seqs=16` for both stages.
|
|
||||||
- `enforce_eager=true` for both stages (CUDA graph available for stage 0 but no throughput gain at batch>1).
|
|
||||||
- Known limitations:
|
|
||||||
- CUDA graph for stage 0 helps batch=1 (-13% RTF) but not batch>1 due to model-owned sampler running outside graph.
|
|
||||||
- Stage 1 (code2wav) must use `enforce_eager=true` (`@torch.inference_mode` incompatible with graph capture).
|
|
||||||
- Async streaming (chunk-based) not yet implemented; sync-only pipeline.
|
|
||||||
+1
-3
@@ -36,8 +36,7 @@ recipes/
|
|||||||
| [`LTX/LTX-2.md`](./LTX/LTX-2.md) | Text-to-video and image-to-video serving | 1x H200 141GB |
|
| [`LTX/LTX-2.md`](./LTX/LTX-2.md) | Text-to-video and image-to-video serving | 1x H200 141GB |
|
||||||
| [`LTX/LTX-2.3.md`](./LTX/LTX-2.3.md) | Text-to-video with audio generation (22B) | 1x GPU (96GB VRAM) |
|
| [`LTX/LTX-2.3.md`](./LTX/LTX-2.3.md) | Text-to-video with audio generation (22B) | 1x GPU (96GB VRAM) |
|
||||||
| [`mistralai/Voxtral-TTS.md`](./mistralai/Voxtral-TTS.md) | Online serving for TTS | 1x RTX 4090 24GB |
|
| [`mistralai/Voxtral-TTS.md`](./mistralai/Voxtral-TTS.md) | Online serving for TTS | 1x RTX 4090 24GB |
|
||||||
| [`cosmos3/Cosmos3-Nano.md`](./cosmos3/Cosmos3-Nano.md) | Text-to-image, text-to-video, image-to-video, video-to-video generation, text to video with sound, action policy | 1x H200 141GB / B300 |
|
| [`nvidia/Cosmos3-Nano.md`](./nvidia/Cosmos3-Nano.md) | Text-to-image, text-to-video, and image-to-video generation | 1x H200 141GB / B300 |
|
||||||
| [`cosmos3/Cosmos3-Super.md`](./cosmos3/Cosmos3-Super.md) | 64B T2I / T2V / I2V / V2V generation (+ optional audio) / Action policy | 8x H200/H100/A100 / 2x H200 / B300 |
|
|
||||||
| [`OpenBMB/MiniCPM-o-4_5.md`](./OpenBMB/MiniCPM-o-4_5.md) | Online serving for omni multimodal chat (text / image / audio / video → text + 24 kHz speech) | 2x A100/H100 80GB / 3x mid-tier GPU / 8x RTX 4090 24GB |
|
| [`OpenBMB/MiniCPM-o-4_5.md`](./OpenBMB/MiniCPM-o-4_5.md) | Online serving for omni multimodal chat (text / image / audio / video → text + 24 kHz speech) | 2x A100/H100 80GB / 3x mid-tier GPU / 8x RTX 4090 24GB |
|
||||||
| [`OpenBMB/VoxCPM2.md`](./OpenBMB/VoxCPM2.md) | Online + offline TTS with native AR pipeline (48 kHz, 30+ languages) | 1x RTX 4090 24GB |
|
| [`OpenBMB/VoxCPM2.md`](./OpenBMB/VoxCPM2.md) | Online + offline TTS with native AR pipeline (48 kHz, 30+ languages) | 1x RTX 4090 24GB |
|
||||||
| [`Qwen/Qwen-Image.md`](./Qwen/Qwen-Image.md) | Text-to-image serving with step-wise continuous batching replay and ModelOpt mixed FP8/NVFP4 | 1x A100 80GB / 2x B200 |
|
| [`Qwen/Qwen-Image.md`](./Qwen/Qwen-Image.md) | Text-to-image serving with step-wise continuous batching replay and ModelOpt mixed FP8/NVFP4 | 1x A100 80GB / 2x B200 |
|
||||||
@@ -50,7 +49,6 @@ recipes/
|
|||||||
| [`Tencent/HunyuanImage-3.0-Instruct.md`](./Tencent/HunyuanImage-3.0-Instruct.md) | DiT-only text-to-image serving and benchmark, including ModelOpt mixed FP8/NVFP4 | 4x H100/H800 80GB / 2x B200 |
|
| [`Tencent/HunyuanImage-3.0-Instruct.md`](./Tencent/HunyuanImage-3.0-Instruct.md) | DiT-only text-to-image serving and benchmark, including ModelOpt mixed FP8/NVFP4 | 4x H100/H800 80GB / 2x B200 |
|
||||||
| [`Wan-AI/Wan2.2-I2V.md`](./Wan-AI/Wan2.2-I2V.md) | Image-to-video serving (Wan2.2 14B) | 8x Ascend NPU (A2/A3) |
|
| [`Wan-AI/Wan2.2-I2V.md`](./Wan-AI/Wan2.2-I2V.md) | Image-to-video serving (Wan2.2 14B) | 8x Ascend NPU (A2/A3) |
|
||||||
| [`Wan-AI/Wan2.2-S2V.md`](./Wan-AI/Wan2.2-S2V.md) | Speech-to-video serving (Wan2.2 14B) | 2x A100/H100 80GB |
|
| [`Wan-AI/Wan2.2-S2V.md`](./Wan-AI/Wan2.2-S2V.md) | Speech-to-video serving (Wan2.2 14B) | 2x A100/H100 80GB |
|
||||||
| [`StabilityAI/Stable-Diffusion-3.5.md`](./StabilityAI/Stable-Diffusion-3.5.md) | Text-to-image serving (SD 3.5-medium and SD 3.5-large) | 1x RTX A6000 48GB |
|
|
||||||
| [`zai-org/GLM-TTS.md`](./zai-org/GLM-TTS.md) | Online serving for Chinese/English zero-shot voice-cloned TTS | 1x A40 48GB |
|
| [`zai-org/GLM-TTS.md`](./zai-org/GLM-TTS.md) | Online serving for Chinese/English zero-shot voice-cloned TTS | 1x A40 48GB |
|
||||||
| [`GLM/GLM-Image.md`](./GLM/GLM-Image.md) | Online serving for image generation | 1x A800 80GB / 2x A800 80GB |
|
| [`GLM/GLM-Image.md`](./GLM/GLM-Image.md) | Online serving for image generation | 1x A800 80GB / 2x A800 80GB |
|
||||||
|
|
|
|
||||||
|
|||||||
@@ -1,139 +0,0 @@
|
|||||||
# Stable Diffusion 3.5
|
|
||||||
|
|
||||||
> Text-to-image serving with Stability AI's SD 3.5 family
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
|
|
||||||
- Vendor: Stability AI
|
|
||||||
- Model: `stabilityai/stable-diffusion-3.5-medium`, `stabilityai/stable-diffusion-3.5-large`
|
|
||||||
- Task: Text-to-image generation
|
|
||||||
- Mode: Online serving with the OpenAI-compatible images API
|
|
||||||
- Maintainer: Community
|
|
||||||
|
|
||||||
## When to use this recipe
|
|
||||||
|
|
||||||
Use this recipe when you want a lightweight, fast text-to-image model that
|
|
||||||
fits on a single consumer or workstation GPU. SD 3.5-medium requires only
|
|
||||||
~17 GiB VRAM and generates 1024×1024 images in ~4 seconds, making it one of
|
|
||||||
the most accessible diffusion models in vLLM-Omni. SD 3.5-large produces
|
|
||||||
noticeably sharper images at the cost of ~34 GiB VRAM and ~21 seconds per
|
|
||||||
image.
|
|
||||||
|
|
||||||
## References
|
|
||||||
|
|
||||||
- Model (medium): <https://huggingface.co/stabilityai/stable-diffusion-3.5-medium>
|
|
||||||
- Model (large): <https://huggingface.co/stabilityai/stable-diffusion-3.5-large>
|
|
||||||
- Related example under `examples/`:
|
|
||||||
[`examples/online_serving/text_to_image/README.md`](../../examples/online_serving/text_to_image/README.md)
|
|
||||||
- User guide:
|
|
||||||
[`docs/user_guide/examples/online_serving/text_to_image.md`](../../docs/user_guide/examples/online_serving/text_to_image.md)
|
|
||||||
|
|
||||||
## Hardware Support
|
|
||||||
|
|
||||||
This recipe documents two CUDA GPU configurations on a single RTX A6000 48 GB,
|
|
||||||
one for each model variant. Extend it with more hardware sections as community
|
|
||||||
validation lands.
|
|
||||||
|
|
||||||
## GPU
|
|
||||||
|
|
||||||
### 1x NVIDIA RTX A6000 48GB — stable-diffusion-3.5-medium
|
|
||||||
|
|
||||||
#### Environment
|
|
||||||
|
|
||||||
- OS: Ubuntu 22.04
|
|
||||||
- Python: 3.12
|
|
||||||
- Driver / runtime: NVIDIA driver 580.126.09, CUDA 13.0
|
|
||||||
- vLLM version: 0.22.0
|
|
||||||
- vLLM-Omni version or commit: 0.22.0rc2.dev8+g41a795b5d
|
|
||||||
|
|
||||||
#### Command
|
|
||||||
|
|
||||||
```bash
|
|
||||||
vllm serve stabilityai/stable-diffusion-3.5-medium --omni --port 8091
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Verification
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl -X POST http://localhost:8091/v1/images/generations \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{
|
|
||||||
"model": "stabilityai/stable-diffusion-3.5-medium",
|
|
||||||
"prompt": "a cute cat sitting on a windowsill, highly detailed, sharp focus",
|
|
||||||
"negative_prompt": "blurry, out of focus",
|
|
||||||
"guidance_scale": 4.5,
|
|
||||||
"n": 1,
|
|
||||||
"size": "1024x1024"
|
|
||||||
}' | python3 -c "
|
|
||||||
import json, base64, sys
|
|
||||||
data = json.load(sys.stdin)
|
|
||||||
with open('output.png', 'wb') as f:
|
|
||||||
f.write(base64.b64decode(data['data'][0]['b64_json']))
|
|
||||||
print('saved output.png')
|
|
||||||
"
|
|
||||||
```
|
|
||||||
|
|
||||||
Or use the example client:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python examples/online_serving/text_to_image/openai_chat_client.py \
|
|
||||||
--server http://localhost:8091 \
|
|
||||||
--prompt "a cute cat sitting on a windowsill" \
|
|
||||||
--output output.png
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Notes
|
|
||||||
|
|
||||||
- Memory usage: Model weights ~15.6 GiB, peak VRAM ~17.0 GiB (no CFG), ~20.9 GiB with `guidance_scale > 1` (CFG enabled).
|
|
||||||
- Generation time: ~4 seconds without CFG, ~8 seconds with `guidance_scale=4.5`.
|
|
||||||
- Model loading time: ~55 seconds (including weight download on first run).
|
|
||||||
- `negative_prompt` requires `guidance_scale > 1` to take effect; SD3 defaults to 1.0 (no CFG).
|
|
||||||
|
|
||||||
### 1x NVIDIA RTX A6000 48GB — stable-diffusion-3.5-large
|
|
||||||
|
|
||||||
#### Environment
|
|
||||||
|
|
||||||
- OS: Ubuntu 22.04
|
|
||||||
- Python: 3.12
|
|
||||||
- Driver / runtime: NVIDIA driver 580.126.09, CUDA 13.0
|
|
||||||
- vLLM version: 0.22.0
|
|
||||||
- vLLM-Omni version or commit: 0.22.0rc2.dev8+g41a795b5d
|
|
||||||
|
|
||||||
#### Command
|
|
||||||
|
|
||||||
```bash
|
|
||||||
vllm serve stabilityai/stable-diffusion-3.5-large --omni --port 8091
|
|
||||||
```
|
|
||||||
|
|
||||||
> **Note**: `stable-diffusion-3.5-large` is a gated model. Accept the license
|
|
||||||
> on [Hugging Face](https://huggingface.co/stabilityai/stable-diffusion-3.5-large)
|
|
||||||
> and set `HF_TOKEN` before running.
|
|
||||||
|
|
||||||
#### Verification
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl -X POST http://localhost:8091/v1/images/generations \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{
|
|
||||||
"model": "stabilityai/stable-diffusion-3.5-large",
|
|
||||||
"prompt": "a cute cat sitting on a windowsill, highly detailed, sharp focus",
|
|
||||||
"negative_prompt": "blurry, out of focus",
|
|
||||||
"guidance_scale": 4.5,
|
|
||||||
"n": 1,
|
|
||||||
"size": "1024x1024"
|
|
||||||
}' | python3 -c "
|
|
||||||
import json, base64, sys
|
|
||||||
data = json.load(sys.stdin)
|
|
||||||
with open('output.png', 'wb') as f:
|
|
||||||
f.write(base64.b64decode(data['data'][0]['b64_json']))
|
|
||||||
print('saved output.png')
|
|
||||||
"
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Notes
|
|
||||||
|
|
||||||
- Memory usage: Peak VRAM ~31.6 GiB at 1024×1024, ~45.5 GiB at 2048×2048 with `guidance_scale=4.5`.
|
|
||||||
- Generation time: ~19 seconds at 1024×1024, ~102 seconds at 2048×2048.
|
|
||||||
- Image quality is noticeably better than medium, with sharper details.
|
|
||||||
- `negative_prompt` requires `guidance_scale > 1` to take effect; SD3 defaults to 1.0 (no CFG).
|
|
||||||
- 2048×2048 fits within 48 GB but leaves little headroom (~45.5 GiB used out of 49 GiB).
|
|
||||||
@@ -1,353 +0,0 @@
|
|||||||
# Cosmos3-Nano
|
|
||||||
|
|
||||||
> Text-to-image, text-to-video, image-to-video, and video-to-video serving
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
|
|
||||||
- Vendor: NVIDIA
|
|
||||||
- Model: `nvidia/Cosmos3-Nano`
|
|
||||||
- Task: Text-to-image (T2I), text-to-video (T2V), image-to-video (I2V), and video-to-video (V2V) generation, with optional synchronized audio (video + sound), action policy
|
|
||||||
- Mode: Online serving with the OpenAI-compatible image/video APIs, plus offline generation via the `Omni` API
|
|
||||||
- Maintainer: Community
|
|
||||||
|
|
||||||
## When to use this recipe
|
|
||||||
|
|
||||||
Use this recipe to deploy `nvidia/Cosmos3-Nano` for image and video generation.
|
|
||||||
A single pipeline class (`Cosmos3OmniDiffusersPipeline`) serves these modes; the
|
|
||||||
mode is selected per request:
|
|
||||||
|
|
||||||
- **T2I** — `POST /v1/images/generations` (or a prompt carrying `modalities=["image"]`).
|
|
||||||
- **T2V** — `POST /v1/videos/sync` with `num_frames > 1` and no reference image/video.
|
|
||||||
- **I2V** — `POST /v1/videos/sync` with a reference image (`input_reference` file
|
|
||||||
upload, or `image_reference` JSON).
|
|
||||||
- **V2V** — `POST /v1/videos/sync` with a reference video (`input_reference`
|
|
||||||
video upload, or `video_reference` JSON). Cosmos3 conditions on selected
|
|
||||||
reference-video latent frames; use `extra_params.condition_frame_indexes_vision`
|
|
||||||
and `extra_params.condition_video_keep` to choose which prefix/tail frames guide
|
|
||||||
generation.
|
|
||||||
- **T2VS / I2VS** — add `generate_sound=true` (and optional `sound_duration`) to a
|
|
||||||
T2V/I2V `/v1/videos/sync` request to also generate synchronized audio, muxed into
|
|
||||||
the mp4 as AAC 48 kHz stereo. See the official model card's "Video + Audio" examples.
|
|
||||||
- **Action** — pass `extra_params={"action_mode": ...}` to drive Physical-AI tasks:
|
|
||||||
- `forward_dynamics` — given a first frame or video **and** an action trajectory,
|
|
||||||
roll out the resulting video. Synchronous: `POST /v1/videos/sync`.
|
|
||||||
- `policy` — given a first frame or video and a language instruction,
|
|
||||||
**predict** the action trajectory (and a rollout video). Use the async
|
|
||||||
`POST /v1/videos` endpoint and read the predicted action from the top-level
|
|
||||||
`action` field.
|
|
||||||
- `inverse_dynamics` — given a video, **recover** the action trajectory. Use
|
|
||||||
the async `POST /v1/videos` endpoint and read the recovered action from
|
|
||||||
the top-level `action` field
|
|
||||||
(`{data, shape, dtype, raw_action_dim, domain_id}`).
|
|
||||||
|
|
||||||
Action requests also take `domain_name` (e.g. `av`, `bridge_orig_lerobot`,
|
|
||||||
`droid_lerobot`, `agibotworld`, …; or a numeric `domain_id`), `raw_action_dim`,
|
|
||||||
and `action_chunk_size` (must equal `num_frames` or `num_frames - 1`). For
|
|
||||||
`forward_dynamics` also pass the `action` array. The dedicated policy checkpoint
|
|
||||||
**`nvidia/Cosmos3-Nano-Policy-DROID`** is served the same way
|
|
||||||
(`domain_name=droid_lerobot`).
|
|
||||||
|
|
||||||
Action requests can use `input_reference` or `video_reference` for video input.
|
|
||||||
`policy` and `forward_dynamics` can also use an image reference; `inverse_dynamics`
|
|
||||||
requires a video reference.
|
|
||||||
|
|
||||||
## References
|
|
||||||
|
|
||||||
- Model card (authoritative usage + example assets): <https://huggingface.co/nvidia/Cosmos3-Nano>
|
|
||||||
- Example inputs/outputs live in the repo's `assets/` (`example_t2v_prompt.json`,
|
|
||||||
`example_i2v_prompt.json`, `example_i2v_input.jpg`, `negative_prompt.json`;
|
|
||||||
audio examples: `example_t2vs_prompt.json`, `example_t2vs_output.mp4`,
|
|
||||||
`example_i2vs_output.mp4`).
|
|
||||||
- Prompt upsampling (recommended for quality): the model expects JSON-upsampled
|
|
||||||
structured prompts; see NVIDIA's `cosmos-framework` prompt-upsampling docs.
|
|
||||||
- Pipeline: [`vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py`](../../vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py)
|
|
||||||
- Smoke tests (canonical request formats): [`tests/e2e/accuracy/test_cosmos3_similarity.py`](../../tests/e2e/accuracy/test_cosmos3_similarity.py)
|
|
||||||
|
|
||||||
## Hardware Support
|
|
||||||
|
|
||||||
## GPU
|
|
||||||
|
|
||||||
### 1x H200 141GB / B300 (Online serving)
|
|
||||||
|
|
||||||
#### Environment
|
|
||||||
|
|
||||||
- OS: Ubuntu 22.04+
|
|
||||||
- Python: 3.12+
|
|
||||||
- Driver / runtime: NVIDIA CUDA environment
|
|
||||||
- vLLM version: match the repository requirements from your current checkout
|
|
||||||
- vLLM-Omni version or commit: use the commit you are deploying from
|
|
||||||
|
|
||||||
#### Command
|
|
||||||
|
|
||||||
Requires the `vllm-omni` package (or the `vllm/vllm-omni:cosmos3` container),
|
|
||||||
which provides the `vllm serve … --omni` entrypoint used below.
|
|
||||||
|
|
||||||
Safety guardrails are **on by default** (NVIDIA Open Model License). They load
|
|
||||||
the **gated** `nvidia/Cosmos-1.0-Guardrail` model, so to keep them on you must:
|
|
||||||
|
|
||||||
1. `pip install cosmos-guardrail`
|
|
||||||
2. Accept the license at <https://huggingface.co/nvidia/Cosmos-1.0-Guardrail>
|
|
||||||
3. Export a token with access: `export HF_TOKEN=hf_...`
|
|
||||||
|
|
||||||
Then launch the recommended server:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
vllm serve nvidia/Cosmos3-Nano \
|
|
||||||
--omni \
|
|
||||||
--host 0.0.0.0 --port 8000 \
|
|
||||||
--init-timeout 1800
|
|
||||||
```
|
|
||||||
|
|
||||||
To run **without** guardrails (you are responsible for license compliance),
|
|
||||||
add `--no-guardrails` (no token/`cosmos-guardrail` needed). For extra GPUs use
|
|
||||||
`--ulysses-degree N` (context parallel) or `--tensor-parallel-size N`;
|
|
||||||
`--enable-layerwise-offload` reduces VRAM on smaller GPUs. The pipeline
|
|
||||||
auto-resolves from `model_index.json`; pass
|
|
||||||
`--model-class-name Cosmos3OmniDiffusersPipeline` to force it explicitly.
|
|
||||||
|
|
||||||
#### Verification
|
|
||||||
|
|
||||||
Best quality uses the JSON-upsampled prompts from `assets/` (download with
|
|
||||||
`hf download nvidia/Cosmos3-Nano assets/ --local-dir Cosmos3-Nano`). Minimal
|
|
||||||
self-contained examples:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl http://localhost:8000/v1/models
|
|
||||||
|
|
||||||
# Text-to-image -> /v1/images/generations (1024x1024, 50 steps; base64 PNG)
|
|
||||||
curl -sS -X POST http://localhost:8000/v1/images/generations \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{
|
|
||||||
"model": "nvidia/Cosmos3-Nano",
|
|
||||||
"prompt": "A photorealistic red sports car on a city street at golden hour, cinematic lighting.",
|
|
||||||
"negative_prompt": "blurry, distorted, low quality",
|
|
||||||
"size": "1024x1024", "n": 1, "response_format": "b64_json",
|
|
||||||
"num_inference_steps": 50, "guidance_scale": 7.0, "seed": 42
|
|
||||||
}' | python -c "import sys,json,base64; open('cosmos3_t2i.png','wb').write(base64.b64decode(json.load(sys.stdin)['data'][0]['b64_json']))"
|
|
||||||
|
|
||||||
# Text-to-video -> /v1/videos/sync (720p, 189 frames @ 24fps; official params)
|
|
||||||
curl -sS -X POST http://localhost:8000/v1/videos/sync \
|
|
||||||
-H "Accept: video/mp4" \
|
|
||||||
-F "model=nvidia/Cosmos3-Nano" \
|
|
||||||
-F "prompt=A robot arm is cleaning a plate in the kitchen" \
|
|
||||||
-F "negative_prompt=blurry, distorted, low quality, jittery, deformed" \
|
|
||||||
-F "size=1280x720" -F "num_frames=189" -F "fps=24" \
|
|
||||||
-F "num_inference_steps=35" -F "guidance_scale=6.0" \
|
|
||||||
-F "max_sequence_length=4096" -F "flow_shift=10.0" \
|
|
||||||
-F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":true}' \
|
|
||||||
-F "seed=123" \
|
|
||||||
-o cosmos3_t2v.mp4
|
|
||||||
|
|
||||||
# Image-to-video -> /v1/videos/sync with an uploaded reference image
|
|
||||||
curl -sS -X POST http://localhost:8000/v1/videos/sync \
|
|
||||||
-H "Accept: video/mp4" \
|
|
||||||
-F "model=nvidia/Cosmos3-Nano" \
|
|
||||||
-F "prompt=The scene comes to life with smooth, natural motion." \
|
|
||||||
-F "negative_prompt=blurry, distorted, low quality" \
|
|
||||||
-F "size=1280x720" -F "num_frames=189" -F "fps=24" \
|
|
||||||
-F "num_inference_steps=35" -F "guidance_scale=6.0" \
|
|
||||||
-F "max_sequence_length=4096" -F "flow_shift=10.0" \
|
|
||||||
-F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":true}' \
|
|
||||||
-F "seed=1111" \
|
|
||||||
-F "input_reference=@/path/to/reference.jpg;type=image/jpeg" \
|
|
||||||
-o cosmos3_i2v.mp4
|
|
||||||
|
|
||||||
# Video-to-video -> /v1/videos/sync with an uploaded reference video.
|
|
||||||
# By default Cosmos3 conditions on latent indexes [0, 1]. For the default
|
|
||||||
# temporal VAE stride this decodes only the first 5 input frames.
|
|
||||||
# The model works best when the prompt describes the actual situation happening in the video.
|
|
||||||
# Generic prompts may create sub-standard generations.
|
|
||||||
curl -sS -X POST http://localhost:8000/v1/videos/sync \
|
|
||||||
-H "Accept: video/mp4" \
|
|
||||||
-F "model=nvidia/Cosmos3-Nano" \
|
|
||||||
-F "prompt=Continue the same scene with smooth natural motion and consistent subjects." \
|
|
||||||
-F "negative_prompt=blurry, distorted, low quality, jittery, deformed" \
|
|
||||||
-F "size=1280x720" -F "num_frames=189" -F "fps=24" \
|
|
||||||
-F "num_inference_steps=35" -F "guidance_scale=6.0" \
|
|
||||||
-F "max_sequence_length=4096" -F "flow_shift=10.0" \
|
|
||||||
-F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":true,"condition_frame_indexes_vision":[0,1],"condition_video_keep":"first"}' \
|
|
||||||
-F "seed=2222" \
|
|
||||||
-F "input_reference=@/path/to/reference.mp4;type=video/mp4" \
|
|
||||||
-o cosmos3_v2v.mp4
|
|
||||||
|
|
||||||
# V2V can also use a JSON-safe URL/data-URL video reference. Do not combine
|
|
||||||
# video_reference with input_reference or image_reference.
|
|
||||||
curl -sS -X POST http://localhost:8000/v1/videos/sync \
|
|
||||||
-H "Accept: video/mp4" \
|
|
||||||
-F "model=nvidia/Cosmos3-Nano" \
|
|
||||||
-F "prompt=Continue the same scene with smooth natural motion and consistent subjects." \
|
|
||||||
-F "size=1280x720" -F "num_frames=189" -F "fps=24" \
|
|
||||||
-F "num_inference_steps=35" -F "guidance_scale=6.0" \
|
|
||||||
-F "max_sequence_length=4096" -F "flow_shift=10.0" \
|
|
||||||
-F 'extra_params={"condition_frame_indexes_vision":[0,1],"condition_video_keep":"last"}' \
|
|
||||||
-F 'video_reference={"video_url":"https://example.com/reference.mp4"}' \
|
|
||||||
-o cosmos3_v2v_from_url.mp4
|
|
||||||
|
|
||||||
# Text-to-video-with-sound
|
|
||||||
curl -sS -X POST http://localhost:8000/v1/videos/sync \
|
|
||||||
-H "Accept: video/mp4" \
|
|
||||||
-F "model=nvidia/Cosmos3-Nano" \
|
|
||||||
-F "prompt=The video opens with a view of a well-lit indoor fruit display. A robotic arm picks up a pear, an orange, and a carambola one by one, placing each into a plastic bag in a shopping cart with red handles. The video is 7.875 seconds long, 24 FPS, and 1280x720. Audio description: soft servo whirs, gentle fruit thuds, plastic bag rustling, and a faint refrigeration hum." \
|
|
||||||
-F "negative_prompt=blurry, distorted, low quality" \
|
|
||||||
-F "size=1280x720" \
|
|
||||||
-F "num_frames=189" \
|
|
||||||
-F "fps=24" \
|
|
||||||
-F "num_inference_steps=35" \
|
|
||||||
-F "guidance_scale=6.0" \
|
|
||||||
-F "max_sequence_length=4096" \
|
|
||||||
-F "flow_shift=10.0" \
|
|
||||||
-F "seed=0" \
|
|
||||||
-F "generate_sound=true" \
|
|
||||||
-F "sound_duration=7.875" \
|
|
||||||
-F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":true}' \
|
|
||||||
-o cosmos3_t2v_with_sound.mp4
|
|
||||||
|
|
||||||
# Action — forward dynamics (first frame + action trajectory -> rollout video).
|
|
||||||
# Synchronous; `action` is a JSON array shaped [action_chunk_size, raw_action_dim].
|
|
||||||
curl -sS -X POST http://localhost:8000/v1/videos/sync \
|
|
||||||
-H "Accept: video/mp4" \
|
|
||||||
--form-string "model=nvidia/Cosmos3-Nano" \
|
|
||||||
--form-string "prompt=You are an autonomous vehicle. This video is captured from a first-person perspective." \
|
|
||||||
-F "input_reference=@first_frame.jpg;type=image/jpeg" \
|
|
||||||
-F "size=640x480" -F "num_frames=61" -F "fps=10" \
|
|
||||||
-F "num_inference_steps=30" -F "guidance_scale=1.0" -F "flow_shift=5.0" \
|
|
||||||
--form-string "extra_params={\"action_mode\":\"forward_dynamics\",\"domain_name\":\"av\",\"raw_action_dim\":9,\"action_chunk_size\":60,\"action\":$(cat action.json)}" \
|
|
||||||
-F "seed=0" \
|
|
||||||
-o cosmos3_forward_dynamics.mp4
|
|
||||||
|
|
||||||
# Action — policy (first frame + instruction -> predicted action trajectory + video).
|
|
||||||
# Asynchronous: POST returns a job id; poll, then read the predicted action from
|
|
||||||
# the top-level `action` field ({data, shape, dtype, raw_action_dim, domain_id}).
|
|
||||||
VIDEO_ID=$(curl -sS -X POST http://localhost:8000/v1/videos \
|
|
||||||
-H "Accept: application/json" \
|
|
||||||
--form-string "model=nvidia/Cosmos3-Nano" \
|
|
||||||
--form-string "prompt=Pick up the banana and place it in the bowl." \
|
|
||||||
-F "input_reference=@first_frame.jpg;type=image/jpeg" \
|
|
||||||
-F "size=640x480" -F "num_frames=17" -F "fps=5" \
|
|
||||||
-F "num_inference_steps=30" -F "guidance_scale=1.0" -F "flow_shift=5.0" \
|
|
||||||
--form-string 'extra_params={"action_mode":"policy","domain_name":"bridge_orig_lerobot","raw_action_dim":10,"action_chunk_size":16}' \
|
|
||||||
-F "seed=0" | jq -r '.id')
|
|
||||||
# poll until status == completed, then:
|
|
||||||
curl -sS "http://localhost:8000/v1/videos/$VIDEO_ID" | jq '.action | {shape, dtype, raw_action_dim, domain_id}'
|
|
||||||
curl -sS -L "http://localhost:8000/v1/videos/$VIDEO_ID/content" -o cosmos3_policy.mp4
|
|
||||||
|
|
||||||
# Action — inverse dynamics (video -> recovered action trajectory).
|
|
||||||
# Asynchronous: use the job metadata to read the recovered action.
|
|
||||||
VIDEO_ID=$(curl -sS -X POST http://localhost:8000/v1/videos \
|
|
||||||
-H "Accept: application/json" \
|
|
||||||
--form-string "model=nvidia/Cosmos3-Nano" \
|
|
||||||
--form-string "prompt=Recover the robot action trajectory from this clip." \
|
|
||||||
-F "input_reference=@motion_clip.mp4;type=video/mp4" \
|
|
||||||
-F "size=640x480" -F "num_frames=17" -F "fps=5" \
|
|
||||||
-F "num_inference_steps=30" -F "guidance_scale=1.0" -F "flow_shift=5.0" \
|
|
||||||
--form-string 'extra_params={"action_mode":"inverse_dynamics","domain_name":"bridge_orig_lerobot","raw_action_dim":10,"action_chunk_size":16}' \
|
|
||||||
-F "seed=0" | jq -r '.id')
|
|
||||||
# poll until status == completed, then:
|
|
||||||
curl -sS "http://localhost:8000/v1/videos/$VIDEO_ID" | jq '.action | {shape, dtype, raw_action_dim, domain_id}'
|
|
||||||
curl -sS -L "http://localhost:8000/v1/videos/$VIDEO_ID/content" -o cosmos3_inverse_dynamics.mp4
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Notes
|
|
||||||
|
|
||||||
- **Measured latency (1x B300, bf16, guardrails off):**
|
|
||||||
- T2I 1024² — 10 / 25 / 50 steps → ~0.4 / 0.7 / **1.3 s**
|
|
||||||
- T2V 1280×720 @ 35 steps — 25 / 49 / 93 / **189** frames → ~7 / 15 / 33 / **~93 s**
|
|
||||||
- I2V 1280×720, 189 frames @ 35 steps → ~**99 s**
|
|
||||||
- Action 640×480 @ 30 steps — forward-dynamics 61f ~**4 s**, policy 17f ~**1–3 s**.
|
|
||||||
- Guardrails-on overhead: ~8% on T2I, negligible on video.
|
|
||||||
- **Memory:** transformer ~17 GiB (bf16); peak ~46 GiB for 720p video on 1 GPU;
|
|
||||||
full repo (transformer + Wan VAE + Qwen3-VL vision encoder + audio tokenizer)
|
|
||||||
~33 GB on disk.
|
|
||||||
- **Determinism:** identical seed reproduces identical output on the same
|
|
||||||
hardware; outputs are not bit-identical across different GPU types.
|
|
||||||
- **Supported sizes (per model card):** 256p / 480p / 720p at 16:9, 4:3, 1:1,
|
|
||||||
3:4, 9:16. Defaults: T2I 1024², 50 steps, guidance 7.0; T2V/I2V/V2V
|
|
||||||
1280×720, 189 frames, 35 steps, guidance 6.0, `flow_shift=10.0`.
|
|
||||||
- **Key flags / params:** `--no-guardrails` (server) or
|
|
||||||
`extra_params={"guardrails":false}` (per request) toggles safety. The
|
|
||||||
per-request flag only takes effect when the server was launched **with**
|
|
||||||
guardrails enabled (it cannot re-enable them on a `--no-guardrails` server).
|
|
||||||
`use_resolution_template` / `use_duration_template` are off by default and only
|
|
||||||
needed when not using upsampled prompts that already encode resolution/duration.
|
|
||||||
For V2V, `condition_frame_indexes_vision` selects the clean conditioned latent
|
|
||||||
frame indexes (default `[0, 1]`), and `condition_video_keep` selects whether the
|
|
||||||
API decodes the first or last needed reference frames (`"first"` by default).
|
|
||||||
- **Known limitations:**
|
|
||||||
- Guardrails-on requires `cosmos-guardrail` **and** access to the gated
|
|
||||||
`nvidia/Cosmos-1.0-Guardrail` repo (accept license + `HF_TOKEN`); otherwise
|
|
||||||
the server fails at pipeline build with a gated-repo / safety-checker error.
|
|
||||||
- A guardrail-blocked prompt currently returns HTTP 500
|
|
||||||
(`"Guardrail blocked prompt"`).
|
|
||||||
- Action `forward_dynamics`, `policy`, and `inverse_dynamics` are supported
|
|
||||||
online. Use async `POST /v1/videos` when you need the predicted/recovered
|
|
||||||
action payload under the top-level `action` field; sync `/v1/videos/sync`
|
|
||||||
returns raw MP4 bytes and does not expose action metadata in the response body.
|
|
||||||
|
|
||||||
### 1x GPU (Offline generation)
|
|
||||||
|
|
||||||
#### Environment
|
|
||||||
|
|
||||||
- OS: Ubuntu 22.04+
|
|
||||||
- Python: 3.12+
|
|
||||||
- Driver / runtime: NVIDIA CUDA environment
|
|
||||||
- vLLM-Omni version or commit: use the commit you are deploying from
|
|
||||||
|
|
||||||
#### Command
|
|
||||||
|
|
||||||
```python
|
|
||||||
# cosmos3_offline.py — run with: python cosmos3_offline.py
|
|
||||||
import torch
|
|
||||||
from vllm_omni.entrypoints.omni import Omni
|
|
||||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
omni = Omni(
|
|
||||||
model="nvidia/Cosmos3-Nano",
|
|
||||||
model_class_name="Cosmos3OmniDiffusersPipeline",
|
|
||||||
trust_remote_code=True,
|
|
||||||
enforce_eager=True,
|
|
||||||
# Guardrails are disabled here for a quick local run; install
|
|
||||||
# cosmos-guardrail + gated-repo access and drop this to enable them.
|
|
||||||
model_config={"guardrails": False},
|
|
||||||
)
|
|
||||||
gen = torch.Generator(device="cpu").manual_seed(42)
|
|
||||||
|
|
||||||
# Text-to-image (modalities=["image"]). For T2V use modalities=["video"]
|
|
||||||
# plus num_frames/fps; for I2V add multi_modal_data={"image": <PIL.Image>}.
|
|
||||||
outputs = omni.generate(
|
|
||||||
{
|
|
||||||
"prompt": "A photorealistic red sports car at golden hour, cinematic lighting.",
|
|
||||||
"negative_prompt": "blurry, distorted, low quality",
|
|
||||||
"modalities": ["image"],
|
|
||||||
},
|
|
||||||
OmniDiffusionSamplingParams(
|
|
||||||
height=1024, width=1024, generator=gen,
|
|
||||||
guidance_scale=7.0, num_inference_steps=50, num_outputs_per_prompt=1,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
outputs[0].request_output.images[0].save("cosmos3_t2i.png")
|
|
||||||
omni.close()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Verification
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python cosmos3_offline.py
|
|
||||||
python -c "from PIL import Image; im=Image.open('cosmos3_t2i.png'); print('image', im.size, im.mode)"
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Notes
|
|
||||||
|
|
||||||
- Same `Cosmos3OmniDiffusersPipeline` as online; mode is chosen by
|
|
||||||
`prompt["modalities"]` (`["image"]` → T2I, `["video"]` → T2V) plus
|
|
||||||
`num_frames`/`fps`, `multi_modal_data={"image": ...}` for I2V, and
|
|
||||||
`multi_modal_data={"video": [<PIL frames>]}` or a video tensor/array for V2V.
|
|
||||||
For video, frames are returned in `outputs[0].request_output.images` as an
|
|
||||||
`(B, F, H, W, 3)` array.
|
|
||||||
- The offline entry must be guarded by `if __name__ == "__main__":` — the engine
|
|
||||||
spawns workers with the `spawn` start method.
|
|
||||||
@@ -1,124 +0,0 @@
|
|||||||
# Cosmos3-Super
|
|
||||||
|
|
||||||
> Frontier 64B world model: text-to-image, text-to-video, image-to-video, video-to-video (+ optional audio)
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
|
|
||||||
- Vendor: NVIDIA
|
|
||||||
- Model: `nvidia/Cosmos3-Super` (64B; also `Cosmos3-Super-Text2Image`, `Cosmos3-Super-Image2Video`)
|
|
||||||
- Task: T2I, T2V, I2V, V2V generation, with optional synchronized audio (video + sound)
|
|
||||||
- Mode: Online serving with the OpenAI-compatible image/video APIs
|
|
||||||
- Maintainer: Community
|
|
||||||
|
|
||||||
## When to use this recipe
|
|
||||||
|
|
||||||
Use this recipe to deploy the 64B `nvidia/Cosmos3-Super` for the highest-quality
|
|
||||||
Cosmos3 generation. It shares the same `Cosmos3OmniDiffusersPipeline` and request
|
|
||||||
formats as [Cosmos3-Nano](./Cosmos3-Nano.md) — only the checkpoint size and the
|
|
||||||
recommended parallelism differ. Mode is selected per request (T2I →
|
|
||||||
`/v1/images/generations`; T2V/I2V/V2V → `/v1/videos/sync`; add
|
|
||||||
`generate_sound=true` for audio).
|
|
||||||
|
|
||||||
## References
|
|
||||||
|
|
||||||
- Model card (authoritative usage + example assets): <https://huggingface.co/nvidia/Cosmos3-Super>
|
|
||||||
- Nano recipe (same APIs/params): [`Cosmos3-Nano.md`](./Cosmos3-Nano.md)
|
|
||||||
- Pipeline: [`vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py`](../../vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py)
|
|
||||||
|
|
||||||
## Hardware Support
|
|
||||||
|
|
||||||
## GPU
|
|
||||||
|
|
||||||
Requires the `vllm-omni` package (or the `vllm/vllm-omni:cosmos3` container),
|
|
||||||
which provides the `vllm serve … --omni` entrypoint used below.
|
|
||||||
|
|
||||||
### 8x H200/H100/A100 (recommended, per model card)
|
|
||||||
|
|
||||||
```bash
|
|
||||||
vllm serve nvidia/Cosmos3-Super \
|
|
||||||
--omni \
|
|
||||||
--host 0.0.0.0 --port 8000 \
|
|
||||||
--cfg-parallel-size 2 \
|
|
||||||
--ulysses-degree 4 \
|
|
||||||
--use-hsdp --hsdp-shard-size 8 \
|
|
||||||
--init-timeout 1800
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2x H200 / B300 (minimum)
|
|
||||||
|
|
||||||
```bash
|
|
||||||
vllm serve nvidia/Cosmos3-Super \
|
|
||||||
--omni \
|
|
||||||
--host 0.0.0.0 --port 8000 \
|
|
||||||
--cfg-parallel-size 2 \
|
|
||||||
--use-hsdp --hsdp-shard-size 2 \
|
|
||||||
--init-timeout 1800
|
|
||||||
```
|
|
||||||
|
|
||||||
Guardrails are on by default (gated `nvidia/Cosmos-1.0-Guardrail` — `pip install
|
|
||||||
cosmos-guardrail`, accept the license, set `HF_TOKEN`); add `--no-guardrails` to
|
|
||||||
disable. `--enable-layerwise-offload` reduces VRAM on smaller GPUs.
|
|
||||||
|
|
||||||
#### Verification
|
|
||||||
|
|
||||||
Requests are identical to Nano (see [`Cosmos3-Nano.md`](./Cosmos3-Nano.md) for full
|
|
||||||
T2I/T2V/I2V/V2V/T2VS curls); official params: `size=1280x720, num_frames=189,
|
|
||||||
fps=24, num_inference_steps=35, guidance_scale=6.0, flow_shift=10.0,
|
|
||||||
max_sequence_length=4096`.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl http://localhost:8000/v1/models
|
|
||||||
# T2V (official prompt assets give best quality)
|
|
||||||
curl -sS -X POST http://localhost:8000/v1/videos/sync -H "Accept: video/mp4" \
|
|
||||||
-F "model=nvidia/Cosmos3-Super" -F "prompt=A robot arm is cleaning a plate in the kitchen" \
|
|
||||||
-F "size=1280x720" -F "num_frames=189" -F "fps=24" -F "num_inference_steps=35" \
|
|
||||||
-F "guidance_scale=6.0" -F "max_sequence_length=4096" -F "flow_shift=10.0" \
|
|
||||||
-F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":true}' \
|
|
||||||
-F "seed=17" -o cosmos3_super_t2v.mp4
|
|
||||||
|
|
||||||
# I2V — add an uploaded reference image
|
|
||||||
curl -sS -X POST http://localhost:8000/v1/videos/sync -H "Accept: video/mp4" \
|
|
||||||
-F "model=nvidia/Cosmos3-Super" -F "prompt=The scene comes to life with smooth, natural motion." \
|
|
||||||
-F "size=1280x720" -F "num_frames=189" -F "fps=24" -F "num_inference_steps=35" \
|
|
||||||
-F "guidance_scale=6.0" -F "max_sequence_length=4096" -F "flow_shift=10.0" \
|
|
||||||
-F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":true}' \
|
|
||||||
-F "seed=1111" -F "input_reference=@/path/to/reference.jpg;type=image/jpeg" \
|
|
||||||
-o cosmos3_super_i2v.mp4
|
|
||||||
|
|
||||||
# V2V — add an uploaded reference video. condition_video_keep can be "first" or "last".
|
|
||||||
curl -sS -X POST http://localhost:8000/v1/videos/sync -H "Accept: video/mp4" \
|
|
||||||
-F "model=nvidia/Cosmos3-Super" -F "prompt=Continue the same scene with smooth natural motion." \
|
|
||||||
-F "size=1280x720" -F "num_frames=189" -F "fps=24" -F "num_inference_steps=35" \
|
|
||||||
-F "guidance_scale=6.0" -F "max_sequence_length=4096" -F "flow_shift=10.0" \
|
|
||||||
-F 'extra_params={"condition_frame_indexes_vision":[0,1],"condition_video_keep":"first"}' \
|
|
||||||
-F "seed=2222" -F "input_reference=@/path/to/reference.mp4;type=video/mp4" \
|
|
||||||
-o cosmos3_super_v2v.mp4
|
|
||||||
|
|
||||||
# T2V + sound — add generate_sound/sound_duration (output muxes AAC 48 kHz stereo)
|
|
||||||
curl -sS -X POST http://localhost:8000/v1/videos/sync -H "Accept: video/mp4" \
|
|
||||||
-F "model=nvidia/Cosmos3-Super" -F "prompt=A robot arm is cleaning a plate in the kitchen" \
|
|
||||||
-F "size=1280x720" -F "num_frames=189" -F "fps=24" -F "num_inference_steps=35" \
|
|
||||||
-F "guidance_scale=6.0" -F "max_sequence_length=4096" -F "flow_shift=10.0" \
|
|
||||||
-F "generate_sound=true" -F "sound_duration=7.875" \
|
|
||||||
-F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":true}' \
|
|
||||||
-F "seed=17" -o cosmos3_super_t2vs.mp4
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Notes
|
|
||||||
|
|
||||||
- **Measured (2x B300, bf16, guardrails off, official 2-GPU config above):**
|
|
||||||
- T2I 1024², 50 steps → **~6 s**
|
|
||||||
- T2V 1280×720, 189 frames, 35 steps → **~197 s**
|
|
||||||
- I2V 1280×720, 189 frames, 35 steps → **~200 s**
|
|
||||||
- T2V + sound (189 frames, 35 steps) → **~198 s**, output muxes **AAC 48 kHz stereo**
|
|
||||||
- (NVIDIA's reference: 8×H200 @ 50 steps ≈ 55 s/video; 2×H200 @ 35 steps ≈ 3 min/video.)
|
|
||||||
- **Memory:** ~61.5 GiB per GPU when sharded across 2 GPUs (HSDP shard 2); repo ~135 GB on disk.
|
|
||||||
- Same generation defaults, supported sizes, V2V reference-video controls
|
|
||||||
(`condition_frame_indexes_vision`, `condition_video_keep`), and
|
|
||||||
`generate_sound`/`sound_duration`
|
|
||||||
semantics as Nano, including the **action** modality: `forward_dynamics`,
|
|
||||||
`policy`, and `inverse_dynamics` — see the Cosmos3-Nano recipe for the request
|
|
||||||
shapes. Use async `/v1/videos` when you need predicted/recovered action metadata
|
|
||||||
under the top-level `action` field. Verified on the 64B Super under
|
|
||||||
`--cfg-parallel-size 2`: async `policy` returns the predicted action (`[16, 10]`)
|
|
||||||
and the rollout video reliably.
|
|
||||||
@@ -57,6 +57,12 @@ Adjust `devices` in the YAML to match your hardware.
|
|||||||
|
|
||||||
#### Command
|
#### Command
|
||||||
|
|
||||||
|
Thinker only (text output):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
vllm serve Jonathan1909/Ming-flash-omni-2.0 --omni --port 8091
|
||||||
|
```
|
||||||
|
|
||||||
Thinker + talker (text and/or audio output):
|
Thinker + talker (text and/or audio output):
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -66,15 +72,6 @@ vllm serve Jonathan1909/Ming-flash-omni-2.0 \
|
|||||||
--log-stats
|
--log-stats
|
||||||
```
|
```
|
||||||
|
|
||||||
Thinker only (text-only output):
|
|
||||||
|
|
||||||
```bash
|
|
||||||
vllm serve Jonathan1909/Ming-flash-omni-2.0 \
|
|
||||||
--omni \
|
|
||||||
--deploy-config vllm_omni/deploy/ming_flash_omni_thinker_only.yaml \
|
|
||||||
--port 8091
|
|
||||||
```
|
|
||||||
|
|
||||||
`--log-stats` is optional but recommended while validating the deployment.
|
`--log-stats` is optional but recommended while validating the deployment.
|
||||||
|
|
||||||
#### Verification
|
#### Verification
|
||||||
|
|||||||
@@ -0,0 +1,215 @@
|
|||||||
|
# Cosmos3-Nano
|
||||||
|
|
||||||
|
> Text-to-image, text-to-video, and image-to-video serving
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
- Vendor: NVIDIA
|
||||||
|
- Model: `nvidia/Cosmos3-Nano`
|
||||||
|
- Task: Text-to-image (T2I), text-to-video (T2V), and image-to-video (I2V) generation
|
||||||
|
- Mode: Online serving with the OpenAI-compatible image/video APIs, plus offline generation via the `Omni` API
|
||||||
|
- Maintainer: Community
|
||||||
|
|
||||||
|
## When to use this recipe
|
||||||
|
|
||||||
|
Use this recipe to deploy `nvidia/Cosmos3-Nano` for image and video generation.
|
||||||
|
A single pipeline class (`Cosmos3OmniDiffusersPipeline`) serves all three modes;
|
||||||
|
the mode is selected per request:
|
||||||
|
|
||||||
|
- **T2I** — `POST /v1/images/generations` (or a prompt carrying `modalities=["image"]`).
|
||||||
|
- **T2V** — `POST /v1/videos/sync` with `num_frames > 1` and no reference image.
|
||||||
|
- **I2V** — `POST /v1/videos/sync` with a reference image (`input_reference` file
|
||||||
|
upload, or `image_reference` JSON).
|
||||||
|
|
||||||
|
## References
|
||||||
|
|
||||||
|
- Model card (authoritative usage + example assets): <https://huggingface.co/nvidia/Cosmos3-Nano>
|
||||||
|
- Example inputs/outputs live in the repo's `assets/` (`example_t2v_prompt.json`,
|
||||||
|
`example_i2v_prompt.json`, `example_i2v_input.jpg`, `negative_prompt.json`).
|
||||||
|
- Prompt upsampling (recommended for quality): the model expects JSON-upsampled
|
||||||
|
structured prompts; see NVIDIA's `cosmos-framework` prompt-upsampling docs.
|
||||||
|
- Pipeline: [`vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py`](../../vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py)
|
||||||
|
- Smoke tests (canonical request formats): [`tests/e2e/accuracy/test_cosmos3_similarity.py`](../../tests/e2e/accuracy/test_cosmos3_similarity.py)
|
||||||
|
|
||||||
|
## Hardware Support
|
||||||
|
|
||||||
|
## GPU
|
||||||
|
|
||||||
|
### 1x H200 141GB / B300 (Online serving)
|
||||||
|
|
||||||
|
#### Environment
|
||||||
|
|
||||||
|
- OS: Ubuntu 22.04+
|
||||||
|
- Python: 3.12+
|
||||||
|
- Driver / runtime: NVIDIA CUDA environment
|
||||||
|
- vLLM version: match the repository requirements from your current checkout
|
||||||
|
- vLLM-Omni version or commit: use the commit you are deploying from
|
||||||
|
|
||||||
|
#### Command
|
||||||
|
|
||||||
|
Safety guardrails are **on by default** (NVIDIA Open Model License). They load
|
||||||
|
the **gated** `nvidia/Cosmos-1.0-Guardrail` model, so to keep them on you must:
|
||||||
|
|
||||||
|
1. `pip install cosmos-guardrail`
|
||||||
|
2. Accept the license at <https://huggingface.co/nvidia/Cosmos-1.0-Guardrail>
|
||||||
|
3. Export a token with access: `export HF_TOKEN=hf_...`
|
||||||
|
|
||||||
|
Then launch the recommended server:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
vllm serve nvidia/Cosmos3-Nano \
|
||||||
|
--omni \
|
||||||
|
--host 0.0.0.0 --port 8000 \
|
||||||
|
--init-timeout 1800
|
||||||
|
```
|
||||||
|
|
||||||
|
To run **without** guardrails (you are responsible for license compliance),
|
||||||
|
add `--no-guardrails` (no token/`cosmos-guardrail` needed). For extra GPUs use
|
||||||
|
`--ulysses-degree N` (context parallel) or `--tensor-parallel-size N`;
|
||||||
|
`--enable-layerwise-offload` reduces VRAM on smaller GPUs. The pipeline
|
||||||
|
auto-resolves from `model_index.json`; pass
|
||||||
|
`--model-class-name Cosmos3OmniDiffusersPipeline` to force it explicitly.
|
||||||
|
|
||||||
|
#### Verification
|
||||||
|
|
||||||
|
Best quality uses the JSON-upsampled prompts from `assets/` (download with
|
||||||
|
`hf download nvidia/Cosmos3-Nano assets/ --local-dir Cosmos3-Nano`). Minimal
|
||||||
|
self-contained examples:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl http://localhost:8000/v1/models
|
||||||
|
|
||||||
|
# Text-to-image -> /v1/images/generations (1024x1024, 50 steps; base64 PNG)
|
||||||
|
curl -sS -X POST http://localhost:8000/v1/images/generations \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{
|
||||||
|
"model": "nvidia/Cosmos3-Nano",
|
||||||
|
"prompt": "A photorealistic red sports car on a city street at golden hour, cinematic lighting.",
|
||||||
|
"negative_prompt": "blurry, distorted, low quality",
|
||||||
|
"size": "1024x1024", "n": 1, "response_format": "b64_json",
|
||||||
|
"num_inference_steps": 50, "guidance_scale": 7.0, "seed": 42
|
||||||
|
}' | python -c "import sys,json,base64; open('cosmos3_t2i.png','wb').write(base64.b64decode(json.load(sys.stdin)['data'][0]['b64_json']))"
|
||||||
|
|
||||||
|
# Text-to-video -> /v1/videos/sync (720p, 189 frames @ 24fps; official params)
|
||||||
|
curl -sS -X POST http://localhost:8000/v1/videos/sync \
|
||||||
|
-H "Accept: video/mp4" \
|
||||||
|
-F "model=nvidia/Cosmos3-Nano" \
|
||||||
|
-F "prompt=A robot arm is cleaning a plate in the kitchen" \
|
||||||
|
-F "negative_prompt=blurry, distorted, low quality, jittery, deformed" \
|
||||||
|
-F "size=1280x720" -F "num_frames=189" -F "fps=24" \
|
||||||
|
-F "num_inference_steps=35" -F "guidance_scale=6.0" \
|
||||||
|
-F "max_sequence_length=4096" -F "flow_shift=10.0" \
|
||||||
|
-F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":true}' \
|
||||||
|
-F "seed=123" \
|
||||||
|
-o cosmos3_t2v.mp4
|
||||||
|
|
||||||
|
# Image-to-video -> /v1/videos/sync with an uploaded reference image
|
||||||
|
curl -sS -X POST http://localhost:8000/v1/videos/sync \
|
||||||
|
-H "Accept: video/mp4" \
|
||||||
|
-F "model=nvidia/Cosmos3-Nano" \
|
||||||
|
-F "prompt=The scene comes to life with smooth, natural motion." \
|
||||||
|
-F "negative_prompt=blurry, distorted, low quality" \
|
||||||
|
-F "size=1280x720" -F "num_frames=189" -F "fps=24" \
|
||||||
|
-F "num_inference_steps=35" -F "guidance_scale=6.0" \
|
||||||
|
-F "max_sequence_length=4096" -F "flow_shift=10.0" \
|
||||||
|
-F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":true}' \
|
||||||
|
-F "seed=1111" \
|
||||||
|
-F "input_reference=@/path/to/reference.jpg;type=image/jpeg" \
|
||||||
|
-o cosmos3_i2v.mp4
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Notes
|
||||||
|
|
||||||
|
- **Measured latency (1x B300, bf16, guardrails off):**
|
||||||
|
- T2I 1024² — 10 / 25 / 50 steps → ~0.4 / 0.7 / **1.3 s**
|
||||||
|
- T2V 1280×720 @ 35 steps — 25 / 49 / 93 / **189** frames → ~7 / 15 / 33 / **~93 s**
|
||||||
|
- I2V 1280×720, 189 frames @ 35 steps → ~**99 s**
|
||||||
|
- Guardrails-on overhead: ~8% on T2I, negligible on video.
|
||||||
|
- **Memory:** transformer ~17 GiB (bf16); peak ~46 GiB for 720p video on 1 GPU;
|
||||||
|
full repo (transformer + Wan VAE + Qwen3-VL vision encoder + audio tokenizer)
|
||||||
|
~33 GB on disk.
|
||||||
|
- **Determinism:** identical seed reproduces identical output on the same
|
||||||
|
hardware; outputs are not bit-identical across different GPU types.
|
||||||
|
- **Supported sizes (per model card):** 256p / 480p / 720p at 16:9, 4:3, 1:1,
|
||||||
|
3:4, 9:16. Defaults: T2I 1024², 50 steps, guidance 7.0; T2V/I2V 1280×720,
|
||||||
|
189 frames, 35 steps, guidance 6.0, `flow_shift=10.0`.
|
||||||
|
- **Key flags / params:** `--no-guardrails` (server) or
|
||||||
|
`extra_params={"guardrails":false}` (per request) toggles safety;
|
||||||
|
`use_resolution_template` / `use_duration_template` are off by default and only
|
||||||
|
needed when not using upsampled prompts that already encode resolution/duration.
|
||||||
|
- **Known limitations:**
|
||||||
|
- Guardrails-on requires `cosmos-guardrail` **and** access to the gated
|
||||||
|
`nvidia/Cosmos-1.0-Guardrail` repo (accept license + `HF_TOKEN`); otherwise
|
||||||
|
the server fails at pipeline build with a gated-repo / safety-checker error.
|
||||||
|
- A guardrail-blocked prompt currently returns HTTP 500
|
||||||
|
(`"Guardrail blocked prompt"`).
|
||||||
|
- Video + audio, and action (policy / forward- / inverse-dynamics) modalities
|
||||||
|
are not part of this integration yet.
|
||||||
|
|
||||||
|
### 1x GPU (Offline generation)
|
||||||
|
|
||||||
|
#### Environment
|
||||||
|
|
||||||
|
- OS: Ubuntu 22.04+
|
||||||
|
- Python: 3.12+
|
||||||
|
- Driver / runtime: NVIDIA CUDA environment
|
||||||
|
- vLLM-Omni version or commit: use the commit you are deploying from
|
||||||
|
|
||||||
|
#### Command
|
||||||
|
|
||||||
|
```python
|
||||||
|
# cosmos3_offline.py — run with: python cosmos3_offline.py
|
||||||
|
import torch
|
||||||
|
from vllm_omni.entrypoints.omni import Omni
|
||||||
|
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
omni = Omni(
|
||||||
|
model="nvidia/Cosmos3-Nano",
|
||||||
|
model_class_name="Cosmos3OmniDiffusersPipeline",
|
||||||
|
trust_remote_code=True,
|
||||||
|
enforce_eager=True,
|
||||||
|
# Keep guardrails on by installing cosmos-guardrail + gated-repo access;
|
||||||
|
# this disables them for a quick local run.
|
||||||
|
model_config={"guardrails": False},
|
||||||
|
)
|
||||||
|
gen = torch.Generator(device="cpu").manual_seed(42)
|
||||||
|
|
||||||
|
# Text-to-image (modalities=["image"]). For T2V use modalities=["video"]
|
||||||
|
# plus num_frames/fps; for I2V add multi_modal_data={"image": <PIL.Image>}.
|
||||||
|
outputs = omni.generate(
|
||||||
|
{
|
||||||
|
"prompt": "A photorealistic red sports car at golden hour, cinematic lighting.",
|
||||||
|
"negative_prompt": "blurry, distorted, low quality",
|
||||||
|
"modalities": ["image"],
|
||||||
|
},
|
||||||
|
OmniDiffusionSamplingParams(
|
||||||
|
height=1024, width=1024, generator=gen,
|
||||||
|
guidance_scale=7.0, num_inference_steps=50, num_outputs_per_prompt=1,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
outputs[0].request_output.images[0].save("cosmos3_t2i.png")
|
||||||
|
omni.close()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Verification
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python cosmos3_offline.py
|
||||||
|
python -c "from PIL import Image; im=Image.open('cosmos3_t2i.png'); print('image', im.size, im.mode)"
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Notes
|
||||||
|
|
||||||
|
- Same `Cosmos3OmniDiffusersPipeline` as online; mode is chosen by
|
||||||
|
`prompt["modalities"]` (`["image"]` → T2I, `["video"]` → T2V) plus
|
||||||
|
`num_frames`/`fps`, and `multi_modal_data={"image": ...}` for I2V. For video,
|
||||||
|
frames are returned in `outputs[0].request_output.images` as an
|
||||||
|
`(B, F, H, W, 3)` array.
|
||||||
|
- The offline entry must be guarded by `if __name__ == "__main__":` — the engine
|
||||||
|
spawns workers with the `spawn` start method.
|
||||||
Binary file not shown.
|
Before Width: | Height: | Size: 1.1 MiB |
@@ -345,14 +345,6 @@ def test_parse_score_payload_handles_raw_json_and_delimited_json():
|
|||||||
assert parse_score_payload(wrapped)["score"] == [6]
|
assert parse_score_payload(wrapped)["score"] == [6]
|
||||||
|
|
||||||
|
|
||||||
def test_parse_score_payload_handles_qwen_vl_nested_score_dicts():
|
|
||||||
qwen_style = '{"score": [{"naturalness": 8, "artifact_free": 7}], "reasoning": "good quality"}'
|
|
||||||
assert parse_score_payload(qwen_style)["score"] == [8, 7]
|
|
||||||
|
|
||||||
nested_values = '{"score": [{"score": 9}, {"score": 6}], "reasoning": "ok"}'
|
|
||||||
assert parse_score_payload(nested_values)["score"] == [9, 6]
|
|
||||||
|
|
||||||
|
|
||||||
def test_summarize_gedit_generated_records_groups_by_task_and_language():
|
def test_summarize_gedit_generated_records_groups_by_task_and_language():
|
||||||
records = []
|
records = []
|
||||||
for group in GEDIT_GROUPS[:2]:
|
for group in GEDIT_GROUPS[:2]:
|
||||||
|
|||||||
@@ -61,7 +61,7 @@ def test_stage0_streaming_update_discards_outstanding_async_placeholder_token()
|
|||||||
|
|
||||||
sched._update_request_as_session(session, _make_update([10, 20]))
|
sched._update_request_as_session(session, _make_update([10, 20]))
|
||||||
|
|
||||||
assert session.async_tokens_to_discard == 1
|
assert session.discard_latest_async_tokens is True
|
||||||
assert session.num_output_placeholders == 0
|
assert session.num_output_placeholders == 0
|
||||||
assert session.spec_token_ids == []
|
assert session.spec_token_ids == []
|
||||||
# The async placeholder makes token 9 unconfirmed, so only 7 and 8 are
|
# The async placeholder makes token 9 unconfirmed, so only 7 and 8 are
|
||||||
@@ -83,7 +83,7 @@ def test_stage0_streaming_update_keeps_all_computed_tokens_without_placeholder()
|
|||||||
|
|
||||||
sched._update_request_as_session(session, _make_update([10, 20]))
|
sched._update_request_as_session(session, _make_update([10, 20]))
|
||||||
|
|
||||||
assert getattr(session, "async_tokens_to_discard", 0) == 0
|
assert session.discard_latest_async_tokens is False
|
||||||
assert session.num_output_placeholders == 0
|
assert session.num_output_placeholders == 0
|
||||||
assert session.prompt_token_ids == [1, 2, 3, 7, 8, 9, 10, 20]
|
assert session.prompt_token_ids == [1, 2, 3, 7, 8, 9, 10, 20]
|
||||||
assert list(session._all_token_ids) == [1, 2, 3, 7, 8, 9, 10, 20]
|
assert list(session._all_token_ids) == [1, 2, 3, 7, 8, 9, 10, 20]
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
[
|
[
|
||||||
{
|
{
|
||||||
"test_name": "test_hunyuan_image3_ti2i_inline",
|
"test_name": "test_hunyuan_image3_ti2i_tp4_tp4_inline",
|
||||||
"description": "AR (TP=2) + 2 * DiT (TP=2) - using inline dataset and deploy-config",
|
"description": "AR (TP=4) + DiT (TP=4) - using inline dataset and deploy-config",
|
||||||
"server_type": "vllm-omni",
|
"server_type": "vllm-omni",
|
||||||
"benchmark_endpoint": "/v1/images/edits",
|
"benchmark_endpoint": "/v1/images/edits",
|
||||||
"server_params": {
|
"server_params": {
|
||||||
@@ -21,7 +21,7 @@
|
|||||||
"zmq_port": 50051,
|
"zmq_port": 50051,
|
||||||
"protocol": "rdma",
|
"protocol": "rdma",
|
||||||
"device_name": "",
|
"device_name": "",
|
||||||
"memory_pool_size": 17179869184,
|
"memory_pool_size": 4294967296,
|
||||||
"memory_pool_device": "cpu"
|
"memory_pool_device": "cpu"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -32,13 +32,13 @@
|
|||||||
"is_comprehension": true,
|
"is_comprehension": true,
|
||||||
"final_output": true,
|
"final_output": true,
|
||||||
"final_output_type": "text",
|
"final_output_type": "text",
|
||||||
"max_num_seqs": 16,
|
"max_num_seqs": 4,
|
||||||
"gpu_memory_utilization": 0.95,
|
"gpu_memory_utilization": 0.95,
|
||||||
"enforce_eager": false,
|
"enforce_eager": false,
|
||||||
"enable_prefix_caching": false,
|
"enable_prefix_caching": false,
|
||||||
"max_num_batched_tokens": 32768,
|
"max_num_batched_tokens": 32768,
|
||||||
"devices": "0,1",
|
"devices": "0,1,2,3",
|
||||||
"tensor_parallel_size": 2,
|
"tensor_parallel_size": 4,
|
||||||
"hf_overrides": {
|
"hf_overrides": {
|
||||||
"rope_parameters": {
|
"rope_parameters": {
|
||||||
"mrope_section": [0, 32, 32],
|
"mrope_section": [0, 32, 32],
|
||||||
@@ -65,14 +65,13 @@
|
|||||||
"stage_id": 1,
|
"stage_id": 1,
|
||||||
"max_num_seqs": 1,
|
"max_num_seqs": 1,
|
||||||
"enforce_eager": true,
|
"enforce_eager": true,
|
||||||
"devices": "2,3",
|
"devices": "4,5,6,7",
|
||||||
"distributed_executor_backend": "mp",
|
"distributed_executor_backend": "mp",
|
||||||
"omni_kv_config": {
|
"omni_kv_config": {
|
||||||
"need_recv_cache": true
|
"need_recv_cache": true
|
||||||
},
|
},
|
||||||
"parallel_config": {
|
"parallel_config": {
|
||||||
"tensor_parallel_size": 2,
|
"tensor_parallel_size": 4,
|
||||||
"vae_patch_parallel_size": 2,
|
|
||||||
"enable_expert_parallel": true
|
"enable_expert_parallel": true
|
||||||
},
|
},
|
||||||
"input_connectors": {
|
"input_connectors": {
|
||||||
@@ -115,8 +114,8 @@
|
|||||||
},
|
},
|
||||||
"task": "ti2i",
|
"task": "ti2i",
|
||||||
"bot-task": "think_recaption",
|
"bot-task": "think_recaption",
|
||||||
"max-concurrency": 16,
|
"max-concurrency": 4,
|
||||||
"num-prompts": 96,
|
"num-prompts": 16,
|
||||||
"skip-performance-assertion": true,
|
"skip-performance-assertion": true,
|
||||||
"baseline": {
|
"baseline": {
|
||||||
"throughput_qps": 0.1,
|
"throughput_qps": 0.1,
|
||||||
|
|||||||
@@ -23,10 +23,10 @@
|
|||||||
"ignore_eos": true,
|
"ignore_eos": true,
|
||||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||||
"baseline": {
|
"baseline": {
|
||||||
"mean_ttft_ms": [203, 283, 470, 869],
|
"mean_ttft_ms": [1000, 2000, 3000, 4000],
|
||||||
"mean_tpot_ms": [28, 36, 41, 45],
|
"mean_tpot_ms": [200, 200, 200, 200],
|
||||||
"mean_audio_ttfp_ms": [861, 1200, 1528, 2100],
|
"mean_audio_ttfp_ms": [10000, 15000, 20000, 25000],
|
||||||
"mean_audio_rtf": [0.24, 0.30, 0.35, 0.39]
|
"mean_audio_rtf": [1.5, 1.5, 1.5, 1.5]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -49,9 +49,9 @@
|
|||||||
},
|
},
|
||||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||||
"baseline": {
|
"baseline": {
|
||||||
"mean_ttft_ms": [377],
|
"mean_ttft_ms": [2000],
|
||||||
"mean_audio_ttfp_ms": [684],
|
"mean_audio_ttfp_ms": [2000],
|
||||||
"mean_audio_rtf": [0.14]
|
"mean_audio_rtf": [0.25]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -76,9 +76,9 @@
|
|||||||
},
|
},
|
||||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||||
"baseline": {
|
"baseline": {
|
||||||
"mean_ttft_ms": [228],
|
"mean_ttft_ms": [6000],
|
||||||
"mean_audio_ttfp_ms": [686],
|
"mean_audio_ttfp_ms": [6000],
|
||||||
"mean_audio_rtf": [0.16]
|
"mean_audio_rtf": [0.7]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -105,9 +105,9 @@
|
|||||||
},
|
},
|
||||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||||
"baseline": {
|
"baseline": {
|
||||||
"mean_ttft_ms": [522],
|
"mean_ttft_ms": [12000],
|
||||||
"mean_audio_ttfp_ms": [1119],
|
"mean_audio_ttfp_ms": [12000],
|
||||||
"mean_audio_rtf": [0.21]
|
"mean_audio_rtf": [1.0]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -31,6 +31,11 @@
|
|||||||
],
|
],
|
||||||
"random_input_len": 2500,
|
"random_input_len": 2500,
|
||||||
"random_output_len": 900,
|
"random_output_len": 900,
|
||||||
|
"temperature": 0.4,
|
||||||
|
"top_p": 0.9,
|
||||||
|
"top_k": 1,
|
||||||
|
"seed": 42,
|
||||||
|
"repetition_penalty": 1.05,
|
||||||
"ignore_eos": true,
|
"ignore_eos": true,
|
||||||
"extra_body": {
|
"extra_body": {
|
||||||
"modalities": [
|
"modalities": [
|
||||||
@@ -69,6 +74,11 @@
|
|||||||
"random_input_len": 100,
|
"random_input_len": 100,
|
||||||
"random_output_len": 100,
|
"random_output_len": 100,
|
||||||
"random_range_ratio": 0.0,
|
"random_range_ratio": 0.0,
|
||||||
|
"temperature": 0.4,
|
||||||
|
"top_p": 0.9,
|
||||||
|
"top_k": 1,
|
||||||
|
"seed": 42,
|
||||||
|
"repetition_penalty": 1.05,
|
||||||
"ignore_eos": true,
|
"ignore_eos": true,
|
||||||
"extra_body": {
|
"extra_body": {
|
||||||
"modalities": [
|
"modalities": [
|
||||||
@@ -107,6 +117,11 @@
|
|||||||
"random_input_len": 100,
|
"random_input_len": 100,
|
||||||
"random_output_len": 100,
|
"random_output_len": 100,
|
||||||
"random_range_ratio": 0.0,
|
"random_range_ratio": 0.0,
|
||||||
|
"temperature": 0.4,
|
||||||
|
"top_p": 0.9,
|
||||||
|
"top_k": 1,
|
||||||
|
"seed": 42,
|
||||||
|
"repetition_penalty": 1.05,
|
||||||
"ignore_eos": true,
|
"ignore_eos": true,
|
||||||
"extra_body": {
|
"extra_body": {
|
||||||
"modalities": [
|
"modalities": [
|
||||||
@@ -147,6 +162,11 @@
|
|||||||
"random_input_len": 100,
|
"random_input_len": 100,
|
||||||
"random_output_len": 100,
|
"random_output_len": 100,
|
||||||
"random_range_ratio": 0.0,
|
"random_range_ratio": 0.0,
|
||||||
|
"temperature": 0.4,
|
||||||
|
"top_p": 0.9,
|
||||||
|
"top_k": 1,
|
||||||
|
"seed": 42,
|
||||||
|
"repetition_penalty": 1.05,
|
||||||
"ignore_eos": true,
|
"ignore_eos": true,
|
||||||
"extra_body": {
|
"extra_body": {
|
||||||
"modalities": [
|
"modalities": [
|
||||||
|
|||||||
@@ -1,46 +1,19 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
"""
|
|
||||||
Tests for the DiffusersPipelineLoader.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
from huggingface_hub import snapshot_download
|
|
||||||
from vllm.config.load import LoadConfig
|
|
||||||
|
|
||||||
from vllm_omni.diffusion.config import get_current_diffusion_config, get_current_diffusion_config_or_none
|
from vllm_omni.diffusion.config import get_current_diffusion_config, get_current_diffusion_config_or_none
|
||||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
|
||||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||||
from vllm_omni.diffusion.model_loader.gguf_adapters import get_gguf_adapter
|
from vllm_omni.diffusion.model_loader.gguf_adapters import get_gguf_adapter
|
||||||
from vllm_omni.diffusion.models.helios import HeliosPipeline
|
|
||||||
from vllm_omni.diffusion.registry import initialize_model
|
from vllm_omni.diffusion.registry import initialize_model
|
||||||
|
|
||||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
|
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
|
||||||
|
|
||||||
model_path = "hf-internal-testing/tiny-helios-modular-pipe"
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
|
||||||
def prefetch_helios_model():
|
|
||||||
"""Downloads the tiny helios model prior to running a test."""
|
|
||||||
snapshot_download(model_path)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="function")
|
|
||||||
def mock_tp_group(mocker):
|
|
||||||
"""Mocks the tensor parallel group; this is needed to initialize the Helios model."""
|
|
||||||
mocker.patch("vllm.model_executor.layers.linear.get_tensor_model_parallel_world_size", return_value=1)
|
|
||||||
mocker.patch("vllm.model_executor.layers.linear.get_tensor_model_parallel_rank", return_value=0)
|
|
||||||
mock_group = mocker.MagicMock()
|
|
||||||
mock_group.world_size = 1
|
|
||||||
mock_group.rank_in_group = 0
|
|
||||||
mocker.patch("vllm.distributed.parallel_state.get_tp_group", return_value=mock_group)
|
|
||||||
|
|
||||||
|
|
||||||
class _DummyPipelineModel(nn.Module):
|
class _DummyPipelineModel(nn.Module):
|
||||||
def __init__(self, *, source_prefix: str):
|
def __init__(self, *, source_prefix: str):
|
||||||
@@ -69,13 +42,7 @@ class _DummyPipelineModel(nn.Module):
|
|||||||
|
|
||||||
|
|
||||||
def _make_loader_with_weights(weight_names: list[str]) -> DiffusersPipelineLoader:
|
def _make_loader_with_weights(weight_names: list[str]) -> DiffusersPipelineLoader:
|
||||||
od_config = SimpleNamespace(
|
loader = object.__new__(DiffusersPipelineLoader)
|
||||||
dtype=torch.float32,
|
|
||||||
parallel_config=SimpleNamespace(use_hsdp=False),
|
|
||||||
quantization_config=None,
|
|
||||||
)
|
|
||||||
loader = DiffusersPipelineLoader(LoadConfig(), od_config)
|
|
||||||
|
|
||||||
loader.counter_before_loading_weights = 0.0
|
loader.counter_before_loading_weights = 0.0
|
||||||
loader.counter_after_loading_weights = 0.0
|
loader.counter_after_loading_weights = 0.0
|
||||||
|
|
||||||
@@ -183,15 +150,16 @@ def test_load_model_custom_pipeline_sets_current_diffusion_config(monkeypatch):
|
|||||||
quantization_config=None,
|
quantization_config=None,
|
||||||
)
|
)
|
||||||
|
|
||||||
loader = DiffusersPipelineLoader(LoadConfig(), od_config)
|
loader = object.__new__(DiffusersPipelineLoader)
|
||||||
loader.load_weights = lambda model: None # type: ignore[assignment]
|
loader.load_weights = lambda model: None # type: ignore[assignment]
|
||||||
loader._process_weights_after_loading = lambda model, target_device: None # type: ignore[assignment]
|
loader._process_weights_after_loading = lambda model, target_device: None # type: ignore[assignment]
|
||||||
loader._is_gguf_quantization = lambda: False # type: ignore[assignment]
|
loader._is_gguf_quantization = lambda _od_config: False # type: ignore[assignment]
|
||||||
|
|
||||||
monkeypatch.setattr(loader_mod, "resolve_obj_by_qualname", lambda _name: _ConfigAwareModel)
|
monkeypatch.setattr(loader_mod, "resolve_obj_by_qualname", lambda _name: _ConfigAwareModel)
|
||||||
monkeypatch.setattr(loader_mod.torch, "device", lambda _name: _DeviceContext("cpu"))
|
monkeypatch.setattr(loader_mod.torch, "device", lambda _name: _DeviceContext("cpu"))
|
||||||
|
|
||||||
model = loader.load_model(
|
model = loader.load_model(
|
||||||
|
od_config,
|
||||||
load_device="cpu",
|
load_device="cpu",
|
||||||
load_format="custom_pipeline",
|
load_format="custom_pipeline",
|
||||||
custom_pipeline_name="tests.dummy.ConfigAwarePipeline",
|
custom_pipeline_name="tests.dummy.ConfigAwarePipeline",
|
||||||
@@ -200,33 +168,3 @@ def test_load_model_custom_pipeline_sets_current_diffusion_config(monkeypatch):
|
|||||||
assert model.captured_config is od_config
|
assert model.captured_config is od_config
|
||||||
assert model.seen_config_during_init is od_config
|
assert model.seen_config_during_init is od_config
|
||||||
assert get_current_diffusion_config_or_none() is None
|
assert get_current_diffusion_config_or_none() is None
|
||||||
|
|
||||||
|
|
||||||
def test_get_all_weights(prefetch_helios_model, mock_tp_group):
|
|
||||||
"""Ensure that get all weights on a tiny model resolves to nonempty weights."""
|
|
||||||
od_config = OmniDiffusionConfig(
|
|
||||||
model_class_name="HeliosPipeline",
|
|
||||||
model=model_path,
|
|
||||||
)
|
|
||||||
loader = DiffusersPipelineLoader(
|
|
||||||
load_config=LoadConfig(),
|
|
||||||
od_config=od_config,
|
|
||||||
)
|
|
||||||
pipeline = HeliosPipeline(od_config=od_config)
|
|
||||||
|
|
||||||
weights = list(loader.get_all_weights(pipeline))
|
|
||||||
assert len(weights) > 0
|
|
||||||
|
|
||||||
|
|
||||||
def test_load_model(prefetch_helios_model, mock_tp_group):
|
|
||||||
"""Ensure that load model creates an instance of the expected pipeline class."""
|
|
||||||
od_config = OmniDiffusionConfig(
|
|
||||||
model_class_name="HeliosPipeline",
|
|
||||||
model=model_path,
|
|
||||||
)
|
|
||||||
loader = DiffusersPipelineLoader(
|
|
||||||
load_config=LoadConfig(),
|
|
||||||
od_config=od_config,
|
|
||||||
)
|
|
||||||
model = loader.load_model(load_device="cpu")
|
|
||||||
assert isinstance(model, HeliosPipeline)
|
|
||||||
@@ -62,7 +62,6 @@ def _make_loader() -> DiffusersPipelineLoader:
|
|||||||
ignore_patterns=["*.tmp"],
|
ignore_patterns=["*.tmp"],
|
||||||
)
|
)
|
||||||
loader.od_config = None
|
loader.od_config = None
|
||||||
loader.quant_config = None
|
|
||||||
loader.counter_before_loading_weights = 0.0
|
loader.counter_before_loading_weights = 0.0
|
||||||
loader.counter_after_loading_weights = 0.0
|
loader.counter_after_loading_weights = 0.0
|
||||||
return loader
|
return loader
|
||||||
@@ -70,47 +69,53 @@ def _make_loader() -> DiffusersPipelineLoader:
|
|||||||
|
|
||||||
def test_is_gguf_quantization_accepts_dict_config():
|
def test_is_gguf_quantization_accepts_dict_config():
|
||||||
loader = _make_loader()
|
loader = _make_loader()
|
||||||
loader.quant_config = {
|
od_config = SimpleNamespace(
|
||||||
"method": "gguf",
|
quantization_config={
|
||||||
"gguf_model": "weights.gguf",
|
"method": "gguf",
|
||||||
}
|
"gguf_model": "weights.gguf",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
assert loader._is_gguf_quantization() is True
|
assert loader._is_gguf_quantization(od_config) is True
|
||||||
|
|
||||||
|
|
||||||
def test_is_gguf_quantization_rejects_non_gguf_dict_config():
|
def test_is_gguf_quantization_rejects_non_gguf_dict_config():
|
||||||
loader = _make_loader()
|
loader = _make_loader()
|
||||||
loader.quant_config = {
|
od_config = SimpleNamespace(
|
||||||
"method": "fp8",
|
quantization_config={
|
||||||
"gguf_model": "weights.gguf",
|
"method": "fp8",
|
||||||
}
|
"gguf_model": "weights.gguf",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
assert loader._is_gguf_quantization() is False
|
assert loader._is_gguf_quantization(od_config) is False
|
||||||
|
|
||||||
|
|
||||||
def test_is_gguf_quantization_requires_gguf_model_for_dict_config():
|
def test_is_gguf_quantization_requires_gguf_model_for_dict_config():
|
||||||
loader = _make_loader()
|
loader = _make_loader()
|
||||||
loader.quant_config = {"method": "gguf"}
|
od_config = SimpleNamespace(quantization_config={"method": "gguf"})
|
||||||
|
|
||||||
with pytest.raises(ValueError, match="gguf_model"):
|
with pytest.raises(ValueError, match="gguf_model"):
|
||||||
loader._is_gguf_quantization()
|
loader._is_gguf_quantization(od_config)
|
||||||
|
|
||||||
|
|
||||||
def test_is_gguf_quantization_accepts_object_config():
|
def test_is_gguf_quantization_accepts_object_config():
|
||||||
loader = _make_loader()
|
loader = _make_loader()
|
||||||
loader.quant_config = SimpleNamespace(
|
od_config = SimpleNamespace(
|
||||||
get_name=lambda: "gguf",
|
quantization_config=SimpleNamespace(
|
||||||
gguf_model="weights.gguf",
|
get_name=lambda: "gguf",
|
||||||
|
gguf_model="weights.gguf",
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
assert loader._is_gguf_quantization() is True
|
assert loader._is_gguf_quantization(od_config) is True
|
||||||
|
|
||||||
|
|
||||||
def test_is_gguf_quantization_uses_fallback_object_without_get_name():
|
def test_is_gguf_quantization_uses_fallback_object_without_get_name():
|
||||||
loader = _make_loader()
|
loader = _make_loader()
|
||||||
loader.quant_config = SimpleNamespace(gguf_model="weights.gguf")
|
od_config = SimpleNamespace(quantization_config=SimpleNamespace(gguf_model="weights.gguf"))
|
||||||
|
|
||||||
assert loader._is_gguf_quantization() is True
|
assert loader._is_gguf_quantization(od_config) is True
|
||||||
|
|
||||||
|
|
||||||
def test_get_model_loadable_names_collects_parameters_and_buffers():
|
def test_get_model_loadable_names_collects_parameters_and_buffers():
|
||||||
@@ -166,14 +171,16 @@ def test_resolve_gguf_model_path_downloads_by_quant_type(monkeypatch: pytest.Mon
|
|||||||
|
|
||||||
def test_get_gguf_weights_iterator_prefixes_source_names(monkeypatch: pytest.MonkeyPatch):
|
def test_get_gguf_weights_iterator_prefixes_source_names(monkeypatch: pytest.MonkeyPatch):
|
||||||
loader = _make_loader()
|
loader = _make_loader()
|
||||||
loader.quant_config = SimpleNamespace(gguf_model="weights.gguf")
|
|
||||||
loader.od_config = SimpleNamespace(revision="main")
|
|
||||||
source = DiffusersPipelineLoader.ComponentSource(
|
source = DiffusersPipelineLoader.ComponentSource(
|
||||||
model_or_path="dummy",
|
model_or_path="dummy",
|
||||||
subfolder="transformer",
|
subfolder="transformer",
|
||||||
revision=None,
|
revision=None,
|
||||||
prefix="transformer.",
|
prefix="transformer.",
|
||||||
)
|
)
|
||||||
|
od_config = SimpleNamespace(
|
||||||
|
revision="main",
|
||||||
|
quantization_config=SimpleNamespace(gguf_model="weights.gguf"),
|
||||||
|
)
|
||||||
|
|
||||||
class _Adapter:
|
class _Adapter:
|
||||||
def weights_iterator(self):
|
def weights_iterator(self):
|
||||||
@@ -183,7 +190,7 @@ def test_get_gguf_weights_iterator_prefixes_source_names(monkeypatch: pytest.Mon
|
|||||||
monkeypatch.setattr(loader, "_resolve_gguf_model_path", lambda gguf_model, revision: "resolved.gguf")
|
monkeypatch.setattr(loader, "_resolve_gguf_model_path", lambda gguf_model, revision: "resolved.gguf")
|
||||||
monkeypatch.setattr(loader_module, "get_gguf_adapter", lambda gguf_file, model, source, od_config: _Adapter())
|
monkeypatch.setattr(loader_module, "get_gguf_adapter", lambda gguf_file, model, source, od_config: _Adapter())
|
||||||
|
|
||||||
weights = list(loader._get_gguf_weights_iterator(source, object()))
|
weights = list(loader._get_gguf_weights_iterator(source, object(), od_config))
|
||||||
|
|
||||||
assert weights[0][0] == "transformer.weight"
|
assert weights[0][0] == "transformer.weight"
|
||||||
assert torch.equal(weights[0][1], torch.ones((2, 2)))
|
assert torch.equal(weights[0][1], torch.ones((2, 2)))
|
||||||
@@ -194,12 +201,13 @@ def test_get_gguf_weights_iterator_prefixes_source_names(monkeypatch: pytest.Mon
|
|||||||
def test_load_weights_with_gguf_falls_back_only_for_missing_transformer_weights(monkeypatch: pytest.MonkeyPatch):
|
def test_load_weights_with_gguf_falls_back_only_for_missing_transformer_weights(monkeypatch: pytest.MonkeyPatch):
|
||||||
loader = _make_loader()
|
loader = _make_loader()
|
||||||
model = _DummyModel()
|
model = _DummyModel()
|
||||||
|
od_config = SimpleNamespace()
|
||||||
|
|
||||||
monkeypatch.setattr(loader, "_get_weight_sources", lambda _model: tuple(model.weights_sources))
|
monkeypatch.setattr(loader, "_get_weight_sources", lambda _model: tuple(model.weights_sources))
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
loader,
|
loader,
|
||||||
"_get_gguf_weights_iterator",
|
"_get_gguf_weights_iterator",
|
||||||
lambda source, model: iter([("transformer.weight", torch.ones((2, 2), dtype=torch.float32))]),
|
lambda source, model, od_config: iter([("transformer.weight", torch.ones((2, 2), dtype=torch.float32))]),
|
||||||
)
|
)
|
||||||
|
|
||||||
def _hf_iter(source):
|
def _hf_iter(source):
|
||||||
@@ -214,7 +222,7 @@ def test_load_weights_with_gguf_falls_back_only_for_missing_transformer_weights(
|
|||||||
lambda _model: {"transformer.weight", "transformer.bias", "vae.weight"},
|
lambda _model: {"transformer.weight", "transformer.bias", "vae.weight"},
|
||||||
)
|
)
|
||||||
|
|
||||||
loaded = loader._load_weights_with_gguf(model)
|
loaded = loader._load_weights_with_gguf(model, od_config)
|
||||||
|
|
||||||
assert loaded == {"transformer.weight", "transformer.bias", "vae.weight"}
|
assert loaded == {"transformer.weight", "transformer.bias", "vae.weight"}
|
||||||
assert model.calls[0] == ["transformer.weight"]
|
assert model.calls[0] == ["transformer.weight"]
|
||||||
@@ -225,12 +233,13 @@ def test_load_weights_with_gguf_falls_back_only_for_missing_transformer_weights(
|
|||||||
def test_load_weights_with_gguf_skips_transformer_fallback_when_gguf_is_complete(monkeypatch: pytest.MonkeyPatch):
|
def test_load_weights_with_gguf_skips_transformer_fallback_when_gguf_is_complete(monkeypatch: pytest.MonkeyPatch):
|
||||||
loader = _make_loader()
|
loader = _make_loader()
|
||||||
model = _DummyModel()
|
model = _DummyModel()
|
||||||
|
od_config = SimpleNamespace()
|
||||||
|
|
||||||
monkeypatch.setattr(loader, "_get_weight_sources", lambda _model: tuple(model.weights_sources))
|
monkeypatch.setattr(loader, "_get_weight_sources", lambda _model: tuple(model.weights_sources))
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
loader,
|
loader,
|
||||||
"_get_gguf_weights_iterator",
|
"_get_gguf_weights_iterator",
|
||||||
lambda source, model: iter(
|
lambda source, model, od_config: iter(
|
||||||
[
|
[
|
||||||
("transformer.weight", torch.ones((2, 2), dtype=torch.float32)),
|
("transformer.weight", torch.ones((2, 2), dtype=torch.float32)),
|
||||||
("transformer.bias", torch.zeros(2, dtype=torch.float32)),
|
("transformer.bias", torch.zeros(2, dtype=torch.float32)),
|
||||||
@@ -253,7 +262,7 @@ def test_load_weights_with_gguf_skips_transformer_fallback_when_gguf_is_complete
|
|||||||
lambda _model: {"transformer.weight", "transformer.bias", "vae.weight"},
|
lambda _model: {"transformer.weight", "transformer.bias", "vae.weight"},
|
||||||
)
|
)
|
||||||
|
|
||||||
loaded = loader._load_weights_with_gguf(model)
|
loaded = loader._load_weights_with_gguf(model, od_config)
|
||||||
|
|
||||||
assert loaded == {"transformer.weight", "transformer.bias", "vae.weight"}
|
assert loaded == {"transformer.weight", "transformer.bias", "vae.weight"}
|
||||||
assert hf_calls == ["vae"]
|
assert hf_calls == ["vae"]
|
||||||
@@ -73,42 +73,14 @@ class StubCosmos3VAE:
|
|||||||
return (latents,)
|
return (latents,)
|
||||||
|
|
||||||
|
|
||||||
class StubCosmos3AVAE:
|
|
||||||
def __init__(self, **kwargs: Any) -> None:
|
|
||||||
self.kwargs = kwargs
|
|
||||||
self.sample_rate = int(kwargs["sample_rate"])
|
|
||||||
self.audio_channels = int(kwargs["audio_channels"])
|
|
||||||
self.latent_ch = int(kwargs["io_channels"])
|
|
||||||
self.temporal_compression_factor = int(kwargs["hop_size"])
|
|
||||||
|
|
||||||
def get_latent_num_samples(self, num_audio_samples: int) -> int:
|
|
||||||
return int(num_audio_samples) // self.temporal_compression_factor
|
|
||||||
|
|
||||||
def get_audio_num_samples(self, num_latent_samples: int) -> int:
|
|
||||||
return int(num_latent_samples) * self.temporal_compression_factor
|
|
||||||
|
|
||||||
def decode(self, latents: torch.Tensor) -> torch.Tensor:
|
|
||||||
return torch.zeros(latents.shape[0], self.audio_channels, 8)
|
|
||||||
|
|
||||||
|
|
||||||
class StubCosmos3Transformer(nn.Module):
|
class StubCosmos3Transformer(nn.Module):
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
latent_channel_size: int = 2,
|
latent_channel_size: int = 2,
|
||||||
sound_gen: bool = False,
|
|
||||||
sound_dim: int = 3,
|
|
||||||
sound_latent_fps: float = 25.0,
|
|
||||||
action_gen: bool = False,
|
|
||||||
action_dim: int = 4,
|
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.latent_channel_size = latent_channel_size
|
self.latent_channel_size = latent_channel_size
|
||||||
self.sound_gen = sound_gen
|
|
||||||
self.sound_dim = sound_dim
|
|
||||||
self.sound_latent_fps = sound_latent_fps
|
|
||||||
self.action_gen = action_gen
|
|
||||||
self.action_dim = action_dim
|
|
||||||
self.cached_kv: Any | None = None
|
self.cached_kv: Any | None = None
|
||||||
self.cached_freqs_gen: Any | None = None
|
self.cached_freqs_gen: Any | None = None
|
||||||
self.calls: list[dict[str, Any]] = []
|
self.calls: list[dict[str, Any]] = []
|
||||||
@@ -127,9 +99,8 @@ class StubCosmos3Transformer(nn.Module):
|
|||||||
text_ids: torch.Tensor,
|
text_ids: torch.Tensor,
|
||||||
text_mask: torch.Tensor,
|
text_mask: torch.Tensor,
|
||||||
**kwargs: Any,
|
**kwargs: Any,
|
||||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
) -> torch.Tensor:
|
||||||
token = int(text_ids.reshape(-1)[0].item()) if text_ids.numel() else 0
|
token = int(text_ids.reshape(-1)[0].item()) if text_ids.numel() else 0
|
||||||
sound_latents = kwargs.get("sound_latents")
|
|
||||||
self.calls.append(
|
self.calls.append(
|
||||||
{
|
{
|
||||||
"token": token,
|
"token": token,
|
||||||
@@ -143,13 +114,7 @@ class StubCosmos3Transformer(nn.Module):
|
|||||||
marker = torch.tensor([token], dtype=torch.float32)
|
marker = torch.tensor([token], dtype=torch.float32)
|
||||||
self.cached_kv = [(marker, marker + 100)]
|
self.cached_kv = [(marker, marker + 100)]
|
||||||
self.cached_freqs_gen = (marker + 200, marker + 300)
|
self.cached_freqs_gen = (marker + 200, marker + 300)
|
||||||
action_latents = kwargs.get("action_latents")
|
return torch.full_like(hidden_states, float(token))
|
||||||
outputs: list[torch.Tensor] = [torch.full_like(hidden_states, float(token))]
|
|
||||||
if action_latents is not None:
|
|
||||||
outputs.append(torch.full_like(action_latents, float(token + 20)))
|
|
||||||
if sound_latents is not None:
|
|
||||||
outputs.append(torch.full_like(sound_latents, float(token + 10)))
|
|
||||||
return outputs[0] if len(outputs) == 1 else tuple(outputs)
|
|
||||||
|
|
||||||
|
|
||||||
def passthrough_progress_bar(iterable):
|
def passthrough_progress_bar(iterable):
|
||||||
@@ -190,7 +155,6 @@ def make_cosmos3_pipeline():
|
|||||||
pipeline._guidance_scale = None
|
pipeline._guidance_scale = None
|
||||||
pipeline._num_timesteps = None
|
pipeline._num_timesteps = None
|
||||||
pipeline._cache_dit_requires_paired_cfg = False
|
pipeline._cache_dit_requires_paired_cfg = False
|
||||||
pipeline._sound_tokenizer = None
|
|
||||||
pipeline.progress_bar = passthrough_progress_bar
|
pipeline.progress_bar = passthrough_progress_bar
|
||||||
return pipeline
|
return pipeline
|
||||||
|
|
||||||
@@ -249,109 +213,7 @@ def test_pipeline_registered_and_exported() -> None:
|
|||||||
assert "Cosmos3OmniDiffusersPipeline" in cosmos3.__all__
|
assert "Cosmos3OmniDiffusersPipeline" in cosmos3.__all__
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
def test_preprocess_i2v_image_input() -> None:
|
||||||
def stub_real_pipeline_init(monkeypatch: pytest.MonkeyPatch):
|
|
||||||
from vllm_omni.diffusion.models.cosmos3 import pipeline_cosmos3
|
|
||||||
|
|
||||||
class _StubAutoTokenizer:
|
|
||||||
@classmethod
|
|
||||||
def from_pretrained(cls, *args, **kwargs):
|
|
||||||
return SimpleNamespace()
|
|
||||||
|
|
||||||
class _StubDiffusersVAE:
|
|
||||||
config = SimpleNamespace(scale_factor_temporal=4, scale_factor_spatial=8)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_pretrained(cls, *args, **kwargs):
|
|
||||||
return cls()
|
|
||||||
|
|
||||||
def to(self, _device):
|
|
||||||
return self
|
|
||||||
|
|
||||||
class _StubDiffusersScheduler:
|
|
||||||
config = SimpleNamespace(flow_shift=1.0)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_pretrained(cls, *args, **kwargs):
|
|
||||||
return cls()
|
|
||||||
|
|
||||||
class _StubVideoProcessor:
|
|
||||||
def __init__(self, *args, **kwargs) -> None:
|
|
||||||
pass
|
|
||||||
|
|
||||||
monkeypatch.setattr(pipeline_cosmos3, "AutoTokenizer", _StubAutoTokenizer)
|
|
||||||
monkeypatch.setattr(pipeline_cosmos3, "DistributedAutoencoderKLWan", _StubDiffusersVAE)
|
|
||||||
monkeypatch.setattr(pipeline_cosmos3, "UniPCMultistepScheduler", _StubDiffusersScheduler)
|
|
||||||
monkeypatch.setattr(pipeline_cosmos3, "VideoProcessor", _StubVideoProcessor)
|
|
||||||
monkeypatch.setattr(pipeline_cosmos3, "get_local_device", lambda: torch.device("cpu"))
|
|
||||||
|
|
||||||
|
|
||||||
def _make_od_config(*, sound_gen: bool) -> SimpleNamespace:
|
|
||||||
tf_model_config = {
|
|
||||||
"hidden_size": 8,
|
|
||||||
"num_hidden_layers": 0,
|
|
||||||
"num_attention_heads": 2,
|
|
||||||
"num_key_value_heads": 2,
|
|
||||||
"head_dim": 4,
|
|
||||||
"intermediate_size": 16,
|
|
||||||
"vocab_size": 32,
|
|
||||||
"latent_patch_size": 1,
|
|
||||||
"latent_channel": 2,
|
|
||||||
"rope_scaling": {"mrope_section": [1, 1, 0]},
|
|
||||||
}
|
|
||||||
if sound_gen:
|
|
||||||
tf_model_config["sound_gen"] = True
|
|
||||||
return SimpleNamespace(
|
|
||||||
enable_cpu_offload=False,
|
|
||||||
enable_diffusion_pipeline_profiler=False,
|
|
||||||
model="/nonexistent/model/path",
|
|
||||||
dtype=torch.float32,
|
|
||||||
flow_shift=None,
|
|
||||||
quantization_config=None,
|
|
||||||
custom_pipeline_args={},
|
|
||||||
model_config={},
|
|
||||||
tf_model_config=tf_model_config,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_pipeline_init_skips_tokenizer_when_sound_disabled(stub_real_pipeline_init) -> None:
|
|
||||||
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import Cosmos3OmniDiffusersPipeline
|
|
||||||
|
|
||||||
pipeline = Cosmos3OmniDiffusersPipeline(od_config=_make_od_config(sound_gen=False))
|
|
||||||
|
|
||||||
assert pipeline._sound_tokenizer is None
|
|
||||||
assert pipeline.transformer.sound_gen is False
|
|
||||||
assert not hasattr(pipeline.transformer, "audio_proj_in")
|
|
||||||
assert not hasattr(pipeline.transformer, "audio_proj_out")
|
|
||||||
|
|
||||||
|
|
||||||
def test_pipeline_init_passes_tokenizer_attrs_into_transformer(
|
|
||||||
stub_real_pipeline_init,
|
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
|
||||||
) -> None:
|
|
||||||
from vllm_omni.diffusion.models.cosmos3 import sound_tokenizer
|
|
||||||
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import Cosmos3OmniDiffusersPipeline
|
|
||||||
|
|
||||||
stub_tokenizer = sound_tokenizer.Cosmos3SoundTokenizer(
|
|
||||||
StubCosmos3AVAE(sample_rate=32000, audio_channels=2, io_channels=5, hop_size=800)
|
|
||||||
)
|
|
||||||
monkeypatch.setattr(
|
|
||||||
sound_tokenizer.Cosmos3SoundTokenizer,
|
|
||||||
"from_config",
|
|
||||||
classmethod(lambda cls, od_config: stub_tokenizer),
|
|
||||||
)
|
|
||||||
|
|
||||||
pipeline = Cosmos3OmniDiffusersPipeline(od_config=_make_od_config(sound_gen=True))
|
|
||||||
|
|
||||||
assert pipeline._sound_tokenizer is stub_tokenizer
|
|
||||||
assert pipeline.transformer.sound_gen is True
|
|
||||||
assert pipeline.transformer.sound_dim == pipeline._sound_tokenizer.latent_ch == 5
|
|
||||||
assert pipeline.transformer.sound_latent_fps == pipeline._sound_tokenizer.latent_fps == 40.0
|
|
||||||
assert pipeline.transformer.audio_proj_in.in_features == 5
|
|
||||||
assert pipeline.transformer.audio_proj_out.out_features == 5
|
|
||||||
|
|
||||||
|
|
||||||
def test_preprocess_i2v_image_and_action_video_inputs() -> None:
|
|
||||||
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import get_cosmos3_pre_process_func
|
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import get_cosmos3_pre_process_func
|
||||||
|
|
||||||
preprocess = get_cosmos3_pre_process_func(SimpleNamespace())
|
preprocess = get_cosmos3_pre_process_func(SimpleNamespace())
|
||||||
@@ -364,31 +226,8 @@ def test_preprocess_i2v_image_and_action_video_inputs() -> None:
|
|||||||
assert (result.sampling_params.height, result.sampling_params.width) == (672, 1344)
|
assert (result.sampling_params.height, result.sampling_params.width) == (672, 1344)
|
||||||
assert tuple(result.prompts[0]["additional_information"]["preprocessed_image"].shape[-2:]) == (672, 1344)
|
assert tuple(result.prompts[0]["additional_information"]["preprocessed_image"].shape[-2:]) == (672, 1344)
|
||||||
|
|
||||||
frames = [Image.new("RGB", (8, 4), color) for color in ("red", "green", "blue")]
|
|
||||||
action = SimpleNamespace(
|
|
||||||
prompts=[{"prompt": "Move.", "multi_modal_data": {"video": frames}}],
|
|
||||||
sampling_params=SimpleNamespace(height=16, width=32, extra_args={"action_mode": "forward_dynamics"}),
|
|
||||||
)
|
|
||||||
|
|
||||||
additional = preprocess(action).prompts[0]["additional_information"]
|
def test_postprocess_handles_image_video_and_validation() -> None:
|
||||||
assert tuple(additional["preprocessed_image"].shape) == (1, 3, 16, 32)
|
|
||||||
assert tuple(additional["preprocessed_video"].shape) == (1, 3, 3, 16, 32)
|
|
||||||
|
|
||||||
frames = [Image.new("RGB", (8, 4), color) for color in ("red", "green", "blue", "yellow", "purple", "black")]
|
|
||||||
v2v = SimpleNamespace(
|
|
||||||
prompts=[{"prompt": "Continue.", "multi_modal_data": {"video": frames}}],
|
|
||||||
sampling_params=SimpleNamespace(
|
|
||||||
height=16,
|
|
||||||
width=32,
|
|
||||||
extra_args={"condition_frame_indexes_vision": [0, 1], "condition_video_keep": "last"},
|
|
||||||
),
|
|
||||||
)
|
|
||||||
additional = preprocess(v2v).prompts[0]["additional_information"]
|
|
||||||
assert tuple(additional["preprocessed_video"].shape) == (1, 3, 5, 16, 32)
|
|
||||||
assert additional["condition_frame_indexes_vision"] == [0, 1]
|
|
||||||
|
|
||||||
|
|
||||||
def test_postprocess_handles_image_video_audio_and_validation() -> None:
|
|
||||||
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import get_cosmos3_post_process_func
|
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import get_cosmos3_post_process_func
|
||||||
|
|
||||||
func = get_cosmos3_post_process_func(SimpleNamespace())
|
func = get_cosmos3_post_process_func(SimpleNamespace())
|
||||||
@@ -396,16 +235,6 @@ def test_postprocess_handles_image_video_audio_and_validation() -> None:
|
|||||||
|
|
||||||
assert func(video, output_type="latent") is video
|
assert func(video, output_type="latent") is video
|
||||||
assert func({"image": video})[0].size == (4, 4)
|
assert func({"image": video})[0].size == (4, 4)
|
||||||
# Video-only postprocess returns the bare processed video (not a dict),
|
|
||||||
# matching the image/latent branches and peer audio-capable pipelines.
|
|
||||||
assert not isinstance(func({"video": video}), dict)
|
|
||||||
assert (
|
|
||||||
func(
|
|
||||||
{"video": video, "audio": torch.ones(1, 2, 16), "audio_sample_rate": 48000},
|
|
||||||
sampling_params=SimpleNamespace(extra_args={"resolved_frame_rate": 12}),
|
|
||||||
)["audio_sample_rate"]
|
|
||||||
== 48000
|
|
||||||
)
|
|
||||||
|
|
||||||
with pytest.raises(ValueError, match="text-to-image postprocess expects"):
|
with pytest.raises(ValueError, match="text-to-image postprocess expects"):
|
||||||
func({"image": torch.zeros(1, 3, 2, 4, 4)})
|
func({"image": torch.zeros(1, 3, 2, 4, 4)})
|
||||||
@@ -464,7 +293,7 @@ def test_prompt_formatting_and_checkpoint_key_remap(make_cosmos3_pipeline) -> No
|
|||||||
assert {key: Cosmos3OmniDiffusersPipeline._remap_ckpt_key(key) for key in remaps} == remaps
|
assert {key: Cosmos3OmniDiffusersPipeline._remap_ckpt_key(key) for key in remaps} == remaps
|
||||||
|
|
||||||
|
|
||||||
def test_prepare_latents_for_video_image_sound_and_action(make_cosmos3_pipeline) -> None:
|
def test_prepare_latents_for_video_and_image(make_cosmos3_pipeline) -> None:
|
||||||
pipeline = make_cosmos3_pipeline()
|
pipeline = make_cosmos3_pipeline()
|
||||||
latents = pipeline._prepare_latents(16, 24, 5, torch.Generator(device="cpu").manual_seed(0))
|
latents = pipeline._prepare_latents(16, 24, 5, torch.Generator(device="cpu").manual_seed(0))
|
||||||
assert latents.shape == (1, 2, 2, 2, 3)
|
assert latents.shape == (1, 2, 2, 2, 3)
|
||||||
@@ -477,49 +306,8 @@ def test_prepare_latents_for_video_image_sound_and_action(make_cosmos3_pipeline)
|
|||||||
assert velocity_mask.tolist() == [[[[[0.0]], [[1.0]]]]]
|
assert velocity_mask.tolist() == [[[[[0.0]], [[1.0]]]]]
|
||||||
assert image_latent.shape == (1, 2, 1, 2, 3)
|
assert image_latent.shape == (1, 2, 1, 2, 3)
|
||||||
|
|
||||||
pipeline._encode_video_tensor = lambda *args, **kwargs: torch.full((1, 2, 3, 2, 3), 6.0)
|
|
||||||
v2v_latents, v2v_velocity_mask, v2v_condition = pipeline._prepare_latents_v2v(
|
|
||||||
torch.zeros(1, 3, 5, 16, 24),
|
|
||||||
16,
|
|
||||||
24,
|
|
||||||
9,
|
|
||||||
torch.Generator(device="cpu").manual_seed(0),
|
|
||||||
[0, 1],
|
|
||||||
)
|
|
||||||
torch.testing.assert_close(v2v_latents[:, :, 0:2], torch.full((1, 2, 2, 2, 3), 6.0))
|
|
||||||
assert v2v_velocity_mask.tolist() == [[[[[0.0]], [[0.0]], [[1.0]]]]]
|
|
||||||
assert v2v_condition.shape == (1, 2, 3, 2, 3)
|
|
||||||
|
|
||||||
pipeline.transformer = pipeline.transformer.__class__(latent_channel_size=2, sound_gen=True, sound_dim=3)
|
def test_diffuse_covers_cfg_and_i2v_steps(make_cosmos3_pipeline) -> None:
|
||||||
pipeline._sound_tokenizer = SimpleNamespace(
|
|
||||||
sample_rate=10,
|
|
||||||
latent_ch=3,
|
|
||||||
hop_size=4,
|
|
||||||
decode=lambda x: torch.ones(x.shape[0], 2, 24),
|
|
||||||
)
|
|
||||||
assert pipeline._resolve_sound_target_samples(SimpleNamespace(extra_args={"sound_duration": 2.0}), 9, 3.0) == (
|
|
||||||
20,
|
|
||||||
2.0,
|
|
||||||
10,
|
|
||||||
)
|
|
||||||
sound_latents, latent_frames = pipeline._prepare_sound_latents(21, torch.Generator(device="cpu").manual_seed(0))
|
|
||||||
assert (sound_latents.shape, latent_frames) == (torch.Size([1, 3, 6]), 6)
|
|
||||||
assert pipeline._decode_sound_latents(torch.zeros(1, 3, 6), target_audio_samples=21).shape == (1, 2, 21)
|
|
||||||
|
|
||||||
pipeline.transformer = pipeline.transformer.__class__(action_gen=True, action_dim=4)
|
|
||||||
action, action_mask, clean, raw_dim = pipeline._prepare_action_latents(
|
|
||||||
mode="forward_dynamics",
|
|
||||||
action_chunk_size=2,
|
|
||||||
raw_action_dim=None,
|
|
||||||
generator=torch.Generator(device="cpu").manual_seed(0),
|
|
||||||
sp=SimpleNamespace(extra_args={"action": [[1.0, 2.0], [3.0, 4.0]]}),
|
|
||||||
)
|
|
||||||
assert raw_dim == 2
|
|
||||||
assert action_mask.tolist() == [[[0.0], [0.0]]]
|
|
||||||
torch.testing.assert_close(action, clean)
|
|
||||||
|
|
||||||
|
|
||||||
def test_diffuse_covers_cfg_i2v_and_multimodal_steps(make_cosmos3_pipeline) -> None:
|
|
||||||
pipeline = make_cosmos3_pipeline()
|
pipeline = make_cosmos3_pipeline()
|
||||||
latents = torch.zeros(1, 2, 1, 1, 1)
|
latents = torch.zeros(1, 2, 1, 1, 1)
|
||||||
|
|
||||||
@@ -551,23 +339,6 @@ def test_diffuse_covers_cfg_i2v_and_multimodal_steps(make_cosmos3_pipeline) -> N
|
|||||||
)
|
)
|
||||||
torch.testing.assert_close(i2v[:, :, 0:1], torch.full((1, 2, 1, 1, 1), 7.0))
|
torch.testing.assert_close(i2v[:, :, 0:1], torch.full((1, 2, 1, 1, 1), 7.0))
|
||||||
|
|
||||||
pipeline.transformer = pipeline.transformer.__class__(latent_channel_size=2, action_gen=True, action_dim=4)
|
|
||||||
video_result, action_result = pipeline.diffuse(
|
|
||||||
latents=latents,
|
|
||||||
action_latents=torch.zeros(1, 3, 4),
|
|
||||||
action_velocity_mask=torch.ones(1, 3, 1),
|
|
||||||
action_condition_latents=torch.zeros(1, 3, 4),
|
|
||||||
timesteps=torch.tensor([7, 3]),
|
|
||||||
cond_ids=_ids(2),
|
|
||||||
cond_mask=_mask(),
|
|
||||||
uncond_ids=_ids(1),
|
|
||||||
uncond_mask=_mask(),
|
|
||||||
guidance_scale=1.0,
|
|
||||||
shared_kwargs={"video_shape": (1, 1, 1), "fps": 24.0, "action_domain_ids": torch.tensor([0])},
|
|
||||||
)
|
|
||||||
torch.testing.assert_close(video_result, torch.full_like(latents, 4.0))
|
|
||||||
torch.testing.assert_close(action_result, torch.full((), 44.0).expand_as(action_result))
|
|
||||||
|
|
||||||
|
|
||||||
def test_diffuse_keeps_paired_cfg_when_cache_dit_active(make_cosmos3_pipeline) -> None:
|
def test_diffuse_keeps_paired_cfg_when_cache_dit_active(make_cosmos3_pipeline) -> None:
|
||||||
"""With cache-dit active the uncond pass is kept even outside the guidance
|
"""With cache-dit active the uncond pass is kept even outside the guidance
|
||||||
@@ -624,12 +395,7 @@ class TestForwardRouting:
|
|||||||
|
|
||||||
def fake_diffuse(**kwargs):
|
def fake_diffuse(**kwargs):
|
||||||
captured["diffuse_calls"].append(kwargs)
|
captured["diffuse_calls"].append(kwargs)
|
||||||
outputs = [kwargs["latents"] + len(captured["diffuse_calls"])]
|
return kwargs["latents"] + len(captured["diffuse_calls"])
|
||||||
if kwargs.get("action_latents") is not None:
|
|
||||||
outputs.append(kwargs["action_latents"] + 3.0)
|
|
||||||
if kwargs.get("sound_latents") is not None:
|
|
||||||
outputs.append(kwargs["sound_latents"] + 2.0)
|
|
||||||
return outputs[0] if len(outputs) == 1 else tuple(outputs)
|
|
||||||
|
|
||||||
pipeline._format_and_tokenize_prompts = fake_format
|
pipeline._format_and_tokenize_prompts = fake_format
|
||||||
pipeline._prepare_latents = fake_prepare
|
pipeline._prepare_latents = fake_prepare
|
||||||
@@ -671,7 +437,7 @@ class TestForwardRouting:
|
|||||||
assert captured["flow_shifts"] == expected["flow"]
|
assert captured["flow_shifts"] == expected["flow"]
|
||||||
assert [call[0] for call in pipeline.scheduler.set_timesteps_calls] == expected["steps"]
|
assert [call[0] for call in pipeline.scheduler.set_timesteps_calls] == expected["steps"]
|
||||||
|
|
||||||
def test_forward_i2v_sound_and_action_routes(self, make_cosmos3_pipeline) -> None:
|
def test_forward_i2v_route(self, make_cosmos3_pipeline) -> None:
|
||||||
pipeline = make_cosmos3_pipeline()
|
pipeline = make_cosmos3_pipeline()
|
||||||
captured = self._install_forward_stubs(pipeline)
|
captured = self._install_forward_stubs(pipeline)
|
||||||
image_tensor = torch.zeros(1, 3, 16, 16)
|
image_tensor = torch.zeros(1, 3, 16, 16)
|
||||||
@@ -696,83 +462,11 @@ class TestForwardRouting:
|
|||||||
)
|
)
|
||||||
assert captured["diffuse_calls"][-1]["shared_kwargs"]["noisy_frame_mask"] is velocity_mask
|
assert captured["diffuse_calls"][-1]["shared_kwargs"]["noisy_frame_mask"] is velocity_mask
|
||||||
|
|
||||||
video_tensor = torch.zeros(1, 3, 5, 16, 16)
|
|
||||||
v2v_condition = torch.full((1, 2, 2, 1, 1), 4.0)
|
|
||||||
v2v_mask = torch.tensor([[[[[0.0]], [[1.0]]]]])
|
|
||||||
pipeline._prepare_latents_v2v = lambda *args, **kwargs: (
|
|
||||||
torch.zeros(1, 2, 2, 1, 1),
|
|
||||||
v2v_mask,
|
|
||||||
v2v_condition,
|
|
||||||
)
|
|
||||||
pipeline.forward(
|
|
||||||
SimpleNamespace(
|
|
||||||
prompts=[
|
|
||||||
{
|
|
||||||
"prompt": "continue",
|
|
||||||
"modalities": ["video"],
|
|
||||||
"additional_information": {
|
|
||||||
"preprocessed_video": video_tensor,
|
|
||||||
"condition_frame_indexes_vision": [0],
|
|
||||||
},
|
|
||||||
}
|
|
||||||
],
|
|
||||||
sampling_params=make_sampling_params(height=16, width=16, num_frames=5),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
assert captured["flow_shifts"][-1] == 10.0
|
|
||||||
assert captured["format"]["negative_prompt"] == ""
|
|
||||||
assert captured["diffuse_calls"][-1]["shared_kwargs"]["noisy_frame_mask"] is v2v_mask
|
|
||||||
assert captured["diffuse_calls"][-1]["condition_latents"] is v2v_condition
|
|
||||||
|
|
||||||
pipeline.transformer = pipeline.transformer.__class__(latent_channel_size=2, sound_gen=True, sound_dim=3)
|
|
||||||
sound_latents = torch.zeros(1, 3, 4)
|
|
||||||
pipeline._resolve_sound_target_samples = lambda *args: (20, 2.0, 10)
|
|
||||||
pipeline._prepare_sound_latents = lambda *args: (sound_latents, 4)
|
|
||||||
pipeline._decode_sound_latents = lambda *args: torch.ones(1, 2, 20)
|
|
||||||
output = pipeline.forward(
|
|
||||||
SimpleNamespace(
|
|
||||||
prompts=[{"prompt": "A robot", "modalities": ["video"], "generate_sound": True}],
|
|
||||||
sampling_params=make_sampling_params(num_frames=9, frame_rate=3.0),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
assert captured["diffuse_calls"][-1]["sound_latents"] is sound_latents
|
|
||||||
assert output.output["audio_sample_rate"] == 10
|
|
||||||
|
|
||||||
pipeline.transformer = pipeline.transformer.__class__(latent_channel_size=2, action_gen=True, action_dim=4)
|
|
||||||
output = pipeline.forward(
|
|
||||||
SimpleNamespace(
|
|
||||||
prompts=[
|
|
||||||
{
|
|
||||||
"prompt": "Pick the block.",
|
|
||||||
"modalities": ["video"],
|
|
||||||
"additional_information": {"preprocessed_image": image_tensor},
|
|
||||||
}
|
|
||||||
],
|
|
||||||
sampling_params=make_sampling_params(
|
|
||||||
height=16,
|
|
||||||
width=16,
|
|
||||||
extra_args={
|
|
||||||
"action_mode": "policy",
|
|
||||||
"action_chunk_size": 2,
|
|
||||||
"raw_action_dim": 2,
|
|
||||||
"domain_name": "bridge_orig_lerobot",
|
|
||||||
},
|
|
||||||
),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
assert captured["diffuse_calls"][-1]["shared_kwargs"]["action_domain_ids"].tolist() == [7]
|
|
||||||
assert output.custom_output["action"].shape == (1, 2, 2)
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("prompt", "sampling_params", "message"),
|
("prompt", "sampling_params", "message"),
|
||||||
[
|
[
|
||||||
(["one", "two"], make_sampling_params(), "single prompt"),
|
(["one", "two"], make_sampling_params(), "single prompt"),
|
||||||
([{"prompt": "one", "modalities": ["image", "video"]}], make_sampling_params(), "both image and video"),
|
([{"prompt": "one", "modalities": ["image", "video"]}], make_sampling_params(), "both image and video"),
|
||||||
(
|
|
||||||
[{"prompt": "x", "modalities": ["image"], "generate_sound": True}],
|
|
||||||
make_sampling_params(),
|
|
||||||
"only for video",
|
|
||||||
),
|
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
def test_forward_rejects_invalid_public_requests(
|
def test_forward_rejects_invalid_public_requests(
|
||||||
@@ -783,7 +477,6 @@ class TestForwardRouting:
|
|||||||
message,
|
message,
|
||||||
) -> None:
|
) -> None:
|
||||||
pipeline = make_cosmos3_pipeline()
|
pipeline = make_cosmos3_pipeline()
|
||||||
pipeline.transformer = pipeline.transformer.__class__(latent_channel_size=2, sound_gen=True, sound_dim=3)
|
|
||||||
|
|
||||||
with pytest.raises(ValueError, match=message):
|
with pytest.raises(ValueError, match=message):
|
||||||
pipeline.forward(SimpleNamespace(prompts=prompt, sampling_params=sampling_params))
|
pipeline.forward(SimpleNamespace(prompts=prompt, sampling_params=sampling_params))
|
||||||
@@ -1,236 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import json
|
|
||||||
from pathlib import Path
|
|
||||||
from types import SimpleNamespace
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
import torch
|
|
||||||
|
|
||||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
|
||||||
|
|
||||||
DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME = "diffusion_pytorch_model.safetensors"
|
|
||||||
|
|
||||||
|
|
||||||
class _FakeAVAEAudioTokenizer:
|
|
||||||
def __init__(self, **kwargs) -> None:
|
|
||||||
self.kwargs = kwargs
|
|
||||||
self.sample_rate = int(kwargs["sample_rate"])
|
|
||||||
self.audio_channels = int(kwargs["audio_channels"])
|
|
||||||
self.latent_ch = int(kwargs["io_channels"])
|
|
||||||
self.temporal_compression_factor = int(kwargs["hop_size"])
|
|
||||||
|
|
||||||
def get_latent_num_samples(self, num_audio_samples: int) -> int:
|
|
||||||
return int(num_audio_samples) // self.temporal_compression_factor
|
|
||||||
|
|
||||||
def get_audio_num_samples(self, num_latent_samples: int) -> int:
|
|
||||||
return int(num_latent_samples) * self.temporal_compression_factor
|
|
||||||
|
|
||||||
def decode(self, latents: torch.Tensor) -> torch.Tensor:
|
|
||||||
return torch.zeros(latents.shape[0], self.audio_channels, 8)
|
|
||||||
|
|
||||||
|
|
||||||
def _write_component(root: Path, config: dict | None = None, checkpoint_name: str | None = None) -> Path:
|
|
||||||
tokenizer_dir = root / "sound_tokenizer"
|
|
||||||
tokenizer_dir.mkdir(parents=True)
|
|
||||||
if checkpoint_name:
|
|
||||||
(tokenizer_dir / checkpoint_name).write_bytes(b"stub")
|
|
||||||
(tokenizer_dir / "config.json").write_text(json.dumps(config or {}), encoding="utf-8")
|
|
||||||
return tokenizer_dir
|
|
||||||
|
|
||||||
|
|
||||||
def _patch_fake_avae(monkeypatch: pytest.MonkeyPatch, created: dict) -> None:
|
|
||||||
from vllm_omni.diffusion.models.cosmos3 import sound_tokenizer
|
|
||||||
|
|
||||||
class FakeAVAE(_FakeAVAEAudioTokenizer):
|
|
||||||
def __init__(self, **kwargs) -> None:
|
|
||||||
created.update(kwargs)
|
|
||||||
super().__init__(**kwargs)
|
|
||||||
|
|
||||||
monkeypatch.setattr(sound_tokenizer, "Cosmos3AVAEAudioTokenizer", FakeAVAE)
|
|
||||||
monkeypatch.setattr(sound_tokenizer, "get_local_device", lambda: torch.device("cpu"))
|
|
||||||
|
|
||||||
|
|
||||||
def test_from_config_loads_local_diffusers_component(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
||||||
from vllm_omni.diffusion.models.cosmos3 import sound_tokenizer
|
|
||||||
|
|
||||||
model_dir = tmp_path / "model"
|
|
||||||
tokenizer_dir = _write_component(model_dir, checkpoint_name=DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME)
|
|
||||||
created = {}
|
|
||||||
_patch_fake_avae(monkeypatch, created)
|
|
||||||
|
|
||||||
tokenizer = sound_tokenizer.Cosmos3SoundTokenizer.from_config(
|
|
||||||
SimpleNamespace(
|
|
||||||
model=str(model_dir),
|
|
||||||
custom_pipeline_args={"sound_sample_rate": 32000, "sound_hop_size": 800, "sound_dim": 3},
|
|
||||||
dtype=torch.float32,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
assert created["checkpoint_path"] == str(tokenizer_dir / DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME)
|
|
||||||
assert created["config_path"] == str(tokenizer_dir / "config.json")
|
|
||||||
assert (tokenizer.sample_rate, tokenizer.latent_ch, tokenizer.hop_size, tokenizer.latent_fps) == (
|
|
||||||
32000,
|
|
||||||
3,
|
|
||||||
800,
|
|
||||||
40.0,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_from_config_downloads_component_from_hf_repo(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
||||||
import huggingface_hub
|
|
||||||
|
|
||||||
from vllm_omni.diffusion.models.cosmos3 import sound_tokenizer
|
|
||||||
|
|
||||||
cache_dir = tmp_path / "hf"
|
|
||||||
_write_component(cache_dir, checkpoint_name=DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME)
|
|
||||||
calls = []
|
|
||||||
created = {}
|
|
||||||
_patch_fake_avae(monkeypatch, created)
|
|
||||||
|
|
||||||
def fake_snapshot_download(repo_id: str, *, revision: str | None, allow_patterns: list[str]) -> str:
|
|
||||||
calls.append((repo_id, revision, allow_patterns))
|
|
||||||
return str(cache_dir)
|
|
||||||
|
|
||||||
monkeypatch.setattr(huggingface_hub, "snapshot_download", fake_snapshot_download)
|
|
||||||
|
|
||||||
sound_tokenizer.Cosmos3SoundTokenizer.from_config(
|
|
||||||
SimpleNamespace(
|
|
||||||
model="nvidia/cosmos3",
|
|
||||||
revision="test-rev",
|
|
||||||
custom_pipeline_args={"sound_sample_rate": 32000, "sound_hop_size": 800, "sound_dim": 3},
|
|
||||||
dtype=torch.float32,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
assert created["checkpoint_path"].endswith(DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME)
|
|
||||||
assert calls == [
|
|
||||||
(
|
|
||||||
"nvidia/cosmos3",
|
|
||||||
"test-rev",
|
|
||||||
["sound_tokenizer/config.json", f"sound_tokenizer/{DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME}"],
|
|
||||||
)
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
("checkpoint_name", "message"),
|
|
||||||
[
|
|
||||||
(None, "no AVAE sound tokenizer checkpoint"),
|
|
||||||
("model.safetensors", DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_default_component_requires_diffusers_checkpoint_name(tmp_path, checkpoint_name, message) -> None:
|
|
||||||
from vllm_omni.diffusion.models.cosmos3 import sound_tokenizer
|
|
||||||
|
|
||||||
model_dir = tmp_path / "model"
|
|
||||||
_write_component(model_dir, checkpoint_name=checkpoint_name)
|
|
||||||
|
|
||||||
with pytest.raises(ValueError, match=message):
|
|
||||||
sound_tokenizer.Cosmos3SoundTokenizer.from_config(
|
|
||||||
SimpleNamespace(model=str(model_dir), custom_pipeline_args={}, dtype=torch.float32)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_component_config_precedence_and_conflict_detection(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
||||||
from vllm_omni.diffusion.models.cosmos3 import sound_tokenizer
|
|
||||||
|
|
||||||
component_config = {
|
|
||||||
"sampling_rate": 48000,
|
|
||||||
"dec_out_channels": 2,
|
|
||||||
"vocoder_input_dim": 64,
|
|
||||||
"hop_size": 1920,
|
|
||||||
}
|
|
||||||
model_dir = tmp_path / "model"
|
|
||||||
_write_component(model_dir, component_config, DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME)
|
|
||||||
created = {}
|
|
||||||
_patch_fake_avae(monkeypatch, created)
|
|
||||||
|
|
||||||
tokenizer = sound_tokenizer.Cosmos3SoundTokenizer.from_config(
|
|
||||||
SimpleNamespace(
|
|
||||||
model=str(model_dir),
|
|
||||||
custom_pipeline_args={
|
|
||||||
"sound_normalize_latents": True,
|
|
||||||
"sound_normalization_type": "tanh",
|
|
||||||
"sound_tanh_input_scale": 2.0,
|
|
||||||
},
|
|
||||||
model_config={
|
|
||||||
"sound_tokenizer": {
|
|
||||||
"sample_rate": 32000,
|
|
||||||
"audio_channels": 1,
|
|
||||||
"io_channels": 3,
|
|
||||||
"hop_size": 800,
|
|
||||||
"normalize_latents": False,
|
|
||||||
"normalization_type": "none",
|
|
||||||
}
|
|
||||||
},
|
|
||||||
dtype=torch.float32,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
assert (created["sample_rate"], created["audio_channels"], created["io_channels"], created["hop_size"]) == (
|
|
||||||
48000,
|
|
||||||
2,
|
|
||||||
64,
|
|
||||||
1920,
|
|
||||||
)
|
|
||||||
assert (created["normalize_latents"], created["normalization_type"], created["tanh_input_scale"]) == (
|
|
||||||
True,
|
|
||||||
"tanh",
|
|
||||||
2.0,
|
|
||||||
)
|
|
||||||
assert (tokenizer.sample_rate, tokenizer.latent_ch, tokenizer.hop_size, tokenizer.latent_fps) == (
|
|
||||||
48000,
|
|
||||||
64,
|
|
||||||
1920,
|
|
||||||
25.0,
|
|
||||||
)
|
|
||||||
|
|
||||||
with pytest.raises(ValueError, match=r"sample_rate.*48000.*32000"):
|
|
||||||
sound_tokenizer.Cosmos3SoundTokenizer.from_config(
|
|
||||||
SimpleNamespace(
|
|
||||||
model=str(model_dir),
|
|
||||||
custom_pipeline_args={"sound_sample_rate": 32000},
|
|
||||||
dtype=torch.float32,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_avae_uses_diffusers_decoder_state_dict_layout(tmp_path) -> None:
|
|
||||||
from safetensors.torch import save_file
|
|
||||||
|
|
||||||
from vllm_omni.diffusion.models.cosmos3.audio_tokenizer import avae
|
|
||||||
|
|
||||||
config = {
|
|
||||||
"sampling_rate": 8000,
|
|
||||||
"hop_size": 2,
|
|
||||||
"dec_dim": 4,
|
|
||||||
"dec_c_mults": [1],
|
|
||||||
"dec_strides": [2],
|
|
||||||
"dec_out_channels": 1,
|
|
||||||
"vocoder_input_dim": 2,
|
|
||||||
"normalization_type": "none",
|
|
||||||
}
|
|
||||||
checkpoint_path = tmp_path / DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME
|
|
||||||
config_path = tmp_path / "config.json"
|
|
||||||
config_path.write_text(json.dumps(config), encoding="utf-8")
|
|
||||||
|
|
||||||
decoder = avae.OobleckDecoder(4, 2, 1, [2], [1])
|
|
||||||
save_file({f"decoder.{key}": value for key, value in decoder.state_dict().items()}, str(checkpoint_path))
|
|
||||||
|
|
||||||
tokenizer = avae.Cosmos3AVAEAudioTokenizer(
|
|
||||||
checkpoint_path=checkpoint_path,
|
|
||||||
config_path=config_path,
|
|
||||||
dtype=torch.float32,
|
|
||||||
device="cpu",
|
|
||||||
)
|
|
||||||
|
|
||||||
keys = set(tokenizer.state_dict())
|
|
||||||
assert {"decoder.conv1.weight_g", "decoder.block.0.conv_t1.weight_g", "decoder.conv2.weight_g"} <= keys
|
|
||||||
assert not any(key.startswith(("decoder.layers.", "model.decoder.")) for key in keys)
|
|
||||||
assert tokenizer.decode(torch.zeros(1, 2, 3)).shape == (1, 1, 6)
|
|
||||||
with pytest.raises(NotImplementedError, match="decoder-only"):
|
|
||||||
tokenizer.encode(torch.zeros(1, 1, 6))
|
|
||||||
@@ -4,7 +4,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
@@ -30,10 +29,8 @@ def _tiny_cosmos3_config(**overrides):
|
|||||||
return config
|
return config
|
||||||
|
|
||||||
|
|
||||||
def test_mrope_position_ids_cover_text_video_sound_and_action() -> None:
|
def test_mrope_position_ids_cover_text_and_video() -> None:
|
||||||
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import (
|
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import (
|
||||||
compute_mrope_position_ids_action,
|
|
||||||
compute_mrope_position_ids_sound,
|
|
||||||
compute_mrope_position_ids_text,
|
compute_mrope_position_ids_text,
|
||||||
compute_mrope_position_ids_vision,
|
compute_mrope_position_ids_vision,
|
||||||
)
|
)
|
||||||
@@ -59,14 +56,6 @@ def test_mrope_position_ids_cover_text_video_sound_and_action() -> None:
|
|||||||
torch.testing.assert_close(modulated_ids[0], torch.tensor([10.0, 12.0]))
|
torch.testing.assert_close(modulated_ids[0], torch.tensor([10.0, 12.0]))
|
||||||
assert modulated_offset == 13
|
assert modulated_offset == 13
|
||||||
|
|
||||||
sound_ids, sound_offset = compute_mrope_position_ids_sound(3, temporal_offset=10, sound_latent_fps=25.0)
|
|
||||||
torch.testing.assert_close(sound_ids[0], torch.tensor([10.0, 10.96, 11.92]))
|
|
||||||
assert sound_offset == 12
|
|
||||||
|
|
||||||
action_ids, action_offset = compute_mrope_position_ids_action(3, temporal_offset=10, action_fps=None)
|
|
||||||
assert action_ids.tolist() == [[11, 12, 13], [0, 0, 0], [0, 0, 0]]
|
|
||||||
assert action_offset == 14
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("key", "value"),
|
("key", "value"),
|
||||||
@@ -131,142 +120,7 @@ def test_forward_returns_video_prediction(monkeypatch: pytest.MonkeyPatch) -> No
|
|||||||
assert tuple(output.shape) == (1, 2, 1, 2, 2)
|
assert tuple(output.shape) == (1, 2, 1, 2, 2)
|
||||||
|
|
||||||
|
|
||||||
def test_sound_and_action_modules_follow_config() -> None:
|
def test_compute_rope_freqs_places_text_and_video_positions() -> None:
|
||||||
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3VFMTransformer
|
|
||||||
|
|
||||||
tiny = _tiny_cosmos3_config()
|
|
||||||
no_modal = Cosmos3VFMTransformer(SimpleNamespace(tf_model_config=tiny, dtype=torch.float32))
|
|
||||||
with_sound = Cosmos3VFMTransformer(
|
|
||||||
SimpleNamespace(tf_model_config=tiny, dtype=torch.float32),
|
|
||||||
sound_gen=True,
|
|
||||||
sound_dim=5,
|
|
||||||
sound_latent_fps=40.0,
|
|
||||||
)
|
|
||||||
with_action = Cosmos3VFMTransformer(
|
|
||||||
SimpleNamespace(
|
|
||||||
tf_model_config={**tiny, "action_gen": True, "max_action_dim": 6, "num_embodiment_domains": 9},
|
|
||||||
dtype=torch.float32,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
assert no_modal.sound_gen is False
|
|
||||||
assert no_modal.action_gen is False
|
|
||||||
assert not hasattr(no_modal, "audio_proj_in")
|
|
||||||
assert not hasattr(no_modal, "action_proj_in")
|
|
||||||
assert with_sound.sound_dim == 5
|
|
||||||
assert with_sound.sound_latent_fps == 40.0
|
|
||||||
assert with_sound.audio_proj_in.in_features == 5
|
|
||||||
assert with_action.action_dim == 6
|
|
||||||
assert with_action.action_proj_in.num_domains == 9
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"kwargs",
|
|
||||||
[
|
|
||||||
{"sound_gen": True},
|
|
||||||
{"sound_gen": True, "sound_dim": 5},
|
|
||||||
{"sound_gen": True, "sound_latent_fps": 40.0},
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_transformer_requires_sound_dim_and_fps_when_sound_gen_true(kwargs: dict[str, Any]) -> None:
|
|
||||||
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3VFMTransformer
|
|
||||||
|
|
||||||
with pytest.raises(ValueError, match=r"requires an explicit sound_dim and sound_latent_fps"):
|
|
||||||
Cosmos3VFMTransformer(
|
|
||||||
SimpleNamespace(tf_model_config=_tiny_cosmos3_config(), dtype=torch.float32),
|
|
||||||
**kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_sound_and_action_pack_unpack_validate_shapes() -> None:
|
|
||||||
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3VFMTransformer
|
|
||||||
|
|
||||||
model = object.__new__(Cosmos3VFMTransformer)
|
|
||||||
nn.Module.__init__(model)
|
|
||||||
model.sound_dim = 3
|
|
||||||
model.action_dim = 3
|
|
||||||
|
|
||||||
sound = torch.arange(2 * 3 * 4, dtype=torch.float32).reshape(2, 3, 4)
|
|
||||||
action = torch.arange(2 * 4 * 3, dtype=torch.float32).reshape(2, 4, 3)
|
|
||||||
torch.testing.assert_close(model.unpack_sound(model.pack_sound(sound)), sound)
|
|
||||||
torch.testing.assert_close(model.unpack_action(model.pack_action(action)), action)
|
|
||||||
|
|
||||||
with pytest.raises(ValueError, match="channel mismatch"):
|
|
||||||
model.pack_sound(torch.zeros(1, 4, 2))
|
|
||||||
with pytest.raises(ValueError, match="dimension mismatch"):
|
|
||||||
model.pack_action(torch.zeros(1, 2, 4))
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
("config", "transformer_kwargs", "extra_kwargs", "expected_shapes"),
|
|
||||||
[
|
|
||||||
(
|
|
||||||
_tiny_cosmos3_config(),
|
|
||||||
{"sound_gen": True, "sound_dim": 3, "sound_latent_fps": 24.0},
|
|
||||||
{"sound_latents": torch.zeros(1, 3, 4)},
|
|
||||||
[(1, 2, 1, 2, 2), (1, 3, 4)],
|
|
||||||
),
|
|
||||||
(
|
|
||||||
_tiny_cosmos3_config(action_gen=True, max_action_dim=3, num_embodiment_domains=4),
|
|
||||||
{},
|
|
||||||
{"action_latents": torch.zeros(1, 5, 3), "action_domain_ids": torch.tensor([2])},
|
|
||||||
[(1, 2, 1, 2, 2), (1, 5, 3)],
|
|
||||||
),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_forward_returns_video_plus_optional_modality_predictions(
|
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
|
||||||
config,
|
|
||||||
transformer_kwargs,
|
|
||||||
extra_kwargs,
|
|
||||||
expected_shapes,
|
|
||||||
) -> None:
|
|
||||||
from vllm_omni.diffusion.models.cosmos3 import transformer_cosmos3
|
|
||||||
|
|
||||||
monkeypatch.setattr(transformer_cosmos3, "_get_ulysses_state", lambda: (1, 0, None))
|
|
||||||
|
|
||||||
output = transformer_cosmos3.Cosmos3VFMTransformer(
|
|
||||||
SimpleNamespace(tf_model_config=config, dtype=torch.float32),
|
|
||||||
**transformer_kwargs,
|
|
||||||
)(
|
|
||||||
hidden_states=torch.zeros(1, 2, 1, 2, 2),
|
|
||||||
timestep=torch.tensor([1.0]),
|
|
||||||
text_ids=torch.tensor([[1, 2]], dtype=torch.long),
|
|
||||||
text_mask=torch.ones(1, 2, dtype=torch.long),
|
|
||||||
video_shape=(1, 2, 2),
|
|
||||||
fps=24.0,
|
|
||||||
action_noisy_mask=torch.ones(1, 5, 1),
|
|
||||||
**extra_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert isinstance(output, tuple)
|
|
||||||
assert [tuple(tensor.shape) for tensor in output] == expected_shapes
|
|
||||||
|
|
||||||
|
|
||||||
def test_forward_with_sound_ulysses_error_mentions_combined_sequence(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
||||||
import vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 as cosmos3_module
|
|
||||||
|
|
||||||
model = cosmos3_module.Cosmos3VFMTransformer(
|
|
||||||
SimpleNamespace(tf_model_config=_tiny_cosmos3_config(), dtype=torch.float32),
|
|
||||||
sound_gen=True,
|
|
||||||
sound_dim=3,
|
|
||||||
sound_latent_fps=40.0,
|
|
||||||
)
|
|
||||||
monkeypatch.setattr(cosmos3_module, "_get_ulysses_state", lambda: (2, 0, None))
|
|
||||||
|
|
||||||
with pytest.raises(ValueError, match=r"GEN sequence length \(3 = video tokens 2 \+ sound tokens 1\)"):
|
|
||||||
model(
|
|
||||||
hidden_states=torch.zeros(1, 2, 1, 1, 2),
|
|
||||||
timestep=torch.tensor([1.0]),
|
|
||||||
text_ids=torch.tensor([[1, 2]], dtype=torch.long),
|
|
||||||
text_mask=torch.ones(1, 2, dtype=torch.long),
|
|
||||||
video_shape=(1, 1, 2),
|
|
||||||
fps=24.0,
|
|
||||||
sound_latents=torch.zeros(1, 3, 1),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_compute_rope_freqs_places_text_video_action_and_sound_positions() -> None:
|
|
||||||
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3VFMTransformer
|
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3VFMTransformer
|
||||||
|
|
||||||
class FakeRotary:
|
class FakeRotary:
|
||||||
@@ -286,8 +140,6 @@ def test_compute_rope_freqs_places_text_video_action_and_sound_positions() -> No
|
|||||||
model.temporal_modality_margin = 100
|
model.temporal_modality_margin = 100
|
||||||
model.base_fps = 24.0
|
model.base_fps = 24.0
|
||||||
model.temporal_compression_factor = 4
|
model.temporal_compression_factor = 4
|
||||||
model.temporal_compression_factor_sound = 1
|
|
||||||
model.sound_latent_fps = 25.0
|
|
||||||
model.enable_fps_modulation = False
|
model.enable_fps_modulation = False
|
||||||
|
|
||||||
freqs_und, freqs_gen = model._compute_rope_freqs(
|
freqs_und, freqs_gen = model._compute_rope_freqs(
|
||||||
@@ -304,21 +156,3 @@ def test_compute_rope_freqs_places_text_video_action_and_sound_positions() -> No
|
|||||||
assert vision_pos[0, 0].tolist() == [102, 103]
|
assert vision_pos[0, 0].tolist() == [102, 103]
|
||||||
assert freqs_und[0].shape == (2, 3, 1, 4)
|
assert freqs_und[0].shape == (2, 3, 1, 4)
|
||||||
assert freqs_gen[0].shape == (2, 2, 1, 4)
|
assert freqs_gen[0].shape == (2, 2, 1, 4)
|
||||||
|
|
||||||
rotary.position_ids.clear()
|
|
||||||
model._compute_rope_freqs(
|
|
||||||
text_mask=torch.tensor([[1, 1]], dtype=torch.long),
|
|
||||||
t=2,
|
|
||||||
hp=1,
|
|
||||||
wp=1,
|
|
||||||
fps=24.0,
|
|
||||||
device=torch.device("cpu"),
|
|
||||||
dtype=torch.float32,
|
|
||||||
t_action=2,
|
|
||||||
action_start_frame_offset=1,
|
|
||||||
t_sound=1,
|
|
||||||
)
|
|
||||||
|
|
||||||
_, gen_pos = rotary.position_ids
|
|
||||||
assert gen_pos.shape == (3, 1, 5)
|
|
||||||
assert gen_pos[0, 0].tolist() == [102, 103, 103, 104, 102]
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
@@ -1,160 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
"""Unit tests for HunyuanVideo-1.5 quant_config propagation through transformer creation.
|
|
||||||
|
|
||||||
Tests cover:
|
|
||||||
- HunyuanVideo15Pipeline passes quant_config to HunyuanVideo15Transformer3DModel
|
|
||||||
- HunyuanVideo15I2VPipeline passes quant_config to HunyuanVideo15Transformer3DModel
|
|
||||||
"""
|
|
||||||
|
|
||||||
from types import SimpleNamespace
|
|
||||||
from unittest.mock import MagicMock, patch
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
import torch
|
|
||||||
|
|
||||||
import vllm_omni.diffusion.models.hunyuan_video.pipeline_hunyuan_video_1_5 as t2v_module
|
|
||||||
import vllm_omni.diffusion.models.hunyuan_video.pipeline_hunyuan_video_1_5_i2v as i2v_module
|
|
||||||
|
|
||||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
|
||||||
|
|
||||||
|
|
||||||
class TestHunyuanVideoQuantConfigPropagation:
|
|
||||||
"""Verify quant_config is propagated to the transformer model in HunyuanVideo-1.5 pipelines."""
|
|
||||||
|
|
||||||
@patch(
|
|
||||||
"vllm_omni.diffusion.models.hunyuan_video.pipeline_hunyuan_video_1_5.get_local_device",
|
|
||||||
return_value="cpu",
|
|
||||||
)
|
|
||||||
@patch("vllm_omni.diffusion.models.hunyuan_video.pipeline_hunyuan_video_1_5.prefetch_subfolders")
|
|
||||||
@patch(
|
|
||||||
"vllm_omni.diffusion.models.hunyuan_video.pipeline_hunyuan_video_1_5.from_pretrained_with_prefetch",
|
|
||||||
return_value=MagicMock(),
|
|
||||||
)
|
|
||||||
@patch(
|
|
||||||
"vllm_omni.diffusion.models.hunyuan_video.pipeline_hunyuan_video_1_5.Qwen2Tokenizer.from_pretrained",
|
|
||||||
return_value=MagicMock(),
|
|
||||||
)
|
|
||||||
@patch(
|
|
||||||
"vllm_omni.diffusion.models.hunyuan_video.pipeline_hunyuan_video_1_5.ByT5Tokenizer.from_pretrained",
|
|
||||||
return_value=MagicMock(),
|
|
||||||
)
|
|
||||||
@patch(
|
|
||||||
"vllm_omni.diffusion.models.hunyuan_video.pipeline_hunyuan_video_1_5.AutoConfig.from_pretrained",
|
|
||||||
return_value=MagicMock(),
|
|
||||||
)
|
|
||||||
@patch(
|
|
||||||
"vllm_omni.diffusion.models.hunyuan_video.pipeline_hunyuan_video_1_5.T5EncoderModel",
|
|
||||||
return_value=MagicMock(),
|
|
||||||
)
|
|
||||||
@patch(
|
|
||||||
"vllm_omni.diffusion.models.hunyuan_video.pipeline_hunyuan_video_1_5.FlowMatchEulerDiscreteScheduler.from_pretrained",
|
|
||||||
return_value=MagicMock(),
|
|
||||||
)
|
|
||||||
@patch(
|
|
||||||
"vllm_omni.diffusion.models.hunyuan_video.pipeline_hunyuan_video_1_5.get_transformer_config_kwargs",
|
|
||||||
return_value={},
|
|
||||||
)
|
|
||||||
def test_t2v_quant_config_passed_through(
|
|
||||||
self,
|
|
||||||
mock_get_kwargs,
|
|
||||||
mock_scheduler,
|
|
||||||
mock_t5_encoder,
|
|
||||||
mock_auto_config,
|
|
||||||
mock_byt5_tok,
|
|
||||||
mock_qwen2_tok,
|
|
||||||
mock_from_pretrained,
|
|
||||||
mock_prefetch,
|
|
||||||
mock_get_device,
|
|
||||||
):
|
|
||||||
captured_kwargs = {}
|
|
||||||
|
|
||||||
class FakeTransformer:
|
|
||||||
def __init__(self, **kwargs):
|
|
||||||
captured_kwargs.update(kwargs)
|
|
||||||
|
|
||||||
with patch.object(t2v_module, "HunyuanVideo15Transformer3DModel", FakeTransformer):
|
|
||||||
fake_qc = MagicMock()
|
|
||||||
od_config = SimpleNamespace(
|
|
||||||
model="fake-t2v-model",
|
|
||||||
dtype=torch.bfloat16,
|
|
||||||
flow_shift=None,
|
|
||||||
tf_model_config=SimpleNamespace(),
|
|
||||||
quantization_config=fake_qc,
|
|
||||||
enable_diffusion_pipeline_profiler=False,
|
|
||||||
)
|
|
||||||
|
|
||||||
t2v_module.HunyuanVideo15Pipeline(od_config=od_config)
|
|
||||||
|
|
||||||
assert captured_kwargs.get("quant_config") is fake_qc
|
|
||||||
|
|
||||||
@patch(
|
|
||||||
"vllm_omni.diffusion.models.hunyuan_video.pipeline_hunyuan_video_1_5_i2v.get_local_device",
|
|
||||||
return_value="cpu",
|
|
||||||
)
|
|
||||||
@patch("vllm_omni.diffusion.models.hunyuan_video.pipeline_hunyuan_video_1_5_i2v.prefetch_subfolders")
|
|
||||||
@patch(
|
|
||||||
"vllm_omni.diffusion.models.hunyuan_video.pipeline_hunyuan_video_1_5_i2v.from_pretrained_with_prefetch",
|
|
||||||
return_value=MagicMock(),
|
|
||||||
)
|
|
||||||
@patch(
|
|
||||||
"vllm_omni.diffusion.models.hunyuan_video.pipeline_hunyuan_video_1_5_i2v.Qwen2Tokenizer.from_pretrained",
|
|
||||||
return_value=MagicMock(),
|
|
||||||
)
|
|
||||||
@patch(
|
|
||||||
"vllm_omni.diffusion.models.hunyuan_video.pipeline_hunyuan_video_1_5_i2v.ByT5Tokenizer.from_pretrained",
|
|
||||||
return_value=MagicMock(),
|
|
||||||
)
|
|
||||||
@patch(
|
|
||||||
"vllm_omni.diffusion.models.hunyuan_video.pipeline_hunyuan_video_1_5_i2v.AutoConfig.from_pretrained",
|
|
||||||
return_value=MagicMock(),
|
|
||||||
)
|
|
||||||
@patch(
|
|
||||||
"vllm_omni.diffusion.models.hunyuan_video.pipeline_hunyuan_video_1_5_i2v.T5EncoderModel",
|
|
||||||
return_value=MagicMock(),
|
|
||||||
)
|
|
||||||
@patch(
|
|
||||||
"vllm_omni.diffusion.models.hunyuan_video.pipeline_hunyuan_video_1_5_i2v.SiglipImageProcessor.from_pretrained",
|
|
||||||
return_value=MagicMock(),
|
|
||||||
)
|
|
||||||
@patch(
|
|
||||||
"vllm_omni.diffusion.models.hunyuan_video.pipeline_hunyuan_video_1_5_i2v.FlowMatchEulerDiscreteScheduler.from_pretrained",
|
|
||||||
return_value=MagicMock(),
|
|
||||||
)
|
|
||||||
@patch(
|
|
||||||
"vllm_omni.diffusion.models.hunyuan_video.pipeline_hunyuan_video_1_5_i2v.get_transformer_config_kwargs",
|
|
||||||
return_value={},
|
|
||||||
)
|
|
||||||
def test_i2v_quant_config_passed_through(
|
|
||||||
self,
|
|
||||||
mock_get_kwargs,
|
|
||||||
mock_scheduler,
|
|
||||||
mock_siglip_processor,
|
|
||||||
mock_t5_encoder,
|
|
||||||
mock_auto_config,
|
|
||||||
mock_byt5_tok,
|
|
||||||
mock_qwen2_tok,
|
|
||||||
mock_from_pretrained,
|
|
||||||
mock_prefetch,
|
|
||||||
mock_get_device,
|
|
||||||
):
|
|
||||||
captured_kwargs = {}
|
|
||||||
|
|
||||||
class FakeTransformer:
|
|
||||||
def __init__(self, **kwargs):
|
|
||||||
captured_kwargs.update(kwargs)
|
|
||||||
|
|
||||||
with patch.object(i2v_module, "HunyuanVideo15Transformer3DModel", FakeTransformer):
|
|
||||||
fake_qc = MagicMock()
|
|
||||||
od_config = SimpleNamespace(
|
|
||||||
model="fake-i2v-model",
|
|
||||||
dtype=torch.bfloat16,
|
|
||||||
flow_shift=None,
|
|
||||||
tf_model_config=SimpleNamespace(),
|
|
||||||
quantization_config=fake_qc,
|
|
||||||
enable_diffusion_pipeline_profiler=False,
|
|
||||||
)
|
|
||||||
|
|
||||||
i2v_module.HunyuanVideo15I2VPipeline(od_config=od_config)
|
|
||||||
|
|
||||||
assert captured_kwargs.get("quant_config") is fake_qc
|
|
||||||
@@ -72,31 +72,6 @@ class TestPipelineIndependence:
|
|||||||
|
|
||||||
assert issubclass(LTX23Pipeline, CFGParallelMixin)
|
assert issubclass(LTX23Pipeline, CFGParallelMixin)
|
||||||
|
|
||||||
def test_ltx23_pipeline_declares_offload_components(self):
|
|
||||||
"""LTX23Pipeline must expose LTX-2.3-specific modules to offload discovery."""
|
|
||||||
from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_3 import LTX23Pipeline
|
|
||||||
from vllm_omni.diffusion.offloader.module_collector import ModuleDiscovery
|
|
||||||
|
|
||||||
pipe = object.__new__(LTX23Pipeline)
|
|
||||||
torch.nn.Module.__init__(pipe)
|
|
||||||
pipe.transformer = torch.nn.Linear(1, 1)
|
|
||||||
pipe.text_encoder = torch.nn.Linear(1, 1)
|
|
||||||
pipe.connectors = torch.nn.Linear(1, 1)
|
|
||||||
pipe.vae = torch.nn.Linear(1, 1)
|
|
||||||
pipe.audio_vae = torch.nn.Linear(1, 1)
|
|
||||||
pipe.vocoder = torch.nn.Linear(1, 1)
|
|
||||||
|
|
||||||
modules = ModuleDiscovery.discover(pipe)
|
|
||||||
|
|
||||||
assert LTX23Pipeline._dit_modules == ["transformer"]
|
|
||||||
assert LTX23Pipeline._encoder_modules == ["text_encoder", "connectors"]
|
|
||||||
assert LTX23Pipeline._vae_modules == ["vae", "audio_vae"]
|
|
||||||
assert LTX23Pipeline._resident_modules == ["vocoder"]
|
|
||||||
assert modules.dit_names == ["transformer"]
|
|
||||||
assert modules.encoder_names == ["text_encoder", "connectors"]
|
|
||||||
assert modules.resident_names == ["vocoder"]
|
|
||||||
assert len(modules.vaes) == 2
|
|
||||||
|
|
||||||
|
|
||||||
class TestCFGParallelHelpers:
|
class TestCFGParallelHelpers:
|
||||||
"""Test LTX-2.3 CFG helper math without loading model weights."""
|
"""Test LTX-2.3 CFG helper math without loading model weights."""
|
||||||
|
|||||||
@@ -1,22 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
from torch import nn
|
|
||||||
|
|
||||||
from vllm_omni.diffusion.models.ltx2.ltx2_transformer import _make_rms_norm
|
|
||||||
|
|
||||||
|
|
||||||
def test_ltx_rms_norm_no_affine_identity_weight_is_non_persistent_buffer():
|
|
||||||
norm = _make_rms_norm(8, eps=1e-6, elementwise_affine=False)
|
|
||||||
|
|
||||||
assert "weight" not in dict(norm.named_parameters())
|
|
||||||
assert "weight" in dict(norm.named_buffers())
|
|
||||||
assert "weight" not in norm.state_dict()
|
|
||||||
|
|
||||||
|
|
||||||
def test_ltx_rms_norm_affine_weight_remains_parameter():
|
|
||||||
norm = _make_rms_norm(8, eps=1e-6, elementwise_affine=True)
|
|
||||||
|
|
||||||
assert isinstance(dict(norm.named_parameters())["weight"], nn.Parameter)
|
|
||||||
assert "weight" not in dict(norm.named_buffers())
|
|
||||||
assert "weight" in norm.state_dict()
|
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import queue
|
import queue
|
||||||
import threading
|
import threading
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
@@ -446,7 +447,11 @@ class TestDiffusionEngine:
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_step_raises_aborted_error(self, mocker: MockerFixture) -> None:
|
async def test_step_raises_aborted_error(self, mocker: MockerFixture) -> None:
|
||||||
engine = DiffusionEngine.__new__(DiffusionEngine)
|
engine = DiffusionEngine.__new__(DiffusionEngine)
|
||||||
engine._check_and_start_background_loop = mocker.AsyncMock()
|
engine._closed = False
|
||||||
|
engine._loop_started = True
|
||||||
|
engine._init_lock = asyncio.Lock()
|
||||||
|
engine.main_loop = asyncio.get_running_loop()
|
||||||
|
engine.stop_event = threading.Event()
|
||||||
engine.pre_process_func = None
|
engine.pre_process_func = None
|
||||||
engine.async_add_req_and_wait_for_response = mocker.AsyncMock(
|
engine.async_add_req_and_wait_for_response = mocker.AsyncMock(
|
||||||
return_value=DiffusionOutput(aborted=True, abort_message="Request req-abort aborted.")
|
return_value=DiffusionOutput(aborted=True, abort_message="Request req-abort aborted.")
|
||||||
@@ -521,52 +526,6 @@ class TestDiffusionEngine:
|
|||||||
with pytest.raises(RuntimeError, match="Dummy run failed: boom"):
|
with pytest.raises(RuntimeError, match="Dummy run failed: boom"):
|
||||||
engine._dummy_run()
|
engine._dummy_run()
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_step_multi_request_reuses_multimodal_slice_logic(self, mocker: MockerFixture) -> None:
|
|
||||||
engine = DiffusionEngine.__new__(DiffusionEngine)
|
|
||||||
engine.od_config = SimpleNamespace(
|
|
||||||
model_class_name="mock_model",
|
|
||||||
enable_cpu_offload=False,
|
|
||||||
)
|
|
||||||
engine.pre_process_func = None
|
|
||||||
engine.post_process_func = None
|
|
||||||
engine._check_and_start_background_loop = mocker.AsyncMock()
|
|
||||||
engine.async_add_req_and_wait_for_response = mocker.AsyncMock(
|
|
||||||
return_value=DiffusionOutput(
|
|
||||||
output={
|
|
||||||
"video": ["frame-0", "frame-1"],
|
|
||||||
"audio": ["audio-0", "audio-1"],
|
|
||||||
"actions": torch.tensor([[1.0, 2.0], [3.0, 4.0]]),
|
|
||||||
}
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
request = OmniDiffusionRequest(
|
|
||||||
prompts=["prompt-0", "prompt-1"],
|
|
||||||
sampling_params=OmniDiffusionSamplingParams(
|
|
||||||
num_inference_steps=1,
|
|
||||||
num_outputs_per_prompt=1,
|
|
||||||
),
|
|
||||||
request_id="req-batch",
|
|
||||||
)
|
|
||||||
|
|
||||||
mocker.patch("vllm_omni.diffusion.diffusion_engine.supports_audio_output", return_value=False)
|
|
||||||
outputs = await engine.step(request)
|
|
||||||
|
|
||||||
assert len(outputs) == 2
|
|
||||||
assert outputs[0].images == ["frame-0"]
|
|
||||||
assert outputs[1].images == ["frame-1"]
|
|
||||||
assert outputs[0].multimodal_output["audio"] == "audio-0"
|
|
||||||
assert outputs[1].multimodal_output["audio"] == "audio-1"
|
|
||||||
torch.testing.assert_close(
|
|
||||||
outputs[0].multimodal_output["actions"],
|
|
||||||
torch.tensor([1.0, 2.0]),
|
|
||||||
)
|
|
||||||
torch.testing.assert_close(
|
|
||||||
outputs[1].multimodal_output["actions"],
|
|
||||||
torch.tensor([3.0, 4.0]),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class TestStepScheduler:
|
class TestStepScheduler:
|
||||||
def setup_method(self) -> None:
|
def setup_method(self) -> None:
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ def mock_od_config(mocker: MockerFixture):
|
|||||||
config.cache_backend = None
|
config.cache_backend = None
|
||||||
config.cache_config = None
|
config.cache_config = None
|
||||||
config.model = "test-model"
|
config.model = "test-model"
|
||||||
config.diffusion_load_format = "default"
|
config.diffusion_load_format = None
|
||||||
config.dtype = "float32"
|
config.dtype = "float32"
|
||||||
config.max_cpu_loras = 0
|
config.max_cpu_loras = 0
|
||||||
config.lora_path = None
|
config.lora_path = None
|
||||||
|
|||||||
@@ -32,8 +32,8 @@ BAGEL_MOONCAKE_CI_DEPLOY = get_deploy_config_path("ci/bagel_mooncake.yaml")
|
|||||||
|
|
||||||
# Reference pixel data extracted from the known-good output image
|
# Reference pixel data extracted from the known-good output image
|
||||||
# Each entry contains (x, y) position and expected (R, G, B) values
|
# Each entry contains (x, y) position and expected (R, G, B) values
|
||||||
# "Generated with seed=52, num_inference_steps=14,
|
# "Generated with seed=52, num_inference_steps=15,
|
||||||
# prompt='A cute cat'"
|
# prompt='A futuristic city skyline at twilight, cyberpunk style'"
|
||||||
REFERENCE_PIXELS = [
|
REFERENCE_PIXELS = [
|
||||||
{"position": (100, 100), "rgb": (115, 113, 94)},
|
{"position": (100, 100), "rgb": (115, 113, 94)},
|
||||||
{"position": (400, 50), "rgb": (159, 160, 144)},
|
{"position": (400, 50), "rgb": (159, 160, 144)},
|
||||||
@@ -63,7 +63,7 @@ def _find_free_port() -> int:
|
|||||||
return port
|
return port
|
||||||
|
|
||||||
|
|
||||||
def _configure_sampling_params(omni: Omni, num_inference_steps: int = 14) -> list:
|
def _configure_sampling_params(omni: Omni, num_inference_steps: int = 15) -> list:
|
||||||
"""Configure sampling parameters for Bagel text2img generation.
|
"""Configure sampling parameters for Bagel text2img generation.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -269,6 +269,7 @@ def _load_mooncake_config(host: str, rpc_port: int, http_port: int) -> str:
|
|||||||
@pytest.mark.core_model
|
@pytest.mark.core_model
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
@pytest.mark.diffusion
|
@pytest.mark.diffusion
|
||||||
|
@pytest.mark.skip(reason="Skip failed CI issue 3977: https://github.com/vllm-project/vllm-omni/issues/3977")
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||||
def test_bagel_text2img_mooncake_connector(run_level):
|
def test_bagel_text2img_mooncake_connector(run_level):
|
||||||
"""Test Bagel text2img with Mooncake connector for inter-stage communication."""
|
"""Test Bagel text2img with Mooncake connector for inter-stage communication."""
|
||||||
|
|||||||
@@ -24,13 +24,14 @@ from tests.helpers.mark import hardware_test
|
|||||||
from tests.helpers.runtime import OmniRunner
|
from tests.helpers.runtime import OmniRunner
|
||||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||||
from vllm_omni.entrypoints.omni import Omni
|
from vllm_omni.entrypoints.omni import Omni
|
||||||
|
from vllm_omni.platforms import current_omni_platform
|
||||||
|
|
||||||
pytestmark = [pytest.mark.usefixtures("clean_gpu_memory_between_tests")]
|
pytestmark = [pytest.mark.usefixtures("clean_gpu_memory_between_tests")]
|
||||||
|
|
||||||
BAGEL_CI_DEPLOY = get_deploy_config_path("ci/bagel.yaml")
|
BAGEL_CI_DEPLOY = get_deploy_config_path("ci/bagel.yaml")
|
||||||
|
|
||||||
# Reference pixel data extracted from the known-good output image
|
# Reference pixel data extracted from the known-good output image
|
||||||
# Generated with seed=52, num_inference_steps=14,
|
# Generated with seed=52, num_inference_steps=15,
|
||||||
# prompt='Change the grass color to red',
|
# prompt='Change the grass color to red',
|
||||||
# input image: 2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg
|
# input image: 2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg
|
||||||
REFERENCE_PIXELS = [
|
REFERENCE_PIXELS = [
|
||||||
@@ -46,10 +47,23 @@ REFERENCE_PIXELS = [
|
|||||||
{"position": (256, 256), "rgb": (181, 201, 221)},
|
{"position": (256, 256), "rgb": (181, 201, 221)},
|
||||||
]
|
]
|
||||||
|
|
||||||
|
if current_omni_platform.is_rocm():
|
||||||
|
REFERENCE_PIXELS = [
|
||||||
|
{"position": (100, 100), "rgb": (156, 172, 217)},
|
||||||
|
{"position": (400, 50), "rgb": (105, 144, 217)},
|
||||||
|
{"position": (700, 100), "rgb": (118, 159, 232)},
|
||||||
|
{"position": (150, 400), "rgb": (180, 22, 52)},
|
||||||
|
{"position": (512, 336), "rgb": (221, 211, 194)},
|
||||||
|
{"position": (700, 400), "rgb": (192, 10, 46)},
|
||||||
|
{"position": (100, 600), "rgb": (102, 12, 22)},
|
||||||
|
{"position": (400, 600), "rgb": (161, 28, 47)},
|
||||||
|
{"position": (700, 600), "rgb": (100, 87, 94)},
|
||||||
|
{"position": (256, 256), "rgb": (181, 201, 221)},
|
||||||
|
]
|
||||||
|
|
||||||
# text2img reference pixels (aligned with offline `bagel/end2end.py` text2img, 15 steps)
|
# text2img reference pixels (aligned with offline `bagel/end2end.py` text2img, 15 steps)
|
||||||
# "Generated with seed=52, num_inference_steps=14,
|
# "Generated with seed=52, num_inference_steps=15,
|
||||||
# prompt='A cute cat'"
|
# prompt='A futuristic city skyline at twilight, cyberpunk style'"
|
||||||
TEXT2IMG_REFERENCE_PIXELS = [
|
TEXT2IMG_REFERENCE_PIXELS = [
|
||||||
{"position": (100, 100), "rgb": (115, 113, 94)},
|
{"position": (100, 100), "rgb": (115, 113, 94)},
|
||||||
{"position": (400, 50), "rgb": (159, 160, 144)},
|
{"position": (400, 50), "rgb": (159, 160, 144)},
|
||||||
@@ -63,6 +77,19 @@ TEXT2IMG_REFERENCE_PIXELS = [
|
|||||||
{"position": (256, 256), "rgb": (167, 156, 150)},
|
{"position": (256, 256), "rgb": (167, 156, 150)},
|
||||||
]
|
]
|
||||||
|
|
||||||
|
if current_omni_platform.is_rocm():
|
||||||
|
TEXT2IMG_REFERENCE_PIXELS = [
|
||||||
|
{"position": (100, 100), "rgb": (115, 113, 94)},
|
||||||
|
{"position": (400, 50), "rgb": (159, 160, 144)},
|
||||||
|
{"position": (700, 100), "rgb": (164, 151, 123)},
|
||||||
|
{"position": (150, 400), "rgb": (120, 121, 107)},
|
||||||
|
{"position": (512, 512), "rgb": (165, 133, 127)},
|
||||||
|
{"position": (700, 400), "rgb": (217, 130, 66)},
|
||||||
|
{"position": (100, 700), "rgb": (191, 168, 152)},
|
||||||
|
{"position": (400, 700), "rgb": (130, 96, 77)},
|
||||||
|
{"position": (700, 700), "rgb": (247, 203, 140)},
|
||||||
|
{"position": (256, 256), "rgb": (167, 156, 150)},
|
||||||
|
]
|
||||||
|
|
||||||
PIXEL_TOLERANCE = 10
|
PIXEL_TOLERANCE = 10
|
||||||
TEXT2IMG_PIXEL_TOLERANCE = 5
|
TEXT2IMG_PIXEL_TOLERANCE = 5
|
||||||
@@ -88,7 +115,7 @@ def _find_free_port() -> int:
|
|||||||
return port
|
return port
|
||||||
|
|
||||||
|
|
||||||
def _configure_sampling_params(omni: Omni, num_inference_steps: int = 14) -> list:
|
def _configure_sampling_params(omni: Omni, num_inference_steps: int = 15) -> list:
|
||||||
"""Configure sampling parameters for Bagel img2img generation.
|
"""Configure sampling parameters for Bagel img2img generation.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -231,6 +258,7 @@ def _resolve_deploy_config(config_path: str, run_level: str) -> str:
|
|||||||
@pytest.mark.core_model
|
@pytest.mark.core_model
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
@pytest.mark.diffusion
|
@pytest.mark.diffusion
|
||||||
|
@pytest.mark.skip(reason="Skip failed CI issue 3977: https://github.com/vllm-project/vllm-omni/issues/3977")
|
||||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
|
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
|
||||||
def test_bagel_img2img_shared_memory_connector(run_level):
|
def test_bagel_img2img_shared_memory_connector(run_level):
|
||||||
"""Test Bagel img2img with shared memory connector."""
|
"""Test Bagel img2img with shared memory connector."""
|
||||||
@@ -248,6 +276,7 @@ def test_bagel_img2img_shared_memory_connector(run_level):
|
|||||||
@pytest.mark.core_model
|
@pytest.mark.core_model
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
@pytest.mark.diffusion
|
@pytest.mark.diffusion
|
||||||
|
@pytest.mark.skip(reason="Skip failed CI issue 3977: https://github.com/vllm-project/vllm-omni/issues/3977")
|
||||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
|
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
|
||||||
def test_bagel_text2img_shared_memory_connector(run_level):
|
def test_bagel_text2img_shared_memory_connector(run_level):
|
||||||
"""Test Bagel text2img with shared memory connector."""
|
"""Test Bagel text2img with shared memory connector."""
|
||||||
|
|||||||
@@ -1,307 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import uuid
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
try:
|
|
||||||
import cv2
|
|
||||||
except ImportError: # pragma: no cover - optional e2e dependency
|
|
||||||
cv2 = None
|
|
||||||
|
|
||||||
try:
|
|
||||||
import websockets.sync.client as websockets_client
|
|
||||||
except ImportError: # pragma: no cover - optional e2e dependency
|
|
||||||
websockets_client = None
|
|
||||||
|
|
||||||
try:
|
|
||||||
from openpi_client import msgpack_numpy
|
|
||||||
except ImportError: # pragma: no cover - optional e2e dependency
|
|
||||||
msgpack_numpy = None
|
|
||||||
|
|
||||||
PING_INTERVAL_SECS = 300
|
|
||||||
PING_TIMEOUT_SECS = 3600
|
|
||||||
DEFAULT_HOST = "127.0.0.1"
|
|
||||||
DEFAULT_PORT = 8000
|
|
||||||
DEFAULT_PATH = "/v1/realtime/robot/openpi"
|
|
||||||
DEFAULT_PROMPT = "Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan"
|
|
||||||
ACTION_HORIZON = 24
|
|
||||||
DEFAULT_ACTION_DIM = 8
|
|
||||||
RELATIVE_OFFSETS = [-23, -16, -8, 0]
|
|
||||||
CAMERA_FILES = {
|
|
||||||
"observation/exterior_image_0_left": "exterior_image_1_left.mp4",
|
|
||||||
"observation/exterior_image_1_left": "exterior_image_2_left.mp4",
|
|
||||||
"observation/wrist_image_left": "wrist_image_left.mp4",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _decode_action_response(response: bytes | str) -> np.ndarray:
|
|
||||||
if isinstance(response, str):
|
|
||||||
raise RuntimeError(f"Inference failed: {response}")
|
|
||||||
decoded = msgpack_numpy.unpackb(response)
|
|
||||||
if isinstance(decoded, dict) and decoded.get("type") == "error":
|
|
||||||
message = decoded.get("message", decoded)
|
|
||||||
raise RuntimeError(f"Inference failed: {message}")
|
|
||||||
return np.asarray(decoded, dtype=np.float32)
|
|
||||||
|
|
||||||
|
|
||||||
def require_dependencies() -> None:
|
|
||||||
missing = []
|
|
||||||
if cv2 is None:
|
|
||||||
missing.append("opencv-python")
|
|
||||||
if websockets_client is None:
|
|
||||||
missing.append("websockets")
|
|
||||||
if msgpack_numpy is None:
|
|
||||||
missing.append("openpi-client")
|
|
||||||
if missing:
|
|
||||||
raise ModuleNotFoundError(f"DreamZero OpenPI test dependencies are missing: {', '.join(missing)}")
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class DreamZeroServerMetadata:
|
|
||||||
image_resolution: tuple[int, int]
|
|
||||||
n_external_cameras: int
|
|
||||||
needs_wrist_camera: bool
|
|
||||||
needs_stereo_camera: bool
|
|
||||||
needs_session_id: bool
|
|
||||||
action_space: str
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_dict(cls, payload: dict[str, Any]) -> DreamZeroServerMetadata:
|
|
||||||
required_keys = (
|
|
||||||
"image_resolution",
|
|
||||||
"n_external_cameras",
|
|
||||||
"needs_wrist_camera",
|
|
||||||
"needs_stereo_camera",
|
|
||||||
"needs_session_id",
|
|
||||||
"action_space",
|
|
||||||
)
|
|
||||||
missing_keys = [key for key in required_keys if key not in payload]
|
|
||||||
if missing_keys:
|
|
||||||
raise ValueError(f"Missing DreamZero metadata keys: {missing_keys}")
|
|
||||||
|
|
||||||
image_resolution = payload["image_resolution"]
|
|
||||||
if not isinstance(image_resolution, (list, tuple)) or len(image_resolution) != 2:
|
|
||||||
raise ValueError(f"Invalid image_resolution: {image_resolution!r}")
|
|
||||||
|
|
||||||
return cls(
|
|
||||||
image_resolution=(int(image_resolution[0]), int(image_resolution[1])),
|
|
||||||
n_external_cameras=int(payload["n_external_cameras"]),
|
|
||||||
needs_wrist_camera=bool(payload["needs_wrist_camera"]),
|
|
||||||
needs_stereo_camera=bool(payload["needs_stereo_camera"]),
|
|
||||||
needs_session_id=bool(payload["needs_session_id"]),
|
|
||||||
action_space=str(payload["action_space"]),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class OpenPIWebsocketClient:
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
host: str = DEFAULT_HOST,
|
|
||||||
port: int = DEFAULT_PORT,
|
|
||||||
path: str = DEFAULT_PATH,
|
|
||||||
) -> None:
|
|
||||||
require_dependencies()
|
|
||||||
self._uri = f"ws://{host}:{port}{path}"
|
|
||||||
self._packer = msgpack_numpy.Packer()
|
|
||||||
self._ws, self._server_metadata = self._connect()
|
|
||||||
|
|
||||||
def _connect(self):
|
|
||||||
conn = websockets_client.connect(
|
|
||||||
self._uri,
|
|
||||||
compression=None,
|
|
||||||
max_size=None,
|
|
||||||
ping_interval=PING_INTERVAL_SECS,
|
|
||||||
ping_timeout=PING_TIMEOUT_SECS,
|
|
||||||
)
|
|
||||||
metadata = msgpack_numpy.unpackb(conn.recv())
|
|
||||||
if not isinstance(metadata, dict):
|
|
||||||
raise TypeError(f"Expected dict metadata from server, got {type(metadata)!r}")
|
|
||||||
return conn, metadata
|
|
||||||
|
|
||||||
def get_server_metadata(self) -> dict[str, Any]:
|
|
||||||
return dict(self._server_metadata)
|
|
||||||
|
|
||||||
def infer(self, obs: dict[str, Any]) -> np.ndarray:
|
|
||||||
payload = dict(obs)
|
|
||||||
payload["endpoint"] = "infer"
|
|
||||||
self._ws.send(self._packer.pack(payload))
|
|
||||||
response = self._ws.recv()
|
|
||||||
return _decode_action_response(response)
|
|
||||||
|
|
||||||
def reset(self, reset_info: dict[str, Any] | None = None) -> str:
|
|
||||||
payload = dict(reset_info or {})
|
|
||||||
payload["endpoint"] = "reset"
|
|
||||||
self._ws.send(self._packer.pack(payload))
|
|
||||||
response = self._ws.recv()
|
|
||||||
if isinstance(response, str):
|
|
||||||
return response
|
|
||||||
decoded = msgpack_numpy.unpackb(response)
|
|
||||||
if not isinstance(decoded, dict) or decoded.get("status") != "reset successful":
|
|
||||||
raise RuntimeError(f"Unexpected reset response: {decoded!r}")
|
|
||||||
return str(decoded["status"])
|
|
||||||
|
|
||||||
def close(self) -> None:
|
|
||||||
self._ws.close()
|
|
||||||
|
|
||||||
|
|
||||||
def load_all_frames(video_path: Path) -> np.ndarray:
|
|
||||||
require_dependencies()
|
|
||||||
cap = cv2.VideoCapture(str(video_path))
|
|
||||||
frames = []
|
|
||||||
while True:
|
|
||||||
ok, frame = cap.read()
|
|
||||||
if not ok:
|
|
||||||
break
|
|
||||||
frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
|
|
||||||
cap.release()
|
|
||||||
if not frames:
|
|
||||||
raise RuntimeError(f"No frames loaded from {video_path}")
|
|
||||||
return np.stack(frames, axis=0)
|
|
||||||
|
|
||||||
|
|
||||||
def load_camera_frames(video_dir: Path) -> dict[str, np.ndarray]:
|
|
||||||
camera_frames: dict[str, np.ndarray] = {}
|
|
||||||
for camera_key, file_name in CAMERA_FILES.items():
|
|
||||||
video_path = video_dir / file_name
|
|
||||||
if not video_path.exists():
|
|
||||||
raise FileNotFoundError(f"Missing DreamZero test asset: {video_path}")
|
|
||||||
camera_frames[camera_key] = load_all_frames(video_path)
|
|
||||||
return camera_frames
|
|
||||||
|
|
||||||
|
|
||||||
def build_frame_schedule(total_frames: int, num_chunks: int) -> list[list[int]]:
|
|
||||||
chunks: list[list[int]] = []
|
|
||||||
current_frame = 23
|
|
||||||
for _ in range(num_chunks):
|
|
||||||
indices = [max(current_frame + offset, 0) for offset in RELATIVE_OFFSETS]
|
|
||||||
if indices[-1] >= total_frames:
|
|
||||||
break
|
|
||||||
chunks.append(indices)
|
|
||||||
current_frame += ACTION_HORIZON
|
|
||||||
return chunks
|
|
||||||
|
|
||||||
|
|
||||||
def make_obs_from_video(
|
|
||||||
camera_frames: dict[str, np.ndarray],
|
|
||||||
frame_indices: list[int],
|
|
||||||
*,
|
|
||||||
prompt: str,
|
|
||||||
session_id: str,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
obs: dict[str, Any] = {}
|
|
||||||
for camera_key, all_frames in camera_frames.items():
|
|
||||||
selected = all_frames[frame_indices]
|
|
||||||
obs[camera_key] = selected[0] if len(frame_indices) == 1 else selected
|
|
||||||
|
|
||||||
obs["observation/joint_position"] = np.zeros(7, dtype=np.float32)
|
|
||||||
obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32)
|
|
||||||
obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32)
|
|
||||||
obs["prompt"] = prompt
|
|
||||||
obs["session_id"] = session_id
|
|
||||||
return obs
|
|
||||||
|
|
||||||
|
|
||||||
def build_demo_observations(
|
|
||||||
camera_frames: dict[str, np.ndarray],
|
|
||||||
*,
|
|
||||||
prompt: str,
|
|
||||||
session_id: str,
|
|
||||||
num_chunks: int = 2,
|
|
||||||
) -> list[dict[str, Any]]:
|
|
||||||
if num_chunks < 1:
|
|
||||||
raise ValueError("num_chunks must be at least 1")
|
|
||||||
|
|
||||||
total_frames = min(frames.shape[0] for frames in camera_frames.values())
|
|
||||||
observations = [
|
|
||||||
make_obs_from_video(
|
|
||||||
camera_frames,
|
|
||||||
[0],
|
|
||||||
prompt=prompt,
|
|
||||||
session_id=session_id,
|
|
||||||
)
|
|
||||||
]
|
|
||||||
for indices in build_frame_schedule(total_frames, num_chunks - 1):
|
|
||||||
observations.append(
|
|
||||||
make_obs_from_video(
|
|
||||||
camera_frames,
|
|
||||||
indices,
|
|
||||||
prompt=prompt,
|
|
||||||
session_id=session_id,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return observations
|
|
||||||
|
|
||||||
|
|
||||||
def validate_session_result(
|
|
||||||
result: dict[str, Any],
|
|
||||||
*,
|
|
||||||
expected_action_horizon: int = ACTION_HORIZON,
|
|
||||||
expected_action_dim: int = DEFAULT_ACTION_DIM,
|
|
||||||
) -> None:
|
|
||||||
metadata = DreamZeroServerMetadata.from_dict(result["metadata"])
|
|
||||||
if metadata.image_resolution != (180, 320):
|
|
||||||
raise AssertionError(f"Unexpected image_resolution: {metadata.image_resolution}")
|
|
||||||
if metadata.n_external_cameras != 2:
|
|
||||||
raise AssertionError(f"Unexpected n_external_cameras: {metadata.n_external_cameras}")
|
|
||||||
if not metadata.needs_wrist_camera:
|
|
||||||
raise AssertionError("DreamZero test expects wrist camera metadata")
|
|
||||||
if metadata.action_space != "joint_position":
|
|
||||||
raise AssertionError(f"Unexpected action_space: {metadata.action_space}")
|
|
||||||
|
|
||||||
actions = result["actions"]
|
|
||||||
if len(actions) != 3:
|
|
||||||
raise AssertionError(f"Expected 3 action tensors, got {len(actions)}")
|
|
||||||
for index, action in enumerate(actions):
|
|
||||||
if action.shape != (expected_action_horizon, expected_action_dim):
|
|
||||||
raise AssertionError(
|
|
||||||
f"Action {index} shape mismatch: expected "
|
|
||||||
f"{(expected_action_horizon, expected_action_dim)}, got {action.shape}"
|
|
||||||
)
|
|
||||||
if not np.isfinite(action).all():
|
|
||||||
raise AssertionError(f"Action {index} contains non-finite values")
|
|
||||||
|
|
||||||
if result["reset_status"] != "reset successful":
|
|
||||||
raise AssertionError(f"Unexpected reset status: {result['reset_status']!r}")
|
|
||||||
|
|
||||||
|
|
||||||
def run_policy_session(
|
|
||||||
*,
|
|
||||||
host: str = DEFAULT_HOST,
|
|
||||||
port: int = DEFAULT_PORT,
|
|
||||||
path: str = DEFAULT_PATH,
|
|
||||||
video_dir: Path,
|
|
||||||
prompt: str = DEFAULT_PROMPT,
|
|
||||||
session_id: str | None = None,
|
|
||||||
num_chunks: int = 2,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
session_id = session_id or str(uuid.uuid4())
|
|
||||||
camera_frames = load_camera_frames(video_dir)
|
|
||||||
observations = build_demo_observations(
|
|
||||||
camera_frames,
|
|
||||||
prompt=prompt,
|
|
||||||
session_id=session_id,
|
|
||||||
num_chunks=num_chunks,
|
|
||||||
)
|
|
||||||
|
|
||||||
client = OpenPIWebsocketClient(host=host, port=port, path=path)
|
|
||||||
try:
|
|
||||||
metadata = client.get_server_metadata()
|
|
||||||
actions = [client.infer(obs) for obs in observations]
|
|
||||||
reset_status = client.reset({})
|
|
||||||
actions.append(client.infer(observations[0]))
|
|
||||||
return {
|
|
||||||
"metadata": metadata,
|
|
||||||
"actions": actions,
|
|
||||||
"reset_status": reset_status,
|
|
||||||
"session_id": session_id,
|
|
||||||
}
|
|
||||||
finally:
|
|
||||||
client.close()
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from vllm_omni.diffusion.models.dreamzero.utils import (
|
|
||||||
DEFAULT_CFG_SCALE,
|
|
||||||
DEFAULT_EMBODIMENT_NAME_TO_ID,
|
|
||||||
DEFAULT_NEGATIVE_PROMPT,
|
|
||||||
DEFAULT_NUM_INFERENCE_STEPS,
|
|
||||||
DEFAULT_SEED,
|
|
||||||
DEFAULT_SIGMA_SHIFT,
|
|
||||||
)
|
|
||||||
|
|
||||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
|
||||||
|
|
||||||
|
|
||||||
def test_dreamzero_default_constants_match_source_baseline():
|
|
||||||
assert DEFAULT_NUM_INFERENCE_STEPS == 16
|
|
||||||
assert DEFAULT_CFG_SCALE == 5.0
|
|
||||||
assert DEFAULT_SIGMA_SHIFT == 5.0
|
|
||||||
assert DEFAULT_SEED == 1140
|
|
||||||
assert "worst quality" in DEFAULT_NEGATIVE_PROMPT
|
|
||||||
assert DEFAULT_EMBODIMENT_NAME_TO_ID["oxe_droid"] == 17
|
|
||||||
@@ -1,46 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from tests.dreamzero import openpi_client_helper
|
|
||||||
|
|
||||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
|
||||||
|
|
||||||
|
|
||||||
class FakeMsgpackNumpy:
|
|
||||||
payloads = {}
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def packb(obj):
|
|
||||||
key = str(len(FakeMsgpackNumpy.payloads)).encode()
|
|
||||||
FakeMsgpackNumpy.payloads[key] = obj
|
|
||||||
return key
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def unpackb(data):
|
|
||||||
return FakeMsgpackNumpy.payloads[data]
|
|
||||||
|
|
||||||
|
|
||||||
def test_decode_action_response_surfaces_structured_error(monkeypatch):
|
|
||||||
monkeypatch.setattr(openpi_client_helper, "msgpack_numpy", FakeMsgpackNumpy)
|
|
||||||
payload = FakeMsgpackNumpy.packb(
|
|
||||||
{
|
|
||||||
"type": "error",
|
|
||||||
"message": "Internal inference error",
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
with pytest.raises(RuntimeError, match="Internal inference error"):
|
|
||||||
openpi_client_helper._decode_action_response(payload)
|
|
||||||
|
|
||||||
|
|
||||||
def test_decode_action_response_converts_action_payload_to_float32(monkeypatch):
|
|
||||||
monkeypatch.setattr(openpi_client_helper, "msgpack_numpy", FakeMsgpackNumpy)
|
|
||||||
payload = FakeMsgpackNumpy.packb(np.asarray([[1.0, 2.0]], dtype=np.float64))
|
|
||||||
|
|
||||||
actions = openpi_client_helper._decode_action_response(payload)
|
|
||||||
|
|
||||||
assert actions.dtype == np.float32
|
|
||||||
np.testing.assert_array_equal(actions, np.asarray([[1.0, 2.0]], dtype=np.float32))
|
|
||||||
@@ -1,59 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
from collections import OrderedDict
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
import torch
|
|
||||||
|
|
||||||
from vllm_omni.diffusion.models.dreamzero.pipeline_dreamzero import DreamZeroPipeline
|
|
||||||
from vllm_omni.diffusion.models.dreamzero.state_dreamzero import DreamZeroState
|
|
||||||
|
|
||||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
|
||||||
|
|
||||||
|
|
||||||
def _empty_pipeline() -> DreamZeroPipeline:
|
|
||||||
pipeline = DreamZeroPipeline.__new__(DreamZeroPipeline)
|
|
||||||
pipeline._states = OrderedDict()
|
|
||||||
pipeline._max_session_states = 2
|
|
||||||
return pipeline
|
|
||||||
|
|
||||||
|
|
||||||
def test_dreamzero_pipeline_state_is_session_keyed() -> None:
|
|
||||||
pipeline = _empty_pipeline()
|
|
||||||
|
|
||||||
session_a = pipeline._get_or_create_state("session-a")
|
|
||||||
session_b = pipeline._get_or_create_state("session-b")
|
|
||||||
session_a.call_count = 7
|
|
||||||
session_b.call_count = 3
|
|
||||||
|
|
||||||
assert pipeline._get_or_create_state("session-a") is session_a
|
|
||||||
assert pipeline._get_or_create_state("session-b") is session_b
|
|
||||||
assert session_a.call_count == 7
|
|
||||||
assert session_b.call_count == 3
|
|
||||||
|
|
||||||
|
|
||||||
def test_dreamzero_pipeline_state_lru_caps_retained_sessions() -> None:
|
|
||||||
pipeline = _empty_pipeline()
|
|
||||||
|
|
||||||
session_a = pipeline._get_or_create_state("session-a")
|
|
||||||
pipeline._get_or_create_state("session-b")
|
|
||||||
assert pipeline._get_or_create_state("session-a") is session_a
|
|
||||||
|
|
||||||
pipeline._get_or_create_state("session-c")
|
|
||||||
|
|
||||||
assert list(pipeline._states) == ["session-a", "session-c"]
|
|
||||||
assert "session-b" not in pipeline._states
|
|
||||||
|
|
||||||
|
|
||||||
def test_dreamzero_state_cache_access_requires_initialization() -> None:
|
|
||||||
state = DreamZeroState()
|
|
||||||
|
|
||||||
with pytest.raises(RuntimeError, match="KV caches not initialized"):
|
|
||||||
state.get_kv_caches()
|
|
||||||
|
|
||||||
with pytest.raises(RuntimeError, match="Cross-attn caches not initialized"):
|
|
||||||
state.get_crossattn_caches()
|
|
||||||
|
|
||||||
with pytest.raises(RuntimeError, match="create_kv_caches first"):
|
|
||||||
state.update_kv_cache(0, torch.empty(0))
|
|
||||||
@@ -1,381 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""
|
|
||||||
Copied from the DreamZero repository's `test_client_AR.py`:
|
|
||||||
https://github.com/dreamzero0/dreamzero/blob/main/test_client_AR.py
|
|
||||||
|
|
||||||
Kept here for end-to-end compatibility / parity testing against the vLLM
|
|
||||||
OpenPI server.
|
|
||||||
|
|
||||||
Test client for AR_droid policy server using roboarena interface.
|
|
||||||
|
|
||||||
Sends real video frames from debug_image/ directory instead of zero dummy images.
|
|
||||||
|
|
||||||
Frame schedule (matching debug_inference.py):
|
|
||||||
- Step 0 (initial): send frame [0] (1 frame, H W 3)
|
|
||||||
- Step 1: send frames [0, 7, 15, 23] (4 frames, 4 H W 3)
|
|
||||||
- Step 2: send frames [24, 31, 39, 47] (4 frames)
|
|
||||||
- Step 3: send frames [48, 55, 63, 71] (4 frames)
|
|
||||||
- ...
|
|
||||||
|
|
||||||
Expected server configuration:
|
|
||||||
- image_resolution: (180, 320)
|
|
||||||
- n_external_cameras: 2
|
|
||||||
- needs_wrist_camera: True
|
|
||||||
- action_space: "joint_position"
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
# Start server with roboarena interface:
|
|
||||||
torchrun --nproc_per_node=8 socket_test_optimized_AR.py --port 8000
|
|
||||||
|
|
||||||
# Run against the original DreamZero websocket server (default path):
|
|
||||||
python test_client_AR.py --host <server_host> --port 8000
|
|
||||||
|
|
||||||
# Run against the vLLM OpenPI server:
|
|
||||||
python test_client_AR.py --host <server_host> --port 8000 --path /v1/realtime/robot/openpi
|
|
||||||
|
|
||||||
# Use zero images instead of real video (old behavior):
|
|
||||||
python test_client_AR.py --host <server_host> --port 8000 --use-zero-images
|
|
||||||
"""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import logging
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
import uuid
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import cv2
|
|
||||||
import numpy as np
|
|
||||||
from openpi_client import msgpack_numpy
|
|
||||||
|
|
||||||
_DREAMZERO_REPO_ENV = os.environ.get("DREAMZERO_REPO")
|
|
||||||
DREAMZERO_REPO = Path(_DREAMZERO_REPO_ENV).expanduser() if _DREAMZERO_REPO_ENV else None
|
|
||||||
|
|
||||||
|
|
||||||
def _import_upstream_policy_modules():
|
|
||||||
if DREAMZERO_REPO is None:
|
|
||||||
raise ImportError("Set DREAMZERO_REPO to an upstream DreamZero checkout before using this helper.")
|
|
||||||
if DREAMZERO_REPO.exists() and str(DREAMZERO_REPO) not in sys.path:
|
|
||||||
sys.path.insert(0, str(DREAMZERO_REPO))
|
|
||||||
|
|
||||||
import eval_utils.policy_server as policy_server
|
|
||||||
from eval_utils.policy_client import WebsocketClientPolicy
|
|
||||||
|
|
||||||
return policy_server, WebsocketClientPolicy
|
|
||||||
|
|
||||||
|
|
||||||
policy_server, WebsocketClientPolicy = _import_upstream_policy_modules()
|
|
||||||
|
|
||||||
VIDEO_DIR = os.environ.get(
|
|
||||||
"DREAMZERO_VIDEO_DIR",
|
|
||||||
str(DREAMZERO_REPO / "debug_image") if DREAMZERO_REPO is not None else "debug_image",
|
|
||||||
)
|
|
||||||
|
|
||||||
# roboarena key -> video filename
|
|
||||||
CAMERA_FILES = {
|
|
||||||
"observation/exterior_image_0_left": "exterior_image_1_left.mp4",
|
|
||||||
"observation/exterior_image_1_left": "exterior_image_2_left.mp4",
|
|
||||||
"observation/wrist_image_left": "wrist_image_left.mp4",
|
|
||||||
}
|
|
||||||
|
|
||||||
# Frame schedule constants (matching debug_inference.py)
|
|
||||||
RELATIVE_OFFSETS = [-23, -16, -8, 0]
|
|
||||||
ACTION_HORIZON = 24
|
|
||||||
DEFAULT_WEBSOCKET_PATH = ""
|
|
||||||
|
|
||||||
|
|
||||||
class OpenPIWebsocketClientPolicy(WebsocketClientPolicy):
|
|
||||||
"""DreamZero websocket client with a configurable path suffix.
|
|
||||||
|
|
||||||
The original DreamZero client connects to ``ws://host:port``.
|
|
||||||
vLLM serves the compatible robot policy endpoint at
|
|
||||||
``/v1/realtime/robot/openpi`` when ``path`` is set accordingly.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
host: str = "0.0.0.0",
|
|
||||||
port: int = 8000,
|
|
||||||
path: str = DEFAULT_WEBSOCKET_PATH,
|
|
||||||
) -> None:
|
|
||||||
self._uri = f"ws://{host}:{port}{path}"
|
|
||||||
self._packer = msgpack_numpy.Packer()
|
|
||||||
self._ws, self._server_metadata = self._wait_for_server()
|
|
||||||
|
|
||||||
|
|
||||||
def load_all_frames(video_path: str) -> np.ndarray:
|
|
||||||
"""Load all frames from a video file. Returns (N, H, W, 3) uint8 array (RGB)."""
|
|
||||||
cap = cv2.VideoCapture(video_path)
|
|
||||||
frames = []
|
|
||||||
while True:
|
|
||||||
ret, frame = cap.read()
|
|
||||||
if not ret:
|
|
||||||
break
|
|
||||||
frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
|
|
||||||
cap.release()
|
|
||||||
if not frames:
|
|
||||||
raise RuntimeError(f"No frames loaded from {video_path}")
|
|
||||||
return np.stack(frames, axis=0)
|
|
||||||
|
|
||||||
|
|
||||||
def load_camera_frames() -> dict[str, np.ndarray]:
|
|
||||||
"""Load all video frames for each camera from the debug_image/ directory.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Dict mapping roboarena camera keys to (N, H, W, 3) uint8 arrays.
|
|
||||||
"""
|
|
||||||
camera_frames: dict[str, np.ndarray] = {}
|
|
||||||
for cam_key, fname in CAMERA_FILES.items():
|
|
||||||
path = os.path.join(VIDEO_DIR, fname)
|
|
||||||
camera_frames[cam_key] = load_all_frames(path)
|
|
||||||
logging.info(f"Loaded {cam_key}: {camera_frames[cam_key].shape}")
|
|
||||||
return camera_frames
|
|
||||||
|
|
||||||
|
|
||||||
def build_frame_schedule(total_frames: int, num_chunks: int) -> list[list[int]]:
|
|
||||||
"""Build the frame index schedule for multi-frame chunks.
|
|
||||||
|
|
||||||
Returns a list of frame-index lists. Each inner list has 4 indices.
|
|
||||||
"""
|
|
||||||
chunks: list[list[int]] = []
|
|
||||||
current_frame = 23 # first anchor frame
|
|
||||||
for _ in range(num_chunks):
|
|
||||||
indices = [max(current_frame + off, 0) for off in RELATIVE_OFFSETS]
|
|
||||||
if indices[-1] >= total_frames:
|
|
||||||
logging.info(f"Frame {indices[-1]} >= {total_frames}, stopping at {len(chunks)} chunks")
|
|
||||||
break
|
|
||||||
chunks.append(indices)
|
|
||||||
current_frame += ACTION_HORIZON
|
|
||||||
return chunks
|
|
||||||
|
|
||||||
|
|
||||||
def _make_obs_from_video(
|
|
||||||
camera_frames: dict[str, np.ndarray],
|
|
||||||
frame_indices: list[int],
|
|
||||||
prompt: str,
|
|
||||||
session_id: str,
|
|
||||||
) -> dict:
|
|
||||||
"""Build an observation dict from real video frames.
|
|
||||||
|
|
||||||
For 1 frame: each image key is (H, W, 3).
|
|
||||||
For 4 frames: each image key is (4, H, W, 3).
|
|
||||||
"""
|
|
||||||
obs: dict = {}
|
|
||||||
for cam_key, all_frames in camera_frames.items():
|
|
||||||
selected = all_frames[frame_indices] # (T, H, W, 3)
|
|
||||||
if len(frame_indices) == 1:
|
|
||||||
selected = selected[0] # (H, W, 3)
|
|
||||||
obs[cam_key] = selected
|
|
||||||
|
|
||||||
obs["observation/joint_position"] = np.zeros(7, dtype=np.float32)
|
|
||||||
obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32)
|
|
||||||
obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32)
|
|
||||||
obs["prompt"] = prompt
|
|
||||||
obs["session_id"] = session_id
|
|
||||||
return obs
|
|
||||||
|
|
||||||
|
|
||||||
def _make_zero_observation(
|
|
||||||
server_config: policy_server.PolicyServerConfig,
|
|
||||||
prompt: str = "pick up the object",
|
|
||||||
session_id: str | None = None,
|
|
||||||
) -> dict:
|
|
||||||
"""Create a dummy observation matching AR_droid expectations.
|
|
||||||
|
|
||||||
AR_droid expects:
|
|
||||||
- 2 external cameras (exterior_image_0_left, exterior_image_1_left)
|
|
||||||
- 1 wrist camera (wrist_image_left)
|
|
||||||
- Image resolution: 180x320 (H x W)
|
|
||||||
- joint_position: 7 DoF
|
|
||||||
- gripper_position: 1 DoF
|
|
||||||
"""
|
|
||||||
obs = {}
|
|
||||||
|
|
||||||
# Determine image resolution
|
|
||||||
if server_config.image_resolution is not None:
|
|
||||||
h, w = server_config.image_resolution
|
|
||||||
else:
|
|
||||||
# Default for AR_droid
|
|
||||||
h, w = 180, 320
|
|
||||||
|
|
||||||
# External cameras (0-indexed in roboarena)
|
|
||||||
for i in range(server_config.n_external_cameras):
|
|
||||||
obs[f"observation/exterior_image_{i}_left"] = np.zeros((h, w, 3), dtype=np.uint8)
|
|
||||||
if server_config.needs_stereo_camera:
|
|
||||||
obs[f"observation/exterior_image_{i}_right"] = np.zeros((h, w, 3), dtype=np.uint8)
|
|
||||||
|
|
||||||
# Wrist camera
|
|
||||||
if server_config.needs_wrist_camera:
|
|
||||||
obs["observation/wrist_image_left"] = np.zeros((h, w, 3), dtype=np.uint8)
|
|
||||||
if server_config.needs_stereo_camera:
|
|
||||||
obs["observation/wrist_image_right"] = np.zeros((h, w, 3), dtype=np.uint8)
|
|
||||||
|
|
||||||
# Session ID - should be passed in to ensure consistency within a session
|
|
||||||
if server_config.needs_session_id:
|
|
||||||
import uuid
|
|
||||||
|
|
||||||
# Generate unique session ID if not provided
|
|
||||||
obs["session_id"] = session_id if session_id else str(uuid.uuid4())
|
|
||||||
|
|
||||||
# State observations (AR_droid: 7 DoF arm + 1 gripper)
|
|
||||||
obs["observation/joint_position"] = np.zeros(7, dtype=np.float32)
|
|
||||||
obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32)
|
|
||||||
obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32)
|
|
||||||
|
|
||||||
# Language prompt
|
|
||||||
obs["prompt"] = prompt
|
|
||||||
|
|
||||||
return obs
|
|
||||||
|
|
||||||
|
|
||||||
def test_ar_droid_policy_server(
|
|
||||||
host: str = "localhost",
|
|
||||||
port: int = 8000,
|
|
||||||
path: str = DEFAULT_WEBSOCKET_PATH,
|
|
||||||
num_chunks: int = 15,
|
|
||||||
prompt: str = "Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan",
|
|
||||||
use_zero_images: bool = False,
|
|
||||||
):
|
|
||||||
"""Test the AR_droid policy server with roboarena interface.
|
|
||||||
|
|
||||||
When use_zero_images is False (default), loads real video frames from
|
|
||||||
debug_image/ and follows the frame schedule from debug_inference.py.
|
|
||||||
"""
|
|
||||||
logging.info(f"Connecting to AR_droid server at ws://{host}:{port}{path} ...")
|
|
||||||
|
|
||||||
client = OpenPIWebsocketClientPolicy(host=host, port=port, path=path)
|
|
||||||
|
|
||||||
# Validate server metadata
|
|
||||||
metadata = client.get_server_metadata()
|
|
||||||
logging.info(f"Server metadata: {metadata}")
|
|
||||||
assert isinstance(metadata, dict), "Metadata should be a dict"
|
|
||||||
|
|
||||||
try:
|
|
||||||
server_config = policy_server.PolicyServerConfig(**metadata)
|
|
||||||
except Exception as e:
|
|
||||||
logging.error(f"Error parsing metadata: {e}")
|
|
||||||
raise e
|
|
||||||
|
|
||||||
# Validate expected AR_droid configuration
|
|
||||||
logging.info(f"Server config: {server_config}")
|
|
||||||
assert server_config.n_external_cameras == 2, f"Expected 2 external cameras, got {server_config.n_external_cameras}"
|
|
||||||
assert server_config.needs_wrist_camera, "Expected wrist camera to be enabled"
|
|
||||||
assert server_config.action_space == "joint_position", (
|
|
||||||
f"Expected joint_position action space, got {server_config.action_space}"
|
|
||||||
)
|
|
||||||
|
|
||||||
logging.info("Server configuration validated for AR_droid")
|
|
||||||
|
|
||||||
# Generate unique session ID for this test run
|
|
||||||
session_id = str(uuid.uuid4())
|
|
||||||
logging.info(f"Session ID: {session_id}")
|
|
||||||
|
|
||||||
# ── Zero-image fallback mode ──────────────────────────────────────
|
|
||||||
if use_zero_images:
|
|
||||||
logging.info("Using ZERO dummy images (legacy mode)")
|
|
||||||
for i in range(num_chunks):
|
|
||||||
obs = _make_zero_observation(server_config, prompt=prompt, session_id=session_id)
|
|
||||||
logging.info(f"Inference {i + 1}/{num_chunks}: prompt='{prompt}'")
|
|
||||||
t0 = time.time()
|
|
||||||
actions = client.infer(obs)
|
|
||||||
dt = time.time() - t0
|
|
||||||
_log_action(actions, dt)
|
|
||||||
|
|
||||||
logging.info("Sending reset...")
|
|
||||||
client.reset({})
|
|
||||||
logging.info("Done (zero-image mode).")
|
|
||||||
return
|
|
||||||
|
|
||||||
# ── Real video frame mode ─────────────────────────────────────────
|
|
||||||
logging.info("Loading real video frames from debug_image/ directory")
|
|
||||||
camera_frames = load_camera_frames()
|
|
||||||
|
|
||||||
total_frames = min(v.shape[0] for v in camera_frames.values())
|
|
||||||
logging.info(f"Total frames available: {total_frames}")
|
|
||||||
|
|
||||||
# Build frame schedule
|
|
||||||
chunks = build_frame_schedule(total_frames, num_chunks)
|
|
||||||
|
|
||||||
logging.info("Frame schedule:")
|
|
||||||
logging.info(" Initial: [0]")
|
|
||||||
for i, indices in enumerate(chunks):
|
|
||||||
logging.info(f" Chunk {i}: {indices}")
|
|
||||||
|
|
||||||
# Step 0: initial single frame
|
|
||||||
logging.info("=== Initial: frame [0] ===")
|
|
||||||
obs = _make_obs_from_video(camera_frames, [0], prompt, session_id)
|
|
||||||
t0 = time.time()
|
|
||||||
actions = client.infer(obs)
|
|
||||||
dt = time.time() - t0
|
|
||||||
_log_action(actions, dt)
|
|
||||||
|
|
||||||
# Subsequent chunks: send 4 frames at a time
|
|
||||||
for chunk_idx, frame_indices in enumerate(chunks):
|
|
||||||
logging.info(f"=== Chunk {chunk_idx}: frames {frame_indices} ===")
|
|
||||||
obs = _make_obs_from_video(camera_frames, frame_indices, prompt, session_id)
|
|
||||||
t0 = time.time()
|
|
||||||
actions = client.infer(obs)
|
|
||||||
dt = time.time() - t0
|
|
||||||
_log_action(actions, dt)
|
|
||||||
|
|
||||||
# Reset triggers video save on the server
|
|
||||||
logging.info("Sending reset to save video...")
|
|
||||||
client.reset({})
|
|
||||||
|
|
||||||
logging.info("Done.")
|
|
||||||
|
|
||||||
|
|
||||||
def _log_action(actions: np.ndarray, dt: float) -> None:
|
|
||||||
"""Pretty-print action shape, range, and timing."""
|
|
||||||
assert isinstance(actions, np.ndarray), f"Expected numpy array, got {type(actions)}"
|
|
||||||
assert actions.ndim == 2, f"Expected 2D array, got shape {actions.shape}"
|
|
||||||
assert actions.shape[-1] == 8, f"Expected 8 action dims (7 joints + 1 gripper), got {actions.shape[-1]}"
|
|
||||||
logging.info(f" Action shape: {actions.shape}, range: [{actions.min():.4f}, {actions.max():.4f}], time: {dt:.2f}s")
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
parser = argparse.ArgumentParser(description="Test AR_droid policy server with real video frames from debug_image/")
|
|
||||||
parser.add_argument("--host", default="localhost", help="Server hostname")
|
|
||||||
parser.add_argument("--port", type=int, default=8000, help="Server port")
|
|
||||||
parser.add_argument(
|
|
||||||
"--path",
|
|
||||||
default=DEFAULT_WEBSOCKET_PATH,
|
|
||||||
help="WebSocket path suffix (default: empty string for the original DreamZero server)",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--num-chunks",
|
|
||||||
type=int,
|
|
||||||
default=15,
|
|
||||||
help="Number of 4-frame chunks to send after the initial frame (default: 15)",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--prompt",
|
|
||||||
default="Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan",
|
|
||||||
help="Language prompt for the policy",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--use-zero-images",
|
|
||||||
action="store_true",
|
|
||||||
help="Use zero dummy images instead of real video frames (legacy mode)",
|
|
||||||
)
|
|
||||||
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
logging.basicConfig(
|
|
||||||
level=logging.INFO,
|
|
||||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
|
||||||
)
|
|
||||||
|
|
||||||
test_ar_droid_policy_server(
|
|
||||||
host=args.host,
|
|
||||||
port=args.port,
|
|
||||||
path=args.path,
|
|
||||||
num_chunks=args.num_chunks,
|
|
||||||
prompt=args.prompt,
|
|
||||||
use_zero_images=args.use_zero_images,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -1,358 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
"""Formal OpenPI end-to-end parity: upstream DreamZero server vs `vllm serve`.
|
|
||||||
|
|
||||||
This test uses DreamZero's own client-side observation builders from
|
|
||||||
`${DREAMZERO_REPO}/test_client_AR.py`, and client-side websocket protocol from
|
|
||||||
`${DREAMZERO_REPO}/eval_utils/policy_client.py`.
|
|
||||||
|
|
||||||
The only client-side adaptation for vLLM is the websocket path:
|
|
||||||
DreamZero's upstream server serves at `/`, while vLLM serves OpenPI at
|
|
||||||
`/v1/realtime/robot/openpi`.
|
|
||||||
|
|
||||||
Current scope for this test:
|
|
||||||
- default two-GPU run (`nproc_per_node=2` on upstream, `--cfg-parallel-size 2` on `vllm serve`)
|
|
||||||
- non-`torch.compile` (upstream launched through
|
|
||||||
`upstream_socket_server_no_compile.py`, vLLM with `--enforce-eager`)
|
|
||||||
- non-DiT-cache / non-skip-schedule (`NUM_DIT_STEPS=16`)
|
|
||||||
|
|
||||||
Serving contract locked by this test:
|
|
||||||
- upstream DreamZero still boots from the local checkpoint directory
|
|
||||||
- vLLM boots from the official DreamZero HF repo name (`GEAR-Dreams/DreamZero-DROID`)
|
|
||||||
rather than a prepared local bundle path
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import os
|
|
||||||
import shutil
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
import pytest
|
|
||||||
import torch
|
|
||||||
|
|
||||||
from tests.helpers.runtime import get_open_port
|
|
||||||
|
|
||||||
msgpack_numpy = pytest.importorskip("openpi_client.msgpack_numpy")
|
|
||||||
|
|
||||||
_DREAMZERO_REPO_ENV = os.environ.get("DREAMZERO_REPO")
|
|
||||||
DREAMZERO_REPO = Path(_DREAMZERO_REPO_ENV).expanduser() if _DREAMZERO_REPO_ENV else None
|
|
||||||
if DREAMZERO_REPO is not None and str(DREAMZERO_REPO) not in sys.path:
|
|
||||||
sys.path.insert(0, str(DREAMZERO_REPO))
|
|
||||||
|
|
||||||
try:
|
|
||||||
import test_client_AR as dreamzero_client
|
|
||||||
from eval_utils.policy_client import WebsocketClientPolicy
|
|
||||||
except Exception: # pragma: no cover - guarded by pytest skip below
|
|
||||||
dreamzero_client = None
|
|
||||||
WebsocketClientPolicy = None
|
|
||||||
_BaseWebsocketClientPolicy = WebsocketClientPolicy if WebsocketClientPolicy is not None else object
|
|
||||||
|
|
||||||
CHECKPOINT_DIR = DREAMZERO_REPO / "checkpoints" / "dreamzero" if DREAMZERO_REPO is not None else None
|
|
||||||
VLLM_MODEL = os.environ.get("VLLM_DREAMZERO_MODEL", "GEAR-Dreams/DreamZero-DROID")
|
|
||||||
SERVICE_READY_TIMEOUT_S = int(os.environ.get("OPENPI_SERVICE_READY_TIMEOUT_S", "900"))
|
|
||||||
PROMPT = "Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan"
|
|
||||||
SESSION_ID = "openpi-e2e-parity-session"
|
|
||||||
|
|
||||||
pytestmark = [
|
|
||||||
pytest.mark.skipif(not torch.cuda.is_available(), reason="GPU required"),
|
|
||||||
pytest.mark.skipif(
|
|
||||||
dreamzero_client is None or WebsocketClientPolicy is None,
|
|
||||||
reason="DreamZero client modules are required on PYTHONPATH",
|
|
||||||
),
|
|
||||||
pytest.mark.skipif(
|
|
||||||
DREAMZERO_REPO is None or not DREAMZERO_REPO.exists(),
|
|
||||||
reason="DreamZero source repo is required at DREAMZERO_REPO",
|
|
||||||
),
|
|
||||||
pytest.mark.skipif(
|
|
||||||
CHECKPOINT_DIR is None or not CHECKPOINT_DIR.exists(), reason="DreamZero local checkpoint is required"
|
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
class OpenPIWebsocketClientPolicy(_BaseWebsocketClientPolicy):
|
|
||||||
"""DreamZero client protocol with an OpenPI websocket path suffix."""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
host: str = "127.0.0.1",
|
|
||||||
port: int = 8000,
|
|
||||||
path: str = "/v1/realtime/robot/openpi",
|
|
||||||
) -> None:
|
|
||||||
self._uri = f"ws://{host}:{port}{path}"
|
|
||||||
self._packer = msgpack_numpy.Packer()
|
|
||||||
self._ws, self._server_metadata = self._wait_for_server()
|
|
||||||
|
|
||||||
|
|
||||||
def _vllm_executable() -> str:
|
|
||||||
fallback = Path(sys.executable).with_name("vllm")
|
|
||||||
if fallback.exists():
|
|
||||||
return str(fallback)
|
|
||||||
exe = shutil.which("vllm")
|
|
||||||
if exe:
|
|
||||||
return exe
|
|
||||||
raise FileNotFoundError("Unable to locate `vllm` executable in current environment.")
|
|
||||||
|
|
||||||
|
|
||||||
def _cfg_parallel_size() -> int:
|
|
||||||
return int(os.environ.get("OPENPI_E2E_CFG_PARALLEL_SIZE", "2"))
|
|
||||||
|
|
||||||
|
|
||||||
def _pick_test_gpus() -> list[str]:
|
|
||||||
cfg_parallel_size = _cfg_parallel_size()
|
|
||||||
override = os.environ.get("OPENPI_E2E_GPUS") or os.environ.get("OPENPI_E2E_GPU")
|
|
||||||
if override is not None:
|
|
||||||
gpus = [part.strip() for part in override.split(",") if part.strip()]
|
|
||||||
if not gpus:
|
|
||||||
raise ValueError("OPENPI_E2E_GPUS is set but empty.")
|
|
||||||
if len(gpus) < cfg_parallel_size:
|
|
||||||
raise RuntimeError(f"Need {cfg_parallel_size} GPUs, but OPENPI_E2E_GPUS only provided {gpus}.")
|
|
||||||
return gpus
|
|
||||||
|
|
||||||
query = subprocess.check_output(
|
|
||||||
[
|
|
||||||
"nvidia-smi",
|
|
||||||
"--query-gpu=index,memory.used",
|
|
||||||
"--format=csv,noheader,nounits",
|
|
||||||
],
|
|
||||||
text=True,
|
|
||||||
)
|
|
||||||
gpu_rows = []
|
|
||||||
for line in query.strip().splitlines():
|
|
||||||
gpu_index, used_mb = [part.strip() for part in line.split(",", maxsplit=1)]
|
|
||||||
gpu_rows.append((int(used_mb), gpu_index))
|
|
||||||
gpu_rows.sort()
|
|
||||||
gpus = [gpu_index for _, gpu_index in gpu_rows[: max(cfg_parallel_size, 1)]]
|
|
||||||
if len(gpus) < cfg_parallel_size:
|
|
||||||
raise RuntimeError(
|
|
||||||
f"Need {cfg_parallel_size} GPUs for cfg_parallel_size={cfg_parallel_size}, "
|
|
||||||
f"but found only {len(gpus)} candidates."
|
|
||||||
)
|
|
||||||
return gpus
|
|
||||||
|
|
||||||
|
|
||||||
def _torchrun_argv(script: str, port: int) -> list[str]:
|
|
||||||
return [
|
|
||||||
sys.executable,
|
|
||||||
"-m",
|
|
||||||
"torch.distributed.run",
|
|
||||||
"--standalone",
|
|
||||||
"--nproc_per_node",
|
|
||||||
str(_cfg_parallel_size()),
|
|
||||||
script,
|
|
||||||
"--port",
|
|
||||||
str(port),
|
|
||||||
"--model_path",
|
|
||||||
str(CHECKPOINT_DIR),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def _run_upstream_service(port: int, log_path: Path) -> subprocess.Popen[str]:
|
|
||||||
env = os.environ.copy()
|
|
||||||
env.setdefault("PYTHONPATH", "")
|
|
||||||
env["PYTHONPATH"] = f"{Path.cwd()}:{DREAMZERO_REPO}:{env['PYTHONPATH']}".rstrip(":")
|
|
||||||
env["CUDA_VISIBLE_DEVICES"] = ",".join(_pick_test_gpus())
|
|
||||||
env.setdefault("NO_ALBUMENTATIONS_UPDATE", "1")
|
|
||||||
env["ATTENTION_BACKEND"] = "torch"
|
|
||||||
env.setdefault("ENABLE_TENSORRT", "false")
|
|
||||||
env["ENABLE_DIT_CACHE"] = "false"
|
|
||||||
env["NUM_DIT_STEPS"] = "16"
|
|
||||||
env["DYNAMIC_CACHE_SCHEDULE"] = "false"
|
|
||||||
argv = _torchrun_argv(
|
|
||||||
str(Path("tests/dreamzero/upstream/upstream_socket_server_no_compile.py")),
|
|
||||||
port,
|
|
||||||
)
|
|
||||||
log_file = log_path.open("w")
|
|
||||||
proc = subprocess.Popen(
|
|
||||||
argv,
|
|
||||||
stdout=log_file,
|
|
||||||
stderr=subprocess.STDOUT,
|
|
||||||
text=True,
|
|
||||||
cwd=str(Path.cwd()),
|
|
||||||
env=env,
|
|
||||||
)
|
|
||||||
proc._codex_log_file = log_file # type: ignore[attr-defined]
|
|
||||||
return proc
|
|
||||||
|
|
||||||
|
|
||||||
def _run_vllm_service(port: int, log_path: Path) -> subprocess.Popen[str]:
|
|
||||||
env = os.environ.copy()
|
|
||||||
gpus = _pick_test_gpus()
|
|
||||||
cfg_parallel_size = _cfg_parallel_size()
|
|
||||||
if cfg_parallel_size > len(gpus):
|
|
||||||
raise RuntimeError(
|
|
||||||
f"cfg_parallel_size={cfg_parallel_size} requires at least {cfg_parallel_size} GPUs, but only got {gpus}."
|
|
||||||
)
|
|
||||||
env["CUDA_VISIBLE_DEVICES"] = ",".join(gpus[:cfg_parallel_size])
|
|
||||||
env.setdefault("ATTENTION_BACKEND", "torch")
|
|
||||||
env.setdefault("DIFFUSION_ATTENTION_BACKEND", "TORCH_SDPA")
|
|
||||||
env.setdefault("MASTER_PORT", str(get_open_port()))
|
|
||||||
argv = [
|
|
||||||
_vllm_executable(),
|
|
||||||
"serve",
|
|
||||||
VLLM_MODEL,
|
|
||||||
"--omni",
|
|
||||||
"--host",
|
|
||||||
"127.0.0.1",
|
|
||||||
"--port",
|
|
||||||
str(port),
|
|
||||||
"--served-model-name",
|
|
||||||
"dreamzero-droid",
|
|
||||||
"--enforce-eager",
|
|
||||||
]
|
|
||||||
if cfg_parallel_size > 1:
|
|
||||||
argv.extend(["--cfg-parallel-size", str(cfg_parallel_size)])
|
|
||||||
log_file = log_path.open("w")
|
|
||||||
proc = subprocess.Popen(
|
|
||||||
argv,
|
|
||||||
stdout=log_file,
|
|
||||||
stderr=subprocess.STDOUT,
|
|
||||||
text=True,
|
|
||||||
env=env,
|
|
||||||
cwd=str(Path.cwd()),
|
|
||||||
)
|
|
||||||
proc._codex_log_file = log_file # type: ignore[attr-defined]
|
|
||||||
return proc
|
|
||||||
|
|
||||||
|
|
||||||
def _stop_process(proc: subprocess.Popen[str]) -> None:
|
|
||||||
log_file = getattr(proc, "_codex_log_file", None)
|
|
||||||
if proc.poll() is None:
|
|
||||||
proc.terminate()
|
|
||||||
try:
|
|
||||||
proc.wait(timeout=30)
|
|
||||||
except subprocess.TimeoutExpired: # pragma: no cover - cleanup path
|
|
||||||
proc.kill()
|
|
||||||
proc.wait(timeout=10)
|
|
||||||
if log_file is not None:
|
|
||||||
log_file.close()
|
|
||||||
|
|
||||||
|
|
||||||
def _build_obs_sequence() -> tuple[dict, dict]:
|
|
||||||
camera_frames = dreamzero_client.load_camera_frames()
|
|
||||||
chunks = dreamzero_client.build_frame_schedule(
|
|
||||||
min(v.shape[0] for v in camera_frames.values()),
|
|
||||||
1,
|
|
||||||
)
|
|
||||||
obs0 = dreamzero_client._make_obs_from_video(camera_frames, [0], PROMPT, SESSION_ID)
|
|
||||||
obs1 = dreamzero_client._make_obs_from_video(camera_frames, chunks[0], PROMPT, SESSION_ID)
|
|
||||||
return obs0, obs1
|
|
||||||
|
|
||||||
|
|
||||||
def _wait_for_client_ready(client_factory, timeout_s: float, proc=None, log_path: Path | None = None):
|
|
||||||
deadline = time.time() + timeout_s
|
|
||||||
last_err: Exception | None = None
|
|
||||||
while time.time() < deadline:
|
|
||||||
if proc is not None and proc.poll() is not None:
|
|
||||||
details = ""
|
|
||||||
if log_path is not None and log_path.exists():
|
|
||||||
details = log_path.read_text(errors="replace")[-8000:]
|
|
||||||
raise RuntimeError(f"Service exited before becoming ready with code {proc.returncode}.\n{details}")
|
|
||||||
try:
|
|
||||||
return client_factory()
|
|
||||||
except Exception as exc: # pragma: no cover - retry path
|
|
||||||
last_err = exc
|
|
||||||
time.sleep(1)
|
|
||||||
raise TimeoutError(f"Timed out waiting for websocket service: {last_err}")
|
|
||||||
|
|
||||||
|
|
||||||
def _collect_outputs_with_client(client) -> tuple[dict, list[np.ndarray]]:
|
|
||||||
metadata = client.get_server_metadata()
|
|
||||||
obs0, obs1 = _build_obs_sequence()
|
|
||||||
outputs = [
|
|
||||||
client.infer(dict(obs0)),
|
|
||||||
client.infer(dict(obs1)),
|
|
||||||
]
|
|
||||||
assert _normalize_reset_response(client.reset({})) == "reset successful"
|
|
||||||
outputs.append(client.infer(dict(obs0)))
|
|
||||||
client._ws.close()
|
|
||||||
return metadata, outputs
|
|
||||||
|
|
||||||
|
|
||||||
def _normalize_reset_response(response) -> str:
|
|
||||||
if isinstance(response, str):
|
|
||||||
return response
|
|
||||||
decoded = msgpack_numpy.unpackb(response)
|
|
||||||
if isinstance(decoded, dict):
|
|
||||||
return str(decoded.get("status"))
|
|
||||||
return str(decoded)
|
|
||||||
|
|
||||||
|
|
||||||
def _normalize_metadata(metadata: dict) -> dict:
|
|
||||||
normalized = dict(metadata)
|
|
||||||
if isinstance(normalized.get("image_resolution"), tuple):
|
|
||||||
normalized["image_resolution"] = list(normalized["image_resolution"])
|
|
||||||
return normalized
|
|
||||||
|
|
||||||
|
|
||||||
def _assert_logs_clean(log_path: Path) -> None:
|
|
||||||
text = log_path.read_text(errors="replace")
|
|
||||||
if "SignalException: Process" in text and "got signal: 15" in text:
|
|
||||||
text = text.split("Traceback (most recent call last):", 1)[0]
|
|
||||||
assert "Traceback" not in text, text
|
|
||||||
assert "RuntimeError:" not in text, text
|
|
||||||
|
|
||||||
|
|
||||||
def _assert_upstream_log_matches_vllm_baseline(log_path: Path) -> None:
|
|
||||||
text = log_path.read_text(errors="replace")
|
|
||||||
assert "DIT Compute Steps 8 steps" not in text, text
|
|
||||||
assert "DIT Compute Steps 16 steps" in text, text
|
|
||||||
|
|
||||||
|
|
||||||
def test_openpi_service_matches_upstream_server_noncompile(tmp_path: Path) -> None:
|
|
||||||
expected_metadata = {
|
|
||||||
"image_resolution": [180, 320],
|
|
||||||
"n_external_cameras": 2,
|
|
||||||
"needs_wrist_camera": True,
|
|
||||||
"needs_stereo_camera": False,
|
|
||||||
"needs_session_id": True,
|
|
||||||
"action_space": "joint_position",
|
|
||||||
}
|
|
||||||
|
|
||||||
upstream_port = get_open_port()
|
|
||||||
upstream_log = tmp_path / "dreamzero_upstream.log"
|
|
||||||
upstream_proc = _run_upstream_service(upstream_port, upstream_log)
|
|
||||||
try:
|
|
||||||
upstream_client = _wait_for_client_ready(
|
|
||||||
lambda: WebsocketClientPolicy(host="127.0.0.1", port=upstream_port),
|
|
||||||
timeout_s=SERVICE_READY_TIMEOUT_S,
|
|
||||||
proc=upstream_proc,
|
|
||||||
log_path=upstream_log,
|
|
||||||
)
|
|
||||||
upstream_metadata, upstream_outputs = _collect_outputs_with_client(upstream_client)
|
|
||||||
finally:
|
|
||||||
_stop_process(upstream_proc)
|
|
||||||
_assert_logs_clean(upstream_log)
|
|
||||||
_assert_upstream_log_matches_vllm_baseline(upstream_log)
|
|
||||||
|
|
||||||
vllm_port = get_open_port()
|
|
||||||
vllm_log = tmp_path / "vllm_openpi.log"
|
|
||||||
vllm_proc = _run_vllm_service(vllm_port, vllm_log)
|
|
||||||
try:
|
|
||||||
vllm_client = _wait_for_client_ready(
|
|
||||||
lambda: OpenPIWebsocketClientPolicy(host="127.0.0.1", port=vllm_port),
|
|
||||||
timeout_s=SERVICE_READY_TIMEOUT_S,
|
|
||||||
proc=vllm_proc,
|
|
||||||
log_path=vllm_log,
|
|
||||||
)
|
|
||||||
vllm_metadata, vllm_outputs = _collect_outputs_with_client(vllm_client)
|
|
||||||
finally:
|
|
||||||
_stop_process(vllm_proc)
|
|
||||||
_assert_logs_clean(vllm_log)
|
|
||||||
|
|
||||||
assert _normalize_metadata(upstream_metadata) == expected_metadata
|
|
||||||
assert _normalize_metadata(vllm_metadata) == expected_metadata
|
|
||||||
|
|
||||||
for idx, (actual, expected) in enumerate(zip(vllm_outputs, upstream_outputs, strict=True)):
|
|
||||||
np.testing.assert_allclose(
|
|
||||||
actual,
|
|
||||||
expected,
|
|
||||||
rtol=1e-2,
|
|
||||||
atol=1e-3,
|
|
||||||
err_msg=f"OpenPI step {idx} output mismatch",
|
|
||||||
)
|
|
||||||
@@ -1,184 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
"""Launch the upstream DreamZero websocket server with `torch.compile` disabled.
|
|
||||||
|
|
||||||
This wrapper is meant for formal parity tests against `vllm serve --omni`.
|
|
||||||
It monkeypatches `torch.compile` before importing DreamZero modules so that
|
|
||||||
all import-time decorators and `post_initialize()` compile calls become eager.
|
|
||||||
|
|
||||||
For the current DreamZero port baseline we also disable all upstream DiT cache
|
|
||||||
and step-skipping behavior so the reference server matches vLLM's current
|
|
||||||
eager/no-skip implementation:
|
|
||||||
- `ENABLE_DIT_CACHE=false`
|
|
||||||
- `NUM_DIT_STEPS=16`
|
|
||||||
- `DYNAMIC_CACHE_SCHEDULE=false`
|
|
||||||
|
|
||||||
In our CI/dev environment we also do not install Transformer Engine or
|
|
||||||
FlashAttention, so upstream's default `socket_test_optimized_AR.main()` path
|
|
||||||
(`ATTENTION_BACKEND=TE` and then fallback to `FA2`) cannot execute. To keep the
|
|
||||||
formal server-vs-server test runnable without pulling in those heavyweight
|
|
||||||
optional deps, this wrapper reproduces upstream `main()` but pins attention to
|
|
||||||
PyTorch SDPA (`ATTENTION_BACKEND=torch`) for this subprocess only.
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
PYTHONPATH="${DREAMZERO_REPO}" \\
|
|
||||||
.venv/bin/python -m torch.distributed.run --standalone --nproc_per_node=2 \\
|
|
||||||
tests/dreamzero/upstream/upstream_socket_server_no_compile.py --port 18081 \\
|
|
||||||
--model_path "${DREAMZERO_REPO}/checkpoints/dreamzero"
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import torch
|
|
||||||
import torch.nn.functional as F
|
|
||||||
|
|
||||||
|
|
||||||
def _identity_compile(*args, **kwargs):
|
|
||||||
if args and callable(args[0]) and len(args) == 1 and not kwargs:
|
|
||||||
return args[0]
|
|
||||||
|
|
||||||
def deco(fn):
|
|
||||||
return fn
|
|
||||||
|
|
||||||
return deco
|
|
||||||
|
|
||||||
|
|
||||||
torch.compile = _identity_compile
|
|
||||||
|
|
||||||
os.environ.setdefault("NO_ALBUMENTATIONS_UPDATE", "1")
|
|
||||||
|
|
||||||
DREAMZERO_REPO_ENV = os.environ.get("DREAMZERO_REPO")
|
|
||||||
if not DREAMZERO_REPO_ENV:
|
|
||||||
raise RuntimeError("Set DREAMZERO_REPO to an upstream DreamZero checkout before launching this helper.")
|
|
||||||
DREAMZERO_REPO = Path(DREAMZERO_REPO_ENV).expanduser()
|
|
||||||
if str(DREAMZERO_REPO) not in sys.path:
|
|
||||||
sys.path.insert(0, str(DREAMZERO_REPO))
|
|
||||||
|
|
||||||
import socket_test_optimized_AR as upstream # noqa: E402
|
|
||||||
import tyro # noqa: E402
|
|
||||||
from groot.vla.model.dreamzero.modules import attention as upstream_attention # noqa: E402
|
|
||||||
from groot.vla.model.dreamzero.modules import wan2_1_submodule as upstream_submodule # noqa: E402
|
|
||||||
from groot.vla.model.dreamzero.modules import wan_video_dit as upstream_wan_video_dit # noqa: E402
|
|
||||||
|
|
||||||
|
|
||||||
def _torch_varlen_flash_attention(
|
|
||||||
q: torch.Tensor,
|
|
||||||
k: torch.Tensor,
|
|
||||||
v: torch.Tensor,
|
|
||||||
q_lens: torch.Tensor | None = None,
|
|
||||||
k_lens: torch.Tensor | None = None,
|
|
||||||
dropout_p: float = 0.0,
|
|
||||||
causal: bool = False,
|
|
||||||
dtype: torch.dtype = torch.bfloat16,
|
|
||||||
**_: object,
|
|
||||||
) -> torch.Tensor:
|
|
||||||
if q_lens is not None or k_lens is not None:
|
|
||||||
upstream_attention.warnings.warn(
|
|
||||||
"Padding mask is disabled in the test-only SDPA fallback.",
|
|
||||||
)
|
|
||||||
out_dtype = q.dtype
|
|
||||||
q = q.transpose(1, 2).to(dtype)
|
|
||||||
k = k.transpose(1, 2).to(dtype)
|
|
||||||
v = v.transpose(1, 2).to(dtype)
|
|
||||||
out = F.scaled_dot_product_attention(
|
|
||||||
q,
|
|
||||||
k,
|
|
||||||
v,
|
|
||||||
attn_mask=None,
|
|
||||||
is_causal=causal,
|
|
||||||
dropout_p=dropout_p,
|
|
||||||
)
|
|
||||||
return out.transpose(1, 2).contiguous().to(out_dtype)
|
|
||||||
|
|
||||||
|
|
||||||
upstream_attention.flash_attention = _torch_varlen_flash_attention
|
|
||||||
upstream_submodule.flash_attention = _torch_varlen_flash_attention
|
|
||||||
upstream_wan_video_dit.FLASH_ATTN_3_AVAILABLE = False
|
|
||||||
upstream_wan_video_dit.FLASH_ATTN_2_AVAILABLE = False
|
|
||||||
upstream_wan_video_dit.SAGE_ATTN_AVAILABLE = False
|
|
||||||
|
|
||||||
|
|
||||||
def main(args: upstream.Args) -> None:
|
|
||||||
os.environ["ENABLE_DIT_CACHE"] = "false"
|
|
||||||
os.environ["ATTENTION_BACKEND"] = "torch"
|
|
||||||
os.environ["NUM_DIT_STEPS"] = "16"
|
|
||||||
os.environ["DYNAMIC_CACHE_SCHEDULE"] = "false"
|
|
||||||
torch._dynamo.config.recompile_limit = 800
|
|
||||||
|
|
||||||
embodiment_tag = "oxe_droid"
|
|
||||||
model_path = args.model_path
|
|
||||||
|
|
||||||
device_mesh = upstream.init_mesh()
|
|
||||||
rank = upstream.dist.get_rank()
|
|
||||||
|
|
||||||
timeout_delta = upstream.datetime.timedelta(seconds=args.timeout_seconds)
|
|
||||||
signal_group = upstream.dist.new_group(backend="gloo", timeout=timeout_delta)
|
|
||||||
upstream.logger.info("Rank %s initialized signal_group (gloo)", rank)
|
|
||||||
|
|
||||||
policy = upstream.GrootSimPolicy(
|
|
||||||
embodiment_tag=upstream.EmbodimentTag(embodiment_tag),
|
|
||||||
model_path=model_path,
|
|
||||||
device="cuda" if torch.cuda.is_available() else "cpu",
|
|
||||||
device_mesh=device_mesh,
|
|
||||||
)
|
|
||||||
|
|
||||||
hostname = upstream.socket.gethostname()
|
|
||||||
local_ip = upstream.socket.gethostbyname(hostname)
|
|
||||||
|
|
||||||
if rank == 0:
|
|
||||||
upstream.logging.info("Creating server (host: %s, ip: %s)", hostname, local_ip)
|
|
||||||
parent_dir = os.path.dirname(model_path)
|
|
||||||
date_suffix = upstream.datetime.datetime.now().strftime("%Y%m%d")
|
|
||||||
checkpoint_name = os.path.basename(model_path)
|
|
||||||
output_dir = os.path.join(
|
|
||||||
parent_dir,
|
|
||||||
f"real_world_eval_gen_{date_suffix}_{args.index}",
|
|
||||||
checkpoint_name,
|
|
||||||
)
|
|
||||||
os.makedirs(output_dir, exist_ok=True)
|
|
||||||
upstream.logging.info("Videos will be saved to: %s", output_dir)
|
|
||||||
else:
|
|
||||||
output_dir = None
|
|
||||||
upstream.logging.info("Rank %s starting as worker for distributed inference...", rank)
|
|
||||||
|
|
||||||
wrapper_policy = upstream.ARDroidRoboarenaPolicy(
|
|
||||||
groot_policy=policy,
|
|
||||||
signal_group=signal_group,
|
|
||||||
output_dir=output_dir,
|
|
||||||
)
|
|
||||||
|
|
||||||
server_config = upstream.PolicyServerConfig(
|
|
||||||
image_resolution=(180, 320),
|
|
||||||
needs_wrist_camera=True,
|
|
||||||
n_external_cameras=2,
|
|
||||||
needs_stereo_camera=False,
|
|
||||||
needs_session_id=True,
|
|
||||||
action_space="joint_position",
|
|
||||||
)
|
|
||||||
|
|
||||||
if rank == 0:
|
|
||||||
upstream.logging.info("Using roboarena policy server interface")
|
|
||||||
upstream.logging.info("Server config: %s", server_config)
|
|
||||||
roboarena_server = upstream.RoboarenaServer(
|
|
||||||
policy=wrapper_policy,
|
|
||||||
server_config=server_config,
|
|
||||||
host="0.0.0.0",
|
|
||||||
port=args.port,
|
|
||||||
)
|
|
||||||
roboarena_server.serve_forever()
|
|
||||||
else:
|
|
||||||
worker = upstream.DistributedWorker(
|
|
||||||
policy=policy,
|
|
||||||
signal_group=signal_group,
|
|
||||||
)
|
|
||||||
upstream.asyncio.run(worker.run())
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
upstream.logging.basicConfig(level=upstream.logging.INFO, force=True)
|
|
||||||
main(tyro.cli(upstream.Args))
|
|
||||||
@@ -8,7 +8,7 @@ L4-style smoke runs against an already-running server. The pytest wrappers in
|
|||||||
H100 / MI325) because they launch the live Omni server inside the test.
|
H100 / MI325) because they launch the live Omni server inside the test.
|
||||||
|
|
||||||
1. **Daily-Omni** — MCQ accuracy fields in the saved JSON (``daily_omni_accuracy``, …); by default the
|
1. **Daily-Omni** — MCQ accuracy fields in the saved JSON (``daily_omni_accuracy``, …); by default the
|
||||||
run **fails** if accuracy is strictly below **0.69** (``--min-daily-omni-accuracy`` / ``ACC_BENCH_MIN_DAILY_OMNI_ACCURACY``).
|
run **fails** if accuracy is strictly below **0.67** (``--min-daily-omni-accuracy`` / ``ACC_BENCH_MIN_DAILY_OMNI_ACCURACY``).
|
||||||
2. **Seed-TTS** — ``seed-tts-eval``-style metrics when ``--seed-tts-wer-eval`` is used
|
2. **Seed-TTS** — ``seed-tts-eval``-style metrics when ``--seed-tts-wer-eval`` is used
|
||||||
(WER / SIM / UTMOS keys from :func:`compute_seed_tts_wer_metrics`).
|
(WER / SIM / UTMOS keys from :func:`compute_seed_tts_wer_metrics`).
|
||||||
|
|
||||||
@@ -308,9 +308,9 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
|||||||
p.add_argument(
|
p.add_argument(
|
||||||
"--min-daily-omni-accuracy",
|
"--min-daily-omni-accuracy",
|
||||||
type=float,
|
type=float,
|
||||||
default=float((os.environ.get("ACC_BENCH_MIN_DAILY_OMNI_ACCURACY") or "0.69").strip() or "0.69"),
|
default=float((os.environ.get("ACC_BENCH_MIN_DAILY_OMNI_ACCURACY") or "0.67").strip() or "0.67"),
|
||||||
help="Fail when daily_omni_accuracy is strictly below this threshold (0–1). "
|
help="Fail when daily_omni_accuracy is strictly below this threshold (0–1). "
|
||||||
"Default baseline 0.69; override with env ACC_BENCH_MIN_DAILY_OMNI_ACCURACY or pass 0 to disable the floor.",
|
"Default baseline 0.67; override with env ACC_BENCH_MIN_DAILY_OMNI_ACCURACY or pass 0 to disable the floor.",
|
||||||
)
|
)
|
||||||
|
|
||||||
p.add_argument(
|
p.add_argument(
|
||||||
@@ -340,7 +340,7 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
|||||||
p.add_argument(
|
p.add_argument(
|
||||||
"--max-seed-tts-mean-wer",
|
"--max-seed-tts-mean-wer",
|
||||||
type=float,
|
type=float,
|
||||||
default=0.35,
|
default=0.5,
|
||||||
help="If set, fail when seed_tts_content_error_mean is strictly above this value.",
|
help="If set, fail when seed_tts_content_error_mean is strictly above this value.",
|
||||||
)
|
)
|
||||||
p.add_argument(
|
p.add_argument(
|
||||||
|
|||||||
@@ -218,7 +218,7 @@ def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> tuple[Image.I
|
|||||||
torch_dtype=torch.bfloat16,
|
torch_dtype=torch.bfloat16,
|
||||||
trust_remote_code=True,
|
trust_remote_code=True,
|
||||||
).to("cuda")
|
).to("cuda")
|
||||||
pipe.transformer.set_attention_backend("_flash_3_hub")
|
|
||||||
_diffusers_dummy_run(pipe)
|
_diffusers_dummy_run(pipe)
|
||||||
|
|
||||||
generator = torch.Generator(device="cuda").manual_seed(SEED)
|
generator = torch.Generator(device="cuda").manual_seed(SEED)
|
||||||
|
|||||||
@@ -1,218 +0,0 @@
|
|||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import base64
|
|
||||||
import copy
|
|
||||||
import io
|
|
||||||
import os
|
|
||||||
import tempfile
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
import requests
|
|
||||||
import yaml
|
|
||||||
from PIL import Image
|
|
||||||
|
|
||||||
from tests.e2e.accuracy.helpers import assert_images_pixel_close, assert_similarity, model_output_dir
|
|
||||||
from tests.helpers.mark import hardware_test
|
|
||||||
from tests.helpers.runtime import OmniServer
|
|
||||||
|
|
||||||
pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]
|
|
||||||
|
|
||||||
MODEL_NAME = "tencent/HunyuanImage-3.0-Instruct"
|
|
||||||
SEED = 42
|
|
||||||
NUM_INFERENCE_STEPS = 50
|
|
||||||
GUIDANCE_SCALE = 2.5
|
|
||||||
HEIGHT = 1024
|
|
||||||
WIDTH = 1024
|
|
||||||
PROMPT = "A brown and white dog is running on the grass."
|
|
||||||
MEAN_THRESHOLD = 3e-2
|
|
||||||
P99_THRESHOLD = 3e-1
|
|
||||||
SSIM_THRESHOLD = 0.97
|
|
||||||
PSNR_THRESHOLD = 30.0
|
|
||||||
|
|
||||||
_REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent
|
|
||||||
BASELINE_PATH = _REPO_ROOT / "tests" / "assets" / "hunyuan" / "hunyuan_baseline.png"
|
|
||||||
_OFFLINE_SCRIPT = _REPO_ROOT / "examples" / "offline_inference" / "hunyuan_image3" / "end2end.py"
|
|
||||||
|
|
||||||
# DiT-only deploy config with trust_remote_code (based on hunyuan_image3_dit.yaml).
|
|
||||||
_DEPLOY_CONFIG = {
|
|
||||||
"pipeline": "hunyuan_image3_dit",
|
|
||||||
"async_chunk": False,
|
|
||||||
"trust_remote_code": True,
|
|
||||||
"stages": [
|
|
||||||
{
|
|
||||||
"stage_id": 0,
|
|
||||||
"max_num_seqs": 1,
|
|
||||||
"gpu_memory_utilization": 0.9,
|
|
||||||
"enforce_eager": True,
|
|
||||||
"trust_remote_code": True,
|
|
||||||
"devices": "0,1,2,3", # set dynamically by _write_deploy_config
|
|
||||||
"vae_use_slicing": False,
|
|
||||||
"vae_use_tiling": False,
|
|
||||||
"parallel_config": {
|
|
||||||
"pipeline_parallel_size": 1,
|
|
||||||
"data_parallel_size": 1,
|
|
||||||
"tensor_parallel_size": 4,
|
|
||||||
"enable_expert_parallel": True,
|
|
||||||
"sequence_parallel_size": 1,
|
|
||||||
"ulysses_degree": 1,
|
|
||||||
"ring_degree": 1,
|
|
||||||
"cfg_parallel_size": 1,
|
|
||||||
"vae_patch_parallel_size": 1,
|
|
||||||
"use_hsdp": False,
|
|
||||||
"hsdp_shard_size": -1,
|
|
||||||
"hsdp_replicate_size": 1,
|
|
||||||
},
|
|
||||||
"default_sampling_params": {
|
|
||||||
"seed": SEED,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _model_name() -> str:
|
|
||||||
return os.environ.get("HUNYUAN_IMAGE3_MODEL", MODEL_NAME)
|
|
||||||
|
|
||||||
|
|
||||||
def _devices() -> str:
|
|
||||||
return os.environ.get("HUNYUAN_IMAGE3_DEVICES", "0,1,2,3")
|
|
||||||
|
|
||||||
|
|
||||||
def _write_deploy_config(path: Path) -> None:
|
|
||||||
config = copy.deepcopy(_DEPLOY_CONFIG)
|
|
||||||
devices = _devices()
|
|
||||||
config["stages"][0]["devices"] = devices
|
|
||||||
config["stages"][0]["parallel_config"]["tensor_parallel_size"] = len(devices.split(","))
|
|
||||||
path.write_text(yaml.dump(config, default_flow_style=False, sort_keys=False))
|
|
||||||
|
|
||||||
|
|
||||||
def _run_vllm_omni_hunyuan_image3_online(*, model: str, deploy_config: str, output_path: Path) -> Image.Image:
|
|
||||||
server_args = [
|
|
||||||
"--deploy-config",
|
|
||||||
deploy_config,
|
|
||||||
"--stage-init-timeout",
|
|
||||||
"300",
|
|
||||||
"--init-timeout",
|
|
||||||
"900",
|
|
||||||
"--enforce-eager",
|
|
||||||
"--trust-remote-code",
|
|
||||||
]
|
|
||||||
with OmniServer(model, server_args, use_omni=True) as omni_server:
|
|
||||||
response = requests.post(
|
|
||||||
f"http://{omni_server.host}:{omni_server.port}/v1/images/generations",
|
|
||||||
json={
|
|
||||||
"model": omni_server.model,
|
|
||||||
"prompt": PROMPT,
|
|
||||||
"size": f"{WIDTH}x{HEIGHT}",
|
|
||||||
"n": 1,
|
|
||||||
"response_format": "b64_json",
|
|
||||||
"num_inference_steps": NUM_INFERENCE_STEPS,
|
|
||||||
"guidance_scale": GUIDANCE_SCALE,
|
|
||||||
"seed": SEED,
|
|
||||||
"bot_task": "none",
|
|
||||||
"use_system_prompt": "en_unified",
|
|
||||||
},
|
|
||||||
timeout=600,
|
|
||||||
)
|
|
||||||
response.raise_for_status()
|
|
||||||
payload = response.json()
|
|
||||||
assert len(payload["data"]) == 1
|
|
||||||
image_bytes = base64.b64decode(payload["data"][0]["b64_json"])
|
|
||||||
image = Image.open(io.BytesIO(image_bytes)).convert("RGB")
|
|
||||||
image.load()
|
|
||||||
image.save(output_path)
|
|
||||||
return image
|
|
||||||
|
|
||||||
|
|
||||||
def _run_vllm_omni_hunyuan_image3_offline(*, model: str, deploy_config: str, output_path: Path) -> Image.Image:
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
|
|
||||||
output_dir = str(output_path.parent)
|
|
||||||
subprocess.run(
|
|
||||||
[
|
|
||||||
sys.executable,
|
|
||||||
str(_OFFLINE_SCRIPT),
|
|
||||||
"--modality",
|
|
||||||
"text2img",
|
|
||||||
"--deploy-config",
|
|
||||||
deploy_config,
|
|
||||||
"--prompts",
|
|
||||||
PROMPT,
|
|
||||||
"--output",
|
|
||||||
output_dir,
|
|
||||||
"--steps",
|
|
||||||
str(NUM_INFERENCE_STEPS),
|
|
||||||
"--guidance-scale",
|
|
||||||
str(GUIDANCE_SCALE),
|
|
||||||
"--seed",
|
|
||||||
str(SEED),
|
|
||||||
"--height",
|
|
||||||
str(HEIGHT),
|
|
||||||
"--width",
|
|
||||||
str(WIDTH),
|
|
||||||
"--bot-task",
|
|
||||||
"none",
|
|
||||||
"--sys-type",
|
|
||||||
"en_unified",
|
|
||||||
"--model",
|
|
||||||
model,
|
|
||||||
"--enforce-eager",
|
|
||||||
],
|
|
||||||
check=True,
|
|
||||||
)
|
|
||||||
images = sorted(Path(output_dir).glob("output_*.png"))
|
|
||||||
assert images, f"No output image found in {output_dir}"
|
|
||||||
image = Image.open(images[0]).convert("RGB")
|
|
||||||
image.load()
|
|
||||||
image.save(output_path)
|
|
||||||
return image
|
|
||||||
|
|
||||||
|
|
||||||
def _assert_against_baseline(image: Image.Image, label: str) -> None:
|
|
||||||
assert BASELINE_PATH.exists(), f"Baseline image not found at {BASELINE_PATH}"
|
|
||||||
baseline_image = Image.open(BASELINE_PATH).convert("RGB")
|
|
||||||
|
|
||||||
assert_images_pixel_close(
|
|
||||||
model_name=f"{MODEL_NAME} ({label} vs baseline)",
|
|
||||||
vllm_image=image,
|
|
||||||
diffusers_image=baseline_image,
|
|
||||||
mean_threshold=MEAN_THRESHOLD,
|
|
||||||
p99_threshold=P99_THRESHOLD,
|
|
||||||
)
|
|
||||||
assert_similarity(
|
|
||||||
model_name=f"{MODEL_NAME} ({label} vs baseline)",
|
|
||||||
vllm_image=image,
|
|
||||||
diffusers_image=baseline_image,
|
|
||||||
ssim_threshold=SSIM_THRESHOLD,
|
|
||||||
psnr_threshold=PSNR_THRESHOLD,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
|
||||||
def test_hunyuan_image3_pixel_accuracy_online(accuracy_artifact_root: Path) -> None:
|
|
||||||
model = _model_name()
|
|
||||||
output_dir = model_output_dir(accuracy_artifact_root, MODEL_NAME)
|
|
||||||
|
|
||||||
with tempfile.TemporaryDirectory() as tmpdir:
|
|
||||||
deploy_config_path = Path(tmpdir) / "deploy.yaml"
|
|
||||||
_write_deploy_config(deploy_config_path)
|
|
||||||
image = _run_vllm_omni_hunyuan_image3_online(
|
|
||||||
model=model, deploy_config=str(deploy_config_path), output_path=output_dir / "vllm_omni_online.png"
|
|
||||||
)
|
|
||||||
_assert_against_baseline(image, "online")
|
|
||||||
|
|
||||||
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
|
||||||
def test_hunyuan_image3_pixel_accuracy_offline(accuracy_artifact_root: Path) -> None:
|
|
||||||
model = _model_name()
|
|
||||||
output_dir = model_output_dir(accuracy_artifact_root, MODEL_NAME)
|
|
||||||
|
|
||||||
with tempfile.TemporaryDirectory() as tmpdir:
|
|
||||||
deploy_config_path = Path(tmpdir) / "deploy.yaml"
|
|
||||||
_write_deploy_config(deploy_config_path)
|
|
||||||
image = _run_vllm_omni_hunyuan_image3_offline(
|
|
||||||
model=model, deploy_config=str(deploy_config_path), output_path=output_dir / "vllm_omni_offline.png"
|
|
||||||
)
|
|
||||||
_assert_against_baseline(image, "offline")
|
|
||||||
@@ -106,7 +106,6 @@ def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> Image.Image:
|
|||||||
trust_remote_code=True,
|
trust_remote_code=True,
|
||||||
local_files_only=_local_files_only(model),
|
local_files_only=_local_files_only(model),
|
||||||
).to("cuda")
|
).to("cuda")
|
||||||
pipe.transformer.set_attention_backend("_flash_3_hub")
|
|
||||||
generator = torch.Generator(device="cuda").manual_seed(SEED)
|
generator = torch.Generator(device="cuda").manual_seed(SEED)
|
||||||
result = pipe( # pyright: ignore[reportCallIssue]
|
result = pipe( # pyright: ignore[reportCallIssue]
|
||||||
prompt=PROMPT,
|
prompt=PROMPT,
|
||||||
@@ -168,7 +167,6 @@ def _run_diffusers_qwen_image_2512(*, model: str, output_path: Path) -> Image.Im
|
|||||||
trust_remote_code=True,
|
trust_remote_code=True,
|
||||||
local_files_only=_local_files_only(model),
|
local_files_only=_local_files_only(model),
|
||||||
).to("cuda")
|
).to("cuda")
|
||||||
pipe.transformer.set_attention_backend("_flash_3_hub")
|
|
||||||
generator = torch.Generator(device="cuda").manual_seed(SEED_2512)
|
generator = torch.Generator(device="cuda").manual_seed(SEED_2512)
|
||||||
result = pipe( # pyright: ignore[reportCallIssue]
|
result = pipe( # pyright: ignore[reportCallIssue]
|
||||||
prompt=PROMPT_2512,
|
prompt=PROMPT_2512,
|
||||||
|
|||||||
@@ -1,273 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
"""End-to-end ModelOpt FP8 quality accuracy for video-gen DiTs.
|
|
||||||
|
|
||||||
Modeled on ``test_hunyuan_image3.py``'s ``test_quantized_dit_matches_bf16_accuracy``:
|
|
||||||
for each calibrated FP8 checkpoint the test generates one video with BF16 and one
|
|
||||||
with the FP8 DiT under the same seed, scores prompt-faithfulness with CLIP on the
|
|
||||||
middle frame, then asserts both ``CLIP >= absolute_floor`` and
|
|
||||||
``CLIP_drop <= drop_threshold``. The test is gated by an opt-in env var (model
|
|
||||||
paths are local) so it never runs in generic CI.
|
|
||||||
|
|
||||||
To run::
|
|
||||||
|
|
||||||
VIDEOGEN_RUN_QUANT_ACCURACY=1 \\
|
|
||||||
WAN22_A14B_BF16_MODEL=Wan-AI/Wan2.2-T2V-A14B-Diffusers \\
|
|
||||||
WAN22_A14B_FP8_MODEL=/path/to/wan22-a14b-modelopt-fp8 \\
|
|
||||||
HV15_BF16_MODEL=hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v \\
|
|
||||||
HV15_FP8_MODEL=/path/to/hv15-modelopt-fp8 \\
|
|
||||||
pytest -s -v tests/e2e/accuracy/test_videogen_modelopt_quant.py
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import gc
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import tempfile
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
import pytest
|
|
||||||
import torch
|
|
||||||
from PIL import Image
|
|
||||||
|
|
||||||
from tests.e2e.accuracy.helpers import CLIPScorer, model_output_dir
|
|
||||||
|
|
||||||
pytestmark = [pytest.mark.local_model, pytest.mark.diffusion]
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Tunables (override via env to suit local hardware / calibration budget)
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
SEED = 42
|
|
||||||
HEIGHT = int(os.environ.get("VIDEOGEN_QUANT_HEIGHT", "480"))
|
|
||||||
WIDTH = int(os.environ.get("VIDEOGEN_QUANT_WIDTH", "832"))
|
|
||||||
NUM_FRAMES = int(os.environ.get("VIDEOGEN_QUANT_NUM_FRAMES", "17"))
|
|
||||||
NUM_INFERENCE_STEPS = int(os.environ.get("VIDEOGEN_QUANT_STEPS", "20"))
|
|
||||||
|
|
||||||
WAN_PROMPT = "A red fox running through a snowy forest at sunrise, cinematic."
|
|
||||||
HV_PROMPT = "A red fox running through a snowy forest at sunrise, cinematic."
|
|
||||||
|
|
||||||
# CLIP gates: absolute floor + max drop from the BF16 reference.
|
|
||||||
CLIP_ABSOLUTE_FLOOR = float(os.environ.get("VIDEOGEN_QUANT_CLIP_FLOOR", "20.0"))
|
|
||||||
CLIP_DROP_FP8 = float(os.environ.get("VIDEOGEN_QUANT_CLIP_DROP_FP8", "7.0"))
|
|
||||||
|
|
||||||
RUN_ENV = "VIDEOGEN_RUN_QUANT_ACCURACY"
|
|
||||||
_TRUE = {"1", "true", "yes", "on"}
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class _Spec:
|
|
||||||
"""A model under test (BF16 + its FP8 variant)."""
|
|
||||||
|
|
||||||
family: str
|
|
||||||
bf16_env: str
|
|
||||||
fp8_env: str
|
|
||||||
prompt: str
|
|
||||||
guidance_scale: float
|
|
||||||
guidance_scale_2: float | None = None
|
|
||||||
boundary_ratio: float | None = None
|
|
||||||
|
|
||||||
|
|
||||||
WAN_SPEC = _Spec(
|
|
||||||
family="wan22_a14b_t2v",
|
|
||||||
bf16_env="WAN22_A14B_BF16_MODEL",
|
|
||||||
fp8_env="WAN22_A14B_FP8_MODEL",
|
|
||||||
prompt=WAN_PROMPT,
|
|
||||||
guidance_scale=3.5,
|
|
||||||
guidance_scale_2=4.0,
|
|
||||||
boundary_ratio=0.875,
|
|
||||||
)
|
|
||||||
|
|
||||||
HV_SPEC = _Spec(
|
|
||||||
family="hunyuanvideo15_t2v",
|
|
||||||
bf16_env="HV15_BF16_MODEL",
|
|
||||||
fp8_env="HV15_FP8_MODEL",
|
|
||||||
prompt=HV_PROMPT,
|
|
||||||
guidance_scale=6.0,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class _QuantCase:
|
|
||||||
name: str
|
|
||||||
spec: _Spec
|
|
||||||
clip_drop: float
|
|
||||||
|
|
||||||
|
|
||||||
def _cases() -> list[pytest.ParameterSet]:
|
|
||||||
cases = [
|
|
||||||
_QuantCase("wan_fp8", WAN_SPEC, CLIP_DROP_FP8),
|
|
||||||
_QuantCase("hv_fp8", HV_SPEC, CLIP_DROP_FP8),
|
|
||||||
]
|
|
||||||
params: list[pytest.ParameterSet] = []
|
|
||||||
opted_in = os.environ.get(RUN_ENV, "").lower() in _TRUE
|
|
||||||
for case in cases:
|
|
||||||
marks: list[Any] = []
|
|
||||||
if not opted_in:
|
|
||||||
marks.append(pytest.mark.skip(reason=f"Set {RUN_ENV}=1 to run video-gen quant accuracy."))
|
|
||||||
if not os.environ.get(case.spec.bf16_env):
|
|
||||||
marks.append(pytest.mark.skip(reason=f"Set {case.spec.bf16_env} to run {case.name}."))
|
|
||||||
if not os.environ.get(case.spec.fp8_env):
|
|
||||||
marks.append(pytest.mark.skip(reason=f"Set {case.spec.fp8_env} to run {case.name}."))
|
|
||||||
params.append(pytest.param(case, id=case.name, marks=marks))
|
|
||||||
return params
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
def _middle_frame(frames: list[Image.Image] | list[np.ndarray] | np.ndarray) -> Image.Image:
|
|
||||||
if isinstance(frames, np.ndarray):
|
|
||||||
idx = frames.shape[0] // 2
|
|
||||||
arr = frames[idx]
|
|
||||||
if arr.dtype != np.uint8:
|
|
||||||
arr = (arr * 255).clip(0, 255).astype(np.uint8)
|
|
||||||
return Image.fromarray(arr).convert("RGB")
|
|
||||||
if not frames:
|
|
||||||
raise AssertionError("Empty video output")
|
|
||||||
mid = frames[len(frames) // 2]
|
|
||||||
if isinstance(mid, np.ndarray):
|
|
||||||
if mid.dtype != np.uint8:
|
|
||||||
mid = (mid * 255).clip(0, 255).astype(np.uint8)
|
|
||||||
return Image.fromarray(mid).convert("RGB")
|
|
||||||
return mid.convert("RGB")
|
|
||||||
|
|
||||||
|
|
||||||
def _extract_frames(result) -> list[Image.Image]:
|
|
||||||
"""Pull a frame list out of an Omni text-to-video result."""
|
|
||||||
from vllm_omni.entrypoints.omni import OmniRequestOutput
|
|
||||||
|
|
||||||
if isinstance(result, list):
|
|
||||||
result = result[0] if result else None
|
|
||||||
if isinstance(result, OmniRequestOutput) and result.is_pipeline_output and result.request_output is not None:
|
|
||||||
result = result.request_output
|
|
||||||
images = getattr(result, "images", None)
|
|
||||||
if not images:
|
|
||||||
raise AssertionError("Pipeline output had no images")
|
|
||||||
if isinstance(images, list) and images and isinstance(images[0], tuple):
|
|
||||||
# (frames, fps)
|
|
||||||
images = images[0][0]
|
|
||||||
return list(images)
|
|
||||||
|
|
||||||
|
|
||||||
def _generate_video(
|
|
||||||
*,
|
|
||||||
model: str,
|
|
||||||
spec: _Spec,
|
|
||||||
quantization: str | None,
|
|
||||||
output_path: Path,
|
|
||||||
) -> tuple[Image.Image, float]:
|
|
||||||
"""Generate one video and return its middle frame + elapsed wall time."""
|
|
||||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
|
||||||
from vllm_omni.entrypoints.omni import Omni
|
|
||||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
|
||||||
from vllm_omni.platforms import current_omni_platform
|
|
||||||
|
|
||||||
kwargs: dict[str, Any] = dict(
|
|
||||||
model=model,
|
|
||||||
parallel_config=DiffusionParallelConfig(),
|
|
||||||
enforce_eager=True,
|
|
||||||
vae_use_tiling=True,
|
|
||||||
)
|
|
||||||
if quantization:
|
|
||||||
kwargs["quantization"] = quantization
|
|
||||||
if spec.boundary_ratio is not None:
|
|
||||||
kwargs["boundary_ratio"] = spec.boundary_ratio
|
|
||||||
omni = Omni(**kwargs)
|
|
||||||
|
|
||||||
try:
|
|
||||||
gen = torch.Generator(device=current_omni_platform.device_type).manual_seed(SEED)
|
|
||||||
sampling = dict(
|
|
||||||
height=HEIGHT,
|
|
||||||
width=WIDTH,
|
|
||||||
generator=gen,
|
|
||||||
guidance_scale=spec.guidance_scale,
|
|
||||||
num_inference_steps=NUM_INFERENCE_STEPS,
|
|
||||||
num_frames=NUM_FRAMES,
|
|
||||||
seed=SEED,
|
|
||||||
)
|
|
||||||
if spec.guidance_scale_2 is not None:
|
|
||||||
sampling["guidance_scale_2"] = spec.guidance_scale_2
|
|
||||||
|
|
||||||
import time as _time
|
|
||||||
|
|
||||||
t0 = _time.perf_counter()
|
|
||||||
out = omni.generate({"prompt": spec.prompt}, OmniDiffusionSamplingParams(**sampling))
|
|
||||||
elapsed = _time.perf_counter() - t0
|
|
||||||
|
|
||||||
frames = _extract_frames(out)
|
|
||||||
mid = _middle_frame(frames)
|
|
||||||
mid.save(output_path)
|
|
||||||
return mid, elapsed
|
|
||||||
finally:
|
|
||||||
del omni
|
|
||||||
gc.collect()
|
|
||||||
if torch.cuda.is_available():
|
|
||||||
torch.accelerator.empty_cache()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("case", _cases())
|
|
||||||
@pytest.mark.skipif(
|
|
||||||
not torch.cuda.is_available() or torch.accelerator.device_count() < 1,
|
|
||||||
reason="Needs at least 1 GPU.",
|
|
||||||
)
|
|
||||||
def test_quantized_videogen_dit_matches_bf16_clip(case: _QuantCase, accuracy_artifact_root: Path) -> None:
|
|
||||||
"""FP8 DiT should preserve prompt-aligned video quality vs BF16."""
|
|
||||||
out_dir = model_output_dir(accuracy_artifact_root, case.spec.family + "-quant")
|
|
||||||
|
|
||||||
with tempfile.TemporaryDirectory():
|
|
||||||
bf16_model = os.environ[case.spec.bf16_env]
|
|
||||||
quant_model = os.environ[case.spec.fp8_env]
|
|
||||||
|
|
||||||
bf16_frame, bf16_time = _generate_video(
|
|
||||||
model=bf16_model,
|
|
||||||
spec=case.spec,
|
|
||||||
quantization=None,
|
|
||||||
output_path=out_dir / "bf16.png",
|
|
||||||
)
|
|
||||||
quant_frame, quant_time = _generate_video(
|
|
||||||
model=quant_model,
|
|
||||||
spec=case.spec,
|
|
||||||
quantization="fp8",
|
|
||||||
output_path=out_dir / f"{case.name}.png",
|
|
||||||
)
|
|
||||||
|
|
||||||
clip = CLIPScorer()
|
|
||||||
bf16_clip = clip.score(bf16_frame, case.spec.prompt)
|
|
||||||
quant_clip = clip.score(quant_frame, case.spec.prompt)
|
|
||||||
clip_drop = bf16_clip - quant_clip
|
|
||||||
speedup = (bf16_time / quant_time) if quant_time else float("nan")
|
|
||||||
|
|
||||||
metrics = {
|
|
||||||
"case": case.name,
|
|
||||||
"family": case.spec.family,
|
|
||||||
"quantization": "fp8",
|
|
||||||
"bf16_model": bf16_model,
|
|
||||||
"quant_model": quant_model,
|
|
||||||
"prompt": case.spec.prompt,
|
|
||||||
"height": HEIGHT,
|
|
||||||
"width": WIDTH,
|
|
||||||
"num_frames": NUM_FRAMES,
|
|
||||||
"num_inference_steps": NUM_INFERENCE_STEPS,
|
|
||||||
"bf16_elapsed_s": bf16_time,
|
|
||||||
"quant_elapsed_s": quant_time,
|
|
||||||
"speedup_vs_bf16": speedup,
|
|
||||||
"bf16_clip_score": bf16_clip,
|
|
||||||
"quant_clip_score": quant_clip,
|
|
||||||
"clip_score_drop": clip_drop,
|
|
||||||
}
|
|
||||||
metrics_path = out_dir / f"{case.name}_metrics.json"
|
|
||||||
metrics_path.write_text(json.dumps(metrics, indent=2, sort_keys=True), encoding="utf-8")
|
|
||||||
|
|
||||||
print(f"\n[{case.name}] BF16 {bf16_time:.2f}s | FP8 {quant_time:.2f}s (x{speedup:.2f})")
|
|
||||||
print(f" CLIP BF16={bf16_clip:.2f} FP8={quant_clip:.2f} drop={clip_drop:.2f}")
|
|
||||||
print(f" metrics={metrics_path}")
|
|
||||||
|
|
||||||
assert quant_clip >= CLIP_ABSOLUTE_FLOOR, (
|
|
||||||
f"{case.name} CLIP below floor: got {quant_clip:.2f}, expected >= {CLIP_ABSOLUTE_FLOOR:.2f}"
|
|
||||||
)
|
|
||||||
assert clip_drop <= case.clip_drop, (
|
|
||||||
f"{case.name} CLIP drop too large: got {clip_drop:.2f}, expected <= {case.clip_drop:.2f}"
|
|
||||||
)
|
|
||||||
@@ -32,8 +32,7 @@ from vllm_omni import Omni
|
|||||||
|
|
||||||
# Selected by the nightly "TTS · Function Test" (-m "full_model and L4 and tts").
|
# Selected by the nightly "TTS · Function Test" (-m "full_model and L4 and tts").
|
||||||
# Without the tts + run-level marks these tests are collected but never run.
|
# Without the tts + run-level marks these tests are collected but never run.
|
||||||
# TODO: Fix this test
|
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
||||||
# pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
|
||||||
|
|
||||||
SAMPLE_RATE = 24_000 # All MOSS-TTS full variants output 24 kHz
|
SAMPLE_RATE = 24_000 # All MOSS-TTS full variants output 24 kHz
|
||||||
|
|
||||||
@@ -129,6 +128,7 @@ def delay_engine():
|
|||||||
return Omni(model=_DELAY_MODEL, stage_init_timeout=300)
|
return Omni(model=_DELAY_MODEL, stage_init_timeout=300)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_delay_english(delay_engine, ref_audio_path):
|
def test_moss_tts_delay_english(delay_engine, ref_audio_path):
|
||||||
"""MossTTSDelayModel: English voice_clone produces non-empty 24 kHz audio."""
|
"""MossTTSDelayModel: English voice_clone produces non-empty 24 kHz audio."""
|
||||||
@@ -140,6 +140,7 @@ def test_moss_tts_delay_english(delay_engine, ref_audio_path):
|
|||||||
assert not torch.all(audio == 0), "Audio is silence"
|
assert not torch.all(audio == 0), "Audio is silence"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_delay_chinese(delay_engine, ref_audio_path):
|
def test_moss_tts_delay_chinese(delay_engine, ref_audio_path):
|
||||||
"""MossTTSDelayModel: Chinese input produces non-empty audio."""
|
"""MossTTSDelayModel: Chinese input produces non-empty audio."""
|
||||||
@@ -151,6 +152,7 @@ def test_moss_tts_delay_chinese(delay_engine, ref_audio_path):
|
|||||||
assert not torch.all(audio == 0)
|
assert not torch.all(audio == 0)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_delay_deterministic(delay_engine, ref_audio_path):
|
def test_moss_tts_delay_deterministic(delay_engine, ref_audio_path):
|
||||||
"""MossTTSDelayModel: same seed yields identical waveforms."""
|
"""MossTTSDelayModel: same seed yields identical waveforms."""
|
||||||
@@ -162,6 +164,7 @@ def test_moss_tts_delay_deterministic(delay_engine, ref_audio_path):
|
|||||||
assert torch.allclose(audio1, audio2, atol=1e-4), "Waveforms differ across identical seeds"
|
assert torch.allclose(audio1, audio2, atol=1e-4), "Waveforms differ across identical seeds"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_delay_batch(delay_engine, ref_audio_path):
|
def test_moss_tts_delay_batch(delay_engine, ref_audio_path):
|
||||||
"""MossTTSDelayModel: batch of two requests each returns non-empty audio."""
|
"""MossTTSDelayModel: batch of two requests each returns non-empty audio."""
|
||||||
@@ -203,6 +206,7 @@ def realtime_engine():
|
|||||||
return Omni(model=_REALTIME_MODEL, stage_init_timeout=300)
|
return Omni(model=_REALTIME_MODEL, stage_init_timeout=300)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_realtime_english(realtime_engine, ref_audio_path):
|
def test_moss_tts_realtime_english(realtime_engine, ref_audio_path):
|
||||||
"""MossTTSRealtime: English voice_clone produces non-empty 24 kHz audio."""
|
"""MossTTSRealtime: English voice_clone produces non-empty 24 kHz audio."""
|
||||||
@@ -214,6 +218,7 @@ def test_moss_tts_realtime_english(realtime_engine, ref_audio_path):
|
|||||||
assert not torch.all(audio == 0), "Audio is silence"
|
assert not torch.all(audio == 0), "Audio is silence"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_realtime_deterministic(realtime_engine, ref_audio_path):
|
def test_moss_tts_realtime_deterministic(realtime_engine, ref_audio_path):
|
||||||
"""MossTTSRealtime: same seed yields identical waveforms."""
|
"""MossTTSRealtime: same seed yields identical waveforms."""
|
||||||
|
|||||||
@@ -120,6 +120,7 @@ def _collect_audio(omni: Omni, request: dict) -> tuple[torch.Tensor, int]:
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_nano_english(omni_runner: OmniRunner, ref_audio_path) -> None:
|
def test_moss_tts_nano_english(omni_runner: OmniRunner, ref_audio_path) -> None:
|
||||||
"""English TTS produces non-empty 48 kHz stereo audio."""
|
"""English TTS produces non-empty 48 kHz stereo audio."""
|
||||||
@@ -132,6 +133,7 @@ def test_moss_tts_nano_english(omni_runner: OmniRunner, ref_audio_path) -> None:
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_nano_chinese(omni_runner: OmniRunner, ref_audio_path) -> None:
|
def test_moss_tts_nano_chinese(omni_runner: OmniRunner, ref_audio_path) -> None:
|
||||||
"""Chinese TTS produces non-empty audio."""
|
"""Chinese TTS produces non-empty audio."""
|
||||||
@@ -144,6 +146,7 @@ def test_moss_tts_nano_chinese(omni_runner: OmniRunner, ref_audio_path) -> None:
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_nano_deterministic(omni_runner: OmniRunner, ref_audio_path) -> None:
|
def test_moss_tts_nano_deterministic(omni_runner: OmniRunner, ref_audio_path) -> None:
|
||||||
"""Same seed produces identical waveforms."""
|
"""Same seed produces identical waveforms."""
|
||||||
@@ -156,6 +159,7 @@ def test_moss_tts_nano_deterministic(omni_runner: OmniRunner, ref_audio_path) ->
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_nano_batch(omni_runner: OmniRunner, ref_audio_path) -> None:
|
def test_moss_tts_nano_batch(omni_runner: OmniRunner, ref_audio_path) -> None:
|
||||||
"""Batch of two requests returns audio for each."""
|
"""Batch of two requests returns audio for each."""
|
||||||
|
|||||||
@@ -24,11 +24,10 @@ from vllm import SamplingParams
|
|||||||
from tests.helpers.mark import hardware_test
|
from tests.helpers.mark import hardware_test
|
||||||
from vllm_omni import Omni
|
from vllm_omni import Omni
|
||||||
|
|
||||||
# H100-gated (8B). Picked up by the nightly "TTS · Function Test with H100"
|
# H100-gated (8B). Picked up by the nightly H100 omni job
|
||||||
# job (-m "full_model and H100 and tts"); the L4 MOSS-TTS tests land in the
|
# (-m "full_model and H100 and omni"); the tts mark keeps it consistent with
|
||||||
# sibling L4 tts lane instead.
|
# the other MOSS-TTS tests and any tts-scoped sweeps.
|
||||||
# TODO: Fix this test
|
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
||||||
# pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
|
||||||
|
|
||||||
SAMPLE_RATE = 24_000
|
SAMPLE_RATE = 24_000
|
||||||
REF_AUDIO_URL = "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS/main/assets/audio/reference_zh_1.wav"
|
REF_AUDIO_URL = "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS/main/assets/audio/reference_zh_1.wav"
|
||||||
@@ -102,6 +101,7 @@ def _collect_audio(omni: Omni, request: dict) -> tuple[torch.Tensor, int]:
|
|||||||
raise AssertionError("No stage outputs received")
|
raise AssertionError("No stage outputs received")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "H100"})
|
@hardware_test(res={"cuda": "H100"})
|
||||||
def test_moss_tts_v15_voice_clone(v15_engine, ref_audio_path):
|
def test_moss_tts_v15_voice_clone(v15_engine, ref_audio_path):
|
||||||
"""MOSS-TTS-v1.5 (8B): voice_clone produces non-empty 24 kHz audio."""
|
"""MOSS-TTS-v1.5 (8B): voice_clone produces non-empty 24 kHz audio."""
|
||||||
|
|||||||
@@ -1,90 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
"""E2E online serving test for DreamZero OpenPI websocket serving."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import os
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from tests.dreamzero import openpi_client_helper as openpi_client
|
|
||||||
from tests.helpers.mark import hardware_test
|
|
||||||
from tests.helpers.runtime import OmniServerParams, get_open_port
|
|
||||||
|
|
||||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
|
||||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
|
||||||
|
|
||||||
MODEL = "GEAR-Dreams/DreamZero-DROID"
|
|
||||||
|
|
||||||
|
|
||||||
test_params = [
|
|
||||||
OmniServerParams(
|
|
||||||
model=MODEL,
|
|
||||||
port=8091,
|
|
||||||
server_args=[
|
|
||||||
"--deploy-config",
|
|
||||||
"vllm_omni/deploy/dreamzero_tp1_cfg2.yaml",
|
|
||||||
"--enforce-eager",
|
|
||||||
"--disable-log-stats",
|
|
||||||
],
|
|
||||||
env_dict={
|
|
||||||
"ATTENTION_BACKEND": "torch",
|
|
||||||
"DIFFUSION_ATTENTION_BACKEND": "TORCH_SDPA",
|
|
||||||
"VLLM_DISABLE_COMPILE_CACHE": "1",
|
|
||||||
"MASTER_PORT": str(get_open_port()),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def _write_synthetic_video(path: Path, cv2_module, *, channel: int) -> None:
|
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
height, width, num_frames = 180, 320, 24
|
|
||||||
writer = cv2_module.VideoWriter(str(path), cv2_module.VideoWriter_fourcc(*"mp4v"), 15.0, (width, height))
|
|
||||||
if not writer.isOpened():
|
|
||||||
raise RuntimeError(f"Failed to open video writer for {path}")
|
|
||||||
try:
|
|
||||||
for frame_idx in range(num_frames):
|
|
||||||
frame = np.zeros((height, width, 3), dtype=np.uint8)
|
|
||||||
frame[..., channel] = (frame_idx * 7) % 255
|
|
||||||
frame[..., (channel + 1) % 3] = 64
|
|
||||||
writer.write(cv2_module.cvtColor(frame, cv2_module.COLOR_RGB2BGR))
|
|
||||||
finally:
|
|
||||||
writer.release()
|
|
||||||
|
|
||||||
|
|
||||||
def _write_synthetic_dreamzero_videos(client_mod, video_dir: Path) -> None:
|
|
||||||
for channel, file_name in enumerate(client_mod.CAMERA_FILES.values()):
|
|
||||||
_write_synthetic_video(video_dir / file_name, client_mod.cv2, channel=channel)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.full_model
|
|
||||||
@pytest.mark.diffusion
|
|
||||||
@pytest.mark.distributed_cuda
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=2)
|
|
||||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
|
||||||
def test_dreamzero_openpi_online(omni_server, tmp_path: Path) -> None:
|
|
||||||
try:
|
|
||||||
openpi_client.require_dependencies()
|
|
||||||
except ModuleNotFoundError as exc:
|
|
||||||
pytest.skip(str(exc))
|
|
||||||
|
|
||||||
video_dir = tmp_path / "dreamzero_videos"
|
|
||||||
_write_synthetic_dreamzero_videos(openpi_client, video_dir)
|
|
||||||
result = openpi_client.run_policy_session(
|
|
||||||
host=omni_server.host,
|
|
||||||
port=omni_server.port,
|
|
||||||
video_dir=video_dir,
|
|
||||||
session_id="dreamzero-online-e2e",
|
|
||||||
)
|
|
||||||
|
|
||||||
openpi_client.validate_session_result(result)
|
|
||||||
|
|
||||||
metadata = result["metadata"]
|
|
||||||
assert metadata["needs_session_id"] is True
|
|
||||||
assert metadata["needs_stereo_camera"] is False
|
|
||||||
assert tuple(metadata["image_resolution"]) == (180, 320)
|
|
||||||
@@ -1,183 +0,0 @@
|
|||||||
"""
|
|
||||||
Comprehensive tests of diffusion features that are available in online serving mode
|
|
||||||
and are supported by the following model:
|
|
||||||
- zai-org/GLM-Image (t2i and i2i)
|
|
||||||
|
|
||||||
Coverage (per mode):
|
|
||||||
- Baseline
|
|
||||||
- Cache-DiT
|
|
||||||
- Tensor-Parallel
|
|
||||||
- HSDP
|
|
||||||
- Sequence parallel
|
|
||||||
|
|
||||||
Topology for multi-GPU cases uses runtime deploy YAML (except TP)
|
|
||||||
so stage-1 parallel fields reach the diffusion worker without relying on CLI
|
|
||||||
|
|
||||||
assert_diffusion_response validates successful generation and the expected resolution.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import os
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from tests.helpers.mark import hardware_marks
|
|
||||||
from tests.helpers.media import generate_synthetic_image
|
|
||||||
from tests.helpers.runtime import (
|
|
||||||
OmniServer,
|
|
||||||
OmniServerParams,
|
|
||||||
OpenAIClientHandler,
|
|
||||||
dummy_messages_from_mix_data,
|
|
||||||
)
|
|
||||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
|
||||||
|
|
||||||
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
|
|
||||||
|
|
||||||
MODEL = os.environ.get("GLM_IMAGE_MODEL_PATH", "zai-org/GLM-Image")
|
|
||||||
|
|
||||||
T2I_PROMPT = "A Vincent van Gogh style impressionist painting."
|
|
||||||
I2I_PROMPT = "Transform this modern, geometric image into a Vincent van Gogh style impressionist painting."
|
|
||||||
NEGATIVE_PROMPT = "low quality, blurry, distorted, unnatural colors"
|
|
||||||
|
|
||||||
TWO_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}, num_cards=2)
|
|
||||||
|
|
||||||
GLM_DEPLOY = get_deploy_config_path("glm_image.yaml")
|
|
||||||
GLM_CACHEDIT_DEPLOY = modify_stage_config(
|
|
||||||
GLM_DEPLOY,
|
|
||||||
updates={
|
|
||||||
"stages": {
|
|
||||||
1: {"cache_backend": "cache_dit"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
GLM_TP_2_DEPLOY = modify_stage_config(
|
|
||||||
GLM_DEPLOY,
|
|
||||||
updates={
|
|
||||||
"stages": {
|
|
||||||
0: {"devices": "0", "tensor_parallel_size": 1},
|
|
||||||
1: {"devices": "0,1", "tensor_parallel_size": 2},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
GLM_HSDP_2_DEPLOY = modify_stage_config(
|
|
||||||
GLM_DEPLOY,
|
|
||||||
updates={
|
|
||||||
"stages": {
|
|
||||||
0: {"devices": "0"},
|
|
||||||
1: {
|
|
||||||
"devices": "0,1",
|
|
||||||
"parallel_config": {
|
|
||||||
"use_hsdp": True,
|
|
||||||
"hsdp_shard_size": 2,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
GLM_SP_2_DEPLOY = modify_stage_config(
|
|
||||||
GLM_DEPLOY,
|
|
||||||
updates={
|
|
||||||
"stages": {
|
|
||||||
0: {"devices": "0"},
|
|
||||||
1: {
|
|
||||||
"devices": "0,1",
|
|
||||||
"parallel_config": {
|
|
||||||
"ulysses_degree": 2,
|
|
||||||
"ring_degree": 1,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _get_diffusion_feature_cases(model: str):
|
|
||||||
return [
|
|
||||||
# Baseline: default deploy (stage 0 -> GPU 0, stage 1 -> GPU 1)
|
|
||||||
pytest.param(
|
|
||||||
OmniServerParams(
|
|
||||||
model=model,
|
|
||||||
stage_config_path=GLM_DEPLOY,
|
|
||||||
),
|
|
||||||
id="baseline",
|
|
||||||
marks=TWO_CARD_FEATURE_MARKS,
|
|
||||||
),
|
|
||||||
# Cache-DiT via runtime deploy patch (cache_backend -> engine_extras on stage 1)
|
|
||||||
pytest.param(
|
|
||||||
OmniServerParams(
|
|
||||||
model=model,
|
|
||||||
stage_config_path=GLM_CACHEDIT_DEPLOY,
|
|
||||||
),
|
|
||||||
id="cachedit",
|
|
||||||
marks=TWO_CARD_FEATURE_MARKS,
|
|
||||||
),
|
|
||||||
# Tensor-Parallel (2 GPUs on diffusion stage)
|
|
||||||
pytest.param(
|
|
||||||
OmniServerParams(
|
|
||||||
model=model,
|
|
||||||
stage_config_path=GLM_TP_2_DEPLOY,
|
|
||||||
),
|
|
||||||
id="tensor_parallel_2",
|
|
||||||
marks=TWO_CARD_FEATURE_MARKS,
|
|
||||||
),
|
|
||||||
# HSDP (2 GPUs on diffusion stage)
|
|
||||||
pytest.param(
|
|
||||||
OmniServerParams(
|
|
||||||
model=model,
|
|
||||||
stage_config_path=GLM_HSDP_2_DEPLOY,
|
|
||||||
),
|
|
||||||
id="hsdp_2",
|
|
||||||
marks=TWO_CARD_FEATURE_MARKS,
|
|
||||||
),
|
|
||||||
# Ulysses sequence parallel degree=2 (2 GPUs)
|
|
||||||
pytest.param(
|
|
||||||
OmniServerParams(
|
|
||||||
model=model,
|
|
||||||
stage_config_path=GLM_SP_2_DEPLOY,
|
|
||||||
),
|
|
||||||
id="sequence_parallel_2",
|
|
||||||
marks=TWO_CARD_FEATURE_MARKS,
|
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
MODES = ["t2i", "i2i"]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("mode", MODES)
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"omni_server",
|
|
||||||
_get_diffusion_feature_cases(MODEL),
|
|
||||||
indirect=True,
|
|
||||||
)
|
|
||||||
def test_glm_image(
|
|
||||||
omni_server: OmniServer,
|
|
||||||
mode: str,
|
|
||||||
openai_client: OpenAIClientHandler,
|
|
||||||
):
|
|
||||||
"""GLM-Image diffusion feature coverage for t2i and i2i on two H100s."""
|
|
||||||
|
|
||||||
if mode == "t2i":
|
|
||||||
messages = dummy_messages_from_mix_data(content_text=T2I_PROMPT)
|
|
||||||
else:
|
|
||||||
image_size = 1024
|
|
||||||
image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(image_size, image_size)['base64']}"
|
|
||||||
messages = dummy_messages_from_mix_data(
|
|
||||||
image_data_url=image_data_url,
|
|
||||||
content_text=I2I_PROMPT,
|
|
||||||
)
|
|
||||||
|
|
||||||
request_config = {
|
|
||||||
"model": omni_server.model,
|
|
||||||
"messages": messages,
|
|
||||||
"extra_body": {
|
|
||||||
"height": 1024,
|
|
||||||
"width": 1024,
|
|
||||||
"num_inference_steps": 2,
|
|
||||||
"guidance_scale": 1.5,
|
|
||||||
"true_cfg_scale": 4.0,
|
|
||||||
"negative_prompt": NEGATIVE_PROMPT,
|
|
||||||
"seed": 42,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
openai_client.send_diffusion_request(request_config)
|
|
||||||
@@ -1,388 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
"""End-to-end online tests for higgs-audio v3 against /v1/audio/speech.
|
|
||||||
|
|
||||||
Mirrors the higgs_audio_v2 test layout. Covers the plain-text-in / audio-out
|
|
||||||
happy path, a small concurrent burst (Stage 0 prefix caching has to coexist
|
|
||||||
with batching), the voice-clone path via ``ref_audio`` + ``ref_text``, the
|
|
||||||
upstream ``references[]`` cookbook alias added by
|
|
||||||
``normalize_references_alias`` in ``protocol/audio.py``, and the few
|
|
||||||
validator rejections that should remain 4xx.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import os
|
|
||||||
|
|
||||||
os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
|
||||||
os.environ.setdefault("VLLM_USE_DEEP_GEMM", "0")
|
|
||||||
os.environ.setdefault("VLLM_MOE_USE_DEEP_GEMM", "0")
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from tests.helpers.mark import hardware_test
|
|
||||||
from tests.helpers.media import load_test_audio_data_url
|
|
||||||
from tests.helpers.runtime import OmniServerParams
|
|
||||||
from tests.helpers.stage_config import get_deploy_config_path
|
|
||||||
|
|
||||||
MODEL = "bosonai/higgs-audio-v3-tts-4b"
|
|
||||||
STAGE_CONFIG = get_deploy_config_path("higgs_multimodal_qwen3.yaml")
|
|
||||||
SERVER_ARGS = ["--trust-remote-code", "--disable-log-stats"]
|
|
||||||
SERVER_ENV = {"VLLM_USE_DEEP_GEMM": "0", "VLLM_MOE_USE_DEEP_GEMM": "0"}
|
|
||||||
|
|
||||||
TEST_PARAMS = [
|
|
||||||
pytest.param(
|
|
||||||
OmniServerParams(
|
|
||||||
model=MODEL,
|
|
||||||
stage_config_path=STAGE_CONFIG,
|
|
||||||
server_args=SERVER_ARGS,
|
|
||||||
env_dict=SERVER_ENV,
|
|
||||||
),
|
|
||||||
id="higgs_audio_v3_plain_text",
|
|
||||||
)
|
|
||||||
]
|
|
||||||
|
|
||||||
DEFAULT_SPEECH_TIMEOUT_S = 180.0
|
|
||||||
# Floor for ~0.5 s of 24 kHz mono PCM_16: 24000 * 0.5 * 2 bytes ~= 24 KiB.
|
|
||||||
# A WAV header adds 44 bytes; conservative floor catches truncated /
|
|
||||||
# silence-only outputs without flagging short legitimate clips.
|
|
||||||
_MIN_AUDIO_BYTES = 20_000
|
|
||||||
|
|
||||||
# Reuse the shared TTS reference clip (clean ~5 s 24 kHz mono human speech)
|
|
||||||
# vendored under tests/assets/qwen3_tts/. Keeps a single WAV across TTS
|
|
||||||
# tests rather than duplicating asset bytes.
|
|
||||||
_REF_AUDIO_URL = load_test_audio_data_url("qwen3_tts/clone_2.wav")
|
|
||||||
_REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
|
|
||||||
class TestHiggsAudioV3OnlineHappyPath:
|
|
||||||
"""Plain-text -> audio happy paths against the live HTTP server."""
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_plain_text_wav(self, omni_server, openai_client) -> None:
|
|
||||||
"""Single non-streaming WAV request - canonical TTS happy path."""
|
|
||||||
openai_client.send_audio_speech_request(
|
|
||||||
{
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": "Hello world.",
|
|
||||||
"stream": False,
|
|
||||||
"response_format": "wav",
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_plain_text_with_max_new_tokens(self, omni_server, openai_client) -> None:
|
|
||||||
"""``max_new_tokens`` is one of the few extra fields the v3 validator accepts."""
|
|
||||||
openai_client.send_audio_speech_request(
|
|
||||||
{
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": "Innovation distinguishes between a leader and a follower.",
|
|
||||||
"stream": False,
|
|
||||||
"response_format": "wav",
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"max_new_tokens": 500,
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_concurrent_plain_text(self, omni_server, openai_client) -> None:
|
|
||||||
"""Three concurrent non-streaming requests - guards per-slot audio state and Stage-0 PC under batching."""
|
|
||||||
openai_client.send_audio_speech_request(
|
|
||||||
{
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": "It was the night before my birthday.",
|
|
||||||
"stream": False,
|
|
||||||
"response_format": "wav",
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
},
|
|
||||||
request_num=3,
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_plain_text_pcm_streaming(self, omni_server, openai_client) -> None:
|
|
||||||
"""Streaming PCM via talker2code2wav_async_chunk + the connector's codec_streaming path.
|
|
||||||
|
|
||||||
The deploy YAML pins ``async_chunk: true`` and ``codec_streaming: true`` so chunks
|
|
||||||
flow Stage 0 -> Stage 1 per AR loop. The Stage-1 codec honors
|
|
||||||
``meta.left_context_size`` and ``meta.right_holdback_size`` so per-chunk codec
|
|
||||||
windows stitch into a coherent PCM stream. The byte-count gate is the same as
|
|
||||||
the sync paths; per-chunk audio content is verified offline against Whisper.
|
|
||||||
|
|
||||||
NOTE: ``min_hnr_db=0.0`` sits below the typical speech-noise floor so the
|
|
||||||
check still catches catastrophic codec failure (silence, white noise, sample
|
|
||||||
scramble all give HNR << 0) while allowing for the sliding-window codec's
|
|
||||||
slightly-noisier-than-sync output. The default 1.0 dB threshold has only
|
|
||||||
~0.16 dB margin over measured single-request output (1.16 dB on L4), so
|
|
||||||
flake risk is too high there.
|
|
||||||
"""
|
|
||||||
openai_client.send_audio_speech_request(
|
|
||||||
{
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": "The quick brown fox jumps over the lazy dog.",
|
|
||||||
"stream": True,
|
|
||||||
"response_format": "pcm",
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
"min_hnr_db": 0.0,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_concurrent_pcm_streaming(self, omni_server, openai_client) -> None:
|
|
||||||
"""Three concurrent streaming requests - guards per-request frame cursors
|
|
||||||
in ``talker2code2wav_async_chunk`` and per-slot delay-pattern state under batched AR.
|
|
||||||
|
|
||||||
NOTE: ``min_hnr_db=-2.0`` is looser than the single-request streaming test
|
|
||||||
because batched AR adds per-request codec quality variance (DAC boundary
|
|
||||||
artifacts compounded across the 3-way batch); -2.0 dB gives ~1.6 dB margin
|
|
||||||
over the measured worst-of-3 on L4 (-0.42 dB) while still well above the
|
|
||||||
catastrophic-failure region.
|
|
||||||
"""
|
|
||||||
openai_client.send_audio_speech_request(
|
|
||||||
{
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": "She sells seashells by the seashore.",
|
|
||||||
"stream": True,
|
|
||||||
"response_format": "pcm",
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
"min_hnr_db": -2.0,
|
|
||||||
},
|
|
||||||
request_num=3,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
|
|
||||||
class TestHiggsAudioV3OnlineInlineControlTokens:
|
|
||||||
"""Inline control-token surface from the bosonai cookbook.
|
|
||||||
|
|
||||||
The model exposes four categories of ``<|category:value|>`` tokens
|
|
||||||
(emotion, style, prosody, sfx). Delivery tokens (emotion / style /
|
|
||||||
prosody speed-pitch-expressive) go at the START of input; positional
|
|
||||||
tokens (pause, sfx) go inline. SFX tokens must pair with their written
|
|
||||||
onomatopoeia.
|
|
||||||
|
|
||||||
These tests check the *serving surface* - the validator accepts the
|
|
||||||
payload, the engine produces audio, and the WAV has non-trivial size.
|
|
||||||
They do not assert audio quality (out of scope for CI).
|
|
||||||
"""
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_inline_emotion_and_expressive(self, omni_server, openai_client) -> None:
|
|
||||||
"""Delivery tokens (emotion + expressive_high) at the start of input."""
|
|
||||||
openai_client.send_audio_speech_request(
|
|
||||||
{
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": ("<|emotion:amusement|><|prosody:expressive_high|>Wait, that was actually hilarious."),
|
|
||||||
"stream": False,
|
|
||||||
"response_format": "wav",
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_inline_style_whispering(self, omni_server, openai_client) -> None:
|
|
||||||
"""Style token at the start - ``whispering``."""
|
|
||||||
openai_client.send_audio_speech_request(
|
|
||||||
{
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": "<|style:whispering|>It is just between you and me, alright?",
|
|
||||||
"stream": False,
|
|
||||||
"response_format": "wav",
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_inline_prosody_speed_and_pitch(self, omni_server, openai_client) -> None:
|
|
||||||
"""Two prosody tokens at the start - slow speed plus low pitch."""
|
|
||||||
openai_client.send_audio_speech_request(
|
|
||||||
{
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": (
|
|
||||||
"<|prosody:speed_slow|><|prosody:pitch_low|>The radar shows a storm approaching from the east."
|
|
||||||
),
|
|
||||||
"stream": False,
|
|
||||||
"response_format": "wav",
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_inline_pause_mid_text(self, omni_server, openai_client) -> None:
|
|
||||||
"""Positional pause token placed inline between two clauses."""
|
|
||||||
openai_client.send_audio_speech_request(
|
|
||||||
{
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": "Hold on a moment <|prosody:pause|> let me think about it.",
|
|
||||||
"stream": False,
|
|
||||||
"response_format": "wav",
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_inline_sfx_with_onomatopoeia(self, omni_server, openai_client) -> None:
|
|
||||||
"""SFX token paired with its written onomatopoeia (``<|sfx:laughter|>Hehe``)."""
|
|
||||||
openai_client.send_audio_speech_request(
|
|
||||||
{
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": (
|
|
||||||
"<|emotion:amusement|>I cannot believe that just happened. "
|
|
||||||
"<|sfx:laughter|>Hehe, I am still recovering from it."
|
|
||||||
),
|
|
||||||
"stream": False,
|
|
||||||
"response_format": "wav",
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
|
|
||||||
class TestHiggsAudioV3OnlineVoiceClone:
|
|
||||||
"""Voice clone via the two payload shapes the model serves."""
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_voice_clone_ref_audio_ref_text(self, omni_server, openai_client) -> None:
|
|
||||||
"""Canonical vllm-omni voice clone via ``ref_audio`` + ``ref_text``."""
|
|
||||||
openai_client.send_audio_speech_request(
|
|
||||||
{
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": "Hello world.",
|
|
||||||
"ref_audio": _REF_AUDIO_URL,
|
|
||||||
"ref_text": _REF_TEXT,
|
|
||||||
"stream": False,
|
|
||||||
"response_format": "wav",
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_voice_clone_references_alias(self, omni_server, openai_client) -> None:
|
|
||||||
"""BosonAI cookbook payload: ``references=[{audio_path, text}]``.
|
|
||||||
|
|
||||||
``normalize_references_alias`` (``protocol/audio.py``) is supposed to
|
|
||||||
translate the cookbook field into ``ref_audio`` / ``ref_text``; without
|
|
||||||
it the request would silently fall through to zero-shot synthesis.
|
|
||||||
"""
|
|
||||||
openai_client.send_audio_speech_request(
|
|
||||||
{
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": "Hello world.",
|
|
||||||
"references": [{"audio_path": _REF_AUDIO_URL, "text": _REF_TEXT}],
|
|
||||||
"stream": False,
|
|
||||||
"response_format": "wav",
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
|
|
||||||
class TestHiggsAudioV3OnlineValidatorRejections:
|
|
||||||
"""Out-of-scope shapes must come back as 4xx.
|
|
||||||
|
|
||||||
These cases use ``send_audio_speech_http_request`` (raw HTTP POST against
|
|
||||||
``/v1/audio/speech``) rather than the OpenAI SDK helper because:
|
|
||||||
|
|
||||||
- We need the failure-path matcher; the SDK helper asserts success.
|
|
||||||
- The cookbook ``references`` field is not in the OpenAI SDK schema, so
|
|
||||||
the SDK strips it before the request leaves the client and the server
|
|
||||||
sees a plain TTS request that returns 200. Raw HTTP keeps the JSON body
|
|
||||||
intact so the validator actually fires.
|
|
||||||
"""
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_rejects_empty_input(self, omni_server, openai_client) -> None:
|
|
||||||
"""Empty text input - validator should reject before reaching the engine."""
|
|
||||||
openai_client.send_audio_speech_http_request(
|
|
||||||
{
|
|
||||||
"json": {
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": "",
|
|
||||||
"response_format": "wav",
|
|
||||||
},
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"err_code": (400, 422),
|
|
||||||
"err_message": ("input", "empty"),
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_rejects_multi_reference_payload(self, omni_server, openai_client) -> None:
|
|
||||||
"""``references[]`` with more than one entry - multi-shot voice clone is not supported."""
|
|
||||||
openai_client.send_audio_speech_http_request(
|
|
||||||
{
|
|
||||||
"json": {
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": "Hello world.",
|
|
||||||
"references": [
|
|
||||||
{"audio_path": _REF_AUDIO_URL, "text": _REF_TEXT},
|
|
||||||
{"audio_path": _REF_AUDIO_URL, "text": _REF_TEXT},
|
|
||||||
],
|
|
||||||
"response_format": "wav",
|
|
||||||
},
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"err_code": (400, 422),
|
|
||||||
"err_message": "references",
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_rejects_conflicting_ref_audio_and_references(self, omni_server, openai_client) -> None:
|
|
||||||
"""``ref_audio`` and ``references`` cannot both be set to different values."""
|
|
||||||
openai_client.send_audio_speech_http_request(
|
|
||||||
{
|
|
||||||
"json": {
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": "Hello world.",
|
|
||||||
"ref_audio": _REF_AUDIO_URL,
|
|
||||||
"references": [{"audio_path": "https://example.com/other.wav"}],
|
|
||||||
"response_format": "wav",
|
|
||||||
},
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"err_code": (400, 422),
|
|
||||||
"err_message": "mutually exclusive",
|
|
||||||
}
|
|
||||||
)
|
|
||||||
@@ -30,8 +30,7 @@ from tests.helpers.mark import hardware_test
|
|||||||
from tests.helpers.runtime import OmniServerParams
|
from tests.helpers.runtime import OmniServerParams
|
||||||
from tests.helpers.stage_config import get_deploy_config_path
|
from tests.helpers.stage_config import get_deploy_config_path
|
||||||
|
|
||||||
# TODO: Fix this test
|
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
||||||
# pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
|
||||||
|
|
||||||
MODEL = "OpenMOSS-Team/MOSS-TTS-Realtime"
|
MODEL = "OpenMOSS-Team/MOSS-TTS-Realtime"
|
||||||
REF_AUDIO_URL = "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS/HEAD/assets/audio/reference_zh_1.wav"
|
REF_AUDIO_URL = "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS/HEAD/assets/audio/reference_zh_1.wav"
|
||||||
|
|||||||
@@ -9,40 +9,63 @@ import pytest
|
|||||||
from tests.helpers.mark import hardware_test
|
from tests.helpers.mark import hardware_test
|
||||||
from tests.helpers.media import generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video
|
from tests.helpers.media import generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video
|
||||||
from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data
|
from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data
|
||||||
from tests.helpers.stage_config import get_deploy_config_path
|
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||||
|
from vllm_omni.platforms import current_omni_platform
|
||||||
|
|
||||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||||
|
|
||||||
|
|
||||||
|
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
|
||||||
|
|
||||||
# Set VLLM_TEST_PD_MODE=1 to test PD disaggregation (follow-up — deploy overlay not yet migrated).
|
# Set VLLM_TEST_PD_MODE=1 to test PD disaggregation (follow-up — deploy overlay not yet migrated).
|
||||||
_USE_PD = os.environ.get("VLLM_TEST_PD_MODE", "0") == "1"
|
_USE_PD = os.environ.get("VLLM_TEST_PD_MODE", "0") == "1"
|
||||||
|
|
||||||
_MODEL = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
|
|
||||||
_CI_DEPLOY = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
|
_CI_DEPLOY = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
|
||||||
|
|
||||||
|
|
||||||
# For prefix caching checks against we enable it on the thinker and talker via CLI override
|
def get_chunk_config(config_path: str | None = None):
|
||||||
# and enable prompt token details so that we can determine if any tokens were cached.
|
"""Load the qwen3_omni CI deploy yaml with async_chunk modifications for streaming mode."""
|
||||||
# We also explicitly set block size so that we can make sure the cached token counts are a
|
if config_path is None:
|
||||||
# multiple of the block size.
|
config_path = _CI_DEPLOY
|
||||||
BLOCK_SIZE = 16
|
# TODO: remove this workaround once legacy `stage_args` path is deleted.
|
||||||
|
# The pipeline (qwen3_omni/pipeline.py) already wires
|
||||||
|
# thinker2talker_async_chunk / talker2code2wav_async_chunk on stage 0/1,
|
||||||
|
# so only async_chunk needs flipping. Writing nested `engine_args:` into
|
||||||
|
# the new-schema overlay trips _parse_stage_deploy's legacy branch and
|
||||||
|
# drops flat fields (load_format, max_num_seqs, ...).
|
||||||
|
return modify_stage_config(config_path, updates={"async_chunk": True})
|
||||||
|
|
||||||
|
|
||||||
|
# Platform-specific overrides live inside the new deploy yaml's ``platforms:``
|
||||||
|
# section, so a single ``_CI_DEPLOY`` path serves CUDA, ROCm, and XPU.
|
||||||
|
# TODO: re-add VLLM_TEST_PD_MODE branch once the PD-disaggregation deploy
|
||||||
|
# overlay has been migrated to the new schema (previously used the deleted
|
||||||
|
# ``qwen3_omni_moe_pd_ci.yaml`` stage-configs file).
|
||||||
|
if current_omni_platform.is_xpu():
|
||||||
|
stage_configs = [_CI_DEPLOY]
|
||||||
|
else: # CUDA + ROCm MI325 share the same deploy config
|
||||||
|
stage_configs = [get_chunk_config()]
|
||||||
|
|
||||||
|
# Create parameter combinations for model and stage config
|
||||||
test_params = [
|
test_params = [
|
||||||
pytest.param(
|
OmniServerParams(model=model, stage_config_path=stage_config) for model in models for stage_config in stage_configs
|
||||||
OmniServerParams(
|
]
|
||||||
model=_MODEL,
|
# For prefix caching, we enable it on the thinker (stage 0) via CLI override
|
||||||
stage_config_path=_CI_DEPLOY,
|
# and enable prompt token details so that we can determine if any tokens were cached.
|
||||||
use_stage_cli=True,
|
BLOCK_SIZE = 16
|
||||||
server_args=[
|
prefix_test_params = [
|
||||||
"--no-async-chunk",
|
OmniServerParams(
|
||||||
"--block-size",
|
model=model,
|
||||||
str(BLOCK_SIZE),
|
stage_config_path=_CI_DEPLOY,
|
||||||
"--stage-overrides",
|
server_args=[
|
||||||
'{"0": {"enable_prefix_caching": true}, "1": {"enable_prefix_caching": true}}',
|
"--block-size",
|
||||||
"--enable-prompt-tokens-details",
|
str(BLOCK_SIZE),
|
||||||
],
|
"--stage-overrides",
|
||||||
),
|
'{"0": {"enable_prefix_caching": true}}',
|
||||||
id="default",
|
"--enable-prompt-tokens-details",
|
||||||
|
],
|
||||||
)
|
)
|
||||||
|
for model in models
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
@@ -143,14 +166,45 @@ def test_text_to_text_001(omni_server, openai_client) -> None:
|
|||||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||||
|
|
||||||
|
|
||||||
def _run_prefix_cache_check(openai_client, request_config: dict):
|
@pytest.mark.advanced_model
|
||||||
"""Make two requests given a request config, and validate that:
|
@pytest.mark.core_model
|
||||||
1. The second request actually had cached tokens
|
@pytest.mark.omni
|
||||||
2. The number of cached tokens is divisible by the block size used in
|
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||||
test_params, because currently upstream vLLM does not cache partial
|
@pytest.mark.parametrize("omni_server", prefix_test_params, indirect=True)
|
||||||
blocks.
|
def test_thinker_prefix_caching(omni_server, openai_client, run_level) -> None:
|
||||||
"""
|
"""
|
||||||
openai_client.send_omni_request(request_config, request_num=1)[0]
|
Test thinker prefix caching by sending identical requests with an image (i.e.,
|
||||||
|
a large shared prefix) and verifying that the second request uses cached tokens
|
||||||
|
& produces the same output with greedy decoding.
|
||||||
|
|
||||||
|
NOTE: The reason that we check against logprobs instead of direct text here is that
|
||||||
|
the outputs may still diverge a bit even though we set the seed and temperature.
|
||||||
|
This is mostly because the GEMM algorithm may vary based on the input tensors dims.
|
||||||
|
Because of this, we don't check the logprobs if it's a dummy load, since in that case
|
||||||
|
the top logprobs will all be very close.
|
||||||
|
"""
|
||||||
|
seed = 10
|
||||||
|
img_res = generate_synthetic_image(224, 224, seed=seed)
|
||||||
|
image_data_url = f"data:image/jpeg;base64,{img_res['base64']}"
|
||||||
|
messages = dummy_messages_from_mix_data(
|
||||||
|
system_prompt=get_system_prompt(),
|
||||||
|
image_data_url=image_data_url,
|
||||||
|
content_text=get_prompt("text_image"),
|
||||||
|
)
|
||||||
|
|
||||||
|
top_k = 10
|
||||||
|
sampling_params = {"seed": seed, "temperature": 0, "max_tokens": 8, "logprobs": top_k}
|
||||||
|
request_config = {
|
||||||
|
"model": omni_server.model,
|
||||||
|
"messages": messages,
|
||||||
|
"stream": False,
|
||||||
|
"modalities": ["text"],
|
||||||
|
"logprobs": True,
|
||||||
|
"top_logprobs": top_k,
|
||||||
|
"sampling_params_list": [sampling_params] * 3,
|
||||||
|
}
|
||||||
|
|
||||||
|
uncached_response = openai_client.send_omni_request(request_config, request_num=1)[0]
|
||||||
cached_response = openai_client.send_omni_request(request_config, request_num=1)[0]
|
cached_response = openai_client.send_omni_request(request_config, request_num=1)[0]
|
||||||
|
|
||||||
# Ensure that we have a prefix cache hit on the second request and that only the last
|
# Ensure that we have a prefix cache hit on the second request and that only the last
|
||||||
@@ -163,66 +217,16 @@ def _run_prefix_cache_check(openai_client, request_config: dict):
|
|||||||
assert num_cached_tokens % BLOCK_SIZE == 0
|
assert num_cached_tokens % BLOCK_SIZE == 0
|
||||||
assert (num_cached_tokens + num_uncached_tokens) == num_prompt_tokens
|
assert (num_cached_tokens + num_uncached_tokens) == num_prompt_tokens
|
||||||
|
|
||||||
|
# Ensure that we have logprobs and tokens were generated for both requests
|
||||||
|
assert uncached_response.logprobs is not None
|
||||||
|
assert cached_response.logprobs is not None
|
||||||
|
n_tokens = min(len(uncached_response.logprobs), len(cached_response.logprobs))
|
||||||
|
assert n_tokens > 0
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
if run_level == "advanced_model":
|
||||||
@pytest.mark.core_model
|
# For each token index where both responses have an output, ensure that the greedy token
|
||||||
@pytest.mark.omni
|
# predicted in the uncached case is in the top k logprobs for the cached case
|
||||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
for idx in range(n_tokens):
|
||||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
greedy_token = uncached_response.logprobs[idx].token
|
||||||
def test_thinker_prefix_caching_text_output(omni_server, openai_client) -> None:
|
cached_top_k = {lp.token for lp in cached_response.logprobs[idx].top_logprobs}
|
||||||
"""
|
assert greedy_token in cached_top_k
|
||||||
Test thinker prefix caching by sending identical requests with an image (i.e.,
|
|
||||||
a large shared prefix) and verifying that the second request uses cached tokens
|
|
||||||
& produces the same output with greedy decoding.
|
|
||||||
|
|
||||||
NOTE: Checking the output of prefix caching directly can be a bit unstable
|
|
||||||
due to slight numerical differences as a result of running different scheduled
|
|
||||||
sequence lengths. As such, for E2E tests on prefix cache, we only check the cached
|
|
||||||
token count and not the output, since the omni tensor cache has solid unit tests,
|
|
||||||
and the core prefix cache algorithm is largely tested by upstream vLLM.
|
|
||||||
"""
|
|
||||||
img_res = generate_synthetic_image(224, 224)
|
|
||||||
image_data_url = f"data:image/jpeg;base64,{img_res['base64']}"
|
|
||||||
messages = dummy_messages_from_mix_data(
|
|
||||||
system_prompt=get_system_prompt(),
|
|
||||||
image_data_url=image_data_url,
|
|
||||||
content_text=get_prompt("text_image"),
|
|
||||||
)
|
|
||||||
|
|
||||||
request_config = {
|
|
||||||
"model": omni_server.model,
|
|
||||||
"messages": messages,
|
|
||||||
"stream": False,
|
|
||||||
"modalities": ["text"],
|
|
||||||
}
|
|
||||||
_run_prefix_cache_check(openai_client, request_config)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.omni
|
|
||||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
|
||||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
|
||||||
def test_thinker_prefix_caching_audio_output(omni_server, openai_client) -> None:
|
|
||||||
"""
|
|
||||||
Verify that thinker prefix caching does not hang when the request
|
|
||||||
produces audio output (text + audio modalities). Sends two identical
|
|
||||||
requests so the second exercises the prefix-cached path through the
|
|
||||||
full thinker -> talker -> code2wav pipeline.
|
|
||||||
|
|
||||||
Regression test for https://github.com/vllm-project/vllm-omni/issues/3510
|
|
||||||
"""
|
|
||||||
messages = dummy_messages_from_mix_data(
|
|
||||||
system_prompt=get_system_prompt(),
|
|
||||||
content_text=get_prompt(),
|
|
||||||
)
|
|
||||||
request_config = {
|
|
||||||
"model": omni_server.model,
|
|
||||||
"messages": messages,
|
|
||||||
"stream": True,
|
|
||||||
"stream_options": {
|
|
||||||
"include_usage": True,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
_run_prefix_cache_check(openai_client, request_config)
|
|
||||||
@@ -1,97 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
"""
|
|
||||||
E2E Online tests for VoxCPM2 with text input and audio output.
|
|
||||||
|
|
||||||
These tests verify the /v1/audio/speech endpoint works correctly with
|
|
||||||
actual model inference, not mocks. VoxCPM2 zero-shot needs only ``input``
|
|
||||||
and a placeholder ``voice`` (``default``); no ``task_type`` / ``ref_text``.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import os
|
|
||||||
|
|
||||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from tests.helpers.mark import hardware_test
|
|
||||||
from tests.helpers.runtime import OmniServerParams
|
|
||||||
from tests.helpers.stage_config import get_deploy_config_path
|
|
||||||
|
|
||||||
MODEL = "openbmb/VoxCPM2"
|
|
||||||
DEFAULT_AUDIO_SPEECH_TIMEOUT_S = 300.0
|
|
||||||
MAX_CONCURRENT = 4
|
|
||||||
|
|
||||||
# ~0.5 s of 48 kHz mono PCM_16 in WAV (~48k payload + header).
|
|
||||||
_MIN_AUDIO_BYTES = 40_000
|
|
||||||
|
|
||||||
|
|
||||||
def get_prompt(prompt_type="text"):
|
|
||||||
"""English prompt for zero-shot TTS (Whisper-friendly for advanced_model accuracy)."""
|
|
||||||
prompts = {
|
|
||||||
"text": "The weather is nice today, perfect for a walk in the park.",
|
|
||||||
}
|
|
||||||
return prompts.get(prompt_type, prompts["text"])
|
|
||||||
|
|
||||||
|
|
||||||
tts_server_params = [
|
|
||||||
pytest.param(
|
|
||||||
OmniServerParams(
|
|
||||||
model=MODEL,
|
|
||||||
stage_config_path=get_deploy_config_path("voxcpm2.yaml"),
|
|
||||||
server_args=["--trust-remote-code", "--disable-log-stats"],
|
|
||||||
),
|
|
||||||
id="voxcpm2",
|
|
||||||
)
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
|
||||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
|
||||||
def test_text_to_audio_001(omni_server, openai_client) -> None:
|
|
||||||
"""
|
|
||||||
Test zero-shot text-to-audio via OpenAI API.
|
|
||||||
Deploy Setting: voxcpm2.yaml
|
|
||||||
Input Modal: text
|
|
||||||
Output Modal: audio
|
|
||||||
Input Setting: stream=False
|
|
||||||
Datasets: few requests (max_num_seqs=4)
|
|
||||||
"""
|
|
||||||
request_config = {
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": get_prompt(),
|
|
||||||
"stream": False,
|
|
||||||
"timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S,
|
|
||||||
"response_format": "wav",
|
|
||||||
"voice": "default",
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
}
|
|
||||||
openai_client.send_audio_speech_request(request_config, request_num=MAX_CONCURRENT)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
|
||||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
|
||||||
def test_text_to_audio_002(omni_server, openai_client) -> None:
|
|
||||||
"""
|
|
||||||
Test zero-shot streaming text-to-audio via OpenAI API.
|
|
||||||
Deploy Setting: voxcpm2.yaml
|
|
||||||
Input Modal: text
|
|
||||||
Output Modal: audio
|
|
||||||
Input Setting: stream=True
|
|
||||||
Datasets: single request
|
|
||||||
"""
|
|
||||||
request_config = {
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": get_prompt(),
|
|
||||||
"stream": True,
|
|
||||||
"timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S,
|
|
||||||
"response_format": "wav",
|
|
||||||
"voice": "default",
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
}
|
|
||||||
openai_client.send_audio_speech_request(request_config)
|
|
||||||
@@ -1,95 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
"""
|
|
||||||
E2E Online expansion tests for VoxCPM2 (voice clone, PCM, concurrency).
|
|
||||||
|
|
||||||
Voice clone uses ``ref_audio`` only (no ``ref_text``). Reference clip is
|
|
||||||
vendored under tests/assets/qwen3_tts/clone_2.wav — see test_qwen3_tts_base.py.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import os
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from tests.helpers.mark import hardware_test
|
|
||||||
from tests.helpers.media import load_test_audio_data_url
|
|
||||||
from tests.helpers.runtime import OmniServerParams
|
|
||||||
from tests.helpers.stage_config import get_deploy_config_path
|
|
||||||
|
|
||||||
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
|
||||||
|
|
||||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
|
||||||
|
|
||||||
MODEL = "openbmb/VoxCPM2"
|
|
||||||
DEFAULT_AUDIO_SPEECH_TIMEOUT_S = 300.0
|
|
||||||
_MIN_AUDIO_BYTES = 40_000
|
|
||||||
MAX_CONCURRENT = 4
|
|
||||||
|
|
||||||
REF_AUDIO_URL = load_test_audio_data_url("qwen3_tts/clone_2.wav")
|
|
||||||
|
|
||||||
|
|
||||||
def get_prompt(prompt_type="text"):
|
|
||||||
prompts = {
|
|
||||||
"text": "The weather is nice today, perfect for a walk in the park.",
|
|
||||||
}
|
|
||||||
return prompts.get(prompt_type, prompts["text"])
|
|
||||||
|
|
||||||
|
|
||||||
tts_server_params = [
|
|
||||||
pytest.param(
|
|
||||||
OmniServerParams(
|
|
||||||
model=MODEL,
|
|
||||||
stage_config_path=get_deploy_config_path("voxcpm2.yaml"),
|
|
||||||
server_args=["--trust-remote-code", "--disable-log-stats"],
|
|
||||||
),
|
|
||||||
id="voxcpm2",
|
|
||||||
)
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
|
||||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
|
||||||
def test_voice_clone_streaming_001(omni_server, openai_client) -> None:
|
|
||||||
"""
|
|
||||||
Test voice-clone text-to-audio via OpenAI API.
|
|
||||||
Deploy Setting: voxcpm2.yaml
|
|
||||||
Input Modal: text + ref_audio
|
|
||||||
Output Modal: audio
|
|
||||||
Input Setting: stream=True
|
|
||||||
Datasets: few requests (max_num_seqs=4)
|
|
||||||
"""
|
|
||||||
request_config = {
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": get_prompt(),
|
|
||||||
"stream": True,
|
|
||||||
"timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S,
|
|
||||||
"response_format": "wav",
|
|
||||||
"voice": "default",
|
|
||||||
"ref_audio": REF_AUDIO_URL,
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
}
|
|
||||||
openai_client.send_audio_speech_request(request_config, request_num=MAX_CONCURRENT)
|
|
||||||
|
|
||||||
|
|
||||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
|
||||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
|
||||||
def test_response_format_001(omni_server, openai_client) -> None:
|
|
||||||
"""
|
|
||||||
Test voice-clone non-stream PCM output via OpenAI API.
|
|
||||||
Deploy Setting: voxcpm2.yaml
|
|
||||||
Input Modal: text + ref_audio
|
|
||||||
Output Modal: audio (pcm)
|
|
||||||
Input Setting: stream=False
|
|
||||||
Datasets: single request
|
|
||||||
"""
|
|
||||||
request_config = {
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": get_prompt(),
|
|
||||||
"response_format": "pcm",
|
|
||||||
"stream": False,
|
|
||||||
"timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S,
|
|
||||||
"voice": "default",
|
|
||||||
"ref_audio": REF_AUDIO_URL,
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
}
|
|
||||||
openai_client.send_audio_speech_request(request_config)
|
|
||||||
@@ -427,7 +427,6 @@ async def _enqueue_add_request(
|
|||||||
sampling_params_list=sampling_params_list,
|
sampling_params_list=sampling_params_list,
|
||||||
final_stage_id=final_stage_id,
|
final_stage_id=final_stage_id,
|
||||||
preprocess_ms=0.0,
|
preprocess_ms=0.0,
|
||||||
request_timestamp=time.time(),
|
|
||||||
enqueue_ts=time.perf_counter(),
|
enqueue_ts=time.perf_counter(),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
@@ -985,7 +984,6 @@ async def test_handle_streaming_update_passes_prompt_text_to_stage_pool() -> Non
|
|||||||
sampling_params_list=[_sampling_params()],
|
sampling_params_list=[_sampling_params()],
|
||||||
final_stage_id=0,
|
final_stage_id=0,
|
||||||
preprocess_ms=0.0,
|
preprocess_ms=0.0,
|
||||||
request_timestamp=time.time(),
|
|
||||||
enqueue_ts=time.perf_counter(),
|
enqueue_ts=time.perf_counter(),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -156,7 +156,7 @@ class FakeAsyncOmni:
|
|||||||
self.captured_prompt = None
|
self.captured_prompt = None
|
||||||
self._images = images or [Image.new("RGB", (64, 64), color="green")]
|
self._images = images or [Image.new("RGB", (64, 64), color="green")]
|
||||||
|
|
||||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None):
|
||||||
if sampling_params_list is not None:
|
if sampling_params_list is not None:
|
||||||
self.captured_sampling_params_list = sampling_params_list
|
self.captured_sampling_params_list = sampling_params_list
|
||||||
else:
|
else:
|
||||||
@@ -254,7 +254,7 @@ def async_omni_test_client():
|
|||||||
self._images = [Image.new("RGB", (64, 64), color="green")]
|
self._images = [Image.new("RGB", (64, 64), color="green")]
|
||||||
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
||||||
|
|
||||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None):
|
||||||
if sampling_params_list is not None:
|
if sampling_params_list is not None:
|
||||||
self.captured_sampling_params_list = sampling_params_list
|
self.captured_sampling_params_list = sampling_params_list
|
||||||
else:
|
else:
|
||||||
@@ -318,7 +318,7 @@ def async_omni_rgba_test_client():
|
|||||||
self._images = [Image.new("RGBA", (64, 64), color=(0, 255, 0, 128))]
|
self._images = [Image.new("RGBA", (64, 64), color=(0, 255, 0, 128))]
|
||||||
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
||||||
|
|
||||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None):
|
||||||
if sampling_params_list is not None:
|
if sampling_params_list is not None:
|
||||||
self.captured_sampling_params_list = sampling_params_list
|
self.captured_sampling_params_list = sampling_params_list
|
||||||
else:
|
else:
|
||||||
@@ -382,7 +382,7 @@ def async_omni_stage_configs_only_client():
|
|||||||
self._images = [Image.new("RGB", (64, 64), color="green")]
|
self._images = [Image.new("RGB", (64, 64), color="green")]
|
||||||
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
||||||
|
|
||||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None):
|
||||||
if sampling_params_list is not None:
|
if sampling_params_list is not None:
|
||||||
self.captured_sampling_params_list = sampling_params_list
|
self.captured_sampling_params_list = sampling_params_list
|
||||||
else:
|
else:
|
||||||
@@ -442,7 +442,7 @@ def streaming_image_edit_client():
|
|||||||
self.captured_prompt = None
|
self.captured_prompt = None
|
||||||
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
||||||
|
|
||||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None):
|
||||||
self.captured_prompt = prompt
|
self.captured_prompt = prompt
|
||||||
self.captured_sampling_params_list = sampling_params_list or [sampling_params]
|
self.captured_sampling_params_list = sampling_params_list or [sampling_params]
|
||||||
assert self.captured_sampling_params_list[0].output_kind == RequestOutputKind.DELTA
|
assert self.captured_sampling_params_list[0].output_kind == RequestOutputKind.DELTA
|
||||||
@@ -696,7 +696,7 @@ def test_generate_images_async_omni_glm_image_sets_stage0_max_tokens():
|
|||||||
self._images = [Image.new("RGB", (64, 64), color="green")]
|
self._images = [Image.new("RGB", (64, 64), color="green")]
|
||||||
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
||||||
|
|
||||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None):
|
||||||
self.captured_sampling_params_list = (
|
self.captured_sampling_params_list = (
|
||||||
sampling_params_list if sampling_params_list is not None else [sampling_params]
|
sampling_params_list if sampling_params_list is not None else [sampling_params]
|
||||||
)
|
)
|
||||||
@@ -815,9 +815,7 @@ def test_image_edits_streaming_returns_ar_delta_then_image(streaming_image_edit_
|
|||||||
|
|
||||||
|
|
||||||
def test_image_edits_streaming_ar_delta_chunks_include_index(streaming_image_edit_client):
|
def test_image_edits_streaming_ar_delta_chunks_include_index(streaming_image_edit_client):
|
||||||
async def generate_multi_output_delta(
|
async def generate_multi_output_delta(prompt, request_id, sampling_params=None, sampling_params_list=None):
|
||||||
prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs
|
|
||||||
):
|
|
||||||
yield MockStageResult(stage_id=0, final_output_type="text", texts=["first", "second"])
|
yield MockStageResult(stage_id=0, final_output_type="text", texts=["first", "second"])
|
||||||
yield MockStageResult(
|
yield MockStageResult(
|
||||||
stage_id=1,
|
stage_id=1,
|
||||||
@@ -845,7 +843,7 @@ def test_image_edits_streaming_ar_delta_chunks_include_index(streaming_image_edi
|
|||||||
|
|
||||||
|
|
||||||
def test_image_edits_streaming_errors_without_final_image(streaming_image_edit_client):
|
def test_image_edits_streaming_errors_without_final_image(streaming_image_edit_client):
|
||||||
async def generate_without_image(prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
async def generate_without_image(prompt, request_id, sampling_params=None, sampling_params_list=None):
|
||||||
yield MockStageResult(stage_id=0, final_output_type="text", text="recap")
|
yield MockStageResult(stage_id=0, final_output_type="text", text="recap")
|
||||||
|
|
||||||
streaming_image_edit_client.app.state.engine_client.generate = generate_without_image
|
streaming_image_edit_client.app.state.engine_client.generate = generate_without_image
|
||||||
@@ -868,7 +866,7 @@ def test_image_edits_streaming_errors_without_final_image(streaming_image_edit_c
|
|||||||
|
|
||||||
|
|
||||||
def test_image_edits_streaming_errors_on_empty_final_image(streaming_image_edit_client):
|
def test_image_edits_streaming_errors_on_empty_final_image(streaming_image_edit_client):
|
||||||
async def generate_empty_image(prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
async def generate_empty_image(prompt, request_id, sampling_params=None, sampling_params_list=None):
|
||||||
yield MockStageResult(stage_id=0, final_output_type="text", text="recap")
|
yield MockStageResult(stage_id=0, final_output_type="text", text="recap")
|
||||||
yield MockStageResult(stage_id=1, final_output_type="image", images=[])
|
yield MockStageResult(stage_id=1, final_output_type="image", images=[])
|
||||||
|
|
||||||
|
|||||||
@@ -2,11 +2,6 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
"""
|
"""
|
||||||
E2E online tests for Qwen3-Omni /v1/realtime WebSocket (streaming PCM in, audio out).
|
E2E online tests for Qwen3-Omni /v1/realtime WebSocket (streaming PCM in, audio out).
|
||||||
|
|
||||||
Three scenarios:
|
|
||||||
- Ready CI: async_chunk on, smoke only (no send delay, no accuracy check).
|
|
||||||
- Merge CI: async_chunk on + send delay, full accuracy check.
|
|
||||||
- Merge CI: async_chunk off, no send delay, full accuracy check.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -39,32 +34,18 @@ REALTIME_SYNTH_PHRASE_TEXT = (
|
|||||||
"Translate into Chinese: Beijing is the Capital of China. It is the center of culture and politics"
|
"Translate into Chinese: Beijing is the Capital of China. It is the center of culture and politics"
|
||||||
)
|
)
|
||||||
|
|
||||||
# Simulate realtime upload pacing (``openai_realtime_client.py --send-delay-ms``).
|
# The new-schema CI overlay bakes in async_chunk: False and covers CUDA/ROCm/XPU
|
||||||
SEND_DELAY_MS = 200
|
# via its ``platforms:`` section, so one path serves all three.
|
||||||
|
|
||||||
# CI overlay bakes in async_chunk: False and covers CUDA/ROCm/XPU via ``platforms:``.
|
|
||||||
default_stage_config = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
|
default_stage_config = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
|
||||||
|
|
||||||
realtime_sync_server_params = [
|
realtime_server_params = [
|
||||||
pytest.param(
|
pytest.param(
|
||||||
OmniServerParams(
|
OmniServerParams(
|
||||||
model=MODEL,
|
model=MODEL,
|
||||||
stage_config_path=default_stage_config,
|
stage_config_path=default_stage_config,
|
||||||
use_stage_cli=True,
|
use_stage_cli=True,
|
||||||
),
|
),
|
||||||
id="sync",
|
id="default",
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
realtime_async_chunk_server_params = [
|
|
||||||
pytest.param(
|
|
||||||
OmniServerParams(
|
|
||||||
model=MODEL,
|
|
||||||
stage_config_path=default_stage_config,
|
|
||||||
use_stage_cli=True,
|
|
||||||
server_args=["--async-chunk"],
|
|
||||||
),
|
|
||||||
id="async_chunk",
|
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -99,7 +80,6 @@ async def _run_realtime_audio_roundtrip(
|
|||||||
pcm16: bytes,
|
pcm16: bytes,
|
||||||
*,
|
*,
|
||||||
chunk_ms: int = 100,
|
chunk_ms: int = 100,
|
||||||
send_delay_ms: int = 0,
|
|
||||||
) -> dict:
|
) -> dict:
|
||||||
uri = f"ws://{host}:{port}/v1/realtime"
|
uri = f"ws://{host}:{port}/v1/realtime"
|
||||||
incremental: list[bytes] = []
|
incremental: list[bytes] = []
|
||||||
@@ -125,8 +105,6 @@ async def _run_realtime_audio_roundtrip(
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
if send_delay_ms > 0:
|
|
||||||
await asyncio.sleep(send_delay_ms / 1000.0)
|
|
||||||
|
|
||||||
await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True}))
|
await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True}))
|
||||||
|
|
||||||
@@ -178,92 +156,26 @@ async def _run_realtime_audio_roundtrip(
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def _synthetic_pcm16_input() -> bytes:
|
|
||||||
syn = generate_synthetic_audio(
|
|
||||||
10,
|
|
||||||
1,
|
|
||||||
sample_rate=16000,
|
|
||||||
phrase_text=REALTIME_SYNTH_PHRASE_TEXT,
|
|
||||||
)
|
|
||||||
wav_bytes = base64.b64decode(syn["base64"])
|
|
||||||
return _pcm16_mono_16k_from_wav_bytes(wav_bytes)
|
|
||||||
|
|
||||||
|
|
||||||
def _assert_realtime_smoke(result: dict) -> None:
|
|
||||||
out_pcm = result["output_pcm"]
|
|
||||||
assert result["delta_events"] >= 1
|
|
||||||
assert out_pcm, "No output PCM from response.audio.delta"
|
|
||||||
assert len(out_pcm) % 2 == 0
|
|
||||||
assert len(out_pcm) >= 4096, "Output audio unexpectedly small"
|
|
||||||
assert result["output_sample_rate"] > 0
|
|
||||||
|
|
||||||
|
|
||||||
def _assert_realtime_accuracy(result: dict) -> None:
|
|
||||||
final_text = (result["transcription_text"] or "").strip()
|
|
||||||
assert final_text, "Expected non-empty transcription (model text stream)"
|
|
||||||
|
|
||||||
wav_out = _wav_bytes_from_pcm16(result["output_pcm"], result["output_sample_rate"])
|
|
||||||
whisper_text = convert_audio_bytes_to_text(wav_out).strip()
|
|
||||||
assert whisper_text, "Whisper returned empty string for synthesized output audio"
|
|
||||||
|
|
||||||
sim = cosine_similarity_text(whisper_text.lower(), final_text.lower())
|
|
||||||
assert sim > 0.8, (
|
|
||||||
f"Output audio transcript should match model text (sim={sim:.3f}): "
|
|
||||||
f"whisper={whisper_text!r}, model_text={final_text!r}"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class TestQwen3OmniRealtimeWebSocket:
|
class TestQwen3OmniRealtimeWebSocket:
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.omni
|
|
||||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
|
||||||
@pytest.mark.parametrize("omni_server", realtime_async_chunk_server_params, indirect=True)
|
|
||||||
def test_streaming_audio_input_pcm_output_async_chunk_smoke(self, omni_server) -> None:
|
|
||||||
"""Ready CI: async_chunk on, no send delay, smoke only."""
|
|
||||||
pcm16 = _synthetic_pcm16_input()
|
|
||||||
|
|
||||||
result = asyncio.run(
|
|
||||||
_run_realtime_audio_roundtrip(
|
|
||||||
omni_server.host,
|
|
||||||
omni_server.port,
|
|
||||||
omni_server.model,
|
|
||||||
pcm16,
|
|
||||||
chunk_ms=100,
|
|
||||||
send_delay_ms=0,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
_assert_realtime_smoke(result)
|
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
@pytest.mark.omni
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||||
@pytest.mark.parametrize("omni_server", realtime_async_chunk_server_params, indirect=True)
|
@pytest.mark.parametrize("omni_server", realtime_server_params, indirect=True)
|
||||||
def test_streaming_audio_input_pcm_output_async_chunk(self, omni_server) -> None:
|
|
||||||
"""Merge CI: async_chunk on, paced upload, full accuracy check."""
|
|
||||||
pcm16 = _synthetic_pcm16_input()
|
|
||||||
|
|
||||||
result = asyncio.run(
|
|
||||||
_run_realtime_audio_roundtrip(
|
|
||||||
omni_server.host,
|
|
||||||
omni_server.port,
|
|
||||||
omni_server.model,
|
|
||||||
pcm16,
|
|
||||||
chunk_ms=100,
|
|
||||||
send_delay_ms=SEND_DELAY_MS,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
_assert_realtime_smoke(result)
|
|
||||||
_assert_realtime_accuracy(result)
|
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
|
||||||
@pytest.mark.omni
|
|
||||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
|
||||||
@pytest.mark.parametrize("omni_server", realtime_sync_server_params, indirect=True)
|
|
||||||
def test_streaming_audio_input_pcm_output(self, omni_server) -> None:
|
def test_streaming_audio_input_pcm_output(self, omni_server) -> None:
|
||||||
"""Merge CI: async_chunk off, no send delay, full accuracy check."""
|
"""
|
||||||
pcm16 = _synthetic_pcm16_input()
|
Short streamed 16 kHz mono PCM16 input; expect streamed PCM16 audio deltas and
|
||||||
|
transcription. Verify Whisper(output audio) aligns with model text (same idea
|
||||||
|
as multimodal omni e2e). Input speech is synthesized from
|
||||||
|
``REALTIME_SYNTH_PHRASE_TEXT``.
|
||||||
|
"""
|
||||||
|
syn = generate_synthetic_audio(
|
||||||
|
10,
|
||||||
|
1,
|
||||||
|
sample_rate=16000,
|
||||||
|
phrase_text=REALTIME_SYNTH_PHRASE_TEXT,
|
||||||
|
)
|
||||||
|
wav_bytes = base64.b64decode(syn["base64"])
|
||||||
|
pcm16 = _pcm16_mono_16k_from_wav_bytes(wav_bytes)
|
||||||
|
|
||||||
result = asyncio.run(
|
result = asyncio.run(
|
||||||
_run_realtime_audio_roundtrip(
|
_run_realtime_audio_roundtrip(
|
||||||
@@ -272,9 +184,25 @@ class TestQwen3OmniRealtimeWebSocket:
|
|||||||
omni_server.model,
|
omni_server.model,
|
||||||
pcm16,
|
pcm16,
|
||||||
chunk_ms=100,
|
chunk_ms=100,
|
||||||
send_delay_ms=0,
|
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
_assert_realtime_smoke(result)
|
out_pcm = result["output_pcm"]
|
||||||
_assert_realtime_accuracy(result)
|
assert result["delta_events"] >= 1
|
||||||
|
assert out_pcm, "No output PCM from response.audio.delta"
|
||||||
|
assert len(out_pcm) % 2 == 0
|
||||||
|
assert len(out_pcm) >= 4096, "Output audio unexpectedly small"
|
||||||
|
assert result["output_sample_rate"] > 0
|
||||||
|
|
||||||
|
final_text = (result["transcription_text"] or "").strip()
|
||||||
|
assert final_text, "Expected non-empty transcription (model text stream)"
|
||||||
|
|
||||||
|
wav_out = _wav_bytes_from_pcm16(out_pcm, result["output_sample_rate"])
|
||||||
|
whisper_text = convert_audio_bytes_to_text(wav_out).strip()
|
||||||
|
assert whisper_text, "Whisper returned empty string for synthesized output audio"
|
||||||
|
|
||||||
|
sim = cosine_similarity_text(whisper_text.lower(), final_text.lower())
|
||||||
|
assert sim > 0.8, (
|
||||||
|
f"Output audio transcript should match model text (sim={sim:.3f}): "
|
||||||
|
f"whisper={whisper_text!r}, model_text={final_text!r}"
|
||||||
|
)
|
||||||
@@ -61,7 +61,7 @@ def test_build_multistage_generation_inputs_applies_stage_specific_overrides(ser
|
|||||||
assert engine_prompt["prompt"] == "draw a robot"
|
assert engine_prompt["prompt"] == "draw a robot"
|
||||||
assert engine_prompt["modalities"] == ["img2img"]
|
assert engine_prompt["modalities"] == ["img2img"]
|
||||||
assert engine_prompt["negative_prompt"] == "blurry"
|
assert engine_prompt["negative_prompt"] == "blurry"
|
||||||
assert engine_prompt["mm_processor_kwargs"] == {"target_h": 768, "target_w": 1024, "vae_generator_seed": 0}
|
assert engine_prompt["mm_processor_kwargs"] == {"target_h": 768, "target_w": 1024}
|
||||||
assert engine_prompt["multi_modal_data"]["img2img"].size == (24, 24)
|
assert engine_prompt["multi_modal_data"]["img2img"].size == (24, 24)
|
||||||
|
|
||||||
assert len(sampling_params_list) == 3
|
assert len(sampling_params_list) == 3
|
||||||
|
|||||||
@@ -33,7 +33,6 @@ from vllm_omni.entrypoints.openai.protocol.audio import (
|
|||||||
SpeechBatchItem,
|
SpeechBatchItem,
|
||||||
)
|
)
|
||||||
from vllm_omni.entrypoints.openai.serving_speech import (
|
from vllm_omni.entrypoints.openai.serving_speech import (
|
||||||
_TTS_LANGUAGES,
|
|
||||||
OmniOpenAIServingSpeech,
|
OmniOpenAIServingSpeech,
|
||||||
_create_wav_header,
|
_create_wav_header,
|
||||||
)
|
)
|
||||||
@@ -1258,67 +1257,6 @@ class TestTTSMethods:
|
|||||||
# Verify speakers are normalized to lowercase
|
# Verify speakers are normalized to lowercase
|
||||||
assert server.supported_speakers == {"ryan", "vivian", "aiden"}
|
assert server.supported_speakers == {"ryan", "vivian", "aiden"}
|
||||||
|
|
||||||
def test_load_supported_languages_from_config(self, speech_server):
|
|
||||||
"""Languages/dialects from codec_language_id are loaded title-cased; 'Auto' is added."""
|
|
||||||
speech_server._tts_model_type = "qwen3_tts"
|
|
||||||
speech_server.engine_client.model_config = SimpleNamespace(
|
|
||||||
hf_config=SimpleNamespace(
|
|
||||||
talker_config=SimpleNamespace(
|
|
||||||
codec_language_id={"chinese": 2055, "english": 2050, "beijing_dialect": 2074}
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
assert speech_server._load_supported_languages() == {
|
|
||||||
"Chinese",
|
|
||||||
"English",
|
|
||||||
"Beijing_Dialect",
|
|
||||||
"Auto",
|
|
||||||
}
|
|
||||||
|
|
||||||
def test_load_supported_languages_from_dict_config(self, speech_server):
|
|
||||||
"""talker_config provided as a plain dict is handled the same as an object."""
|
|
||||||
speech_server._tts_model_type = "qwen3_tts"
|
|
||||||
speech_server.engine_client.model_config = SimpleNamespace(
|
|
||||||
hf_config=SimpleNamespace(talker_config={"codec_language_id": {"chinese": 2055, "english": 2050}})
|
|
||||||
)
|
|
||||||
assert speech_server._load_supported_languages() == {"Chinese", "English", "Auto"}
|
|
||||||
|
|
||||||
def test_validate_language_custom_dialect_accepted(self, speech_server):
|
|
||||||
"""A language present in the model config passes validation, case-insensitively."""
|
|
||||||
speech_server.supported_languages = {"Chinese", "English", "Beijing_Dialect", "Auto"}
|
|
||||||
for language in ("Beijing_Dialect", "beijing_dialect", "English", "english", "Auto", "AUTO"):
|
|
||||||
req = OpenAICreateSpeechRequest(input="Hello", language=language)
|
|
||||||
result = speech_server._validate_tts_request(req)
|
|
||||||
assert result is None or "Invalid language" not in result
|
|
||||||
# Language is normalized to the title-cased config form.
|
|
||||||
assert req.language == language.title()
|
|
||||||
|
|
||||||
def test_validate_language_unknown_rejected(self, speech_server):
|
|
||||||
"""A language not in the configured set is rejected."""
|
|
||||||
speech_server.supported_languages = {"Chinese", "English", "Auto"}
|
|
||||||
for language in ("Klingon", "klingon"):
|
|
||||||
req = OpenAICreateSpeechRequest(input="Hello", language=language)
|
|
||||||
assert "Invalid language" in speech_server._validate_tts_request(req)
|
|
||||||
|
|
||||||
def test_load_supported_languages_default_when_no_config(self, speech_server):
|
|
||||||
"""Empty/missing codec_language_id on a Qwen3-TTS model falls back to the default list."""
|
|
||||||
speech_server._tts_model_type = "qwen3_tts"
|
|
||||||
speech_server.engine_client.model_config = SimpleNamespace(
|
|
||||||
hf_config=SimpleNamespace(talker_config=SimpleNamespace(codec_language_id={}))
|
|
||||||
)
|
|
||||||
assert speech_server._load_supported_languages() == _TTS_LANGUAGES
|
|
||||||
|
|
||||||
def test_load_supported_languages_default_on_config_error(self, speech_server):
|
|
||||||
"""If the model config cannot be read, fall back to the default list."""
|
|
||||||
speech_server._tts_model_type = "qwen3_tts"
|
|
||||||
speech_server.engine_client = SimpleNamespace() # no model_config -> AttributeError
|
|
||||||
assert speech_server._load_supported_languages() == _TTS_LANGUAGES
|
|
||||||
|
|
||||||
def test_load_supported_languages_default_for_non_qwen(self, speech_server):
|
|
||||||
"""Non-qwen3_tts model types get the default language set."""
|
|
||||||
speech_server._tts_model_type = None
|
|
||||||
assert speech_server._load_supported_languages() == _TTS_LANGUAGES
|
|
||||||
|
|
||||||
def test_build_tts_params_with_uploaded_voice(self, speech_server, mocker: MockerFixture):
|
def test_build_tts_params_with_uploaded_voice(self, speech_server, mocker: MockerFixture):
|
||||||
"""Test _build_tts_params auto-sets ref_audio for uploaded voices (x_vector only)."""
|
"""Test _build_tts_params auto-sets ref_audio for uploaded voices (x_vector only)."""
|
||||||
speech_server.uploaded_speakers = {
|
speech_server.uploaded_speakers = {
|
||||||
|
|||||||
@@ -13,14 +13,12 @@ import threading
|
|||||||
import time
|
import time
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
import pytest
|
import pytest
|
||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
from pytest_mock import MockerFixture
|
from pytest_mock import MockerFixture
|
||||||
|
|
||||||
from vllm_omni.diffusion.utils.media_utils import mux_video_audio_bytes
|
|
||||||
from vllm_omni.entrypoints.openai import api_server
|
from vllm_omni.entrypoints.openai import api_server
|
||||||
from vllm_omni.entrypoints.openai.api_server import router
|
from vllm_omni.entrypoints.openai.api_server import router
|
||||||
from vllm_omni.entrypoints.openai.protocol.videos import (
|
from vllm_omni.entrypoints.openai.protocol.videos import (
|
||||||
@@ -86,8 +84,7 @@ class BlockingVideoHandler:
|
|||||||
if self.stage_configs is None:
|
if self.stage_configs is None:
|
||||||
self.stage_configs = stage_configs
|
self.stage_configs = stage_configs
|
||||||
|
|
||||||
async def generate_video_bytes(self, request, reference_id, *, reference_image=None, reference_video=None):
|
async def generate_video_bytes(self, request, reference_id, *, reference_image=None):
|
||||||
del request, reference_id, reference_image, reference_video
|
|
||||||
self.started.set()
|
self.started.set()
|
||||||
try:
|
try:
|
||||||
await asyncio.Future()
|
await asyncio.Future()
|
||||||
@@ -133,30 +130,6 @@ def _make_test_image_data_url(size=(64, 64)) -> str:
|
|||||||
return f"data:image/png;base64,{encoded}"
|
return f"data:image/png;base64,{encoded}"
|
||||||
|
|
||||||
|
|
||||||
def _make_test_video_bytes(size=(32, 24), num_frames=3) -> bytes:
|
|
||||||
width, height = size
|
|
||||||
frames = np.zeros((num_frames, height, width, 3), dtype=np.uint8)
|
|
||||||
for idx in range(num_frames):
|
|
||||||
frames[idx, :, :, 0] = idx * 40
|
|
||||||
frames[idx, :, :, 1] = 128
|
|
||||||
frames[idx, :, :, 2] = 255 - idx * 40
|
|
||||||
return mux_video_audio_bytes(frames, fps=8, video_codec_options={"preset": "ultrafast", "threads": "0"})
|
|
||||||
|
|
||||||
|
|
||||||
def _make_test_video_data_url(size=(32, 24), num_frames=3) -> str:
|
|
||||||
encoded = base64.b64encode(_make_test_video_bytes(size, num_frames)).decode("utf-8")
|
|
||||||
return f"data:video/mp4;base64,{encoded}"
|
|
||||||
|
|
||||||
|
|
||||||
def _cosmos3_stage_configs():
|
|
||||||
return [
|
|
||||||
SimpleNamespace(
|
|
||||||
stage_type="diffusion",
|
|
||||||
engine_args=SimpleNamespace(model_class_name="Cosmos3OmniDiffusersPipeline"),
|
|
||||||
)
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def _wait_for_status(client: TestClient, video_id: str, status: str, timeout_s: float = 2.0):
|
def _wait_for_status(client: TestClient, video_id: str, status: str, timeout_s: float = 2.0):
|
||||||
deadline = time.time() + timeout_s
|
deadline = time.time() + timeout_s
|
||||||
last_payload = None
|
last_payload = None
|
||||||
@@ -359,104 +332,6 @@ def test_i2v_video_generation_with_image_reference_form(test_client, mocker: Moc
|
|||||||
assert input_image.size == (40, 24)
|
assert input_image.size == (40, 24)
|
||||||
|
|
||||||
|
|
||||||
def test_v2v_video_generation_form(test_client, mocker: MockerFixture):
|
|
||||||
video_bytes = _make_test_video_bytes((32, 24), num_frames=3)
|
|
||||||
|
|
||||||
mocker.patch(
|
|
||||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
|
||||||
return_value=b"fake-video",
|
|
||||||
)
|
|
||||||
response = test_client.post(
|
|
||||||
"/v1/videos",
|
|
||||||
data={"prompt": "Continue this motion."},
|
|
||||||
files={"input_reference": ("input.mp4", video_bytes, "video/mp4")},
|
|
||||||
)
|
|
||||||
|
|
||||||
assert response.status_code == 200
|
|
||||||
video_id = response.json()["id"]
|
|
||||||
_wait_for_status(test_client, video_id, VideoGenerationStatus.COMPLETED.value)
|
|
||||||
|
|
||||||
engine = test_client.app.state.openai_serving_video._engine_client
|
|
||||||
prompt = engine.captured_prompt
|
|
||||||
assert "multi_modal_data" in prompt
|
|
||||||
assert "video" in prompt["multi_modal_data"]
|
|
||||||
input_video = prompt["multi_modal_data"]["video"]
|
|
||||||
assert len(input_video) == 3
|
|
||||||
assert all(isinstance(frame, Image.Image) for frame in input_video)
|
|
||||||
assert input_video[0].size == (32, 24)
|
|
||||||
|
|
||||||
|
|
||||||
def test_v2v_video_generation_with_video_reference_form(test_client, mocker: MockerFixture):
|
|
||||||
mocker.patch(
|
|
||||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
|
||||||
return_value=b"fake-video",
|
|
||||||
)
|
|
||||||
response = test_client.post(
|
|
||||||
"/v1/videos",
|
|
||||||
data={
|
|
||||||
"prompt": "Continue this motion.",
|
|
||||||
"video_reference": json.dumps({"video_url": _make_test_video_data_url((32, 24), 2)}),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
assert response.status_code == 200
|
|
||||||
video_id = response.json()["id"]
|
|
||||||
_wait_for_status(test_client, video_id, VideoGenerationStatus.COMPLETED.value)
|
|
||||||
|
|
||||||
engine = test_client.app.state.openai_serving_video._engine_client
|
|
||||||
input_video = engine.captured_prompt["multi_modal_data"]["video"]
|
|
||||||
assert len(input_video) == 2
|
|
||||||
assert input_video[0].size == (32, 24)
|
|
||||||
|
|
||||||
|
|
||||||
def test_decode_video_bytes_can_keep_last_frames():
|
|
||||||
from vllm_omni.entrypoints.openai.video_api_utils import _decode_video_bytes
|
|
||||||
|
|
||||||
frames = _decode_video_bytes(
|
|
||||||
_make_test_video_bytes((32, 24), num_frames=6),
|
|
||||||
source="input_reference",
|
|
||||||
max_frames=2,
|
|
||||||
keep="last",
|
|
||||||
)
|
|
||||||
|
|
||||||
assert len(frames) == 2
|
|
||||||
red_means = [np.asarray(frame)[:, :, 0].mean() for frame in frames]
|
|
||||||
assert red_means[0] > 100
|
|
||||||
assert red_means[1] > red_means[0]
|
|
||||||
|
|
||||||
|
|
||||||
def test_cosmos3_reference_video_limit_uses_v2v_condition_frames():
|
|
||||||
request = VideoGenerationRequest(
|
|
||||||
prompt="Continue this motion.",
|
|
||||||
num_frames=189,
|
|
||||||
extra_params={"condition_frame_indexes_vision": [0, 2]},
|
|
||||||
)
|
|
||||||
|
|
||||||
spec = api_server._reference_video_decode_spec(request, _cosmos3_stage_configs())
|
|
||||||
assert spec.max_frames == 9
|
|
||||||
assert spec.keep == "first"
|
|
||||||
|
|
||||||
|
|
||||||
def test_cosmos3_reference_video_limit_preserves_action_frames():
|
|
||||||
request = VideoGenerationRequest(
|
|
||||||
prompt="Predict the action.",
|
|
||||||
num_frames=17,
|
|
||||||
extra_params={"action_mode": "inverse_dynamics", "action_chunk_size": 16},
|
|
||||||
)
|
|
||||||
|
|
||||||
assert api_server._reference_video_decode_spec(request, _cosmos3_stage_configs()).max_frames == 17
|
|
||||||
|
|
||||||
|
|
||||||
def test_cosmos3_reference_video_limit_caps_condition_frames_to_output_frames():
|
|
||||||
request = VideoGenerationRequest(
|
|
||||||
prompt="Continue this motion.",
|
|
||||||
num_frames=5,
|
|
||||||
extra_params={"condition_frame_indexes_vision": [0, 20]},
|
|
||||||
)
|
|
||||||
|
|
||||||
assert api_server._reference_video_decode_spec(request, _cosmos3_stage_configs()).max_frames == 5
|
|
||||||
|
|
||||||
|
|
||||||
def test_seconds_defaults_fps_and_frames(test_client, mocker: MockerFixture):
|
def test_seconds_defaults_fps_and_frames(test_client, mocker: MockerFixture):
|
||||||
fps_values = []
|
fps_values = []
|
||||||
|
|
||||||
@@ -524,8 +399,6 @@ def test_sampling_params_pass_through(test_client, mocker: MockerFixture):
|
|||||||
"true_cfg_scale": "4.0",
|
"true_cfg_scale": "4.0",
|
||||||
"boundary_ratio": "0.7",
|
"boundary_ratio": "0.7",
|
||||||
"flow_shift": "0.25",
|
"flow_shift": "0.25",
|
||||||
"generate_sound": "true",
|
|
||||||
"sound_duration": "2.5",
|
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -540,8 +413,6 @@ def test_sampling_params_pass_through(test_client, mocker: MockerFixture):
|
|||||||
assert captured.true_cfg_scale == 4.0
|
assert captured.true_cfg_scale == 4.0
|
||||||
assert captured.boundary_ratio == 0.7
|
assert captured.boundary_ratio == 0.7
|
||||||
assert captured.extra_args["flow_shift"] == 0.25
|
assert captured.extra_args["flow_shift"] == 0.25
|
||||||
assert captured.extra_args["generate_sound"] is True
|
|
||||||
assert captured.extra_args["sound_duration"] == 2.5
|
|
||||||
|
|
||||||
|
|
||||||
def test_frame_interpolation_params_pass_to_diffusion_sampling_params(test_client, mocker: MockerFixture):
|
def test_frame_interpolation_params_pass_to_diffusion_sampling_params(test_client, mocker: MockerFixture):
|
||||||
@@ -752,115 +623,6 @@ def test_video_job_persists_profiler_metadata(test_client, mocker: MockerFixture
|
|||||||
|
|
||||||
assert completed["stage_durations"] == {"diffuse": 2.5, "vae.decode": 0.3}
|
assert completed["stage_durations"] == {"diffuse": 2.5, "vae.decode": 0.3}
|
||||||
assert completed["peak_memory_mb"] == 4096.5
|
assert completed["peak_memory_mb"] == 4096.5
|
||||||
assert completed["action"] is None
|
|
||||||
|
|
||||||
|
|
||||||
def test_video_generation_response_exposes_action_payload(mocker: MockerFixture):
|
|
||||||
engine = FakeAsyncOmni()
|
|
||||||
handler = OmniOpenAIServingVideo.for_diffusion(
|
|
||||||
diffusion_engine=engine,
|
|
||||||
model_name="Cosmos3-8B-UVA",
|
|
||||||
)
|
|
||||||
|
|
||||||
async def _generate(prompt, request_id, sampling_params_list):
|
|
||||||
del prompt, request_id, sampling_params_list
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
yield MockVideoResult(
|
|
||||||
[object()],
|
|
||||||
custom_output={
|
|
||||||
"action": np.array([[[1.5, 2.5], [3.5, 4.5]]], dtype=np.float32),
|
|
||||||
"raw_action_dim": 2,
|
|
||||||
"action_mode": "policy",
|
|
||||||
"domain_id": 7,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
engine.generate = _generate
|
|
||||||
mocker.patch(
|
|
||||||
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
|
|
||||||
return_value="encoded-video",
|
|
||||||
)
|
|
||||||
|
|
||||||
response = asyncio.run(
|
|
||||||
handler.generate_videos(
|
|
||||||
VideoGenerationRequest(prompt="predict actions"),
|
|
||||||
"action-json",
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
action = response.data[0].action
|
|
||||||
assert action is not None
|
|
||||||
assert action.data == [[1.5, 2.5], [3.5, 4.5]]
|
|
||||||
assert action.shape == [2, 2]
|
|
||||||
assert action.dtype == "float32"
|
|
||||||
assert action.raw_action_dim == 2
|
|
||||||
assert action.action_mode == "policy"
|
|
||||||
assert action.domain_id == 7
|
|
||||||
assert response.model_dump(mode="json")["data"][0]["action"]["data"] == [[1.5, 2.5], [3.5, 4.5]]
|
|
||||||
|
|
||||||
|
|
||||||
def test_video_job_persists_action_metadata(test_client, mocker: MockerFixture):
|
|
||||||
engine = test_client.app.state.openai_serving_video._engine_client
|
|
||||||
|
|
||||||
async def _generate(prompt, request_id, sampling_params_list):
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
engine.captured_prompt = prompt
|
|
||||||
engine.captured_sampling_params_list = sampling_params_list
|
|
||||||
yield MockVideoResult(
|
|
||||||
[object()],
|
|
||||||
custom_output={
|
|
||||||
"action": np.array([[[1.0, 2.0], [3.0, 4.0]]], dtype=np.float32),
|
|
||||||
"raw_action_dim": 2,
|
|
||||||
"action_mode": "policy",
|
|
||||||
"domain_id": 7,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
engine.generate = _generate
|
|
||||||
mocker.patch(
|
|
||||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
|
||||||
return_value=b"fake-video",
|
|
||||||
)
|
|
||||||
|
|
||||||
response = test_client.post("/v1/videos", data={"prompt": "profile me"})
|
|
||||||
assert response.status_code == 200
|
|
||||||
video_id = response.json()["id"]
|
|
||||||
completed = _wait_for_status(test_client, video_id, VideoGenerationStatus.COMPLETED.value)
|
|
||||||
|
|
||||||
expected_action = {
|
|
||||||
"data": [[1.0, 2.0], [3.0, 4.0]],
|
|
||||||
"shape": [2, 2],
|
|
||||||
"dtype": "float32",
|
|
||||||
"raw_action_dim": 2,
|
|
||||||
"action_mode": "policy",
|
|
||||||
"domain_id": 7,
|
|
||||||
}
|
|
||||||
assert completed["action"] == expected_action
|
|
||||||
|
|
||||||
listed = test_client.get("/v1/videos").json()
|
|
||||||
assert listed["data"][0]["action"] == expected_action
|
|
||||||
|
|
||||||
|
|
||||||
def test_action_extraction_accepts_unbatched_action():
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
result = MockVideoResult(
|
|
||||||
[object()],
|
|
||||||
custom_output={
|
|
||||||
"action": np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32),
|
|
||||||
"raw_action_dim": 2,
|
|
||||||
"action_mode": "policy",
|
|
||||||
"domain_id": 7,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
actions = OmniOpenAIServingVideo._extract_action_outputs(result, expected_count=1)
|
|
||||||
|
|
||||||
assert actions[0] is not None
|
|
||||||
assert actions[0].data == [[1.0, 2.0], [3.0, 4.0]]
|
|
||||||
assert actions[0].shape == [2, 2]
|
|
||||||
|
|
||||||
|
|
||||||
def test_missing_handler_returns_503():
|
def test_missing_handler_returns_503():
|
||||||
@@ -919,20 +681,7 @@ def test_rejects_input_reference_and_image_reference_together(test_client):
|
|||||||
files={"input_reference": ("input.png", _make_test_image_bytes(), "image/png")},
|
files={"input_reference": ("input.png", _make_test_image_bytes(), "image/png")},
|
||||||
)
|
)
|
||||||
assert response.status_code == 400
|
assert response.status_code == 400
|
||||||
assert "only one of input_reference, image_reference, or video_reference" in response.json()["detail"].lower()
|
assert "either input_reference or image_reference" in response.json()["detail"].lower()
|
||||||
|
|
||||||
|
|
||||||
def test_rejects_image_reference_and_video_reference_together(test_client):
|
|
||||||
response = test_client.post(
|
|
||||||
"/v1/videos",
|
|
||||||
data={
|
|
||||||
"prompt": "bad refs",
|
|
||||||
"image_reference": '{"image_url": "https://example.com/cat.png"}',
|
|
||||||
"video_reference": '{"video_url": "https://example.com/cat.mp4"}',
|
|
||||||
},
|
|
||||||
)
|
|
||||||
assert response.status_code == 400
|
|
||||||
assert "only one of input_reference, image_reference, or video_reference" in response.json()["detail"].lower()
|
|
||||||
|
|
||||||
|
|
||||||
def test_invalid_seconds_returns_422(test_client):
|
def test_invalid_seconds_returns_422(test_client):
|
||||||
@@ -994,18 +743,6 @@ def test_unsupported_image_reference_file_id_returns_400(test_client):
|
|||||||
assert response.json()["detail"] == "Invalid image_reference: file_id is not supported yet."
|
assert response.json()["detail"] == "Invalid image_reference: file_id is not supported yet."
|
||||||
|
|
||||||
|
|
||||||
def test_unsupported_video_reference_file_id_returns_400(test_client):
|
|
||||||
response = test_client.post(
|
|
||||||
"/v1/videos",
|
|
||||||
data={
|
|
||||||
"prompt": "unsupported ref",
|
|
||||||
"video_reference": '{"file_id": "file-123"}',
|
|
||||||
},
|
|
||||||
)
|
|
||||||
assert response.status_code == 400
|
|
||||||
assert response.json()["detail"] == "Invalid video_reference: file_id is not supported yet."
|
|
||||||
|
|
||||||
|
|
||||||
def test_invalid_uploaded_input_reference_returns_400(test_client):
|
def test_invalid_uploaded_input_reference_returns_400(test_client):
|
||||||
response = test_client.post(
|
response = test_client.post(
|
||||||
"/v1/videos",
|
"/v1/videos",
|
||||||
@@ -1013,15 +750,12 @@ def test_invalid_uploaded_input_reference_returns_400(test_client):
|
|||||||
files={"input_reference": ("input.png", b"not-an-image", "image/png")},
|
files={"input_reference": ("input.png", b"not-an-image", "image/png")},
|
||||||
)
|
)
|
||||||
assert response.status_code == 400
|
assert response.status_code == 400
|
||||||
assert response.json()["detail"] == "Invalid input_reference: provided content is not a valid image or video."
|
assert response.json()["detail"] == "Invalid input_reference: provided content is not a valid image."
|
||||||
|
|
||||||
|
|
||||||
def test_video_request_validation():
|
def test_video_request_validation():
|
||||||
req = VideoGenerationRequest(prompt="test")
|
req = VideoGenerationRequest(prompt="test")
|
||||||
assert req.prompt == "test"
|
assert req.prompt == "test"
|
||||||
assert req.generate_sound is False
|
|
||||||
assert req.sound_duration is None
|
|
||||||
assert VideoGenerationRequest(prompt="test", generate_sound=True, sound_duration=1.5).generate_sound is True
|
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
VideoGenerationRequest(prompt="test", size="invalid")
|
VideoGenerationRequest(prompt="test", size="invalid")
|
||||||
|
|
||||||
@@ -1030,14 +764,10 @@ def test_video_request_validation():
|
|||||||
|
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
VideoGenerationRequest(prompt="test", image_reference={"file_id": "file-1", "image_url": "https://example.com"})
|
VideoGenerationRequest(prompt="test", image_reference={"file_id": "file-1", "image_url": "https://example.com"})
|
||||||
with pytest.raises(ValueError):
|
|
||||||
VideoGenerationRequest(prompt="test", video_reference={"file_id": "file-1", "video_url": "https://example.com"})
|
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
VideoGenerationRequest(prompt="test", frame_interpolation_exp=0)
|
VideoGenerationRequest(prompt="test", frame_interpolation_exp=0)
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
VideoGenerationRequest(prompt="test", frame_interpolation_scale=0)
|
VideoGenerationRequest(prompt="test", frame_interpolation_scale=0)
|
||||||
with pytest.raises(ValueError):
|
|
||||||
VideoGenerationRequest(prompt="test", sound_duration=0)
|
|
||||||
|
|
||||||
|
|
||||||
def test_list_videos_supports_order_after_and_limit(test_client, mocker: MockerFixture):
|
def test_list_videos_supports_order_after_and_limit(test_client, mocker: MockerFixture):
|
||||||
@@ -1299,31 +1029,6 @@ def test_sample_solver_forwarded_via_extra_params(test_client, mocker: MockerFix
|
|||||||
assert captured.extra_args["sample_solver"] == "euler"
|
assert captured.extra_args["sample_solver"] == "euler"
|
||||||
|
|
||||||
|
|
||||||
def test_extra_params_allows_inline_action(test_client, mocker: MockerFixture):
|
|
||||||
"""Inline ``action`` data is accepted and forwarded verbatim to
|
|
||||||
``extra_args`` (the supported way to pass forward-dynamics actions)."""
|
|
||||||
mocker.patch(
|
|
||||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
|
||||||
return_value=b"fake-video",
|
|
||||||
)
|
|
||||||
action = [[0.1, 0.2], [0.3, 0.4]]
|
|
||||||
response = test_client.post(
|
|
||||||
"/v1/videos",
|
|
||||||
data={
|
|
||||||
"prompt": "forward dynamics inline",
|
|
||||||
"extra_params": json.dumps({"action_mode": "forward_dynamics", "action": action}),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
assert response.status_code == 200
|
|
||||||
video_id = response.json()["id"]
|
|
||||||
_wait_for_status(test_client, video_id, VideoGenerationStatus.COMPLETED.value)
|
|
||||||
engine = test_client.app.state.openai_serving_video._engine_client
|
|
||||||
captured = engine.captured_sampling_params_list[0]
|
|
||||||
assert captured.extra_args["action"] == action
|
|
||||||
assert captured.extra_args["action_mode"] == "forward_dynamics"
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Sync endpoint tests (POST /v1/videos/sync)
|
# Sync endpoint tests (POST /v1/videos/sync)
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -1417,23 +1122,6 @@ def test_sync_i2v_with_image_reference(test_client, mocker: MockerFixture):
|
|||||||
assert response.content == b"ref-video"
|
assert response.content == b"ref-video"
|
||||||
|
|
||||||
|
|
||||||
def test_sync_v2v_returns_video_bytes(test_client, mocker: MockerFixture):
|
|
||||||
video_bytes = _make_test_video_bytes((32, 24), num_frames=3)
|
|
||||||
_mock_encode_video_bytes(mocker, b"v2v-video-data")
|
|
||||||
response = test_client.post(
|
|
||||||
"/v1/videos/sync",
|
|
||||||
data={"prompt": "Continue this motion."},
|
|
||||||
files={"input_reference": ("input.mp4", video_bytes, "video/mp4")},
|
|
||||||
)
|
|
||||||
|
|
||||||
assert response.status_code == 200
|
|
||||||
assert response.content == b"v2v-video-data"
|
|
||||||
engine = test_client.app.state.openai_serving_video._engine_client
|
|
||||||
input_video = engine.captured_prompt["multi_modal_data"]["video"]
|
|
||||||
assert len(input_video) == 3
|
|
||||||
assert input_video[0].size == (32, 24)
|
|
||||||
|
|
||||||
|
|
||||||
def test_sync_missing_handler_returns_503():
|
def test_sync_missing_handler_returns_503():
|
||||||
app = FastAPI()
|
app = FastAPI()
|
||||||
app.include_router(router)
|
app.include_router(router)
|
||||||
@@ -1466,7 +1154,7 @@ def test_sync_rejects_both_references(test_client):
|
|||||||
files={"input_reference": ("input.png", _make_test_image_bytes(), "image/png")},
|
files={"input_reference": ("input.png", _make_test_image_bytes(), "image/png")},
|
||||||
)
|
)
|
||||||
assert response.status_code == 400
|
assert response.status_code == 400
|
||||||
assert "only one of input_reference, image_reference, or video_reference" in response.json()["detail"].lower()
|
assert "either input_reference or image_reference" in response.json()["detail"].lower()
|
||||||
|
|
||||||
|
|
||||||
def test_sync_generation_error_returns_500(test_client, mocker: MockerFixture):
|
def test_sync_generation_error_returns_500(test_client, mocker: MockerFixture):
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ OMNI_MODEL = "Qwen/Qwen2.5-Omni-7B"
|
|||||||
OMNI_STAGE_CONFIG = get_deploy_config_path("ci/qwen2_5_omni_thinker_only.yaml")
|
OMNI_STAGE_CONFIG = get_deploy_config_path("ci/qwen2_5_omni_thinker_only.yaml")
|
||||||
|
|
||||||
|
|
||||||
async def _noop(*args, **kw):
|
async def _noop(**kw):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
@@ -61,7 +61,6 @@ def get_async_omni_instance(fake_add_request=_noop, fake_abort_request=_noop) ->
|
|||||||
omni._final_output_handler = lambda: None
|
omni._final_output_handler = lambda: None
|
||||||
omni.resolve_sampling_params_list = lambda params, allow_delta_coercion: params
|
omni.resolve_sampling_params_list = lambda params, allow_delta_coercion: params
|
||||||
omni._compute_final_stage_id = lambda output_modalities: 0
|
omni._compute_final_stage_id = lambda output_modalities: 0
|
||||||
omni._compute_final_output_stage_ids = lambda output_modalities: [0]
|
|
||||||
omni._process_orchestrator_results = fake_process_results
|
omni._process_orchestrator_results = fake_process_results
|
||||||
omni._log_summary_and_cleanup = lambda request_id: omni.request_states.pop(request_id, None)
|
omni._log_summary_and_cleanup = lambda request_id: omni.request_states.pop(request_id, None)
|
||||||
return omni
|
return omni
|
||||||
|
|||||||
@@ -413,19 +413,6 @@ def test_openai_serving_models_can_consume_async_omni_compat_attrs():
|
|||||||
|
|
||||||
|
|
||||||
def test_get_diffusion_od_config_returns_diffusion_stage_config():
|
def test_get_diffusion_od_config_returns_diffusion_stage_config():
|
||||||
diffusion_od_config = object()
|
|
||||||
omni = object.__new__(AsyncOmni)
|
|
||||||
omni.engine = SimpleNamespace(
|
|
||||||
stage_clients=[
|
|
||||||
SimpleNamespace(stage_type="llm"),
|
|
||||||
SimpleNamespace(stage_type="diffusion", od_config=diffusion_od_config),
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
assert omni.get_diffusion_od_config() is diffusion_od_config
|
|
||||||
|
|
||||||
|
|
||||||
def test_get_diffusion_od_config_falls_back_to_inner_engine():
|
|
||||||
diffusion_od_config = object()
|
diffusion_od_config = object()
|
||||||
omni = object.__new__(AsyncOmni)
|
omni = object.__new__(AsyncOmni)
|
||||||
omni.engine = SimpleNamespace(
|
omni.engine = SimpleNamespace(
|
||||||
|
|||||||
Loaded 100 of 240 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user