Compare commits
50
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3bda7ebd8b | ||
|
|
969c9b5a61 | ||
|
|
e592d29f4c | ||
|
|
82cce633a3 | ||
|
|
fce0f51089 | ||
|
|
82e74ce972 | ||
|
|
963ba1ab7a | ||
|
|
e25239c2ec | ||
|
|
2414b6f94d | ||
|
|
b3dd0a4baf | ||
|
|
6cefe91370 | ||
|
|
84569280b1 | ||
|
|
1e43656ab1 | ||
|
|
3875b81c18 | ||
|
|
b63b36ded3 | ||
|
|
95d56cf062 | ||
|
|
6e8ef213d8 | ||
|
|
31f882935d | ||
|
|
fccd95b12b | ||
|
|
469c1dc87e | ||
|
|
519cf3ecf1 | ||
|
|
bfc7cedcf1 | ||
|
|
e10aca30ec | ||
|
|
c6e8d78eda | ||
|
|
e5030c7524 | ||
|
|
a1cc583665 | ||
|
|
a57b8f858b | ||
|
|
9a9c8f7503 | ||
|
|
af02713952 | ||
|
|
d8dc1defa7 | ||
|
|
800c2b8251 | ||
|
|
221584a489 | ||
|
|
4e1c13236e | ||
|
|
706bad253d | ||
|
|
8949fdc440 | ||
|
|
3818ba4c65 | ||
|
|
c33ca60208 | ||
|
|
0fc4032a71 | ||
|
|
2e108a4109 | ||
|
|
ce69eb35e9 | ||
|
|
07737e6c61 | ||
|
|
d2e65a92f1 | ||
|
|
3842970a82 | ||
|
|
9bb4974198 | ||
|
|
0e2d55f46d | ||
|
|
40b2959183 | ||
|
|
3329f1b674 | ||
|
|
23484d8a11 | ||
|
|
bdfc77147d | ||
|
|
e35cd213d7 |
No files matched your search
@@ -389,6 +389,29 @@ steps:
|
|||||||
volumes:
|
volumes:
|
||||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||||
|
|
||||||
|
- label: "TTS · VoxCPM2 Online Test"
|
||||||
|
timeout_in_minutes: 30
|
||||||
|
commands:
|
||||||
|
- |
|
||||||
|
timeout 30m bash -c "
|
||||||
|
export VLLM_LOGGING_LEVEL=DEBUG
|
||||||
|
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
|
pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||||
|
"
|
||||||
|
agents:
|
||||||
|
queue: "gpu_1_queue"
|
||||||
|
plugins:
|
||||||
|
- docker#v5.2.0:
|
||||||
|
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||||
|
always-pull: true
|
||||||
|
propagate-environment: true
|
||||||
|
shm-size: "8gb"
|
||||||
|
environment:
|
||||||
|
- "HF_HOME=/fsx/hf_cache"
|
||||||
|
- "HF_TOKEN"
|
||||||
|
volumes:
|
||||||
|
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||||
|
|
||||||
- label: "Omni · Higgs-Audio V2 Test"
|
- label: "Omni · Higgs-Audio V2 Test"
|
||||||
timeout_in_minutes: 30
|
timeout_in_minutes: 30
|
||||||
commands:
|
commands:
|
||||||
|
|||||||
+188
-4
@@ -394,11 +394,11 @@ steps:
|
|||||||
depends_on: upload-nightly-pipeline
|
depends_on: upload-nightly-pipeline
|
||||||
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
|
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
|
||||||
steps:
|
steps:
|
||||||
- label: ":full_moon: TTS · Function Test"
|
- label: ":full_moon: TTS · Function Test with L4"
|
||||||
timeout_in_minutes: 90
|
timeout_in_minutes: 90
|
||||||
commands:
|
commands:
|
||||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||||
- pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy
|
- pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_voxcpm2_tts_expansion.py
|
||||||
agents:
|
agents:
|
||||||
queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
|
queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
|
||||||
plugins:
|
plugins:
|
||||||
@@ -413,6 +413,69 @@ steps:
|
|||||||
volumes:
|
volumes:
|
||||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||||
|
|
||||||
|
# - label: ":full_moon: TTS · Function Test with H100"
|
||||||
|
# timeout_in_minutes: 90
|
||||||
|
# commands:
|
||||||
|
# - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||||
|
# - pytest -s -v tests/e2e/ -m "full_model and H100 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy
|
||||||
|
# agents:
|
||||||
|
# queue: "mithril-h100-pool"
|
||||||
|
# plugins:
|
||||||
|
# - kubernetes:
|
||||||
|
# podSpec:
|
||||||
|
# containers:
|
||||||
|
# - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||||
|
# resources:
|
||||||
|
# limits:
|
||||||
|
# nvidia.com/gpu: 1
|
||||||
|
# volumeMounts:
|
||||||
|
# - name: devshm
|
||||||
|
# mountPath: /dev/shm
|
||||||
|
# - name: hf-cache
|
||||||
|
# mountPath: /root/.cache/huggingface
|
||||||
|
# env:
|
||||||
|
# - name: HF_HOME
|
||||||
|
# value: /root/.cache/huggingface
|
||||||
|
# - name: HF_TOKEN
|
||||||
|
# valueFrom:
|
||||||
|
# secretKeyRef:
|
||||||
|
# name: hf-token-secret
|
||||||
|
# key: token
|
||||||
|
# nodeSelector:
|
||||||
|
# node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||||
|
# volumes:
|
||||||
|
# - name: devshm
|
||||||
|
# emptyDir:
|
||||||
|
# medium: Memory
|
||||||
|
# - name: hf-cache
|
||||||
|
# hostPath:
|
||||||
|
# path: /mnt/hf-cache
|
||||||
|
# type: DirectoryOrCreate
|
||||||
|
|
||||||
|
- label: ":full_moon: TTS · Accuracy Test · VoxCPM2"
|
||||||
|
timeout_in_minutes: 30
|
||||||
|
commands:
|
||||||
|
- |
|
||||||
|
timeout 30m bash -c "
|
||||||
|
export VLLM_LOGGING_LEVEL=DEBUG
|
||||||
|
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
|
export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
||||||
|
pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts_expansion.py -m 'full_model and cuda and tts' --run-level 'full_model'
|
||||||
|
"
|
||||||
|
agents:
|
||||||
|
queue: "gpu_1_queue"
|
||||||
|
plugins:
|
||||||
|
- docker#v5.2.0:
|
||||||
|
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||||
|
always-pull: true
|
||||||
|
propagate-environment: true
|
||||||
|
shm-size: "8gb"
|
||||||
|
environment:
|
||||||
|
- "HF_HOME=/fsx/hf_cache"
|
||||||
|
- "HF_TOKEN"
|
||||||
|
volumes:
|
||||||
|
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||||
|
|
||||||
- label: ":full_moon: TTS · Perf Test"
|
- label: ":full_moon: TTS · Perf Test"
|
||||||
key: nightly-tts-performance
|
key: nightly-tts-performance
|
||||||
timeout_in_minutes: 180
|
timeout_in_minutes: 180
|
||||||
@@ -459,8 +522,8 @@ steps:
|
|||||||
path: /mnt/hf-cache
|
path: /mnt/hf-cache
|
||||||
type: DirectoryOrCreate
|
type: DirectoryOrCreate
|
||||||
|
|
||||||
- label: ":full_moon: VoxCPM2 · Perf Test"
|
- label: ":full_moon: TTS · VoxCPM2 · Perf Test"
|
||||||
key: nightly-voxcpm2-performance
|
key: nightly-tts-performance-voxcpm2
|
||||||
timeout_in_minutes: 180
|
timeout_in_minutes: 180
|
||||||
commands:
|
commands:
|
||||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||||
@@ -695,6 +758,87 @@ steps:
|
|||||||
path: /mnt/hf-cache
|
path: /mnt/hf-cache
|
||||||
type: DirectoryOrCreate
|
type: DirectoryOrCreate
|
||||||
|
|
||||||
|
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU DreamZero"
|
||||||
|
timeout_in_minutes: 120
|
||||||
|
commands:
|
||||||
|
- >-
|
||||||
|
pytest -sv
|
||||||
|
tests/e2e/online_serving/test_dreamzero_expansion.py
|
||||||
|
-m "full_model and diffusion and H100 and distributed_cuda"
|
||||||
|
--run-level "full_model"
|
||||||
|
agents:
|
||||||
|
queue: "mithril-h100-pool"
|
||||||
|
plugins:
|
||||||
|
- kubernetes:
|
||||||
|
podSpec:
|
||||||
|
containers:
|
||||||
|
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
nvidia.com/gpu: 2
|
||||||
|
volumeMounts:
|
||||||
|
- name: devshm
|
||||||
|
mountPath: /dev/shm
|
||||||
|
- name: hf-cache
|
||||||
|
mountPath: /root/.cache/huggingface
|
||||||
|
env:
|
||||||
|
- name: HF_HOME
|
||||||
|
value: /root/.cache/huggingface
|
||||||
|
- name: HF_TOKEN
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: hf-token-secret
|
||||||
|
key: token
|
||||||
|
nodeSelector:
|
||||||
|
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||||
|
volumes:
|
||||||
|
- name: devshm
|
||||||
|
emptyDir:
|
||||||
|
medium: Memory
|
||||||
|
- name: hf-cache
|
||||||
|
hostPath:
|
||||||
|
path: /mnt/hf-cache
|
||||||
|
type: DirectoryOrCreate
|
||||||
|
|
||||||
|
|
||||||
|
- label: ":full_moon: Diffusion X2I(&A&T) · GLM-Image Function Test with H100"
|
||||||
|
timeout_in_minutes: 120
|
||||||
|
commands:
|
||||||
|
- pytest -sv tests/e2e/online_serving/test_glm_image_expansion.py -m "full_model and diffusion and H100" --run-level "full_model"
|
||||||
|
agents:
|
||||||
|
queue: "mithril-h100-pool"
|
||||||
|
plugins:
|
||||||
|
- kubernetes:
|
||||||
|
podSpec:
|
||||||
|
containers:
|
||||||
|
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
nvidia.com/gpu: 2
|
||||||
|
volumeMounts:
|
||||||
|
- name: devshm
|
||||||
|
mountPath: /dev/shm
|
||||||
|
- name: hf-cache
|
||||||
|
mountPath: /root/.cache/huggingface
|
||||||
|
env:
|
||||||
|
- name: HF_HOME
|
||||||
|
value: /root/.cache/huggingface
|
||||||
|
- name: HF_TOKEN
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: hf-token-secret
|
||||||
|
key: token
|
||||||
|
nodeSelector:
|
||||||
|
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||||
|
volumes:
|
||||||
|
- name: devshm
|
||||||
|
emptyDir:
|
||||||
|
medium: Memory
|
||||||
|
- name: hf-cache
|
||||||
|
hostPath:
|
||||||
|
path: /mnt/hf-cache
|
||||||
|
type: DirectoryOrCreate
|
||||||
|
|
||||||
|
|
||||||
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4"
|
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4"
|
||||||
timeout_in_minutes: 60
|
timeout_in_minutes: 60
|
||||||
@@ -793,6 +937,46 @@ steps:
|
|||||||
path: /mnt/hf-cache
|
path: /mnt/hf-cache
|
||||||
type: DirectoryOrCreate
|
type: DirectoryOrCreate
|
||||||
|
|
||||||
|
- label: ":full_moon: HunyuanImage3-DIT · Accuracy Test"
|
||||||
|
timeout_in_minutes: 180
|
||||||
|
commands:
|
||||||
|
- export HUNYUAN_IMAGE3_MODEL="tencent/HunyuanImage-3.0-Instruct"
|
||||||
|
- export HUNYUAN_IMAGE3_DEVICES="0,1,2,3"
|
||||||
|
- pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m full_model --run-level full_model
|
||||||
|
agents:
|
||||||
|
queue: "mithril-h100-pool"
|
||||||
|
plugins:
|
||||||
|
- kubernetes:
|
||||||
|
podSpec:
|
||||||
|
containers:
|
||||||
|
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
nvidia.com/gpu: 4
|
||||||
|
volumeMounts:
|
||||||
|
- name: devshm
|
||||||
|
mountPath: /dev/shm
|
||||||
|
- name: hf-cache
|
||||||
|
mountPath: /root/.cache/huggingface
|
||||||
|
env:
|
||||||
|
- name: HF_HOME
|
||||||
|
value: /root/.cache/huggingface
|
||||||
|
- name: HF_TOKEN
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: hf-token-secret
|
||||||
|
key: token
|
||||||
|
nodeSelector:
|
||||||
|
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||||
|
volumes:
|
||||||
|
- name: devshm
|
||||||
|
emptyDir:
|
||||||
|
medium: Memory
|
||||||
|
- name: hf-cache
|
||||||
|
hostPath:
|
||||||
|
path: /mnt/hf-cache
|
||||||
|
type: DirectoryOrCreate
|
||||||
|
|
||||||
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image"
|
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image"
|
||||||
key: nightly-diffusion-x2iat-performance-qwen-image
|
key: nightly-diffusion-x2iat-performance-qwen-image
|
||||||
timeout_in_minutes: 180
|
timeout_in_minutes: 180
|
||||||
|
|||||||
@@ -273,6 +273,42 @@ steps:
|
|||||||
path: /mnt/hf-cache
|
path: /mnt/hf-cache
|
||||||
type: DirectoryOrCreate
|
type: DirectoryOrCreate
|
||||||
|
|
||||||
|
- label: "Omni · Qwen3-Omni Realtime WebSocket"
|
||||||
|
commands:
|
||||||
|
- |
|
||||||
|
timeout 30m bash -c "
|
||||||
|
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
|
pytest -s -v tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py::TestQwen3OmniRealtimeWebSocket::test_streaming_audio_input_pcm_output_async_chunk_smoke -m 'core_model' --run-level 'core_model'
|
||||||
|
"
|
||||||
|
agents:
|
||||||
|
queue: "mithril-h100-pool"
|
||||||
|
plugins:
|
||||||
|
- kubernetes:
|
||||||
|
podSpec:
|
||||||
|
containers:
|
||||||
|
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
nvidia.com/gpu: 2
|
||||||
|
volumeMounts:
|
||||||
|
- name: devshm
|
||||||
|
mountPath: /dev/shm
|
||||||
|
- name: hf-cache
|
||||||
|
mountPath: /root/.cache/huggingface
|
||||||
|
env:
|
||||||
|
- name: HF_HOME
|
||||||
|
value: /root/.cache/huggingface
|
||||||
|
nodeSelector:
|
||||||
|
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||||
|
volumes:
|
||||||
|
- name: devshm
|
||||||
|
emptyDir:
|
||||||
|
medium: Memory
|
||||||
|
- name: hf-cache
|
||||||
|
hostPath:
|
||||||
|
path: /mnt/hf-cache
|
||||||
|
type: DirectoryOrCreate
|
||||||
|
|
||||||
- label: "Omni · VoxCPM2 Test"
|
- label: "Omni · VoxCPM2 Test"
|
||||||
timeout_in_minutes: 20
|
timeout_in_minutes: 20
|
||||||
commands:
|
commands:
|
||||||
@@ -296,6 +332,29 @@ steps:
|
|||||||
volumes:
|
volumes:
|
||||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||||
|
|
||||||
|
- label: "TTS · VoxCPM2 Online Test"
|
||||||
|
timeout_in_minutes: 30
|
||||||
|
commands:
|
||||||
|
- |
|
||||||
|
timeout 30m bash -c "
|
||||||
|
export VLLM_LOGGING_LEVEL=DEBUG
|
||||||
|
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
|
pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts.py -m 'core_model and tts' --run-level 'core_model'
|
||||||
|
"
|
||||||
|
agents:
|
||||||
|
queue: "gpu_1_queue"
|
||||||
|
plugins:
|
||||||
|
- docker#v5.2.0:
|
||||||
|
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||||
|
always-pull: true
|
||||||
|
propagate-environment: true
|
||||||
|
shm-size: "8gb"
|
||||||
|
environment:
|
||||||
|
- "HF_HOME=/fsx/hf_cache"
|
||||||
|
- "HF_TOKEN"
|
||||||
|
volumes:
|
||||||
|
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||||
|
|
||||||
- label: "TTS · Qwen3-TTS CustomVoice Test"
|
- label: "TTS · Qwen3-TTS CustomVoice Test"
|
||||||
commands:
|
commands:
|
||||||
- |
|
- |
|
||||||
@@ -340,6 +399,29 @@ steps:
|
|||||||
volumes:
|
volumes:
|
||||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||||
|
|
||||||
|
- label: "TTS · Higgs-Audio-V3 Test"
|
||||||
|
commands:
|
||||||
|
- |
|
||||||
|
timeout 20m bash -c "
|
||||||
|
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
|
export VLLM_USE_DEEP_GEMM=0
|
||||||
|
export VLLM_MOE_USE_DEEP_GEMM=0
|
||||||
|
pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'core_model' --run-level 'core_model'
|
||||||
|
"
|
||||||
|
agents:
|
||||||
|
queue: "gpu_4_queue"
|
||||||
|
plugins:
|
||||||
|
- docker#v5.2.0:
|
||||||
|
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||||
|
always-pull: true
|
||||||
|
propagate-environment: true
|
||||||
|
shm-size: "8gb"
|
||||||
|
environment:
|
||||||
|
- "HF_HOME=/fsx/hf_cache"
|
||||||
|
- "HF_TOKEN"
|
||||||
|
volumes:
|
||||||
|
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||||
|
|
||||||
- label: "Diffusion · Qwen Image Edit Test"
|
- label: "Diffusion · Qwen Image Edit Test"
|
||||||
commands:
|
commands:
|
||||||
- |
|
- |
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ Easy, fast, and cheap omni-modality model serving for everyone
|
|||||||
---
|
---
|
||||||
|
|
||||||
*Latest News* 🔥
|
*Latest News* 🔥
|
||||||
|
- [2026/06] We released [0.22.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.22.0) - an **omnimodal world-model** release aligned with vLLM 0.22, featuring [Nvidia Cosmos3](recipes/cosmos3/Cosmos3-Nano.md)/DreamZero world model support, expanded quantization coverage across Blackwell/NPU/XPU, TTS production improvements, new models including MiniCPM-o 4.5, MOSS-TTS, and Lance, plus RL integration with veRL-Omni.
|
||||||
- [2026/05] We released [0.20.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.20.0) - refreshes the serving/runtime stack for large-scale omni workloads, and improves diffusion model performance, quantization, and hardware readiness across CUDA, ROCm, MUSA, NPU, and XPU backends.
|
- [2026/05] We released [0.20.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.20.0) - refreshes the serving/runtime stack for large-scale omni workloads, and improves diffusion model performance, quantization, and hardware readiness across CUDA, ROCm, MUSA, NPU, and XPU backends.
|
||||||
- [2026/03] We released [0.18.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.18.0) - strengthens the core runtime through a large entrypoint refactor and scheduler/runtime cleanups, expands unified quantization and diffusion execution, broadens multimodal model coverage, and improves production readiness across audio, omni, image, video, RL, and multi-platform deployments.
|
- [2026/03] We released [0.18.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.18.0) - strengthens the core runtime through a large entrypoint refactor and scheduler/runtime cleanups, expands unified quantization and diffusion execution, broadens multimodal model coverage, and improves production readiness across audio, omni, image, video, RL, and multi-platform deployments.
|
||||||
- [2026/03] Check out our first public [project deepdive](https://youtu.be/sgwNfsNnR9I) at the vLLM Hong Kong Meetup!
|
- [2026/03] Check out our first public [project deepdive](https://youtu.be/sgwNfsNnR9I) at the vLLM Hong Kong Meetup!
|
||||||
|
|||||||
@@ -151,6 +151,9 @@ class VllmOmniImageClient:
|
|||||||
seed: int | None = None,
|
seed: int | None = None,
|
||||||
negative_prompt: str | None = None,
|
negative_prompt: str | None = None,
|
||||||
output_compression: int | None = None,
|
output_compression: int | None = None,
|
||||||
|
bot_task: str | None = None,
|
||||||
|
sys_type: str | None = None,
|
||||||
|
system_prompt: str | None = None,
|
||||||
) -> Image.Image:
|
) -> Image.Image:
|
||||||
if not isinstance(images, list):
|
if not isinstance(images, list):
|
||||||
images = [images]
|
images = [images]
|
||||||
@@ -170,6 +173,12 @@ class VllmOmniImageClient:
|
|||||||
data["negative_prompt"] = negative_prompt
|
data["negative_prompt"] = negative_prompt
|
||||||
if output_compression is not None:
|
if output_compression is not None:
|
||||||
data["output_compression"] = str(output_compression)
|
data["output_compression"] = str(output_compression)
|
||||||
|
if bot_task is not None:
|
||||||
|
data["bot_task"] = bot_task
|
||||||
|
if sys_type is not None:
|
||||||
|
data["sys_type"] = sys_type
|
||||||
|
if system_prompt is not None:
|
||||||
|
data["system_prompt"] = system_prompt
|
||||||
|
|
||||||
files = [
|
files = [
|
||||||
(
|
(
|
||||||
|
|||||||
@@ -64,6 +64,47 @@ def resolve_model_name(*, model_name: str | None, model: str | None = None, outp
|
|||||||
raise ValueError("model-name is required when it cannot be inferred from model or output-root")
|
raise ValueError("model-name is required when it cannot be inferred from model or output-root")
|
||||||
|
|
||||||
|
|
||||||
|
def _coerce_score_int(value: Any) -> int:
|
||||||
|
if isinstance(value, bool):
|
||||||
|
raise TypeError(f"Expected numeric score, got bool: {value!r}")
|
||||||
|
if isinstance(value, (int, float)):
|
||||||
|
return int(value)
|
||||||
|
if isinstance(value, str):
|
||||||
|
stripped = value.strip()
|
||||||
|
if stripped.lstrip("-").isdigit():
|
||||||
|
return int(stripped)
|
||||||
|
raise TypeError(f"Expected numeric score, got {type(value).__name__}: {value!r}")
|
||||||
|
|
||||||
|
|
||||||
|
def _expand_score_entry(entry: Any) -> list[int]:
|
||||||
|
if isinstance(entry, dict):
|
||||||
|
if "score" in entry:
|
||||||
|
return [_coerce_score_int(entry["score"])]
|
||||||
|
if "value" in entry:
|
||||||
|
return [_coerce_score_int(entry["value"])]
|
||||||
|
numeric_values = [_coerce_score_int(value) for value in entry.values()]
|
||||||
|
if numeric_values:
|
||||||
|
return numeric_values
|
||||||
|
raise TypeError(f"Score dict has no numeric values: {entry!r}")
|
||||||
|
return [_coerce_score_int(entry)]
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_score_list(score: Any) -> list[int]:
|
||||||
|
if isinstance(score, dict):
|
||||||
|
entries: list[Any] = [score]
|
||||||
|
elif isinstance(score, list):
|
||||||
|
entries = score
|
||||||
|
else:
|
||||||
|
entries = [score]
|
||||||
|
|
||||||
|
normalized: list[int] = []
|
||||||
|
for entry in entries:
|
||||||
|
normalized.extend(_expand_score_entry(entry))
|
||||||
|
if not normalized:
|
||||||
|
raise ValueError(f"Empty score list after normalization: {score!r}")
|
||||||
|
return normalized
|
||||||
|
|
||||||
|
|
||||||
def parse_score_payload(raw_text: str) -> dict[str, Any]:
|
def parse_score_payload(raw_text: str) -> dict[str, Any]:
|
||||||
try:
|
try:
|
||||||
parsed = extract_json_object(raw_text)
|
parsed = extract_json_object(raw_text)
|
||||||
@@ -73,10 +114,7 @@ def parse_score_payload(raw_text: str) -> dict[str, Any]:
|
|||||||
parsed = {"score": json.loads(stripped), "reasoning": ""}
|
parsed = {"score": json.loads(stripped), "reasoning": ""}
|
||||||
else:
|
else:
|
||||||
raise
|
raise
|
||||||
score = parsed.get("score", [])
|
parsed["score"] = _normalize_score_list(parsed.get("score", []))
|
||||||
if not isinstance(score, list):
|
|
||||||
score = [score]
|
|
||||||
parsed["score"] = [int(value) for value in score]
|
|
||||||
return parsed
|
return parsed
|
||||||
|
|
||||||
|
|
||||||
@@ -391,6 +429,9 @@ class GEditBenchRunner:
|
|||||||
num_inference_steps: int = 20,
|
num_inference_steps: int = 20,
|
||||||
guidance_scale: float | None = None,
|
guidance_scale: float | None = None,
|
||||||
seed: int | None = 42,
|
seed: int | None = 42,
|
||||||
|
bot_task: str | None = None,
|
||||||
|
sys_type: str | None = None,
|
||||||
|
system_prompt: str | None = None,
|
||||||
):
|
):
|
||||||
self.dataset_ref = dataset_ref
|
self.dataset_ref = dataset_ref
|
||||||
self.output_root = output_root
|
self.output_root = output_root
|
||||||
@@ -400,6 +441,9 @@ class GEditBenchRunner:
|
|||||||
self.num_inference_steps = num_inference_steps
|
self.num_inference_steps = num_inference_steps
|
||||||
self.guidance_scale = guidance_scale
|
self.guidance_scale = guidance_scale
|
||||||
self.seed = seed
|
self.seed = seed
|
||||||
|
self.bot_task = bot_task
|
||||||
|
self.sys_type = sys_type
|
||||||
|
self.system_prompt = system_prompt
|
||||||
self.client = VllmOmniImageClient(base_url=base_url, api_key=api_key)
|
self.client = VllmOmniImageClient(base_url=base_url, api_key=api_key)
|
||||||
|
|
||||||
def generate(
|
def generate(
|
||||||
@@ -478,6 +522,9 @@ class GEditBenchRunner:
|
|||||||
num_inference_steps=self.num_inference_steps,
|
num_inference_steps=self.num_inference_steps,
|
||||||
guidance_scale=self.guidance_scale,
|
guidance_scale=self.guidance_scale,
|
||||||
seed=self.seed,
|
seed=self.seed,
|
||||||
|
bot_task=self.bot_task,
|
||||||
|
sys_type=self.sys_type,
|
||||||
|
system_prompt=self.system_prompt,
|
||||||
)
|
)
|
||||||
edited_image.save(output_path)
|
edited_image.save(output_path)
|
||||||
return {
|
return {
|
||||||
@@ -633,6 +680,25 @@ def build_parser() -> argparse.ArgumentParser:
|
|||||||
generate.add_argument("--num-inference-steps", type=int, default=20)
|
generate.add_argument("--num-inference-steps", type=int, default=20)
|
||||||
generate.add_argument("--guidance-scale", type=float, default=None)
|
generate.add_argument("--guidance-scale", type=float, default=None)
|
||||||
generate.add_argument("--seed", type=int, default=42)
|
generate.add_argument("--seed", type=int, default=42)
|
||||||
|
generate.add_argument(
|
||||||
|
"--bot-task",
|
||||||
|
type=str,
|
||||||
|
default=None,
|
||||||
|
choices=["think", "recaption", "think_recaption", "vanilla"],
|
||||||
|
help="HunyuanImage-3 prompting mode for /v1/images/edits (optional).",
|
||||||
|
)
|
||||||
|
generate.add_argument(
|
||||||
|
"--sys-type",
|
||||||
|
type=str,
|
||||||
|
default=None,
|
||||||
|
help="Hunyuan system-prompt type (e.g. en_unified). Defaults from --bot-task when omitted.",
|
||||||
|
)
|
||||||
|
generate.add_argument(
|
||||||
|
"--system-prompt",
|
||||||
|
type=str,
|
||||||
|
default=None,
|
||||||
|
help="Custom Hunyuan system prompt body (sys_type=custom).",
|
||||||
|
)
|
||||||
generate.add_argument("--workers", type=int, default=1)
|
generate.add_argument("--workers", type=int, default=1)
|
||||||
generate.add_argument("--max-samples", type=int, default=None)
|
generate.add_argument("--max-samples", type=int, default=None)
|
||||||
generate.add_argument("--samples-per-group", type=int, default=None)
|
generate.add_argument("--samples-per-group", type=int, default=None)
|
||||||
@@ -675,6 +741,9 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
num_inference_steps=args.num_inference_steps,
|
num_inference_steps=args.num_inference_steps,
|
||||||
guidance_scale=args.guidance_scale,
|
guidance_scale=args.guidance_scale,
|
||||||
seed=args.seed,
|
seed=args.seed,
|
||||||
|
bot_task=args.bot_task,
|
||||||
|
sys_type=args.sys_type,
|
||||||
|
system_prompt=args.system_prompt,
|
||||||
)
|
)
|
||||||
records = runner.generate(
|
records = runner.generate(
|
||||||
model_name=model_name,
|
model_name=model_name,
|
||||||
|
|||||||
+111
-3
@@ -2,7 +2,7 @@
|
|||||||
The vllm bench command launches the vLLM-Omni benchmark to evaluate the performance of multimodal models.
|
The vllm bench command launches the vLLM-Omni benchmark to evaluate the performance of multimodal models.
|
||||||
|
|
||||||
## Notes
|
## Notes
|
||||||
We currently only support using the "openai-chat-omni" backend.
|
We currently only support using the "openai-chat-omni" and "openai-image-edits-omni" backend.
|
||||||
|
|
||||||
## Basic Parameter Description
|
## Basic Parameter Description
|
||||||
You can use `vllm bench serve --omni --help=all` to get descriptions of all parameters. The commonly used parameters are described below:
|
You can use `vllm bench serve --omni --help=all` to get descriptions of all parameters. The commonly used parameters are described below:
|
||||||
@@ -52,7 +52,11 @@ You can use `vllm bench serve --omni --help=all` to get descriptions of all para
|
|||||||
- `--percentile-metrics`
|
- `--percentile-metrics`
|
||||||
"Comma-separated list of selected metrics to report percentiles."
|
"Comma-separated list of selected metrics to report percentiles."
|
||||||
"This argument specifies the metrics to report percentiles."
|
"This argument specifies the metrics to report percentiles."
|
||||||
'Allowed metric names are "ttft", "tpot", "itl", "e2el", "audio_ttfp", "audio_rtf", "audio_duration". '
|
'Allowed metric names are "ttft", "tpot", "itl", "ttfc", "tpoc", "icl", '
|
||||||
|
'"tpop", "e2el", "audio_ttfp", "audio_rtf", "audio_duration". '
|
||||||
|
|
||||||
|
- `-print-stage`
|
||||||
|
Print per-stage benchmark metrics for --omni serving when stage metrics are returned by the server. Disabled by default.
|
||||||
|
|
||||||
- `--save-result`
|
- `--save-result`
|
||||||
Specify to save benchmark results to a json file
|
Specify to save benchmark results to a json file
|
||||||
@@ -276,7 +280,9 @@ Generate synthetic image、video、audio inputs alongside random text prompts to
|
|||||||
|
|
||||||
Notes:
|
Notes:
|
||||||
|
|
||||||
- Works only with online benchmark via the OpenAI backend (`--backend openai-chat-omni`) and endpoint `/v1/chat/completions`.
|
- Works only with online benchmark via:
|
||||||
|
- the OpenAI chat backend (`--backend openai-chat-omni`) and endpoint `/v1/chat/completions`.
|
||||||
|
- the OpenAI edit image backend (`--backend openai-image-edits-omni`) and endpoint `/v1/images/edits`.
|
||||||
|
|
||||||
Start the server (example):
|
Start the server (example):
|
||||||
|
|
||||||
@@ -357,3 +363,105 @@ How sampling works:
|
|||||||
This should be seen as an edge case, and if this behavior can be avoided by setting `--random-mm-limit-mm-per-prompt` to a large number. Note that this might result in errors due to engine config `--limit-mm-per-prompt`.
|
This should be seen as an edge case, and if this behavior can be avoided by setting `--random-mm-limit-mm-per-prompt` to a large number. Note that this might result in errors due to engine config `--limit-mm-per-prompt`.
|
||||||
- The resulting request contains synthetic image data in `multi_modal_data` (OpenAI Chat format). When `random-mm` is used with the OpenAI Chat backend, prompts remain text and MM content is attached via `multi_modal_data`.
|
- The resulting request contains synthetic image data in `multi_modal_data` (OpenAI Chat format). When `random-mm` is used with the OpenAI Chat backend, prompts remain text and MM content is attached via `multi_modal_data`.
|
||||||
</details>
|
</details>
|
||||||
|
|
||||||
|
### Multi-Stage Benchmark
|
||||||
|
|
||||||
|
<details class="admonition abstract" markdown="1">
|
||||||
|
<summary>Show more</summary>
|
||||||
|
|
||||||
|
Start the server (change the port and model):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
vllm serve --omni --port 29999 --model ~/Qwen3-Omni-30B-A3B-Instruct
|
||||||
|
```
|
||||||
|
|
||||||
|
Then run the benchmarking script (remember to add the flag `--print-stage`):
|
||||||
|
```bash
|
||||||
|
vllm bench serve --omni \
|
||||||
|
--dataset-name random \
|
||||||
|
--port 29999 \
|
||||||
|
--max-concurrency 1 \
|
||||||
|
--num-warmups 2 \
|
||||||
|
--model ~/Qwen3-Omni-30B-A3B-Instruct \
|
||||||
|
--endpoint /v1/chat/completions \
|
||||||
|
--backend openai-chat-omni \
|
||||||
|
--num-prompts 3 \
|
||||||
|
--random-input-len 2500 \
|
||||||
|
--ignore-eos \
|
||||||
|
--percentile-metrics ttft,tpot,itl,e2el,audio_ttfp,audio_rtf,ttfc,tpoc,icl \
|
||||||
|
--random-output-len 900 \
|
||||||
|
--extra_body '{"modalities": ["text","audio"]}' \
|
||||||
|
--print-stage
|
||||||
|
```
|
||||||
|
|
||||||
|
Besides the "Serving Benchmark Result", there will be "Stage Benchmark Result" below:
|
||||||
|
```text
|
||||||
|
============= Stage Benchmark Result =============
|
||||||
|
=============== Stage 0 (thinker) ================
|
||||||
|
-------------------Stage Timing-------------------
|
||||||
|
Mean stage_gen_time (ms): 18243.16
|
||||||
|
Median stage_gen_time (ms): 18220.83
|
||||||
|
P99 stage_gen_time (ms): 18296.75
|
||||||
|
================== Text Result ===================
|
||||||
|
Stage generated tokens: 2700
|
||||||
|
-----------Serving Time to First Token------------
|
||||||
|
Mean Serving TTFT (ms): 816.31
|
||||||
|
Median Serving TTFT (ms): 818.94
|
||||||
|
P99 Serving TTFT (ms): 829.76
|
||||||
|
-----Time per Output Token (excl. 1st token)------
|
||||||
|
Mean TPOP (ms): 19.41
|
||||||
|
Median TPOP (ms): 19.40
|
||||||
|
P99 TPOP (ms): 19.47
|
||||||
|
---------------Inter-token Latency----------------
|
||||||
|
Mean ITL (ms): 19.41
|
||||||
|
Median ITL (ms): 19.12
|
||||||
|
P99 ITL (ms): 23.58
|
||||||
|
================ Stage 1 (talker) ================
|
||||||
|
-------------------Stage Timing-------------------
|
||||||
|
Mean stage_gen_time (ms): 18849.64
|
||||||
|
Median stage_gen_time (ms): 13641.15
|
||||||
|
P99 stage_gen_time (ms): 33655.58
|
||||||
|
============= Internal Stream Result =============
|
||||||
|
-----------Serving Time to First Chunk------------
|
||||||
|
Mean Serving TTFC (ms): 1029.14
|
||||||
|
Median Serving TTFC (ms): 1027.98
|
||||||
|
P99 Serving TTFC (ms): 1032.59
|
||||||
|
-----Time per Output Chunk (excl. 1st chunk)------
|
||||||
|
Mean TPOP (ms): 17.53
|
||||||
|
Median TPOP (ms): 18.70
|
||||||
|
P99 TPOP (ms): 18.88
|
||||||
|
---------------Inter-chunk Latency----------------
|
||||||
|
Mean ICL (ms): 16.27
|
||||||
|
Median ICL (ms): 18.19
|
||||||
|
P99 ICL (ms): 25.86
|
||||||
|
=============== Stage 2 (code2wav) ===============
|
||||||
|
-------------------Stage Timing-------------------
|
||||||
|
Mean stage_gen_time (ms): 18860.74
|
||||||
|
Median stage_gen_time (ms): 13655.84
|
||||||
|
P99 stage_gen_time (ms): 33659.79
|
||||||
|
================== Audio Result ==================
|
||||||
|
Stage audio duration generated(s): 260.11
|
||||||
|
Stage audio frames generated: 6242730
|
||||||
|
-----------Serving Time to First Packet-----------
|
||||||
|
Mean Serving AUDIO_TTFP (ms): 1366.16
|
||||||
|
Median Serving AUDIO_TTFP (ms): 1367.20
|
||||||
|
P99 Serving AUDIO_TTFP (ms): 1368.26
|
||||||
|
-----------------Real Time Factor-----------------
|
||||||
|
Mean AUDIO_RTF: 0.24
|
||||||
|
Median AUDIO_RTF: 0.26
|
||||||
|
P99 AUDIO_RTF: 0.27
|
||||||
|
```
|
||||||
|
Explanation:
|
||||||
|
- stage_gen_time: Time from submitting a request to a specific stage to that stage finishing generation.
|
||||||
|
|
||||||
|
- Serving TTFC (Time to First Chunk): Time from the HTTP request being accepted by the serving frontend to
|
||||||
|
the stage producing its first non-empty output chunk. Keep the name TTFT in text stage, TTFP in audio stage
|
||||||
|
for easier mapping.
|
||||||
|
|
||||||
|
- TPOP (Time per Output Chunk): Average time from the first output chunk to stage completion, divided by the
|
||||||
|
number of remaining output chunks. The TPOP abbreviation follows Qwen3.5-Omni Technical Report.
|
||||||
|
|
||||||
|
- ICL (Inter-Chunk Latency): Time between two consecutive output chunks produced by the same stage. Keep the
|
||||||
|
name ITL in text stage for easier mapping.
|
||||||
|
|
||||||
|
</details>
|
||||||
@@ -45,7 +45,7 @@ If you do not need to modify source code of vLLM, you can directly install the s
|
|||||||
uv pip install vllm==0.22.0 --torch-backend=auto
|
uv pip install vllm==0.22.0 --torch-backend=auto
|
||||||
```
|
```
|
||||||
|
|
||||||
The 0.22.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
The 0.22.0 release of vLLM ships CUDA 12.9-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
||||||
|
|
||||||
#### Installation of vLLM-Omni
|
#### Installation of vLLM-Omni
|
||||||
Since vllm-omni is rapidly evolving, it's recommended to install it from source
|
Since vllm-omni is rapidly evolving, it's recommended to install it from source
|
||||||
@@ -70,8 +70,8 @@ git checkout v0.22.0
|
|||||||
```
|
```
|
||||||
Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set `VLLM_PRECOMPILED_WHEEL_LOCATION` as your local absolute path of whl file.
|
Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set `VLLM_PRECOMPILED_WHEEL_LOCATION` as your local absolute path of whl file.
|
||||||
```bash
|
```bash
|
||||||
#For CUDA 13.0 (the default for v0.22.0; the wheel filename has no `+cu130` suffix)
|
#For CUDA 12.9 (the default for v0.22.0; the wheel filename has no `+cu129` suffix)
|
||||||
export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.22.0/vllm-0.22.0-cp38-abi3-manylinux_2_35_x86_64.whl
|
export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.22.0/vllm-0.22.0-cp38-abi3-manylinux_2_28_x86_64.whl
|
||||||
```
|
```
|
||||||
Install vllm with command below (If you have no existing PyTorch).
|
Install vllm with command below (If you have no existing PyTorch).
|
||||||
```bash
|
```bash
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ source .venv/bin/activate
|
|||||||
uv pip install vllm==0.22.0 --torch-backend=auto
|
uv pip install vllm==0.22.0 --torch-backend=auto
|
||||||
|
|
||||||
# On ROCm
|
# On ROCm
|
||||||
uv pip install vllm==0.22.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.22.0/rocm721
|
uv pip install vllm==0.22.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.22.0/rocm722
|
||||||
|
|
||||||
git clone https://github.com/vllm-project/vllm-omni.git
|
git clone https://github.com/vllm-project/vllm-omni.git
|
||||||
cd vllm-omni
|
cd vllm-omni
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ th {
|
|||||||
| `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
| `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
| `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
| `Cosmos3OmniDiffusersPipeline` | Cosmos3 T2I, T2V, I2V | `nvidia/Cosmos3-Nano` | ✅︎ | | | |
|
| `Cosmos3OmniDiffusersPipeline` | Cosmos3 T2I, T2V, I2V, T2V with sound, action policy | `nvidia/Cosmos3-Nano` | ✅︎ | | | |
|
||||||
| `WanSpeechToVideoPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `WanSpeechToVideoPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
| `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
| `LTX2Pipeline` | LTX-2-T2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | |
|
| `LTX2Pipeline` | LTX-2-T2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | |
|
||||||
@@ -42,6 +42,7 @@ th {
|
|||||||
| `LTX2ImageToVideoTwoStagesPipeline` | LTX-2-I2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | |
|
| `LTX2ImageToVideoTwoStagesPipeline` | LTX-2-I2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | |
|
||||||
| `LTX23Pipeline` | LTX-2.3-T2V | `dg845/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | |
|
| `LTX23Pipeline` | LTX-2.3-T2V | `dg845/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | |
|
||||||
| `LTX23ImageToVideoPipeline` | LTX-2.3-I2V | `dg845/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | |
|
| `LTX23ImageToVideoPipeline` | LTX-2.3-I2V | `dg845/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | |
|
||||||
|
| `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | | | |
|
||||||
| `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | |
|
| `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | |
|
||||||
| `MagiHumanPipeline` | MagiHuman | `SII-GAIR/daVinci-MagiHuman-Base-1080p` | ✅︎ | ✅︎ | | |
|
| `MagiHumanPipeline` | MagiHuman | `SII-GAIR/daVinci-MagiHuman-Base-1080p` | ✅︎ | ✅︎ | | |
|
||||||
| `OvisImagePipeline` | Ovis-Image | `OvisAI/Ovis-Image` | ✅︎ | ✅︎ | | ✅︎ |
|
| `OvisImagePipeline` | Ovis-Image | `OvisAI/Ovis-Image` | ✅︎ | ✅︎ | | ✅︎ |
|
||||||
@@ -49,6 +50,8 @@ th {
|
|||||||
| `LongCatImageEditPipeline` | LongCat-Image-Edit | `meituan-longcat/LongCat-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `LongCatImageEditPipeline` | LongCat-Image-Edit | `meituan-longcat/LongCat-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
| `StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` | ✅︎ | ✅︎ | | ✅︎ |
|
| `StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` | ✅︎ | ✅︎ | | ✅︎ |
|
||||||
| `CosyVoice3Model` | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✅︎ | ✅︎ | | ✅︎ |
|
| `CosyVoice3Model` | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✅︎ | ✅︎ | | ✅︎ |
|
||||||
|
| `OmniVoiceModel` | OmniVoice | `k2-fsa/OmniVoice` | ✅︎ | | | |
|
||||||
|
| `VoxCPM2TalkerForConditionalGeneration` | VoxCPM2 | `openbmb/VoxCPM2` | ✅︎ | | | |
|
||||||
| `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | |
|
| `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | |
|
||||||
| `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
| `FluxKontextPipeline` | FLUX.1-Kontext-dev | `black-forest-labs/FLUX.1-Kontext-dev` | ✅︎ | ✅︎ | | |
|
| `FluxKontextPipeline` | FLUX.1-Kontext-dev | `black-forest-labs/FLUX.1-Kontext-dev` | ✅︎ | ✅︎ | | |
|
||||||
@@ -61,6 +64,10 @@ th {
|
|||||||
| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-0.6B-Base` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-0.6B-Base` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
| `GLMTTSForConditionalGeneration` | GLM-TTS | `zai-org/GLM-TTS` | ✅︎ | | | |
|
| `GLMTTSForConditionalGeneration` | GLM-TTS | `zai-org/GLM-TTS` | ✅︎ | | | |
|
||||||
|
| `MossTTSNanoForCausalLM` | MOSS-TTS-Nano | `OpenMOSS-Team/MOSS-TTS-Nano` | ✅︎ | | | |
|
||||||
|
| `MossTTSDelayModel` | MOSS-TTS, MOSS-TTSD, MOSS-SoundEffect, MOSS-VoiceGenerator | `OpenMOSS-Team/MOSS-VoiceGenerator` | ✅︎ | | | |
|
||||||
|
| `MossTTSRealtime` | MOSS-TTS-Realtime | `OpenMOSS-Team/MOSS-TTS-Realtime` | ✅︎ | | | |
|
||||||
|
| `HiggsAudioV2ForConditionalGeneration` | Higgs-Audio v2 | `bosonai/higgs-audio-v2-generation-3B-base` | ✅︎ | | | |
|
||||||
| `NextStep11Pipeline` | NextStep-1.1 | `stepfun-ai/NextStep-1.1` | ✅︎ | ✅︎ | | ✅︎ |
|
| `NextStep11Pipeline` | NextStep-1.1 | `stepfun-ai/NextStep-1.1` | ✅︎ | ✅︎ | | ✅︎ |
|
||||||
| `MiMoAudioModel` | MiMo-Audio-7B-Instruct | `XiaomiMiMo/MiMo-Audio-7B-Instruct` | ✅︎ | ✅︎ | | |
|
| `MiMoAudioModel` | MiMo-Audio-7B-Instruct | `XiaomiMiMo/MiMo-Audio-7B-Instruct` | ✅︎ | ✅︎ | | |
|
||||||
| `MiMoV2ASRForCausalLM` | MiMo-V2.5-ASR | `XiaomiMiMo/MiMo-V2.5-ASR` | ✅︎ | ✅︎ | | |
|
| `MiMoV2ASRForCausalLM` | MiMo-V2.5-ASR | `XiaomiMiMo/MiMo-V2.5-ASR` | ✅︎ | ✅︎ | | |
|
||||||
@@ -68,11 +75,13 @@ th {
|
|||||||
| `FishSpeechSlowARForConditionalGeneration` | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✅︎ | ✅︎ | | |
|
| `FishSpeechSlowARForConditionalGeneration` | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✅︎ | ✅︎ | | |
|
||||||
| `DreamIDOmniPipeline` | DreamID-Omni | `XuGuo699/DreamID-Omni` | ✅︎ | ✅︎ | | |
|
| `DreamIDOmniPipeline` | DreamID-Omni | `XuGuo699/DreamID-Omni` | ✅︎ | ✅︎ | | |
|
||||||
| `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only) | `SenseNova/SenseNova-U1-8B-MoT` | ✅︎ | | | |
|
| `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only) | `SenseNova/SenseNova-U1-8B-MoT` | ✅︎ | | | |
|
||||||
|
| `LancePipeline` | Lance | `bytedance-research/Lance` | ✅︎ | | | |
|
||||||
| `HunyuanVideo15Pipeline` | HunyuanVideo-1.5-T2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | ✅︎ | ✅︎ | | |
|
| `HunyuanVideo15Pipeline` | HunyuanVideo-1.5-T2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | ✅︎ | ✅︎ | | |
|
||||||
| `HunyuanVideo15ImageToVideoPipeline` | HunyuanVideo-1.5-I2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v` | ✅︎ | ✅︎ | | |
|
| `HunyuanVideo15ImageToVideoPipeline` | HunyuanVideo-1.5-I2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v` | ✅︎ | ✅︎ | | |
|
||||||
| `VoxtralTTSForConditionalGeneration` | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✅︎ | ✅︎ | | |
|
| `VoxtralTTSForConditionalGeneration` | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✅︎ | ✅︎ | | |
|
||||||
| `CovoAudioForConditionalGeneration` | Covo-Audio-Chat | `tencent/Covo-Audio-Chat` | ✅︎ | | | |
|
| `CovoAudioForConditionalGeneration` | Covo-Audio-Chat | `tencent/Covo-Audio-Chat` | ✅︎ | | | |
|
||||||
|`DyninOmniForConditionalGeneration` | Dynin-Omni | `snu-aidas/Dynin-Omni` | ✅︎ | | | |
|
|`DyninOmniForConditionalGeneration` | Dynin-Omni | `snu-aidas/Dynin-Omni` | ✅︎ | | | |
|
||||||
|
| `MiniCPMO45OmniForConditionalGeneration` | MiniCPM-o 4.5 | `openbmb/MiniCPM-o-4_5` | ✅︎ | | ✅︎ | |
|
||||||
| `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
|`HiDreamImagePipeline` | HiDream-I1-Full | `HiDream-ai/HiDream-I1-Full` | ✅︎ | ✅︎ | | |
|
|`HiDreamImagePipeline` | HiDream-I1-Full | `HiDream-ai/HiDream-I1-Full` | ✅︎ | ✅︎ | | |
|
||||||
|
|
||||||
|
|||||||
@@ -145,6 +145,7 @@ The following tables show which models support each feature:
|
|||||||
| Model | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | Pipeline-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
|
| Model | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | Pipeline-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
|
||||||
|------------------------------|:---------:|:----------:|:---------------------:|:--------------:|:-----------------:|:-----------------:|:------:|:-------------------------:|:--------------------:|:--------------:|:----------------:|
|
|------------------------------|:---------:|:----------:|:---------------------:|:--------------:|:-----------------:|:-----------------:|:------:|:-------------------------:|:--------------------:|:--------------:|:----------------:|
|
||||||
| **Wan2.2** | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (encode/decode) | ❌ | ❌ |
|
| **Wan2.2** | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (encode/decode) | ❌ | ❌ |
|
||||||
|
| **Wan2.2-S2V** | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ (encode/decode) | ❌ | ❌ |
|
||||||
| **Wan2.1-VACE** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
|
| **Wan2.1-VACE** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
|
||||||
| **LTX-2** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
| **LTX-2** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||||
| **LTX-2.3** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
| **LTX-2.3** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||||
|
|||||||
@@ -0,0 +1,304 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import uuid
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import cv2
|
||||||
|
import numpy as np
|
||||||
|
import torch
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
from vllm_omni import Omni
|
||||||
|
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||||
|
from vllm_omni.outputs import OmniRequestOutput
|
||||||
|
|
||||||
|
WORKER_EXTENSION = "vllm_omni.diffusion.models.dreamzero.video_export_worker.DreamZeroVideoExportWorkerExtension"
|
||||||
|
ASSET_REPO_ID = "YangshenDeng/vllm-omni-dreamzero-assets"
|
||||||
|
DEFAULT_SESSION_PREFIX = "dreamzero-export"
|
||||||
|
RELATIVE_OFFSETS = [-23, -16, -8, 0]
|
||||||
|
ACTION_HORIZON = 24
|
||||||
|
CAMERA_FILES = {
|
||||||
|
"observation/exterior_image_0_left": "exterior_image_1_left.mp4",
|
||||||
|
"observation/exterior_image_1_left": "exterior_image_2_left.mp4",
|
||||||
|
"observation/wrist_image_left": "wrist_image_left.mp4",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_args() -> argparse.Namespace:
|
||||||
|
repo_root = Path(__file__).resolve().parents[3]
|
||||||
|
parser = argparse.ArgumentParser(description="Export DreamZero prediction video from downloaded example inputs.")
|
||||||
|
parser.add_argument("--model", default="GEAR-Dreams/DreamZero-DROID")
|
||||||
|
parser.add_argument("--deploy-config", type=Path, required=True)
|
||||||
|
parser.add_argument(
|
||||||
|
"--video-dir",
|
||||||
|
type=Path,
|
||||||
|
default=repo_root / "outputs" / "dreamzero" / "assets",
|
||||||
|
help="Directory containing the three camera MP4 files.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--output-dir", type=Path, default=repo_root / "outputs" / "dreamzero" / "generated_predictions"
|
||||||
|
)
|
||||||
|
parser.add_argument("--output-stem", default="dreamzero_prediction")
|
||||||
|
parser.add_argument(
|
||||||
|
"--prompt",
|
||||||
|
default="Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan",
|
||||||
|
)
|
||||||
|
parser.add_argument("--session-id", default=None)
|
||||||
|
parser.add_argument("--save-input-video", action="store_true")
|
||||||
|
parser.add_argument("--save-gif", action="store_true")
|
||||||
|
parser.add_argument("--save-actions", action="store_true")
|
||||||
|
parser.add_argument("--fps", type=int, default=5)
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def _load_all_frames(video_path: Path) -> np.ndarray:
|
||||||
|
cap = cv2.VideoCapture(str(video_path))
|
||||||
|
frames = []
|
||||||
|
while True:
|
||||||
|
ok, frame = cap.read()
|
||||||
|
if not ok:
|
||||||
|
break
|
||||||
|
frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
|
||||||
|
cap.release()
|
||||||
|
if not frames:
|
||||||
|
raise RuntimeError(f"No frames loaded from {video_path}")
|
||||||
|
return np.stack(frames, axis=0)
|
||||||
|
|
||||||
|
|
||||||
|
def _load_camera_frames(video_dir: Path) -> dict[str, np.ndarray]:
|
||||||
|
camera_frames: dict[str, np.ndarray] = {}
|
||||||
|
for camera_key, file_name in CAMERA_FILES.items():
|
||||||
|
video_path = video_dir / file_name
|
||||||
|
if not video_path.exists():
|
||||||
|
raise FileNotFoundError(
|
||||||
|
f"Missing DreamZero example asset: {video_path}. "
|
||||||
|
"Download the example videos with: "
|
||||||
|
f"`hf download {ASSET_REPO_ID} --repo-type dataset --local-dir {video_dir}`"
|
||||||
|
)
|
||||||
|
camera_frames[camera_key] = _load_all_frames(video_path)
|
||||||
|
return camera_frames
|
||||||
|
|
||||||
|
|
||||||
|
def _build_frame_schedule(total_frames: int, num_chunks: int) -> list[list[int]]:
|
||||||
|
chunks: list[list[int]] = []
|
||||||
|
current_frame = 23
|
||||||
|
for _ in range(num_chunks):
|
||||||
|
indices = [max(current_frame + offset, 0) for offset in RELATIVE_OFFSETS]
|
||||||
|
if indices[-1] >= total_frames:
|
||||||
|
break
|
||||||
|
chunks.append(indices)
|
||||||
|
current_frame += ACTION_HORIZON
|
||||||
|
return chunks
|
||||||
|
|
||||||
|
|
||||||
|
def _make_obs_from_video(
|
||||||
|
camera_frames: dict[str, np.ndarray],
|
||||||
|
frame_indices: list[int],
|
||||||
|
*,
|
||||||
|
prompt: str,
|
||||||
|
session_id: str,
|
||||||
|
) -> dict:
|
||||||
|
obs: dict = {}
|
||||||
|
for camera_key, all_frames in camera_frames.items():
|
||||||
|
selected = all_frames[frame_indices]
|
||||||
|
obs[camera_key] = selected[0] if len(frame_indices) == 1 else selected
|
||||||
|
|
||||||
|
obs["observation/joint_position"] = np.zeros(7, dtype=np.float32)
|
||||||
|
obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32)
|
||||||
|
obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32)
|
||||||
|
obs["prompt"] = prompt
|
||||||
|
obs["session_id"] = session_id
|
||||||
|
return obs
|
||||||
|
|
||||||
|
|
||||||
|
def _build_observations(video_dir: Path, prompt: str, session_id: str) -> tuple[dict[str, np.ndarray], list[dict]]:
|
||||||
|
camera_frames = _load_camera_frames(video_dir)
|
||||||
|
total_frames = min(frames.shape[0] for frames in camera_frames.values())
|
||||||
|
chunks = _build_frame_schedule(total_frames, 1)
|
||||||
|
observations = [
|
||||||
|
_make_obs_from_video(camera_frames, [0], prompt=prompt, session_id=session_id),
|
||||||
|
]
|
||||||
|
if chunks:
|
||||||
|
observations.append(
|
||||||
|
_make_obs_from_video(
|
||||||
|
camera_frames,
|
||||||
|
chunks[0],
|
||||||
|
prompt=prompt,
|
||||||
|
session_id=session_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if len(observations) < 2:
|
||||||
|
raise RuntimeError("Need at least two DreamZero example observations to export a prediction video.")
|
||||||
|
return camera_frames, observations[:2]
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_latents(output: OmniRequestOutput) -> torch.Tensor:
|
||||||
|
if not isinstance(output, OmniRequestOutput):
|
||||||
|
raise TypeError(f"Expected OmniRequestOutput, got {type(output)!r}")
|
||||||
|
if not output.images:
|
||||||
|
raise RuntimeError("DreamZero output does not contain video latents in `images`.")
|
||||||
|
|
||||||
|
latents = output.images[0]
|
||||||
|
if not isinstance(latents, torch.Tensor):
|
||||||
|
raise TypeError(f"Expected tensor latents, got {type(latents)!r}")
|
||||||
|
|
||||||
|
latents = latents.detach().cpu()
|
||||||
|
if latents.dim() == 4:
|
||||||
|
latents = latents.unsqueeze(0)
|
||||||
|
if latents.dim() != 5:
|
||||||
|
raise ValueError(f"Unexpected latent shape: {tuple(latents.shape)}")
|
||||||
|
|
||||||
|
if latents.shape[1] < latents.shape[2]:
|
||||||
|
latents = latents.transpose(1, 2).contiguous()
|
||||||
|
return latents
|
||||||
|
|
||||||
|
|
||||||
|
def _write_mp4(path: Path, frames: np.ndarray, fps: int) -> None:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
height, width = frames.shape[1:3]
|
||||||
|
writer = cv2.VideoWriter(
|
||||||
|
str(path),
|
||||||
|
cv2.VideoWriter_fourcc(*"mp4v"),
|
||||||
|
float(fps),
|
||||||
|
(width, height),
|
||||||
|
)
|
||||||
|
if not writer.isOpened():
|
||||||
|
raise RuntimeError(f"Failed to open video writer for {path}")
|
||||||
|
try:
|
||||||
|
for frame in frames:
|
||||||
|
writer.write(cv2.cvtColor(frame, cv2.COLOR_RGB2BGR))
|
||||||
|
finally:
|
||||||
|
writer.release()
|
||||||
|
|
||||||
|
|
||||||
|
def _write_gif(path: Path, frames: np.ndarray, fps: int) -> None:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
images = [Image.fromarray(frame) for frame in frames]
|
||||||
|
duration_ms = max(int(round(1000 / max(fps, 1))), 1)
|
||||||
|
images[0].save(
|
||||||
|
path,
|
||||||
|
save_all=True,
|
||||||
|
append_images=images[1:],
|
||||||
|
duration=duration_ms,
|
||||||
|
loop=0,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _stitch_input_frames(camera_frames: dict[str, np.ndarray]) -> np.ndarray:
|
||||||
|
total_frames = min(frames.shape[0] for frames in camera_frames.values())
|
||||||
|
stitched = []
|
||||||
|
for frame_index in range(total_frames):
|
||||||
|
left = camera_frames["observation/exterior_image_0_left"][frame_index]
|
||||||
|
right = camera_frames["observation/exterior_image_1_left"][frame_index]
|
||||||
|
wrist = camera_frames["observation/wrist_image_left"][frame_index]
|
||||||
|
pad = np.zeros((left.shape[0], left.shape[1], 3), dtype=np.uint8)
|
||||||
|
canvas = np.concatenate([left, right], axis=1)
|
||||||
|
bottom = np.concatenate([wrist, pad], axis=1)
|
||||||
|
stitched.append(np.concatenate([canvas, bottom], axis=0))
|
||||||
|
return np.stack(stitched, axis=0)
|
||||||
|
|
||||||
|
|
||||||
|
def _run_generation(
|
||||||
|
model: str, deploy_config_path: Path, observations: list[dict]
|
||||||
|
) -> tuple[Omni, list[OmniRequestOutput]]:
|
||||||
|
omni = Omni(
|
||||||
|
model=model,
|
||||||
|
deploy_config=str(deploy_config_path),
|
||||||
|
enforce_eager=True,
|
||||||
|
worker_extension_cls=WORKER_EXTENSION,
|
||||||
|
)
|
||||||
|
|
||||||
|
outputs: list[OmniRequestOutput] = []
|
||||||
|
for index, obs in enumerate(observations):
|
||||||
|
sampling_params = OmniDiffusionSamplingParams(
|
||||||
|
extra_args={
|
||||||
|
"reset": index == 0,
|
||||||
|
"session_id": obs["session_id"],
|
||||||
|
"robot_obs": obs,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
result = omni.generate(obs["prompt"], sampling_params_list=[sampling_params])
|
||||||
|
if not result:
|
||||||
|
raise RuntimeError(f"No output returned for DreamZero request {index}")
|
||||||
|
outputs.append(result[0])
|
||||||
|
return omni, outputs
|
||||||
|
|
||||||
|
|
||||||
|
def _decode_with_worker(omni: Omni, full_latents: torch.Tensor) -> np.ndarray:
|
||||||
|
stage_client = omni.engine.stage_clients[0]
|
||||||
|
engine = getattr(stage_client, "_engine", None)
|
||||||
|
if engine is None:
|
||||||
|
raise RuntimeError("DreamZero export requires inline diffusion stage access.")
|
||||||
|
|
||||||
|
decoded = engine.executor.collective_rpc(
|
||||||
|
"decode_video_latents_to_uint8",
|
||||||
|
args=(full_latents,),
|
||||||
|
unique_reply_rank=0,
|
||||||
|
exec_all_ranks=True,
|
||||||
|
)
|
||||||
|
if isinstance(decoded, torch.Tensor):
|
||||||
|
decoded = decoded.numpy()
|
||||||
|
if not isinstance(decoded, np.ndarray):
|
||||||
|
raise TypeError(f"Unexpected decoded output type: {type(decoded)!r}")
|
||||||
|
return decoded
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
args = _parse_args()
|
||||||
|
session_id = args.session_id or f"{DEFAULT_SESSION_PREFIX}-{uuid.uuid4()}"
|
||||||
|
|
||||||
|
camera_frames, observations = _build_observations(
|
||||||
|
video_dir=args.video_dir,
|
||||||
|
prompt=args.prompt,
|
||||||
|
session_id=session_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
if args.save_input_video:
|
||||||
|
input_frames = _stitch_input_frames(camera_frames)
|
||||||
|
_write_mp4(args.output_dir / f"{args.output_stem}_input.mp4", input_frames, fps=15)
|
||||||
|
if args.save_gif:
|
||||||
|
_write_gif(args.output_dir / f"{args.output_stem}_input.gif", input_frames[::3], fps=5)
|
||||||
|
|
||||||
|
omni = None
|
||||||
|
try:
|
||||||
|
omni, outputs = _run_generation(
|
||||||
|
model=args.model,
|
||||||
|
deploy_config_path=args.deploy_config,
|
||||||
|
observations=observations,
|
||||||
|
)
|
||||||
|
latent_steps = [_extract_latents(output) for output in outputs]
|
||||||
|
full_latents = torch.cat(latent_steps, dim=2)
|
||||||
|
frames = _decode_with_worker(omni, full_latents)
|
||||||
|
finally:
|
||||||
|
if omni is not None:
|
||||||
|
omni.close()
|
||||||
|
|
||||||
|
mp4_path = args.output_dir / f"{args.output_stem}.mp4"
|
||||||
|
|
||||||
|
_write_mp4(mp4_path, frames, fps=args.fps)
|
||||||
|
print(f"SAVED_MP4={mp4_path}")
|
||||||
|
|
||||||
|
if args.save_gif:
|
||||||
|
gif_path = args.output_dir / f"{args.output_stem}.gif"
|
||||||
|
_write_gif(gif_path, frames, fps=args.fps)
|
||||||
|
print(f"SAVED_GIF={gif_path}")
|
||||||
|
|
||||||
|
if args.save_actions:
|
||||||
|
npz_path = args.output_dir / f"{args.output_stem}_actions.npz"
|
||||||
|
np.savez(
|
||||||
|
npz_path,
|
||||||
|
step0=np.asarray(outputs[0].multimodal_output.get("actions")),
|
||||||
|
step1=np.asarray(outputs[1].multimodal_output.get("actions")),
|
||||||
|
)
|
||||||
|
print(f"SAVED_ACTIONS={npz_path}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -42,7 +42,7 @@ pip install \
|
|||||||
## 3. Hardware and VRAM Requirements
|
## 3. Hardware and VRAM Requirements
|
||||||
|
|
||||||
This example uses a 3-stage pipeline on one GPU by default
|
This example uses a 3-stage pipeline on one GPU by default
|
||||||
([`dynin_omni.yaml`](../../../vllm_omni/model_executor/stage_configs/dynin_omni.yaml)):
|
([`dynin_omni.yaml`](../../../vllm_omni/deploy/dynin_omni.yaml)):
|
||||||
|
|
||||||
- Stage-0 (`token2text`): `gpu_memory_utilization: 0.5`
|
- Stage-0 (`token2text`): `gpu_memory_utilization: 0.5`
|
||||||
- Stage-1 (`token2image`): `gpu_memory_utilization: 0.1`
|
- Stage-1 (`token2image`): `gpu_memory_utilization: 0.1`
|
||||||
|
|||||||
@@ -878,8 +878,8 @@ def parse_args(repo_root: Path) -> argparse.Namespace:
|
|||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--stage-config-path",
|
"--stage-config-path",
|
||||||
type=str,
|
type=str,
|
||||||
default=str(repo_root / "vllm_omni/model_executor/stage_configs/dynin_omni.yaml"),
|
default=str(repo_root / "vllm_omni/deploy/dynin_omni.yaml"),
|
||||||
help="Path to stage config yaml.",
|
help="Path to deploy config yaml.",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--dynin-config-path",
|
"--dynin-config-path",
|
||||||
|
|||||||
@@ -16,9 +16,10 @@ Usage:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import os
|
import json
|
||||||
import time
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
@@ -29,6 +30,16 @@ from vllm_omni.outputs import OmniRequestOutput
|
|||||||
from vllm_omni.platforms import current_omni_platform
|
from vllm_omni.platforms import current_omni_platform
|
||||||
|
|
||||||
|
|
||||||
|
def parse_profiler_config(value: str) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
config = json.loads(value)
|
||||||
|
except json.JSONDecodeError as e:
|
||||||
|
raise argparse.ArgumentTypeError(f"--profiler-config must be valid JSON: {e}") from e
|
||||||
|
if not isinstance(config, dict):
|
||||||
|
raise argparse.ArgumentTypeError("--profiler-config must be a JSON object")
|
||||||
|
return config
|
||||||
|
|
||||||
|
|
||||||
def parse_args() -> argparse.Namespace:
|
def parse_args() -> argparse.Namespace:
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
description="Generate a talking-head video from a reference image and audio (Wan2.2 S2V)."
|
description="Generate a talking-head video from a reference image and audio (Wan2.2 S2V)."
|
||||||
@@ -87,6 +98,12 @@ def parse_args() -> argparse.Namespace:
|
|||||||
default=3.0,
|
default=3.0,
|
||||||
help="Scheduler flow shift (default: 3.0).",
|
help="Scheduler flow shift (default: 3.0).",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--boundary-ratio",
|
||||||
|
type=float,
|
||||||
|
default=None,
|
||||||
|
help="Boundary split ratio for low/high DiT stages (S2V). Default varies by model.",
|
||||||
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--output",
|
"--output",
|
||||||
type=str,
|
type=str,
|
||||||
@@ -178,30 +195,10 @@ def parse_args() -> argparse.Namespace:
|
|||||||
help="Enable cache-dit summary logging after diffusion forward passes.",
|
help="Enable cache-dit summary logging after diffusion forward passes.",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--profile-dir",
|
"--profiler-config",
|
||||||
type=str,
|
type=parse_profiler_config,
|
||||||
default=None,
|
default=None,
|
||||||
help="Enable torch profiler and save traces to this directory.",
|
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--profile-record-shapes",
|
|
||||||
action="store_true",
|
|
||||||
help="Record tensor shapes in profiler (increases trace size).",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--profile-with-stack",
|
|
||||||
action="store_true",
|
|
||||||
help="Record stack traces in profiler (increases overhead).",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--profile-with-memory",
|
|
||||||
action="store_true",
|
|
||||||
help="Profile memory usage.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--profile-with-flops",
|
|
||||||
action="store_true",
|
|
||||||
help="Estimate FLOPs for operations.",
|
|
||||||
)
|
)
|
||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
@@ -239,23 +236,9 @@ def main():
|
|||||||
ulysses_degree=args.ulysses_degree,
|
ulysses_degree=args.ulysses_degree,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Check if profiling is requested via CLI args or environment variable
|
profiler_enabled = args.profiler_config is not None
|
||||||
profile_dir = args.profile_dir or os.getenv("VLLM_TORCH_PROFILER_DIR")
|
|
||||||
profiler_enabled = bool(profile_dir)
|
|
||||||
profiler_config = None
|
|
||||||
if profiler_enabled:
|
|
||||||
from vllm.config import ProfilerConfig
|
|
||||||
|
|
||||||
profiler_config = ProfilerConfig(
|
omni_kwargs = dict(
|
||||||
profiler="torch",
|
|
||||||
torch_profiler_dir=profile_dir,
|
|
||||||
torch_profiler_record_shapes=args.profile_record_shapes,
|
|
||||||
torch_profiler_with_stack=args.profile_with_stack,
|
|
||||||
torch_profiler_with_memory=args.profile_with_memory,
|
|
||||||
torch_profiler_with_flops=args.profile_with_flops,
|
|
||||||
)
|
|
||||||
|
|
||||||
omni = Omni(
|
|
||||||
model=args.model,
|
model=args.model,
|
||||||
model_class_name="WanS2VPipeline",
|
model_class_name="WanS2VPipeline",
|
||||||
flow_shift=args.flow_shift,
|
flow_shift=args.flow_shift,
|
||||||
@@ -269,8 +252,12 @@ def main():
|
|||||||
cache_backend=args.cache_backend,
|
cache_backend=args.cache_backend,
|
||||||
cache_config=cache_config,
|
cache_config=cache_config,
|
||||||
enable_cache_dit_summary=args.enable_cache_dit_summary,
|
enable_cache_dit_summary=args.enable_cache_dit_summary,
|
||||||
profiler_config=profiler_config,
|
profiler_config=args.profiler_config,
|
||||||
)
|
)
|
||||||
|
if args.boundary_ratio is not None:
|
||||||
|
omni_kwargs["boundary_ratio"] = args.boundary_ratio
|
||||||
|
|
||||||
|
omni = Omni(**omni_kwargs)
|
||||||
|
|
||||||
# Print generation configuration
|
# Print generation configuration
|
||||||
print(f"\n{'=' * 60}")
|
print(f"\n{'=' * 60}")
|
||||||
@@ -325,13 +312,16 @@ def main():
|
|||||||
if profiler_enabled:
|
if profiler_enabled:
|
||||||
print("\n[Profiler] Stopping profiler and collecting results...")
|
print("\n[Profiler] Stopping profiler and collecting results...")
|
||||||
profile_results = omni.stop_profile()
|
profile_results = omni.stop_profile()
|
||||||
if profile_results and isinstance(profile_results, list):
|
if profile_results and isinstance(profile_results, dict):
|
||||||
|
traces = profile_results.get("traces", [])
|
||||||
print("\n" + "=" * 60)
|
print("\n" + "=" * 60)
|
||||||
print("PROFILING RESULTS:")
|
print("PROFILING RESULTS:")
|
||||||
for i, result in enumerate(profile_results):
|
for rank, trace in enumerate(traces):
|
||||||
print(f"\nStage {i}:")
|
print(f"\nRank {rank}:")
|
||||||
if result:
|
if trace:
|
||||||
print(f" {result}")
|
print(f" • Trace: {trace}")
|
||||||
|
if not traces:
|
||||||
|
print(" No traces collected.")
|
||||||
print("=" * 60)
|
print("=" * 60)
|
||||||
else:
|
else:
|
||||||
print("[Profiler] No valid profiling data returned.")
|
print("[Profiler] No valid profiling data returned.")
|
||||||
|
|||||||
@@ -0,0 +1,204 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Offline higgs-audio v3 inference example.
|
||||||
|
|
||||||
|
Runs Stage 0 (Qwen3 talker) + Stage 1 (HiggsAudio codec) end-to-end
|
||||||
|
through the vLLM-Omni engine without going through the HTTP server, and
|
||||||
|
saves a 24 kHz mono WAV per prompt.
|
||||||
|
|
||||||
|
Example (plain TTS):
|
||||||
|
|
||||||
|
python examples/offline_inference/text_to_speech/higgs_audio_v3/end2end.py \\
|
||||||
|
--texts "Hello world." \\
|
||||||
|
--output-dir results/higgs_v3_wavs
|
||||||
|
|
||||||
|
Example (voice clone):
|
||||||
|
|
||||||
|
python examples/offline_inference/text_to_speech/higgs_audio_v3/end2end.py \\
|
||||||
|
--texts "Hello world." \\
|
||||||
|
--ref-audio path/to/reference.wav \\
|
||||||
|
--ref-text "Transcript of the reference clip." \\
|
||||||
|
--output-dir results/higgs_v3_wavs
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
os.environ.setdefault("VLLM_USE_DEEP_GEMM", "0")
|
||||||
|
os.environ.setdefault("VLLM_MOE_USE_DEEP_GEMM", "0")
|
||||||
|
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import soundfile as sf
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from vllm_omni import Omni
|
||||||
|
|
||||||
|
SAMPLE_RATE = 24_000
|
||||||
|
DEFAULT_TEXTS = (
|
||||||
|
"Hello world.",
|
||||||
|
"The quick brown fox jumps over the lazy dog.",
|
||||||
|
"Today is a beautiful day for a walk in the park.",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args():
|
||||||
|
import argparse
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser(description="Offline higgs-audio v3 inference")
|
||||||
|
parser.add_argument(
|
||||||
|
"--model",
|
||||||
|
type=str,
|
||||||
|
default="bosonai/higgs-audio-v3-tts-4b",
|
||||||
|
help="Stage-0 talker model id or path.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--texts",
|
||||||
|
type=str,
|
||||||
|
nargs="+",
|
||||||
|
default=list(DEFAULT_TEXTS),
|
||||||
|
help="One or more plain-text prompts to synthesize.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--output-dir",
|
||||||
|
type=str,
|
||||||
|
default="results/higgs_v3_wavs",
|
||||||
|
help="Directory to write per-prompt WAV files.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--deploy-config",
|
||||||
|
type=str,
|
||||||
|
default=None,
|
||||||
|
help="Override the deploy config path.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--max-new-tokens",
|
||||||
|
type=int,
|
||||||
|
default=2048,
|
||||||
|
help="Cap on Stage-0 codec frames per request.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--ref-audio",
|
||||||
|
type=str,
|
||||||
|
default=None,
|
||||||
|
help="Reference audio for voice clone (WAV/FLAC/MP3 path).",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--ref-text",
|
||||||
|
type=str,
|
||||||
|
default=None,
|
||||||
|
help="Transcript of the reference audio. Optional but improves fidelity.",
|
||||||
|
)
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def _slugify(text: str) -> str:
|
||||||
|
slug = re.sub(r"\s+", "_", text.strip().lower())
|
||||||
|
slug = re.sub(r"[^a-z0-9_]+", "", slug)
|
||||||
|
return slug[:48] or "prompt"
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_pcm(multimodal_output: dict) -> torch.Tensor:
|
||||||
|
audio = multimodal_output.get("model_outputs")
|
||||||
|
if audio is None:
|
||||||
|
audio = multimodal_output.get("audio")
|
||||||
|
if audio is None:
|
||||||
|
raise ValueError(f"no audio key in multimodal_output: {list(multimodal_output.keys())}")
|
||||||
|
if isinstance(audio, list):
|
||||||
|
valid = [torch.as_tensor(a).float().cpu().reshape(-1) for a in audio if a is not None]
|
||||||
|
if not valid:
|
||||||
|
raise ValueError("audio list is empty")
|
||||||
|
return torch.cat(valid, dim=0) if len(valid) > 1 else valid[0]
|
||||||
|
return torch.as_tensor(audio).float().cpu().reshape(-1)
|
||||||
|
|
||||||
|
|
||||||
|
def _pcm_to_int16(pcm: torch.Tensor) -> np.ndarray:
|
||||||
|
arr = pcm.numpy()
|
||||||
|
if arr.dtype.kind == "f":
|
||||||
|
arr = np.clip(arr, -1.0, 1.0)
|
||||||
|
arr = (arr * 32767.0).astype(np.int16)
|
||||||
|
else:
|
||||||
|
arr = arr.astype(np.int16)
|
||||||
|
return arr
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
args = parse_args()
|
||||||
|
output_dir = Path(args.output_dir)
|
||||||
|
output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
engine = Omni(model=args.model, deploy_config=args.deploy_config, trust_remote_code=True)
|
||||||
|
|
||||||
|
from transformers import AutoTokenizer
|
||||||
|
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_tokenizer import (
|
||||||
|
HiggsAudioV3TokenizerAdapter,
|
||||||
|
apply_delay_pattern,
|
||||||
|
encode_reference_audio,
|
||||||
|
)
|
||||||
|
|
||||||
|
tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
|
||||||
|
adapter = HiggsAudioV3TokenizerAdapter(tokenizer)
|
||||||
|
|
||||||
|
# Load and encode reference audio once (if voice cloning)
|
||||||
|
ref_codes_delayed: torch.Tensor | None = None
|
||||||
|
if args.ref_audio is not None:
|
||||||
|
ref_wav, ref_sr = sf.read(args.ref_audio, always_2d=False)
|
||||||
|
if ref_wav.ndim == 2:
|
||||||
|
ref_wav = ref_wav.mean(axis=1)
|
||||||
|
ref_codes_raw = encode_reference_audio(ref_wav, int(ref_sr))
|
||||||
|
ref_codes_delayed = apply_delay_pattern(ref_codes_raw)
|
||||||
|
print(f"Reference : {args.ref_audio}")
|
||||||
|
print(f"Ref codes : {ref_codes_raw.shape[0]} frames -> {ref_codes_delayed.shape[0]} delayed")
|
||||||
|
if args.ref_text:
|
||||||
|
print(f"Ref text : {args.ref_text}")
|
||||||
|
|
||||||
|
print(f"Model : {args.model}")
|
||||||
|
print(f"Prompts : {len(args.texts)}")
|
||||||
|
print(f"Output dir : {output_dir}")
|
||||||
|
print(f"Voice clone : {'yes' if ref_codes_delayed is not None else 'no'}")
|
||||||
|
|
||||||
|
total_elapsed = 0.0
|
||||||
|
total_dur = 0.0
|
||||||
|
for text in args.texts:
|
||||||
|
if ref_codes_delayed is not None:
|
||||||
|
prompt_ids = adapter.build_prompt(
|
||||||
|
text,
|
||||||
|
num_ref_tokens=int(ref_codes_delayed.shape[0]),
|
||||||
|
reference_text=args.ref_text,
|
||||||
|
)
|
||||||
|
prompt = {
|
||||||
|
"prompt_token_ids": prompt_ids,
|
||||||
|
"additional_information": {
|
||||||
|
"audio_input_ids": ref_codes_delayed.to(torch.long),
|
||||||
|
"audio_input_ids_mask": torch.ones(ref_codes_delayed.shape[0], dtype=torch.bool),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
prompt_ids = adapter.build_prompt(text)
|
||||||
|
prompt = {"prompt_token_ids": prompt_ids}
|
||||||
|
|
||||||
|
t_start = time.perf_counter()
|
||||||
|
outputs = engine.generate([prompt])
|
||||||
|
elapsed = time.perf_counter() - t_start
|
||||||
|
total_elapsed += elapsed
|
||||||
|
|
||||||
|
mm = outputs[0].outputs[0].multimodal_output
|
||||||
|
pcm = _extract_pcm(mm)
|
||||||
|
slug = _slugify(text)
|
||||||
|
out_path = output_dir / f"{slug}.wav"
|
||||||
|
sf.write(str(out_path), _pcm_to_int16(pcm), SAMPLE_RATE, format="WAV", subtype="PCM_16")
|
||||||
|
dur = pcm.numel() / SAMPLE_RATE
|
||||||
|
total_dur += dur
|
||||||
|
print(f" {slug:<50} dur={dur:5.2f}s -> {out_path}")
|
||||||
|
|
||||||
|
rtf = total_elapsed / total_dur if total_dur > 0 else float("inf")
|
||||||
|
print(f"Total infer : {total_elapsed:.2f}s total audio: {total_dur:.2f}s RTF: {rtf:.3f}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,132 @@
|
|||||||
|
# DreamZero
|
||||||
|
|
||||||
|
DreamZero is the robot-policy serving example for the OpenPI-compatible `/v1/realtime/robot/openpi` endpoint.
|
||||||
|
|
||||||
|
## Supported checkpoints
|
||||||
|
|
||||||
|
| Model | HuggingFace repo |
|
||||||
|
|---|---|
|
||||||
|
| DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` |
|
||||||
|
|
||||||
|
## Quick start
|
||||||
|
|
||||||
|
### Start the server
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bash examples/online_serving/dreamzero/run_server.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
By default this launches:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
vllm serve GEAR-Dreams/DreamZero-DROID --omni --port 8000 \
|
||||||
|
--served-model-name dreamzero-droid \
|
||||||
|
--deploy-config vllm_omni/deploy/dreamzero_tp1_cfg2.yaml \
|
||||||
|
--enforce-eager --disable-log-stats
|
||||||
|
```
|
||||||
|
|
||||||
|
Override `MODEL`, `PORT`, `HOST`, `DEPLOY_CONFIG`, or `SERVED_MODEL_NAME` through the script environment if needed.
|
||||||
|
|
||||||
|
### Install example dependencies
|
||||||
|
|
||||||
|
The core `pip install -e .` setup does not include the extra packages used by the DreamZero example scripts.
|
||||||
|
|
||||||
|
- `openpi_client.py`:
|
||||||
|
`openpi-client`, `websockets`, `opencv-python`
|
||||||
|
- `droid_sim_eval_client.py`:
|
||||||
|
`mediapy`, `websockets`, `openpi-client`
|
||||||
|
|
||||||
|
The DROID sim-eval script also needs an Isaac Lab environment that provides `isaaclab`, `isaaclab_tasks`, `sim_evals`, and `gymnasium`.
|
||||||
|
|
||||||
|
If you run the DROID client on Python < 3.12, also install `typing-extensions`.
|
||||||
|
|
||||||
|
### Configure TP and CFG parallelism
|
||||||
|
|
||||||
|
The bundled DreamZero configs intentionally keep only:
|
||||||
|
|
||||||
|
| Config | Purpose |
|
||||||
|
|---|---|
|
||||||
|
| `vllm_omni/deploy/dreamzero.yaml` | Default TP=1, CFG parallel disabled |
|
||||||
|
| `vllm_omni/deploy/dreamzero_tp1_cfg2.yaml` | TP=1, CFG parallel size=2 |
|
||||||
|
|
||||||
|
For other topologies, use CLI parallelism flags and update stage 0 `devices` with `--stage-overrides`. The number of listed devices must match `tensor_parallel_size * cfg_parallel_size`.
|
||||||
|
|
||||||
|
TP=2 with CFG parallel disabled:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
vllm serve GEAR-Dreams/DreamZero-DROID --omni --port 8000 \
|
||||||
|
--served-model-name dreamzero-droid \
|
||||||
|
--deploy-config vllm_omni/deploy/dreamzero.yaml \
|
||||||
|
--tensor-parallel-size 2 \
|
||||||
|
--stage-overrides '{"0": {"devices": "0,1"}}' \
|
||||||
|
--enforce-eager --disable-log-stats
|
||||||
|
```
|
||||||
|
|
||||||
|
TP=2 with CFG parallel size=2:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
vllm serve GEAR-Dreams/DreamZero-DROID --omni --port 8000 \
|
||||||
|
--served-model-name dreamzero-droid \
|
||||||
|
--deploy-config vllm_omni/deploy/dreamzero.yaml \
|
||||||
|
--tensor-parallel-size 2 \
|
||||||
|
--cfg-parallel-size 2 \
|
||||||
|
--stage-overrides '{"0": {"devices": "0,1,2,3"}}' \
|
||||||
|
--enforce-eager --disable-log-stats
|
||||||
|
```
|
||||||
|
|
||||||
|
### Download example assets
|
||||||
|
|
||||||
|
The OpenPI client and DROID sim-eval example expect the three camera MP4 files in `outputs/dreamzero/assets`.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
hf download YangshenDeng/vllm-omni-dreamzero-assets --repo-type dataset --local-dir outputs/dreamzero/assets
|
||||||
|
```
|
||||||
|
|
||||||
|
### Run the OpenPI client
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python examples/online_serving/dreamzero/openpi_client.py \
|
||||||
|
--host 127.0.0.1 \
|
||||||
|
--port 8000 \
|
||||||
|
--video-dir outputs/dreamzero/assets
|
||||||
|
```
|
||||||
|
|
||||||
|
This client uses downloaded example videos and talks to the OpenPI websocket server.
|
||||||
|
|
||||||
|
### Run DROID sim eval
|
||||||
|
|
||||||
|
```bash
|
||||||
|
${ISAACLAB_LAUNCHER} -p examples/online_serving/dreamzero/droid_sim_eval_client.py \
|
||||||
|
--host 127.0.0.1 \
|
||||||
|
--port 8000 \
|
||||||
|
--scene 1 \
|
||||||
|
--episodes 1 \
|
||||||
|
--headless \
|
||||||
|
--device cuda:0
|
||||||
|
```
|
||||||
|
|
||||||
|
Set `ISAACLAB_LAUNCHER=path/to/isaaclab.sh` from the vLLM-Omni repository root before running the command.
|
||||||
|
This launches Isaac Lab / sim-evals and runs the DROID benchmark loop against the same websocket endpoint.
|
||||||
|
|
||||||
|
### Export comparison videos offline
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python examples/offline_inference/dreamzero/export_prediction_video.py \
|
||||||
|
--deploy-config vllm_omni/deploy/dreamzero_tp1_cfg2.yaml \
|
||||||
|
--save-input-video \
|
||||||
|
--save-gif
|
||||||
|
```
|
||||||
|
|
||||||
|
The export script uses local `Omni` inference, not the websocket server. It writes the input rollout video and the predicted output video artifacts for side-by-side inspection.
|
||||||
|
|
||||||
|
### MolmoSpace demo
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python examples/online_serving/dreamzero/molmospace_dreamzero_eval_demo.py \
|
||||||
|
--host 127.0.0.1 \
|
||||||
|
--port 8000 \
|
||||||
|
--benchmark_dir /path/to/benchmark \
|
||||||
|
--output_dir /path/to/output
|
||||||
|
```
|
||||||
|
|
||||||
|
This demo adapts DreamZero to the MolmoSpace-style remote policy eval loop.
|
||||||
@@ -0,0 +1,819 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
"""Run one or more DROID sim-eval rollouts against the vLLM DreamZero server.
|
||||||
|
|
||||||
|
This script is the vLLM/OpenPI adaptation of the upstream DreamZero
|
||||||
|
`eval_utils/run_sim_eval.py` sim-eval client.
|
||||||
|
|
||||||
|
Behavior intentionally kept close to upstream:
|
||||||
|
- same DROID observation extraction (`external_cam`, `external_cam_2`,
|
||||||
|
`wrist_cam`, joint position, gripper position)
|
||||||
|
- same resize-with-pad preprocessing to `(180, 320)`
|
||||||
|
- same `open_loop_horizon=8`
|
||||||
|
- same gripper binarization rule (`> 0.5 -> 1`, else `0`)
|
||||||
|
- same per-scene language prompts
|
||||||
|
|
||||||
|
Unlike upstream DreamZero, vLLM serves the compatible websocket policy endpoint
|
||||||
|
at `/v1/realtime/robot/openpi`, so this script includes the path suffix in the
|
||||||
|
client URI.
|
||||||
|
|
||||||
|
Run this script through Isaac Lab's launcher from the vLLM-Omni repository
|
||||||
|
root, for example:
|
||||||
|
|
||||||
|
"${ISAACLAB_LAUNCHER}" -p \
|
||||||
|
examples/online_serving/dreamzero/droid_sim_eval_client.py \
|
||||||
|
--host 127.0.0.1 \
|
||||||
|
--port 8000 \
|
||||||
|
--scene 1 \
|
||||||
|
--episodes 1 \
|
||||||
|
--headless \
|
||||||
|
--device cuda:1
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import torch
|
||||||
|
|
||||||
|
try:
|
||||||
|
import mediapy
|
||||||
|
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
||||||
|
raise ImportError("Optional DROID sim-eval client requires `mediapy`.") from exc
|
||||||
|
|
||||||
|
try:
|
||||||
|
from typing import override
|
||||||
|
except ImportError:
|
||||||
|
try:
|
||||||
|
from typing_extensions import override
|
||||||
|
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
||||||
|
raise ImportError("Optional DROID sim-eval client requires `typing-extensions` on Python < 3.12.") from exc
|
||||||
|
|
||||||
|
try:
|
||||||
|
import websockets.sync.client
|
||||||
|
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
||||||
|
raise ImportError("Optional DROID sim-eval client requires `websockets`.") from exc
|
||||||
|
|
||||||
|
# NOTE:
|
||||||
|
# This directory already contains a local file named `openpi_client.py`.
|
||||||
|
# However, what we want here is the *installed* `openpi_client` package from
|
||||||
|
# upstream OpenPI, not the sibling example file. When a script is executed
|
||||||
|
# directly, Python often puts the script directory at `sys.path[0]`, which
|
||||||
|
# would cause `import openpi_client` to resolve to the local example file and
|
||||||
|
# create a circular import.
|
||||||
|
#
|
||||||
|
# To avoid that ambiguity, temporarily remove the current example directory
|
||||||
|
# from the front of `sys.path`, import the real package, and then restore the
|
||||||
|
# path afterwards.
|
||||||
|
example_dir = str(Path(__file__).resolve().parent)
|
||||||
|
removed_path = False
|
||||||
|
if sys.path and sys.path[0] == example_dir:
|
||||||
|
sys.path.pop(0)
|
||||||
|
removed_path = True
|
||||||
|
try:
|
||||||
|
from openpi_client import image_tools, msgpack_numpy
|
||||||
|
from openpi_client.base_policy import BasePolicy
|
||||||
|
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
||||||
|
raise ImportError("Optional DROID sim-eval client requires the `openpi-client` package.") from exc
|
||||||
|
finally:
|
||||||
|
if removed_path:
|
||||||
|
sys.path.insert(0, example_dir)
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------
|
||||||
|
# Constants
|
||||||
|
# -----------------------------------------------------------------------------
|
||||||
|
#
|
||||||
|
# These values intentionally mirror the upstream DreamZero sim-eval client
|
||||||
|
# where possible. The important distinction is:
|
||||||
|
#
|
||||||
|
# - ACTION_HORIZON = 24
|
||||||
|
# The model returns 24 future actions per inference call.
|
||||||
|
# - DEFAULT_OPEN_LOOP_HORIZON = 8
|
||||||
|
# The sim client only executes the first 8 actions locally before asking
|
||||||
|
# the server to replan from a fresh observation.
|
||||||
|
#
|
||||||
|
# So a single server call predicts 24x8 actions, but the rollout consumes only
|
||||||
|
# 8 of them before replanning.
|
||||||
|
PING_INTERVAL_SECS = 60
|
||||||
|
PING_TIMEOUT_SECS = 600
|
||||||
|
DEFAULT_PATH = "/v1/realtime/robot/openpi"
|
||||||
|
DEFAULT_OPEN_LOOP_HORIZON = 8
|
||||||
|
ACTION_HORIZON = 24
|
||||||
|
ACTION_DIM = 8
|
||||||
|
DEFAULT_OUTPUT_ROOT = Path("runs") / "dreamzero_sim_eval"
|
||||||
|
SCENE_PROMPTS = {
|
||||||
|
1: "put the cube in the bowl",
|
||||||
|
2: "pick up the can and put it in the mug",
|
||||||
|
3: "put the banana in the bin",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _decode_action_response(response: bytes | str) -> np.ndarray:
|
||||||
|
if isinstance(response, str):
|
||||||
|
raise RuntimeError(f"Error in inference server:\n{response}")
|
||||||
|
decoded = msgpack_numpy.unpackb(response)
|
||||||
|
if isinstance(decoded, dict) and decoded.get("type") == "error":
|
||||||
|
message = decoded.get("message", decoded)
|
||||||
|
raise RuntimeError(f"Error in inference server:\n{message}")
|
||||||
|
return np.asarray(decoded, dtype=np.float32)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class StepRecord:
|
||||||
|
"""One fully materialized rollout step for later JSON export.
|
||||||
|
|
||||||
|
The `episode_00.json` artifact is intended to be human-readable and
|
||||||
|
post-process-friendly. Instead of keeping raw tensors around, each step is
|
||||||
|
flattened into plain Python types so it can be serialized directly.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Index of this control step within the episode.
|
||||||
|
step_index: int
|
||||||
|
# Whether this step triggered a fresh model call (as opposed to reusing the
|
||||||
|
# cached open-loop chunk from the previous server response).
|
||||||
|
used_server_call: bool
|
||||||
|
# End-to-end latency of the server call that produced the current chunk.
|
||||||
|
# This is `None` on steps that only reuse cached actions.
|
||||||
|
chunk_latency_s: float | None
|
||||||
|
# The concrete 8-D action sent into the simulator at this step.
|
||||||
|
action: list[float]
|
||||||
|
# Observed 7-DoF arm joint positions before the next environment step.
|
||||||
|
joint_position: list[float]
|
||||||
|
# Observed gripper scalar before the next environment step.
|
||||||
|
gripper_position: list[float]
|
||||||
|
# Reward and termination signals directly returned by the simulator.
|
||||||
|
reward: float
|
||||||
|
terminated: bool
|
||||||
|
truncated: bool
|
||||||
|
# Optional scene object positions for downstream debugging / success
|
||||||
|
# heuristics. This may be empty if the environment does not expose them.
|
||||||
|
scene_objects: dict[str, list[float]]
|
||||||
|
|
||||||
|
|
||||||
|
class OpenPIWebsocketClientPolicy(BasePolicy):
|
||||||
|
"""Minimal websocket client for the DreamZero/OpenPI policy protocol.
|
||||||
|
|
||||||
|
Protocol shape:
|
||||||
|
- connect -> server immediately sends a metadata dict
|
||||||
|
- infer -> send msgpack observation, receive action chunk
|
||||||
|
- reset -> send msgpack reset command, receive confirmation string
|
||||||
|
|
||||||
|
This class intentionally stays very small because the more interesting
|
||||||
|
DreamZero-specific behavior lives one layer above, in
|
||||||
|
`DreamZeroJointPosClient`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
host: str = "127.0.0.1",
|
||||||
|
port: int = 8000,
|
||||||
|
path: str = DEFAULT_PATH,
|
||||||
|
) -> None:
|
||||||
|
# vLLM serves the robot endpoint under `/v1/realtime/robot/openpi`.
|
||||||
|
self._uri = f"ws://{host}:{port}{path}"
|
||||||
|
# Upstream protocol uses msgpack with numpy support, not JSON.
|
||||||
|
self._packer = msgpack_numpy.Packer()
|
||||||
|
# Connect immediately and cache the server handshake metadata.
|
||||||
|
self._ws, self._server_metadata = self._wait_for_server()
|
||||||
|
|
||||||
|
def get_server_metadata(self) -> dict[str, Any]:
|
||||||
|
"""Return a copy of the server handshake metadata."""
|
||||||
|
|
||||||
|
return dict(self._server_metadata)
|
||||||
|
|
||||||
|
def _wait_for_server(self):
|
||||||
|
"""Connect to the websocket server and read the initial metadata frame."""
|
||||||
|
|
||||||
|
logging.info("Connecting to %s", self._uri)
|
||||||
|
conn = websockets.sync.client.connect(
|
||||||
|
self._uri,
|
||||||
|
compression=None,
|
||||||
|
max_size=None,
|
||||||
|
ping_interval=PING_INTERVAL_SECS,
|
||||||
|
ping_timeout=PING_TIMEOUT_SECS,
|
||||||
|
)
|
||||||
|
metadata = msgpack_numpy.unpackb(conn.recv())
|
||||||
|
if not isinstance(metadata, dict):
|
||||||
|
raise TypeError(f"Expected dict metadata from server, got {type(metadata)!r}")
|
||||||
|
return conn, metadata
|
||||||
|
|
||||||
|
@override
|
||||||
|
def infer(self, obs: dict[str, Any]) -> np.ndarray:
|
||||||
|
"""Send an inference request and return the decoded action chunk."""
|
||||||
|
|
||||||
|
# Keep the upstream DreamZero/OpenPI convention that the request itself
|
||||||
|
# tells the server which logical endpoint is being called.
|
||||||
|
payload = dict(obs)
|
||||||
|
payload["endpoint"] = "infer"
|
||||||
|
self._ws.send(self._packer.pack(payload))
|
||||||
|
response = self._ws.recv()
|
||||||
|
return _decode_action_response(response)
|
||||||
|
|
||||||
|
@override
|
||||||
|
def reset(self, reset_info: dict[str, Any] | None = None) -> str:
|
||||||
|
"""Tell the server to reset its session-side state."""
|
||||||
|
|
||||||
|
payload = dict(reset_info or {})
|
||||||
|
payload["endpoint"] = "reset"
|
||||||
|
self._ws.send(self._packer.pack(payload))
|
||||||
|
response = self._ws.recv()
|
||||||
|
if isinstance(response, str):
|
||||||
|
return response
|
||||||
|
decoded = msgpack_numpy.unpackb(response)
|
||||||
|
if not isinstance(decoded, dict) or decoded.get("status") != "reset successful":
|
||||||
|
raise RuntimeError(f"Unexpected reset response: {decoded!r}")
|
||||||
|
return str(decoded["status"])
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
"""Close the websocket connection explicitly."""
|
||||||
|
|
||||||
|
self._ws.close()
|
||||||
|
|
||||||
|
|
||||||
|
class DreamZeroJointPosClient:
|
||||||
|
"""DROID sim-eval client that talks to the vLLM OpenPI websocket server.
|
||||||
|
|
||||||
|
This is the main compatibility layer between:
|
||||||
|
- Isaac Lab DROID observations (`obs["policy"][...]`)
|
||||||
|
- DreamZero/OpenPI websocket payloads
|
||||||
|
- local open-loop action reuse across several simulator steps
|
||||||
|
|
||||||
|
In other words:
|
||||||
|
simulator obs -> websocket request -> action chunk -> one action per step
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
remote_host: str = "127.0.0.1",
|
||||||
|
remote_port: int = 8000,
|
||||||
|
path: str = DEFAULT_PATH,
|
||||||
|
open_loop_horizon: int = DEFAULT_OPEN_LOOP_HORIZON,
|
||||||
|
) -> None:
|
||||||
|
# Low-level transport client.
|
||||||
|
self.client = OpenPIWebsocketClientPolicy(remote_host, remote_port, path=path)
|
||||||
|
# Number of actions to execute locally before replanning.
|
||||||
|
self.open_loop_horizon = open_loop_horizon
|
||||||
|
# Cursor into the currently cached action chunk.
|
||||||
|
self.actions_from_chunk_completed = 0
|
||||||
|
# Most recent `(ACTION_HORIZON, ACTION_DIM)` server response.
|
||||||
|
self.pred_action_chunk: np.ndarray | None = None
|
||||||
|
# Session id is part of the DreamZero serving contract. Changing it
|
||||||
|
# causes the server side to treat the rollout as a fresh episode.
|
||||||
|
self.session_id = str(uuid.uuid4())
|
||||||
|
# Simple runtime stats for reporting.
|
||||||
|
self.server_calls = 0
|
||||||
|
self.last_chunk_latency_s: float | None = None
|
||||||
|
self.last_used_server_call = False
|
||||||
|
|
||||||
|
def metadata(self) -> dict[str, Any]:
|
||||||
|
"""Expose the server metadata to callers / logs."""
|
||||||
|
|
||||||
|
return self.client.get_server_metadata()
|
||||||
|
|
||||||
|
def reset(self) -> str:
|
||||||
|
"""Reset local chunk state and remote session state.
|
||||||
|
|
||||||
|
Local reset:
|
||||||
|
- drop cached action chunk
|
||||||
|
- rewind chunk cursor
|
||||||
|
- allocate a fresh session id
|
||||||
|
|
||||||
|
Remote reset:
|
||||||
|
- send a websocket `reset` message so the server can clear any
|
||||||
|
request/session-side state it associates with this client
|
||||||
|
"""
|
||||||
|
|
||||||
|
self.actions_from_chunk_completed = 0
|
||||||
|
self.pred_action_chunk = None
|
||||||
|
self.session_id = str(uuid.uuid4())
|
||||||
|
self.last_chunk_latency_s = None
|
||||||
|
self.last_used_server_call = False
|
||||||
|
return self.client.reset({})
|
||||||
|
|
||||||
|
def infer(self, obs: dict[str, Any], instruction: str) -> dict[str, Any]:
|
||||||
|
"""Turn one simulator observation into one executable 8-D action.
|
||||||
|
|
||||||
|
Key behavior:
|
||||||
|
- call the server only when the local chunk cache is empty/exhausted
|
||||||
|
- otherwise, keep consuming the cached chunk open-loop
|
||||||
|
- always return exactly one 8-D action for the current simulator step
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Convert Isaac Lab observation structure into a plain numpy-friendly
|
||||||
|
# record that is easier to serialize and visualize.
|
||||||
|
curr_obs = self._extract_observation(obs)
|
||||||
|
self.last_used_server_call = False
|
||||||
|
|
||||||
|
# Replan if:
|
||||||
|
# 1. this is the first step of a rollout / chunk
|
||||||
|
# 2. we already consumed `open_loop_horizon` actions from the current chunk
|
||||||
|
# 3. no cached chunk is currently available
|
||||||
|
if (
|
||||||
|
self.actions_from_chunk_completed == 0
|
||||||
|
or self.actions_from_chunk_completed >= self.open_loop_horizon
|
||||||
|
or self.pred_action_chunk is None
|
||||||
|
):
|
||||||
|
self.actions_from_chunk_completed = 0
|
||||||
|
# Build the exact DreamZero/OpenPI payload expected by the server.
|
||||||
|
#
|
||||||
|
# Notes:
|
||||||
|
# - images are resized/padded to the serving contract's 180x320
|
||||||
|
# - proprio is cast to float64 to match upstream client behavior
|
||||||
|
# - cartesian_position is currently unused by DreamZero DROID, so
|
||||||
|
# a dummy zero vector is sent for protocol completeness
|
||||||
|
request_data = {
|
||||||
|
"observation/exterior_image_0_left": image_tools.resize_with_pad(curr_obs["right_image"], 180, 320),
|
||||||
|
"observation/exterior_image_1_left": image_tools.resize_with_pad(curr_obs["left_image"], 180, 320),
|
||||||
|
"observation/wrist_image_left": image_tools.resize_with_pad(curr_obs["wrist_image"], 180, 320),
|
||||||
|
"observation/joint_position": curr_obs["joint_position"].astype(np.float64),
|
||||||
|
"observation/cartesian_position": np.zeros((6,), dtype=np.float64),
|
||||||
|
"observation/gripper_position": curr_obs["gripper_position"].astype(np.float64),
|
||||||
|
"prompt": instruction,
|
||||||
|
"session_id": self.session_id,
|
||||||
|
}
|
||||||
|
|
||||||
|
# Measure end-to-end server latency for this chunk request.
|
||||||
|
start = time.perf_counter()
|
||||||
|
actions = self.client.infer(request_data)
|
||||||
|
self.last_chunk_latency_s = time.perf_counter() - start
|
||||||
|
self.last_used_server_call = True
|
||||||
|
self.server_calls += 1
|
||||||
|
|
||||||
|
# DreamZero DROID serving is expected to return an action chunk with
|
||||||
|
# 24 future actions, each action being 8-D.
|
||||||
|
if actions.ndim != 2:
|
||||||
|
raise AssertionError(f"Expected 2D action array, got shape {actions.shape}")
|
||||||
|
if actions.shape != (ACTION_HORIZON, ACTION_DIM):
|
||||||
|
raise AssertionError(f"Expected action shape {(ACTION_HORIZON, ACTION_DIM)}, got {actions.shape}")
|
||||||
|
self.pred_action_chunk = actions
|
||||||
|
|
||||||
|
# Consume exactly one action row from the cached chunk for this
|
||||||
|
# simulator step.
|
||||||
|
action = np.array(self.pred_action_chunk[self.actions_from_chunk_completed], copy=True)
|
||||||
|
self.actions_from_chunk_completed += 1
|
||||||
|
|
||||||
|
# Upstream DreamZero sim-eval binarizes the gripper command.
|
||||||
|
action[-1] = 1.0 if action[-1].item() > 0.5 else 0.0
|
||||||
|
|
||||||
|
# Produce a human-friendly visualization strip for videos:
|
||||||
|
# right external | wrist | left external
|
||||||
|
img1 = image_tools.resize_with_pad(curr_obs["right_image"], 224, 224)
|
||||||
|
img2 = image_tools.resize_with_pad(curr_obs["wrist_image"], 224, 224)
|
||||||
|
img3 = image_tools.resize_with_pad(curr_obs["left_image"], 224, 224)
|
||||||
|
viz = np.concatenate([img1, img2, img3], axis=1)
|
||||||
|
|
||||||
|
# Return both the executable action and auxiliary debug info.
|
||||||
|
return {
|
||||||
|
"action": action,
|
||||||
|
"viz": viz,
|
||||||
|
"joint_position": curr_obs["joint_position"],
|
||||||
|
"gripper_position": curr_obs["gripper_position"],
|
||||||
|
"used_server_call": self.last_used_server_call,
|
||||||
|
"chunk_latency_s": self.last_chunk_latency_s if self.last_used_server_call else None,
|
||||||
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _extract_observation(obs_dict: dict[str, Any]) -> dict[str, np.ndarray]:
|
||||||
|
"""Extract the pieces DreamZero cares about from Isaac Lab observations.
|
||||||
|
|
||||||
|
`sim-evals` exposes camera frames and robot state inside the
|
||||||
|
`obs["policy"]` group. This helper converts those tensors into numpy
|
||||||
|
arrays so they can be fed into image preprocessing / websocket packing.
|
||||||
|
"""
|
||||||
|
|
||||||
|
policy = obs_dict["policy"]
|
||||||
|
# Isaac Lab stores camera observations as batched tensors; use env 0.
|
||||||
|
right_image = policy["external_cam"][0].clone().detach().cpu().numpy()
|
||||||
|
left_image = policy["external_cam_2"][0].clone().detach().cpu().numpy()
|
||||||
|
wrist_image = policy["wrist_cam"][0].clone().detach().cpu().numpy()
|
||||||
|
# Robot proprioception.
|
||||||
|
joint_position = policy["arm_joint_pos"].clone().detach().cpu().numpy()
|
||||||
|
gripper_position = policy["gripper_pos"].clone().detach().cpu().numpy()
|
||||||
|
|
||||||
|
return {
|
||||||
|
"right_image": right_image,
|
||||||
|
"left_image": left_image,
|
||||||
|
"wrist_image": wrist_image,
|
||||||
|
"joint_position": joint_position,
|
||||||
|
"gripper_position": gripper_position,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _scene_instruction(scene: int) -> str:
|
||||||
|
"""Map a numeric scene id onto the fixed language prompt used for rollout."""
|
||||||
|
|
||||||
|
try:
|
||||||
|
return SCENE_PROMPTS[scene]
|
||||||
|
except KeyError as exc:
|
||||||
|
raise ValueError(f"Unsupported scene {scene}. Available scenes: {sorted(SCENE_PROMPTS)}") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def _capture_scene_objects(env: Any) -> dict[str, list[float]]:
|
||||||
|
"""Best-effort extraction of scene object root positions.
|
||||||
|
|
||||||
|
This is only for debugging / reporting. The rollout logic itself does not
|
||||||
|
depend on these positions.
|
||||||
|
"""
|
||||||
|
|
||||||
|
objects: dict[str, list[float]] = {}
|
||||||
|
scene = getattr(env, "scene", None)
|
||||||
|
if scene is None:
|
||||||
|
return objects
|
||||||
|
|
||||||
|
# Skip non-task entities such as cameras, the robot, and lighting.
|
||||||
|
for name in scene.keys():
|
||||||
|
if name in {"robot", "external_cam", "external_cam_2", "wrist_cam", "sphere_light", "scene"}:
|
||||||
|
continue
|
||||||
|
entity = scene[name]
|
||||||
|
data = getattr(entity, "data", None)
|
||||||
|
root_pos_w = getattr(data, "root_pos_w", None)
|
||||||
|
if root_pos_w is None:
|
||||||
|
continue
|
||||||
|
# Convert tensors to plain lists for JSON serialization.
|
||||||
|
value = root_pos_w[0].detach().cpu().to(torch.float32).tolist()
|
||||||
|
objects[str(name)] = [float(x) for x in value]
|
||||||
|
return objects
|
||||||
|
|
||||||
|
|
||||||
|
def _maybe_infer_success(scene: int, final_objects: dict[str, list[float]]) -> dict[str, Any]:
|
||||||
|
"""Best-effort geometric heuristic.
|
||||||
|
|
||||||
|
The simulator itself does not expose a built-in success term; this function
|
||||||
|
provides a transparent fallback for human-readable reporting only.
|
||||||
|
"""
|
||||||
|
|
||||||
|
task_pairs = {
|
||||||
|
1: ("cube", "bowl"),
|
||||||
|
2: ("can", "mug"),
|
||||||
|
3: ("banana", "bin"),
|
||||||
|
}
|
||||||
|
source_name, target_name = task_pairs.get(scene, (None, None))
|
||||||
|
if source_name not in final_objects or target_name not in final_objects:
|
||||||
|
return {
|
||||||
|
"has_builtin_success": False,
|
||||||
|
"heuristic_success": None,
|
||||||
|
"reason": "scene object names unavailable for heuristic",
|
||||||
|
}
|
||||||
|
|
||||||
|
source = np.asarray(final_objects[source_name], dtype=np.float32)
|
||||||
|
target = np.asarray(final_objects[target_name], dtype=np.float32)
|
||||||
|
xy_distance = float(np.linalg.norm(source[:2] - target[:2]))
|
||||||
|
z_delta = float(source[2] - target[2])
|
||||||
|
heuristic_success = bool(xy_distance < 0.12 and z_delta > -0.08)
|
||||||
|
return {
|
||||||
|
"has_builtin_success": False,
|
||||||
|
"heuristic_success": heuristic_success,
|
||||||
|
"xy_distance": xy_distance,
|
||||||
|
"z_delta": z_delta,
|
||||||
|
"source_object": source_name,
|
||||||
|
"target_object": target_name,
|
||||||
|
"reason": "no built-in env success flag; using final object pose heuristic",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _dump_json(path: Path, payload: dict[str, Any]) -> None:
|
||||||
|
"""Write a JSON file with stable UTF-8 formatting."""
|
||||||
|
|
||||||
|
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _scalar_from_env_value(value: Any) -> float:
|
||||||
|
"""Normalize simulator scalar outputs into a plain Python float.
|
||||||
|
|
||||||
|
Isaac Lab / Gym values may come back as tensors, numpy arrays, tuples, or
|
||||||
|
direct Python scalars depending on the wrapper stack. Centralizing the
|
||||||
|
conversion here makes the rollout loop cleaner and more robust.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if isinstance(value, torch.Tensor):
|
||||||
|
return float(value.reshape(-1)[0].detach().cpu().item())
|
||||||
|
if isinstance(value, np.ndarray):
|
||||||
|
return float(value.reshape(-1)[0])
|
||||||
|
if isinstance(value, (list, tuple)):
|
||||||
|
return float(np.asarray(value).reshape(-1)[0])
|
||||||
|
return float(value)
|
||||||
|
|
||||||
|
|
||||||
|
def _bool_from_env_value(value: Any) -> bool:
|
||||||
|
"""Normalize simulator boolean-like outputs into a plain Python bool."""
|
||||||
|
|
||||||
|
if isinstance(value, torch.Tensor):
|
||||||
|
return bool(value.reshape(-1)[0].detach().cpu().item())
|
||||||
|
if isinstance(value, np.ndarray):
|
||||||
|
return bool(value.reshape(-1)[0])
|
||||||
|
if isinstance(value, (list, tuple)):
|
||||||
|
return bool(np.asarray(value).reshape(-1)[0])
|
||||||
|
return bool(value)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_output_dir(output_root: Path, scene: int) -> Path:
|
||||||
|
"""Create a timestamped output directory for one sim-eval run."""
|
||||||
|
|
||||||
|
timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
||||||
|
output_dir = output_root / f"scene{scene}_{timestamp}"
|
||||||
|
output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
return output_dir
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
"""Entry point for one or more DROID simulation rollouts.
|
||||||
|
|
||||||
|
High-level flow:
|
||||||
|
1. parse command-line flags
|
||||||
|
2. bootstrap Isaac Lab / sim-evals imports
|
||||||
|
3. create the DROID environment
|
||||||
|
4. connect the DreamZero websocket client
|
||||||
|
5. run `episodes` rollouts
|
||||||
|
6. export videos + JSON summaries
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Script-level arguments.
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--episodes", type=int, default=1, help="Number of episodes to run.")
|
||||||
|
parser.add_argument("--scene", type=int, default=1, help="DROID scene id (1/2/3).")
|
||||||
|
parser.add_argument("--host", type=str, default="127.0.0.1", help="vLLM DreamZero server host.")
|
||||||
|
parser.add_argument("--port", type=int, default=8000, help="vLLM DreamZero server port.")
|
||||||
|
parser.add_argument("--path", type=str, default=DEFAULT_PATH, help="Websocket path suffix.")
|
||||||
|
parser.add_argument(
|
||||||
|
"--open-loop-horizon",
|
||||||
|
type=int,
|
||||||
|
default=DEFAULT_OPEN_LOOP_HORIZON,
|
||||||
|
help="How many actions to consume locally before requesting the next chunk.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--output-root",
|
||||||
|
type=Path,
|
||||||
|
default=DEFAULT_OUTPUT_ROOT,
|
||||||
|
help="Directory where videos and trajectory logs are stored.",
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
from isaaclab.app import AppLauncher
|
||||||
|
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
||||||
|
raise ImportError(
|
||||||
|
"Optional DROID sim-eval client requires Isaac Lab (`isaaclab`). "
|
||||||
|
"Launch it from an Isaac Lab environment, e.g. via `isaaclab.sh -p`."
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
# Let Isaac Lab inject its own runtime flags (e.g. headless, device).
|
||||||
|
AppLauncher.add_app_launcher_args(parser)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
# DreamZero sim-eval always needs camera observations enabled.
|
||||||
|
args.enable_cameras = True
|
||||||
|
# Boot Isaac Sim / Isaac Lab.
|
||||||
|
app_launcher = AppLauncher(args)
|
||||||
|
simulation_app = app_launcher.app
|
||||||
|
# Set defaults so the `finally` block can clean up safely even if an
|
||||||
|
# earlier step fails.
|
||||||
|
env = None
|
||||||
|
client = None
|
||||||
|
|
||||||
|
# Import simulator modules only *after* the app is launched. This matches
|
||||||
|
# Isaac Lab's required import ordering.
|
||||||
|
try:
|
||||||
|
import gymnasium as gym
|
||||||
|
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
||||||
|
raise ImportError("Optional DROID sim-eval client requires `gymnasium`.") from exc
|
||||||
|
|
||||||
|
try:
|
||||||
|
import sim_evals.environments # noqa: F401
|
||||||
|
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
||||||
|
raise ImportError(
|
||||||
|
"Optional DROID sim-eval client requires the external `sim-evals` package or checkout to be importable."
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
try:
|
||||||
|
from isaaclab_tasks.utils import parse_env_cfg
|
||||||
|
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
||||||
|
raise ImportError("Optional DROID sim-eval client requires `isaaclab_tasks`.") from exc
|
||||||
|
|
||||||
|
# Resolve output location and scene prompt.
|
||||||
|
output_dir = _make_output_dir(args.output_root.expanduser().resolve(), args.scene)
|
||||||
|
instruction = _scene_instruction(args.scene)
|
||||||
|
|
||||||
|
# Build the DROID environment configuration from `sim-evals`.
|
||||||
|
env_cfg = parse_env_cfg(
|
||||||
|
"DROID",
|
||||||
|
device=args.device,
|
||||||
|
num_envs=1,
|
||||||
|
use_fabric=True,
|
||||||
|
)
|
||||||
|
# Select one of the pre-authored scenes/tasks.
|
||||||
|
env_cfg.set_scene(args.scene)
|
||||||
|
env = gym.make("DROID", cfg=env_cfg)
|
||||||
|
|
||||||
|
# Upstream sim-evals resets twice so materials / cameras are fully ready.
|
||||||
|
obs, _ = env.reset()
|
||||||
|
obs, _ = env.reset()
|
||||||
|
|
||||||
|
# Connect the websocket policy client.
|
||||||
|
client = DreamZeroJointPosClient(
|
||||||
|
remote_host=args.host,
|
||||||
|
remote_port=args.port,
|
||||||
|
path=args.path,
|
||||||
|
open_loop_horizon=args.open_loop_horizon,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Aggregated per-run results.
|
||||||
|
all_episode_summaries: list[dict[str, Any]] = []
|
||||||
|
max_steps = int(env.env.max_episode_length)
|
||||||
|
logging.info("DreamZero metadata: %s", client.metadata())
|
||||||
|
logging.info("Scene %s prompt: %s", args.scene, instruction)
|
||||||
|
logging.info("Writing outputs to %s", output_dir)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# No gradients are needed in inference-only rollout mode.
|
||||||
|
with torch.no_grad():
|
||||||
|
for episode_index in range(args.episodes):
|
||||||
|
# Per-episode collectors.
|
||||||
|
frames: list[np.ndarray] = []
|
||||||
|
step_records: list[StepRecord] = []
|
||||||
|
episode_start = time.perf_counter()
|
||||||
|
server_time_s = 0.0
|
||||||
|
final_reward = 0.0
|
||||||
|
terminated = False
|
||||||
|
truncated = False
|
||||||
|
|
||||||
|
for step_index in range(max_steps):
|
||||||
|
# Ask the policy for the next action. Internally this may or
|
||||||
|
# may not trigger a real server request depending on whether
|
||||||
|
# the local chunk cache has been exhausted.
|
||||||
|
logging.debug("Episode %d step %d: requesting action", episode_index, step_index)
|
||||||
|
result = client.infer(obs, instruction)
|
||||||
|
logging.debug(
|
||||||
|
"Episode %d step %d: got action (server_call=%s latency=%s)",
|
||||||
|
episode_index,
|
||||||
|
step_index,
|
||||||
|
result["used_server_call"],
|
||||||
|
result["chunk_latency_s"],
|
||||||
|
)
|
||||||
|
# Save one visualization frame per simulator step.
|
||||||
|
frames.append(result["viz"])
|
||||||
|
|
||||||
|
# Isaac Lab expects batched actions, hence `[None]`.
|
||||||
|
action_tensor = torch.tensor(result["action"], dtype=torch.float32)[None]
|
||||||
|
logging.debug("Episode %d step %d: stepping env", episode_index, step_index)
|
||||||
|
obs, reward, term, trunc, info = env.step(action_tensor)
|
||||||
|
logging.debug("Episode %d step %d: env.step returned", episode_index, step_index)
|
||||||
|
logging.debug("Episode %d step %d: parsing reward/flags", episode_index, step_index)
|
||||||
|
logging.debug(
|
||||||
|
"Episode %d step %d: raw types reward=%s term=%s trunc=%s",
|
||||||
|
episode_index,
|
||||||
|
step_index,
|
||||||
|
type(reward).__name__,
|
||||||
|
type(term).__name__,
|
||||||
|
type(trunc).__name__,
|
||||||
|
)
|
||||||
|
# Normalize environment outputs into plain Python scalars so
|
||||||
|
# the rest of the code does not depend on wrapper-specific types.
|
||||||
|
reward_value = _scalar_from_env_value(reward)
|
||||||
|
term_value = _bool_from_env_value(term)
|
||||||
|
trunc_value = _bool_from_env_value(trunc)
|
||||||
|
logging.debug(
|
||||||
|
"Episode %d step %d: parsed reward=%s term=%s trunc=%s",
|
||||||
|
episode_index,
|
||||||
|
step_index,
|
||||||
|
reward_value,
|
||||||
|
term_value,
|
||||||
|
trunc_value,
|
||||||
|
)
|
||||||
|
# Keep scene-object capture optional. It is useful for
|
||||||
|
# debugging / success heuristics, but the rollout should not
|
||||||
|
# fail if the environment does not expose object roots.
|
||||||
|
scene_objects = _capture_scene_objects(env)
|
||||||
|
|
||||||
|
# Accumulate total server-side time only on steps that
|
||||||
|
# triggered a fresh chunk inference.
|
||||||
|
if result["chunk_latency_s"] is not None:
|
||||||
|
server_time_s += float(result["chunk_latency_s"])
|
||||||
|
|
||||||
|
# Materialize one JSON-serializable trajectory record.
|
||||||
|
logging.debug("Episode %d step %d: appending trajectory", episode_index, step_index)
|
||||||
|
step_records.append(
|
||||||
|
StepRecord(
|
||||||
|
step_index=step_index,
|
||||||
|
used_server_call=bool(result["used_server_call"]),
|
||||||
|
chunk_latency_s=(
|
||||||
|
float(result["chunk_latency_s"]) if result["chunk_latency_s"] is not None else None
|
||||||
|
),
|
||||||
|
action=[float(x) for x in np.asarray(result["action"], dtype=np.float32).tolist()],
|
||||||
|
joint_position=[
|
||||||
|
float(x) for x in np.asarray(result["joint_position"], dtype=np.float32).tolist()
|
||||||
|
],
|
||||||
|
gripper_position=[
|
||||||
|
float(x) for x in np.asarray(result["gripper_position"], dtype=np.float32).tolist()
|
||||||
|
],
|
||||||
|
reward=reward_value,
|
||||||
|
terminated=term_value,
|
||||||
|
truncated=trunc_value,
|
||||||
|
scene_objects=scene_objects,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
logging.debug("Episode %d step %d: trajectory appended", episode_index, step_index)
|
||||||
|
|
||||||
|
# Track final status for summary export.
|
||||||
|
final_reward = reward_value
|
||||||
|
terminated = term_value
|
||||||
|
truncated = trunc_value
|
||||||
|
if term_value or trunc_value:
|
||||||
|
# End the rollout early if the environment terminates.
|
||||||
|
break
|
||||||
|
|
||||||
|
# Episode-level timing and video export.
|
||||||
|
episode_wall_time_s = time.perf_counter() - episode_start
|
||||||
|
video_path = output_dir / f"episode_{episode_index:02d}.mp4"
|
||||||
|
logging.info("Episode %d: writing video to %s", episode_index, video_path)
|
||||||
|
mediapy.write_video(video_path, frames, fps=15)
|
||||||
|
|
||||||
|
# Reset the policy server between episodes.
|
||||||
|
logging.info("Episode %d: sending reset", episode_index)
|
||||||
|
reset_response = client.reset()
|
||||||
|
final_objects = step_records[-1].scene_objects if step_records else {}
|
||||||
|
success_report = _maybe_infer_success(args.scene, final_objects)
|
||||||
|
|
||||||
|
# Assemble the per-episode summary that is written to
|
||||||
|
# `episode_XX.json`.
|
||||||
|
episode_summary = {
|
||||||
|
"episode_index": episode_index,
|
||||||
|
"prompt": instruction,
|
||||||
|
"video_path": str(video_path),
|
||||||
|
"steps_executed": len(step_records),
|
||||||
|
"max_steps": max_steps,
|
||||||
|
"terminated": terminated,
|
||||||
|
"truncated": truncated,
|
||||||
|
"final_reward": final_reward,
|
||||||
|
"server_calls": client.server_calls,
|
||||||
|
"server_time_s": server_time_s,
|
||||||
|
"episode_wall_time_s": episode_wall_time_s,
|
||||||
|
"avg_server_time_per_call_s": (
|
||||||
|
server_time_s / client.server_calls if client.server_calls else None
|
||||||
|
),
|
||||||
|
"reset_response": reset_response,
|
||||||
|
"success_report": success_report,
|
||||||
|
"server_metadata": client.metadata(),
|
||||||
|
"trajectory": [record.__dict__ for record in step_records],
|
||||||
|
}
|
||||||
|
_dump_json(output_dir / f"episode_{episode_index:02d}.json", episode_summary)
|
||||||
|
all_episode_summaries.append(episode_summary)
|
||||||
|
|
||||||
|
logging.info(
|
||||||
|
"Episode %d done: steps=%d wall=%.2fs server_calls=%d heuristic_success=%s",
|
||||||
|
episode_index,
|
||||||
|
len(step_records),
|
||||||
|
episode_wall_time_s,
|
||||||
|
client.server_calls,
|
||||||
|
success_report.get("heuristic_success"),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Reset per-episode counters while keeping the client alive.
|
||||||
|
client.server_calls = 0
|
||||||
|
|
||||||
|
# Top-level run summary across all episodes.
|
||||||
|
summary = {
|
||||||
|
"scene": args.scene,
|
||||||
|
"prompt": instruction,
|
||||||
|
"episodes": args.episodes,
|
||||||
|
"host": args.host,
|
||||||
|
"port": args.port,
|
||||||
|
"path": args.path,
|
||||||
|
"device": args.device,
|
||||||
|
"output_dir": str(output_dir),
|
||||||
|
"server_metadata": client.metadata(),
|
||||||
|
"episodes_summary": all_episode_summaries,
|
||||||
|
}
|
||||||
|
_dump_json(output_dir / "summary.json", summary)
|
||||||
|
# Also print the summary to stdout so the caller can capture it in logs.
|
||||||
|
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||||
|
finally:
|
||||||
|
# Best-effort cleanup. Avoid masking the main error if cleanup fails.
|
||||||
|
try:
|
||||||
|
if client is not None:
|
||||||
|
client.client.close()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
if env is not None:
|
||||||
|
env.close()
|
||||||
|
simulation_app.close()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
# Keep the script-level logs readable. Per-step rollout details are still
|
||||||
|
# available via `DEBUG` if needed, but websocket / asyncio internals are
|
||||||
|
# usually too noisy for normal usage.
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
logging.getLogger("websockets").setLevel(logging.WARNING)
|
||||||
|
logging.getLogger("asyncio").setLevel(logging.WARNING)
|
||||||
|
main()
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
os.environ.setdefault("MUJOCO_GL", "egl")
|
||||||
|
os.environ.setdefault("PYOPENGL_PLATFORM", "egl")
|
||||||
|
|
||||||
|
# Import base configs at module top level so the subclasses below are pickle-
|
||||||
|
# resolvable (worker processes import this module fresh via __main__).
|
||||||
|
from molmo_spaces.configs.policy_configs_baselines import ( # noqa: E402
|
||||||
|
DreamZeroPolicyConfig,
|
||||||
|
)
|
||||||
|
from molmo_spaces.evaluation.configs.evaluation_configs import ( # noqa: E402
|
||||||
|
DreamZeroPolicyEvalConfig,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# We only need to change the backend host and port to the vllm-host!
|
||||||
|
class DreamZeroVllmOmniPolicyConfig(DreamZeroPolicyConfig):
|
||||||
|
remote_config: dict = dict(host="127.0.0.1", port=8000)
|
||||||
|
|
||||||
|
|
||||||
|
class DreamZeroVllmOmniEvalConfig(DreamZeroPolicyEvalConfig):
|
||||||
|
policy_config: DreamZeroVllmOmniPolicyConfig = DreamZeroVllmOmniPolicyConfig()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--host", default="127.0.0.1")
|
||||||
|
parser.add_argument("--port", type=int, default=8000)
|
||||||
|
parser.add_argument(
|
||||||
|
"--benchmark_dir",
|
||||||
|
required=True,
|
||||||
|
help=(
|
||||||
|
"Path to a MolmoSpaces benchmark directory, for example "
|
||||||
|
"$MOLMOSPACES_BENCHMARK_DIR/20260327/ithor/FrankaCloseHardBench/"
|
||||||
|
"FrankaCloseHardBench_20260206_json_benchmark"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
parser.add_argument("--max_episodes", type=int, default=1)
|
||||||
|
parser.add_argument("--task_horizon_steps", type=int, default=80)
|
||||||
|
parser.add_argument(
|
||||||
|
"--output_dir",
|
||||||
|
required=True,
|
||||||
|
help="Directory to write evaluation outputs (created if missing).",
|
||||||
|
)
|
||||||
|
parser.add_argument("--episode_idx", type=int, default=None)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
policy_config = DreamZeroVllmOmniPolicyConfig(remote_config=dict(host=args.host, port=args.port))
|
||||||
|
DreamZeroVllmOmniPolicyConfig.model_fields["remote_config"].default = policy_config.remote_config
|
||||||
|
DreamZeroVllmOmniEvalConfig.model_fields["policy_config"].default = policy_config
|
||||||
|
|
||||||
|
# Import after env vars are set so MuJoCo picks EGL.
|
||||||
|
from molmo_spaces.evaluation import run_evaluation
|
||||||
|
|
||||||
|
cfg_cls = DreamZeroVllmOmniEvalConfig
|
||||||
|
|
||||||
|
output_dir = args.output_dir
|
||||||
|
Path(output_dir).mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
print(f"[eval] benchmark_dir={args.benchmark_dir}")
|
||||||
|
print(f"[eval] max_episodes={args.max_episodes} task_horizon_steps={args.task_horizon_steps}")
|
||||||
|
print(f"[eval] remote policy: ws://{args.host}:{args.port}/v1/realtime/robot/openpi")
|
||||||
|
|
||||||
|
results = run_evaluation(
|
||||||
|
eval_config_cls=cfg_cls,
|
||||||
|
benchmark_dir=Path(args.benchmark_dir),
|
||||||
|
max_episodes=args.max_episodes,
|
||||||
|
task_horizon_steps=args.task_horizon_steps,
|
||||||
|
num_workers=1,
|
||||||
|
use_wandb=False,
|
||||||
|
output_dir=output_dir,
|
||||||
|
episode_idx=args.episode_idx,
|
||||||
|
)
|
||||||
|
|
||||||
|
print(f"[eval] success={results.success_count}/{results.total_count} ({results.success_rate:.1%})")
|
||||||
|
print(f"[eval] output_dir={results.output_dir}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
+311
@@ -0,0 +1,311 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import sys
|
||||||
|
import uuid
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
try:
|
||||||
|
import cv2
|
||||||
|
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
||||||
|
raise ImportError("DreamZero OpenPI example requires `opencv-python`.") from exc
|
||||||
|
|
||||||
|
try:
|
||||||
|
example_dir = str(Path(__file__).resolve().parent)
|
||||||
|
removed_path = False
|
||||||
|
if sys.path and sys.path[0] == example_dir:
|
||||||
|
sys.path.pop(0)
|
||||||
|
removed_path = True
|
||||||
|
try:
|
||||||
|
from openpi_client.websocket_client_policy import WebsocketClientPolicy
|
||||||
|
finally:
|
||||||
|
if removed_path:
|
||||||
|
sys.path.insert(0, example_dir)
|
||||||
|
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
||||||
|
raise ImportError("DreamZero OpenPI example requires `openpi-client`.") from exc
|
||||||
|
|
||||||
|
DEFAULT_HOST = "127.0.0.1"
|
||||||
|
DEFAULT_PORT = 8000
|
||||||
|
DEFAULT_PATH = "/v1/realtime/robot/openpi"
|
||||||
|
DEFAULT_PROMPT = "Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan"
|
||||||
|
ACTION_HORIZON = 24
|
||||||
|
DEFAULT_ACTION_DIM = 8
|
||||||
|
RELATIVE_OFFSETS = [-23, -16, -8, 0]
|
||||||
|
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||||
|
ASSET_REPO_ID = "YangshenDeng/vllm-omni-dreamzero-assets"
|
||||||
|
DEFAULT_VIDEO_DIR = REPO_ROOT / "outputs" / "dreamzero" / "assets"
|
||||||
|
CAMERA_FILES = {
|
||||||
|
"observation/exterior_image_0_left": "exterior_image_1_left.mp4",
|
||||||
|
"observation/exterior_image_1_left": "exterior_image_2_left.mp4",
|
||||||
|
"observation/wrist_image_left": "wrist_image_left.mp4",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _as_action_array(response: Any) -> np.ndarray:
|
||||||
|
if isinstance(response, dict) and response.get("type") == "error":
|
||||||
|
message = response.get("message", response)
|
||||||
|
raise RuntimeError(f"Inference failed: {message}")
|
||||||
|
return np.asarray(response, dtype=np.float32)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class DreamZeroServerMetadata:
|
||||||
|
image_resolution: tuple[int, int]
|
||||||
|
n_external_cameras: int
|
||||||
|
needs_wrist_camera: bool
|
||||||
|
needs_stereo_camera: bool
|
||||||
|
needs_session_id: bool
|
||||||
|
action_space: str
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_dict(cls, payload: dict[str, Any]) -> DreamZeroServerMetadata:
|
||||||
|
required_keys = (
|
||||||
|
"image_resolution",
|
||||||
|
"n_external_cameras",
|
||||||
|
"needs_wrist_camera",
|
||||||
|
"needs_stereo_camera",
|
||||||
|
"needs_session_id",
|
||||||
|
"action_space",
|
||||||
|
)
|
||||||
|
missing_keys = [key for key in required_keys if key not in payload]
|
||||||
|
if missing_keys:
|
||||||
|
raise ValueError(f"Missing DreamZero metadata keys: {missing_keys}")
|
||||||
|
|
||||||
|
image_resolution = payload["image_resolution"]
|
||||||
|
if not isinstance(image_resolution, (list, tuple)) or len(image_resolution) != 2:
|
||||||
|
raise ValueError(f"Invalid image_resolution: {image_resolution!r}")
|
||||||
|
|
||||||
|
return cls(
|
||||||
|
image_resolution=(int(image_resolution[0]), int(image_resolution[1])),
|
||||||
|
n_external_cameras=int(payload["n_external_cameras"]),
|
||||||
|
needs_wrist_camera=bool(payload["needs_wrist_camera"]),
|
||||||
|
needs_stereo_camera=bool(payload["needs_stereo_camera"]),
|
||||||
|
needs_session_id=bool(payload["needs_session_id"]),
|
||||||
|
action_space=str(payload["action_space"]),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _openpi_uri(host: str, port: int, path: str) -> str:
|
||||||
|
if host.startswith(("ws://", "wss://")):
|
||||||
|
return host
|
||||||
|
return f"ws://{host}:{port}{path}"
|
||||||
|
|
||||||
|
|
||||||
|
def _close_policy_client(client: WebsocketClientPolicy) -> None:
|
||||||
|
ws = getattr(client, "_ws", None)
|
||||||
|
if ws is not None:
|
||||||
|
ws.close()
|
||||||
|
|
||||||
|
|
||||||
|
def load_all_frames(video_path: Path) -> np.ndarray:
|
||||||
|
cap = cv2.VideoCapture(str(video_path))
|
||||||
|
frames = []
|
||||||
|
while True:
|
||||||
|
ok, frame = cap.read()
|
||||||
|
if not ok:
|
||||||
|
break
|
||||||
|
frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
|
||||||
|
cap.release()
|
||||||
|
if not frames:
|
||||||
|
raise RuntimeError(f"No frames loaded from {video_path}")
|
||||||
|
return np.stack(frames, axis=0)
|
||||||
|
|
||||||
|
|
||||||
|
def load_camera_frames(video_dir: Path) -> dict[str, np.ndarray]:
|
||||||
|
camera_frames: dict[str, np.ndarray] = {}
|
||||||
|
for camera_key, file_name in CAMERA_FILES.items():
|
||||||
|
video_path = video_dir / file_name
|
||||||
|
if not video_path.exists():
|
||||||
|
raise FileNotFoundError(
|
||||||
|
f"Missing DreamZero example asset: {video_path}. "
|
||||||
|
"Download the example videos with: "
|
||||||
|
f"`hf download {ASSET_REPO_ID} --repo-type dataset --local-dir {video_dir}`"
|
||||||
|
)
|
||||||
|
camera_frames[camera_key] = load_all_frames(video_path)
|
||||||
|
return camera_frames
|
||||||
|
|
||||||
|
|
||||||
|
def build_frame_schedule(total_frames: int, num_chunks: int) -> list[list[int]]:
|
||||||
|
chunks: list[list[int]] = []
|
||||||
|
current_frame = 23
|
||||||
|
for _ in range(num_chunks):
|
||||||
|
indices = [max(current_frame + offset, 0) for offset in RELATIVE_OFFSETS]
|
||||||
|
if indices[-1] >= total_frames:
|
||||||
|
break
|
||||||
|
chunks.append(indices)
|
||||||
|
current_frame += ACTION_HORIZON
|
||||||
|
return chunks
|
||||||
|
|
||||||
|
|
||||||
|
def make_obs_from_video(
|
||||||
|
camera_frames: dict[str, np.ndarray],
|
||||||
|
frame_indices: list[int],
|
||||||
|
*,
|
||||||
|
prompt: str,
|
||||||
|
session_id: str,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
obs: dict[str, Any] = {}
|
||||||
|
for camera_key, all_frames in camera_frames.items():
|
||||||
|
selected = all_frames[frame_indices]
|
||||||
|
obs[camera_key] = selected[0] if len(frame_indices) == 1 else selected
|
||||||
|
|
||||||
|
obs["observation/joint_position"] = np.zeros(7, dtype=np.float32)
|
||||||
|
obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32)
|
||||||
|
obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32)
|
||||||
|
obs["prompt"] = prompt
|
||||||
|
obs["session_id"] = session_id
|
||||||
|
return obs
|
||||||
|
|
||||||
|
|
||||||
|
def build_demo_observations(
|
||||||
|
camera_frames: dict[str, np.ndarray],
|
||||||
|
*,
|
||||||
|
prompt: str,
|
||||||
|
session_id: str,
|
||||||
|
num_chunks: int = 2,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
if num_chunks < 1:
|
||||||
|
raise ValueError("num_chunks must be at least 1")
|
||||||
|
|
||||||
|
total_frames = min(frames.shape[0] for frames in camera_frames.values())
|
||||||
|
observations = [
|
||||||
|
make_obs_from_video(
|
||||||
|
camera_frames,
|
||||||
|
[0],
|
||||||
|
prompt=prompt,
|
||||||
|
session_id=session_id,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
for indices in build_frame_schedule(total_frames, num_chunks - 1):
|
||||||
|
observations.append(
|
||||||
|
make_obs_from_video(
|
||||||
|
camera_frames,
|
||||||
|
indices,
|
||||||
|
prompt=prompt,
|
||||||
|
session_id=session_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return observations
|
||||||
|
|
||||||
|
|
||||||
|
def validate_session_result(
|
||||||
|
result: dict[str, Any],
|
||||||
|
*,
|
||||||
|
expected_action_horizon: int = ACTION_HORIZON,
|
||||||
|
expected_action_dim: int = DEFAULT_ACTION_DIM,
|
||||||
|
expected_action_count: int | None = None,
|
||||||
|
) -> None:
|
||||||
|
metadata = DreamZeroServerMetadata.from_dict(result["metadata"])
|
||||||
|
if metadata.image_resolution != (180, 320):
|
||||||
|
raise AssertionError(f"Unexpected image_resolution: {metadata.image_resolution}")
|
||||||
|
if metadata.n_external_cameras != 2:
|
||||||
|
raise AssertionError(f"Unexpected n_external_cameras: {metadata.n_external_cameras}")
|
||||||
|
if not metadata.needs_wrist_camera:
|
||||||
|
raise AssertionError("DreamZero example expects wrist camera metadata")
|
||||||
|
if metadata.action_space != "joint_position":
|
||||||
|
raise AssertionError(f"Unexpected action_space: {metadata.action_space}")
|
||||||
|
|
||||||
|
actions = result["actions"]
|
||||||
|
if expected_action_count is not None and len(actions) != expected_action_count:
|
||||||
|
raise AssertionError(f"Expected {expected_action_count} action tensors, got {len(actions)}")
|
||||||
|
if not actions:
|
||||||
|
raise AssertionError("Expected at least one action tensor")
|
||||||
|
for index, action in enumerate(actions):
|
||||||
|
if action.shape != (expected_action_horizon, expected_action_dim):
|
||||||
|
raise AssertionError(
|
||||||
|
f"Action {index} shape mismatch: expected "
|
||||||
|
f"{(expected_action_horizon, expected_action_dim)}, got {action.shape}"
|
||||||
|
)
|
||||||
|
if not np.isfinite(action).all():
|
||||||
|
raise AssertionError(f"Action {index} contains non-finite values")
|
||||||
|
|
||||||
|
|
||||||
|
def run_policy_session(
|
||||||
|
*,
|
||||||
|
host: str = DEFAULT_HOST,
|
||||||
|
port: int = DEFAULT_PORT,
|
||||||
|
path: str = DEFAULT_PATH,
|
||||||
|
video_dir: Path = DEFAULT_VIDEO_DIR,
|
||||||
|
prompt: str = DEFAULT_PROMPT,
|
||||||
|
session_id: str | None = None,
|
||||||
|
num_chunks: int = 2,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
session_id = session_id or str(uuid.uuid4())
|
||||||
|
camera_frames = load_camera_frames(video_dir)
|
||||||
|
observations = build_demo_observations(
|
||||||
|
camera_frames,
|
||||||
|
prompt=prompt,
|
||||||
|
session_id=session_id,
|
||||||
|
num_chunks=num_chunks,
|
||||||
|
)
|
||||||
|
|
||||||
|
uri = _openpi_uri(host, port, path)
|
||||||
|
logging.info("Connecting to %s", uri)
|
||||||
|
client = WebsocketClientPolicy(host=uri)
|
||||||
|
try:
|
||||||
|
metadata = dict(client.get_server_metadata())
|
||||||
|
actions = [_as_action_array(client.infer(obs)) for obs in observations]
|
||||||
|
return {
|
||||||
|
"metadata": metadata,
|
||||||
|
"actions": actions,
|
||||||
|
"session_id": session_id,
|
||||||
|
}
|
||||||
|
finally:
|
||||||
|
_close_policy_client(client)
|
||||||
|
|
||||||
|
|
||||||
|
def format_action_summary(index: int, action: np.ndarray) -> str:
|
||||||
|
return (
|
||||||
|
f"Action {index}: shape={tuple(action.shape)} dtype={action.dtype} "
|
||||||
|
f"min={action.min():.6f} max={action.max():.6f}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args() -> argparse.Namespace:
|
||||||
|
parser = argparse.ArgumentParser(description="DreamZero OpenPI client example with downloaded real videos.")
|
||||||
|
parser.add_argument("--host", default=DEFAULT_HOST)
|
||||||
|
parser.add_argument("--port", type=int, default=DEFAULT_PORT)
|
||||||
|
parser.add_argument("--path", default=DEFAULT_PATH)
|
||||||
|
parser.add_argument(
|
||||||
|
"--video-dir", type=Path, default=DEFAULT_VIDEO_DIR, help="Directory containing the three camera MP4 files."
|
||||||
|
)
|
||||||
|
parser.add_argument("--prompt", default=DEFAULT_PROMPT)
|
||||||
|
parser.add_argument("--session-id", default=None)
|
||||||
|
parser.add_argument("--num-chunks", type=int, default=2)
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
args = parse_args()
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
|
||||||
|
result = run_policy_session(
|
||||||
|
host=args.host,
|
||||||
|
port=args.port,
|
||||||
|
path=args.path,
|
||||||
|
video_dir=args.video_dir,
|
||||||
|
prompt=args.prompt,
|
||||||
|
session_id=args.session_id,
|
||||||
|
num_chunks=args.num_chunks,
|
||||||
|
)
|
||||||
|
validate_session_result(result, expected_action_count=args.num_chunks)
|
||||||
|
|
||||||
|
print("Server metadata:", json.dumps(result["metadata"], sort_keys=True))
|
||||||
|
for index, action in enumerate(result["actions"]):
|
||||||
|
print(format_action_summary(index, action))
|
||||||
|
print("Session ID:", result["session_id"])
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
+24
@@ -0,0 +1,24 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
MODEL="${MODEL:-GEAR-Dreams/DreamZero-DROID}"
|
||||||
|
HOST="${HOST:-127.0.0.1}"
|
||||||
|
PORT="${PORT:-8000}"
|
||||||
|
DEPLOY_CONFIG="${DEPLOY_CONFIG:-vllm_omni/deploy/dreamzero_tp1_cfg2.yaml}"
|
||||||
|
SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-dreamzero-droid}"
|
||||||
|
|
||||||
|
args=(
|
||||||
|
serve
|
||||||
|
"$MODEL"
|
||||||
|
--omni
|
||||||
|
--host "$HOST"
|
||||||
|
--port "$PORT"
|
||||||
|
--served-model-name "$SERVED_MODEL_NAME"
|
||||||
|
--deploy-config "$DEPLOY_CONFIG"
|
||||||
|
--enforce-eager
|
||||||
|
--disable-log-stats
|
||||||
|
)
|
||||||
|
|
||||||
|
ATTENTION_BACKEND="${ATTENTION_BACKEND:-torch}" \
|
||||||
|
DIFFUSION_ATTENTION_BACKEND="${DIFFUSION_ATTENTION_BACKEND:-TORCH_SDPA}" \
|
||||||
|
vllm "${args[@]}"
|
||||||
@@ -22,13 +22,14 @@ export PYTHONPATH=<transformers_modules_path>:$PYTHONPATH
|
|||||||
export HF_MODULES_CACHE=<transformers_modules_path>
|
export HF_MODULES_CACHE=<transformers_modules_path>
|
||||||
```
|
```
|
||||||
|
|
||||||
Run from repository root:
|
Run from repository root. The 3-stage pipeline is resolved automatically from
|
||||||
|
the model type (`vllm_omni/deploy/dynin_omni.yaml`), so no config path is
|
||||||
|
required. Pass `--deploy-config` only to override the defaults:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
vllm-omni serve snu-aidas/Dynin-Omni \
|
vllm-omni serve snu-aidas/Dynin-Omni \
|
||||||
--omni \
|
--omni \
|
||||||
--port 8091 \
|
--port 8091
|
||||||
--stage-configs-path "$(pwd)/vllm_omni/model_executor/stage_configs/dynin_omni.yaml"
|
|
||||||
```
|
```
|
||||||
|
|
||||||
If `vllm-omni` is not in PATH, run:
|
If `vllm-omni` is not in PATH, run:
|
||||||
@@ -36,8 +37,7 @@ If `vllm-omni` is not in PATH, run:
|
|||||||
```bash
|
```bash
|
||||||
PYTHONPATH="$(pwd)" python -m vllm_omni.entrypoints.cli.main serve snu-aidas/Dynin-Omni \
|
PYTHONPATH="$(pwd)" python -m vllm_omni.entrypoints.cli.main serve snu-aidas/Dynin-Omni \
|
||||||
--omni \
|
--omni \
|
||||||
--port 8091 \
|
--port 8091
|
||||||
--stage-configs-path "$(pwd)/vllm_omni/model_executor/stage_configs/dynin_omni.yaml"
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Wait until the server logs show both `All stages initialized successfully` and
|
Wait until the server logs show both `All stages initialized successfully` and
|
||||||
|
|||||||
@@ -198,7 +198,14 @@ def _build_prompt_for_query_type(
|
|||||||
return query_func(custom_prompt=custom_prompt)
|
return query_func(custom_prompt=custom_prompt)
|
||||||
if query_type == "use_audio_in_video":
|
if query_type == "use_audio_in_video":
|
||||||
return query_func(video_path=video_path, custom_prompt=custom_prompt)
|
return query_func(video_path=video_path, custom_prompt=custom_prompt)
|
||||||
# use_mixed_modalities / use_multi_audios
|
if query_type == "use_mixed_modalities":
|
||||||
|
return query_func(
|
||||||
|
video_path=video_path,
|
||||||
|
image_path=image_path,
|
||||||
|
audio_path=audio_path,
|
||||||
|
custom_prompt=custom_prompt,
|
||||||
|
)
|
||||||
|
# use_multi_audios
|
||||||
return query_func(custom_prompt=custom_prompt)
|
return query_func(custom_prompt=custom_prompt)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,40 @@
|
|||||||
|
# Higgs-Audio V3 Online Serving
|
||||||
|
|
||||||
|
## Start the server
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Default: GPU 0, port 8095
|
||||||
|
./examples/online_serving/text_to_speech/higgs_audio_v3/run_server.sh
|
||||||
|
|
||||||
|
# Custom GPU / port
|
||||||
|
PORT=8096 GPUS=0,1 ./examples/online_serving/text_to_speech/higgs_audio_v3/run_server.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
## Plain text TTS
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python examples/online_serving/text_to_speech/higgs_audio_v3/batch_speech_client.py \
|
||||||
|
--base-url http://localhost:8095 \
|
||||||
|
--output-dir /tmp/higgs_v3_batch \
|
||||||
|
--prompts "Hello world." "The quick brown fox jumps over the lazy dog."
|
||||||
|
```
|
||||||
|
|
||||||
|
## Voice clone
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python examples/online_serving/text_to_speech/higgs_audio_v3/batch_speech_client.py \
|
||||||
|
--base-url http://localhost:8095 \
|
||||||
|
--output-dir /tmp/higgs_v3_clone \
|
||||||
|
--ref-audio path/to/reference.wav \
|
||||||
|
--ref-text "transcript of the reference clip" \
|
||||||
|
--prompts "Text to synthesize in the cloned voice."
|
||||||
|
```
|
||||||
|
|
||||||
|
## curl example
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -X POST http://localhost:8095/v1/audio/speech \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{"model": "higgs_audio_v3", "input": "Hello world."}' \
|
||||||
|
--output hello.wav
|
||||||
|
```
|
||||||
@@ -0,0 +1,115 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Batch client for the higgs-audio v3 online server.
|
||||||
|
|
||||||
|
Sends prompts to ``/v1/audio/speech`` and saves the returned WAV files.
|
||||||
|
|
||||||
|
Usage (plain text -> speech):
|
||||||
|
|
||||||
|
python examples/online_serving/text_to_speech/higgs_audio_v3/batch_speech_client.py \
|
||||||
|
--base-url http://localhost:8095 \
|
||||||
|
--output-dir /tmp/higgs_v3_batch \
|
||||||
|
--prompts "Hello world." "The quick brown fox jumps over the lazy dog."
|
||||||
|
|
||||||
|
Usage (voice clone):
|
||||||
|
|
||||||
|
python examples/online_serving/text_to_speech/higgs_audio_v3/batch_speech_client.py \
|
||||||
|
--base-url http://localhost:8095 \
|
||||||
|
--output-dir /tmp/higgs_v3_clone \
|
||||||
|
--ref-audio path/to/reference.wav \
|
||||||
|
--ref-text "the transcript of the reference clip" \
|
||||||
|
--prompts "Hello world."
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import base64
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
DEFAULT_PROMPTS = (
|
||||||
|
"Hello world.",
|
||||||
|
"The quick brown fox jumps over the lazy dog.",
|
||||||
|
"Today is a beautiful day for a walk in the park.",
|
||||||
|
"Innovation distinguishes between a leader and a follower.",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _slug(text: str) -> str:
|
||||||
|
import re
|
||||||
|
|
||||||
|
s = re.sub(r"\s+", "_", text.strip().lower())
|
||||||
|
return re.sub(r"[^a-z0-9_]+", "", s)[:32] or "prompt"
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||||
|
parser.add_argument("--base-url", default="http://localhost:8095")
|
||||||
|
parser.add_argument("--model", default="higgs_audio_v3")
|
||||||
|
parser.add_argument("--prompts", nargs="+", default=list(DEFAULT_PROMPTS))
|
||||||
|
parser.add_argument("--output-dir", type=Path, default=Path("/tmp/higgs_v3_batch"))
|
||||||
|
parser.add_argument("--format", choices=("wav", "pcm"), default="wav")
|
||||||
|
parser.add_argument("--max-new-tokens", type=int, default=2048)
|
||||||
|
parser.add_argument("--seed", type=int, default=42)
|
||||||
|
parser.add_argument("--timeout-s", type=float, default=120.0)
|
||||||
|
parser.add_argument(
|
||||||
|
"--ref-audio",
|
||||||
|
type=Path,
|
||||||
|
default=None,
|
||||||
|
help="Reference clip for voice clone (WAV/FLAC/MP3 path). Pair with --ref-text.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--ref-text",
|
||||||
|
type=str,
|
||||||
|
default=None,
|
||||||
|
help="Transcript of the reference clip. Optional but improves fidelity.",
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
ref_audio_data_url: str | None = None
|
||||||
|
if args.ref_audio is not None:
|
||||||
|
if not args.ref_audio.exists():
|
||||||
|
print(f"ref-audio file not found: {args.ref_audio}", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
mime = "audio/wav" if args.ref_audio.suffix.lower() == ".wav" else "audio/mpeg"
|
||||||
|
ref_b64 = base64.b64encode(args.ref_audio.read_bytes()).decode("ascii")
|
||||||
|
ref_audio_data_url = f"data:{mime};base64,{ref_b64}"
|
||||||
|
|
||||||
|
try:
|
||||||
|
import httpx
|
||||||
|
except ImportError:
|
||||||
|
print("this client needs `httpx`. Install with `pip install httpx`.", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
url = args.base_url.rstrip("/") + "/v1/audio/speech"
|
||||||
|
failures = 0
|
||||||
|
with httpx.Client(timeout=args.timeout_s) as client:
|
||||||
|
for prompt in args.prompts:
|
||||||
|
payload = {
|
||||||
|
"model": args.model,
|
||||||
|
"input": prompt,
|
||||||
|
"response_format": args.format,
|
||||||
|
"max_new_tokens": args.max_new_tokens,
|
||||||
|
"seed": args.seed,
|
||||||
|
}
|
||||||
|
if ref_audio_data_url is not None:
|
||||||
|
payload["ref_audio"] = ref_audio_data_url
|
||||||
|
if args.ref_text:
|
||||||
|
payload["ref_text"] = args.ref_text
|
||||||
|
resp = client.post(url, json=payload)
|
||||||
|
if resp.status_code != 200:
|
||||||
|
print(f"[FAIL] {prompt!r} -> {resp.status_code}: {resp.text[:200]}", file=sys.stderr)
|
||||||
|
failures += 1
|
||||||
|
continue
|
||||||
|
suffix = ".wav" if args.format == "wav" else ".pcm"
|
||||||
|
out = args.output_dir / f"{_slug(prompt)}{suffix}"
|
||||||
|
out.write_bytes(resp.content)
|
||||||
|
print(f"[ ok ] {prompt!r} -> {out} ({len(resp.content)} bytes)")
|
||||||
|
|
||||||
|
return 1 if failures else 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Launch vLLM-Omni server for higgs-audio v3.
|
||||||
|
#
|
||||||
|
# Supports plain text TTS and voice cloning via /v1/audio/speech.
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# ./run_server.sh # default port 8095, GPU 0
|
||||||
|
# PORT=8096 GPUS=0,1 ./run_server.sh
|
||||||
|
# MODEL=/path/to/local/checkpoint ./run_server.sh
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
MODEL="${MODEL:-bosonai/higgs-audio-v3-tts-4b}"
|
||||||
|
PORT="${PORT:-8095}"
|
||||||
|
GPUS="${GPUS:-0}"
|
||||||
|
GPU_UTIL="${GPU_UTIL:-0.6}"
|
||||||
|
|
||||||
|
echo "Starting higgs-audio v3 server"
|
||||||
|
echo " MODEL=$MODEL"
|
||||||
|
echo " PORT=$PORT"
|
||||||
|
echo " CUDA_VISIBLE_DEVICES=$GPUS"
|
||||||
|
|
||||||
|
CUDA_VISIBLE_DEVICES="$GPUS" \
|
||||||
|
VLLM_USE_DEEP_GEMM=0 \
|
||||||
|
VLLM_MOE_USE_DEEP_GEMM=0 \
|
||||||
|
vllm-omni serve "$MODEL" \
|
||||||
|
--deploy-config vllm_omni/deploy/higgs_multimodal_qwen3.yaml \
|
||||||
|
--host 0.0.0.0 \
|
||||||
|
--port "$PORT" \
|
||||||
|
--gpu-memory-utilization "$GPU_UTIL" \
|
||||||
|
--trust-remote-code \
|
||||||
|
--omni
|
||||||
+3
-1
@@ -64,7 +64,9 @@ dev = [
|
|||||||
"lpips>=0.1.4", # for quantization fp8
|
"lpips>=0.1.4", # for quantization fp8
|
||||||
"orjson>=3.11.7", # to make v1/embeddings API fast
|
"orjson>=3.11.7", # to make v1/embeddings API fast
|
||||||
"voxcpm>=2.0.2", # for VoxCPM2 path-locating in tests
|
"voxcpm>=2.0.2", # for VoxCPM2 path-locating in tests
|
||||||
"torchdiffeq>=0.2.5" # for covo_audio Test
|
"torchdiffeq>=0.2.5", # for covo_audio Test
|
||||||
|
"FlagEmbedding", # for hunyuan_image3 tests
|
||||||
|
"kernels<0.15.0" # for accuracy tests comparison with diffusers output (0.15.0 incompatible with diffusers 0.38.0)
|
||||||
]
|
]
|
||||||
|
|
||||||
demo = [
|
demo = [
|
||||||
|
|||||||
@@ -0,0 +1,110 @@
|
|||||||
|
# Higgs-Audio V3 TTS
|
||||||
|
|
||||||
|
> Multilingual text-to-speech with voice cloning on 1×H100
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
- Vendor: Boson AI
|
||||||
|
- Model: `bosonai/higgs-audio-v3-tts-4b`
|
||||||
|
- Task: Text-to-speech synthesis with optional voice cloning (100+ languages)
|
||||||
|
- Mode: Online serving with the OpenAI-compatible `/v1/audio/speech` API; offline batch inference
|
||||||
|
- Maintainer: @yuekaiz
|
||||||
|
|
||||||
|
## When to use this recipe
|
||||||
|
|
||||||
|
Use this recipe to serve `bosonai/higgs-audio-v3-tts-4b` for high-quality
|
||||||
|
multilingual TTS. The model generates 24 kHz speech, supports zero-shot voice
|
||||||
|
cloning from a reference clip, and handles 100+ languages with inline control
|
||||||
|
tokens for emotion, style, and prosody. The architecture is a ~4B Qwen3
|
||||||
|
backbone with fused multi-codebook embedding/head (8 codebooks × 1026 vocab,
|
||||||
|
MusicGen-style delay pattern).
|
||||||
|
|
||||||
|
## References
|
||||||
|
|
||||||
|
- Model card: [bosonai/higgs-audio-v3-tts-4b](https://huggingface.co/bosonai/higgs-audio-v3-tts-4b)
|
||||||
|
- Offline example: [`examples/offline_inference/text_to_speech/higgs_audio_v3/end2end.py`](../../examples/offline_inference/text_to_speech/higgs_audio_v3/end2end.py)
|
||||||
|
- Online example: [`examples/online_serving/text_to_speech/higgs_audio_v3/`](../../examples/online_serving/text_to_speech/higgs_audio_v3/)
|
||||||
|
- Benchmark results: see Performance section below
|
||||||
|
|
||||||
|
## Hardware Support
|
||||||
|
|
||||||
|
## GPU
|
||||||
|
|
||||||
|
### 1×H100 80GB
|
||||||
|
|
||||||
|
#### Environment
|
||||||
|
|
||||||
|
- OS: Linux
|
||||||
|
- Python: 3.12+
|
||||||
|
- CUDA: 12.x
|
||||||
|
- vLLM version: 0.22.0
|
||||||
|
- vLLM-Omni version or commit: `36e048fd` (branch `higgs-v3`)
|
||||||
|
|
||||||
|
#### Command
|
||||||
|
|
||||||
|
**Online serving:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
vllm-omni serve bosonai/higgs-audio-v3-tts-4b \
|
||||||
|
--host 0.0.0.0 --port 8095 \
|
||||||
|
--trust-remote-code --omni
|
||||||
|
```
|
||||||
|
|
||||||
|
The default deploy config `vllm_omni/deploy/higgs_multimodal_qwen3.yaml` is
|
||||||
|
loaded automatically by model registry (HF `model_type=higgs_multimodal_qwen3`).
|
||||||
|
|
||||||
|
**Offline batch inference:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python examples/offline_inference/text_to_speech/higgs_audio_v3/end2end.py \
|
||||||
|
--texts "Hello world." "The quick brown fox jumps over the lazy dog." \
|
||||||
|
--output-dir results/higgs_v3_wavs
|
||||||
|
```
|
||||||
|
|
||||||
|
**Offline voice clone:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python examples/offline_inference/text_to_speech/higgs_audio_v3/end2end.py \
|
||||||
|
--texts "Text to synthesize in the cloned voice." \
|
||||||
|
--ref-audio path/to/reference.wav \
|
||||||
|
--ref-text "Transcript of the reference clip." \
|
||||||
|
--output-dir results/higgs_v3_clone
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Verification
|
||||||
|
|
||||||
|
Basic TTS via curl:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -X POST http://localhost:8095/v1/audio/speech \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{
|
||||||
|
"model": "bosonai/higgs-audio-v3-tts-4b",
|
||||||
|
"input": "Hello, how are you?"
|
||||||
|
}' --output hello.wav
|
||||||
|
```
|
||||||
|
|
||||||
|
Voice clone via Python client:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python examples/online_serving/text_to_speech/higgs_audio_v3/batch_speech_client.py \
|
||||||
|
--base-url http://localhost:8095 \
|
||||||
|
--model bosonai/higgs-audio-v3-tts-4b \
|
||||||
|
--ref-audio path/to/reference.wav \
|
||||||
|
--ref-text "Transcript of the reference." \
|
||||||
|
--prompts "Text to clone."
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Notes
|
||||||
|
|
||||||
|
- Memory usage: Stage 0 (talker, ~4B) uses ~60% GPU memory; Stage 1 (codec decoder) uses ~25%.
|
||||||
|
- Key flags: `--trust-remote-code` and `--omni` are required.
|
||||||
|
- Output: 24 kHz mono WAV.
|
||||||
|
- Voice cloning: `ref_audio` accepts WAV/FLAC/MP3; `ref_text` is optional but improves fidelity.
|
||||||
|
- Deploy config: `vllm_omni/deploy/higgs_multimodal_qwen3.yaml` (auto-discovered from `model_type`).
|
||||||
|
- `max_num_seqs=16` for both stages.
|
||||||
|
- `enforce_eager=true` for both stages (CUDA graph available for stage 0 but no throughput gain at batch>1).
|
||||||
|
- Known limitations:
|
||||||
|
- CUDA graph for stage 0 helps batch=1 (-13% RTF) but not batch>1 due to model-owned sampler running outside graph.
|
||||||
|
- Stage 1 (code2wav) must use `enforce_eager=true` (`@torch.inference_mode` incompatible with graph capture).
|
||||||
|
- Async streaming (chunk-based) not yet implemented; sync-only pipeline.
|
||||||
+2
-1
@@ -36,7 +36,8 @@ recipes/
|
|||||||
| [`LTX/LTX-2.md`](./LTX/LTX-2.md) | Text-to-video and image-to-video serving | 1x H200 141GB |
|
| [`LTX/LTX-2.md`](./LTX/LTX-2.md) | Text-to-video and image-to-video serving | 1x H200 141GB |
|
||||||
| [`LTX/LTX-2.3.md`](./LTX/LTX-2.3.md) | Text-to-video with audio generation (22B) | 1x GPU (96GB VRAM) |
|
| [`LTX/LTX-2.3.md`](./LTX/LTX-2.3.md) | Text-to-video with audio generation (22B) | 1x GPU (96GB VRAM) |
|
||||||
| [`mistralai/Voxtral-TTS.md`](./mistralai/Voxtral-TTS.md) | Online serving for TTS | 1x RTX 4090 24GB |
|
| [`mistralai/Voxtral-TTS.md`](./mistralai/Voxtral-TTS.md) | Online serving for TTS | 1x RTX 4090 24GB |
|
||||||
| [`nvidia/Cosmos3-Nano.md`](./nvidia/Cosmos3-Nano.md) | Text-to-image, text-to-video, and image-to-video generation | 1x H200 141GB / B300 |
|
| [`cosmos3/Cosmos3-Nano.md`](./cosmos3/Cosmos3-Nano.md) | Text-to-image, text-to-video, image-to-video generation, text to video with sound, action policy | 1x H200 141GB / B300 |
|
||||||
|
| [`cosmos3/Cosmos3-Super.md`](./cosmos3/Cosmos3-Super.md) | 64B T2I / T2V / I2V generation (+ optional audio) / Action policy | 8x H200/H100/A100 / 2x H200 / B300 |
|
||||||
| [`OpenBMB/MiniCPM-o-4_5.md`](./OpenBMB/MiniCPM-o-4_5.md) | Online serving for omni multimodal chat (text / image / audio / video → text + 24 kHz speech) | 2x A100/H100 80GB / 3x mid-tier GPU / 8x RTX 4090 24GB |
|
| [`OpenBMB/MiniCPM-o-4_5.md`](./OpenBMB/MiniCPM-o-4_5.md) | Online serving for omni multimodal chat (text / image / audio / video → text + 24 kHz speech) | 2x A100/H100 80GB / 3x mid-tier GPU / 8x RTX 4090 24GB |
|
||||||
| [`OpenBMB/VoxCPM2.md`](./OpenBMB/VoxCPM2.md) | Online + offline TTS with native AR pipeline (48 kHz, 30+ languages) | 1x RTX 4090 24GB |
|
| [`OpenBMB/VoxCPM2.md`](./OpenBMB/VoxCPM2.md) | Online + offline TTS with native AR pipeline (48 kHz, 30+ languages) | 1x RTX 4090 24GB |
|
||||||
| [`Qwen/Qwen-Image.md`](./Qwen/Qwen-Image.md) | Text-to-image serving with step-wise continuous batching replay and ModelOpt mixed FP8/NVFP4 | 1x A100 80GB / 2x B200 |
|
| [`Qwen/Qwen-Image.md`](./Qwen/Qwen-Image.md) | Text-to-image serving with step-wise continuous batching replay and ModelOpt mixed FP8/NVFP4 | 1x A100 80GB / 2x B200 |
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
- Vendor: NVIDIA
|
- Vendor: NVIDIA
|
||||||
- Model: `nvidia/Cosmos3-Nano`
|
- Model: `nvidia/Cosmos3-Nano`
|
||||||
- Task: Text-to-image (T2I), text-to-video (T2V), and image-to-video (I2V) generation
|
- Task: Text-to-image (T2I), text-to-video (T2V), and image-to-video (I2V) generation, with optional synchronized audio (video + sound), action policy
|
||||||
- Mode: Online serving with the OpenAI-compatible image/video APIs, plus offline generation via the `Omni` API
|
- Mode: Online serving with the OpenAI-compatible image/video APIs, plus offline generation via the `Omni` API
|
||||||
- Maintainer: Community
|
- Maintainer: Community
|
||||||
|
|
||||||
@@ -20,12 +20,34 @@ the mode is selected per request:
|
|||||||
- **T2V** — `POST /v1/videos/sync` with `num_frames > 1` and no reference image.
|
- **T2V** — `POST /v1/videos/sync` with `num_frames > 1` and no reference image.
|
||||||
- **I2V** — `POST /v1/videos/sync` with a reference image (`input_reference` file
|
- **I2V** — `POST /v1/videos/sync` with a reference image (`input_reference` file
|
||||||
upload, or `image_reference` JSON).
|
upload, or `image_reference` JSON).
|
||||||
|
- **T2VS / I2VS** — add `generate_sound=true` (and optional `sound_duration`) to a
|
||||||
|
T2V/I2V `/v1/videos/sync` request to also generate synchronized audio, muxed into
|
||||||
|
the mp4 as AAC 48 kHz stereo. See the official model card's "Video + Audio" examples.
|
||||||
|
- **Action** — pass `extra_params={"action_mode": ...}` to drive Physical-AI tasks:
|
||||||
|
- `forward_dynamics` — given a first frame **and** an action trajectory, roll out
|
||||||
|
the resulting video. Synchronous: `POST /v1/videos/sync`.
|
||||||
|
- `policy` — given a first frame and a language instruction, **predict** the action
|
||||||
|
trajectory (and a rollout video). Use the async `POST /v1/videos` endpoint and
|
||||||
|
read the predicted action from the top-level `action` field
|
||||||
|
(`{data, shape, dtype, raw_action_dim, domain_id}`).
|
||||||
|
|
||||||
|
Action requests also take `domain_name` (e.g. `av`, `bridge_orig_lerobot`,
|
||||||
|
`droid_lerobot`, `agibotworld`, …; or a numeric `domain_id`), `raw_action_dim`,
|
||||||
|
and `action_chunk_size` (must equal `num_frames` or `num_frames - 1`). For
|
||||||
|
`forward_dynamics` also pass the `action` array. The dedicated policy checkpoint
|
||||||
|
**`nvidia/Cosmos3-Nano-Policy-DROID`** is served the same way
|
||||||
|
(`domain_name=droid_lerobot`).
|
||||||
|
|
||||||
|
`inverse_dynamics` (recover the action from a given video) is supported by the
|
||||||
|
pipeline; **online inference of inverse dynamics will be added in a follow-up MR.**
|
||||||
|
|
||||||
## References
|
## References
|
||||||
|
|
||||||
- Model card (authoritative usage + example assets): <https://huggingface.co/nvidia/Cosmos3-Nano>
|
- Model card (authoritative usage + example assets): <https://huggingface.co/nvidia/Cosmos3-Nano>
|
||||||
- Example inputs/outputs live in the repo's `assets/` (`example_t2v_prompt.json`,
|
- Example inputs/outputs live in the repo's `assets/` (`example_t2v_prompt.json`,
|
||||||
`example_i2v_prompt.json`, `example_i2v_input.jpg`, `negative_prompt.json`).
|
`example_i2v_prompt.json`, `example_i2v_input.jpg`, `negative_prompt.json`;
|
||||||
|
audio examples: `example_t2vs_prompt.json`, `example_t2vs_output.mp4`,
|
||||||
|
`example_i2vs_output.mp4`).
|
||||||
- Prompt upsampling (recommended for quality): the model expects JSON-upsampled
|
- Prompt upsampling (recommended for quality): the model expects JSON-upsampled
|
||||||
structured prompts; see NVIDIA's `cosmos-framework` prompt-upsampling docs.
|
structured prompts; see NVIDIA's `cosmos-framework` prompt-upsampling docs.
|
||||||
- Pipeline: [`vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py`](../../vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py)
|
- Pipeline: [`vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py`](../../vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py)
|
||||||
@@ -47,6 +69,9 @@ the mode is selected per request:
|
|||||||
|
|
||||||
#### Command
|
#### Command
|
||||||
|
|
||||||
|
Requires the `vllm-omni` package (or the `vllm/vllm-omni:cosmos3` container),
|
||||||
|
which provides the `vllm serve … --omni` entrypoint used below.
|
||||||
|
|
||||||
Safety guardrails are **on by default** (NVIDIA Open Model License). They load
|
Safety guardrails are **on by default** (NVIDIA Open Model License). They load
|
||||||
the **gated** `nvidia/Cosmos-1.0-Guardrail` model, so to keep them on you must:
|
the **gated** `nvidia/Cosmos-1.0-Guardrail` model, so to keep them on you must:
|
||||||
|
|
||||||
@@ -116,6 +141,55 @@ curl -sS -X POST http://localhost:8000/v1/videos/sync \
|
|||||||
-F "seed=1111" \
|
-F "seed=1111" \
|
||||||
-F "input_reference=@/path/to/reference.jpg;type=image/jpeg" \
|
-F "input_reference=@/path/to/reference.jpg;type=image/jpeg" \
|
||||||
-o cosmos3_i2v.mp4
|
-o cosmos3_i2v.mp4
|
||||||
|
|
||||||
|
|
||||||
|
# Text-to-video-with-sound
|
||||||
|
curl -sS -X POST http://localhost:8000/v1/videos/sync \
|
||||||
|
-H "Accept: video/mp4" \
|
||||||
|
-F "model=nvidia/Cosmos3-Nano" \
|
||||||
|
-F "prompt=The video opens with a view of a well-lit indoor fruit display. A robotic arm picks up a pear, an orange, and a carambola one by one, placing each into a plastic bag in a shopping cart with red handles. The video is 7.875 seconds long, 24 FPS, and 1280x720. Audio description: soft servo whirs, gentle fruit thuds, plastic bag rustling, and a faint refrigeration hum." \
|
||||||
|
-F "negative_prompt=blurry, distorted, low quality" \
|
||||||
|
-F "size=1280x720" \
|
||||||
|
-F "num_frames=189" \
|
||||||
|
-F "fps=24" \
|
||||||
|
-F "num_inference_steps=35" \
|
||||||
|
-F "guidance_scale=6.0" \
|
||||||
|
-F "max_sequence_length=4096" \
|
||||||
|
-F "flow_shift=10.0" \
|
||||||
|
-F "seed=0" \
|
||||||
|
-F "generate_sound=true" \
|
||||||
|
-F "sound_duration=7.875" \
|
||||||
|
-F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":true}' \
|
||||||
|
-o cosmos3_t2v_with_sound.mp4
|
||||||
|
|
||||||
|
# Action — forward dynamics (first frame + action trajectory -> rollout video).
|
||||||
|
# Synchronous; `action` is a JSON array shaped [action_chunk_size, raw_action_dim].
|
||||||
|
curl -sS -X POST http://localhost:8000/v1/videos/sync \
|
||||||
|
-H "Accept: video/mp4" \
|
||||||
|
--form-string "model=nvidia/Cosmos3-Nano" \
|
||||||
|
--form-string "prompt=You are an autonomous vehicle. This video is captured from a first-person perspective." \
|
||||||
|
-F "input_reference=@first_frame.jpg;type=image/jpeg" \
|
||||||
|
-F "size=640x480" -F "num_frames=61" -F "fps=10" \
|
||||||
|
-F "num_inference_steps=30" -F "guidance_scale=1.0" -F "flow_shift=5.0" \
|
||||||
|
--form-string "extra_params={\"action_mode\":\"forward_dynamics\",\"domain_name\":\"av\",\"raw_action_dim\":9,\"action_chunk_size\":60,\"action\":$(cat action.json)}" \
|
||||||
|
-F "seed=0" \
|
||||||
|
-o cosmos3_forward_dynamics.mp4
|
||||||
|
|
||||||
|
# Action — policy (first frame + instruction -> predicted action trajectory + video).
|
||||||
|
# Asynchronous: POST returns a job id; poll, then read the predicted action from
|
||||||
|
# the top-level `action` field ({data, shape, dtype, raw_action_dim, domain_id}).
|
||||||
|
VIDEO_ID=$(curl -sS -X POST http://localhost:8000/v1/videos \
|
||||||
|
-H "Accept: application/json" \
|
||||||
|
--form-string "model=nvidia/Cosmos3-Nano" \
|
||||||
|
--form-string "prompt=Pick up the banana and place it in the bowl." \
|
||||||
|
-F "input_reference=@first_frame.jpg;type=image/jpeg" \
|
||||||
|
-F "size=640x480" -F "num_frames=17" -F "fps=5" \
|
||||||
|
-F "num_inference_steps=30" -F "guidance_scale=1.0" -F "flow_shift=5.0" \
|
||||||
|
--form-string 'extra_params={"action_mode":"policy","domain_name":"bridge_orig_lerobot","raw_action_dim":10,"action_chunk_size":16}' \
|
||||||
|
-F "seed=0" | jq -r '.id')
|
||||||
|
# poll until status == completed, then:
|
||||||
|
curl -sS "http://localhost:8000/v1/videos/$VIDEO_ID" | jq '.action | {shape, dtype, raw_action_dim, domain_id}'
|
||||||
|
curl -sS -L "http://localhost:8000/v1/videos/$VIDEO_ID/content" -o cosmos3_policy.mp4
|
||||||
```
|
```
|
||||||
|
|
||||||
#### Notes
|
#### Notes
|
||||||
@@ -124,6 +198,7 @@ curl -sS -X POST http://localhost:8000/v1/videos/sync \
|
|||||||
- T2I 1024² — 10 / 25 / 50 steps → ~0.4 / 0.7 / **1.3 s**
|
- T2I 1024² — 10 / 25 / 50 steps → ~0.4 / 0.7 / **1.3 s**
|
||||||
- T2V 1280×720 @ 35 steps — 25 / 49 / 93 / **189** frames → ~7 / 15 / 33 / **~93 s**
|
- T2V 1280×720 @ 35 steps — 25 / 49 / 93 / **189** frames → ~7 / 15 / 33 / **~93 s**
|
||||||
- I2V 1280×720, 189 frames @ 35 steps → ~**99 s**
|
- I2V 1280×720, 189 frames @ 35 steps → ~**99 s**
|
||||||
|
- Action 640×480 @ 30 steps — forward-dynamics 61f ~**4 s**, policy 17f ~**1–3 s**.
|
||||||
- Guardrails-on overhead: ~8% on T2I, negligible on video.
|
- Guardrails-on overhead: ~8% on T2I, negligible on video.
|
||||||
- **Memory:** transformer ~17 GiB (bf16); peak ~46 GiB for 720p video on 1 GPU;
|
- **Memory:** transformer ~17 GiB (bf16); peak ~46 GiB for 720p video on 1 GPU;
|
||||||
full repo (transformer + Wan VAE + Qwen3-VL vision encoder + audio tokenizer)
|
full repo (transformer + Wan VAE + Qwen3-VL vision encoder + audio tokenizer)
|
||||||
@@ -134,7 +209,9 @@ curl -sS -X POST http://localhost:8000/v1/videos/sync \
|
|||||||
3:4, 9:16. Defaults: T2I 1024², 50 steps, guidance 7.0; T2V/I2V 1280×720,
|
3:4, 9:16. Defaults: T2I 1024², 50 steps, guidance 7.0; T2V/I2V 1280×720,
|
||||||
189 frames, 35 steps, guidance 6.0, `flow_shift=10.0`.
|
189 frames, 35 steps, guidance 6.0, `flow_shift=10.0`.
|
||||||
- **Key flags / params:** `--no-guardrails` (server) or
|
- **Key flags / params:** `--no-guardrails` (server) or
|
||||||
`extra_params={"guardrails":false}` (per request) toggles safety;
|
`extra_params={"guardrails":false}` (per request) toggles safety. The
|
||||||
|
per-request flag only takes effect when the server was launched **with**
|
||||||
|
guardrails enabled (it cannot re-enable them on a `--no-guardrails` server).
|
||||||
`use_resolution_template` / `use_duration_template` are off by default and only
|
`use_resolution_template` / `use_duration_template` are off by default and only
|
||||||
needed when not using upsampled prompts that already encode resolution/duration.
|
needed when not using upsampled prompts that already encode resolution/duration.
|
||||||
- **Known limitations:**
|
- **Known limitations:**
|
||||||
@@ -143,8 +220,10 @@ curl -sS -X POST http://localhost:8000/v1/videos/sync \
|
|||||||
the server fails at pipeline build with a gated-repo / safety-checker error.
|
the server fails at pipeline build with a gated-repo / safety-checker error.
|
||||||
- A guardrail-blocked prompt currently returns HTTP 500
|
- A guardrail-blocked prompt currently returns HTTP 500
|
||||||
(`"Guardrail blocked prompt"`).
|
(`"Guardrail blocked prompt"`).
|
||||||
- Video + audio, and action (policy / forward- / inverse-dynamics) modalities
|
- Action `forward_dynamics` (sync `/v1/videos/sync`) and `policy` (async
|
||||||
are not part of this integration yet.
|
`/v1/videos`, returns the predicted action under the top-level `action`
|
||||||
|
field) are supported online. **Online inference of inverse dynamics will be
|
||||||
|
added in a follow-up MR.**
|
||||||
|
|
||||||
### 1x GPU (Offline generation)
|
### 1x GPU (Offline generation)
|
||||||
|
|
||||||
@@ -170,8 +249,8 @@ def main():
|
|||||||
model_class_name="Cosmos3OmniDiffusersPipeline",
|
model_class_name="Cosmos3OmniDiffusersPipeline",
|
||||||
trust_remote_code=True,
|
trust_remote_code=True,
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
# Keep guardrails on by installing cosmos-guardrail + gated-repo access;
|
# Guardrails are disabled here for a quick local run; install
|
||||||
# this disables them for a quick local run.
|
# cosmos-guardrail + gated-repo access and drop this to enable them.
|
||||||
model_config={"guardrails": False},
|
model_config={"guardrails": False},
|
||||||
)
|
)
|
||||||
gen = torch.Generator(device="cpu").manual_seed(42)
|
gen = torch.Generator(device="cpu").manual_seed(42)
|
||||||
@@ -0,0 +1,112 @@
|
|||||||
|
# Cosmos3-Super
|
||||||
|
|
||||||
|
> Frontier 64B world model: text-to-image, text-to-video, image-to-video (+ optional audio)
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
- Vendor: NVIDIA
|
||||||
|
- Model: `nvidia/Cosmos3-Super` (64B; also `Cosmos3-Super-Text2Image`, `Cosmos3-Super-Image2Video`)
|
||||||
|
- Task: T2I, T2V, I2V generation, with optional synchronized audio (video + sound)
|
||||||
|
- Mode: Online serving with the OpenAI-compatible image/video APIs
|
||||||
|
- Maintainer: Community
|
||||||
|
|
||||||
|
## When to use this recipe
|
||||||
|
|
||||||
|
Use this recipe to deploy the 64B `nvidia/Cosmos3-Super` for the highest-quality
|
||||||
|
Cosmos3 generation. It shares the same `Cosmos3OmniDiffusersPipeline` and request
|
||||||
|
formats as [Cosmos3-Nano](./Cosmos3-Nano.md) — only the checkpoint size and the
|
||||||
|
recommended parallelism differ. Mode is selected per request (T2I →
|
||||||
|
`/v1/images/generations`; T2V/I2V → `/v1/videos/sync`; add `generate_sound=true`
|
||||||
|
for audio).
|
||||||
|
|
||||||
|
## References
|
||||||
|
|
||||||
|
- Model card (authoritative usage + example assets): <https://huggingface.co/nvidia/Cosmos3-Super>
|
||||||
|
- Nano recipe (same APIs/params): [`Cosmos3-Nano.md`](./Cosmos3-Nano.md)
|
||||||
|
- Pipeline: [`vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py`](../../vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py)
|
||||||
|
|
||||||
|
## Hardware Support
|
||||||
|
|
||||||
|
## GPU
|
||||||
|
|
||||||
|
Requires the `vllm-omni` package (or the `vllm/vllm-omni:cosmos3` container),
|
||||||
|
which provides the `vllm serve … --omni` entrypoint used below.
|
||||||
|
|
||||||
|
### 8x H200/H100/A100 (recommended, per model card)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
vllm serve nvidia/Cosmos3-Super \
|
||||||
|
--omni \
|
||||||
|
--host 0.0.0.0 --port 8000 \
|
||||||
|
--cfg-parallel-size 2 \
|
||||||
|
--ulysses-degree 4 \
|
||||||
|
--use-hsdp --hsdp-shard-size 8 \
|
||||||
|
--init-timeout 1800
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2x H200 / B300 (minimum)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
vllm serve nvidia/Cosmos3-Super \
|
||||||
|
--omni \
|
||||||
|
--host 0.0.0.0 --port 8000 \
|
||||||
|
--cfg-parallel-size 2 \
|
||||||
|
--use-hsdp --hsdp-shard-size 2 \
|
||||||
|
--init-timeout 1800
|
||||||
|
```
|
||||||
|
|
||||||
|
Guardrails are on by default (gated `nvidia/Cosmos-1.0-Guardrail` — `pip install
|
||||||
|
cosmos-guardrail`, accept the license, set `HF_TOKEN`); add `--no-guardrails` to
|
||||||
|
disable. `--enable-layerwise-offload` reduces VRAM on smaller GPUs.
|
||||||
|
|
||||||
|
#### Verification
|
||||||
|
|
||||||
|
Requests are identical to Nano (see [`Cosmos3-Nano.md`](./Cosmos3-Nano.md) for full
|
||||||
|
T2I/T2V/I2V/T2VS curls); official params: `size=1280x720, num_frames=189, fps=24,
|
||||||
|
num_inference_steps=35, guidance_scale=6.0, flow_shift=10.0, max_sequence_length=4096`.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl http://localhost:8000/v1/models
|
||||||
|
# T2V (official prompt assets give best quality)
|
||||||
|
curl -sS -X POST http://localhost:8000/v1/videos/sync -H "Accept: video/mp4" \
|
||||||
|
-F "model=nvidia/Cosmos3-Super" -F "prompt=A robot arm is cleaning a plate in the kitchen" \
|
||||||
|
-F "size=1280x720" -F "num_frames=189" -F "fps=24" -F "num_inference_steps=35" \
|
||||||
|
-F "guidance_scale=6.0" -F "max_sequence_length=4096" -F "flow_shift=10.0" \
|
||||||
|
-F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":true}' \
|
||||||
|
-F "seed=17" -o cosmos3_super_t2v.mp4
|
||||||
|
|
||||||
|
# I2V — add an uploaded reference image
|
||||||
|
curl -sS -X POST http://localhost:8000/v1/videos/sync -H "Accept: video/mp4" \
|
||||||
|
-F "model=nvidia/Cosmos3-Super" -F "prompt=The scene comes to life with smooth, natural motion." \
|
||||||
|
-F "size=1280x720" -F "num_frames=189" -F "fps=24" -F "num_inference_steps=35" \
|
||||||
|
-F "guidance_scale=6.0" -F "max_sequence_length=4096" -F "flow_shift=10.0" \
|
||||||
|
-F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":true}' \
|
||||||
|
-F "seed=1111" -F "input_reference=@/path/to/reference.jpg;type=image/jpeg" \
|
||||||
|
-o cosmos3_super_i2v.mp4
|
||||||
|
|
||||||
|
# T2V + sound — add generate_sound/sound_duration (output muxes AAC 48 kHz stereo)
|
||||||
|
curl -sS -X POST http://localhost:8000/v1/videos/sync -H "Accept: video/mp4" \
|
||||||
|
-F "model=nvidia/Cosmos3-Super" -F "prompt=A robot arm is cleaning a plate in the kitchen" \
|
||||||
|
-F "size=1280x720" -F "num_frames=189" -F "fps=24" -F "num_inference_steps=35" \
|
||||||
|
-F "guidance_scale=6.0" -F "max_sequence_length=4096" -F "flow_shift=10.0" \
|
||||||
|
-F "generate_sound=true" -F "sound_duration=7.875" \
|
||||||
|
-F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":true}' \
|
||||||
|
-F "seed=17" -o cosmos3_super_t2vs.mp4
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Notes
|
||||||
|
|
||||||
|
- **Measured (2x B300, bf16, guardrails off, official 2-GPU config above):**
|
||||||
|
- T2I 1024², 50 steps → **~6 s**
|
||||||
|
- T2V 1280×720, 189 frames, 35 steps → **~197 s**
|
||||||
|
- I2V 1280×720, 189 frames, 35 steps → **~200 s**
|
||||||
|
- T2V + sound (189 frames, 35 steps) → **~198 s**, output muxes **AAC 48 kHz stereo**
|
||||||
|
- (NVIDIA's reference: 8×H200 @ 50 steps ≈ 55 s/video; 2×H200 @ 35 steps ≈ 3 min/video.)
|
||||||
|
- **Memory:** ~61.5 GiB per GPU when sharded across 2 GPUs (HSDP shard 2); repo ~135 GB on disk.
|
||||||
|
- Same generation defaults, supported sizes, and `generate_sound`/`sound_duration`
|
||||||
|
semantics as Nano, including the **action** modality: `forward_dynamics`
|
||||||
|
(sync `/v1/videos/sync`) and `policy` (async `/v1/videos`, predicted action under
|
||||||
|
the top-level `action` field) — see the Cosmos3-Nano recipe for the request shape.
|
||||||
|
Online inference of inverse dynamics will be added in a follow-up MR. Verified on
|
||||||
|
the 64B Super under `--cfg-parallel-size 2`: async `policy` returns the predicted
|
||||||
|
action (`[16, 10]`) and the rollout video reliably.
|
||||||
@@ -57,12 +57,6 @@ Adjust `devices` in the YAML to match your hardware.
|
|||||||
|
|
||||||
#### Command
|
#### Command
|
||||||
|
|
||||||
Thinker only (text output):
|
|
||||||
|
|
||||||
```bash
|
|
||||||
vllm serve Jonathan1909/Ming-flash-omni-2.0 --omni --port 8091
|
|
||||||
```
|
|
||||||
|
|
||||||
Thinker + talker (text and/or audio output):
|
Thinker + talker (text and/or audio output):
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -72,6 +66,15 @@ vllm serve Jonathan1909/Ming-flash-omni-2.0 \
|
|||||||
--log-stats
|
--log-stats
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Thinker only (text-only output):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
vllm serve Jonathan1909/Ming-flash-omni-2.0 \
|
||||||
|
--omni \
|
||||||
|
--deploy-config vllm_omni/deploy/ming_flash_omni_thinker_only.yaml \
|
||||||
|
--port 8091
|
||||||
|
```
|
||||||
|
|
||||||
`--log-stats` is optional but recommended while validating the deployment.
|
`--log-stats` is optional but recommended while validating the deployment.
|
||||||
|
|
||||||
#### Verification
|
#### Verification
|
||||||
|
|||||||
Binary file not shown.
|
After Width: | Height: | Size: 1.1 MiB |
@@ -345,6 +345,14 @@ def test_parse_score_payload_handles_raw_json_and_delimited_json():
|
|||||||
assert parse_score_payload(wrapped)["score"] == [6]
|
assert parse_score_payload(wrapped)["score"] == [6]
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_score_payload_handles_qwen_vl_nested_score_dicts():
|
||||||
|
qwen_style = '{"score": [{"naturalness": 8, "artifact_free": 7}], "reasoning": "good quality"}'
|
||||||
|
assert parse_score_payload(qwen_style)["score"] == [8, 7]
|
||||||
|
|
||||||
|
nested_values = '{"score": [{"score": 9}, {"score": 6}], "reasoning": "ok"}'
|
||||||
|
assert parse_score_payload(nested_values)["score"] == [9, 6]
|
||||||
|
|
||||||
|
|
||||||
def test_summarize_gedit_generated_records_groups_by_task_and_language():
|
def test_summarize_gedit_generated_records_groups_by_task_and_language():
|
||||||
records = []
|
records = []
|
||||||
for group in GEDIT_GROUPS[:2]:
|
for group in GEDIT_GROUPS[:2]:
|
||||||
|
|||||||
@@ -61,7 +61,7 @@ def test_stage0_streaming_update_discards_outstanding_async_placeholder_token()
|
|||||||
|
|
||||||
sched._update_request_as_session(session, _make_update([10, 20]))
|
sched._update_request_as_session(session, _make_update([10, 20]))
|
||||||
|
|
||||||
assert session.discard_latest_async_tokens is True
|
assert session.async_tokens_to_discard == 1
|
||||||
assert session.num_output_placeholders == 0
|
assert session.num_output_placeholders == 0
|
||||||
assert session.spec_token_ids == []
|
assert session.spec_token_ids == []
|
||||||
# The async placeholder makes token 9 unconfirmed, so only 7 and 8 are
|
# The async placeholder makes token 9 unconfirmed, so only 7 and 8 are
|
||||||
@@ -83,7 +83,7 @@ def test_stage0_streaming_update_keeps_all_computed_tokens_without_placeholder()
|
|||||||
|
|
||||||
sched._update_request_as_session(session, _make_update([10, 20]))
|
sched._update_request_as_session(session, _make_update([10, 20]))
|
||||||
|
|
||||||
assert session.discard_latest_async_tokens is False
|
assert getattr(session, "async_tokens_to_discard", 0) == 0
|
||||||
assert session.num_output_placeholders == 0
|
assert session.num_output_placeholders == 0
|
||||||
assert session.prompt_token_ids == [1, 2, 3, 7, 8, 9, 10, 20]
|
assert session.prompt_token_ids == [1, 2, 3, 7, 8, 9, 10, 20]
|
||||||
assert list(session._all_token_ids) == [1, 2, 3, 7, 8, 9, 10, 20]
|
assert list(session._all_token_ids) == [1, 2, 3, 7, 8, 9, 10, 20]
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
[
|
[
|
||||||
{
|
{
|
||||||
"test_name": "test_hunyuan_image3_ti2i_tp4_tp4_inline",
|
"test_name": "test_hunyuan_image3_ti2i_inline",
|
||||||
"description": "AR (TP=4) + DiT (TP=4) - using inline dataset and deploy-config",
|
"description": "AR (TP=2) + 2 * DiT (TP=2) - using inline dataset and deploy-config",
|
||||||
"server_type": "vllm-omni",
|
"server_type": "vllm-omni",
|
||||||
"benchmark_endpoint": "/v1/images/edits",
|
"benchmark_endpoint": "/v1/images/edits",
|
||||||
"server_params": {
|
"server_params": {
|
||||||
@@ -21,7 +21,7 @@
|
|||||||
"zmq_port": 50051,
|
"zmq_port": 50051,
|
||||||
"protocol": "rdma",
|
"protocol": "rdma",
|
||||||
"device_name": "",
|
"device_name": "",
|
||||||
"memory_pool_size": 4294967296,
|
"memory_pool_size": 17179869184,
|
||||||
"memory_pool_device": "cpu"
|
"memory_pool_device": "cpu"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -32,13 +32,13 @@
|
|||||||
"is_comprehension": true,
|
"is_comprehension": true,
|
||||||
"final_output": true,
|
"final_output": true,
|
||||||
"final_output_type": "text",
|
"final_output_type": "text",
|
||||||
"max_num_seqs": 4,
|
"max_num_seqs": 16,
|
||||||
"gpu_memory_utilization": 0.95,
|
"gpu_memory_utilization": 0.95,
|
||||||
"enforce_eager": false,
|
"enforce_eager": false,
|
||||||
"enable_prefix_caching": false,
|
"enable_prefix_caching": false,
|
||||||
"max_num_batched_tokens": 32768,
|
"max_num_batched_tokens": 32768,
|
||||||
"devices": "0,1,2,3",
|
"devices": "0,1",
|
||||||
"tensor_parallel_size": 4,
|
"tensor_parallel_size": 2,
|
||||||
"hf_overrides": {
|
"hf_overrides": {
|
||||||
"rope_parameters": {
|
"rope_parameters": {
|
||||||
"mrope_section": [0, 32, 32],
|
"mrope_section": [0, 32, 32],
|
||||||
@@ -65,13 +65,14 @@
|
|||||||
"stage_id": 1,
|
"stage_id": 1,
|
||||||
"max_num_seqs": 1,
|
"max_num_seqs": 1,
|
||||||
"enforce_eager": true,
|
"enforce_eager": true,
|
||||||
"devices": "4,5,6,7",
|
"devices": "2,3",
|
||||||
"distributed_executor_backend": "mp",
|
"distributed_executor_backend": "mp",
|
||||||
"omni_kv_config": {
|
"omni_kv_config": {
|
||||||
"need_recv_cache": true
|
"need_recv_cache": true
|
||||||
},
|
},
|
||||||
"parallel_config": {
|
"parallel_config": {
|
||||||
"tensor_parallel_size": 4,
|
"tensor_parallel_size": 2,
|
||||||
|
"vae_patch_parallel_size": 2,
|
||||||
"enable_expert_parallel": true
|
"enable_expert_parallel": true
|
||||||
},
|
},
|
||||||
"input_connectors": {
|
"input_connectors": {
|
||||||
@@ -114,8 +115,8 @@
|
|||||||
},
|
},
|
||||||
"task": "ti2i",
|
"task": "ti2i",
|
||||||
"bot-task": "think_recaption",
|
"bot-task": "think_recaption",
|
||||||
"max-concurrency": 4,
|
"max-concurrency": 16,
|
||||||
"num-prompts": 16,
|
"num-prompts": 96,
|
||||||
"skip-performance-assertion": true,
|
"skip-performance-assertion": true,
|
||||||
"baseline": {
|
"baseline": {
|
||||||
"throughput_qps": 0.1,
|
"throughput_qps": 0.1,
|
||||||
|
|||||||
@@ -23,10 +23,10 @@
|
|||||||
"ignore_eos": true,
|
"ignore_eos": true,
|
||||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||||
"baseline": {
|
"baseline": {
|
||||||
"mean_ttft_ms": [1000, 2000, 3000, 4000],
|
"mean_ttft_ms": [203, 283, 470, 869],
|
||||||
"mean_tpot_ms": [200, 200, 200, 200],
|
"mean_tpot_ms": [28, 36, 41, 45],
|
||||||
"mean_audio_ttfp_ms": [10000, 15000, 20000, 25000],
|
"mean_audio_ttfp_ms": [861, 1200, 1528, 2100],
|
||||||
"mean_audio_rtf": [1.5, 1.5, 1.5, 1.5]
|
"mean_audio_rtf": [0.24, 0.30, 0.35, 0.39]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -49,9 +49,9 @@
|
|||||||
},
|
},
|
||||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||||
"baseline": {
|
"baseline": {
|
||||||
"mean_ttft_ms": [2000],
|
"mean_ttft_ms": [377],
|
||||||
"mean_audio_ttfp_ms": [2000],
|
"mean_audio_ttfp_ms": [684],
|
||||||
"mean_audio_rtf": [0.25]
|
"mean_audio_rtf": [0.14]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -76,9 +76,9 @@
|
|||||||
},
|
},
|
||||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||||
"baseline": {
|
"baseline": {
|
||||||
"mean_ttft_ms": [6000],
|
"mean_ttft_ms": [228],
|
||||||
"mean_audio_ttfp_ms": [6000],
|
"mean_audio_ttfp_ms": [686],
|
||||||
"mean_audio_rtf": [0.7]
|
"mean_audio_rtf": [0.16]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -105,9 +105,9 @@
|
|||||||
},
|
},
|
||||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||||
"baseline": {
|
"baseline": {
|
||||||
"mean_ttft_ms": [12000],
|
"mean_ttft_ms": [522],
|
||||||
"mean_audio_ttfp_ms": [12000],
|
"mean_audio_ttfp_ms": [1119],
|
||||||
"mean_audio_rtf": [1.0]
|
"mean_audio_rtf": [0.21]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -31,11 +31,6 @@
|
|||||||
],
|
],
|
||||||
"random_input_len": 2500,
|
"random_input_len": 2500,
|
||||||
"random_output_len": 900,
|
"random_output_len": 900,
|
||||||
"temperature": 0.4,
|
|
||||||
"top_p": 0.9,
|
|
||||||
"top_k": 1,
|
|
||||||
"seed": 42,
|
|
||||||
"repetition_penalty": 1.05,
|
|
||||||
"ignore_eos": true,
|
"ignore_eos": true,
|
||||||
"extra_body": {
|
"extra_body": {
|
||||||
"modalities": [
|
"modalities": [
|
||||||
@@ -74,11 +69,6 @@
|
|||||||
"random_input_len": 100,
|
"random_input_len": 100,
|
||||||
"random_output_len": 100,
|
"random_output_len": 100,
|
||||||
"random_range_ratio": 0.0,
|
"random_range_ratio": 0.0,
|
||||||
"temperature": 0.4,
|
|
||||||
"top_p": 0.9,
|
|
||||||
"top_k": 1,
|
|
||||||
"seed": 42,
|
|
||||||
"repetition_penalty": 1.05,
|
|
||||||
"ignore_eos": true,
|
"ignore_eos": true,
|
||||||
"extra_body": {
|
"extra_body": {
|
||||||
"modalities": [
|
"modalities": [
|
||||||
@@ -117,11 +107,6 @@
|
|||||||
"random_input_len": 100,
|
"random_input_len": 100,
|
||||||
"random_output_len": 100,
|
"random_output_len": 100,
|
||||||
"random_range_ratio": 0.0,
|
"random_range_ratio": 0.0,
|
||||||
"temperature": 0.4,
|
|
||||||
"top_p": 0.9,
|
|
||||||
"top_k": 1,
|
|
||||||
"seed": 42,
|
|
||||||
"repetition_penalty": 1.05,
|
|
||||||
"ignore_eos": true,
|
"ignore_eos": true,
|
||||||
"extra_body": {
|
"extra_body": {
|
||||||
"modalities": [
|
"modalities": [
|
||||||
@@ -162,11 +147,6 @@
|
|||||||
"random_input_len": 100,
|
"random_input_len": 100,
|
||||||
"random_output_len": 100,
|
"random_output_len": 100,
|
||||||
"random_range_ratio": 0.0,
|
"random_range_ratio": 0.0,
|
||||||
"temperature": 0.4,
|
|
||||||
"top_p": 0.9,
|
|
||||||
"top_k": 1,
|
|
||||||
"seed": 42,
|
|
||||||
"repetition_penalty": 1.05,
|
|
||||||
"ignore_eos": true,
|
"ignore_eos": true,
|
||||||
"extra_body": {
|
"extra_body": {
|
||||||
"modalities": [
|
"modalities": [
|
||||||
|
|||||||
@@ -73,14 +73,42 @@ class StubCosmos3VAE:
|
|||||||
return (latents,)
|
return (latents,)
|
||||||
|
|
||||||
|
|
||||||
|
class StubCosmos3AVAE:
|
||||||
|
def __init__(self, **kwargs: Any) -> None:
|
||||||
|
self.kwargs = kwargs
|
||||||
|
self.sample_rate = int(kwargs["sample_rate"])
|
||||||
|
self.audio_channels = int(kwargs["audio_channels"])
|
||||||
|
self.latent_ch = int(kwargs["io_channels"])
|
||||||
|
self.temporal_compression_factor = int(kwargs["hop_size"])
|
||||||
|
|
||||||
|
def get_latent_num_samples(self, num_audio_samples: int) -> int:
|
||||||
|
return int(num_audio_samples) // self.temporal_compression_factor
|
||||||
|
|
||||||
|
def get_audio_num_samples(self, num_latent_samples: int) -> int:
|
||||||
|
return int(num_latent_samples) * self.temporal_compression_factor
|
||||||
|
|
||||||
|
def decode(self, latents: torch.Tensor) -> torch.Tensor:
|
||||||
|
return torch.zeros(latents.shape[0], self.audio_channels, 8)
|
||||||
|
|
||||||
|
|
||||||
class StubCosmos3Transformer(nn.Module):
|
class StubCosmos3Transformer(nn.Module):
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
latent_channel_size: int = 2,
|
latent_channel_size: int = 2,
|
||||||
|
sound_gen: bool = False,
|
||||||
|
sound_dim: int = 3,
|
||||||
|
sound_latent_fps: float = 25.0,
|
||||||
|
action_gen: bool = False,
|
||||||
|
action_dim: int = 4,
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.latent_channel_size = latent_channel_size
|
self.latent_channel_size = latent_channel_size
|
||||||
|
self.sound_gen = sound_gen
|
||||||
|
self.sound_dim = sound_dim
|
||||||
|
self.sound_latent_fps = sound_latent_fps
|
||||||
|
self.action_gen = action_gen
|
||||||
|
self.action_dim = action_dim
|
||||||
self.cached_kv: Any | None = None
|
self.cached_kv: Any | None = None
|
||||||
self.cached_freqs_gen: Any | None = None
|
self.cached_freqs_gen: Any | None = None
|
||||||
self.calls: list[dict[str, Any]] = []
|
self.calls: list[dict[str, Any]] = []
|
||||||
@@ -99,8 +127,9 @@ class StubCosmos3Transformer(nn.Module):
|
|||||||
text_ids: torch.Tensor,
|
text_ids: torch.Tensor,
|
||||||
text_mask: torch.Tensor,
|
text_mask: torch.Tensor,
|
||||||
**kwargs: Any,
|
**kwargs: Any,
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||||
token = int(text_ids.reshape(-1)[0].item()) if text_ids.numel() else 0
|
token = int(text_ids.reshape(-1)[0].item()) if text_ids.numel() else 0
|
||||||
|
sound_latents = kwargs.get("sound_latents")
|
||||||
self.calls.append(
|
self.calls.append(
|
||||||
{
|
{
|
||||||
"token": token,
|
"token": token,
|
||||||
@@ -114,7 +143,13 @@ class StubCosmos3Transformer(nn.Module):
|
|||||||
marker = torch.tensor([token], dtype=torch.float32)
|
marker = torch.tensor([token], dtype=torch.float32)
|
||||||
self.cached_kv = [(marker, marker + 100)]
|
self.cached_kv = [(marker, marker + 100)]
|
||||||
self.cached_freqs_gen = (marker + 200, marker + 300)
|
self.cached_freqs_gen = (marker + 200, marker + 300)
|
||||||
return torch.full_like(hidden_states, float(token))
|
action_latents = kwargs.get("action_latents")
|
||||||
|
outputs: list[torch.Tensor] = [torch.full_like(hidden_states, float(token))]
|
||||||
|
if action_latents is not None:
|
||||||
|
outputs.append(torch.full_like(action_latents, float(token + 20)))
|
||||||
|
if sound_latents is not None:
|
||||||
|
outputs.append(torch.full_like(sound_latents, float(token + 10)))
|
||||||
|
return outputs[0] if len(outputs) == 1 else tuple(outputs)
|
||||||
|
|
||||||
|
|
||||||
def passthrough_progress_bar(iterable):
|
def passthrough_progress_bar(iterable):
|
||||||
@@ -155,6 +190,7 @@ def make_cosmos3_pipeline():
|
|||||||
pipeline._guidance_scale = None
|
pipeline._guidance_scale = None
|
||||||
pipeline._num_timesteps = None
|
pipeline._num_timesteps = None
|
||||||
pipeline._cache_dit_requires_paired_cfg = False
|
pipeline._cache_dit_requires_paired_cfg = False
|
||||||
|
pipeline._sound_tokenizer = None
|
||||||
pipeline.progress_bar = passthrough_progress_bar
|
pipeline.progress_bar = passthrough_progress_bar
|
||||||
return pipeline
|
return pipeline
|
||||||
|
|
||||||
@@ -213,7 +249,109 @@ def test_pipeline_registered_and_exported() -> None:
|
|||||||
assert "Cosmos3OmniDiffusersPipeline" in cosmos3.__all__
|
assert "Cosmos3OmniDiffusersPipeline" in cosmos3.__all__
|
||||||
|
|
||||||
|
|
||||||
def test_preprocess_i2v_image_input() -> None:
|
@pytest.fixture
|
||||||
|
def stub_real_pipeline_init(monkeypatch: pytest.MonkeyPatch):
|
||||||
|
from vllm_omni.diffusion.models.cosmos3 import pipeline_cosmos3
|
||||||
|
|
||||||
|
class _StubAutoTokenizer:
|
||||||
|
@classmethod
|
||||||
|
def from_pretrained(cls, *args, **kwargs):
|
||||||
|
return SimpleNamespace()
|
||||||
|
|
||||||
|
class _StubDiffusersVAE:
|
||||||
|
config = SimpleNamespace(scale_factor_temporal=4, scale_factor_spatial=8)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_pretrained(cls, *args, **kwargs):
|
||||||
|
return cls()
|
||||||
|
|
||||||
|
def to(self, _device):
|
||||||
|
return self
|
||||||
|
|
||||||
|
class _StubDiffusersScheduler:
|
||||||
|
config = SimpleNamespace(flow_shift=1.0)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_pretrained(cls, *args, **kwargs):
|
||||||
|
return cls()
|
||||||
|
|
||||||
|
class _StubVideoProcessor:
|
||||||
|
def __init__(self, *args, **kwargs) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
monkeypatch.setattr(pipeline_cosmos3, "AutoTokenizer", _StubAutoTokenizer)
|
||||||
|
monkeypatch.setattr(pipeline_cosmos3, "DistributedAutoencoderKLWan", _StubDiffusersVAE)
|
||||||
|
monkeypatch.setattr(pipeline_cosmos3, "UniPCMultistepScheduler", _StubDiffusersScheduler)
|
||||||
|
monkeypatch.setattr(pipeline_cosmos3, "VideoProcessor", _StubVideoProcessor)
|
||||||
|
monkeypatch.setattr(pipeline_cosmos3, "get_local_device", lambda: torch.device("cpu"))
|
||||||
|
|
||||||
|
|
||||||
|
def _make_od_config(*, sound_gen: bool) -> SimpleNamespace:
|
||||||
|
tf_model_config = {
|
||||||
|
"hidden_size": 8,
|
||||||
|
"num_hidden_layers": 0,
|
||||||
|
"num_attention_heads": 2,
|
||||||
|
"num_key_value_heads": 2,
|
||||||
|
"head_dim": 4,
|
||||||
|
"intermediate_size": 16,
|
||||||
|
"vocab_size": 32,
|
||||||
|
"latent_patch_size": 1,
|
||||||
|
"latent_channel": 2,
|
||||||
|
"rope_scaling": {"mrope_section": [1, 1, 0]},
|
||||||
|
}
|
||||||
|
if sound_gen:
|
||||||
|
tf_model_config["sound_gen"] = True
|
||||||
|
return SimpleNamespace(
|
||||||
|
enable_cpu_offload=False,
|
||||||
|
enable_diffusion_pipeline_profiler=False,
|
||||||
|
model="/nonexistent/model/path",
|
||||||
|
dtype=torch.float32,
|
||||||
|
flow_shift=None,
|
||||||
|
quantization_config=None,
|
||||||
|
custom_pipeline_args={},
|
||||||
|
model_config={},
|
||||||
|
tf_model_config=tf_model_config,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_pipeline_init_skips_tokenizer_when_sound_disabled(stub_real_pipeline_init) -> None:
|
||||||
|
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import Cosmos3OmniDiffusersPipeline
|
||||||
|
|
||||||
|
pipeline = Cosmos3OmniDiffusersPipeline(od_config=_make_od_config(sound_gen=False))
|
||||||
|
|
||||||
|
assert pipeline._sound_tokenizer is None
|
||||||
|
assert pipeline.transformer.sound_gen is False
|
||||||
|
assert not hasattr(pipeline.transformer, "audio_proj_in")
|
||||||
|
assert not hasattr(pipeline.transformer, "audio_proj_out")
|
||||||
|
|
||||||
|
|
||||||
|
def test_pipeline_init_passes_tokenizer_attrs_into_transformer(
|
||||||
|
stub_real_pipeline_init,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
from vllm_omni.diffusion.models.cosmos3 import sound_tokenizer
|
||||||
|
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import Cosmos3OmniDiffusersPipeline
|
||||||
|
|
||||||
|
stub_tokenizer = sound_tokenizer.Cosmos3SoundTokenizer(
|
||||||
|
StubCosmos3AVAE(sample_rate=32000, audio_channels=2, io_channels=5, hop_size=800)
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
sound_tokenizer.Cosmos3SoundTokenizer,
|
||||||
|
"from_config",
|
||||||
|
classmethod(lambda cls, od_config: stub_tokenizer),
|
||||||
|
)
|
||||||
|
|
||||||
|
pipeline = Cosmos3OmniDiffusersPipeline(od_config=_make_od_config(sound_gen=True))
|
||||||
|
|
||||||
|
assert pipeline._sound_tokenizer is stub_tokenizer
|
||||||
|
assert pipeline.transformer.sound_gen is True
|
||||||
|
assert pipeline.transformer.sound_dim == pipeline._sound_tokenizer.latent_ch == 5
|
||||||
|
assert pipeline.transformer.sound_latent_fps == pipeline._sound_tokenizer.latent_fps == 40.0
|
||||||
|
assert pipeline.transformer.audio_proj_in.in_features == 5
|
||||||
|
assert pipeline.transformer.audio_proj_out.out_features == 5
|
||||||
|
|
||||||
|
|
||||||
|
def test_preprocess_i2v_image_and_action_video_inputs() -> None:
|
||||||
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import get_cosmos3_pre_process_func
|
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import get_cosmos3_pre_process_func
|
||||||
|
|
||||||
preprocess = get_cosmos3_pre_process_func(SimpleNamespace())
|
preprocess = get_cosmos3_pre_process_func(SimpleNamespace())
|
||||||
@@ -226,8 +364,18 @@ def test_preprocess_i2v_image_input() -> None:
|
|||||||
assert (result.sampling_params.height, result.sampling_params.width) == (672, 1344)
|
assert (result.sampling_params.height, result.sampling_params.width) == (672, 1344)
|
||||||
assert tuple(result.prompts[0]["additional_information"]["preprocessed_image"].shape[-2:]) == (672, 1344)
|
assert tuple(result.prompts[0]["additional_information"]["preprocessed_image"].shape[-2:]) == (672, 1344)
|
||||||
|
|
||||||
|
frames = [Image.new("RGB", (8, 4), color) for color in ("red", "green", "blue")]
|
||||||
|
action = SimpleNamespace(
|
||||||
|
prompts=[{"prompt": "Move.", "multi_modal_data": {"video": frames}}],
|
||||||
|
sampling_params=SimpleNamespace(height=16, width=32, extra_args={"action_mode": "forward_dynamics"}),
|
||||||
|
)
|
||||||
|
|
||||||
def test_postprocess_handles_image_video_and_validation() -> None:
|
additional = preprocess(action).prompts[0]["additional_information"]
|
||||||
|
assert tuple(additional["preprocessed_image"].shape) == (1, 3, 16, 32)
|
||||||
|
assert tuple(additional["preprocessed_video"].shape) == (1, 3, 3, 16, 32)
|
||||||
|
|
||||||
|
|
||||||
|
def test_postprocess_handles_image_video_audio_and_validation() -> None:
|
||||||
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import get_cosmos3_post_process_func
|
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import get_cosmos3_post_process_func
|
||||||
|
|
||||||
func = get_cosmos3_post_process_func(SimpleNamespace())
|
func = get_cosmos3_post_process_func(SimpleNamespace())
|
||||||
@@ -235,6 +383,16 @@ def test_postprocess_handles_image_video_and_validation() -> None:
|
|||||||
|
|
||||||
assert func(video, output_type="latent") is video
|
assert func(video, output_type="latent") is video
|
||||||
assert func({"image": video})[0].size == (4, 4)
|
assert func({"image": video})[0].size == (4, 4)
|
||||||
|
# Video-only postprocess returns the bare processed video (not a dict),
|
||||||
|
# matching the image/latent branches and peer audio-capable pipelines.
|
||||||
|
assert not isinstance(func({"video": video}), dict)
|
||||||
|
assert (
|
||||||
|
func(
|
||||||
|
{"video": video, "audio": torch.ones(1, 2, 16), "audio_sample_rate": 48000},
|
||||||
|
sampling_params=SimpleNamespace(extra_args={"resolved_frame_rate": 12}),
|
||||||
|
)["audio_sample_rate"]
|
||||||
|
== 48000
|
||||||
|
)
|
||||||
|
|
||||||
with pytest.raises(ValueError, match="text-to-image postprocess expects"):
|
with pytest.raises(ValueError, match="text-to-image postprocess expects"):
|
||||||
func({"image": torch.zeros(1, 3, 2, 4, 4)})
|
func({"image": torch.zeros(1, 3, 2, 4, 4)})
|
||||||
@@ -293,7 +451,7 @@ def test_prompt_formatting_and_checkpoint_key_remap(make_cosmos3_pipeline) -> No
|
|||||||
assert {key: Cosmos3OmniDiffusersPipeline._remap_ckpt_key(key) for key in remaps} == remaps
|
assert {key: Cosmos3OmniDiffusersPipeline._remap_ckpt_key(key) for key in remaps} == remaps
|
||||||
|
|
||||||
|
|
||||||
def test_prepare_latents_for_video_and_image(make_cosmos3_pipeline) -> None:
|
def test_prepare_latents_for_video_image_sound_and_action(make_cosmos3_pipeline) -> None:
|
||||||
pipeline = make_cosmos3_pipeline()
|
pipeline = make_cosmos3_pipeline()
|
||||||
latents = pipeline._prepare_latents(16, 24, 5, torch.Generator(device="cpu").manual_seed(0))
|
latents = pipeline._prepare_latents(16, 24, 5, torch.Generator(device="cpu").manual_seed(0))
|
||||||
assert latents.shape == (1, 2, 2, 2, 3)
|
assert latents.shape == (1, 2, 2, 2, 3)
|
||||||
@@ -306,8 +464,36 @@ def test_prepare_latents_for_video_and_image(make_cosmos3_pipeline) -> None:
|
|||||||
assert velocity_mask.tolist() == [[[[[0.0]], [[1.0]]]]]
|
assert velocity_mask.tolist() == [[[[[0.0]], [[1.0]]]]]
|
||||||
assert image_latent.shape == (1, 2, 1, 2, 3)
|
assert image_latent.shape == (1, 2, 1, 2, 3)
|
||||||
|
|
||||||
|
pipeline.transformer = pipeline.transformer.__class__(latent_channel_size=2, sound_gen=True, sound_dim=3)
|
||||||
|
pipeline._sound_tokenizer = SimpleNamespace(
|
||||||
|
sample_rate=10,
|
||||||
|
latent_ch=3,
|
||||||
|
hop_size=4,
|
||||||
|
decode=lambda x: torch.ones(x.shape[0], 2, 24),
|
||||||
|
)
|
||||||
|
assert pipeline._resolve_sound_target_samples(SimpleNamespace(extra_args={"sound_duration": 2.0}), 9, 3.0) == (
|
||||||
|
20,
|
||||||
|
2.0,
|
||||||
|
10,
|
||||||
|
)
|
||||||
|
sound_latents, latent_frames = pipeline._prepare_sound_latents(21, torch.Generator(device="cpu").manual_seed(0))
|
||||||
|
assert (sound_latents.shape, latent_frames) == (torch.Size([1, 3, 6]), 6)
|
||||||
|
assert pipeline._decode_sound_latents(torch.zeros(1, 3, 6), target_audio_samples=21).shape == (1, 2, 21)
|
||||||
|
|
||||||
def test_diffuse_covers_cfg_and_i2v_steps(make_cosmos3_pipeline) -> None:
|
pipeline.transformer = pipeline.transformer.__class__(action_gen=True, action_dim=4)
|
||||||
|
action, action_mask, clean, raw_dim = pipeline._prepare_action_latents(
|
||||||
|
mode="forward_dynamics",
|
||||||
|
action_chunk_size=2,
|
||||||
|
raw_action_dim=None,
|
||||||
|
generator=torch.Generator(device="cpu").manual_seed(0),
|
||||||
|
sp=SimpleNamespace(extra_args={"action": [[1.0, 2.0], [3.0, 4.0]]}),
|
||||||
|
)
|
||||||
|
assert raw_dim == 2
|
||||||
|
assert action_mask.tolist() == [[[0.0], [0.0]]]
|
||||||
|
torch.testing.assert_close(action, clean)
|
||||||
|
|
||||||
|
|
||||||
|
def test_diffuse_covers_cfg_i2v_and_multimodal_steps(make_cosmos3_pipeline) -> None:
|
||||||
pipeline = make_cosmos3_pipeline()
|
pipeline = make_cosmos3_pipeline()
|
||||||
latents = torch.zeros(1, 2, 1, 1, 1)
|
latents = torch.zeros(1, 2, 1, 1, 1)
|
||||||
|
|
||||||
@@ -339,6 +525,23 @@ def test_diffuse_covers_cfg_and_i2v_steps(make_cosmos3_pipeline) -> None:
|
|||||||
)
|
)
|
||||||
torch.testing.assert_close(i2v[:, :, 0:1], torch.full((1, 2, 1, 1, 1), 7.0))
|
torch.testing.assert_close(i2v[:, :, 0:1], torch.full((1, 2, 1, 1, 1), 7.0))
|
||||||
|
|
||||||
|
pipeline.transformer = pipeline.transformer.__class__(latent_channel_size=2, action_gen=True, action_dim=4)
|
||||||
|
video_result, action_result = pipeline.diffuse(
|
||||||
|
latents=latents,
|
||||||
|
action_latents=torch.zeros(1, 3, 4),
|
||||||
|
action_velocity_mask=torch.ones(1, 3, 1),
|
||||||
|
action_condition_latents=torch.zeros(1, 3, 4),
|
||||||
|
timesteps=torch.tensor([7, 3]),
|
||||||
|
cond_ids=_ids(2),
|
||||||
|
cond_mask=_mask(),
|
||||||
|
uncond_ids=_ids(1),
|
||||||
|
uncond_mask=_mask(),
|
||||||
|
guidance_scale=1.0,
|
||||||
|
shared_kwargs={"video_shape": (1, 1, 1), "fps": 24.0, "action_domain_ids": torch.tensor([0])},
|
||||||
|
)
|
||||||
|
torch.testing.assert_close(video_result, torch.full_like(latents, 4.0))
|
||||||
|
torch.testing.assert_close(action_result, torch.full((), 44.0).expand_as(action_result))
|
||||||
|
|
||||||
|
|
||||||
def test_diffuse_keeps_paired_cfg_when_cache_dit_active(make_cosmos3_pipeline) -> None:
|
def test_diffuse_keeps_paired_cfg_when_cache_dit_active(make_cosmos3_pipeline) -> None:
|
||||||
"""With cache-dit active the uncond pass is kept even outside the guidance
|
"""With cache-dit active the uncond pass is kept even outside the guidance
|
||||||
@@ -395,7 +598,12 @@ class TestForwardRouting:
|
|||||||
|
|
||||||
def fake_diffuse(**kwargs):
|
def fake_diffuse(**kwargs):
|
||||||
captured["diffuse_calls"].append(kwargs)
|
captured["diffuse_calls"].append(kwargs)
|
||||||
return kwargs["latents"] + len(captured["diffuse_calls"])
|
outputs = [kwargs["latents"] + len(captured["diffuse_calls"])]
|
||||||
|
if kwargs.get("action_latents") is not None:
|
||||||
|
outputs.append(kwargs["action_latents"] + 3.0)
|
||||||
|
if kwargs.get("sound_latents") is not None:
|
||||||
|
outputs.append(kwargs["sound_latents"] + 2.0)
|
||||||
|
return outputs[0] if len(outputs) == 1 else tuple(outputs)
|
||||||
|
|
||||||
pipeline._format_and_tokenize_prompts = fake_format
|
pipeline._format_and_tokenize_prompts = fake_format
|
||||||
pipeline._prepare_latents = fake_prepare
|
pipeline._prepare_latents = fake_prepare
|
||||||
@@ -437,7 +645,7 @@ class TestForwardRouting:
|
|||||||
assert captured["flow_shifts"] == expected["flow"]
|
assert captured["flow_shifts"] == expected["flow"]
|
||||||
assert [call[0] for call in pipeline.scheduler.set_timesteps_calls] == expected["steps"]
|
assert [call[0] for call in pipeline.scheduler.set_timesteps_calls] == expected["steps"]
|
||||||
|
|
||||||
def test_forward_i2v_route(self, make_cosmos3_pipeline) -> None:
|
def test_forward_i2v_sound_and_action_routes(self, make_cosmos3_pipeline) -> None:
|
||||||
pipeline = make_cosmos3_pipeline()
|
pipeline = make_cosmos3_pipeline()
|
||||||
captured = self._install_forward_stubs(pipeline)
|
captured = self._install_forward_stubs(pipeline)
|
||||||
image_tensor = torch.zeros(1, 3, 16, 16)
|
image_tensor = torch.zeros(1, 3, 16, 16)
|
||||||
@@ -462,11 +670,55 @@ class TestForwardRouting:
|
|||||||
)
|
)
|
||||||
assert captured["diffuse_calls"][-1]["shared_kwargs"]["noisy_frame_mask"] is velocity_mask
|
assert captured["diffuse_calls"][-1]["shared_kwargs"]["noisy_frame_mask"] is velocity_mask
|
||||||
|
|
||||||
|
pipeline.transformer = pipeline.transformer.__class__(latent_channel_size=2, sound_gen=True, sound_dim=3)
|
||||||
|
sound_latents = torch.zeros(1, 3, 4)
|
||||||
|
pipeline._resolve_sound_target_samples = lambda *args: (20, 2.0, 10)
|
||||||
|
pipeline._prepare_sound_latents = lambda *args: (sound_latents, 4)
|
||||||
|
pipeline._decode_sound_latents = lambda *args: torch.ones(1, 2, 20)
|
||||||
|
output = pipeline.forward(
|
||||||
|
SimpleNamespace(
|
||||||
|
prompts=[{"prompt": "A robot", "modalities": ["video"], "generate_sound": True}],
|
||||||
|
sampling_params=make_sampling_params(num_frames=9, frame_rate=3.0),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assert captured["diffuse_calls"][-1]["sound_latents"] is sound_latents
|
||||||
|
assert output.output["audio_sample_rate"] == 10
|
||||||
|
|
||||||
|
pipeline.transformer = pipeline.transformer.__class__(latent_channel_size=2, action_gen=True, action_dim=4)
|
||||||
|
output = pipeline.forward(
|
||||||
|
SimpleNamespace(
|
||||||
|
prompts=[
|
||||||
|
{
|
||||||
|
"prompt": "Pick the block.",
|
||||||
|
"modalities": ["video"],
|
||||||
|
"additional_information": {"preprocessed_image": image_tensor},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
sampling_params=make_sampling_params(
|
||||||
|
height=16,
|
||||||
|
width=16,
|
||||||
|
extra_args={
|
||||||
|
"action_mode": "policy",
|
||||||
|
"action_chunk_size": 2,
|
||||||
|
"raw_action_dim": 2,
|
||||||
|
"domain_name": "bridge_orig_lerobot",
|
||||||
|
},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assert captured["diffuse_calls"][-1]["shared_kwargs"]["action_domain_ids"].tolist() == [7]
|
||||||
|
assert output.custom_output["action"].shape == (1, 2, 2)
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("prompt", "sampling_params", "message"),
|
("prompt", "sampling_params", "message"),
|
||||||
[
|
[
|
||||||
(["one", "two"], make_sampling_params(), "single prompt"),
|
(["one", "two"], make_sampling_params(), "single prompt"),
|
||||||
([{"prompt": "one", "modalities": ["image", "video"]}], make_sampling_params(), "both image and video"),
|
([{"prompt": "one", "modalities": ["image", "video"]}], make_sampling_params(), "both image and video"),
|
||||||
|
(
|
||||||
|
[{"prompt": "x", "modalities": ["image"], "generate_sound": True}],
|
||||||
|
make_sampling_params(),
|
||||||
|
"only for video",
|
||||||
|
),
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
def test_forward_rejects_invalid_public_requests(
|
def test_forward_rejects_invalid_public_requests(
|
||||||
@@ -477,6 +729,7 @@ class TestForwardRouting:
|
|||||||
message,
|
message,
|
||||||
) -> None:
|
) -> None:
|
||||||
pipeline = make_cosmos3_pipeline()
|
pipeline = make_cosmos3_pipeline()
|
||||||
|
pipeline.transformer = pipeline.transformer.__class__(latent_channel_size=2, sound_gen=True, sound_dim=3)
|
||||||
|
|
||||||
with pytest.raises(ValueError, match=message):
|
with pytest.raises(ValueError, match=message):
|
||||||
pipeline.forward(SimpleNamespace(prompts=prompt, sampling_params=sampling_params))
|
pipeline.forward(SimpleNamespace(prompts=prompt, sampling_params=sampling_params))
|
||||||
@@ -0,0 +1,236 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import torch
|
||||||
|
|
||||||
|
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||||
|
|
||||||
|
DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME = "diffusion_pytorch_model.safetensors"
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeAVAEAudioTokenizer:
|
||||||
|
def __init__(self, **kwargs) -> None:
|
||||||
|
self.kwargs = kwargs
|
||||||
|
self.sample_rate = int(kwargs["sample_rate"])
|
||||||
|
self.audio_channels = int(kwargs["audio_channels"])
|
||||||
|
self.latent_ch = int(kwargs["io_channels"])
|
||||||
|
self.temporal_compression_factor = int(kwargs["hop_size"])
|
||||||
|
|
||||||
|
def get_latent_num_samples(self, num_audio_samples: int) -> int:
|
||||||
|
return int(num_audio_samples) // self.temporal_compression_factor
|
||||||
|
|
||||||
|
def get_audio_num_samples(self, num_latent_samples: int) -> int:
|
||||||
|
return int(num_latent_samples) * self.temporal_compression_factor
|
||||||
|
|
||||||
|
def decode(self, latents: torch.Tensor) -> torch.Tensor:
|
||||||
|
return torch.zeros(latents.shape[0], self.audio_channels, 8)
|
||||||
|
|
||||||
|
|
||||||
|
def _write_component(root: Path, config: dict | None = None, checkpoint_name: str | None = None) -> Path:
|
||||||
|
tokenizer_dir = root / "sound_tokenizer"
|
||||||
|
tokenizer_dir.mkdir(parents=True)
|
||||||
|
if checkpoint_name:
|
||||||
|
(tokenizer_dir / checkpoint_name).write_bytes(b"stub")
|
||||||
|
(tokenizer_dir / "config.json").write_text(json.dumps(config or {}), encoding="utf-8")
|
||||||
|
return tokenizer_dir
|
||||||
|
|
||||||
|
|
||||||
|
def _patch_fake_avae(monkeypatch: pytest.MonkeyPatch, created: dict) -> None:
|
||||||
|
from vllm_omni.diffusion.models.cosmos3 import sound_tokenizer
|
||||||
|
|
||||||
|
class FakeAVAE(_FakeAVAEAudioTokenizer):
|
||||||
|
def __init__(self, **kwargs) -> None:
|
||||||
|
created.update(kwargs)
|
||||||
|
super().__init__(**kwargs)
|
||||||
|
|
||||||
|
monkeypatch.setattr(sound_tokenizer, "Cosmos3AVAEAudioTokenizer", FakeAVAE)
|
||||||
|
monkeypatch.setattr(sound_tokenizer, "get_local_device", lambda: torch.device("cpu"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_from_config_loads_local_diffusers_component(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
from vllm_omni.diffusion.models.cosmos3 import sound_tokenizer
|
||||||
|
|
||||||
|
model_dir = tmp_path / "model"
|
||||||
|
tokenizer_dir = _write_component(model_dir, checkpoint_name=DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME)
|
||||||
|
created = {}
|
||||||
|
_patch_fake_avae(monkeypatch, created)
|
||||||
|
|
||||||
|
tokenizer = sound_tokenizer.Cosmos3SoundTokenizer.from_config(
|
||||||
|
SimpleNamespace(
|
||||||
|
model=str(model_dir),
|
||||||
|
custom_pipeline_args={"sound_sample_rate": 32000, "sound_hop_size": 800, "sound_dim": 3},
|
||||||
|
dtype=torch.float32,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert created["checkpoint_path"] == str(tokenizer_dir / DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME)
|
||||||
|
assert created["config_path"] == str(tokenizer_dir / "config.json")
|
||||||
|
assert (tokenizer.sample_rate, tokenizer.latent_ch, tokenizer.hop_size, tokenizer.latent_fps) == (
|
||||||
|
32000,
|
||||||
|
3,
|
||||||
|
800,
|
||||||
|
40.0,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_from_config_downloads_component_from_hf_repo(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
import huggingface_hub
|
||||||
|
|
||||||
|
from vllm_omni.diffusion.models.cosmos3 import sound_tokenizer
|
||||||
|
|
||||||
|
cache_dir = tmp_path / "hf"
|
||||||
|
_write_component(cache_dir, checkpoint_name=DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME)
|
||||||
|
calls = []
|
||||||
|
created = {}
|
||||||
|
_patch_fake_avae(monkeypatch, created)
|
||||||
|
|
||||||
|
def fake_snapshot_download(repo_id: str, *, revision: str | None, allow_patterns: list[str]) -> str:
|
||||||
|
calls.append((repo_id, revision, allow_patterns))
|
||||||
|
return str(cache_dir)
|
||||||
|
|
||||||
|
monkeypatch.setattr(huggingface_hub, "snapshot_download", fake_snapshot_download)
|
||||||
|
|
||||||
|
sound_tokenizer.Cosmos3SoundTokenizer.from_config(
|
||||||
|
SimpleNamespace(
|
||||||
|
model="nvidia/cosmos3",
|
||||||
|
revision="test-rev",
|
||||||
|
custom_pipeline_args={"sound_sample_rate": 32000, "sound_hop_size": 800, "sound_dim": 3},
|
||||||
|
dtype=torch.float32,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert created["checkpoint_path"].endswith(DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME)
|
||||||
|
assert calls == [
|
||||||
|
(
|
||||||
|
"nvidia/cosmos3",
|
||||||
|
"test-rev",
|
||||||
|
["sound_tokenizer/config.json", f"sound_tokenizer/{DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME}"],
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("checkpoint_name", "message"),
|
||||||
|
[
|
||||||
|
(None, "no AVAE sound tokenizer checkpoint"),
|
||||||
|
("model.safetensors", DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_default_component_requires_diffusers_checkpoint_name(tmp_path, checkpoint_name, message) -> None:
|
||||||
|
from vllm_omni.diffusion.models.cosmos3 import sound_tokenizer
|
||||||
|
|
||||||
|
model_dir = tmp_path / "model"
|
||||||
|
_write_component(model_dir, checkpoint_name=checkpoint_name)
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match=message):
|
||||||
|
sound_tokenizer.Cosmos3SoundTokenizer.from_config(
|
||||||
|
SimpleNamespace(model=str(model_dir), custom_pipeline_args={}, dtype=torch.float32)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_component_config_precedence_and_conflict_detection(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
from vllm_omni.diffusion.models.cosmos3 import sound_tokenizer
|
||||||
|
|
||||||
|
component_config = {
|
||||||
|
"sampling_rate": 48000,
|
||||||
|
"dec_out_channels": 2,
|
||||||
|
"vocoder_input_dim": 64,
|
||||||
|
"hop_size": 1920,
|
||||||
|
}
|
||||||
|
model_dir = tmp_path / "model"
|
||||||
|
_write_component(model_dir, component_config, DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME)
|
||||||
|
created = {}
|
||||||
|
_patch_fake_avae(monkeypatch, created)
|
||||||
|
|
||||||
|
tokenizer = sound_tokenizer.Cosmos3SoundTokenizer.from_config(
|
||||||
|
SimpleNamespace(
|
||||||
|
model=str(model_dir),
|
||||||
|
custom_pipeline_args={
|
||||||
|
"sound_normalize_latents": True,
|
||||||
|
"sound_normalization_type": "tanh",
|
||||||
|
"sound_tanh_input_scale": 2.0,
|
||||||
|
},
|
||||||
|
model_config={
|
||||||
|
"sound_tokenizer": {
|
||||||
|
"sample_rate": 32000,
|
||||||
|
"audio_channels": 1,
|
||||||
|
"io_channels": 3,
|
||||||
|
"hop_size": 800,
|
||||||
|
"normalize_latents": False,
|
||||||
|
"normalization_type": "none",
|
||||||
|
}
|
||||||
|
},
|
||||||
|
dtype=torch.float32,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert (created["sample_rate"], created["audio_channels"], created["io_channels"], created["hop_size"]) == (
|
||||||
|
48000,
|
||||||
|
2,
|
||||||
|
64,
|
||||||
|
1920,
|
||||||
|
)
|
||||||
|
assert (created["normalize_latents"], created["normalization_type"], created["tanh_input_scale"]) == (
|
||||||
|
True,
|
||||||
|
"tanh",
|
||||||
|
2.0,
|
||||||
|
)
|
||||||
|
assert (tokenizer.sample_rate, tokenizer.latent_ch, tokenizer.hop_size, tokenizer.latent_fps) == (
|
||||||
|
48000,
|
||||||
|
64,
|
||||||
|
1920,
|
||||||
|
25.0,
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match=r"sample_rate.*48000.*32000"):
|
||||||
|
sound_tokenizer.Cosmos3SoundTokenizer.from_config(
|
||||||
|
SimpleNamespace(
|
||||||
|
model=str(model_dir),
|
||||||
|
custom_pipeline_args={"sound_sample_rate": 32000},
|
||||||
|
dtype=torch.float32,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_avae_uses_diffusers_decoder_state_dict_layout(tmp_path) -> None:
|
||||||
|
from safetensors.torch import save_file
|
||||||
|
|
||||||
|
from vllm_omni.diffusion.models.cosmos3.audio_tokenizer import avae
|
||||||
|
|
||||||
|
config = {
|
||||||
|
"sampling_rate": 8000,
|
||||||
|
"hop_size": 2,
|
||||||
|
"dec_dim": 4,
|
||||||
|
"dec_c_mults": [1],
|
||||||
|
"dec_strides": [2],
|
||||||
|
"dec_out_channels": 1,
|
||||||
|
"vocoder_input_dim": 2,
|
||||||
|
"normalization_type": "none",
|
||||||
|
}
|
||||||
|
checkpoint_path = tmp_path / DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME
|
||||||
|
config_path = tmp_path / "config.json"
|
||||||
|
config_path.write_text(json.dumps(config), encoding="utf-8")
|
||||||
|
|
||||||
|
decoder = avae.OobleckDecoder(4, 2, 1, [2], [1])
|
||||||
|
save_file({f"decoder.{key}": value for key, value in decoder.state_dict().items()}, str(checkpoint_path))
|
||||||
|
|
||||||
|
tokenizer = avae.Cosmos3AVAEAudioTokenizer(
|
||||||
|
checkpoint_path=checkpoint_path,
|
||||||
|
config_path=config_path,
|
||||||
|
dtype=torch.float32,
|
||||||
|
device="cpu",
|
||||||
|
)
|
||||||
|
|
||||||
|
keys = set(tokenizer.state_dict())
|
||||||
|
assert {"decoder.conv1.weight_g", "decoder.block.0.conv_t1.weight_g", "decoder.conv2.weight_g"} <= keys
|
||||||
|
assert not any(key.startswith(("decoder.layers.", "model.decoder.")) for key in keys)
|
||||||
|
assert tokenizer.decode(torch.zeros(1, 2, 3)).shape == (1, 1, 6)
|
||||||
|
with pytest.raises(NotImplementedError, match="decoder-only"):
|
||||||
|
tokenizer.encode(torch.zeros(1, 1, 6))
|
||||||
@@ -4,6 +4,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
@@ -29,8 +30,10 @@ def _tiny_cosmos3_config(**overrides):
|
|||||||
return config
|
return config
|
||||||
|
|
||||||
|
|
||||||
def test_mrope_position_ids_cover_text_and_video() -> None:
|
def test_mrope_position_ids_cover_text_video_sound_and_action() -> None:
|
||||||
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import (
|
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import (
|
||||||
|
compute_mrope_position_ids_action,
|
||||||
|
compute_mrope_position_ids_sound,
|
||||||
compute_mrope_position_ids_text,
|
compute_mrope_position_ids_text,
|
||||||
compute_mrope_position_ids_vision,
|
compute_mrope_position_ids_vision,
|
||||||
)
|
)
|
||||||
@@ -56,6 +59,14 @@ def test_mrope_position_ids_cover_text_and_video() -> None:
|
|||||||
torch.testing.assert_close(modulated_ids[0], torch.tensor([10.0, 12.0]))
|
torch.testing.assert_close(modulated_ids[0], torch.tensor([10.0, 12.0]))
|
||||||
assert modulated_offset == 13
|
assert modulated_offset == 13
|
||||||
|
|
||||||
|
sound_ids, sound_offset = compute_mrope_position_ids_sound(3, temporal_offset=10, sound_latent_fps=25.0)
|
||||||
|
torch.testing.assert_close(sound_ids[0], torch.tensor([10.0, 10.96, 11.92]))
|
||||||
|
assert sound_offset == 12
|
||||||
|
|
||||||
|
action_ids, action_offset = compute_mrope_position_ids_action(3, temporal_offset=10, action_fps=None)
|
||||||
|
assert action_ids.tolist() == [[11, 12, 13], [0, 0, 0], [0, 0, 0]]
|
||||||
|
assert action_offset == 14
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("key", "value"),
|
("key", "value"),
|
||||||
@@ -120,7 +131,142 @@ def test_forward_returns_video_prediction(monkeypatch: pytest.MonkeyPatch) -> No
|
|||||||
assert tuple(output.shape) == (1, 2, 1, 2, 2)
|
assert tuple(output.shape) == (1, 2, 1, 2, 2)
|
||||||
|
|
||||||
|
|
||||||
def test_compute_rope_freqs_places_text_and_video_positions() -> None:
|
def test_sound_and_action_modules_follow_config() -> None:
|
||||||
|
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3VFMTransformer
|
||||||
|
|
||||||
|
tiny = _tiny_cosmos3_config()
|
||||||
|
no_modal = Cosmos3VFMTransformer(SimpleNamespace(tf_model_config=tiny, dtype=torch.float32))
|
||||||
|
with_sound = Cosmos3VFMTransformer(
|
||||||
|
SimpleNamespace(tf_model_config=tiny, dtype=torch.float32),
|
||||||
|
sound_gen=True,
|
||||||
|
sound_dim=5,
|
||||||
|
sound_latent_fps=40.0,
|
||||||
|
)
|
||||||
|
with_action = Cosmos3VFMTransformer(
|
||||||
|
SimpleNamespace(
|
||||||
|
tf_model_config={**tiny, "action_gen": True, "max_action_dim": 6, "num_embodiment_domains": 9},
|
||||||
|
dtype=torch.float32,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert no_modal.sound_gen is False
|
||||||
|
assert no_modal.action_gen is False
|
||||||
|
assert not hasattr(no_modal, "audio_proj_in")
|
||||||
|
assert not hasattr(no_modal, "action_proj_in")
|
||||||
|
assert with_sound.sound_dim == 5
|
||||||
|
assert with_sound.sound_latent_fps == 40.0
|
||||||
|
assert with_sound.audio_proj_in.in_features == 5
|
||||||
|
assert with_action.action_dim == 6
|
||||||
|
assert with_action.action_proj_in.num_domains == 9
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"kwargs",
|
||||||
|
[
|
||||||
|
{"sound_gen": True},
|
||||||
|
{"sound_gen": True, "sound_dim": 5},
|
||||||
|
{"sound_gen": True, "sound_latent_fps": 40.0},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_transformer_requires_sound_dim_and_fps_when_sound_gen_true(kwargs: dict[str, Any]) -> None:
|
||||||
|
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3VFMTransformer
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match=r"requires an explicit sound_dim and sound_latent_fps"):
|
||||||
|
Cosmos3VFMTransformer(
|
||||||
|
SimpleNamespace(tf_model_config=_tiny_cosmos3_config(), dtype=torch.float32),
|
||||||
|
**kwargs,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_sound_and_action_pack_unpack_validate_shapes() -> None:
|
||||||
|
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3VFMTransformer
|
||||||
|
|
||||||
|
model = object.__new__(Cosmos3VFMTransformer)
|
||||||
|
nn.Module.__init__(model)
|
||||||
|
model.sound_dim = 3
|
||||||
|
model.action_dim = 3
|
||||||
|
|
||||||
|
sound = torch.arange(2 * 3 * 4, dtype=torch.float32).reshape(2, 3, 4)
|
||||||
|
action = torch.arange(2 * 4 * 3, dtype=torch.float32).reshape(2, 4, 3)
|
||||||
|
torch.testing.assert_close(model.unpack_sound(model.pack_sound(sound)), sound)
|
||||||
|
torch.testing.assert_close(model.unpack_action(model.pack_action(action)), action)
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="channel mismatch"):
|
||||||
|
model.pack_sound(torch.zeros(1, 4, 2))
|
||||||
|
with pytest.raises(ValueError, match="dimension mismatch"):
|
||||||
|
model.pack_action(torch.zeros(1, 2, 4))
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("config", "transformer_kwargs", "extra_kwargs", "expected_shapes"),
|
||||||
|
[
|
||||||
|
(
|
||||||
|
_tiny_cosmos3_config(),
|
||||||
|
{"sound_gen": True, "sound_dim": 3, "sound_latent_fps": 24.0},
|
||||||
|
{"sound_latents": torch.zeros(1, 3, 4)},
|
||||||
|
[(1, 2, 1, 2, 2), (1, 3, 4)],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
_tiny_cosmos3_config(action_gen=True, max_action_dim=3, num_embodiment_domains=4),
|
||||||
|
{},
|
||||||
|
{"action_latents": torch.zeros(1, 5, 3), "action_domain_ids": torch.tensor([2])},
|
||||||
|
[(1, 2, 1, 2, 2), (1, 5, 3)],
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_forward_returns_video_plus_optional_modality_predictions(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
config,
|
||||||
|
transformer_kwargs,
|
||||||
|
extra_kwargs,
|
||||||
|
expected_shapes,
|
||||||
|
) -> None:
|
||||||
|
from vllm_omni.diffusion.models.cosmos3 import transformer_cosmos3
|
||||||
|
|
||||||
|
monkeypatch.setattr(transformer_cosmos3, "_get_ulysses_state", lambda: (1, 0, None))
|
||||||
|
|
||||||
|
output = transformer_cosmos3.Cosmos3VFMTransformer(
|
||||||
|
SimpleNamespace(tf_model_config=config, dtype=torch.float32),
|
||||||
|
**transformer_kwargs,
|
||||||
|
)(
|
||||||
|
hidden_states=torch.zeros(1, 2, 1, 2, 2),
|
||||||
|
timestep=torch.tensor([1.0]),
|
||||||
|
text_ids=torch.tensor([[1, 2]], dtype=torch.long),
|
||||||
|
text_mask=torch.ones(1, 2, dtype=torch.long),
|
||||||
|
video_shape=(1, 2, 2),
|
||||||
|
fps=24.0,
|
||||||
|
action_noisy_mask=torch.ones(1, 5, 1),
|
||||||
|
**extra_kwargs,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert isinstance(output, tuple)
|
||||||
|
assert [tuple(tensor.shape) for tensor in output] == expected_shapes
|
||||||
|
|
||||||
|
|
||||||
|
def test_forward_with_sound_ulysses_error_mentions_combined_sequence(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
import vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 as cosmos3_module
|
||||||
|
|
||||||
|
model = cosmos3_module.Cosmos3VFMTransformer(
|
||||||
|
SimpleNamespace(tf_model_config=_tiny_cosmos3_config(), dtype=torch.float32),
|
||||||
|
sound_gen=True,
|
||||||
|
sound_dim=3,
|
||||||
|
sound_latent_fps=40.0,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(cosmos3_module, "_get_ulysses_state", lambda: (2, 0, None))
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match=r"GEN sequence length \(3 = video tokens 2 \+ sound tokens 1\)"):
|
||||||
|
model(
|
||||||
|
hidden_states=torch.zeros(1, 2, 1, 1, 2),
|
||||||
|
timestep=torch.tensor([1.0]),
|
||||||
|
text_ids=torch.tensor([[1, 2]], dtype=torch.long),
|
||||||
|
text_mask=torch.ones(1, 2, dtype=torch.long),
|
||||||
|
video_shape=(1, 1, 2),
|
||||||
|
fps=24.0,
|
||||||
|
sound_latents=torch.zeros(1, 3, 1),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_compute_rope_freqs_places_text_video_action_and_sound_positions() -> None:
|
||||||
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3VFMTransformer
|
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3VFMTransformer
|
||||||
|
|
||||||
class FakeRotary:
|
class FakeRotary:
|
||||||
@@ -140,6 +286,8 @@ def test_compute_rope_freqs_places_text_and_video_positions() -> None:
|
|||||||
model.temporal_modality_margin = 100
|
model.temporal_modality_margin = 100
|
||||||
model.base_fps = 24.0
|
model.base_fps = 24.0
|
||||||
model.temporal_compression_factor = 4
|
model.temporal_compression_factor = 4
|
||||||
|
model.temporal_compression_factor_sound = 1
|
||||||
|
model.sound_latent_fps = 25.0
|
||||||
model.enable_fps_modulation = False
|
model.enable_fps_modulation = False
|
||||||
|
|
||||||
freqs_und, freqs_gen = model._compute_rope_freqs(
|
freqs_und, freqs_gen = model._compute_rope_freqs(
|
||||||
@@ -156,3 +304,21 @@ def test_compute_rope_freqs_places_text_and_video_positions() -> None:
|
|||||||
assert vision_pos[0, 0].tolist() == [102, 103]
|
assert vision_pos[0, 0].tolist() == [102, 103]
|
||||||
assert freqs_und[0].shape == (2, 3, 1, 4)
|
assert freqs_und[0].shape == (2, 3, 1, 4)
|
||||||
assert freqs_gen[0].shape == (2, 2, 1, 4)
|
assert freqs_gen[0].shape == (2, 2, 1, 4)
|
||||||
|
|
||||||
|
rotary.position_ids.clear()
|
||||||
|
model._compute_rope_freqs(
|
||||||
|
text_mask=torch.tensor([[1, 1]], dtype=torch.long),
|
||||||
|
t=2,
|
||||||
|
hp=1,
|
||||||
|
wp=1,
|
||||||
|
fps=24.0,
|
||||||
|
device=torch.device("cpu"),
|
||||||
|
dtype=torch.float32,
|
||||||
|
t_action=2,
|
||||||
|
action_start_frame_offset=1,
|
||||||
|
t_sound=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
_, gen_pos = rotary.position_ids
|
||||||
|
assert gen_pos.shape == (3, 1, 5)
|
||||||
|
assert gen_pos[0, 0].tolist() == [102, 103, 103, 104, 102]
|
||||||
@@ -1,7 +1,6 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import queue
|
import queue
|
||||||
import threading
|
import threading
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
@@ -447,11 +446,7 @@ class TestDiffusionEngine:
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_step_raises_aborted_error(self, mocker: MockerFixture) -> None:
|
async def test_step_raises_aborted_error(self, mocker: MockerFixture) -> None:
|
||||||
engine = DiffusionEngine.__new__(DiffusionEngine)
|
engine = DiffusionEngine.__new__(DiffusionEngine)
|
||||||
engine._closed = False
|
engine._check_and_start_background_loop = mocker.AsyncMock()
|
||||||
engine._loop_started = True
|
|
||||||
engine._init_lock = asyncio.Lock()
|
|
||||||
engine.main_loop = asyncio.get_running_loop()
|
|
||||||
engine.stop_event = threading.Event()
|
|
||||||
engine.pre_process_func = None
|
engine.pre_process_func = None
|
||||||
engine.async_add_req_and_wait_for_response = mocker.AsyncMock(
|
engine.async_add_req_and_wait_for_response = mocker.AsyncMock(
|
||||||
return_value=DiffusionOutput(aborted=True, abort_message="Request req-abort aborted.")
|
return_value=DiffusionOutput(aborted=True, abort_message="Request req-abort aborted.")
|
||||||
@@ -526,6 +521,52 @@ class TestDiffusionEngine:
|
|||||||
with pytest.raises(RuntimeError, match="Dummy run failed: boom"):
|
with pytest.raises(RuntimeError, match="Dummy run failed: boom"):
|
||||||
engine._dummy_run()
|
engine._dummy_run()
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_step_multi_request_reuses_multimodal_slice_logic(self, mocker: MockerFixture) -> None:
|
||||||
|
engine = DiffusionEngine.__new__(DiffusionEngine)
|
||||||
|
engine.od_config = SimpleNamespace(
|
||||||
|
model_class_name="mock_model",
|
||||||
|
enable_cpu_offload=False,
|
||||||
|
)
|
||||||
|
engine.pre_process_func = None
|
||||||
|
engine.post_process_func = None
|
||||||
|
engine._check_and_start_background_loop = mocker.AsyncMock()
|
||||||
|
engine.async_add_req_and_wait_for_response = mocker.AsyncMock(
|
||||||
|
return_value=DiffusionOutput(
|
||||||
|
output={
|
||||||
|
"video": ["frame-0", "frame-1"],
|
||||||
|
"audio": ["audio-0", "audio-1"],
|
||||||
|
"actions": torch.tensor([[1.0, 2.0], [3.0, 4.0]]),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
request = OmniDiffusionRequest(
|
||||||
|
prompts=["prompt-0", "prompt-1"],
|
||||||
|
sampling_params=OmniDiffusionSamplingParams(
|
||||||
|
num_inference_steps=1,
|
||||||
|
num_outputs_per_prompt=1,
|
||||||
|
),
|
||||||
|
request_id="req-batch",
|
||||||
|
)
|
||||||
|
|
||||||
|
mocker.patch("vllm_omni.diffusion.diffusion_engine.supports_audio_output", return_value=False)
|
||||||
|
outputs = await engine.step(request)
|
||||||
|
|
||||||
|
assert len(outputs) == 2
|
||||||
|
assert outputs[0].images == ["frame-0"]
|
||||||
|
assert outputs[1].images == ["frame-1"]
|
||||||
|
assert outputs[0].multimodal_output["audio"] == "audio-0"
|
||||||
|
assert outputs[1].multimodal_output["audio"] == "audio-1"
|
||||||
|
torch.testing.assert_close(
|
||||||
|
outputs[0].multimodal_output["actions"],
|
||||||
|
torch.tensor([1.0, 2.0]),
|
||||||
|
)
|
||||||
|
torch.testing.assert_close(
|
||||||
|
outputs[1].multimodal_output["actions"],
|
||||||
|
torch.tensor([3.0, 4.0]),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class TestStepScheduler:
|
class TestStepScheduler:
|
||||||
def setup_method(self) -> None:
|
def setup_method(self) -> None:
|
||||||
|
|||||||
@@ -32,8 +32,8 @@ BAGEL_MOONCAKE_CI_DEPLOY = get_deploy_config_path("ci/bagel_mooncake.yaml")
|
|||||||
|
|
||||||
# Reference pixel data extracted from the known-good output image
|
# Reference pixel data extracted from the known-good output image
|
||||||
# Each entry contains (x, y) position and expected (R, G, B) values
|
# Each entry contains (x, y) position and expected (R, G, B) values
|
||||||
# "Generated with seed=52, num_inference_steps=15,
|
# "Generated with seed=52, num_inference_steps=14,
|
||||||
# prompt='A futuristic city skyline at twilight, cyberpunk style'"
|
# prompt='A cute cat'"
|
||||||
REFERENCE_PIXELS = [
|
REFERENCE_PIXELS = [
|
||||||
{"position": (100, 100), "rgb": (115, 113, 94)},
|
{"position": (100, 100), "rgb": (115, 113, 94)},
|
||||||
{"position": (400, 50), "rgb": (159, 160, 144)},
|
{"position": (400, 50), "rgb": (159, 160, 144)},
|
||||||
@@ -63,7 +63,7 @@ def _find_free_port() -> int:
|
|||||||
return port
|
return port
|
||||||
|
|
||||||
|
|
||||||
def _configure_sampling_params(omni: Omni, num_inference_steps: int = 15) -> list:
|
def _configure_sampling_params(omni: Omni, num_inference_steps: int = 14) -> list:
|
||||||
"""Configure sampling parameters for Bagel text2img generation.
|
"""Configure sampling parameters for Bagel text2img generation.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -269,7 +269,6 @@ def _load_mooncake_config(host: str, rpc_port: int, http_port: int) -> str:
|
|||||||
@pytest.mark.core_model
|
@pytest.mark.core_model
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
@pytest.mark.diffusion
|
@pytest.mark.diffusion
|
||||||
@pytest.mark.skip(reason="Skip failed CI issue 3977: https://github.com/vllm-project/vllm-omni/issues/3977")
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||||
def test_bagel_text2img_mooncake_connector(run_level):
|
def test_bagel_text2img_mooncake_connector(run_level):
|
||||||
"""Test Bagel text2img with Mooncake connector for inter-stage communication."""
|
"""Test Bagel text2img with Mooncake connector for inter-stage communication."""
|
||||||
|
|||||||
@@ -24,14 +24,13 @@ from tests.helpers.mark import hardware_test
|
|||||||
from tests.helpers.runtime import OmniRunner
|
from tests.helpers.runtime import OmniRunner
|
||||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||||
from vllm_omni.entrypoints.omni import Omni
|
from vllm_omni.entrypoints.omni import Omni
|
||||||
from vllm_omni.platforms import current_omni_platform
|
|
||||||
|
|
||||||
pytestmark = [pytest.mark.usefixtures("clean_gpu_memory_between_tests")]
|
pytestmark = [pytest.mark.usefixtures("clean_gpu_memory_between_tests")]
|
||||||
|
|
||||||
BAGEL_CI_DEPLOY = get_deploy_config_path("ci/bagel.yaml")
|
BAGEL_CI_DEPLOY = get_deploy_config_path("ci/bagel.yaml")
|
||||||
|
|
||||||
# Reference pixel data extracted from the known-good output image
|
# Reference pixel data extracted from the known-good output image
|
||||||
# Generated with seed=52, num_inference_steps=15,
|
# Generated with seed=52, num_inference_steps=14,
|
||||||
# prompt='Change the grass color to red',
|
# prompt='Change the grass color to red',
|
||||||
# input image: 2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg
|
# input image: 2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg
|
||||||
REFERENCE_PIXELS = [
|
REFERENCE_PIXELS = [
|
||||||
@@ -47,23 +46,10 @@ REFERENCE_PIXELS = [
|
|||||||
{"position": (256, 256), "rgb": (181, 201, 221)},
|
{"position": (256, 256), "rgb": (181, 201, 221)},
|
||||||
]
|
]
|
||||||
|
|
||||||
if current_omni_platform.is_rocm():
|
|
||||||
REFERENCE_PIXELS = [
|
|
||||||
{"position": (100, 100), "rgb": (156, 172, 217)},
|
|
||||||
{"position": (400, 50), "rgb": (105, 144, 217)},
|
|
||||||
{"position": (700, 100), "rgb": (118, 159, 232)},
|
|
||||||
{"position": (150, 400), "rgb": (180, 22, 52)},
|
|
||||||
{"position": (512, 336), "rgb": (221, 211, 194)},
|
|
||||||
{"position": (700, 400), "rgb": (192, 10, 46)},
|
|
||||||
{"position": (100, 600), "rgb": (102, 12, 22)},
|
|
||||||
{"position": (400, 600), "rgb": (161, 28, 47)},
|
|
||||||
{"position": (700, 600), "rgb": (100, 87, 94)},
|
|
||||||
{"position": (256, 256), "rgb": (181, 201, 221)},
|
|
||||||
]
|
|
||||||
|
|
||||||
# text2img reference pixels (aligned with offline `bagel/end2end.py` text2img, 15 steps)
|
# text2img reference pixels (aligned with offline `bagel/end2end.py` text2img, 15 steps)
|
||||||
# "Generated with seed=52, num_inference_steps=15,
|
# "Generated with seed=52, num_inference_steps=14,
|
||||||
# prompt='A futuristic city skyline at twilight, cyberpunk style'"
|
# prompt='A cute cat'"
|
||||||
TEXT2IMG_REFERENCE_PIXELS = [
|
TEXT2IMG_REFERENCE_PIXELS = [
|
||||||
{"position": (100, 100), "rgb": (115, 113, 94)},
|
{"position": (100, 100), "rgb": (115, 113, 94)},
|
||||||
{"position": (400, 50), "rgb": (159, 160, 144)},
|
{"position": (400, 50), "rgb": (159, 160, 144)},
|
||||||
@@ -77,19 +63,6 @@ TEXT2IMG_REFERENCE_PIXELS = [
|
|||||||
{"position": (256, 256), "rgb": (167, 156, 150)},
|
{"position": (256, 256), "rgb": (167, 156, 150)},
|
||||||
]
|
]
|
||||||
|
|
||||||
if current_omni_platform.is_rocm():
|
|
||||||
TEXT2IMG_REFERENCE_PIXELS = [
|
|
||||||
{"position": (100, 100), "rgb": (115, 113, 94)},
|
|
||||||
{"position": (400, 50), "rgb": (159, 160, 144)},
|
|
||||||
{"position": (700, 100), "rgb": (164, 151, 123)},
|
|
||||||
{"position": (150, 400), "rgb": (120, 121, 107)},
|
|
||||||
{"position": (512, 512), "rgb": (165, 133, 127)},
|
|
||||||
{"position": (700, 400), "rgb": (217, 130, 66)},
|
|
||||||
{"position": (100, 700), "rgb": (191, 168, 152)},
|
|
||||||
{"position": (400, 700), "rgb": (130, 96, 77)},
|
|
||||||
{"position": (700, 700), "rgb": (247, 203, 140)},
|
|
||||||
{"position": (256, 256), "rgb": (167, 156, 150)},
|
|
||||||
]
|
|
||||||
|
|
||||||
PIXEL_TOLERANCE = 10
|
PIXEL_TOLERANCE = 10
|
||||||
TEXT2IMG_PIXEL_TOLERANCE = 5
|
TEXT2IMG_PIXEL_TOLERANCE = 5
|
||||||
@@ -115,7 +88,7 @@ def _find_free_port() -> int:
|
|||||||
return port
|
return port
|
||||||
|
|
||||||
|
|
||||||
def _configure_sampling_params(omni: Omni, num_inference_steps: int = 15) -> list:
|
def _configure_sampling_params(omni: Omni, num_inference_steps: int = 14) -> list:
|
||||||
"""Configure sampling parameters for Bagel img2img generation.
|
"""Configure sampling parameters for Bagel img2img generation.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -258,7 +231,6 @@ def _resolve_deploy_config(config_path: str, run_level: str) -> str:
|
|||||||
@pytest.mark.core_model
|
@pytest.mark.core_model
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
@pytest.mark.diffusion
|
@pytest.mark.diffusion
|
||||||
@pytest.mark.skip(reason="Skip failed CI issue 3977: https://github.com/vllm-project/vllm-omni/issues/3977")
|
|
||||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
|
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
|
||||||
def test_bagel_img2img_shared_memory_connector(run_level):
|
def test_bagel_img2img_shared_memory_connector(run_level):
|
||||||
"""Test Bagel img2img with shared memory connector."""
|
"""Test Bagel img2img with shared memory connector."""
|
||||||
@@ -276,7 +248,6 @@ def test_bagel_img2img_shared_memory_connector(run_level):
|
|||||||
@pytest.mark.core_model
|
@pytest.mark.core_model
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
@pytest.mark.diffusion
|
@pytest.mark.diffusion
|
||||||
@pytest.mark.skip(reason="Skip failed CI issue 3977: https://github.com/vllm-project/vllm-omni/issues/3977")
|
|
||||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
|
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
|
||||||
def test_bagel_text2img_shared_memory_connector(run_level):
|
def test_bagel_text2img_shared_memory_connector(run_level):
|
||||||
"""Test Bagel text2img with shared memory connector."""
|
"""Test Bagel text2img with shared memory connector."""
|
||||||
|
|||||||
@@ -0,0 +1,307 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
try:
|
||||||
|
import cv2
|
||||||
|
except ImportError: # pragma: no cover - optional e2e dependency
|
||||||
|
cv2 = None
|
||||||
|
|
||||||
|
try:
|
||||||
|
import websockets.sync.client as websockets_client
|
||||||
|
except ImportError: # pragma: no cover - optional e2e dependency
|
||||||
|
websockets_client = None
|
||||||
|
|
||||||
|
try:
|
||||||
|
from openpi_client import msgpack_numpy
|
||||||
|
except ImportError: # pragma: no cover - optional e2e dependency
|
||||||
|
msgpack_numpy = None
|
||||||
|
|
||||||
|
PING_INTERVAL_SECS = 300
|
||||||
|
PING_TIMEOUT_SECS = 3600
|
||||||
|
DEFAULT_HOST = "127.0.0.1"
|
||||||
|
DEFAULT_PORT = 8000
|
||||||
|
DEFAULT_PATH = "/v1/realtime/robot/openpi"
|
||||||
|
DEFAULT_PROMPT = "Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan"
|
||||||
|
ACTION_HORIZON = 24
|
||||||
|
DEFAULT_ACTION_DIM = 8
|
||||||
|
RELATIVE_OFFSETS = [-23, -16, -8, 0]
|
||||||
|
CAMERA_FILES = {
|
||||||
|
"observation/exterior_image_0_left": "exterior_image_1_left.mp4",
|
||||||
|
"observation/exterior_image_1_left": "exterior_image_2_left.mp4",
|
||||||
|
"observation/wrist_image_left": "wrist_image_left.mp4",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _decode_action_response(response: bytes | str) -> np.ndarray:
|
||||||
|
if isinstance(response, str):
|
||||||
|
raise RuntimeError(f"Inference failed: {response}")
|
||||||
|
decoded = msgpack_numpy.unpackb(response)
|
||||||
|
if isinstance(decoded, dict) and decoded.get("type") == "error":
|
||||||
|
message = decoded.get("message", decoded)
|
||||||
|
raise RuntimeError(f"Inference failed: {message}")
|
||||||
|
return np.asarray(decoded, dtype=np.float32)
|
||||||
|
|
||||||
|
|
||||||
|
def require_dependencies() -> None:
|
||||||
|
missing = []
|
||||||
|
if cv2 is None:
|
||||||
|
missing.append("opencv-python")
|
||||||
|
if websockets_client is None:
|
||||||
|
missing.append("websockets")
|
||||||
|
if msgpack_numpy is None:
|
||||||
|
missing.append("openpi-client")
|
||||||
|
if missing:
|
||||||
|
raise ModuleNotFoundError(f"DreamZero OpenPI test dependencies are missing: {', '.join(missing)}")
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class DreamZeroServerMetadata:
|
||||||
|
image_resolution: tuple[int, int]
|
||||||
|
n_external_cameras: int
|
||||||
|
needs_wrist_camera: bool
|
||||||
|
needs_stereo_camera: bool
|
||||||
|
needs_session_id: bool
|
||||||
|
action_space: str
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_dict(cls, payload: dict[str, Any]) -> DreamZeroServerMetadata:
|
||||||
|
required_keys = (
|
||||||
|
"image_resolution",
|
||||||
|
"n_external_cameras",
|
||||||
|
"needs_wrist_camera",
|
||||||
|
"needs_stereo_camera",
|
||||||
|
"needs_session_id",
|
||||||
|
"action_space",
|
||||||
|
)
|
||||||
|
missing_keys = [key for key in required_keys if key not in payload]
|
||||||
|
if missing_keys:
|
||||||
|
raise ValueError(f"Missing DreamZero metadata keys: {missing_keys}")
|
||||||
|
|
||||||
|
image_resolution = payload["image_resolution"]
|
||||||
|
if not isinstance(image_resolution, (list, tuple)) or len(image_resolution) != 2:
|
||||||
|
raise ValueError(f"Invalid image_resolution: {image_resolution!r}")
|
||||||
|
|
||||||
|
return cls(
|
||||||
|
image_resolution=(int(image_resolution[0]), int(image_resolution[1])),
|
||||||
|
n_external_cameras=int(payload["n_external_cameras"]),
|
||||||
|
needs_wrist_camera=bool(payload["needs_wrist_camera"]),
|
||||||
|
needs_stereo_camera=bool(payload["needs_stereo_camera"]),
|
||||||
|
needs_session_id=bool(payload["needs_session_id"]),
|
||||||
|
action_space=str(payload["action_space"]),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class OpenPIWebsocketClient:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
host: str = DEFAULT_HOST,
|
||||||
|
port: int = DEFAULT_PORT,
|
||||||
|
path: str = DEFAULT_PATH,
|
||||||
|
) -> None:
|
||||||
|
require_dependencies()
|
||||||
|
self._uri = f"ws://{host}:{port}{path}"
|
||||||
|
self._packer = msgpack_numpy.Packer()
|
||||||
|
self._ws, self._server_metadata = self._connect()
|
||||||
|
|
||||||
|
def _connect(self):
|
||||||
|
conn = websockets_client.connect(
|
||||||
|
self._uri,
|
||||||
|
compression=None,
|
||||||
|
max_size=None,
|
||||||
|
ping_interval=PING_INTERVAL_SECS,
|
||||||
|
ping_timeout=PING_TIMEOUT_SECS,
|
||||||
|
)
|
||||||
|
metadata = msgpack_numpy.unpackb(conn.recv())
|
||||||
|
if not isinstance(metadata, dict):
|
||||||
|
raise TypeError(f"Expected dict metadata from server, got {type(metadata)!r}")
|
||||||
|
return conn, metadata
|
||||||
|
|
||||||
|
def get_server_metadata(self) -> dict[str, Any]:
|
||||||
|
return dict(self._server_metadata)
|
||||||
|
|
||||||
|
def infer(self, obs: dict[str, Any]) -> np.ndarray:
|
||||||
|
payload = dict(obs)
|
||||||
|
payload["endpoint"] = "infer"
|
||||||
|
self._ws.send(self._packer.pack(payload))
|
||||||
|
response = self._ws.recv()
|
||||||
|
return _decode_action_response(response)
|
||||||
|
|
||||||
|
def reset(self, reset_info: dict[str, Any] | None = None) -> str:
|
||||||
|
payload = dict(reset_info or {})
|
||||||
|
payload["endpoint"] = "reset"
|
||||||
|
self._ws.send(self._packer.pack(payload))
|
||||||
|
response = self._ws.recv()
|
||||||
|
if isinstance(response, str):
|
||||||
|
return response
|
||||||
|
decoded = msgpack_numpy.unpackb(response)
|
||||||
|
if not isinstance(decoded, dict) or decoded.get("status") != "reset successful":
|
||||||
|
raise RuntimeError(f"Unexpected reset response: {decoded!r}")
|
||||||
|
return str(decoded["status"])
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
self._ws.close()
|
||||||
|
|
||||||
|
|
||||||
|
def load_all_frames(video_path: Path) -> np.ndarray:
|
||||||
|
require_dependencies()
|
||||||
|
cap = cv2.VideoCapture(str(video_path))
|
||||||
|
frames = []
|
||||||
|
while True:
|
||||||
|
ok, frame = cap.read()
|
||||||
|
if not ok:
|
||||||
|
break
|
||||||
|
frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
|
||||||
|
cap.release()
|
||||||
|
if not frames:
|
||||||
|
raise RuntimeError(f"No frames loaded from {video_path}")
|
||||||
|
return np.stack(frames, axis=0)
|
||||||
|
|
||||||
|
|
||||||
|
def load_camera_frames(video_dir: Path) -> dict[str, np.ndarray]:
|
||||||
|
camera_frames: dict[str, np.ndarray] = {}
|
||||||
|
for camera_key, file_name in CAMERA_FILES.items():
|
||||||
|
video_path = video_dir / file_name
|
||||||
|
if not video_path.exists():
|
||||||
|
raise FileNotFoundError(f"Missing DreamZero test asset: {video_path}")
|
||||||
|
camera_frames[camera_key] = load_all_frames(video_path)
|
||||||
|
return camera_frames
|
||||||
|
|
||||||
|
|
||||||
|
def build_frame_schedule(total_frames: int, num_chunks: int) -> list[list[int]]:
|
||||||
|
chunks: list[list[int]] = []
|
||||||
|
current_frame = 23
|
||||||
|
for _ in range(num_chunks):
|
||||||
|
indices = [max(current_frame + offset, 0) for offset in RELATIVE_OFFSETS]
|
||||||
|
if indices[-1] >= total_frames:
|
||||||
|
break
|
||||||
|
chunks.append(indices)
|
||||||
|
current_frame += ACTION_HORIZON
|
||||||
|
return chunks
|
||||||
|
|
||||||
|
|
||||||
|
def make_obs_from_video(
|
||||||
|
camera_frames: dict[str, np.ndarray],
|
||||||
|
frame_indices: list[int],
|
||||||
|
*,
|
||||||
|
prompt: str,
|
||||||
|
session_id: str,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
obs: dict[str, Any] = {}
|
||||||
|
for camera_key, all_frames in camera_frames.items():
|
||||||
|
selected = all_frames[frame_indices]
|
||||||
|
obs[camera_key] = selected[0] if len(frame_indices) == 1 else selected
|
||||||
|
|
||||||
|
obs["observation/joint_position"] = np.zeros(7, dtype=np.float32)
|
||||||
|
obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32)
|
||||||
|
obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32)
|
||||||
|
obs["prompt"] = prompt
|
||||||
|
obs["session_id"] = session_id
|
||||||
|
return obs
|
||||||
|
|
||||||
|
|
||||||
|
def build_demo_observations(
|
||||||
|
camera_frames: dict[str, np.ndarray],
|
||||||
|
*,
|
||||||
|
prompt: str,
|
||||||
|
session_id: str,
|
||||||
|
num_chunks: int = 2,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
if num_chunks < 1:
|
||||||
|
raise ValueError("num_chunks must be at least 1")
|
||||||
|
|
||||||
|
total_frames = min(frames.shape[0] for frames in camera_frames.values())
|
||||||
|
observations = [
|
||||||
|
make_obs_from_video(
|
||||||
|
camera_frames,
|
||||||
|
[0],
|
||||||
|
prompt=prompt,
|
||||||
|
session_id=session_id,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
for indices in build_frame_schedule(total_frames, num_chunks - 1):
|
||||||
|
observations.append(
|
||||||
|
make_obs_from_video(
|
||||||
|
camera_frames,
|
||||||
|
indices,
|
||||||
|
prompt=prompt,
|
||||||
|
session_id=session_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return observations
|
||||||
|
|
||||||
|
|
||||||
|
def validate_session_result(
|
||||||
|
result: dict[str, Any],
|
||||||
|
*,
|
||||||
|
expected_action_horizon: int = ACTION_HORIZON,
|
||||||
|
expected_action_dim: int = DEFAULT_ACTION_DIM,
|
||||||
|
) -> None:
|
||||||
|
metadata = DreamZeroServerMetadata.from_dict(result["metadata"])
|
||||||
|
if metadata.image_resolution != (180, 320):
|
||||||
|
raise AssertionError(f"Unexpected image_resolution: {metadata.image_resolution}")
|
||||||
|
if metadata.n_external_cameras != 2:
|
||||||
|
raise AssertionError(f"Unexpected n_external_cameras: {metadata.n_external_cameras}")
|
||||||
|
if not metadata.needs_wrist_camera:
|
||||||
|
raise AssertionError("DreamZero test expects wrist camera metadata")
|
||||||
|
if metadata.action_space != "joint_position":
|
||||||
|
raise AssertionError(f"Unexpected action_space: {metadata.action_space}")
|
||||||
|
|
||||||
|
actions = result["actions"]
|
||||||
|
if len(actions) != 3:
|
||||||
|
raise AssertionError(f"Expected 3 action tensors, got {len(actions)}")
|
||||||
|
for index, action in enumerate(actions):
|
||||||
|
if action.shape != (expected_action_horizon, expected_action_dim):
|
||||||
|
raise AssertionError(
|
||||||
|
f"Action {index} shape mismatch: expected "
|
||||||
|
f"{(expected_action_horizon, expected_action_dim)}, got {action.shape}"
|
||||||
|
)
|
||||||
|
if not np.isfinite(action).all():
|
||||||
|
raise AssertionError(f"Action {index} contains non-finite values")
|
||||||
|
|
||||||
|
if result["reset_status"] != "reset successful":
|
||||||
|
raise AssertionError(f"Unexpected reset status: {result['reset_status']!r}")
|
||||||
|
|
||||||
|
|
||||||
|
def run_policy_session(
|
||||||
|
*,
|
||||||
|
host: str = DEFAULT_HOST,
|
||||||
|
port: int = DEFAULT_PORT,
|
||||||
|
path: str = DEFAULT_PATH,
|
||||||
|
video_dir: Path,
|
||||||
|
prompt: str = DEFAULT_PROMPT,
|
||||||
|
session_id: str | None = None,
|
||||||
|
num_chunks: int = 2,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
session_id = session_id or str(uuid.uuid4())
|
||||||
|
camera_frames = load_camera_frames(video_dir)
|
||||||
|
observations = build_demo_observations(
|
||||||
|
camera_frames,
|
||||||
|
prompt=prompt,
|
||||||
|
session_id=session_id,
|
||||||
|
num_chunks=num_chunks,
|
||||||
|
)
|
||||||
|
|
||||||
|
client = OpenPIWebsocketClient(host=host, port=port, path=path)
|
||||||
|
try:
|
||||||
|
metadata = client.get_server_metadata()
|
||||||
|
actions = [client.infer(obs) for obs in observations]
|
||||||
|
reset_status = client.reset({})
|
||||||
|
actions.append(client.infer(observations[0]))
|
||||||
|
return {
|
||||||
|
"metadata": metadata,
|
||||||
|
"actions": actions,
|
||||||
|
"reset_status": reset_status,
|
||||||
|
"session_id": session_id,
|
||||||
|
}
|
||||||
|
finally:
|
||||||
|
client.close()
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from vllm_omni.diffusion.models.dreamzero.utils import (
|
||||||
|
DEFAULT_CFG_SCALE,
|
||||||
|
DEFAULT_EMBODIMENT_NAME_TO_ID,
|
||||||
|
DEFAULT_NEGATIVE_PROMPT,
|
||||||
|
DEFAULT_NUM_INFERENCE_STEPS,
|
||||||
|
DEFAULT_SEED,
|
||||||
|
DEFAULT_SIGMA_SHIFT,
|
||||||
|
)
|
||||||
|
|
||||||
|
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||||
|
|
||||||
|
|
||||||
|
def test_dreamzero_default_constants_match_source_baseline():
|
||||||
|
assert DEFAULT_NUM_INFERENCE_STEPS == 16
|
||||||
|
assert DEFAULT_CFG_SCALE == 5.0
|
||||||
|
assert DEFAULT_SIGMA_SHIFT == 5.0
|
||||||
|
assert DEFAULT_SEED == 1140
|
||||||
|
assert "worst quality" in DEFAULT_NEGATIVE_PROMPT
|
||||||
|
assert DEFAULT_EMBODIMENT_NAME_TO_ID["oxe_droid"] == 17
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from tests.dreamzero import openpi_client_helper
|
||||||
|
|
||||||
|
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||||
|
|
||||||
|
|
||||||
|
class FakeMsgpackNumpy:
|
||||||
|
payloads = {}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def packb(obj):
|
||||||
|
key = str(len(FakeMsgpackNumpy.payloads)).encode()
|
||||||
|
FakeMsgpackNumpy.payloads[key] = obj
|
||||||
|
return key
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def unpackb(data):
|
||||||
|
return FakeMsgpackNumpy.payloads[data]
|
||||||
|
|
||||||
|
|
||||||
|
def test_decode_action_response_surfaces_structured_error(monkeypatch):
|
||||||
|
monkeypatch.setattr(openpi_client_helper, "msgpack_numpy", FakeMsgpackNumpy)
|
||||||
|
payload = FakeMsgpackNumpy.packb(
|
||||||
|
{
|
||||||
|
"type": "error",
|
||||||
|
"message": "Internal inference error",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(RuntimeError, match="Internal inference error"):
|
||||||
|
openpi_client_helper._decode_action_response(payload)
|
||||||
|
|
||||||
|
|
||||||
|
def test_decode_action_response_converts_action_payload_to_float32(monkeypatch):
|
||||||
|
monkeypatch.setattr(openpi_client_helper, "msgpack_numpy", FakeMsgpackNumpy)
|
||||||
|
payload = FakeMsgpackNumpy.packb(np.asarray([[1.0, 2.0]], dtype=np.float64))
|
||||||
|
|
||||||
|
actions = openpi_client_helper._decode_action_response(payload)
|
||||||
|
|
||||||
|
assert actions.dtype == np.float32
|
||||||
|
np.testing.assert_array_equal(actions, np.asarray([[1.0, 2.0]], dtype=np.float32))
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
from collections import OrderedDict
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from vllm_omni.diffusion.models.dreamzero.pipeline_dreamzero import DreamZeroPipeline
|
||||||
|
from vllm_omni.diffusion.models.dreamzero.state_dreamzero import DreamZeroState
|
||||||
|
|
||||||
|
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||||
|
|
||||||
|
|
||||||
|
def _empty_pipeline() -> DreamZeroPipeline:
|
||||||
|
pipeline = DreamZeroPipeline.__new__(DreamZeroPipeline)
|
||||||
|
pipeline._states = OrderedDict()
|
||||||
|
pipeline._max_session_states = 2
|
||||||
|
return pipeline
|
||||||
|
|
||||||
|
|
||||||
|
def test_dreamzero_pipeline_state_is_session_keyed() -> None:
|
||||||
|
pipeline = _empty_pipeline()
|
||||||
|
|
||||||
|
session_a = pipeline._get_or_create_state("session-a")
|
||||||
|
session_b = pipeline._get_or_create_state("session-b")
|
||||||
|
session_a.call_count = 7
|
||||||
|
session_b.call_count = 3
|
||||||
|
|
||||||
|
assert pipeline._get_or_create_state("session-a") is session_a
|
||||||
|
assert pipeline._get_or_create_state("session-b") is session_b
|
||||||
|
assert session_a.call_count == 7
|
||||||
|
assert session_b.call_count == 3
|
||||||
|
|
||||||
|
|
||||||
|
def test_dreamzero_pipeline_state_lru_caps_retained_sessions() -> None:
|
||||||
|
pipeline = _empty_pipeline()
|
||||||
|
|
||||||
|
session_a = pipeline._get_or_create_state("session-a")
|
||||||
|
pipeline._get_or_create_state("session-b")
|
||||||
|
assert pipeline._get_or_create_state("session-a") is session_a
|
||||||
|
|
||||||
|
pipeline._get_or_create_state("session-c")
|
||||||
|
|
||||||
|
assert list(pipeline._states) == ["session-a", "session-c"]
|
||||||
|
assert "session-b" not in pipeline._states
|
||||||
|
|
||||||
|
|
||||||
|
def test_dreamzero_state_cache_access_requires_initialization() -> None:
|
||||||
|
state = DreamZeroState()
|
||||||
|
|
||||||
|
with pytest.raises(RuntimeError, match="KV caches not initialized"):
|
||||||
|
state.get_kv_caches()
|
||||||
|
|
||||||
|
with pytest.raises(RuntimeError, match="Cross-attn caches not initialized"):
|
||||||
|
state.get_crossattn_caches()
|
||||||
|
|
||||||
|
with pytest.raises(RuntimeError, match="create_kv_caches first"):
|
||||||
|
state.update_kv_cache(0, torch.empty(0))
|
||||||
@@ -0,0 +1,381 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Copied from the DreamZero repository's `test_client_AR.py`:
|
||||||
|
https://github.com/dreamzero0/dreamzero/blob/main/test_client_AR.py
|
||||||
|
|
||||||
|
Kept here for end-to-end compatibility / parity testing against the vLLM
|
||||||
|
OpenPI server.
|
||||||
|
|
||||||
|
Test client for AR_droid policy server using roboarena interface.
|
||||||
|
|
||||||
|
Sends real video frames from debug_image/ directory instead of zero dummy images.
|
||||||
|
|
||||||
|
Frame schedule (matching debug_inference.py):
|
||||||
|
- Step 0 (initial): send frame [0] (1 frame, H W 3)
|
||||||
|
- Step 1: send frames [0, 7, 15, 23] (4 frames, 4 H W 3)
|
||||||
|
- Step 2: send frames [24, 31, 39, 47] (4 frames)
|
||||||
|
- Step 3: send frames [48, 55, 63, 71] (4 frames)
|
||||||
|
- ...
|
||||||
|
|
||||||
|
Expected server configuration:
|
||||||
|
- image_resolution: (180, 320)
|
||||||
|
- n_external_cameras: 2
|
||||||
|
- needs_wrist_camera: True
|
||||||
|
- action_space: "joint_position"
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
# Start server with roboarena interface:
|
||||||
|
torchrun --nproc_per_node=8 socket_test_optimized_AR.py --port 8000
|
||||||
|
|
||||||
|
# Run against the original DreamZero websocket server (default path):
|
||||||
|
python test_client_AR.py --host <server_host> --port 8000
|
||||||
|
|
||||||
|
# Run against the vLLM OpenPI server:
|
||||||
|
python test_client_AR.py --host <server_host> --port 8000 --path /v1/realtime/robot/openpi
|
||||||
|
|
||||||
|
# Use zero images instead of real video (old behavior):
|
||||||
|
python test_client_AR.py --host <server_host> --port 8000 --use-zero-images
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import cv2
|
||||||
|
import numpy as np
|
||||||
|
from openpi_client import msgpack_numpy
|
||||||
|
|
||||||
|
_DREAMZERO_REPO_ENV = os.environ.get("DREAMZERO_REPO")
|
||||||
|
DREAMZERO_REPO = Path(_DREAMZERO_REPO_ENV).expanduser() if _DREAMZERO_REPO_ENV else None
|
||||||
|
|
||||||
|
|
||||||
|
def _import_upstream_policy_modules():
|
||||||
|
if DREAMZERO_REPO is None:
|
||||||
|
raise ImportError("Set DREAMZERO_REPO to an upstream DreamZero checkout before using this helper.")
|
||||||
|
if DREAMZERO_REPO.exists() and str(DREAMZERO_REPO) not in sys.path:
|
||||||
|
sys.path.insert(0, str(DREAMZERO_REPO))
|
||||||
|
|
||||||
|
import eval_utils.policy_server as policy_server
|
||||||
|
from eval_utils.policy_client import WebsocketClientPolicy
|
||||||
|
|
||||||
|
return policy_server, WebsocketClientPolicy
|
||||||
|
|
||||||
|
|
||||||
|
policy_server, WebsocketClientPolicy = _import_upstream_policy_modules()
|
||||||
|
|
||||||
|
VIDEO_DIR = os.environ.get(
|
||||||
|
"DREAMZERO_VIDEO_DIR",
|
||||||
|
str(DREAMZERO_REPO / "debug_image") if DREAMZERO_REPO is not None else "debug_image",
|
||||||
|
)
|
||||||
|
|
||||||
|
# roboarena key -> video filename
|
||||||
|
CAMERA_FILES = {
|
||||||
|
"observation/exterior_image_0_left": "exterior_image_1_left.mp4",
|
||||||
|
"observation/exterior_image_1_left": "exterior_image_2_left.mp4",
|
||||||
|
"observation/wrist_image_left": "wrist_image_left.mp4",
|
||||||
|
}
|
||||||
|
|
||||||
|
# Frame schedule constants (matching debug_inference.py)
|
||||||
|
RELATIVE_OFFSETS = [-23, -16, -8, 0]
|
||||||
|
ACTION_HORIZON = 24
|
||||||
|
DEFAULT_WEBSOCKET_PATH = ""
|
||||||
|
|
||||||
|
|
||||||
|
class OpenPIWebsocketClientPolicy(WebsocketClientPolicy):
|
||||||
|
"""DreamZero websocket client with a configurable path suffix.
|
||||||
|
|
||||||
|
The original DreamZero client connects to ``ws://host:port``.
|
||||||
|
vLLM serves the compatible robot policy endpoint at
|
||||||
|
``/v1/realtime/robot/openpi`` when ``path`` is set accordingly.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
host: str = "0.0.0.0",
|
||||||
|
port: int = 8000,
|
||||||
|
path: str = DEFAULT_WEBSOCKET_PATH,
|
||||||
|
) -> None:
|
||||||
|
self._uri = f"ws://{host}:{port}{path}"
|
||||||
|
self._packer = msgpack_numpy.Packer()
|
||||||
|
self._ws, self._server_metadata = self._wait_for_server()
|
||||||
|
|
||||||
|
|
||||||
|
def load_all_frames(video_path: str) -> np.ndarray:
|
||||||
|
"""Load all frames from a video file. Returns (N, H, W, 3) uint8 array (RGB)."""
|
||||||
|
cap = cv2.VideoCapture(video_path)
|
||||||
|
frames = []
|
||||||
|
while True:
|
||||||
|
ret, frame = cap.read()
|
||||||
|
if not ret:
|
||||||
|
break
|
||||||
|
frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
|
||||||
|
cap.release()
|
||||||
|
if not frames:
|
||||||
|
raise RuntimeError(f"No frames loaded from {video_path}")
|
||||||
|
return np.stack(frames, axis=0)
|
||||||
|
|
||||||
|
|
||||||
|
def load_camera_frames() -> dict[str, np.ndarray]:
|
||||||
|
"""Load all video frames for each camera from the debug_image/ directory.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict mapping roboarena camera keys to (N, H, W, 3) uint8 arrays.
|
||||||
|
"""
|
||||||
|
camera_frames: dict[str, np.ndarray] = {}
|
||||||
|
for cam_key, fname in CAMERA_FILES.items():
|
||||||
|
path = os.path.join(VIDEO_DIR, fname)
|
||||||
|
camera_frames[cam_key] = load_all_frames(path)
|
||||||
|
logging.info(f"Loaded {cam_key}: {camera_frames[cam_key].shape}")
|
||||||
|
return camera_frames
|
||||||
|
|
||||||
|
|
||||||
|
def build_frame_schedule(total_frames: int, num_chunks: int) -> list[list[int]]:
|
||||||
|
"""Build the frame index schedule for multi-frame chunks.
|
||||||
|
|
||||||
|
Returns a list of frame-index lists. Each inner list has 4 indices.
|
||||||
|
"""
|
||||||
|
chunks: list[list[int]] = []
|
||||||
|
current_frame = 23 # first anchor frame
|
||||||
|
for _ in range(num_chunks):
|
||||||
|
indices = [max(current_frame + off, 0) for off in RELATIVE_OFFSETS]
|
||||||
|
if indices[-1] >= total_frames:
|
||||||
|
logging.info(f"Frame {indices[-1]} >= {total_frames}, stopping at {len(chunks)} chunks")
|
||||||
|
break
|
||||||
|
chunks.append(indices)
|
||||||
|
current_frame += ACTION_HORIZON
|
||||||
|
return chunks
|
||||||
|
|
||||||
|
|
||||||
|
def _make_obs_from_video(
|
||||||
|
camera_frames: dict[str, np.ndarray],
|
||||||
|
frame_indices: list[int],
|
||||||
|
prompt: str,
|
||||||
|
session_id: str,
|
||||||
|
) -> dict:
|
||||||
|
"""Build an observation dict from real video frames.
|
||||||
|
|
||||||
|
For 1 frame: each image key is (H, W, 3).
|
||||||
|
For 4 frames: each image key is (4, H, W, 3).
|
||||||
|
"""
|
||||||
|
obs: dict = {}
|
||||||
|
for cam_key, all_frames in camera_frames.items():
|
||||||
|
selected = all_frames[frame_indices] # (T, H, W, 3)
|
||||||
|
if len(frame_indices) == 1:
|
||||||
|
selected = selected[0] # (H, W, 3)
|
||||||
|
obs[cam_key] = selected
|
||||||
|
|
||||||
|
obs["observation/joint_position"] = np.zeros(7, dtype=np.float32)
|
||||||
|
obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32)
|
||||||
|
obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32)
|
||||||
|
obs["prompt"] = prompt
|
||||||
|
obs["session_id"] = session_id
|
||||||
|
return obs
|
||||||
|
|
||||||
|
|
||||||
|
def _make_zero_observation(
|
||||||
|
server_config: policy_server.PolicyServerConfig,
|
||||||
|
prompt: str = "pick up the object",
|
||||||
|
session_id: str | None = None,
|
||||||
|
) -> dict:
|
||||||
|
"""Create a dummy observation matching AR_droid expectations.
|
||||||
|
|
||||||
|
AR_droid expects:
|
||||||
|
- 2 external cameras (exterior_image_0_left, exterior_image_1_left)
|
||||||
|
- 1 wrist camera (wrist_image_left)
|
||||||
|
- Image resolution: 180x320 (H x W)
|
||||||
|
- joint_position: 7 DoF
|
||||||
|
- gripper_position: 1 DoF
|
||||||
|
"""
|
||||||
|
obs = {}
|
||||||
|
|
||||||
|
# Determine image resolution
|
||||||
|
if server_config.image_resolution is not None:
|
||||||
|
h, w = server_config.image_resolution
|
||||||
|
else:
|
||||||
|
# Default for AR_droid
|
||||||
|
h, w = 180, 320
|
||||||
|
|
||||||
|
# External cameras (0-indexed in roboarena)
|
||||||
|
for i in range(server_config.n_external_cameras):
|
||||||
|
obs[f"observation/exterior_image_{i}_left"] = np.zeros((h, w, 3), dtype=np.uint8)
|
||||||
|
if server_config.needs_stereo_camera:
|
||||||
|
obs[f"observation/exterior_image_{i}_right"] = np.zeros((h, w, 3), dtype=np.uint8)
|
||||||
|
|
||||||
|
# Wrist camera
|
||||||
|
if server_config.needs_wrist_camera:
|
||||||
|
obs["observation/wrist_image_left"] = np.zeros((h, w, 3), dtype=np.uint8)
|
||||||
|
if server_config.needs_stereo_camera:
|
||||||
|
obs["observation/wrist_image_right"] = np.zeros((h, w, 3), dtype=np.uint8)
|
||||||
|
|
||||||
|
# Session ID - should be passed in to ensure consistency within a session
|
||||||
|
if server_config.needs_session_id:
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
# Generate unique session ID if not provided
|
||||||
|
obs["session_id"] = session_id if session_id else str(uuid.uuid4())
|
||||||
|
|
||||||
|
# State observations (AR_droid: 7 DoF arm + 1 gripper)
|
||||||
|
obs["observation/joint_position"] = np.zeros(7, dtype=np.float32)
|
||||||
|
obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32)
|
||||||
|
obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32)
|
||||||
|
|
||||||
|
# Language prompt
|
||||||
|
obs["prompt"] = prompt
|
||||||
|
|
||||||
|
return obs
|
||||||
|
|
||||||
|
|
||||||
|
def test_ar_droid_policy_server(
|
||||||
|
host: str = "localhost",
|
||||||
|
port: int = 8000,
|
||||||
|
path: str = DEFAULT_WEBSOCKET_PATH,
|
||||||
|
num_chunks: int = 15,
|
||||||
|
prompt: str = "Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan",
|
||||||
|
use_zero_images: bool = False,
|
||||||
|
):
|
||||||
|
"""Test the AR_droid policy server with roboarena interface.
|
||||||
|
|
||||||
|
When use_zero_images is False (default), loads real video frames from
|
||||||
|
debug_image/ and follows the frame schedule from debug_inference.py.
|
||||||
|
"""
|
||||||
|
logging.info(f"Connecting to AR_droid server at ws://{host}:{port}{path} ...")
|
||||||
|
|
||||||
|
client = OpenPIWebsocketClientPolicy(host=host, port=port, path=path)
|
||||||
|
|
||||||
|
# Validate server metadata
|
||||||
|
metadata = client.get_server_metadata()
|
||||||
|
logging.info(f"Server metadata: {metadata}")
|
||||||
|
assert isinstance(metadata, dict), "Metadata should be a dict"
|
||||||
|
|
||||||
|
try:
|
||||||
|
server_config = policy_server.PolicyServerConfig(**metadata)
|
||||||
|
except Exception as e:
|
||||||
|
logging.error(f"Error parsing metadata: {e}")
|
||||||
|
raise e
|
||||||
|
|
||||||
|
# Validate expected AR_droid configuration
|
||||||
|
logging.info(f"Server config: {server_config}")
|
||||||
|
assert server_config.n_external_cameras == 2, f"Expected 2 external cameras, got {server_config.n_external_cameras}"
|
||||||
|
assert server_config.needs_wrist_camera, "Expected wrist camera to be enabled"
|
||||||
|
assert server_config.action_space == "joint_position", (
|
||||||
|
f"Expected joint_position action space, got {server_config.action_space}"
|
||||||
|
)
|
||||||
|
|
||||||
|
logging.info("Server configuration validated for AR_droid")
|
||||||
|
|
||||||
|
# Generate unique session ID for this test run
|
||||||
|
session_id = str(uuid.uuid4())
|
||||||
|
logging.info(f"Session ID: {session_id}")
|
||||||
|
|
||||||
|
# ── Zero-image fallback mode ──────────────────────────────────────
|
||||||
|
if use_zero_images:
|
||||||
|
logging.info("Using ZERO dummy images (legacy mode)")
|
||||||
|
for i in range(num_chunks):
|
||||||
|
obs = _make_zero_observation(server_config, prompt=prompt, session_id=session_id)
|
||||||
|
logging.info(f"Inference {i + 1}/{num_chunks}: prompt='{prompt}'")
|
||||||
|
t0 = time.time()
|
||||||
|
actions = client.infer(obs)
|
||||||
|
dt = time.time() - t0
|
||||||
|
_log_action(actions, dt)
|
||||||
|
|
||||||
|
logging.info("Sending reset...")
|
||||||
|
client.reset({})
|
||||||
|
logging.info("Done (zero-image mode).")
|
||||||
|
return
|
||||||
|
|
||||||
|
# ── Real video frame mode ─────────────────────────────────────────
|
||||||
|
logging.info("Loading real video frames from debug_image/ directory")
|
||||||
|
camera_frames = load_camera_frames()
|
||||||
|
|
||||||
|
total_frames = min(v.shape[0] for v in camera_frames.values())
|
||||||
|
logging.info(f"Total frames available: {total_frames}")
|
||||||
|
|
||||||
|
# Build frame schedule
|
||||||
|
chunks = build_frame_schedule(total_frames, num_chunks)
|
||||||
|
|
||||||
|
logging.info("Frame schedule:")
|
||||||
|
logging.info(" Initial: [0]")
|
||||||
|
for i, indices in enumerate(chunks):
|
||||||
|
logging.info(f" Chunk {i}: {indices}")
|
||||||
|
|
||||||
|
# Step 0: initial single frame
|
||||||
|
logging.info("=== Initial: frame [0] ===")
|
||||||
|
obs = _make_obs_from_video(camera_frames, [0], prompt, session_id)
|
||||||
|
t0 = time.time()
|
||||||
|
actions = client.infer(obs)
|
||||||
|
dt = time.time() - t0
|
||||||
|
_log_action(actions, dt)
|
||||||
|
|
||||||
|
# Subsequent chunks: send 4 frames at a time
|
||||||
|
for chunk_idx, frame_indices in enumerate(chunks):
|
||||||
|
logging.info(f"=== Chunk {chunk_idx}: frames {frame_indices} ===")
|
||||||
|
obs = _make_obs_from_video(camera_frames, frame_indices, prompt, session_id)
|
||||||
|
t0 = time.time()
|
||||||
|
actions = client.infer(obs)
|
||||||
|
dt = time.time() - t0
|
||||||
|
_log_action(actions, dt)
|
||||||
|
|
||||||
|
# Reset triggers video save on the server
|
||||||
|
logging.info("Sending reset to save video...")
|
||||||
|
client.reset({})
|
||||||
|
|
||||||
|
logging.info("Done.")
|
||||||
|
|
||||||
|
|
||||||
|
def _log_action(actions: np.ndarray, dt: float) -> None:
|
||||||
|
"""Pretty-print action shape, range, and timing."""
|
||||||
|
assert isinstance(actions, np.ndarray), f"Expected numpy array, got {type(actions)}"
|
||||||
|
assert actions.ndim == 2, f"Expected 2D array, got shape {actions.shape}"
|
||||||
|
assert actions.shape[-1] == 8, f"Expected 8 action dims (7 joints + 1 gripper), got {actions.shape[-1]}"
|
||||||
|
logging.info(f" Action shape: {actions.shape}, range: [{actions.min():.4f}, {actions.max():.4f}], time: {dt:.2f}s")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description="Test AR_droid policy server with real video frames from debug_image/")
|
||||||
|
parser.add_argument("--host", default="localhost", help="Server hostname")
|
||||||
|
parser.add_argument("--port", type=int, default=8000, help="Server port")
|
||||||
|
parser.add_argument(
|
||||||
|
"--path",
|
||||||
|
default=DEFAULT_WEBSOCKET_PATH,
|
||||||
|
help="WebSocket path suffix (default: empty string for the original DreamZero server)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--num-chunks",
|
||||||
|
type=int,
|
||||||
|
default=15,
|
||||||
|
help="Number of 4-frame chunks to send after the initial frame (default: 15)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--prompt",
|
||||||
|
default="Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan",
|
||||||
|
help="Language prompt for the policy",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--use-zero-images",
|
||||||
|
action="store_true",
|
||||||
|
help="Use zero dummy images instead of real video frames (legacy mode)",
|
||||||
|
)
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
logging.basicConfig(
|
||||||
|
level=logging.INFO,
|
||||||
|
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||||
|
)
|
||||||
|
|
||||||
|
test_ar_droid_policy_server(
|
||||||
|
host=args.host,
|
||||||
|
port=args.port,
|
||||||
|
path=args.path,
|
||||||
|
num_chunks=args.num_chunks,
|
||||||
|
prompt=args.prompt,
|
||||||
|
use_zero_images=args.use_zero_images,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,358 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
"""Formal OpenPI end-to-end parity: upstream DreamZero server vs `vllm serve`.
|
||||||
|
|
||||||
|
This test uses DreamZero's own client-side observation builders from
|
||||||
|
`${DREAMZERO_REPO}/test_client_AR.py`, and client-side websocket protocol from
|
||||||
|
`${DREAMZERO_REPO}/eval_utils/policy_client.py`.
|
||||||
|
|
||||||
|
The only client-side adaptation for vLLM is the websocket path:
|
||||||
|
DreamZero's upstream server serves at `/`, while vLLM serves OpenPI at
|
||||||
|
`/v1/realtime/robot/openpi`.
|
||||||
|
|
||||||
|
Current scope for this test:
|
||||||
|
- default two-GPU run (`nproc_per_node=2` on upstream, `--cfg-parallel-size 2` on `vllm serve`)
|
||||||
|
- non-`torch.compile` (upstream launched through
|
||||||
|
`upstream_socket_server_no_compile.py`, vLLM with `--enforce-eager`)
|
||||||
|
- non-DiT-cache / non-skip-schedule (`NUM_DIT_STEPS=16`)
|
||||||
|
|
||||||
|
Serving contract locked by this test:
|
||||||
|
- upstream DreamZero still boots from the local checkpoint directory
|
||||||
|
- vLLM boots from the official DreamZero HF repo name (`GEAR-Dreams/DreamZero-DROID`)
|
||||||
|
rather than a prepared local bundle path
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import pytest
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from tests.helpers.runtime import get_open_port
|
||||||
|
|
||||||
|
msgpack_numpy = pytest.importorskip("openpi_client.msgpack_numpy")
|
||||||
|
|
||||||
|
_DREAMZERO_REPO_ENV = os.environ.get("DREAMZERO_REPO")
|
||||||
|
DREAMZERO_REPO = Path(_DREAMZERO_REPO_ENV).expanduser() if _DREAMZERO_REPO_ENV else None
|
||||||
|
if DREAMZERO_REPO is not None and str(DREAMZERO_REPO) not in sys.path:
|
||||||
|
sys.path.insert(0, str(DREAMZERO_REPO))
|
||||||
|
|
||||||
|
try:
|
||||||
|
import test_client_AR as dreamzero_client
|
||||||
|
from eval_utils.policy_client import WebsocketClientPolicy
|
||||||
|
except Exception: # pragma: no cover - guarded by pytest skip below
|
||||||
|
dreamzero_client = None
|
||||||
|
WebsocketClientPolicy = None
|
||||||
|
_BaseWebsocketClientPolicy = WebsocketClientPolicy if WebsocketClientPolicy is not None else object
|
||||||
|
|
||||||
|
CHECKPOINT_DIR = DREAMZERO_REPO / "checkpoints" / "dreamzero" if DREAMZERO_REPO is not None else None
|
||||||
|
VLLM_MODEL = os.environ.get("VLLM_DREAMZERO_MODEL", "GEAR-Dreams/DreamZero-DROID")
|
||||||
|
SERVICE_READY_TIMEOUT_S = int(os.environ.get("OPENPI_SERVICE_READY_TIMEOUT_S", "900"))
|
||||||
|
PROMPT = "Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan"
|
||||||
|
SESSION_ID = "openpi-e2e-parity-session"
|
||||||
|
|
||||||
|
pytestmark = [
|
||||||
|
pytest.mark.skipif(not torch.cuda.is_available(), reason="GPU required"),
|
||||||
|
pytest.mark.skipif(
|
||||||
|
dreamzero_client is None or WebsocketClientPolicy is None,
|
||||||
|
reason="DreamZero client modules are required on PYTHONPATH",
|
||||||
|
),
|
||||||
|
pytest.mark.skipif(
|
||||||
|
DREAMZERO_REPO is None or not DREAMZERO_REPO.exists(),
|
||||||
|
reason="DreamZero source repo is required at DREAMZERO_REPO",
|
||||||
|
),
|
||||||
|
pytest.mark.skipif(
|
||||||
|
CHECKPOINT_DIR is None or not CHECKPOINT_DIR.exists(), reason="DreamZero local checkpoint is required"
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class OpenPIWebsocketClientPolicy(_BaseWebsocketClientPolicy):
|
||||||
|
"""DreamZero client protocol with an OpenPI websocket path suffix."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
host: str = "127.0.0.1",
|
||||||
|
port: int = 8000,
|
||||||
|
path: str = "/v1/realtime/robot/openpi",
|
||||||
|
) -> None:
|
||||||
|
self._uri = f"ws://{host}:{port}{path}"
|
||||||
|
self._packer = msgpack_numpy.Packer()
|
||||||
|
self._ws, self._server_metadata = self._wait_for_server()
|
||||||
|
|
||||||
|
|
||||||
|
def _vllm_executable() -> str:
|
||||||
|
fallback = Path(sys.executable).with_name("vllm")
|
||||||
|
if fallback.exists():
|
||||||
|
return str(fallback)
|
||||||
|
exe = shutil.which("vllm")
|
||||||
|
if exe:
|
||||||
|
return exe
|
||||||
|
raise FileNotFoundError("Unable to locate `vllm` executable in current environment.")
|
||||||
|
|
||||||
|
|
||||||
|
def _cfg_parallel_size() -> int:
|
||||||
|
return int(os.environ.get("OPENPI_E2E_CFG_PARALLEL_SIZE", "2"))
|
||||||
|
|
||||||
|
|
||||||
|
def _pick_test_gpus() -> list[str]:
|
||||||
|
cfg_parallel_size = _cfg_parallel_size()
|
||||||
|
override = os.environ.get("OPENPI_E2E_GPUS") or os.environ.get("OPENPI_E2E_GPU")
|
||||||
|
if override is not None:
|
||||||
|
gpus = [part.strip() for part in override.split(",") if part.strip()]
|
||||||
|
if not gpus:
|
||||||
|
raise ValueError("OPENPI_E2E_GPUS is set but empty.")
|
||||||
|
if len(gpus) < cfg_parallel_size:
|
||||||
|
raise RuntimeError(f"Need {cfg_parallel_size} GPUs, but OPENPI_E2E_GPUS only provided {gpus}.")
|
||||||
|
return gpus
|
||||||
|
|
||||||
|
query = subprocess.check_output(
|
||||||
|
[
|
||||||
|
"nvidia-smi",
|
||||||
|
"--query-gpu=index,memory.used",
|
||||||
|
"--format=csv,noheader,nounits",
|
||||||
|
],
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
gpu_rows = []
|
||||||
|
for line in query.strip().splitlines():
|
||||||
|
gpu_index, used_mb = [part.strip() for part in line.split(",", maxsplit=1)]
|
||||||
|
gpu_rows.append((int(used_mb), gpu_index))
|
||||||
|
gpu_rows.sort()
|
||||||
|
gpus = [gpu_index for _, gpu_index in gpu_rows[: max(cfg_parallel_size, 1)]]
|
||||||
|
if len(gpus) < cfg_parallel_size:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Need {cfg_parallel_size} GPUs for cfg_parallel_size={cfg_parallel_size}, "
|
||||||
|
f"but found only {len(gpus)} candidates."
|
||||||
|
)
|
||||||
|
return gpus
|
||||||
|
|
||||||
|
|
||||||
|
def _torchrun_argv(script: str, port: int) -> list[str]:
|
||||||
|
return [
|
||||||
|
sys.executable,
|
||||||
|
"-m",
|
||||||
|
"torch.distributed.run",
|
||||||
|
"--standalone",
|
||||||
|
"--nproc_per_node",
|
||||||
|
str(_cfg_parallel_size()),
|
||||||
|
script,
|
||||||
|
"--port",
|
||||||
|
str(port),
|
||||||
|
"--model_path",
|
||||||
|
str(CHECKPOINT_DIR),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _run_upstream_service(port: int, log_path: Path) -> subprocess.Popen[str]:
|
||||||
|
env = os.environ.copy()
|
||||||
|
env.setdefault("PYTHONPATH", "")
|
||||||
|
env["PYTHONPATH"] = f"{Path.cwd()}:{DREAMZERO_REPO}:{env['PYTHONPATH']}".rstrip(":")
|
||||||
|
env["CUDA_VISIBLE_DEVICES"] = ",".join(_pick_test_gpus())
|
||||||
|
env.setdefault("NO_ALBUMENTATIONS_UPDATE", "1")
|
||||||
|
env["ATTENTION_BACKEND"] = "torch"
|
||||||
|
env.setdefault("ENABLE_TENSORRT", "false")
|
||||||
|
env["ENABLE_DIT_CACHE"] = "false"
|
||||||
|
env["NUM_DIT_STEPS"] = "16"
|
||||||
|
env["DYNAMIC_CACHE_SCHEDULE"] = "false"
|
||||||
|
argv = _torchrun_argv(
|
||||||
|
str(Path("tests/dreamzero/upstream/upstream_socket_server_no_compile.py")),
|
||||||
|
port,
|
||||||
|
)
|
||||||
|
log_file = log_path.open("w")
|
||||||
|
proc = subprocess.Popen(
|
||||||
|
argv,
|
||||||
|
stdout=log_file,
|
||||||
|
stderr=subprocess.STDOUT,
|
||||||
|
text=True,
|
||||||
|
cwd=str(Path.cwd()),
|
||||||
|
env=env,
|
||||||
|
)
|
||||||
|
proc._codex_log_file = log_file # type: ignore[attr-defined]
|
||||||
|
return proc
|
||||||
|
|
||||||
|
|
||||||
|
def _run_vllm_service(port: int, log_path: Path) -> subprocess.Popen[str]:
|
||||||
|
env = os.environ.copy()
|
||||||
|
gpus = _pick_test_gpus()
|
||||||
|
cfg_parallel_size = _cfg_parallel_size()
|
||||||
|
if cfg_parallel_size > len(gpus):
|
||||||
|
raise RuntimeError(
|
||||||
|
f"cfg_parallel_size={cfg_parallel_size} requires at least {cfg_parallel_size} GPUs, but only got {gpus}."
|
||||||
|
)
|
||||||
|
env["CUDA_VISIBLE_DEVICES"] = ",".join(gpus[:cfg_parallel_size])
|
||||||
|
env.setdefault("ATTENTION_BACKEND", "torch")
|
||||||
|
env.setdefault("DIFFUSION_ATTENTION_BACKEND", "TORCH_SDPA")
|
||||||
|
env.setdefault("MASTER_PORT", str(get_open_port()))
|
||||||
|
argv = [
|
||||||
|
_vllm_executable(),
|
||||||
|
"serve",
|
||||||
|
VLLM_MODEL,
|
||||||
|
"--omni",
|
||||||
|
"--host",
|
||||||
|
"127.0.0.1",
|
||||||
|
"--port",
|
||||||
|
str(port),
|
||||||
|
"--served-model-name",
|
||||||
|
"dreamzero-droid",
|
||||||
|
"--enforce-eager",
|
||||||
|
]
|
||||||
|
if cfg_parallel_size > 1:
|
||||||
|
argv.extend(["--cfg-parallel-size", str(cfg_parallel_size)])
|
||||||
|
log_file = log_path.open("w")
|
||||||
|
proc = subprocess.Popen(
|
||||||
|
argv,
|
||||||
|
stdout=log_file,
|
||||||
|
stderr=subprocess.STDOUT,
|
||||||
|
text=True,
|
||||||
|
env=env,
|
||||||
|
cwd=str(Path.cwd()),
|
||||||
|
)
|
||||||
|
proc._codex_log_file = log_file # type: ignore[attr-defined]
|
||||||
|
return proc
|
||||||
|
|
||||||
|
|
||||||
|
def _stop_process(proc: subprocess.Popen[str]) -> None:
|
||||||
|
log_file = getattr(proc, "_codex_log_file", None)
|
||||||
|
if proc.poll() is None:
|
||||||
|
proc.terminate()
|
||||||
|
try:
|
||||||
|
proc.wait(timeout=30)
|
||||||
|
except subprocess.TimeoutExpired: # pragma: no cover - cleanup path
|
||||||
|
proc.kill()
|
||||||
|
proc.wait(timeout=10)
|
||||||
|
if log_file is not None:
|
||||||
|
log_file.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _build_obs_sequence() -> tuple[dict, dict]:
|
||||||
|
camera_frames = dreamzero_client.load_camera_frames()
|
||||||
|
chunks = dreamzero_client.build_frame_schedule(
|
||||||
|
min(v.shape[0] for v in camera_frames.values()),
|
||||||
|
1,
|
||||||
|
)
|
||||||
|
obs0 = dreamzero_client._make_obs_from_video(camera_frames, [0], PROMPT, SESSION_ID)
|
||||||
|
obs1 = dreamzero_client._make_obs_from_video(camera_frames, chunks[0], PROMPT, SESSION_ID)
|
||||||
|
return obs0, obs1
|
||||||
|
|
||||||
|
|
||||||
|
def _wait_for_client_ready(client_factory, timeout_s: float, proc=None, log_path: Path | None = None):
|
||||||
|
deadline = time.time() + timeout_s
|
||||||
|
last_err: Exception | None = None
|
||||||
|
while time.time() < deadline:
|
||||||
|
if proc is not None and proc.poll() is not None:
|
||||||
|
details = ""
|
||||||
|
if log_path is not None and log_path.exists():
|
||||||
|
details = log_path.read_text(errors="replace")[-8000:]
|
||||||
|
raise RuntimeError(f"Service exited before becoming ready with code {proc.returncode}.\n{details}")
|
||||||
|
try:
|
||||||
|
return client_factory()
|
||||||
|
except Exception as exc: # pragma: no cover - retry path
|
||||||
|
last_err = exc
|
||||||
|
time.sleep(1)
|
||||||
|
raise TimeoutError(f"Timed out waiting for websocket service: {last_err}")
|
||||||
|
|
||||||
|
|
||||||
|
def _collect_outputs_with_client(client) -> tuple[dict, list[np.ndarray]]:
|
||||||
|
metadata = client.get_server_metadata()
|
||||||
|
obs0, obs1 = _build_obs_sequence()
|
||||||
|
outputs = [
|
||||||
|
client.infer(dict(obs0)),
|
||||||
|
client.infer(dict(obs1)),
|
||||||
|
]
|
||||||
|
assert _normalize_reset_response(client.reset({})) == "reset successful"
|
||||||
|
outputs.append(client.infer(dict(obs0)))
|
||||||
|
client._ws.close()
|
||||||
|
return metadata, outputs
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_reset_response(response) -> str:
|
||||||
|
if isinstance(response, str):
|
||||||
|
return response
|
||||||
|
decoded = msgpack_numpy.unpackb(response)
|
||||||
|
if isinstance(decoded, dict):
|
||||||
|
return str(decoded.get("status"))
|
||||||
|
return str(decoded)
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_metadata(metadata: dict) -> dict:
|
||||||
|
normalized = dict(metadata)
|
||||||
|
if isinstance(normalized.get("image_resolution"), tuple):
|
||||||
|
normalized["image_resolution"] = list(normalized["image_resolution"])
|
||||||
|
return normalized
|
||||||
|
|
||||||
|
|
||||||
|
def _assert_logs_clean(log_path: Path) -> None:
|
||||||
|
text = log_path.read_text(errors="replace")
|
||||||
|
if "SignalException: Process" in text and "got signal: 15" in text:
|
||||||
|
text = text.split("Traceback (most recent call last):", 1)[0]
|
||||||
|
assert "Traceback" not in text, text
|
||||||
|
assert "RuntimeError:" not in text, text
|
||||||
|
|
||||||
|
|
||||||
|
def _assert_upstream_log_matches_vllm_baseline(log_path: Path) -> None:
|
||||||
|
text = log_path.read_text(errors="replace")
|
||||||
|
assert "DIT Compute Steps 8 steps" not in text, text
|
||||||
|
assert "DIT Compute Steps 16 steps" in text, text
|
||||||
|
|
||||||
|
|
||||||
|
def test_openpi_service_matches_upstream_server_noncompile(tmp_path: Path) -> None:
|
||||||
|
expected_metadata = {
|
||||||
|
"image_resolution": [180, 320],
|
||||||
|
"n_external_cameras": 2,
|
||||||
|
"needs_wrist_camera": True,
|
||||||
|
"needs_stereo_camera": False,
|
||||||
|
"needs_session_id": True,
|
||||||
|
"action_space": "joint_position",
|
||||||
|
}
|
||||||
|
|
||||||
|
upstream_port = get_open_port()
|
||||||
|
upstream_log = tmp_path / "dreamzero_upstream.log"
|
||||||
|
upstream_proc = _run_upstream_service(upstream_port, upstream_log)
|
||||||
|
try:
|
||||||
|
upstream_client = _wait_for_client_ready(
|
||||||
|
lambda: WebsocketClientPolicy(host="127.0.0.1", port=upstream_port),
|
||||||
|
timeout_s=SERVICE_READY_TIMEOUT_S,
|
||||||
|
proc=upstream_proc,
|
||||||
|
log_path=upstream_log,
|
||||||
|
)
|
||||||
|
upstream_metadata, upstream_outputs = _collect_outputs_with_client(upstream_client)
|
||||||
|
finally:
|
||||||
|
_stop_process(upstream_proc)
|
||||||
|
_assert_logs_clean(upstream_log)
|
||||||
|
_assert_upstream_log_matches_vllm_baseline(upstream_log)
|
||||||
|
|
||||||
|
vllm_port = get_open_port()
|
||||||
|
vllm_log = tmp_path / "vllm_openpi.log"
|
||||||
|
vllm_proc = _run_vllm_service(vllm_port, vllm_log)
|
||||||
|
try:
|
||||||
|
vllm_client = _wait_for_client_ready(
|
||||||
|
lambda: OpenPIWebsocketClientPolicy(host="127.0.0.1", port=vllm_port),
|
||||||
|
timeout_s=SERVICE_READY_TIMEOUT_S,
|
||||||
|
proc=vllm_proc,
|
||||||
|
log_path=vllm_log,
|
||||||
|
)
|
||||||
|
vllm_metadata, vllm_outputs = _collect_outputs_with_client(vllm_client)
|
||||||
|
finally:
|
||||||
|
_stop_process(vllm_proc)
|
||||||
|
_assert_logs_clean(vllm_log)
|
||||||
|
|
||||||
|
assert _normalize_metadata(upstream_metadata) == expected_metadata
|
||||||
|
assert _normalize_metadata(vllm_metadata) == expected_metadata
|
||||||
|
|
||||||
|
for idx, (actual, expected) in enumerate(zip(vllm_outputs, upstream_outputs, strict=True)):
|
||||||
|
np.testing.assert_allclose(
|
||||||
|
actual,
|
||||||
|
expected,
|
||||||
|
rtol=1e-2,
|
||||||
|
atol=1e-3,
|
||||||
|
err_msg=f"OpenPI step {idx} output mismatch",
|
||||||
|
)
|
||||||
@@ -0,0 +1,184 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
"""Launch the upstream DreamZero websocket server with `torch.compile` disabled.
|
||||||
|
|
||||||
|
This wrapper is meant for formal parity tests against `vllm serve --omni`.
|
||||||
|
It monkeypatches `torch.compile` before importing DreamZero modules so that
|
||||||
|
all import-time decorators and `post_initialize()` compile calls become eager.
|
||||||
|
|
||||||
|
For the current DreamZero port baseline we also disable all upstream DiT cache
|
||||||
|
and step-skipping behavior so the reference server matches vLLM's current
|
||||||
|
eager/no-skip implementation:
|
||||||
|
- `ENABLE_DIT_CACHE=false`
|
||||||
|
- `NUM_DIT_STEPS=16`
|
||||||
|
- `DYNAMIC_CACHE_SCHEDULE=false`
|
||||||
|
|
||||||
|
In our CI/dev environment we also do not install Transformer Engine or
|
||||||
|
FlashAttention, so upstream's default `socket_test_optimized_AR.main()` path
|
||||||
|
(`ATTENTION_BACKEND=TE` and then fallback to `FA2`) cannot execute. To keep the
|
||||||
|
formal server-vs-server test runnable without pulling in those heavyweight
|
||||||
|
optional deps, this wrapper reproduces upstream `main()` but pins attention to
|
||||||
|
PyTorch SDPA (`ATTENTION_BACKEND=torch`) for this subprocess only.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
PYTHONPATH="${DREAMZERO_REPO}" \\
|
||||||
|
.venv/bin/python -m torch.distributed.run --standalone --nproc_per_node=2 \\
|
||||||
|
tests/dreamzero/upstream/upstream_socket_server_no_compile.py --port 18081 \\
|
||||||
|
--model_path "${DREAMZERO_REPO}/checkpoints/dreamzero"
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import torch
|
||||||
|
import torch.nn.functional as F
|
||||||
|
|
||||||
|
|
||||||
|
def _identity_compile(*args, **kwargs):
|
||||||
|
if args and callable(args[0]) and len(args) == 1 and not kwargs:
|
||||||
|
return args[0]
|
||||||
|
|
||||||
|
def deco(fn):
|
||||||
|
return fn
|
||||||
|
|
||||||
|
return deco
|
||||||
|
|
||||||
|
|
||||||
|
torch.compile = _identity_compile
|
||||||
|
|
||||||
|
os.environ.setdefault("NO_ALBUMENTATIONS_UPDATE", "1")
|
||||||
|
|
||||||
|
DREAMZERO_REPO_ENV = os.environ.get("DREAMZERO_REPO")
|
||||||
|
if not DREAMZERO_REPO_ENV:
|
||||||
|
raise RuntimeError("Set DREAMZERO_REPO to an upstream DreamZero checkout before launching this helper.")
|
||||||
|
DREAMZERO_REPO = Path(DREAMZERO_REPO_ENV).expanduser()
|
||||||
|
if str(DREAMZERO_REPO) not in sys.path:
|
||||||
|
sys.path.insert(0, str(DREAMZERO_REPO))
|
||||||
|
|
||||||
|
import socket_test_optimized_AR as upstream # noqa: E402
|
||||||
|
import tyro # noqa: E402
|
||||||
|
from groot.vla.model.dreamzero.modules import attention as upstream_attention # noqa: E402
|
||||||
|
from groot.vla.model.dreamzero.modules import wan2_1_submodule as upstream_submodule # noqa: E402
|
||||||
|
from groot.vla.model.dreamzero.modules import wan_video_dit as upstream_wan_video_dit # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
def _torch_varlen_flash_attention(
|
||||||
|
q: torch.Tensor,
|
||||||
|
k: torch.Tensor,
|
||||||
|
v: torch.Tensor,
|
||||||
|
q_lens: torch.Tensor | None = None,
|
||||||
|
k_lens: torch.Tensor | None = None,
|
||||||
|
dropout_p: float = 0.0,
|
||||||
|
causal: bool = False,
|
||||||
|
dtype: torch.dtype = torch.bfloat16,
|
||||||
|
**_: object,
|
||||||
|
) -> torch.Tensor:
|
||||||
|
if q_lens is not None or k_lens is not None:
|
||||||
|
upstream_attention.warnings.warn(
|
||||||
|
"Padding mask is disabled in the test-only SDPA fallback.",
|
||||||
|
)
|
||||||
|
out_dtype = q.dtype
|
||||||
|
q = q.transpose(1, 2).to(dtype)
|
||||||
|
k = k.transpose(1, 2).to(dtype)
|
||||||
|
v = v.transpose(1, 2).to(dtype)
|
||||||
|
out = F.scaled_dot_product_attention(
|
||||||
|
q,
|
||||||
|
k,
|
||||||
|
v,
|
||||||
|
attn_mask=None,
|
||||||
|
is_causal=causal,
|
||||||
|
dropout_p=dropout_p,
|
||||||
|
)
|
||||||
|
return out.transpose(1, 2).contiguous().to(out_dtype)
|
||||||
|
|
||||||
|
|
||||||
|
upstream_attention.flash_attention = _torch_varlen_flash_attention
|
||||||
|
upstream_submodule.flash_attention = _torch_varlen_flash_attention
|
||||||
|
upstream_wan_video_dit.FLASH_ATTN_3_AVAILABLE = False
|
||||||
|
upstream_wan_video_dit.FLASH_ATTN_2_AVAILABLE = False
|
||||||
|
upstream_wan_video_dit.SAGE_ATTN_AVAILABLE = False
|
||||||
|
|
||||||
|
|
||||||
|
def main(args: upstream.Args) -> None:
|
||||||
|
os.environ["ENABLE_DIT_CACHE"] = "false"
|
||||||
|
os.environ["ATTENTION_BACKEND"] = "torch"
|
||||||
|
os.environ["NUM_DIT_STEPS"] = "16"
|
||||||
|
os.environ["DYNAMIC_CACHE_SCHEDULE"] = "false"
|
||||||
|
torch._dynamo.config.recompile_limit = 800
|
||||||
|
|
||||||
|
embodiment_tag = "oxe_droid"
|
||||||
|
model_path = args.model_path
|
||||||
|
|
||||||
|
device_mesh = upstream.init_mesh()
|
||||||
|
rank = upstream.dist.get_rank()
|
||||||
|
|
||||||
|
timeout_delta = upstream.datetime.timedelta(seconds=args.timeout_seconds)
|
||||||
|
signal_group = upstream.dist.new_group(backend="gloo", timeout=timeout_delta)
|
||||||
|
upstream.logger.info("Rank %s initialized signal_group (gloo)", rank)
|
||||||
|
|
||||||
|
policy = upstream.GrootSimPolicy(
|
||||||
|
embodiment_tag=upstream.EmbodimentTag(embodiment_tag),
|
||||||
|
model_path=model_path,
|
||||||
|
device="cuda" if torch.cuda.is_available() else "cpu",
|
||||||
|
device_mesh=device_mesh,
|
||||||
|
)
|
||||||
|
|
||||||
|
hostname = upstream.socket.gethostname()
|
||||||
|
local_ip = upstream.socket.gethostbyname(hostname)
|
||||||
|
|
||||||
|
if rank == 0:
|
||||||
|
upstream.logging.info("Creating server (host: %s, ip: %s)", hostname, local_ip)
|
||||||
|
parent_dir = os.path.dirname(model_path)
|
||||||
|
date_suffix = upstream.datetime.datetime.now().strftime("%Y%m%d")
|
||||||
|
checkpoint_name = os.path.basename(model_path)
|
||||||
|
output_dir = os.path.join(
|
||||||
|
parent_dir,
|
||||||
|
f"real_world_eval_gen_{date_suffix}_{args.index}",
|
||||||
|
checkpoint_name,
|
||||||
|
)
|
||||||
|
os.makedirs(output_dir, exist_ok=True)
|
||||||
|
upstream.logging.info("Videos will be saved to: %s", output_dir)
|
||||||
|
else:
|
||||||
|
output_dir = None
|
||||||
|
upstream.logging.info("Rank %s starting as worker for distributed inference...", rank)
|
||||||
|
|
||||||
|
wrapper_policy = upstream.ARDroidRoboarenaPolicy(
|
||||||
|
groot_policy=policy,
|
||||||
|
signal_group=signal_group,
|
||||||
|
output_dir=output_dir,
|
||||||
|
)
|
||||||
|
|
||||||
|
server_config = upstream.PolicyServerConfig(
|
||||||
|
image_resolution=(180, 320),
|
||||||
|
needs_wrist_camera=True,
|
||||||
|
n_external_cameras=2,
|
||||||
|
needs_stereo_camera=False,
|
||||||
|
needs_session_id=True,
|
||||||
|
action_space="joint_position",
|
||||||
|
)
|
||||||
|
|
||||||
|
if rank == 0:
|
||||||
|
upstream.logging.info("Using roboarena policy server interface")
|
||||||
|
upstream.logging.info("Server config: %s", server_config)
|
||||||
|
roboarena_server = upstream.RoboarenaServer(
|
||||||
|
policy=wrapper_policy,
|
||||||
|
server_config=server_config,
|
||||||
|
host="0.0.0.0",
|
||||||
|
port=args.port,
|
||||||
|
)
|
||||||
|
roboarena_server.serve_forever()
|
||||||
|
else:
|
||||||
|
worker = upstream.DistributedWorker(
|
||||||
|
policy=policy,
|
||||||
|
signal_group=signal_group,
|
||||||
|
)
|
||||||
|
upstream.asyncio.run(worker.run())
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
upstream.logging.basicConfig(level=upstream.logging.INFO, force=True)
|
||||||
|
main(tyro.cli(upstream.Args))
|
||||||
@@ -8,7 +8,7 @@ L4-style smoke runs against an already-running server. The pytest wrappers in
|
|||||||
H100 / MI325) because they launch the live Omni server inside the test.
|
H100 / MI325) because they launch the live Omni server inside the test.
|
||||||
|
|
||||||
1. **Daily-Omni** — MCQ accuracy fields in the saved JSON (``daily_omni_accuracy``, …); by default the
|
1. **Daily-Omni** — MCQ accuracy fields in the saved JSON (``daily_omni_accuracy``, …); by default the
|
||||||
run **fails** if accuracy is strictly below **0.67** (``--min-daily-omni-accuracy`` / ``ACC_BENCH_MIN_DAILY_OMNI_ACCURACY``).
|
run **fails** if accuracy is strictly below **0.69** (``--min-daily-omni-accuracy`` / ``ACC_BENCH_MIN_DAILY_OMNI_ACCURACY``).
|
||||||
2. **Seed-TTS** — ``seed-tts-eval``-style metrics when ``--seed-tts-wer-eval`` is used
|
2. **Seed-TTS** — ``seed-tts-eval``-style metrics when ``--seed-tts-wer-eval`` is used
|
||||||
(WER / SIM / UTMOS keys from :func:`compute_seed_tts_wer_metrics`).
|
(WER / SIM / UTMOS keys from :func:`compute_seed_tts_wer_metrics`).
|
||||||
|
|
||||||
@@ -308,9 +308,9 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
|||||||
p.add_argument(
|
p.add_argument(
|
||||||
"--min-daily-omni-accuracy",
|
"--min-daily-omni-accuracy",
|
||||||
type=float,
|
type=float,
|
||||||
default=float((os.environ.get("ACC_BENCH_MIN_DAILY_OMNI_ACCURACY") or "0.67").strip() or "0.67"),
|
default=float((os.environ.get("ACC_BENCH_MIN_DAILY_OMNI_ACCURACY") or "0.69").strip() or "0.69"),
|
||||||
help="Fail when daily_omni_accuracy is strictly below this threshold (0–1). "
|
help="Fail when daily_omni_accuracy is strictly below this threshold (0–1). "
|
||||||
"Default baseline 0.67; override with env ACC_BENCH_MIN_DAILY_OMNI_ACCURACY or pass 0 to disable the floor.",
|
"Default baseline 0.69; override with env ACC_BENCH_MIN_DAILY_OMNI_ACCURACY or pass 0 to disable the floor.",
|
||||||
)
|
)
|
||||||
|
|
||||||
p.add_argument(
|
p.add_argument(
|
||||||
@@ -340,7 +340,7 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
|||||||
p.add_argument(
|
p.add_argument(
|
||||||
"--max-seed-tts-mean-wer",
|
"--max-seed-tts-mean-wer",
|
||||||
type=float,
|
type=float,
|
||||||
default=0.5,
|
default=0.35,
|
||||||
help="If set, fail when seed_tts_content_error_mean is strictly above this value.",
|
help="If set, fail when seed_tts_content_error_mean is strictly above this value.",
|
||||||
)
|
)
|
||||||
p.add_argument(
|
p.add_argument(
|
||||||
|
|||||||
@@ -218,7 +218,7 @@ def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> tuple[Image.I
|
|||||||
torch_dtype=torch.bfloat16,
|
torch_dtype=torch.bfloat16,
|
||||||
trust_remote_code=True,
|
trust_remote_code=True,
|
||||||
).to("cuda")
|
).to("cuda")
|
||||||
|
pipe.transformer.set_attention_backend("_flash_3_hub")
|
||||||
_diffusers_dummy_run(pipe)
|
_diffusers_dummy_run(pipe)
|
||||||
|
|
||||||
generator = torch.Generator(device="cuda").manual_seed(SEED)
|
generator = torch.Generator(device="cuda").manual_seed(SEED)
|
||||||
|
|||||||
@@ -0,0 +1,218 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import copy
|
||||||
|
import io
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import requests
|
||||||
|
import yaml
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
from tests.e2e.accuracy.helpers import assert_images_pixel_close, assert_similarity, model_output_dir
|
||||||
|
from tests.helpers.mark import hardware_test
|
||||||
|
from tests.helpers.runtime import OmniServer
|
||||||
|
|
||||||
|
pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]
|
||||||
|
|
||||||
|
MODEL_NAME = "tencent/HunyuanImage-3.0-Instruct"
|
||||||
|
SEED = 42
|
||||||
|
NUM_INFERENCE_STEPS = 50
|
||||||
|
GUIDANCE_SCALE = 2.5
|
||||||
|
HEIGHT = 1024
|
||||||
|
WIDTH = 1024
|
||||||
|
PROMPT = "A brown and white dog is running on the grass."
|
||||||
|
MEAN_THRESHOLD = 3e-2
|
||||||
|
P99_THRESHOLD = 3e-1
|
||||||
|
SSIM_THRESHOLD = 0.97
|
||||||
|
PSNR_THRESHOLD = 30.0
|
||||||
|
|
||||||
|
_REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent
|
||||||
|
BASELINE_PATH = _REPO_ROOT / "tests" / "assets" / "hunyuan" / "hunyuan_baseline.png"
|
||||||
|
_OFFLINE_SCRIPT = _REPO_ROOT / "examples" / "offline_inference" / "hunyuan_image3" / "end2end.py"
|
||||||
|
|
||||||
|
# DiT-only deploy config with trust_remote_code (based on hunyuan_image3_dit.yaml).
|
||||||
|
_DEPLOY_CONFIG = {
|
||||||
|
"pipeline": "hunyuan_image3_dit",
|
||||||
|
"async_chunk": False,
|
||||||
|
"trust_remote_code": True,
|
||||||
|
"stages": [
|
||||||
|
{
|
||||||
|
"stage_id": 0,
|
||||||
|
"max_num_seqs": 1,
|
||||||
|
"gpu_memory_utilization": 0.9,
|
||||||
|
"enforce_eager": True,
|
||||||
|
"trust_remote_code": True,
|
||||||
|
"devices": "0,1,2,3", # set dynamically by _write_deploy_config
|
||||||
|
"vae_use_slicing": False,
|
||||||
|
"vae_use_tiling": False,
|
||||||
|
"parallel_config": {
|
||||||
|
"pipeline_parallel_size": 1,
|
||||||
|
"data_parallel_size": 1,
|
||||||
|
"tensor_parallel_size": 4,
|
||||||
|
"enable_expert_parallel": True,
|
||||||
|
"sequence_parallel_size": 1,
|
||||||
|
"ulysses_degree": 1,
|
||||||
|
"ring_degree": 1,
|
||||||
|
"cfg_parallel_size": 1,
|
||||||
|
"vae_patch_parallel_size": 1,
|
||||||
|
"use_hsdp": False,
|
||||||
|
"hsdp_shard_size": -1,
|
||||||
|
"hsdp_replicate_size": 1,
|
||||||
|
},
|
||||||
|
"default_sampling_params": {
|
||||||
|
"seed": SEED,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _model_name() -> str:
|
||||||
|
return os.environ.get("HUNYUAN_IMAGE3_MODEL", MODEL_NAME)
|
||||||
|
|
||||||
|
|
||||||
|
def _devices() -> str:
|
||||||
|
return os.environ.get("HUNYUAN_IMAGE3_DEVICES", "0,1,2,3")
|
||||||
|
|
||||||
|
|
||||||
|
def _write_deploy_config(path: Path) -> None:
|
||||||
|
config = copy.deepcopy(_DEPLOY_CONFIG)
|
||||||
|
devices = _devices()
|
||||||
|
config["stages"][0]["devices"] = devices
|
||||||
|
config["stages"][0]["parallel_config"]["tensor_parallel_size"] = len(devices.split(","))
|
||||||
|
path.write_text(yaml.dump(config, default_flow_style=False, sort_keys=False))
|
||||||
|
|
||||||
|
|
||||||
|
def _run_vllm_omni_hunyuan_image3_online(*, model: str, deploy_config: str, output_path: Path) -> Image.Image:
|
||||||
|
server_args = [
|
||||||
|
"--deploy-config",
|
||||||
|
deploy_config,
|
||||||
|
"--stage-init-timeout",
|
||||||
|
"300",
|
||||||
|
"--init-timeout",
|
||||||
|
"900",
|
||||||
|
"--enforce-eager",
|
||||||
|
"--trust-remote-code",
|
||||||
|
]
|
||||||
|
with OmniServer(model, server_args, use_omni=True) as omni_server:
|
||||||
|
response = requests.post(
|
||||||
|
f"http://{omni_server.host}:{omni_server.port}/v1/images/generations",
|
||||||
|
json={
|
||||||
|
"model": omni_server.model,
|
||||||
|
"prompt": PROMPT,
|
||||||
|
"size": f"{WIDTH}x{HEIGHT}",
|
||||||
|
"n": 1,
|
||||||
|
"response_format": "b64_json",
|
||||||
|
"num_inference_steps": NUM_INFERENCE_STEPS,
|
||||||
|
"guidance_scale": GUIDANCE_SCALE,
|
||||||
|
"seed": SEED,
|
||||||
|
"bot_task": "none",
|
||||||
|
"use_system_prompt": "en_unified",
|
||||||
|
},
|
||||||
|
timeout=600,
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
payload = response.json()
|
||||||
|
assert len(payload["data"]) == 1
|
||||||
|
image_bytes = base64.b64decode(payload["data"][0]["b64_json"])
|
||||||
|
image = Image.open(io.BytesIO(image_bytes)).convert("RGB")
|
||||||
|
image.load()
|
||||||
|
image.save(output_path)
|
||||||
|
return image
|
||||||
|
|
||||||
|
|
||||||
|
def _run_vllm_omni_hunyuan_image3_offline(*, model: str, deploy_config: str, output_path: Path) -> Image.Image:
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
output_dir = str(output_path.parent)
|
||||||
|
subprocess.run(
|
||||||
|
[
|
||||||
|
sys.executable,
|
||||||
|
str(_OFFLINE_SCRIPT),
|
||||||
|
"--modality",
|
||||||
|
"text2img",
|
||||||
|
"--deploy-config",
|
||||||
|
deploy_config,
|
||||||
|
"--prompts",
|
||||||
|
PROMPT,
|
||||||
|
"--output",
|
||||||
|
output_dir,
|
||||||
|
"--steps",
|
||||||
|
str(NUM_INFERENCE_STEPS),
|
||||||
|
"--guidance-scale",
|
||||||
|
str(GUIDANCE_SCALE),
|
||||||
|
"--seed",
|
||||||
|
str(SEED),
|
||||||
|
"--height",
|
||||||
|
str(HEIGHT),
|
||||||
|
"--width",
|
||||||
|
str(WIDTH),
|
||||||
|
"--bot-task",
|
||||||
|
"none",
|
||||||
|
"--sys-type",
|
||||||
|
"en_unified",
|
||||||
|
"--model",
|
||||||
|
model,
|
||||||
|
"--enforce-eager",
|
||||||
|
],
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
images = sorted(Path(output_dir).glob("output_*.png"))
|
||||||
|
assert images, f"No output image found in {output_dir}"
|
||||||
|
image = Image.open(images[0]).convert("RGB")
|
||||||
|
image.load()
|
||||||
|
image.save(output_path)
|
||||||
|
return image
|
||||||
|
|
||||||
|
|
||||||
|
def _assert_against_baseline(image: Image.Image, label: str) -> None:
|
||||||
|
assert BASELINE_PATH.exists(), f"Baseline image not found at {BASELINE_PATH}"
|
||||||
|
baseline_image = Image.open(BASELINE_PATH).convert("RGB")
|
||||||
|
|
||||||
|
assert_images_pixel_close(
|
||||||
|
model_name=f"{MODEL_NAME} ({label} vs baseline)",
|
||||||
|
vllm_image=image,
|
||||||
|
diffusers_image=baseline_image,
|
||||||
|
mean_threshold=MEAN_THRESHOLD,
|
||||||
|
p99_threshold=P99_THRESHOLD,
|
||||||
|
)
|
||||||
|
assert_similarity(
|
||||||
|
model_name=f"{MODEL_NAME} ({label} vs baseline)",
|
||||||
|
vllm_image=image,
|
||||||
|
diffusers_image=baseline_image,
|
||||||
|
ssim_threshold=SSIM_THRESHOLD,
|
||||||
|
psnr_threshold=PSNR_THRESHOLD,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
||||||
|
def test_hunyuan_image3_pixel_accuracy_online(accuracy_artifact_root: Path) -> None:
|
||||||
|
model = _model_name()
|
||||||
|
output_dir = model_output_dir(accuracy_artifact_root, MODEL_NAME)
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
deploy_config_path = Path(tmpdir) / "deploy.yaml"
|
||||||
|
_write_deploy_config(deploy_config_path)
|
||||||
|
image = _run_vllm_omni_hunyuan_image3_online(
|
||||||
|
model=model, deploy_config=str(deploy_config_path), output_path=output_dir / "vllm_omni_online.png"
|
||||||
|
)
|
||||||
|
_assert_against_baseline(image, "online")
|
||||||
|
|
||||||
|
|
||||||
|
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
||||||
|
def test_hunyuan_image3_pixel_accuracy_offline(accuracy_artifact_root: Path) -> None:
|
||||||
|
model = _model_name()
|
||||||
|
output_dir = model_output_dir(accuracy_artifact_root, MODEL_NAME)
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
deploy_config_path = Path(tmpdir) / "deploy.yaml"
|
||||||
|
_write_deploy_config(deploy_config_path)
|
||||||
|
image = _run_vllm_omni_hunyuan_image3_offline(
|
||||||
|
model=model, deploy_config=str(deploy_config_path), output_path=output_dir / "vllm_omni_offline.png"
|
||||||
|
)
|
||||||
|
_assert_against_baseline(image, "offline")
|
||||||
@@ -106,6 +106,7 @@ def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> Image.Image:
|
|||||||
trust_remote_code=True,
|
trust_remote_code=True,
|
||||||
local_files_only=_local_files_only(model),
|
local_files_only=_local_files_only(model),
|
||||||
).to("cuda")
|
).to("cuda")
|
||||||
|
pipe.transformer.set_attention_backend("_flash_3_hub")
|
||||||
generator = torch.Generator(device="cuda").manual_seed(SEED)
|
generator = torch.Generator(device="cuda").manual_seed(SEED)
|
||||||
result = pipe( # pyright: ignore[reportCallIssue]
|
result = pipe( # pyright: ignore[reportCallIssue]
|
||||||
prompt=PROMPT,
|
prompt=PROMPT,
|
||||||
@@ -167,6 +168,7 @@ def _run_diffusers_qwen_image_2512(*, model: str, output_path: Path) -> Image.Im
|
|||||||
trust_remote_code=True,
|
trust_remote_code=True,
|
||||||
local_files_only=_local_files_only(model),
|
local_files_only=_local_files_only(model),
|
||||||
).to("cuda")
|
).to("cuda")
|
||||||
|
pipe.transformer.set_attention_backend("_flash_3_hub")
|
||||||
generator = torch.Generator(device="cuda").manual_seed(SEED_2512)
|
generator = torch.Generator(device="cuda").manual_seed(SEED_2512)
|
||||||
result = pipe( # pyright: ignore[reportCallIssue]
|
result = pipe( # pyright: ignore[reportCallIssue]
|
||||||
prompt=PROMPT_2512,
|
prompt=PROMPT_2512,
|
||||||
|
|||||||
@@ -32,7 +32,8 @@ from vllm_omni import Omni
|
|||||||
|
|
||||||
# Selected by the nightly "TTS · Function Test" (-m "full_model and L4 and tts").
|
# Selected by the nightly "TTS · Function Test" (-m "full_model and L4 and tts").
|
||||||
# Without the tts + run-level marks these tests are collected but never run.
|
# Without the tts + run-level marks these tests are collected but never run.
|
||||||
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
# TODO: Fix this test
|
||||||
|
# pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
||||||
|
|
||||||
SAMPLE_RATE = 24_000 # All MOSS-TTS full variants output 24 kHz
|
SAMPLE_RATE = 24_000 # All MOSS-TTS full variants output 24 kHz
|
||||||
|
|
||||||
@@ -128,7 +129,6 @@ def delay_engine():
|
|||||||
return Omni(model=_DELAY_MODEL, stage_init_timeout=300)
|
return Omni(model=_DELAY_MODEL, stage_init_timeout=300)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.omni
|
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_delay_english(delay_engine, ref_audio_path):
|
def test_moss_tts_delay_english(delay_engine, ref_audio_path):
|
||||||
"""MossTTSDelayModel: English voice_clone produces non-empty 24 kHz audio."""
|
"""MossTTSDelayModel: English voice_clone produces non-empty 24 kHz audio."""
|
||||||
@@ -140,7 +140,6 @@ def test_moss_tts_delay_english(delay_engine, ref_audio_path):
|
|||||||
assert not torch.all(audio == 0), "Audio is silence"
|
assert not torch.all(audio == 0), "Audio is silence"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.omni
|
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_delay_chinese(delay_engine, ref_audio_path):
|
def test_moss_tts_delay_chinese(delay_engine, ref_audio_path):
|
||||||
"""MossTTSDelayModel: Chinese input produces non-empty audio."""
|
"""MossTTSDelayModel: Chinese input produces non-empty audio."""
|
||||||
@@ -152,7 +151,6 @@ def test_moss_tts_delay_chinese(delay_engine, ref_audio_path):
|
|||||||
assert not torch.all(audio == 0)
|
assert not torch.all(audio == 0)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.omni
|
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_delay_deterministic(delay_engine, ref_audio_path):
|
def test_moss_tts_delay_deterministic(delay_engine, ref_audio_path):
|
||||||
"""MossTTSDelayModel: same seed yields identical waveforms."""
|
"""MossTTSDelayModel: same seed yields identical waveforms."""
|
||||||
@@ -164,7 +162,6 @@ def test_moss_tts_delay_deterministic(delay_engine, ref_audio_path):
|
|||||||
assert torch.allclose(audio1, audio2, atol=1e-4), "Waveforms differ across identical seeds"
|
assert torch.allclose(audio1, audio2, atol=1e-4), "Waveforms differ across identical seeds"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.omni
|
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_delay_batch(delay_engine, ref_audio_path):
|
def test_moss_tts_delay_batch(delay_engine, ref_audio_path):
|
||||||
"""MossTTSDelayModel: batch of two requests each returns non-empty audio."""
|
"""MossTTSDelayModel: batch of two requests each returns non-empty audio."""
|
||||||
@@ -206,7 +203,6 @@ def realtime_engine():
|
|||||||
return Omni(model=_REALTIME_MODEL, stage_init_timeout=300)
|
return Omni(model=_REALTIME_MODEL, stage_init_timeout=300)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.omni
|
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_realtime_english(realtime_engine, ref_audio_path):
|
def test_moss_tts_realtime_english(realtime_engine, ref_audio_path):
|
||||||
"""MossTTSRealtime: English voice_clone produces non-empty 24 kHz audio."""
|
"""MossTTSRealtime: English voice_clone produces non-empty 24 kHz audio."""
|
||||||
@@ -218,7 +214,6 @@ def test_moss_tts_realtime_english(realtime_engine, ref_audio_path):
|
|||||||
assert not torch.all(audio == 0), "Audio is silence"
|
assert not torch.all(audio == 0), "Audio is silence"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.omni
|
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_realtime_deterministic(realtime_engine, ref_audio_path):
|
def test_moss_tts_realtime_deterministic(realtime_engine, ref_audio_path):
|
||||||
"""MossTTSRealtime: same seed yields identical waveforms."""
|
"""MossTTSRealtime: same seed yields identical waveforms."""
|
||||||
|
|||||||
@@ -120,7 +120,6 @@ def _collect_audio(omni: Omni, request: dict) -> tuple[torch.Tensor, int]:
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
@pytest.mark.omni
|
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_nano_english(omni_runner: OmniRunner, ref_audio_path) -> None:
|
def test_moss_tts_nano_english(omni_runner: OmniRunner, ref_audio_path) -> None:
|
||||||
"""English TTS produces non-empty 48 kHz stereo audio."""
|
"""English TTS produces non-empty 48 kHz stereo audio."""
|
||||||
@@ -133,7 +132,6 @@ def test_moss_tts_nano_english(omni_runner: OmniRunner, ref_audio_path) -> None:
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
@pytest.mark.omni
|
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_nano_chinese(omni_runner: OmniRunner, ref_audio_path) -> None:
|
def test_moss_tts_nano_chinese(omni_runner: OmniRunner, ref_audio_path) -> None:
|
||||||
"""Chinese TTS produces non-empty audio."""
|
"""Chinese TTS produces non-empty audio."""
|
||||||
@@ -146,7 +144,6 @@ def test_moss_tts_nano_chinese(omni_runner: OmniRunner, ref_audio_path) -> None:
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
@pytest.mark.omni
|
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_nano_deterministic(omni_runner: OmniRunner, ref_audio_path) -> None:
|
def test_moss_tts_nano_deterministic(omni_runner: OmniRunner, ref_audio_path) -> None:
|
||||||
"""Same seed produces identical waveforms."""
|
"""Same seed produces identical waveforms."""
|
||||||
@@ -159,7 +156,6 @@ def test_moss_tts_nano_deterministic(omni_runner: OmniRunner, ref_audio_path) ->
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
@pytest.mark.omni
|
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_nano_batch(omni_runner: OmniRunner, ref_audio_path) -> None:
|
def test_moss_tts_nano_batch(omni_runner: OmniRunner, ref_audio_path) -> None:
|
||||||
"""Batch of two requests returns audio for each."""
|
"""Batch of two requests returns audio for each."""
|
||||||
|
|||||||
@@ -24,10 +24,11 @@ from vllm import SamplingParams
|
|||||||
from tests.helpers.mark import hardware_test
|
from tests.helpers.mark import hardware_test
|
||||||
from vllm_omni import Omni
|
from vllm_omni import Omni
|
||||||
|
|
||||||
# H100-gated (8B). Picked up by the nightly H100 omni job
|
# H100-gated (8B). Picked up by the nightly "TTS · Function Test with H100"
|
||||||
# (-m "full_model and H100 and omni"); the tts mark keeps it consistent with
|
# job (-m "full_model and H100 and tts"); the L4 MOSS-TTS tests land in the
|
||||||
# the other MOSS-TTS tests and any tts-scoped sweeps.
|
# sibling L4 tts lane instead.
|
||||||
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
# TODO: Fix this test
|
||||||
|
# pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
||||||
|
|
||||||
SAMPLE_RATE = 24_000
|
SAMPLE_RATE = 24_000
|
||||||
REF_AUDIO_URL = "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS/main/assets/audio/reference_zh_1.wav"
|
REF_AUDIO_URL = "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS/main/assets/audio/reference_zh_1.wav"
|
||||||
@@ -101,7 +102,6 @@ def _collect_audio(omni: Omni, request: dict) -> tuple[torch.Tensor, int]:
|
|||||||
raise AssertionError("No stage outputs received")
|
raise AssertionError("No stage outputs received")
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.omni
|
|
||||||
@hardware_test(res={"cuda": "H100"})
|
@hardware_test(res={"cuda": "H100"})
|
||||||
def test_moss_tts_v15_voice_clone(v15_engine, ref_audio_path):
|
def test_moss_tts_v15_voice_clone(v15_engine, ref_audio_path):
|
||||||
"""MOSS-TTS-v1.5 (8B): voice_clone produces non-empty 24 kHz audio."""
|
"""MOSS-TTS-v1.5 (8B): voice_clone produces non-empty 24 kHz audio."""
|
||||||
|
|||||||
@@ -0,0 +1,90 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
"""E2E online serving test for DreamZero OpenPI websocket serving."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from tests.dreamzero import openpi_client_helper as openpi_client
|
||||||
|
from tests.helpers.mark import hardware_test
|
||||||
|
from tests.helpers.runtime import OmniServerParams, get_open_port
|
||||||
|
|
||||||
|
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||||
|
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||||
|
|
||||||
|
MODEL = "GEAR-Dreams/DreamZero-DROID"
|
||||||
|
|
||||||
|
|
||||||
|
test_params = [
|
||||||
|
OmniServerParams(
|
||||||
|
model=MODEL,
|
||||||
|
port=8091,
|
||||||
|
server_args=[
|
||||||
|
"--deploy-config",
|
||||||
|
"vllm_omni/deploy/dreamzero_tp1_cfg2.yaml",
|
||||||
|
"--enforce-eager",
|
||||||
|
"--disable-log-stats",
|
||||||
|
],
|
||||||
|
env_dict={
|
||||||
|
"ATTENTION_BACKEND": "torch",
|
||||||
|
"DIFFUSION_ATTENTION_BACKEND": "TORCH_SDPA",
|
||||||
|
"VLLM_DISABLE_COMPILE_CACHE": "1",
|
||||||
|
"MASTER_PORT": str(get_open_port()),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _write_synthetic_video(path: Path, cv2_module, *, channel: int) -> None:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
height, width, num_frames = 180, 320, 24
|
||||||
|
writer = cv2_module.VideoWriter(str(path), cv2_module.VideoWriter_fourcc(*"mp4v"), 15.0, (width, height))
|
||||||
|
if not writer.isOpened():
|
||||||
|
raise RuntimeError(f"Failed to open video writer for {path}")
|
||||||
|
try:
|
||||||
|
for frame_idx in range(num_frames):
|
||||||
|
frame = np.zeros((height, width, 3), dtype=np.uint8)
|
||||||
|
frame[..., channel] = (frame_idx * 7) % 255
|
||||||
|
frame[..., (channel + 1) % 3] = 64
|
||||||
|
writer.write(cv2_module.cvtColor(frame, cv2_module.COLOR_RGB2BGR))
|
||||||
|
finally:
|
||||||
|
writer.release()
|
||||||
|
|
||||||
|
|
||||||
|
def _write_synthetic_dreamzero_videos(client_mod, video_dir: Path) -> None:
|
||||||
|
for channel, file_name in enumerate(client_mod.CAMERA_FILES.values()):
|
||||||
|
_write_synthetic_video(video_dir / file_name, client_mod.cv2, channel=channel)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.full_model
|
||||||
|
@pytest.mark.diffusion
|
||||||
|
@pytest.mark.distributed_cuda
|
||||||
|
@hardware_test(res={"cuda": "H100"}, num_cards=2)
|
||||||
|
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||||
|
def test_dreamzero_openpi_online(omni_server, tmp_path: Path) -> None:
|
||||||
|
try:
|
||||||
|
openpi_client.require_dependencies()
|
||||||
|
except ModuleNotFoundError as exc:
|
||||||
|
pytest.skip(str(exc))
|
||||||
|
|
||||||
|
video_dir = tmp_path / "dreamzero_videos"
|
||||||
|
_write_synthetic_dreamzero_videos(openpi_client, video_dir)
|
||||||
|
result = openpi_client.run_policy_session(
|
||||||
|
host=omni_server.host,
|
||||||
|
port=omni_server.port,
|
||||||
|
video_dir=video_dir,
|
||||||
|
session_id="dreamzero-online-e2e",
|
||||||
|
)
|
||||||
|
|
||||||
|
openpi_client.validate_session_result(result)
|
||||||
|
|
||||||
|
metadata = result["metadata"]
|
||||||
|
assert metadata["needs_session_id"] is True
|
||||||
|
assert metadata["needs_stereo_camera"] is False
|
||||||
|
assert tuple(metadata["image_resolution"]) == (180, 320)
|
||||||
@@ -0,0 +1,183 @@
|
|||||||
|
"""
|
||||||
|
Comprehensive tests of diffusion features that are available in online serving mode
|
||||||
|
and are supported by the following model:
|
||||||
|
- zai-org/GLM-Image (t2i and i2i)
|
||||||
|
|
||||||
|
Coverage (per mode):
|
||||||
|
- Baseline
|
||||||
|
- Cache-DiT
|
||||||
|
- Tensor-Parallel
|
||||||
|
- HSDP
|
||||||
|
- Sequence parallel
|
||||||
|
|
||||||
|
Topology for multi-GPU cases uses runtime deploy YAML (except TP)
|
||||||
|
so stage-1 parallel fields reach the diffusion worker without relying on CLI
|
||||||
|
|
||||||
|
assert_diffusion_response validates successful generation and the expected resolution.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from tests.helpers.mark import hardware_marks
|
||||||
|
from tests.helpers.media import generate_synthetic_image
|
||||||
|
from tests.helpers.runtime import (
|
||||||
|
OmniServer,
|
||||||
|
OmniServerParams,
|
||||||
|
OpenAIClientHandler,
|
||||||
|
dummy_messages_from_mix_data,
|
||||||
|
)
|
||||||
|
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||||
|
|
||||||
|
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
|
||||||
|
|
||||||
|
MODEL = os.environ.get("GLM_IMAGE_MODEL_PATH", "zai-org/GLM-Image")
|
||||||
|
|
||||||
|
T2I_PROMPT = "A Vincent van Gogh style impressionist painting."
|
||||||
|
I2I_PROMPT = "Transform this modern, geometric image into a Vincent van Gogh style impressionist painting."
|
||||||
|
NEGATIVE_PROMPT = "low quality, blurry, distorted, unnatural colors"
|
||||||
|
|
||||||
|
TWO_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}, num_cards=2)
|
||||||
|
|
||||||
|
GLM_DEPLOY = get_deploy_config_path("glm_image.yaml")
|
||||||
|
GLM_CACHEDIT_DEPLOY = modify_stage_config(
|
||||||
|
GLM_DEPLOY,
|
||||||
|
updates={
|
||||||
|
"stages": {
|
||||||
|
1: {"cache_backend": "cache_dit"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
GLM_TP_2_DEPLOY = modify_stage_config(
|
||||||
|
GLM_DEPLOY,
|
||||||
|
updates={
|
||||||
|
"stages": {
|
||||||
|
0: {"devices": "0", "tensor_parallel_size": 1},
|
||||||
|
1: {"devices": "0,1", "tensor_parallel_size": 2},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
GLM_HSDP_2_DEPLOY = modify_stage_config(
|
||||||
|
GLM_DEPLOY,
|
||||||
|
updates={
|
||||||
|
"stages": {
|
||||||
|
0: {"devices": "0"},
|
||||||
|
1: {
|
||||||
|
"devices": "0,1",
|
||||||
|
"parallel_config": {
|
||||||
|
"use_hsdp": True,
|
||||||
|
"hsdp_shard_size": 2,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
GLM_SP_2_DEPLOY = modify_stage_config(
|
||||||
|
GLM_DEPLOY,
|
||||||
|
updates={
|
||||||
|
"stages": {
|
||||||
|
0: {"devices": "0"},
|
||||||
|
1: {
|
||||||
|
"devices": "0,1",
|
||||||
|
"parallel_config": {
|
||||||
|
"ulysses_degree": 2,
|
||||||
|
"ring_degree": 1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _get_diffusion_feature_cases(model: str):
|
||||||
|
return [
|
||||||
|
# Baseline: default deploy (stage 0 -> GPU 0, stage 1 -> GPU 1)
|
||||||
|
pytest.param(
|
||||||
|
OmniServerParams(
|
||||||
|
model=model,
|
||||||
|
stage_config_path=GLM_DEPLOY,
|
||||||
|
),
|
||||||
|
id="baseline",
|
||||||
|
marks=TWO_CARD_FEATURE_MARKS,
|
||||||
|
),
|
||||||
|
# Cache-DiT via runtime deploy patch (cache_backend -> engine_extras on stage 1)
|
||||||
|
pytest.param(
|
||||||
|
OmniServerParams(
|
||||||
|
model=model,
|
||||||
|
stage_config_path=GLM_CACHEDIT_DEPLOY,
|
||||||
|
),
|
||||||
|
id="cachedit",
|
||||||
|
marks=TWO_CARD_FEATURE_MARKS,
|
||||||
|
),
|
||||||
|
# Tensor-Parallel (2 GPUs on diffusion stage)
|
||||||
|
pytest.param(
|
||||||
|
OmniServerParams(
|
||||||
|
model=model,
|
||||||
|
stage_config_path=GLM_TP_2_DEPLOY,
|
||||||
|
),
|
||||||
|
id="tensor_parallel_2",
|
||||||
|
marks=TWO_CARD_FEATURE_MARKS,
|
||||||
|
),
|
||||||
|
# HSDP (2 GPUs on diffusion stage)
|
||||||
|
pytest.param(
|
||||||
|
OmniServerParams(
|
||||||
|
model=model,
|
||||||
|
stage_config_path=GLM_HSDP_2_DEPLOY,
|
||||||
|
),
|
||||||
|
id="hsdp_2",
|
||||||
|
marks=TWO_CARD_FEATURE_MARKS,
|
||||||
|
),
|
||||||
|
# Ulysses sequence parallel degree=2 (2 GPUs)
|
||||||
|
pytest.param(
|
||||||
|
OmniServerParams(
|
||||||
|
model=model,
|
||||||
|
stage_config_path=GLM_SP_2_DEPLOY,
|
||||||
|
),
|
||||||
|
id="sequence_parallel_2",
|
||||||
|
marks=TWO_CARD_FEATURE_MARKS,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
MODES = ["t2i", "i2i"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("mode", MODES)
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"omni_server",
|
||||||
|
_get_diffusion_feature_cases(MODEL),
|
||||||
|
indirect=True,
|
||||||
|
)
|
||||||
|
def test_glm_image(
|
||||||
|
omni_server: OmniServer,
|
||||||
|
mode: str,
|
||||||
|
openai_client: OpenAIClientHandler,
|
||||||
|
):
|
||||||
|
"""GLM-Image diffusion feature coverage for t2i and i2i on two H100s."""
|
||||||
|
|
||||||
|
if mode == "t2i":
|
||||||
|
messages = dummy_messages_from_mix_data(content_text=T2I_PROMPT)
|
||||||
|
else:
|
||||||
|
image_size = 1024
|
||||||
|
image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(image_size, image_size)['base64']}"
|
||||||
|
messages = dummy_messages_from_mix_data(
|
||||||
|
image_data_url=image_data_url,
|
||||||
|
content_text=I2I_PROMPT,
|
||||||
|
)
|
||||||
|
|
||||||
|
request_config = {
|
||||||
|
"model": omni_server.model,
|
||||||
|
"messages": messages,
|
||||||
|
"extra_body": {
|
||||||
|
"height": 1024,
|
||||||
|
"width": 1024,
|
||||||
|
"num_inference_steps": 2,
|
||||||
|
"guidance_scale": 1.5,
|
||||||
|
"true_cfg_scale": 4.0,
|
||||||
|
"negative_prompt": NEGATIVE_PROMPT,
|
||||||
|
"seed": 42,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
openai_client.send_diffusion_request(request_config)
|
||||||
@@ -0,0 +1,388 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""End-to-end online tests for higgs-audio v3 against /v1/audio/speech.
|
||||||
|
|
||||||
|
Mirrors the higgs_audio_v2 test layout. Covers the plain-text-in / audio-out
|
||||||
|
happy path, a small concurrent burst (Stage 0 prefix caching has to coexist
|
||||||
|
with batching), the voice-clone path via ``ref_audio`` + ``ref_text``, the
|
||||||
|
upstream ``references[]`` cookbook alias added by
|
||||||
|
``normalize_references_alias`` in ``protocol/audio.py``, and the few
|
||||||
|
validator rejections that should remain 4xx.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||||
|
os.environ.setdefault("VLLM_USE_DEEP_GEMM", "0")
|
||||||
|
os.environ.setdefault("VLLM_MOE_USE_DEEP_GEMM", "0")
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from tests.helpers.mark import hardware_test
|
||||||
|
from tests.helpers.media import load_test_audio_data_url
|
||||||
|
from tests.helpers.runtime import OmniServerParams
|
||||||
|
from tests.helpers.stage_config import get_deploy_config_path
|
||||||
|
|
||||||
|
MODEL = "bosonai/higgs-audio-v3-tts-4b"
|
||||||
|
STAGE_CONFIG = get_deploy_config_path("higgs_multimodal_qwen3.yaml")
|
||||||
|
SERVER_ARGS = ["--trust-remote-code", "--disable-log-stats"]
|
||||||
|
SERVER_ENV = {"VLLM_USE_DEEP_GEMM": "0", "VLLM_MOE_USE_DEEP_GEMM": "0"}
|
||||||
|
|
||||||
|
TEST_PARAMS = [
|
||||||
|
pytest.param(
|
||||||
|
OmniServerParams(
|
||||||
|
model=MODEL,
|
||||||
|
stage_config_path=STAGE_CONFIG,
|
||||||
|
server_args=SERVER_ARGS,
|
||||||
|
env_dict=SERVER_ENV,
|
||||||
|
),
|
||||||
|
id="higgs_audio_v3_plain_text",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
DEFAULT_SPEECH_TIMEOUT_S = 180.0
|
||||||
|
# Floor for ~0.5 s of 24 kHz mono PCM_16: 24000 * 0.5 * 2 bytes ~= 24 KiB.
|
||||||
|
# A WAV header adds 44 bytes; conservative floor catches truncated /
|
||||||
|
# silence-only outputs without flagging short legitimate clips.
|
||||||
|
_MIN_AUDIO_BYTES = 20_000
|
||||||
|
|
||||||
|
# Reuse the shared TTS reference clip (clean ~5 s 24 kHz mono human speech)
|
||||||
|
# vendored under tests/assets/qwen3_tts/. Keeps a single WAV across TTS
|
||||||
|
# tests rather than duplicating asset bytes.
|
||||||
|
_REF_AUDIO_URL = load_test_audio_data_url("qwen3_tts/clone_2.wav")
|
||||||
|
_REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
|
||||||
|
class TestHiggsAudioV3OnlineHappyPath:
|
||||||
|
"""Plain-text -> audio happy paths against the live HTTP server."""
|
||||||
|
|
||||||
|
@pytest.mark.core_model
|
||||||
|
@pytest.mark.tts
|
||||||
|
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||||
|
def test_plain_text_wav(self, omni_server, openai_client) -> None:
|
||||||
|
"""Single non-streaming WAV request - canonical TTS happy path."""
|
||||||
|
openai_client.send_audio_speech_request(
|
||||||
|
{
|
||||||
|
"model": omni_server.model,
|
||||||
|
"input": "Hello world.",
|
||||||
|
"stream": False,
|
||||||
|
"response_format": "wav",
|
||||||
|
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
||||||
|
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
@pytest.mark.core_model
|
||||||
|
@pytest.mark.tts
|
||||||
|
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||||
|
def test_plain_text_with_max_new_tokens(self, omni_server, openai_client) -> None:
|
||||||
|
"""``max_new_tokens`` is one of the few extra fields the v3 validator accepts."""
|
||||||
|
openai_client.send_audio_speech_request(
|
||||||
|
{
|
||||||
|
"model": omni_server.model,
|
||||||
|
"input": "Innovation distinguishes between a leader and a follower.",
|
||||||
|
"stream": False,
|
||||||
|
"response_format": "wav",
|
||||||
|
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
||||||
|
"max_new_tokens": 500,
|
||||||
|
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
@pytest.mark.core_model
|
||||||
|
@pytest.mark.tts
|
||||||
|
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||||
|
def test_concurrent_plain_text(self, omni_server, openai_client) -> None:
|
||||||
|
"""Three concurrent non-streaming requests - guards per-slot audio state and Stage-0 PC under batching."""
|
||||||
|
openai_client.send_audio_speech_request(
|
||||||
|
{
|
||||||
|
"model": omni_server.model,
|
||||||
|
"input": "It was the night before my birthday.",
|
||||||
|
"stream": False,
|
||||||
|
"response_format": "wav",
|
||||||
|
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
||||||
|
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
||||||
|
},
|
||||||
|
request_num=3,
|
||||||
|
)
|
||||||
|
|
||||||
|
@pytest.mark.core_model
|
||||||
|
@pytest.mark.tts
|
||||||
|
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||||
|
def test_plain_text_pcm_streaming(self, omni_server, openai_client) -> None:
|
||||||
|
"""Streaming PCM via talker2code2wav_async_chunk + the connector's codec_streaming path.
|
||||||
|
|
||||||
|
The deploy YAML pins ``async_chunk: true`` and ``codec_streaming: true`` so chunks
|
||||||
|
flow Stage 0 -> Stage 1 per AR loop. The Stage-1 codec honors
|
||||||
|
``meta.left_context_size`` and ``meta.right_holdback_size`` so per-chunk codec
|
||||||
|
windows stitch into a coherent PCM stream. The byte-count gate is the same as
|
||||||
|
the sync paths; per-chunk audio content is verified offline against Whisper.
|
||||||
|
|
||||||
|
NOTE: ``min_hnr_db=0.0`` sits below the typical speech-noise floor so the
|
||||||
|
check still catches catastrophic codec failure (silence, white noise, sample
|
||||||
|
scramble all give HNR << 0) while allowing for the sliding-window codec's
|
||||||
|
slightly-noisier-than-sync output. The default 1.0 dB threshold has only
|
||||||
|
~0.16 dB margin over measured single-request output (1.16 dB on L4), so
|
||||||
|
flake risk is too high there.
|
||||||
|
"""
|
||||||
|
openai_client.send_audio_speech_request(
|
||||||
|
{
|
||||||
|
"model": omni_server.model,
|
||||||
|
"input": "The quick brown fox jumps over the lazy dog.",
|
||||||
|
"stream": True,
|
||||||
|
"response_format": "pcm",
|
||||||
|
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
||||||
|
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
||||||
|
"min_hnr_db": 0.0,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
@pytest.mark.core_model
|
||||||
|
@pytest.mark.tts
|
||||||
|
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||||
|
def test_concurrent_pcm_streaming(self, omni_server, openai_client) -> None:
|
||||||
|
"""Three concurrent streaming requests - guards per-request frame cursors
|
||||||
|
in ``talker2code2wav_async_chunk`` and per-slot delay-pattern state under batched AR.
|
||||||
|
|
||||||
|
NOTE: ``min_hnr_db=-2.0`` is looser than the single-request streaming test
|
||||||
|
because batched AR adds per-request codec quality variance (DAC boundary
|
||||||
|
artifacts compounded across the 3-way batch); -2.0 dB gives ~1.6 dB margin
|
||||||
|
over the measured worst-of-3 on L4 (-0.42 dB) while still well above the
|
||||||
|
catastrophic-failure region.
|
||||||
|
"""
|
||||||
|
openai_client.send_audio_speech_request(
|
||||||
|
{
|
||||||
|
"model": omni_server.model,
|
||||||
|
"input": "She sells seashells by the seashore.",
|
||||||
|
"stream": True,
|
||||||
|
"response_format": "pcm",
|
||||||
|
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
||||||
|
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
||||||
|
"min_hnr_db": -2.0,
|
||||||
|
},
|
||||||
|
request_num=3,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
|
||||||
|
class TestHiggsAudioV3OnlineInlineControlTokens:
|
||||||
|
"""Inline control-token surface from the bosonai cookbook.
|
||||||
|
|
||||||
|
The model exposes four categories of ``<|category:value|>`` tokens
|
||||||
|
(emotion, style, prosody, sfx). Delivery tokens (emotion / style /
|
||||||
|
prosody speed-pitch-expressive) go at the START of input; positional
|
||||||
|
tokens (pause, sfx) go inline. SFX tokens must pair with their written
|
||||||
|
onomatopoeia.
|
||||||
|
|
||||||
|
These tests check the *serving surface* - the validator accepts the
|
||||||
|
payload, the engine produces audio, and the WAV has non-trivial size.
|
||||||
|
They do not assert audio quality (out of scope for CI).
|
||||||
|
"""
|
||||||
|
|
||||||
|
@pytest.mark.core_model
|
||||||
|
@pytest.mark.tts
|
||||||
|
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||||
|
def test_inline_emotion_and_expressive(self, omni_server, openai_client) -> None:
|
||||||
|
"""Delivery tokens (emotion + expressive_high) at the start of input."""
|
||||||
|
openai_client.send_audio_speech_request(
|
||||||
|
{
|
||||||
|
"model": omni_server.model,
|
||||||
|
"input": ("<|emotion:amusement|><|prosody:expressive_high|>Wait, that was actually hilarious."),
|
||||||
|
"stream": False,
|
||||||
|
"response_format": "wav",
|
||||||
|
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
||||||
|
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
@pytest.mark.core_model
|
||||||
|
@pytest.mark.tts
|
||||||
|
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||||
|
def test_inline_style_whispering(self, omni_server, openai_client) -> None:
|
||||||
|
"""Style token at the start - ``whispering``."""
|
||||||
|
openai_client.send_audio_speech_request(
|
||||||
|
{
|
||||||
|
"model": omni_server.model,
|
||||||
|
"input": "<|style:whispering|>It is just between you and me, alright?",
|
||||||
|
"stream": False,
|
||||||
|
"response_format": "wav",
|
||||||
|
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
||||||
|
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
@pytest.mark.core_model
|
||||||
|
@pytest.mark.tts
|
||||||
|
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||||
|
def test_inline_prosody_speed_and_pitch(self, omni_server, openai_client) -> None:
|
||||||
|
"""Two prosody tokens at the start - slow speed plus low pitch."""
|
||||||
|
openai_client.send_audio_speech_request(
|
||||||
|
{
|
||||||
|
"model": omni_server.model,
|
||||||
|
"input": (
|
||||||
|
"<|prosody:speed_slow|><|prosody:pitch_low|>The radar shows a storm approaching from the east."
|
||||||
|
),
|
||||||
|
"stream": False,
|
||||||
|
"response_format": "wav",
|
||||||
|
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
||||||
|
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
@pytest.mark.core_model
|
||||||
|
@pytest.mark.tts
|
||||||
|
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||||
|
def test_inline_pause_mid_text(self, omni_server, openai_client) -> None:
|
||||||
|
"""Positional pause token placed inline between two clauses."""
|
||||||
|
openai_client.send_audio_speech_request(
|
||||||
|
{
|
||||||
|
"model": omni_server.model,
|
||||||
|
"input": "Hold on a moment <|prosody:pause|> let me think about it.",
|
||||||
|
"stream": False,
|
||||||
|
"response_format": "wav",
|
||||||
|
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
||||||
|
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
@pytest.mark.core_model
|
||||||
|
@pytest.mark.tts
|
||||||
|
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||||
|
def test_inline_sfx_with_onomatopoeia(self, omni_server, openai_client) -> None:
|
||||||
|
"""SFX token paired with its written onomatopoeia (``<|sfx:laughter|>Hehe``)."""
|
||||||
|
openai_client.send_audio_speech_request(
|
||||||
|
{
|
||||||
|
"model": omni_server.model,
|
||||||
|
"input": (
|
||||||
|
"<|emotion:amusement|>I cannot believe that just happened. "
|
||||||
|
"<|sfx:laughter|>Hehe, I am still recovering from it."
|
||||||
|
),
|
||||||
|
"stream": False,
|
||||||
|
"response_format": "wav",
|
||||||
|
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
||||||
|
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
|
||||||
|
class TestHiggsAudioV3OnlineVoiceClone:
|
||||||
|
"""Voice clone via the two payload shapes the model serves."""
|
||||||
|
|
||||||
|
@pytest.mark.core_model
|
||||||
|
@pytest.mark.tts
|
||||||
|
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||||
|
def test_voice_clone_ref_audio_ref_text(self, omni_server, openai_client) -> None:
|
||||||
|
"""Canonical vllm-omni voice clone via ``ref_audio`` + ``ref_text``."""
|
||||||
|
openai_client.send_audio_speech_request(
|
||||||
|
{
|
||||||
|
"model": omni_server.model,
|
||||||
|
"input": "Hello world.",
|
||||||
|
"ref_audio": _REF_AUDIO_URL,
|
||||||
|
"ref_text": _REF_TEXT,
|
||||||
|
"stream": False,
|
||||||
|
"response_format": "wav",
|
||||||
|
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
||||||
|
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
@pytest.mark.core_model
|
||||||
|
@pytest.mark.tts
|
||||||
|
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||||
|
def test_voice_clone_references_alias(self, omni_server, openai_client) -> None:
|
||||||
|
"""BosonAI cookbook payload: ``references=[{audio_path, text}]``.
|
||||||
|
|
||||||
|
``normalize_references_alias`` (``protocol/audio.py``) is supposed to
|
||||||
|
translate the cookbook field into ``ref_audio`` / ``ref_text``; without
|
||||||
|
it the request would silently fall through to zero-shot synthesis.
|
||||||
|
"""
|
||||||
|
openai_client.send_audio_speech_request(
|
||||||
|
{
|
||||||
|
"model": omni_server.model,
|
||||||
|
"input": "Hello world.",
|
||||||
|
"references": [{"audio_path": _REF_AUDIO_URL, "text": _REF_TEXT}],
|
||||||
|
"stream": False,
|
||||||
|
"response_format": "wav",
|
||||||
|
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
||||||
|
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
|
||||||
|
class TestHiggsAudioV3OnlineValidatorRejections:
|
||||||
|
"""Out-of-scope shapes must come back as 4xx.
|
||||||
|
|
||||||
|
These cases use ``send_audio_speech_http_request`` (raw HTTP POST against
|
||||||
|
``/v1/audio/speech``) rather than the OpenAI SDK helper because:
|
||||||
|
|
||||||
|
- We need the failure-path matcher; the SDK helper asserts success.
|
||||||
|
- The cookbook ``references`` field is not in the OpenAI SDK schema, so
|
||||||
|
the SDK strips it before the request leaves the client and the server
|
||||||
|
sees a plain TTS request that returns 200. Raw HTTP keeps the JSON body
|
||||||
|
intact so the validator actually fires.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@pytest.mark.core_model
|
||||||
|
@pytest.mark.tts
|
||||||
|
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||||
|
def test_rejects_empty_input(self, omni_server, openai_client) -> None:
|
||||||
|
"""Empty text input - validator should reject before reaching the engine."""
|
||||||
|
openai_client.send_audio_speech_http_request(
|
||||||
|
{
|
||||||
|
"json": {
|
||||||
|
"model": omni_server.model,
|
||||||
|
"input": "",
|
||||||
|
"response_format": "wav",
|
||||||
|
},
|
||||||
|
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
||||||
|
"err_code": (400, 422),
|
||||||
|
"err_message": ("input", "empty"),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
@pytest.mark.core_model
|
||||||
|
@pytest.mark.tts
|
||||||
|
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||||
|
def test_rejects_multi_reference_payload(self, omni_server, openai_client) -> None:
|
||||||
|
"""``references[]`` with more than one entry - multi-shot voice clone is not supported."""
|
||||||
|
openai_client.send_audio_speech_http_request(
|
||||||
|
{
|
||||||
|
"json": {
|
||||||
|
"model": omni_server.model,
|
||||||
|
"input": "Hello world.",
|
||||||
|
"references": [
|
||||||
|
{"audio_path": _REF_AUDIO_URL, "text": _REF_TEXT},
|
||||||
|
{"audio_path": _REF_AUDIO_URL, "text": _REF_TEXT},
|
||||||
|
],
|
||||||
|
"response_format": "wav",
|
||||||
|
},
|
||||||
|
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
||||||
|
"err_code": (400, 422),
|
||||||
|
"err_message": "references",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
@pytest.mark.core_model
|
||||||
|
@pytest.mark.tts
|
||||||
|
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||||
|
def test_rejects_conflicting_ref_audio_and_references(self, omni_server, openai_client) -> None:
|
||||||
|
"""``ref_audio`` and ``references`` cannot both be set to different values."""
|
||||||
|
openai_client.send_audio_speech_http_request(
|
||||||
|
{
|
||||||
|
"json": {
|
||||||
|
"model": omni_server.model,
|
||||||
|
"input": "Hello world.",
|
||||||
|
"ref_audio": _REF_AUDIO_URL,
|
||||||
|
"references": [{"audio_path": "https://example.com/other.wav"}],
|
||||||
|
"response_format": "wav",
|
||||||
|
},
|
||||||
|
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
||||||
|
"err_code": (400, 422),
|
||||||
|
"err_message": "mutually exclusive",
|
||||||
|
}
|
||||||
|
)
|
||||||
@@ -30,7 +30,8 @@ from tests.helpers.mark import hardware_test
|
|||||||
from tests.helpers.runtime import OmniServerParams
|
from tests.helpers.runtime import OmniServerParams
|
||||||
from tests.helpers.stage_config import get_deploy_config_path
|
from tests.helpers.stage_config import get_deploy_config_path
|
||||||
|
|
||||||
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
# TODO: Fix this test
|
||||||
|
# pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
||||||
|
|
||||||
MODEL = "OpenMOSS-Team/MOSS-TTS-Realtime"
|
MODEL = "OpenMOSS-Team/MOSS-TTS-Realtime"
|
||||||
REF_AUDIO_URL = "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS/HEAD/assets/audio/reference_zh_1.wav"
|
REF_AUDIO_URL = "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS/HEAD/assets/audio/reference_zh_1.wav"
|
||||||
|
|||||||
@@ -9,63 +9,40 @@ import pytest
|
|||||||
from tests.helpers.mark import hardware_test
|
from tests.helpers.mark import hardware_test
|
||||||
from tests.helpers.media import generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video
|
from tests.helpers.media import generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video
|
||||||
from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data
|
from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data
|
||||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
from tests.helpers.stage_config import get_deploy_config_path
|
||||||
from vllm_omni.platforms import current_omni_platform
|
|
||||||
|
|
||||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||||
|
|
||||||
|
|
||||||
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
|
|
||||||
|
|
||||||
# Set VLLM_TEST_PD_MODE=1 to test PD disaggregation (follow-up — deploy overlay not yet migrated).
|
# Set VLLM_TEST_PD_MODE=1 to test PD disaggregation (follow-up — deploy overlay not yet migrated).
|
||||||
_USE_PD = os.environ.get("VLLM_TEST_PD_MODE", "0") == "1"
|
_USE_PD = os.environ.get("VLLM_TEST_PD_MODE", "0") == "1"
|
||||||
|
|
||||||
|
_MODEL = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
|
||||||
_CI_DEPLOY = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
|
_CI_DEPLOY = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
|
||||||
|
|
||||||
|
|
||||||
def get_chunk_config(config_path: str | None = None):
|
# For prefix caching checks against we enable it on the thinker and talker via CLI override
|
||||||
"""Load the qwen3_omni CI deploy yaml with async_chunk modifications for streaming mode."""
|
|
||||||
if config_path is None:
|
|
||||||
config_path = _CI_DEPLOY
|
|
||||||
# TODO: remove this workaround once legacy `stage_args` path is deleted.
|
|
||||||
# The pipeline (qwen3_omni/pipeline.py) already wires
|
|
||||||
# thinker2talker_async_chunk / talker2code2wav_async_chunk on stage 0/1,
|
|
||||||
# so only async_chunk needs flipping. Writing nested `engine_args:` into
|
|
||||||
# the new-schema overlay trips _parse_stage_deploy's legacy branch and
|
|
||||||
# drops flat fields (load_format, max_num_seqs, ...).
|
|
||||||
return modify_stage_config(config_path, updates={"async_chunk": True})
|
|
||||||
|
|
||||||
|
|
||||||
# Platform-specific overrides live inside the new deploy yaml's ``platforms:``
|
|
||||||
# section, so a single ``_CI_DEPLOY`` path serves CUDA, ROCm, and XPU.
|
|
||||||
# TODO: re-add VLLM_TEST_PD_MODE branch once the PD-disaggregation deploy
|
|
||||||
# overlay has been migrated to the new schema (previously used the deleted
|
|
||||||
# ``qwen3_omni_moe_pd_ci.yaml`` stage-configs file).
|
|
||||||
if current_omni_platform.is_xpu():
|
|
||||||
stage_configs = [_CI_DEPLOY]
|
|
||||||
else: # CUDA + ROCm MI325 share the same deploy config
|
|
||||||
stage_configs = [get_chunk_config()]
|
|
||||||
|
|
||||||
# Create parameter combinations for model and stage config
|
|
||||||
test_params = [
|
|
||||||
OmniServerParams(model=model, stage_config_path=stage_config) for model in models for stage_config in stage_configs
|
|
||||||
]
|
|
||||||
# For prefix caching, we enable it on the thinker (stage 0) via CLI override
|
|
||||||
# and enable prompt token details so that we can determine if any tokens were cached.
|
# and enable prompt token details so that we can determine if any tokens were cached.
|
||||||
|
# We also explicitly set block size so that we can make sure the cached token counts are a
|
||||||
|
# multiple of the block size.
|
||||||
BLOCK_SIZE = 16
|
BLOCK_SIZE = 16
|
||||||
prefix_test_params = [
|
test_params = [
|
||||||
OmniServerParams(
|
pytest.param(
|
||||||
model=model,
|
OmniServerParams(
|
||||||
stage_config_path=_CI_DEPLOY,
|
model=_MODEL,
|
||||||
server_args=[
|
stage_config_path=_CI_DEPLOY,
|
||||||
"--block-size",
|
use_stage_cli=True,
|
||||||
str(BLOCK_SIZE),
|
server_args=[
|
||||||
"--stage-overrides",
|
"--no-async-chunk",
|
||||||
'{"0": {"enable_prefix_caching": true}}',
|
"--block-size",
|
||||||
"--enable-prompt-tokens-details",
|
str(BLOCK_SIZE),
|
||||||
],
|
"--stage-overrides",
|
||||||
|
'{"0": {"enable_prefix_caching": true}, "1": {"enable_prefix_caching": true}}',
|
||||||
|
"--enable-prompt-tokens-details",
|
||||||
|
],
|
||||||
|
),
|
||||||
|
id="default",
|
||||||
)
|
)
|
||||||
for model in models
|
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
@@ -166,45 +143,14 @@ def test_text_to_text_001(omni_server, openai_client) -> None:
|
|||||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
def _run_prefix_cache_check(openai_client, request_config: dict):
|
||||||
@pytest.mark.core_model
|
"""Make two requests given a request config, and validate that:
|
||||||
@pytest.mark.omni
|
1. The second request actually had cached tokens
|
||||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
2. The number of cached tokens is divisible by the block size used in
|
||||||
@pytest.mark.parametrize("omni_server", prefix_test_params, indirect=True)
|
test_params, because currently upstream vLLM does not cache partial
|
||||||
def test_thinker_prefix_caching(omni_server, openai_client, run_level) -> None:
|
blocks.
|
||||||
"""
|
"""
|
||||||
Test thinker prefix caching by sending identical requests with an image (i.e.,
|
openai_client.send_omni_request(request_config, request_num=1)[0]
|
||||||
a large shared prefix) and verifying that the second request uses cached tokens
|
|
||||||
& produces the same output with greedy decoding.
|
|
||||||
|
|
||||||
NOTE: The reason that we check against logprobs instead of direct text here is that
|
|
||||||
the outputs may still diverge a bit even though we set the seed and temperature.
|
|
||||||
This is mostly because the GEMM algorithm may vary based on the input tensors dims.
|
|
||||||
Because of this, we don't check the logprobs if it's a dummy load, since in that case
|
|
||||||
the top logprobs will all be very close.
|
|
||||||
"""
|
|
||||||
seed = 10
|
|
||||||
img_res = generate_synthetic_image(224, 224, seed=seed)
|
|
||||||
image_data_url = f"data:image/jpeg;base64,{img_res['base64']}"
|
|
||||||
messages = dummy_messages_from_mix_data(
|
|
||||||
system_prompt=get_system_prompt(),
|
|
||||||
image_data_url=image_data_url,
|
|
||||||
content_text=get_prompt("text_image"),
|
|
||||||
)
|
|
||||||
|
|
||||||
top_k = 10
|
|
||||||
sampling_params = {"seed": seed, "temperature": 0, "max_tokens": 8, "logprobs": top_k}
|
|
||||||
request_config = {
|
|
||||||
"model": omni_server.model,
|
|
||||||
"messages": messages,
|
|
||||||
"stream": False,
|
|
||||||
"modalities": ["text"],
|
|
||||||
"logprobs": True,
|
|
||||||
"top_logprobs": top_k,
|
|
||||||
"sampling_params_list": [sampling_params] * 3,
|
|
||||||
}
|
|
||||||
|
|
||||||
uncached_response = openai_client.send_omni_request(request_config, request_num=1)[0]
|
|
||||||
cached_response = openai_client.send_omni_request(request_config, request_num=1)[0]
|
cached_response = openai_client.send_omni_request(request_config, request_num=1)[0]
|
||||||
|
|
||||||
# Ensure that we have a prefix cache hit on the second request and that only the last
|
# Ensure that we have a prefix cache hit on the second request and that only the last
|
||||||
@@ -217,16 +163,66 @@ def test_thinker_prefix_caching(omni_server, openai_client, run_level) -> None:
|
|||||||
assert num_cached_tokens % BLOCK_SIZE == 0
|
assert num_cached_tokens % BLOCK_SIZE == 0
|
||||||
assert (num_cached_tokens + num_uncached_tokens) == num_prompt_tokens
|
assert (num_cached_tokens + num_uncached_tokens) == num_prompt_tokens
|
||||||
|
|
||||||
# Ensure that we have logprobs and tokens were generated for both requests
|
|
||||||
assert uncached_response.logprobs is not None
|
|
||||||
assert cached_response.logprobs is not None
|
|
||||||
n_tokens = min(len(uncached_response.logprobs), len(cached_response.logprobs))
|
|
||||||
assert n_tokens > 0
|
|
||||||
|
|
||||||
if run_level == "advanced_model":
|
@pytest.mark.advanced_model
|
||||||
# For each token index where both responses have an output, ensure that the greedy token
|
@pytest.mark.core_model
|
||||||
# predicted in the uncached case is in the top k logprobs for the cached case
|
@pytest.mark.omni
|
||||||
for idx in range(n_tokens):
|
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||||
greedy_token = uncached_response.logprobs[idx].token
|
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||||
cached_top_k = {lp.token for lp in cached_response.logprobs[idx].top_logprobs}
|
def test_thinker_prefix_caching_text_output(omni_server, openai_client) -> None:
|
||||||
assert greedy_token in cached_top_k
|
"""
|
||||||
|
Test thinker prefix caching by sending identical requests with an image (i.e.,
|
||||||
|
a large shared prefix) and verifying that the second request uses cached tokens
|
||||||
|
& produces the same output with greedy decoding.
|
||||||
|
|
||||||
|
NOTE: Checking the output of prefix caching directly can be a bit unstable
|
||||||
|
due to slight numerical differences as a result of running different scheduled
|
||||||
|
sequence lengths. As such, for E2E tests on prefix cache, we only check the cached
|
||||||
|
token count and not the output, since the omni tensor cache has solid unit tests,
|
||||||
|
and the core prefix cache algorithm is largely tested by upstream vLLM.
|
||||||
|
"""
|
||||||
|
img_res = generate_synthetic_image(224, 224)
|
||||||
|
image_data_url = f"data:image/jpeg;base64,{img_res['base64']}"
|
||||||
|
messages = dummy_messages_from_mix_data(
|
||||||
|
system_prompt=get_system_prompt(),
|
||||||
|
image_data_url=image_data_url,
|
||||||
|
content_text=get_prompt("text_image"),
|
||||||
|
)
|
||||||
|
|
||||||
|
request_config = {
|
||||||
|
"model": omni_server.model,
|
||||||
|
"messages": messages,
|
||||||
|
"stream": False,
|
||||||
|
"modalities": ["text"],
|
||||||
|
}
|
||||||
|
_run_prefix_cache_check(openai_client, request_config)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.advanced_model
|
||||||
|
@pytest.mark.core_model
|
||||||
|
@pytest.mark.omni
|
||||||
|
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||||
|
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||||
|
def test_thinker_prefix_caching_audio_output(omni_server, openai_client) -> None:
|
||||||
|
"""
|
||||||
|
Verify that thinker prefix caching does not hang when the request
|
||||||
|
produces audio output (text + audio modalities). Sends two identical
|
||||||
|
requests so the second exercises the prefix-cached path through the
|
||||||
|
full thinker -> talker -> code2wav pipeline.
|
||||||
|
|
||||||
|
Regression test for https://github.com/vllm-project/vllm-omni/issues/3510
|
||||||
|
"""
|
||||||
|
messages = dummy_messages_from_mix_data(
|
||||||
|
system_prompt=get_system_prompt(),
|
||||||
|
content_text=get_prompt(),
|
||||||
|
)
|
||||||
|
request_config = {
|
||||||
|
"model": omni_server.model,
|
||||||
|
"messages": messages,
|
||||||
|
"stream": True,
|
||||||
|
"stream_options": {
|
||||||
|
"include_usage": True,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
_run_prefix_cache_check(openai_client, request_config)
|
||||||
@@ -0,0 +1,97 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""
|
||||||
|
E2E Online tests for VoxCPM2 with text input and audio output.
|
||||||
|
|
||||||
|
These tests verify the /v1/audio/speech endpoint works correctly with
|
||||||
|
actual model inference, not mocks. VoxCPM2 zero-shot needs only ``input``
|
||||||
|
and a placeholder ``voice`` (``default``); no ``task_type`` / ``ref_text``.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from tests.helpers.mark import hardware_test
|
||||||
|
from tests.helpers.runtime import OmniServerParams
|
||||||
|
from tests.helpers.stage_config import get_deploy_config_path
|
||||||
|
|
||||||
|
MODEL = "openbmb/VoxCPM2"
|
||||||
|
DEFAULT_AUDIO_SPEECH_TIMEOUT_S = 300.0
|
||||||
|
MAX_CONCURRENT = 4
|
||||||
|
|
||||||
|
# ~0.5 s of 48 kHz mono PCM_16 in WAV (~48k payload + header).
|
||||||
|
_MIN_AUDIO_BYTES = 40_000
|
||||||
|
|
||||||
|
|
||||||
|
def get_prompt(prompt_type="text"):
|
||||||
|
"""English prompt for zero-shot TTS (Whisper-friendly for advanced_model accuracy)."""
|
||||||
|
prompts = {
|
||||||
|
"text": "The weather is nice today, perfect for a walk in the park.",
|
||||||
|
}
|
||||||
|
return prompts.get(prompt_type, prompts["text"])
|
||||||
|
|
||||||
|
|
||||||
|
tts_server_params = [
|
||||||
|
pytest.param(
|
||||||
|
OmniServerParams(
|
||||||
|
model=MODEL,
|
||||||
|
stage_config_path=get_deploy_config_path("voxcpm2.yaml"),
|
||||||
|
server_args=["--trust-remote-code", "--disable-log-stats"],
|
||||||
|
),
|
||||||
|
id="voxcpm2",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.advanced_model
|
||||||
|
@pytest.mark.core_model
|
||||||
|
@pytest.mark.tts
|
||||||
|
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||||
|
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
||||||
|
def test_text_to_audio_001(omni_server, openai_client) -> None:
|
||||||
|
"""
|
||||||
|
Test zero-shot text-to-audio via OpenAI API.
|
||||||
|
Deploy Setting: voxcpm2.yaml
|
||||||
|
Input Modal: text
|
||||||
|
Output Modal: audio
|
||||||
|
Input Setting: stream=False
|
||||||
|
Datasets: few requests (max_num_seqs=4)
|
||||||
|
"""
|
||||||
|
request_config = {
|
||||||
|
"model": omni_server.model,
|
||||||
|
"input": get_prompt(),
|
||||||
|
"stream": False,
|
||||||
|
"timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S,
|
||||||
|
"response_format": "wav",
|
||||||
|
"voice": "default",
|
||||||
|
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
||||||
|
}
|
||||||
|
openai_client.send_audio_speech_request(request_config, request_num=MAX_CONCURRENT)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.advanced_model
|
||||||
|
@pytest.mark.tts
|
||||||
|
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||||
|
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
||||||
|
def test_text_to_audio_002(omni_server, openai_client) -> None:
|
||||||
|
"""
|
||||||
|
Test zero-shot streaming text-to-audio via OpenAI API.
|
||||||
|
Deploy Setting: voxcpm2.yaml
|
||||||
|
Input Modal: text
|
||||||
|
Output Modal: audio
|
||||||
|
Input Setting: stream=True
|
||||||
|
Datasets: single request
|
||||||
|
"""
|
||||||
|
request_config = {
|
||||||
|
"model": omni_server.model,
|
||||||
|
"input": get_prompt(),
|
||||||
|
"stream": True,
|
||||||
|
"timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S,
|
||||||
|
"response_format": "wav",
|
||||||
|
"voice": "default",
|
||||||
|
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
||||||
|
}
|
||||||
|
openai_client.send_audio_speech_request(request_config)
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""
|
||||||
|
E2E Online expansion tests for VoxCPM2 (voice clone, PCM, concurrency).
|
||||||
|
|
||||||
|
Voice clone uses ``ref_audio`` only (no ``ref_text``). Reference clip is
|
||||||
|
vendored under tests/assets/qwen3_tts/clone_2.wav — see test_qwen3_tts_base.py.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from tests.helpers.mark import hardware_test
|
||||||
|
from tests.helpers.media import load_test_audio_data_url
|
||||||
|
from tests.helpers.runtime import OmniServerParams
|
||||||
|
from tests.helpers.stage_config import get_deploy_config_path
|
||||||
|
|
||||||
|
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
||||||
|
|
||||||
|
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||||
|
|
||||||
|
MODEL = "openbmb/VoxCPM2"
|
||||||
|
DEFAULT_AUDIO_SPEECH_TIMEOUT_S = 300.0
|
||||||
|
_MIN_AUDIO_BYTES = 40_000
|
||||||
|
MAX_CONCURRENT = 4
|
||||||
|
|
||||||
|
REF_AUDIO_URL = load_test_audio_data_url("qwen3_tts/clone_2.wav")
|
||||||
|
|
||||||
|
|
||||||
|
def get_prompt(prompt_type="text"):
|
||||||
|
prompts = {
|
||||||
|
"text": "The weather is nice today, perfect for a walk in the park.",
|
||||||
|
}
|
||||||
|
return prompts.get(prompt_type, prompts["text"])
|
||||||
|
|
||||||
|
|
||||||
|
tts_server_params = [
|
||||||
|
pytest.param(
|
||||||
|
OmniServerParams(
|
||||||
|
model=MODEL,
|
||||||
|
stage_config_path=get_deploy_config_path("voxcpm2.yaml"),
|
||||||
|
server_args=["--trust-remote-code", "--disable-log-stats"],
|
||||||
|
),
|
||||||
|
id="voxcpm2",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||||
|
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
||||||
|
def test_voice_clone_streaming_001(omni_server, openai_client) -> None:
|
||||||
|
"""
|
||||||
|
Test voice-clone text-to-audio via OpenAI API.
|
||||||
|
Deploy Setting: voxcpm2.yaml
|
||||||
|
Input Modal: text + ref_audio
|
||||||
|
Output Modal: audio
|
||||||
|
Input Setting: stream=True
|
||||||
|
Datasets: few requests (max_num_seqs=4)
|
||||||
|
"""
|
||||||
|
request_config = {
|
||||||
|
"model": omni_server.model,
|
||||||
|
"input": get_prompt(),
|
||||||
|
"stream": True,
|
||||||
|
"timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S,
|
||||||
|
"response_format": "wav",
|
||||||
|
"voice": "default",
|
||||||
|
"ref_audio": REF_AUDIO_URL,
|
||||||
|
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
||||||
|
}
|
||||||
|
openai_client.send_audio_speech_request(request_config, request_num=MAX_CONCURRENT)
|
||||||
|
|
||||||
|
|
||||||
|
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||||
|
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
||||||
|
def test_response_format_001(omni_server, openai_client) -> None:
|
||||||
|
"""
|
||||||
|
Test voice-clone non-stream PCM output via OpenAI API.
|
||||||
|
Deploy Setting: voxcpm2.yaml
|
||||||
|
Input Modal: text + ref_audio
|
||||||
|
Output Modal: audio (pcm)
|
||||||
|
Input Setting: stream=False
|
||||||
|
Datasets: single request
|
||||||
|
"""
|
||||||
|
request_config = {
|
||||||
|
"model": omni_server.model,
|
||||||
|
"input": get_prompt(),
|
||||||
|
"response_format": "pcm",
|
||||||
|
"stream": False,
|
||||||
|
"timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S,
|
||||||
|
"voice": "default",
|
||||||
|
"ref_audio": REF_AUDIO_URL,
|
||||||
|
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
||||||
|
}
|
||||||
|
openai_client.send_audio_speech_request(request_config)
|
||||||
@@ -427,6 +427,7 @@ async def _enqueue_add_request(
|
|||||||
sampling_params_list=sampling_params_list,
|
sampling_params_list=sampling_params_list,
|
||||||
final_stage_id=final_stage_id,
|
final_stage_id=final_stage_id,
|
||||||
preprocess_ms=0.0,
|
preprocess_ms=0.0,
|
||||||
|
request_timestamp=time.time(),
|
||||||
enqueue_ts=time.perf_counter(),
|
enqueue_ts=time.perf_counter(),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
@@ -984,6 +985,7 @@ async def test_handle_streaming_update_passes_prompt_text_to_stage_pool() -> Non
|
|||||||
sampling_params_list=[_sampling_params()],
|
sampling_params_list=[_sampling_params()],
|
||||||
final_stage_id=0,
|
final_stage_id=0,
|
||||||
preprocess_ms=0.0,
|
preprocess_ms=0.0,
|
||||||
|
request_timestamp=time.time(),
|
||||||
enqueue_ts=time.perf_counter(),
|
enqueue_ts=time.perf_counter(),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -156,7 +156,7 @@ class FakeAsyncOmni:
|
|||||||
self.captured_prompt = None
|
self.captured_prompt = None
|
||||||
self._images = images or [Image.new("RGB", (64, 64), color="green")]
|
self._images = images or [Image.new("RGB", (64, 64), color="green")]
|
||||||
|
|
||||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None):
|
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
||||||
if sampling_params_list is not None:
|
if sampling_params_list is not None:
|
||||||
self.captured_sampling_params_list = sampling_params_list
|
self.captured_sampling_params_list = sampling_params_list
|
||||||
else:
|
else:
|
||||||
@@ -254,7 +254,7 @@ def async_omni_test_client():
|
|||||||
self._images = [Image.new("RGB", (64, 64), color="green")]
|
self._images = [Image.new("RGB", (64, 64), color="green")]
|
||||||
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
||||||
|
|
||||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None):
|
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
||||||
if sampling_params_list is not None:
|
if sampling_params_list is not None:
|
||||||
self.captured_sampling_params_list = sampling_params_list
|
self.captured_sampling_params_list = sampling_params_list
|
||||||
else:
|
else:
|
||||||
@@ -318,7 +318,7 @@ def async_omni_rgba_test_client():
|
|||||||
self._images = [Image.new("RGBA", (64, 64), color=(0, 255, 0, 128))]
|
self._images = [Image.new("RGBA", (64, 64), color=(0, 255, 0, 128))]
|
||||||
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
||||||
|
|
||||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None):
|
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
||||||
if sampling_params_list is not None:
|
if sampling_params_list is not None:
|
||||||
self.captured_sampling_params_list = sampling_params_list
|
self.captured_sampling_params_list = sampling_params_list
|
||||||
else:
|
else:
|
||||||
@@ -382,7 +382,7 @@ def async_omni_stage_configs_only_client():
|
|||||||
self._images = [Image.new("RGB", (64, 64), color="green")]
|
self._images = [Image.new("RGB", (64, 64), color="green")]
|
||||||
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
||||||
|
|
||||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None):
|
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
||||||
if sampling_params_list is not None:
|
if sampling_params_list is not None:
|
||||||
self.captured_sampling_params_list = sampling_params_list
|
self.captured_sampling_params_list = sampling_params_list
|
||||||
else:
|
else:
|
||||||
@@ -442,7 +442,7 @@ def streaming_image_edit_client():
|
|||||||
self.captured_prompt = None
|
self.captured_prompt = None
|
||||||
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
||||||
|
|
||||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None):
|
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
||||||
self.captured_prompt = prompt
|
self.captured_prompt = prompt
|
||||||
self.captured_sampling_params_list = sampling_params_list or [sampling_params]
|
self.captured_sampling_params_list = sampling_params_list or [sampling_params]
|
||||||
assert self.captured_sampling_params_list[0].output_kind == RequestOutputKind.DELTA
|
assert self.captured_sampling_params_list[0].output_kind == RequestOutputKind.DELTA
|
||||||
@@ -696,7 +696,7 @@ def test_generate_images_async_omni_glm_image_sets_stage0_max_tokens():
|
|||||||
self._images = [Image.new("RGB", (64, 64), color="green")]
|
self._images = [Image.new("RGB", (64, 64), color="green")]
|
||||||
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
||||||
|
|
||||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None):
|
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
||||||
self.captured_sampling_params_list = (
|
self.captured_sampling_params_list = (
|
||||||
sampling_params_list if sampling_params_list is not None else [sampling_params]
|
sampling_params_list if sampling_params_list is not None else [sampling_params]
|
||||||
)
|
)
|
||||||
@@ -815,7 +815,9 @@ def test_image_edits_streaming_returns_ar_delta_then_image(streaming_image_edit_
|
|||||||
|
|
||||||
|
|
||||||
def test_image_edits_streaming_ar_delta_chunks_include_index(streaming_image_edit_client):
|
def test_image_edits_streaming_ar_delta_chunks_include_index(streaming_image_edit_client):
|
||||||
async def generate_multi_output_delta(prompt, request_id, sampling_params=None, sampling_params_list=None):
|
async def generate_multi_output_delta(
|
||||||
|
prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs
|
||||||
|
):
|
||||||
yield MockStageResult(stage_id=0, final_output_type="text", texts=["first", "second"])
|
yield MockStageResult(stage_id=0, final_output_type="text", texts=["first", "second"])
|
||||||
yield MockStageResult(
|
yield MockStageResult(
|
||||||
stage_id=1,
|
stage_id=1,
|
||||||
@@ -843,7 +845,7 @@ def test_image_edits_streaming_ar_delta_chunks_include_index(streaming_image_edi
|
|||||||
|
|
||||||
|
|
||||||
def test_image_edits_streaming_errors_without_final_image(streaming_image_edit_client):
|
def test_image_edits_streaming_errors_without_final_image(streaming_image_edit_client):
|
||||||
async def generate_without_image(prompt, request_id, sampling_params=None, sampling_params_list=None):
|
async def generate_without_image(prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
||||||
yield MockStageResult(stage_id=0, final_output_type="text", text="recap")
|
yield MockStageResult(stage_id=0, final_output_type="text", text="recap")
|
||||||
|
|
||||||
streaming_image_edit_client.app.state.engine_client.generate = generate_without_image
|
streaming_image_edit_client.app.state.engine_client.generate = generate_without_image
|
||||||
@@ -866,7 +868,7 @@ def test_image_edits_streaming_errors_without_final_image(streaming_image_edit_c
|
|||||||
|
|
||||||
|
|
||||||
def test_image_edits_streaming_errors_on_empty_final_image(streaming_image_edit_client):
|
def test_image_edits_streaming_errors_on_empty_final_image(streaming_image_edit_client):
|
||||||
async def generate_empty_image(prompt, request_id, sampling_params=None, sampling_params_list=None):
|
async def generate_empty_image(prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
||||||
yield MockStageResult(stage_id=0, final_output_type="text", text="recap")
|
yield MockStageResult(stage_id=0, final_output_type="text", text="recap")
|
||||||
yield MockStageResult(stage_id=1, final_output_type="image", images=[])
|
yield MockStageResult(stage_id=1, final_output_type="image", images=[])
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,11 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
"""
|
"""
|
||||||
E2E online tests for Qwen3-Omni /v1/realtime WebSocket (streaming PCM in, audio out).
|
E2E online tests for Qwen3-Omni /v1/realtime WebSocket (streaming PCM in, audio out).
|
||||||
|
|
||||||
|
Three scenarios:
|
||||||
|
- Ready CI: async_chunk on, smoke only (no send delay, no accuracy check).
|
||||||
|
- Merge CI: async_chunk on + send delay, full accuracy check.
|
||||||
|
- Merge CI: async_chunk off, no send delay, full accuracy check.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -34,18 +39,32 @@ REALTIME_SYNTH_PHRASE_TEXT = (
|
|||||||
"Translate into Chinese: Beijing is the Capital of China. It is the center of culture and politics"
|
"Translate into Chinese: Beijing is the Capital of China. It is the center of culture and politics"
|
||||||
)
|
)
|
||||||
|
|
||||||
# The new-schema CI overlay bakes in async_chunk: False and covers CUDA/ROCm/XPU
|
# Simulate realtime upload pacing (``openai_realtime_client.py --send-delay-ms``).
|
||||||
# via its ``platforms:`` section, so one path serves all three.
|
SEND_DELAY_MS = 200
|
||||||
|
|
||||||
|
# CI overlay bakes in async_chunk: False and covers CUDA/ROCm/XPU via ``platforms:``.
|
||||||
default_stage_config = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
|
default_stage_config = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
|
||||||
|
|
||||||
realtime_server_params = [
|
realtime_sync_server_params = [
|
||||||
pytest.param(
|
pytest.param(
|
||||||
OmniServerParams(
|
OmniServerParams(
|
||||||
model=MODEL,
|
model=MODEL,
|
||||||
stage_config_path=default_stage_config,
|
stage_config_path=default_stage_config,
|
||||||
use_stage_cli=True,
|
use_stage_cli=True,
|
||||||
),
|
),
|
||||||
id="default",
|
id="sync",
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
realtime_async_chunk_server_params = [
|
||||||
|
pytest.param(
|
||||||
|
OmniServerParams(
|
||||||
|
model=MODEL,
|
||||||
|
stage_config_path=default_stage_config,
|
||||||
|
use_stage_cli=True,
|
||||||
|
server_args=["--async-chunk"],
|
||||||
|
),
|
||||||
|
id="async_chunk",
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -80,6 +99,7 @@ async def _run_realtime_audio_roundtrip(
|
|||||||
pcm16: bytes,
|
pcm16: bytes,
|
||||||
*,
|
*,
|
||||||
chunk_ms: int = 100,
|
chunk_ms: int = 100,
|
||||||
|
send_delay_ms: int = 0,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
uri = f"ws://{host}:{port}/v1/realtime"
|
uri = f"ws://{host}:{port}/v1/realtime"
|
||||||
incremental: list[bytes] = []
|
incremental: list[bytes] = []
|
||||||
@@ -105,6 +125,8 @@ async def _run_realtime_audio_roundtrip(
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
if send_delay_ms > 0:
|
||||||
|
await asyncio.sleep(send_delay_ms / 1000.0)
|
||||||
|
|
||||||
await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True}))
|
await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True}))
|
||||||
|
|
||||||
@@ -156,26 +178,49 @@ async def _run_realtime_audio_roundtrip(
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _synthetic_pcm16_input() -> bytes:
|
||||||
|
syn = generate_synthetic_audio(
|
||||||
|
10,
|
||||||
|
1,
|
||||||
|
sample_rate=16000,
|
||||||
|
phrase_text=REALTIME_SYNTH_PHRASE_TEXT,
|
||||||
|
)
|
||||||
|
wav_bytes = base64.b64decode(syn["base64"])
|
||||||
|
return _pcm16_mono_16k_from_wav_bytes(wav_bytes)
|
||||||
|
|
||||||
|
|
||||||
|
def _assert_realtime_smoke(result: dict) -> None:
|
||||||
|
out_pcm = result["output_pcm"]
|
||||||
|
assert result["delta_events"] >= 1
|
||||||
|
assert out_pcm, "No output PCM from response.audio.delta"
|
||||||
|
assert len(out_pcm) % 2 == 0
|
||||||
|
assert len(out_pcm) >= 4096, "Output audio unexpectedly small"
|
||||||
|
assert result["output_sample_rate"] > 0
|
||||||
|
|
||||||
|
|
||||||
|
def _assert_realtime_accuracy(result: dict) -> None:
|
||||||
|
final_text = (result["transcription_text"] or "").strip()
|
||||||
|
assert final_text, "Expected non-empty transcription (model text stream)"
|
||||||
|
|
||||||
|
wav_out = _wav_bytes_from_pcm16(result["output_pcm"], result["output_sample_rate"])
|
||||||
|
whisper_text = convert_audio_bytes_to_text(wav_out).strip()
|
||||||
|
assert whisper_text, "Whisper returned empty string for synthesized output audio"
|
||||||
|
|
||||||
|
sim = cosine_similarity_text(whisper_text.lower(), final_text.lower())
|
||||||
|
assert sim > 0.8, (
|
||||||
|
f"Output audio transcript should match model text (sim={sim:.3f}): "
|
||||||
|
f"whisper={whisper_text!r}, model_text={final_text!r}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class TestQwen3OmniRealtimeWebSocket:
|
class TestQwen3OmniRealtimeWebSocket:
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.core_model
|
||||||
@pytest.mark.omni
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||||
@pytest.mark.parametrize("omni_server", realtime_server_params, indirect=True)
|
@pytest.mark.parametrize("omni_server", realtime_async_chunk_server_params, indirect=True)
|
||||||
def test_streaming_audio_input_pcm_output(self, omni_server) -> None:
|
def test_streaming_audio_input_pcm_output_async_chunk_smoke(self, omni_server) -> None:
|
||||||
"""
|
"""Ready CI: async_chunk on, no send delay, smoke only."""
|
||||||
Short streamed 16 kHz mono PCM16 input; expect streamed PCM16 audio deltas and
|
pcm16 = _synthetic_pcm16_input()
|
||||||
transcription. Verify Whisper(output audio) aligns with model text (same idea
|
|
||||||
as multimodal omni e2e). Input speech is synthesized from
|
|
||||||
``REALTIME_SYNTH_PHRASE_TEXT``.
|
|
||||||
"""
|
|
||||||
syn = generate_synthetic_audio(
|
|
||||||
10,
|
|
||||||
1,
|
|
||||||
sample_rate=16000,
|
|
||||||
phrase_text=REALTIME_SYNTH_PHRASE_TEXT,
|
|
||||||
)
|
|
||||||
wav_bytes = base64.b64decode(syn["base64"])
|
|
||||||
pcm16 = _pcm16_mono_16k_from_wav_bytes(wav_bytes)
|
|
||||||
|
|
||||||
result = asyncio.run(
|
result = asyncio.run(
|
||||||
_run_realtime_audio_roundtrip(
|
_run_realtime_audio_roundtrip(
|
||||||
@@ -184,25 +229,52 @@ class TestQwen3OmniRealtimeWebSocket:
|
|||||||
omni_server.model,
|
omni_server.model,
|
||||||
pcm16,
|
pcm16,
|
||||||
chunk_ms=100,
|
chunk_ms=100,
|
||||||
|
send_delay_ms=0,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
out_pcm = result["output_pcm"]
|
_assert_realtime_smoke(result)
|
||||||
assert result["delta_events"] >= 1
|
|
||||||
assert out_pcm, "No output PCM from response.audio.delta"
|
|
||||||
assert len(out_pcm) % 2 == 0
|
|
||||||
assert len(out_pcm) >= 4096, "Output audio unexpectedly small"
|
|
||||||
assert result["output_sample_rate"] > 0
|
|
||||||
|
|
||||||
final_text = (result["transcription_text"] or "").strip()
|
@pytest.mark.advanced_model
|
||||||
assert final_text, "Expected non-empty transcription (model text stream)"
|
@pytest.mark.omni
|
||||||
|
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||||
|
@pytest.mark.parametrize("omni_server", realtime_async_chunk_server_params, indirect=True)
|
||||||
|
def test_streaming_audio_input_pcm_output_async_chunk(self, omni_server) -> None:
|
||||||
|
"""Merge CI: async_chunk on, paced upload, full accuracy check."""
|
||||||
|
pcm16 = _synthetic_pcm16_input()
|
||||||
|
|
||||||
wav_out = _wav_bytes_from_pcm16(out_pcm, result["output_sample_rate"])
|
result = asyncio.run(
|
||||||
whisper_text = convert_audio_bytes_to_text(wav_out).strip()
|
_run_realtime_audio_roundtrip(
|
||||||
assert whisper_text, "Whisper returned empty string for synthesized output audio"
|
omni_server.host,
|
||||||
|
omni_server.port,
|
||||||
sim = cosine_similarity_text(whisper_text.lower(), final_text.lower())
|
omni_server.model,
|
||||||
assert sim > 0.8, (
|
pcm16,
|
||||||
f"Output audio transcript should match model text (sim={sim:.3f}): "
|
chunk_ms=100,
|
||||||
f"whisper={whisper_text!r}, model_text={final_text!r}"
|
send_delay_ms=SEND_DELAY_MS,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
_assert_realtime_smoke(result)
|
||||||
|
_assert_realtime_accuracy(result)
|
||||||
|
|
||||||
|
@pytest.mark.advanced_model
|
||||||
|
@pytest.mark.omni
|
||||||
|
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||||
|
@pytest.mark.parametrize("omni_server", realtime_sync_server_params, indirect=True)
|
||||||
|
def test_streaming_audio_input_pcm_output(self, omni_server) -> None:
|
||||||
|
"""Merge CI: async_chunk off, no send delay, full accuracy check."""
|
||||||
|
pcm16 = _synthetic_pcm16_input()
|
||||||
|
|
||||||
|
result = asyncio.run(
|
||||||
|
_run_realtime_audio_roundtrip(
|
||||||
|
omni_server.host,
|
||||||
|
omni_server.port,
|
||||||
|
omni_server.model,
|
||||||
|
pcm16,
|
||||||
|
chunk_ms=100,
|
||||||
|
send_delay_ms=0,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
_assert_realtime_smoke(result)
|
||||||
|
_assert_realtime_accuracy(result)
|
||||||
@@ -61,7 +61,7 @@ def test_build_multistage_generation_inputs_applies_stage_specific_overrides(ser
|
|||||||
assert engine_prompt["prompt"] == "draw a robot"
|
assert engine_prompt["prompt"] == "draw a robot"
|
||||||
assert engine_prompt["modalities"] == ["img2img"]
|
assert engine_prompt["modalities"] == ["img2img"]
|
||||||
assert engine_prompt["negative_prompt"] == "blurry"
|
assert engine_prompt["negative_prompt"] == "blurry"
|
||||||
assert engine_prompt["mm_processor_kwargs"] == {"target_h": 768, "target_w": 1024}
|
assert engine_prompt["mm_processor_kwargs"] == {"target_h": 768, "target_w": 1024, "vae_generator_seed": 0}
|
||||||
assert engine_prompt["multi_modal_data"]["img2img"].size == (24, 24)
|
assert engine_prompt["multi_modal_data"]["img2img"].size == (24, 24)
|
||||||
|
|
||||||
assert len(sampling_params_list) == 3
|
assert len(sampling_params_list) == 3
|
||||||
|
|||||||
@@ -399,6 +399,8 @@ def test_sampling_params_pass_through(test_client, mocker: MockerFixture):
|
|||||||
"true_cfg_scale": "4.0",
|
"true_cfg_scale": "4.0",
|
||||||
"boundary_ratio": "0.7",
|
"boundary_ratio": "0.7",
|
||||||
"flow_shift": "0.25",
|
"flow_shift": "0.25",
|
||||||
|
"generate_sound": "true",
|
||||||
|
"sound_duration": "2.5",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -413,6 +415,8 @@ def test_sampling_params_pass_through(test_client, mocker: MockerFixture):
|
|||||||
assert captured.true_cfg_scale == 4.0
|
assert captured.true_cfg_scale == 4.0
|
||||||
assert captured.boundary_ratio == 0.7
|
assert captured.boundary_ratio == 0.7
|
||||||
assert captured.extra_args["flow_shift"] == 0.25
|
assert captured.extra_args["flow_shift"] == 0.25
|
||||||
|
assert captured.extra_args["generate_sound"] is True
|
||||||
|
assert captured.extra_args["sound_duration"] == 2.5
|
||||||
|
|
||||||
|
|
||||||
def test_frame_interpolation_params_pass_to_diffusion_sampling_params(test_client, mocker: MockerFixture):
|
def test_frame_interpolation_params_pass_to_diffusion_sampling_params(test_client, mocker: MockerFixture):
|
||||||
@@ -623,6 +627,115 @@ def test_video_job_persists_profiler_metadata(test_client, mocker: MockerFixture
|
|||||||
|
|
||||||
assert completed["stage_durations"] == {"diffuse": 2.5, "vae.decode": 0.3}
|
assert completed["stage_durations"] == {"diffuse": 2.5, "vae.decode": 0.3}
|
||||||
assert completed["peak_memory_mb"] == 4096.5
|
assert completed["peak_memory_mb"] == 4096.5
|
||||||
|
assert completed["action"] is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_video_generation_response_exposes_action_payload(mocker: MockerFixture):
|
||||||
|
engine = FakeAsyncOmni()
|
||||||
|
handler = OmniOpenAIServingVideo.for_diffusion(
|
||||||
|
diffusion_engine=engine,
|
||||||
|
model_name="Cosmos3-8B-UVA",
|
||||||
|
)
|
||||||
|
|
||||||
|
async def _generate(prompt, request_id, sampling_params_list):
|
||||||
|
del prompt, request_id, sampling_params_list
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
yield MockVideoResult(
|
||||||
|
[object()],
|
||||||
|
custom_output={
|
||||||
|
"action": np.array([[[1.5, 2.5], [3.5, 4.5]]], dtype=np.float32),
|
||||||
|
"raw_action_dim": 2,
|
||||||
|
"action_mode": "policy",
|
||||||
|
"domain_id": 7,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
engine.generate = _generate
|
||||||
|
mocker.patch(
|
||||||
|
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
|
||||||
|
return_value="encoded-video",
|
||||||
|
)
|
||||||
|
|
||||||
|
response = asyncio.run(
|
||||||
|
handler.generate_videos(
|
||||||
|
VideoGenerationRequest(prompt="predict actions"),
|
||||||
|
"action-json",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
action = response.data[0].action
|
||||||
|
assert action is not None
|
||||||
|
assert action.data == [[1.5, 2.5], [3.5, 4.5]]
|
||||||
|
assert action.shape == [2, 2]
|
||||||
|
assert action.dtype == "float32"
|
||||||
|
assert action.raw_action_dim == 2
|
||||||
|
assert action.action_mode == "policy"
|
||||||
|
assert action.domain_id == 7
|
||||||
|
assert response.model_dump(mode="json")["data"][0]["action"]["data"] == [[1.5, 2.5], [3.5, 4.5]]
|
||||||
|
|
||||||
|
|
||||||
|
def test_video_job_persists_action_metadata(test_client, mocker: MockerFixture):
|
||||||
|
engine = test_client.app.state.openai_serving_video._engine_client
|
||||||
|
|
||||||
|
async def _generate(prompt, request_id, sampling_params_list):
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
engine.captured_prompt = prompt
|
||||||
|
engine.captured_sampling_params_list = sampling_params_list
|
||||||
|
yield MockVideoResult(
|
||||||
|
[object()],
|
||||||
|
custom_output={
|
||||||
|
"action": np.array([[[1.0, 2.0], [3.0, 4.0]]], dtype=np.float32),
|
||||||
|
"raw_action_dim": 2,
|
||||||
|
"action_mode": "policy",
|
||||||
|
"domain_id": 7,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
engine.generate = _generate
|
||||||
|
mocker.patch(
|
||||||
|
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
||||||
|
return_value=b"fake-video",
|
||||||
|
)
|
||||||
|
|
||||||
|
response = test_client.post("/v1/videos", data={"prompt": "profile me"})
|
||||||
|
assert response.status_code == 200
|
||||||
|
video_id = response.json()["id"]
|
||||||
|
completed = _wait_for_status(test_client, video_id, VideoGenerationStatus.COMPLETED.value)
|
||||||
|
|
||||||
|
expected_action = {
|
||||||
|
"data": [[1.0, 2.0], [3.0, 4.0]],
|
||||||
|
"shape": [2, 2],
|
||||||
|
"dtype": "float32",
|
||||||
|
"raw_action_dim": 2,
|
||||||
|
"action_mode": "policy",
|
||||||
|
"domain_id": 7,
|
||||||
|
}
|
||||||
|
assert completed["action"] == expected_action
|
||||||
|
|
||||||
|
listed = test_client.get("/v1/videos").json()
|
||||||
|
assert listed["data"][0]["action"] == expected_action
|
||||||
|
|
||||||
|
|
||||||
|
def test_action_extraction_accepts_unbatched_action():
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
result = MockVideoResult(
|
||||||
|
[object()],
|
||||||
|
custom_output={
|
||||||
|
"action": np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32),
|
||||||
|
"raw_action_dim": 2,
|
||||||
|
"action_mode": "policy",
|
||||||
|
"domain_id": 7,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
actions = OmniOpenAIServingVideo._extract_action_outputs(result, expected_count=1)
|
||||||
|
|
||||||
|
assert actions[0] is not None
|
||||||
|
assert actions[0].data == [[1.0, 2.0], [3.0, 4.0]]
|
||||||
|
assert actions[0].shape == [2, 2]
|
||||||
|
|
||||||
|
|
||||||
def test_missing_handler_returns_503():
|
def test_missing_handler_returns_503():
|
||||||
@@ -756,6 +869,9 @@ def test_invalid_uploaded_input_reference_returns_400(test_client):
|
|||||||
def test_video_request_validation():
|
def test_video_request_validation():
|
||||||
req = VideoGenerationRequest(prompt="test")
|
req = VideoGenerationRequest(prompt="test")
|
||||||
assert req.prompt == "test"
|
assert req.prompt == "test"
|
||||||
|
assert req.generate_sound is False
|
||||||
|
assert req.sound_duration is None
|
||||||
|
assert VideoGenerationRequest(prompt="test", generate_sound=True, sound_duration=1.5).generate_sound is True
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
VideoGenerationRequest(prompt="test", size="invalid")
|
VideoGenerationRequest(prompt="test", size="invalid")
|
||||||
|
|
||||||
@@ -768,6 +884,8 @@ def test_video_request_validation():
|
|||||||
VideoGenerationRequest(prompt="test", frame_interpolation_exp=0)
|
VideoGenerationRequest(prompt="test", frame_interpolation_exp=0)
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
VideoGenerationRequest(prompt="test", frame_interpolation_scale=0)
|
VideoGenerationRequest(prompt="test", frame_interpolation_scale=0)
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
VideoGenerationRequest(prompt="test", sound_duration=0)
|
||||||
|
|
||||||
|
|
||||||
def test_list_videos_supports_order_after_and_limit(test_client, mocker: MockerFixture):
|
def test_list_videos_supports_order_after_and_limit(test_client, mocker: MockerFixture):
|
||||||
@@ -1029,6 +1147,31 @@ def test_sample_solver_forwarded_via_extra_params(test_client, mocker: MockerFix
|
|||||||
assert captured.extra_args["sample_solver"] == "euler"
|
assert captured.extra_args["sample_solver"] == "euler"
|
||||||
|
|
||||||
|
|
||||||
|
def test_extra_params_allows_inline_action(test_client, mocker: MockerFixture):
|
||||||
|
"""Inline ``action`` data is accepted and forwarded verbatim to
|
||||||
|
``extra_args`` (the supported way to pass forward-dynamics actions)."""
|
||||||
|
mocker.patch(
|
||||||
|
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
||||||
|
return_value=b"fake-video",
|
||||||
|
)
|
||||||
|
action = [[0.1, 0.2], [0.3, 0.4]]
|
||||||
|
response = test_client.post(
|
||||||
|
"/v1/videos",
|
||||||
|
data={
|
||||||
|
"prompt": "forward dynamics inline",
|
||||||
|
"extra_params": json.dumps({"action_mode": "forward_dynamics", "action": action}),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
video_id = response.json()["id"]
|
||||||
|
_wait_for_status(test_client, video_id, VideoGenerationStatus.COMPLETED.value)
|
||||||
|
engine = test_client.app.state.openai_serving_video._engine_client
|
||||||
|
captured = engine.captured_sampling_params_list[0]
|
||||||
|
assert captured.extra_args["action"] == action
|
||||||
|
assert captured.extra_args["action_mode"] == "forward_dynamics"
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Sync endpoint tests (POST /v1/videos/sync)
|
# Sync endpoint tests (POST /v1/videos/sync)
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ OMNI_MODEL = "Qwen/Qwen2.5-Omni-7B"
|
|||||||
OMNI_STAGE_CONFIG = get_deploy_config_path("ci/qwen2_5_omni_thinker_only.yaml")
|
OMNI_STAGE_CONFIG = get_deploy_config_path("ci/qwen2_5_omni_thinker_only.yaml")
|
||||||
|
|
||||||
|
|
||||||
async def _noop(**kw):
|
async def _noop(*args, **kw):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
@@ -61,6 +61,7 @@ def get_async_omni_instance(fake_add_request=_noop, fake_abort_request=_noop) ->
|
|||||||
omni._final_output_handler = lambda: None
|
omni._final_output_handler = lambda: None
|
||||||
omni.resolve_sampling_params_list = lambda params, allow_delta_coercion: params
|
omni.resolve_sampling_params_list = lambda params, allow_delta_coercion: params
|
||||||
omni._compute_final_stage_id = lambda output_modalities: 0
|
omni._compute_final_stage_id = lambda output_modalities: 0
|
||||||
|
omni._compute_final_output_stage_ids = lambda output_modalities: [0]
|
||||||
omni._process_orchestrator_results = fake_process_results
|
omni._process_orchestrator_results = fake_process_results
|
||||||
omni._log_summary_and_cleanup = lambda request_id: omni.request_states.pop(request_id, None)
|
omni._log_summary_and_cleanup = lambda request_id: omni.request_states.pop(request_id, None)
|
||||||
return omni
|
return omni
|
||||||
|
|||||||
@@ -413,6 +413,19 @@ def test_openai_serving_models_can_consume_async_omni_compat_attrs():
|
|||||||
|
|
||||||
|
|
||||||
def test_get_diffusion_od_config_returns_diffusion_stage_config():
|
def test_get_diffusion_od_config_returns_diffusion_stage_config():
|
||||||
|
diffusion_od_config = object()
|
||||||
|
omni = object.__new__(AsyncOmni)
|
||||||
|
omni.engine = SimpleNamespace(
|
||||||
|
stage_clients=[
|
||||||
|
SimpleNamespace(stage_type="llm"),
|
||||||
|
SimpleNamespace(stage_type="diffusion", od_config=diffusion_od_config),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
assert omni.get_diffusion_od_config() is diffusion_od_config
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_diffusion_od_config_falls_back_to_inner_engine():
|
||||||
diffusion_od_config = object()
|
diffusion_od_config = object()
|
||||||
omni = object.__new__(AsyncOmni)
|
omni = object.__new__(AsyncOmni)
|
||||||
omni.engine = SimpleNamespace(
|
omni.engine = SimpleNamespace(
|
||||||
|
|||||||
@@ -0,0 +1,56 @@
|
|||||||
|
import pytest
|
||||||
|
|
||||||
|
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||||
|
from vllm_omni.diffusion.stage_diffusion_proc import StageDiffusionProc
|
||||||
|
from vllm_omni.entrypoints.utils import load_stage_configs_from_model, resolve_model_config_path
|
||||||
|
|
||||||
|
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||||
|
|
||||||
|
|
||||||
|
def test_dreamzero_vla_resolves_to_dreamzero_config(monkeypatch):
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"vllm_omni.entrypoints.utils.get_config",
|
||||||
|
lambda _model, trust_remote_code=True: type("Cfg", (), {"model_type": "vla"})(),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"vllm_omni.entrypoints.utils._looks_like_dreamzero",
|
||||||
|
lambda _model: True,
|
||||||
|
)
|
||||||
|
result = resolve_model_config_path("GEAR-Dreams/DreamZero-DROID")
|
||||||
|
|
||||||
|
assert result is not None
|
||||||
|
assert result.endswith("vllm_omni/deploy/dreamzero.yaml")
|
||||||
|
|
||||||
|
|
||||||
|
def test_dreamzero_config_sets_model_class_and_policy_config(monkeypatch):
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"vllm_omni.config.stage_config.StageConfigFactory._auto_detect_model_type",
|
||||||
|
classmethod(lambda _cls, _model, trust_remote_code=True: ("vla", None)),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"vllm_omni.diffusion.utils.hf_utils._looks_like_dreamzero",
|
||||||
|
lambda _model: True,
|
||||||
|
)
|
||||||
|
|
||||||
|
stage_configs = load_stage_configs_from_model("GEAR-Dreams/DreamZero-DROID")
|
||||||
|
engine_args = stage_configs[0].engine_args
|
||||||
|
|
||||||
|
assert engine_args.model_class_name == "DreamZeroPipeline"
|
||||||
|
assert engine_args.model_config.policy_server_config.action_space == "joint_position"
|
||||||
|
|
||||||
|
|
||||||
|
def test_dreamzero_enrich_config_preserves_explicit_model_class_name(monkeypatch):
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"vllm.transformers_utils.config.get_hf_file_to_dict",
|
||||||
|
lambda path, _model: None if path == "model_index.json" else {"model_type": "vla", "architectures": ["VLA"]},
|
||||||
|
)
|
||||||
|
|
||||||
|
od_config = OmniDiffusionConfig(
|
||||||
|
model="GEAR-Dreams/DreamZero-DROID",
|
||||||
|
model_class_name="DreamZeroPipeline",
|
||||||
|
)
|
||||||
|
proc = StageDiffusionProc(od_config.model, od_config)
|
||||||
|
|
||||||
|
proc._enrich_config()
|
||||||
|
|
||||||
|
assert od_config.model_class_name == "DreamZeroPipeline"
|
||||||
@@ -135,22 +135,6 @@ def test_set_stage_devices_npu_platform(mocker: MockerFixture, monkeypatch: pyte
|
|||||||
# ---- _map_device_list unit tests ----
|
# ---- _map_device_list unit tests ----
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.cpu
|
|
||||||
def test_map_device_list_idempotency():
|
|
||||||
"""Device IDs already in the visible set are returned as-is (idempotency)."""
|
|
||||||
result = _map_device_list(0, ["0", "1"], ["0", "1", "2", "3"])
|
|
||||||
assert result == ["0", "1"]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.cpu
|
|
||||||
def test_map_device_list_idempotency_single():
|
|
||||||
"""Single device ID in visible set passes through."""
|
|
||||||
result = _map_device_list(1, ["1"], ["0", "1"])
|
|
||||||
assert result == ["1"]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
@pytest.mark.core_model
|
||||||
@pytest.mark.cpu
|
@pytest.mark.cpu
|
||||||
def test_map_device_list_index_mapping():
|
def test_map_device_list_index_mapping():
|
||||||
@@ -211,6 +195,18 @@ def test_map_device_list_partial_mapping():
|
|||||||
assert result == ["5", "6"]
|
assert result == ["5", "6"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.core_model
|
||||||
|
@pytest.mark.cpu
|
||||||
|
def test_map_device_list_multi_replica_offset_cvd():
|
||||||
|
"""Logical indices from split_devices_for_replicas must map through an offset
|
||||||
|
CUDA_VISIBLE_DEVICES, not be treated as physical IDs.
|
||||||
|
"""
|
||||||
|
visible = ["2", "3", "4", "5"]
|
||||||
|
assert _map_device_list(1, ["1"], visible) == ["3"]
|
||||||
|
assert _map_device_list(1, ["2"], visible) == ["4"]
|
||||||
|
assert _map_device_list(1, ["2", "3"], visible) == ["4", "5"]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
@pytest.mark.core_model
|
||||||
@pytest.mark.cpu
|
@pytest.mark.cpu
|
||||||
def test_map_device_list_raises_on_non_numeric():
|
def test_map_device_list_raises_on_non_numeric():
|
||||||
|
|||||||
@@ -888,6 +888,12 @@ class OpenAIClientHandler:
|
|||||||
audio_data.append(content)
|
audio_data.append(content)
|
||||||
elif modality == "text" and content:
|
elif modality == "text" and content:
|
||||||
text_content += content
|
text_content += content
|
||||||
|
# Usage is yielded after the last token
|
||||||
|
if chunk.usage:
|
||||||
|
result.prompt_tokens = chunk.usage.prompt_tokens
|
||||||
|
if details := getattr(chunk.usage, "prompt_tokens_details", None):
|
||||||
|
result.cached_tokens = details.cached_tokens
|
||||||
|
|
||||||
audio_content = None
|
audio_content = None
|
||||||
if audio_data:
|
if audio_data:
|
||||||
merged_seg = _merge_base64_audio_to_segment(audio_data)
|
merged_seg = _merge_base64_audio_to_segment(audio_data)
|
||||||
@@ -1582,6 +1588,8 @@ class OpenAIClientHandler:
|
|||||||
create_kwargs["logprobs"] = request_config["logprobs"]
|
create_kwargs["logprobs"] = request_config["logprobs"]
|
||||||
if "top_logprobs" in request_config:
|
if "top_logprobs" in request_config:
|
||||||
create_kwargs["top_logprobs"] = request_config["top_logprobs"]
|
create_kwargs["top_logprobs"] = request_config["top_logprobs"]
|
||||||
|
if "stream_options" in request_config:
|
||||||
|
create_kwargs["stream_options"] = request_config["stream_options"]
|
||||||
if extra_body:
|
if extra_body:
|
||||||
create_kwargs["extra_body"] = extra_body
|
create_kwargs["extra_body"] = extra_body
|
||||||
|
|
||||||
|
|||||||
@@ -154,6 +154,7 @@ class TestRequestLifecycleGauges:
|
|||||||
obj.engine = SimpleNamespace(_running_counter=OmniRequestCounter())
|
obj.engine = SimpleNamespace(_running_counter=OmniRequestCounter())
|
||||||
obj.prom_metrics = OmniPrometheusMetrics(model_name="lifecycle-test")
|
obj.prom_metrics = OmniPrometheusMetrics(model_name="lifecycle-test")
|
||||||
obj.request_states = {}
|
obj.request_states = {}
|
||||||
|
obj._consumed_metric_messages = {}
|
||||||
obj.log_stats = False
|
obj.log_stats = False
|
||||||
|
|
||||||
# Simulate request lifecycle: start (counter 0→1, dict {} → {req}),
|
# Simulate request lifecycle: start (counter 0→1, dict {} → {req}),
|
||||||
@@ -182,6 +183,7 @@ class TestRequestLifecycleGauges:
|
|||||||
obj.engine = SimpleNamespace(_running_counter=OmniRequestCounter())
|
obj.engine = SimpleNamespace(_running_counter=OmniRequestCounter())
|
||||||
obj.prom_metrics = OmniPrometheusMetrics(model_name="lifecycle-test-2")
|
obj.prom_metrics = OmniPrometheusMetrics(model_name="lifecycle-test-2")
|
||||||
obj.request_states = {}
|
obj.request_states = {}
|
||||||
|
obj._consumed_metric_messages = {}
|
||||||
obj.log_stats = False
|
obj.log_stats = False
|
||||||
|
|
||||||
# Two in flight, one finalizes — running should report 1, waiting 0.
|
# Two in flight, one finalizes — running should report 1, waiting 0.
|
||||||
|
|||||||
@@ -5,6 +5,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import functools
|
import functools
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
@@ -17,13 +18,14 @@ def _voxcpm2_talker_mod():
|
|||||||
from vllm_omni.model_executor.models.voxcpm2.voxcpm2_talker import (
|
from vllm_omni.model_executor.models.voxcpm2.voxcpm2_talker import (
|
||||||
VoxCPM2TalkerForConditionalGeneration,
|
VoxCPM2TalkerForConditionalGeneration,
|
||||||
_RequestState,
|
_RequestState,
|
||||||
|
_VoxCPM2RuntimeConfig,
|
||||||
)
|
)
|
||||||
|
|
||||||
return VoxCPM2TalkerForConditionalGeneration, _RequestState
|
return VoxCPM2TalkerForConditionalGeneration, _RequestState, _VoxCPM2RuntimeConfig
|
||||||
|
|
||||||
|
|
||||||
def _make_bare_talker():
|
def _make_bare_talker():
|
||||||
VoxCPM2TalkerForConditionalGeneration, _ = _voxcpm2_talker_mod()
|
VoxCPM2TalkerForConditionalGeneration, _, _ = _voxcpm2_talker_mod()
|
||||||
talker = VoxCPM2TalkerForConditionalGeneration.__new__(VoxCPM2TalkerForConditionalGeneration)
|
talker = VoxCPM2TalkerForConditionalGeneration.__new__(VoxCPM2TalkerForConditionalGeneration)
|
||||||
talker._active_states = {}
|
talker._active_states = {}
|
||||||
talker._current_request_id = None
|
talker._current_request_id = None
|
||||||
@@ -34,11 +36,27 @@ def _make_bare_talker():
|
|||||||
talker._max_batch_size = 4
|
talker._max_batch_size = 4
|
||||||
talker._active_state_warn_threshold = 512
|
talker._active_state_warn_threshold = 512
|
||||||
talker._active_state_warned = False
|
talker._active_state_warned = False
|
||||||
|
talker._enable_profiling = False
|
||||||
|
talker._audio_emit_every = 1
|
||||||
|
talker._vae_decode_every = 1
|
||||||
|
talker._enable_delayed_audio_copy = False
|
||||||
|
talker._delayed_audio_copy_use_events = False
|
||||||
|
talker._coalesce_audio_d2h = False
|
||||||
|
talker._enable_batched_vae_decode = False
|
||||||
|
talker._audio_copy_stream = None
|
||||||
|
talker._enable_vae_cuda_graph = False
|
||||||
|
talker._enable_cfm_cuda_graph = False
|
||||||
|
talker._enable_cfm_prealloc_output = False
|
||||||
|
talker._enable_batched_cfm = True
|
||||||
|
talker._deterministic_cfm_noise = False
|
||||||
|
talker._cfm_buffers = None
|
||||||
|
talker._last_audio_output_req_ids = []
|
||||||
|
talker._batched_fsq_fusion_max_batch = 32
|
||||||
return talker
|
return talker
|
||||||
|
|
||||||
|
|
||||||
def _seed_cached_decode(talker, req_id: str):
|
def _seed_cached_decode(talker, req_id: str):
|
||||||
_, _RequestState = _voxcpm2_talker_mod()
|
_, _RequestState, _ = _voxcpm2_talker_mod()
|
||||||
state = _RequestState(request_id=req_id)
|
state = _RequestState(request_id=req_id)
|
||||||
state.prefill_completed = True
|
state.prefill_completed = True
|
||||||
state.decode_step_count = 5
|
state.decode_step_count = 5
|
||||||
@@ -47,6 +65,23 @@ def _seed_cached_decode(talker, req_id: str):
|
|||||||
|
|
||||||
|
|
||||||
class TestStateEvictionContract:
|
class TestStateEvictionContract:
|
||||||
|
def test_runtime_config_normalizes_mutually_exclusive_paths(self) -> None:
|
||||||
|
_, _, RuntimeConfig = _voxcpm2_talker_mod()
|
||||||
|
|
||||||
|
cfg = RuntimeConfig(
|
||||||
|
enable_batched_cfm=True,
|
||||||
|
enable_cfm_cuda_graph=True,
|
||||||
|
enable_cfm_prealloc_output=True,
|
||||||
|
enable_batched_vae_decode=True,
|
||||||
|
enable_delayed_audio_copy=True,
|
||||||
|
)._normalized()
|
||||||
|
|
||||||
|
assert cfg.enable_batched_cfm is True
|
||||||
|
assert cfg.enable_cfm_cuda_graph is False
|
||||||
|
assert cfg.enable_cfm_prealloc_output is False
|
||||||
|
assert cfg.enable_batched_vae_decode is False
|
||||||
|
assert cfg.enable_delayed_audio_copy is True
|
||||||
|
|
||||||
def test_pending_requests_is_not_used_for_eviction(self) -> None:
|
def test_pending_requests_is_not_used_for_eviction(self) -> None:
|
||||||
talker = _make_bare_talker()
|
talker = _make_bare_talker()
|
||||||
|
|
||||||
@@ -61,6 +96,61 @@ class TestStateEvictionContract:
|
|||||||
assert rid in talker._active_states
|
assert rid in talker._active_states
|
||||||
assert talker._active_states[rid].prefill_completed is True
|
assert talker._active_states[rid].prefill_completed is True
|
||||||
|
|
||||||
|
def test_compute_logits_does_not_sync_stop_bool_on_default_path(self) -> None:
|
||||||
|
_, RState, _ = _voxcpm2_talker_mod()
|
||||||
|
talker = _make_bare_talker()
|
||||||
|
talker.config = SimpleNamespace(vocab_size=4)
|
||||||
|
state = RState(request_id="req", precomputed_stop_logits=torch.tensor([[0.0, 1.0]]))
|
||||||
|
talker._active_states["req"] = state
|
||||||
|
talker._results_queue = [("req", state.precomputed_stop_logits)]
|
||||||
|
|
||||||
|
logits = talker.compute_logits(torch.zeros(1, 1))
|
||||||
|
|
||||||
|
assert logits[0, 0] == 0.0
|
||||||
|
assert logits[0, 1] == 1.0
|
||||||
|
assert state.is_stopping is False
|
||||||
|
assert state.precomputed_stop_logits is None
|
||||||
|
assert state.precomputed_is_stopping is None
|
||||||
|
|
||||||
|
def test_compute_logits_reuses_cached_stop_bool_from_sparse_audio_path(self) -> None:
|
||||||
|
_, RState, _ = _voxcpm2_talker_mod()
|
||||||
|
talker = _make_bare_talker()
|
||||||
|
talker.config = SimpleNamespace(vocab_size=4)
|
||||||
|
state = RState(
|
||||||
|
request_id="req",
|
||||||
|
precomputed_stop_logits=torch.tensor([[0.0, 1.0]]),
|
||||||
|
precomputed_is_stopping=True,
|
||||||
|
)
|
||||||
|
talker._active_states["req"] = state
|
||||||
|
talker._results_queue = [("req", state.precomputed_stop_logits)]
|
||||||
|
|
||||||
|
logits = talker.compute_logits(torch.zeros(1, 1))
|
||||||
|
|
||||||
|
assert logits[0, 0] == 0.0
|
||||||
|
assert logits[0, 1] == 1.0
|
||||||
|
assert state.is_stopping is True
|
||||||
|
assert state.precomputed_stop_logits is None
|
||||||
|
assert state.precomputed_is_stopping is None
|
||||||
|
|
||||||
|
def test_batched_stop_precompute_sets_audio_collect_cache(self) -> None:
|
||||||
|
if not torch.cuda.is_available():
|
||||||
|
pytest.skip("CUDA is required to exercise batched stop D2H precompute")
|
||||||
|
|
||||||
|
_, RState, _ = _voxcpm2_talker_mod()
|
||||||
|
talker = _make_bare_talker()
|
||||||
|
talker._audio_emit_every = 2
|
||||||
|
states = [
|
||||||
|
RState(request_id="stop", precomputed_stop_logits=torch.tensor([[0.0, 1.0]], device="cuda")),
|
||||||
|
RState(request_id="keep", precomputed_stop_logits=torch.tensor([[1.0, 0.0]], device="cuda")),
|
||||||
|
]
|
||||||
|
|
||||||
|
talker._precompute_stop_flags_for_audio_collect(states)
|
||||||
|
|
||||||
|
assert states[0].precomputed_is_stopping is True
|
||||||
|
assert states[0].is_stopping is True
|
||||||
|
assert states[1].precomputed_is_stopping is False
|
||||||
|
assert states[1].is_stopping is False
|
||||||
|
|
||||||
def test_on_requests_finished_defers_cleanup(self) -> None:
|
def test_on_requests_finished_defers_cleanup(self) -> None:
|
||||||
talker = _make_bare_talker()
|
talker = _make_bare_talker()
|
||||||
_seed_cached_decode(talker, "req-A")
|
_seed_cached_decode(talker, "req-A")
|
||||||
@@ -119,7 +209,7 @@ class TestLeakWarnGuard:
|
|||||||
talker = _make_bare_talker()
|
talker = _make_bare_talker()
|
||||||
talker._active_state_warn_threshold = 3
|
talker._active_state_warn_threshold = 3
|
||||||
|
|
||||||
_, RState = _voxcpm2_talker_mod()
|
_, RState, _ = _voxcpm2_talker_mod()
|
||||||
for i in range(4):
|
for i in range(4):
|
||||||
talker._active_states[f"seed-{i}"] = RState(request_id=f"seed-{i}")
|
talker._active_states[f"seed-{i}"] = RState(request_id=f"seed-{i}")
|
||||||
|
|
||||||
@@ -129,3 +219,548 @@ class TestLeakWarnGuard:
|
|||||||
leak_warnings = [m for m in calls if "cleanup path leak" in m]
|
leak_warnings = [m for m in calls if "cleanup path leak" in m]
|
||||||
assert len(leak_warnings) == 1
|
assert len(leak_warnings) == 1
|
||||||
assert talker._active_state_warned is True
|
assert talker._active_state_warned is True
|
||||||
|
|
||||||
|
|
||||||
|
class _NoopPerf:
|
||||||
|
def start(self, name: str) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def stop(self, name: str) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeTTS:
|
||||||
|
feat_decoder = SimpleNamespace(estimator=SimpleNamespace(_compiled=False))
|
||||||
|
|
||||||
|
def fsq_layer(self, x: torch.Tensor) -> torch.Tensor:
|
||||||
|
return x + 10
|
||||||
|
|
||||||
|
def fusion_concat_proj(self, x: torch.Tensor) -> torch.Tensor:
|
||||||
|
half = x.shape[-1] // 2
|
||||||
|
return x[:, :half] * 2 + x[:, half:] * 3
|
||||||
|
|
||||||
|
def feat_encoder(self, x: torch.Tensor) -> torch.Tensor:
|
||||||
|
return x
|
||||||
|
|
||||||
|
def enc_to_lm_proj(self, x: torch.Tensor) -> torch.Tensor:
|
||||||
|
return x.squeeze(1).mean(dim=1).repeat(1, 4)
|
||||||
|
|
||||||
|
|
||||||
|
class TestDecodeBatchContract:
|
||||||
|
def test_forward_skip_pending_prefill_keeps_audio_queue_aligned(self) -> None:
|
||||||
|
class FakeScaffold:
|
||||||
|
def __call__(self, input_ids, positions, intermediate_tensors, inputs_embeds):
|
||||||
|
return torch.zeros(input_ids.shape[0], 4)
|
||||||
|
|
||||||
|
talker = _make_bare_talker()
|
||||||
|
talker._perf = _NoopPerf()
|
||||||
|
talker._tts = _FakeTTS()
|
||||||
|
talker.model = FakeScaffold()
|
||||||
|
talker._enable_cuda_graph = False
|
||||||
|
talker._cuda_graph_warmup_steps = 0
|
||||||
|
talker._cuda_graph_warmup_threshold = 3
|
||||||
|
talker._max_cached_graphs = 4
|
||||||
|
talker._scaffold_graphs = {}
|
||||||
|
talker._residual_graphs = {}
|
||||||
|
talker._pending_requests = [("req", False, None, 1)]
|
||||||
|
|
||||||
|
out = talker.forward(
|
||||||
|
torch.tensor([1]),
|
||||||
|
torch.tensor([0]),
|
||||||
|
inputs_embeds=torch.zeros(1, 4),
|
||||||
|
)
|
||||||
|
|
||||||
|
torch.testing.assert_close(out, torch.zeros(1, 4))
|
||||||
|
assert talker._results_queue == [("req", None)]
|
||||||
|
assert talker._audio_queue == [("req", None)]
|
||||||
|
|
||||||
|
def _make_decode_talker(self) -> object:
|
||||||
|
talker = _make_bare_talker()
|
||||||
|
talker._perf = _NoopPerf()
|
||||||
|
talker._tts = _FakeTTS()
|
||||||
|
talker._side_dtype = torch.float32
|
||||||
|
talker._patch_size = 2
|
||||||
|
talker._feat_dim = 3
|
||||||
|
|
||||||
|
def dit_proj(lm_h: torch.Tensor, res_h: torch.Tensor) -> torch.Tensor:
|
||||||
|
return lm_h + 2 * res_h
|
||||||
|
|
||||||
|
def stop_fn(lm_h: torch.Tensor) -> torch.Tensor:
|
||||||
|
return torch.stack([lm_h.sum(dim=1), -lm_h.sum(dim=1)], dim=1)
|
||||||
|
|
||||||
|
def run_cfm(dit_h: torch.Tensor, cond: torch.Tensor) -> torch.Tensor:
|
||||||
|
return dit_h[:, :1].view(-1, 1, 1) + cond.transpose(1, 2) * 0.1
|
||||||
|
|
||||||
|
talker._dit_proj_fn = dit_proj
|
||||||
|
talker._stop_fn = stop_fn
|
||||||
|
talker._run_cfm = run_cfm
|
||||||
|
talker._run_cfm_for_state = lambda _state, dit_h, cond: talker._run_cfm(dit_h, cond)
|
||||||
|
return talker
|
||||||
|
|
||||||
|
def _make_decode_state(self, request_id: str, offset: float):
|
||||||
|
_, RState, _ = _voxcpm2_talker_mod()
|
||||||
|
state = RState(request_id=request_id)
|
||||||
|
state.curr_prefix_feat_cond = torch.arange(6, dtype=torch.float32).reshape(2, 3) + offset
|
||||||
|
return state
|
||||||
|
|
||||||
|
def test_batched_fsq_fusion_matches_per_request_decode_prepare(self) -> None:
|
||||||
|
seq_talker = self._make_decode_talker()
|
||||||
|
batch_talker = self._make_decode_talker()
|
||||||
|
seq_talker._max_decode_steps = 2000
|
||||||
|
batch_talker._max_decode_steps = 2000
|
||||||
|
|
||||||
|
seq_states = [self._make_decode_state("req-0", 0.0), self._make_decode_state("req-1", 10.0)]
|
||||||
|
batch_states = [self._make_decode_state("req-0", 0.0), self._make_decode_state("req-1", 10.0)]
|
||||||
|
for idx, (seq_state, batch_state) in enumerate(zip(seq_states, batch_states, strict=True)):
|
||||||
|
prev = torch.full((1, 4), float(idx + 1))
|
||||||
|
seq_state.prev_feat_embed = prev.clone()
|
||||||
|
batch_state.prev_feat_embed = prev.clone()
|
||||||
|
|
||||||
|
hiddens = [
|
||||||
|
torch.tensor([[1.0, 2.0, 3.0, 4.0]]),
|
||||||
|
torch.tensor([[5.0, 6.0, 7.0, 8.0]]),
|
||||||
|
]
|
||||||
|
|
||||||
|
expected = [
|
||||||
|
seq_talker._prepare_residual_decode(state, hidden, torch.device("cpu"))
|
||||||
|
for state, hidden in zip(seq_states, hiddens, strict=True)
|
||||||
|
]
|
||||||
|
actual = batch_talker._prepare_residual_decode_batch(batch_states, hiddens, torch.device("cpu"))
|
||||||
|
|
||||||
|
for (expected_res, expected_meta), (actual_res, actual_meta) in zip(expected, actual, strict=True):
|
||||||
|
torch.testing.assert_close(actual_res, expected_res)
|
||||||
|
torch.testing.assert_close(actual_meta["new_lm_hidden"], expected_meta["new_lm_hidden"])
|
||||||
|
assert [state.decode_step_count for state in batch_states] == [1, 1]
|
||||||
|
|
||||||
|
def test_batch_decode_matches_per_request_state_updates(self) -> None:
|
||||||
|
seq_talker = self._make_decode_talker()
|
||||||
|
batch_talker = self._make_decode_talker()
|
||||||
|
|
||||||
|
seq_states = [self._make_decode_state("req-0", 0.0), self._make_decode_state("req-1", 10.0)]
|
||||||
|
batch_states = [self._make_decode_state("req-0", 0.0), self._make_decode_state("req-1", 10.0)]
|
||||||
|
metas = [
|
||||||
|
{"new_lm_hidden": torch.tensor([[1.0, 2.0, 3.0, 4.0]])},
|
||||||
|
{"new_lm_hidden": torch.tensor([[5.0, 6.0, 7.0, 8.0]])},
|
||||||
|
]
|
||||||
|
batch_out = torch.tensor([[0.5, 1.0, 1.5, 2.0], [2.5, 3.0, 3.5, 4.0]])
|
||||||
|
|
||||||
|
for state, meta, res_out in zip(seq_states, metas, batch_out, strict=True):
|
||||||
|
seq_talker._finish_decode(state, meta, res_out.unsqueeze(0), torch.device("cpu"))
|
||||||
|
|
||||||
|
batch_talker._finish_decode_batch(
|
||||||
|
[(state, False, meta) for state, meta in zip(batch_states, metas, strict=True)],
|
||||||
|
batch_out,
|
||||||
|
)
|
||||||
|
|
||||||
|
for expected, actual in zip(seq_states, batch_states, strict=True):
|
||||||
|
torch.testing.assert_close(actual.precomputed_stop_logits, expected.precomputed_stop_logits)
|
||||||
|
torch.testing.assert_close(actual.curr_embed_for_next, expected.curr_embed_for_next)
|
||||||
|
torch.testing.assert_close(actual.prev_feat_embed, expected.prev_feat_embed)
|
||||||
|
torch.testing.assert_close(actual.curr_prefix_feat_cond, expected.curr_prefix_feat_cond)
|
||||||
|
torch.testing.assert_close(actual.last_audio_patch_gpu, expected.last_audio_patch_gpu)
|
||||||
|
|
||||||
|
def test_batch_decode_preserves_per_request_cfm_rng_order(self) -> None:
|
||||||
|
talker = self._make_decode_talker()
|
||||||
|
talker._enable_batched_cfm = False
|
||||||
|
states = [self._make_decode_state("req-0", 0.0), self._make_decode_state("req-1", 10.0)]
|
||||||
|
metas = [
|
||||||
|
{"new_lm_hidden": torch.tensor([[1.0, 2.0, 3.0, 4.0]])},
|
||||||
|
{"new_lm_hidden": torch.tensor([[5.0, 6.0, 7.0, 8.0]])},
|
||||||
|
]
|
||||||
|
batch_out = torch.tensor([[0.5, 1.0, 1.5, 2.0], [2.5, 3.0, 3.5, 4.0]])
|
||||||
|
|
||||||
|
captured_noise: list[torch.Tensor] = []
|
||||||
|
torch.manual_seed(1234)
|
||||||
|
expected_noise = [torch.empty(1, 3, 2).normal_().detach().clone() for _ in states]
|
||||||
|
|
||||||
|
def run_cfm(
|
||||||
|
dit_h: torch.Tensor,
|
||||||
|
cond: torch.Tensor,
|
||||||
|
) -> torch.Tensor:
|
||||||
|
captured_noise.append(torch.empty(1, 3, 2).normal_().detach().clone())
|
||||||
|
return dit_h[:, :1].view(-1, 1, 1) + cond.transpose(1, 2) * 0.1
|
||||||
|
|
||||||
|
talker._run_cfm = run_cfm
|
||||||
|
|
||||||
|
torch.manual_seed(1234)
|
||||||
|
talker._finish_decode_batch(
|
||||||
|
[(state, False, meta) for state, meta in zip(states, metas, strict=True)],
|
||||||
|
batch_out,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert len(captured_noise) == len(states)
|
||||||
|
for actual, expected in zip(captured_noise, expected_noise, strict=True):
|
||||||
|
torch.testing.assert_close(actual, expected)
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeAudioVAE:
|
||||||
|
decode_chunk_size = 2
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeAudioTTS:
|
||||||
|
audio_vae = _FakeAudioVAE()
|
||||||
|
|
||||||
|
|
||||||
|
class TestAudioEmitCoalescing:
|
||||||
|
def test_audio_emit_every_accumulates_gpu_chunks_without_changing_order(self) -> None:
|
||||||
|
_, RState, _ = _voxcpm2_talker_mod()
|
||||||
|
talker = _make_bare_talker()
|
||||||
|
talker._audio_emit_every = 2
|
||||||
|
talker._feat_dim = 1
|
||||||
|
talker._n_decode_pad_frames = 1
|
||||||
|
talker._device = torch.device("cpu")
|
||||||
|
talker._tts = _FakeAudioTTS()
|
||||||
|
talker._perf = _NoopPerf()
|
||||||
|
|
||||||
|
values = [torch.tensor([10.0, 11.0]), torch.tensor([99.0, 99.0, 20.0, 21.0])]
|
||||||
|
|
||||||
|
def run_vae_decode(feat: torch.Tensor) -> torch.Tensor:
|
||||||
|
return values.pop(0)
|
||||||
|
|
||||||
|
talker._run_vae_decode = run_vae_decode
|
||||||
|
state = RState(request_id="req")
|
||||||
|
|
||||||
|
state.last_audio_patch_gpu = torch.tensor([[1.0]])
|
||||||
|
assert talker._collect_audio(state) is None
|
||||||
|
assert len(state.pending_audio_chunks_gpu) == 1
|
||||||
|
|
||||||
|
state.last_audio_patch_gpu = torch.tensor([[2.0]])
|
||||||
|
audio = talker._collect_audio(state)
|
||||||
|
|
||||||
|
torch.testing.assert_close(audio, torch.tensor([10.0, 11.0, 20.0, 21.0]))
|
||||||
|
assert state.pending_audio_chunks_gpu == []
|
||||||
|
|
||||||
|
def test_audio_emit_every_flushes_on_stop(self) -> None:
|
||||||
|
_, RState, _ = _voxcpm2_talker_mod()
|
||||||
|
talker = _make_bare_talker()
|
||||||
|
talker._audio_emit_every = 8
|
||||||
|
talker._feat_dim = 1
|
||||||
|
talker._n_decode_pad_frames = 1
|
||||||
|
talker._device = torch.device("cpu")
|
||||||
|
talker._tts = _FakeAudioTTS()
|
||||||
|
talker._perf = _NoopPerf()
|
||||||
|
talker._run_vae_decode = lambda feat: torch.tensor([3.0, 4.0])
|
||||||
|
state = RState(request_id="req", precomputed_stop_logits=torch.tensor([[0.0, 1.0]]))
|
||||||
|
|
||||||
|
state.last_audio_patch_gpu = torch.tensor([[1.0]])
|
||||||
|
audio = talker._collect_audio(state)
|
||||||
|
|
||||||
|
torch.testing.assert_close(audio, torch.tensor([3.0, 4.0]))
|
||||||
|
assert state.pending_audio_chunks_gpu == []
|
||||||
|
|
||||||
|
def test_vae_decode_every_accumulates_latents_before_decode(self) -> None:
|
||||||
|
_, RState, _ = _voxcpm2_talker_mod()
|
||||||
|
talker = _make_bare_talker()
|
||||||
|
talker._vae_decode_every = 2
|
||||||
|
talker._feat_dim = 1
|
||||||
|
talker._n_decode_pad_frames = 1
|
||||||
|
talker._device = torch.device("cpu")
|
||||||
|
talker._tts = _FakeAudioTTS()
|
||||||
|
talker._perf = _NoopPerf()
|
||||||
|
|
||||||
|
seen_feats = []
|
||||||
|
values = [torch.tensor([10.0, 11.0, 20.0, 21.0])]
|
||||||
|
|
||||||
|
def run_vae_decode(feat: torch.Tensor) -> torch.Tensor:
|
||||||
|
seen_feats.append(feat.detach().clone())
|
||||||
|
return values.pop(0)
|
||||||
|
|
||||||
|
talker._run_vae_decode = run_vae_decode
|
||||||
|
state = RState(request_id="req")
|
||||||
|
|
||||||
|
state.last_audio_patch_gpu = torch.tensor([[1.0]])
|
||||||
|
assert talker._collect_audio(state) is None
|
||||||
|
assert len(state.pending_vae_latents_gpu) == 1
|
||||||
|
assert seen_feats == []
|
||||||
|
|
||||||
|
state.last_audio_patch_gpu = torch.tensor([[2.0]])
|
||||||
|
audio = talker._collect_audio(state)
|
||||||
|
|
||||||
|
torch.testing.assert_close(audio, torch.tensor([10.0, 11.0, 20.0, 21.0]))
|
||||||
|
assert state.pending_vae_latents_gpu == []
|
||||||
|
torch.testing.assert_close(seen_feats[0].reshape(-1), torch.tensor([1.0, 2.0]))
|
||||||
|
|
||||||
|
def test_vae_decode_every_flushes_pending_latent_on_stop(self) -> None:
|
||||||
|
_, RState, _ = _voxcpm2_talker_mod()
|
||||||
|
talker = _make_bare_talker()
|
||||||
|
talker._vae_decode_every = 4
|
||||||
|
talker._feat_dim = 1
|
||||||
|
talker._n_decode_pad_frames = 1
|
||||||
|
talker._device = torch.device("cpu")
|
||||||
|
talker._tts = _FakeAudioTTS()
|
||||||
|
talker._perf = _NoopPerf()
|
||||||
|
talker._run_vae_decode = lambda feat: torch.tensor([7.0, 8.0])
|
||||||
|
state = RState(request_id="req", precomputed_stop_logits=torch.tensor([[0.0, 1.0]]))
|
||||||
|
|
||||||
|
state.last_audio_patch_gpu = torch.tensor([[1.0]])
|
||||||
|
audio = talker._collect_audio(state)
|
||||||
|
|
||||||
|
torch.testing.assert_close(audio, torch.tensor([7.0, 8.0]))
|
||||||
|
assert state.pending_vae_latents_gpu == []
|
||||||
|
assert state.is_stopping is True
|
||||||
|
|
||||||
|
def test_batched_vae_decode_collects_ready_requests_in_order(self) -> None:
|
||||||
|
_, RState, _ = _voxcpm2_talker_mod()
|
||||||
|
talker = _make_bare_talker()
|
||||||
|
talker._enable_batched_vae_decode = True
|
||||||
|
talker._coalesce_audio_d2h = True
|
||||||
|
talker._vae_decode_every = 2
|
||||||
|
talker._feat_dim = 1
|
||||||
|
talker._n_decode_pad_frames = 1
|
||||||
|
talker._device = torch.device("cpu")
|
||||||
|
talker._tts = _FakeAudioTTS()
|
||||||
|
talker._perf = _NoopPerf()
|
||||||
|
|
||||||
|
seen_feats: list[torch.Tensor] = []
|
||||||
|
|
||||||
|
def run_vae_decode(feat: torch.Tensor) -> torch.Tensor:
|
||||||
|
seen_feats.append(feat.detach().clone())
|
||||||
|
# One output row per request. decode_chunk_size=2, and each ready
|
||||||
|
# request has two new latent frames, so the full decoded length is 4.
|
||||||
|
return torch.tensor(
|
||||||
|
[
|
||||||
|
[[10.0, 11.0, 12.0, 13.0]],
|
||||||
|
[[20.0, 21.0, 22.0, 23.0]],
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
talker._run_vae_decode = run_vae_decode
|
||||||
|
|
||||||
|
ready0 = RState(request_id="ready-0")
|
||||||
|
ready0.pending_vae_latents_gpu.append(torch.tensor([[1.0]]))
|
||||||
|
ready0.last_audio_patch_gpu = torch.tensor([[2.0]])
|
||||||
|
|
||||||
|
pending = RState(request_id="pending")
|
||||||
|
pending.last_audio_patch_gpu = torch.tensor([[99.0]])
|
||||||
|
|
||||||
|
ready1 = RState(request_id="ready-1")
|
||||||
|
ready1.pending_vae_latents_gpu.append(torch.tensor([[3.0]]))
|
||||||
|
ready1.last_audio_patch_gpu = torch.tensor([[4.0]])
|
||||||
|
|
||||||
|
outputs = talker._collect_audio_batch([ready0, pending, ready1])
|
||||||
|
|
||||||
|
assert list(outputs) == ["ready-0", "pending", "ready-1"]
|
||||||
|
torch.testing.assert_close(outputs["ready-0"], torch.tensor([10.0, 11.0, 12.0, 13.0]))
|
||||||
|
assert outputs["pending"] is None
|
||||||
|
torch.testing.assert_close(outputs["ready-1"], torch.tensor([20.0, 21.0, 22.0, 23.0]))
|
||||||
|
assert len(seen_feats) == 1
|
||||||
|
torch.testing.assert_close(seen_feats[0].reshape(2, -1), torch.tensor([[1.0, 2.0], [3.0, 4.0]]))
|
||||||
|
assert pending.pending_vae_latents_gpu
|
||||||
|
assert ready0.pending_vae_latents_gpu == []
|
||||||
|
assert ready1.pending_vae_latents_gpu == []
|
||||||
|
|
||||||
|
def test_audio_emit_every_keeps_private_chunk_when_vae_output_storage_is_reused(self) -> None:
|
||||||
|
_, RState, _ = _voxcpm2_talker_mod()
|
||||||
|
talker = _make_bare_talker()
|
||||||
|
talker._audio_emit_every = 2
|
||||||
|
talker._enable_vae_cuda_graph = True
|
||||||
|
talker._feat_dim = 1
|
||||||
|
talker._n_decode_pad_frames = 1
|
||||||
|
talker._device = torch.device("cpu")
|
||||||
|
talker._tts = _FakeAudioTTS()
|
||||||
|
talker._perf = _NoopPerf()
|
||||||
|
|
||||||
|
shared_output = torch.empty(4)
|
||||||
|
values = [torch.tensor([10.0, 11.0, -1.0, -1.0]), torch.tensor([99.0, 99.0, 20.0, 21.0])]
|
||||||
|
|
||||||
|
def run_vae_decode(feat: torch.Tensor) -> torch.Tensor:
|
||||||
|
shared_output.copy_(values.pop(0))
|
||||||
|
return shared_output
|
||||||
|
|
||||||
|
talker._run_vae_decode = run_vae_decode
|
||||||
|
state = RState(request_id="req")
|
||||||
|
|
||||||
|
state.last_audio_patch_gpu = torch.tensor([[1.0]])
|
||||||
|
assert talker._collect_audio(state) is None
|
||||||
|
|
||||||
|
state.last_audio_patch_gpu = torch.tensor([[2.0]])
|
||||||
|
audio = talker._collect_audio(state)
|
||||||
|
|
||||||
|
torch.testing.assert_close(audio, torch.tensor([10.0, 11.0, 20.0, 21.0]))
|
||||||
|
|
||||||
|
def test_audio_emit_every_keeps_private_chunk_when_compiled_vae_reuses_storage(self) -> None:
|
||||||
|
_, RState, _ = _voxcpm2_talker_mod()
|
||||||
|
talker = _make_bare_talker()
|
||||||
|
talker._audio_emit_every = 2
|
||||||
|
talker._feat_dim = 1
|
||||||
|
talker._n_decode_pad_frames = 1
|
||||||
|
talker._device = torch.device("cpu")
|
||||||
|
talker._tts = _FakeAudioTTS()
|
||||||
|
talker._tts.audio_vae._compiled = True
|
||||||
|
talker._perf = _NoopPerf()
|
||||||
|
|
||||||
|
shared_output = torch.empty(4)
|
||||||
|
values = [torch.tensor([10.0, 11.0, -1.0, -1.0]), torch.tensor([99.0, 99.0, 20.0, 21.0])]
|
||||||
|
|
||||||
|
def run_vae_decode(feat: torch.Tensor) -> torch.Tensor:
|
||||||
|
shared_output.copy_(values.pop(0))
|
||||||
|
return shared_output
|
||||||
|
|
||||||
|
talker._run_vae_decode = run_vae_decode
|
||||||
|
state = RState(request_id="req")
|
||||||
|
|
||||||
|
state.last_audio_patch_gpu = torch.tensor([[1.0]])
|
||||||
|
assert talker._collect_audio(state) is None
|
||||||
|
|
||||||
|
state.last_audio_patch_gpu = torch.tensor([[2.0]])
|
||||||
|
audio = talker._collect_audio(state)
|
||||||
|
|
||||||
|
torch.testing.assert_close(audio, torch.tensor([10.0, 11.0, 20.0, 21.0]))
|
||||||
|
|
||||||
|
def test_delayed_audio_copy_emits_previous_chunk_and_flushes_on_stop(self) -> None:
|
||||||
|
_, RState, _ = _voxcpm2_talker_mod()
|
||||||
|
talker = _make_bare_talker()
|
||||||
|
talker._enable_delayed_audio_copy = True
|
||||||
|
talker._audio_emit_every = 1
|
||||||
|
talker._feat_dim = 1
|
||||||
|
talker._n_decode_pad_frames = 1
|
||||||
|
talker._device = torch.device("cpu")
|
||||||
|
talker._tts = _FakeAudioTTS()
|
||||||
|
talker._perf = _NoopPerf()
|
||||||
|
|
||||||
|
values = [
|
||||||
|
torch.tensor([10.0, 11.0]),
|
||||||
|
torch.tensor([99.0, 99.0, 20.0, 21.0]),
|
||||||
|
torch.tensor([88.0, 88.0, 30.0, 31.0]),
|
||||||
|
]
|
||||||
|
|
||||||
|
def run_vae_decode(feat: torch.Tensor) -> torch.Tensor:
|
||||||
|
return values.pop(0)
|
||||||
|
|
||||||
|
talker._run_vae_decode = run_vae_decode
|
||||||
|
state = RState(request_id="req")
|
||||||
|
|
||||||
|
state.last_audio_patch_gpu = torch.tensor([[1.0]])
|
||||||
|
assert talker._collect_audio(state) is None
|
||||||
|
assert len(state.pending_audio_copies) == 1
|
||||||
|
|
||||||
|
state.last_audio_patch_gpu = torch.tensor([[2.0]])
|
||||||
|
audio = talker._collect_audio(state)
|
||||||
|
torch.testing.assert_close(audio, torch.tensor([10.0, 11.0]))
|
||||||
|
assert len(state.pending_audio_copies) == 1
|
||||||
|
|
||||||
|
state.precomputed_stop_logits = torch.tensor([[0.0, 1.0]])
|
||||||
|
state.last_audio_patch_gpu = torch.tensor([[3.0]])
|
||||||
|
audio = talker._collect_audio(state)
|
||||||
|
torch.testing.assert_close(audio, torch.tensor([20.0, 21.0, 30.0, 31.0]))
|
||||||
|
assert state.pending_audio_copies == []
|
||||||
|
assert state.is_stopping is True
|
||||||
|
|
||||||
|
def test_delayed_audio_copy_flushes_on_forced_stop(self) -> None:
|
||||||
|
_, RState, _ = _voxcpm2_talker_mod()
|
||||||
|
talker = _make_bare_talker()
|
||||||
|
talker._enable_delayed_audio_copy = True
|
||||||
|
talker._audio_emit_every = 1
|
||||||
|
talker._feat_dim = 1
|
||||||
|
talker._n_decode_pad_frames = 1
|
||||||
|
talker._device = torch.device("cpu")
|
||||||
|
talker._tts = _FakeAudioTTS()
|
||||||
|
talker._perf = _NoopPerf()
|
||||||
|
|
||||||
|
values = [
|
||||||
|
torch.tensor([10.0, 11.0]),
|
||||||
|
torch.tensor([99.0, 99.0, 20.0, 21.0]),
|
||||||
|
]
|
||||||
|
|
||||||
|
def run_vae_decode(feat: torch.Tensor) -> torch.Tensor:
|
||||||
|
return values.pop(0)
|
||||||
|
|
||||||
|
talker._run_vae_decode = run_vae_decode
|
||||||
|
state = RState(request_id="req")
|
||||||
|
|
||||||
|
state.last_audio_patch_gpu = torch.tensor([[1.0]])
|
||||||
|
assert talker._collect_audio(state) is None
|
||||||
|
|
||||||
|
state.is_stopping = True
|
||||||
|
state.last_audio_patch_gpu = torch.tensor([[2.0]])
|
||||||
|
audio = talker._collect_audio(state)
|
||||||
|
|
||||||
|
torch.testing.assert_close(audio, torch.tensor([10.0, 11.0, 20.0, 21.0]))
|
||||||
|
assert state.pending_audio_copies == []
|
||||||
|
|
||||||
|
def test_make_omni_output_uses_sparse_req_ids_for_deferred_chunks(self) -> None:
|
||||||
|
talker = _make_bare_talker()
|
||||||
|
talker._audio_emit_every = 4
|
||||||
|
talker._sample_rate = 16000
|
||||||
|
talker._last_audio_output_req_ids = ["req-0", "req-1", "req-2"]
|
||||||
|
talker._audio_queue = [
|
||||||
|
("req-0", None),
|
||||||
|
("req-1", torch.tensor([1.0, 2.0])),
|
||||||
|
("req-2", None),
|
||||||
|
]
|
||||||
|
|
||||||
|
out = talker.make_omni_output(torch.zeros(3, 1))
|
||||||
|
|
||||||
|
audio = out.multimodal_outputs["model_outputs"]
|
||||||
|
sr = out.multimodal_outputs["sr"]
|
||||||
|
assert out.multimodal_outputs["meta"]["req_id"] == ["req-1"]
|
||||||
|
assert out.multimodal_outputs["meta"]["sparse_audio"] == ["1"]
|
||||||
|
assert len(audio) == 1
|
||||||
|
torch.testing.assert_close(audio[0], torch.tensor([1.0, 2.0]))
|
||||||
|
assert int(sr[0]) == 16000
|
||||||
|
|
||||||
|
def test_make_omni_output_emits_empty_sparse_marker_for_deferred_chunks(self) -> None:
|
||||||
|
talker = _make_bare_talker()
|
||||||
|
talker._audio_emit_every = 4
|
||||||
|
talker._sample_rate = 16000
|
||||||
|
talker._audio_queue = [("req-0", None), ("req-1", None)]
|
||||||
|
|
||||||
|
out = talker.make_omni_output(torch.zeros(2, 1))
|
||||||
|
|
||||||
|
assert out.multimodal_outputs["model_outputs"] == []
|
||||||
|
assert out.multimodal_outputs["sr"] == []
|
||||||
|
assert out.multimodal_outputs["meta"]["req_id"] == []
|
||||||
|
assert out.multimodal_outputs["meta"]["sparse_audio"] == ["1"]
|
||||||
|
|
||||||
|
def test_make_omni_output_uses_sparse_req_ids_for_delayed_audio_copy(self) -> None:
|
||||||
|
talker = _make_bare_talker()
|
||||||
|
talker._enable_delayed_audio_copy = True
|
||||||
|
talker._sample_rate = 16000
|
||||||
|
talker._audio_queue = [
|
||||||
|
("req-0", None),
|
||||||
|
("req-1", torch.tensor([1.0, 2.0])),
|
||||||
|
("req-2", None),
|
||||||
|
]
|
||||||
|
|
||||||
|
out = talker.make_omni_output(torch.zeros(3, 1))
|
||||||
|
|
||||||
|
assert out.multimodal_outputs["meta"]["req_id"] == ["req-1"]
|
||||||
|
assert out.multimodal_outputs["meta"]["sparse_audio"] == ["1"]
|
||||||
|
assert len(out.multimodal_outputs["model_outputs"]) == 1
|
||||||
|
torch.testing.assert_close(out.multimodal_outputs["model_outputs"][0], torch.tensor([1.0, 2.0]))
|
||||||
|
|
||||||
|
def test_make_omni_output_uses_sparse_req_ids_for_deferred_vae_decode(self) -> None:
|
||||||
|
talker = _make_bare_talker()
|
||||||
|
talker._vae_decode_every = 2
|
||||||
|
talker._sample_rate = 16000
|
||||||
|
talker._audio_queue = [
|
||||||
|
("req-0", None),
|
||||||
|
("req-1", torch.tensor([1.0, 2.0])),
|
||||||
|
("req-2", None),
|
||||||
|
]
|
||||||
|
|
||||||
|
out = talker.make_omni_output(torch.zeros(3, 1))
|
||||||
|
|
||||||
|
assert out.multimodal_outputs["meta"]["req_id"] == ["req-1"]
|
||||||
|
assert out.multimodal_outputs["meta"]["sparse_audio"] == ["1"]
|
||||||
|
assert len(out.multimodal_outputs["model_outputs"]) == 1
|
||||||
|
torch.testing.assert_close(out.multimodal_outputs["model_outputs"][0], torch.tensor([1.0, 2.0]))
|
||||||
|
|
||||||
|
def test_make_omni_output_coalesces_gpu_audio_to_cpu(self) -> None:
|
||||||
|
if not torch.cuda.is_available():
|
||||||
|
pytest.skip("CUDA is required to exercise D2H coalescing")
|
||||||
|
talker = _make_bare_talker()
|
||||||
|
talker._coalesce_audio_d2h = True
|
||||||
|
talker._sample_rate = 16000
|
||||||
|
talker._audio_queue = [
|
||||||
|
("req-0", torch.tensor([1.0, 2.0], device="cuda")),
|
||||||
|
("req-1", torch.tensor([3.0], device="cuda")),
|
||||||
|
]
|
||||||
|
|
||||||
|
out = talker.make_omni_output(torch.zeros(2, 1))
|
||||||
|
|
||||||
|
audio = out.multimodal_outputs["model_outputs"]
|
||||||
|
assert len(audio) == 2
|
||||||
|
assert all(not chunk.is_cuda for chunk in audio)
|
||||||
|
torch.testing.assert_close(audio[0], torch.tensor([1.0, 2.0]))
|
||||||
|
torch.testing.assert_close(audio[1], torch.tensor([3.0]))
|
||||||
@@ -1638,9 +1638,6 @@ class TestBaseConfigInheritance:
|
|||||||
s0 = deploy.stages[0].default_sampling_params
|
s0 = deploy.stages[0].default_sampling_params
|
||||||
# CI overrides max_tokens
|
# CI overrides max_tokens
|
||||||
assert s0["max_tokens"] == 150
|
assert s0["max_tokens"] == 150
|
||||||
# Inherited from base
|
|
||||||
assert s0["temperature"] == 0.4
|
|
||||||
assert s0["seed"] == 42
|
|
||||||
|
|
||||||
def test_pure_inheritance_overlay(self, tmp_path):
|
def test_pure_inheritance_overlay(self, tmp_path):
|
||||||
"""An overlay with only ``base_config`` inherits everything."""
|
"""An overlay with only ``base_config`` inherits everything."""
|
||||||
@@ -1960,5 +1957,3 @@ class TestSamplingConstraintsPrecedence:
|
|||||||
assert stages[0].yaml_extras["default_sampling_params"]["detokenize"] is True
|
assert stages[0].yaml_extras["default_sampling_params"]["detokenize"] is True
|
||||||
# Pipeline says stop_token_ids=[2150] for talker
|
# Pipeline says stop_token_ids=[2150] for talker
|
||||||
assert stages[1].yaml_extras["default_sampling_params"]["stop_token_ids"] == [2150]
|
assert stages[1].yaml_extras["default_sampling_params"]["stop_token_ids"] == [2150]
|
||||||
# Deploy temperature still flows through
|
|
||||||
assert stages[0].yaml_extras["default_sampling_params"]["temperature"] == 0.4
|
|
||||||
Whitespace-only changes.
@@ -0,0 +1,910 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
"""Deterministic unit tests for higgs-audio v3.
|
||||||
|
|
||||||
|
These tests verify AC-1 (config), AC-3 (prompt), AC-4 (fused modules),
|
||||||
|
AC-5 (delay pattern), AC-7 (stage processor), and AC-10 (registry)
|
||||||
|
without requiring the actual checkpoint or GPU.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import torch
|
||||||
|
|
||||||
|
# ---- AC-1: Configuration ----
|
||||||
|
|
||||||
|
|
||||||
|
class TestHiggsAudioV3Config:
|
||||||
|
def test_default_config_loads(self):
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
||||||
|
HiggsAudioV3Config,
|
||||||
|
)
|
||||||
|
|
||||||
|
config = HiggsAudioV3Config()
|
||||||
|
assert config.num_codebooks == 8
|
||||||
|
assert config.codebook_size == 1026
|
||||||
|
assert config.audio_stream_bos_id == 1024
|
||||||
|
assert config.audio_stream_eos_id == 1025
|
||||||
|
assert config.sample_rate == 24000
|
||||||
|
assert config.frame_rate == 25
|
||||||
|
assert config.num_real_codes == 1024
|
||||||
|
assert config.tie_modality_embeddings is True
|
||||||
|
|
||||||
|
def test_custom_audio_encoder_config(self):
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
||||||
|
HiggsAudioV3Config,
|
||||||
|
)
|
||||||
|
|
||||||
|
config = HiggsAudioV3Config(
|
||||||
|
audio_encoder_config={
|
||||||
|
"encoder_type": "discrete",
|
||||||
|
"num_codebooks": 4,
|
||||||
|
"vocab_size": 512,
|
||||||
|
"tie_word_embeddings": False,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
assert config.num_codebooks == 4
|
||||||
|
assert config.codebook_size == 512
|
||||||
|
assert config.tie_modality_embeddings is False
|
||||||
|
|
||||||
|
def test_invalid_num_codebooks_rejected(self):
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
||||||
|
HiggsAudioV3Config,
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="num_codebooks must be > 0"):
|
||||||
|
HiggsAudioV3Config(audio_encoder_config={"num_codebooks": 0, "vocab_size": 1026})
|
||||||
|
|
||||||
|
def test_negative_num_codebooks_rejected(self):
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
||||||
|
HiggsAudioV3Config,
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="num_codebooks must be > 0"):
|
||||||
|
HiggsAudioV3Config(audio_encoder_config={"num_codebooks": -1, "vocab_size": 1026})
|
||||||
|
|
||||||
|
def test_special_token_ids_initially_none(self):
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
||||||
|
HiggsAudioV3Config,
|
||||||
|
)
|
||||||
|
|
||||||
|
config = HiggsAudioV3Config()
|
||||||
|
assert config.tts_token_id is None
|
||||||
|
assert config.text_token_id is None
|
||||||
|
assert config.audio_continuation_id is None
|
||||||
|
|
||||||
|
def test_hidden_size_from_text_config(self):
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
||||||
|
HiggsAudioV3Config,
|
||||||
|
)
|
||||||
|
|
||||||
|
config = HiggsAudioV3Config(text_config={"model_type": "qwen3", "hidden_size": 2560})
|
||||||
|
assert config.hidden_size == 2560
|
||||||
|
assert config.audio_hidden_size == 2560
|
||||||
|
|
||||||
|
|
||||||
|
# ---- AC-4: Fused Multi-Codebook Modules ----
|
||||||
|
|
||||||
|
|
||||||
|
class TestFusedModules:
|
||||||
|
def test_embedding_shape(self):
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
||||||
|
HiggsFusedMultiTextEmbedding,
|
||||||
|
)
|
||||||
|
|
||||||
|
embed = HiggsFusedMultiTextEmbedding(num_codebooks=8, vocab_size=1026, hidden_size=256)
|
||||||
|
assert embed.weight.shape == (8 * 1026, 256)
|
||||||
|
|
||||||
|
codes = torch.randint(0, 1024, (5, 8))
|
||||||
|
out = embed(codes)
|
||||||
|
assert out.shape == (5, 256)
|
||||||
|
|
||||||
|
def test_embedding_offset_indexing(self):
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
||||||
|
HiggsFusedMultiTextEmbedding,
|
||||||
|
)
|
||||||
|
|
||||||
|
embed = HiggsFusedMultiTextEmbedding(num_codebooks=2, vocab_size=4, hidden_size=3)
|
||||||
|
# Set weights to identity-like pattern so we can verify offsets
|
||||||
|
with torch.no_grad():
|
||||||
|
embed.weight.zero_()
|
||||||
|
# Codebook 0, vocab [0..3] -> row 0..3
|
||||||
|
# Codebook 1, vocab [0..3] -> row 4..7
|
||||||
|
for i in range(8):
|
||||||
|
embed.weight[i, i % 3] = float(i)
|
||||||
|
|
||||||
|
# codes[0] = [1, 2] -> embed[1] + embed[4+2=6]
|
||||||
|
codes = torch.tensor([[1, 2]])
|
||||||
|
out = embed(codes)
|
||||||
|
expected = embed.weight[1] + embed.weight[6]
|
||||||
|
assert torch.allclose(out[0], expected)
|
||||||
|
|
||||||
|
def test_head_shape(self):
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
||||||
|
HiggsFusedMultiTextHead,
|
||||||
|
)
|
||||||
|
|
||||||
|
head = HiggsFusedMultiTextHead(num_codebooks=8, vocab_size=1026, hidden_size=256)
|
||||||
|
assert head.weight.shape == (8 * 1026, 256)
|
||||||
|
|
||||||
|
hidden = torch.randn(3, 256)
|
||||||
|
out = head.generate(hidden)
|
||||||
|
assert out.shape == (3, 8, 1026)
|
||||||
|
|
||||||
|
def test_tying(self):
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
||||||
|
HiggsFusedMultiTextEmbedding,
|
||||||
|
HiggsFusedMultiTextHead,
|
||||||
|
)
|
||||||
|
|
||||||
|
embed = HiggsFusedMultiTextEmbedding(num_codebooks=8, vocab_size=1026, hidden_size=64)
|
||||||
|
head = HiggsFusedMultiTextHead(num_codebooks=8, vocab_size=1026, hidden_size=64)
|
||||||
|
head.weight = embed.weight
|
||||||
|
assert head.weight is embed.weight
|
||||||
|
# Modification to one should reflect in the other
|
||||||
|
with torch.no_grad():
|
||||||
|
embed.weight[0, 0] = 42.0
|
||||||
|
assert head.weight[0, 0] == 42.0
|
||||||
|
|
||||||
|
def test_codes_out_of_range(self):
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
||||||
|
HiggsFusedMultiTextEmbedding,
|
||||||
|
)
|
||||||
|
|
||||||
|
embed = HiggsFusedMultiTextEmbedding(num_codebooks=8, vocab_size=1026, hidden_size=16)
|
||||||
|
# Code 1026 + offset for codebook 7 = 1026 + 7*1026 = 8208
|
||||||
|
# But weight only has 8*1026 = 8208 rows, so index 8208 is out of bounds
|
||||||
|
codes = torch.tensor([[0, 0, 0, 0, 0, 0, 0, 1026]]) # Last cb: 1026 is OOB
|
||||||
|
with pytest.raises(IndexError):
|
||||||
|
embed(codes)
|
||||||
|
|
||||||
|
|
||||||
|
# ---- AC-5: Delay Pattern Behavior ----
|
||||||
|
|
||||||
|
|
||||||
|
class TestDelayPatternBehavior:
|
||||||
|
"""Test the delay pattern masking logic extracted from the talker."""
|
||||||
|
|
||||||
|
def test_boc_eoc_ids(self):
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
||||||
|
BOC_ID,
|
||||||
|
EOC_ID,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert BOC_ID == 1024
|
||||||
|
assert EOC_ID == 1025
|
||||||
|
|
||||||
|
def test_delay_phase_boc_masking(self):
|
||||||
|
"""During delay phase, codebooks beyond delay_count are forced to BOC."""
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import BOC_ID
|
||||||
|
|
||||||
|
num_codebooks = 8
|
||||||
|
# Simulate delay phase at step 3 (delay_count=2, so CBs 3-7 should be BOC)
|
||||||
|
codes = torch.randint(0, 1024, (num_codebooks,))
|
||||||
|
delay_count = 2
|
||||||
|
next_cb = delay_count + 1
|
||||||
|
if next_cb < num_codebooks:
|
||||||
|
codes[next_cb:] = BOC_ID
|
||||||
|
# CBs 0-2 should have original codes, CBs 3-7 should be BOC
|
||||||
|
assert all(codes[i] != BOC_ID or i >= 3 for i in range(num_codebooks))
|
||||||
|
assert all(codes[i] == BOC_ID for i in range(3, num_codebooks))
|
||||||
|
|
||||||
|
def test_cb0_eoc_triggers_rampdown(self):
|
||||||
|
"""EOC on codebook 0 starts ramp-down; EOC on other codebooks does not."""
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
||||||
|
EOC_ID,
|
||||||
|
)
|
||||||
|
|
||||||
|
num_codebooks = 8
|
||||||
|
codes = torch.randint(0, 1024, (num_codebooks,))
|
||||||
|
|
||||||
|
# Simulate: cb0 emits EOC
|
||||||
|
codes[0] = EOC_ID
|
||||||
|
assert int(codes[0].item()) == EOC_ID
|
||||||
|
# Ramp-down should start with countdown = N-2 = 6
|
||||||
|
eoc_countdown = num_codebooks - 2
|
||||||
|
assert eoc_countdown == 6
|
||||||
|
|
||||||
|
# If cb3 emits EOC but cb0 doesn't, no ramp-down
|
||||||
|
codes2 = torch.randint(0, 1024, (num_codebooks,))
|
||||||
|
codes2[3] = EOC_ID
|
||||||
|
assert int(codes2[0].item()) != EOC_ID # cb0 is not EOC
|
||||||
|
|
||||||
|
def test_rampdown_termination(self):
|
||||||
|
"""After N-2 steps of ramp-down, generation_done becomes True."""
|
||||||
|
num_codebooks = 8
|
||||||
|
eoc_countdown = num_codebooks - 2 # = 6
|
||||||
|
generation_done = False
|
||||||
|
for _ in range(6):
|
||||||
|
eoc_countdown -= 1
|
||||||
|
if eoc_countdown <= 0:
|
||||||
|
generation_done = True
|
||||||
|
assert generation_done is True
|
||||||
|
|
||||||
|
def test_all_boc_seed(self):
|
||||||
|
"""First audio step should seed last_codes with all-BOC."""
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import BOC_ID
|
||||||
|
|
||||||
|
seeded = torch.full((8,), BOC_ID, dtype=torch.long)
|
||||||
|
assert seeded.shape == (8,)
|
||||||
|
assert all(seeded[i] == BOC_ID for i in range(8))
|
||||||
|
|
||||||
|
|
||||||
|
# ---- AC-5: Real Sampler Method Tests ----
|
||||||
|
|
||||||
|
|
||||||
|
class TestSamplerMethods:
|
||||||
|
"""Test the actual _sample_audio_codes and _apply_delay_pattern_masking methods."""
|
||||||
|
|
||||||
|
def _make_minimal_talker(self):
|
||||||
|
"""Create a minimal talker-like object with sampler/masking methods."""
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod
|
||||||
|
|
||||||
|
class FakeTalker:
|
||||||
|
num_codebooks = 8
|
||||||
|
codebook_size = 1026
|
||||||
|
_audio_state = {}
|
||||||
|
|
||||||
|
t = FakeTalker()
|
||||||
|
t._sample_audio_codes = mod.HiggsAudioV3TalkerForConditionalGeneration._sample_audio_codes.__get__(t)
|
||||||
|
t._apply_delay_pattern_masking = (
|
||||||
|
mod.HiggsAudioV3TalkerForConditionalGeneration._apply_delay_pattern_masking.__get__(t)
|
||||||
|
)
|
||||||
|
return t
|
||||||
|
|
||||||
|
def test_sample_respects_mask(self):
|
||||||
|
"""Tokens masked to -inf must never be sampled."""
|
||||||
|
t = self._make_minimal_talker()
|
||||||
|
logits = torch.full((10, 1026), float("-inf"))
|
||||||
|
# Only token 500 is allowed for each row
|
||||||
|
logits[:, 500] = 0.0
|
||||||
|
result = t._sample_audio_codes(logits)
|
||||||
|
assert result.shape == (10,)
|
||||||
|
assert (result == 500).all(), f"Expected all 500, got {result.tolist()}"
|
||||||
|
|
||||||
|
def test_sample_all_masked_falls_back_to_argmax(self):
|
||||||
|
"""All-masked row should fall back to argmax (least-negative logit)."""
|
||||||
|
t = self._make_minimal_talker()
|
||||||
|
logits = torch.full((2, 1026), float("-inf"))
|
||||||
|
# Row 0: all masked
|
||||||
|
# Row 1: only token 42 allowed
|
||||||
|
logits[1, 42] = 0.0
|
||||||
|
result = t._sample_audio_codes(logits)
|
||||||
|
assert result.shape == (2,)
|
||||||
|
assert result[1].item() == 42
|
||||||
|
|
||||||
|
def test_delay_masking_forces_boc_during_delay(self):
|
||||||
|
"""During delay phase, codebooks beyond delay_count must have only BOC allowed."""
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import BOC_ID
|
||||||
|
|
||||||
|
t = self._make_minimal_talker()
|
||||||
|
# Simulate delay_count=2 for batch row 0
|
||||||
|
t._audio_state[0] = {"num_delay": 2, "num_remaining_delays": None}
|
||||||
|
cb_logits = torch.zeros(1, 8, 1026) # [1 audio row, 8 codebooks, 1026 vocab]
|
||||||
|
t._apply_delay_pattern_masking(cb_logits, [0])
|
||||||
|
# CBs 3-7 should have only BOC allowed (everything else -inf)
|
||||||
|
for q in range(3, 8):
|
||||||
|
row = cb_logits[0, q]
|
||||||
|
assert row[BOC_ID].item() == 0.0 # BOC kept
|
||||||
|
# All non-BOC should be -inf
|
||||||
|
non_boc = torch.cat([row[:BOC_ID], row[BOC_ID + 1 :]])
|
||||||
|
assert (non_boc == float("-inf")).all(), f"CB{q} has non-inf non-BOC values"
|
||||||
|
|
||||||
|
def test_delay_masking_disallows_boc_for_active_codebooks(self):
|
||||||
|
"""Active codebooks during delay should have BOC disallowed."""
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import BOC_ID
|
||||||
|
|
||||||
|
t = self._make_minimal_talker()
|
||||||
|
t._audio_state[0] = {"num_delay": 3, "num_remaining_delays": None}
|
||||||
|
cb_logits = torch.zeros(1, 8, 1026)
|
||||||
|
t._apply_delay_pattern_masking(cb_logits, [0])
|
||||||
|
# CBs 0-3 are active: BOC should be -inf
|
||||||
|
for q in range(4):
|
||||||
|
assert cb_logits[0, q, BOC_ID].item() == float("-inf")
|
||||||
|
|
||||||
|
def test_delay_masking_only_cb0_allows_eoc(self):
|
||||||
|
"""Only codebook 0 should allow EOC during normal generation."""
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import EOC_ID
|
||||||
|
|
||||||
|
t = self._make_minimal_talker()
|
||||||
|
t._audio_state[0] = {"num_delay": 8, "num_remaining_delays": None}
|
||||||
|
cb_logits = torch.zeros(1, 8, 1026)
|
||||||
|
t._apply_delay_pattern_masking(cb_logits, [0])
|
||||||
|
# CB0 should keep EOC
|
||||||
|
assert cb_logits[0, 0, EOC_ID].item() != float("-inf")
|
||||||
|
# CB1-7 should have EOC masked
|
||||||
|
for q in range(1, 8):
|
||||||
|
assert cb_logits[0, q, EOC_ID].item() == float("-inf"), f"CB{q} EOC not masked"
|
||||||
|
|
||||||
|
def test_rampdown_masking_locks_to_eoc(self):
|
||||||
|
"""During ramp-down, locked codebooks should only allow EOC."""
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
||||||
|
BOC_ID,
|
||||||
|
EOC_ID,
|
||||||
|
)
|
||||||
|
|
||||||
|
t = self._make_minimal_talker()
|
||||||
|
# Ramp-down with 4 remaining delays: lock CBs 0-3 to EOC
|
||||||
|
t._audio_state[0] = {"num_delay": 8, "num_remaining_delays": 4}
|
||||||
|
cb_logits = torch.zeros(1, 8, 1026)
|
||||||
|
t._apply_delay_pattern_masking(cb_logits, [0])
|
||||||
|
# CBs 0-3 locked: only EOC allowed
|
||||||
|
for q in range(4):
|
||||||
|
row = cb_logits[0, q]
|
||||||
|
assert row[EOC_ID].item() == 0.0
|
||||||
|
non_eoc = torch.cat([row[:EOC_ID], row[EOC_ID + 1 :]])
|
||||||
|
assert (non_eoc == float("-inf")).all(), f"Locked CB{q} has non-inf non-EOC"
|
||||||
|
# CBs 4-7 active: BOC and EOC disallowed
|
||||||
|
for q in range(4, 8):
|
||||||
|
assert cb_logits[0, q, BOC_ID].item() == float("-inf")
|
||||||
|
assert cb_logits[0, q, EOC_ID].item() == float("-inf")
|
||||||
|
|
||||||
|
|
||||||
|
# ---- AC-6: Feedback Method Tests ----
|
||||||
|
|
||||||
|
|
||||||
|
class TestFeedbackMethods:
|
||||||
|
def test_postprocess_emits_audio_codes(self):
|
||||||
|
"""postprocess() should return codes from _last_audio_codes."""
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod
|
||||||
|
|
||||||
|
class FakeTalker:
|
||||||
|
_last_audio_codes = torch.tensor([[100, 200, 300, 400, 500, 600, 700, 800]])
|
||||||
|
_postprocess_cursor = 0
|
||||||
|
|
||||||
|
t = FakeTalker()
|
||||||
|
t.postprocess = mod.HiggsAudioV3TalkerForConditionalGeneration.postprocess.__get__(t)
|
||||||
|
result = t.postprocess(torch.zeros(1, 64))
|
||||||
|
assert "codes" in result
|
||||||
|
assert "audio" in result["codes"]
|
||||||
|
assert result["codes"]["audio"].shape == (1, 8)
|
||||||
|
|
||||||
|
def test_postprocess_skips_negative_rows(self):
|
||||||
|
"""postprocess() should skip rows with -1 (no audio)."""
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod
|
||||||
|
|
||||||
|
class FakeTalker:
|
||||||
|
_last_audio_codes = torch.tensor([[-1, -1, -1, -1, -1, -1, -1, -1]])
|
||||||
|
_postprocess_cursor = 0
|
||||||
|
|
||||||
|
t = FakeTalker()
|
||||||
|
t.postprocess = mod.HiggsAudioV3TalkerForConditionalGeneration.postprocess.__get__(t)
|
||||||
|
result = t.postprocess(torch.zeros(1, 64))
|
||||||
|
assert result == {}
|
||||||
|
|
||||||
|
def test_postprocess_advances_cursor(self):
|
||||||
|
"""postprocess() should advance cursor by 1 per call."""
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod
|
||||||
|
|
||||||
|
class FakeTalker:
|
||||||
|
_last_audio_codes = torch.tensor(
|
||||||
|
[[100, 200, 300, 400, 500, 600, 700, 800], [-1, -1, -1, -1, -1, -1, -1, -1]]
|
||||||
|
)
|
||||||
|
_postprocess_cursor = 0
|
||||||
|
|
||||||
|
t = FakeTalker()
|
||||||
|
t.postprocess = mod.HiggsAudioV3TalkerForConditionalGeneration.postprocess.__get__(t)
|
||||||
|
r1 = t.postprocess(torch.zeros(1, 64))
|
||||||
|
assert "codes" in r1
|
||||||
|
r2 = t.postprocess(torch.zeros(1, 64))
|
||||||
|
assert r2 == {} # Second row is all -1
|
||||||
|
|
||||||
|
|
||||||
|
# ---- AC-6: Audio Feedback Method Tests ----
|
||||||
|
|
||||||
|
|
||||||
|
class TestAudioFeedback:
|
||||||
|
"""Test _apply_audio_feedback with minimal fake talker."""
|
||||||
|
|
||||||
|
def _make_feedback_talker(self):
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod
|
||||||
|
|
||||||
|
embed = mod.HiggsFusedMultiTextEmbedding(num_codebooks=8, vocab_size=1026, hidden_size=16)
|
||||||
|
torch.nn.init.ones_(embed.weight)
|
||||||
|
|
||||||
|
class FakeTalker:
|
||||||
|
num_codebooks = 8
|
||||||
|
codebook_size = 1026
|
||||||
|
_audio_continuation_id = 99999 # fake audio token
|
||||||
|
_last_step_query_start_loc = None
|
||||||
|
_audio_state = {}
|
||||||
|
multimodal_embedding = embed
|
||||||
|
model = type("M", (), {"embed_tokens": lambda self, ids: torch.zeros(ids.shape[0], 16)})()
|
||||||
|
|
||||||
|
t = FakeTalker()
|
||||||
|
t._apply_audio_feedback = mod.HiggsAudioV3TalkerForConditionalGeneration._apply_audio_feedback.__get__(t)
|
||||||
|
return t
|
||||||
|
|
||||||
|
def test_text_positions_unchanged(self):
|
||||||
|
"""Non-audio positions should not be modified."""
|
||||||
|
t = self._make_feedback_talker()
|
||||||
|
# No audio state → all positions unchanged
|
||||||
|
input_ids = torch.tensor([1, 2, 3, 4])
|
||||||
|
hidden = torch.randn(4, 16)
|
||||||
|
result = t._apply_audio_feedback(hidden, input_ids)
|
||||||
|
assert torch.equal(result, hidden)
|
||||||
|
|
||||||
|
def test_audio_position_replaced_with_state(self):
|
||||||
|
"""Audio position with state should have its embedding replaced."""
|
||||||
|
t = self._make_feedback_talker()
|
||||||
|
audio_id = t._audio_continuation_id
|
||||||
|
t._audio_state[1] = {"last_codes": torch.zeros(8, dtype=torch.long)}
|
||||||
|
input_ids = torch.tensor([1, audio_id, 3])
|
||||||
|
hidden = torch.zeros(3, 16)
|
||||||
|
result = t._apply_audio_feedback(hidden, input_ids)
|
||||||
|
# Position 1 (audio) should be non-zero (embedding of all-0 codes)
|
||||||
|
assert result[1].abs().sum() > 0
|
||||||
|
# Positions 0 and 2 should remain zero
|
||||||
|
assert result[0].abs().sum() == 0
|
||||||
|
assert result[2].abs().sum() == 0
|
||||||
|
|
||||||
|
def test_no_state_no_replacement(self):
|
||||||
|
"""Audio position without state should not be replaced."""
|
||||||
|
t = self._make_feedback_talker()
|
||||||
|
audio_id = t._audio_continuation_id
|
||||||
|
# No state for row 0
|
||||||
|
input_ids = torch.tensor([audio_id])
|
||||||
|
hidden = torch.zeros(1, 16)
|
||||||
|
result = t._apply_audio_feedback(hidden, input_ids)
|
||||||
|
assert torch.equal(result, hidden)
|
||||||
|
|
||||||
|
|
||||||
|
# ---- AC-8: Codec Strictness ----
|
||||||
|
|
||||||
|
|
||||||
|
class TestCodecStrictness:
|
||||||
|
def test_bundled_missing_quantizer_key_raises(self):
|
||||||
|
"""Bundled codec load must fail when a quantizer codebook key is missing."""
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
||||||
|
HiggsAudioV3Config,
|
||||||
|
)
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_code2wav import (
|
||||||
|
HiggsAudioV3Code2Wav,
|
||||||
|
)
|
||||||
|
|
||||||
|
config = HiggsAudioV3Config()
|
||||||
|
c2w = HiggsAudioV3Code2Wav(config=config)
|
||||||
|
|
||||||
|
# State with only 7 quantizers (missing quantizer.quantizers.7.*)
|
||||||
|
codec_state = {}
|
||||||
|
for i in range(7):
|
||||||
|
codec_state[f"quantizer.quantizers.{i}.codebook.embed"] = torch.randn(1024, 64)
|
||||||
|
codec_state[f"quantizer.quantizers.{i}.project_out.weight"] = torch.randn(1024, 64)
|
||||||
|
codec_state[f"quantizer.quantizers.{i}.project_out.bias"] = torch.randn(1024)
|
||||||
|
codec_state["fc2.weight"] = torch.randn(256, 1024)
|
||||||
|
codec_state["fc2.bias"] = torch.randn(256)
|
||||||
|
|
||||||
|
with pytest.raises(KeyError, match="quantizer 7"):
|
||||||
|
c2w._load_from_bundled_state(codec_state, device=torch.device("cpu"))
|
||||||
|
|
||||||
|
def test_bundled_missing_fc2_raises(self):
|
||||||
|
"""Bundled codec load must fail when fc2 keys are missing."""
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
||||||
|
HiggsAudioV3Config,
|
||||||
|
)
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_code2wav import (
|
||||||
|
HiggsAudioV3Code2Wav,
|
||||||
|
)
|
||||||
|
|
||||||
|
config = HiggsAudioV3Config()
|
||||||
|
c2w = HiggsAudioV3Code2Wav(config=config)
|
||||||
|
|
||||||
|
codec_state = {}
|
||||||
|
for i in range(8):
|
||||||
|
codec_state[f"quantizer.quantizers.{i}.codebook.embed"] = torch.randn(1024, 64)
|
||||||
|
codec_state[f"quantizer.quantizers.{i}.project_out.weight"] = torch.randn(1024, 64)
|
||||||
|
codec_state[f"quantizer.quantizers.{i}.project_out.bias"] = torch.randn(1024)
|
||||||
|
# No fc2 keys
|
||||||
|
|
||||||
|
with pytest.raises(KeyError, match="fc2"):
|
||||||
|
c2w._load_from_bundled_state(codec_state, device=torch.device("cpu"))
|
||||||
|
|
||||||
|
def test_bundled_unknown_decoder_key_rejected(self, monkeypatch):
|
||||||
|
"""Unknown decoder-side key should be rejected as unconsumed."""
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_code2wav as c2w_mod
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
||||||
|
HiggsAudioV3Config,
|
||||||
|
)
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_code2wav import (
|
||||||
|
HiggsAudioV3Code2Wav,
|
||||||
|
)
|
||||||
|
|
||||||
|
class FakeDAC(torch.nn.Module):
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__()
|
||||||
|
self.dummy = torch.nn.Parameter(torch.zeros(1))
|
||||||
|
|
||||||
|
monkeypatch.setattr(c2w_mod, "build_boson_dac_decoder", lambda device: FakeDAC().to(device))
|
||||||
|
monkeypatch.setattr(c2w_mod, "build_higgs_audio_acoustic_decoder", lambda cfg, device: FakeDAC().to(device))
|
||||||
|
|
||||||
|
config = HiggsAudioV3Config()
|
||||||
|
c2w = HiggsAudioV3Code2Wav(config=config)
|
||||||
|
|
||||||
|
codec_state = _build_full_codec_state()
|
||||||
|
codec_state["unknown_decoder_module.weight"] = torch.randn(10)
|
||||||
|
|
||||||
|
with pytest.raises(RuntimeError, match="unexpected decoder-side keys"):
|
||||||
|
c2w._load_from_bundled_state(codec_state, device=torch.device("cpu"))
|
||||||
|
|
||||||
|
def test_bundled_encoder_side_keys_accepted(self, monkeypatch):
|
||||||
|
"""Known encoder-side keys should not trigger unconsumed-key rejection."""
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_code2wav as c2w_mod
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
||||||
|
HiggsAudioV3Config,
|
||||||
|
)
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_code2wav import (
|
||||||
|
HiggsAudioV3Code2Wav,
|
||||||
|
)
|
||||||
|
|
||||||
|
class FakeDAC(torch.nn.Module):
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__()
|
||||||
|
self.dummy = torch.nn.Parameter(torch.zeros(1))
|
||||||
|
|
||||||
|
monkeypatch.setattr(c2w_mod, "build_boson_dac_decoder", lambda device: FakeDAC().to(device))
|
||||||
|
monkeypatch.setattr(c2w_mod, "build_higgs_audio_acoustic_decoder", lambda cfg, device: FakeDAC().to(device))
|
||||||
|
|
||||||
|
config = HiggsAudioV3Config()
|
||||||
|
c2w = HiggsAudioV3Code2Wav(config=config)
|
||||||
|
|
||||||
|
codec_state = _build_full_codec_state()
|
||||||
|
codec_state["acoustic_encoder.block.0.conv_t1.weight"] = torch.randn(10)
|
||||||
|
codec_state["semantic_model.encoder.layers.0.weight"] = torch.randn(10)
|
||||||
|
codec_state["fc.weight"] = torch.randn(10)
|
||||||
|
|
||||||
|
# Should NOT raise — encoder-side keys are allowed
|
||||||
|
c2w._load_from_bundled_state(codec_state, device=torch.device("cpu"))
|
||||||
|
assert c2w._loaded
|
||||||
|
|
||||||
|
|
||||||
|
def _build_minimal_codec_state() -> dict[str, torch.Tensor]:
|
||||||
|
"""Build a minimal codec state dict (quantizer + fc2, no DAC)."""
|
||||||
|
state: dict[str, torch.Tensor] = {}
|
||||||
|
for i in range(8):
|
||||||
|
state[f"quantizer.quantizers.{i}.codebook.embed"] = torch.randn(1024, 64)
|
||||||
|
state[f"quantizer.quantizers.{i}.project_out.weight"] = torch.randn(1024, 64)
|
||||||
|
state[f"quantizer.quantizers.{i}.project_out.bias"] = torch.randn(1024)
|
||||||
|
state["fc2.weight"] = torch.randn(256, 1024)
|
||||||
|
state["fc2.bias"] = torch.randn(256)
|
||||||
|
return state
|
||||||
|
|
||||||
|
|
||||||
|
def _build_full_codec_state() -> dict[str, torch.Tensor]:
|
||||||
|
"""Build a full codec state dict including acoustic_decoder keys.
|
||||||
|
|
||||||
|
The acoustic_decoder keys here match what a FakeDAC module expects
|
||||||
|
(just 'dummy'). Real DAC has many more keys.
|
||||||
|
"""
|
||||||
|
state = _build_minimal_codec_state()
|
||||||
|
state["acoustic_decoder.dummy"] = torch.zeros(1)
|
||||||
|
return state
|
||||||
|
|
||||||
|
|
||||||
|
# ---- AC-2: Loader Required-Key Validation ----
|
||||||
|
|
||||||
|
|
||||||
|
class TestLoaderRequiredKeys:
|
||||||
|
"""Test exact required-key validation in load_weights."""
|
||||||
|
|
||||||
|
def test_build_required_keys_count(self):
|
||||||
|
"""Required key set should have 1 text_emb + 36*11 layers + 1 norm + 1 modality = 399."""
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
||||||
|
HiggsAudioV3TalkerForConditionalGeneration,
|
||||||
|
)
|
||||||
|
|
||||||
|
keys = HiggsAudioV3TalkerForConditionalGeneration._build_required_keys(36)
|
||||||
|
assert len(keys) == 1 + 36 * 11 + 1 + 1 # text + layers + norm + modality = 399
|
||||||
|
|
||||||
|
def test_build_required_keys_contains_norm(self):
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
||||||
|
HiggsAudioV3TalkerForConditionalGeneration,
|
||||||
|
)
|
||||||
|
|
||||||
|
keys = HiggsAudioV3TalkerForConditionalGeneration._build_required_keys(2)
|
||||||
|
assert "body.norm.weight" in keys
|
||||||
|
|
||||||
|
def test_build_required_keys_contains_layer_subkeys(self):
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
||||||
|
HiggsAudioV3TalkerForConditionalGeneration,
|
||||||
|
)
|
||||||
|
|
||||||
|
keys = HiggsAudioV3TalkerForConditionalGeneration._build_required_keys(2)
|
||||||
|
assert "body.layers.0.self_attn.k_proj.weight" in keys
|
||||||
|
assert "body.layers.1.mlp.gate_proj.weight" in keys
|
||||||
|
|
||||||
|
def test_build_required_keys_contains_modality_embedding(self):
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
||||||
|
HiggsAudioV3TalkerForConditionalGeneration,
|
||||||
|
)
|
||||||
|
|
||||||
|
keys = HiggsAudioV3TalkerForConditionalGeneration._build_required_keys(1)
|
||||||
|
assert "tied.embedding.modality_embeddings.0.embedding.weight" in keys
|
||||||
|
|
||||||
|
def test_build_required_keys_contains_text_embedding(self):
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
||||||
|
HiggsAudioV3TalkerForConditionalGeneration,
|
||||||
|
)
|
||||||
|
|
||||||
|
keys = HiggsAudioV3TalkerForConditionalGeneration._build_required_keys(1)
|
||||||
|
assert "tied.embedding.text_embedding.weight" in keys
|
||||||
|
|
||||||
|
|
||||||
|
class TestLoadWeightsEnforcement:
|
||||||
|
"""Test load_weights() itself with synthetic key sets via a minimal fake talker."""
|
||||||
|
|
||||||
|
def _make_fake_talker_and_keys(self):
|
||||||
|
"""Build a minimal talker-like object and the full 1-layer key set."""
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod
|
||||||
|
|
||||||
|
# Minimal fake talker with just enough to run load_weights
|
||||||
|
talker = object.__new__(mod.HiggsAudioV3TalkerForConditionalGeneration)
|
||||||
|
torch.nn.Module.__init__(talker)
|
||||||
|
|
||||||
|
# Attach minimal attributes load_weights needs
|
||||||
|
talker.multimodal_embedding = mod.HiggsFusedMultiTextEmbedding(8, 1026, 16)
|
||||||
|
talker.modality_head = mod.HiggsFusedMultiTextHead(8, 1026, 16)
|
||||||
|
talker.tie_modality = True
|
||||||
|
talker.modality_head.weight = talker.multimodal_embedding.weight
|
||||||
|
|
||||||
|
# Fake backbone config with 1 layer
|
||||||
|
class FakeConfig:
|
||||||
|
num_hidden_layers = 1
|
||||||
|
tie_word_embeddings = True
|
||||||
|
|
||||||
|
talker._backbone_config = FakeConfig()
|
||||||
|
|
||||||
|
# Minimal model and lm_head so _BackboneWrapper can be constructed
|
||||||
|
talker.model = torch.nn.Module()
|
||||||
|
talker.lm_head = torch.nn.Module()
|
||||||
|
|
||||||
|
# Fake vllm_config for resolve_special_tokens (no-op)
|
||||||
|
class FakeModelConfig:
|
||||||
|
model = None
|
||||||
|
|
||||||
|
class FakeVllmConfig:
|
||||||
|
model_config = FakeModelConfig()
|
||||||
|
|
||||||
|
talker.vllm_config = FakeVllmConfig()
|
||||||
|
talker.config = type("C", (), {"resolve_special_tokens": lambda self, p: None})()
|
||||||
|
talker._resolved_tokens = True
|
||||||
|
talker._audio_continuation_id = None
|
||||||
|
talker._eos_token_id = None
|
||||||
|
|
||||||
|
# Build the full required key set for 1 layer
|
||||||
|
required = mod.HiggsAudioV3TalkerForConditionalGeneration._build_required_keys(1)
|
||||||
|
|
||||||
|
# Build synthetic weights: tiny tensors for every required key
|
||||||
|
weights = {}
|
||||||
|
for key in required:
|
||||||
|
if "modality_embeddings.0.embedding" in key:
|
||||||
|
weights[key] = torch.randn(8 * 1026, 16)
|
||||||
|
else:
|
||||||
|
weights[key] = torch.randn(4) # tiny placeholder
|
||||||
|
|
||||||
|
# Monkeypatch _BackboneWrapper.load_weights to accept and return keys
|
||||||
|
def fake_backbone_load(self_wrapper, ws):
|
||||||
|
consumed = set()
|
||||||
|
for name, _ in ws:
|
||||||
|
consumed.add(name)
|
||||||
|
return consumed
|
||||||
|
|
||||||
|
talker._fake_backbone_load = fake_backbone_load
|
||||||
|
return talker, weights, mod, fake_backbone_load
|
||||||
|
|
||||||
|
def test_load_weights_succeeds_with_all_keys(self, monkeypatch):
|
||||||
|
"""load_weights succeeds when all required keys are present."""
|
||||||
|
talker, weights, mod, fake_load = self._make_fake_talker_and_keys()
|
||||||
|
monkeypatch.setattr(mod._BackboneWrapper, "load_weights", fake_load)
|
||||||
|
result = talker.load_weights(iter(weights.items()))
|
||||||
|
assert isinstance(result, set)
|
||||||
|
assert "multimodal_embedding.weight" in result
|
||||||
|
|
||||||
|
def test_load_weights_fails_missing_norm(self, monkeypatch):
|
||||||
|
"""load_weights raises when body.norm.weight is missing."""
|
||||||
|
talker, weights, mod, fake_load = self._make_fake_talker_and_keys()
|
||||||
|
monkeypatch.setattr(mod._BackboneWrapper, "load_weights", fake_load)
|
||||||
|
del weights["body.norm.weight"]
|
||||||
|
with pytest.raises(RuntimeError, match="body.norm.weight"):
|
||||||
|
talker.load_weights(iter(weights.items()))
|
||||||
|
|
||||||
|
def test_load_weights_fails_missing_layer_subkey(self, monkeypatch):
|
||||||
|
"""load_weights raises when a specific layer subkey is missing."""
|
||||||
|
talker, weights, mod, fake_load = self._make_fake_talker_and_keys()
|
||||||
|
monkeypatch.setattr(mod._BackboneWrapper, "load_weights", fake_load)
|
||||||
|
del weights["body.layers.0.self_attn.k_proj.weight"]
|
||||||
|
with pytest.raises(RuntimeError, match="body.layers.0.self_attn.k_proj.weight"):
|
||||||
|
talker.load_weights(iter(weights.items()))
|
||||||
|
|
||||||
|
def test_load_weights_fails_missing_modality_embedding(self, monkeypatch):
|
||||||
|
"""load_weights raises when modality embedding is missing."""
|
||||||
|
talker, weights, mod, fake_load = self._make_fake_talker_and_keys()
|
||||||
|
monkeypatch.setattr(mod._BackboneWrapper, "load_weights", fake_load)
|
||||||
|
del weights["tied.embedding.modality_embeddings.0.embedding.weight"]
|
||||||
|
with pytest.raises(RuntimeError, match="modality_embeddings"):
|
||||||
|
talker.load_weights(iter(weights.items()))
|
||||||
|
|
||||||
|
def test_load_weights_fails_unknown_prefix(self, monkeypatch):
|
||||||
|
"""load_weights raises on unknown non-codec checkpoint prefix."""
|
||||||
|
talker, weights, mod, fake_load = self._make_fake_talker_and_keys()
|
||||||
|
monkeypatch.setattr(mod._BackboneWrapper, "load_weights", fake_load)
|
||||||
|
weights["unknown.prefix.weight"] = torch.randn(4)
|
||||||
|
with pytest.raises(ValueError, match="Unexpected checkpoint key"):
|
||||||
|
talker.load_weights(iter(weights.items()))
|
||||||
|
|
||||||
|
|
||||||
|
# ---- AC-7: Stage Input Processor ----
|
||||||
|
|
||||||
|
|
||||||
|
class TestStageInputProcessor:
|
||||||
|
def test_revert_delay_pattern(self):
|
||||||
|
from vllm_omni.model_executor.stage_input_processors.higgs_audio_v3 import (
|
||||||
|
_revert_delay_pattern,
|
||||||
|
)
|
||||||
|
|
||||||
|
# 8 codebooks, 3 real frames -> delayed shape [8, 3+8-1=10]
|
||||||
|
Q, T = 8, 3
|
||||||
|
delayed = torch.full((Q, T + Q - 1), 1024) # Fill with BOC
|
||||||
|
for i in range(Q):
|
||||||
|
for t in range(T):
|
||||||
|
delayed[i, i + t] = i * 100 + t # Real codes at shifted positions
|
||||||
|
result = _revert_delay_pattern(delayed)
|
||||||
|
assert result.shape == (Q, T)
|
||||||
|
for i in range(Q):
|
||||||
|
for t in range(T):
|
||||||
|
assert result[i, t].item() == i * 100 + t
|
||||||
|
|
||||||
|
def test_revert_delay_pattern_rejects_wrong_codebooks(self):
|
||||||
|
from vllm_omni.model_executor.stage_input_processors.higgs_audio_v3 import (
|
||||||
|
_revert_delay_pattern,
|
||||||
|
)
|
||||||
|
|
||||||
|
# 7 codebooks should be rejected
|
||||||
|
codes = torch.zeros(7, 20)
|
||||||
|
with pytest.raises(ValueError, match="Expected exactly 8 codebook rows"):
|
||||||
|
_revert_delay_pattern(codes)
|
||||||
|
|
||||||
|
def test_revert_delay_pattern_rejects_too_few_frames(self):
|
||||||
|
from vllm_omni.model_executor.stage_input_processors.higgs_audio_v3 import (
|
||||||
|
_revert_delay_pattern,
|
||||||
|
)
|
||||||
|
|
||||||
|
# 8 codebooks but only 5 frames (need at least 8)
|
||||||
|
codes = torch.zeros(8, 5)
|
||||||
|
with pytest.raises(ValueError, match="Not enough frames"):
|
||||||
|
_revert_delay_pattern(codes)
|
||||||
|
|
||||||
|
def test_filter_real_code_frames(self):
|
||||||
|
from vllm_omni.model_executor.stage_input_processors.higgs_audio_v3 import (
|
||||||
|
_filter_real_code_frames,
|
||||||
|
)
|
||||||
|
|
||||||
|
# 8 codebooks, 4 frames
|
||||||
|
codes = torch.tensor(
|
||||||
|
[
|
||||||
|
[100, 200, 1024, 300], # cb0: frame 2 has BOC
|
||||||
|
[101, 201, 1024, 301],
|
||||||
|
[102, 202, 1024, 302],
|
||||||
|
[103, 203, 1024, 303],
|
||||||
|
[104, 204, 1024, 304],
|
||||||
|
[105, 205, 1024, 305],
|
||||||
|
[106, 206, 1024, 306],
|
||||||
|
[107, 207, 1024, 307],
|
||||||
|
]
|
||||||
|
)
|
||||||
|
result = _filter_real_code_frames(codes)
|
||||||
|
# Frame 2 (column 2) has BOC in all codebooks -> filtered out
|
||||||
|
assert result.shape == (8, 3)
|
||||||
|
assert result[0, 0].item() == 100
|
||||||
|
assert result[0, 1].item() == 200
|
||||||
|
assert result[0, 2].item() == 300
|
||||||
|
|
||||||
|
|
||||||
|
# ---- AC-10: Registry ----
|
||||||
|
|
||||||
|
|
||||||
|
class TestRegistry:
|
||||||
|
def test_talker_registered(self):
|
||||||
|
from vllm_omni.model_executor.models.registry import _OMNI_MODELS
|
||||||
|
|
||||||
|
assert "HiggsMultimodalQwen3ForConditionalGeneration" in _OMNI_MODELS
|
||||||
|
assert "HiggsAudioV3TalkerForConditionalGeneration" in _OMNI_MODELS
|
||||||
|
|
||||||
|
def test_code2wav_registered(self):
|
||||||
|
from vllm_omni.model_executor.models.registry import _OMNI_MODELS
|
||||||
|
|
||||||
|
assert "HiggsAudioV3Code2WavForConditionalGeneration" in _OMNI_MODELS
|
||||||
|
|
||||||
|
def test_pipeline_registered(self):
|
||||||
|
from vllm_omni.config.pipeline_registry import _OMNI_PIPELINES
|
||||||
|
|
||||||
|
assert "higgs_multimodal_qwen3" in _OMNI_PIPELINES
|
||||||
|
|
||||||
|
def test_deploy_yaml_exists(self):
|
||||||
|
import os
|
||||||
|
|
||||||
|
# __file__ = tests/unit/higgs_audio_v3/test_*.py → 4 dirnames to repo root
|
||||||
|
repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
|
||||||
|
yaml_path = os.path.join(repo_root, "vllm_omni", "deploy", "higgs_multimodal_qwen3.yaml")
|
||||||
|
assert os.path.isfile(yaml_path), f"Deploy YAML not found at {yaml_path}"
|
||||||
|
|
||||||
|
|
||||||
|
# ---- AC-3: Prompt Builder ----
|
||||||
|
|
||||||
|
|
||||||
|
class TestPromptBuilder:
|
||||||
|
def _make_mock_tokenizer(self):
|
||||||
|
"""Create a mock tokenizer with the required special tokens."""
|
||||||
|
|
||||||
|
class MockTokenizer:
|
||||||
|
def __init__(self):
|
||||||
|
self._added_vocab = {
|
||||||
|
"<|tts|>": 151700,
|
||||||
|
"<|text|>": 151701,
|
||||||
|
"<|audio|>": 151702,
|
||||||
|
"<|ref_audio|>": 151703,
|
||||||
|
"<|ref_text|>": 151704,
|
||||||
|
}
|
||||||
|
|
||||||
|
def get_added_vocab(self):
|
||||||
|
return self._added_vocab
|
||||||
|
|
||||||
|
def encode(self, text, add_special_tokens=True):
|
||||||
|
# Simple word-level tokenization for testing
|
||||||
|
return list(range(100, 100 + len(text.split())))
|
||||||
|
|
||||||
|
return MockTokenizer()
|
||||||
|
|
||||||
|
def test_plain_tts_prompt(self):
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_tokenizer import (
|
||||||
|
HiggsAudioV3TokenizerAdapter,
|
||||||
|
)
|
||||||
|
|
||||||
|
tok = self._make_mock_tokenizer()
|
||||||
|
adapter = HiggsAudioV3TokenizerAdapter(tok)
|
||||||
|
ids = adapter.build_prompt("Hello world")
|
||||||
|
assert ids[0] == 151700 # <|tts|>
|
||||||
|
assert ids[1] == 151701 # <|text|>
|
||||||
|
assert ids[-1] == 151702 # <|audio|>
|
||||||
|
assert len(ids) == 2 + 2 + 1 # tts + text + 2 word tokens + audio
|
||||||
|
|
||||||
|
def test_empty_text_rejected(self):
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_tokenizer import (
|
||||||
|
HiggsAudioV3TokenizerAdapter,
|
||||||
|
)
|
||||||
|
|
||||||
|
tok = self._make_mock_tokenizer()
|
||||||
|
adapter = HiggsAudioV3TokenizerAdapter(tok)
|
||||||
|
with pytest.raises(ValueError, match="non-empty"):
|
||||||
|
adapter.build_prompt("")
|
||||||
|
|
||||||
|
def test_whitespace_only_rejected(self):
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_tokenizer import (
|
||||||
|
HiggsAudioV3TokenizerAdapter,
|
||||||
|
)
|
||||||
|
|
||||||
|
tok = self._make_mock_tokenizer()
|
||||||
|
adapter = HiggsAudioV3TokenizerAdapter(tok)
|
||||||
|
with pytest.raises(ValueError, match="non-empty"):
|
||||||
|
adapter.build_prompt(" ")
|
||||||
|
|
||||||
|
def test_missing_specials_rejected(self):
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_tokenizer import (
|
||||||
|
HiggsAudioV3TokenizerAdapter,
|
||||||
|
)
|
||||||
|
|
||||||
|
class BadTokenizer:
|
||||||
|
def get_added_vocab(self):
|
||||||
|
return {"<|tts|>": 1} # Missing <|text|> and <|audio|>
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="missing"):
|
||||||
|
HiggsAudioV3TokenizerAdapter(BadTokenizer())
|
||||||
|
|
||||||
|
def test_no_voice_clone_tokens_in_plain_tts(self):
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_tokenizer import (
|
||||||
|
HiggsAudioV3TokenizerAdapter,
|
||||||
|
)
|
||||||
|
|
||||||
|
tok = self._make_mock_tokenizer()
|
||||||
|
adapter = HiggsAudioV3TokenizerAdapter(tok)
|
||||||
|
ids = adapter.build_prompt("Hello")
|
||||||
|
# Should not contain ref_audio or ref_text token IDs
|
||||||
|
assert 151703 not in ids # <|ref_audio|>
|
||||||
|
assert 151704 not in ids # <|ref_text|>
|
||||||
@@ -44,6 +44,14 @@ def test_resolve_pooler_payload_req_ids_downstream_stage_uses_filtered_requests(
|
|||||||
assert payload_req_ids == ["r2"]
|
assert payload_req_ids == ["r2"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_sparse_mm_req_ids_requires_sparse_audio_marker():
|
||||||
|
assert GPUARModelRunner._sparse_mm_req_ids({"meta": {"req_id": ["r1"]}}) is None
|
||||||
|
assert GPUARModelRunner._sparse_mm_req_ids({"meta.req_id": ["r1"]}) is None
|
||||||
|
|
||||||
|
assert GPUARModelRunner._sparse_mm_req_ids({"meta": {"req_id": ["r1"], "sparse_audio": ["1"]}}) == ["r1"]
|
||||||
|
assert GPUARModelRunner._sparse_mm_req_ids({"meta.req_id": ["r1"], "meta.sparse_audio": ["1"]}) == ["r1"]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("query_start_loc_attr", ["method", "tensor_attr"])
|
@pytest.mark.parametrize("query_start_loc_attr", ["method", "tensor_attr"])
|
||||||
def test_sample_tokens_tail_only_prefix_cache_uses_staged_cpu_hidden_states(monkeypatch, query_start_loc_attr):
|
def test_sample_tokens_tail_only_prefix_cache_uses_staged_cpu_hidden_states(monkeypatch, query_start_loc_attr):
|
||||||
runner = object.__new__(GPUARModelRunner)
|
runner = object.__new__(GPUARModelRunner)
|
||||||
|
|||||||
@@ -1,16 +0,0 @@
|
|||||||
import numpy as np
|
|
||||||
from vllm.assets.video import VideoAsset
|
|
||||||
from vllm.multimodal.media.audio import load_audio
|
|
||||||
|
|
||||||
|
|
||||||
def extract_video_audio(path: str = None, sampling_rate: int = 16000) -> np.ndarray:
|
|
||||||
"""This function extracts the audio from a video file path and returns the audio as a numpy array.
|
|
||||||
Args:
|
|
||||||
path: The path to the video file.
|
|
||||||
Returns:
|
|
||||||
The audio as a numpy array.
|
|
||||||
"""
|
|
||||||
if not path:
|
|
||||||
path = VideoAsset(name="baby_reading").video_path
|
|
||||||
audio_signal, sr = load_audio(path, sr=sampling_rate)
|
|
||||||
return audio_signal
|
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
import warnings
|
import warnings
|
||||||
from dataclasses import dataclass
|
from collections import defaultdict
|
||||||
|
from dataclasses import field, make_dataclass
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
from vllm.benchmarks.datasets import SampleRequest
|
from vllm.benchmarks.datasets import SampleRequest
|
||||||
@@ -9,30 +10,172 @@ from vllm.tokenizers import TokenizerLike
|
|||||||
|
|
||||||
from vllm_omni.metrics import definitions as defs
|
from vllm_omni.metrics import definitions as defs
|
||||||
|
|
||||||
|
_PERCENTILE_ROWS_TYPE = list[tuple[float, float]] | None
|
||||||
|
_FLOAT_LIST_TYPE = list[float]
|
||||||
|
_INT_LIST_TYPE = list[int]
|
||||||
|
|
||||||
@dataclass
|
_MULTIMODAL_BENCHMARK_FIELDS = [
|
||||||
class MultiModalsBenchmarkMetrics(BenchmarkMetrics):
|
(defs.MEAN_AUDIO_TTFP_MS, float, field(default=0.0)),
|
||||||
mean_audio_ttfp_ms: float = 0.0
|
(defs.MEDIAN_AUDIO_TTFP_MS, float, field(default=0.0)),
|
||||||
median_audio_ttfp_ms: float = 0.0
|
(defs.STD_AUDIO_TTFP_MS, float, field(default=0.0)),
|
||||||
std_audio_ttfp_ms: float = 0.0
|
(defs.PERCENTILES_AUDIO_TTFP_MS, _PERCENTILE_ROWS_TYPE, field(default=None)),
|
||||||
percentiles_audio_ttfp_ms: list[tuple[float, float]] = None
|
(defs.TOTAL_AUDIO_DURATION_S, float, field(default=0.0)),
|
||||||
total_audio_duration_s: float = 0.0
|
(defs.TOTAL_AUDIO_FRAMES, int, field(default=0)),
|
||||||
total_audio_frames: int = 0
|
(defs.AUDIO_THROUGHPUT, float, field(default=0.0)),
|
||||||
audio_throughput: float = 0.0
|
(defs.MEAN_AUDIO_RTF, float, field(default=0.0)),
|
||||||
mean_audio_rtf: float = 0.0
|
(defs.MEDIAN_AUDIO_RTF, float, field(default=0.0)),
|
||||||
median_audio_rtf: float = 0.0
|
(defs.STD_AUDIO_RTF, float, field(default=0.0)),
|
||||||
std_audio_rtf: float = 0.0
|
(defs.PERCENTILES_AUDIO_RTF, _PERCENTILE_ROWS_TYPE, field(default=None)),
|
||||||
percentiles_audio_rtf: list[tuple[float, float]] = None
|
(defs.MEAN_AUDIO_DURATION_S, float, field(default=0.0)),
|
||||||
mean_audio_duration_s: float = 0.0
|
(defs.MEDIAN_AUDIO_DURATION_S, float, field(default=0.0)),
|
||||||
median_audio_duration_s: float = 0.0
|
(defs.STD_AUDIO_DURATION_S, float, field(default=0.0)),
|
||||||
std_audio_duration_s: float = 0.0
|
(defs.PERCENTILES_AUDIO_DURATION_S, _PERCENTILE_ROWS_TYPE, field(default=None)),
|
||||||
percentiles_audio_duration_s: list[tuple[float, float]] = None
|
(defs.MEAN_AUDIO_UNDERRUN_S, float, field(default=0.0)),
|
||||||
# Streaming-audio continuity (see vllm_omni/benchmarks/audio_continuity.py).
|
(defs.MEDIAN_AUDIO_UNDERRUN_S, float, field(default=0.0)),
|
||||||
mean_audio_underrun_s: float = 0.0
|
(defs.STD_AUDIO_UNDERRUN_S, float, field(default=0.0)),
|
||||||
median_audio_underrun_s: float = 0.0
|
(defs.PERCENTILES_AUDIO_UNDERRUN_S, _PERCENTILE_ROWS_TYPE, field(default=None)),
|
||||||
std_audio_underrun_s: float = 0.0
|
(defs.AUDIO_CONTINUITY_OK_RATE, float, field(default=1.0)),
|
||||||
percentiles_audio_underrun_s: list[tuple[float, float]] = None
|
(defs.TOTAL_IMAGES, int, field(default=0)),
|
||||||
audio_continuity_ok_rate: float = 1.0
|
(defs.IMAGE_THROUGHPUT, float, field(default=0.0)),
|
||||||
|
(defs.AVERAGE_PIXELS_PER_IMAGE, float, field(default=0.0)),
|
||||||
|
(defs.MEAN_DENOISE_STEP_LATENCY_MS, float, field(default=0.0)),
|
||||||
|
(defs.MEAN_IMAGE_GENERATION_MS, float, field(default=0.0)),
|
||||||
|
(defs.MEDIAN_IMAGE_GENERATION_MS, float, field(default=0.0)),
|
||||||
|
(defs.STD_IMAGE_GENERATION_MS, float, field(default=0.0)),
|
||||||
|
(defs.PERCENTILES_IMAGE_GENERATION_MS, _PERCENTILE_ROWS_TYPE, field(default=None)),
|
||||||
|
]
|
||||||
|
|
||||||
|
MultiModalsBenchmarkMetrics = make_dataclass(
|
||||||
|
"MultiModalsBenchmarkMetrics",
|
||||||
|
_MULTIMODAL_BENCHMARK_FIELDS,
|
||||||
|
bases=(BenchmarkMetrics,),
|
||||||
|
)
|
||||||
|
|
||||||
|
_STAGE_BENCHMARK_FIELDS = [
|
||||||
|
("stage_id", int, field(default=0)),
|
||||||
|
("stage_name", str, field(default="")),
|
||||||
|
("final_output_type", str, field(default="")),
|
||||||
|
(defs.TOTAL_OUTPUT, int, field(default=0)),
|
||||||
|
(defs.TTFTS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
||||||
|
(defs.TPOTS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
||||||
|
(defs.ITLS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
||||||
|
(defs.VLLM_TTFTS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
||||||
|
(defs.VLLM_TPOTS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
||||||
|
(defs.VLLM_ITLS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
||||||
|
(defs.AUDIO_TTFPS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
||||||
|
(defs.AUDIO_RTFS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
||||||
|
(defs.AUDIO_DURATIONS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
||||||
|
(defs.AUDIO_FRAMES, _INT_LIST_TYPE, field(default_factory=list)),
|
||||||
|
(defs.MISSING_AUDIO_DURATION_COUNT, int, field(default=0)),
|
||||||
|
(defs.MISSING_AUDIO_RTF_COUNT, int, field(default=0)),
|
||||||
|
(defs.STAGE_GEN_TIMES_MS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
||||||
|
(defs.POSTPROCESS_TIMES_MS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
||||||
|
("output_unit_type", str, field(default="")),
|
||||||
|
(defs.OUTPUT_UNIT_COUNT, int, field(default=0)),
|
||||||
|
(defs.SERVING_TIME_TO_FIRST_OUTPUTS_MS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
||||||
|
(defs.TIME_PER_OUTPUT_UNITS_MS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
||||||
|
(defs.INTER_OUTPUT_LATENCIES_MS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
||||||
|
]
|
||||||
|
|
||||||
|
StageBenchmarkMetrics = make_dataclass(
|
||||||
|
"StageBenchmarkMetrics",
|
||||||
|
_STAGE_BENCHMARK_FIELDS,
|
||||||
|
namespace={"__doc__": "Aggregated metrics for one pipeline stage (for printing only)."},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _percentile_rows_seconds(
|
||||||
|
values: list[float], selected_percentiles: list[float], *, to_ms: bool
|
||||||
|
) -> tuple[float, float, list[tuple[float, float]]]:
|
||||||
|
arr = np.asarray(values, dtype=float)
|
||||||
|
scale = 1000.0 if to_ms else 1.0
|
||||||
|
mean_v = float(np.mean(arr)) * scale
|
||||||
|
median_v = float(np.median(arr)) * scale
|
||||||
|
rows = [(float(p), float(np.percentile(arr, p)) * scale) for p in selected_percentiles]
|
||||||
|
return mean_v, median_v, rows
|
||||||
|
|
||||||
|
|
||||||
|
def _p_label(p: float) -> str:
|
||||||
|
pf = float(p)
|
||||||
|
if abs(pf - round(pf)) < 1e-9:
|
||||||
|
return str(int(round(pf)))
|
||||||
|
return str(pf)
|
||||||
|
|
||||||
|
|
||||||
|
_STREAMING_OUTPUT_UNIT_TYPES = frozenset(
|
||||||
|
{
|
||||||
|
"text",
|
||||||
|
"stream",
|
||||||
|
"audio",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
_AGGREGATE_PERCENTILE_FIELD_NAMES = {
|
||||||
|
defs.AUDIO_TTFP: (
|
||||||
|
defs.MEAN_AUDIO_TTFP_MS,
|
||||||
|
defs.MEDIAN_AUDIO_TTFP_MS,
|
||||||
|
defs.PERCENTILES_AUDIO_TTFP_MS,
|
||||||
|
),
|
||||||
|
defs.AUDIO_RTF: (
|
||||||
|
defs.MEAN_AUDIO_RTF,
|
||||||
|
defs.MEDIAN_AUDIO_RTF,
|
||||||
|
defs.PERCENTILES_AUDIO_RTF,
|
||||||
|
),
|
||||||
|
defs.AUDIO_DURATION: (
|
||||||
|
defs.MEAN_AUDIO_DURATION_S,
|
||||||
|
defs.MEDIAN_AUDIO_DURATION_S,
|
||||||
|
defs.PERCENTILES_AUDIO_DURATION_S,
|
||||||
|
),
|
||||||
|
defs.AUDIO_UNDERRUN: (
|
||||||
|
defs.MEAN_AUDIO_UNDERRUN_S,
|
||||||
|
defs.MEDIAN_AUDIO_UNDERRUN_S,
|
||||||
|
defs.PERCENTILES_AUDIO_UNDERRUN_S,
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _wants_text_ttft(metrics: list[str]) -> bool:
|
||||||
|
return "ttft" in metrics
|
||||||
|
|
||||||
|
|
||||||
|
def _wants_text_tpot(metrics: list[str]) -> bool:
|
||||||
|
return "tpot" in metrics or "tpop" in metrics
|
||||||
|
|
||||||
|
|
||||||
|
def _wants_text_itl(metrics: list[str]) -> bool:
|
||||||
|
return "itl" in metrics
|
||||||
|
|
||||||
|
|
||||||
|
def _wants_stream_ttfc(metrics: list[str]) -> bool:
|
||||||
|
return "ttfc" in metrics
|
||||||
|
|
||||||
|
|
||||||
|
def _wants_stream_tpoc(metrics: list[str]) -> bool:
|
||||||
|
return "tpoc" in metrics or "tpop" in metrics
|
||||||
|
|
||||||
|
|
||||||
|
def _wants_stream_icl(metrics: list[str]) -> bool:
|
||||||
|
return "icl" in metrics
|
||||||
|
|
||||||
|
|
||||||
|
def _stage_modality_flags(
|
||||||
|
final_output_type: str,
|
||||||
|
output_unit_type: str,
|
||||||
|
) -> tuple[bool, bool, bool, bool, bool]:
|
||||||
|
is_text_stage = final_output_type == "text" or output_unit_type == "text"
|
||||||
|
is_audio_stage = final_output_type == "audio" or output_unit_type == "audio"
|
||||||
|
is_image_stage = final_output_type in {"image", "images"} or output_unit_type == "image"
|
||||||
|
is_video_stage = final_output_type in {"video", "videos"} or output_unit_type == "video"
|
||||||
|
is_internal_stream_stage = (
|
||||||
|
output_unit_type in _STREAMING_OUTPUT_UNIT_TYPES and not is_text_stage and not is_audio_stage
|
||||||
|
)
|
||||||
|
return (
|
||||||
|
is_text_stage,
|
||||||
|
is_audio_stage,
|
||||||
|
is_image_stage,
|
||||||
|
is_video_stage,
|
||||||
|
is_internal_stream_stage,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def print_metrics(
|
def print_metrics(
|
||||||
@@ -43,6 +186,10 @@ def print_metrics(
|
|||||||
benchmark_duration,
|
benchmark_duration,
|
||||||
goodput_config_dict,
|
goodput_config_dict,
|
||||||
metrics: MultiModalsBenchmarkMetrics,
|
metrics: MultiModalsBenchmarkMetrics,
|
||||||
|
*,
|
||||||
|
outputs: list[RequestFuncOutput] | None = None,
|
||||||
|
selected_percentiles: list[float] | None = None,
|
||||||
|
print_stage: bool = False,
|
||||||
):
|
):
|
||||||
print("{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="="))
|
print("{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="="))
|
||||||
print("{:<40} {:<10}".format("Successful requests:", metrics.completed))
|
print("{:<40} {:<10}".format("Successful requests:", metrics.completed))
|
||||||
@@ -61,7 +208,19 @@ def print_metrics(
|
|||||||
process_one_metric("e2el", metrics)
|
process_one_metric("e2el", metrics)
|
||||||
print_text_metrics(task_type, selected_percentile_metrics, metrics)
|
print_text_metrics(task_type, selected_percentile_metrics, metrics)
|
||||||
if task_type == TaskType.GENERATION:
|
if task_type == TaskType.GENERATION:
|
||||||
print_audio_metrics(selected_percentile_metrics, metrics)
|
if _has_audio_output(metrics):
|
||||||
|
print_audio_metrics(selected_percentile_metrics, metrics)
|
||||||
|
if _has_image_output(metrics):
|
||||||
|
print_image_metrics(selected_percentiles or [], metrics)
|
||||||
|
if print_stage and outputs and selected_percentiles is not None:
|
||||||
|
print("\n{s:{c}^{n}}".format(s=" Stage Benchmark Result ", n=50, c="="))
|
||||||
|
for sm in _build_stage_metrics_from_outputs(outputs):
|
||||||
|
print_stage_metrics(
|
||||||
|
task_type,
|
||||||
|
selected_percentile_metrics,
|
||||||
|
selected_percentiles,
|
||||||
|
sm,
|
||||||
|
)
|
||||||
print("=" * 50)
|
print("=" * 50)
|
||||||
|
|
||||||
|
|
||||||
@@ -79,22 +238,36 @@ def print_text_metrics(task_type, selected_percentile_metrics, metrics: MultiMod
|
|||||||
# No text tokens generated (e.g. pure TTS speech endpoint): per-token
|
# No text tokens generated (e.g. pure TTS speech endpoint): per-token
|
||||||
# latency metrics (ttft/tpot/itl) are undefined, so skip them.
|
# latency metrics (ttft/tpot/itl) are undefined, so skip them.
|
||||||
has_text_output = metrics.total_output > 0
|
has_text_output = metrics.total_output > 0
|
||||||
for metric in selected_percentile_metrics:
|
if has_text_output:
|
||||||
if metric == "e2el":
|
if _wants_text_ttft(selected_percentile_metrics):
|
||||||
continue
|
process_one_metric("ttft", metrics)
|
||||||
if not metric.startswith("audio") and has_text_output:
|
if _wants_text_tpot(selected_percentile_metrics):
|
||||||
process_one_metric(metric, metrics)
|
process_one_metric("tpot", metrics)
|
||||||
|
if _wants_text_itl(selected_percentile_metrics):
|
||||||
|
process_one_metric("itl", metrics)
|
||||||
|
|
||||||
|
|
||||||
|
def _has_audio_output(metrics: MultiModalsBenchmarkMetrics) -> bool:
|
||||||
|
return bool(
|
||||||
|
getattr(metrics, defs.TOTAL_AUDIO_DURATION_S, 0.0) > 0 or getattr(metrics, defs.TOTAL_AUDIO_FRAMES, 0) > 0
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _has_image_output(metrics: MultiModalsBenchmarkMetrics) -> bool:
|
||||||
|
return int(getattr(metrics, defs.TOTAL_IMAGES, 0) or 0) > 0
|
||||||
|
|
||||||
|
|
||||||
def print_audio_metrics(selected_percentile_metrics, metrics: MultiModalsBenchmarkMetrics):
|
def print_audio_metrics(selected_percentile_metrics, metrics: MultiModalsBenchmarkMetrics):
|
||||||
print("{s:{c}^{n}}".format(s=" Audio Result ", n=50, c="="))
|
print("{s:{c}^{n}}".format(s=" Audio Result ", n=50, c="="))
|
||||||
print("{:<40} {:<10.2f}".format("Total audio duration generated(s):", metrics.total_audio_duration_s))
|
print(
|
||||||
print("{:<40} {:<10}".format("Total audio frames generated:", metrics.total_audio_frames))
|
"{:<40} {:<10.2f}".format("Total audio duration generated(s):", getattr(metrics, defs.TOTAL_AUDIO_DURATION_S))
|
||||||
print("{:<40} {:<10.2f}".format("Audio throughput(audio duration/s):", metrics.audio_throughput))
|
)
|
||||||
|
print("{:<40} {:<10}".format("Total audio frames generated:", getattr(metrics, defs.TOTAL_AUDIO_FRAMES)))
|
||||||
|
print("{:<40} {:<10.2f}".format("Audio throughput(audio duration/s):", getattr(metrics, defs.AUDIO_THROUGHPUT)))
|
||||||
print(
|
print(
|
||||||
"{:<40} {:<10.2%}".format(
|
"{:<40} {:<10.2%}".format(
|
||||||
"Streaming continuity OK rate:",
|
"Streaming continuity OK rate:",
|
||||||
metrics.audio_continuity_ok_rate,
|
getattr(metrics, defs.AUDIO_CONTINUITY_OK_RATE),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
for metric in selected_percentile_metrics:
|
for metric in selected_percentile_metrics:
|
||||||
@@ -102,6 +275,37 @@ def print_audio_metrics(selected_percentile_metrics, metrics: MultiModalsBenchma
|
|||||||
process_one_metric(metric, metrics)
|
process_one_metric(metric, metrics)
|
||||||
|
|
||||||
|
|
||||||
|
def print_image_metrics(selected_percentiles: list[float], metrics: MultiModalsBenchmarkMetrics):
|
||||||
|
print("{s:{c}^{n}}".format(s=" Image Result ", n=50, c="="))
|
||||||
|
print("{:<40} {:<10}".format("Total images generated:", getattr(metrics, defs.TOTAL_IMAGES)))
|
||||||
|
print("{:<40} {:<10.2f}".format("Image throughput (img/s):", getattr(metrics, defs.IMAGE_THROUGHPUT)))
|
||||||
|
print(
|
||||||
|
"{:<40} {:<10.2f}".format(
|
||||||
|
"Average pixels per image:",
|
||||||
|
getattr(metrics, defs.AVERAGE_PIXELS_PER_IMAGE),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if getattr(metrics, defs.MEAN_DENOISE_STEP_LATENCY_MS) > 0:
|
||||||
|
print(
|
||||||
|
"{:<40} {:<10.2f}".format(
|
||||||
|
"Mean denoise step latency (ms):",
|
||||||
|
getattr(metrics, defs.MEAN_DENOISE_STEP_LATENCY_MS),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if getattr(metrics, defs.MEAN_IMAGE_GENERATION_MS) > 0:
|
||||||
|
print("-----------------Image Generation-----------------")
|
||||||
|
print("{:<40} {:<10.2f}".format("Mean IMAGE_GENERATION (ms):", getattr(metrics, defs.MEAN_IMAGE_GENERATION_MS)))
|
||||||
|
print(
|
||||||
|
"{:<40} {:<10.2f}".format(
|
||||||
|
"Median IMAGE_GENERATION (ms):",
|
||||||
|
getattr(metrics, defs.MEDIAN_IMAGE_GENERATION_MS),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
for p, value in getattr(metrics, defs.PERCENTILES_IMAGE_GENERATION_MS) or []:
|
||||||
|
if p in selected_percentiles:
|
||||||
|
print("{:<40} {:<10.2f}".format(f"P{_p_label(p)} IMAGE_GENERATION (ms):", value))
|
||||||
|
|
||||||
|
|
||||||
def process_one_metric(
|
def process_one_metric(
|
||||||
metric_attribute_name: str,
|
metric_attribute_name: str,
|
||||||
metrics: MultiModalsBenchmarkMetrics,
|
metrics: MultiModalsBenchmarkMetrics,
|
||||||
@@ -131,15 +335,16 @@ def process_one_metric(
|
|||||||
elif is_audio_rtf:
|
elif is_audio_rtf:
|
||||||
suffix = ""
|
suffix = ""
|
||||||
unit_suffix = ""
|
unit_suffix = ""
|
||||||
mean_attr_name = f"mean_{metric_attribute_name}{suffix}"
|
field_names = _AGGREGATE_PERCENTILE_FIELD_NAMES.get(metric_attribute_name)
|
||||||
|
mean_attr_name = field_names[0] if field_names else f"mean_{metric_attribute_name}{suffix}"
|
||||||
mean_value = getattr(metrics, mean_attr_name, 0.0)
|
mean_value = getattr(metrics, mean_attr_name, 0.0)
|
||||||
print(f"{f'Mean {metric_attribute_name.upper()}{unit_suffix}:':<40} {mean_value:<10.2f}")
|
print(f"{f'Mean {metric_attribute_name.upper()}{unit_suffix}:':<40} {mean_value:<10.2f}")
|
||||||
|
|
||||||
median_attr_name = f"median_{metric_attribute_name}{suffix}"
|
median_attr_name = field_names[1] if field_names else f"median_{metric_attribute_name}{suffix}"
|
||||||
median_value = getattr(metrics, median_attr_name, 0.0)
|
median_value = getattr(metrics, median_attr_name, 0.0)
|
||||||
print(f"{f'Median {metric_attribute_name.upper()}{unit_suffix}:':<40} {median_value:<10.2f}")
|
print(f"{f'Median {metric_attribute_name.upper()}{unit_suffix}:':<40} {median_value:<10.2f}")
|
||||||
|
|
||||||
percentiles_attr_name = f"percentiles_{metric_attribute_name}{suffix}"
|
percentiles_attr_name = field_names[2] if field_names else f"percentiles_{metric_attribute_name}{suffix}"
|
||||||
percentiles = getattr(metrics, percentiles_attr_name, [])
|
percentiles = getattr(metrics, percentiles_attr_name, [])
|
||||||
|
|
||||||
for percentile, value in percentiles:
|
for percentile, value in percentiles:
|
||||||
@@ -148,6 +353,356 @@ def process_one_metric(
|
|||||||
print(f"{label:<40} {value:<10.2f}")
|
print(f"{label:<40} {value:<10.2f}")
|
||||||
|
|
||||||
|
|
||||||
|
def _print_percentile_metric(
|
||||||
|
header: str,
|
||||||
|
label: str,
|
||||||
|
values: list[float],
|
||||||
|
selected_percentiles: list[float],
|
||||||
|
*,
|
||||||
|
values_are_ms: bool = False,
|
||||||
|
to_ms: bool = True,
|
||||||
|
) -> None:
|
||||||
|
if not values:
|
||||||
|
return
|
||||||
|
values_s = [v / 1000.0 for v in values] if values_are_ms else values
|
||||||
|
mean_v, med_v, pcts = _percentile_rows_seconds(values_s, selected_percentiles, to_ms=to_ms)
|
||||||
|
unit_suffix = " (ms)" if to_ms else ""
|
||||||
|
print("{s:{c}^{n}}".format(s=header, n=50, c="-"))
|
||||||
|
print(f"{f'Mean {label}{unit_suffix}:':<40} {mean_v:<10.2f}")
|
||||||
|
print(f"{f'Median {label}{unit_suffix}:':<40} {med_v:<10.2f}")
|
||||||
|
for p, v in pcts:
|
||||||
|
p_str = _p_label(p)
|
||||||
|
print(f"{f'P{p_str} {label}{unit_suffix}:':<40} {v:<10.2f}")
|
||||||
|
|
||||||
|
|
||||||
|
def _print_stage_timing(sm: StageBenchmarkMetrics, selected_percentiles: list[float]) -> None:
|
||||||
|
_print_percentile_metric(
|
||||||
|
"Stage Timing",
|
||||||
|
"stage_gen_time",
|
||||||
|
getattr(sm, defs.STAGE_GEN_TIMES_MS),
|
||||||
|
selected_percentiles,
|
||||||
|
values_are_ms=True,
|
||||||
|
to_ms=True,
|
||||||
|
)
|
||||||
|
postprocess_times_ms = getattr(sm, defs.POSTPROCESS_TIMES_MS)
|
||||||
|
if postprocess_times_ms and any(v > 0 for v in postprocess_times_ms):
|
||||||
|
_print_percentile_metric(
|
||||||
|
"Stage Postprocess",
|
||||||
|
"postprocess",
|
||||||
|
postprocess_times_ms,
|
||||||
|
selected_percentiles,
|
||||||
|
values_are_ms=True,
|
||||||
|
to_ms=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _print_text_stage_metrics(
|
||||||
|
task_type: TaskType,
|
||||||
|
selected_percentile_metrics: list[str],
|
||||||
|
selected_percentiles: list[float],
|
||||||
|
sm: StageBenchmarkMetrics,
|
||||||
|
) -> None:
|
||||||
|
print("{s:{c}^{n}}".format(s=" Text Result ", n=50, c="="))
|
||||||
|
print("{:<40} {:<10}".format("Stage generated tokens:", getattr(sm, defs.TOTAL_OUTPUT)))
|
||||||
|
|
||||||
|
if task_type != TaskType.GENERATION:
|
||||||
|
return
|
||||||
|
if _wants_text_ttft(selected_percentile_metrics):
|
||||||
|
_print_percentile_metric(
|
||||||
|
"Serving Time to First Token",
|
||||||
|
"Serving TTFT",
|
||||||
|
getattr(sm, defs.TTFTS),
|
||||||
|
selected_percentiles,
|
||||||
|
)
|
||||||
|
_print_percentile_metric("Time to First Token", "TTFT", getattr(sm, defs.VLLM_TTFTS), selected_percentiles)
|
||||||
|
if _wants_text_tpot(selected_percentile_metrics):
|
||||||
|
_print_percentile_metric(
|
||||||
|
"Time per Output Token (excl. 1st token)",
|
||||||
|
"TPOT",
|
||||||
|
getattr(sm, defs.VLLM_TPOTS),
|
||||||
|
selected_percentiles,
|
||||||
|
)
|
||||||
|
if _wants_text_itl(selected_percentile_metrics):
|
||||||
|
_print_percentile_metric("Inter-token Latency", "ITL", getattr(sm, defs.VLLM_ITLS), selected_percentiles)
|
||||||
|
|
||||||
|
|
||||||
|
def _print_audio_stage_metrics(
|
||||||
|
selected_percentile_metrics: list[str],
|
||||||
|
selected_percentiles: list[float],
|
||||||
|
sm: StageBenchmarkMetrics,
|
||||||
|
) -> None:
|
||||||
|
total_duration = float(sum(getattr(sm, defs.AUDIO_DURATIONS)))
|
||||||
|
total_frames = int(sum(getattr(sm, defs.AUDIO_FRAMES)))
|
||||||
|
print("{s:{c}^{n}}".format(s=" Audio Result ", n=50, c="="))
|
||||||
|
print("{:<40} {:<10.2f}".format("Stage audio duration generated(s):", total_duration))
|
||||||
|
print("{:<40} {:<10}".format("Stage audio frames generated:", total_frames))
|
||||||
|
if defs.AUDIO_TTFP in selected_percentile_metrics:
|
||||||
|
_print_percentile_metric(
|
||||||
|
"Serving Time to First Packet",
|
||||||
|
"Serving AUDIO_TTFP",
|
||||||
|
getattr(sm, defs.AUDIO_TTFPS),
|
||||||
|
selected_percentiles,
|
||||||
|
)
|
||||||
|
if defs.AUDIO_RTF in selected_percentile_metrics:
|
||||||
|
audio_rtfs = getattr(sm, defs.AUDIO_RTFS)
|
||||||
|
if audio_rtfs:
|
||||||
|
_print_percentile_metric("Real Time Factor", "AUDIO_RTF", audio_rtfs, selected_percentiles, to_ms=False)
|
||||||
|
elif getattr(sm, defs.MISSING_AUDIO_RTF_COUNT) > 0:
|
||||||
|
print("{:<40} {:<10}".format("AUDIO_RTF skipped:", "missing stage-local audio duration/sample rate"))
|
||||||
|
if defs.AUDIO_DURATION in selected_percentile_metrics:
|
||||||
|
audio_durations = getattr(sm, defs.AUDIO_DURATIONS)
|
||||||
|
if audio_durations:
|
||||||
|
_print_percentile_metric(
|
||||||
|
"Audio Duration",
|
||||||
|
"AUDIO_DURATION",
|
||||||
|
audio_durations,
|
||||||
|
selected_percentiles,
|
||||||
|
to_ms=False,
|
||||||
|
)
|
||||||
|
elif getattr(sm, defs.MISSING_AUDIO_DURATION_COUNT) > 0:
|
||||||
|
print("{:<40} {:<10}".format("AUDIO_DURATION skipped:", "missing stage-local audio duration/sample rate"))
|
||||||
|
|
||||||
|
|
||||||
|
def _print_image_stage_metrics(
|
||||||
|
selected_percentiles: list[float],
|
||||||
|
sm: StageBenchmarkMetrics,
|
||||||
|
) -> None:
|
||||||
|
image_num = getattr(sm, defs.OUTPUT_UNIT_COUNT) if getattr(sm, "output_unit_type") == "image" else 0
|
||||||
|
print("{s:{c}^{n}}".format(s=" Image Result ", n=50, c="="))
|
||||||
|
print("{:<40} {:<10}".format("Total images generated:", image_num))
|
||||||
|
_print_percentile_metric(
|
||||||
|
"Image Generation",
|
||||||
|
"IMAGE_GENERATION",
|
||||||
|
getattr(sm, defs.STAGE_GEN_TIMES_MS),
|
||||||
|
selected_percentiles,
|
||||||
|
values_are_ms=True,
|
||||||
|
to_ms=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _print_internal_stream_stage_metrics(
|
||||||
|
selected_percentile_metrics: list[str],
|
||||||
|
selected_percentiles: list[float],
|
||||||
|
sm: StageBenchmarkMetrics,
|
||||||
|
) -> None:
|
||||||
|
print("{s:{c}^{n}}".format(s=" Internal Stream Result ", n=50, c="="))
|
||||||
|
serving_time_to_first_outputs_ms = getattr(sm, defs.SERVING_TIME_TO_FIRST_OUTPUTS_MS)
|
||||||
|
if _wants_stream_ttfc(selected_percentile_metrics) and serving_time_to_first_outputs_ms:
|
||||||
|
_print_percentile_metric(
|
||||||
|
"Serving Time to First Chunk",
|
||||||
|
"Serving TTFC",
|
||||||
|
serving_time_to_first_outputs_ms,
|
||||||
|
selected_percentiles,
|
||||||
|
values_are_ms=True,
|
||||||
|
to_ms=True,
|
||||||
|
)
|
||||||
|
time_per_output_units_ms = getattr(sm, defs.TIME_PER_OUTPUT_UNITS_MS)
|
||||||
|
if _wants_stream_tpoc(selected_percentile_metrics) and time_per_output_units_ms:
|
||||||
|
_print_percentile_metric(
|
||||||
|
"Time per Output Chunk (excl. 1st chunk)",
|
||||||
|
"TPOP",
|
||||||
|
time_per_output_units_ms,
|
||||||
|
selected_percentiles,
|
||||||
|
values_are_ms=True,
|
||||||
|
to_ms=True,
|
||||||
|
)
|
||||||
|
inter_output_latencies_ms = getattr(sm, defs.INTER_OUTPUT_LATENCIES_MS)
|
||||||
|
if _wants_stream_icl(selected_percentile_metrics) and inter_output_latencies_ms:
|
||||||
|
_print_percentile_metric(
|
||||||
|
"Inter-chunk Latency",
|
||||||
|
"ICL",
|
||||||
|
inter_output_latencies_ms,
|
||||||
|
selected_percentiles,
|
||||||
|
values_are_ms=True,
|
||||||
|
to_ms=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def print_stage_metrics(
|
||||||
|
task_type: TaskType,
|
||||||
|
selected_percentile_metrics: list[str],
|
||||||
|
selected_percentiles: list[float],
|
||||||
|
sm: StageBenchmarkMetrics,
|
||||||
|
):
|
||||||
|
title = f" Stage {getattr(sm, 'stage_id')} ({getattr(sm, 'stage_name')}) "
|
||||||
|
(
|
||||||
|
is_text_stage,
|
||||||
|
is_audio_stage,
|
||||||
|
is_image_stage,
|
||||||
|
is_video_stage,
|
||||||
|
is_internal_stream_stage,
|
||||||
|
) = _stage_modality_flags(getattr(sm, "final_output_type"), getattr(sm, "output_unit_type"))
|
||||||
|
|
||||||
|
print("{s:{c}^{n}}".format(s=title, n=50, c="="))
|
||||||
|
if is_image_stage:
|
||||||
|
_print_image_stage_metrics(selected_percentiles, sm)
|
||||||
|
return
|
||||||
|
|
||||||
|
if is_video_stage:
|
||||||
|
return
|
||||||
|
|
||||||
|
_print_stage_timing(sm, selected_percentiles)
|
||||||
|
if is_text_stage:
|
||||||
|
_print_text_stage_metrics(task_type, selected_percentile_metrics, selected_percentiles, sm)
|
||||||
|
elif is_audio_stage:
|
||||||
|
_print_audio_stage_metrics(
|
||||||
|
selected_percentile_metrics,
|
||||||
|
selected_percentiles,
|
||||||
|
sm,
|
||||||
|
)
|
||||||
|
elif is_internal_stream_stage:
|
||||||
|
_print_internal_stream_stage_metrics(selected_percentile_metrics, selected_percentiles, sm)
|
||||||
|
|
||||||
|
|
||||||
|
def _build_stage_metrics_from_outputs(
|
||||||
|
outputs: list[RequestFuncOutput],
|
||||||
|
) -> list[StageBenchmarkMetrics]:
|
||||||
|
"""Aggregate per ``stage_id`` using ``stage_metrics`` snapshots from the client."""
|
||||||
|
buckets: dict[str, list[tuple[RequestFuncOutput, dict]]] = defaultdict(list)
|
||||||
|
for out in outputs:
|
||||||
|
if not getattr(out, "success", False):
|
||||||
|
continue
|
||||||
|
smap = getattr(out, "stage_metrics", None) or {}
|
||||||
|
if not isinstance(smap, dict) or not smap:
|
||||||
|
continue
|
||||||
|
for sid, info in smap.items():
|
||||||
|
if isinstance(info, dict):
|
||||||
|
buckets[str(sid)].append((out, info))
|
||||||
|
|
||||||
|
result: list[StageBenchmarkMetrics] = []
|
||||||
|
for sid in sorted(buckets.keys(), key=lambda x: int(x)):
|
||||||
|
rows = buckets[sid]
|
||||||
|
stage_id_int = int(sid)
|
||||||
|
stage_name = str((rows[0][1] or {}).get("stage_name") or f"stage_{stage_id_int}")
|
||||||
|
final_output_type = str((rows[0][1] or {}).get("final_output_type") or "unknown")
|
||||||
|
output_unit_types = [
|
||||||
|
str((info or {}).get("output_unit_type") or "") for _, info in rows if (info or {}).get("output_unit_type")
|
||||||
|
]
|
||||||
|
output_unit_type = output_unit_types[0] if output_unit_types else "other"
|
||||||
|
|
||||||
|
is_text_stage, is_audio_stage, _, _, is_internal_stream_stage = _stage_modality_flags(
|
||||||
|
final_output_type, output_unit_type
|
||||||
|
)
|
||||||
|
total_output = 0
|
||||||
|
|
||||||
|
ttfts: list[float] = []
|
||||||
|
tpots: list[float] = []
|
||||||
|
itls: list[float] = []
|
||||||
|
vllm_ttfts: list[float] = []
|
||||||
|
vllm_tpots: list[float] = []
|
||||||
|
vllm_itls: list[float] = []
|
||||||
|
stage_gen_times_ms = [float((info or {}).get(defs.STAGE_GEN_TIME_MS) or 0.0) for _, info in rows]
|
||||||
|
postprocess_times_ms = [float((info or {}).get(defs.POSTPROCESS_TIME_MS) or 0.0) for _, info in rows]
|
||||||
|
output_unit_count = sum(int((info or {}).get(defs.OUTPUT_UNIT_COUNT) or 0) for _, info in rows)
|
||||||
|
inter_output_latencies_ms: list[float] = []
|
||||||
|
serving_time_to_first_outputs_ms: list[float] = []
|
||||||
|
time_per_output_units_ms: list[float] = []
|
||||||
|
if is_text_stage or is_internal_stream_stage:
|
||||||
|
serving_time_to_first_outputs_ms = [
|
||||||
|
float((info or {}).get(defs.SERVING_TIME_TO_FIRST_OUTPUT_MS) or 0.0)
|
||||||
|
for _, info in rows
|
||||||
|
if (info or {}).get(defs.SERVING_TIME_TO_FIRST_OUTPUT_MS) is not None
|
||||||
|
]
|
||||||
|
time_per_output_units_ms = [
|
||||||
|
float((info or {}).get(defs.TIME_PER_OUTPUT_UNIT_MS) or 0.0)
|
||||||
|
for _, info in rows
|
||||||
|
if int((info or {}).get(defs.OUTPUT_UNIT_COUNT) or 0) > 1
|
||||||
|
and (info or {}).get(defs.TIME_PER_OUTPUT_UNIT_MS) is not None
|
||||||
|
]
|
||||||
|
for _, info in rows:
|
||||||
|
values = (info or {}).get(defs.INTER_OUTPUT_LATENCIES_MS)
|
||||||
|
if isinstance(values, list):
|
||||||
|
inter_output_latencies_ms.extend(float(v or 0.0) for v in values)
|
||||||
|
elif (info or {}).get(defs.INTER_OUTPUT_LATENCY_MS) is not None:
|
||||||
|
inter_output_latencies_ms.append(float((info or {}).get(defs.INTER_OUTPUT_LATENCY_MS) or 0.0))
|
||||||
|
|
||||||
|
if is_text_stage:
|
||||||
|
for _, info in rows:
|
||||||
|
ttft_ms = float((info or {}).get(defs.SERVING_TIME_TO_FIRST_OUTPUT_MS) or 0.0)
|
||||||
|
if ttft_ms > 0:
|
||||||
|
ttfts.append(ttft_ms / 1000.0)
|
||||||
|
tpot_ms = float((info or {}).get(defs.TIME_PER_OUTPUT_UNIT_MS) or 0.0)
|
||||||
|
if int((info or {}).get(defs.OUTPUT_UNIT_COUNT) or 0) > 1 and tpot_ms > 0:
|
||||||
|
tpots.append(tpot_ms / 1000.0)
|
||||||
|
values = (info or {}).get(defs.INTER_OUTPUT_LATENCIES_MS)
|
||||||
|
if isinstance(values, list):
|
||||||
|
itls.extend(float(v or 0.0) / 1000.0 for v in values)
|
||||||
|
elif (info or {}).get(defs.INTER_OUTPUT_LATENCY_MS) is not None:
|
||||||
|
itls.append(float((info or {}).get(defs.INTER_OUTPUT_LATENCY_MS) or 0.0) / 1000.0)
|
||||||
|
vllm_ttft_ms = float((info or {}).get(defs.VLLM_TTFT_MS) or 0.0)
|
||||||
|
if vllm_ttft_ms > 0:
|
||||||
|
vllm_ttfts.append(vllm_ttft_ms / 1000.0)
|
||||||
|
vllm_tpot_ms = float((info or {}).get(defs.VLLM_TPOT_MS) or 0.0)
|
||||||
|
if vllm_tpot_ms > 0:
|
||||||
|
vllm_tpots.append(vllm_tpot_ms / 1000.0)
|
||||||
|
vllm_values = (info or {}).get(defs.VLLM_ITLS_MS)
|
||||||
|
if isinstance(vllm_values, list):
|
||||||
|
vllm_itls.extend(float(v or 0.0) / 1000.0 for v in vllm_values)
|
||||||
|
elif (info or {}).get(defs.VLLM_ITL_MS) is not None:
|
||||||
|
vllm_itls.append(float((info or {}).get(defs.VLLM_ITL_MS) or 0.0) / 1000.0)
|
||||||
|
total_output = sum(int((info or {}).get(defs.NUM_TOKENS_OUT) or 0) for _, info in rows)
|
||||||
|
|
||||||
|
audio_ttfps: list[float] = []
|
||||||
|
audio_rtfs: list[float] = []
|
||||||
|
audio_durations: list[float] = []
|
||||||
|
audio_frames: list[int] = []
|
||||||
|
missing_audio_duration_count = 0
|
||||||
|
missing_audio_rtf_count = 0
|
||||||
|
if is_audio_stage:
|
||||||
|
for _, info in rows:
|
||||||
|
audio_ttfp_ms = float((info or {}).get(defs.SERVING_TIME_TO_FIRST_OUTPUT_MS) or 0.0)
|
||||||
|
if audio_ttfp_ms > 0:
|
||||||
|
audio_ttfps.append(audio_ttfp_ms / 1000.0)
|
||||||
|
|
||||||
|
frame_count = int((info or {}).get(defs.AUDIO_FRAMES) or 0)
|
||||||
|
if frame_count <= 0 and (info or {}).get("output_unit_type") == "audio":
|
||||||
|
frame_count = int((info or {}).get(defs.OUTPUT_UNIT_COUNT) or 0)
|
||||||
|
audio_frames.append(frame_count)
|
||||||
|
|
||||||
|
duration_s = float((info or {}).get(f"{defs.AUDIO_DURATION}_s") or 0.0)
|
||||||
|
if duration_s > 0:
|
||||||
|
audio_durations.append(duration_s)
|
||||||
|
else:
|
||||||
|
missing_audio_duration_count += 1
|
||||||
|
|
||||||
|
rtf = float((info or {}).get(defs.AUDIO_RTF) or 0.0)
|
||||||
|
if rtf > 0:
|
||||||
|
audio_rtfs.append(rtf)
|
||||||
|
else:
|
||||||
|
missing_audio_rtf_count += 1
|
||||||
|
|
||||||
|
result.append(
|
||||||
|
StageBenchmarkMetrics(
|
||||||
|
**{
|
||||||
|
"stage_id": stage_id_int,
|
||||||
|
"stage_name": stage_name,
|
||||||
|
"final_output_type": final_output_type,
|
||||||
|
defs.TOTAL_OUTPUT: total_output,
|
||||||
|
defs.TTFTS: ttfts,
|
||||||
|
defs.TPOTS: tpots,
|
||||||
|
defs.ITLS: itls,
|
||||||
|
defs.VLLM_TTFTS: vllm_ttfts,
|
||||||
|
defs.VLLM_TPOTS: vllm_tpots,
|
||||||
|
defs.VLLM_ITLS: vllm_itls,
|
||||||
|
defs.AUDIO_TTFPS: audio_ttfps,
|
||||||
|
defs.AUDIO_RTFS: audio_rtfs,
|
||||||
|
defs.AUDIO_DURATIONS: audio_durations,
|
||||||
|
defs.AUDIO_FRAMES: audio_frames,
|
||||||
|
defs.MISSING_AUDIO_DURATION_COUNT: missing_audio_duration_count,
|
||||||
|
defs.MISSING_AUDIO_RTF_COUNT: missing_audio_rtf_count,
|
||||||
|
defs.STAGE_GEN_TIMES_MS: stage_gen_times_ms,
|
||||||
|
defs.POSTPROCESS_TIMES_MS: postprocess_times_ms,
|
||||||
|
"output_unit_type": output_unit_type,
|
||||||
|
defs.OUTPUT_UNIT_COUNT: output_unit_count,
|
||||||
|
defs.SERVING_TIME_TO_FIRST_OUTPUTS_MS: serving_time_to_first_outputs_ms,
|
||||||
|
defs.TIME_PER_OUTPUT_UNITS_MS: time_per_output_units_ms,
|
||||||
|
defs.INTER_OUTPUT_LATENCIES_MS: inter_output_latencies_ms,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
def calculate_metrics(
|
def calculate_metrics(
|
||||||
input_requests: list[SampleRequest],
|
input_requests: list[SampleRequest],
|
||||||
outputs: list[RequestFuncOutput],
|
outputs: list[RequestFuncOutput],
|
||||||
@@ -160,6 +715,7 @@ def calculate_metrics(
|
|||||||
max_concurrency,
|
max_concurrency,
|
||||||
request_rate,
|
request_rate,
|
||||||
benchmark_duration,
|
benchmark_duration,
|
||||||
|
print_stage: bool = False,
|
||||||
) -> tuple[BenchmarkMetrics, list[int]]:
|
) -> tuple[BenchmarkMetrics, list[int]]:
|
||||||
"""Calculate the metrics for the benchmark.
|
"""Calculate the metrics for the benchmark.
|
||||||
|
|
||||||
@@ -187,6 +743,10 @@ def calculate_metrics(
|
|||||||
audio_rtfs: list[float] = []
|
audio_rtfs: list[float] = []
|
||||||
audio_duration: list[float] = []
|
audio_duration: list[float] = []
|
||||||
audio_frames: list[int] = []
|
audio_frames: list[int] = []
|
||||||
|
image_generation_times_ms: list[float] = []
|
||||||
|
denoise_step_latencies_ms: list[float] = []
|
||||||
|
total_images = 0
|
||||||
|
total_image_pixels = 0
|
||||||
audio_underruns: list[float] = []
|
audio_underruns: list[float] = []
|
||||||
audio_continuity_ok: list[bool] = []
|
audio_continuity_ok: list[bool] = []
|
||||||
input_audio_duration = 0.0
|
input_audio_duration = 0.0
|
||||||
@@ -222,6 +782,15 @@ def calculate_metrics(
|
|||||||
audio_rtfs.append(getattr(outputs[i], defs.AUDIO_RTF, 0.0))
|
audio_rtfs.append(getattr(outputs[i], defs.AUDIO_RTF, 0.0))
|
||||||
audio_duration.append(getattr(outputs[i], defs.AUDIO_DURATION, 0.0))
|
audio_duration.append(getattr(outputs[i], defs.AUDIO_DURATION, 0.0))
|
||||||
audio_frames.append(getattr(outputs[i], defs.AUDIO_FRAMES, 0.0))
|
audio_frames.append(getattr(outputs[i], defs.AUDIO_FRAMES, 0.0))
|
||||||
|
image_count = int(getattr(outputs[i], defs.IMAGE_COUNT, 0) or 0)
|
||||||
|
total_images += image_count
|
||||||
|
image_generation_time_ms = float(getattr(outputs[i], defs.IMAGE_GENERATION_TIME_MS, 0.0) or 0.0)
|
||||||
|
if image_generation_time_ms > 0:
|
||||||
|
image_generation_times_ms.append(image_generation_time_ms)
|
||||||
|
total_image_pixels += int(getattr(outputs[i], defs.IMAGE_PIXELS, 0) or 0)
|
||||||
|
denoise_step_latency_ms = float(getattr(outputs[i], defs.DENOISE_STEP_LATENCY_MS, 0.0) or 0.0)
|
||||||
|
if denoise_step_latency_ms > 0:
|
||||||
|
denoise_step_latencies_ms.append(denoise_step_latency_ms)
|
||||||
audio_underruns.append(getattr(outputs[i], f"{defs.AUDIO_UNDERRUN}_s", 0.0))
|
audio_underruns.append(getattr(outputs[i], f"{defs.AUDIO_UNDERRUN}_s", 0.0))
|
||||||
audio_continuity_ok.append(bool(getattr(outputs[i], defs.AUDIO_CONTINUITY_OK, True)))
|
audio_continuity_ok.append(bool(getattr(outputs[i], defs.AUDIO_CONTINUITY_OK, True)))
|
||||||
e2els.append(outputs[i].latency)
|
e2els.append(outputs[i].latency)
|
||||||
@@ -336,26 +905,6 @@ def calculate_metrics(
|
|||||||
std_ttft_ms=np.std(ttfts or 0) * 1000,
|
std_ttft_ms=np.std(ttfts or 0) * 1000,
|
||||||
median_ttft_ms=np.median(ttfts or 0) * 1000,
|
median_ttft_ms=np.median(ttfts or 0) * 1000,
|
||||||
percentiles_ttft_ms=[(p, np.percentile(ttfts or 0, p) * 1000) for p in selected_percentiles],
|
percentiles_ttft_ms=[(p, np.percentile(ttfts or 0, p) * 1000) for p in selected_percentiles],
|
||||||
mean_audio_ttfp_ms=np.mean(audio_ttfps or 0) * 1000,
|
|
||||||
std_audio_ttfp_ms=np.std(audio_ttfps or 0) * 1000,
|
|
||||||
median_audio_ttfp_ms=np.median(audio_ttfps or 0) * 1000,
|
|
||||||
percentiles_audio_ttfp_ms=[(p, np.percentile(audio_ttfps or 0, p) * 1000) for p in selected_percentiles],
|
|
||||||
mean_audio_duration_s=np.mean(audio_duration or 0),
|
|
||||||
std_audio_duration_s=np.std(audio_duration or 0),
|
|
||||||
median_audio_duration_s=np.median(audio_duration or 0),
|
|
||||||
percentiles_audio_duration_s=[(p, np.percentile(audio_duration or 0, p)) for p in selected_percentiles],
|
|
||||||
total_audio_duration_s=sum(audio_duration),
|
|
||||||
total_audio_frames=sum(audio_frames),
|
|
||||||
audio_throughput=sum(audio_duration) / dur_s,
|
|
||||||
mean_audio_rtf=np.mean(audio_rtfs or 0),
|
|
||||||
std_audio_rtf=np.std(audio_rtfs or 0),
|
|
||||||
median_audio_rtf=np.median(audio_rtfs or 0),
|
|
||||||
percentiles_audio_rtf=[(p, np.percentile(audio_rtfs or 0, p)) for p in selected_percentiles],
|
|
||||||
mean_audio_underrun_s=np.mean(audio_underruns or 0),
|
|
||||||
std_audio_underrun_s=np.std(audio_underruns or 0),
|
|
||||||
median_audio_underrun_s=np.median(audio_underruns or 0),
|
|
||||||
percentiles_audio_underrun_s=[(p, np.percentile(audio_underruns or 0, p)) for p in selected_percentiles],
|
|
||||||
audio_continuity_ok_rate=(sum(audio_continuity_ok) / len(audio_continuity_ok)) if audio_continuity_ok else 1.0,
|
|
||||||
mean_tpot_ms=np.mean(tpots or 0) * 1000,
|
mean_tpot_ms=np.mean(tpots or 0) * 1000,
|
||||||
std_tpot_ms=np.std(tpots or 0) * 1000,
|
std_tpot_ms=np.std(tpots or 0) * 1000,
|
||||||
median_tpot_ms=np.median(tpots or 0) * 1000,
|
median_tpot_ms=np.median(tpots or 0) * 1000,
|
||||||
@@ -371,6 +920,46 @@ def calculate_metrics(
|
|||||||
max_output_tokens_per_s=max_output_tokens_per_s,
|
max_output_tokens_per_s=max_output_tokens_per_s,
|
||||||
max_concurrent_requests=max_concurrent_requests,
|
max_concurrent_requests=max_concurrent_requests,
|
||||||
rtfx=input_audio_duration / dur_s,
|
rtfx=input_audio_duration / dur_s,
|
||||||
|
**{
|
||||||
|
defs.MEAN_AUDIO_TTFP_MS: np.mean(audio_ttfps or 0) * 1000,
|
||||||
|
defs.STD_AUDIO_TTFP_MS: np.std(audio_ttfps or 0) * 1000,
|
||||||
|
defs.MEDIAN_AUDIO_TTFP_MS: np.median(audio_ttfps or 0) * 1000,
|
||||||
|
defs.PERCENTILES_AUDIO_TTFP_MS: [
|
||||||
|
(p, np.percentile(audio_ttfps or 0, p) * 1000) for p in selected_percentiles
|
||||||
|
],
|
||||||
|
defs.MEAN_AUDIO_DURATION_S: np.mean(audio_duration or 0),
|
||||||
|
defs.STD_AUDIO_DURATION_S: np.std(audio_duration or 0),
|
||||||
|
defs.MEDIAN_AUDIO_DURATION_S: np.median(audio_duration or 0),
|
||||||
|
defs.PERCENTILES_AUDIO_DURATION_S: [
|
||||||
|
(p, np.percentile(audio_duration or 0, p)) for p in selected_percentiles
|
||||||
|
],
|
||||||
|
defs.TOTAL_AUDIO_DURATION_S: sum(audio_duration),
|
||||||
|
defs.TOTAL_AUDIO_FRAMES: sum(audio_frames),
|
||||||
|
defs.AUDIO_THROUGHPUT: sum(audio_duration) / dur_s,
|
||||||
|
defs.MEAN_AUDIO_RTF: np.mean(audio_rtfs or 0),
|
||||||
|
defs.STD_AUDIO_RTF: np.std(audio_rtfs or 0),
|
||||||
|
defs.MEDIAN_AUDIO_RTF: np.median(audio_rtfs or 0),
|
||||||
|
defs.PERCENTILES_AUDIO_RTF: [(p, np.percentile(audio_rtfs or 0, p)) for p in selected_percentiles],
|
||||||
|
defs.TOTAL_IMAGES: total_images,
|
||||||
|
defs.IMAGE_THROUGHPUT: total_images / dur_s,
|
||||||
|
defs.AVERAGE_PIXELS_PER_IMAGE: (total_image_pixels / total_images) if total_images > 0 else 0.0,
|
||||||
|
defs.MEAN_DENOISE_STEP_LATENCY_MS: np.mean(denoise_step_latencies_ms or 0),
|
||||||
|
defs.MEAN_IMAGE_GENERATION_MS: np.mean(image_generation_times_ms or 0),
|
||||||
|
defs.STD_IMAGE_GENERATION_MS: np.std(image_generation_times_ms or 0),
|
||||||
|
defs.MEDIAN_IMAGE_GENERATION_MS: np.median(image_generation_times_ms or 0),
|
||||||
|
defs.PERCENTILES_IMAGE_GENERATION_MS: [
|
||||||
|
(p, np.percentile(image_generation_times_ms or 0, p)) for p in selected_percentiles
|
||||||
|
],
|
||||||
|
defs.MEAN_AUDIO_UNDERRUN_S: np.mean(audio_underruns or 0),
|
||||||
|
defs.STD_AUDIO_UNDERRUN_S: np.std(audio_underruns or 0),
|
||||||
|
defs.MEDIAN_AUDIO_UNDERRUN_S: np.median(audio_underruns or 0),
|
||||||
|
defs.PERCENTILES_AUDIO_UNDERRUN_S: [
|
||||||
|
(p, np.percentile(audio_underruns or 0, p)) for p in selected_percentiles
|
||||||
|
],
|
||||||
|
defs.AUDIO_CONTINUITY_OK_RATE: (
|
||||||
|
(sum(audio_continuity_ok) / len(audio_continuity_ok)) if audio_continuity_ok else 1.0
|
||||||
|
),
|
||||||
|
},
|
||||||
)
|
)
|
||||||
print_metrics(
|
print_metrics(
|
||||||
task_type,
|
task_type,
|
||||||
@@ -380,5 +969,8 @@ def calculate_metrics(
|
|||||||
benchmark_duration,
|
benchmark_duration,
|
||||||
goodput_config_dict,
|
goodput_config_dict,
|
||||||
metrics,
|
metrics,
|
||||||
|
outputs=outputs,
|
||||||
|
selected_percentiles=selected_percentiles,
|
||||||
|
print_stage=print_stage,
|
||||||
)
|
)
|
||||||
return metrics, actual_output_lens
|
return metrics, actual_output_lens
|
||||||
@@ -2,6 +2,7 @@ import asyncio
|
|||||||
import contextlib
|
import contextlib
|
||||||
import io
|
import io
|
||||||
import json
|
import json
|
||||||
|
import mimetypes
|
||||||
import os
|
import os
|
||||||
import random
|
import random
|
||||||
import ssl
|
import ssl
|
||||||
@@ -12,7 +13,7 @@ import wave
|
|||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Literal
|
from typing import Any, Literal
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
import pybase64 as base64
|
import pybase64 as base64
|
||||||
@@ -48,9 +49,41 @@ from vllm_omni.benchmarks.data_modules.seed_tts_dataset import (
|
|||||||
)
|
)
|
||||||
from vllm_omni.benchmarks.data_modules.sound_effect_dataset import SoundEffectDataset
|
from vllm_omni.benchmarks.data_modules.sound_effect_dataset import SoundEffectDataset
|
||||||
from vllm_omni.benchmarks.data_modules.ttsd_dataset import TTSDDataset
|
from vllm_omni.benchmarks.data_modules.ttsd_dataset import TTSDDataset
|
||||||
|
from vllm_omni.metrics import definitions as defs
|
||||||
|
|
||||||
_AUDIO_CONTINUITY_THRESHOLD_ENV = "VLLM_OMNI_BENCH_AUDIO_CONTINUITY_THRESHOLD_S"
|
_AUDIO_CONTINUITY_THRESHOLD_ENV = "VLLM_OMNI_BENCH_AUDIO_CONTINUITY_THRESHOLD_S"
|
||||||
_DEFAULT_AUDIO_CONTINUITY_THRESHOLD_S = 0.1
|
RETURN_STAGE_METRICS_FIELD = "return_stage_metrics"
|
||||||
|
_IMAGE_STAGE_METRICS_BACKENDS = frozenset({"openai-image-edits-omni"})
|
||||||
|
_PRINT_STAGE = False
|
||||||
|
|
||||||
|
|
||||||
|
def maybe_enable_stage_metrics(extra_body: dict[str, Any] | None, *, enabled: bool) -> dict[str, Any] | None:
|
||||||
|
"""Return extra_body with stage-metric opt-in when benchmark metrics need it."""
|
||||||
|
if not enabled:
|
||||||
|
return extra_body
|
||||||
|
body = dict(extra_body or {})
|
||||||
|
body.setdefault(RETURN_STAGE_METRICS_FIELD, True)
|
||||||
|
return body
|
||||||
|
|
||||||
|
|
||||||
|
def should_request_stage_metrics(args: Any) -> bool:
|
||||||
|
"""Whether this benchmark run needs server-side stage metrics in responses."""
|
||||||
|
if getattr(args, "print_stage", False):
|
||||||
|
return True
|
||||||
|
|
||||||
|
backend = getattr(args, "backend", None)
|
||||||
|
if backend in _IMAGE_STAGE_METRICS_BACKENDS:
|
||||||
|
return True
|
||||||
|
|
||||||
|
extra_body = getattr(args, "extra_body", None) or {}
|
||||||
|
modalities = extra_body.get("modalities") if isinstance(extra_body, dict) else None
|
||||||
|
return backend == "openai-chat-omni" and "image" in (modalities or [])
|
||||||
|
|
||||||
|
|
||||||
|
def set_print_stage(enabled: bool) -> None:
|
||||||
|
"""Set whether this benchmark run prints the stage benchmark section."""
|
||||||
|
global _PRINT_STAGE
|
||||||
|
_PRINT_STAGE = bool(enabled)
|
||||||
|
|
||||||
|
|
||||||
def _audio_continuity_threshold_s() -> float:
|
def _audio_continuity_threshold_s() -> float:
|
||||||
@@ -62,7 +95,7 @@ def _audio_continuity_threshold_s() -> float:
|
|||||||
"""
|
"""
|
||||||
raw = os.environ.get(_AUDIO_CONTINUITY_THRESHOLD_ENV)
|
raw = os.environ.get(_AUDIO_CONTINUITY_THRESHOLD_ENV)
|
||||||
if not raw:
|
if not raw:
|
||||||
return _DEFAULT_AUDIO_CONTINUITY_THRESHOLD_S
|
return defs.AUDIO_CONTINUITY_DEFAULT_THRESHOLD_S
|
||||||
try:
|
try:
|
||||||
value = float(raw)
|
value = float(raw)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
@@ -70,9 +103,9 @@ def _audio_continuity_threshold_s() -> float:
|
|||||||
"Invalid %s=%r; using default %.3fs",
|
"Invalid %s=%r; using default %.3fs",
|
||||||
_AUDIO_CONTINUITY_THRESHOLD_ENV,
|
_AUDIO_CONTINUITY_THRESHOLD_ENV,
|
||||||
raw,
|
raw,
|
||||||
_DEFAULT_AUDIO_CONTINUITY_THRESHOLD_S,
|
defs.AUDIO_CONTINUITY_DEFAULT_THRESHOLD_S,
|
||||||
)
|
)
|
||||||
return _DEFAULT_AUDIO_CONTINUITY_THRESHOLD_S
|
return defs.AUDIO_CONTINUITY_DEFAULT_THRESHOLD_S
|
||||||
return max(value, 0.0)
|
return max(value, 0.0)
|
||||||
|
|
||||||
|
|
||||||
@@ -352,6 +385,10 @@ class MixRequestFuncOutput(RequestFuncOutput):
|
|||||||
audio_duration: float = 0.0
|
audio_duration: float = 0.0
|
||||||
audio_frames: int = 0
|
audio_frames: int = 0
|
||||||
audio_rtf: float = 0.0
|
audio_rtf: float = 0.0
|
||||||
|
image_count: int = 0
|
||||||
|
image_generation_time_ms: float = 0.0
|
||||||
|
image_pixels: int = 0
|
||||||
|
denoise_step_latency_ms: float = 0.0
|
||||||
text_latency: float = 0.0
|
text_latency: float = 0.0
|
||||||
#: Worst-case streaming-audio underrun (wall-clock seconds the player
|
#: Worst-case streaming-audio underrun (wall-clock seconds the player
|
||||||
#: would have been starved). Populated by the audio-speech backend; ``0.0``
|
#: would have been starved). Populated by the audio-speech backend; ``0.0``
|
||||||
@@ -366,6 +403,200 @@ class MixRequestFuncOutput(RequestFuncOutput):
|
|||||||
#: Raw PCM s16le mono at 24 kHz for Seed-TTS WER: from ``/v1/audio/speech`` stream or
|
#: Raw PCM s16le mono at 24 kHz for Seed-TTS WER: from ``/v1/audio/speech`` stream or
|
||||||
#: resampled export after ``openai-chat-omni`` audio deltas.
|
#: resampled export after ``openai-chat-omni`` audio deltas.
|
||||||
tts_output_pcm_bytes: bytes | None = None
|
tts_output_pcm_bytes: bytes | None = None
|
||||||
|
#: Per-stage snapshot from orchestrator ``metrics["stage_metrics"]`` (merged across SSE chunks).
|
||||||
|
stage_metrics: dict[str, dict] | None = None
|
||||||
|
stage_id: int | None = None
|
||||||
|
final_output_type: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
_IMAGE_EDITS_EXTRA_BODY_FORM_FIELDS = (
|
||||||
|
"negative_prompt",
|
||||||
|
"num_inference_steps",
|
||||||
|
"guidance_scale",
|
||||||
|
"strength",
|
||||||
|
"true_cfg_scale",
|
||||||
|
"seed",
|
||||||
|
"generator_device",
|
||||||
|
"lora",
|
||||||
|
"layers",
|
||||||
|
"resolution",
|
||||||
|
"bot_task",
|
||||||
|
"sys_type",
|
||||||
|
"system_prompt",
|
||||||
|
RETURN_STAGE_METRICS_FIELD,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _guess_mime_type(path: str) -> str:
|
||||||
|
mime, _ = mimetypes.guess_type(path)
|
||||||
|
return mime or "application/octet-stream"
|
||||||
|
|
||||||
|
|
||||||
|
def _iter_image_edit_inputs(value: Any) -> Iterable[Any]:
|
||||||
|
"""Yield image references from benchmark multimodal content."""
|
||||||
|
if value is None:
|
||||||
|
return
|
||||||
|
if isinstance(value, list):
|
||||||
|
for item in value:
|
||||||
|
yield from _iter_image_edit_inputs(item)
|
||||||
|
return
|
||||||
|
if not isinstance(value, dict):
|
||||||
|
yield value
|
||||||
|
return
|
||||||
|
|
||||||
|
content_type = value.get("type")
|
||||||
|
if content_type == "image_url":
|
||||||
|
image_url = value.get("image_url")
|
||||||
|
if isinstance(image_url, dict):
|
||||||
|
url = image_url.get("url")
|
||||||
|
if url:
|
||||||
|
yield url
|
||||||
|
elif image_url:
|
||||||
|
yield image_url
|
||||||
|
return
|
||||||
|
|
||||||
|
for key in ("image", "images"):
|
||||||
|
if key in value:
|
||||||
|
yield from _iter_image_edit_inputs(value[key])
|
||||||
|
|
||||||
|
|
||||||
|
def _add_image_edit_input_to_form(form: aiohttp.FormData, image_input: Any) -> None:
|
||||||
|
if isinstance(image_input, dict) and "bytes" in image_input:
|
||||||
|
form.add_field(
|
||||||
|
"image",
|
||||||
|
image_input["bytes"],
|
||||||
|
filename="benchmark.png",
|
||||||
|
content_type="image/png",
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
if isinstance(image_input, str):
|
||||||
|
if image_input.startswith(("data:image", "http://", "https://")):
|
||||||
|
form.add_field("url", image_input)
|
||||||
|
return
|
||||||
|
local_path = image_input.removeprefix("file://")
|
||||||
|
if os.path.exists(local_path):
|
||||||
|
with open(local_path, "rb") as f:
|
||||||
|
image_bytes = f.read()
|
||||||
|
form.add_field(
|
||||||
|
"image",
|
||||||
|
image_bytes,
|
||||||
|
filename=os.path.basename(local_path),
|
||||||
|
content_type=_guess_mime_type(local_path),
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
raise ValueError(f"Unsupported image edit input: {type(image_input).__name__}")
|
||||||
|
|
||||||
|
|
||||||
|
def _add_image_edit_extra_body_to_form(form: aiohttp.FormData, extra_body: dict[str, Any]) -> None:
|
||||||
|
for key in _IMAGE_EDITS_EXTRA_BODY_FORM_FIELDS:
|
||||||
|
value = extra_body.get(key)
|
||||||
|
if value is None:
|
||||||
|
continue
|
||||||
|
if isinstance(value, (dict, list)):
|
||||||
|
form.add_field(key, json.dumps(value))
|
||||||
|
else:
|
||||||
|
form.add_field(key, str(value))
|
||||||
|
|
||||||
|
|
||||||
|
def _coerce_positive_int(value: Any) -> int | None:
|
||||||
|
try:
|
||||||
|
coerced = int(value)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
return coerced if coerced > 0 else None
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_output_tokens_from_metrics(metrics: dict[str, Any]) -> int | None:
|
||||||
|
top_level_tokens = _coerce_positive_int(metrics.get(defs.NUM_TOKENS_OUT))
|
||||||
|
if top_level_tokens is not None:
|
||||||
|
return top_level_tokens
|
||||||
|
|
||||||
|
stage_snapshot = metrics.get("stage_metrics")
|
||||||
|
if not isinstance(stage_snapshot, dict):
|
||||||
|
return None
|
||||||
|
|
||||||
|
fallback_tokens: list[int] = []
|
||||||
|
for info in stage_snapshot.values():
|
||||||
|
if not isinstance(info, dict):
|
||||||
|
continue
|
||||||
|
num_tokens_out = _coerce_positive_int(info.get(defs.NUM_TOKENS_OUT))
|
||||||
|
if num_tokens_out is None:
|
||||||
|
continue
|
||||||
|
if info.get("final_output_type") == "text" or info.get("output_unit_type") == "token":
|
||||||
|
return num_tokens_out
|
||||||
|
fallback_tokens.append(num_tokens_out)
|
||||||
|
return max(fallback_tokens, default=None)
|
||||||
|
|
||||||
|
|
||||||
|
def _update_output_stage_metrics_from_payload(
|
||||||
|
output: MixRequestFuncOutput,
|
||||||
|
data: dict[str, Any],
|
||||||
|
*,
|
||||||
|
update_output_tokens: bool = True,
|
||||||
|
) -> None:
|
||||||
|
metrics = data.get("metrics")
|
||||||
|
if not isinstance(metrics, dict):
|
||||||
|
return
|
||||||
|
if update_output_tokens:
|
||||||
|
if (num_tokens_out := _extract_output_tokens_from_metrics(metrics)) is not None:
|
||||||
|
output.output_tokens = num_tokens_out
|
||||||
|
if isinstance(sid := metrics.get("stage_id"), int):
|
||||||
|
output.stage_id = sid
|
||||||
|
if isinstance(final_output_type := metrics.get("final_output_type"), str):
|
||||||
|
output.final_output_type = final_output_type
|
||||||
|
stage_snapshot = metrics.get("stage_metrics")
|
||||||
|
if isinstance(stage_snapshot, dict):
|
||||||
|
if output.stage_metrics is None:
|
||||||
|
output.stage_metrics = {}
|
||||||
|
output.stage_metrics.update(stage_snapshot)
|
||||||
|
|
||||||
|
|
||||||
|
def _image_metrics_from_stage_metrics(metrics: dict[str, Any] | None) -> tuple[int, float, int, float]:
|
||||||
|
if not isinstance(metrics, dict):
|
||||||
|
return 0, 0.0, 0, 0.0
|
||||||
|
stage_snapshot = metrics.get("stage_metrics")
|
||||||
|
if not isinstance(stage_snapshot, dict):
|
||||||
|
return 0, 0.0, 0, 0.0
|
||||||
|
image_count = 0
|
||||||
|
image_generation_ms = 0.0
|
||||||
|
image_pixels = 0
|
||||||
|
denoise_step_latency_ms = 0.0
|
||||||
|
for info in stage_snapshot.values():
|
||||||
|
if not isinstance(info, dict):
|
||||||
|
continue
|
||||||
|
final_output_type = info.get("final_output_type")
|
||||||
|
output_unit_type = info.get("output_unit_type")
|
||||||
|
if final_output_type not in {"image", "images"} and output_unit_type != "image":
|
||||||
|
continue
|
||||||
|
image_count += int(info.get(defs.OUTPUT_UNIT_COUNT) or 0)
|
||||||
|
image_generation_ms += float(info.get(defs.STAGE_GEN_TIME_MS) or 0.0)
|
||||||
|
image_pixels += int(info.get(defs.IMAGE_PIXELS) or 0)
|
||||||
|
denoise_step_latency_ms = max(
|
||||||
|
denoise_step_latency_ms,
|
||||||
|
float(info.get(defs.DENOISE_STEP_LATENCY_MS) or 0.0),
|
||||||
|
)
|
||||||
|
return image_count, image_generation_ms, image_pixels, denoise_step_latency_ms
|
||||||
|
|
||||||
|
|
||||||
|
def _image_generation_ms_from_content(content: Any) -> float:
|
||||||
|
if not isinstance(content, list):
|
||||||
|
return 0.0
|
||||||
|
for item in content:
|
||||||
|
if not isinstance(item, dict):
|
||||||
|
continue
|
||||||
|
stage_durations = item.get("stage_durations")
|
||||||
|
if not isinstance(stage_durations, dict):
|
||||||
|
continue
|
||||||
|
gen_values = [
|
||||||
|
float(value)
|
||||||
|
for key, value in stage_durations.items()
|
||||||
|
if str(key).endswith("_gen_ms") and isinstance(value, (int, float))
|
||||||
|
]
|
||||||
|
if gen_values:
|
||||||
|
return max(gen_values)
|
||||||
|
return 0.0
|
||||||
|
|
||||||
|
|
||||||
async def async_request_openai_chat_omni_completions(
|
async def async_request_openai_chat_omni_completions(
|
||||||
@@ -448,6 +679,13 @@ async def async_request_openai_chat_omni_completions(
|
|||||||
output.output_tokens = 0
|
output.output_tokens = 0
|
||||||
output.error = ""
|
output.error = ""
|
||||||
output.success = False
|
output.success = False
|
||||||
|
output.stage_metrics = {}
|
||||||
|
output.stage_id = None
|
||||||
|
output.final_output_type = None
|
||||||
|
output.image_count = 0
|
||||||
|
output.image_generation_time_ms = 0.0
|
||||||
|
output.image_pixels = 0
|
||||||
|
output.denoise_step_latency_ms = 0.0
|
||||||
try:
|
try:
|
||||||
async with session.post(url=api_url, json=payload, headers=headers) as response:
|
async with session.post(url=api_url, json=payload, headers=headers) as response:
|
||||||
if response.status == 200:
|
if response.status == 200:
|
||||||
@@ -519,9 +757,27 @@ async def async_request_openai_chat_omni_completions(
|
|||||||
logger.warning("Failed to parse wav audio chunk: %s", ex)
|
logger.warning("Failed to parse wav audio chunk: %s", ex)
|
||||||
else:
|
else:
|
||||||
audio_bytes_buffer.extend(audio_bytes)
|
audio_bytes_buffer.extend(audio_bytes)
|
||||||
|
elif modality == "image":
|
||||||
|
output.image_count += 1
|
||||||
|
content_image_ms = _image_generation_ms_from_content(content)
|
||||||
|
if content_image_ms > 0:
|
||||||
|
output.image_generation_time_ms += content_image_ms
|
||||||
|
|
||||||
if metrics := data.get("metrics"):
|
_update_output_stage_metrics_from_payload(output, data)
|
||||||
output.output_tokens = metrics.get("num_tokens_out", 0)
|
(
|
||||||
|
metrics_image_count,
|
||||||
|
metrics_image_ms,
|
||||||
|
metrics_image_pixels,
|
||||||
|
metrics_denoise_step_ms,
|
||||||
|
) = _image_metrics_from_stage_metrics(data.get("metrics"))
|
||||||
|
if metrics_image_count > output.image_count:
|
||||||
|
output.image_count = metrics_image_count
|
||||||
|
if metrics_image_ms > output.image_generation_time_ms:
|
||||||
|
output.image_generation_time_ms = metrics_image_ms
|
||||||
|
if metrics_image_pixels > output.image_pixels:
|
||||||
|
output.image_pixels = metrics_image_pixels
|
||||||
|
if metrics_denoise_step_ms > output.denoise_step_latency_ms:
|
||||||
|
output.denoise_step_latency_ms = metrics_denoise_step_ms
|
||||||
|
|
||||||
if usage := data.get("usage"):
|
if usage := data.get("usage"):
|
||||||
if (pt := usage.get("prompt_tokens")) is not None:
|
if (pt := usage.get("prompt_tokens")) is not None:
|
||||||
@@ -610,6 +866,138 @@ async def async_request_openai_chat_omni_completions(
|
|||||||
return output
|
return output
|
||||||
|
|
||||||
|
|
||||||
|
async def async_request_openai_image_edits_omni(
|
||||||
|
request_func_input: RequestFuncInput,
|
||||||
|
session: aiohttp.ClientSession,
|
||||||
|
pbar: tqdm | None = None,
|
||||||
|
) -> MixRequestFuncOutput:
|
||||||
|
"""Streaming request to /v1/images/edits for multi-stage image-edit benchmarks."""
|
||||||
|
api_url = request_func_input.api_url
|
||||||
|
_validate_api_url(api_url, "OpenAI Image Edits API", "images/edits")
|
||||||
|
|
||||||
|
extra_body = dict(request_func_input.extra_body or {})
|
||||||
|
model = request_func_input.model_name if request_func_input.model_name else request_func_input.model
|
||||||
|
output = MixRequestFuncOutput()
|
||||||
|
output.prompt_len = request_func_input.prompt_len
|
||||||
|
output.itl = []
|
||||||
|
output.stage_metrics = {}
|
||||||
|
output.output_tokens = 0
|
||||||
|
output.image_count = 0
|
||||||
|
output.image_generation_time_ms = 0.0
|
||||||
|
output.image_pixels = 0
|
||||||
|
output.denoise_step_latency_ms = 0.0
|
||||||
|
|
||||||
|
form = aiohttp.FormData()
|
||||||
|
form.add_field("model", model)
|
||||||
|
form.add_field("prompt", request_func_input.prompt)
|
||||||
|
form.add_field("response_format", "b64_json")
|
||||||
|
form.add_field("output_format", str(extra_body.get("output_format", "png")))
|
||||||
|
form.add_field("stream", "true")
|
||||||
|
|
||||||
|
size = extra_body.get("size")
|
||||||
|
if size is None:
|
||||||
|
width, height = extra_body.get("width"), extra_body.get("height")
|
||||||
|
size = f"{width}x{height}" if width is not None and height is not None else "auto"
|
||||||
|
form.add_field("size", str(size))
|
||||||
|
|
||||||
|
_add_image_edit_extra_body_to_form(form, extra_body)
|
||||||
|
|
||||||
|
try:
|
||||||
|
image_inputs = list(_iter_image_edit_inputs(request_func_input.multi_modal_content))
|
||||||
|
if not image_inputs:
|
||||||
|
raise ValueError(
|
||||||
|
"openai-image-edits-omni requires image multimodal content. "
|
||||||
|
"For synthetic inputs, use --dataset-name random-mm with an image bucket."
|
||||||
|
)
|
||||||
|
for image_input in image_inputs:
|
||||||
|
_add_image_edit_input_to_form(form, image_input)
|
||||||
|
except Exception:
|
||||||
|
output.success = False
|
||||||
|
output.error = traceback.format_exc()
|
||||||
|
if pbar:
|
||||||
|
pbar.update(1)
|
||||||
|
return output
|
||||||
|
|
||||||
|
headers = {
|
||||||
|
"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}",
|
||||||
|
}
|
||||||
|
_update_headers_common(headers, request_func_input)
|
||||||
|
|
||||||
|
st = time.perf_counter()
|
||||||
|
output.start_time = st
|
||||||
|
timestamp = st
|
||||||
|
most_recent_text_timestamp = st
|
||||||
|
generated_text = ""
|
||||||
|
try:
|
||||||
|
async with session.post(url=api_url, data=form, headers=headers) as response:
|
||||||
|
if response.status == 200:
|
||||||
|
handler = StreamedResponseHandler()
|
||||||
|
async for chunk_bytes in response.content.iter_any():
|
||||||
|
if not chunk_bytes:
|
||||||
|
continue
|
||||||
|
for message in handler.add_chunk(chunk_bytes):
|
||||||
|
if type(message) is bytes:
|
||||||
|
message = message.decode("utf-8")
|
||||||
|
if message.startswith(":"):
|
||||||
|
continue
|
||||||
|
chunk = message.removeprefix("data: ")
|
||||||
|
if chunk == "[DONE]":
|
||||||
|
continue
|
||||||
|
|
||||||
|
timestamp = time.perf_counter()
|
||||||
|
data = json.loads(chunk)
|
||||||
|
_update_output_stage_metrics_from_payload(
|
||||||
|
output,
|
||||||
|
data,
|
||||||
|
update_output_tokens=(data.get("type") == "ar_delta"),
|
||||||
|
)
|
||||||
|
|
||||||
|
chunk_type = data.get("type")
|
||||||
|
if chunk_type == "ar_delta":
|
||||||
|
if output.ttft == 0.0:
|
||||||
|
output.ttft = timestamp - st
|
||||||
|
else:
|
||||||
|
output.itl.append(timestamp - most_recent_text_timestamp)
|
||||||
|
delta = data.get("delta") or ""
|
||||||
|
generated_text += delta
|
||||||
|
most_recent_text_timestamp = timestamp
|
||||||
|
output.text_latency = timestamp - st
|
||||||
|
elif chunk_type == "image":
|
||||||
|
image_data = data.get("data")
|
||||||
|
output.image_count += len(image_data) if isinstance(image_data, list) else 1
|
||||||
|
content_image_ms = _image_generation_ms_from_content(data.get("data"))
|
||||||
|
if content_image_ms > 0:
|
||||||
|
output.image_generation_time_ms += content_image_ms
|
||||||
|
(
|
||||||
|
metrics_image_count,
|
||||||
|
metrics_image_ms,
|
||||||
|
metrics_image_pixels,
|
||||||
|
metrics_denoise_step_ms,
|
||||||
|
) = _image_metrics_from_stage_metrics(data.get("metrics"))
|
||||||
|
if metrics_image_count > output.image_count:
|
||||||
|
output.image_count = metrics_image_count
|
||||||
|
if metrics_image_ms > output.image_generation_time_ms:
|
||||||
|
output.image_generation_time_ms = metrics_image_ms
|
||||||
|
if metrics_image_pixels > output.image_pixels:
|
||||||
|
output.image_pixels = metrics_image_pixels
|
||||||
|
if metrics_denoise_step_ms > output.denoise_step_latency_ms:
|
||||||
|
output.denoise_step_latency_ms = metrics_denoise_step_ms
|
||||||
|
output.latency = timestamp - st
|
||||||
|
output.generated_text = generated_text
|
||||||
|
output.success = True
|
||||||
|
else:
|
||||||
|
output.error = f"HTTP {response.status}: {await response.text()}"
|
||||||
|
output.success = False
|
||||||
|
except Exception:
|
||||||
|
output.success = False
|
||||||
|
output.error = traceback.format_exc()
|
||||||
|
logger.error(f"ERROR: send image edit request failed, reason is: {output.error}")
|
||||||
|
|
||||||
|
if pbar:
|
||||||
|
pbar.update(1)
|
||||||
|
return output
|
||||||
|
|
||||||
|
|
||||||
async def async_request_openai_audio_speech(
|
async def async_request_openai_audio_speech(
|
||||||
request_func_input: RequestFuncInput, session: aiohttp.ClientSession, pbar: tqdm | None = None
|
request_func_input: RequestFuncInput, session: aiohttp.ClientSession, pbar: tqdm | None = None
|
||||||
) -> MixRequestFuncOutput:
|
) -> MixRequestFuncOutput:
|
||||||
@@ -727,6 +1115,10 @@ ASYNC_REQUEST_FUNCS["openai-audio-speech"] = async_request_openai_audio_speech
|
|||||||
if "openai-audio-speech" not in OPENAI_COMPATIBLE_BACKENDS:
|
if "openai-audio-speech" not in OPENAI_COMPATIBLE_BACKENDS:
|
||||||
OPENAI_COMPATIBLE_BACKENDS.append("openai-audio-speech")
|
OPENAI_COMPATIBLE_BACKENDS.append("openai-audio-speech")
|
||||||
|
|
||||||
|
ASYNC_REQUEST_FUNCS["openai-image-edits-omni"] = async_request_openai_image_edits_omni
|
||||||
|
if "openai-image-edits-omni" not in OPENAI_COMPATIBLE_BACKENDS:
|
||||||
|
OPENAI_COMPATIBLE_BACKENDS.append("openai-image-edits-omni")
|
||||||
|
|
||||||
# Daily-Omni backend for audio-visual reasoning benchmark
|
# Daily-Omni backend for audio-visual reasoning benchmark
|
||||||
# Reuses openai-chat-omni completions for video+text understanding
|
# Reuses openai-chat-omni completions for video+text understanding
|
||||||
ASYNC_REQUEST_FUNCS["daily-omni"] = async_request_openai_chat_omni_completions
|
ASYNC_REQUEST_FUNCS["daily-omni"] = async_request_openai_chat_omni_completions
|
||||||
@@ -739,7 +1131,10 @@ from vllm.benchmarks import serve
|
|||||||
from vllm.benchmarks.lib.ready_checker import wait_for_endpoint
|
from vllm.benchmarks.lib.ready_checker import wait_for_endpoint
|
||||||
from vllm.benchmarks.serve import TaskType, calculate_metrics_for_embeddings, get_request
|
from vllm.benchmarks.serve import TaskType, calculate_metrics_for_embeddings, get_request
|
||||||
|
|
||||||
from vllm_omni.benchmarks.metrics.metrics import MultiModalsBenchmarkMetrics, calculate_metrics
|
from vllm_omni.benchmarks.metrics.metrics import (
|
||||||
|
MultiModalsBenchmarkMetrics,
|
||||||
|
calculate_metrics,
|
||||||
|
)
|
||||||
|
|
||||||
# ruff: noqa: E402
|
# ruff: noqa: E402
|
||||||
|
|
||||||
@@ -996,6 +1391,7 @@ async def benchmark(
|
|||||||
max_concurrency=max_concurrency,
|
max_concurrency=max_concurrency,
|
||||||
request_rate=request_rate,
|
request_rate=request_rate,
|
||||||
benchmark_duration=benchmark_duration,
|
benchmark_duration=benchmark_duration,
|
||||||
|
print_stage=_PRINT_STAGE,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
metrics = calculate_metrics_for_embeddings(
|
metrics = calculate_metrics_for_embeddings(
|
||||||
@@ -1016,9 +1412,13 @@ async def benchmark(
|
|||||||
"request_goodput": metrics.request_goodput if goodput_config_dict else None,
|
"request_goodput": metrics.request_goodput if goodput_config_dict else None,
|
||||||
"output_throughput": metrics.output_throughput,
|
"output_throughput": metrics.output_throughput,
|
||||||
"total_token_throughput": metrics.total_token_throughput,
|
"total_token_throughput": metrics.total_token_throughput,
|
||||||
"total_audio_duration_s": metrics.total_audio_duration_s,
|
defs.TOTAL_AUDIO_DURATION_S: getattr(metrics, defs.TOTAL_AUDIO_DURATION_S),
|
||||||
"total_audio_frames": metrics.total_audio_frames,
|
defs.TOTAL_AUDIO_FRAMES: getattr(metrics, defs.TOTAL_AUDIO_FRAMES),
|
||||||
"audio_throughput": metrics.audio_throughput,
|
defs.AUDIO_THROUGHPUT: getattr(metrics, defs.AUDIO_THROUGHPUT),
|
||||||
|
defs.TOTAL_IMAGES: getattr(metrics, defs.TOTAL_IMAGES),
|
||||||
|
defs.IMAGE_THROUGHPUT: getattr(metrics, defs.IMAGE_THROUGHPUT),
|
||||||
|
defs.AVERAGE_PIXELS_PER_IMAGE: getattr(metrics, defs.AVERAGE_PIXELS_PER_IMAGE),
|
||||||
|
defs.MEAN_DENOISE_STEP_LATENCY_MS: getattr(metrics, defs.MEAN_DENOISE_STEP_LATENCY_MS),
|
||||||
"input_lens": [output.prompt_len for output in outputs],
|
"input_lens": [output.prompt_len for output in outputs],
|
||||||
"output_lens": actual_output_lens,
|
"output_lens": actual_output_lens,
|
||||||
"ttfts": [output.ttft for output in outputs],
|
"ttfts": [output.ttft for output in outputs],
|
||||||
@@ -1074,21 +1474,34 @@ async def benchmark(
|
|||||||
if rps_change_events:
|
if rps_change_events:
|
||||||
result["rps_change_events"] = rps_change_events
|
result["rps_change_events"] = rps_change_events
|
||||||
|
|
||||||
|
result_percentile_metrics: list[str] = []
|
||||||
|
if "ttft" in selected_percentile_metrics:
|
||||||
|
result_percentile_metrics.append("ttft")
|
||||||
|
if "tpot" in selected_percentile_metrics or "tpop" in selected_percentile_metrics:
|
||||||
|
result_percentile_metrics.append("tpot")
|
||||||
|
if "itl" in selected_percentile_metrics:
|
||||||
|
result_percentile_metrics.append("itl")
|
||||||
|
if "e2el" in selected_percentile_metrics:
|
||||||
|
result_percentile_metrics.append("e2el")
|
||||||
|
for metric in selected_percentile_metrics:
|
||||||
|
if metric.startswith("audio") and metric not in result_percentile_metrics:
|
||||||
|
result_percentile_metrics.append(metric)
|
||||||
|
|
||||||
def process_one_metric(
|
def process_one_metric(
|
||||||
# E.g., "ttft"
|
# E.g., "ttft"
|
||||||
metric_attribute_name: str,
|
metric_attribute_name: str,
|
||||||
):
|
):
|
||||||
# This function prints and adds statistics of the specified
|
# This function prints and adds statistics of the specified
|
||||||
# metric.
|
# metric.
|
||||||
if metric_attribute_name not in selected_percentile_metrics:
|
if metric_attribute_name not in result_percentile_metrics:
|
||||||
return
|
return
|
||||||
# No text tokens generated (e.g. pure TTS speech endpoint): per-token
|
# No text tokens generated (e.g. pure TTS speech endpoint): per-token
|
||||||
# latency metrics (ttft/tpot/itl) are undefined, so skip them.
|
# latency metrics (ttft/tpot/itl) are undefined, so skip them.
|
||||||
is_text_token_metric = not (metric_attribute_name == "e2el" or metric_attribute_name.startswith("audio"))
|
is_text_token_metric = not (metric_attribute_name == "e2el" or metric_attribute_name.startswith("audio"))
|
||||||
if is_text_token_metric and getattr(metrics, "total_output", 0) == 0:
|
if is_text_token_metric and getattr(metrics, "total_output", 0) == 0:
|
||||||
return
|
return
|
||||||
is_audio_rtf = metric_attribute_name == "audio_rtf"
|
is_audio_rtf = metric_attribute_name == defs.AUDIO_RTF
|
||||||
is_audio_duration_or_underrun = metric_attribute_name in ("audio_duration", "audio_underrun")
|
is_audio_duration_or_underrun = metric_attribute_name in (defs.AUDIO_DURATION, defs.AUDIO_UNDERRUN)
|
||||||
|
|
||||||
suffix = "_ms"
|
suffix = "_ms"
|
||||||
if is_audio_duration_or_underrun:
|
if is_audio_duration_or_underrun:
|
||||||
@@ -1107,9 +1520,10 @@ async def benchmark(
|
|||||||
result[f"p{p_word}_{metric_attribute_name}{suffix}"] = value
|
result[f"p{p_word}_{metric_attribute_name}{suffix}"] = value
|
||||||
|
|
||||||
if task_type == TaskType.GENERATION:
|
if task_type == TaskType.GENERATION:
|
||||||
for metric in selected_percentile_metrics:
|
for metric in result_percentile_metrics:
|
||||||
process_one_metric(metric)
|
process_one_metric(metric)
|
||||||
else:
|
else:
|
||||||
|
result_percentile_metrics.append("e2el")
|
||||||
process_one_metric("e2el")
|
process_one_metric("e2el")
|
||||||
|
|
||||||
if profile:
|
if profile:
|
||||||
|
|||||||
@@ -9,6 +9,11 @@ from vllm.benchmarks.serve import main_async
|
|||||||
# This monkey-patches vllm.benchmarks.datasets.get_samples before it's used
|
# This monkey-patches vllm.benchmarks.datasets.get_samples before it's used
|
||||||
# Must be imported before any vllm.benchmarks module usage
|
# Must be imported before any vllm.benchmarks module usage
|
||||||
import vllm_omni.benchmarks.patch.patch # noqa: F401
|
import vllm_omni.benchmarks.patch.patch # noqa: F401
|
||||||
|
from vllm_omni.benchmarks.patch.patch import (
|
||||||
|
maybe_enable_stage_metrics,
|
||||||
|
set_print_stage,
|
||||||
|
should_request_stage_metrics,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def main(args: argparse.Namespace) -> dict[str, Any]:
|
def main(args: argparse.Namespace) -> dict[str, Any]:
|
||||||
@@ -18,4 +23,9 @@ def main(args: argparse.Namespace) -> dict[str, Any]:
|
|||||||
os.environ["SEED_TTS_WER_SAVE_ITEMS"] = "1"
|
os.environ["SEED_TTS_WER_SAVE_ITEMS"] = "1"
|
||||||
if getattr(args, "daily_omni_save_eval_items", False):
|
if getattr(args, "daily_omni_save_eval_items", False):
|
||||||
os.environ["DAILY_OMNI_SAVE_EVAL_ITEMS"] = "1"
|
os.environ["DAILY_OMNI_SAVE_EVAL_ITEMS"] = "1"
|
||||||
|
set_print_stage(getattr(args, "print_stage", False))
|
||||||
|
args.extra_body = maybe_enable_stage_metrics(
|
||||||
|
getattr(args, "extra_body", None),
|
||||||
|
enabled=should_request_stage_metrics(args),
|
||||||
|
)
|
||||||
return asyncio.run(main_async(args))
|
return asyncio.run(main_async(args))
|
||||||
@@ -17,11 +17,10 @@ Adding a new pipeline:
|
|||||||
``vllm_omni/.../pipeline.py``.
|
``vllm_omni/.../pipeline.py``.
|
||||||
2. Add one line to ``_OMNI_PIPELINES`` below.
|
2. Add one line to ``_OMNI_PIPELINES`` below.
|
||||||
|
|
||||||
Single-stage diffusion models continue to use the
|
Plain single-stage diffusion models continue to use the
|
||||||
``_create_default_diffusion_stage_cfg`` fallback in
|
``_create_default_diffusion_stage_cfg`` fallback in ``async_omni_engine.py``.
|
||||||
``async_omni_engine.py`` — they don't need a registry entry. The empty
|
The empty ``_DIFFUSION_PIPELINES`` placeholder previously here (#2915) was
|
||||||
``_DIFFUSION_PIPELINES`` placeholder previously here (#2915) was removed
|
removed once #2987 (which would have populated it) was deferred.
|
||||||
once #2987 (which would have populated it) was deferred.
|
|
||||||
|
|
||||||
``register_pipeline(config)`` in ``stage_config`` is still supported for
|
``register_pipeline(config)`` in ``stage_config`` is still supported for
|
||||||
out-of-tree plugins and tests that create pipelines at runtime; those override
|
out-of-tree plugins and tests that create pipelines at runtime; those override
|
||||||
@@ -70,6 +69,10 @@ _OMNI_PIPELINES: dict[str, tuple[str, str]] = {
|
|||||||
"vllm_omni.model_executor.models.lance.pipeline",
|
"vllm_omni.model_executor.models.lance.pipeline",
|
||||||
"LANCE_PIPELINE",
|
"LANCE_PIPELINE",
|
||||||
),
|
),
|
||||||
|
"dreamzero": (
|
||||||
|
"vllm_omni.model_executor.models.dreamzero.pipeline",
|
||||||
|
"DREAMZERO_PIPELINE",
|
||||||
|
),
|
||||||
"glm_image": (
|
"glm_image": (
|
||||||
"vllm_omni.model_executor.models.glm_image.pipeline",
|
"vllm_omni.model_executor.models.glm_image.pipeline",
|
||||||
"GLM_IMAGE_PIPELINE",
|
"GLM_IMAGE_PIPELINE",
|
||||||
@@ -146,4 +149,12 @@ _OMNI_PIPELINES: dict[str, tuple[str, str]] = {
|
|||||||
"vllm_omni.model_executor.models.higgs_audio_v2.pipeline",
|
"vllm_omni.model_executor.models.higgs_audio_v2.pipeline",
|
||||||
"HIGGS_AUDIO_V2_PIPELINE",
|
"HIGGS_AUDIO_V2_PIPELINE",
|
||||||
),
|
),
|
||||||
|
"higgs_multimodal_qwen3": (
|
||||||
|
"vllm_omni.model_executor.models.higgs_audio_v3.pipeline",
|
||||||
|
"HIGGS_AUDIO_V3_PIPELINE",
|
||||||
|
),
|
||||||
|
"dynin_omni": (
|
||||||
|
"vllm_omni.model_executor.models.dynin_omni.pipeline",
|
||||||
|
"DYNIN_OMNI_PIPELINE",
|
||||||
|
),
|
||||||
}
|
}
|
||||||
@@ -1165,6 +1165,11 @@ class StageConfigFactory:
|
|||||||
|
|
||||||
# --- New path: check pipeline registry by model_type first ---
|
# --- New path: check pipeline registry by model_type first ---
|
||||||
model_type, hf_config = cls._auto_detect_model_type(model, trust_remote_code=trust_remote_code)
|
model_type, hf_config = cls._auto_detect_model_type(model, trust_remote_code=trust_remote_code)
|
||||||
|
if model_type == "vla":
|
||||||
|
from vllm_omni.diffusion.utils.hf_utils import _looks_like_dreamzero
|
||||||
|
|
||||||
|
if _looks_like_dreamzero(model):
|
||||||
|
model_type = "dreamzero"
|
||||||
if model_type and model_type in _PIPELINE_REGISTRY:
|
if model_type and model_type in _PIPELINE_REGISTRY:
|
||||||
return cls._create_from_registry(model_type, cli_overrides, deploy_config_path)
|
return cls._create_from_registry(model_type, cli_overrides, deploy_config_path)
|
||||||
|
|
||||||
|
|||||||
@@ -391,27 +391,33 @@ class OmniTensorPrefixCache:
|
|||||||
self,
|
self,
|
||||||
query_start_loc: torch.Tensor,
|
query_start_loc: torch.Tensor,
|
||||||
input_batch: InputBatch,
|
input_batch: InputBatch,
|
||||||
multimodal_outputs: dict,
|
multimodal_outputs: dict | None,
|
||||||
num_scheduled_tokens: dict[str, int],
|
num_scheduled_tokens: dict[str, int],
|
||||||
):
|
):
|
||||||
"""Get the merged multimodal states if hidden state prefix caching is enabled."""
|
"""Get the merged multimodal states if hidden state prefix caching is enabled."""
|
||||||
combined_multimodal_outputs = {}
|
combined_multimodal_outputs = {}
|
||||||
|
# Talkers that produce multimodal outputs only at decode (e.g. Higgs
|
||||||
|
# Audio v3's audio codes) have ``multimodal_outputs is None`` at the
|
||||||
|
# initial prefill step. Treat it as an empty mapping so the merger
|
||||||
|
# short-circuits cleanly and the cache merge still runs once codes
|
||||||
|
# start arriving.
|
||||||
|
mm_outputs = multimodal_outputs if multimodal_outputs is not None else {}
|
||||||
# First get the prefix cached tensors that are present in the mm data
|
# First get the prefix cached tensors that are present in the mm data
|
||||||
for mm_key in self.mm_cache_keys:
|
for mm_key in self.mm_cache_keys:
|
||||||
if mm_key in multimodal_outputs:
|
if mm_key in mm_outputs:
|
||||||
combined_multimodal_outputs[mm_key] = self._get_merged_tensors(
|
combined_multimodal_outputs[mm_key] = self._get_merged_tensors(
|
||||||
query_start_loc=query_start_loc,
|
query_start_loc=query_start_loc,
|
||||||
input_batch=input_batch,
|
input_batch=input_batch,
|
||||||
cache=self.mm_outputs_cache[mm_key],
|
cache=self.mm_outputs_cache[mm_key],
|
||||||
hidden_states=multimodal_outputs[mm_key],
|
hidden_states=mm_outputs[mm_key],
|
||||||
num_scheduled_tokens=num_scheduled_tokens,
|
num_scheduled_tokens=num_scheduled_tokens,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Then, get everything else (passthrough data); first, convert to CPU
|
# Then, get everything else (passthrough data); first, convert to CPU
|
||||||
# tensors similarly to the non prefix cached path, and then populate
|
# tensors similarly to the non prefix cached path, and then populate
|
||||||
# the subdicts mapping request IDs -> payload objects
|
# the subdicts mapping request IDs -> payload objects
|
||||||
passthrough_keys = set(multimodal_outputs.keys()) - self.mm_cache_keys
|
passthrough_keys = set(mm_outputs.keys()) - self.mm_cache_keys
|
||||||
passthrough_mm_data = {k: v for k, v in multimodal_outputs.items() if k in passthrough_keys}
|
passthrough_mm_data = {k: v for k, v in mm_outputs.items() if k in passthrough_keys}
|
||||||
mm_cpu = build_mm_cpu(multimodal_outputs=passthrough_mm_data)
|
mm_cpu = build_mm_cpu(multimodal_outputs=passthrough_mm_data)
|
||||||
|
|
||||||
for mm_key, mm_val in mm_cpu.items():
|
for mm_key, mm_val in mm_cpu.items():
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ from vllm.v1.core.sched.async_scheduler import AsyncScheduler as AsyncVLLMSchedu
|
|||||||
from vllm.v1.core.sched.output import SchedulerOutput
|
from vllm.v1.core.sched.output import SchedulerOutput
|
||||||
from vllm.v1.core.sched.scheduler import Scheduler as VLLMScheduler
|
from vllm.v1.core.sched.scheduler import Scheduler as VLLMScheduler
|
||||||
from vllm.v1.core.sched.utils import remove_all
|
from vllm.v1.core.sched.utils import remove_all
|
||||||
from vllm.v1.engine import EngineCoreEventType, EngineCoreOutput, EngineCoreOutputs
|
from vllm.v1.engine import EngineCoreEventType, EngineCoreOutput, EngineCoreOutputs, FinishReason
|
||||||
from vllm.v1.metrics.perf import PerfStats
|
from vllm.v1.metrics.perf import PerfStats
|
||||||
from vllm.v1.outputs import ModelRunnerOutput
|
from vllm.v1.outputs import ModelRunnerOutput
|
||||||
from vllm.v1.request import Request, RequestStatus, StreamingUpdate
|
from vllm.v1.request import Request, RequestStatus, StreamingUpdate
|
||||||
@@ -69,6 +69,8 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler):
|
|||||||
# alive. Stopped on the first scheduler step after extraction ack.
|
# alive. Stopped on the first scheduler step after extraction ack.
|
||||||
self.pending_stop_after_extraction: set[str] = set()
|
self.pending_stop_after_extraction: set[str] = set()
|
||||||
|
|
||||||
|
self.finished_req_ids_dict = defaultdict(set)
|
||||||
|
|
||||||
# [Omni] Pre-parse KV transfer criteria
|
# [Omni] Pre-parse KV transfer criteria
|
||||||
self.kv_transfer_criteria = self._get_kv_transfer_criteria()
|
self.kv_transfer_criteria = self._get_kv_transfer_criteria()
|
||||||
|
|
||||||
@@ -434,7 +436,7 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler):
|
|||||||
# Downstream async-chunk stages receive real payloads from the
|
# Downstream async-chunk stages receive real payloads from the
|
||||||
# connector. This update only resumes polling for the next segment.
|
# connector. This update only resumes polling for the next segment.
|
||||||
self.chunk_transfer_adapter.segment_finished_requests.discard(request.request_id)
|
self.chunk_transfer_adapter.segment_finished_requests.discard(request.request_id)
|
||||||
request.discard_latest_async_tokens = True
|
request.async_tokens_to_discard = 1
|
||||||
request.num_output_placeholders = 0
|
request.num_output_placeholders = 0
|
||||||
request.spec_token_ids = []
|
request.spec_token_ids = []
|
||||||
request._output_token_ids.clear()
|
request._output_token_ids.clear()
|
||||||
@@ -556,16 +558,24 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler):
|
|||||||
# outputs in this step.
|
# outputs in this step.
|
||||||
engine_core_outputs = {client_index: EngineCoreOutputs(outputs=outs) for client_index, outs in outputs.items()}
|
engine_core_outputs = {client_index: EngineCoreOutputs(outputs=outs) for client_index, outs in outputs.items()}
|
||||||
|
|
||||||
|
# FIXME: finished_req_ids_dict is unconditionally initialized as
|
||||||
|
# defaultdict(set) in __init__ (not gated by include_finished_set).
|
||||||
|
# This branch is therefore always eligible once any client_index is
|
||||||
|
# populated; revisit when wiring streaming-only / upstream semantics.
|
||||||
finished_req_ids = self.finished_req_ids_dict
|
finished_req_ids = self.finished_req_ids_dict
|
||||||
if finished_req_ids:
|
if finished_req_ids:
|
||||||
# Include ids of requests that finished since last outputs
|
# Include ids of requests that finished since last outputs
|
||||||
# were sent.
|
# were sent.
|
||||||
for client_index, finished_set in finished_req_ids.items():
|
for client_index, finished_set in finished_req_ids.items():
|
||||||
# Set finished request set in EngineCoreOutputs for this client.
|
eco = engine_core_outputs.get(client_index)
|
||||||
if (eco := engine_core_outputs.get(client_index)) is not None:
|
if eco is None:
|
||||||
eco.finished_requests = finished_set
|
eco = EngineCoreOutputs()
|
||||||
else:
|
engine_core_outputs[client_index] = eco
|
||||||
engine_core_outputs[client_index] = EngineCoreOutputs(finished_requests=finished_set)
|
emitted = {o.request_id for o in eco.outputs}
|
||||||
|
for req_id in finished_set:
|
||||||
|
if req_id not in emitted:
|
||||||
|
eco.outputs.append(EngineCoreOutput(req_id, [], finish_reason=FinishReason.ABORT))
|
||||||
|
eco.finished_requests = finished_set
|
||||||
finished_req_ids.clear()
|
finished_req_ids.clear()
|
||||||
|
|
||||||
if (stats := self.make_stats(spec_decoding_stats, kv_connector_stats, cudagraph_stats, perf_stats)) is not None:
|
if (stats := self.make_stats(spec_decoding_stats, kv_connector_stats, cudagraph_stats, perf_stats)) is not None:
|
||||||
@@ -632,13 +642,12 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler):
|
|||||||
"""
|
"""
|
||||||
req_id = session.request_id
|
req_id = session.request_id
|
||||||
self._new_prompt_len_snapshot[req_id] = len(update.prompt_token_ids)
|
self._new_prompt_len_snapshot[req_id] = len(update.prompt_token_ids)
|
||||||
session.discard_latest_async_tokens = False
|
|
||||||
outstanding_async_tokens = getattr(session, "num_output_placeholders", 0)
|
outstanding_async_tokens = getattr(session, "num_output_placeholders", 0)
|
||||||
if outstanding_async_tokens > 0:
|
if outstanding_async_tokens > 0:
|
||||||
# Async scheduling may already have sampled the previous
|
# Async scheduling may already have sampled the previous
|
||||||
# segment's next token. Drop that late token instead of
|
# segment's next token. Drop that late token instead of
|
||||||
# appending it to the new streaming segment.
|
# appending it to the new streaming segment.
|
||||||
session.discard_latest_async_tokens = True
|
session.async_tokens_to_discard = 1
|
||||||
session.num_computed_tokens -= session.num_output_placeholders
|
session.num_computed_tokens -= session.num_output_placeholders
|
||||||
session.num_output_placeholders = 0
|
session.num_output_placeholders = 0
|
||||||
session.spec_token_ids = []
|
session.spec_token_ids = []
|
||||||
|
|||||||
@@ -63,6 +63,7 @@ class OmniPayloadMeta(TypedDict, total=False):
|
|||||||
stream_finished: torch.Tensor
|
stream_finished: torch.Tensor
|
||||||
req_id: list[str]
|
req_id: list[str]
|
||||||
left_context_size: int
|
left_context_size: int
|
||||||
|
right_holdback_size: int
|
||||||
override_keys: list[tuple[str, str]]
|
override_keys: list[tuple[str, str]]
|
||||||
num_processed_tokens: int
|
num_processed_tokens: int
|
||||||
next_stage_prompt_len: int
|
next_stage_prompt_len: int
|
||||||
@@ -150,6 +151,7 @@ class MetaStruct(_StructBase):
|
|||||||
stream_finished: torch.Tensor | None = None
|
stream_finished: torch.Tensor | None = None
|
||||||
req_id: list[str] | None = None
|
req_id: list[str] | None = None
|
||||||
left_context_size: int | None = None
|
left_context_size: int | None = None
|
||||||
|
right_holdback_size: int | None = None
|
||||||
override_keys: list[tuple[str, str]] | None = None
|
override_keys: list[tuple[str, str]] | None = None
|
||||||
num_processed_tokens: int | None = None
|
num_processed_tokens: int | None = None
|
||||||
next_stage_prompt_len: int | None = None
|
next_stage_prompt_len: int | None = None
|
||||||
|
|||||||
@@ -0,0 +1,25 @@
|
|||||||
|
# DreamZero-DROID deploy: single diffusion stage.
|
||||||
|
#
|
||||||
|
# Topology is declared in vllm_omni/model_executor/models/dreamzero/pipeline.py.
|
||||||
|
# This default uses one GPU with TP=1 and CFG parallel disabled.
|
||||||
|
|
||||||
|
pipeline: dreamzero
|
||||||
|
async_chunk: false
|
||||||
|
distributed_executor_backend: mp
|
||||||
|
dtype: bfloat16
|
||||||
|
|
||||||
|
stages:
|
||||||
|
- stage_id: 0
|
||||||
|
devices: "0"
|
||||||
|
max_num_seqs: 1
|
||||||
|
enforce_eager: true
|
||||||
|
model_class_name: DreamZeroPipeline
|
||||||
|
model_config:
|
||||||
|
default_robot_embodiment: roboarena
|
||||||
|
policy_server_config:
|
||||||
|
image_resolution: [180, 320]
|
||||||
|
n_external_cameras: 2
|
||||||
|
needs_wrist_camera: true
|
||||||
|
needs_stereo_camera: false
|
||||||
|
needs_session_id: true
|
||||||
|
action_space: joint_position
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
# DreamZero-DROID deploy: TP=1, CFG parallel size=2.
|
||||||
|
|
||||||
|
pipeline: dreamzero
|
||||||
|
async_chunk: false
|
||||||
|
distributed_executor_backend: mp
|
||||||
|
dtype: bfloat16
|
||||||
|
|
||||||
|
stages:
|
||||||
|
- stage_id: 0
|
||||||
|
devices: "0,1"
|
||||||
|
max_num_seqs: 1
|
||||||
|
enforce_eager: true
|
||||||
|
model_class_name: DreamZeroPipeline
|
||||||
|
parallel_config:
|
||||||
|
tensor_parallel_size: 1
|
||||||
|
cfg_parallel_size: 2
|
||||||
|
model_config:
|
||||||
|
default_robot_embodiment: roboarena
|
||||||
|
policy_server_config:
|
||||||
|
image_resolution: [180, 320]
|
||||||
|
n_external_cameras: 2
|
||||||
|
needs_wrist_camera: true
|
||||||
|
needs_stereo_camera: false
|
||||||
|
needs_session_id: true
|
||||||
|
action_space: joint_position
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
# Dynin-Omni 3-stage deploy: token2text → token2image → token2audio.
|
||||||
|
#
|
||||||
|
# Topology (stages, processor functions, model_arch) is frozen in
|
||||||
|
# vllm_omni/model_executor/models/dynin_omni/pipeline.py. This file carries
|
||||||
|
# only the deploy-time knobs (devices, GPU memory, batched tokens).
|
||||||
|
#
|
||||||
|
# async_chunk=false: dynin hands the full token payload across stages through
|
||||||
|
# the worker-connector data plane (``*_full_payload`` producers), not chunked
|
||||||
|
# streaming. Pipeline-wide ``trust_remote_code`` is required by the custom
|
||||||
|
# Dynin HF modeling code.
|
||||||
|
async_chunk: false
|
||||||
|
trust_remote_code: true
|
||||||
|
enable_prefix_caching: false
|
||||||
|
|
||||||
|
stages:
|
||||||
|
- stage_id: 0
|
||||||
|
devices: "0"
|
||||||
|
max_num_seqs: 1
|
||||||
|
gpu_memory_utilization: 0.5
|
||||||
|
max_num_batched_tokens: 32768
|
||||||
|
enforce_eager: true
|
||||||
|
async_scheduling: false
|
||||||
|
|
||||||
|
- stage_id: 1
|
||||||
|
devices: "0"
|
||||||
|
max_num_seqs: 1
|
||||||
|
gpu_memory_utilization: 0.1
|
||||||
|
max_num_batched_tokens: 32768
|
||||||
|
enforce_eager: true
|
||||||
|
async_scheduling: false
|
||||||
|
|
||||||
|
- stage_id: 2
|
||||||
|
devices: "0"
|
||||||
|
max_num_seqs: 1
|
||||||
|
gpu_memory_utilization: 0.1
|
||||||
|
max_num_batched_tokens: 32768
|
||||||
|
enforce_eager: true
|
||||||
|
async_scheduling: false
|
||||||
@@ -1,81 +1,35 @@
|
|||||||
# stage config for running dynin_omni with a 3-stage architecture.
|
# Dynin-Omni deploy for e2e smoke tests (3-stage, one GPU).
|
||||||
# this config is intended for e2e smoke tests.
|
#
|
||||||
|
# Topology is frozen in models/dynin_omni/pipeline.py; ``pipeline: dynin_omni``
|
||||||
|
# pins it explicitly so this CI YAML resolves regardless of model_type
|
||||||
|
# auto-detection. Differs from the default deploy only in reduced GPU memory
|
||||||
|
# and a smaller max_num_batched_tokens so the pipeline fits L4-class CI GPUs.
|
||||||
|
pipeline: dynin_omni
|
||||||
|
async_chunk: false
|
||||||
|
trust_remote_code: true
|
||||||
|
enable_prefix_caching: false
|
||||||
|
|
||||||
stage_args:
|
stages:
|
||||||
- stage_id: 0
|
- stage_id: 0
|
||||||
stage_type: llm
|
devices: "0"
|
||||||
runtime:
|
max_num_seqs: 1
|
||||||
process: true
|
gpu_memory_utilization: 0.5
|
||||||
devices: "0"
|
max_num_batched_tokens: 4096
|
||||||
max_batch_size: 1
|
enforce_eager: true
|
||||||
engine_args:
|
async_scheduling: false
|
||||||
model_stage: token2text
|
|
||||||
model_arch: DyninOmniForConditionalGeneration
|
|
||||||
worker_type: generation
|
|
||||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
|
||||||
engine_output_type: latent
|
|
||||||
custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.dynin_omni.token2text_to_token2image_full_payload
|
|
||||||
trust_remote_code: true
|
|
||||||
gpu_memory_utilization: 0.5
|
|
||||||
enforce_eager: true
|
|
||||||
enable_prefix_caching: false
|
|
||||||
async_scheduling: false
|
|
||||||
max_num_batched_tokens: 4096
|
|
||||||
is_comprehension: true
|
|
||||||
final_output: true
|
|
||||||
final_output_type: text
|
|
||||||
|
|
||||||
- stage_id: 1
|
- stage_id: 1
|
||||||
stage_type: llm
|
devices: "0"
|
||||||
runtime:
|
max_num_seqs: 1
|
||||||
process: true
|
gpu_memory_utilization: 0.2
|
||||||
devices: "0"
|
max_num_batched_tokens: 4096
|
||||||
max_batch_size: 1
|
enforce_eager: true
|
||||||
engine_args:
|
async_scheduling: false
|
||||||
model_stage: token2image
|
|
||||||
model_arch: DyninOmniForConditionalGeneration
|
|
||||||
worker_type: generation
|
|
||||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
|
||||||
engine_output_type: latent
|
|
||||||
custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.dynin_omni.token2image_to_token2audio_full_payload
|
|
||||||
trust_remote_code: true
|
|
||||||
gpu_memory_utilization: 0.2
|
|
||||||
enforce_eager: true
|
|
||||||
enable_prefix_caching: false
|
|
||||||
async_scheduling: false
|
|
||||||
max_num_batched_tokens: 4096
|
|
||||||
engine_input_source: [0]
|
|
||||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.dynin_omni.token2text_to_token2image
|
|
||||||
final_output: true
|
|
||||||
final_output_type: image
|
|
||||||
|
|
||||||
- stage_id: 2
|
- stage_id: 2
|
||||||
stage_type: llm
|
devices: "0"
|
||||||
runtime:
|
max_num_seqs: 1
|
||||||
process: true
|
gpu_memory_utilization: 0.2
|
||||||
devices: "0"
|
max_num_batched_tokens: 4096
|
||||||
max_batch_size: 1
|
enforce_eager: true
|
||||||
engine_args:
|
async_scheduling: false
|
||||||
model_stage: token2audio
|
|
||||||
model_arch: DyninOmniForConditionalGeneration
|
|
||||||
worker_type: generation
|
|
||||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
|
||||||
engine_output_type: latent
|
|
||||||
trust_remote_code: true
|
|
||||||
gpu_memory_utilization: 0.2
|
|
||||||
enforce_eager: true
|
|
||||||
enable_prefix_caching: false
|
|
||||||
async_scheduling: false
|
|
||||||
max_num_batched_tokens: 4096
|
|
||||||
engine_input_source: [1]
|
|
||||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.dynin_omni.token2image_to_token2audio
|
|
||||||
final_output: true
|
|
||||||
final_output_type: audio
|
|
||||||
|
|
||||||
runtime:
|
|
||||||
enabled: true
|
|
||||||
edges:
|
|
||||||
- from: 0
|
|
||||||
to: 1
|
|
||||||
- from: 1
|
|
||||||
to: 2
|
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
# Dynin-Omni 3-stage deploy with cross-node connectors (Mooncake demo).
|
||||||
|
#
|
||||||
|
# Topology is frozen in models/dynin_omni/pipeline.py (``pipeline: dynin_omni``).
|
||||||
|
# This variant wires a MooncakeConnector for each stage hand-off in place of
|
||||||
|
# the default SharedMemoryConnector. Update the ``extra`` host/master/metadata
|
||||||
|
# fields to match your Mooncake deployment before use.
|
||||||
|
pipeline: dynin_omni
|
||||||
|
async_chunk: false
|
||||||
|
trust_remote_code: false
|
||||||
|
enable_prefix_caching: false
|
||||||
|
|
||||||
|
connectors:
|
||||||
|
# Mooncake connector for cross-node/intra-node communication.
|
||||||
|
mooncake_connector:
|
||||||
|
name: MooncakeConnector
|
||||||
|
extra:
|
||||||
|
host: "127.0.0.1"
|
||||||
|
metadata_server: "http://10.90.67.86:8080/metadata"
|
||||||
|
master: "10.90.67.86:50051"
|
||||||
|
segment: 512000000 # 512MB
|
||||||
|
localbuf: 64000000 # 64MB
|
||||||
|
proto: "tcp"
|
||||||
|
|
||||||
|
# Yuanrong connector for cross-node/intra-node communication.
|
||||||
|
yuanrong_connector:
|
||||||
|
name: YuanrongConnector
|
||||||
|
extra:
|
||||||
|
host: "127.0.0.1"
|
||||||
|
port: "35000"
|
||||||
|
|
||||||
|
# Alternative SHM connector with an explicit threshold.
|
||||||
|
shared_memory_connector:
|
||||||
|
name: SharedMemoryConnector
|
||||||
|
extra:
|
||||||
|
shm_threshold_bytes: 65536 # 64KB threshold
|
||||||
|
|
||||||
|
stages:
|
||||||
|
- stage_id: 0
|
||||||
|
devices: "0"
|
||||||
|
max_num_seqs: 1
|
||||||
|
max_num_batched_tokens: 32768
|
||||||
|
enforce_eager: true
|
||||||
|
async_scheduling: false
|
||||||
|
output_connectors:
|
||||||
|
to_stage_1: mooncake_connector
|
||||||
|
|
||||||
|
- stage_id: 1
|
||||||
|
devices: "0"
|
||||||
|
max_num_seqs: 1
|
||||||
|
max_num_batched_tokens: 32768
|
||||||
|
enforce_eager: true
|
||||||
|
async_scheduling: false
|
||||||
|
input_connectors:
|
||||||
|
from_stage_0: mooncake_connector
|
||||||
|
output_connectors:
|
||||||
|
to_stage_2: mooncake_connector
|
||||||
|
|
||||||
|
- stage_id: 2
|
||||||
|
devices: "0"
|
||||||
|
max_num_seqs: 1
|
||||||
|
max_num_batched_tokens: 32768
|
||||||
|
enforce_eager: true
|
||||||
|
async_scheduling: false
|
||||||
|
input_connectors:
|
||||||
|
from_stage_1: mooncake_connector
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
# Higgs-audio v3 deploy: Stage 0 (talker) -> Stage 1 (code2wav) via shared memory.
|
||||||
|
#
|
||||||
|
# Plain TTS only (no voice cloning). async_chunk streams codec chunks from
|
||||||
|
# Stage 0 to Stage 1 as they are produced (TTFA approaches one AR loop's
|
||||||
|
# worth of work). Stage 1 honors meta.left_context_size + meta.right_holdback_size
|
||||||
|
# so the per-chunk codec windows stitch into a coherent PCM stream.
|
||||||
|
async_chunk: true
|
||||||
|
|
||||||
|
connectors:
|
||||||
|
connector_of_shared_memory:
|
||||||
|
name: SharedMemoryConnector
|
||||||
|
extra:
|
||||||
|
shm_threshold_bytes: 65536
|
||||||
|
# Stream PCM bytes back to the client as Stage 1 emits each chunk.
|
||||||
|
codec_streaming: true
|
||||||
|
# Streaming chunk shape for talker2code2wav_async_chunk:
|
||||||
|
# codec_chunk_frames steady-state new frames per emit
|
||||||
|
# codec_left_context_frames sliding past context to avoid leading-edge artifacts
|
||||||
|
# codec_right_holdback_frames trailing frames held back so the codec
|
||||||
|
# has future context for emit-region tail samples
|
||||||
|
# initial_codec_chunk_frames smaller first chunk to minimize TTFA
|
||||||
|
codec_chunk_frames: 25
|
||||||
|
codec_left_context_frames: 25
|
||||||
|
codec_right_holdback_frames: 4
|
||||||
|
initial_codec_chunk_frames: 1
|
||||||
|
connector_get_sleep_s: 0.01
|
||||||
|
connector_get_max_wait_first_chunk: 5000
|
||||||
|
connector_get_max_wait: 500
|
||||||
|
|
||||||
|
stages:
|
||||||
|
- stage_id: 0
|
||||||
|
max_num_seqs: 16
|
||||||
|
gpu_memory_utilization: 0.6
|
||||||
|
enforce_eager: true
|
||||||
|
trust_remote_code: true
|
||||||
|
# Stage-0 prefix caching is on by default. The talker's HS opt-out and
|
||||||
|
# deferred ``codes.audio`` mm-output write (see higgs_audio_v3_talker.py)
|
||||||
|
# keep per-step bookkeeping negligible; any shared text prefix across
|
||||||
|
# concurrent requests is a net throughput win. Stage 1 (codec decoder)
|
||||||
|
# has no KV cache and stays disabled.
|
||||||
|
enable_prefix_caching: true
|
||||||
|
async_scheduling: false
|
||||||
|
max_num_batched_tokens: 4096
|
||||||
|
max_model_len: 8192
|
||||||
|
devices: "0"
|
||||||
|
output_connectors:
|
||||||
|
to_stage_1: connector_of_shared_memory
|
||||||
|
default_sampling_params:
|
||||||
|
temperature: 1.0
|
||||||
|
top_p: 0.95
|
||||||
|
top_k: 50
|
||||||
|
max_tokens: 2048
|
||||||
|
seed: 42
|
||||||
|
repetition_penalty: 1.0
|
||||||
|
|
||||||
|
- stage_id: 1
|
||||||
|
max_num_seqs: 16
|
||||||
|
gpu_memory_utilization: 0.25
|
||||||
|
enforce_eager: true
|
||||||
|
trust_remote_code: true
|
||||||
|
enable_prefix_caching: false
|
||||||
|
async_scheduling: false
|
||||||
|
max_num_batched_tokens: 65536
|
||||||
|
max_model_len: 65536
|
||||||
|
devices: "0"
|
||||||
|
input_connectors:
|
||||||
|
from_stage_0: connector_of_shared_memory
|
||||||
|
default_sampling_params:
|
||||||
|
temperature: 0.0
|
||||||
|
top_p: 1.0
|
||||||
|
top_k: -1
|
||||||
|
max_tokens: 65536
|
||||||
|
seed: 42
|
||||||
|
repetition_penalty: 1.0
|
||||||
@@ -18,6 +18,7 @@ connectors:
|
|||||||
connector_of_shared_memory:
|
connector_of_shared_memory:
|
||||||
name: SharedMemoryConnector
|
name: SharedMemoryConnector
|
||||||
extra:
|
extra:
|
||||||
|
initial_codec_chunk_frames: 4
|
||||||
codec_chunk_frames: 25
|
codec_chunk_frames: 25
|
||||||
codec_left_context_frames: 25
|
codec_left_context_frames: 25
|
||||||
|
|
||||||
@@ -30,12 +31,8 @@ stages:
|
|||||||
enable_prefix_caching: false
|
enable_prefix_caching: false
|
||||||
devices: "0"
|
devices: "0"
|
||||||
default_sampling_params:
|
default_sampling_params:
|
||||||
temperature: 0.4
|
temperature: 0.0
|
||||||
top_p: 0.9
|
|
||||||
top_k: 1
|
|
||||||
max_tokens: 2048
|
max_tokens: 2048
|
||||||
seed: 42
|
|
||||||
repetition_penalty: 1.05
|
|
||||||
|
|
||||||
- stage_id: 1
|
- stage_id: 1
|
||||||
max_num_batched_tokens: 32768
|
max_num_batched_tokens: 32768
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ connectors:
|
|||||||
connector_of_shared_memory:
|
connector_of_shared_memory:
|
||||||
name: SharedMemoryConnector
|
name: SharedMemoryConnector
|
||||||
extra:
|
extra:
|
||||||
|
initial_codec_chunk_frames: 4
|
||||||
codec_chunk_frames: 25
|
codec_chunk_frames: 25
|
||||||
codec_left_context_frames: 25
|
codec_left_context_frames: 25
|
||||||
|
|
||||||
|
|||||||
@@ -1,19 +1,13 @@
|
|||||||
# VoxCPM2 deploy: single-stage AR pipeline with per-request state batching.
|
# VoxCPM2 deploy: single-stage AR pipeline with per-request state batching.
|
||||||
# Verified on 1x H20 141GB.
|
# Verified on 1x H20 141GB.
|
||||||
#
|
#
|
||||||
# Fields omitted from a stage fall back to StageDeployConfig defaults (see
|
# VoxCPM2-local runtime knobs are passed through hf_overrides.
|
||||||
# vllm_omni/config/stage_config.py). Notable choices:
|
|
||||||
# * enforce_eager=true because the talker's KV-cache save/restore loop is
|
|
||||||
# not cudagraph-compatible (captured separately via voxcpm2_talker's own
|
|
||||||
# _CapturedGraph path).
|
|
||||||
# * max_num_seqs=4 matches the legacy max_batch_size=4 contract: concurrent
|
|
||||||
# requests are supported via per-request StaticKVCache save/restore.
|
|
||||||
async_chunk: false
|
async_chunk: false
|
||||||
dtype: bfloat16
|
dtype: bfloat16
|
||||||
|
|
||||||
stages:
|
stages:
|
||||||
- stage_id: 0
|
- stage_id: 0
|
||||||
max_num_seqs: 4
|
max_num_seqs: 32
|
||||||
gpu_memory_utilization: 0.9
|
gpu_memory_utilization: 0.9
|
||||||
enforce_eager: true
|
enforce_eager: true
|
||||||
trust_remote_code: true
|
trust_remote_code: true
|
||||||
@@ -22,6 +16,18 @@ stages:
|
|||||||
max_num_batched_tokens: 4096
|
max_num_batched_tokens: 4096
|
||||||
max_model_len: 4096
|
max_model_len: 4096
|
||||||
devices: "0"
|
devices: "0"
|
||||||
|
engine_extras:
|
||||||
|
hf_overrides:
|
||||||
|
voxcpm2_runtime_config:
|
||||||
|
audio_emit_every: 1
|
||||||
|
vae_decode_every: 3
|
||||||
|
enable_batched_cfm: true
|
||||||
|
enable_batched_vae_decode: true
|
||||||
|
coalesce_audio_d2h: true
|
||||||
|
enable_loc_dit_fused_qkv: true
|
||||||
|
enable_loc_dit_fused_mlp: true
|
||||||
|
enable_loc_dit_zero_dt_cache: true
|
||||||
|
enable_loc_dit_skip_qkv_contig: true
|
||||||
default_sampling_params:
|
default_sampling_params:
|
||||||
temperature: 0.0
|
temperature: 0.0
|
||||||
top_p: 1.0
|
top_p: 1.0
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ from vllm_omni.diffusion.attention.backends.abstract import (
|
|||||||
AttentionImpl,
|
AttentionImpl,
|
||||||
AttentionMetadata,
|
AttentionMetadata,
|
||||||
)
|
)
|
||||||
|
from vllm_omni.diffusion.attention.backends.sdpa import _maybe_reshape_attn_mask
|
||||||
from vllm_omni.diffusion.attention.backends.utils.piecewise_attn import (
|
from vllm_omni.diffusion.attention.backends.utils.piecewise_attn import (
|
||||||
piecewise_attn,
|
piecewise_attn,
|
||||||
)
|
)
|
||||||
@@ -309,6 +310,13 @@ class FlashAttentionImpl(AttentionImpl):
|
|||||||
"Otherwise, use SDPA backend by setting DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA"
|
"Otherwise, use SDPA backend by setting DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA"
|
||||||
)
|
)
|
||||||
attention_mask = attn_metadata.attn_mask if attn_metadata else None
|
attention_mask = attn_metadata.attn_mask if attn_metadata else None
|
||||||
|
|
||||||
|
# NPU aclnnFlashAttentionScore requires mask shape to be one of:
|
||||||
|
# [B, N, Sq, Skv], [B, 1, Sq, Skv], [1, 1, Sq, Skv], or [Sq, Skv]
|
||||||
|
# But the incoming mask is 2D [B, S] — reshape to [B, 1, 1, S]
|
||||||
|
# So reuse SDPA's mask reshape logic: [B, S] -> [B, 1, Sq, Skv]
|
||||||
|
attention_mask = _maybe_reshape_attn_mask(query, key, attention_mask, mask_mode="full_qk")
|
||||||
|
|
||||||
layout = self.qkv_layout or "BNSD"
|
layout = self.qkv_layout or "BNSD"
|
||||||
return attention_forward(
|
return attention_forward(
|
||||||
query,
|
query,
|
||||||
|
|||||||
Loaded 100 of 212 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user