Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
69fd466fba | ||
|
|
cb08af976b | ||
|
|
617c6ca55f |
No files matched your search
@@ -389,29 +389,6 @@ steps:
|
|||||||
volumes:
|
volumes:
|
||||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||||
|
|
||||||
- label: "TTS · VoxCPM2 Online Test"
|
|
||||||
timeout_in_minutes: 30
|
|
||||||
commands:
|
|
||||||
- |
|
|
||||||
timeout 30m bash -c "
|
|
||||||
export VLLM_LOGGING_LEVEL=DEBUG
|
|
||||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
||||||
pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
|
||||||
"
|
|
||||||
agents:
|
|
||||||
queue: "gpu_1_queue"
|
|
||||||
plugins:
|
|
||||||
- docker#v5.2.0:
|
|
||||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
|
||||||
always-pull: true
|
|
||||||
propagate-environment: true
|
|
||||||
shm-size: "8gb"
|
|
||||||
environment:
|
|
||||||
- "HF_HOME=/fsx/hf_cache"
|
|
||||||
- "HF_TOKEN"
|
|
||||||
volumes:
|
|
||||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
|
||||||
|
|
||||||
- label: "Omni · Higgs-Audio V2 Test"
|
- label: "Omni · Higgs-Audio V2 Test"
|
||||||
timeout_in_minutes: 30
|
timeout_in_minutes: 30
|
||||||
commands:
|
commands:
|
||||||
|
|||||||
+4
-188
@@ -394,11 +394,11 @@ steps:
|
|||||||
depends_on: upload-nightly-pipeline
|
depends_on: upload-nightly-pipeline
|
||||||
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
|
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
|
||||||
steps:
|
steps:
|
||||||
- label: ":full_moon: TTS · Function Test with L4"
|
- label: ":full_moon: TTS · Function Test"
|
||||||
timeout_in_minutes: 90
|
timeout_in_minutes: 90
|
||||||
commands:
|
commands:
|
||||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||||
- pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_voxcpm2_tts_expansion.py
|
- pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy
|
||||||
agents:
|
agents:
|
||||||
queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
|
queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
|
||||||
plugins:
|
plugins:
|
||||||
@@ -413,69 +413,6 @@ steps:
|
|||||||
volumes:
|
volumes:
|
||||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||||
|
|
||||||
# - label: ":full_moon: TTS · Function Test with H100"
|
|
||||||
# timeout_in_minutes: 90
|
|
||||||
# commands:
|
|
||||||
# - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
|
||||||
# - pytest -s -v tests/e2e/ -m "full_model and H100 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy
|
|
||||||
# agents:
|
|
||||||
# queue: "mithril-h100-pool"
|
|
||||||
# plugins:
|
|
||||||
# - kubernetes:
|
|
||||||
# podSpec:
|
|
||||||
# containers:
|
|
||||||
# - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
|
||||||
# resources:
|
|
||||||
# limits:
|
|
||||||
# nvidia.com/gpu: 1
|
|
||||||
# volumeMounts:
|
|
||||||
# - name: devshm
|
|
||||||
# mountPath: /dev/shm
|
|
||||||
# - name: hf-cache
|
|
||||||
# mountPath: /root/.cache/huggingface
|
|
||||||
# env:
|
|
||||||
# - name: HF_HOME
|
|
||||||
# value: /root/.cache/huggingface
|
|
||||||
# - name: HF_TOKEN
|
|
||||||
# valueFrom:
|
|
||||||
# secretKeyRef:
|
|
||||||
# name: hf-token-secret
|
|
||||||
# key: token
|
|
||||||
# nodeSelector:
|
|
||||||
# node.kubernetes.io/instance-type: gpu-h100-sxm
|
|
||||||
# volumes:
|
|
||||||
# - name: devshm
|
|
||||||
# emptyDir:
|
|
||||||
# medium: Memory
|
|
||||||
# - name: hf-cache
|
|
||||||
# hostPath:
|
|
||||||
# path: /mnt/hf-cache
|
|
||||||
# type: DirectoryOrCreate
|
|
||||||
|
|
||||||
- label: ":full_moon: TTS · Accuracy Test · VoxCPM2"
|
|
||||||
timeout_in_minutes: 30
|
|
||||||
commands:
|
|
||||||
- |
|
|
||||||
timeout 30m bash -c "
|
|
||||||
export VLLM_LOGGING_LEVEL=DEBUG
|
|
||||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
||||||
export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
|
||||||
pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts_expansion.py -m 'full_model and cuda and tts' --run-level 'full_model'
|
|
||||||
"
|
|
||||||
agents:
|
|
||||||
queue: "gpu_1_queue"
|
|
||||||
plugins:
|
|
||||||
- docker#v5.2.0:
|
|
||||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
|
||||||
always-pull: true
|
|
||||||
propagate-environment: true
|
|
||||||
shm-size: "8gb"
|
|
||||||
environment:
|
|
||||||
- "HF_HOME=/fsx/hf_cache"
|
|
||||||
- "HF_TOKEN"
|
|
||||||
volumes:
|
|
||||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
|
||||||
|
|
||||||
- label: ":full_moon: TTS · Perf Test"
|
- label: ":full_moon: TTS · Perf Test"
|
||||||
key: nightly-tts-performance
|
key: nightly-tts-performance
|
||||||
timeout_in_minutes: 180
|
timeout_in_minutes: 180
|
||||||
@@ -522,8 +459,8 @@ steps:
|
|||||||
path: /mnt/hf-cache
|
path: /mnt/hf-cache
|
||||||
type: DirectoryOrCreate
|
type: DirectoryOrCreate
|
||||||
|
|
||||||
- label: ":full_moon: TTS · VoxCPM2 · Perf Test"
|
- label: ":full_moon: VoxCPM2 · Perf Test"
|
||||||
key: nightly-tts-performance-voxcpm2
|
key: nightly-voxcpm2-performance
|
||||||
timeout_in_minutes: 180
|
timeout_in_minutes: 180
|
||||||
commands:
|
commands:
|
||||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||||
@@ -758,87 +695,6 @@ steps:
|
|||||||
path: /mnt/hf-cache
|
path: /mnt/hf-cache
|
||||||
type: DirectoryOrCreate
|
type: DirectoryOrCreate
|
||||||
|
|
||||||
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU DreamZero"
|
|
||||||
timeout_in_minutes: 120
|
|
||||||
commands:
|
|
||||||
- >-
|
|
||||||
pytest -sv
|
|
||||||
tests/e2e/online_serving/test_dreamzero_expansion.py
|
|
||||||
-m "full_model and diffusion and H100 and distributed_cuda"
|
|
||||||
--run-level "full_model"
|
|
||||||
agents:
|
|
||||||
queue: "mithril-h100-pool"
|
|
||||||
plugins:
|
|
||||||
- kubernetes:
|
|
||||||
podSpec:
|
|
||||||
containers:
|
|
||||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
|
||||||
resources:
|
|
||||||
limits:
|
|
||||||
nvidia.com/gpu: 2
|
|
||||||
volumeMounts:
|
|
||||||
- name: devshm
|
|
||||||
mountPath: /dev/shm
|
|
||||||
- name: hf-cache
|
|
||||||
mountPath: /root/.cache/huggingface
|
|
||||||
env:
|
|
||||||
- name: HF_HOME
|
|
||||||
value: /root/.cache/huggingface
|
|
||||||
- name: HF_TOKEN
|
|
||||||
valueFrom:
|
|
||||||
secretKeyRef:
|
|
||||||
name: hf-token-secret
|
|
||||||
key: token
|
|
||||||
nodeSelector:
|
|
||||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
|
||||||
volumes:
|
|
||||||
- name: devshm
|
|
||||||
emptyDir:
|
|
||||||
medium: Memory
|
|
||||||
- name: hf-cache
|
|
||||||
hostPath:
|
|
||||||
path: /mnt/hf-cache
|
|
||||||
type: DirectoryOrCreate
|
|
||||||
|
|
||||||
|
|
||||||
- label: ":full_moon: Diffusion X2I(&A&T) · GLM-Image Function Test with H100"
|
|
||||||
timeout_in_minutes: 120
|
|
||||||
commands:
|
|
||||||
- pytest -sv tests/e2e/online_serving/test_glm_image_expansion.py -m "full_model and diffusion and H100" --run-level "full_model"
|
|
||||||
agents:
|
|
||||||
queue: "mithril-h100-pool"
|
|
||||||
plugins:
|
|
||||||
- kubernetes:
|
|
||||||
podSpec:
|
|
||||||
containers:
|
|
||||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
|
||||||
resources:
|
|
||||||
limits:
|
|
||||||
nvidia.com/gpu: 2
|
|
||||||
volumeMounts:
|
|
||||||
- name: devshm
|
|
||||||
mountPath: /dev/shm
|
|
||||||
- name: hf-cache
|
|
||||||
mountPath: /root/.cache/huggingface
|
|
||||||
env:
|
|
||||||
- name: HF_HOME
|
|
||||||
value: /root/.cache/huggingface
|
|
||||||
- name: HF_TOKEN
|
|
||||||
valueFrom:
|
|
||||||
secretKeyRef:
|
|
||||||
name: hf-token-secret
|
|
||||||
key: token
|
|
||||||
nodeSelector:
|
|
||||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
|
||||||
volumes:
|
|
||||||
- name: devshm
|
|
||||||
emptyDir:
|
|
||||||
medium: Memory
|
|
||||||
- name: hf-cache
|
|
||||||
hostPath:
|
|
||||||
path: /mnt/hf-cache
|
|
||||||
type: DirectoryOrCreate
|
|
||||||
|
|
||||||
|
|
||||||
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4"
|
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4"
|
||||||
timeout_in_minutes: 60
|
timeout_in_minutes: 60
|
||||||
@@ -937,46 +793,6 @@ steps:
|
|||||||
path: /mnt/hf-cache
|
path: /mnt/hf-cache
|
||||||
type: DirectoryOrCreate
|
type: DirectoryOrCreate
|
||||||
|
|
||||||
- label: ":full_moon: HunyuanImage3-DIT · Accuracy Test"
|
|
||||||
timeout_in_minutes: 180
|
|
||||||
commands:
|
|
||||||
- export HUNYUAN_IMAGE3_MODEL="tencent/HunyuanImage-3.0-Instruct"
|
|
||||||
- export HUNYUAN_IMAGE3_DEVICES="0,1,2,3"
|
|
||||||
- pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m full_model --run-level full_model
|
|
||||||
agents:
|
|
||||||
queue: "mithril-h100-pool"
|
|
||||||
plugins:
|
|
||||||
- kubernetes:
|
|
||||||
podSpec:
|
|
||||||
containers:
|
|
||||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
|
||||||
resources:
|
|
||||||
limits:
|
|
||||||
nvidia.com/gpu: 4
|
|
||||||
volumeMounts:
|
|
||||||
- name: devshm
|
|
||||||
mountPath: /dev/shm
|
|
||||||
- name: hf-cache
|
|
||||||
mountPath: /root/.cache/huggingface
|
|
||||||
env:
|
|
||||||
- name: HF_HOME
|
|
||||||
value: /root/.cache/huggingface
|
|
||||||
- name: HF_TOKEN
|
|
||||||
valueFrom:
|
|
||||||
secretKeyRef:
|
|
||||||
name: hf-token-secret
|
|
||||||
key: token
|
|
||||||
nodeSelector:
|
|
||||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
|
||||||
volumes:
|
|
||||||
- name: devshm
|
|
||||||
emptyDir:
|
|
||||||
medium: Memory
|
|
||||||
- name: hf-cache
|
|
||||||
hostPath:
|
|
||||||
path: /mnt/hf-cache
|
|
||||||
type: DirectoryOrCreate
|
|
||||||
|
|
||||||
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image"
|
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image"
|
||||||
key: nightly-diffusion-x2iat-performance-qwen-image
|
key: nightly-diffusion-x2iat-performance-qwen-image
|
||||||
timeout_in_minutes: 180
|
timeout_in_minutes: 180
|
||||||
|
|||||||
@@ -273,42 +273,6 @@ steps:
|
|||||||
path: /mnt/hf-cache
|
path: /mnt/hf-cache
|
||||||
type: DirectoryOrCreate
|
type: DirectoryOrCreate
|
||||||
|
|
||||||
- label: "Omni · Qwen3-Omni Realtime WebSocket"
|
|
||||||
commands:
|
|
||||||
- |
|
|
||||||
timeout 30m bash -c "
|
|
||||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
||||||
pytest -s -v tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py::TestQwen3OmniRealtimeWebSocket::test_streaming_audio_input_pcm_output_async_chunk_smoke -m 'core_model' --run-level 'core_model'
|
|
||||||
"
|
|
||||||
agents:
|
|
||||||
queue: "mithril-h100-pool"
|
|
||||||
plugins:
|
|
||||||
- kubernetes:
|
|
||||||
podSpec:
|
|
||||||
containers:
|
|
||||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
|
||||||
resources:
|
|
||||||
limits:
|
|
||||||
nvidia.com/gpu: 2
|
|
||||||
volumeMounts:
|
|
||||||
- name: devshm
|
|
||||||
mountPath: /dev/shm
|
|
||||||
- name: hf-cache
|
|
||||||
mountPath: /root/.cache/huggingface
|
|
||||||
env:
|
|
||||||
- name: HF_HOME
|
|
||||||
value: /root/.cache/huggingface
|
|
||||||
nodeSelector:
|
|
||||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
|
||||||
volumes:
|
|
||||||
- name: devshm
|
|
||||||
emptyDir:
|
|
||||||
medium: Memory
|
|
||||||
- name: hf-cache
|
|
||||||
hostPath:
|
|
||||||
path: /mnt/hf-cache
|
|
||||||
type: DirectoryOrCreate
|
|
||||||
|
|
||||||
- label: "Omni · VoxCPM2 Test"
|
- label: "Omni · VoxCPM2 Test"
|
||||||
timeout_in_minutes: 20
|
timeout_in_minutes: 20
|
||||||
commands:
|
commands:
|
||||||
@@ -332,29 +296,6 @@ steps:
|
|||||||
volumes:
|
volumes:
|
||||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||||
|
|
||||||
- label: "TTS · VoxCPM2 Online Test"
|
|
||||||
timeout_in_minutes: 30
|
|
||||||
commands:
|
|
||||||
- |
|
|
||||||
timeout 30m bash -c "
|
|
||||||
export VLLM_LOGGING_LEVEL=DEBUG
|
|
||||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
||||||
pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts.py -m 'core_model and tts' --run-level 'core_model'
|
|
||||||
"
|
|
||||||
agents:
|
|
||||||
queue: "gpu_1_queue"
|
|
||||||
plugins:
|
|
||||||
- docker#v5.2.0:
|
|
||||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
|
||||||
always-pull: true
|
|
||||||
propagate-environment: true
|
|
||||||
shm-size: "8gb"
|
|
||||||
environment:
|
|
||||||
- "HF_HOME=/fsx/hf_cache"
|
|
||||||
- "HF_TOKEN"
|
|
||||||
volumes:
|
|
||||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
|
||||||
|
|
||||||
- label: "TTS · Qwen3-TTS CustomVoice Test"
|
- label: "TTS · Qwen3-TTS CustomVoice Test"
|
||||||
commands:
|
commands:
|
||||||
- |
|
- |
|
||||||
@@ -399,29 +340,6 @@ steps:
|
|||||||
volumes:
|
volumes:
|
||||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||||
|
|
||||||
- label: "TTS · Higgs-Audio-V3 Test"
|
|
||||||
commands:
|
|
||||||
- |
|
|
||||||
timeout 20m bash -c "
|
|
||||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
||||||
export VLLM_USE_DEEP_GEMM=0
|
|
||||||
export VLLM_MOE_USE_DEEP_GEMM=0
|
|
||||||
pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'core_model' --run-level 'core_model'
|
|
||||||
"
|
|
||||||
agents:
|
|
||||||
queue: "gpu_4_queue"
|
|
||||||
plugins:
|
|
||||||
- docker#v5.2.0:
|
|
||||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
|
||||||
always-pull: true
|
|
||||||
propagate-environment: true
|
|
||||||
shm-size: "8gb"
|
|
||||||
environment:
|
|
||||||
- "HF_HOME=/fsx/hf_cache"
|
|
||||||
- "HF_TOKEN"
|
|
||||||
volumes:
|
|
||||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
|
||||||
|
|
||||||
- label: "Diffusion · Qwen Image Edit Test"
|
- label: "Diffusion · Qwen Image Edit Test"
|
||||||
commands:
|
commands:
|
||||||
- |
|
- |
|
||||||
|
|||||||
@@ -16,7 +16,6 @@ Easy, fast, and cheap omni-modality model serving for everyone
|
|||||||
---
|
---
|
||||||
|
|
||||||
*Latest News* 🔥
|
*Latest News* 🔥
|
||||||
- [2026/06] We released [0.22.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.22.0) - an **omnimodal world-model** release aligned with vLLM 0.22, featuring [Nvidia Cosmos3](recipes/cosmos3/Cosmos3-Nano.md)/DreamZero world model support, expanded quantization coverage across Blackwell/NPU/XPU, TTS production improvements, new models including MiniCPM-o 4.5, MOSS-TTS, and Lance, plus RL integration with veRL-Omni.
|
|
||||||
- [2026/05] We released [0.20.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.20.0) - refreshes the serving/runtime stack for large-scale omni workloads, and improves diffusion model performance, quantization, and hardware readiness across CUDA, ROCm, MUSA, NPU, and XPU backends.
|
- [2026/05] We released [0.20.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.20.0) - refreshes the serving/runtime stack for large-scale omni workloads, and improves diffusion model performance, quantization, and hardware readiness across CUDA, ROCm, MUSA, NPU, and XPU backends.
|
||||||
- [2026/03] We released [0.18.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.18.0) - strengthens the core runtime through a large entrypoint refactor and scheduler/runtime cleanups, expands unified quantization and diffusion execution, broadens multimodal model coverage, and improves production readiness across audio, omni, image, video, RL, and multi-platform deployments.
|
- [2026/03] We released [0.18.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.18.0) - strengthens the core runtime through a large entrypoint refactor and scheduler/runtime cleanups, expands unified quantization and diffusion execution, broadens multimodal model coverage, and improves production readiness across audio, omni, image, video, RL, and multi-platform deployments.
|
||||||
- [2026/03] Check out our first public [project deepdive](https://youtu.be/sgwNfsNnR9I) at the vLLM Hong Kong Meetup!
|
- [2026/03] Check out our first public [project deepdive](https://youtu.be/sgwNfsNnR9I) at the vLLM Hong Kong Meetup!
|
||||||
|
|||||||
@@ -151,9 +151,6 @@ class VllmOmniImageClient:
|
|||||||
seed: int | None = None,
|
seed: int | None = None,
|
||||||
negative_prompt: str | None = None,
|
negative_prompt: str | None = None,
|
||||||
output_compression: int | None = None,
|
output_compression: int | None = None,
|
||||||
bot_task: str | None = None,
|
|
||||||
sys_type: str | None = None,
|
|
||||||
system_prompt: str | None = None,
|
|
||||||
) -> Image.Image:
|
) -> Image.Image:
|
||||||
if not isinstance(images, list):
|
if not isinstance(images, list):
|
||||||
images = [images]
|
images = [images]
|
||||||
@@ -173,12 +170,6 @@ class VllmOmniImageClient:
|
|||||||
data["negative_prompt"] = negative_prompt
|
data["negative_prompt"] = negative_prompt
|
||||||
if output_compression is not None:
|
if output_compression is not None:
|
||||||
data["output_compression"] = str(output_compression)
|
data["output_compression"] = str(output_compression)
|
||||||
if bot_task is not None:
|
|
||||||
data["bot_task"] = bot_task
|
|
||||||
if sys_type is not None:
|
|
||||||
data["sys_type"] = sys_type
|
|
||||||
if system_prompt is not None:
|
|
||||||
data["system_prompt"] = system_prompt
|
|
||||||
|
|
||||||
files = [
|
files = [
|
||||||
(
|
(
|
||||||
|
|||||||
@@ -64,47 +64,6 @@ def resolve_model_name(*, model_name: str | None, model: str | None = None, outp
|
|||||||
raise ValueError("model-name is required when it cannot be inferred from model or output-root")
|
raise ValueError("model-name is required when it cannot be inferred from model or output-root")
|
||||||
|
|
||||||
|
|
||||||
def _coerce_score_int(value: Any) -> int:
|
|
||||||
if isinstance(value, bool):
|
|
||||||
raise TypeError(f"Expected numeric score, got bool: {value!r}")
|
|
||||||
if isinstance(value, (int, float)):
|
|
||||||
return int(value)
|
|
||||||
if isinstance(value, str):
|
|
||||||
stripped = value.strip()
|
|
||||||
if stripped.lstrip("-").isdigit():
|
|
||||||
return int(stripped)
|
|
||||||
raise TypeError(f"Expected numeric score, got {type(value).__name__}: {value!r}")
|
|
||||||
|
|
||||||
|
|
||||||
def _expand_score_entry(entry: Any) -> list[int]:
|
|
||||||
if isinstance(entry, dict):
|
|
||||||
if "score" in entry:
|
|
||||||
return [_coerce_score_int(entry["score"])]
|
|
||||||
if "value" in entry:
|
|
||||||
return [_coerce_score_int(entry["value"])]
|
|
||||||
numeric_values = [_coerce_score_int(value) for value in entry.values()]
|
|
||||||
if numeric_values:
|
|
||||||
return numeric_values
|
|
||||||
raise TypeError(f"Score dict has no numeric values: {entry!r}")
|
|
||||||
return [_coerce_score_int(entry)]
|
|
||||||
|
|
||||||
|
|
||||||
def _normalize_score_list(score: Any) -> list[int]:
|
|
||||||
if isinstance(score, dict):
|
|
||||||
entries: list[Any] = [score]
|
|
||||||
elif isinstance(score, list):
|
|
||||||
entries = score
|
|
||||||
else:
|
|
||||||
entries = [score]
|
|
||||||
|
|
||||||
normalized: list[int] = []
|
|
||||||
for entry in entries:
|
|
||||||
normalized.extend(_expand_score_entry(entry))
|
|
||||||
if not normalized:
|
|
||||||
raise ValueError(f"Empty score list after normalization: {score!r}")
|
|
||||||
return normalized
|
|
||||||
|
|
||||||
|
|
||||||
def parse_score_payload(raw_text: str) -> dict[str, Any]:
|
def parse_score_payload(raw_text: str) -> dict[str, Any]:
|
||||||
try:
|
try:
|
||||||
parsed = extract_json_object(raw_text)
|
parsed = extract_json_object(raw_text)
|
||||||
@@ -114,7 +73,10 @@ def parse_score_payload(raw_text: str) -> dict[str, Any]:
|
|||||||
parsed = {"score": json.loads(stripped), "reasoning": ""}
|
parsed = {"score": json.loads(stripped), "reasoning": ""}
|
||||||
else:
|
else:
|
||||||
raise
|
raise
|
||||||
parsed["score"] = _normalize_score_list(parsed.get("score", []))
|
score = parsed.get("score", [])
|
||||||
|
if not isinstance(score, list):
|
||||||
|
score = [score]
|
||||||
|
parsed["score"] = [int(value) for value in score]
|
||||||
return parsed
|
return parsed
|
||||||
|
|
||||||
|
|
||||||
@@ -429,9 +391,6 @@ class GEditBenchRunner:
|
|||||||
num_inference_steps: int = 20,
|
num_inference_steps: int = 20,
|
||||||
guidance_scale: float | None = None,
|
guidance_scale: float | None = None,
|
||||||
seed: int | None = 42,
|
seed: int | None = 42,
|
||||||
bot_task: str | None = None,
|
|
||||||
sys_type: str | None = None,
|
|
||||||
system_prompt: str | None = None,
|
|
||||||
):
|
):
|
||||||
self.dataset_ref = dataset_ref
|
self.dataset_ref = dataset_ref
|
||||||
self.output_root = output_root
|
self.output_root = output_root
|
||||||
@@ -441,9 +400,6 @@ class GEditBenchRunner:
|
|||||||
self.num_inference_steps = num_inference_steps
|
self.num_inference_steps = num_inference_steps
|
||||||
self.guidance_scale = guidance_scale
|
self.guidance_scale = guidance_scale
|
||||||
self.seed = seed
|
self.seed = seed
|
||||||
self.bot_task = bot_task
|
|
||||||
self.sys_type = sys_type
|
|
||||||
self.system_prompt = system_prompt
|
|
||||||
self.client = VllmOmniImageClient(base_url=base_url, api_key=api_key)
|
self.client = VllmOmniImageClient(base_url=base_url, api_key=api_key)
|
||||||
|
|
||||||
def generate(
|
def generate(
|
||||||
@@ -522,9 +478,6 @@ class GEditBenchRunner:
|
|||||||
num_inference_steps=self.num_inference_steps,
|
num_inference_steps=self.num_inference_steps,
|
||||||
guidance_scale=self.guidance_scale,
|
guidance_scale=self.guidance_scale,
|
||||||
seed=self.seed,
|
seed=self.seed,
|
||||||
bot_task=self.bot_task,
|
|
||||||
sys_type=self.sys_type,
|
|
||||||
system_prompt=self.system_prompt,
|
|
||||||
)
|
)
|
||||||
edited_image.save(output_path)
|
edited_image.save(output_path)
|
||||||
return {
|
return {
|
||||||
@@ -680,25 +633,6 @@ def build_parser() -> argparse.ArgumentParser:
|
|||||||
generate.add_argument("--num-inference-steps", type=int, default=20)
|
generate.add_argument("--num-inference-steps", type=int, default=20)
|
||||||
generate.add_argument("--guidance-scale", type=float, default=None)
|
generate.add_argument("--guidance-scale", type=float, default=None)
|
||||||
generate.add_argument("--seed", type=int, default=42)
|
generate.add_argument("--seed", type=int, default=42)
|
||||||
generate.add_argument(
|
|
||||||
"--bot-task",
|
|
||||||
type=str,
|
|
||||||
default=None,
|
|
||||||
choices=["think", "recaption", "think_recaption", "vanilla"],
|
|
||||||
help="HunyuanImage-3 prompting mode for /v1/images/edits (optional).",
|
|
||||||
)
|
|
||||||
generate.add_argument(
|
|
||||||
"--sys-type",
|
|
||||||
type=str,
|
|
||||||
default=None,
|
|
||||||
help="Hunyuan system-prompt type (e.g. en_unified). Defaults from --bot-task when omitted.",
|
|
||||||
)
|
|
||||||
generate.add_argument(
|
|
||||||
"--system-prompt",
|
|
||||||
type=str,
|
|
||||||
default=None,
|
|
||||||
help="Custom Hunyuan system prompt body (sys_type=custom).",
|
|
||||||
)
|
|
||||||
generate.add_argument("--workers", type=int, default=1)
|
generate.add_argument("--workers", type=int, default=1)
|
||||||
generate.add_argument("--max-samples", type=int, default=None)
|
generate.add_argument("--max-samples", type=int, default=None)
|
||||||
generate.add_argument("--samples-per-group", type=int, default=None)
|
generate.add_argument("--samples-per-group", type=int, default=None)
|
||||||
@@ -741,9 +675,6 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
num_inference_steps=args.num_inference_steps,
|
num_inference_steps=args.num_inference_steps,
|
||||||
guidance_scale=args.guidance_scale,
|
guidance_scale=args.guidance_scale,
|
||||||
seed=args.seed,
|
seed=args.seed,
|
||||||
bot_task=args.bot_task,
|
|
||||||
sys_type=args.sys_type,
|
|
||||||
system_prompt=args.system_prompt,
|
|
||||||
)
|
)
|
||||||
records = runner.generate(
|
records = runner.generate(
|
||||||
model_name=model_name,
|
model_name=model_name,
|
||||||
|
|||||||
+26
-134
@@ -2,29 +2,29 @@
|
|||||||
The vllm bench command launches the vLLM-Omni benchmark to evaluate the performance of multimodal models.
|
The vllm bench command launches the vLLM-Omni benchmark to evaluate the performance of multimodal models.
|
||||||
|
|
||||||
## Notes
|
## Notes
|
||||||
We currently only support using the "openai-chat-omni" and "openai-image-edits-omni" backend.
|
We currently only support using the "openai-chat-omni" backend.
|
||||||
|
|
||||||
## Basic Parameter Description
|
## Basic Parameter Description
|
||||||
You can use `vllm bench serve --omni --help=all` to get descriptions of all parameters. The commonly used parameters are described below:
|
You can use `vllm bench serve --omni --help=all` to get descriptions of all parameters. The commonly used parameters are described below:
|
||||||
- `--omni`
|
- `--omni`
|
||||||
Enable Omni (multimodal) mode, supporting multimodal inputs and outputs such as images, videos, and audio.
|
Enable Omni (multimodal) mode, supporting multimodal inputs and outputs such as images, videos, and audio.
|
||||||
|
|
||||||
- `--backend`
|
- `--backend`
|
||||||
Specify the backend adapter as openai-chat-omni, using OpenAI Chat compatible API behavior as the protocol. Currently only openai-chat-omni is supported.
|
Specify the backend adapter as openai-chat-omni, using OpenAI Chat compatible API behavior as the protocol. Currently only openai-chat-omni is supported.
|
||||||
|
|
||||||
- `--model`
|
- `--model`
|
||||||
The model identifier to load, filled according to the models supported by vLLM-Omni.
|
The model identifier to load, filled according to the models supported by vLLM-Omni.
|
||||||
|
|
||||||
- `--endpoint`
|
- `--endpoint`
|
||||||
The API endpoint exposed externally, to which clients send their requests.
|
The API endpoint exposed externally, to which clients send their requests.
|
||||||
|
|
||||||
- `--dataset-name`
|
- `--dataset-name`
|
||||||
The name of the dataset used; random-mm indicates generating random multimodal inputs (images, videos, audio).
|
The name of the dataset used; random-mm indicates generating random multimodal inputs (images, videos, audio).
|
||||||
|
|
||||||
- `--num-prompts`
|
- `--num-prompts`
|
||||||
The total number of requests to send, an integer.
|
The total number of requests to send, an integer.
|
||||||
|
|
||||||
- `--max-concurrency`
|
- `--max-concurrency`
|
||||||
"Maximum number of concurrent requests. This can be used "
|
"Maximum number of concurrent requests. This can be used "
|
||||||
"to help simulate an environment where a higher level component "
|
"to help simulate an environment where a higher level component "
|
||||||
"is enforcing a maximum number of concurrent requests. While the "
|
"is enforcing a maximum number of concurrent requests. While the "
|
||||||
@@ -34,73 +34,69 @@ You can use `vllm bench serve --omni --help=all` to get descriptions of all para
|
|||||||
"actual request rate may be lower than specified with --request-rate, "
|
"actual request rate may be lower than specified with --request-rate, "
|
||||||
"if the server is not processing requests fast enough to keep up."
|
"if the server is not processing requests fast enough to keep up."
|
||||||
|
|
||||||
- `--request-rate`
|
- `--request-rate`
|
||||||
"Number of requests per second. If this is inf, "
|
"Number of requests per second. If this is inf, "
|
||||||
"then all the requests are sent at time 0. "
|
"then all the requests are sent at time 0. "
|
||||||
"Otherwise, we use Poisson process or gamma distribution "
|
"Otherwise, we use Poisson process or gamma distribution "
|
||||||
"to synthesize the request arrival times."
|
"to synthesize the request arrival times."
|
||||||
|
|
||||||
- `--ignore-eos`
|
- `--ignore-eos`
|
||||||
"Set ignore_eos flag when sending the benchmark request."
|
"Set ignore_eos flag when sending the benchmark request."
|
||||||
|
|
||||||
- `--metric-percentiles`
|
- `--metric-percentiles`
|
||||||
Comma-separated list of percentiles for selected metrics. "
|
Comma-separated list of percentiles for selected metrics. "
|
||||||
"To report 25-th, 50-th, and 75-th percentiles, use \"25,50,75\". "
|
"To report 25-th, 50-th, and 75-th percentiles, use \"25,50,75\". "
|
||||||
"Default value is \"99\"."
|
"Default value is \"99\"."
|
||||||
"Use \"--percentile-metrics\" to select metrics.
|
"Use \"--percentile-metrics\" to select metrics.
|
||||||
|
|
||||||
- `--percentile-metrics`
|
- `--percentile-metrics`
|
||||||
"Comma-separated list of selected metrics to report percentiles."
|
"Comma-separated list of selected metrics to report percentiles."
|
||||||
"This argument specifies the metrics to report percentiles."
|
"This argument specifies the metrics to report percentiles."
|
||||||
'Allowed metric names are "ttft", "tpot", "itl", "ttfc", "tpoc", "icl", '
|
'Allowed metric names are "ttft", "tpot", "itl", "e2el", "audio_ttfp", "audio_rtf", "audio_duration". '
|
||||||
'"tpop", "e2el", "audio_ttfp", "audio_rtf", "audio_duration". '
|
|
||||||
|
|
||||||
- `-print-stage`
|
- `--save-result`
|
||||||
Print per-stage benchmark metrics for --omni serving when stage metrics are returned by the server. Disabled by default.
|
|
||||||
|
|
||||||
- `--save-result`
|
|
||||||
Specify to save benchmark results to a json file
|
Specify to save benchmark results to a json file
|
||||||
|
|
||||||
- `--save-detailed`
|
- `--save-detailed`
|
||||||
"When saving the results, whether to include per request "
|
"When saving the results, whether to include per request "
|
||||||
"information such as response, error, ttfs, tpots, etc."
|
"information such as response, error, ttfs, tpots, etc."
|
||||||
|
|
||||||
- `--result-dir`
|
- `--result-dir`
|
||||||
"Specify directory to save benchmark json results."
|
"Specify directory to save benchmark json results."
|
||||||
"If not specified, results are saved in the current directory."
|
"If not specified, results are saved in the current directory."
|
||||||
|
|
||||||
- `--result-filename`
|
- `--result-filename`
|
||||||
"Specify the filename to save benchmark json results."
|
"Specify the filename to save benchmark json results."
|
||||||
"If not specified, results will be saved in "
|
"If not specified, results will be saved in "
|
||||||
"{label}-{args.request_rate}qps-{base_model_id}-{current_dt}.json"
|
"{label}-{args.request_rate}qps-{base_model_id}-{current_dt}.json"
|
||||||
|
|
||||||
- `--random-prefix-len`
|
- `--random-prefix-len`
|
||||||
Number of fixed prefix tokens before the random context in a request.
|
Number of fixed prefix tokens before the random context in a request.
|
||||||
The total input length is the sum of random-prefix-len and a random
|
The total input length is the sum of random-prefix-len and a random
|
||||||
context length sampled from [input_len * (1 - range_ratio),
|
context length sampled from [input_len * (1 - range_ratio),
|
||||||
input_len * (1 + range_ratio)].Only the random and random-mm modes
|
input_len * (1 + range_ratio)].Only the random and random-mm modes
|
||||||
support this parameter.
|
support this parameter.
|
||||||
|
|
||||||
- `--random-input-len`
|
- `--random-input-len`
|
||||||
Number of input tokens per request.Only the random and random-mm modes support this parameter.
|
Number of input tokens per request.Only the random and random-mm modes support this parameter.
|
||||||
|
|
||||||
- `--random-output-len`
|
- `--random-output-len`
|
||||||
Number of output tokens per request.Only the random and random-mm modes support this parameter.
|
Number of output tokens per request.Only the random and random-mm modes support this parameter.
|
||||||
|
|
||||||
- `--random-range-ratio`
|
- `--random-range-ratio`
|
||||||
Range ratio for sampling input/output length,
|
Range ratio for sampling input/output length,
|
||||||
used only for random sampling. Must be in the range [0, 1) to define
|
used only for random sampling. Must be in the range [0, 1) to define
|
||||||
a symmetric sampling range
|
a symmetric sampling range
|
||||||
[length * (1 - range_ratio), length * (1 + range_ratio)].
|
[length * (1 - range_ratio), length * (1 + range_ratio)].
|
||||||
Only the random and random-mm modes support this parameter.
|
Only the random and random-mm modes support this parameter.
|
||||||
|
|
||||||
- `--random-mm-base-items-per-request`
|
- `--random-mm-base-items-per-request`
|
||||||
Base number of multimodal items per request for random-mm.
|
Base number of multimodal items per request for random-mm.
|
||||||
Actual per-request count is sampled around this base using
|
Actual per-request count is sampled around this base using
|
||||||
--random-mm-num-mm-items-range-ratio.
|
--random-mm-num-mm-items-range-ratio.
|
||||||
Only the random-mm mode supports this parameter.
|
Only the random-mm mode supports this parameter.
|
||||||
|
|
||||||
- `--random-mm-limit-mm-per-prompt`
|
- `--random-mm-limit-mm-per-prompt`
|
||||||
Per-modality hard caps for items attached per request, e.g.
|
Per-modality hard caps for items attached per request, e.g.
|
||||||
'{"image": 3, "video": 1, "audio": 1}'. The sampled per-request item
|
'{"image": 3, "video": 1, "audio": 1}'. The sampled per-request item
|
||||||
count is clamped to the sum of these limits. When a modality
|
count is clamped to the sum of these limits. When a modality
|
||||||
@@ -108,7 +104,7 @@ Specify to save benchmark results to a json file
|
|||||||
renormalized.
|
renormalized.
|
||||||
Only the random-mm mode supports this parameter.
|
Only the random-mm mode supports this parameter.
|
||||||
|
|
||||||
- `--random-mm-num-mm-items-range-ratio`
|
- `--random-mm-num-mm-items-range-ratio`
|
||||||
Range ratio r in [0, 1] for sampling items per request.
|
Range ratio r in [0, 1] for sampling items per request.
|
||||||
We sample uniformly from the closed integer range
|
We sample uniformly from the closed integer range
|
||||||
[floor(n*(1-r)), ceil(n*(1+r))]
|
[floor(n*(1-r)), ceil(n*(1+r))]
|
||||||
@@ -119,7 +115,7 @@ Specify to save benchmark results to a json file
|
|||||||
An error is raised if the computed min exceeds the max.
|
An error is raised if the computed min exceeds the max.
|
||||||
Only the random-mm mode supports this parameter.
|
Only the random-mm mode supports this parameter.
|
||||||
|
|
||||||
- `--random-mm-bucket-config`
|
- `--random-mm-bucket-config`
|
||||||
The bucket config is a dictionary mapping a multimodal item
|
The bucket config is a dictionary mapping a multimodal item
|
||||||
sampling configuration to a probability.
|
sampling configuration to a probability.
|
||||||
Currently allows for 3 modalities: audio, images and videos.
|
Currently allows for 3 modalities: audio, images and videos.
|
||||||
@@ -280,9 +276,7 @@ Generate synthetic image、video、audio inputs alongside random text prompts to
|
|||||||
|
|
||||||
Notes:
|
Notes:
|
||||||
|
|
||||||
- Works only with online benchmark via:
|
- Works only with online benchmark via the OpenAI backend (`--backend openai-chat-omni`) and endpoint `/v1/chat/completions`.
|
||||||
- the OpenAI chat backend (`--backend openai-chat-omni`) and endpoint `/v1/chat/completions`.
|
|
||||||
- the OpenAI edit image backend (`--backend openai-image-edits-omni`) and endpoint `/v1/images/edits`.
|
|
||||||
|
|
||||||
Start the server (example):
|
Start the server (example):
|
||||||
|
|
||||||
@@ -363,105 +357,3 @@ How sampling works:
|
|||||||
This should be seen as an edge case, and if this behavior can be avoided by setting `--random-mm-limit-mm-per-prompt` to a large number. Note that this might result in errors due to engine config `--limit-mm-per-prompt`.
|
This should be seen as an edge case, and if this behavior can be avoided by setting `--random-mm-limit-mm-per-prompt` to a large number. Note that this might result in errors due to engine config `--limit-mm-per-prompt`.
|
||||||
- The resulting request contains synthetic image data in `multi_modal_data` (OpenAI Chat format). When `random-mm` is used with the OpenAI Chat backend, prompts remain text and MM content is attached via `multi_modal_data`.
|
- The resulting request contains synthetic image data in `multi_modal_data` (OpenAI Chat format). When `random-mm` is used with the OpenAI Chat backend, prompts remain text and MM content is attached via `multi_modal_data`.
|
||||||
</details>
|
</details>
|
||||||
|
|
||||||
### Multi-Stage Benchmark
|
|
||||||
|
|
||||||
<details class="admonition abstract" markdown="1">
|
|
||||||
<summary>Show more</summary>
|
|
||||||
|
|
||||||
Start the server (change the port and model):
|
|
||||||
|
|
||||||
```bash
|
|
||||||
vllm serve --omni --port 29999 --model ~/Qwen3-Omni-30B-A3B-Instruct
|
|
||||||
```
|
|
||||||
|
|
||||||
Then run the benchmarking script (remember to add the flag `--print-stage`):
|
|
||||||
```bash
|
|
||||||
vllm bench serve --omni \
|
|
||||||
--dataset-name random \
|
|
||||||
--port 29999 \
|
|
||||||
--max-concurrency 1 \
|
|
||||||
--num-warmups 2 \
|
|
||||||
--model ~/Qwen3-Omni-30B-A3B-Instruct \
|
|
||||||
--endpoint /v1/chat/completions \
|
|
||||||
--backend openai-chat-omni \
|
|
||||||
--num-prompts 3 \
|
|
||||||
--random-input-len 2500 \
|
|
||||||
--ignore-eos \
|
|
||||||
--percentile-metrics ttft,tpot,itl,e2el,audio_ttfp,audio_rtf,ttfc,tpoc,icl \
|
|
||||||
--random-output-len 900 \
|
|
||||||
--extra_body '{"modalities": ["text","audio"]}' \
|
|
||||||
--print-stage
|
|
||||||
```
|
|
||||||
|
|
||||||
Besides the "Serving Benchmark Result", there will be "Stage Benchmark Result" below:
|
|
||||||
```text
|
|
||||||
============= Stage Benchmark Result =============
|
|
||||||
=============== Stage 0 (thinker) ================
|
|
||||||
-------------------Stage Timing-------------------
|
|
||||||
Mean stage_gen_time (ms): 18243.16
|
|
||||||
Median stage_gen_time (ms): 18220.83
|
|
||||||
P99 stage_gen_time (ms): 18296.75
|
|
||||||
================== Text Result ===================
|
|
||||||
Stage generated tokens: 2700
|
|
||||||
-----------Serving Time to First Token------------
|
|
||||||
Mean Serving TTFT (ms): 816.31
|
|
||||||
Median Serving TTFT (ms): 818.94
|
|
||||||
P99 Serving TTFT (ms): 829.76
|
|
||||||
-----Time per Output Token (excl. 1st token)------
|
|
||||||
Mean TPOP (ms): 19.41
|
|
||||||
Median TPOP (ms): 19.40
|
|
||||||
P99 TPOP (ms): 19.47
|
|
||||||
---------------Inter-token Latency----------------
|
|
||||||
Mean ITL (ms): 19.41
|
|
||||||
Median ITL (ms): 19.12
|
|
||||||
P99 ITL (ms): 23.58
|
|
||||||
================ Stage 1 (talker) ================
|
|
||||||
-------------------Stage Timing-------------------
|
|
||||||
Mean stage_gen_time (ms): 18849.64
|
|
||||||
Median stage_gen_time (ms): 13641.15
|
|
||||||
P99 stage_gen_time (ms): 33655.58
|
|
||||||
============= Internal Stream Result =============
|
|
||||||
-----------Serving Time to First Chunk------------
|
|
||||||
Mean Serving TTFC (ms): 1029.14
|
|
||||||
Median Serving TTFC (ms): 1027.98
|
|
||||||
P99 Serving TTFC (ms): 1032.59
|
|
||||||
-----Time per Output Chunk (excl. 1st chunk)------
|
|
||||||
Mean TPOP (ms): 17.53
|
|
||||||
Median TPOP (ms): 18.70
|
|
||||||
P99 TPOP (ms): 18.88
|
|
||||||
---------------Inter-chunk Latency----------------
|
|
||||||
Mean ICL (ms): 16.27
|
|
||||||
Median ICL (ms): 18.19
|
|
||||||
P99 ICL (ms): 25.86
|
|
||||||
=============== Stage 2 (code2wav) ===============
|
|
||||||
-------------------Stage Timing-------------------
|
|
||||||
Mean stage_gen_time (ms): 18860.74
|
|
||||||
Median stage_gen_time (ms): 13655.84
|
|
||||||
P99 stage_gen_time (ms): 33659.79
|
|
||||||
================== Audio Result ==================
|
|
||||||
Stage audio duration generated(s): 260.11
|
|
||||||
Stage audio frames generated: 6242730
|
|
||||||
-----------Serving Time to First Packet-----------
|
|
||||||
Mean Serving AUDIO_TTFP (ms): 1366.16
|
|
||||||
Median Serving AUDIO_TTFP (ms): 1367.20
|
|
||||||
P99 Serving AUDIO_TTFP (ms): 1368.26
|
|
||||||
-----------------Real Time Factor-----------------
|
|
||||||
Mean AUDIO_RTF: 0.24
|
|
||||||
Median AUDIO_RTF: 0.26
|
|
||||||
P99 AUDIO_RTF: 0.27
|
|
||||||
```
|
|
||||||
Explanation:
|
|
||||||
- stage_gen_time: Time from submitting a request to a specific stage to that stage finishing generation.
|
|
||||||
|
|
||||||
- Serving TTFC (Time to First Chunk): Time from the HTTP request being accepted by the serving frontend to
|
|
||||||
the stage producing its first non-empty output chunk. Keep the name TTFT in text stage, TTFP in audio stage
|
|
||||||
for easier mapping.
|
|
||||||
|
|
||||||
- TPOP (Time per Output Chunk): Average time from the first output chunk to stage completion, divided by the
|
|
||||||
number of remaining output chunks. The TPOP abbreviation follows Qwen3.5-Omni Technical Report.
|
|
||||||
|
|
||||||
- ICL (Inter-Chunk Latency): Time between two consecutive output chunks produced by the same stage. Keep the
|
|
||||||
name ITL in text stage for easier mapping.
|
|
||||||
|
|
||||||
</details>
|
|
||||||
@@ -45,7 +45,7 @@ If you do not need to modify source code of vLLM, you can directly install the s
|
|||||||
uv pip install vllm==0.22.0 --torch-backend=auto
|
uv pip install vllm==0.22.0 --torch-backend=auto
|
||||||
```
|
```
|
||||||
|
|
||||||
The 0.22.0 release of vLLM ships CUDA 12.9-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
The 0.22.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
||||||
|
|
||||||
#### Installation of vLLM-Omni
|
#### Installation of vLLM-Omni
|
||||||
Since vllm-omni is rapidly evolving, it's recommended to install it from source
|
Since vllm-omni is rapidly evolving, it's recommended to install it from source
|
||||||
@@ -70,8 +70,8 @@ git checkout v0.22.0
|
|||||||
```
|
```
|
||||||
Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set `VLLM_PRECOMPILED_WHEEL_LOCATION` as your local absolute path of whl file.
|
Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set `VLLM_PRECOMPILED_WHEEL_LOCATION` as your local absolute path of whl file.
|
||||||
```bash
|
```bash
|
||||||
#For CUDA 12.9 (the default for v0.22.0; the wheel filename has no `+cu129` suffix)
|
#For CUDA 13.0 (the default for v0.22.0; the wheel filename has no `+cu130` suffix)
|
||||||
export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.22.0/vllm-0.22.0-cp38-abi3-manylinux_2_28_x86_64.whl
|
export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.22.0/vllm-0.22.0-cp38-abi3-manylinux_2_35_x86_64.whl
|
||||||
```
|
```
|
||||||
Install vllm with command below (If you have no existing PyTorch).
|
Install vllm with command below (If you have no existing PyTorch).
|
||||||
```bash
|
```bash
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ source .venv/bin/activate
|
|||||||
uv pip install vllm==0.22.0 --torch-backend=auto
|
uv pip install vllm==0.22.0 --torch-backend=auto
|
||||||
|
|
||||||
# On ROCm
|
# On ROCm
|
||||||
uv pip install vllm==0.22.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.22.0/rocm722
|
uv pip install vllm==0.22.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.22.0/rocm721
|
||||||
|
|
||||||
git clone https://github.com/vllm-project/vllm-omni.git
|
git clone https://github.com/vllm-project/vllm-omni.git
|
||||||
cd vllm-omni
|
cd vllm-omni
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ th {
|
|||||||
| `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
| `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
| `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
| `Cosmos3OmniDiffusersPipeline` | Cosmos3 T2I, T2V, I2V, T2V with sound, action policy | `nvidia/Cosmos3-Nano` | ✅︎ | | | |
|
| `Cosmos3OmniDiffusersPipeline` | Cosmos3 T2I, T2V, I2V | `nvidia/Cosmos3-Nano` | ✅︎ | | | |
|
||||||
| `WanSpeechToVideoPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `WanSpeechToVideoPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
| `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
| `LTX2Pipeline` | LTX-2-T2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | |
|
| `LTX2Pipeline` | LTX-2-T2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | |
|
||||||
@@ -42,7 +42,6 @@ th {
|
|||||||
| `LTX2ImageToVideoTwoStagesPipeline` | LTX-2-I2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | |
|
| `LTX2ImageToVideoTwoStagesPipeline` | LTX-2-I2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | |
|
||||||
| `LTX23Pipeline` | LTX-2.3-T2V | `dg845/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | |
|
| `LTX23Pipeline` | LTX-2.3-T2V | `dg845/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | |
|
||||||
| `LTX23ImageToVideoPipeline` | LTX-2.3-I2V | `dg845/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | |
|
| `LTX23ImageToVideoPipeline` | LTX-2.3-I2V | `dg845/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | |
|
||||||
| `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | | | |
|
|
||||||
| `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | |
|
| `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | |
|
||||||
| `MagiHumanPipeline` | MagiHuman | `SII-GAIR/daVinci-MagiHuman-Base-1080p` | ✅︎ | ✅︎ | | |
|
| `MagiHumanPipeline` | MagiHuman | `SII-GAIR/daVinci-MagiHuman-Base-1080p` | ✅︎ | ✅︎ | | |
|
||||||
| `OvisImagePipeline` | Ovis-Image | `OvisAI/Ovis-Image` | ✅︎ | ✅︎ | | ✅︎ |
|
| `OvisImagePipeline` | Ovis-Image | `OvisAI/Ovis-Image` | ✅︎ | ✅︎ | | ✅︎ |
|
||||||
@@ -50,8 +49,6 @@ th {
|
|||||||
| `LongCatImageEditPipeline` | LongCat-Image-Edit | `meituan-longcat/LongCat-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `LongCatImageEditPipeline` | LongCat-Image-Edit | `meituan-longcat/LongCat-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
| `StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` | ✅︎ | ✅︎ | | ✅︎ |
|
| `StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` | ✅︎ | ✅︎ | | ✅︎ |
|
||||||
| `CosyVoice3Model` | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✅︎ | ✅︎ | | ✅︎ |
|
| `CosyVoice3Model` | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✅︎ | ✅︎ | | ✅︎ |
|
||||||
| `OmniVoiceModel` | OmniVoice | `k2-fsa/OmniVoice` | ✅︎ | | | |
|
|
||||||
| `VoxCPM2TalkerForConditionalGeneration` | VoxCPM2 | `openbmb/VoxCPM2` | ✅︎ | | | |
|
|
||||||
| `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | |
|
| `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | |
|
||||||
| `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
| `FluxKontextPipeline` | FLUX.1-Kontext-dev | `black-forest-labs/FLUX.1-Kontext-dev` | ✅︎ | ✅︎ | | |
|
| `FluxKontextPipeline` | FLUX.1-Kontext-dev | `black-forest-labs/FLUX.1-Kontext-dev` | ✅︎ | ✅︎ | | |
|
||||||
@@ -64,10 +61,6 @@ th {
|
|||||||
| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-0.6B-Base` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-0.6B-Base` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
| `GLMTTSForConditionalGeneration` | GLM-TTS | `zai-org/GLM-TTS` | ✅︎ | | | |
|
| `GLMTTSForConditionalGeneration` | GLM-TTS | `zai-org/GLM-TTS` | ✅︎ | | | |
|
||||||
| `MossTTSNanoForCausalLM` | MOSS-TTS-Nano | `OpenMOSS-Team/MOSS-TTS-Nano` | ✅︎ | | | |
|
|
||||||
| `MossTTSDelayModel` | MOSS-TTS, MOSS-TTSD, MOSS-SoundEffect, MOSS-VoiceGenerator | `OpenMOSS-Team/MOSS-VoiceGenerator` | ✅︎ | | | |
|
|
||||||
| `MossTTSRealtime` | MOSS-TTS-Realtime | `OpenMOSS-Team/MOSS-TTS-Realtime` | ✅︎ | | | |
|
|
||||||
| `HiggsAudioV2ForConditionalGeneration` | Higgs-Audio v2 | `bosonai/higgs-audio-v2-generation-3B-base` | ✅︎ | | | |
|
|
||||||
| `NextStep11Pipeline` | NextStep-1.1 | `stepfun-ai/NextStep-1.1` | ✅︎ | ✅︎ | | ✅︎ |
|
| `NextStep11Pipeline` | NextStep-1.1 | `stepfun-ai/NextStep-1.1` | ✅︎ | ✅︎ | | ✅︎ |
|
||||||
| `MiMoAudioModel` | MiMo-Audio-7B-Instruct | `XiaomiMiMo/MiMo-Audio-7B-Instruct` | ✅︎ | ✅︎ | | |
|
| `MiMoAudioModel` | MiMo-Audio-7B-Instruct | `XiaomiMiMo/MiMo-Audio-7B-Instruct` | ✅︎ | ✅︎ | | |
|
||||||
| `MiMoV2ASRForCausalLM` | MiMo-V2.5-ASR | `XiaomiMiMo/MiMo-V2.5-ASR` | ✅︎ | ✅︎ | | |
|
| `MiMoV2ASRForCausalLM` | MiMo-V2.5-ASR | `XiaomiMiMo/MiMo-V2.5-ASR` | ✅︎ | ✅︎ | | |
|
||||||
@@ -75,13 +68,11 @@ th {
|
|||||||
| `FishSpeechSlowARForConditionalGeneration` | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✅︎ | ✅︎ | | |
|
| `FishSpeechSlowARForConditionalGeneration` | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✅︎ | ✅︎ | | |
|
||||||
| `DreamIDOmniPipeline` | DreamID-Omni | `XuGuo699/DreamID-Omni` | ✅︎ | ✅︎ | | |
|
| `DreamIDOmniPipeline` | DreamID-Omni | `XuGuo699/DreamID-Omni` | ✅︎ | ✅︎ | | |
|
||||||
| `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only) | `SenseNova/SenseNova-U1-8B-MoT` | ✅︎ | | | |
|
| `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only) | `SenseNova/SenseNova-U1-8B-MoT` | ✅︎ | | | |
|
||||||
| `LancePipeline` | Lance | `bytedance-research/Lance` | ✅︎ | | | |
|
|
||||||
| `HunyuanVideo15Pipeline` | HunyuanVideo-1.5-T2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | ✅︎ | ✅︎ | | |
|
| `HunyuanVideo15Pipeline` | HunyuanVideo-1.5-T2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | ✅︎ | ✅︎ | | |
|
||||||
| `HunyuanVideo15ImageToVideoPipeline` | HunyuanVideo-1.5-I2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v` | ✅︎ | ✅︎ | | |
|
| `HunyuanVideo15ImageToVideoPipeline` | HunyuanVideo-1.5-I2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v` | ✅︎ | ✅︎ | | |
|
||||||
| `VoxtralTTSForConditionalGeneration` | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✅︎ | ✅︎ | | |
|
| `VoxtralTTSForConditionalGeneration` | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✅︎ | ✅︎ | | |
|
||||||
| `CovoAudioForConditionalGeneration` | Covo-Audio-Chat | `tencent/Covo-Audio-Chat` | ✅︎ | | | |
|
| `CovoAudioForConditionalGeneration` | Covo-Audio-Chat | `tencent/Covo-Audio-Chat` | ✅︎ | | | |
|
||||||
|`DyninOmniForConditionalGeneration` | Dynin-Omni | `snu-aidas/Dynin-Omni` | ✅︎ | | | |
|
|`DyninOmniForConditionalGeneration` | Dynin-Omni | `snu-aidas/Dynin-Omni` | ✅︎ | | | |
|
||||||
| `MiniCPMO45OmniForConditionalGeneration` | MiniCPM-o 4.5 | `openbmb/MiniCPM-o-4_5` | ✅︎ | | ✅︎ | |
|
|
||||||
| `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
| `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||||
|`HiDreamImagePipeline` | HiDream-I1-Full | `HiDream-ai/HiDream-I1-Full` | ✅︎ | ✅︎ | | |
|
|`HiDreamImagePipeline` | HiDream-I1-Full | `HiDream-ai/HiDream-I1-Full` | ✅︎ | ✅︎ | | |
|
||||||
|
|
||||||
|
|||||||
@@ -145,7 +145,6 @@ The following tables show which models support each feature:
|
|||||||
| Model | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | Pipeline-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
|
| Model | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | Pipeline-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
|
||||||
|------------------------------|:---------:|:----------:|:---------------------:|:--------------:|:-----------------:|:-----------------:|:------:|:-------------------------:|:--------------------:|:--------------:|:----------------:|
|
|------------------------------|:---------:|:----------:|:---------------------:|:--------------:|:-----------------:|:-----------------:|:------:|:-------------------------:|:--------------------:|:--------------:|:----------------:|
|
||||||
| **Wan2.2** | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (encode/decode) | ❌ | ❌ |
|
| **Wan2.2** | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (encode/decode) | ❌ | ❌ |
|
||||||
| **Wan2.2-S2V** | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ (encode/decode) | ❌ | ❌ |
|
|
||||||
| **Wan2.1-VACE** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
|
| **Wan2.1-VACE** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
|
||||||
| **LTX-2** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
| **LTX-2** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||||
| **LTX-2.3** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
| **LTX-2.3** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||||
|
|||||||
@@ -1,304 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import uuid
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import cv2
|
|
||||||
import numpy as np
|
|
||||||
import torch
|
|
||||||
from PIL import Image
|
|
||||||
|
|
||||||
from vllm_omni import Omni
|
|
||||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
|
||||||
from vllm_omni.outputs import OmniRequestOutput
|
|
||||||
|
|
||||||
WORKER_EXTENSION = "vllm_omni.diffusion.models.dreamzero.video_export_worker.DreamZeroVideoExportWorkerExtension"
|
|
||||||
ASSET_REPO_ID = "YangshenDeng/vllm-omni-dreamzero-assets"
|
|
||||||
DEFAULT_SESSION_PREFIX = "dreamzero-export"
|
|
||||||
RELATIVE_OFFSETS = [-23, -16, -8, 0]
|
|
||||||
ACTION_HORIZON = 24
|
|
||||||
CAMERA_FILES = {
|
|
||||||
"observation/exterior_image_0_left": "exterior_image_1_left.mp4",
|
|
||||||
"observation/exterior_image_1_left": "exterior_image_2_left.mp4",
|
|
||||||
"observation/wrist_image_left": "wrist_image_left.mp4",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _parse_args() -> argparse.Namespace:
|
|
||||||
repo_root = Path(__file__).resolve().parents[3]
|
|
||||||
parser = argparse.ArgumentParser(description="Export DreamZero prediction video from downloaded example inputs.")
|
|
||||||
parser.add_argument("--model", default="GEAR-Dreams/DreamZero-DROID")
|
|
||||||
parser.add_argument("--deploy-config", type=Path, required=True)
|
|
||||||
parser.add_argument(
|
|
||||||
"--video-dir",
|
|
||||||
type=Path,
|
|
||||||
default=repo_root / "outputs" / "dreamzero" / "assets",
|
|
||||||
help="Directory containing the three camera MP4 files.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--output-dir", type=Path, default=repo_root / "outputs" / "dreamzero" / "generated_predictions"
|
|
||||||
)
|
|
||||||
parser.add_argument("--output-stem", default="dreamzero_prediction")
|
|
||||||
parser.add_argument(
|
|
||||||
"--prompt",
|
|
||||||
default="Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan",
|
|
||||||
)
|
|
||||||
parser.add_argument("--session-id", default=None)
|
|
||||||
parser.add_argument("--save-input-video", action="store_true")
|
|
||||||
parser.add_argument("--save-gif", action="store_true")
|
|
||||||
parser.add_argument("--save-actions", action="store_true")
|
|
||||||
parser.add_argument("--fps", type=int, default=5)
|
|
||||||
return parser.parse_args()
|
|
||||||
|
|
||||||
|
|
||||||
def _load_all_frames(video_path: Path) -> np.ndarray:
|
|
||||||
cap = cv2.VideoCapture(str(video_path))
|
|
||||||
frames = []
|
|
||||||
while True:
|
|
||||||
ok, frame = cap.read()
|
|
||||||
if not ok:
|
|
||||||
break
|
|
||||||
frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
|
|
||||||
cap.release()
|
|
||||||
if not frames:
|
|
||||||
raise RuntimeError(f"No frames loaded from {video_path}")
|
|
||||||
return np.stack(frames, axis=0)
|
|
||||||
|
|
||||||
|
|
||||||
def _load_camera_frames(video_dir: Path) -> dict[str, np.ndarray]:
|
|
||||||
camera_frames: dict[str, np.ndarray] = {}
|
|
||||||
for camera_key, file_name in CAMERA_FILES.items():
|
|
||||||
video_path = video_dir / file_name
|
|
||||||
if not video_path.exists():
|
|
||||||
raise FileNotFoundError(
|
|
||||||
f"Missing DreamZero example asset: {video_path}. "
|
|
||||||
"Download the example videos with: "
|
|
||||||
f"`hf download {ASSET_REPO_ID} --repo-type dataset --local-dir {video_dir}`"
|
|
||||||
)
|
|
||||||
camera_frames[camera_key] = _load_all_frames(video_path)
|
|
||||||
return camera_frames
|
|
||||||
|
|
||||||
|
|
||||||
def _build_frame_schedule(total_frames: int, num_chunks: int) -> list[list[int]]:
|
|
||||||
chunks: list[list[int]] = []
|
|
||||||
current_frame = 23
|
|
||||||
for _ in range(num_chunks):
|
|
||||||
indices = [max(current_frame + offset, 0) for offset in RELATIVE_OFFSETS]
|
|
||||||
if indices[-1] >= total_frames:
|
|
||||||
break
|
|
||||||
chunks.append(indices)
|
|
||||||
current_frame += ACTION_HORIZON
|
|
||||||
return chunks
|
|
||||||
|
|
||||||
|
|
||||||
def _make_obs_from_video(
|
|
||||||
camera_frames: dict[str, np.ndarray],
|
|
||||||
frame_indices: list[int],
|
|
||||||
*,
|
|
||||||
prompt: str,
|
|
||||||
session_id: str,
|
|
||||||
) -> dict:
|
|
||||||
obs: dict = {}
|
|
||||||
for camera_key, all_frames in camera_frames.items():
|
|
||||||
selected = all_frames[frame_indices]
|
|
||||||
obs[camera_key] = selected[0] if len(frame_indices) == 1 else selected
|
|
||||||
|
|
||||||
obs["observation/joint_position"] = np.zeros(7, dtype=np.float32)
|
|
||||||
obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32)
|
|
||||||
obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32)
|
|
||||||
obs["prompt"] = prompt
|
|
||||||
obs["session_id"] = session_id
|
|
||||||
return obs
|
|
||||||
|
|
||||||
|
|
||||||
def _build_observations(video_dir: Path, prompt: str, session_id: str) -> tuple[dict[str, np.ndarray], list[dict]]:
|
|
||||||
camera_frames = _load_camera_frames(video_dir)
|
|
||||||
total_frames = min(frames.shape[0] for frames in camera_frames.values())
|
|
||||||
chunks = _build_frame_schedule(total_frames, 1)
|
|
||||||
observations = [
|
|
||||||
_make_obs_from_video(camera_frames, [0], prompt=prompt, session_id=session_id),
|
|
||||||
]
|
|
||||||
if chunks:
|
|
||||||
observations.append(
|
|
||||||
_make_obs_from_video(
|
|
||||||
camera_frames,
|
|
||||||
chunks[0],
|
|
||||||
prompt=prompt,
|
|
||||||
session_id=session_id,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
if len(observations) < 2:
|
|
||||||
raise RuntimeError("Need at least two DreamZero example observations to export a prediction video.")
|
|
||||||
return camera_frames, observations[:2]
|
|
||||||
|
|
||||||
|
|
||||||
def _extract_latents(output: OmniRequestOutput) -> torch.Tensor:
|
|
||||||
if not isinstance(output, OmniRequestOutput):
|
|
||||||
raise TypeError(f"Expected OmniRequestOutput, got {type(output)!r}")
|
|
||||||
if not output.images:
|
|
||||||
raise RuntimeError("DreamZero output does not contain video latents in `images`.")
|
|
||||||
|
|
||||||
latents = output.images[0]
|
|
||||||
if not isinstance(latents, torch.Tensor):
|
|
||||||
raise TypeError(f"Expected tensor latents, got {type(latents)!r}")
|
|
||||||
|
|
||||||
latents = latents.detach().cpu()
|
|
||||||
if latents.dim() == 4:
|
|
||||||
latents = latents.unsqueeze(0)
|
|
||||||
if latents.dim() != 5:
|
|
||||||
raise ValueError(f"Unexpected latent shape: {tuple(latents.shape)}")
|
|
||||||
|
|
||||||
if latents.shape[1] < latents.shape[2]:
|
|
||||||
latents = latents.transpose(1, 2).contiguous()
|
|
||||||
return latents
|
|
||||||
|
|
||||||
|
|
||||||
def _write_mp4(path: Path, frames: np.ndarray, fps: int) -> None:
|
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
height, width = frames.shape[1:3]
|
|
||||||
writer = cv2.VideoWriter(
|
|
||||||
str(path),
|
|
||||||
cv2.VideoWriter_fourcc(*"mp4v"),
|
|
||||||
float(fps),
|
|
||||||
(width, height),
|
|
||||||
)
|
|
||||||
if not writer.isOpened():
|
|
||||||
raise RuntimeError(f"Failed to open video writer for {path}")
|
|
||||||
try:
|
|
||||||
for frame in frames:
|
|
||||||
writer.write(cv2.cvtColor(frame, cv2.COLOR_RGB2BGR))
|
|
||||||
finally:
|
|
||||||
writer.release()
|
|
||||||
|
|
||||||
|
|
||||||
def _write_gif(path: Path, frames: np.ndarray, fps: int) -> None:
|
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
images = [Image.fromarray(frame) for frame in frames]
|
|
||||||
duration_ms = max(int(round(1000 / max(fps, 1))), 1)
|
|
||||||
images[0].save(
|
|
||||||
path,
|
|
||||||
save_all=True,
|
|
||||||
append_images=images[1:],
|
|
||||||
duration=duration_ms,
|
|
||||||
loop=0,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _stitch_input_frames(camera_frames: dict[str, np.ndarray]) -> np.ndarray:
|
|
||||||
total_frames = min(frames.shape[0] for frames in camera_frames.values())
|
|
||||||
stitched = []
|
|
||||||
for frame_index in range(total_frames):
|
|
||||||
left = camera_frames["observation/exterior_image_0_left"][frame_index]
|
|
||||||
right = camera_frames["observation/exterior_image_1_left"][frame_index]
|
|
||||||
wrist = camera_frames["observation/wrist_image_left"][frame_index]
|
|
||||||
pad = np.zeros((left.shape[0], left.shape[1], 3), dtype=np.uint8)
|
|
||||||
canvas = np.concatenate([left, right], axis=1)
|
|
||||||
bottom = np.concatenate([wrist, pad], axis=1)
|
|
||||||
stitched.append(np.concatenate([canvas, bottom], axis=0))
|
|
||||||
return np.stack(stitched, axis=0)
|
|
||||||
|
|
||||||
|
|
||||||
def _run_generation(
|
|
||||||
model: str, deploy_config_path: Path, observations: list[dict]
|
|
||||||
) -> tuple[Omni, list[OmniRequestOutput]]:
|
|
||||||
omni = Omni(
|
|
||||||
model=model,
|
|
||||||
deploy_config=str(deploy_config_path),
|
|
||||||
enforce_eager=True,
|
|
||||||
worker_extension_cls=WORKER_EXTENSION,
|
|
||||||
)
|
|
||||||
|
|
||||||
outputs: list[OmniRequestOutput] = []
|
|
||||||
for index, obs in enumerate(observations):
|
|
||||||
sampling_params = OmniDiffusionSamplingParams(
|
|
||||||
extra_args={
|
|
||||||
"reset": index == 0,
|
|
||||||
"session_id": obs["session_id"],
|
|
||||||
"robot_obs": obs,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
result = omni.generate(obs["prompt"], sampling_params_list=[sampling_params])
|
|
||||||
if not result:
|
|
||||||
raise RuntimeError(f"No output returned for DreamZero request {index}")
|
|
||||||
outputs.append(result[0])
|
|
||||||
return omni, outputs
|
|
||||||
|
|
||||||
|
|
||||||
def _decode_with_worker(omni: Omni, full_latents: torch.Tensor) -> np.ndarray:
|
|
||||||
stage_client = omni.engine.stage_clients[0]
|
|
||||||
engine = getattr(stage_client, "_engine", None)
|
|
||||||
if engine is None:
|
|
||||||
raise RuntimeError("DreamZero export requires inline diffusion stage access.")
|
|
||||||
|
|
||||||
decoded = engine.executor.collective_rpc(
|
|
||||||
"decode_video_latents_to_uint8",
|
|
||||||
args=(full_latents,),
|
|
||||||
unique_reply_rank=0,
|
|
||||||
exec_all_ranks=True,
|
|
||||||
)
|
|
||||||
if isinstance(decoded, torch.Tensor):
|
|
||||||
decoded = decoded.numpy()
|
|
||||||
if not isinstance(decoded, np.ndarray):
|
|
||||||
raise TypeError(f"Unexpected decoded output type: {type(decoded)!r}")
|
|
||||||
return decoded
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
args = _parse_args()
|
|
||||||
session_id = args.session_id or f"{DEFAULT_SESSION_PREFIX}-{uuid.uuid4()}"
|
|
||||||
|
|
||||||
camera_frames, observations = _build_observations(
|
|
||||||
video_dir=args.video_dir,
|
|
||||||
prompt=args.prompt,
|
|
||||||
session_id=session_id,
|
|
||||||
)
|
|
||||||
|
|
||||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
if args.save_input_video:
|
|
||||||
input_frames = _stitch_input_frames(camera_frames)
|
|
||||||
_write_mp4(args.output_dir / f"{args.output_stem}_input.mp4", input_frames, fps=15)
|
|
||||||
if args.save_gif:
|
|
||||||
_write_gif(args.output_dir / f"{args.output_stem}_input.gif", input_frames[::3], fps=5)
|
|
||||||
|
|
||||||
omni = None
|
|
||||||
try:
|
|
||||||
omni, outputs = _run_generation(
|
|
||||||
model=args.model,
|
|
||||||
deploy_config_path=args.deploy_config,
|
|
||||||
observations=observations,
|
|
||||||
)
|
|
||||||
latent_steps = [_extract_latents(output) for output in outputs]
|
|
||||||
full_latents = torch.cat(latent_steps, dim=2)
|
|
||||||
frames = _decode_with_worker(omni, full_latents)
|
|
||||||
finally:
|
|
||||||
if omni is not None:
|
|
||||||
omni.close()
|
|
||||||
|
|
||||||
mp4_path = args.output_dir / f"{args.output_stem}.mp4"
|
|
||||||
|
|
||||||
_write_mp4(mp4_path, frames, fps=args.fps)
|
|
||||||
print(f"SAVED_MP4={mp4_path}")
|
|
||||||
|
|
||||||
if args.save_gif:
|
|
||||||
gif_path = args.output_dir / f"{args.output_stem}.gif"
|
|
||||||
_write_gif(gif_path, frames, fps=args.fps)
|
|
||||||
print(f"SAVED_GIF={gif_path}")
|
|
||||||
|
|
||||||
if args.save_actions:
|
|
||||||
npz_path = args.output_dir / f"{args.output_stem}_actions.npz"
|
|
||||||
np.savez(
|
|
||||||
npz_path,
|
|
||||||
step0=np.asarray(outputs[0].multimodal_output.get("actions")),
|
|
||||||
step1=np.asarray(outputs[1].multimodal_output.get("actions")),
|
|
||||||
)
|
|
||||||
print(f"SAVED_ACTIONS={npz_path}")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -42,7 +42,7 @@ pip install \
|
|||||||
## 3. Hardware and VRAM Requirements
|
## 3. Hardware and VRAM Requirements
|
||||||
|
|
||||||
This example uses a 3-stage pipeline on one GPU by default
|
This example uses a 3-stage pipeline on one GPU by default
|
||||||
([`dynin_omni.yaml`](../../../vllm_omni/deploy/dynin_omni.yaml)):
|
([`dynin_omni.yaml`](../../../vllm_omni/model_executor/stage_configs/dynin_omni.yaml)):
|
||||||
|
|
||||||
- Stage-0 (`token2text`): `gpu_memory_utilization: 0.5`
|
- Stage-0 (`token2text`): `gpu_memory_utilization: 0.5`
|
||||||
- Stage-1 (`token2image`): `gpu_memory_utilization: 0.1`
|
- Stage-1 (`token2image`): `gpu_memory_utilization: 0.1`
|
||||||
|
|||||||
@@ -878,8 +878,8 @@ def parse_args(repo_root: Path) -> argparse.Namespace:
|
|||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--stage-config-path",
|
"--stage-config-path",
|
||||||
type=str,
|
type=str,
|
||||||
default=str(repo_root / "vllm_omni/deploy/dynin_omni.yaml"),
|
default=str(repo_root / "vllm_omni/model_executor/stage_configs/dynin_omni.yaml"),
|
||||||
help="Path to deploy config yaml.",
|
help="Path to stage config yaml.",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--dynin-config-path",
|
"--dynin-config-path",
|
||||||
|
|||||||
@@ -16,10 +16,9 @@ Usage:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import json
|
import os
|
||||||
import time
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
@@ -30,16 +29,6 @@ from vllm_omni.outputs import OmniRequestOutput
|
|||||||
from vllm_omni.platforms import current_omni_platform
|
from vllm_omni.platforms import current_omni_platform
|
||||||
|
|
||||||
|
|
||||||
def parse_profiler_config(value: str) -> dict[str, Any]:
|
|
||||||
try:
|
|
||||||
config = json.loads(value)
|
|
||||||
except json.JSONDecodeError as e:
|
|
||||||
raise argparse.ArgumentTypeError(f"--profiler-config must be valid JSON: {e}") from e
|
|
||||||
if not isinstance(config, dict):
|
|
||||||
raise argparse.ArgumentTypeError("--profiler-config must be a JSON object")
|
|
||||||
return config
|
|
||||||
|
|
||||||
|
|
||||||
def parse_args() -> argparse.Namespace:
|
def parse_args() -> argparse.Namespace:
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
description="Generate a talking-head video from a reference image and audio (Wan2.2 S2V)."
|
description="Generate a talking-head video from a reference image and audio (Wan2.2 S2V)."
|
||||||
@@ -98,12 +87,6 @@ def parse_args() -> argparse.Namespace:
|
|||||||
default=3.0,
|
default=3.0,
|
||||||
help="Scheduler flow shift (default: 3.0).",
|
help="Scheduler flow shift (default: 3.0).",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
|
||||||
"--boundary-ratio",
|
|
||||||
type=float,
|
|
||||||
default=None,
|
|
||||||
help="Boundary split ratio for low/high DiT stages (S2V). Default varies by model.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--output",
|
"--output",
|
||||||
type=str,
|
type=str,
|
||||||
@@ -195,10 +178,30 @@ def parse_args() -> argparse.Namespace:
|
|||||||
help="Enable cache-dit summary logging after diffusion forward passes.",
|
help="Enable cache-dit summary logging after diffusion forward passes.",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--profiler-config",
|
"--profile-dir",
|
||||||
type=parse_profiler_config,
|
type=str,
|
||||||
default=None,
|
default=None,
|
||||||
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
|
help="Enable torch profiler and save traces to this directory.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--profile-record-shapes",
|
||||||
|
action="store_true",
|
||||||
|
help="Record tensor shapes in profiler (increases trace size).",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--profile-with-stack",
|
||||||
|
action="store_true",
|
||||||
|
help="Record stack traces in profiler (increases overhead).",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--profile-with-memory",
|
||||||
|
action="store_true",
|
||||||
|
help="Profile memory usage.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--profile-with-flops",
|
||||||
|
action="store_true",
|
||||||
|
help="Estimate FLOPs for operations.",
|
||||||
)
|
)
|
||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
@@ -236,9 +239,23 @@ def main():
|
|||||||
ulysses_degree=args.ulysses_degree,
|
ulysses_degree=args.ulysses_degree,
|
||||||
)
|
)
|
||||||
|
|
||||||
profiler_enabled = args.profiler_config is not None
|
# Check if profiling is requested via CLI args or environment variable
|
||||||
|
profile_dir = args.profile_dir or os.getenv("VLLM_TORCH_PROFILER_DIR")
|
||||||
|
profiler_enabled = bool(profile_dir)
|
||||||
|
profiler_config = None
|
||||||
|
if profiler_enabled:
|
||||||
|
from vllm.config import ProfilerConfig
|
||||||
|
|
||||||
omni_kwargs = dict(
|
profiler_config = ProfilerConfig(
|
||||||
|
profiler="torch",
|
||||||
|
torch_profiler_dir=profile_dir,
|
||||||
|
torch_profiler_record_shapes=args.profile_record_shapes,
|
||||||
|
torch_profiler_with_stack=args.profile_with_stack,
|
||||||
|
torch_profiler_with_memory=args.profile_with_memory,
|
||||||
|
torch_profiler_with_flops=args.profile_with_flops,
|
||||||
|
)
|
||||||
|
|
||||||
|
omni = Omni(
|
||||||
model=args.model,
|
model=args.model,
|
||||||
model_class_name="WanS2VPipeline",
|
model_class_name="WanS2VPipeline",
|
||||||
flow_shift=args.flow_shift,
|
flow_shift=args.flow_shift,
|
||||||
@@ -252,12 +269,8 @@ def main():
|
|||||||
cache_backend=args.cache_backend,
|
cache_backend=args.cache_backend,
|
||||||
cache_config=cache_config,
|
cache_config=cache_config,
|
||||||
enable_cache_dit_summary=args.enable_cache_dit_summary,
|
enable_cache_dit_summary=args.enable_cache_dit_summary,
|
||||||
profiler_config=args.profiler_config,
|
profiler_config=profiler_config,
|
||||||
)
|
)
|
||||||
if args.boundary_ratio is not None:
|
|
||||||
omni_kwargs["boundary_ratio"] = args.boundary_ratio
|
|
||||||
|
|
||||||
omni = Omni(**omni_kwargs)
|
|
||||||
|
|
||||||
# Print generation configuration
|
# Print generation configuration
|
||||||
print(f"\n{'=' * 60}")
|
print(f"\n{'=' * 60}")
|
||||||
@@ -312,16 +325,13 @@ def main():
|
|||||||
if profiler_enabled:
|
if profiler_enabled:
|
||||||
print("\n[Profiler] Stopping profiler and collecting results...")
|
print("\n[Profiler] Stopping profiler and collecting results...")
|
||||||
profile_results = omni.stop_profile()
|
profile_results = omni.stop_profile()
|
||||||
if profile_results and isinstance(profile_results, dict):
|
if profile_results and isinstance(profile_results, list):
|
||||||
traces = profile_results.get("traces", [])
|
|
||||||
print("\n" + "=" * 60)
|
print("\n" + "=" * 60)
|
||||||
print("PROFILING RESULTS:")
|
print("PROFILING RESULTS:")
|
||||||
for rank, trace in enumerate(traces):
|
for i, result in enumerate(profile_results):
|
||||||
print(f"\nRank {rank}:")
|
print(f"\nStage {i}:")
|
||||||
if trace:
|
if result:
|
||||||
print(f" • Trace: {trace}")
|
print(f" {result}")
|
||||||
if not traces:
|
|
||||||
print(" No traces collected.")
|
|
||||||
print("=" * 60)
|
print("=" * 60)
|
||||||
else:
|
else:
|
||||||
print("[Profiler] No valid profiling data returned.")
|
print("[Profiler] No valid profiling data returned.")
|
||||||
|
|||||||
@@ -1,204 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
"""Offline higgs-audio v3 inference example.
|
|
||||||
|
|
||||||
Runs Stage 0 (Qwen3 talker) + Stage 1 (HiggsAudio codec) end-to-end
|
|
||||||
through the vLLM-Omni engine without going through the HTTP server, and
|
|
||||||
saves a 24 kHz mono WAV per prompt.
|
|
||||||
|
|
||||||
Example (plain TTS):
|
|
||||||
|
|
||||||
python examples/offline_inference/text_to_speech/higgs_audio_v3/end2end.py \\
|
|
||||||
--texts "Hello world." \\
|
|
||||||
--output-dir results/higgs_v3_wavs
|
|
||||||
|
|
||||||
Example (voice clone):
|
|
||||||
|
|
||||||
python examples/offline_inference/text_to_speech/higgs_audio_v3/end2end.py \\
|
|
||||||
--texts "Hello world." \\
|
|
||||||
--ref-audio path/to/reference.wav \\
|
|
||||||
--ref-text "Transcript of the reference clip." \\
|
|
||||||
--output-dir results/higgs_v3_wavs
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import os
|
|
||||||
|
|
||||||
os.environ.setdefault("VLLM_USE_DEEP_GEMM", "0")
|
|
||||||
os.environ.setdefault("VLLM_MOE_USE_DEEP_GEMM", "0")
|
|
||||||
|
|
||||||
import re
|
|
||||||
import time
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
import soundfile as sf
|
|
||||||
import torch
|
|
||||||
|
|
||||||
from vllm_omni import Omni
|
|
||||||
|
|
||||||
SAMPLE_RATE = 24_000
|
|
||||||
DEFAULT_TEXTS = (
|
|
||||||
"Hello world.",
|
|
||||||
"The quick brown fox jumps over the lazy dog.",
|
|
||||||
"Today is a beautiful day for a walk in the park.",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def parse_args():
|
|
||||||
import argparse
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(description="Offline higgs-audio v3 inference")
|
|
||||||
parser.add_argument(
|
|
||||||
"--model",
|
|
||||||
type=str,
|
|
||||||
default="bosonai/higgs-audio-v3-tts-4b",
|
|
||||||
help="Stage-0 talker model id or path.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--texts",
|
|
||||||
type=str,
|
|
||||||
nargs="+",
|
|
||||||
default=list(DEFAULT_TEXTS),
|
|
||||||
help="One or more plain-text prompts to synthesize.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--output-dir",
|
|
||||||
type=str,
|
|
||||||
default="results/higgs_v3_wavs",
|
|
||||||
help="Directory to write per-prompt WAV files.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--deploy-config",
|
|
||||||
type=str,
|
|
||||||
default=None,
|
|
||||||
help="Override the deploy config path.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--max-new-tokens",
|
|
||||||
type=int,
|
|
||||||
default=2048,
|
|
||||||
help="Cap on Stage-0 codec frames per request.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--ref-audio",
|
|
||||||
type=str,
|
|
||||||
default=None,
|
|
||||||
help="Reference audio for voice clone (WAV/FLAC/MP3 path).",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--ref-text",
|
|
||||||
type=str,
|
|
||||||
default=None,
|
|
||||||
help="Transcript of the reference audio. Optional but improves fidelity.",
|
|
||||||
)
|
|
||||||
return parser.parse_args()
|
|
||||||
|
|
||||||
|
|
||||||
def _slugify(text: str) -> str:
|
|
||||||
slug = re.sub(r"\s+", "_", text.strip().lower())
|
|
||||||
slug = re.sub(r"[^a-z0-9_]+", "", slug)
|
|
||||||
return slug[:48] or "prompt"
|
|
||||||
|
|
||||||
|
|
||||||
def _extract_pcm(multimodal_output: dict) -> torch.Tensor:
|
|
||||||
audio = multimodal_output.get("model_outputs")
|
|
||||||
if audio is None:
|
|
||||||
audio = multimodal_output.get("audio")
|
|
||||||
if audio is None:
|
|
||||||
raise ValueError(f"no audio key in multimodal_output: {list(multimodal_output.keys())}")
|
|
||||||
if isinstance(audio, list):
|
|
||||||
valid = [torch.as_tensor(a).float().cpu().reshape(-1) for a in audio if a is not None]
|
|
||||||
if not valid:
|
|
||||||
raise ValueError("audio list is empty")
|
|
||||||
return torch.cat(valid, dim=0) if len(valid) > 1 else valid[0]
|
|
||||||
return torch.as_tensor(audio).float().cpu().reshape(-1)
|
|
||||||
|
|
||||||
|
|
||||||
def _pcm_to_int16(pcm: torch.Tensor) -> np.ndarray:
|
|
||||||
arr = pcm.numpy()
|
|
||||||
if arr.dtype.kind == "f":
|
|
||||||
arr = np.clip(arr, -1.0, 1.0)
|
|
||||||
arr = (arr * 32767.0).astype(np.int16)
|
|
||||||
else:
|
|
||||||
arr = arr.astype(np.int16)
|
|
||||||
return arr
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
args = parse_args()
|
|
||||||
output_dir = Path(args.output_dir)
|
|
||||||
output_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
engine = Omni(model=args.model, deploy_config=args.deploy_config, trust_remote_code=True)
|
|
||||||
|
|
||||||
from transformers import AutoTokenizer
|
|
||||||
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_tokenizer import (
|
|
||||||
HiggsAudioV3TokenizerAdapter,
|
|
||||||
apply_delay_pattern,
|
|
||||||
encode_reference_audio,
|
|
||||||
)
|
|
||||||
|
|
||||||
tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
|
|
||||||
adapter = HiggsAudioV3TokenizerAdapter(tokenizer)
|
|
||||||
|
|
||||||
# Load and encode reference audio once (if voice cloning)
|
|
||||||
ref_codes_delayed: torch.Tensor | None = None
|
|
||||||
if args.ref_audio is not None:
|
|
||||||
ref_wav, ref_sr = sf.read(args.ref_audio, always_2d=False)
|
|
||||||
if ref_wav.ndim == 2:
|
|
||||||
ref_wav = ref_wav.mean(axis=1)
|
|
||||||
ref_codes_raw = encode_reference_audio(ref_wav, int(ref_sr))
|
|
||||||
ref_codes_delayed = apply_delay_pattern(ref_codes_raw)
|
|
||||||
print(f"Reference : {args.ref_audio}")
|
|
||||||
print(f"Ref codes : {ref_codes_raw.shape[0]} frames -> {ref_codes_delayed.shape[0]} delayed")
|
|
||||||
if args.ref_text:
|
|
||||||
print(f"Ref text : {args.ref_text}")
|
|
||||||
|
|
||||||
print(f"Model : {args.model}")
|
|
||||||
print(f"Prompts : {len(args.texts)}")
|
|
||||||
print(f"Output dir : {output_dir}")
|
|
||||||
print(f"Voice clone : {'yes' if ref_codes_delayed is not None else 'no'}")
|
|
||||||
|
|
||||||
total_elapsed = 0.0
|
|
||||||
total_dur = 0.0
|
|
||||||
for text in args.texts:
|
|
||||||
if ref_codes_delayed is not None:
|
|
||||||
prompt_ids = adapter.build_prompt(
|
|
||||||
text,
|
|
||||||
num_ref_tokens=int(ref_codes_delayed.shape[0]),
|
|
||||||
reference_text=args.ref_text,
|
|
||||||
)
|
|
||||||
prompt = {
|
|
||||||
"prompt_token_ids": prompt_ids,
|
|
||||||
"additional_information": {
|
|
||||||
"audio_input_ids": ref_codes_delayed.to(torch.long),
|
|
||||||
"audio_input_ids_mask": torch.ones(ref_codes_delayed.shape[0], dtype=torch.bool),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
else:
|
|
||||||
prompt_ids = adapter.build_prompt(text)
|
|
||||||
prompt = {"prompt_token_ids": prompt_ids}
|
|
||||||
|
|
||||||
t_start = time.perf_counter()
|
|
||||||
outputs = engine.generate([prompt])
|
|
||||||
elapsed = time.perf_counter() - t_start
|
|
||||||
total_elapsed += elapsed
|
|
||||||
|
|
||||||
mm = outputs[0].outputs[0].multimodal_output
|
|
||||||
pcm = _extract_pcm(mm)
|
|
||||||
slug = _slugify(text)
|
|
||||||
out_path = output_dir / f"{slug}.wav"
|
|
||||||
sf.write(str(out_path), _pcm_to_int16(pcm), SAMPLE_RATE, format="WAV", subtype="PCM_16")
|
|
||||||
dur = pcm.numel() / SAMPLE_RATE
|
|
||||||
total_dur += dur
|
|
||||||
print(f" {slug:<50} dur={dur:5.2f}s -> {out_path}")
|
|
||||||
|
|
||||||
rtf = total_elapsed / total_dur if total_dur > 0 else float("inf")
|
|
||||||
print(f"Total infer : {total_elapsed:.2f}s total audio: {total_dur:.2f}s RTF: {rtf:.3f}")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -1,132 +0,0 @@
|
|||||||
# DreamZero
|
|
||||||
|
|
||||||
DreamZero is the robot-policy serving example for the OpenPI-compatible `/v1/realtime/robot/openpi` endpoint.
|
|
||||||
|
|
||||||
## Supported checkpoints
|
|
||||||
|
|
||||||
| Model | HuggingFace repo |
|
|
||||||
|---|---|
|
|
||||||
| DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` |
|
|
||||||
|
|
||||||
## Quick start
|
|
||||||
|
|
||||||
### Start the server
|
|
||||||
|
|
||||||
```bash
|
|
||||||
bash examples/online_serving/dreamzero/run_server.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
By default this launches:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
vllm serve GEAR-Dreams/DreamZero-DROID --omni --port 8000 \
|
|
||||||
--served-model-name dreamzero-droid \
|
|
||||||
--deploy-config vllm_omni/deploy/dreamzero_tp1_cfg2.yaml \
|
|
||||||
--enforce-eager --disable-log-stats
|
|
||||||
```
|
|
||||||
|
|
||||||
Override `MODEL`, `PORT`, `HOST`, `DEPLOY_CONFIG`, or `SERVED_MODEL_NAME` through the script environment if needed.
|
|
||||||
|
|
||||||
### Install example dependencies
|
|
||||||
|
|
||||||
The core `pip install -e .` setup does not include the extra packages used by the DreamZero example scripts.
|
|
||||||
|
|
||||||
- `openpi_client.py`:
|
|
||||||
`openpi-client`, `websockets`, `opencv-python`
|
|
||||||
- `droid_sim_eval_client.py`:
|
|
||||||
`mediapy`, `websockets`, `openpi-client`
|
|
||||||
|
|
||||||
The DROID sim-eval script also needs an Isaac Lab environment that provides `isaaclab`, `isaaclab_tasks`, `sim_evals`, and `gymnasium`.
|
|
||||||
|
|
||||||
If you run the DROID client on Python < 3.12, also install `typing-extensions`.
|
|
||||||
|
|
||||||
### Configure TP and CFG parallelism
|
|
||||||
|
|
||||||
The bundled DreamZero configs intentionally keep only:
|
|
||||||
|
|
||||||
| Config | Purpose |
|
|
||||||
|---|---|
|
|
||||||
| `vllm_omni/deploy/dreamzero.yaml` | Default TP=1, CFG parallel disabled |
|
|
||||||
| `vllm_omni/deploy/dreamzero_tp1_cfg2.yaml` | TP=1, CFG parallel size=2 |
|
|
||||||
|
|
||||||
For other topologies, use CLI parallelism flags and update stage 0 `devices` with `--stage-overrides`. The number of listed devices must match `tensor_parallel_size * cfg_parallel_size`.
|
|
||||||
|
|
||||||
TP=2 with CFG parallel disabled:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
vllm serve GEAR-Dreams/DreamZero-DROID --omni --port 8000 \
|
|
||||||
--served-model-name dreamzero-droid \
|
|
||||||
--deploy-config vllm_omni/deploy/dreamzero.yaml \
|
|
||||||
--tensor-parallel-size 2 \
|
|
||||||
--stage-overrides '{"0": {"devices": "0,1"}}' \
|
|
||||||
--enforce-eager --disable-log-stats
|
|
||||||
```
|
|
||||||
|
|
||||||
TP=2 with CFG parallel size=2:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
vllm serve GEAR-Dreams/DreamZero-DROID --omni --port 8000 \
|
|
||||||
--served-model-name dreamzero-droid \
|
|
||||||
--deploy-config vllm_omni/deploy/dreamzero.yaml \
|
|
||||||
--tensor-parallel-size 2 \
|
|
||||||
--cfg-parallel-size 2 \
|
|
||||||
--stage-overrides '{"0": {"devices": "0,1,2,3"}}' \
|
|
||||||
--enforce-eager --disable-log-stats
|
|
||||||
```
|
|
||||||
|
|
||||||
### Download example assets
|
|
||||||
|
|
||||||
The OpenPI client and DROID sim-eval example expect the three camera MP4 files in `outputs/dreamzero/assets`.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
hf download YangshenDeng/vllm-omni-dreamzero-assets --repo-type dataset --local-dir outputs/dreamzero/assets
|
|
||||||
```
|
|
||||||
|
|
||||||
### Run the OpenPI client
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python examples/online_serving/dreamzero/openpi_client.py \
|
|
||||||
--host 127.0.0.1 \
|
|
||||||
--port 8000 \
|
|
||||||
--video-dir outputs/dreamzero/assets
|
|
||||||
```
|
|
||||||
|
|
||||||
This client uses downloaded example videos and talks to the OpenPI websocket server.
|
|
||||||
|
|
||||||
### Run DROID sim eval
|
|
||||||
|
|
||||||
```bash
|
|
||||||
${ISAACLAB_LAUNCHER} -p examples/online_serving/dreamzero/droid_sim_eval_client.py \
|
|
||||||
--host 127.0.0.1 \
|
|
||||||
--port 8000 \
|
|
||||||
--scene 1 \
|
|
||||||
--episodes 1 \
|
|
||||||
--headless \
|
|
||||||
--device cuda:0
|
|
||||||
```
|
|
||||||
|
|
||||||
Set `ISAACLAB_LAUNCHER=path/to/isaaclab.sh` from the vLLM-Omni repository root before running the command.
|
|
||||||
This launches Isaac Lab / sim-evals and runs the DROID benchmark loop against the same websocket endpoint.
|
|
||||||
|
|
||||||
### Export comparison videos offline
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python examples/offline_inference/dreamzero/export_prediction_video.py \
|
|
||||||
--deploy-config vllm_omni/deploy/dreamzero_tp1_cfg2.yaml \
|
|
||||||
--save-input-video \
|
|
||||||
--save-gif
|
|
||||||
```
|
|
||||||
|
|
||||||
The export script uses local `Omni` inference, not the websocket server. It writes the input rollout video and the predicted output video artifacts for side-by-side inspection.
|
|
||||||
|
|
||||||
### MolmoSpace demo
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python examples/online_serving/dreamzero/molmospace_dreamzero_eval_demo.py \
|
|
||||||
--host 127.0.0.1 \
|
|
||||||
--port 8000 \
|
|
||||||
--benchmark_dir /path/to/benchmark \
|
|
||||||
--output_dir /path/to/output
|
|
||||||
```
|
|
||||||
|
|
||||||
This demo adapts DreamZero to the MolmoSpace-style remote policy eval loop.
|
|
||||||
@@ -1,819 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
"""Run one or more DROID sim-eval rollouts against the vLLM DreamZero server.
|
|
||||||
|
|
||||||
This script is the vLLM/OpenPI adaptation of the upstream DreamZero
|
|
||||||
`eval_utils/run_sim_eval.py` sim-eval client.
|
|
||||||
|
|
||||||
Behavior intentionally kept close to upstream:
|
|
||||||
- same DROID observation extraction (`external_cam`, `external_cam_2`,
|
|
||||||
`wrist_cam`, joint position, gripper position)
|
|
||||||
- same resize-with-pad preprocessing to `(180, 320)`
|
|
||||||
- same `open_loop_horizon=8`
|
|
||||||
- same gripper binarization rule (`> 0.5 -> 1`, else `0`)
|
|
||||||
- same per-scene language prompts
|
|
||||||
|
|
||||||
Unlike upstream DreamZero, vLLM serves the compatible websocket policy endpoint
|
|
||||||
at `/v1/realtime/robot/openpi`, so this script includes the path suffix in the
|
|
||||||
client URI.
|
|
||||||
|
|
||||||
Run this script through Isaac Lab's launcher from the vLLM-Omni repository
|
|
||||||
root, for example:
|
|
||||||
|
|
||||||
"${ISAACLAB_LAUNCHER}" -p \
|
|
||||||
examples/online_serving/dreamzero/droid_sim_eval_client.py \
|
|
||||||
--host 127.0.0.1 \
|
|
||||||
--port 8000 \
|
|
||||||
--scene 1 \
|
|
||||||
--episodes 1 \
|
|
||||||
--headless \
|
|
||||||
--device cuda:1
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import json
|
|
||||||
import logging
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
import uuid
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from datetime import UTC, datetime
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
import torch
|
|
||||||
|
|
||||||
try:
|
|
||||||
import mediapy
|
|
||||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
|
||||||
raise ImportError("Optional DROID sim-eval client requires `mediapy`.") from exc
|
|
||||||
|
|
||||||
try:
|
|
||||||
from typing import override
|
|
||||||
except ImportError:
|
|
||||||
try:
|
|
||||||
from typing_extensions import override
|
|
||||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
|
||||||
raise ImportError("Optional DROID sim-eval client requires `typing-extensions` on Python < 3.12.") from exc
|
|
||||||
|
|
||||||
try:
|
|
||||||
import websockets.sync.client
|
|
||||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
|
||||||
raise ImportError("Optional DROID sim-eval client requires `websockets`.") from exc
|
|
||||||
|
|
||||||
# NOTE:
|
|
||||||
# This directory already contains a local file named `openpi_client.py`.
|
|
||||||
# However, what we want here is the *installed* `openpi_client` package from
|
|
||||||
# upstream OpenPI, not the sibling example file. When a script is executed
|
|
||||||
# directly, Python often puts the script directory at `sys.path[0]`, which
|
|
||||||
# would cause `import openpi_client` to resolve to the local example file and
|
|
||||||
# create a circular import.
|
|
||||||
#
|
|
||||||
# To avoid that ambiguity, temporarily remove the current example directory
|
|
||||||
# from the front of `sys.path`, import the real package, and then restore the
|
|
||||||
# path afterwards.
|
|
||||||
example_dir = str(Path(__file__).resolve().parent)
|
|
||||||
removed_path = False
|
|
||||||
if sys.path and sys.path[0] == example_dir:
|
|
||||||
sys.path.pop(0)
|
|
||||||
removed_path = True
|
|
||||||
try:
|
|
||||||
from openpi_client import image_tools, msgpack_numpy
|
|
||||||
from openpi_client.base_policy import BasePolicy
|
|
||||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
|
||||||
raise ImportError("Optional DROID sim-eval client requires the `openpi-client` package.") from exc
|
|
||||||
finally:
|
|
||||||
if removed_path:
|
|
||||||
sys.path.insert(0, example_dir)
|
|
||||||
|
|
||||||
# -----------------------------------------------------------------------------
|
|
||||||
# Constants
|
|
||||||
# -----------------------------------------------------------------------------
|
|
||||||
#
|
|
||||||
# These values intentionally mirror the upstream DreamZero sim-eval client
|
|
||||||
# where possible. The important distinction is:
|
|
||||||
#
|
|
||||||
# - ACTION_HORIZON = 24
|
|
||||||
# The model returns 24 future actions per inference call.
|
|
||||||
# - DEFAULT_OPEN_LOOP_HORIZON = 8
|
|
||||||
# The sim client only executes the first 8 actions locally before asking
|
|
||||||
# the server to replan from a fresh observation.
|
|
||||||
#
|
|
||||||
# So a single server call predicts 24x8 actions, but the rollout consumes only
|
|
||||||
# 8 of them before replanning.
|
|
||||||
PING_INTERVAL_SECS = 60
|
|
||||||
PING_TIMEOUT_SECS = 600
|
|
||||||
DEFAULT_PATH = "/v1/realtime/robot/openpi"
|
|
||||||
DEFAULT_OPEN_LOOP_HORIZON = 8
|
|
||||||
ACTION_HORIZON = 24
|
|
||||||
ACTION_DIM = 8
|
|
||||||
DEFAULT_OUTPUT_ROOT = Path("runs") / "dreamzero_sim_eval"
|
|
||||||
SCENE_PROMPTS = {
|
|
||||||
1: "put the cube in the bowl",
|
|
||||||
2: "pick up the can and put it in the mug",
|
|
||||||
3: "put the banana in the bin",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _decode_action_response(response: bytes | str) -> np.ndarray:
|
|
||||||
if isinstance(response, str):
|
|
||||||
raise RuntimeError(f"Error in inference server:\n{response}")
|
|
||||||
decoded = msgpack_numpy.unpackb(response)
|
|
||||||
if isinstance(decoded, dict) and decoded.get("type") == "error":
|
|
||||||
message = decoded.get("message", decoded)
|
|
||||||
raise RuntimeError(f"Error in inference server:\n{message}")
|
|
||||||
return np.asarray(decoded, dtype=np.float32)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class StepRecord:
|
|
||||||
"""One fully materialized rollout step for later JSON export.
|
|
||||||
|
|
||||||
The `episode_00.json` artifact is intended to be human-readable and
|
|
||||||
post-process-friendly. Instead of keeping raw tensors around, each step is
|
|
||||||
flattened into plain Python types so it can be serialized directly.
|
|
||||||
"""
|
|
||||||
|
|
||||||
# Index of this control step within the episode.
|
|
||||||
step_index: int
|
|
||||||
# Whether this step triggered a fresh model call (as opposed to reusing the
|
|
||||||
# cached open-loop chunk from the previous server response).
|
|
||||||
used_server_call: bool
|
|
||||||
# End-to-end latency of the server call that produced the current chunk.
|
|
||||||
# This is `None` on steps that only reuse cached actions.
|
|
||||||
chunk_latency_s: float | None
|
|
||||||
# The concrete 8-D action sent into the simulator at this step.
|
|
||||||
action: list[float]
|
|
||||||
# Observed 7-DoF arm joint positions before the next environment step.
|
|
||||||
joint_position: list[float]
|
|
||||||
# Observed gripper scalar before the next environment step.
|
|
||||||
gripper_position: list[float]
|
|
||||||
# Reward and termination signals directly returned by the simulator.
|
|
||||||
reward: float
|
|
||||||
terminated: bool
|
|
||||||
truncated: bool
|
|
||||||
# Optional scene object positions for downstream debugging / success
|
|
||||||
# heuristics. This may be empty if the environment does not expose them.
|
|
||||||
scene_objects: dict[str, list[float]]
|
|
||||||
|
|
||||||
|
|
||||||
class OpenPIWebsocketClientPolicy(BasePolicy):
|
|
||||||
"""Minimal websocket client for the DreamZero/OpenPI policy protocol.
|
|
||||||
|
|
||||||
Protocol shape:
|
|
||||||
- connect -> server immediately sends a metadata dict
|
|
||||||
- infer -> send msgpack observation, receive action chunk
|
|
||||||
- reset -> send msgpack reset command, receive confirmation string
|
|
||||||
|
|
||||||
This class intentionally stays very small because the more interesting
|
|
||||||
DreamZero-specific behavior lives one layer above, in
|
|
||||||
`DreamZeroJointPosClient`.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
host: str = "127.0.0.1",
|
|
||||||
port: int = 8000,
|
|
||||||
path: str = DEFAULT_PATH,
|
|
||||||
) -> None:
|
|
||||||
# vLLM serves the robot endpoint under `/v1/realtime/robot/openpi`.
|
|
||||||
self._uri = f"ws://{host}:{port}{path}"
|
|
||||||
# Upstream protocol uses msgpack with numpy support, not JSON.
|
|
||||||
self._packer = msgpack_numpy.Packer()
|
|
||||||
# Connect immediately and cache the server handshake metadata.
|
|
||||||
self._ws, self._server_metadata = self._wait_for_server()
|
|
||||||
|
|
||||||
def get_server_metadata(self) -> dict[str, Any]:
|
|
||||||
"""Return a copy of the server handshake metadata."""
|
|
||||||
|
|
||||||
return dict(self._server_metadata)
|
|
||||||
|
|
||||||
def _wait_for_server(self):
|
|
||||||
"""Connect to the websocket server and read the initial metadata frame."""
|
|
||||||
|
|
||||||
logging.info("Connecting to %s", self._uri)
|
|
||||||
conn = websockets.sync.client.connect(
|
|
||||||
self._uri,
|
|
||||||
compression=None,
|
|
||||||
max_size=None,
|
|
||||||
ping_interval=PING_INTERVAL_SECS,
|
|
||||||
ping_timeout=PING_TIMEOUT_SECS,
|
|
||||||
)
|
|
||||||
metadata = msgpack_numpy.unpackb(conn.recv())
|
|
||||||
if not isinstance(metadata, dict):
|
|
||||||
raise TypeError(f"Expected dict metadata from server, got {type(metadata)!r}")
|
|
||||||
return conn, metadata
|
|
||||||
|
|
||||||
@override
|
|
||||||
def infer(self, obs: dict[str, Any]) -> np.ndarray:
|
|
||||||
"""Send an inference request and return the decoded action chunk."""
|
|
||||||
|
|
||||||
# Keep the upstream DreamZero/OpenPI convention that the request itself
|
|
||||||
# tells the server which logical endpoint is being called.
|
|
||||||
payload = dict(obs)
|
|
||||||
payload["endpoint"] = "infer"
|
|
||||||
self._ws.send(self._packer.pack(payload))
|
|
||||||
response = self._ws.recv()
|
|
||||||
return _decode_action_response(response)
|
|
||||||
|
|
||||||
@override
|
|
||||||
def reset(self, reset_info: dict[str, Any] | None = None) -> str:
|
|
||||||
"""Tell the server to reset its session-side state."""
|
|
||||||
|
|
||||||
payload = dict(reset_info or {})
|
|
||||||
payload["endpoint"] = "reset"
|
|
||||||
self._ws.send(self._packer.pack(payload))
|
|
||||||
response = self._ws.recv()
|
|
||||||
if isinstance(response, str):
|
|
||||||
return response
|
|
||||||
decoded = msgpack_numpy.unpackb(response)
|
|
||||||
if not isinstance(decoded, dict) or decoded.get("status") != "reset successful":
|
|
||||||
raise RuntimeError(f"Unexpected reset response: {decoded!r}")
|
|
||||||
return str(decoded["status"])
|
|
||||||
|
|
||||||
def close(self) -> None:
|
|
||||||
"""Close the websocket connection explicitly."""
|
|
||||||
|
|
||||||
self._ws.close()
|
|
||||||
|
|
||||||
|
|
||||||
class DreamZeroJointPosClient:
|
|
||||||
"""DROID sim-eval client that talks to the vLLM OpenPI websocket server.
|
|
||||||
|
|
||||||
This is the main compatibility layer between:
|
|
||||||
- Isaac Lab DROID observations (`obs["policy"][...]`)
|
|
||||||
- DreamZero/OpenPI websocket payloads
|
|
||||||
- local open-loop action reuse across several simulator steps
|
|
||||||
|
|
||||||
In other words:
|
|
||||||
simulator obs -> websocket request -> action chunk -> one action per step
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
remote_host: str = "127.0.0.1",
|
|
||||||
remote_port: int = 8000,
|
|
||||||
path: str = DEFAULT_PATH,
|
|
||||||
open_loop_horizon: int = DEFAULT_OPEN_LOOP_HORIZON,
|
|
||||||
) -> None:
|
|
||||||
# Low-level transport client.
|
|
||||||
self.client = OpenPIWebsocketClientPolicy(remote_host, remote_port, path=path)
|
|
||||||
# Number of actions to execute locally before replanning.
|
|
||||||
self.open_loop_horizon = open_loop_horizon
|
|
||||||
# Cursor into the currently cached action chunk.
|
|
||||||
self.actions_from_chunk_completed = 0
|
|
||||||
# Most recent `(ACTION_HORIZON, ACTION_DIM)` server response.
|
|
||||||
self.pred_action_chunk: np.ndarray | None = None
|
|
||||||
# Session id is part of the DreamZero serving contract. Changing it
|
|
||||||
# causes the server side to treat the rollout as a fresh episode.
|
|
||||||
self.session_id = str(uuid.uuid4())
|
|
||||||
# Simple runtime stats for reporting.
|
|
||||||
self.server_calls = 0
|
|
||||||
self.last_chunk_latency_s: float | None = None
|
|
||||||
self.last_used_server_call = False
|
|
||||||
|
|
||||||
def metadata(self) -> dict[str, Any]:
|
|
||||||
"""Expose the server metadata to callers / logs."""
|
|
||||||
|
|
||||||
return self.client.get_server_metadata()
|
|
||||||
|
|
||||||
def reset(self) -> str:
|
|
||||||
"""Reset local chunk state and remote session state.
|
|
||||||
|
|
||||||
Local reset:
|
|
||||||
- drop cached action chunk
|
|
||||||
- rewind chunk cursor
|
|
||||||
- allocate a fresh session id
|
|
||||||
|
|
||||||
Remote reset:
|
|
||||||
- send a websocket `reset` message so the server can clear any
|
|
||||||
request/session-side state it associates with this client
|
|
||||||
"""
|
|
||||||
|
|
||||||
self.actions_from_chunk_completed = 0
|
|
||||||
self.pred_action_chunk = None
|
|
||||||
self.session_id = str(uuid.uuid4())
|
|
||||||
self.last_chunk_latency_s = None
|
|
||||||
self.last_used_server_call = False
|
|
||||||
return self.client.reset({})
|
|
||||||
|
|
||||||
def infer(self, obs: dict[str, Any], instruction: str) -> dict[str, Any]:
|
|
||||||
"""Turn one simulator observation into one executable 8-D action.
|
|
||||||
|
|
||||||
Key behavior:
|
|
||||||
- call the server only when the local chunk cache is empty/exhausted
|
|
||||||
- otherwise, keep consuming the cached chunk open-loop
|
|
||||||
- always return exactly one 8-D action for the current simulator step
|
|
||||||
"""
|
|
||||||
|
|
||||||
# Convert Isaac Lab observation structure into a plain numpy-friendly
|
|
||||||
# record that is easier to serialize and visualize.
|
|
||||||
curr_obs = self._extract_observation(obs)
|
|
||||||
self.last_used_server_call = False
|
|
||||||
|
|
||||||
# Replan if:
|
|
||||||
# 1. this is the first step of a rollout / chunk
|
|
||||||
# 2. we already consumed `open_loop_horizon` actions from the current chunk
|
|
||||||
# 3. no cached chunk is currently available
|
|
||||||
if (
|
|
||||||
self.actions_from_chunk_completed == 0
|
|
||||||
or self.actions_from_chunk_completed >= self.open_loop_horizon
|
|
||||||
or self.pred_action_chunk is None
|
|
||||||
):
|
|
||||||
self.actions_from_chunk_completed = 0
|
|
||||||
# Build the exact DreamZero/OpenPI payload expected by the server.
|
|
||||||
#
|
|
||||||
# Notes:
|
|
||||||
# - images are resized/padded to the serving contract's 180x320
|
|
||||||
# - proprio is cast to float64 to match upstream client behavior
|
|
||||||
# - cartesian_position is currently unused by DreamZero DROID, so
|
|
||||||
# a dummy zero vector is sent for protocol completeness
|
|
||||||
request_data = {
|
|
||||||
"observation/exterior_image_0_left": image_tools.resize_with_pad(curr_obs["right_image"], 180, 320),
|
|
||||||
"observation/exterior_image_1_left": image_tools.resize_with_pad(curr_obs["left_image"], 180, 320),
|
|
||||||
"observation/wrist_image_left": image_tools.resize_with_pad(curr_obs["wrist_image"], 180, 320),
|
|
||||||
"observation/joint_position": curr_obs["joint_position"].astype(np.float64),
|
|
||||||
"observation/cartesian_position": np.zeros((6,), dtype=np.float64),
|
|
||||||
"observation/gripper_position": curr_obs["gripper_position"].astype(np.float64),
|
|
||||||
"prompt": instruction,
|
|
||||||
"session_id": self.session_id,
|
|
||||||
}
|
|
||||||
|
|
||||||
# Measure end-to-end server latency for this chunk request.
|
|
||||||
start = time.perf_counter()
|
|
||||||
actions = self.client.infer(request_data)
|
|
||||||
self.last_chunk_latency_s = time.perf_counter() - start
|
|
||||||
self.last_used_server_call = True
|
|
||||||
self.server_calls += 1
|
|
||||||
|
|
||||||
# DreamZero DROID serving is expected to return an action chunk with
|
|
||||||
# 24 future actions, each action being 8-D.
|
|
||||||
if actions.ndim != 2:
|
|
||||||
raise AssertionError(f"Expected 2D action array, got shape {actions.shape}")
|
|
||||||
if actions.shape != (ACTION_HORIZON, ACTION_DIM):
|
|
||||||
raise AssertionError(f"Expected action shape {(ACTION_HORIZON, ACTION_DIM)}, got {actions.shape}")
|
|
||||||
self.pred_action_chunk = actions
|
|
||||||
|
|
||||||
# Consume exactly one action row from the cached chunk for this
|
|
||||||
# simulator step.
|
|
||||||
action = np.array(self.pred_action_chunk[self.actions_from_chunk_completed], copy=True)
|
|
||||||
self.actions_from_chunk_completed += 1
|
|
||||||
|
|
||||||
# Upstream DreamZero sim-eval binarizes the gripper command.
|
|
||||||
action[-1] = 1.0 if action[-1].item() > 0.5 else 0.0
|
|
||||||
|
|
||||||
# Produce a human-friendly visualization strip for videos:
|
|
||||||
# right external | wrist | left external
|
|
||||||
img1 = image_tools.resize_with_pad(curr_obs["right_image"], 224, 224)
|
|
||||||
img2 = image_tools.resize_with_pad(curr_obs["wrist_image"], 224, 224)
|
|
||||||
img3 = image_tools.resize_with_pad(curr_obs["left_image"], 224, 224)
|
|
||||||
viz = np.concatenate([img1, img2, img3], axis=1)
|
|
||||||
|
|
||||||
# Return both the executable action and auxiliary debug info.
|
|
||||||
return {
|
|
||||||
"action": action,
|
|
||||||
"viz": viz,
|
|
||||||
"joint_position": curr_obs["joint_position"],
|
|
||||||
"gripper_position": curr_obs["gripper_position"],
|
|
||||||
"used_server_call": self.last_used_server_call,
|
|
||||||
"chunk_latency_s": self.last_chunk_latency_s if self.last_used_server_call else None,
|
|
||||||
}
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _extract_observation(obs_dict: dict[str, Any]) -> dict[str, np.ndarray]:
|
|
||||||
"""Extract the pieces DreamZero cares about from Isaac Lab observations.
|
|
||||||
|
|
||||||
`sim-evals` exposes camera frames and robot state inside the
|
|
||||||
`obs["policy"]` group. This helper converts those tensors into numpy
|
|
||||||
arrays so they can be fed into image preprocessing / websocket packing.
|
|
||||||
"""
|
|
||||||
|
|
||||||
policy = obs_dict["policy"]
|
|
||||||
# Isaac Lab stores camera observations as batched tensors; use env 0.
|
|
||||||
right_image = policy["external_cam"][0].clone().detach().cpu().numpy()
|
|
||||||
left_image = policy["external_cam_2"][0].clone().detach().cpu().numpy()
|
|
||||||
wrist_image = policy["wrist_cam"][0].clone().detach().cpu().numpy()
|
|
||||||
# Robot proprioception.
|
|
||||||
joint_position = policy["arm_joint_pos"].clone().detach().cpu().numpy()
|
|
||||||
gripper_position = policy["gripper_pos"].clone().detach().cpu().numpy()
|
|
||||||
|
|
||||||
return {
|
|
||||||
"right_image": right_image,
|
|
||||||
"left_image": left_image,
|
|
||||||
"wrist_image": wrist_image,
|
|
||||||
"joint_position": joint_position,
|
|
||||||
"gripper_position": gripper_position,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _scene_instruction(scene: int) -> str:
|
|
||||||
"""Map a numeric scene id onto the fixed language prompt used for rollout."""
|
|
||||||
|
|
||||||
try:
|
|
||||||
return SCENE_PROMPTS[scene]
|
|
||||||
except KeyError as exc:
|
|
||||||
raise ValueError(f"Unsupported scene {scene}. Available scenes: {sorted(SCENE_PROMPTS)}") from exc
|
|
||||||
|
|
||||||
|
|
||||||
def _capture_scene_objects(env: Any) -> dict[str, list[float]]:
|
|
||||||
"""Best-effort extraction of scene object root positions.
|
|
||||||
|
|
||||||
This is only for debugging / reporting. The rollout logic itself does not
|
|
||||||
depend on these positions.
|
|
||||||
"""
|
|
||||||
|
|
||||||
objects: dict[str, list[float]] = {}
|
|
||||||
scene = getattr(env, "scene", None)
|
|
||||||
if scene is None:
|
|
||||||
return objects
|
|
||||||
|
|
||||||
# Skip non-task entities such as cameras, the robot, and lighting.
|
|
||||||
for name in scene.keys():
|
|
||||||
if name in {"robot", "external_cam", "external_cam_2", "wrist_cam", "sphere_light", "scene"}:
|
|
||||||
continue
|
|
||||||
entity = scene[name]
|
|
||||||
data = getattr(entity, "data", None)
|
|
||||||
root_pos_w = getattr(data, "root_pos_w", None)
|
|
||||||
if root_pos_w is None:
|
|
||||||
continue
|
|
||||||
# Convert tensors to plain lists for JSON serialization.
|
|
||||||
value = root_pos_w[0].detach().cpu().to(torch.float32).tolist()
|
|
||||||
objects[str(name)] = [float(x) for x in value]
|
|
||||||
return objects
|
|
||||||
|
|
||||||
|
|
||||||
def _maybe_infer_success(scene: int, final_objects: dict[str, list[float]]) -> dict[str, Any]:
|
|
||||||
"""Best-effort geometric heuristic.
|
|
||||||
|
|
||||||
The simulator itself does not expose a built-in success term; this function
|
|
||||||
provides a transparent fallback for human-readable reporting only.
|
|
||||||
"""
|
|
||||||
|
|
||||||
task_pairs = {
|
|
||||||
1: ("cube", "bowl"),
|
|
||||||
2: ("can", "mug"),
|
|
||||||
3: ("banana", "bin"),
|
|
||||||
}
|
|
||||||
source_name, target_name = task_pairs.get(scene, (None, None))
|
|
||||||
if source_name not in final_objects or target_name not in final_objects:
|
|
||||||
return {
|
|
||||||
"has_builtin_success": False,
|
|
||||||
"heuristic_success": None,
|
|
||||||
"reason": "scene object names unavailable for heuristic",
|
|
||||||
}
|
|
||||||
|
|
||||||
source = np.asarray(final_objects[source_name], dtype=np.float32)
|
|
||||||
target = np.asarray(final_objects[target_name], dtype=np.float32)
|
|
||||||
xy_distance = float(np.linalg.norm(source[:2] - target[:2]))
|
|
||||||
z_delta = float(source[2] - target[2])
|
|
||||||
heuristic_success = bool(xy_distance < 0.12 and z_delta > -0.08)
|
|
||||||
return {
|
|
||||||
"has_builtin_success": False,
|
|
||||||
"heuristic_success": heuristic_success,
|
|
||||||
"xy_distance": xy_distance,
|
|
||||||
"z_delta": z_delta,
|
|
||||||
"source_object": source_name,
|
|
||||||
"target_object": target_name,
|
|
||||||
"reason": "no built-in env success flag; using final object pose heuristic",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _dump_json(path: Path, payload: dict[str, Any]) -> None:
|
|
||||||
"""Write a JSON file with stable UTF-8 formatting."""
|
|
||||||
|
|
||||||
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
||||||
|
|
||||||
|
|
||||||
def _scalar_from_env_value(value: Any) -> float:
|
|
||||||
"""Normalize simulator scalar outputs into a plain Python float.
|
|
||||||
|
|
||||||
Isaac Lab / Gym values may come back as tensors, numpy arrays, tuples, or
|
|
||||||
direct Python scalars depending on the wrapper stack. Centralizing the
|
|
||||||
conversion here makes the rollout loop cleaner and more robust.
|
|
||||||
"""
|
|
||||||
|
|
||||||
if isinstance(value, torch.Tensor):
|
|
||||||
return float(value.reshape(-1)[0].detach().cpu().item())
|
|
||||||
if isinstance(value, np.ndarray):
|
|
||||||
return float(value.reshape(-1)[0])
|
|
||||||
if isinstance(value, (list, tuple)):
|
|
||||||
return float(np.asarray(value).reshape(-1)[0])
|
|
||||||
return float(value)
|
|
||||||
|
|
||||||
|
|
||||||
def _bool_from_env_value(value: Any) -> bool:
|
|
||||||
"""Normalize simulator boolean-like outputs into a plain Python bool."""
|
|
||||||
|
|
||||||
if isinstance(value, torch.Tensor):
|
|
||||||
return bool(value.reshape(-1)[0].detach().cpu().item())
|
|
||||||
if isinstance(value, np.ndarray):
|
|
||||||
return bool(value.reshape(-1)[0])
|
|
||||||
if isinstance(value, (list, tuple)):
|
|
||||||
return bool(np.asarray(value).reshape(-1)[0])
|
|
||||||
return bool(value)
|
|
||||||
|
|
||||||
|
|
||||||
def _make_output_dir(output_root: Path, scene: int) -> Path:
|
|
||||||
"""Create a timestamped output directory for one sim-eval run."""
|
|
||||||
|
|
||||||
timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
|
||||||
output_dir = output_root / f"scene{scene}_{timestamp}"
|
|
||||||
output_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
return output_dir
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
"""Entry point for one or more DROID simulation rollouts.
|
|
||||||
|
|
||||||
High-level flow:
|
|
||||||
1. parse command-line flags
|
|
||||||
2. bootstrap Isaac Lab / sim-evals imports
|
|
||||||
3. create the DROID environment
|
|
||||||
4. connect the DreamZero websocket client
|
|
||||||
5. run `episodes` rollouts
|
|
||||||
6. export videos + JSON summaries
|
|
||||||
"""
|
|
||||||
|
|
||||||
# Script-level arguments.
|
|
||||||
parser = argparse.ArgumentParser(description=__doc__)
|
|
||||||
parser.add_argument("--episodes", type=int, default=1, help="Number of episodes to run.")
|
|
||||||
parser.add_argument("--scene", type=int, default=1, help="DROID scene id (1/2/3).")
|
|
||||||
parser.add_argument("--host", type=str, default="127.0.0.1", help="vLLM DreamZero server host.")
|
|
||||||
parser.add_argument("--port", type=int, default=8000, help="vLLM DreamZero server port.")
|
|
||||||
parser.add_argument("--path", type=str, default=DEFAULT_PATH, help="Websocket path suffix.")
|
|
||||||
parser.add_argument(
|
|
||||||
"--open-loop-horizon",
|
|
||||||
type=int,
|
|
||||||
default=DEFAULT_OPEN_LOOP_HORIZON,
|
|
||||||
help="How many actions to consume locally before requesting the next chunk.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--output-root",
|
|
||||||
type=Path,
|
|
||||||
default=DEFAULT_OUTPUT_ROOT,
|
|
||||||
help="Directory where videos and trajectory logs are stored.",
|
|
||||||
)
|
|
||||||
|
|
||||||
try:
|
|
||||||
from isaaclab.app import AppLauncher
|
|
||||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
|
||||||
raise ImportError(
|
|
||||||
"Optional DROID sim-eval client requires Isaac Lab (`isaaclab`). "
|
|
||||||
"Launch it from an Isaac Lab environment, e.g. via `isaaclab.sh -p`."
|
|
||||||
) from exc
|
|
||||||
|
|
||||||
# Let Isaac Lab inject its own runtime flags (e.g. headless, device).
|
|
||||||
AppLauncher.add_app_launcher_args(parser)
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
# DreamZero sim-eval always needs camera observations enabled.
|
|
||||||
args.enable_cameras = True
|
|
||||||
# Boot Isaac Sim / Isaac Lab.
|
|
||||||
app_launcher = AppLauncher(args)
|
|
||||||
simulation_app = app_launcher.app
|
|
||||||
# Set defaults so the `finally` block can clean up safely even if an
|
|
||||||
# earlier step fails.
|
|
||||||
env = None
|
|
||||||
client = None
|
|
||||||
|
|
||||||
# Import simulator modules only *after* the app is launched. This matches
|
|
||||||
# Isaac Lab's required import ordering.
|
|
||||||
try:
|
|
||||||
import gymnasium as gym
|
|
||||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
|
||||||
raise ImportError("Optional DROID sim-eval client requires `gymnasium`.") from exc
|
|
||||||
|
|
||||||
try:
|
|
||||||
import sim_evals.environments # noqa: F401
|
|
||||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
|
||||||
raise ImportError(
|
|
||||||
"Optional DROID sim-eval client requires the external `sim-evals` package or checkout to be importable."
|
|
||||||
) from exc
|
|
||||||
|
|
||||||
try:
|
|
||||||
from isaaclab_tasks.utils import parse_env_cfg
|
|
||||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
|
||||||
raise ImportError("Optional DROID sim-eval client requires `isaaclab_tasks`.") from exc
|
|
||||||
|
|
||||||
# Resolve output location and scene prompt.
|
|
||||||
output_dir = _make_output_dir(args.output_root.expanduser().resolve(), args.scene)
|
|
||||||
instruction = _scene_instruction(args.scene)
|
|
||||||
|
|
||||||
# Build the DROID environment configuration from `sim-evals`.
|
|
||||||
env_cfg = parse_env_cfg(
|
|
||||||
"DROID",
|
|
||||||
device=args.device,
|
|
||||||
num_envs=1,
|
|
||||||
use_fabric=True,
|
|
||||||
)
|
|
||||||
# Select one of the pre-authored scenes/tasks.
|
|
||||||
env_cfg.set_scene(args.scene)
|
|
||||||
env = gym.make("DROID", cfg=env_cfg)
|
|
||||||
|
|
||||||
# Upstream sim-evals resets twice so materials / cameras are fully ready.
|
|
||||||
obs, _ = env.reset()
|
|
||||||
obs, _ = env.reset()
|
|
||||||
|
|
||||||
# Connect the websocket policy client.
|
|
||||||
client = DreamZeroJointPosClient(
|
|
||||||
remote_host=args.host,
|
|
||||||
remote_port=args.port,
|
|
||||||
path=args.path,
|
|
||||||
open_loop_horizon=args.open_loop_horizon,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Aggregated per-run results.
|
|
||||||
all_episode_summaries: list[dict[str, Any]] = []
|
|
||||||
max_steps = int(env.env.max_episode_length)
|
|
||||||
logging.info("DreamZero metadata: %s", client.metadata())
|
|
||||||
logging.info("Scene %s prompt: %s", args.scene, instruction)
|
|
||||||
logging.info("Writing outputs to %s", output_dir)
|
|
||||||
|
|
||||||
try:
|
|
||||||
# No gradients are needed in inference-only rollout mode.
|
|
||||||
with torch.no_grad():
|
|
||||||
for episode_index in range(args.episodes):
|
|
||||||
# Per-episode collectors.
|
|
||||||
frames: list[np.ndarray] = []
|
|
||||||
step_records: list[StepRecord] = []
|
|
||||||
episode_start = time.perf_counter()
|
|
||||||
server_time_s = 0.0
|
|
||||||
final_reward = 0.0
|
|
||||||
terminated = False
|
|
||||||
truncated = False
|
|
||||||
|
|
||||||
for step_index in range(max_steps):
|
|
||||||
# Ask the policy for the next action. Internally this may or
|
|
||||||
# may not trigger a real server request depending on whether
|
|
||||||
# the local chunk cache has been exhausted.
|
|
||||||
logging.debug("Episode %d step %d: requesting action", episode_index, step_index)
|
|
||||||
result = client.infer(obs, instruction)
|
|
||||||
logging.debug(
|
|
||||||
"Episode %d step %d: got action (server_call=%s latency=%s)",
|
|
||||||
episode_index,
|
|
||||||
step_index,
|
|
||||||
result["used_server_call"],
|
|
||||||
result["chunk_latency_s"],
|
|
||||||
)
|
|
||||||
# Save one visualization frame per simulator step.
|
|
||||||
frames.append(result["viz"])
|
|
||||||
|
|
||||||
# Isaac Lab expects batched actions, hence `[None]`.
|
|
||||||
action_tensor = torch.tensor(result["action"], dtype=torch.float32)[None]
|
|
||||||
logging.debug("Episode %d step %d: stepping env", episode_index, step_index)
|
|
||||||
obs, reward, term, trunc, info = env.step(action_tensor)
|
|
||||||
logging.debug("Episode %d step %d: env.step returned", episode_index, step_index)
|
|
||||||
logging.debug("Episode %d step %d: parsing reward/flags", episode_index, step_index)
|
|
||||||
logging.debug(
|
|
||||||
"Episode %d step %d: raw types reward=%s term=%s trunc=%s",
|
|
||||||
episode_index,
|
|
||||||
step_index,
|
|
||||||
type(reward).__name__,
|
|
||||||
type(term).__name__,
|
|
||||||
type(trunc).__name__,
|
|
||||||
)
|
|
||||||
# Normalize environment outputs into plain Python scalars so
|
|
||||||
# the rest of the code does not depend on wrapper-specific types.
|
|
||||||
reward_value = _scalar_from_env_value(reward)
|
|
||||||
term_value = _bool_from_env_value(term)
|
|
||||||
trunc_value = _bool_from_env_value(trunc)
|
|
||||||
logging.debug(
|
|
||||||
"Episode %d step %d: parsed reward=%s term=%s trunc=%s",
|
|
||||||
episode_index,
|
|
||||||
step_index,
|
|
||||||
reward_value,
|
|
||||||
term_value,
|
|
||||||
trunc_value,
|
|
||||||
)
|
|
||||||
# Keep scene-object capture optional. It is useful for
|
|
||||||
# debugging / success heuristics, but the rollout should not
|
|
||||||
# fail if the environment does not expose object roots.
|
|
||||||
scene_objects = _capture_scene_objects(env)
|
|
||||||
|
|
||||||
# Accumulate total server-side time only on steps that
|
|
||||||
# triggered a fresh chunk inference.
|
|
||||||
if result["chunk_latency_s"] is not None:
|
|
||||||
server_time_s += float(result["chunk_latency_s"])
|
|
||||||
|
|
||||||
# Materialize one JSON-serializable trajectory record.
|
|
||||||
logging.debug("Episode %d step %d: appending trajectory", episode_index, step_index)
|
|
||||||
step_records.append(
|
|
||||||
StepRecord(
|
|
||||||
step_index=step_index,
|
|
||||||
used_server_call=bool(result["used_server_call"]),
|
|
||||||
chunk_latency_s=(
|
|
||||||
float(result["chunk_latency_s"]) if result["chunk_latency_s"] is not None else None
|
|
||||||
),
|
|
||||||
action=[float(x) for x in np.asarray(result["action"], dtype=np.float32).tolist()],
|
|
||||||
joint_position=[
|
|
||||||
float(x) for x in np.asarray(result["joint_position"], dtype=np.float32).tolist()
|
|
||||||
],
|
|
||||||
gripper_position=[
|
|
||||||
float(x) for x in np.asarray(result["gripper_position"], dtype=np.float32).tolist()
|
|
||||||
],
|
|
||||||
reward=reward_value,
|
|
||||||
terminated=term_value,
|
|
||||||
truncated=trunc_value,
|
|
||||||
scene_objects=scene_objects,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
logging.debug("Episode %d step %d: trajectory appended", episode_index, step_index)
|
|
||||||
|
|
||||||
# Track final status for summary export.
|
|
||||||
final_reward = reward_value
|
|
||||||
terminated = term_value
|
|
||||||
truncated = trunc_value
|
|
||||||
if term_value or trunc_value:
|
|
||||||
# End the rollout early if the environment terminates.
|
|
||||||
break
|
|
||||||
|
|
||||||
# Episode-level timing and video export.
|
|
||||||
episode_wall_time_s = time.perf_counter() - episode_start
|
|
||||||
video_path = output_dir / f"episode_{episode_index:02d}.mp4"
|
|
||||||
logging.info("Episode %d: writing video to %s", episode_index, video_path)
|
|
||||||
mediapy.write_video(video_path, frames, fps=15)
|
|
||||||
|
|
||||||
# Reset the policy server between episodes.
|
|
||||||
logging.info("Episode %d: sending reset", episode_index)
|
|
||||||
reset_response = client.reset()
|
|
||||||
final_objects = step_records[-1].scene_objects if step_records else {}
|
|
||||||
success_report = _maybe_infer_success(args.scene, final_objects)
|
|
||||||
|
|
||||||
# Assemble the per-episode summary that is written to
|
|
||||||
# `episode_XX.json`.
|
|
||||||
episode_summary = {
|
|
||||||
"episode_index": episode_index,
|
|
||||||
"prompt": instruction,
|
|
||||||
"video_path": str(video_path),
|
|
||||||
"steps_executed": len(step_records),
|
|
||||||
"max_steps": max_steps,
|
|
||||||
"terminated": terminated,
|
|
||||||
"truncated": truncated,
|
|
||||||
"final_reward": final_reward,
|
|
||||||
"server_calls": client.server_calls,
|
|
||||||
"server_time_s": server_time_s,
|
|
||||||
"episode_wall_time_s": episode_wall_time_s,
|
|
||||||
"avg_server_time_per_call_s": (
|
|
||||||
server_time_s / client.server_calls if client.server_calls else None
|
|
||||||
),
|
|
||||||
"reset_response": reset_response,
|
|
||||||
"success_report": success_report,
|
|
||||||
"server_metadata": client.metadata(),
|
|
||||||
"trajectory": [record.__dict__ for record in step_records],
|
|
||||||
}
|
|
||||||
_dump_json(output_dir / f"episode_{episode_index:02d}.json", episode_summary)
|
|
||||||
all_episode_summaries.append(episode_summary)
|
|
||||||
|
|
||||||
logging.info(
|
|
||||||
"Episode %d done: steps=%d wall=%.2fs server_calls=%d heuristic_success=%s",
|
|
||||||
episode_index,
|
|
||||||
len(step_records),
|
|
||||||
episode_wall_time_s,
|
|
||||||
client.server_calls,
|
|
||||||
success_report.get("heuristic_success"),
|
|
||||||
)
|
|
||||||
|
|
||||||
# Reset per-episode counters while keeping the client alive.
|
|
||||||
client.server_calls = 0
|
|
||||||
|
|
||||||
# Top-level run summary across all episodes.
|
|
||||||
summary = {
|
|
||||||
"scene": args.scene,
|
|
||||||
"prompt": instruction,
|
|
||||||
"episodes": args.episodes,
|
|
||||||
"host": args.host,
|
|
||||||
"port": args.port,
|
|
||||||
"path": args.path,
|
|
||||||
"device": args.device,
|
|
||||||
"output_dir": str(output_dir),
|
|
||||||
"server_metadata": client.metadata(),
|
|
||||||
"episodes_summary": all_episode_summaries,
|
|
||||||
}
|
|
||||||
_dump_json(output_dir / "summary.json", summary)
|
|
||||||
# Also print the summary to stdout so the caller can capture it in logs.
|
|
||||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
|
||||||
finally:
|
|
||||||
# Best-effort cleanup. Avoid masking the main error if cleanup fails.
|
|
||||||
try:
|
|
||||||
if client is not None:
|
|
||||||
client.client.close()
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
if env is not None:
|
|
||||||
env.close()
|
|
||||||
simulation_app.close()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
# Keep the script-level logs readable. Per-step rollout details are still
|
|
||||||
# available via `DEBUG` if needed, but websocket / asyncio internals are
|
|
||||||
# usually too noisy for normal usage.
|
|
||||||
logging.basicConfig(level=logging.INFO)
|
|
||||||
logging.getLogger("websockets").setLevel(logging.WARNING)
|
|
||||||
logging.getLogger("asyncio").setLevel(logging.WARNING)
|
|
||||||
main()
|
|
||||||
@@ -1,86 +0,0 @@
|
|||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
os.environ.setdefault("MUJOCO_GL", "egl")
|
|
||||||
os.environ.setdefault("PYOPENGL_PLATFORM", "egl")
|
|
||||||
|
|
||||||
# Import base configs at module top level so the subclasses below are pickle-
|
|
||||||
# resolvable (worker processes import this module fresh via __main__).
|
|
||||||
from molmo_spaces.configs.policy_configs_baselines import ( # noqa: E402
|
|
||||||
DreamZeroPolicyConfig,
|
|
||||||
)
|
|
||||||
from molmo_spaces.evaluation.configs.evaluation_configs import ( # noqa: E402
|
|
||||||
DreamZeroPolicyEvalConfig,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
# We only need to change the backend host and port to the vllm-host!
|
|
||||||
class DreamZeroVllmOmniPolicyConfig(DreamZeroPolicyConfig):
|
|
||||||
remote_config: dict = dict(host="127.0.0.1", port=8000)
|
|
||||||
|
|
||||||
|
|
||||||
class DreamZeroVllmOmniEvalConfig(DreamZeroPolicyEvalConfig):
|
|
||||||
policy_config: DreamZeroVllmOmniPolicyConfig = DreamZeroVllmOmniPolicyConfig()
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
parser = argparse.ArgumentParser()
|
|
||||||
parser.add_argument("--host", default="127.0.0.1")
|
|
||||||
parser.add_argument("--port", type=int, default=8000)
|
|
||||||
parser.add_argument(
|
|
||||||
"--benchmark_dir",
|
|
||||||
required=True,
|
|
||||||
help=(
|
|
||||||
"Path to a MolmoSpaces benchmark directory, for example "
|
|
||||||
"$MOLMOSPACES_BENCHMARK_DIR/20260327/ithor/FrankaCloseHardBench/"
|
|
||||||
"FrankaCloseHardBench_20260206_json_benchmark"
|
|
||||||
),
|
|
||||||
)
|
|
||||||
parser.add_argument("--max_episodes", type=int, default=1)
|
|
||||||
parser.add_argument("--task_horizon_steps", type=int, default=80)
|
|
||||||
parser.add_argument(
|
|
||||||
"--output_dir",
|
|
||||||
required=True,
|
|
||||||
help="Directory to write evaluation outputs (created if missing).",
|
|
||||||
)
|
|
||||||
parser.add_argument("--episode_idx", type=int, default=None)
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
policy_config = DreamZeroVllmOmniPolicyConfig(remote_config=dict(host=args.host, port=args.port))
|
|
||||||
DreamZeroVllmOmniPolicyConfig.model_fields["remote_config"].default = policy_config.remote_config
|
|
||||||
DreamZeroVllmOmniEvalConfig.model_fields["policy_config"].default = policy_config
|
|
||||||
|
|
||||||
# Import after env vars are set so MuJoCo picks EGL.
|
|
||||||
from molmo_spaces.evaluation import run_evaluation
|
|
||||||
|
|
||||||
cfg_cls = DreamZeroVllmOmniEvalConfig
|
|
||||||
|
|
||||||
output_dir = args.output_dir
|
|
||||||
Path(output_dir).mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
print(f"[eval] benchmark_dir={args.benchmark_dir}")
|
|
||||||
print(f"[eval] max_episodes={args.max_episodes} task_horizon_steps={args.task_horizon_steps}")
|
|
||||||
print(f"[eval] remote policy: ws://{args.host}:{args.port}/v1/realtime/robot/openpi")
|
|
||||||
|
|
||||||
results = run_evaluation(
|
|
||||||
eval_config_cls=cfg_cls,
|
|
||||||
benchmark_dir=Path(args.benchmark_dir),
|
|
||||||
max_episodes=args.max_episodes,
|
|
||||||
task_horizon_steps=args.task_horizon_steps,
|
|
||||||
num_workers=1,
|
|
||||||
use_wandb=False,
|
|
||||||
output_dir=output_dir,
|
|
||||||
episode_idx=args.episode_idx,
|
|
||||||
)
|
|
||||||
|
|
||||||
print(f"[eval] success={results.success_count}/{results.total_count} ({results.success_rate:.1%})")
|
|
||||||
print(f"[eval] output_dir={results.output_dir}")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
@@ -1,311 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import json
|
|
||||||
import logging
|
|
||||||
import sys
|
|
||||||
import uuid
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
try:
|
|
||||||
import cv2
|
|
||||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
|
||||||
raise ImportError("DreamZero OpenPI example requires `opencv-python`.") from exc
|
|
||||||
|
|
||||||
try:
|
|
||||||
example_dir = str(Path(__file__).resolve().parent)
|
|
||||||
removed_path = False
|
|
||||||
if sys.path and sys.path[0] == example_dir:
|
|
||||||
sys.path.pop(0)
|
|
||||||
removed_path = True
|
|
||||||
try:
|
|
||||||
from openpi_client.websocket_client_policy import WebsocketClientPolicy
|
|
||||||
finally:
|
|
||||||
if removed_path:
|
|
||||||
sys.path.insert(0, example_dir)
|
|
||||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
|
||||||
raise ImportError("DreamZero OpenPI example requires `openpi-client`.") from exc
|
|
||||||
|
|
||||||
DEFAULT_HOST = "127.0.0.1"
|
|
||||||
DEFAULT_PORT = 8000
|
|
||||||
DEFAULT_PATH = "/v1/realtime/robot/openpi"
|
|
||||||
DEFAULT_PROMPT = "Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan"
|
|
||||||
ACTION_HORIZON = 24
|
|
||||||
DEFAULT_ACTION_DIM = 8
|
|
||||||
RELATIVE_OFFSETS = [-23, -16, -8, 0]
|
|
||||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
|
||||||
ASSET_REPO_ID = "YangshenDeng/vllm-omni-dreamzero-assets"
|
|
||||||
DEFAULT_VIDEO_DIR = REPO_ROOT / "outputs" / "dreamzero" / "assets"
|
|
||||||
CAMERA_FILES = {
|
|
||||||
"observation/exterior_image_0_left": "exterior_image_1_left.mp4",
|
|
||||||
"observation/exterior_image_1_left": "exterior_image_2_left.mp4",
|
|
||||||
"observation/wrist_image_left": "wrist_image_left.mp4",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _as_action_array(response: Any) -> np.ndarray:
|
|
||||||
if isinstance(response, dict) and response.get("type") == "error":
|
|
||||||
message = response.get("message", response)
|
|
||||||
raise RuntimeError(f"Inference failed: {message}")
|
|
||||||
return np.asarray(response, dtype=np.float32)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class DreamZeroServerMetadata:
|
|
||||||
image_resolution: tuple[int, int]
|
|
||||||
n_external_cameras: int
|
|
||||||
needs_wrist_camera: bool
|
|
||||||
needs_stereo_camera: bool
|
|
||||||
needs_session_id: bool
|
|
||||||
action_space: str
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_dict(cls, payload: dict[str, Any]) -> DreamZeroServerMetadata:
|
|
||||||
required_keys = (
|
|
||||||
"image_resolution",
|
|
||||||
"n_external_cameras",
|
|
||||||
"needs_wrist_camera",
|
|
||||||
"needs_stereo_camera",
|
|
||||||
"needs_session_id",
|
|
||||||
"action_space",
|
|
||||||
)
|
|
||||||
missing_keys = [key for key in required_keys if key not in payload]
|
|
||||||
if missing_keys:
|
|
||||||
raise ValueError(f"Missing DreamZero metadata keys: {missing_keys}")
|
|
||||||
|
|
||||||
image_resolution = payload["image_resolution"]
|
|
||||||
if not isinstance(image_resolution, (list, tuple)) or len(image_resolution) != 2:
|
|
||||||
raise ValueError(f"Invalid image_resolution: {image_resolution!r}")
|
|
||||||
|
|
||||||
return cls(
|
|
||||||
image_resolution=(int(image_resolution[0]), int(image_resolution[1])),
|
|
||||||
n_external_cameras=int(payload["n_external_cameras"]),
|
|
||||||
needs_wrist_camera=bool(payload["needs_wrist_camera"]),
|
|
||||||
needs_stereo_camera=bool(payload["needs_stereo_camera"]),
|
|
||||||
needs_session_id=bool(payload["needs_session_id"]),
|
|
||||||
action_space=str(payload["action_space"]),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _openpi_uri(host: str, port: int, path: str) -> str:
|
|
||||||
if host.startswith(("ws://", "wss://")):
|
|
||||||
return host
|
|
||||||
return f"ws://{host}:{port}{path}"
|
|
||||||
|
|
||||||
|
|
||||||
def _close_policy_client(client: WebsocketClientPolicy) -> None:
|
|
||||||
ws = getattr(client, "_ws", None)
|
|
||||||
if ws is not None:
|
|
||||||
ws.close()
|
|
||||||
|
|
||||||
|
|
||||||
def load_all_frames(video_path: Path) -> np.ndarray:
|
|
||||||
cap = cv2.VideoCapture(str(video_path))
|
|
||||||
frames = []
|
|
||||||
while True:
|
|
||||||
ok, frame = cap.read()
|
|
||||||
if not ok:
|
|
||||||
break
|
|
||||||
frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
|
|
||||||
cap.release()
|
|
||||||
if not frames:
|
|
||||||
raise RuntimeError(f"No frames loaded from {video_path}")
|
|
||||||
return np.stack(frames, axis=0)
|
|
||||||
|
|
||||||
|
|
||||||
def load_camera_frames(video_dir: Path) -> dict[str, np.ndarray]:
|
|
||||||
camera_frames: dict[str, np.ndarray] = {}
|
|
||||||
for camera_key, file_name in CAMERA_FILES.items():
|
|
||||||
video_path = video_dir / file_name
|
|
||||||
if not video_path.exists():
|
|
||||||
raise FileNotFoundError(
|
|
||||||
f"Missing DreamZero example asset: {video_path}. "
|
|
||||||
"Download the example videos with: "
|
|
||||||
f"`hf download {ASSET_REPO_ID} --repo-type dataset --local-dir {video_dir}`"
|
|
||||||
)
|
|
||||||
camera_frames[camera_key] = load_all_frames(video_path)
|
|
||||||
return camera_frames
|
|
||||||
|
|
||||||
|
|
||||||
def build_frame_schedule(total_frames: int, num_chunks: int) -> list[list[int]]:
|
|
||||||
chunks: list[list[int]] = []
|
|
||||||
current_frame = 23
|
|
||||||
for _ in range(num_chunks):
|
|
||||||
indices = [max(current_frame + offset, 0) for offset in RELATIVE_OFFSETS]
|
|
||||||
if indices[-1] >= total_frames:
|
|
||||||
break
|
|
||||||
chunks.append(indices)
|
|
||||||
current_frame += ACTION_HORIZON
|
|
||||||
return chunks
|
|
||||||
|
|
||||||
|
|
||||||
def make_obs_from_video(
|
|
||||||
camera_frames: dict[str, np.ndarray],
|
|
||||||
frame_indices: list[int],
|
|
||||||
*,
|
|
||||||
prompt: str,
|
|
||||||
session_id: str,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
obs: dict[str, Any] = {}
|
|
||||||
for camera_key, all_frames in camera_frames.items():
|
|
||||||
selected = all_frames[frame_indices]
|
|
||||||
obs[camera_key] = selected[0] if len(frame_indices) == 1 else selected
|
|
||||||
|
|
||||||
obs["observation/joint_position"] = np.zeros(7, dtype=np.float32)
|
|
||||||
obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32)
|
|
||||||
obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32)
|
|
||||||
obs["prompt"] = prompt
|
|
||||||
obs["session_id"] = session_id
|
|
||||||
return obs
|
|
||||||
|
|
||||||
|
|
||||||
def build_demo_observations(
|
|
||||||
camera_frames: dict[str, np.ndarray],
|
|
||||||
*,
|
|
||||||
prompt: str,
|
|
||||||
session_id: str,
|
|
||||||
num_chunks: int = 2,
|
|
||||||
) -> list[dict[str, Any]]:
|
|
||||||
if num_chunks < 1:
|
|
||||||
raise ValueError("num_chunks must be at least 1")
|
|
||||||
|
|
||||||
total_frames = min(frames.shape[0] for frames in camera_frames.values())
|
|
||||||
observations = [
|
|
||||||
make_obs_from_video(
|
|
||||||
camera_frames,
|
|
||||||
[0],
|
|
||||||
prompt=prompt,
|
|
||||||
session_id=session_id,
|
|
||||||
)
|
|
||||||
]
|
|
||||||
for indices in build_frame_schedule(total_frames, num_chunks - 1):
|
|
||||||
observations.append(
|
|
||||||
make_obs_from_video(
|
|
||||||
camera_frames,
|
|
||||||
indices,
|
|
||||||
prompt=prompt,
|
|
||||||
session_id=session_id,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return observations
|
|
||||||
|
|
||||||
|
|
||||||
def validate_session_result(
|
|
||||||
result: dict[str, Any],
|
|
||||||
*,
|
|
||||||
expected_action_horizon: int = ACTION_HORIZON,
|
|
||||||
expected_action_dim: int = DEFAULT_ACTION_DIM,
|
|
||||||
expected_action_count: int | None = None,
|
|
||||||
) -> None:
|
|
||||||
metadata = DreamZeroServerMetadata.from_dict(result["metadata"])
|
|
||||||
if metadata.image_resolution != (180, 320):
|
|
||||||
raise AssertionError(f"Unexpected image_resolution: {metadata.image_resolution}")
|
|
||||||
if metadata.n_external_cameras != 2:
|
|
||||||
raise AssertionError(f"Unexpected n_external_cameras: {metadata.n_external_cameras}")
|
|
||||||
if not metadata.needs_wrist_camera:
|
|
||||||
raise AssertionError("DreamZero example expects wrist camera metadata")
|
|
||||||
if metadata.action_space != "joint_position":
|
|
||||||
raise AssertionError(f"Unexpected action_space: {metadata.action_space}")
|
|
||||||
|
|
||||||
actions = result["actions"]
|
|
||||||
if expected_action_count is not None and len(actions) != expected_action_count:
|
|
||||||
raise AssertionError(f"Expected {expected_action_count} action tensors, got {len(actions)}")
|
|
||||||
if not actions:
|
|
||||||
raise AssertionError("Expected at least one action tensor")
|
|
||||||
for index, action in enumerate(actions):
|
|
||||||
if action.shape != (expected_action_horizon, expected_action_dim):
|
|
||||||
raise AssertionError(
|
|
||||||
f"Action {index} shape mismatch: expected "
|
|
||||||
f"{(expected_action_horizon, expected_action_dim)}, got {action.shape}"
|
|
||||||
)
|
|
||||||
if not np.isfinite(action).all():
|
|
||||||
raise AssertionError(f"Action {index} contains non-finite values")
|
|
||||||
|
|
||||||
|
|
||||||
def run_policy_session(
|
|
||||||
*,
|
|
||||||
host: str = DEFAULT_HOST,
|
|
||||||
port: int = DEFAULT_PORT,
|
|
||||||
path: str = DEFAULT_PATH,
|
|
||||||
video_dir: Path = DEFAULT_VIDEO_DIR,
|
|
||||||
prompt: str = DEFAULT_PROMPT,
|
|
||||||
session_id: str | None = None,
|
|
||||||
num_chunks: int = 2,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
session_id = session_id or str(uuid.uuid4())
|
|
||||||
camera_frames = load_camera_frames(video_dir)
|
|
||||||
observations = build_demo_observations(
|
|
||||||
camera_frames,
|
|
||||||
prompt=prompt,
|
|
||||||
session_id=session_id,
|
|
||||||
num_chunks=num_chunks,
|
|
||||||
)
|
|
||||||
|
|
||||||
uri = _openpi_uri(host, port, path)
|
|
||||||
logging.info("Connecting to %s", uri)
|
|
||||||
client = WebsocketClientPolicy(host=uri)
|
|
||||||
try:
|
|
||||||
metadata = dict(client.get_server_metadata())
|
|
||||||
actions = [_as_action_array(client.infer(obs)) for obs in observations]
|
|
||||||
return {
|
|
||||||
"metadata": metadata,
|
|
||||||
"actions": actions,
|
|
||||||
"session_id": session_id,
|
|
||||||
}
|
|
||||||
finally:
|
|
||||||
_close_policy_client(client)
|
|
||||||
|
|
||||||
|
|
||||||
def format_action_summary(index: int, action: np.ndarray) -> str:
|
|
||||||
return (
|
|
||||||
f"Action {index}: shape={tuple(action.shape)} dtype={action.dtype} "
|
|
||||||
f"min={action.min():.6f} max={action.max():.6f}"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def parse_args() -> argparse.Namespace:
|
|
||||||
parser = argparse.ArgumentParser(description="DreamZero OpenPI client example with downloaded real videos.")
|
|
||||||
parser.add_argument("--host", default=DEFAULT_HOST)
|
|
||||||
parser.add_argument("--port", type=int, default=DEFAULT_PORT)
|
|
||||||
parser.add_argument("--path", default=DEFAULT_PATH)
|
|
||||||
parser.add_argument(
|
|
||||||
"--video-dir", type=Path, default=DEFAULT_VIDEO_DIR, help="Directory containing the three camera MP4 files."
|
|
||||||
)
|
|
||||||
parser.add_argument("--prompt", default=DEFAULT_PROMPT)
|
|
||||||
parser.add_argument("--session-id", default=None)
|
|
||||||
parser.add_argument("--num-chunks", type=int, default=2)
|
|
||||||
return parser.parse_args()
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
args = parse_args()
|
|
||||||
logging.basicConfig(level=logging.INFO)
|
|
||||||
|
|
||||||
result = run_policy_session(
|
|
||||||
host=args.host,
|
|
||||||
port=args.port,
|
|
||||||
path=args.path,
|
|
||||||
video_dir=args.video_dir,
|
|
||||||
prompt=args.prompt,
|
|
||||||
session_id=args.session_id,
|
|
||||||
num_chunks=args.num_chunks,
|
|
||||||
)
|
|
||||||
validate_session_result(result, expected_action_count=args.num_chunks)
|
|
||||||
|
|
||||||
print("Server metadata:", json.dumps(result["metadata"], sort_keys=True))
|
|
||||||
for index, action in enumerate(result["actions"]):
|
|
||||||
print(format_action_summary(index, action))
|
|
||||||
print("Session ID:", result["session_id"])
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
MODEL="${MODEL:-GEAR-Dreams/DreamZero-DROID}"
|
|
||||||
HOST="${HOST:-127.0.0.1}"
|
|
||||||
PORT="${PORT:-8000}"
|
|
||||||
DEPLOY_CONFIG="${DEPLOY_CONFIG:-vllm_omni/deploy/dreamzero_tp1_cfg2.yaml}"
|
|
||||||
SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-dreamzero-droid}"
|
|
||||||
|
|
||||||
args=(
|
|
||||||
serve
|
|
||||||
"$MODEL"
|
|
||||||
--omni
|
|
||||||
--host "$HOST"
|
|
||||||
--port "$PORT"
|
|
||||||
--served-model-name "$SERVED_MODEL_NAME"
|
|
||||||
--deploy-config "$DEPLOY_CONFIG"
|
|
||||||
--enforce-eager
|
|
||||||
--disable-log-stats
|
|
||||||
)
|
|
||||||
|
|
||||||
ATTENTION_BACKEND="${ATTENTION_BACKEND:-torch}" \
|
|
||||||
DIFFUSION_ATTENTION_BACKEND="${DIFFUSION_ATTENTION_BACKEND:-TORCH_SDPA}" \
|
|
||||||
vllm "${args[@]}"
|
|
||||||
@@ -22,14 +22,13 @@ export PYTHONPATH=<transformers_modules_path>:$PYTHONPATH
|
|||||||
export HF_MODULES_CACHE=<transformers_modules_path>
|
export HF_MODULES_CACHE=<transformers_modules_path>
|
||||||
```
|
```
|
||||||
|
|
||||||
Run from repository root. The 3-stage pipeline is resolved automatically from
|
Run from repository root:
|
||||||
the model type (`vllm_omni/deploy/dynin_omni.yaml`), so no config path is
|
|
||||||
required. Pass `--deploy-config` only to override the defaults:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
vllm-omni serve snu-aidas/Dynin-Omni \
|
vllm-omni serve snu-aidas/Dynin-Omni \
|
||||||
--omni \
|
--omni \
|
||||||
--port 8091
|
--port 8091 \
|
||||||
|
--stage-configs-path "$(pwd)/vllm_omni/model_executor/stage_configs/dynin_omni.yaml"
|
||||||
```
|
```
|
||||||
|
|
||||||
If `vllm-omni` is not in PATH, run:
|
If `vllm-omni` is not in PATH, run:
|
||||||
@@ -37,7 +36,8 @@ If `vllm-omni` is not in PATH, run:
|
|||||||
```bash
|
```bash
|
||||||
PYTHONPATH="$(pwd)" python -m vllm_omni.entrypoints.cli.main serve snu-aidas/Dynin-Omni \
|
PYTHONPATH="$(pwd)" python -m vllm_omni.entrypoints.cli.main serve snu-aidas/Dynin-Omni \
|
||||||
--omni \
|
--omni \
|
||||||
--port 8091
|
--port 8091 \
|
||||||
|
--stage-configs-path "$(pwd)/vllm_omni/model_executor/stage_configs/dynin_omni.yaml"
|
||||||
```
|
```
|
||||||
|
|
||||||
Wait until the server logs show both `All stages initialized successfully` and
|
Wait until the server logs show both `All stages initialized successfully` and
|
||||||
|
|||||||
@@ -198,14 +198,7 @@ def _build_prompt_for_query_type(
|
|||||||
return query_func(custom_prompt=custom_prompt)
|
return query_func(custom_prompt=custom_prompt)
|
||||||
if query_type == "use_audio_in_video":
|
if query_type == "use_audio_in_video":
|
||||||
return query_func(video_path=video_path, custom_prompt=custom_prompt)
|
return query_func(video_path=video_path, custom_prompt=custom_prompt)
|
||||||
if query_type == "use_mixed_modalities":
|
# use_mixed_modalities / use_multi_audios
|
||||||
return query_func(
|
|
||||||
video_path=video_path,
|
|
||||||
image_path=image_path,
|
|
||||||
audio_path=audio_path,
|
|
||||||
custom_prompt=custom_prompt,
|
|
||||||
)
|
|
||||||
# use_multi_audios
|
|
||||||
return query_func(custom_prompt=custom_prompt)
|
return query_func(custom_prompt=custom_prompt)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,40 +0,0 @@
|
|||||||
# Higgs-Audio V3 Online Serving
|
|
||||||
|
|
||||||
## Start the server
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Default: GPU 0, port 8095
|
|
||||||
./examples/online_serving/text_to_speech/higgs_audio_v3/run_server.sh
|
|
||||||
|
|
||||||
# Custom GPU / port
|
|
||||||
PORT=8096 GPUS=0,1 ./examples/online_serving/text_to_speech/higgs_audio_v3/run_server.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
## Plain text TTS
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python examples/online_serving/text_to_speech/higgs_audio_v3/batch_speech_client.py \
|
|
||||||
--base-url http://localhost:8095 \
|
|
||||||
--output-dir /tmp/higgs_v3_batch \
|
|
||||||
--prompts "Hello world." "The quick brown fox jumps over the lazy dog."
|
|
||||||
```
|
|
||||||
|
|
||||||
## Voice clone
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python examples/online_serving/text_to_speech/higgs_audio_v3/batch_speech_client.py \
|
|
||||||
--base-url http://localhost:8095 \
|
|
||||||
--output-dir /tmp/higgs_v3_clone \
|
|
||||||
--ref-audio path/to/reference.wav \
|
|
||||||
--ref-text "transcript of the reference clip" \
|
|
||||||
--prompts "Text to synthesize in the cloned voice."
|
|
||||||
```
|
|
||||||
|
|
||||||
## curl example
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl -X POST http://localhost:8095/v1/audio/speech \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{"model": "higgs_audio_v3", "input": "Hello world."}' \
|
|
||||||
--output hello.wav
|
|
||||||
```
|
|
||||||
@@ -1,115 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
"""Batch client for the higgs-audio v3 online server.
|
|
||||||
|
|
||||||
Sends prompts to ``/v1/audio/speech`` and saves the returned WAV files.
|
|
||||||
|
|
||||||
Usage (plain text -> speech):
|
|
||||||
|
|
||||||
python examples/online_serving/text_to_speech/higgs_audio_v3/batch_speech_client.py \
|
|
||||||
--base-url http://localhost:8095 \
|
|
||||||
--output-dir /tmp/higgs_v3_batch \
|
|
||||||
--prompts "Hello world." "The quick brown fox jumps over the lazy dog."
|
|
||||||
|
|
||||||
Usage (voice clone):
|
|
||||||
|
|
||||||
python examples/online_serving/text_to_speech/higgs_audio_v3/batch_speech_client.py \
|
|
||||||
--base-url http://localhost:8095 \
|
|
||||||
--output-dir /tmp/higgs_v3_clone \
|
|
||||||
--ref-audio path/to/reference.wav \
|
|
||||||
--ref-text "the transcript of the reference clip" \
|
|
||||||
--prompts "Hello world."
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import base64
|
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
DEFAULT_PROMPTS = (
|
|
||||||
"Hello world.",
|
|
||||||
"The quick brown fox jumps over the lazy dog.",
|
|
||||||
"Today is a beautiful day for a walk in the park.",
|
|
||||||
"Innovation distinguishes between a leader and a follower.",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _slug(text: str) -> str:
|
|
||||||
import re
|
|
||||||
|
|
||||||
s = re.sub(r"\s+", "_", text.strip().lower())
|
|
||||||
return re.sub(r"[^a-z0-9_]+", "", s)[:32] or "prompt"
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
||||||
parser.add_argument("--base-url", default="http://localhost:8095")
|
|
||||||
parser.add_argument("--model", default="higgs_audio_v3")
|
|
||||||
parser.add_argument("--prompts", nargs="+", default=list(DEFAULT_PROMPTS))
|
|
||||||
parser.add_argument("--output-dir", type=Path, default=Path("/tmp/higgs_v3_batch"))
|
|
||||||
parser.add_argument("--format", choices=("wav", "pcm"), default="wav")
|
|
||||||
parser.add_argument("--max-new-tokens", type=int, default=2048)
|
|
||||||
parser.add_argument("--seed", type=int, default=42)
|
|
||||||
parser.add_argument("--timeout-s", type=float, default=120.0)
|
|
||||||
parser.add_argument(
|
|
||||||
"--ref-audio",
|
|
||||||
type=Path,
|
|
||||||
default=None,
|
|
||||||
help="Reference clip for voice clone (WAV/FLAC/MP3 path). Pair with --ref-text.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--ref-text",
|
|
||||||
type=str,
|
|
||||||
default=None,
|
|
||||||
help="Transcript of the reference clip. Optional but improves fidelity.",
|
|
||||||
)
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
ref_audio_data_url: str | None = None
|
|
||||||
if args.ref_audio is not None:
|
|
||||||
if not args.ref_audio.exists():
|
|
||||||
print(f"ref-audio file not found: {args.ref_audio}", file=sys.stderr)
|
|
||||||
return 2
|
|
||||||
mime = "audio/wav" if args.ref_audio.suffix.lower() == ".wav" else "audio/mpeg"
|
|
||||||
ref_b64 = base64.b64encode(args.ref_audio.read_bytes()).decode("ascii")
|
|
||||||
ref_audio_data_url = f"data:{mime};base64,{ref_b64}"
|
|
||||||
|
|
||||||
try:
|
|
||||||
import httpx
|
|
||||||
except ImportError:
|
|
||||||
print("this client needs `httpx`. Install with `pip install httpx`.", file=sys.stderr)
|
|
||||||
return 2
|
|
||||||
|
|
||||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
url = args.base_url.rstrip("/") + "/v1/audio/speech"
|
|
||||||
failures = 0
|
|
||||||
with httpx.Client(timeout=args.timeout_s) as client:
|
|
||||||
for prompt in args.prompts:
|
|
||||||
payload = {
|
|
||||||
"model": args.model,
|
|
||||||
"input": prompt,
|
|
||||||
"response_format": args.format,
|
|
||||||
"max_new_tokens": args.max_new_tokens,
|
|
||||||
"seed": args.seed,
|
|
||||||
}
|
|
||||||
if ref_audio_data_url is not None:
|
|
||||||
payload["ref_audio"] = ref_audio_data_url
|
|
||||||
if args.ref_text:
|
|
||||||
payload["ref_text"] = args.ref_text
|
|
||||||
resp = client.post(url, json=payload)
|
|
||||||
if resp.status_code != 200:
|
|
||||||
print(f"[FAIL] {prompt!r} -> {resp.status_code}: {resp.text[:200]}", file=sys.stderr)
|
|
||||||
failures += 1
|
|
||||||
continue
|
|
||||||
suffix = ".wav" if args.format == "wav" else ".pcm"
|
|
||||||
out = args.output_dir / f"{_slug(prompt)}{suffix}"
|
|
||||||
out.write_bytes(resp.content)
|
|
||||||
print(f"[ ok ] {prompt!r} -> {out} ({len(resp.content)} bytes)")
|
|
||||||
|
|
||||||
return 1 if failures else 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
@@ -1,32 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
# Launch vLLM-Omni server for higgs-audio v3.
|
|
||||||
#
|
|
||||||
# Supports plain text TTS and voice cloning via /v1/audio/speech.
|
|
||||||
#
|
|
||||||
# Usage:
|
|
||||||
# ./run_server.sh # default port 8095, GPU 0
|
|
||||||
# PORT=8096 GPUS=0,1 ./run_server.sh
|
|
||||||
# MODEL=/path/to/local/checkpoint ./run_server.sh
|
|
||||||
|
|
||||||
set -e
|
|
||||||
|
|
||||||
MODEL="${MODEL:-bosonai/higgs-audio-v3-tts-4b}"
|
|
||||||
PORT="${PORT:-8095}"
|
|
||||||
GPUS="${GPUS:-0}"
|
|
||||||
GPU_UTIL="${GPU_UTIL:-0.6}"
|
|
||||||
|
|
||||||
echo "Starting higgs-audio v3 server"
|
|
||||||
echo " MODEL=$MODEL"
|
|
||||||
echo " PORT=$PORT"
|
|
||||||
echo " CUDA_VISIBLE_DEVICES=$GPUS"
|
|
||||||
|
|
||||||
CUDA_VISIBLE_DEVICES="$GPUS" \
|
|
||||||
VLLM_USE_DEEP_GEMM=0 \
|
|
||||||
VLLM_MOE_USE_DEEP_GEMM=0 \
|
|
||||||
vllm-omni serve "$MODEL" \
|
|
||||||
--deploy-config vllm_omni/deploy/higgs_multimodal_qwen3.yaml \
|
|
||||||
--host 0.0.0.0 \
|
|
||||||
--port "$PORT" \
|
|
||||||
--gpu-memory-utilization "$GPU_UTIL" \
|
|
||||||
--trust-remote-code \
|
|
||||||
--omni
|
|
||||||
+1
-3
@@ -64,9 +64,7 @@ dev = [
|
|||||||
"lpips>=0.1.4", # for quantization fp8
|
"lpips>=0.1.4", # for quantization fp8
|
||||||
"orjson>=3.11.7", # to make v1/embeddings API fast
|
"orjson>=3.11.7", # to make v1/embeddings API fast
|
||||||
"voxcpm>=2.0.2", # for VoxCPM2 path-locating in tests
|
"voxcpm>=2.0.2", # for VoxCPM2 path-locating in tests
|
||||||
"torchdiffeq>=0.2.5", # for covo_audio Test
|
"torchdiffeq>=0.2.5" # for covo_audio Test
|
||||||
"FlagEmbedding", # for hunyuan_image3 tests
|
|
||||||
"kernels<0.15.0" # for accuracy tests comparison with diffusers output (0.15.0 incompatible with diffusers 0.38.0)
|
|
||||||
]
|
]
|
||||||
|
|
||||||
demo = [
|
demo = [
|
||||||
|
|||||||
@@ -1,110 +0,0 @@
|
|||||||
# Higgs-Audio V3 TTS
|
|
||||||
|
|
||||||
> Multilingual text-to-speech with voice cloning on 1×H100
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
|
|
||||||
- Vendor: Boson AI
|
|
||||||
- Model: `bosonai/higgs-audio-v3-tts-4b`
|
|
||||||
- Task: Text-to-speech synthesis with optional voice cloning (100+ languages)
|
|
||||||
- Mode: Online serving with the OpenAI-compatible `/v1/audio/speech` API; offline batch inference
|
|
||||||
- Maintainer: @yuekaiz
|
|
||||||
|
|
||||||
## When to use this recipe
|
|
||||||
|
|
||||||
Use this recipe to serve `bosonai/higgs-audio-v3-tts-4b` for high-quality
|
|
||||||
multilingual TTS. The model generates 24 kHz speech, supports zero-shot voice
|
|
||||||
cloning from a reference clip, and handles 100+ languages with inline control
|
|
||||||
tokens for emotion, style, and prosody. The architecture is a ~4B Qwen3
|
|
||||||
backbone with fused multi-codebook embedding/head (8 codebooks × 1026 vocab,
|
|
||||||
MusicGen-style delay pattern).
|
|
||||||
|
|
||||||
## References
|
|
||||||
|
|
||||||
- Model card: [bosonai/higgs-audio-v3-tts-4b](https://huggingface.co/bosonai/higgs-audio-v3-tts-4b)
|
|
||||||
- Offline example: [`examples/offline_inference/text_to_speech/higgs_audio_v3/end2end.py`](../../examples/offline_inference/text_to_speech/higgs_audio_v3/end2end.py)
|
|
||||||
- Online example: [`examples/online_serving/text_to_speech/higgs_audio_v3/`](../../examples/online_serving/text_to_speech/higgs_audio_v3/)
|
|
||||||
- Benchmark results: see Performance section below
|
|
||||||
|
|
||||||
## Hardware Support
|
|
||||||
|
|
||||||
## GPU
|
|
||||||
|
|
||||||
### 1×H100 80GB
|
|
||||||
|
|
||||||
#### Environment
|
|
||||||
|
|
||||||
- OS: Linux
|
|
||||||
- Python: 3.12+
|
|
||||||
- CUDA: 12.x
|
|
||||||
- vLLM version: 0.22.0
|
|
||||||
- vLLM-Omni version or commit: `36e048fd` (branch `higgs-v3`)
|
|
||||||
|
|
||||||
#### Command
|
|
||||||
|
|
||||||
**Online serving:**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
vllm-omni serve bosonai/higgs-audio-v3-tts-4b \
|
|
||||||
--host 0.0.0.0 --port 8095 \
|
|
||||||
--trust-remote-code --omni
|
|
||||||
```
|
|
||||||
|
|
||||||
The default deploy config `vllm_omni/deploy/higgs_multimodal_qwen3.yaml` is
|
|
||||||
loaded automatically by model registry (HF `model_type=higgs_multimodal_qwen3`).
|
|
||||||
|
|
||||||
**Offline batch inference:**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python examples/offline_inference/text_to_speech/higgs_audio_v3/end2end.py \
|
|
||||||
--texts "Hello world." "The quick brown fox jumps over the lazy dog." \
|
|
||||||
--output-dir results/higgs_v3_wavs
|
|
||||||
```
|
|
||||||
|
|
||||||
**Offline voice clone:**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python examples/offline_inference/text_to_speech/higgs_audio_v3/end2end.py \
|
|
||||||
--texts "Text to synthesize in the cloned voice." \
|
|
||||||
--ref-audio path/to/reference.wav \
|
|
||||||
--ref-text "Transcript of the reference clip." \
|
|
||||||
--output-dir results/higgs_v3_clone
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Verification
|
|
||||||
|
|
||||||
Basic TTS via curl:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl -X POST http://localhost:8095/v1/audio/speech \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{
|
|
||||||
"model": "bosonai/higgs-audio-v3-tts-4b",
|
|
||||||
"input": "Hello, how are you?"
|
|
||||||
}' --output hello.wav
|
|
||||||
```
|
|
||||||
|
|
||||||
Voice clone via Python client:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python examples/online_serving/text_to_speech/higgs_audio_v3/batch_speech_client.py \
|
|
||||||
--base-url http://localhost:8095 \
|
|
||||||
--model bosonai/higgs-audio-v3-tts-4b \
|
|
||||||
--ref-audio path/to/reference.wav \
|
|
||||||
--ref-text "Transcript of the reference." \
|
|
||||||
--prompts "Text to clone."
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Notes
|
|
||||||
|
|
||||||
- Memory usage: Stage 0 (talker, ~4B) uses ~60% GPU memory; Stage 1 (codec decoder) uses ~25%.
|
|
||||||
- Key flags: `--trust-remote-code` and `--omni` are required.
|
|
||||||
- Output: 24 kHz mono WAV.
|
|
||||||
- Voice cloning: `ref_audio` accepts WAV/FLAC/MP3; `ref_text` is optional but improves fidelity.
|
|
||||||
- Deploy config: `vllm_omni/deploy/higgs_multimodal_qwen3.yaml` (auto-discovered from `model_type`).
|
|
||||||
- `max_num_seqs=16` for both stages.
|
|
||||||
- `enforce_eager=true` for both stages (CUDA graph available for stage 0 but no throughput gain at batch>1).
|
|
||||||
- Known limitations:
|
|
||||||
- CUDA graph for stage 0 helps batch=1 (-13% RTF) but not batch>1 due to model-owned sampler running outside graph.
|
|
||||||
- Stage 1 (code2wav) must use `enforce_eager=true` (`@torch.inference_mode` incompatible with graph capture).
|
|
||||||
- Async streaming (chunk-based) not yet implemented; sync-only pipeline.
|
|
||||||
+1
-2
@@ -36,8 +36,7 @@ recipes/
|
|||||||
| [`LTX/LTX-2.md`](./LTX/LTX-2.md) | Text-to-video and image-to-video serving | 1x H200 141GB |
|
| [`LTX/LTX-2.md`](./LTX/LTX-2.md) | Text-to-video and image-to-video serving | 1x H200 141GB |
|
||||||
| [`LTX/LTX-2.3.md`](./LTX/LTX-2.3.md) | Text-to-video with audio generation (22B) | 1x GPU (96GB VRAM) |
|
| [`LTX/LTX-2.3.md`](./LTX/LTX-2.3.md) | Text-to-video with audio generation (22B) | 1x GPU (96GB VRAM) |
|
||||||
| [`mistralai/Voxtral-TTS.md`](./mistralai/Voxtral-TTS.md) | Online serving for TTS | 1x RTX 4090 24GB |
|
| [`mistralai/Voxtral-TTS.md`](./mistralai/Voxtral-TTS.md) | Online serving for TTS | 1x RTX 4090 24GB |
|
||||||
| [`cosmos3/Cosmos3-Nano.md`](./cosmos3/Cosmos3-Nano.md) | Text-to-image, text-to-video, image-to-video generation, text to video with sound, action policy | 1x H200 141GB / B300 |
|
| [`nvidia/Cosmos3-Nano.md`](./nvidia/Cosmos3-Nano.md) | Text-to-image, text-to-video, and image-to-video generation | 1x H200 141GB / B300 |
|
||||||
| [`cosmos3/Cosmos3-Super.md`](./cosmos3/Cosmos3-Super.md) | 64B T2I / T2V / I2V generation (+ optional audio) / Action policy | 8x H200/H100/A100 / 2x H200 / B300 |
|
|
||||||
| [`OpenBMB/MiniCPM-o-4_5.md`](./OpenBMB/MiniCPM-o-4_5.md) | Online serving for omni multimodal chat (text / image / audio / video → text + 24 kHz speech) | 2x A100/H100 80GB / 3x mid-tier GPU / 8x RTX 4090 24GB |
|
| [`OpenBMB/MiniCPM-o-4_5.md`](./OpenBMB/MiniCPM-o-4_5.md) | Online serving for omni multimodal chat (text / image / audio / video → text + 24 kHz speech) | 2x A100/H100 80GB / 3x mid-tier GPU / 8x RTX 4090 24GB |
|
||||||
| [`OpenBMB/VoxCPM2.md`](./OpenBMB/VoxCPM2.md) | Online + offline TTS with native AR pipeline (48 kHz, 30+ languages) | 1x RTX 4090 24GB |
|
| [`OpenBMB/VoxCPM2.md`](./OpenBMB/VoxCPM2.md) | Online + offline TTS with native AR pipeline (48 kHz, 30+ languages) | 1x RTX 4090 24GB |
|
||||||
| [`Qwen/Qwen-Image.md`](./Qwen/Qwen-Image.md) | Text-to-image serving with step-wise continuous batching replay and ModelOpt mixed FP8/NVFP4 | 1x A100 80GB / 2x B200 |
|
| [`Qwen/Qwen-Image.md`](./Qwen/Qwen-Image.md) | Text-to-image serving with step-wise continuous batching replay and ModelOpt mixed FP8/NVFP4 | 1x A100 80GB / 2x B200 |
|
||||||
|
|||||||
@@ -1,112 +0,0 @@
|
|||||||
# Cosmos3-Super
|
|
||||||
|
|
||||||
> Frontier 64B world model: text-to-image, text-to-video, image-to-video (+ optional audio)
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
|
|
||||||
- Vendor: NVIDIA
|
|
||||||
- Model: `nvidia/Cosmos3-Super` (64B; also `Cosmos3-Super-Text2Image`, `Cosmos3-Super-Image2Video`)
|
|
||||||
- Task: T2I, T2V, I2V generation, with optional synchronized audio (video + sound)
|
|
||||||
- Mode: Online serving with the OpenAI-compatible image/video APIs
|
|
||||||
- Maintainer: Community
|
|
||||||
|
|
||||||
## When to use this recipe
|
|
||||||
|
|
||||||
Use this recipe to deploy the 64B `nvidia/Cosmos3-Super` for the highest-quality
|
|
||||||
Cosmos3 generation. It shares the same `Cosmos3OmniDiffusersPipeline` and request
|
|
||||||
formats as [Cosmos3-Nano](./Cosmos3-Nano.md) — only the checkpoint size and the
|
|
||||||
recommended parallelism differ. Mode is selected per request (T2I →
|
|
||||||
`/v1/images/generations`; T2V/I2V → `/v1/videos/sync`; add `generate_sound=true`
|
|
||||||
for audio).
|
|
||||||
|
|
||||||
## References
|
|
||||||
|
|
||||||
- Model card (authoritative usage + example assets): <https://huggingface.co/nvidia/Cosmos3-Super>
|
|
||||||
- Nano recipe (same APIs/params): [`Cosmos3-Nano.md`](./Cosmos3-Nano.md)
|
|
||||||
- Pipeline: [`vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py`](../../vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py)
|
|
||||||
|
|
||||||
## Hardware Support
|
|
||||||
|
|
||||||
## GPU
|
|
||||||
|
|
||||||
Requires the `vllm-omni` package (or the `vllm/vllm-omni:cosmos3` container),
|
|
||||||
which provides the `vllm serve … --omni` entrypoint used below.
|
|
||||||
|
|
||||||
### 8x H200/H100/A100 (recommended, per model card)
|
|
||||||
|
|
||||||
```bash
|
|
||||||
vllm serve nvidia/Cosmos3-Super \
|
|
||||||
--omni \
|
|
||||||
--host 0.0.0.0 --port 8000 \
|
|
||||||
--cfg-parallel-size 2 \
|
|
||||||
--ulysses-degree 4 \
|
|
||||||
--use-hsdp --hsdp-shard-size 8 \
|
|
||||||
--init-timeout 1800
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2x H200 / B300 (minimum)
|
|
||||||
|
|
||||||
```bash
|
|
||||||
vllm serve nvidia/Cosmos3-Super \
|
|
||||||
--omni \
|
|
||||||
--host 0.0.0.0 --port 8000 \
|
|
||||||
--cfg-parallel-size 2 \
|
|
||||||
--use-hsdp --hsdp-shard-size 2 \
|
|
||||||
--init-timeout 1800
|
|
||||||
```
|
|
||||||
|
|
||||||
Guardrails are on by default (gated `nvidia/Cosmos-1.0-Guardrail` — `pip install
|
|
||||||
cosmos-guardrail`, accept the license, set `HF_TOKEN`); add `--no-guardrails` to
|
|
||||||
disable. `--enable-layerwise-offload` reduces VRAM on smaller GPUs.
|
|
||||||
|
|
||||||
#### Verification
|
|
||||||
|
|
||||||
Requests are identical to Nano (see [`Cosmos3-Nano.md`](./Cosmos3-Nano.md) for full
|
|
||||||
T2I/T2V/I2V/T2VS curls); official params: `size=1280x720, num_frames=189, fps=24,
|
|
||||||
num_inference_steps=35, guidance_scale=6.0, flow_shift=10.0, max_sequence_length=4096`.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl http://localhost:8000/v1/models
|
|
||||||
# T2V (official prompt assets give best quality)
|
|
||||||
curl -sS -X POST http://localhost:8000/v1/videos/sync -H "Accept: video/mp4" \
|
|
||||||
-F "model=nvidia/Cosmos3-Super" -F "prompt=A robot arm is cleaning a plate in the kitchen" \
|
|
||||||
-F "size=1280x720" -F "num_frames=189" -F "fps=24" -F "num_inference_steps=35" \
|
|
||||||
-F "guidance_scale=6.0" -F "max_sequence_length=4096" -F "flow_shift=10.0" \
|
|
||||||
-F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":true}' \
|
|
||||||
-F "seed=17" -o cosmos3_super_t2v.mp4
|
|
||||||
|
|
||||||
# I2V — add an uploaded reference image
|
|
||||||
curl -sS -X POST http://localhost:8000/v1/videos/sync -H "Accept: video/mp4" \
|
|
||||||
-F "model=nvidia/Cosmos3-Super" -F "prompt=The scene comes to life with smooth, natural motion." \
|
|
||||||
-F "size=1280x720" -F "num_frames=189" -F "fps=24" -F "num_inference_steps=35" \
|
|
||||||
-F "guidance_scale=6.0" -F "max_sequence_length=4096" -F "flow_shift=10.0" \
|
|
||||||
-F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":true}' \
|
|
||||||
-F "seed=1111" -F "input_reference=@/path/to/reference.jpg;type=image/jpeg" \
|
|
||||||
-o cosmos3_super_i2v.mp4
|
|
||||||
|
|
||||||
# T2V + sound — add generate_sound/sound_duration (output muxes AAC 48 kHz stereo)
|
|
||||||
curl -sS -X POST http://localhost:8000/v1/videos/sync -H "Accept: video/mp4" \
|
|
||||||
-F "model=nvidia/Cosmos3-Super" -F "prompt=A robot arm is cleaning a plate in the kitchen" \
|
|
||||||
-F "size=1280x720" -F "num_frames=189" -F "fps=24" -F "num_inference_steps=35" \
|
|
||||||
-F "guidance_scale=6.0" -F "max_sequence_length=4096" -F "flow_shift=10.0" \
|
|
||||||
-F "generate_sound=true" -F "sound_duration=7.875" \
|
|
||||||
-F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":true}' \
|
|
||||||
-F "seed=17" -o cosmos3_super_t2vs.mp4
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Notes
|
|
||||||
|
|
||||||
- **Measured (2x B300, bf16, guardrails off, official 2-GPU config above):**
|
|
||||||
- T2I 1024², 50 steps → **~6 s**
|
|
||||||
- T2V 1280×720, 189 frames, 35 steps → **~197 s**
|
|
||||||
- I2V 1280×720, 189 frames, 35 steps → **~200 s**
|
|
||||||
- T2V + sound (189 frames, 35 steps) → **~198 s**, output muxes **AAC 48 kHz stereo**
|
|
||||||
- (NVIDIA's reference: 8×H200 @ 50 steps ≈ 55 s/video; 2×H200 @ 35 steps ≈ 3 min/video.)
|
|
||||||
- **Memory:** ~61.5 GiB per GPU when sharded across 2 GPUs (HSDP shard 2); repo ~135 GB on disk.
|
|
||||||
- Same generation defaults, supported sizes, and `generate_sound`/`sound_duration`
|
|
||||||
semantics as Nano, including the **action** modality: `forward_dynamics`
|
|
||||||
(sync `/v1/videos/sync`) and `policy` (async `/v1/videos`, predicted action under
|
|
||||||
the top-level `action` field) — see the Cosmos3-Nano recipe for the request shape.
|
|
||||||
Online inference of inverse dynamics will be added in a follow-up MR. Verified on
|
|
||||||
the 64B Super under `--cfg-parallel-size 2`: async `policy` returns the predicted
|
|
||||||
action (`[16, 10]`) and the rollout video reliably.
|
|
||||||
@@ -57,6 +57,12 @@ Adjust `devices` in the YAML to match your hardware.
|
|||||||
|
|
||||||
#### Command
|
#### Command
|
||||||
|
|
||||||
|
Thinker only (text output):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
vllm serve Jonathan1909/Ming-flash-omni-2.0 --omni --port 8091
|
||||||
|
```
|
||||||
|
|
||||||
Thinker + talker (text and/or audio output):
|
Thinker + talker (text and/or audio output):
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -66,15 +72,6 @@ vllm serve Jonathan1909/Ming-flash-omni-2.0 \
|
|||||||
--log-stats
|
--log-stats
|
||||||
```
|
```
|
||||||
|
|
||||||
Thinker only (text-only output):
|
|
||||||
|
|
||||||
```bash
|
|
||||||
vllm serve Jonathan1909/Ming-flash-omni-2.0 \
|
|
||||||
--omni \
|
|
||||||
--deploy-config vllm_omni/deploy/ming_flash_omni_thinker_only.yaml \
|
|
||||||
--port 8091
|
|
||||||
```
|
|
||||||
|
|
||||||
`--log-stats` is optional but recommended while validating the deployment.
|
`--log-stats` is optional but recommended while validating the deployment.
|
||||||
|
|
||||||
#### Verification
|
#### Verification
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
- Vendor: NVIDIA
|
- Vendor: NVIDIA
|
||||||
- Model: `nvidia/Cosmos3-Nano`
|
- Model: `nvidia/Cosmos3-Nano`
|
||||||
- Task: Text-to-image (T2I), text-to-video (T2V), and image-to-video (I2V) generation, with optional synchronized audio (video + sound), action policy
|
- Task: Text-to-image (T2I), text-to-video (T2V), and image-to-video (I2V) generation
|
||||||
- Mode: Online serving with the OpenAI-compatible image/video APIs, plus offline generation via the `Omni` API
|
- Mode: Online serving with the OpenAI-compatible image/video APIs, plus offline generation via the `Omni` API
|
||||||
- Maintainer: Community
|
- Maintainer: Community
|
||||||
|
|
||||||
@@ -20,34 +20,12 @@ the mode is selected per request:
|
|||||||
- **T2V** — `POST /v1/videos/sync` with `num_frames > 1` and no reference image.
|
- **T2V** — `POST /v1/videos/sync` with `num_frames > 1` and no reference image.
|
||||||
- **I2V** — `POST /v1/videos/sync` with a reference image (`input_reference` file
|
- **I2V** — `POST /v1/videos/sync` with a reference image (`input_reference` file
|
||||||
upload, or `image_reference` JSON).
|
upload, or `image_reference` JSON).
|
||||||
- **T2VS / I2VS** — add `generate_sound=true` (and optional `sound_duration`) to a
|
|
||||||
T2V/I2V `/v1/videos/sync` request to also generate synchronized audio, muxed into
|
|
||||||
the mp4 as AAC 48 kHz stereo. See the official model card's "Video + Audio" examples.
|
|
||||||
- **Action** — pass `extra_params={"action_mode": ...}` to drive Physical-AI tasks:
|
|
||||||
- `forward_dynamics` — given a first frame **and** an action trajectory, roll out
|
|
||||||
the resulting video. Synchronous: `POST /v1/videos/sync`.
|
|
||||||
- `policy` — given a first frame and a language instruction, **predict** the action
|
|
||||||
trajectory (and a rollout video). Use the async `POST /v1/videos` endpoint and
|
|
||||||
read the predicted action from the top-level `action` field
|
|
||||||
(`{data, shape, dtype, raw_action_dim, domain_id}`).
|
|
||||||
|
|
||||||
Action requests also take `domain_name` (e.g. `av`, `bridge_orig_lerobot`,
|
|
||||||
`droid_lerobot`, `agibotworld`, …; or a numeric `domain_id`), `raw_action_dim`,
|
|
||||||
and `action_chunk_size` (must equal `num_frames` or `num_frames - 1`). For
|
|
||||||
`forward_dynamics` also pass the `action` array. The dedicated policy checkpoint
|
|
||||||
**`nvidia/Cosmos3-Nano-Policy-DROID`** is served the same way
|
|
||||||
(`domain_name=droid_lerobot`).
|
|
||||||
|
|
||||||
`inverse_dynamics` (recover the action from a given video) is supported by the
|
|
||||||
pipeline; **online inference of inverse dynamics will be added in a follow-up MR.**
|
|
||||||
|
|
||||||
## References
|
## References
|
||||||
|
|
||||||
- Model card (authoritative usage + example assets): <https://huggingface.co/nvidia/Cosmos3-Nano>
|
- Model card (authoritative usage + example assets): <https://huggingface.co/nvidia/Cosmos3-Nano>
|
||||||
- Example inputs/outputs live in the repo's `assets/` (`example_t2v_prompt.json`,
|
- Example inputs/outputs live in the repo's `assets/` (`example_t2v_prompt.json`,
|
||||||
`example_i2v_prompt.json`, `example_i2v_input.jpg`, `negative_prompt.json`;
|
`example_i2v_prompt.json`, `example_i2v_input.jpg`, `negative_prompt.json`).
|
||||||
audio examples: `example_t2vs_prompt.json`, `example_t2vs_output.mp4`,
|
|
||||||
`example_i2vs_output.mp4`).
|
|
||||||
- Prompt upsampling (recommended for quality): the model expects JSON-upsampled
|
- Prompt upsampling (recommended for quality): the model expects JSON-upsampled
|
||||||
structured prompts; see NVIDIA's `cosmos-framework` prompt-upsampling docs.
|
structured prompts; see NVIDIA's `cosmos-framework` prompt-upsampling docs.
|
||||||
- Pipeline: [`vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py`](../../vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py)
|
- Pipeline: [`vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py`](../../vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py)
|
||||||
@@ -69,9 +47,6 @@ the mode is selected per request:
|
|||||||
|
|
||||||
#### Command
|
#### Command
|
||||||
|
|
||||||
Requires the `vllm-omni` package (or the `vllm/vllm-omni:cosmos3` container),
|
|
||||||
which provides the `vllm serve … --omni` entrypoint used below.
|
|
||||||
|
|
||||||
Safety guardrails are **on by default** (NVIDIA Open Model License). They load
|
Safety guardrails are **on by default** (NVIDIA Open Model License). They load
|
||||||
the **gated** `nvidia/Cosmos-1.0-Guardrail` model, so to keep them on you must:
|
the **gated** `nvidia/Cosmos-1.0-Guardrail` model, so to keep them on you must:
|
||||||
|
|
||||||
@@ -141,55 +116,6 @@ curl -sS -X POST http://localhost:8000/v1/videos/sync \
|
|||||||
-F "seed=1111" \
|
-F "seed=1111" \
|
||||||
-F "input_reference=@/path/to/reference.jpg;type=image/jpeg" \
|
-F "input_reference=@/path/to/reference.jpg;type=image/jpeg" \
|
||||||
-o cosmos3_i2v.mp4
|
-o cosmos3_i2v.mp4
|
||||||
|
|
||||||
|
|
||||||
# Text-to-video-with-sound
|
|
||||||
curl -sS -X POST http://localhost:8000/v1/videos/sync \
|
|
||||||
-H "Accept: video/mp4" \
|
|
||||||
-F "model=nvidia/Cosmos3-Nano" \
|
|
||||||
-F "prompt=The video opens with a view of a well-lit indoor fruit display. A robotic arm picks up a pear, an orange, and a carambola one by one, placing each into a plastic bag in a shopping cart with red handles. The video is 7.875 seconds long, 24 FPS, and 1280x720. Audio description: soft servo whirs, gentle fruit thuds, plastic bag rustling, and a faint refrigeration hum." \
|
|
||||||
-F "negative_prompt=blurry, distorted, low quality" \
|
|
||||||
-F "size=1280x720" \
|
|
||||||
-F "num_frames=189" \
|
|
||||||
-F "fps=24" \
|
|
||||||
-F "num_inference_steps=35" \
|
|
||||||
-F "guidance_scale=6.0" \
|
|
||||||
-F "max_sequence_length=4096" \
|
|
||||||
-F "flow_shift=10.0" \
|
|
||||||
-F "seed=0" \
|
|
||||||
-F "generate_sound=true" \
|
|
||||||
-F "sound_duration=7.875" \
|
|
||||||
-F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":true}' \
|
|
||||||
-o cosmos3_t2v_with_sound.mp4
|
|
||||||
|
|
||||||
# Action — forward dynamics (first frame + action trajectory -> rollout video).
|
|
||||||
# Synchronous; `action` is a JSON array shaped [action_chunk_size, raw_action_dim].
|
|
||||||
curl -sS -X POST http://localhost:8000/v1/videos/sync \
|
|
||||||
-H "Accept: video/mp4" \
|
|
||||||
--form-string "model=nvidia/Cosmos3-Nano" \
|
|
||||||
--form-string "prompt=You are an autonomous vehicle. This video is captured from a first-person perspective." \
|
|
||||||
-F "input_reference=@first_frame.jpg;type=image/jpeg" \
|
|
||||||
-F "size=640x480" -F "num_frames=61" -F "fps=10" \
|
|
||||||
-F "num_inference_steps=30" -F "guidance_scale=1.0" -F "flow_shift=5.0" \
|
|
||||||
--form-string "extra_params={\"action_mode\":\"forward_dynamics\",\"domain_name\":\"av\",\"raw_action_dim\":9,\"action_chunk_size\":60,\"action\":$(cat action.json)}" \
|
|
||||||
-F "seed=0" \
|
|
||||||
-o cosmos3_forward_dynamics.mp4
|
|
||||||
|
|
||||||
# Action — policy (first frame + instruction -> predicted action trajectory + video).
|
|
||||||
# Asynchronous: POST returns a job id; poll, then read the predicted action from
|
|
||||||
# the top-level `action` field ({data, shape, dtype, raw_action_dim, domain_id}).
|
|
||||||
VIDEO_ID=$(curl -sS -X POST http://localhost:8000/v1/videos \
|
|
||||||
-H "Accept: application/json" \
|
|
||||||
--form-string "model=nvidia/Cosmos3-Nano" \
|
|
||||||
--form-string "prompt=Pick up the banana and place it in the bowl." \
|
|
||||||
-F "input_reference=@first_frame.jpg;type=image/jpeg" \
|
|
||||||
-F "size=640x480" -F "num_frames=17" -F "fps=5" \
|
|
||||||
-F "num_inference_steps=30" -F "guidance_scale=1.0" -F "flow_shift=5.0" \
|
|
||||||
--form-string 'extra_params={"action_mode":"policy","domain_name":"bridge_orig_lerobot","raw_action_dim":10,"action_chunk_size":16}' \
|
|
||||||
-F "seed=0" | jq -r '.id')
|
|
||||||
# poll until status == completed, then:
|
|
||||||
curl -sS "http://localhost:8000/v1/videos/$VIDEO_ID" | jq '.action | {shape, dtype, raw_action_dim, domain_id}'
|
|
||||||
curl -sS -L "http://localhost:8000/v1/videos/$VIDEO_ID/content" -o cosmos3_policy.mp4
|
|
||||||
```
|
```
|
||||||
|
|
||||||
#### Notes
|
#### Notes
|
||||||
@@ -198,7 +124,6 @@ curl -sS -L "http://localhost:8000/v1/videos/$VIDEO_ID/content" -o cosmos3_polic
|
|||||||
- T2I 1024² — 10 / 25 / 50 steps → ~0.4 / 0.7 / **1.3 s**
|
- T2I 1024² — 10 / 25 / 50 steps → ~0.4 / 0.7 / **1.3 s**
|
||||||
- T2V 1280×720 @ 35 steps — 25 / 49 / 93 / **189** frames → ~7 / 15 / 33 / **~93 s**
|
- T2V 1280×720 @ 35 steps — 25 / 49 / 93 / **189** frames → ~7 / 15 / 33 / **~93 s**
|
||||||
- I2V 1280×720, 189 frames @ 35 steps → ~**99 s**
|
- I2V 1280×720, 189 frames @ 35 steps → ~**99 s**
|
||||||
- Action 640×480 @ 30 steps — forward-dynamics 61f ~**4 s**, policy 17f ~**1–3 s**.
|
|
||||||
- Guardrails-on overhead: ~8% on T2I, negligible on video.
|
- Guardrails-on overhead: ~8% on T2I, negligible on video.
|
||||||
- **Memory:** transformer ~17 GiB (bf16); peak ~46 GiB for 720p video on 1 GPU;
|
- **Memory:** transformer ~17 GiB (bf16); peak ~46 GiB for 720p video on 1 GPU;
|
||||||
full repo (transformer + Wan VAE + Qwen3-VL vision encoder + audio tokenizer)
|
full repo (transformer + Wan VAE + Qwen3-VL vision encoder + audio tokenizer)
|
||||||
@@ -209,9 +134,7 @@ curl -sS -L "http://localhost:8000/v1/videos/$VIDEO_ID/content" -o cosmos3_polic
|
|||||||
3:4, 9:16. Defaults: T2I 1024², 50 steps, guidance 7.0; T2V/I2V 1280×720,
|
3:4, 9:16. Defaults: T2I 1024², 50 steps, guidance 7.0; T2V/I2V 1280×720,
|
||||||
189 frames, 35 steps, guidance 6.0, `flow_shift=10.0`.
|
189 frames, 35 steps, guidance 6.0, `flow_shift=10.0`.
|
||||||
- **Key flags / params:** `--no-guardrails` (server) or
|
- **Key flags / params:** `--no-guardrails` (server) or
|
||||||
`extra_params={"guardrails":false}` (per request) toggles safety. The
|
`extra_params={"guardrails":false}` (per request) toggles safety;
|
||||||
per-request flag only takes effect when the server was launched **with**
|
|
||||||
guardrails enabled (it cannot re-enable them on a `--no-guardrails` server).
|
|
||||||
`use_resolution_template` / `use_duration_template` are off by default and only
|
`use_resolution_template` / `use_duration_template` are off by default and only
|
||||||
needed when not using upsampled prompts that already encode resolution/duration.
|
needed when not using upsampled prompts that already encode resolution/duration.
|
||||||
- **Known limitations:**
|
- **Known limitations:**
|
||||||
@@ -220,10 +143,8 @@ curl -sS -L "http://localhost:8000/v1/videos/$VIDEO_ID/content" -o cosmos3_polic
|
|||||||
the server fails at pipeline build with a gated-repo / safety-checker error.
|
the server fails at pipeline build with a gated-repo / safety-checker error.
|
||||||
- A guardrail-blocked prompt currently returns HTTP 500
|
- A guardrail-blocked prompt currently returns HTTP 500
|
||||||
(`"Guardrail blocked prompt"`).
|
(`"Guardrail blocked prompt"`).
|
||||||
- Action `forward_dynamics` (sync `/v1/videos/sync`) and `policy` (async
|
- Video + audio, and action (policy / forward- / inverse-dynamics) modalities
|
||||||
`/v1/videos`, returns the predicted action under the top-level `action`
|
are not part of this integration yet.
|
||||||
field) are supported online. **Online inference of inverse dynamics will be
|
|
||||||
added in a follow-up MR.**
|
|
||||||
|
|
||||||
### 1x GPU (Offline generation)
|
### 1x GPU (Offline generation)
|
||||||
|
|
||||||
@@ -249,8 +170,8 @@ def main():
|
|||||||
model_class_name="Cosmos3OmniDiffusersPipeline",
|
model_class_name="Cosmos3OmniDiffusersPipeline",
|
||||||
trust_remote_code=True,
|
trust_remote_code=True,
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
# Guardrails are disabled here for a quick local run; install
|
# Keep guardrails on by installing cosmos-guardrail + gated-repo access;
|
||||||
# cosmos-guardrail + gated-repo access and drop this to enable them.
|
# this disables them for a quick local run.
|
||||||
model_config={"guardrails": False},
|
model_config={"guardrails": False},
|
||||||
)
|
)
|
||||||
gen = torch.Generator(device="cpu").manual_seed(42)
|
gen = torch.Generator(device="cpu").manual_seed(42)
|
||||||
Binary file not shown.
|
Before Width: | Height: | Size: 1.1 MiB |
@@ -345,14 +345,6 @@ def test_parse_score_payload_handles_raw_json_and_delimited_json():
|
|||||||
assert parse_score_payload(wrapped)["score"] == [6]
|
assert parse_score_payload(wrapped)["score"] == [6]
|
||||||
|
|
||||||
|
|
||||||
def test_parse_score_payload_handles_qwen_vl_nested_score_dicts():
|
|
||||||
qwen_style = '{"score": [{"naturalness": 8, "artifact_free": 7}], "reasoning": "good quality"}'
|
|
||||||
assert parse_score_payload(qwen_style)["score"] == [8, 7]
|
|
||||||
|
|
||||||
nested_values = '{"score": [{"score": 9}, {"score": 6}], "reasoning": "ok"}'
|
|
||||||
assert parse_score_payload(nested_values)["score"] == [9, 6]
|
|
||||||
|
|
||||||
|
|
||||||
def test_summarize_gedit_generated_records_groups_by_task_and_language():
|
def test_summarize_gedit_generated_records_groups_by_task_and_language():
|
||||||
records = []
|
records = []
|
||||||
for group in GEDIT_GROUPS[:2]:
|
for group in GEDIT_GROUPS[:2]:
|
||||||
|
|||||||
@@ -61,7 +61,7 @@ def test_stage0_streaming_update_discards_outstanding_async_placeholder_token()
|
|||||||
|
|
||||||
sched._update_request_as_session(session, _make_update([10, 20]))
|
sched._update_request_as_session(session, _make_update([10, 20]))
|
||||||
|
|
||||||
assert session.async_tokens_to_discard == 1
|
assert session.discard_latest_async_tokens is True
|
||||||
assert session.num_output_placeholders == 0
|
assert session.num_output_placeholders == 0
|
||||||
assert session.spec_token_ids == []
|
assert session.spec_token_ids == []
|
||||||
# The async placeholder makes token 9 unconfirmed, so only 7 and 8 are
|
# The async placeholder makes token 9 unconfirmed, so only 7 and 8 are
|
||||||
@@ -83,7 +83,7 @@ def test_stage0_streaming_update_keeps_all_computed_tokens_without_placeholder()
|
|||||||
|
|
||||||
sched._update_request_as_session(session, _make_update([10, 20]))
|
sched._update_request_as_session(session, _make_update([10, 20]))
|
||||||
|
|
||||||
assert getattr(session, "async_tokens_to_discard", 0) == 0
|
assert session.discard_latest_async_tokens is False
|
||||||
assert session.num_output_placeholders == 0
|
assert session.num_output_placeholders == 0
|
||||||
assert session.prompt_token_ids == [1, 2, 3, 7, 8, 9, 10, 20]
|
assert session.prompt_token_ids == [1, 2, 3, 7, 8, 9, 10, 20]
|
||||||
assert list(session._all_token_ids) == [1, 2, 3, 7, 8, 9, 10, 20]
|
assert list(session._all_token_ids) == [1, 2, 3, 7, 8, 9, 10, 20]
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
[
|
[
|
||||||
{
|
{
|
||||||
"test_name": "test_hunyuan_image3_ti2i_inline",
|
"test_name": "test_hunyuan_image3_ti2i_tp4_tp4_inline",
|
||||||
"description": "AR (TP=2) + 2 * DiT (TP=2) - using inline dataset and deploy-config",
|
"description": "AR (TP=4) + DiT (TP=4) - using inline dataset and deploy-config",
|
||||||
"server_type": "vllm-omni",
|
"server_type": "vllm-omni",
|
||||||
"benchmark_endpoint": "/v1/images/edits",
|
"benchmark_endpoint": "/v1/images/edits",
|
||||||
"server_params": {
|
"server_params": {
|
||||||
@@ -21,7 +21,7 @@
|
|||||||
"zmq_port": 50051,
|
"zmq_port": 50051,
|
||||||
"protocol": "rdma",
|
"protocol": "rdma",
|
||||||
"device_name": "",
|
"device_name": "",
|
||||||
"memory_pool_size": 17179869184,
|
"memory_pool_size": 4294967296,
|
||||||
"memory_pool_device": "cpu"
|
"memory_pool_device": "cpu"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -32,13 +32,13 @@
|
|||||||
"is_comprehension": true,
|
"is_comprehension": true,
|
||||||
"final_output": true,
|
"final_output": true,
|
||||||
"final_output_type": "text",
|
"final_output_type": "text",
|
||||||
"max_num_seqs": 16,
|
"max_num_seqs": 4,
|
||||||
"gpu_memory_utilization": 0.95,
|
"gpu_memory_utilization": 0.95,
|
||||||
"enforce_eager": false,
|
"enforce_eager": false,
|
||||||
"enable_prefix_caching": false,
|
"enable_prefix_caching": false,
|
||||||
"max_num_batched_tokens": 32768,
|
"max_num_batched_tokens": 32768,
|
||||||
"devices": "0,1",
|
"devices": "0,1,2,3",
|
||||||
"tensor_parallel_size": 2,
|
"tensor_parallel_size": 4,
|
||||||
"hf_overrides": {
|
"hf_overrides": {
|
||||||
"rope_parameters": {
|
"rope_parameters": {
|
||||||
"mrope_section": [0, 32, 32],
|
"mrope_section": [0, 32, 32],
|
||||||
@@ -65,14 +65,13 @@
|
|||||||
"stage_id": 1,
|
"stage_id": 1,
|
||||||
"max_num_seqs": 1,
|
"max_num_seqs": 1,
|
||||||
"enforce_eager": true,
|
"enforce_eager": true,
|
||||||
"devices": "2,3",
|
"devices": "4,5,6,7",
|
||||||
"distributed_executor_backend": "mp",
|
"distributed_executor_backend": "mp",
|
||||||
"omni_kv_config": {
|
"omni_kv_config": {
|
||||||
"need_recv_cache": true
|
"need_recv_cache": true
|
||||||
},
|
},
|
||||||
"parallel_config": {
|
"parallel_config": {
|
||||||
"tensor_parallel_size": 2,
|
"tensor_parallel_size": 4,
|
||||||
"vae_patch_parallel_size": 2,
|
|
||||||
"enable_expert_parallel": true
|
"enable_expert_parallel": true
|
||||||
},
|
},
|
||||||
"input_connectors": {
|
"input_connectors": {
|
||||||
@@ -115,8 +114,8 @@
|
|||||||
},
|
},
|
||||||
"task": "ti2i",
|
"task": "ti2i",
|
||||||
"bot-task": "think_recaption",
|
"bot-task": "think_recaption",
|
||||||
"max-concurrency": 16,
|
"max-concurrency": 4,
|
||||||
"num-prompts": 96,
|
"num-prompts": 16,
|
||||||
"skip-performance-assertion": true,
|
"skip-performance-assertion": true,
|
||||||
"baseline": {
|
"baseline": {
|
||||||
"throughput_qps": 0.1,
|
"throughput_qps": 0.1,
|
||||||
|
|||||||
@@ -23,10 +23,10 @@
|
|||||||
"ignore_eos": true,
|
"ignore_eos": true,
|
||||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||||
"baseline": {
|
"baseline": {
|
||||||
"mean_ttft_ms": [203, 283, 470, 869],
|
"mean_ttft_ms": [1000, 2000, 3000, 4000],
|
||||||
"mean_tpot_ms": [28, 36, 41, 45],
|
"mean_tpot_ms": [200, 200, 200, 200],
|
||||||
"mean_audio_ttfp_ms": [861, 1200, 1528, 2100],
|
"mean_audio_ttfp_ms": [10000, 15000, 20000, 25000],
|
||||||
"mean_audio_rtf": [0.24, 0.30, 0.35, 0.39]
|
"mean_audio_rtf": [1.5, 1.5, 1.5, 1.5]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -49,9 +49,9 @@
|
|||||||
},
|
},
|
||||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||||
"baseline": {
|
"baseline": {
|
||||||
"mean_ttft_ms": [377],
|
"mean_ttft_ms": [2000],
|
||||||
"mean_audio_ttfp_ms": [684],
|
"mean_audio_ttfp_ms": [2000],
|
||||||
"mean_audio_rtf": [0.14]
|
"mean_audio_rtf": [0.25]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -76,9 +76,9 @@
|
|||||||
},
|
},
|
||||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||||
"baseline": {
|
"baseline": {
|
||||||
"mean_ttft_ms": [228],
|
"mean_ttft_ms": [6000],
|
||||||
"mean_audio_ttfp_ms": [686],
|
"mean_audio_ttfp_ms": [6000],
|
||||||
"mean_audio_rtf": [0.16]
|
"mean_audio_rtf": [0.7]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -105,9 +105,9 @@
|
|||||||
},
|
},
|
||||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||||
"baseline": {
|
"baseline": {
|
||||||
"mean_ttft_ms": [522],
|
"mean_ttft_ms": [12000],
|
||||||
"mean_audio_ttfp_ms": [1119],
|
"mean_audio_ttfp_ms": [12000],
|
||||||
"mean_audio_rtf": [0.21]
|
"mean_audio_rtf": [1.0]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -31,6 +31,11 @@
|
|||||||
],
|
],
|
||||||
"random_input_len": 2500,
|
"random_input_len": 2500,
|
||||||
"random_output_len": 900,
|
"random_output_len": 900,
|
||||||
|
"temperature": 0.4,
|
||||||
|
"top_p": 0.9,
|
||||||
|
"top_k": 1,
|
||||||
|
"seed": 42,
|
||||||
|
"repetition_penalty": 1.05,
|
||||||
"ignore_eos": true,
|
"ignore_eos": true,
|
||||||
"extra_body": {
|
"extra_body": {
|
||||||
"modalities": [
|
"modalities": [
|
||||||
@@ -69,6 +74,11 @@
|
|||||||
"random_input_len": 100,
|
"random_input_len": 100,
|
||||||
"random_output_len": 100,
|
"random_output_len": 100,
|
||||||
"random_range_ratio": 0.0,
|
"random_range_ratio": 0.0,
|
||||||
|
"temperature": 0.4,
|
||||||
|
"top_p": 0.9,
|
||||||
|
"top_k": 1,
|
||||||
|
"seed": 42,
|
||||||
|
"repetition_penalty": 1.05,
|
||||||
"ignore_eos": true,
|
"ignore_eos": true,
|
||||||
"extra_body": {
|
"extra_body": {
|
||||||
"modalities": [
|
"modalities": [
|
||||||
@@ -107,6 +117,11 @@
|
|||||||
"random_input_len": 100,
|
"random_input_len": 100,
|
||||||
"random_output_len": 100,
|
"random_output_len": 100,
|
||||||
"random_range_ratio": 0.0,
|
"random_range_ratio": 0.0,
|
||||||
|
"temperature": 0.4,
|
||||||
|
"top_p": 0.9,
|
||||||
|
"top_k": 1,
|
||||||
|
"seed": 42,
|
||||||
|
"repetition_penalty": 1.05,
|
||||||
"ignore_eos": true,
|
"ignore_eos": true,
|
||||||
"extra_body": {
|
"extra_body": {
|
||||||
"modalities": [
|
"modalities": [
|
||||||
@@ -147,6 +162,11 @@
|
|||||||
"random_input_len": 100,
|
"random_input_len": 100,
|
||||||
"random_output_len": 100,
|
"random_output_len": 100,
|
||||||
"random_range_ratio": 0.0,
|
"random_range_ratio": 0.0,
|
||||||
|
"temperature": 0.4,
|
||||||
|
"top_p": 0.9,
|
||||||
|
"top_k": 1,
|
||||||
|
"seed": 42,
|
||||||
|
"repetition_penalty": 1.05,
|
||||||
"ignore_eos": true,
|
"ignore_eos": true,
|
||||||
"extra_body": {
|
"extra_body": {
|
||||||
"modalities": [
|
"modalities": [
|
||||||
|
|||||||
@@ -73,42 +73,14 @@ class StubCosmos3VAE:
|
|||||||
return (latents,)
|
return (latents,)
|
||||||
|
|
||||||
|
|
||||||
class StubCosmos3AVAE:
|
|
||||||
def __init__(self, **kwargs: Any) -> None:
|
|
||||||
self.kwargs = kwargs
|
|
||||||
self.sample_rate = int(kwargs["sample_rate"])
|
|
||||||
self.audio_channels = int(kwargs["audio_channels"])
|
|
||||||
self.latent_ch = int(kwargs["io_channels"])
|
|
||||||
self.temporal_compression_factor = int(kwargs["hop_size"])
|
|
||||||
|
|
||||||
def get_latent_num_samples(self, num_audio_samples: int) -> int:
|
|
||||||
return int(num_audio_samples) // self.temporal_compression_factor
|
|
||||||
|
|
||||||
def get_audio_num_samples(self, num_latent_samples: int) -> int:
|
|
||||||
return int(num_latent_samples) * self.temporal_compression_factor
|
|
||||||
|
|
||||||
def decode(self, latents: torch.Tensor) -> torch.Tensor:
|
|
||||||
return torch.zeros(latents.shape[0], self.audio_channels, 8)
|
|
||||||
|
|
||||||
|
|
||||||
class StubCosmos3Transformer(nn.Module):
|
class StubCosmos3Transformer(nn.Module):
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
latent_channel_size: int = 2,
|
latent_channel_size: int = 2,
|
||||||
sound_gen: bool = False,
|
|
||||||
sound_dim: int = 3,
|
|
||||||
sound_latent_fps: float = 25.0,
|
|
||||||
action_gen: bool = False,
|
|
||||||
action_dim: int = 4,
|
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.latent_channel_size = latent_channel_size
|
self.latent_channel_size = latent_channel_size
|
||||||
self.sound_gen = sound_gen
|
|
||||||
self.sound_dim = sound_dim
|
|
||||||
self.sound_latent_fps = sound_latent_fps
|
|
||||||
self.action_gen = action_gen
|
|
||||||
self.action_dim = action_dim
|
|
||||||
self.cached_kv: Any | None = None
|
self.cached_kv: Any | None = None
|
||||||
self.cached_freqs_gen: Any | None = None
|
self.cached_freqs_gen: Any | None = None
|
||||||
self.calls: list[dict[str, Any]] = []
|
self.calls: list[dict[str, Any]] = []
|
||||||
@@ -127,9 +99,8 @@ class StubCosmos3Transformer(nn.Module):
|
|||||||
text_ids: torch.Tensor,
|
text_ids: torch.Tensor,
|
||||||
text_mask: torch.Tensor,
|
text_mask: torch.Tensor,
|
||||||
**kwargs: Any,
|
**kwargs: Any,
|
||||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
) -> torch.Tensor:
|
||||||
token = int(text_ids.reshape(-1)[0].item()) if text_ids.numel() else 0
|
token = int(text_ids.reshape(-1)[0].item()) if text_ids.numel() else 0
|
||||||
sound_latents = kwargs.get("sound_latents")
|
|
||||||
self.calls.append(
|
self.calls.append(
|
||||||
{
|
{
|
||||||
"token": token,
|
"token": token,
|
||||||
@@ -143,13 +114,7 @@ class StubCosmos3Transformer(nn.Module):
|
|||||||
marker = torch.tensor([token], dtype=torch.float32)
|
marker = torch.tensor([token], dtype=torch.float32)
|
||||||
self.cached_kv = [(marker, marker + 100)]
|
self.cached_kv = [(marker, marker + 100)]
|
||||||
self.cached_freqs_gen = (marker + 200, marker + 300)
|
self.cached_freqs_gen = (marker + 200, marker + 300)
|
||||||
action_latents = kwargs.get("action_latents")
|
return torch.full_like(hidden_states, float(token))
|
||||||
outputs: list[torch.Tensor] = [torch.full_like(hidden_states, float(token))]
|
|
||||||
if action_latents is not None:
|
|
||||||
outputs.append(torch.full_like(action_latents, float(token + 20)))
|
|
||||||
if sound_latents is not None:
|
|
||||||
outputs.append(torch.full_like(sound_latents, float(token + 10)))
|
|
||||||
return outputs[0] if len(outputs) == 1 else tuple(outputs)
|
|
||||||
|
|
||||||
|
|
||||||
def passthrough_progress_bar(iterable):
|
def passthrough_progress_bar(iterable):
|
||||||
@@ -190,7 +155,6 @@ def make_cosmos3_pipeline():
|
|||||||
pipeline._guidance_scale = None
|
pipeline._guidance_scale = None
|
||||||
pipeline._num_timesteps = None
|
pipeline._num_timesteps = None
|
||||||
pipeline._cache_dit_requires_paired_cfg = False
|
pipeline._cache_dit_requires_paired_cfg = False
|
||||||
pipeline._sound_tokenizer = None
|
|
||||||
pipeline.progress_bar = passthrough_progress_bar
|
pipeline.progress_bar = passthrough_progress_bar
|
||||||
return pipeline
|
return pipeline
|
||||||
|
|
||||||
@@ -249,109 +213,7 @@ def test_pipeline_registered_and_exported() -> None:
|
|||||||
assert "Cosmos3OmniDiffusersPipeline" in cosmos3.__all__
|
assert "Cosmos3OmniDiffusersPipeline" in cosmos3.__all__
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
def test_preprocess_i2v_image_input() -> None:
|
||||||
def stub_real_pipeline_init(monkeypatch: pytest.MonkeyPatch):
|
|
||||||
from vllm_omni.diffusion.models.cosmos3 import pipeline_cosmos3
|
|
||||||
|
|
||||||
class _StubAutoTokenizer:
|
|
||||||
@classmethod
|
|
||||||
def from_pretrained(cls, *args, **kwargs):
|
|
||||||
return SimpleNamespace()
|
|
||||||
|
|
||||||
class _StubDiffusersVAE:
|
|
||||||
config = SimpleNamespace(scale_factor_temporal=4, scale_factor_spatial=8)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_pretrained(cls, *args, **kwargs):
|
|
||||||
return cls()
|
|
||||||
|
|
||||||
def to(self, _device):
|
|
||||||
return self
|
|
||||||
|
|
||||||
class _StubDiffusersScheduler:
|
|
||||||
config = SimpleNamespace(flow_shift=1.0)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_pretrained(cls, *args, **kwargs):
|
|
||||||
return cls()
|
|
||||||
|
|
||||||
class _StubVideoProcessor:
|
|
||||||
def __init__(self, *args, **kwargs) -> None:
|
|
||||||
pass
|
|
||||||
|
|
||||||
monkeypatch.setattr(pipeline_cosmos3, "AutoTokenizer", _StubAutoTokenizer)
|
|
||||||
monkeypatch.setattr(pipeline_cosmos3, "DistributedAutoencoderKLWan", _StubDiffusersVAE)
|
|
||||||
monkeypatch.setattr(pipeline_cosmos3, "UniPCMultistepScheduler", _StubDiffusersScheduler)
|
|
||||||
monkeypatch.setattr(pipeline_cosmos3, "VideoProcessor", _StubVideoProcessor)
|
|
||||||
monkeypatch.setattr(pipeline_cosmos3, "get_local_device", lambda: torch.device("cpu"))
|
|
||||||
|
|
||||||
|
|
||||||
def _make_od_config(*, sound_gen: bool) -> SimpleNamespace:
|
|
||||||
tf_model_config = {
|
|
||||||
"hidden_size": 8,
|
|
||||||
"num_hidden_layers": 0,
|
|
||||||
"num_attention_heads": 2,
|
|
||||||
"num_key_value_heads": 2,
|
|
||||||
"head_dim": 4,
|
|
||||||
"intermediate_size": 16,
|
|
||||||
"vocab_size": 32,
|
|
||||||
"latent_patch_size": 1,
|
|
||||||
"latent_channel": 2,
|
|
||||||
"rope_scaling": {"mrope_section": [1, 1, 0]},
|
|
||||||
}
|
|
||||||
if sound_gen:
|
|
||||||
tf_model_config["sound_gen"] = True
|
|
||||||
return SimpleNamespace(
|
|
||||||
enable_cpu_offload=False,
|
|
||||||
enable_diffusion_pipeline_profiler=False,
|
|
||||||
model="/nonexistent/model/path",
|
|
||||||
dtype=torch.float32,
|
|
||||||
flow_shift=None,
|
|
||||||
quantization_config=None,
|
|
||||||
custom_pipeline_args={},
|
|
||||||
model_config={},
|
|
||||||
tf_model_config=tf_model_config,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_pipeline_init_skips_tokenizer_when_sound_disabled(stub_real_pipeline_init) -> None:
|
|
||||||
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import Cosmos3OmniDiffusersPipeline
|
|
||||||
|
|
||||||
pipeline = Cosmos3OmniDiffusersPipeline(od_config=_make_od_config(sound_gen=False))
|
|
||||||
|
|
||||||
assert pipeline._sound_tokenizer is None
|
|
||||||
assert pipeline.transformer.sound_gen is False
|
|
||||||
assert not hasattr(pipeline.transformer, "audio_proj_in")
|
|
||||||
assert not hasattr(pipeline.transformer, "audio_proj_out")
|
|
||||||
|
|
||||||
|
|
||||||
def test_pipeline_init_passes_tokenizer_attrs_into_transformer(
|
|
||||||
stub_real_pipeline_init,
|
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
|
||||||
) -> None:
|
|
||||||
from vllm_omni.diffusion.models.cosmos3 import sound_tokenizer
|
|
||||||
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import Cosmos3OmniDiffusersPipeline
|
|
||||||
|
|
||||||
stub_tokenizer = sound_tokenizer.Cosmos3SoundTokenizer(
|
|
||||||
StubCosmos3AVAE(sample_rate=32000, audio_channels=2, io_channels=5, hop_size=800)
|
|
||||||
)
|
|
||||||
monkeypatch.setattr(
|
|
||||||
sound_tokenizer.Cosmos3SoundTokenizer,
|
|
||||||
"from_config",
|
|
||||||
classmethod(lambda cls, od_config: stub_tokenizer),
|
|
||||||
)
|
|
||||||
|
|
||||||
pipeline = Cosmos3OmniDiffusersPipeline(od_config=_make_od_config(sound_gen=True))
|
|
||||||
|
|
||||||
assert pipeline._sound_tokenizer is stub_tokenizer
|
|
||||||
assert pipeline.transformer.sound_gen is True
|
|
||||||
assert pipeline.transformer.sound_dim == pipeline._sound_tokenizer.latent_ch == 5
|
|
||||||
assert pipeline.transformer.sound_latent_fps == pipeline._sound_tokenizer.latent_fps == 40.0
|
|
||||||
assert pipeline.transformer.audio_proj_in.in_features == 5
|
|
||||||
assert pipeline.transformer.audio_proj_out.out_features == 5
|
|
||||||
|
|
||||||
|
|
||||||
def test_preprocess_i2v_image_and_action_video_inputs() -> None:
|
|
||||||
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import get_cosmos3_pre_process_func
|
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import get_cosmos3_pre_process_func
|
||||||
|
|
||||||
preprocess = get_cosmos3_pre_process_func(SimpleNamespace())
|
preprocess = get_cosmos3_pre_process_func(SimpleNamespace())
|
||||||
@@ -364,18 +226,8 @@ def test_preprocess_i2v_image_and_action_video_inputs() -> None:
|
|||||||
assert (result.sampling_params.height, result.sampling_params.width) == (672, 1344)
|
assert (result.sampling_params.height, result.sampling_params.width) == (672, 1344)
|
||||||
assert tuple(result.prompts[0]["additional_information"]["preprocessed_image"].shape[-2:]) == (672, 1344)
|
assert tuple(result.prompts[0]["additional_information"]["preprocessed_image"].shape[-2:]) == (672, 1344)
|
||||||
|
|
||||||
frames = [Image.new("RGB", (8, 4), color) for color in ("red", "green", "blue")]
|
|
||||||
action = SimpleNamespace(
|
|
||||||
prompts=[{"prompt": "Move.", "multi_modal_data": {"video": frames}}],
|
|
||||||
sampling_params=SimpleNamespace(height=16, width=32, extra_args={"action_mode": "forward_dynamics"}),
|
|
||||||
)
|
|
||||||
|
|
||||||
additional = preprocess(action).prompts[0]["additional_information"]
|
def test_postprocess_handles_image_video_and_validation() -> None:
|
||||||
assert tuple(additional["preprocessed_image"].shape) == (1, 3, 16, 32)
|
|
||||||
assert tuple(additional["preprocessed_video"].shape) == (1, 3, 3, 16, 32)
|
|
||||||
|
|
||||||
|
|
||||||
def test_postprocess_handles_image_video_audio_and_validation() -> None:
|
|
||||||
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import get_cosmos3_post_process_func
|
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import get_cosmos3_post_process_func
|
||||||
|
|
||||||
func = get_cosmos3_post_process_func(SimpleNamespace())
|
func = get_cosmos3_post_process_func(SimpleNamespace())
|
||||||
@@ -383,16 +235,6 @@ def test_postprocess_handles_image_video_audio_and_validation() -> None:
|
|||||||
|
|
||||||
assert func(video, output_type="latent") is video
|
assert func(video, output_type="latent") is video
|
||||||
assert func({"image": video})[0].size == (4, 4)
|
assert func({"image": video})[0].size == (4, 4)
|
||||||
# Video-only postprocess returns the bare processed video (not a dict),
|
|
||||||
# matching the image/latent branches and peer audio-capable pipelines.
|
|
||||||
assert not isinstance(func({"video": video}), dict)
|
|
||||||
assert (
|
|
||||||
func(
|
|
||||||
{"video": video, "audio": torch.ones(1, 2, 16), "audio_sample_rate": 48000},
|
|
||||||
sampling_params=SimpleNamespace(extra_args={"resolved_frame_rate": 12}),
|
|
||||||
)["audio_sample_rate"]
|
|
||||||
== 48000
|
|
||||||
)
|
|
||||||
|
|
||||||
with pytest.raises(ValueError, match="text-to-image postprocess expects"):
|
with pytest.raises(ValueError, match="text-to-image postprocess expects"):
|
||||||
func({"image": torch.zeros(1, 3, 2, 4, 4)})
|
func({"image": torch.zeros(1, 3, 2, 4, 4)})
|
||||||
@@ -451,7 +293,7 @@ def test_prompt_formatting_and_checkpoint_key_remap(make_cosmos3_pipeline) -> No
|
|||||||
assert {key: Cosmos3OmniDiffusersPipeline._remap_ckpt_key(key) for key in remaps} == remaps
|
assert {key: Cosmos3OmniDiffusersPipeline._remap_ckpt_key(key) for key in remaps} == remaps
|
||||||
|
|
||||||
|
|
||||||
def test_prepare_latents_for_video_image_sound_and_action(make_cosmos3_pipeline) -> None:
|
def test_prepare_latents_for_video_and_image(make_cosmos3_pipeline) -> None:
|
||||||
pipeline = make_cosmos3_pipeline()
|
pipeline = make_cosmos3_pipeline()
|
||||||
latents = pipeline._prepare_latents(16, 24, 5, torch.Generator(device="cpu").manual_seed(0))
|
latents = pipeline._prepare_latents(16, 24, 5, torch.Generator(device="cpu").manual_seed(0))
|
||||||
assert latents.shape == (1, 2, 2, 2, 3)
|
assert latents.shape == (1, 2, 2, 2, 3)
|
||||||
@@ -464,36 +306,8 @@ def test_prepare_latents_for_video_image_sound_and_action(make_cosmos3_pipeline)
|
|||||||
assert velocity_mask.tolist() == [[[[[0.0]], [[1.0]]]]]
|
assert velocity_mask.tolist() == [[[[[0.0]], [[1.0]]]]]
|
||||||
assert image_latent.shape == (1, 2, 1, 2, 3)
|
assert image_latent.shape == (1, 2, 1, 2, 3)
|
||||||
|
|
||||||
pipeline.transformer = pipeline.transformer.__class__(latent_channel_size=2, sound_gen=True, sound_dim=3)
|
|
||||||
pipeline._sound_tokenizer = SimpleNamespace(
|
|
||||||
sample_rate=10,
|
|
||||||
latent_ch=3,
|
|
||||||
hop_size=4,
|
|
||||||
decode=lambda x: torch.ones(x.shape[0], 2, 24),
|
|
||||||
)
|
|
||||||
assert pipeline._resolve_sound_target_samples(SimpleNamespace(extra_args={"sound_duration": 2.0}), 9, 3.0) == (
|
|
||||||
20,
|
|
||||||
2.0,
|
|
||||||
10,
|
|
||||||
)
|
|
||||||
sound_latents, latent_frames = pipeline._prepare_sound_latents(21, torch.Generator(device="cpu").manual_seed(0))
|
|
||||||
assert (sound_latents.shape, latent_frames) == (torch.Size([1, 3, 6]), 6)
|
|
||||||
assert pipeline._decode_sound_latents(torch.zeros(1, 3, 6), target_audio_samples=21).shape == (1, 2, 21)
|
|
||||||
|
|
||||||
pipeline.transformer = pipeline.transformer.__class__(action_gen=True, action_dim=4)
|
def test_diffuse_covers_cfg_and_i2v_steps(make_cosmos3_pipeline) -> None:
|
||||||
action, action_mask, clean, raw_dim = pipeline._prepare_action_latents(
|
|
||||||
mode="forward_dynamics",
|
|
||||||
action_chunk_size=2,
|
|
||||||
raw_action_dim=None,
|
|
||||||
generator=torch.Generator(device="cpu").manual_seed(0),
|
|
||||||
sp=SimpleNamespace(extra_args={"action": [[1.0, 2.0], [3.0, 4.0]]}),
|
|
||||||
)
|
|
||||||
assert raw_dim == 2
|
|
||||||
assert action_mask.tolist() == [[[0.0], [0.0]]]
|
|
||||||
torch.testing.assert_close(action, clean)
|
|
||||||
|
|
||||||
|
|
||||||
def test_diffuse_covers_cfg_i2v_and_multimodal_steps(make_cosmos3_pipeline) -> None:
|
|
||||||
pipeline = make_cosmos3_pipeline()
|
pipeline = make_cosmos3_pipeline()
|
||||||
latents = torch.zeros(1, 2, 1, 1, 1)
|
latents = torch.zeros(1, 2, 1, 1, 1)
|
||||||
|
|
||||||
@@ -525,23 +339,6 @@ def test_diffuse_covers_cfg_i2v_and_multimodal_steps(make_cosmos3_pipeline) -> N
|
|||||||
)
|
)
|
||||||
torch.testing.assert_close(i2v[:, :, 0:1], torch.full((1, 2, 1, 1, 1), 7.0))
|
torch.testing.assert_close(i2v[:, :, 0:1], torch.full((1, 2, 1, 1, 1), 7.0))
|
||||||
|
|
||||||
pipeline.transformer = pipeline.transformer.__class__(latent_channel_size=2, action_gen=True, action_dim=4)
|
|
||||||
video_result, action_result = pipeline.diffuse(
|
|
||||||
latents=latents,
|
|
||||||
action_latents=torch.zeros(1, 3, 4),
|
|
||||||
action_velocity_mask=torch.ones(1, 3, 1),
|
|
||||||
action_condition_latents=torch.zeros(1, 3, 4),
|
|
||||||
timesteps=torch.tensor([7, 3]),
|
|
||||||
cond_ids=_ids(2),
|
|
||||||
cond_mask=_mask(),
|
|
||||||
uncond_ids=_ids(1),
|
|
||||||
uncond_mask=_mask(),
|
|
||||||
guidance_scale=1.0,
|
|
||||||
shared_kwargs={"video_shape": (1, 1, 1), "fps": 24.0, "action_domain_ids": torch.tensor([0])},
|
|
||||||
)
|
|
||||||
torch.testing.assert_close(video_result, torch.full_like(latents, 4.0))
|
|
||||||
torch.testing.assert_close(action_result, torch.full((), 44.0).expand_as(action_result))
|
|
||||||
|
|
||||||
|
|
||||||
def test_diffuse_keeps_paired_cfg_when_cache_dit_active(make_cosmos3_pipeline) -> None:
|
def test_diffuse_keeps_paired_cfg_when_cache_dit_active(make_cosmos3_pipeline) -> None:
|
||||||
"""With cache-dit active the uncond pass is kept even outside the guidance
|
"""With cache-dit active the uncond pass is kept even outside the guidance
|
||||||
@@ -598,12 +395,7 @@ class TestForwardRouting:
|
|||||||
|
|
||||||
def fake_diffuse(**kwargs):
|
def fake_diffuse(**kwargs):
|
||||||
captured["diffuse_calls"].append(kwargs)
|
captured["diffuse_calls"].append(kwargs)
|
||||||
outputs = [kwargs["latents"] + len(captured["diffuse_calls"])]
|
return kwargs["latents"] + len(captured["diffuse_calls"])
|
||||||
if kwargs.get("action_latents") is not None:
|
|
||||||
outputs.append(kwargs["action_latents"] + 3.0)
|
|
||||||
if kwargs.get("sound_latents") is not None:
|
|
||||||
outputs.append(kwargs["sound_latents"] + 2.0)
|
|
||||||
return outputs[0] if len(outputs) == 1 else tuple(outputs)
|
|
||||||
|
|
||||||
pipeline._format_and_tokenize_prompts = fake_format
|
pipeline._format_and_tokenize_prompts = fake_format
|
||||||
pipeline._prepare_latents = fake_prepare
|
pipeline._prepare_latents = fake_prepare
|
||||||
@@ -645,7 +437,7 @@ class TestForwardRouting:
|
|||||||
assert captured["flow_shifts"] == expected["flow"]
|
assert captured["flow_shifts"] == expected["flow"]
|
||||||
assert [call[0] for call in pipeline.scheduler.set_timesteps_calls] == expected["steps"]
|
assert [call[0] for call in pipeline.scheduler.set_timesteps_calls] == expected["steps"]
|
||||||
|
|
||||||
def test_forward_i2v_sound_and_action_routes(self, make_cosmos3_pipeline) -> None:
|
def test_forward_i2v_route(self, make_cosmos3_pipeline) -> None:
|
||||||
pipeline = make_cosmos3_pipeline()
|
pipeline = make_cosmos3_pipeline()
|
||||||
captured = self._install_forward_stubs(pipeline)
|
captured = self._install_forward_stubs(pipeline)
|
||||||
image_tensor = torch.zeros(1, 3, 16, 16)
|
image_tensor = torch.zeros(1, 3, 16, 16)
|
||||||
@@ -670,55 +462,11 @@ class TestForwardRouting:
|
|||||||
)
|
)
|
||||||
assert captured["diffuse_calls"][-1]["shared_kwargs"]["noisy_frame_mask"] is velocity_mask
|
assert captured["diffuse_calls"][-1]["shared_kwargs"]["noisy_frame_mask"] is velocity_mask
|
||||||
|
|
||||||
pipeline.transformer = pipeline.transformer.__class__(latent_channel_size=2, sound_gen=True, sound_dim=3)
|
|
||||||
sound_latents = torch.zeros(1, 3, 4)
|
|
||||||
pipeline._resolve_sound_target_samples = lambda *args: (20, 2.0, 10)
|
|
||||||
pipeline._prepare_sound_latents = lambda *args: (sound_latents, 4)
|
|
||||||
pipeline._decode_sound_latents = lambda *args: torch.ones(1, 2, 20)
|
|
||||||
output = pipeline.forward(
|
|
||||||
SimpleNamespace(
|
|
||||||
prompts=[{"prompt": "A robot", "modalities": ["video"], "generate_sound": True}],
|
|
||||||
sampling_params=make_sampling_params(num_frames=9, frame_rate=3.0),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
assert captured["diffuse_calls"][-1]["sound_latents"] is sound_latents
|
|
||||||
assert output.output["audio_sample_rate"] == 10
|
|
||||||
|
|
||||||
pipeline.transformer = pipeline.transformer.__class__(latent_channel_size=2, action_gen=True, action_dim=4)
|
|
||||||
output = pipeline.forward(
|
|
||||||
SimpleNamespace(
|
|
||||||
prompts=[
|
|
||||||
{
|
|
||||||
"prompt": "Pick the block.",
|
|
||||||
"modalities": ["video"],
|
|
||||||
"additional_information": {"preprocessed_image": image_tensor},
|
|
||||||
}
|
|
||||||
],
|
|
||||||
sampling_params=make_sampling_params(
|
|
||||||
height=16,
|
|
||||||
width=16,
|
|
||||||
extra_args={
|
|
||||||
"action_mode": "policy",
|
|
||||||
"action_chunk_size": 2,
|
|
||||||
"raw_action_dim": 2,
|
|
||||||
"domain_name": "bridge_orig_lerobot",
|
|
||||||
},
|
|
||||||
),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
assert captured["diffuse_calls"][-1]["shared_kwargs"]["action_domain_ids"].tolist() == [7]
|
|
||||||
assert output.custom_output["action"].shape == (1, 2, 2)
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("prompt", "sampling_params", "message"),
|
("prompt", "sampling_params", "message"),
|
||||||
[
|
[
|
||||||
(["one", "two"], make_sampling_params(), "single prompt"),
|
(["one", "two"], make_sampling_params(), "single prompt"),
|
||||||
([{"prompt": "one", "modalities": ["image", "video"]}], make_sampling_params(), "both image and video"),
|
([{"prompt": "one", "modalities": ["image", "video"]}], make_sampling_params(), "both image and video"),
|
||||||
(
|
|
||||||
[{"prompt": "x", "modalities": ["image"], "generate_sound": True}],
|
|
||||||
make_sampling_params(),
|
|
||||||
"only for video",
|
|
||||||
),
|
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
def test_forward_rejects_invalid_public_requests(
|
def test_forward_rejects_invalid_public_requests(
|
||||||
@@ -729,7 +477,6 @@ class TestForwardRouting:
|
|||||||
message,
|
message,
|
||||||
) -> None:
|
) -> None:
|
||||||
pipeline = make_cosmos3_pipeline()
|
pipeline = make_cosmos3_pipeline()
|
||||||
pipeline.transformer = pipeline.transformer.__class__(latent_channel_size=2, sound_gen=True, sound_dim=3)
|
|
||||||
|
|
||||||
with pytest.raises(ValueError, match=message):
|
with pytest.raises(ValueError, match=message):
|
||||||
pipeline.forward(SimpleNamespace(prompts=prompt, sampling_params=sampling_params))
|
pipeline.forward(SimpleNamespace(prompts=prompt, sampling_params=sampling_params))
|
||||||
@@ -1,236 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import json
|
|
||||||
from pathlib import Path
|
|
||||||
from types import SimpleNamespace
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
import torch
|
|
||||||
|
|
||||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
|
||||||
|
|
||||||
DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME = "diffusion_pytorch_model.safetensors"
|
|
||||||
|
|
||||||
|
|
||||||
class _FakeAVAEAudioTokenizer:
|
|
||||||
def __init__(self, **kwargs) -> None:
|
|
||||||
self.kwargs = kwargs
|
|
||||||
self.sample_rate = int(kwargs["sample_rate"])
|
|
||||||
self.audio_channels = int(kwargs["audio_channels"])
|
|
||||||
self.latent_ch = int(kwargs["io_channels"])
|
|
||||||
self.temporal_compression_factor = int(kwargs["hop_size"])
|
|
||||||
|
|
||||||
def get_latent_num_samples(self, num_audio_samples: int) -> int:
|
|
||||||
return int(num_audio_samples) // self.temporal_compression_factor
|
|
||||||
|
|
||||||
def get_audio_num_samples(self, num_latent_samples: int) -> int:
|
|
||||||
return int(num_latent_samples) * self.temporal_compression_factor
|
|
||||||
|
|
||||||
def decode(self, latents: torch.Tensor) -> torch.Tensor:
|
|
||||||
return torch.zeros(latents.shape[0], self.audio_channels, 8)
|
|
||||||
|
|
||||||
|
|
||||||
def _write_component(root: Path, config: dict | None = None, checkpoint_name: str | None = None) -> Path:
|
|
||||||
tokenizer_dir = root / "sound_tokenizer"
|
|
||||||
tokenizer_dir.mkdir(parents=True)
|
|
||||||
if checkpoint_name:
|
|
||||||
(tokenizer_dir / checkpoint_name).write_bytes(b"stub")
|
|
||||||
(tokenizer_dir / "config.json").write_text(json.dumps(config or {}), encoding="utf-8")
|
|
||||||
return tokenizer_dir
|
|
||||||
|
|
||||||
|
|
||||||
def _patch_fake_avae(monkeypatch: pytest.MonkeyPatch, created: dict) -> None:
|
|
||||||
from vllm_omni.diffusion.models.cosmos3 import sound_tokenizer
|
|
||||||
|
|
||||||
class FakeAVAE(_FakeAVAEAudioTokenizer):
|
|
||||||
def __init__(self, **kwargs) -> None:
|
|
||||||
created.update(kwargs)
|
|
||||||
super().__init__(**kwargs)
|
|
||||||
|
|
||||||
monkeypatch.setattr(sound_tokenizer, "Cosmos3AVAEAudioTokenizer", FakeAVAE)
|
|
||||||
monkeypatch.setattr(sound_tokenizer, "get_local_device", lambda: torch.device("cpu"))
|
|
||||||
|
|
||||||
|
|
||||||
def test_from_config_loads_local_diffusers_component(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
||||||
from vllm_omni.diffusion.models.cosmos3 import sound_tokenizer
|
|
||||||
|
|
||||||
model_dir = tmp_path / "model"
|
|
||||||
tokenizer_dir = _write_component(model_dir, checkpoint_name=DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME)
|
|
||||||
created = {}
|
|
||||||
_patch_fake_avae(monkeypatch, created)
|
|
||||||
|
|
||||||
tokenizer = sound_tokenizer.Cosmos3SoundTokenizer.from_config(
|
|
||||||
SimpleNamespace(
|
|
||||||
model=str(model_dir),
|
|
||||||
custom_pipeline_args={"sound_sample_rate": 32000, "sound_hop_size": 800, "sound_dim": 3},
|
|
||||||
dtype=torch.float32,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
assert created["checkpoint_path"] == str(tokenizer_dir / DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME)
|
|
||||||
assert created["config_path"] == str(tokenizer_dir / "config.json")
|
|
||||||
assert (tokenizer.sample_rate, tokenizer.latent_ch, tokenizer.hop_size, tokenizer.latent_fps) == (
|
|
||||||
32000,
|
|
||||||
3,
|
|
||||||
800,
|
|
||||||
40.0,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_from_config_downloads_component_from_hf_repo(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
||||||
import huggingface_hub
|
|
||||||
|
|
||||||
from vllm_omni.diffusion.models.cosmos3 import sound_tokenizer
|
|
||||||
|
|
||||||
cache_dir = tmp_path / "hf"
|
|
||||||
_write_component(cache_dir, checkpoint_name=DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME)
|
|
||||||
calls = []
|
|
||||||
created = {}
|
|
||||||
_patch_fake_avae(monkeypatch, created)
|
|
||||||
|
|
||||||
def fake_snapshot_download(repo_id: str, *, revision: str | None, allow_patterns: list[str]) -> str:
|
|
||||||
calls.append((repo_id, revision, allow_patterns))
|
|
||||||
return str(cache_dir)
|
|
||||||
|
|
||||||
monkeypatch.setattr(huggingface_hub, "snapshot_download", fake_snapshot_download)
|
|
||||||
|
|
||||||
sound_tokenizer.Cosmos3SoundTokenizer.from_config(
|
|
||||||
SimpleNamespace(
|
|
||||||
model="nvidia/cosmos3",
|
|
||||||
revision="test-rev",
|
|
||||||
custom_pipeline_args={"sound_sample_rate": 32000, "sound_hop_size": 800, "sound_dim": 3},
|
|
||||||
dtype=torch.float32,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
assert created["checkpoint_path"].endswith(DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME)
|
|
||||||
assert calls == [
|
|
||||||
(
|
|
||||||
"nvidia/cosmos3",
|
|
||||||
"test-rev",
|
|
||||||
["sound_tokenizer/config.json", f"sound_tokenizer/{DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME}"],
|
|
||||||
)
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
("checkpoint_name", "message"),
|
|
||||||
[
|
|
||||||
(None, "no AVAE sound tokenizer checkpoint"),
|
|
||||||
("model.safetensors", DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_default_component_requires_diffusers_checkpoint_name(tmp_path, checkpoint_name, message) -> None:
|
|
||||||
from vllm_omni.diffusion.models.cosmos3 import sound_tokenizer
|
|
||||||
|
|
||||||
model_dir = tmp_path / "model"
|
|
||||||
_write_component(model_dir, checkpoint_name=checkpoint_name)
|
|
||||||
|
|
||||||
with pytest.raises(ValueError, match=message):
|
|
||||||
sound_tokenizer.Cosmos3SoundTokenizer.from_config(
|
|
||||||
SimpleNamespace(model=str(model_dir), custom_pipeline_args={}, dtype=torch.float32)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_component_config_precedence_and_conflict_detection(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
||||||
from vllm_omni.diffusion.models.cosmos3 import sound_tokenizer
|
|
||||||
|
|
||||||
component_config = {
|
|
||||||
"sampling_rate": 48000,
|
|
||||||
"dec_out_channels": 2,
|
|
||||||
"vocoder_input_dim": 64,
|
|
||||||
"hop_size": 1920,
|
|
||||||
}
|
|
||||||
model_dir = tmp_path / "model"
|
|
||||||
_write_component(model_dir, component_config, DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME)
|
|
||||||
created = {}
|
|
||||||
_patch_fake_avae(monkeypatch, created)
|
|
||||||
|
|
||||||
tokenizer = sound_tokenizer.Cosmos3SoundTokenizer.from_config(
|
|
||||||
SimpleNamespace(
|
|
||||||
model=str(model_dir),
|
|
||||||
custom_pipeline_args={
|
|
||||||
"sound_normalize_latents": True,
|
|
||||||
"sound_normalization_type": "tanh",
|
|
||||||
"sound_tanh_input_scale": 2.0,
|
|
||||||
},
|
|
||||||
model_config={
|
|
||||||
"sound_tokenizer": {
|
|
||||||
"sample_rate": 32000,
|
|
||||||
"audio_channels": 1,
|
|
||||||
"io_channels": 3,
|
|
||||||
"hop_size": 800,
|
|
||||||
"normalize_latents": False,
|
|
||||||
"normalization_type": "none",
|
|
||||||
}
|
|
||||||
},
|
|
||||||
dtype=torch.float32,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
assert (created["sample_rate"], created["audio_channels"], created["io_channels"], created["hop_size"]) == (
|
|
||||||
48000,
|
|
||||||
2,
|
|
||||||
64,
|
|
||||||
1920,
|
|
||||||
)
|
|
||||||
assert (created["normalize_latents"], created["normalization_type"], created["tanh_input_scale"]) == (
|
|
||||||
True,
|
|
||||||
"tanh",
|
|
||||||
2.0,
|
|
||||||
)
|
|
||||||
assert (tokenizer.sample_rate, tokenizer.latent_ch, tokenizer.hop_size, tokenizer.latent_fps) == (
|
|
||||||
48000,
|
|
||||||
64,
|
|
||||||
1920,
|
|
||||||
25.0,
|
|
||||||
)
|
|
||||||
|
|
||||||
with pytest.raises(ValueError, match=r"sample_rate.*48000.*32000"):
|
|
||||||
sound_tokenizer.Cosmos3SoundTokenizer.from_config(
|
|
||||||
SimpleNamespace(
|
|
||||||
model=str(model_dir),
|
|
||||||
custom_pipeline_args={"sound_sample_rate": 32000},
|
|
||||||
dtype=torch.float32,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_avae_uses_diffusers_decoder_state_dict_layout(tmp_path) -> None:
|
|
||||||
from safetensors.torch import save_file
|
|
||||||
|
|
||||||
from vllm_omni.diffusion.models.cosmos3.audio_tokenizer import avae
|
|
||||||
|
|
||||||
config = {
|
|
||||||
"sampling_rate": 8000,
|
|
||||||
"hop_size": 2,
|
|
||||||
"dec_dim": 4,
|
|
||||||
"dec_c_mults": [1],
|
|
||||||
"dec_strides": [2],
|
|
||||||
"dec_out_channels": 1,
|
|
||||||
"vocoder_input_dim": 2,
|
|
||||||
"normalization_type": "none",
|
|
||||||
}
|
|
||||||
checkpoint_path = tmp_path / DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME
|
|
||||||
config_path = tmp_path / "config.json"
|
|
||||||
config_path.write_text(json.dumps(config), encoding="utf-8")
|
|
||||||
|
|
||||||
decoder = avae.OobleckDecoder(4, 2, 1, [2], [1])
|
|
||||||
save_file({f"decoder.{key}": value for key, value in decoder.state_dict().items()}, str(checkpoint_path))
|
|
||||||
|
|
||||||
tokenizer = avae.Cosmos3AVAEAudioTokenizer(
|
|
||||||
checkpoint_path=checkpoint_path,
|
|
||||||
config_path=config_path,
|
|
||||||
dtype=torch.float32,
|
|
||||||
device="cpu",
|
|
||||||
)
|
|
||||||
|
|
||||||
keys = set(tokenizer.state_dict())
|
|
||||||
assert {"decoder.conv1.weight_g", "decoder.block.0.conv_t1.weight_g", "decoder.conv2.weight_g"} <= keys
|
|
||||||
assert not any(key.startswith(("decoder.layers.", "model.decoder.")) for key in keys)
|
|
||||||
assert tokenizer.decode(torch.zeros(1, 2, 3)).shape == (1, 1, 6)
|
|
||||||
with pytest.raises(NotImplementedError, match="decoder-only"):
|
|
||||||
tokenizer.encode(torch.zeros(1, 1, 6))
|
|
||||||
@@ -4,7 +4,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
@@ -30,10 +29,8 @@ def _tiny_cosmos3_config(**overrides):
|
|||||||
return config
|
return config
|
||||||
|
|
||||||
|
|
||||||
def test_mrope_position_ids_cover_text_video_sound_and_action() -> None:
|
def test_mrope_position_ids_cover_text_and_video() -> None:
|
||||||
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import (
|
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import (
|
||||||
compute_mrope_position_ids_action,
|
|
||||||
compute_mrope_position_ids_sound,
|
|
||||||
compute_mrope_position_ids_text,
|
compute_mrope_position_ids_text,
|
||||||
compute_mrope_position_ids_vision,
|
compute_mrope_position_ids_vision,
|
||||||
)
|
)
|
||||||
@@ -59,14 +56,6 @@ def test_mrope_position_ids_cover_text_video_sound_and_action() -> None:
|
|||||||
torch.testing.assert_close(modulated_ids[0], torch.tensor([10.0, 12.0]))
|
torch.testing.assert_close(modulated_ids[0], torch.tensor([10.0, 12.0]))
|
||||||
assert modulated_offset == 13
|
assert modulated_offset == 13
|
||||||
|
|
||||||
sound_ids, sound_offset = compute_mrope_position_ids_sound(3, temporal_offset=10, sound_latent_fps=25.0)
|
|
||||||
torch.testing.assert_close(sound_ids[0], torch.tensor([10.0, 10.96, 11.92]))
|
|
||||||
assert sound_offset == 12
|
|
||||||
|
|
||||||
action_ids, action_offset = compute_mrope_position_ids_action(3, temporal_offset=10, action_fps=None)
|
|
||||||
assert action_ids.tolist() == [[11, 12, 13], [0, 0, 0], [0, 0, 0]]
|
|
||||||
assert action_offset == 14
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("key", "value"),
|
("key", "value"),
|
||||||
@@ -131,142 +120,7 @@ def test_forward_returns_video_prediction(monkeypatch: pytest.MonkeyPatch) -> No
|
|||||||
assert tuple(output.shape) == (1, 2, 1, 2, 2)
|
assert tuple(output.shape) == (1, 2, 1, 2, 2)
|
||||||
|
|
||||||
|
|
||||||
def test_sound_and_action_modules_follow_config() -> None:
|
def test_compute_rope_freqs_places_text_and_video_positions() -> None:
|
||||||
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3VFMTransformer
|
|
||||||
|
|
||||||
tiny = _tiny_cosmos3_config()
|
|
||||||
no_modal = Cosmos3VFMTransformer(SimpleNamespace(tf_model_config=tiny, dtype=torch.float32))
|
|
||||||
with_sound = Cosmos3VFMTransformer(
|
|
||||||
SimpleNamespace(tf_model_config=tiny, dtype=torch.float32),
|
|
||||||
sound_gen=True,
|
|
||||||
sound_dim=5,
|
|
||||||
sound_latent_fps=40.0,
|
|
||||||
)
|
|
||||||
with_action = Cosmos3VFMTransformer(
|
|
||||||
SimpleNamespace(
|
|
||||||
tf_model_config={**tiny, "action_gen": True, "max_action_dim": 6, "num_embodiment_domains": 9},
|
|
||||||
dtype=torch.float32,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
assert no_modal.sound_gen is False
|
|
||||||
assert no_modal.action_gen is False
|
|
||||||
assert not hasattr(no_modal, "audio_proj_in")
|
|
||||||
assert not hasattr(no_modal, "action_proj_in")
|
|
||||||
assert with_sound.sound_dim == 5
|
|
||||||
assert with_sound.sound_latent_fps == 40.0
|
|
||||||
assert with_sound.audio_proj_in.in_features == 5
|
|
||||||
assert with_action.action_dim == 6
|
|
||||||
assert with_action.action_proj_in.num_domains == 9
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"kwargs",
|
|
||||||
[
|
|
||||||
{"sound_gen": True},
|
|
||||||
{"sound_gen": True, "sound_dim": 5},
|
|
||||||
{"sound_gen": True, "sound_latent_fps": 40.0},
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_transformer_requires_sound_dim_and_fps_when_sound_gen_true(kwargs: dict[str, Any]) -> None:
|
|
||||||
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3VFMTransformer
|
|
||||||
|
|
||||||
with pytest.raises(ValueError, match=r"requires an explicit sound_dim and sound_latent_fps"):
|
|
||||||
Cosmos3VFMTransformer(
|
|
||||||
SimpleNamespace(tf_model_config=_tiny_cosmos3_config(), dtype=torch.float32),
|
|
||||||
**kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_sound_and_action_pack_unpack_validate_shapes() -> None:
|
|
||||||
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3VFMTransformer
|
|
||||||
|
|
||||||
model = object.__new__(Cosmos3VFMTransformer)
|
|
||||||
nn.Module.__init__(model)
|
|
||||||
model.sound_dim = 3
|
|
||||||
model.action_dim = 3
|
|
||||||
|
|
||||||
sound = torch.arange(2 * 3 * 4, dtype=torch.float32).reshape(2, 3, 4)
|
|
||||||
action = torch.arange(2 * 4 * 3, dtype=torch.float32).reshape(2, 4, 3)
|
|
||||||
torch.testing.assert_close(model.unpack_sound(model.pack_sound(sound)), sound)
|
|
||||||
torch.testing.assert_close(model.unpack_action(model.pack_action(action)), action)
|
|
||||||
|
|
||||||
with pytest.raises(ValueError, match="channel mismatch"):
|
|
||||||
model.pack_sound(torch.zeros(1, 4, 2))
|
|
||||||
with pytest.raises(ValueError, match="dimension mismatch"):
|
|
||||||
model.pack_action(torch.zeros(1, 2, 4))
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
("config", "transformer_kwargs", "extra_kwargs", "expected_shapes"),
|
|
||||||
[
|
|
||||||
(
|
|
||||||
_tiny_cosmos3_config(),
|
|
||||||
{"sound_gen": True, "sound_dim": 3, "sound_latent_fps": 24.0},
|
|
||||||
{"sound_latents": torch.zeros(1, 3, 4)},
|
|
||||||
[(1, 2, 1, 2, 2), (1, 3, 4)],
|
|
||||||
),
|
|
||||||
(
|
|
||||||
_tiny_cosmos3_config(action_gen=True, max_action_dim=3, num_embodiment_domains=4),
|
|
||||||
{},
|
|
||||||
{"action_latents": torch.zeros(1, 5, 3), "action_domain_ids": torch.tensor([2])},
|
|
||||||
[(1, 2, 1, 2, 2), (1, 5, 3)],
|
|
||||||
),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_forward_returns_video_plus_optional_modality_predictions(
|
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
|
||||||
config,
|
|
||||||
transformer_kwargs,
|
|
||||||
extra_kwargs,
|
|
||||||
expected_shapes,
|
|
||||||
) -> None:
|
|
||||||
from vllm_omni.diffusion.models.cosmos3 import transformer_cosmos3
|
|
||||||
|
|
||||||
monkeypatch.setattr(transformer_cosmos3, "_get_ulysses_state", lambda: (1, 0, None))
|
|
||||||
|
|
||||||
output = transformer_cosmos3.Cosmos3VFMTransformer(
|
|
||||||
SimpleNamespace(tf_model_config=config, dtype=torch.float32),
|
|
||||||
**transformer_kwargs,
|
|
||||||
)(
|
|
||||||
hidden_states=torch.zeros(1, 2, 1, 2, 2),
|
|
||||||
timestep=torch.tensor([1.0]),
|
|
||||||
text_ids=torch.tensor([[1, 2]], dtype=torch.long),
|
|
||||||
text_mask=torch.ones(1, 2, dtype=torch.long),
|
|
||||||
video_shape=(1, 2, 2),
|
|
||||||
fps=24.0,
|
|
||||||
action_noisy_mask=torch.ones(1, 5, 1),
|
|
||||||
**extra_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert isinstance(output, tuple)
|
|
||||||
assert [tuple(tensor.shape) for tensor in output] == expected_shapes
|
|
||||||
|
|
||||||
|
|
||||||
def test_forward_with_sound_ulysses_error_mentions_combined_sequence(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
||||||
import vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 as cosmos3_module
|
|
||||||
|
|
||||||
model = cosmos3_module.Cosmos3VFMTransformer(
|
|
||||||
SimpleNamespace(tf_model_config=_tiny_cosmos3_config(), dtype=torch.float32),
|
|
||||||
sound_gen=True,
|
|
||||||
sound_dim=3,
|
|
||||||
sound_latent_fps=40.0,
|
|
||||||
)
|
|
||||||
monkeypatch.setattr(cosmos3_module, "_get_ulysses_state", lambda: (2, 0, None))
|
|
||||||
|
|
||||||
with pytest.raises(ValueError, match=r"GEN sequence length \(3 = video tokens 2 \+ sound tokens 1\)"):
|
|
||||||
model(
|
|
||||||
hidden_states=torch.zeros(1, 2, 1, 1, 2),
|
|
||||||
timestep=torch.tensor([1.0]),
|
|
||||||
text_ids=torch.tensor([[1, 2]], dtype=torch.long),
|
|
||||||
text_mask=torch.ones(1, 2, dtype=torch.long),
|
|
||||||
video_shape=(1, 1, 2),
|
|
||||||
fps=24.0,
|
|
||||||
sound_latents=torch.zeros(1, 3, 1),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_compute_rope_freqs_places_text_video_action_and_sound_positions() -> None:
|
|
||||||
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3VFMTransformer
|
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3VFMTransformer
|
||||||
|
|
||||||
class FakeRotary:
|
class FakeRotary:
|
||||||
@@ -286,8 +140,6 @@ def test_compute_rope_freqs_places_text_video_action_and_sound_positions() -> No
|
|||||||
model.temporal_modality_margin = 100
|
model.temporal_modality_margin = 100
|
||||||
model.base_fps = 24.0
|
model.base_fps = 24.0
|
||||||
model.temporal_compression_factor = 4
|
model.temporal_compression_factor = 4
|
||||||
model.temporal_compression_factor_sound = 1
|
|
||||||
model.sound_latent_fps = 25.0
|
|
||||||
model.enable_fps_modulation = False
|
model.enable_fps_modulation = False
|
||||||
|
|
||||||
freqs_und, freqs_gen = model._compute_rope_freqs(
|
freqs_und, freqs_gen = model._compute_rope_freqs(
|
||||||
@@ -304,21 +156,3 @@ def test_compute_rope_freqs_places_text_video_action_and_sound_positions() -> No
|
|||||||
assert vision_pos[0, 0].tolist() == [102, 103]
|
assert vision_pos[0, 0].tolist() == [102, 103]
|
||||||
assert freqs_und[0].shape == (2, 3, 1, 4)
|
assert freqs_und[0].shape == (2, 3, 1, 4)
|
||||||
assert freqs_gen[0].shape == (2, 2, 1, 4)
|
assert freqs_gen[0].shape == (2, 2, 1, 4)
|
||||||
|
|
||||||
rotary.position_ids.clear()
|
|
||||||
model._compute_rope_freqs(
|
|
||||||
text_mask=torch.tensor([[1, 1]], dtype=torch.long),
|
|
||||||
t=2,
|
|
||||||
hp=1,
|
|
||||||
wp=1,
|
|
||||||
fps=24.0,
|
|
||||||
device=torch.device("cpu"),
|
|
||||||
dtype=torch.float32,
|
|
||||||
t_action=2,
|
|
||||||
action_start_frame_offset=1,
|
|
||||||
t_sound=1,
|
|
||||||
)
|
|
||||||
|
|
||||||
_, gen_pos = rotary.position_ids
|
|
||||||
assert gen_pos.shape == (3, 1, 5)
|
|
||||||
assert gen_pos[0, 0].tolist() == [102, 103, 103, 104, 102]
|
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import queue
|
import queue
|
||||||
import threading
|
import threading
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
@@ -446,7 +447,11 @@ class TestDiffusionEngine:
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_step_raises_aborted_error(self, mocker: MockerFixture) -> None:
|
async def test_step_raises_aborted_error(self, mocker: MockerFixture) -> None:
|
||||||
engine = DiffusionEngine.__new__(DiffusionEngine)
|
engine = DiffusionEngine.__new__(DiffusionEngine)
|
||||||
engine._check_and_start_background_loop = mocker.AsyncMock()
|
engine._closed = False
|
||||||
|
engine._loop_started = True
|
||||||
|
engine._init_lock = asyncio.Lock()
|
||||||
|
engine.main_loop = asyncio.get_running_loop()
|
||||||
|
engine.stop_event = threading.Event()
|
||||||
engine.pre_process_func = None
|
engine.pre_process_func = None
|
||||||
engine.async_add_req_and_wait_for_response = mocker.AsyncMock(
|
engine.async_add_req_and_wait_for_response = mocker.AsyncMock(
|
||||||
return_value=DiffusionOutput(aborted=True, abort_message="Request req-abort aborted.")
|
return_value=DiffusionOutput(aborted=True, abort_message="Request req-abort aborted.")
|
||||||
@@ -521,52 +526,6 @@ class TestDiffusionEngine:
|
|||||||
with pytest.raises(RuntimeError, match="Dummy run failed: boom"):
|
with pytest.raises(RuntimeError, match="Dummy run failed: boom"):
|
||||||
engine._dummy_run()
|
engine._dummy_run()
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_step_multi_request_reuses_multimodal_slice_logic(self, mocker: MockerFixture) -> None:
|
|
||||||
engine = DiffusionEngine.__new__(DiffusionEngine)
|
|
||||||
engine.od_config = SimpleNamespace(
|
|
||||||
model_class_name="mock_model",
|
|
||||||
enable_cpu_offload=False,
|
|
||||||
)
|
|
||||||
engine.pre_process_func = None
|
|
||||||
engine.post_process_func = None
|
|
||||||
engine._check_and_start_background_loop = mocker.AsyncMock()
|
|
||||||
engine.async_add_req_and_wait_for_response = mocker.AsyncMock(
|
|
||||||
return_value=DiffusionOutput(
|
|
||||||
output={
|
|
||||||
"video": ["frame-0", "frame-1"],
|
|
||||||
"audio": ["audio-0", "audio-1"],
|
|
||||||
"actions": torch.tensor([[1.0, 2.0], [3.0, 4.0]]),
|
|
||||||
}
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
request = OmniDiffusionRequest(
|
|
||||||
prompts=["prompt-0", "prompt-1"],
|
|
||||||
sampling_params=OmniDiffusionSamplingParams(
|
|
||||||
num_inference_steps=1,
|
|
||||||
num_outputs_per_prompt=1,
|
|
||||||
),
|
|
||||||
request_id="req-batch",
|
|
||||||
)
|
|
||||||
|
|
||||||
mocker.patch("vllm_omni.diffusion.diffusion_engine.supports_audio_output", return_value=False)
|
|
||||||
outputs = await engine.step(request)
|
|
||||||
|
|
||||||
assert len(outputs) == 2
|
|
||||||
assert outputs[0].images == ["frame-0"]
|
|
||||||
assert outputs[1].images == ["frame-1"]
|
|
||||||
assert outputs[0].multimodal_output["audio"] == "audio-0"
|
|
||||||
assert outputs[1].multimodal_output["audio"] == "audio-1"
|
|
||||||
torch.testing.assert_close(
|
|
||||||
outputs[0].multimodal_output["actions"],
|
|
||||||
torch.tensor([1.0, 2.0]),
|
|
||||||
)
|
|
||||||
torch.testing.assert_close(
|
|
||||||
outputs[1].multimodal_output["actions"],
|
|
||||||
torch.tensor([3.0, 4.0]),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class TestStepScheduler:
|
class TestStepScheduler:
|
||||||
def setup_method(self) -> None:
|
def setup_method(self) -> None:
|
||||||
|
|||||||
@@ -32,8 +32,8 @@ BAGEL_MOONCAKE_CI_DEPLOY = get_deploy_config_path("ci/bagel_mooncake.yaml")
|
|||||||
|
|
||||||
# Reference pixel data extracted from the known-good output image
|
# Reference pixel data extracted from the known-good output image
|
||||||
# Each entry contains (x, y) position and expected (R, G, B) values
|
# Each entry contains (x, y) position and expected (R, G, B) values
|
||||||
# "Generated with seed=52, num_inference_steps=14,
|
# "Generated with seed=52, num_inference_steps=15,
|
||||||
# prompt='A cute cat'"
|
# prompt='A futuristic city skyline at twilight, cyberpunk style'"
|
||||||
REFERENCE_PIXELS = [
|
REFERENCE_PIXELS = [
|
||||||
{"position": (100, 100), "rgb": (115, 113, 94)},
|
{"position": (100, 100), "rgb": (115, 113, 94)},
|
||||||
{"position": (400, 50), "rgb": (159, 160, 144)},
|
{"position": (400, 50), "rgb": (159, 160, 144)},
|
||||||
@@ -63,7 +63,7 @@ def _find_free_port() -> int:
|
|||||||
return port
|
return port
|
||||||
|
|
||||||
|
|
||||||
def _configure_sampling_params(omni: Omni, num_inference_steps: int = 14) -> list:
|
def _configure_sampling_params(omni: Omni, num_inference_steps: int = 15) -> list:
|
||||||
"""Configure sampling parameters for Bagel text2img generation.
|
"""Configure sampling parameters for Bagel text2img generation.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -269,6 +269,7 @@ def _load_mooncake_config(host: str, rpc_port: int, http_port: int) -> str:
|
|||||||
@pytest.mark.core_model
|
@pytest.mark.core_model
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
@pytest.mark.diffusion
|
@pytest.mark.diffusion
|
||||||
|
@pytest.mark.skip(reason="Skip failed CI issue 3977: https://github.com/vllm-project/vllm-omni/issues/3977")
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||||
def test_bagel_text2img_mooncake_connector(run_level):
|
def test_bagel_text2img_mooncake_connector(run_level):
|
||||||
"""Test Bagel text2img with Mooncake connector for inter-stage communication."""
|
"""Test Bagel text2img with Mooncake connector for inter-stage communication."""
|
||||||
|
|||||||
@@ -24,13 +24,14 @@ from tests.helpers.mark import hardware_test
|
|||||||
from tests.helpers.runtime import OmniRunner
|
from tests.helpers.runtime import OmniRunner
|
||||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||||
from vllm_omni.entrypoints.omni import Omni
|
from vllm_omni.entrypoints.omni import Omni
|
||||||
|
from vllm_omni.platforms import current_omni_platform
|
||||||
|
|
||||||
pytestmark = [pytest.mark.usefixtures("clean_gpu_memory_between_tests")]
|
pytestmark = [pytest.mark.usefixtures("clean_gpu_memory_between_tests")]
|
||||||
|
|
||||||
BAGEL_CI_DEPLOY = get_deploy_config_path("ci/bagel.yaml")
|
BAGEL_CI_DEPLOY = get_deploy_config_path("ci/bagel.yaml")
|
||||||
|
|
||||||
# Reference pixel data extracted from the known-good output image
|
# Reference pixel data extracted from the known-good output image
|
||||||
# Generated with seed=52, num_inference_steps=14,
|
# Generated with seed=52, num_inference_steps=15,
|
||||||
# prompt='Change the grass color to red',
|
# prompt='Change the grass color to red',
|
||||||
# input image: 2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg
|
# input image: 2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg
|
||||||
REFERENCE_PIXELS = [
|
REFERENCE_PIXELS = [
|
||||||
@@ -46,10 +47,23 @@ REFERENCE_PIXELS = [
|
|||||||
{"position": (256, 256), "rgb": (181, 201, 221)},
|
{"position": (256, 256), "rgb": (181, 201, 221)},
|
||||||
]
|
]
|
||||||
|
|
||||||
|
if current_omni_platform.is_rocm():
|
||||||
|
REFERENCE_PIXELS = [
|
||||||
|
{"position": (100, 100), "rgb": (156, 172, 217)},
|
||||||
|
{"position": (400, 50), "rgb": (105, 144, 217)},
|
||||||
|
{"position": (700, 100), "rgb": (118, 159, 232)},
|
||||||
|
{"position": (150, 400), "rgb": (180, 22, 52)},
|
||||||
|
{"position": (512, 336), "rgb": (221, 211, 194)},
|
||||||
|
{"position": (700, 400), "rgb": (192, 10, 46)},
|
||||||
|
{"position": (100, 600), "rgb": (102, 12, 22)},
|
||||||
|
{"position": (400, 600), "rgb": (161, 28, 47)},
|
||||||
|
{"position": (700, 600), "rgb": (100, 87, 94)},
|
||||||
|
{"position": (256, 256), "rgb": (181, 201, 221)},
|
||||||
|
]
|
||||||
|
|
||||||
# text2img reference pixels (aligned with offline `bagel/end2end.py` text2img, 15 steps)
|
# text2img reference pixels (aligned with offline `bagel/end2end.py` text2img, 15 steps)
|
||||||
# "Generated with seed=52, num_inference_steps=14,
|
# "Generated with seed=52, num_inference_steps=15,
|
||||||
# prompt='A cute cat'"
|
# prompt='A futuristic city skyline at twilight, cyberpunk style'"
|
||||||
TEXT2IMG_REFERENCE_PIXELS = [
|
TEXT2IMG_REFERENCE_PIXELS = [
|
||||||
{"position": (100, 100), "rgb": (115, 113, 94)},
|
{"position": (100, 100), "rgb": (115, 113, 94)},
|
||||||
{"position": (400, 50), "rgb": (159, 160, 144)},
|
{"position": (400, 50), "rgb": (159, 160, 144)},
|
||||||
@@ -63,6 +77,19 @@ TEXT2IMG_REFERENCE_PIXELS = [
|
|||||||
{"position": (256, 256), "rgb": (167, 156, 150)},
|
{"position": (256, 256), "rgb": (167, 156, 150)},
|
||||||
]
|
]
|
||||||
|
|
||||||
|
if current_omni_platform.is_rocm():
|
||||||
|
TEXT2IMG_REFERENCE_PIXELS = [
|
||||||
|
{"position": (100, 100), "rgb": (115, 113, 94)},
|
||||||
|
{"position": (400, 50), "rgb": (159, 160, 144)},
|
||||||
|
{"position": (700, 100), "rgb": (164, 151, 123)},
|
||||||
|
{"position": (150, 400), "rgb": (120, 121, 107)},
|
||||||
|
{"position": (512, 512), "rgb": (165, 133, 127)},
|
||||||
|
{"position": (700, 400), "rgb": (217, 130, 66)},
|
||||||
|
{"position": (100, 700), "rgb": (191, 168, 152)},
|
||||||
|
{"position": (400, 700), "rgb": (130, 96, 77)},
|
||||||
|
{"position": (700, 700), "rgb": (247, 203, 140)},
|
||||||
|
{"position": (256, 256), "rgb": (167, 156, 150)},
|
||||||
|
]
|
||||||
|
|
||||||
PIXEL_TOLERANCE = 10
|
PIXEL_TOLERANCE = 10
|
||||||
TEXT2IMG_PIXEL_TOLERANCE = 5
|
TEXT2IMG_PIXEL_TOLERANCE = 5
|
||||||
@@ -88,7 +115,7 @@ def _find_free_port() -> int:
|
|||||||
return port
|
return port
|
||||||
|
|
||||||
|
|
||||||
def _configure_sampling_params(omni: Omni, num_inference_steps: int = 14) -> list:
|
def _configure_sampling_params(omni: Omni, num_inference_steps: int = 15) -> list:
|
||||||
"""Configure sampling parameters for Bagel img2img generation.
|
"""Configure sampling parameters for Bagel img2img generation.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -231,6 +258,7 @@ def _resolve_deploy_config(config_path: str, run_level: str) -> str:
|
|||||||
@pytest.mark.core_model
|
@pytest.mark.core_model
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
@pytest.mark.diffusion
|
@pytest.mark.diffusion
|
||||||
|
@pytest.mark.skip(reason="Skip failed CI issue 3977: https://github.com/vllm-project/vllm-omni/issues/3977")
|
||||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
|
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
|
||||||
def test_bagel_img2img_shared_memory_connector(run_level):
|
def test_bagel_img2img_shared_memory_connector(run_level):
|
||||||
"""Test Bagel img2img with shared memory connector."""
|
"""Test Bagel img2img with shared memory connector."""
|
||||||
@@ -248,6 +276,7 @@ def test_bagel_img2img_shared_memory_connector(run_level):
|
|||||||
@pytest.mark.core_model
|
@pytest.mark.core_model
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
@pytest.mark.diffusion
|
@pytest.mark.diffusion
|
||||||
|
@pytest.mark.skip(reason="Skip failed CI issue 3977: https://github.com/vllm-project/vllm-omni/issues/3977")
|
||||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
|
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
|
||||||
def test_bagel_text2img_shared_memory_connector(run_level):
|
def test_bagel_text2img_shared_memory_connector(run_level):
|
||||||
"""Test Bagel text2img with shared memory connector."""
|
"""Test Bagel text2img with shared memory connector."""
|
||||||
|
|||||||
@@ -1,307 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import uuid
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
try:
|
|
||||||
import cv2
|
|
||||||
except ImportError: # pragma: no cover - optional e2e dependency
|
|
||||||
cv2 = None
|
|
||||||
|
|
||||||
try:
|
|
||||||
import websockets.sync.client as websockets_client
|
|
||||||
except ImportError: # pragma: no cover - optional e2e dependency
|
|
||||||
websockets_client = None
|
|
||||||
|
|
||||||
try:
|
|
||||||
from openpi_client import msgpack_numpy
|
|
||||||
except ImportError: # pragma: no cover - optional e2e dependency
|
|
||||||
msgpack_numpy = None
|
|
||||||
|
|
||||||
PING_INTERVAL_SECS = 300
|
|
||||||
PING_TIMEOUT_SECS = 3600
|
|
||||||
DEFAULT_HOST = "127.0.0.1"
|
|
||||||
DEFAULT_PORT = 8000
|
|
||||||
DEFAULT_PATH = "/v1/realtime/robot/openpi"
|
|
||||||
DEFAULT_PROMPT = "Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan"
|
|
||||||
ACTION_HORIZON = 24
|
|
||||||
DEFAULT_ACTION_DIM = 8
|
|
||||||
RELATIVE_OFFSETS = [-23, -16, -8, 0]
|
|
||||||
CAMERA_FILES = {
|
|
||||||
"observation/exterior_image_0_left": "exterior_image_1_left.mp4",
|
|
||||||
"observation/exterior_image_1_left": "exterior_image_2_left.mp4",
|
|
||||||
"observation/wrist_image_left": "wrist_image_left.mp4",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _decode_action_response(response: bytes | str) -> np.ndarray:
|
|
||||||
if isinstance(response, str):
|
|
||||||
raise RuntimeError(f"Inference failed: {response}")
|
|
||||||
decoded = msgpack_numpy.unpackb(response)
|
|
||||||
if isinstance(decoded, dict) and decoded.get("type") == "error":
|
|
||||||
message = decoded.get("message", decoded)
|
|
||||||
raise RuntimeError(f"Inference failed: {message}")
|
|
||||||
return np.asarray(decoded, dtype=np.float32)
|
|
||||||
|
|
||||||
|
|
||||||
def require_dependencies() -> None:
|
|
||||||
missing = []
|
|
||||||
if cv2 is None:
|
|
||||||
missing.append("opencv-python")
|
|
||||||
if websockets_client is None:
|
|
||||||
missing.append("websockets")
|
|
||||||
if msgpack_numpy is None:
|
|
||||||
missing.append("openpi-client")
|
|
||||||
if missing:
|
|
||||||
raise ModuleNotFoundError(f"DreamZero OpenPI test dependencies are missing: {', '.join(missing)}")
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class DreamZeroServerMetadata:
|
|
||||||
image_resolution: tuple[int, int]
|
|
||||||
n_external_cameras: int
|
|
||||||
needs_wrist_camera: bool
|
|
||||||
needs_stereo_camera: bool
|
|
||||||
needs_session_id: bool
|
|
||||||
action_space: str
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_dict(cls, payload: dict[str, Any]) -> DreamZeroServerMetadata:
|
|
||||||
required_keys = (
|
|
||||||
"image_resolution",
|
|
||||||
"n_external_cameras",
|
|
||||||
"needs_wrist_camera",
|
|
||||||
"needs_stereo_camera",
|
|
||||||
"needs_session_id",
|
|
||||||
"action_space",
|
|
||||||
)
|
|
||||||
missing_keys = [key for key in required_keys if key not in payload]
|
|
||||||
if missing_keys:
|
|
||||||
raise ValueError(f"Missing DreamZero metadata keys: {missing_keys}")
|
|
||||||
|
|
||||||
image_resolution = payload["image_resolution"]
|
|
||||||
if not isinstance(image_resolution, (list, tuple)) or len(image_resolution) != 2:
|
|
||||||
raise ValueError(f"Invalid image_resolution: {image_resolution!r}")
|
|
||||||
|
|
||||||
return cls(
|
|
||||||
image_resolution=(int(image_resolution[0]), int(image_resolution[1])),
|
|
||||||
n_external_cameras=int(payload["n_external_cameras"]),
|
|
||||||
needs_wrist_camera=bool(payload["needs_wrist_camera"]),
|
|
||||||
needs_stereo_camera=bool(payload["needs_stereo_camera"]),
|
|
||||||
needs_session_id=bool(payload["needs_session_id"]),
|
|
||||||
action_space=str(payload["action_space"]),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class OpenPIWebsocketClient:
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
host: str = DEFAULT_HOST,
|
|
||||||
port: int = DEFAULT_PORT,
|
|
||||||
path: str = DEFAULT_PATH,
|
|
||||||
) -> None:
|
|
||||||
require_dependencies()
|
|
||||||
self._uri = f"ws://{host}:{port}{path}"
|
|
||||||
self._packer = msgpack_numpy.Packer()
|
|
||||||
self._ws, self._server_metadata = self._connect()
|
|
||||||
|
|
||||||
def _connect(self):
|
|
||||||
conn = websockets_client.connect(
|
|
||||||
self._uri,
|
|
||||||
compression=None,
|
|
||||||
max_size=None,
|
|
||||||
ping_interval=PING_INTERVAL_SECS,
|
|
||||||
ping_timeout=PING_TIMEOUT_SECS,
|
|
||||||
)
|
|
||||||
metadata = msgpack_numpy.unpackb(conn.recv())
|
|
||||||
if not isinstance(metadata, dict):
|
|
||||||
raise TypeError(f"Expected dict metadata from server, got {type(metadata)!r}")
|
|
||||||
return conn, metadata
|
|
||||||
|
|
||||||
def get_server_metadata(self) -> dict[str, Any]:
|
|
||||||
return dict(self._server_metadata)
|
|
||||||
|
|
||||||
def infer(self, obs: dict[str, Any]) -> np.ndarray:
|
|
||||||
payload = dict(obs)
|
|
||||||
payload["endpoint"] = "infer"
|
|
||||||
self._ws.send(self._packer.pack(payload))
|
|
||||||
response = self._ws.recv()
|
|
||||||
return _decode_action_response(response)
|
|
||||||
|
|
||||||
def reset(self, reset_info: dict[str, Any] | None = None) -> str:
|
|
||||||
payload = dict(reset_info or {})
|
|
||||||
payload["endpoint"] = "reset"
|
|
||||||
self._ws.send(self._packer.pack(payload))
|
|
||||||
response = self._ws.recv()
|
|
||||||
if isinstance(response, str):
|
|
||||||
return response
|
|
||||||
decoded = msgpack_numpy.unpackb(response)
|
|
||||||
if not isinstance(decoded, dict) or decoded.get("status") != "reset successful":
|
|
||||||
raise RuntimeError(f"Unexpected reset response: {decoded!r}")
|
|
||||||
return str(decoded["status"])
|
|
||||||
|
|
||||||
def close(self) -> None:
|
|
||||||
self._ws.close()
|
|
||||||
|
|
||||||
|
|
||||||
def load_all_frames(video_path: Path) -> np.ndarray:
|
|
||||||
require_dependencies()
|
|
||||||
cap = cv2.VideoCapture(str(video_path))
|
|
||||||
frames = []
|
|
||||||
while True:
|
|
||||||
ok, frame = cap.read()
|
|
||||||
if not ok:
|
|
||||||
break
|
|
||||||
frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
|
|
||||||
cap.release()
|
|
||||||
if not frames:
|
|
||||||
raise RuntimeError(f"No frames loaded from {video_path}")
|
|
||||||
return np.stack(frames, axis=0)
|
|
||||||
|
|
||||||
|
|
||||||
def load_camera_frames(video_dir: Path) -> dict[str, np.ndarray]:
|
|
||||||
camera_frames: dict[str, np.ndarray] = {}
|
|
||||||
for camera_key, file_name in CAMERA_FILES.items():
|
|
||||||
video_path = video_dir / file_name
|
|
||||||
if not video_path.exists():
|
|
||||||
raise FileNotFoundError(f"Missing DreamZero test asset: {video_path}")
|
|
||||||
camera_frames[camera_key] = load_all_frames(video_path)
|
|
||||||
return camera_frames
|
|
||||||
|
|
||||||
|
|
||||||
def build_frame_schedule(total_frames: int, num_chunks: int) -> list[list[int]]:
|
|
||||||
chunks: list[list[int]] = []
|
|
||||||
current_frame = 23
|
|
||||||
for _ in range(num_chunks):
|
|
||||||
indices = [max(current_frame + offset, 0) for offset in RELATIVE_OFFSETS]
|
|
||||||
if indices[-1] >= total_frames:
|
|
||||||
break
|
|
||||||
chunks.append(indices)
|
|
||||||
current_frame += ACTION_HORIZON
|
|
||||||
return chunks
|
|
||||||
|
|
||||||
|
|
||||||
def make_obs_from_video(
|
|
||||||
camera_frames: dict[str, np.ndarray],
|
|
||||||
frame_indices: list[int],
|
|
||||||
*,
|
|
||||||
prompt: str,
|
|
||||||
session_id: str,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
obs: dict[str, Any] = {}
|
|
||||||
for camera_key, all_frames in camera_frames.items():
|
|
||||||
selected = all_frames[frame_indices]
|
|
||||||
obs[camera_key] = selected[0] if len(frame_indices) == 1 else selected
|
|
||||||
|
|
||||||
obs["observation/joint_position"] = np.zeros(7, dtype=np.float32)
|
|
||||||
obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32)
|
|
||||||
obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32)
|
|
||||||
obs["prompt"] = prompt
|
|
||||||
obs["session_id"] = session_id
|
|
||||||
return obs
|
|
||||||
|
|
||||||
|
|
||||||
def build_demo_observations(
|
|
||||||
camera_frames: dict[str, np.ndarray],
|
|
||||||
*,
|
|
||||||
prompt: str,
|
|
||||||
session_id: str,
|
|
||||||
num_chunks: int = 2,
|
|
||||||
) -> list[dict[str, Any]]:
|
|
||||||
if num_chunks < 1:
|
|
||||||
raise ValueError("num_chunks must be at least 1")
|
|
||||||
|
|
||||||
total_frames = min(frames.shape[0] for frames in camera_frames.values())
|
|
||||||
observations = [
|
|
||||||
make_obs_from_video(
|
|
||||||
camera_frames,
|
|
||||||
[0],
|
|
||||||
prompt=prompt,
|
|
||||||
session_id=session_id,
|
|
||||||
)
|
|
||||||
]
|
|
||||||
for indices in build_frame_schedule(total_frames, num_chunks - 1):
|
|
||||||
observations.append(
|
|
||||||
make_obs_from_video(
|
|
||||||
camera_frames,
|
|
||||||
indices,
|
|
||||||
prompt=prompt,
|
|
||||||
session_id=session_id,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return observations
|
|
||||||
|
|
||||||
|
|
||||||
def validate_session_result(
|
|
||||||
result: dict[str, Any],
|
|
||||||
*,
|
|
||||||
expected_action_horizon: int = ACTION_HORIZON,
|
|
||||||
expected_action_dim: int = DEFAULT_ACTION_DIM,
|
|
||||||
) -> None:
|
|
||||||
metadata = DreamZeroServerMetadata.from_dict(result["metadata"])
|
|
||||||
if metadata.image_resolution != (180, 320):
|
|
||||||
raise AssertionError(f"Unexpected image_resolution: {metadata.image_resolution}")
|
|
||||||
if metadata.n_external_cameras != 2:
|
|
||||||
raise AssertionError(f"Unexpected n_external_cameras: {metadata.n_external_cameras}")
|
|
||||||
if not metadata.needs_wrist_camera:
|
|
||||||
raise AssertionError("DreamZero test expects wrist camera metadata")
|
|
||||||
if metadata.action_space != "joint_position":
|
|
||||||
raise AssertionError(f"Unexpected action_space: {metadata.action_space}")
|
|
||||||
|
|
||||||
actions = result["actions"]
|
|
||||||
if len(actions) != 3:
|
|
||||||
raise AssertionError(f"Expected 3 action tensors, got {len(actions)}")
|
|
||||||
for index, action in enumerate(actions):
|
|
||||||
if action.shape != (expected_action_horizon, expected_action_dim):
|
|
||||||
raise AssertionError(
|
|
||||||
f"Action {index} shape mismatch: expected "
|
|
||||||
f"{(expected_action_horizon, expected_action_dim)}, got {action.shape}"
|
|
||||||
)
|
|
||||||
if not np.isfinite(action).all():
|
|
||||||
raise AssertionError(f"Action {index} contains non-finite values")
|
|
||||||
|
|
||||||
if result["reset_status"] != "reset successful":
|
|
||||||
raise AssertionError(f"Unexpected reset status: {result['reset_status']!r}")
|
|
||||||
|
|
||||||
|
|
||||||
def run_policy_session(
|
|
||||||
*,
|
|
||||||
host: str = DEFAULT_HOST,
|
|
||||||
port: int = DEFAULT_PORT,
|
|
||||||
path: str = DEFAULT_PATH,
|
|
||||||
video_dir: Path,
|
|
||||||
prompt: str = DEFAULT_PROMPT,
|
|
||||||
session_id: str | None = None,
|
|
||||||
num_chunks: int = 2,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
session_id = session_id or str(uuid.uuid4())
|
|
||||||
camera_frames = load_camera_frames(video_dir)
|
|
||||||
observations = build_demo_observations(
|
|
||||||
camera_frames,
|
|
||||||
prompt=prompt,
|
|
||||||
session_id=session_id,
|
|
||||||
num_chunks=num_chunks,
|
|
||||||
)
|
|
||||||
|
|
||||||
client = OpenPIWebsocketClient(host=host, port=port, path=path)
|
|
||||||
try:
|
|
||||||
metadata = client.get_server_metadata()
|
|
||||||
actions = [client.infer(obs) for obs in observations]
|
|
||||||
reset_status = client.reset({})
|
|
||||||
actions.append(client.infer(observations[0]))
|
|
||||||
return {
|
|
||||||
"metadata": metadata,
|
|
||||||
"actions": actions,
|
|
||||||
"reset_status": reset_status,
|
|
||||||
"session_id": session_id,
|
|
||||||
}
|
|
||||||
finally:
|
|
||||||
client.close()
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from vllm_omni.diffusion.models.dreamzero.utils import (
|
|
||||||
DEFAULT_CFG_SCALE,
|
|
||||||
DEFAULT_EMBODIMENT_NAME_TO_ID,
|
|
||||||
DEFAULT_NEGATIVE_PROMPT,
|
|
||||||
DEFAULT_NUM_INFERENCE_STEPS,
|
|
||||||
DEFAULT_SEED,
|
|
||||||
DEFAULT_SIGMA_SHIFT,
|
|
||||||
)
|
|
||||||
|
|
||||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
|
||||||
|
|
||||||
|
|
||||||
def test_dreamzero_default_constants_match_source_baseline():
|
|
||||||
assert DEFAULT_NUM_INFERENCE_STEPS == 16
|
|
||||||
assert DEFAULT_CFG_SCALE == 5.0
|
|
||||||
assert DEFAULT_SIGMA_SHIFT == 5.0
|
|
||||||
assert DEFAULT_SEED == 1140
|
|
||||||
assert "worst quality" in DEFAULT_NEGATIVE_PROMPT
|
|
||||||
assert DEFAULT_EMBODIMENT_NAME_TO_ID["oxe_droid"] == 17
|
|
||||||
@@ -1,46 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from tests.dreamzero import openpi_client_helper
|
|
||||||
|
|
||||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
|
||||||
|
|
||||||
|
|
||||||
class FakeMsgpackNumpy:
|
|
||||||
payloads = {}
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def packb(obj):
|
|
||||||
key = str(len(FakeMsgpackNumpy.payloads)).encode()
|
|
||||||
FakeMsgpackNumpy.payloads[key] = obj
|
|
||||||
return key
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def unpackb(data):
|
|
||||||
return FakeMsgpackNumpy.payloads[data]
|
|
||||||
|
|
||||||
|
|
||||||
def test_decode_action_response_surfaces_structured_error(monkeypatch):
|
|
||||||
monkeypatch.setattr(openpi_client_helper, "msgpack_numpy", FakeMsgpackNumpy)
|
|
||||||
payload = FakeMsgpackNumpy.packb(
|
|
||||||
{
|
|
||||||
"type": "error",
|
|
||||||
"message": "Internal inference error",
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
with pytest.raises(RuntimeError, match="Internal inference error"):
|
|
||||||
openpi_client_helper._decode_action_response(payload)
|
|
||||||
|
|
||||||
|
|
||||||
def test_decode_action_response_converts_action_payload_to_float32(monkeypatch):
|
|
||||||
monkeypatch.setattr(openpi_client_helper, "msgpack_numpy", FakeMsgpackNumpy)
|
|
||||||
payload = FakeMsgpackNumpy.packb(np.asarray([[1.0, 2.0]], dtype=np.float64))
|
|
||||||
|
|
||||||
actions = openpi_client_helper._decode_action_response(payload)
|
|
||||||
|
|
||||||
assert actions.dtype == np.float32
|
|
||||||
np.testing.assert_array_equal(actions, np.asarray([[1.0, 2.0]], dtype=np.float32))
|
|
||||||
@@ -1,59 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
from collections import OrderedDict
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
import torch
|
|
||||||
|
|
||||||
from vllm_omni.diffusion.models.dreamzero.pipeline_dreamzero import DreamZeroPipeline
|
|
||||||
from vllm_omni.diffusion.models.dreamzero.state_dreamzero import DreamZeroState
|
|
||||||
|
|
||||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
|
||||||
|
|
||||||
|
|
||||||
def _empty_pipeline() -> DreamZeroPipeline:
|
|
||||||
pipeline = DreamZeroPipeline.__new__(DreamZeroPipeline)
|
|
||||||
pipeline._states = OrderedDict()
|
|
||||||
pipeline._max_session_states = 2
|
|
||||||
return pipeline
|
|
||||||
|
|
||||||
|
|
||||||
def test_dreamzero_pipeline_state_is_session_keyed() -> None:
|
|
||||||
pipeline = _empty_pipeline()
|
|
||||||
|
|
||||||
session_a = pipeline._get_or_create_state("session-a")
|
|
||||||
session_b = pipeline._get_or_create_state("session-b")
|
|
||||||
session_a.call_count = 7
|
|
||||||
session_b.call_count = 3
|
|
||||||
|
|
||||||
assert pipeline._get_or_create_state("session-a") is session_a
|
|
||||||
assert pipeline._get_or_create_state("session-b") is session_b
|
|
||||||
assert session_a.call_count == 7
|
|
||||||
assert session_b.call_count == 3
|
|
||||||
|
|
||||||
|
|
||||||
def test_dreamzero_pipeline_state_lru_caps_retained_sessions() -> None:
|
|
||||||
pipeline = _empty_pipeline()
|
|
||||||
|
|
||||||
session_a = pipeline._get_or_create_state("session-a")
|
|
||||||
pipeline._get_or_create_state("session-b")
|
|
||||||
assert pipeline._get_or_create_state("session-a") is session_a
|
|
||||||
|
|
||||||
pipeline._get_or_create_state("session-c")
|
|
||||||
|
|
||||||
assert list(pipeline._states) == ["session-a", "session-c"]
|
|
||||||
assert "session-b" not in pipeline._states
|
|
||||||
|
|
||||||
|
|
||||||
def test_dreamzero_state_cache_access_requires_initialization() -> None:
|
|
||||||
state = DreamZeroState()
|
|
||||||
|
|
||||||
with pytest.raises(RuntimeError, match="KV caches not initialized"):
|
|
||||||
state.get_kv_caches()
|
|
||||||
|
|
||||||
with pytest.raises(RuntimeError, match="Cross-attn caches not initialized"):
|
|
||||||
state.get_crossattn_caches()
|
|
||||||
|
|
||||||
with pytest.raises(RuntimeError, match="create_kv_caches first"):
|
|
||||||
state.update_kv_cache(0, torch.empty(0))
|
|
||||||
@@ -1,381 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""
|
|
||||||
Copied from the DreamZero repository's `test_client_AR.py`:
|
|
||||||
https://github.com/dreamzero0/dreamzero/blob/main/test_client_AR.py
|
|
||||||
|
|
||||||
Kept here for end-to-end compatibility / parity testing against the vLLM
|
|
||||||
OpenPI server.
|
|
||||||
|
|
||||||
Test client for AR_droid policy server using roboarena interface.
|
|
||||||
|
|
||||||
Sends real video frames from debug_image/ directory instead of zero dummy images.
|
|
||||||
|
|
||||||
Frame schedule (matching debug_inference.py):
|
|
||||||
- Step 0 (initial): send frame [0] (1 frame, H W 3)
|
|
||||||
- Step 1: send frames [0, 7, 15, 23] (4 frames, 4 H W 3)
|
|
||||||
- Step 2: send frames [24, 31, 39, 47] (4 frames)
|
|
||||||
- Step 3: send frames [48, 55, 63, 71] (4 frames)
|
|
||||||
- ...
|
|
||||||
|
|
||||||
Expected server configuration:
|
|
||||||
- image_resolution: (180, 320)
|
|
||||||
- n_external_cameras: 2
|
|
||||||
- needs_wrist_camera: True
|
|
||||||
- action_space: "joint_position"
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
# Start server with roboarena interface:
|
|
||||||
torchrun --nproc_per_node=8 socket_test_optimized_AR.py --port 8000
|
|
||||||
|
|
||||||
# Run against the original DreamZero websocket server (default path):
|
|
||||||
python test_client_AR.py --host <server_host> --port 8000
|
|
||||||
|
|
||||||
# Run against the vLLM OpenPI server:
|
|
||||||
python test_client_AR.py --host <server_host> --port 8000 --path /v1/realtime/robot/openpi
|
|
||||||
|
|
||||||
# Use zero images instead of real video (old behavior):
|
|
||||||
python test_client_AR.py --host <server_host> --port 8000 --use-zero-images
|
|
||||||
"""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import logging
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
import uuid
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import cv2
|
|
||||||
import numpy as np
|
|
||||||
from openpi_client import msgpack_numpy
|
|
||||||
|
|
||||||
_DREAMZERO_REPO_ENV = os.environ.get("DREAMZERO_REPO")
|
|
||||||
DREAMZERO_REPO = Path(_DREAMZERO_REPO_ENV).expanduser() if _DREAMZERO_REPO_ENV else None
|
|
||||||
|
|
||||||
|
|
||||||
def _import_upstream_policy_modules():
|
|
||||||
if DREAMZERO_REPO is None:
|
|
||||||
raise ImportError("Set DREAMZERO_REPO to an upstream DreamZero checkout before using this helper.")
|
|
||||||
if DREAMZERO_REPO.exists() and str(DREAMZERO_REPO) not in sys.path:
|
|
||||||
sys.path.insert(0, str(DREAMZERO_REPO))
|
|
||||||
|
|
||||||
import eval_utils.policy_server as policy_server
|
|
||||||
from eval_utils.policy_client import WebsocketClientPolicy
|
|
||||||
|
|
||||||
return policy_server, WebsocketClientPolicy
|
|
||||||
|
|
||||||
|
|
||||||
policy_server, WebsocketClientPolicy = _import_upstream_policy_modules()
|
|
||||||
|
|
||||||
VIDEO_DIR = os.environ.get(
|
|
||||||
"DREAMZERO_VIDEO_DIR",
|
|
||||||
str(DREAMZERO_REPO / "debug_image") if DREAMZERO_REPO is not None else "debug_image",
|
|
||||||
)
|
|
||||||
|
|
||||||
# roboarena key -> video filename
|
|
||||||
CAMERA_FILES = {
|
|
||||||
"observation/exterior_image_0_left": "exterior_image_1_left.mp4",
|
|
||||||
"observation/exterior_image_1_left": "exterior_image_2_left.mp4",
|
|
||||||
"observation/wrist_image_left": "wrist_image_left.mp4",
|
|
||||||
}
|
|
||||||
|
|
||||||
# Frame schedule constants (matching debug_inference.py)
|
|
||||||
RELATIVE_OFFSETS = [-23, -16, -8, 0]
|
|
||||||
ACTION_HORIZON = 24
|
|
||||||
DEFAULT_WEBSOCKET_PATH = ""
|
|
||||||
|
|
||||||
|
|
||||||
class OpenPIWebsocketClientPolicy(WebsocketClientPolicy):
|
|
||||||
"""DreamZero websocket client with a configurable path suffix.
|
|
||||||
|
|
||||||
The original DreamZero client connects to ``ws://host:port``.
|
|
||||||
vLLM serves the compatible robot policy endpoint at
|
|
||||||
``/v1/realtime/robot/openpi`` when ``path`` is set accordingly.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
host: str = "0.0.0.0",
|
|
||||||
port: int = 8000,
|
|
||||||
path: str = DEFAULT_WEBSOCKET_PATH,
|
|
||||||
) -> None:
|
|
||||||
self._uri = f"ws://{host}:{port}{path}"
|
|
||||||
self._packer = msgpack_numpy.Packer()
|
|
||||||
self._ws, self._server_metadata = self._wait_for_server()
|
|
||||||
|
|
||||||
|
|
||||||
def load_all_frames(video_path: str) -> np.ndarray:
|
|
||||||
"""Load all frames from a video file. Returns (N, H, W, 3) uint8 array (RGB)."""
|
|
||||||
cap = cv2.VideoCapture(video_path)
|
|
||||||
frames = []
|
|
||||||
while True:
|
|
||||||
ret, frame = cap.read()
|
|
||||||
if not ret:
|
|
||||||
break
|
|
||||||
frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
|
|
||||||
cap.release()
|
|
||||||
if not frames:
|
|
||||||
raise RuntimeError(f"No frames loaded from {video_path}")
|
|
||||||
return np.stack(frames, axis=0)
|
|
||||||
|
|
||||||
|
|
||||||
def load_camera_frames() -> dict[str, np.ndarray]:
|
|
||||||
"""Load all video frames for each camera from the debug_image/ directory.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Dict mapping roboarena camera keys to (N, H, W, 3) uint8 arrays.
|
|
||||||
"""
|
|
||||||
camera_frames: dict[str, np.ndarray] = {}
|
|
||||||
for cam_key, fname in CAMERA_FILES.items():
|
|
||||||
path = os.path.join(VIDEO_DIR, fname)
|
|
||||||
camera_frames[cam_key] = load_all_frames(path)
|
|
||||||
logging.info(f"Loaded {cam_key}: {camera_frames[cam_key].shape}")
|
|
||||||
return camera_frames
|
|
||||||
|
|
||||||
|
|
||||||
def build_frame_schedule(total_frames: int, num_chunks: int) -> list[list[int]]:
|
|
||||||
"""Build the frame index schedule for multi-frame chunks.
|
|
||||||
|
|
||||||
Returns a list of frame-index lists. Each inner list has 4 indices.
|
|
||||||
"""
|
|
||||||
chunks: list[list[int]] = []
|
|
||||||
current_frame = 23 # first anchor frame
|
|
||||||
for _ in range(num_chunks):
|
|
||||||
indices = [max(current_frame + off, 0) for off in RELATIVE_OFFSETS]
|
|
||||||
if indices[-1] >= total_frames:
|
|
||||||
logging.info(f"Frame {indices[-1]} >= {total_frames}, stopping at {len(chunks)} chunks")
|
|
||||||
break
|
|
||||||
chunks.append(indices)
|
|
||||||
current_frame += ACTION_HORIZON
|
|
||||||
return chunks
|
|
||||||
|
|
||||||
|
|
||||||
def _make_obs_from_video(
|
|
||||||
camera_frames: dict[str, np.ndarray],
|
|
||||||
frame_indices: list[int],
|
|
||||||
prompt: str,
|
|
||||||
session_id: str,
|
|
||||||
) -> dict:
|
|
||||||
"""Build an observation dict from real video frames.
|
|
||||||
|
|
||||||
For 1 frame: each image key is (H, W, 3).
|
|
||||||
For 4 frames: each image key is (4, H, W, 3).
|
|
||||||
"""
|
|
||||||
obs: dict = {}
|
|
||||||
for cam_key, all_frames in camera_frames.items():
|
|
||||||
selected = all_frames[frame_indices] # (T, H, W, 3)
|
|
||||||
if len(frame_indices) == 1:
|
|
||||||
selected = selected[0] # (H, W, 3)
|
|
||||||
obs[cam_key] = selected
|
|
||||||
|
|
||||||
obs["observation/joint_position"] = np.zeros(7, dtype=np.float32)
|
|
||||||
obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32)
|
|
||||||
obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32)
|
|
||||||
obs["prompt"] = prompt
|
|
||||||
obs["session_id"] = session_id
|
|
||||||
return obs
|
|
||||||
|
|
||||||
|
|
||||||
def _make_zero_observation(
|
|
||||||
server_config: policy_server.PolicyServerConfig,
|
|
||||||
prompt: str = "pick up the object",
|
|
||||||
session_id: str | None = None,
|
|
||||||
) -> dict:
|
|
||||||
"""Create a dummy observation matching AR_droid expectations.
|
|
||||||
|
|
||||||
AR_droid expects:
|
|
||||||
- 2 external cameras (exterior_image_0_left, exterior_image_1_left)
|
|
||||||
- 1 wrist camera (wrist_image_left)
|
|
||||||
- Image resolution: 180x320 (H x W)
|
|
||||||
- joint_position: 7 DoF
|
|
||||||
- gripper_position: 1 DoF
|
|
||||||
"""
|
|
||||||
obs = {}
|
|
||||||
|
|
||||||
# Determine image resolution
|
|
||||||
if server_config.image_resolution is not None:
|
|
||||||
h, w = server_config.image_resolution
|
|
||||||
else:
|
|
||||||
# Default for AR_droid
|
|
||||||
h, w = 180, 320
|
|
||||||
|
|
||||||
# External cameras (0-indexed in roboarena)
|
|
||||||
for i in range(server_config.n_external_cameras):
|
|
||||||
obs[f"observation/exterior_image_{i}_left"] = np.zeros((h, w, 3), dtype=np.uint8)
|
|
||||||
if server_config.needs_stereo_camera:
|
|
||||||
obs[f"observation/exterior_image_{i}_right"] = np.zeros((h, w, 3), dtype=np.uint8)
|
|
||||||
|
|
||||||
# Wrist camera
|
|
||||||
if server_config.needs_wrist_camera:
|
|
||||||
obs["observation/wrist_image_left"] = np.zeros((h, w, 3), dtype=np.uint8)
|
|
||||||
if server_config.needs_stereo_camera:
|
|
||||||
obs["observation/wrist_image_right"] = np.zeros((h, w, 3), dtype=np.uint8)
|
|
||||||
|
|
||||||
# Session ID - should be passed in to ensure consistency within a session
|
|
||||||
if server_config.needs_session_id:
|
|
||||||
import uuid
|
|
||||||
|
|
||||||
# Generate unique session ID if not provided
|
|
||||||
obs["session_id"] = session_id if session_id else str(uuid.uuid4())
|
|
||||||
|
|
||||||
# State observations (AR_droid: 7 DoF arm + 1 gripper)
|
|
||||||
obs["observation/joint_position"] = np.zeros(7, dtype=np.float32)
|
|
||||||
obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32)
|
|
||||||
obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32)
|
|
||||||
|
|
||||||
# Language prompt
|
|
||||||
obs["prompt"] = prompt
|
|
||||||
|
|
||||||
return obs
|
|
||||||
|
|
||||||
|
|
||||||
def test_ar_droid_policy_server(
|
|
||||||
host: str = "localhost",
|
|
||||||
port: int = 8000,
|
|
||||||
path: str = DEFAULT_WEBSOCKET_PATH,
|
|
||||||
num_chunks: int = 15,
|
|
||||||
prompt: str = "Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan",
|
|
||||||
use_zero_images: bool = False,
|
|
||||||
):
|
|
||||||
"""Test the AR_droid policy server with roboarena interface.
|
|
||||||
|
|
||||||
When use_zero_images is False (default), loads real video frames from
|
|
||||||
debug_image/ and follows the frame schedule from debug_inference.py.
|
|
||||||
"""
|
|
||||||
logging.info(f"Connecting to AR_droid server at ws://{host}:{port}{path} ...")
|
|
||||||
|
|
||||||
client = OpenPIWebsocketClientPolicy(host=host, port=port, path=path)
|
|
||||||
|
|
||||||
# Validate server metadata
|
|
||||||
metadata = client.get_server_metadata()
|
|
||||||
logging.info(f"Server metadata: {metadata}")
|
|
||||||
assert isinstance(metadata, dict), "Metadata should be a dict"
|
|
||||||
|
|
||||||
try:
|
|
||||||
server_config = policy_server.PolicyServerConfig(**metadata)
|
|
||||||
except Exception as e:
|
|
||||||
logging.error(f"Error parsing metadata: {e}")
|
|
||||||
raise e
|
|
||||||
|
|
||||||
# Validate expected AR_droid configuration
|
|
||||||
logging.info(f"Server config: {server_config}")
|
|
||||||
assert server_config.n_external_cameras == 2, f"Expected 2 external cameras, got {server_config.n_external_cameras}"
|
|
||||||
assert server_config.needs_wrist_camera, "Expected wrist camera to be enabled"
|
|
||||||
assert server_config.action_space == "joint_position", (
|
|
||||||
f"Expected joint_position action space, got {server_config.action_space}"
|
|
||||||
)
|
|
||||||
|
|
||||||
logging.info("Server configuration validated for AR_droid")
|
|
||||||
|
|
||||||
# Generate unique session ID for this test run
|
|
||||||
session_id = str(uuid.uuid4())
|
|
||||||
logging.info(f"Session ID: {session_id}")
|
|
||||||
|
|
||||||
# ── Zero-image fallback mode ──────────────────────────────────────
|
|
||||||
if use_zero_images:
|
|
||||||
logging.info("Using ZERO dummy images (legacy mode)")
|
|
||||||
for i in range(num_chunks):
|
|
||||||
obs = _make_zero_observation(server_config, prompt=prompt, session_id=session_id)
|
|
||||||
logging.info(f"Inference {i + 1}/{num_chunks}: prompt='{prompt}'")
|
|
||||||
t0 = time.time()
|
|
||||||
actions = client.infer(obs)
|
|
||||||
dt = time.time() - t0
|
|
||||||
_log_action(actions, dt)
|
|
||||||
|
|
||||||
logging.info("Sending reset...")
|
|
||||||
client.reset({})
|
|
||||||
logging.info("Done (zero-image mode).")
|
|
||||||
return
|
|
||||||
|
|
||||||
# ── Real video frame mode ─────────────────────────────────────────
|
|
||||||
logging.info("Loading real video frames from debug_image/ directory")
|
|
||||||
camera_frames = load_camera_frames()
|
|
||||||
|
|
||||||
total_frames = min(v.shape[0] for v in camera_frames.values())
|
|
||||||
logging.info(f"Total frames available: {total_frames}")
|
|
||||||
|
|
||||||
# Build frame schedule
|
|
||||||
chunks = build_frame_schedule(total_frames, num_chunks)
|
|
||||||
|
|
||||||
logging.info("Frame schedule:")
|
|
||||||
logging.info(" Initial: [0]")
|
|
||||||
for i, indices in enumerate(chunks):
|
|
||||||
logging.info(f" Chunk {i}: {indices}")
|
|
||||||
|
|
||||||
# Step 0: initial single frame
|
|
||||||
logging.info("=== Initial: frame [0] ===")
|
|
||||||
obs = _make_obs_from_video(camera_frames, [0], prompt, session_id)
|
|
||||||
t0 = time.time()
|
|
||||||
actions = client.infer(obs)
|
|
||||||
dt = time.time() - t0
|
|
||||||
_log_action(actions, dt)
|
|
||||||
|
|
||||||
# Subsequent chunks: send 4 frames at a time
|
|
||||||
for chunk_idx, frame_indices in enumerate(chunks):
|
|
||||||
logging.info(f"=== Chunk {chunk_idx}: frames {frame_indices} ===")
|
|
||||||
obs = _make_obs_from_video(camera_frames, frame_indices, prompt, session_id)
|
|
||||||
t0 = time.time()
|
|
||||||
actions = client.infer(obs)
|
|
||||||
dt = time.time() - t0
|
|
||||||
_log_action(actions, dt)
|
|
||||||
|
|
||||||
# Reset triggers video save on the server
|
|
||||||
logging.info("Sending reset to save video...")
|
|
||||||
client.reset({})
|
|
||||||
|
|
||||||
logging.info("Done.")
|
|
||||||
|
|
||||||
|
|
||||||
def _log_action(actions: np.ndarray, dt: float) -> None:
|
|
||||||
"""Pretty-print action shape, range, and timing."""
|
|
||||||
assert isinstance(actions, np.ndarray), f"Expected numpy array, got {type(actions)}"
|
|
||||||
assert actions.ndim == 2, f"Expected 2D array, got shape {actions.shape}"
|
|
||||||
assert actions.shape[-1] == 8, f"Expected 8 action dims (7 joints + 1 gripper), got {actions.shape[-1]}"
|
|
||||||
logging.info(f" Action shape: {actions.shape}, range: [{actions.min():.4f}, {actions.max():.4f}], time: {dt:.2f}s")
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
parser = argparse.ArgumentParser(description="Test AR_droid policy server with real video frames from debug_image/")
|
|
||||||
parser.add_argument("--host", default="localhost", help="Server hostname")
|
|
||||||
parser.add_argument("--port", type=int, default=8000, help="Server port")
|
|
||||||
parser.add_argument(
|
|
||||||
"--path",
|
|
||||||
default=DEFAULT_WEBSOCKET_PATH,
|
|
||||||
help="WebSocket path suffix (default: empty string for the original DreamZero server)",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--num-chunks",
|
|
||||||
type=int,
|
|
||||||
default=15,
|
|
||||||
help="Number of 4-frame chunks to send after the initial frame (default: 15)",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--prompt",
|
|
||||||
default="Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan",
|
|
||||||
help="Language prompt for the policy",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--use-zero-images",
|
|
||||||
action="store_true",
|
|
||||||
help="Use zero dummy images instead of real video frames (legacy mode)",
|
|
||||||
)
|
|
||||||
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
logging.basicConfig(
|
|
||||||
level=logging.INFO,
|
|
||||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
|
||||||
)
|
|
||||||
|
|
||||||
test_ar_droid_policy_server(
|
|
||||||
host=args.host,
|
|
||||||
port=args.port,
|
|
||||||
path=args.path,
|
|
||||||
num_chunks=args.num_chunks,
|
|
||||||
prompt=args.prompt,
|
|
||||||
use_zero_images=args.use_zero_images,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -1,358 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
"""Formal OpenPI end-to-end parity: upstream DreamZero server vs `vllm serve`.
|
|
||||||
|
|
||||||
This test uses DreamZero's own client-side observation builders from
|
|
||||||
`${DREAMZERO_REPO}/test_client_AR.py`, and client-side websocket protocol from
|
|
||||||
`${DREAMZERO_REPO}/eval_utils/policy_client.py`.
|
|
||||||
|
|
||||||
The only client-side adaptation for vLLM is the websocket path:
|
|
||||||
DreamZero's upstream server serves at `/`, while vLLM serves OpenPI at
|
|
||||||
`/v1/realtime/robot/openpi`.
|
|
||||||
|
|
||||||
Current scope for this test:
|
|
||||||
- default two-GPU run (`nproc_per_node=2` on upstream, `--cfg-parallel-size 2` on `vllm serve`)
|
|
||||||
- non-`torch.compile` (upstream launched through
|
|
||||||
`upstream_socket_server_no_compile.py`, vLLM with `--enforce-eager`)
|
|
||||||
- non-DiT-cache / non-skip-schedule (`NUM_DIT_STEPS=16`)
|
|
||||||
|
|
||||||
Serving contract locked by this test:
|
|
||||||
- upstream DreamZero still boots from the local checkpoint directory
|
|
||||||
- vLLM boots from the official DreamZero HF repo name (`GEAR-Dreams/DreamZero-DROID`)
|
|
||||||
rather than a prepared local bundle path
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import os
|
|
||||||
import shutil
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
import pytest
|
|
||||||
import torch
|
|
||||||
|
|
||||||
from tests.helpers.runtime import get_open_port
|
|
||||||
|
|
||||||
msgpack_numpy = pytest.importorskip("openpi_client.msgpack_numpy")
|
|
||||||
|
|
||||||
_DREAMZERO_REPO_ENV = os.environ.get("DREAMZERO_REPO")
|
|
||||||
DREAMZERO_REPO = Path(_DREAMZERO_REPO_ENV).expanduser() if _DREAMZERO_REPO_ENV else None
|
|
||||||
if DREAMZERO_REPO is not None and str(DREAMZERO_REPO) not in sys.path:
|
|
||||||
sys.path.insert(0, str(DREAMZERO_REPO))
|
|
||||||
|
|
||||||
try:
|
|
||||||
import test_client_AR as dreamzero_client
|
|
||||||
from eval_utils.policy_client import WebsocketClientPolicy
|
|
||||||
except Exception: # pragma: no cover - guarded by pytest skip below
|
|
||||||
dreamzero_client = None
|
|
||||||
WebsocketClientPolicy = None
|
|
||||||
_BaseWebsocketClientPolicy = WebsocketClientPolicy if WebsocketClientPolicy is not None else object
|
|
||||||
|
|
||||||
CHECKPOINT_DIR = DREAMZERO_REPO / "checkpoints" / "dreamzero" if DREAMZERO_REPO is not None else None
|
|
||||||
VLLM_MODEL = os.environ.get("VLLM_DREAMZERO_MODEL", "GEAR-Dreams/DreamZero-DROID")
|
|
||||||
SERVICE_READY_TIMEOUT_S = int(os.environ.get("OPENPI_SERVICE_READY_TIMEOUT_S", "900"))
|
|
||||||
PROMPT = "Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan"
|
|
||||||
SESSION_ID = "openpi-e2e-parity-session"
|
|
||||||
|
|
||||||
pytestmark = [
|
|
||||||
pytest.mark.skipif(not torch.cuda.is_available(), reason="GPU required"),
|
|
||||||
pytest.mark.skipif(
|
|
||||||
dreamzero_client is None or WebsocketClientPolicy is None,
|
|
||||||
reason="DreamZero client modules are required on PYTHONPATH",
|
|
||||||
),
|
|
||||||
pytest.mark.skipif(
|
|
||||||
DREAMZERO_REPO is None or not DREAMZERO_REPO.exists(),
|
|
||||||
reason="DreamZero source repo is required at DREAMZERO_REPO",
|
|
||||||
),
|
|
||||||
pytest.mark.skipif(
|
|
||||||
CHECKPOINT_DIR is None or not CHECKPOINT_DIR.exists(), reason="DreamZero local checkpoint is required"
|
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
class OpenPIWebsocketClientPolicy(_BaseWebsocketClientPolicy):
|
|
||||||
"""DreamZero client protocol with an OpenPI websocket path suffix."""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
host: str = "127.0.0.1",
|
|
||||||
port: int = 8000,
|
|
||||||
path: str = "/v1/realtime/robot/openpi",
|
|
||||||
) -> None:
|
|
||||||
self._uri = f"ws://{host}:{port}{path}"
|
|
||||||
self._packer = msgpack_numpy.Packer()
|
|
||||||
self._ws, self._server_metadata = self._wait_for_server()
|
|
||||||
|
|
||||||
|
|
||||||
def _vllm_executable() -> str:
|
|
||||||
fallback = Path(sys.executable).with_name("vllm")
|
|
||||||
if fallback.exists():
|
|
||||||
return str(fallback)
|
|
||||||
exe = shutil.which("vllm")
|
|
||||||
if exe:
|
|
||||||
return exe
|
|
||||||
raise FileNotFoundError("Unable to locate `vllm` executable in current environment.")
|
|
||||||
|
|
||||||
|
|
||||||
def _cfg_parallel_size() -> int:
|
|
||||||
return int(os.environ.get("OPENPI_E2E_CFG_PARALLEL_SIZE", "2"))
|
|
||||||
|
|
||||||
|
|
||||||
def _pick_test_gpus() -> list[str]:
|
|
||||||
cfg_parallel_size = _cfg_parallel_size()
|
|
||||||
override = os.environ.get("OPENPI_E2E_GPUS") or os.environ.get("OPENPI_E2E_GPU")
|
|
||||||
if override is not None:
|
|
||||||
gpus = [part.strip() for part in override.split(",") if part.strip()]
|
|
||||||
if not gpus:
|
|
||||||
raise ValueError("OPENPI_E2E_GPUS is set but empty.")
|
|
||||||
if len(gpus) < cfg_parallel_size:
|
|
||||||
raise RuntimeError(f"Need {cfg_parallel_size} GPUs, but OPENPI_E2E_GPUS only provided {gpus}.")
|
|
||||||
return gpus
|
|
||||||
|
|
||||||
query = subprocess.check_output(
|
|
||||||
[
|
|
||||||
"nvidia-smi",
|
|
||||||
"--query-gpu=index,memory.used",
|
|
||||||
"--format=csv,noheader,nounits",
|
|
||||||
],
|
|
||||||
text=True,
|
|
||||||
)
|
|
||||||
gpu_rows = []
|
|
||||||
for line in query.strip().splitlines():
|
|
||||||
gpu_index, used_mb = [part.strip() for part in line.split(",", maxsplit=1)]
|
|
||||||
gpu_rows.append((int(used_mb), gpu_index))
|
|
||||||
gpu_rows.sort()
|
|
||||||
gpus = [gpu_index for _, gpu_index in gpu_rows[: max(cfg_parallel_size, 1)]]
|
|
||||||
if len(gpus) < cfg_parallel_size:
|
|
||||||
raise RuntimeError(
|
|
||||||
f"Need {cfg_parallel_size} GPUs for cfg_parallel_size={cfg_parallel_size}, "
|
|
||||||
f"but found only {len(gpus)} candidates."
|
|
||||||
)
|
|
||||||
return gpus
|
|
||||||
|
|
||||||
|
|
||||||
def _torchrun_argv(script: str, port: int) -> list[str]:
|
|
||||||
return [
|
|
||||||
sys.executable,
|
|
||||||
"-m",
|
|
||||||
"torch.distributed.run",
|
|
||||||
"--standalone",
|
|
||||||
"--nproc_per_node",
|
|
||||||
str(_cfg_parallel_size()),
|
|
||||||
script,
|
|
||||||
"--port",
|
|
||||||
str(port),
|
|
||||||
"--model_path",
|
|
||||||
str(CHECKPOINT_DIR),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def _run_upstream_service(port: int, log_path: Path) -> subprocess.Popen[str]:
|
|
||||||
env = os.environ.copy()
|
|
||||||
env.setdefault("PYTHONPATH", "")
|
|
||||||
env["PYTHONPATH"] = f"{Path.cwd()}:{DREAMZERO_REPO}:{env['PYTHONPATH']}".rstrip(":")
|
|
||||||
env["CUDA_VISIBLE_DEVICES"] = ",".join(_pick_test_gpus())
|
|
||||||
env.setdefault("NO_ALBUMENTATIONS_UPDATE", "1")
|
|
||||||
env["ATTENTION_BACKEND"] = "torch"
|
|
||||||
env.setdefault("ENABLE_TENSORRT", "false")
|
|
||||||
env["ENABLE_DIT_CACHE"] = "false"
|
|
||||||
env["NUM_DIT_STEPS"] = "16"
|
|
||||||
env["DYNAMIC_CACHE_SCHEDULE"] = "false"
|
|
||||||
argv = _torchrun_argv(
|
|
||||||
str(Path("tests/dreamzero/upstream/upstream_socket_server_no_compile.py")),
|
|
||||||
port,
|
|
||||||
)
|
|
||||||
log_file = log_path.open("w")
|
|
||||||
proc = subprocess.Popen(
|
|
||||||
argv,
|
|
||||||
stdout=log_file,
|
|
||||||
stderr=subprocess.STDOUT,
|
|
||||||
text=True,
|
|
||||||
cwd=str(Path.cwd()),
|
|
||||||
env=env,
|
|
||||||
)
|
|
||||||
proc._codex_log_file = log_file # type: ignore[attr-defined]
|
|
||||||
return proc
|
|
||||||
|
|
||||||
|
|
||||||
def _run_vllm_service(port: int, log_path: Path) -> subprocess.Popen[str]:
|
|
||||||
env = os.environ.copy()
|
|
||||||
gpus = _pick_test_gpus()
|
|
||||||
cfg_parallel_size = _cfg_parallel_size()
|
|
||||||
if cfg_parallel_size > len(gpus):
|
|
||||||
raise RuntimeError(
|
|
||||||
f"cfg_parallel_size={cfg_parallel_size} requires at least {cfg_parallel_size} GPUs, but only got {gpus}."
|
|
||||||
)
|
|
||||||
env["CUDA_VISIBLE_DEVICES"] = ",".join(gpus[:cfg_parallel_size])
|
|
||||||
env.setdefault("ATTENTION_BACKEND", "torch")
|
|
||||||
env.setdefault("DIFFUSION_ATTENTION_BACKEND", "TORCH_SDPA")
|
|
||||||
env.setdefault("MASTER_PORT", str(get_open_port()))
|
|
||||||
argv = [
|
|
||||||
_vllm_executable(),
|
|
||||||
"serve",
|
|
||||||
VLLM_MODEL,
|
|
||||||
"--omni",
|
|
||||||
"--host",
|
|
||||||
"127.0.0.1",
|
|
||||||
"--port",
|
|
||||||
str(port),
|
|
||||||
"--served-model-name",
|
|
||||||
"dreamzero-droid",
|
|
||||||
"--enforce-eager",
|
|
||||||
]
|
|
||||||
if cfg_parallel_size > 1:
|
|
||||||
argv.extend(["--cfg-parallel-size", str(cfg_parallel_size)])
|
|
||||||
log_file = log_path.open("w")
|
|
||||||
proc = subprocess.Popen(
|
|
||||||
argv,
|
|
||||||
stdout=log_file,
|
|
||||||
stderr=subprocess.STDOUT,
|
|
||||||
text=True,
|
|
||||||
env=env,
|
|
||||||
cwd=str(Path.cwd()),
|
|
||||||
)
|
|
||||||
proc._codex_log_file = log_file # type: ignore[attr-defined]
|
|
||||||
return proc
|
|
||||||
|
|
||||||
|
|
||||||
def _stop_process(proc: subprocess.Popen[str]) -> None:
|
|
||||||
log_file = getattr(proc, "_codex_log_file", None)
|
|
||||||
if proc.poll() is None:
|
|
||||||
proc.terminate()
|
|
||||||
try:
|
|
||||||
proc.wait(timeout=30)
|
|
||||||
except subprocess.TimeoutExpired: # pragma: no cover - cleanup path
|
|
||||||
proc.kill()
|
|
||||||
proc.wait(timeout=10)
|
|
||||||
if log_file is not None:
|
|
||||||
log_file.close()
|
|
||||||
|
|
||||||
|
|
||||||
def _build_obs_sequence() -> tuple[dict, dict]:
|
|
||||||
camera_frames = dreamzero_client.load_camera_frames()
|
|
||||||
chunks = dreamzero_client.build_frame_schedule(
|
|
||||||
min(v.shape[0] for v in camera_frames.values()),
|
|
||||||
1,
|
|
||||||
)
|
|
||||||
obs0 = dreamzero_client._make_obs_from_video(camera_frames, [0], PROMPT, SESSION_ID)
|
|
||||||
obs1 = dreamzero_client._make_obs_from_video(camera_frames, chunks[0], PROMPT, SESSION_ID)
|
|
||||||
return obs0, obs1
|
|
||||||
|
|
||||||
|
|
||||||
def _wait_for_client_ready(client_factory, timeout_s: float, proc=None, log_path: Path | None = None):
|
|
||||||
deadline = time.time() + timeout_s
|
|
||||||
last_err: Exception | None = None
|
|
||||||
while time.time() < deadline:
|
|
||||||
if proc is not None and proc.poll() is not None:
|
|
||||||
details = ""
|
|
||||||
if log_path is not None and log_path.exists():
|
|
||||||
details = log_path.read_text(errors="replace")[-8000:]
|
|
||||||
raise RuntimeError(f"Service exited before becoming ready with code {proc.returncode}.\n{details}")
|
|
||||||
try:
|
|
||||||
return client_factory()
|
|
||||||
except Exception as exc: # pragma: no cover - retry path
|
|
||||||
last_err = exc
|
|
||||||
time.sleep(1)
|
|
||||||
raise TimeoutError(f"Timed out waiting for websocket service: {last_err}")
|
|
||||||
|
|
||||||
|
|
||||||
def _collect_outputs_with_client(client) -> tuple[dict, list[np.ndarray]]:
|
|
||||||
metadata = client.get_server_metadata()
|
|
||||||
obs0, obs1 = _build_obs_sequence()
|
|
||||||
outputs = [
|
|
||||||
client.infer(dict(obs0)),
|
|
||||||
client.infer(dict(obs1)),
|
|
||||||
]
|
|
||||||
assert _normalize_reset_response(client.reset({})) == "reset successful"
|
|
||||||
outputs.append(client.infer(dict(obs0)))
|
|
||||||
client._ws.close()
|
|
||||||
return metadata, outputs
|
|
||||||
|
|
||||||
|
|
||||||
def _normalize_reset_response(response) -> str:
|
|
||||||
if isinstance(response, str):
|
|
||||||
return response
|
|
||||||
decoded = msgpack_numpy.unpackb(response)
|
|
||||||
if isinstance(decoded, dict):
|
|
||||||
return str(decoded.get("status"))
|
|
||||||
return str(decoded)
|
|
||||||
|
|
||||||
|
|
||||||
def _normalize_metadata(metadata: dict) -> dict:
|
|
||||||
normalized = dict(metadata)
|
|
||||||
if isinstance(normalized.get("image_resolution"), tuple):
|
|
||||||
normalized["image_resolution"] = list(normalized["image_resolution"])
|
|
||||||
return normalized
|
|
||||||
|
|
||||||
|
|
||||||
def _assert_logs_clean(log_path: Path) -> None:
|
|
||||||
text = log_path.read_text(errors="replace")
|
|
||||||
if "SignalException: Process" in text and "got signal: 15" in text:
|
|
||||||
text = text.split("Traceback (most recent call last):", 1)[0]
|
|
||||||
assert "Traceback" not in text, text
|
|
||||||
assert "RuntimeError:" not in text, text
|
|
||||||
|
|
||||||
|
|
||||||
def _assert_upstream_log_matches_vllm_baseline(log_path: Path) -> None:
|
|
||||||
text = log_path.read_text(errors="replace")
|
|
||||||
assert "DIT Compute Steps 8 steps" not in text, text
|
|
||||||
assert "DIT Compute Steps 16 steps" in text, text
|
|
||||||
|
|
||||||
|
|
||||||
def test_openpi_service_matches_upstream_server_noncompile(tmp_path: Path) -> None:
|
|
||||||
expected_metadata = {
|
|
||||||
"image_resolution": [180, 320],
|
|
||||||
"n_external_cameras": 2,
|
|
||||||
"needs_wrist_camera": True,
|
|
||||||
"needs_stereo_camera": False,
|
|
||||||
"needs_session_id": True,
|
|
||||||
"action_space": "joint_position",
|
|
||||||
}
|
|
||||||
|
|
||||||
upstream_port = get_open_port()
|
|
||||||
upstream_log = tmp_path / "dreamzero_upstream.log"
|
|
||||||
upstream_proc = _run_upstream_service(upstream_port, upstream_log)
|
|
||||||
try:
|
|
||||||
upstream_client = _wait_for_client_ready(
|
|
||||||
lambda: WebsocketClientPolicy(host="127.0.0.1", port=upstream_port),
|
|
||||||
timeout_s=SERVICE_READY_TIMEOUT_S,
|
|
||||||
proc=upstream_proc,
|
|
||||||
log_path=upstream_log,
|
|
||||||
)
|
|
||||||
upstream_metadata, upstream_outputs = _collect_outputs_with_client(upstream_client)
|
|
||||||
finally:
|
|
||||||
_stop_process(upstream_proc)
|
|
||||||
_assert_logs_clean(upstream_log)
|
|
||||||
_assert_upstream_log_matches_vllm_baseline(upstream_log)
|
|
||||||
|
|
||||||
vllm_port = get_open_port()
|
|
||||||
vllm_log = tmp_path / "vllm_openpi.log"
|
|
||||||
vllm_proc = _run_vllm_service(vllm_port, vllm_log)
|
|
||||||
try:
|
|
||||||
vllm_client = _wait_for_client_ready(
|
|
||||||
lambda: OpenPIWebsocketClientPolicy(host="127.0.0.1", port=vllm_port),
|
|
||||||
timeout_s=SERVICE_READY_TIMEOUT_S,
|
|
||||||
proc=vllm_proc,
|
|
||||||
log_path=vllm_log,
|
|
||||||
)
|
|
||||||
vllm_metadata, vllm_outputs = _collect_outputs_with_client(vllm_client)
|
|
||||||
finally:
|
|
||||||
_stop_process(vllm_proc)
|
|
||||||
_assert_logs_clean(vllm_log)
|
|
||||||
|
|
||||||
assert _normalize_metadata(upstream_metadata) == expected_metadata
|
|
||||||
assert _normalize_metadata(vllm_metadata) == expected_metadata
|
|
||||||
|
|
||||||
for idx, (actual, expected) in enumerate(zip(vllm_outputs, upstream_outputs, strict=True)):
|
|
||||||
np.testing.assert_allclose(
|
|
||||||
actual,
|
|
||||||
expected,
|
|
||||||
rtol=1e-2,
|
|
||||||
atol=1e-3,
|
|
||||||
err_msg=f"OpenPI step {idx} output mismatch",
|
|
||||||
)
|
|
||||||
@@ -1,184 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
"""Launch the upstream DreamZero websocket server with `torch.compile` disabled.
|
|
||||||
|
|
||||||
This wrapper is meant for formal parity tests against `vllm serve --omni`.
|
|
||||||
It monkeypatches `torch.compile` before importing DreamZero modules so that
|
|
||||||
all import-time decorators and `post_initialize()` compile calls become eager.
|
|
||||||
|
|
||||||
For the current DreamZero port baseline we also disable all upstream DiT cache
|
|
||||||
and step-skipping behavior so the reference server matches vLLM's current
|
|
||||||
eager/no-skip implementation:
|
|
||||||
- `ENABLE_DIT_CACHE=false`
|
|
||||||
- `NUM_DIT_STEPS=16`
|
|
||||||
- `DYNAMIC_CACHE_SCHEDULE=false`
|
|
||||||
|
|
||||||
In our CI/dev environment we also do not install Transformer Engine or
|
|
||||||
FlashAttention, so upstream's default `socket_test_optimized_AR.main()` path
|
|
||||||
(`ATTENTION_BACKEND=TE` and then fallback to `FA2`) cannot execute. To keep the
|
|
||||||
formal server-vs-server test runnable without pulling in those heavyweight
|
|
||||||
optional deps, this wrapper reproduces upstream `main()` but pins attention to
|
|
||||||
PyTorch SDPA (`ATTENTION_BACKEND=torch`) for this subprocess only.
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
PYTHONPATH="${DREAMZERO_REPO}" \\
|
|
||||||
.venv/bin/python -m torch.distributed.run --standalone --nproc_per_node=2 \\
|
|
||||||
tests/dreamzero/upstream/upstream_socket_server_no_compile.py --port 18081 \\
|
|
||||||
--model_path "${DREAMZERO_REPO}/checkpoints/dreamzero"
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import torch
|
|
||||||
import torch.nn.functional as F
|
|
||||||
|
|
||||||
|
|
||||||
def _identity_compile(*args, **kwargs):
|
|
||||||
if args and callable(args[0]) and len(args) == 1 and not kwargs:
|
|
||||||
return args[0]
|
|
||||||
|
|
||||||
def deco(fn):
|
|
||||||
return fn
|
|
||||||
|
|
||||||
return deco
|
|
||||||
|
|
||||||
|
|
||||||
torch.compile = _identity_compile
|
|
||||||
|
|
||||||
os.environ.setdefault("NO_ALBUMENTATIONS_UPDATE", "1")
|
|
||||||
|
|
||||||
DREAMZERO_REPO_ENV = os.environ.get("DREAMZERO_REPO")
|
|
||||||
if not DREAMZERO_REPO_ENV:
|
|
||||||
raise RuntimeError("Set DREAMZERO_REPO to an upstream DreamZero checkout before launching this helper.")
|
|
||||||
DREAMZERO_REPO = Path(DREAMZERO_REPO_ENV).expanduser()
|
|
||||||
if str(DREAMZERO_REPO) not in sys.path:
|
|
||||||
sys.path.insert(0, str(DREAMZERO_REPO))
|
|
||||||
|
|
||||||
import socket_test_optimized_AR as upstream # noqa: E402
|
|
||||||
import tyro # noqa: E402
|
|
||||||
from groot.vla.model.dreamzero.modules import attention as upstream_attention # noqa: E402
|
|
||||||
from groot.vla.model.dreamzero.modules import wan2_1_submodule as upstream_submodule # noqa: E402
|
|
||||||
from groot.vla.model.dreamzero.modules import wan_video_dit as upstream_wan_video_dit # noqa: E402
|
|
||||||
|
|
||||||
|
|
||||||
def _torch_varlen_flash_attention(
|
|
||||||
q: torch.Tensor,
|
|
||||||
k: torch.Tensor,
|
|
||||||
v: torch.Tensor,
|
|
||||||
q_lens: torch.Tensor | None = None,
|
|
||||||
k_lens: torch.Tensor | None = None,
|
|
||||||
dropout_p: float = 0.0,
|
|
||||||
causal: bool = False,
|
|
||||||
dtype: torch.dtype = torch.bfloat16,
|
|
||||||
**_: object,
|
|
||||||
) -> torch.Tensor:
|
|
||||||
if q_lens is not None or k_lens is not None:
|
|
||||||
upstream_attention.warnings.warn(
|
|
||||||
"Padding mask is disabled in the test-only SDPA fallback.",
|
|
||||||
)
|
|
||||||
out_dtype = q.dtype
|
|
||||||
q = q.transpose(1, 2).to(dtype)
|
|
||||||
k = k.transpose(1, 2).to(dtype)
|
|
||||||
v = v.transpose(1, 2).to(dtype)
|
|
||||||
out = F.scaled_dot_product_attention(
|
|
||||||
q,
|
|
||||||
k,
|
|
||||||
v,
|
|
||||||
attn_mask=None,
|
|
||||||
is_causal=causal,
|
|
||||||
dropout_p=dropout_p,
|
|
||||||
)
|
|
||||||
return out.transpose(1, 2).contiguous().to(out_dtype)
|
|
||||||
|
|
||||||
|
|
||||||
upstream_attention.flash_attention = _torch_varlen_flash_attention
|
|
||||||
upstream_submodule.flash_attention = _torch_varlen_flash_attention
|
|
||||||
upstream_wan_video_dit.FLASH_ATTN_3_AVAILABLE = False
|
|
||||||
upstream_wan_video_dit.FLASH_ATTN_2_AVAILABLE = False
|
|
||||||
upstream_wan_video_dit.SAGE_ATTN_AVAILABLE = False
|
|
||||||
|
|
||||||
|
|
||||||
def main(args: upstream.Args) -> None:
|
|
||||||
os.environ["ENABLE_DIT_CACHE"] = "false"
|
|
||||||
os.environ["ATTENTION_BACKEND"] = "torch"
|
|
||||||
os.environ["NUM_DIT_STEPS"] = "16"
|
|
||||||
os.environ["DYNAMIC_CACHE_SCHEDULE"] = "false"
|
|
||||||
torch._dynamo.config.recompile_limit = 800
|
|
||||||
|
|
||||||
embodiment_tag = "oxe_droid"
|
|
||||||
model_path = args.model_path
|
|
||||||
|
|
||||||
device_mesh = upstream.init_mesh()
|
|
||||||
rank = upstream.dist.get_rank()
|
|
||||||
|
|
||||||
timeout_delta = upstream.datetime.timedelta(seconds=args.timeout_seconds)
|
|
||||||
signal_group = upstream.dist.new_group(backend="gloo", timeout=timeout_delta)
|
|
||||||
upstream.logger.info("Rank %s initialized signal_group (gloo)", rank)
|
|
||||||
|
|
||||||
policy = upstream.GrootSimPolicy(
|
|
||||||
embodiment_tag=upstream.EmbodimentTag(embodiment_tag),
|
|
||||||
model_path=model_path,
|
|
||||||
device="cuda" if torch.cuda.is_available() else "cpu",
|
|
||||||
device_mesh=device_mesh,
|
|
||||||
)
|
|
||||||
|
|
||||||
hostname = upstream.socket.gethostname()
|
|
||||||
local_ip = upstream.socket.gethostbyname(hostname)
|
|
||||||
|
|
||||||
if rank == 0:
|
|
||||||
upstream.logging.info("Creating server (host: %s, ip: %s)", hostname, local_ip)
|
|
||||||
parent_dir = os.path.dirname(model_path)
|
|
||||||
date_suffix = upstream.datetime.datetime.now().strftime("%Y%m%d")
|
|
||||||
checkpoint_name = os.path.basename(model_path)
|
|
||||||
output_dir = os.path.join(
|
|
||||||
parent_dir,
|
|
||||||
f"real_world_eval_gen_{date_suffix}_{args.index}",
|
|
||||||
checkpoint_name,
|
|
||||||
)
|
|
||||||
os.makedirs(output_dir, exist_ok=True)
|
|
||||||
upstream.logging.info("Videos will be saved to: %s", output_dir)
|
|
||||||
else:
|
|
||||||
output_dir = None
|
|
||||||
upstream.logging.info("Rank %s starting as worker for distributed inference...", rank)
|
|
||||||
|
|
||||||
wrapper_policy = upstream.ARDroidRoboarenaPolicy(
|
|
||||||
groot_policy=policy,
|
|
||||||
signal_group=signal_group,
|
|
||||||
output_dir=output_dir,
|
|
||||||
)
|
|
||||||
|
|
||||||
server_config = upstream.PolicyServerConfig(
|
|
||||||
image_resolution=(180, 320),
|
|
||||||
needs_wrist_camera=True,
|
|
||||||
n_external_cameras=2,
|
|
||||||
needs_stereo_camera=False,
|
|
||||||
needs_session_id=True,
|
|
||||||
action_space="joint_position",
|
|
||||||
)
|
|
||||||
|
|
||||||
if rank == 0:
|
|
||||||
upstream.logging.info("Using roboarena policy server interface")
|
|
||||||
upstream.logging.info("Server config: %s", server_config)
|
|
||||||
roboarena_server = upstream.RoboarenaServer(
|
|
||||||
policy=wrapper_policy,
|
|
||||||
server_config=server_config,
|
|
||||||
host="0.0.0.0",
|
|
||||||
port=args.port,
|
|
||||||
)
|
|
||||||
roboarena_server.serve_forever()
|
|
||||||
else:
|
|
||||||
worker = upstream.DistributedWorker(
|
|
||||||
policy=policy,
|
|
||||||
signal_group=signal_group,
|
|
||||||
)
|
|
||||||
upstream.asyncio.run(worker.run())
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
upstream.logging.basicConfig(level=upstream.logging.INFO, force=True)
|
|
||||||
main(tyro.cli(upstream.Args))
|
|
||||||
@@ -8,7 +8,7 @@ L4-style smoke runs against an already-running server. The pytest wrappers in
|
|||||||
H100 / MI325) because they launch the live Omni server inside the test.
|
H100 / MI325) because they launch the live Omni server inside the test.
|
||||||
|
|
||||||
1. **Daily-Omni** — MCQ accuracy fields in the saved JSON (``daily_omni_accuracy``, …); by default the
|
1. **Daily-Omni** — MCQ accuracy fields in the saved JSON (``daily_omni_accuracy``, …); by default the
|
||||||
run **fails** if accuracy is strictly below **0.69** (``--min-daily-omni-accuracy`` / ``ACC_BENCH_MIN_DAILY_OMNI_ACCURACY``).
|
run **fails** if accuracy is strictly below **0.67** (``--min-daily-omni-accuracy`` / ``ACC_BENCH_MIN_DAILY_OMNI_ACCURACY``).
|
||||||
2. **Seed-TTS** — ``seed-tts-eval``-style metrics when ``--seed-tts-wer-eval`` is used
|
2. **Seed-TTS** — ``seed-tts-eval``-style metrics when ``--seed-tts-wer-eval`` is used
|
||||||
(WER / SIM / UTMOS keys from :func:`compute_seed_tts_wer_metrics`).
|
(WER / SIM / UTMOS keys from :func:`compute_seed_tts_wer_metrics`).
|
||||||
|
|
||||||
@@ -308,9 +308,9 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
|||||||
p.add_argument(
|
p.add_argument(
|
||||||
"--min-daily-omni-accuracy",
|
"--min-daily-omni-accuracy",
|
||||||
type=float,
|
type=float,
|
||||||
default=float((os.environ.get("ACC_BENCH_MIN_DAILY_OMNI_ACCURACY") or "0.69").strip() or "0.69"),
|
default=float((os.environ.get("ACC_BENCH_MIN_DAILY_OMNI_ACCURACY") or "0.67").strip() or "0.67"),
|
||||||
help="Fail when daily_omni_accuracy is strictly below this threshold (0–1). "
|
help="Fail when daily_omni_accuracy is strictly below this threshold (0–1). "
|
||||||
"Default baseline 0.69; override with env ACC_BENCH_MIN_DAILY_OMNI_ACCURACY or pass 0 to disable the floor.",
|
"Default baseline 0.67; override with env ACC_BENCH_MIN_DAILY_OMNI_ACCURACY or pass 0 to disable the floor.",
|
||||||
)
|
)
|
||||||
|
|
||||||
p.add_argument(
|
p.add_argument(
|
||||||
@@ -340,7 +340,7 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
|||||||
p.add_argument(
|
p.add_argument(
|
||||||
"--max-seed-tts-mean-wer",
|
"--max-seed-tts-mean-wer",
|
||||||
type=float,
|
type=float,
|
||||||
default=0.35,
|
default=0.5,
|
||||||
help="If set, fail when seed_tts_content_error_mean is strictly above this value.",
|
help="If set, fail when seed_tts_content_error_mean is strictly above this value.",
|
||||||
)
|
)
|
||||||
p.add_argument(
|
p.add_argument(
|
||||||
|
|||||||
@@ -218,7 +218,7 @@ def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> tuple[Image.I
|
|||||||
torch_dtype=torch.bfloat16,
|
torch_dtype=torch.bfloat16,
|
||||||
trust_remote_code=True,
|
trust_remote_code=True,
|
||||||
).to("cuda")
|
).to("cuda")
|
||||||
pipe.transformer.set_attention_backend("_flash_3_hub")
|
|
||||||
_diffusers_dummy_run(pipe)
|
_diffusers_dummy_run(pipe)
|
||||||
|
|
||||||
generator = torch.Generator(device="cuda").manual_seed(SEED)
|
generator = torch.Generator(device="cuda").manual_seed(SEED)
|
||||||
|
|||||||
@@ -1,218 +0,0 @@
|
|||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import base64
|
|
||||||
import copy
|
|
||||||
import io
|
|
||||||
import os
|
|
||||||
import tempfile
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
import requests
|
|
||||||
import yaml
|
|
||||||
from PIL import Image
|
|
||||||
|
|
||||||
from tests.e2e.accuracy.helpers import assert_images_pixel_close, assert_similarity, model_output_dir
|
|
||||||
from tests.helpers.mark import hardware_test
|
|
||||||
from tests.helpers.runtime import OmniServer
|
|
||||||
|
|
||||||
pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]
|
|
||||||
|
|
||||||
MODEL_NAME = "tencent/HunyuanImage-3.0-Instruct"
|
|
||||||
SEED = 42
|
|
||||||
NUM_INFERENCE_STEPS = 50
|
|
||||||
GUIDANCE_SCALE = 2.5
|
|
||||||
HEIGHT = 1024
|
|
||||||
WIDTH = 1024
|
|
||||||
PROMPT = "A brown and white dog is running on the grass."
|
|
||||||
MEAN_THRESHOLD = 3e-2
|
|
||||||
P99_THRESHOLD = 3e-1
|
|
||||||
SSIM_THRESHOLD = 0.97
|
|
||||||
PSNR_THRESHOLD = 30.0
|
|
||||||
|
|
||||||
_REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent
|
|
||||||
BASELINE_PATH = _REPO_ROOT / "tests" / "assets" / "hunyuan" / "hunyuan_baseline.png"
|
|
||||||
_OFFLINE_SCRIPT = _REPO_ROOT / "examples" / "offline_inference" / "hunyuan_image3" / "end2end.py"
|
|
||||||
|
|
||||||
# DiT-only deploy config with trust_remote_code (based on hunyuan_image3_dit.yaml).
|
|
||||||
_DEPLOY_CONFIG = {
|
|
||||||
"pipeline": "hunyuan_image3_dit",
|
|
||||||
"async_chunk": False,
|
|
||||||
"trust_remote_code": True,
|
|
||||||
"stages": [
|
|
||||||
{
|
|
||||||
"stage_id": 0,
|
|
||||||
"max_num_seqs": 1,
|
|
||||||
"gpu_memory_utilization": 0.9,
|
|
||||||
"enforce_eager": True,
|
|
||||||
"trust_remote_code": True,
|
|
||||||
"devices": "0,1,2,3", # set dynamically by _write_deploy_config
|
|
||||||
"vae_use_slicing": False,
|
|
||||||
"vae_use_tiling": False,
|
|
||||||
"parallel_config": {
|
|
||||||
"pipeline_parallel_size": 1,
|
|
||||||
"data_parallel_size": 1,
|
|
||||||
"tensor_parallel_size": 4,
|
|
||||||
"enable_expert_parallel": True,
|
|
||||||
"sequence_parallel_size": 1,
|
|
||||||
"ulysses_degree": 1,
|
|
||||||
"ring_degree": 1,
|
|
||||||
"cfg_parallel_size": 1,
|
|
||||||
"vae_patch_parallel_size": 1,
|
|
||||||
"use_hsdp": False,
|
|
||||||
"hsdp_shard_size": -1,
|
|
||||||
"hsdp_replicate_size": 1,
|
|
||||||
},
|
|
||||||
"default_sampling_params": {
|
|
||||||
"seed": SEED,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _model_name() -> str:
|
|
||||||
return os.environ.get("HUNYUAN_IMAGE3_MODEL", MODEL_NAME)
|
|
||||||
|
|
||||||
|
|
||||||
def _devices() -> str:
|
|
||||||
return os.environ.get("HUNYUAN_IMAGE3_DEVICES", "0,1,2,3")
|
|
||||||
|
|
||||||
|
|
||||||
def _write_deploy_config(path: Path) -> None:
|
|
||||||
config = copy.deepcopy(_DEPLOY_CONFIG)
|
|
||||||
devices = _devices()
|
|
||||||
config["stages"][0]["devices"] = devices
|
|
||||||
config["stages"][0]["parallel_config"]["tensor_parallel_size"] = len(devices.split(","))
|
|
||||||
path.write_text(yaml.dump(config, default_flow_style=False, sort_keys=False))
|
|
||||||
|
|
||||||
|
|
||||||
def _run_vllm_omni_hunyuan_image3_online(*, model: str, deploy_config: str, output_path: Path) -> Image.Image:
|
|
||||||
server_args = [
|
|
||||||
"--deploy-config",
|
|
||||||
deploy_config,
|
|
||||||
"--stage-init-timeout",
|
|
||||||
"300",
|
|
||||||
"--init-timeout",
|
|
||||||
"900",
|
|
||||||
"--enforce-eager",
|
|
||||||
"--trust-remote-code",
|
|
||||||
]
|
|
||||||
with OmniServer(model, server_args, use_omni=True) as omni_server:
|
|
||||||
response = requests.post(
|
|
||||||
f"http://{omni_server.host}:{omni_server.port}/v1/images/generations",
|
|
||||||
json={
|
|
||||||
"model": omni_server.model,
|
|
||||||
"prompt": PROMPT,
|
|
||||||
"size": f"{WIDTH}x{HEIGHT}",
|
|
||||||
"n": 1,
|
|
||||||
"response_format": "b64_json",
|
|
||||||
"num_inference_steps": NUM_INFERENCE_STEPS,
|
|
||||||
"guidance_scale": GUIDANCE_SCALE,
|
|
||||||
"seed": SEED,
|
|
||||||
"bot_task": "none",
|
|
||||||
"use_system_prompt": "en_unified",
|
|
||||||
},
|
|
||||||
timeout=600,
|
|
||||||
)
|
|
||||||
response.raise_for_status()
|
|
||||||
payload = response.json()
|
|
||||||
assert len(payload["data"]) == 1
|
|
||||||
image_bytes = base64.b64decode(payload["data"][0]["b64_json"])
|
|
||||||
image = Image.open(io.BytesIO(image_bytes)).convert("RGB")
|
|
||||||
image.load()
|
|
||||||
image.save(output_path)
|
|
||||||
return image
|
|
||||||
|
|
||||||
|
|
||||||
def _run_vllm_omni_hunyuan_image3_offline(*, model: str, deploy_config: str, output_path: Path) -> Image.Image:
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
|
|
||||||
output_dir = str(output_path.parent)
|
|
||||||
subprocess.run(
|
|
||||||
[
|
|
||||||
sys.executable,
|
|
||||||
str(_OFFLINE_SCRIPT),
|
|
||||||
"--modality",
|
|
||||||
"text2img",
|
|
||||||
"--deploy-config",
|
|
||||||
deploy_config,
|
|
||||||
"--prompts",
|
|
||||||
PROMPT,
|
|
||||||
"--output",
|
|
||||||
output_dir,
|
|
||||||
"--steps",
|
|
||||||
str(NUM_INFERENCE_STEPS),
|
|
||||||
"--guidance-scale",
|
|
||||||
str(GUIDANCE_SCALE),
|
|
||||||
"--seed",
|
|
||||||
str(SEED),
|
|
||||||
"--height",
|
|
||||||
str(HEIGHT),
|
|
||||||
"--width",
|
|
||||||
str(WIDTH),
|
|
||||||
"--bot-task",
|
|
||||||
"none",
|
|
||||||
"--sys-type",
|
|
||||||
"en_unified",
|
|
||||||
"--model",
|
|
||||||
model,
|
|
||||||
"--enforce-eager",
|
|
||||||
],
|
|
||||||
check=True,
|
|
||||||
)
|
|
||||||
images = sorted(Path(output_dir).glob("output_*.png"))
|
|
||||||
assert images, f"No output image found in {output_dir}"
|
|
||||||
image = Image.open(images[0]).convert("RGB")
|
|
||||||
image.load()
|
|
||||||
image.save(output_path)
|
|
||||||
return image
|
|
||||||
|
|
||||||
|
|
||||||
def _assert_against_baseline(image: Image.Image, label: str) -> None:
|
|
||||||
assert BASELINE_PATH.exists(), f"Baseline image not found at {BASELINE_PATH}"
|
|
||||||
baseline_image = Image.open(BASELINE_PATH).convert("RGB")
|
|
||||||
|
|
||||||
assert_images_pixel_close(
|
|
||||||
model_name=f"{MODEL_NAME} ({label} vs baseline)",
|
|
||||||
vllm_image=image,
|
|
||||||
diffusers_image=baseline_image,
|
|
||||||
mean_threshold=MEAN_THRESHOLD,
|
|
||||||
p99_threshold=P99_THRESHOLD,
|
|
||||||
)
|
|
||||||
assert_similarity(
|
|
||||||
model_name=f"{MODEL_NAME} ({label} vs baseline)",
|
|
||||||
vllm_image=image,
|
|
||||||
diffusers_image=baseline_image,
|
|
||||||
ssim_threshold=SSIM_THRESHOLD,
|
|
||||||
psnr_threshold=PSNR_THRESHOLD,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
|
||||||
def test_hunyuan_image3_pixel_accuracy_online(accuracy_artifact_root: Path) -> None:
|
|
||||||
model = _model_name()
|
|
||||||
output_dir = model_output_dir(accuracy_artifact_root, MODEL_NAME)
|
|
||||||
|
|
||||||
with tempfile.TemporaryDirectory() as tmpdir:
|
|
||||||
deploy_config_path = Path(tmpdir) / "deploy.yaml"
|
|
||||||
_write_deploy_config(deploy_config_path)
|
|
||||||
image = _run_vllm_omni_hunyuan_image3_online(
|
|
||||||
model=model, deploy_config=str(deploy_config_path), output_path=output_dir / "vllm_omni_online.png"
|
|
||||||
)
|
|
||||||
_assert_against_baseline(image, "online")
|
|
||||||
|
|
||||||
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
|
||||||
def test_hunyuan_image3_pixel_accuracy_offline(accuracy_artifact_root: Path) -> None:
|
|
||||||
model = _model_name()
|
|
||||||
output_dir = model_output_dir(accuracy_artifact_root, MODEL_NAME)
|
|
||||||
|
|
||||||
with tempfile.TemporaryDirectory() as tmpdir:
|
|
||||||
deploy_config_path = Path(tmpdir) / "deploy.yaml"
|
|
||||||
_write_deploy_config(deploy_config_path)
|
|
||||||
image = _run_vllm_omni_hunyuan_image3_offline(
|
|
||||||
model=model, deploy_config=str(deploy_config_path), output_path=output_dir / "vllm_omni_offline.png"
|
|
||||||
)
|
|
||||||
_assert_against_baseline(image, "offline")
|
|
||||||
@@ -106,7 +106,6 @@ def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> Image.Image:
|
|||||||
trust_remote_code=True,
|
trust_remote_code=True,
|
||||||
local_files_only=_local_files_only(model),
|
local_files_only=_local_files_only(model),
|
||||||
).to("cuda")
|
).to("cuda")
|
||||||
pipe.transformer.set_attention_backend("_flash_3_hub")
|
|
||||||
generator = torch.Generator(device="cuda").manual_seed(SEED)
|
generator = torch.Generator(device="cuda").manual_seed(SEED)
|
||||||
result = pipe( # pyright: ignore[reportCallIssue]
|
result = pipe( # pyright: ignore[reportCallIssue]
|
||||||
prompt=PROMPT,
|
prompt=PROMPT,
|
||||||
@@ -168,7 +167,6 @@ def _run_diffusers_qwen_image_2512(*, model: str, output_path: Path) -> Image.Im
|
|||||||
trust_remote_code=True,
|
trust_remote_code=True,
|
||||||
local_files_only=_local_files_only(model),
|
local_files_only=_local_files_only(model),
|
||||||
).to("cuda")
|
).to("cuda")
|
||||||
pipe.transformer.set_attention_backend("_flash_3_hub")
|
|
||||||
generator = torch.Generator(device="cuda").manual_seed(SEED_2512)
|
generator = torch.Generator(device="cuda").manual_seed(SEED_2512)
|
||||||
result = pipe( # pyright: ignore[reportCallIssue]
|
result = pipe( # pyright: ignore[reportCallIssue]
|
||||||
prompt=PROMPT_2512,
|
prompt=PROMPT_2512,
|
||||||
|
|||||||
@@ -32,8 +32,7 @@ from vllm_omni import Omni
|
|||||||
|
|
||||||
# Selected by the nightly "TTS · Function Test" (-m "full_model and L4 and tts").
|
# Selected by the nightly "TTS · Function Test" (-m "full_model and L4 and tts").
|
||||||
# Without the tts + run-level marks these tests are collected but never run.
|
# Without the tts + run-level marks these tests are collected but never run.
|
||||||
# TODO: Fix this test
|
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
||||||
# pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
|
||||||
|
|
||||||
SAMPLE_RATE = 24_000 # All MOSS-TTS full variants output 24 kHz
|
SAMPLE_RATE = 24_000 # All MOSS-TTS full variants output 24 kHz
|
||||||
|
|
||||||
@@ -129,6 +128,7 @@ def delay_engine():
|
|||||||
return Omni(model=_DELAY_MODEL, stage_init_timeout=300)
|
return Omni(model=_DELAY_MODEL, stage_init_timeout=300)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_delay_english(delay_engine, ref_audio_path):
|
def test_moss_tts_delay_english(delay_engine, ref_audio_path):
|
||||||
"""MossTTSDelayModel: English voice_clone produces non-empty 24 kHz audio."""
|
"""MossTTSDelayModel: English voice_clone produces non-empty 24 kHz audio."""
|
||||||
@@ -140,6 +140,7 @@ def test_moss_tts_delay_english(delay_engine, ref_audio_path):
|
|||||||
assert not torch.all(audio == 0), "Audio is silence"
|
assert not torch.all(audio == 0), "Audio is silence"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_delay_chinese(delay_engine, ref_audio_path):
|
def test_moss_tts_delay_chinese(delay_engine, ref_audio_path):
|
||||||
"""MossTTSDelayModel: Chinese input produces non-empty audio."""
|
"""MossTTSDelayModel: Chinese input produces non-empty audio."""
|
||||||
@@ -151,6 +152,7 @@ def test_moss_tts_delay_chinese(delay_engine, ref_audio_path):
|
|||||||
assert not torch.all(audio == 0)
|
assert not torch.all(audio == 0)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_delay_deterministic(delay_engine, ref_audio_path):
|
def test_moss_tts_delay_deterministic(delay_engine, ref_audio_path):
|
||||||
"""MossTTSDelayModel: same seed yields identical waveforms."""
|
"""MossTTSDelayModel: same seed yields identical waveforms."""
|
||||||
@@ -162,6 +164,7 @@ def test_moss_tts_delay_deterministic(delay_engine, ref_audio_path):
|
|||||||
assert torch.allclose(audio1, audio2, atol=1e-4), "Waveforms differ across identical seeds"
|
assert torch.allclose(audio1, audio2, atol=1e-4), "Waveforms differ across identical seeds"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_delay_batch(delay_engine, ref_audio_path):
|
def test_moss_tts_delay_batch(delay_engine, ref_audio_path):
|
||||||
"""MossTTSDelayModel: batch of two requests each returns non-empty audio."""
|
"""MossTTSDelayModel: batch of two requests each returns non-empty audio."""
|
||||||
@@ -203,6 +206,7 @@ def realtime_engine():
|
|||||||
return Omni(model=_REALTIME_MODEL, stage_init_timeout=300)
|
return Omni(model=_REALTIME_MODEL, stage_init_timeout=300)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_realtime_english(realtime_engine, ref_audio_path):
|
def test_moss_tts_realtime_english(realtime_engine, ref_audio_path):
|
||||||
"""MossTTSRealtime: English voice_clone produces non-empty 24 kHz audio."""
|
"""MossTTSRealtime: English voice_clone produces non-empty 24 kHz audio."""
|
||||||
@@ -214,6 +218,7 @@ def test_moss_tts_realtime_english(realtime_engine, ref_audio_path):
|
|||||||
assert not torch.all(audio == 0), "Audio is silence"
|
assert not torch.all(audio == 0), "Audio is silence"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_realtime_deterministic(realtime_engine, ref_audio_path):
|
def test_moss_tts_realtime_deterministic(realtime_engine, ref_audio_path):
|
||||||
"""MossTTSRealtime: same seed yields identical waveforms."""
|
"""MossTTSRealtime: same seed yields identical waveforms."""
|
||||||
|
|||||||
@@ -120,6 +120,7 @@ def _collect_audio(omni: Omni, request: dict) -> tuple[torch.Tensor, int]:
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_nano_english(omni_runner: OmniRunner, ref_audio_path) -> None:
|
def test_moss_tts_nano_english(omni_runner: OmniRunner, ref_audio_path) -> None:
|
||||||
"""English TTS produces non-empty 48 kHz stereo audio."""
|
"""English TTS produces non-empty 48 kHz stereo audio."""
|
||||||
@@ -132,6 +133,7 @@ def test_moss_tts_nano_english(omni_runner: OmniRunner, ref_audio_path) -> None:
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_nano_chinese(omni_runner: OmniRunner, ref_audio_path) -> None:
|
def test_moss_tts_nano_chinese(omni_runner: OmniRunner, ref_audio_path) -> None:
|
||||||
"""Chinese TTS produces non-empty audio."""
|
"""Chinese TTS produces non-empty audio."""
|
||||||
@@ -144,6 +146,7 @@ def test_moss_tts_nano_chinese(omni_runner: OmniRunner, ref_audio_path) -> None:
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_nano_deterministic(omni_runner: OmniRunner, ref_audio_path) -> None:
|
def test_moss_tts_nano_deterministic(omni_runner: OmniRunner, ref_audio_path) -> None:
|
||||||
"""Same seed produces identical waveforms."""
|
"""Same seed produces identical waveforms."""
|
||||||
@@ -156,6 +159,7 @@ def test_moss_tts_nano_deterministic(omni_runner: OmniRunner, ref_audio_path) ->
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "L4"})
|
@hardware_test(res={"cuda": "L4"})
|
||||||
def test_moss_tts_nano_batch(omni_runner: OmniRunner, ref_audio_path) -> None:
|
def test_moss_tts_nano_batch(omni_runner: OmniRunner, ref_audio_path) -> None:
|
||||||
"""Batch of two requests returns audio for each."""
|
"""Batch of two requests returns audio for each."""
|
||||||
|
|||||||
@@ -24,11 +24,10 @@ from vllm import SamplingParams
|
|||||||
from tests.helpers.mark import hardware_test
|
from tests.helpers.mark import hardware_test
|
||||||
from vllm_omni import Omni
|
from vllm_omni import Omni
|
||||||
|
|
||||||
# H100-gated (8B). Picked up by the nightly "TTS · Function Test with H100"
|
# H100-gated (8B). Picked up by the nightly H100 omni job
|
||||||
# job (-m "full_model and H100 and tts"); the L4 MOSS-TTS tests land in the
|
# (-m "full_model and H100 and omni"); the tts mark keeps it consistent with
|
||||||
# sibling L4 tts lane instead.
|
# the other MOSS-TTS tests and any tts-scoped sweeps.
|
||||||
# TODO: Fix this test
|
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
||||||
# pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
|
||||||
|
|
||||||
SAMPLE_RATE = 24_000
|
SAMPLE_RATE = 24_000
|
||||||
REF_AUDIO_URL = "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS/main/assets/audio/reference_zh_1.wav"
|
REF_AUDIO_URL = "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS/main/assets/audio/reference_zh_1.wav"
|
||||||
@@ -102,6 +101,7 @@ def _collect_audio(omni: Omni, request: dict) -> tuple[torch.Tensor, int]:
|
|||||||
raise AssertionError("No stage outputs received")
|
raise AssertionError("No stage outputs received")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "H100"})
|
@hardware_test(res={"cuda": "H100"})
|
||||||
def test_moss_tts_v15_voice_clone(v15_engine, ref_audio_path):
|
def test_moss_tts_v15_voice_clone(v15_engine, ref_audio_path):
|
||||||
"""MOSS-TTS-v1.5 (8B): voice_clone produces non-empty 24 kHz audio."""
|
"""MOSS-TTS-v1.5 (8B): voice_clone produces non-empty 24 kHz audio."""
|
||||||
|
|||||||
@@ -1,90 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
"""E2E online serving test for DreamZero OpenPI websocket serving."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import os
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from tests.dreamzero import openpi_client_helper as openpi_client
|
|
||||||
from tests.helpers.mark import hardware_test
|
|
||||||
from tests.helpers.runtime import OmniServerParams, get_open_port
|
|
||||||
|
|
||||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
|
||||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
|
||||||
|
|
||||||
MODEL = "GEAR-Dreams/DreamZero-DROID"
|
|
||||||
|
|
||||||
|
|
||||||
test_params = [
|
|
||||||
OmniServerParams(
|
|
||||||
model=MODEL,
|
|
||||||
port=8091,
|
|
||||||
server_args=[
|
|
||||||
"--deploy-config",
|
|
||||||
"vllm_omni/deploy/dreamzero_tp1_cfg2.yaml",
|
|
||||||
"--enforce-eager",
|
|
||||||
"--disable-log-stats",
|
|
||||||
],
|
|
||||||
env_dict={
|
|
||||||
"ATTENTION_BACKEND": "torch",
|
|
||||||
"DIFFUSION_ATTENTION_BACKEND": "TORCH_SDPA",
|
|
||||||
"VLLM_DISABLE_COMPILE_CACHE": "1",
|
|
||||||
"MASTER_PORT": str(get_open_port()),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def _write_synthetic_video(path: Path, cv2_module, *, channel: int) -> None:
|
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
height, width, num_frames = 180, 320, 24
|
|
||||||
writer = cv2_module.VideoWriter(str(path), cv2_module.VideoWriter_fourcc(*"mp4v"), 15.0, (width, height))
|
|
||||||
if not writer.isOpened():
|
|
||||||
raise RuntimeError(f"Failed to open video writer for {path}")
|
|
||||||
try:
|
|
||||||
for frame_idx in range(num_frames):
|
|
||||||
frame = np.zeros((height, width, 3), dtype=np.uint8)
|
|
||||||
frame[..., channel] = (frame_idx * 7) % 255
|
|
||||||
frame[..., (channel + 1) % 3] = 64
|
|
||||||
writer.write(cv2_module.cvtColor(frame, cv2_module.COLOR_RGB2BGR))
|
|
||||||
finally:
|
|
||||||
writer.release()
|
|
||||||
|
|
||||||
|
|
||||||
def _write_synthetic_dreamzero_videos(client_mod, video_dir: Path) -> None:
|
|
||||||
for channel, file_name in enumerate(client_mod.CAMERA_FILES.values()):
|
|
||||||
_write_synthetic_video(video_dir / file_name, client_mod.cv2, channel=channel)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.full_model
|
|
||||||
@pytest.mark.diffusion
|
|
||||||
@pytest.mark.distributed_cuda
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=2)
|
|
||||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
|
||||||
def test_dreamzero_openpi_online(omni_server, tmp_path: Path) -> None:
|
|
||||||
try:
|
|
||||||
openpi_client.require_dependencies()
|
|
||||||
except ModuleNotFoundError as exc:
|
|
||||||
pytest.skip(str(exc))
|
|
||||||
|
|
||||||
video_dir = tmp_path / "dreamzero_videos"
|
|
||||||
_write_synthetic_dreamzero_videos(openpi_client, video_dir)
|
|
||||||
result = openpi_client.run_policy_session(
|
|
||||||
host=omni_server.host,
|
|
||||||
port=omni_server.port,
|
|
||||||
video_dir=video_dir,
|
|
||||||
session_id="dreamzero-online-e2e",
|
|
||||||
)
|
|
||||||
|
|
||||||
openpi_client.validate_session_result(result)
|
|
||||||
|
|
||||||
metadata = result["metadata"]
|
|
||||||
assert metadata["needs_session_id"] is True
|
|
||||||
assert metadata["needs_stereo_camera"] is False
|
|
||||||
assert tuple(metadata["image_resolution"]) == (180, 320)
|
|
||||||
@@ -1,183 +0,0 @@
|
|||||||
"""
|
|
||||||
Comprehensive tests of diffusion features that are available in online serving mode
|
|
||||||
and are supported by the following model:
|
|
||||||
- zai-org/GLM-Image (t2i and i2i)
|
|
||||||
|
|
||||||
Coverage (per mode):
|
|
||||||
- Baseline
|
|
||||||
- Cache-DiT
|
|
||||||
- Tensor-Parallel
|
|
||||||
- HSDP
|
|
||||||
- Sequence parallel
|
|
||||||
|
|
||||||
Topology for multi-GPU cases uses runtime deploy YAML (except TP)
|
|
||||||
so stage-1 parallel fields reach the diffusion worker without relying on CLI
|
|
||||||
|
|
||||||
assert_diffusion_response validates successful generation and the expected resolution.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import os
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from tests.helpers.mark import hardware_marks
|
|
||||||
from tests.helpers.media import generate_synthetic_image
|
|
||||||
from tests.helpers.runtime import (
|
|
||||||
OmniServer,
|
|
||||||
OmniServerParams,
|
|
||||||
OpenAIClientHandler,
|
|
||||||
dummy_messages_from_mix_data,
|
|
||||||
)
|
|
||||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
|
||||||
|
|
||||||
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
|
|
||||||
|
|
||||||
MODEL = os.environ.get("GLM_IMAGE_MODEL_PATH", "zai-org/GLM-Image")
|
|
||||||
|
|
||||||
T2I_PROMPT = "A Vincent van Gogh style impressionist painting."
|
|
||||||
I2I_PROMPT = "Transform this modern, geometric image into a Vincent van Gogh style impressionist painting."
|
|
||||||
NEGATIVE_PROMPT = "low quality, blurry, distorted, unnatural colors"
|
|
||||||
|
|
||||||
TWO_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}, num_cards=2)
|
|
||||||
|
|
||||||
GLM_DEPLOY = get_deploy_config_path("glm_image.yaml")
|
|
||||||
GLM_CACHEDIT_DEPLOY = modify_stage_config(
|
|
||||||
GLM_DEPLOY,
|
|
||||||
updates={
|
|
||||||
"stages": {
|
|
||||||
1: {"cache_backend": "cache_dit"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
GLM_TP_2_DEPLOY = modify_stage_config(
|
|
||||||
GLM_DEPLOY,
|
|
||||||
updates={
|
|
||||||
"stages": {
|
|
||||||
0: {"devices": "0", "tensor_parallel_size": 1},
|
|
||||||
1: {"devices": "0,1", "tensor_parallel_size": 2},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
GLM_HSDP_2_DEPLOY = modify_stage_config(
|
|
||||||
GLM_DEPLOY,
|
|
||||||
updates={
|
|
||||||
"stages": {
|
|
||||||
0: {"devices": "0"},
|
|
||||||
1: {
|
|
||||||
"devices": "0,1",
|
|
||||||
"parallel_config": {
|
|
||||||
"use_hsdp": True,
|
|
||||||
"hsdp_shard_size": 2,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
GLM_SP_2_DEPLOY = modify_stage_config(
|
|
||||||
GLM_DEPLOY,
|
|
||||||
updates={
|
|
||||||
"stages": {
|
|
||||||
0: {"devices": "0"},
|
|
||||||
1: {
|
|
||||||
"devices": "0,1",
|
|
||||||
"parallel_config": {
|
|
||||||
"ulysses_degree": 2,
|
|
||||||
"ring_degree": 1,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _get_diffusion_feature_cases(model: str):
|
|
||||||
return [
|
|
||||||
# Baseline: default deploy (stage 0 -> GPU 0, stage 1 -> GPU 1)
|
|
||||||
pytest.param(
|
|
||||||
OmniServerParams(
|
|
||||||
model=model,
|
|
||||||
stage_config_path=GLM_DEPLOY,
|
|
||||||
),
|
|
||||||
id="baseline",
|
|
||||||
marks=TWO_CARD_FEATURE_MARKS,
|
|
||||||
),
|
|
||||||
# Cache-DiT via runtime deploy patch (cache_backend -> engine_extras on stage 1)
|
|
||||||
pytest.param(
|
|
||||||
OmniServerParams(
|
|
||||||
model=model,
|
|
||||||
stage_config_path=GLM_CACHEDIT_DEPLOY,
|
|
||||||
),
|
|
||||||
id="cachedit",
|
|
||||||
marks=TWO_CARD_FEATURE_MARKS,
|
|
||||||
),
|
|
||||||
# Tensor-Parallel (2 GPUs on diffusion stage)
|
|
||||||
pytest.param(
|
|
||||||
OmniServerParams(
|
|
||||||
model=model,
|
|
||||||
stage_config_path=GLM_TP_2_DEPLOY,
|
|
||||||
),
|
|
||||||
id="tensor_parallel_2",
|
|
||||||
marks=TWO_CARD_FEATURE_MARKS,
|
|
||||||
),
|
|
||||||
# HSDP (2 GPUs on diffusion stage)
|
|
||||||
pytest.param(
|
|
||||||
OmniServerParams(
|
|
||||||
model=model,
|
|
||||||
stage_config_path=GLM_HSDP_2_DEPLOY,
|
|
||||||
),
|
|
||||||
id="hsdp_2",
|
|
||||||
marks=TWO_CARD_FEATURE_MARKS,
|
|
||||||
),
|
|
||||||
# Ulysses sequence parallel degree=2 (2 GPUs)
|
|
||||||
pytest.param(
|
|
||||||
OmniServerParams(
|
|
||||||
model=model,
|
|
||||||
stage_config_path=GLM_SP_2_DEPLOY,
|
|
||||||
),
|
|
||||||
id="sequence_parallel_2",
|
|
||||||
marks=TWO_CARD_FEATURE_MARKS,
|
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
MODES = ["t2i", "i2i"]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("mode", MODES)
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"omni_server",
|
|
||||||
_get_diffusion_feature_cases(MODEL),
|
|
||||||
indirect=True,
|
|
||||||
)
|
|
||||||
def test_glm_image(
|
|
||||||
omni_server: OmniServer,
|
|
||||||
mode: str,
|
|
||||||
openai_client: OpenAIClientHandler,
|
|
||||||
):
|
|
||||||
"""GLM-Image diffusion feature coverage for t2i and i2i on two H100s."""
|
|
||||||
|
|
||||||
if mode == "t2i":
|
|
||||||
messages = dummy_messages_from_mix_data(content_text=T2I_PROMPT)
|
|
||||||
else:
|
|
||||||
image_size = 1024
|
|
||||||
image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(image_size, image_size)['base64']}"
|
|
||||||
messages = dummy_messages_from_mix_data(
|
|
||||||
image_data_url=image_data_url,
|
|
||||||
content_text=I2I_PROMPT,
|
|
||||||
)
|
|
||||||
|
|
||||||
request_config = {
|
|
||||||
"model": omni_server.model,
|
|
||||||
"messages": messages,
|
|
||||||
"extra_body": {
|
|
||||||
"height": 1024,
|
|
||||||
"width": 1024,
|
|
||||||
"num_inference_steps": 2,
|
|
||||||
"guidance_scale": 1.5,
|
|
||||||
"true_cfg_scale": 4.0,
|
|
||||||
"negative_prompt": NEGATIVE_PROMPT,
|
|
||||||
"seed": 42,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
openai_client.send_diffusion_request(request_config)
|
|
||||||
@@ -1,388 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
"""End-to-end online tests for higgs-audio v3 against /v1/audio/speech.
|
|
||||||
|
|
||||||
Mirrors the higgs_audio_v2 test layout. Covers the plain-text-in / audio-out
|
|
||||||
happy path, a small concurrent burst (Stage 0 prefix caching has to coexist
|
|
||||||
with batching), the voice-clone path via ``ref_audio`` + ``ref_text``, the
|
|
||||||
upstream ``references[]`` cookbook alias added by
|
|
||||||
``normalize_references_alias`` in ``protocol/audio.py``, and the few
|
|
||||||
validator rejections that should remain 4xx.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import os
|
|
||||||
|
|
||||||
os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
|
||||||
os.environ.setdefault("VLLM_USE_DEEP_GEMM", "0")
|
|
||||||
os.environ.setdefault("VLLM_MOE_USE_DEEP_GEMM", "0")
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from tests.helpers.mark import hardware_test
|
|
||||||
from tests.helpers.media import load_test_audio_data_url
|
|
||||||
from tests.helpers.runtime import OmniServerParams
|
|
||||||
from tests.helpers.stage_config import get_deploy_config_path
|
|
||||||
|
|
||||||
MODEL = "bosonai/higgs-audio-v3-tts-4b"
|
|
||||||
STAGE_CONFIG = get_deploy_config_path("higgs_multimodal_qwen3.yaml")
|
|
||||||
SERVER_ARGS = ["--trust-remote-code", "--disable-log-stats"]
|
|
||||||
SERVER_ENV = {"VLLM_USE_DEEP_GEMM": "0", "VLLM_MOE_USE_DEEP_GEMM": "0"}
|
|
||||||
|
|
||||||
TEST_PARAMS = [
|
|
||||||
pytest.param(
|
|
||||||
OmniServerParams(
|
|
||||||
model=MODEL,
|
|
||||||
stage_config_path=STAGE_CONFIG,
|
|
||||||
server_args=SERVER_ARGS,
|
|
||||||
env_dict=SERVER_ENV,
|
|
||||||
),
|
|
||||||
id="higgs_audio_v3_plain_text",
|
|
||||||
)
|
|
||||||
]
|
|
||||||
|
|
||||||
DEFAULT_SPEECH_TIMEOUT_S = 180.0
|
|
||||||
# Floor for ~0.5 s of 24 kHz mono PCM_16: 24000 * 0.5 * 2 bytes ~= 24 KiB.
|
|
||||||
# A WAV header adds 44 bytes; conservative floor catches truncated /
|
|
||||||
# silence-only outputs without flagging short legitimate clips.
|
|
||||||
_MIN_AUDIO_BYTES = 20_000
|
|
||||||
|
|
||||||
# Reuse the shared TTS reference clip (clean ~5 s 24 kHz mono human speech)
|
|
||||||
# vendored under tests/assets/qwen3_tts/. Keeps a single WAV across TTS
|
|
||||||
# tests rather than duplicating asset bytes.
|
|
||||||
_REF_AUDIO_URL = load_test_audio_data_url("qwen3_tts/clone_2.wav")
|
|
||||||
_REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
|
|
||||||
class TestHiggsAudioV3OnlineHappyPath:
|
|
||||||
"""Plain-text -> audio happy paths against the live HTTP server."""
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_plain_text_wav(self, omni_server, openai_client) -> None:
|
|
||||||
"""Single non-streaming WAV request - canonical TTS happy path."""
|
|
||||||
openai_client.send_audio_speech_request(
|
|
||||||
{
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": "Hello world.",
|
|
||||||
"stream": False,
|
|
||||||
"response_format": "wav",
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_plain_text_with_max_new_tokens(self, omni_server, openai_client) -> None:
|
|
||||||
"""``max_new_tokens`` is one of the few extra fields the v3 validator accepts."""
|
|
||||||
openai_client.send_audio_speech_request(
|
|
||||||
{
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": "Innovation distinguishes between a leader and a follower.",
|
|
||||||
"stream": False,
|
|
||||||
"response_format": "wav",
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"max_new_tokens": 500,
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_concurrent_plain_text(self, omni_server, openai_client) -> None:
|
|
||||||
"""Three concurrent non-streaming requests - guards per-slot audio state and Stage-0 PC under batching."""
|
|
||||||
openai_client.send_audio_speech_request(
|
|
||||||
{
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": "It was the night before my birthday.",
|
|
||||||
"stream": False,
|
|
||||||
"response_format": "wav",
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
},
|
|
||||||
request_num=3,
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_plain_text_pcm_streaming(self, omni_server, openai_client) -> None:
|
|
||||||
"""Streaming PCM via talker2code2wav_async_chunk + the connector's codec_streaming path.
|
|
||||||
|
|
||||||
The deploy YAML pins ``async_chunk: true`` and ``codec_streaming: true`` so chunks
|
|
||||||
flow Stage 0 -> Stage 1 per AR loop. The Stage-1 codec honors
|
|
||||||
``meta.left_context_size`` and ``meta.right_holdback_size`` so per-chunk codec
|
|
||||||
windows stitch into a coherent PCM stream. The byte-count gate is the same as
|
|
||||||
the sync paths; per-chunk audio content is verified offline against Whisper.
|
|
||||||
|
|
||||||
NOTE: ``min_hnr_db=0.0`` sits below the typical speech-noise floor so the
|
|
||||||
check still catches catastrophic codec failure (silence, white noise, sample
|
|
||||||
scramble all give HNR << 0) while allowing for the sliding-window codec's
|
|
||||||
slightly-noisier-than-sync output. The default 1.0 dB threshold has only
|
|
||||||
~0.16 dB margin over measured single-request output (1.16 dB on L4), so
|
|
||||||
flake risk is too high there.
|
|
||||||
"""
|
|
||||||
openai_client.send_audio_speech_request(
|
|
||||||
{
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": "The quick brown fox jumps over the lazy dog.",
|
|
||||||
"stream": True,
|
|
||||||
"response_format": "pcm",
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
"min_hnr_db": 0.0,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_concurrent_pcm_streaming(self, omni_server, openai_client) -> None:
|
|
||||||
"""Three concurrent streaming requests - guards per-request frame cursors
|
|
||||||
in ``talker2code2wav_async_chunk`` and per-slot delay-pattern state under batched AR.
|
|
||||||
|
|
||||||
NOTE: ``min_hnr_db=-2.0`` is looser than the single-request streaming test
|
|
||||||
because batched AR adds per-request codec quality variance (DAC boundary
|
|
||||||
artifacts compounded across the 3-way batch); -2.0 dB gives ~1.6 dB margin
|
|
||||||
over the measured worst-of-3 on L4 (-0.42 dB) while still well above the
|
|
||||||
catastrophic-failure region.
|
|
||||||
"""
|
|
||||||
openai_client.send_audio_speech_request(
|
|
||||||
{
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": "She sells seashells by the seashore.",
|
|
||||||
"stream": True,
|
|
||||||
"response_format": "pcm",
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
"min_hnr_db": -2.0,
|
|
||||||
},
|
|
||||||
request_num=3,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
|
|
||||||
class TestHiggsAudioV3OnlineInlineControlTokens:
|
|
||||||
"""Inline control-token surface from the bosonai cookbook.
|
|
||||||
|
|
||||||
The model exposes four categories of ``<|category:value|>`` tokens
|
|
||||||
(emotion, style, prosody, sfx). Delivery tokens (emotion / style /
|
|
||||||
prosody speed-pitch-expressive) go at the START of input; positional
|
|
||||||
tokens (pause, sfx) go inline. SFX tokens must pair with their written
|
|
||||||
onomatopoeia.
|
|
||||||
|
|
||||||
These tests check the *serving surface* - the validator accepts the
|
|
||||||
payload, the engine produces audio, and the WAV has non-trivial size.
|
|
||||||
They do not assert audio quality (out of scope for CI).
|
|
||||||
"""
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_inline_emotion_and_expressive(self, omni_server, openai_client) -> None:
|
|
||||||
"""Delivery tokens (emotion + expressive_high) at the start of input."""
|
|
||||||
openai_client.send_audio_speech_request(
|
|
||||||
{
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": ("<|emotion:amusement|><|prosody:expressive_high|>Wait, that was actually hilarious."),
|
|
||||||
"stream": False,
|
|
||||||
"response_format": "wav",
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_inline_style_whispering(self, omni_server, openai_client) -> None:
|
|
||||||
"""Style token at the start - ``whispering``."""
|
|
||||||
openai_client.send_audio_speech_request(
|
|
||||||
{
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": "<|style:whispering|>It is just between you and me, alright?",
|
|
||||||
"stream": False,
|
|
||||||
"response_format": "wav",
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_inline_prosody_speed_and_pitch(self, omni_server, openai_client) -> None:
|
|
||||||
"""Two prosody tokens at the start - slow speed plus low pitch."""
|
|
||||||
openai_client.send_audio_speech_request(
|
|
||||||
{
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": (
|
|
||||||
"<|prosody:speed_slow|><|prosody:pitch_low|>The radar shows a storm approaching from the east."
|
|
||||||
),
|
|
||||||
"stream": False,
|
|
||||||
"response_format": "wav",
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_inline_pause_mid_text(self, omni_server, openai_client) -> None:
|
|
||||||
"""Positional pause token placed inline between two clauses."""
|
|
||||||
openai_client.send_audio_speech_request(
|
|
||||||
{
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": "Hold on a moment <|prosody:pause|> let me think about it.",
|
|
||||||
"stream": False,
|
|
||||||
"response_format": "wav",
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_inline_sfx_with_onomatopoeia(self, omni_server, openai_client) -> None:
|
|
||||||
"""SFX token paired with its written onomatopoeia (``<|sfx:laughter|>Hehe``)."""
|
|
||||||
openai_client.send_audio_speech_request(
|
|
||||||
{
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": (
|
|
||||||
"<|emotion:amusement|>I cannot believe that just happened. "
|
|
||||||
"<|sfx:laughter|>Hehe, I am still recovering from it."
|
|
||||||
),
|
|
||||||
"stream": False,
|
|
||||||
"response_format": "wav",
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
|
|
||||||
class TestHiggsAudioV3OnlineVoiceClone:
|
|
||||||
"""Voice clone via the two payload shapes the model serves."""
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_voice_clone_ref_audio_ref_text(self, omni_server, openai_client) -> None:
|
|
||||||
"""Canonical vllm-omni voice clone via ``ref_audio`` + ``ref_text``."""
|
|
||||||
openai_client.send_audio_speech_request(
|
|
||||||
{
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": "Hello world.",
|
|
||||||
"ref_audio": _REF_AUDIO_URL,
|
|
||||||
"ref_text": _REF_TEXT,
|
|
||||||
"stream": False,
|
|
||||||
"response_format": "wav",
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_voice_clone_references_alias(self, omni_server, openai_client) -> None:
|
|
||||||
"""BosonAI cookbook payload: ``references=[{audio_path, text}]``.
|
|
||||||
|
|
||||||
``normalize_references_alias`` (``protocol/audio.py``) is supposed to
|
|
||||||
translate the cookbook field into ``ref_audio`` / ``ref_text``; without
|
|
||||||
it the request would silently fall through to zero-shot synthesis.
|
|
||||||
"""
|
|
||||||
openai_client.send_audio_speech_request(
|
|
||||||
{
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": "Hello world.",
|
|
||||||
"references": [{"audio_path": _REF_AUDIO_URL, "text": _REF_TEXT}],
|
|
||||||
"stream": False,
|
|
||||||
"response_format": "wav",
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
|
|
||||||
class TestHiggsAudioV3OnlineValidatorRejections:
|
|
||||||
"""Out-of-scope shapes must come back as 4xx.
|
|
||||||
|
|
||||||
These cases use ``send_audio_speech_http_request`` (raw HTTP POST against
|
|
||||||
``/v1/audio/speech``) rather than the OpenAI SDK helper because:
|
|
||||||
|
|
||||||
- We need the failure-path matcher; the SDK helper asserts success.
|
|
||||||
- The cookbook ``references`` field is not in the OpenAI SDK schema, so
|
|
||||||
the SDK strips it before the request leaves the client and the server
|
|
||||||
sees a plain TTS request that returns 200. Raw HTTP keeps the JSON body
|
|
||||||
intact so the validator actually fires.
|
|
||||||
"""
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_rejects_empty_input(self, omni_server, openai_client) -> None:
|
|
||||||
"""Empty text input - validator should reject before reaching the engine."""
|
|
||||||
openai_client.send_audio_speech_http_request(
|
|
||||||
{
|
|
||||||
"json": {
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": "",
|
|
||||||
"response_format": "wav",
|
|
||||||
},
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"err_code": (400, 422),
|
|
||||||
"err_message": ("input", "empty"),
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_rejects_multi_reference_payload(self, omni_server, openai_client) -> None:
|
|
||||||
"""``references[]`` with more than one entry - multi-shot voice clone is not supported."""
|
|
||||||
openai_client.send_audio_speech_http_request(
|
|
||||||
{
|
|
||||||
"json": {
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": "Hello world.",
|
|
||||||
"references": [
|
|
||||||
{"audio_path": _REF_AUDIO_URL, "text": _REF_TEXT},
|
|
||||||
{"audio_path": _REF_AUDIO_URL, "text": _REF_TEXT},
|
|
||||||
],
|
|
||||||
"response_format": "wav",
|
|
||||||
},
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"err_code": (400, 422),
|
|
||||||
"err_message": "references",
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
|
||||||
def test_rejects_conflicting_ref_audio_and_references(self, omni_server, openai_client) -> None:
|
|
||||||
"""``ref_audio`` and ``references`` cannot both be set to different values."""
|
|
||||||
openai_client.send_audio_speech_http_request(
|
|
||||||
{
|
|
||||||
"json": {
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": "Hello world.",
|
|
||||||
"ref_audio": _REF_AUDIO_URL,
|
|
||||||
"references": [{"audio_path": "https://example.com/other.wav"}],
|
|
||||||
"response_format": "wav",
|
|
||||||
},
|
|
||||||
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
|
|
||||||
"err_code": (400, 422),
|
|
||||||
"err_message": "mutually exclusive",
|
|
||||||
}
|
|
||||||
)
|
|
||||||
@@ -30,8 +30,7 @@ from tests.helpers.mark import hardware_test
|
|||||||
from tests.helpers.runtime import OmniServerParams
|
from tests.helpers.runtime import OmniServerParams
|
||||||
from tests.helpers.stage_config import get_deploy_config_path
|
from tests.helpers.stage_config import get_deploy_config_path
|
||||||
|
|
||||||
# TODO: Fix this test
|
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
||||||
# pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
|
||||||
|
|
||||||
MODEL = "OpenMOSS-Team/MOSS-TTS-Realtime"
|
MODEL = "OpenMOSS-Team/MOSS-TTS-Realtime"
|
||||||
REF_AUDIO_URL = "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS/HEAD/assets/audio/reference_zh_1.wav"
|
REF_AUDIO_URL = "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS/HEAD/assets/audio/reference_zh_1.wav"
|
||||||
|
|||||||
@@ -9,40 +9,63 @@ import pytest
|
|||||||
from tests.helpers.mark import hardware_test
|
from tests.helpers.mark import hardware_test
|
||||||
from tests.helpers.media import generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video
|
from tests.helpers.media import generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video
|
||||||
from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data
|
from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data
|
||||||
from tests.helpers.stage_config import get_deploy_config_path
|
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||||
|
from vllm_omni.platforms import current_omni_platform
|
||||||
|
|
||||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||||
|
|
||||||
|
|
||||||
|
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
|
||||||
|
|
||||||
# Set VLLM_TEST_PD_MODE=1 to test PD disaggregation (follow-up — deploy overlay not yet migrated).
|
# Set VLLM_TEST_PD_MODE=1 to test PD disaggregation (follow-up — deploy overlay not yet migrated).
|
||||||
_USE_PD = os.environ.get("VLLM_TEST_PD_MODE", "0") == "1"
|
_USE_PD = os.environ.get("VLLM_TEST_PD_MODE", "0") == "1"
|
||||||
|
|
||||||
_MODEL = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
|
|
||||||
_CI_DEPLOY = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
|
_CI_DEPLOY = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
|
||||||
|
|
||||||
|
|
||||||
# For prefix caching checks against we enable it on the thinker and talker via CLI override
|
def get_chunk_config(config_path: str | None = None):
|
||||||
# and enable prompt token details so that we can determine if any tokens were cached.
|
"""Load the qwen3_omni CI deploy yaml with async_chunk modifications for streaming mode."""
|
||||||
# We also explicitly set block size so that we can make sure the cached token counts are a
|
if config_path is None:
|
||||||
# multiple of the block size.
|
config_path = _CI_DEPLOY
|
||||||
BLOCK_SIZE = 16
|
# TODO: remove this workaround once legacy `stage_args` path is deleted.
|
||||||
|
# The pipeline (qwen3_omni/pipeline.py) already wires
|
||||||
|
# thinker2talker_async_chunk / talker2code2wav_async_chunk on stage 0/1,
|
||||||
|
# so only async_chunk needs flipping. Writing nested `engine_args:` into
|
||||||
|
# the new-schema overlay trips _parse_stage_deploy's legacy branch and
|
||||||
|
# drops flat fields (load_format, max_num_seqs, ...).
|
||||||
|
return modify_stage_config(config_path, updates={"async_chunk": True})
|
||||||
|
|
||||||
|
|
||||||
|
# Platform-specific overrides live inside the new deploy yaml's ``platforms:``
|
||||||
|
# section, so a single ``_CI_DEPLOY`` path serves CUDA, ROCm, and XPU.
|
||||||
|
# TODO: re-add VLLM_TEST_PD_MODE branch once the PD-disaggregation deploy
|
||||||
|
# overlay has been migrated to the new schema (previously used the deleted
|
||||||
|
# ``qwen3_omni_moe_pd_ci.yaml`` stage-configs file).
|
||||||
|
if current_omni_platform.is_xpu():
|
||||||
|
stage_configs = [_CI_DEPLOY]
|
||||||
|
else: # CUDA + ROCm MI325 share the same deploy config
|
||||||
|
stage_configs = [get_chunk_config()]
|
||||||
|
|
||||||
|
# Create parameter combinations for model and stage config
|
||||||
test_params = [
|
test_params = [
|
||||||
pytest.param(
|
OmniServerParams(model=model, stage_config_path=stage_config) for model in models for stage_config in stage_configs
|
||||||
OmniServerParams(
|
]
|
||||||
model=_MODEL,
|
# For prefix caching, we enable it on the thinker (stage 0) via CLI override
|
||||||
stage_config_path=_CI_DEPLOY,
|
# and enable prompt token details so that we can determine if any tokens were cached.
|
||||||
use_stage_cli=True,
|
BLOCK_SIZE = 16
|
||||||
server_args=[
|
prefix_test_params = [
|
||||||
"--no-async-chunk",
|
OmniServerParams(
|
||||||
"--block-size",
|
model=model,
|
||||||
str(BLOCK_SIZE),
|
stage_config_path=_CI_DEPLOY,
|
||||||
"--stage-overrides",
|
server_args=[
|
||||||
'{"0": {"enable_prefix_caching": true}, "1": {"enable_prefix_caching": true}}',
|
"--block-size",
|
||||||
"--enable-prompt-tokens-details",
|
str(BLOCK_SIZE),
|
||||||
],
|
"--stage-overrides",
|
||||||
),
|
'{"0": {"enable_prefix_caching": true}}',
|
||||||
id="default",
|
"--enable-prompt-tokens-details",
|
||||||
|
],
|
||||||
)
|
)
|
||||||
|
for model in models
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
@@ -143,14 +166,45 @@ def test_text_to_text_001(omni_server, openai_client) -> None:
|
|||||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||||
|
|
||||||
|
|
||||||
def _run_prefix_cache_check(openai_client, request_config: dict):
|
@pytest.mark.advanced_model
|
||||||
"""Make two requests given a request config, and validate that:
|
@pytest.mark.core_model
|
||||||
1. The second request actually had cached tokens
|
@pytest.mark.omni
|
||||||
2. The number of cached tokens is divisible by the block size used in
|
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||||
test_params, because currently upstream vLLM does not cache partial
|
@pytest.mark.parametrize("omni_server", prefix_test_params, indirect=True)
|
||||||
blocks.
|
def test_thinker_prefix_caching(omni_server, openai_client, run_level) -> None:
|
||||||
"""
|
"""
|
||||||
openai_client.send_omni_request(request_config, request_num=1)[0]
|
Test thinker prefix caching by sending identical requests with an image (i.e.,
|
||||||
|
a large shared prefix) and verifying that the second request uses cached tokens
|
||||||
|
& produces the same output with greedy decoding.
|
||||||
|
|
||||||
|
NOTE: The reason that we check against logprobs instead of direct text here is that
|
||||||
|
the outputs may still diverge a bit even though we set the seed and temperature.
|
||||||
|
This is mostly because the GEMM algorithm may vary based on the input tensors dims.
|
||||||
|
Because of this, we don't check the logprobs if it's a dummy load, since in that case
|
||||||
|
the top logprobs will all be very close.
|
||||||
|
"""
|
||||||
|
seed = 10
|
||||||
|
img_res = generate_synthetic_image(224, 224, seed=seed)
|
||||||
|
image_data_url = f"data:image/jpeg;base64,{img_res['base64']}"
|
||||||
|
messages = dummy_messages_from_mix_data(
|
||||||
|
system_prompt=get_system_prompt(),
|
||||||
|
image_data_url=image_data_url,
|
||||||
|
content_text=get_prompt("text_image"),
|
||||||
|
)
|
||||||
|
|
||||||
|
top_k = 10
|
||||||
|
sampling_params = {"seed": seed, "temperature": 0, "max_tokens": 8, "logprobs": top_k}
|
||||||
|
request_config = {
|
||||||
|
"model": omni_server.model,
|
||||||
|
"messages": messages,
|
||||||
|
"stream": False,
|
||||||
|
"modalities": ["text"],
|
||||||
|
"logprobs": True,
|
||||||
|
"top_logprobs": top_k,
|
||||||
|
"sampling_params_list": [sampling_params] * 3,
|
||||||
|
}
|
||||||
|
|
||||||
|
uncached_response = openai_client.send_omni_request(request_config, request_num=1)[0]
|
||||||
cached_response = openai_client.send_omni_request(request_config, request_num=1)[0]
|
cached_response = openai_client.send_omni_request(request_config, request_num=1)[0]
|
||||||
|
|
||||||
# Ensure that we have a prefix cache hit on the second request and that only the last
|
# Ensure that we have a prefix cache hit on the second request and that only the last
|
||||||
@@ -163,66 +217,16 @@ def _run_prefix_cache_check(openai_client, request_config: dict):
|
|||||||
assert num_cached_tokens % BLOCK_SIZE == 0
|
assert num_cached_tokens % BLOCK_SIZE == 0
|
||||||
assert (num_cached_tokens + num_uncached_tokens) == num_prompt_tokens
|
assert (num_cached_tokens + num_uncached_tokens) == num_prompt_tokens
|
||||||
|
|
||||||
|
# Ensure that we have logprobs and tokens were generated for both requests
|
||||||
|
assert uncached_response.logprobs is not None
|
||||||
|
assert cached_response.logprobs is not None
|
||||||
|
n_tokens = min(len(uncached_response.logprobs), len(cached_response.logprobs))
|
||||||
|
assert n_tokens > 0
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
if run_level == "advanced_model":
|
||||||
@pytest.mark.core_model
|
# For each token index where both responses have an output, ensure that the greedy token
|
||||||
@pytest.mark.omni
|
# predicted in the uncached case is in the top k logprobs for the cached case
|
||||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
for idx in range(n_tokens):
|
||||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
greedy_token = uncached_response.logprobs[idx].token
|
||||||
def test_thinker_prefix_caching_text_output(omni_server, openai_client) -> None:
|
cached_top_k = {lp.token for lp in cached_response.logprobs[idx].top_logprobs}
|
||||||
"""
|
assert greedy_token in cached_top_k
|
||||||
Test thinker prefix caching by sending identical requests with an image (i.e.,
|
|
||||||
a large shared prefix) and verifying that the second request uses cached tokens
|
|
||||||
& produces the same output with greedy decoding.
|
|
||||||
|
|
||||||
NOTE: Checking the output of prefix caching directly can be a bit unstable
|
|
||||||
due to slight numerical differences as a result of running different scheduled
|
|
||||||
sequence lengths. As such, for E2E tests on prefix cache, we only check the cached
|
|
||||||
token count and not the output, since the omni tensor cache has solid unit tests,
|
|
||||||
and the core prefix cache algorithm is largely tested by upstream vLLM.
|
|
||||||
"""
|
|
||||||
img_res = generate_synthetic_image(224, 224)
|
|
||||||
image_data_url = f"data:image/jpeg;base64,{img_res['base64']}"
|
|
||||||
messages = dummy_messages_from_mix_data(
|
|
||||||
system_prompt=get_system_prompt(),
|
|
||||||
image_data_url=image_data_url,
|
|
||||||
content_text=get_prompt("text_image"),
|
|
||||||
)
|
|
||||||
|
|
||||||
request_config = {
|
|
||||||
"model": omni_server.model,
|
|
||||||
"messages": messages,
|
|
||||||
"stream": False,
|
|
||||||
"modalities": ["text"],
|
|
||||||
}
|
|
||||||
_run_prefix_cache_check(openai_client, request_config)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.omni
|
|
||||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
|
||||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
|
||||||
def test_thinker_prefix_caching_audio_output(omni_server, openai_client) -> None:
|
|
||||||
"""
|
|
||||||
Verify that thinker prefix caching does not hang when the request
|
|
||||||
produces audio output (text + audio modalities). Sends two identical
|
|
||||||
requests so the second exercises the prefix-cached path through the
|
|
||||||
full thinker -> talker -> code2wav pipeline.
|
|
||||||
|
|
||||||
Regression test for https://github.com/vllm-project/vllm-omni/issues/3510
|
|
||||||
"""
|
|
||||||
messages = dummy_messages_from_mix_data(
|
|
||||||
system_prompt=get_system_prompt(),
|
|
||||||
content_text=get_prompt(),
|
|
||||||
)
|
|
||||||
request_config = {
|
|
||||||
"model": omni_server.model,
|
|
||||||
"messages": messages,
|
|
||||||
"stream": True,
|
|
||||||
"stream_options": {
|
|
||||||
"include_usage": True,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
_run_prefix_cache_check(openai_client, request_config)
|
|
||||||
@@ -1,97 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
"""
|
|
||||||
E2E Online tests for VoxCPM2 with text input and audio output.
|
|
||||||
|
|
||||||
These tests verify the /v1/audio/speech endpoint works correctly with
|
|
||||||
actual model inference, not mocks. VoxCPM2 zero-shot needs only ``input``
|
|
||||||
and a placeholder ``voice`` (``default``); no ``task_type`` / ``ref_text``.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import os
|
|
||||||
|
|
||||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from tests.helpers.mark import hardware_test
|
|
||||||
from tests.helpers.runtime import OmniServerParams
|
|
||||||
from tests.helpers.stage_config import get_deploy_config_path
|
|
||||||
|
|
||||||
MODEL = "openbmb/VoxCPM2"
|
|
||||||
DEFAULT_AUDIO_SPEECH_TIMEOUT_S = 300.0
|
|
||||||
MAX_CONCURRENT = 4
|
|
||||||
|
|
||||||
# ~0.5 s of 48 kHz mono PCM_16 in WAV (~48k payload + header).
|
|
||||||
_MIN_AUDIO_BYTES = 40_000
|
|
||||||
|
|
||||||
|
|
||||||
def get_prompt(prompt_type="text"):
|
|
||||||
"""English prompt for zero-shot TTS (Whisper-friendly for advanced_model accuracy)."""
|
|
||||||
prompts = {
|
|
||||||
"text": "The weather is nice today, perfect for a walk in the park.",
|
|
||||||
}
|
|
||||||
return prompts.get(prompt_type, prompts["text"])
|
|
||||||
|
|
||||||
|
|
||||||
tts_server_params = [
|
|
||||||
pytest.param(
|
|
||||||
OmniServerParams(
|
|
||||||
model=MODEL,
|
|
||||||
stage_config_path=get_deploy_config_path("voxcpm2.yaml"),
|
|
||||||
server_args=["--trust-remote-code", "--disable-log-stats"],
|
|
||||||
),
|
|
||||||
id="voxcpm2",
|
|
||||||
)
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
|
||||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
|
||||||
def test_text_to_audio_001(omni_server, openai_client) -> None:
|
|
||||||
"""
|
|
||||||
Test zero-shot text-to-audio via OpenAI API.
|
|
||||||
Deploy Setting: voxcpm2.yaml
|
|
||||||
Input Modal: text
|
|
||||||
Output Modal: audio
|
|
||||||
Input Setting: stream=False
|
|
||||||
Datasets: few requests (max_num_seqs=4)
|
|
||||||
"""
|
|
||||||
request_config = {
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": get_prompt(),
|
|
||||||
"stream": False,
|
|
||||||
"timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S,
|
|
||||||
"response_format": "wav",
|
|
||||||
"voice": "default",
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
}
|
|
||||||
openai_client.send_audio_speech_request(request_config, request_num=MAX_CONCURRENT)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
|
||||||
@pytest.mark.tts
|
|
||||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
|
||||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
|
||||||
def test_text_to_audio_002(omni_server, openai_client) -> None:
|
|
||||||
"""
|
|
||||||
Test zero-shot streaming text-to-audio via OpenAI API.
|
|
||||||
Deploy Setting: voxcpm2.yaml
|
|
||||||
Input Modal: text
|
|
||||||
Output Modal: audio
|
|
||||||
Input Setting: stream=True
|
|
||||||
Datasets: single request
|
|
||||||
"""
|
|
||||||
request_config = {
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": get_prompt(),
|
|
||||||
"stream": True,
|
|
||||||
"timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S,
|
|
||||||
"response_format": "wav",
|
|
||||||
"voice": "default",
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
}
|
|
||||||
openai_client.send_audio_speech_request(request_config)
|
|
||||||
@@ -1,95 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
"""
|
|
||||||
E2E Online expansion tests for VoxCPM2 (voice clone, PCM, concurrency).
|
|
||||||
|
|
||||||
Voice clone uses ``ref_audio`` only (no ``ref_text``). Reference clip is
|
|
||||||
vendored under tests/assets/qwen3_tts/clone_2.wav — see test_qwen3_tts_base.py.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import os
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from tests.helpers.mark import hardware_test
|
|
||||||
from tests.helpers.media import load_test_audio_data_url
|
|
||||||
from tests.helpers.runtime import OmniServerParams
|
|
||||||
from tests.helpers.stage_config import get_deploy_config_path
|
|
||||||
|
|
||||||
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
|
||||||
|
|
||||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
|
||||||
|
|
||||||
MODEL = "openbmb/VoxCPM2"
|
|
||||||
DEFAULT_AUDIO_SPEECH_TIMEOUT_S = 300.0
|
|
||||||
_MIN_AUDIO_BYTES = 40_000
|
|
||||||
MAX_CONCURRENT = 4
|
|
||||||
|
|
||||||
REF_AUDIO_URL = load_test_audio_data_url("qwen3_tts/clone_2.wav")
|
|
||||||
|
|
||||||
|
|
||||||
def get_prompt(prompt_type="text"):
|
|
||||||
prompts = {
|
|
||||||
"text": "The weather is nice today, perfect for a walk in the park.",
|
|
||||||
}
|
|
||||||
return prompts.get(prompt_type, prompts["text"])
|
|
||||||
|
|
||||||
|
|
||||||
tts_server_params = [
|
|
||||||
pytest.param(
|
|
||||||
OmniServerParams(
|
|
||||||
model=MODEL,
|
|
||||||
stage_config_path=get_deploy_config_path("voxcpm2.yaml"),
|
|
||||||
server_args=["--trust-remote-code", "--disable-log-stats"],
|
|
||||||
),
|
|
||||||
id="voxcpm2",
|
|
||||||
)
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
|
||||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
|
||||||
def test_voice_clone_streaming_001(omni_server, openai_client) -> None:
|
|
||||||
"""
|
|
||||||
Test voice-clone text-to-audio via OpenAI API.
|
|
||||||
Deploy Setting: voxcpm2.yaml
|
|
||||||
Input Modal: text + ref_audio
|
|
||||||
Output Modal: audio
|
|
||||||
Input Setting: stream=True
|
|
||||||
Datasets: few requests (max_num_seqs=4)
|
|
||||||
"""
|
|
||||||
request_config = {
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": get_prompt(),
|
|
||||||
"stream": True,
|
|
||||||
"timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S,
|
|
||||||
"response_format": "wav",
|
|
||||||
"voice": "default",
|
|
||||||
"ref_audio": REF_AUDIO_URL,
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
}
|
|
||||||
openai_client.send_audio_speech_request(request_config, request_num=MAX_CONCURRENT)
|
|
||||||
|
|
||||||
|
|
||||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
|
||||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
|
||||||
def test_response_format_001(omni_server, openai_client) -> None:
|
|
||||||
"""
|
|
||||||
Test voice-clone non-stream PCM output via OpenAI API.
|
|
||||||
Deploy Setting: voxcpm2.yaml
|
|
||||||
Input Modal: text + ref_audio
|
|
||||||
Output Modal: audio (pcm)
|
|
||||||
Input Setting: stream=False
|
|
||||||
Datasets: single request
|
|
||||||
"""
|
|
||||||
request_config = {
|
|
||||||
"model": omni_server.model,
|
|
||||||
"input": get_prompt(),
|
|
||||||
"response_format": "pcm",
|
|
||||||
"stream": False,
|
|
||||||
"timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S,
|
|
||||||
"voice": "default",
|
|
||||||
"ref_audio": REF_AUDIO_URL,
|
|
||||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
|
||||||
}
|
|
||||||
openai_client.send_audio_speech_request(request_config)
|
|
||||||
@@ -427,7 +427,6 @@ async def _enqueue_add_request(
|
|||||||
sampling_params_list=sampling_params_list,
|
sampling_params_list=sampling_params_list,
|
||||||
final_stage_id=final_stage_id,
|
final_stage_id=final_stage_id,
|
||||||
preprocess_ms=0.0,
|
preprocess_ms=0.0,
|
||||||
request_timestamp=time.time(),
|
|
||||||
enqueue_ts=time.perf_counter(),
|
enqueue_ts=time.perf_counter(),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
@@ -985,7 +984,6 @@ async def test_handle_streaming_update_passes_prompt_text_to_stage_pool() -> Non
|
|||||||
sampling_params_list=[_sampling_params()],
|
sampling_params_list=[_sampling_params()],
|
||||||
final_stage_id=0,
|
final_stage_id=0,
|
||||||
preprocess_ms=0.0,
|
preprocess_ms=0.0,
|
||||||
request_timestamp=time.time(),
|
|
||||||
enqueue_ts=time.perf_counter(),
|
enqueue_ts=time.perf_counter(),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -156,7 +156,7 @@ class FakeAsyncOmni:
|
|||||||
self.captured_prompt = None
|
self.captured_prompt = None
|
||||||
self._images = images or [Image.new("RGB", (64, 64), color="green")]
|
self._images = images or [Image.new("RGB", (64, 64), color="green")]
|
||||||
|
|
||||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None):
|
||||||
if sampling_params_list is not None:
|
if sampling_params_list is not None:
|
||||||
self.captured_sampling_params_list = sampling_params_list
|
self.captured_sampling_params_list = sampling_params_list
|
||||||
else:
|
else:
|
||||||
@@ -254,7 +254,7 @@ def async_omni_test_client():
|
|||||||
self._images = [Image.new("RGB", (64, 64), color="green")]
|
self._images = [Image.new("RGB", (64, 64), color="green")]
|
||||||
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
||||||
|
|
||||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None):
|
||||||
if sampling_params_list is not None:
|
if sampling_params_list is not None:
|
||||||
self.captured_sampling_params_list = sampling_params_list
|
self.captured_sampling_params_list = sampling_params_list
|
||||||
else:
|
else:
|
||||||
@@ -318,7 +318,7 @@ def async_omni_rgba_test_client():
|
|||||||
self._images = [Image.new("RGBA", (64, 64), color=(0, 255, 0, 128))]
|
self._images = [Image.new("RGBA", (64, 64), color=(0, 255, 0, 128))]
|
||||||
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
||||||
|
|
||||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None):
|
||||||
if sampling_params_list is not None:
|
if sampling_params_list is not None:
|
||||||
self.captured_sampling_params_list = sampling_params_list
|
self.captured_sampling_params_list = sampling_params_list
|
||||||
else:
|
else:
|
||||||
@@ -382,7 +382,7 @@ def async_omni_stage_configs_only_client():
|
|||||||
self._images = [Image.new("RGB", (64, 64), color="green")]
|
self._images = [Image.new("RGB", (64, 64), color="green")]
|
||||||
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
||||||
|
|
||||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None):
|
||||||
if sampling_params_list is not None:
|
if sampling_params_list is not None:
|
||||||
self.captured_sampling_params_list = sampling_params_list
|
self.captured_sampling_params_list = sampling_params_list
|
||||||
else:
|
else:
|
||||||
@@ -442,7 +442,7 @@ def streaming_image_edit_client():
|
|||||||
self.captured_prompt = None
|
self.captured_prompt = None
|
||||||
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
||||||
|
|
||||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None):
|
||||||
self.captured_prompt = prompt
|
self.captured_prompt = prompt
|
||||||
self.captured_sampling_params_list = sampling_params_list or [sampling_params]
|
self.captured_sampling_params_list = sampling_params_list or [sampling_params]
|
||||||
assert self.captured_sampling_params_list[0].output_kind == RequestOutputKind.DELTA
|
assert self.captured_sampling_params_list[0].output_kind == RequestOutputKind.DELTA
|
||||||
@@ -696,7 +696,7 @@ def test_generate_images_async_omni_glm_image_sets_stage0_max_tokens():
|
|||||||
self._images = [Image.new("RGB", (64, 64), color="green")]
|
self._images = [Image.new("RGB", (64, 64), color="green")]
|
||||||
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
||||||
|
|
||||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None):
|
||||||
self.captured_sampling_params_list = (
|
self.captured_sampling_params_list = (
|
||||||
sampling_params_list if sampling_params_list is not None else [sampling_params]
|
sampling_params_list if sampling_params_list is not None else [sampling_params]
|
||||||
)
|
)
|
||||||
@@ -815,9 +815,7 @@ def test_image_edits_streaming_returns_ar_delta_then_image(streaming_image_edit_
|
|||||||
|
|
||||||
|
|
||||||
def test_image_edits_streaming_ar_delta_chunks_include_index(streaming_image_edit_client):
|
def test_image_edits_streaming_ar_delta_chunks_include_index(streaming_image_edit_client):
|
||||||
async def generate_multi_output_delta(
|
async def generate_multi_output_delta(prompt, request_id, sampling_params=None, sampling_params_list=None):
|
||||||
prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs
|
|
||||||
):
|
|
||||||
yield MockStageResult(stage_id=0, final_output_type="text", texts=["first", "second"])
|
yield MockStageResult(stage_id=0, final_output_type="text", texts=["first", "second"])
|
||||||
yield MockStageResult(
|
yield MockStageResult(
|
||||||
stage_id=1,
|
stage_id=1,
|
||||||
@@ -845,7 +843,7 @@ def test_image_edits_streaming_ar_delta_chunks_include_index(streaming_image_edi
|
|||||||
|
|
||||||
|
|
||||||
def test_image_edits_streaming_errors_without_final_image(streaming_image_edit_client):
|
def test_image_edits_streaming_errors_without_final_image(streaming_image_edit_client):
|
||||||
async def generate_without_image(prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
async def generate_without_image(prompt, request_id, sampling_params=None, sampling_params_list=None):
|
||||||
yield MockStageResult(stage_id=0, final_output_type="text", text="recap")
|
yield MockStageResult(stage_id=0, final_output_type="text", text="recap")
|
||||||
|
|
||||||
streaming_image_edit_client.app.state.engine_client.generate = generate_without_image
|
streaming_image_edit_client.app.state.engine_client.generate = generate_without_image
|
||||||
@@ -868,7 +866,7 @@ def test_image_edits_streaming_errors_without_final_image(streaming_image_edit_c
|
|||||||
|
|
||||||
|
|
||||||
def test_image_edits_streaming_errors_on_empty_final_image(streaming_image_edit_client):
|
def test_image_edits_streaming_errors_on_empty_final_image(streaming_image_edit_client):
|
||||||
async def generate_empty_image(prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
async def generate_empty_image(prompt, request_id, sampling_params=None, sampling_params_list=None):
|
||||||
yield MockStageResult(stage_id=0, final_output_type="text", text="recap")
|
yield MockStageResult(stage_id=0, final_output_type="text", text="recap")
|
||||||
yield MockStageResult(stage_id=1, final_output_type="image", images=[])
|
yield MockStageResult(stage_id=1, final_output_type="image", images=[])
|
||||||
|
|
||||||
|
|||||||
@@ -2,11 +2,6 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
"""
|
"""
|
||||||
E2E online tests for Qwen3-Omni /v1/realtime WebSocket (streaming PCM in, audio out).
|
E2E online tests for Qwen3-Omni /v1/realtime WebSocket (streaming PCM in, audio out).
|
||||||
|
|
||||||
Three scenarios:
|
|
||||||
- Ready CI: async_chunk on, smoke only (no send delay, no accuracy check).
|
|
||||||
- Merge CI: async_chunk on + send delay, full accuracy check.
|
|
||||||
- Merge CI: async_chunk off, no send delay, full accuracy check.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -39,32 +34,18 @@ REALTIME_SYNTH_PHRASE_TEXT = (
|
|||||||
"Translate into Chinese: Beijing is the Capital of China. It is the center of culture and politics"
|
"Translate into Chinese: Beijing is the Capital of China. It is the center of culture and politics"
|
||||||
)
|
)
|
||||||
|
|
||||||
# Simulate realtime upload pacing (``openai_realtime_client.py --send-delay-ms``).
|
# The new-schema CI overlay bakes in async_chunk: False and covers CUDA/ROCm/XPU
|
||||||
SEND_DELAY_MS = 200
|
# via its ``platforms:`` section, so one path serves all three.
|
||||||
|
|
||||||
# CI overlay bakes in async_chunk: False and covers CUDA/ROCm/XPU via ``platforms:``.
|
|
||||||
default_stage_config = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
|
default_stage_config = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
|
||||||
|
|
||||||
realtime_sync_server_params = [
|
realtime_server_params = [
|
||||||
pytest.param(
|
pytest.param(
|
||||||
OmniServerParams(
|
OmniServerParams(
|
||||||
model=MODEL,
|
model=MODEL,
|
||||||
stage_config_path=default_stage_config,
|
stage_config_path=default_stage_config,
|
||||||
use_stage_cli=True,
|
use_stage_cli=True,
|
||||||
),
|
),
|
||||||
id="sync",
|
id="default",
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
realtime_async_chunk_server_params = [
|
|
||||||
pytest.param(
|
|
||||||
OmniServerParams(
|
|
||||||
model=MODEL,
|
|
||||||
stage_config_path=default_stage_config,
|
|
||||||
use_stage_cli=True,
|
|
||||||
server_args=["--async-chunk"],
|
|
||||||
),
|
|
||||||
id="async_chunk",
|
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -99,7 +80,6 @@ async def _run_realtime_audio_roundtrip(
|
|||||||
pcm16: bytes,
|
pcm16: bytes,
|
||||||
*,
|
*,
|
||||||
chunk_ms: int = 100,
|
chunk_ms: int = 100,
|
||||||
send_delay_ms: int = 0,
|
|
||||||
) -> dict:
|
) -> dict:
|
||||||
uri = f"ws://{host}:{port}/v1/realtime"
|
uri = f"ws://{host}:{port}/v1/realtime"
|
||||||
incremental: list[bytes] = []
|
incremental: list[bytes] = []
|
||||||
@@ -125,8 +105,6 @@ async def _run_realtime_audio_roundtrip(
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
if send_delay_ms > 0:
|
|
||||||
await asyncio.sleep(send_delay_ms / 1000.0)
|
|
||||||
|
|
||||||
await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True}))
|
await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True}))
|
||||||
|
|
||||||
@@ -178,92 +156,26 @@ async def _run_realtime_audio_roundtrip(
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def _synthetic_pcm16_input() -> bytes:
|
|
||||||
syn = generate_synthetic_audio(
|
|
||||||
10,
|
|
||||||
1,
|
|
||||||
sample_rate=16000,
|
|
||||||
phrase_text=REALTIME_SYNTH_PHRASE_TEXT,
|
|
||||||
)
|
|
||||||
wav_bytes = base64.b64decode(syn["base64"])
|
|
||||||
return _pcm16_mono_16k_from_wav_bytes(wav_bytes)
|
|
||||||
|
|
||||||
|
|
||||||
def _assert_realtime_smoke(result: dict) -> None:
|
|
||||||
out_pcm = result["output_pcm"]
|
|
||||||
assert result["delta_events"] >= 1
|
|
||||||
assert out_pcm, "No output PCM from response.audio.delta"
|
|
||||||
assert len(out_pcm) % 2 == 0
|
|
||||||
assert len(out_pcm) >= 4096, "Output audio unexpectedly small"
|
|
||||||
assert result["output_sample_rate"] > 0
|
|
||||||
|
|
||||||
|
|
||||||
def _assert_realtime_accuracy(result: dict) -> None:
|
|
||||||
final_text = (result["transcription_text"] or "").strip()
|
|
||||||
assert final_text, "Expected non-empty transcription (model text stream)"
|
|
||||||
|
|
||||||
wav_out = _wav_bytes_from_pcm16(result["output_pcm"], result["output_sample_rate"])
|
|
||||||
whisper_text = convert_audio_bytes_to_text(wav_out).strip()
|
|
||||||
assert whisper_text, "Whisper returned empty string for synthesized output audio"
|
|
||||||
|
|
||||||
sim = cosine_similarity_text(whisper_text.lower(), final_text.lower())
|
|
||||||
assert sim > 0.8, (
|
|
||||||
f"Output audio transcript should match model text (sim={sim:.3f}): "
|
|
||||||
f"whisper={whisper_text!r}, model_text={final_text!r}"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class TestQwen3OmniRealtimeWebSocket:
|
class TestQwen3OmniRealtimeWebSocket:
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.omni
|
|
||||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
|
||||||
@pytest.mark.parametrize("omni_server", realtime_async_chunk_server_params, indirect=True)
|
|
||||||
def test_streaming_audio_input_pcm_output_async_chunk_smoke(self, omni_server) -> None:
|
|
||||||
"""Ready CI: async_chunk on, no send delay, smoke only."""
|
|
||||||
pcm16 = _synthetic_pcm16_input()
|
|
||||||
|
|
||||||
result = asyncio.run(
|
|
||||||
_run_realtime_audio_roundtrip(
|
|
||||||
omni_server.host,
|
|
||||||
omni_server.port,
|
|
||||||
omni_server.model,
|
|
||||||
pcm16,
|
|
||||||
chunk_ms=100,
|
|
||||||
send_delay_ms=0,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
_assert_realtime_smoke(result)
|
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
@pytest.mark.omni
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||||
@pytest.mark.parametrize("omni_server", realtime_async_chunk_server_params, indirect=True)
|
@pytest.mark.parametrize("omni_server", realtime_server_params, indirect=True)
|
||||||
def test_streaming_audio_input_pcm_output_async_chunk(self, omni_server) -> None:
|
|
||||||
"""Merge CI: async_chunk on, paced upload, full accuracy check."""
|
|
||||||
pcm16 = _synthetic_pcm16_input()
|
|
||||||
|
|
||||||
result = asyncio.run(
|
|
||||||
_run_realtime_audio_roundtrip(
|
|
||||||
omni_server.host,
|
|
||||||
omni_server.port,
|
|
||||||
omni_server.model,
|
|
||||||
pcm16,
|
|
||||||
chunk_ms=100,
|
|
||||||
send_delay_ms=SEND_DELAY_MS,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
_assert_realtime_smoke(result)
|
|
||||||
_assert_realtime_accuracy(result)
|
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
|
||||||
@pytest.mark.omni
|
|
||||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
|
||||||
@pytest.mark.parametrize("omni_server", realtime_sync_server_params, indirect=True)
|
|
||||||
def test_streaming_audio_input_pcm_output(self, omni_server) -> None:
|
def test_streaming_audio_input_pcm_output(self, omni_server) -> None:
|
||||||
"""Merge CI: async_chunk off, no send delay, full accuracy check."""
|
"""
|
||||||
pcm16 = _synthetic_pcm16_input()
|
Short streamed 16 kHz mono PCM16 input; expect streamed PCM16 audio deltas and
|
||||||
|
transcription. Verify Whisper(output audio) aligns with model text (same idea
|
||||||
|
as multimodal omni e2e). Input speech is synthesized from
|
||||||
|
``REALTIME_SYNTH_PHRASE_TEXT``.
|
||||||
|
"""
|
||||||
|
syn = generate_synthetic_audio(
|
||||||
|
10,
|
||||||
|
1,
|
||||||
|
sample_rate=16000,
|
||||||
|
phrase_text=REALTIME_SYNTH_PHRASE_TEXT,
|
||||||
|
)
|
||||||
|
wav_bytes = base64.b64decode(syn["base64"])
|
||||||
|
pcm16 = _pcm16_mono_16k_from_wav_bytes(wav_bytes)
|
||||||
|
|
||||||
result = asyncio.run(
|
result = asyncio.run(
|
||||||
_run_realtime_audio_roundtrip(
|
_run_realtime_audio_roundtrip(
|
||||||
@@ -272,9 +184,25 @@ class TestQwen3OmniRealtimeWebSocket:
|
|||||||
omni_server.model,
|
omni_server.model,
|
||||||
pcm16,
|
pcm16,
|
||||||
chunk_ms=100,
|
chunk_ms=100,
|
||||||
send_delay_ms=0,
|
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
_assert_realtime_smoke(result)
|
out_pcm = result["output_pcm"]
|
||||||
_assert_realtime_accuracy(result)
|
assert result["delta_events"] >= 1
|
||||||
|
assert out_pcm, "No output PCM from response.audio.delta"
|
||||||
|
assert len(out_pcm) % 2 == 0
|
||||||
|
assert len(out_pcm) >= 4096, "Output audio unexpectedly small"
|
||||||
|
assert result["output_sample_rate"] > 0
|
||||||
|
|
||||||
|
final_text = (result["transcription_text"] or "").strip()
|
||||||
|
assert final_text, "Expected non-empty transcription (model text stream)"
|
||||||
|
|
||||||
|
wav_out = _wav_bytes_from_pcm16(out_pcm, result["output_sample_rate"])
|
||||||
|
whisper_text = convert_audio_bytes_to_text(wav_out).strip()
|
||||||
|
assert whisper_text, "Whisper returned empty string for synthesized output audio"
|
||||||
|
|
||||||
|
sim = cosine_similarity_text(whisper_text.lower(), final_text.lower())
|
||||||
|
assert sim > 0.8, (
|
||||||
|
f"Output audio transcript should match model text (sim={sim:.3f}): "
|
||||||
|
f"whisper={whisper_text!r}, model_text={final_text!r}"
|
||||||
|
)
|
||||||
@@ -61,7 +61,7 @@ def test_build_multistage_generation_inputs_applies_stage_specific_overrides(ser
|
|||||||
assert engine_prompt["prompt"] == "draw a robot"
|
assert engine_prompt["prompt"] == "draw a robot"
|
||||||
assert engine_prompt["modalities"] == ["img2img"]
|
assert engine_prompt["modalities"] == ["img2img"]
|
||||||
assert engine_prompt["negative_prompt"] == "blurry"
|
assert engine_prompt["negative_prompt"] == "blurry"
|
||||||
assert engine_prompt["mm_processor_kwargs"] == {"target_h": 768, "target_w": 1024, "vae_generator_seed": 0}
|
assert engine_prompt["mm_processor_kwargs"] == {"target_h": 768, "target_w": 1024}
|
||||||
assert engine_prompt["multi_modal_data"]["img2img"].size == (24, 24)
|
assert engine_prompt["multi_modal_data"]["img2img"].size == (24, 24)
|
||||||
|
|
||||||
assert len(sampling_params_list) == 3
|
assert len(sampling_params_list) == 3
|
||||||
|
|||||||
@@ -399,8 +399,6 @@ def test_sampling_params_pass_through(test_client, mocker: MockerFixture):
|
|||||||
"true_cfg_scale": "4.0",
|
"true_cfg_scale": "4.0",
|
||||||
"boundary_ratio": "0.7",
|
"boundary_ratio": "0.7",
|
||||||
"flow_shift": "0.25",
|
"flow_shift": "0.25",
|
||||||
"generate_sound": "true",
|
|
||||||
"sound_duration": "2.5",
|
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -415,8 +413,6 @@ def test_sampling_params_pass_through(test_client, mocker: MockerFixture):
|
|||||||
assert captured.true_cfg_scale == 4.0
|
assert captured.true_cfg_scale == 4.0
|
||||||
assert captured.boundary_ratio == 0.7
|
assert captured.boundary_ratio == 0.7
|
||||||
assert captured.extra_args["flow_shift"] == 0.25
|
assert captured.extra_args["flow_shift"] == 0.25
|
||||||
assert captured.extra_args["generate_sound"] is True
|
|
||||||
assert captured.extra_args["sound_duration"] == 2.5
|
|
||||||
|
|
||||||
|
|
||||||
def test_frame_interpolation_params_pass_to_diffusion_sampling_params(test_client, mocker: MockerFixture):
|
def test_frame_interpolation_params_pass_to_diffusion_sampling_params(test_client, mocker: MockerFixture):
|
||||||
@@ -627,115 +623,6 @@ def test_video_job_persists_profiler_metadata(test_client, mocker: MockerFixture
|
|||||||
|
|
||||||
assert completed["stage_durations"] == {"diffuse": 2.5, "vae.decode": 0.3}
|
assert completed["stage_durations"] == {"diffuse": 2.5, "vae.decode": 0.3}
|
||||||
assert completed["peak_memory_mb"] == 4096.5
|
assert completed["peak_memory_mb"] == 4096.5
|
||||||
assert completed["action"] is None
|
|
||||||
|
|
||||||
|
|
||||||
def test_video_generation_response_exposes_action_payload(mocker: MockerFixture):
|
|
||||||
engine = FakeAsyncOmni()
|
|
||||||
handler = OmniOpenAIServingVideo.for_diffusion(
|
|
||||||
diffusion_engine=engine,
|
|
||||||
model_name="Cosmos3-8B-UVA",
|
|
||||||
)
|
|
||||||
|
|
||||||
async def _generate(prompt, request_id, sampling_params_list):
|
|
||||||
del prompt, request_id, sampling_params_list
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
yield MockVideoResult(
|
|
||||||
[object()],
|
|
||||||
custom_output={
|
|
||||||
"action": np.array([[[1.5, 2.5], [3.5, 4.5]]], dtype=np.float32),
|
|
||||||
"raw_action_dim": 2,
|
|
||||||
"action_mode": "policy",
|
|
||||||
"domain_id": 7,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
engine.generate = _generate
|
|
||||||
mocker.patch(
|
|
||||||
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
|
|
||||||
return_value="encoded-video",
|
|
||||||
)
|
|
||||||
|
|
||||||
response = asyncio.run(
|
|
||||||
handler.generate_videos(
|
|
||||||
VideoGenerationRequest(prompt="predict actions"),
|
|
||||||
"action-json",
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
action = response.data[0].action
|
|
||||||
assert action is not None
|
|
||||||
assert action.data == [[1.5, 2.5], [3.5, 4.5]]
|
|
||||||
assert action.shape == [2, 2]
|
|
||||||
assert action.dtype == "float32"
|
|
||||||
assert action.raw_action_dim == 2
|
|
||||||
assert action.action_mode == "policy"
|
|
||||||
assert action.domain_id == 7
|
|
||||||
assert response.model_dump(mode="json")["data"][0]["action"]["data"] == [[1.5, 2.5], [3.5, 4.5]]
|
|
||||||
|
|
||||||
|
|
||||||
def test_video_job_persists_action_metadata(test_client, mocker: MockerFixture):
|
|
||||||
engine = test_client.app.state.openai_serving_video._engine_client
|
|
||||||
|
|
||||||
async def _generate(prompt, request_id, sampling_params_list):
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
engine.captured_prompt = prompt
|
|
||||||
engine.captured_sampling_params_list = sampling_params_list
|
|
||||||
yield MockVideoResult(
|
|
||||||
[object()],
|
|
||||||
custom_output={
|
|
||||||
"action": np.array([[[1.0, 2.0], [3.0, 4.0]]], dtype=np.float32),
|
|
||||||
"raw_action_dim": 2,
|
|
||||||
"action_mode": "policy",
|
|
||||||
"domain_id": 7,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
engine.generate = _generate
|
|
||||||
mocker.patch(
|
|
||||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
|
||||||
return_value=b"fake-video",
|
|
||||||
)
|
|
||||||
|
|
||||||
response = test_client.post("/v1/videos", data={"prompt": "profile me"})
|
|
||||||
assert response.status_code == 200
|
|
||||||
video_id = response.json()["id"]
|
|
||||||
completed = _wait_for_status(test_client, video_id, VideoGenerationStatus.COMPLETED.value)
|
|
||||||
|
|
||||||
expected_action = {
|
|
||||||
"data": [[1.0, 2.0], [3.0, 4.0]],
|
|
||||||
"shape": [2, 2],
|
|
||||||
"dtype": "float32",
|
|
||||||
"raw_action_dim": 2,
|
|
||||||
"action_mode": "policy",
|
|
||||||
"domain_id": 7,
|
|
||||||
}
|
|
||||||
assert completed["action"] == expected_action
|
|
||||||
|
|
||||||
listed = test_client.get("/v1/videos").json()
|
|
||||||
assert listed["data"][0]["action"] == expected_action
|
|
||||||
|
|
||||||
|
|
||||||
def test_action_extraction_accepts_unbatched_action():
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
result = MockVideoResult(
|
|
||||||
[object()],
|
|
||||||
custom_output={
|
|
||||||
"action": np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32),
|
|
||||||
"raw_action_dim": 2,
|
|
||||||
"action_mode": "policy",
|
|
||||||
"domain_id": 7,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
actions = OmniOpenAIServingVideo._extract_action_outputs(result, expected_count=1)
|
|
||||||
|
|
||||||
assert actions[0] is not None
|
|
||||||
assert actions[0].data == [[1.0, 2.0], [3.0, 4.0]]
|
|
||||||
assert actions[0].shape == [2, 2]
|
|
||||||
|
|
||||||
|
|
||||||
def test_missing_handler_returns_503():
|
def test_missing_handler_returns_503():
|
||||||
@@ -869,9 +756,6 @@ def test_invalid_uploaded_input_reference_returns_400(test_client):
|
|||||||
def test_video_request_validation():
|
def test_video_request_validation():
|
||||||
req = VideoGenerationRequest(prompt="test")
|
req = VideoGenerationRequest(prompt="test")
|
||||||
assert req.prompt == "test"
|
assert req.prompt == "test"
|
||||||
assert req.generate_sound is False
|
|
||||||
assert req.sound_duration is None
|
|
||||||
assert VideoGenerationRequest(prompt="test", generate_sound=True, sound_duration=1.5).generate_sound is True
|
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
VideoGenerationRequest(prompt="test", size="invalid")
|
VideoGenerationRequest(prompt="test", size="invalid")
|
||||||
|
|
||||||
@@ -884,8 +768,6 @@ def test_video_request_validation():
|
|||||||
VideoGenerationRequest(prompt="test", frame_interpolation_exp=0)
|
VideoGenerationRequest(prompt="test", frame_interpolation_exp=0)
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
VideoGenerationRequest(prompt="test", frame_interpolation_scale=0)
|
VideoGenerationRequest(prompt="test", frame_interpolation_scale=0)
|
||||||
with pytest.raises(ValueError):
|
|
||||||
VideoGenerationRequest(prompt="test", sound_duration=0)
|
|
||||||
|
|
||||||
|
|
||||||
def test_list_videos_supports_order_after_and_limit(test_client, mocker: MockerFixture):
|
def test_list_videos_supports_order_after_and_limit(test_client, mocker: MockerFixture):
|
||||||
@@ -1147,31 +1029,6 @@ def test_sample_solver_forwarded_via_extra_params(test_client, mocker: MockerFix
|
|||||||
assert captured.extra_args["sample_solver"] == "euler"
|
assert captured.extra_args["sample_solver"] == "euler"
|
||||||
|
|
||||||
|
|
||||||
def test_extra_params_allows_inline_action(test_client, mocker: MockerFixture):
|
|
||||||
"""Inline ``action`` data is accepted and forwarded verbatim to
|
|
||||||
``extra_args`` (the supported way to pass forward-dynamics actions)."""
|
|
||||||
mocker.patch(
|
|
||||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
|
||||||
return_value=b"fake-video",
|
|
||||||
)
|
|
||||||
action = [[0.1, 0.2], [0.3, 0.4]]
|
|
||||||
response = test_client.post(
|
|
||||||
"/v1/videos",
|
|
||||||
data={
|
|
||||||
"prompt": "forward dynamics inline",
|
|
||||||
"extra_params": json.dumps({"action_mode": "forward_dynamics", "action": action}),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
assert response.status_code == 200
|
|
||||||
video_id = response.json()["id"]
|
|
||||||
_wait_for_status(test_client, video_id, VideoGenerationStatus.COMPLETED.value)
|
|
||||||
engine = test_client.app.state.openai_serving_video._engine_client
|
|
||||||
captured = engine.captured_sampling_params_list[0]
|
|
||||||
assert captured.extra_args["action"] == action
|
|
||||||
assert captured.extra_args["action_mode"] == "forward_dynamics"
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Sync endpoint tests (POST /v1/videos/sync)
|
# Sync endpoint tests (POST /v1/videos/sync)
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ OMNI_MODEL = "Qwen/Qwen2.5-Omni-7B"
|
|||||||
OMNI_STAGE_CONFIG = get_deploy_config_path("ci/qwen2_5_omni_thinker_only.yaml")
|
OMNI_STAGE_CONFIG = get_deploy_config_path("ci/qwen2_5_omni_thinker_only.yaml")
|
||||||
|
|
||||||
|
|
||||||
async def _noop(*args, **kw):
|
async def _noop(**kw):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
@@ -61,7 +61,6 @@ def get_async_omni_instance(fake_add_request=_noop, fake_abort_request=_noop) ->
|
|||||||
omni._final_output_handler = lambda: None
|
omni._final_output_handler = lambda: None
|
||||||
omni.resolve_sampling_params_list = lambda params, allow_delta_coercion: params
|
omni.resolve_sampling_params_list = lambda params, allow_delta_coercion: params
|
||||||
omni._compute_final_stage_id = lambda output_modalities: 0
|
omni._compute_final_stage_id = lambda output_modalities: 0
|
||||||
omni._compute_final_output_stage_ids = lambda output_modalities: [0]
|
|
||||||
omni._process_orchestrator_results = fake_process_results
|
omni._process_orchestrator_results = fake_process_results
|
||||||
omni._log_summary_and_cleanup = lambda request_id: omni.request_states.pop(request_id, None)
|
omni._log_summary_and_cleanup = lambda request_id: omni.request_states.pop(request_id, None)
|
||||||
return omni
|
return omni
|
||||||
|
|||||||
@@ -413,19 +413,6 @@ def test_openai_serving_models_can_consume_async_omni_compat_attrs():
|
|||||||
|
|
||||||
|
|
||||||
def test_get_diffusion_od_config_returns_diffusion_stage_config():
|
def test_get_diffusion_od_config_returns_diffusion_stage_config():
|
||||||
diffusion_od_config = object()
|
|
||||||
omni = object.__new__(AsyncOmni)
|
|
||||||
omni.engine = SimpleNamespace(
|
|
||||||
stage_clients=[
|
|
||||||
SimpleNamespace(stage_type="llm"),
|
|
||||||
SimpleNamespace(stage_type="diffusion", od_config=diffusion_od_config),
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
assert omni.get_diffusion_od_config() is diffusion_od_config
|
|
||||||
|
|
||||||
|
|
||||||
def test_get_diffusion_od_config_falls_back_to_inner_engine():
|
|
||||||
diffusion_od_config = object()
|
diffusion_od_config = object()
|
||||||
omni = object.__new__(AsyncOmni)
|
omni = object.__new__(AsyncOmni)
|
||||||
omni.engine = SimpleNamespace(
|
omni.engine = SimpleNamespace(
|
||||||
|
|||||||
@@ -1,56 +0,0 @@
|
|||||||
import pytest
|
|
||||||
|
|
||||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
|
||||||
from vllm_omni.diffusion.stage_diffusion_proc import StageDiffusionProc
|
|
||||||
from vllm_omni.entrypoints.utils import load_stage_configs_from_model, resolve_model_config_path
|
|
||||||
|
|
||||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
|
||||||
|
|
||||||
|
|
||||||
def test_dreamzero_vla_resolves_to_dreamzero_config(monkeypatch):
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"vllm_omni.entrypoints.utils.get_config",
|
|
||||||
lambda _model, trust_remote_code=True: type("Cfg", (), {"model_type": "vla"})(),
|
|
||||||
)
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"vllm_omni.entrypoints.utils._looks_like_dreamzero",
|
|
||||||
lambda _model: True,
|
|
||||||
)
|
|
||||||
result = resolve_model_config_path("GEAR-Dreams/DreamZero-DROID")
|
|
||||||
|
|
||||||
assert result is not None
|
|
||||||
assert result.endswith("vllm_omni/deploy/dreamzero.yaml")
|
|
||||||
|
|
||||||
|
|
||||||
def test_dreamzero_config_sets_model_class_and_policy_config(monkeypatch):
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"vllm_omni.config.stage_config.StageConfigFactory._auto_detect_model_type",
|
|
||||||
classmethod(lambda _cls, _model, trust_remote_code=True: ("vla", None)),
|
|
||||||
)
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"vllm_omni.diffusion.utils.hf_utils._looks_like_dreamzero",
|
|
||||||
lambda _model: True,
|
|
||||||
)
|
|
||||||
|
|
||||||
stage_configs = load_stage_configs_from_model("GEAR-Dreams/DreamZero-DROID")
|
|
||||||
engine_args = stage_configs[0].engine_args
|
|
||||||
|
|
||||||
assert engine_args.model_class_name == "DreamZeroPipeline"
|
|
||||||
assert engine_args.model_config.policy_server_config.action_space == "joint_position"
|
|
||||||
|
|
||||||
|
|
||||||
def test_dreamzero_enrich_config_preserves_explicit_model_class_name(monkeypatch):
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"vllm.transformers_utils.config.get_hf_file_to_dict",
|
|
||||||
lambda path, _model: None if path == "model_index.json" else {"model_type": "vla", "architectures": ["VLA"]},
|
|
||||||
)
|
|
||||||
|
|
||||||
od_config = OmniDiffusionConfig(
|
|
||||||
model="GEAR-Dreams/DreamZero-DROID",
|
|
||||||
model_class_name="DreamZeroPipeline",
|
|
||||||
)
|
|
||||||
proc = StageDiffusionProc(od_config.model, od_config)
|
|
||||||
|
|
||||||
proc._enrich_config()
|
|
||||||
|
|
||||||
assert od_config.model_class_name == "DreamZeroPipeline"
|
|
||||||
@@ -135,6 +135,22 @@ def test_set_stage_devices_npu_platform(mocker: MockerFixture, monkeypatch: pyte
|
|||||||
# ---- _map_device_list unit tests ----
|
# ---- _map_device_list unit tests ----
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.core_model
|
||||||
|
@pytest.mark.cpu
|
||||||
|
def test_map_device_list_idempotency():
|
||||||
|
"""Device IDs already in the visible set are returned as-is (idempotency)."""
|
||||||
|
result = _map_device_list(0, ["0", "1"], ["0", "1", "2", "3"])
|
||||||
|
assert result == ["0", "1"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.core_model
|
||||||
|
@pytest.mark.cpu
|
||||||
|
def test_map_device_list_idempotency_single():
|
||||||
|
"""Single device ID in visible set passes through."""
|
||||||
|
result = _map_device_list(1, ["1"], ["0", "1"])
|
||||||
|
assert result == ["1"]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
@pytest.mark.core_model
|
||||||
@pytest.mark.cpu
|
@pytest.mark.cpu
|
||||||
def test_map_device_list_index_mapping():
|
def test_map_device_list_index_mapping():
|
||||||
@@ -195,18 +211,6 @@ def test_map_device_list_partial_mapping():
|
|||||||
assert result == ["5", "6"]
|
assert result == ["5", "6"]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
|
||||||
@pytest.mark.cpu
|
|
||||||
def test_map_device_list_multi_replica_offset_cvd():
|
|
||||||
"""Logical indices from split_devices_for_replicas must map through an offset
|
|
||||||
CUDA_VISIBLE_DEVICES, not be treated as physical IDs.
|
|
||||||
"""
|
|
||||||
visible = ["2", "3", "4", "5"]
|
|
||||||
assert _map_device_list(1, ["1"], visible) == ["3"]
|
|
||||||
assert _map_device_list(1, ["2"], visible) == ["4"]
|
|
||||||
assert _map_device_list(1, ["2", "3"], visible) == ["4", "5"]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
@pytest.mark.core_model
|
||||||
@pytest.mark.cpu
|
@pytest.mark.cpu
|
||||||
def test_map_device_list_raises_on_non_numeric():
|
def test_map_device_list_raises_on_non_numeric():
|
||||||
|
|||||||
@@ -888,12 +888,6 @@ class OpenAIClientHandler:
|
|||||||
audio_data.append(content)
|
audio_data.append(content)
|
||||||
elif modality == "text" and content:
|
elif modality == "text" and content:
|
||||||
text_content += content
|
text_content += content
|
||||||
# Usage is yielded after the last token
|
|
||||||
if chunk.usage:
|
|
||||||
result.prompt_tokens = chunk.usage.prompt_tokens
|
|
||||||
if details := getattr(chunk.usage, "prompt_tokens_details", None):
|
|
||||||
result.cached_tokens = details.cached_tokens
|
|
||||||
|
|
||||||
audio_content = None
|
audio_content = None
|
||||||
if audio_data:
|
if audio_data:
|
||||||
merged_seg = _merge_base64_audio_to_segment(audio_data)
|
merged_seg = _merge_base64_audio_to_segment(audio_data)
|
||||||
@@ -1588,8 +1582,6 @@ class OpenAIClientHandler:
|
|||||||
create_kwargs["logprobs"] = request_config["logprobs"]
|
create_kwargs["logprobs"] = request_config["logprobs"]
|
||||||
if "top_logprobs" in request_config:
|
if "top_logprobs" in request_config:
|
||||||
create_kwargs["top_logprobs"] = request_config["top_logprobs"]
|
create_kwargs["top_logprobs"] = request_config["top_logprobs"]
|
||||||
if "stream_options" in request_config:
|
|
||||||
create_kwargs["stream_options"] = request_config["stream_options"]
|
|
||||||
if extra_body:
|
if extra_body:
|
||||||
create_kwargs["extra_body"] = extra_body
|
create_kwargs["extra_body"] = extra_body
|
||||||
|
|
||||||
|
|||||||
@@ -154,7 +154,6 @@ class TestRequestLifecycleGauges:
|
|||||||
obj.engine = SimpleNamespace(_running_counter=OmniRequestCounter())
|
obj.engine = SimpleNamespace(_running_counter=OmniRequestCounter())
|
||||||
obj.prom_metrics = OmniPrometheusMetrics(model_name="lifecycle-test")
|
obj.prom_metrics = OmniPrometheusMetrics(model_name="lifecycle-test")
|
||||||
obj.request_states = {}
|
obj.request_states = {}
|
||||||
obj._consumed_metric_messages = {}
|
|
||||||
obj.log_stats = False
|
obj.log_stats = False
|
||||||
|
|
||||||
# Simulate request lifecycle: start (counter 0→1, dict {} → {req}),
|
# Simulate request lifecycle: start (counter 0→1, dict {} → {req}),
|
||||||
@@ -183,7 +182,6 @@ class TestRequestLifecycleGauges:
|
|||||||
obj.engine = SimpleNamespace(_running_counter=OmniRequestCounter())
|
obj.engine = SimpleNamespace(_running_counter=OmniRequestCounter())
|
||||||
obj.prom_metrics = OmniPrometheusMetrics(model_name="lifecycle-test-2")
|
obj.prom_metrics = OmniPrometheusMetrics(model_name="lifecycle-test-2")
|
||||||
obj.request_states = {}
|
obj.request_states = {}
|
||||||
obj._consumed_metric_messages = {}
|
|
||||||
obj.log_stats = False
|
obj.log_stats = False
|
||||||
|
|
||||||
# Two in flight, one finalizes — running should report 1, waiting 0.
|
# Two in flight, one finalizes — running should report 1, waiting 0.
|
||||||
|
|||||||
@@ -5,7 +5,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import functools
|
import functools
|
||||||
from types import SimpleNamespace
|
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
@@ -18,14 +17,13 @@ def _voxcpm2_talker_mod():
|
|||||||
from vllm_omni.model_executor.models.voxcpm2.voxcpm2_talker import (
|
from vllm_omni.model_executor.models.voxcpm2.voxcpm2_talker import (
|
||||||
VoxCPM2TalkerForConditionalGeneration,
|
VoxCPM2TalkerForConditionalGeneration,
|
||||||
_RequestState,
|
_RequestState,
|
||||||
_VoxCPM2RuntimeConfig,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
return VoxCPM2TalkerForConditionalGeneration, _RequestState, _VoxCPM2RuntimeConfig
|
return VoxCPM2TalkerForConditionalGeneration, _RequestState
|
||||||
|
|
||||||
|
|
||||||
def _make_bare_talker():
|
def _make_bare_talker():
|
||||||
VoxCPM2TalkerForConditionalGeneration, _, _ = _voxcpm2_talker_mod()
|
VoxCPM2TalkerForConditionalGeneration, _ = _voxcpm2_talker_mod()
|
||||||
talker = VoxCPM2TalkerForConditionalGeneration.__new__(VoxCPM2TalkerForConditionalGeneration)
|
talker = VoxCPM2TalkerForConditionalGeneration.__new__(VoxCPM2TalkerForConditionalGeneration)
|
||||||
talker._active_states = {}
|
talker._active_states = {}
|
||||||
talker._current_request_id = None
|
talker._current_request_id = None
|
||||||
@@ -36,27 +34,11 @@ def _make_bare_talker():
|
|||||||
talker._max_batch_size = 4
|
talker._max_batch_size = 4
|
||||||
talker._active_state_warn_threshold = 512
|
talker._active_state_warn_threshold = 512
|
||||||
talker._active_state_warned = False
|
talker._active_state_warned = False
|
||||||
talker._enable_profiling = False
|
|
||||||
talker._audio_emit_every = 1
|
|
||||||
talker._vae_decode_every = 1
|
|
||||||
talker._enable_delayed_audio_copy = False
|
|
||||||
talker._delayed_audio_copy_use_events = False
|
|
||||||
talker._coalesce_audio_d2h = False
|
|
||||||
talker._enable_batched_vae_decode = False
|
|
||||||
talker._audio_copy_stream = None
|
|
||||||
talker._enable_vae_cuda_graph = False
|
|
||||||
talker._enable_cfm_cuda_graph = False
|
|
||||||
talker._enable_cfm_prealloc_output = False
|
|
||||||
talker._enable_batched_cfm = True
|
|
||||||
talker._deterministic_cfm_noise = False
|
|
||||||
talker._cfm_buffers = None
|
|
||||||
talker._last_audio_output_req_ids = []
|
|
||||||
talker._batched_fsq_fusion_max_batch = 32
|
|
||||||
return talker
|
return talker
|
||||||
|
|
||||||
|
|
||||||
def _seed_cached_decode(talker, req_id: str):
|
def _seed_cached_decode(talker, req_id: str):
|
||||||
_, _RequestState, _ = _voxcpm2_talker_mod()
|
_, _RequestState = _voxcpm2_talker_mod()
|
||||||
state = _RequestState(request_id=req_id)
|
state = _RequestState(request_id=req_id)
|
||||||
state.prefill_completed = True
|
state.prefill_completed = True
|
||||||
state.decode_step_count = 5
|
state.decode_step_count = 5
|
||||||
@@ -65,23 +47,6 @@ def _seed_cached_decode(talker, req_id: str):
|
|||||||
|
|
||||||
|
|
||||||
class TestStateEvictionContract:
|
class TestStateEvictionContract:
|
||||||
def test_runtime_config_normalizes_mutually_exclusive_paths(self) -> None:
|
|
||||||
_, _, RuntimeConfig = _voxcpm2_talker_mod()
|
|
||||||
|
|
||||||
cfg = RuntimeConfig(
|
|
||||||
enable_batched_cfm=True,
|
|
||||||
enable_cfm_cuda_graph=True,
|
|
||||||
enable_cfm_prealloc_output=True,
|
|
||||||
enable_batched_vae_decode=True,
|
|
||||||
enable_delayed_audio_copy=True,
|
|
||||||
)._normalized()
|
|
||||||
|
|
||||||
assert cfg.enable_batched_cfm is True
|
|
||||||
assert cfg.enable_cfm_cuda_graph is False
|
|
||||||
assert cfg.enable_cfm_prealloc_output is False
|
|
||||||
assert cfg.enable_batched_vae_decode is False
|
|
||||||
assert cfg.enable_delayed_audio_copy is True
|
|
||||||
|
|
||||||
def test_pending_requests_is_not_used_for_eviction(self) -> None:
|
def test_pending_requests_is_not_used_for_eviction(self) -> None:
|
||||||
talker = _make_bare_talker()
|
talker = _make_bare_talker()
|
||||||
|
|
||||||
@@ -96,61 +61,6 @@ class TestStateEvictionContract:
|
|||||||
assert rid in talker._active_states
|
assert rid in talker._active_states
|
||||||
assert talker._active_states[rid].prefill_completed is True
|
assert talker._active_states[rid].prefill_completed is True
|
||||||
|
|
||||||
def test_compute_logits_does_not_sync_stop_bool_on_default_path(self) -> None:
|
|
||||||
_, RState, _ = _voxcpm2_talker_mod()
|
|
||||||
talker = _make_bare_talker()
|
|
||||||
talker.config = SimpleNamespace(vocab_size=4)
|
|
||||||
state = RState(request_id="req", precomputed_stop_logits=torch.tensor([[0.0, 1.0]]))
|
|
||||||
talker._active_states["req"] = state
|
|
||||||
talker._results_queue = [("req", state.precomputed_stop_logits)]
|
|
||||||
|
|
||||||
logits = talker.compute_logits(torch.zeros(1, 1))
|
|
||||||
|
|
||||||
assert logits[0, 0] == 0.0
|
|
||||||
assert logits[0, 1] == 1.0
|
|
||||||
assert state.is_stopping is False
|
|
||||||
assert state.precomputed_stop_logits is None
|
|
||||||
assert state.precomputed_is_stopping is None
|
|
||||||
|
|
||||||
def test_compute_logits_reuses_cached_stop_bool_from_sparse_audio_path(self) -> None:
|
|
||||||
_, RState, _ = _voxcpm2_talker_mod()
|
|
||||||
talker = _make_bare_talker()
|
|
||||||
talker.config = SimpleNamespace(vocab_size=4)
|
|
||||||
state = RState(
|
|
||||||
request_id="req",
|
|
||||||
precomputed_stop_logits=torch.tensor([[0.0, 1.0]]),
|
|
||||||
precomputed_is_stopping=True,
|
|
||||||
)
|
|
||||||
talker._active_states["req"] = state
|
|
||||||
talker._results_queue = [("req", state.precomputed_stop_logits)]
|
|
||||||
|
|
||||||
logits = talker.compute_logits(torch.zeros(1, 1))
|
|
||||||
|
|
||||||
assert logits[0, 0] == 0.0
|
|
||||||
assert logits[0, 1] == 1.0
|
|
||||||
assert state.is_stopping is True
|
|
||||||
assert state.precomputed_stop_logits is None
|
|
||||||
assert state.precomputed_is_stopping is None
|
|
||||||
|
|
||||||
def test_batched_stop_precompute_sets_audio_collect_cache(self) -> None:
|
|
||||||
if not torch.cuda.is_available():
|
|
||||||
pytest.skip("CUDA is required to exercise batched stop D2H precompute")
|
|
||||||
|
|
||||||
_, RState, _ = _voxcpm2_talker_mod()
|
|
||||||
talker = _make_bare_talker()
|
|
||||||
talker._audio_emit_every = 2
|
|
||||||
states = [
|
|
||||||
RState(request_id="stop", precomputed_stop_logits=torch.tensor([[0.0, 1.0]], device="cuda")),
|
|
||||||
RState(request_id="keep", precomputed_stop_logits=torch.tensor([[1.0, 0.0]], device="cuda")),
|
|
||||||
]
|
|
||||||
|
|
||||||
talker._precompute_stop_flags_for_audio_collect(states)
|
|
||||||
|
|
||||||
assert states[0].precomputed_is_stopping is True
|
|
||||||
assert states[0].is_stopping is True
|
|
||||||
assert states[1].precomputed_is_stopping is False
|
|
||||||
assert states[1].is_stopping is False
|
|
||||||
|
|
||||||
def test_on_requests_finished_defers_cleanup(self) -> None:
|
def test_on_requests_finished_defers_cleanup(self) -> None:
|
||||||
talker = _make_bare_talker()
|
talker = _make_bare_talker()
|
||||||
_seed_cached_decode(talker, "req-A")
|
_seed_cached_decode(talker, "req-A")
|
||||||
@@ -209,7 +119,7 @@ class TestLeakWarnGuard:
|
|||||||
talker = _make_bare_talker()
|
talker = _make_bare_talker()
|
||||||
talker._active_state_warn_threshold = 3
|
talker._active_state_warn_threshold = 3
|
||||||
|
|
||||||
_, RState, _ = _voxcpm2_talker_mod()
|
_, RState = _voxcpm2_talker_mod()
|
||||||
for i in range(4):
|
for i in range(4):
|
||||||
talker._active_states[f"seed-{i}"] = RState(request_id=f"seed-{i}")
|
talker._active_states[f"seed-{i}"] = RState(request_id=f"seed-{i}")
|
||||||
|
|
||||||
@@ -219,548 +129,3 @@ class TestLeakWarnGuard:
|
|||||||
leak_warnings = [m for m in calls if "cleanup path leak" in m]
|
leak_warnings = [m for m in calls if "cleanup path leak" in m]
|
||||||
assert len(leak_warnings) == 1
|
assert len(leak_warnings) == 1
|
||||||
assert talker._active_state_warned is True
|
assert talker._active_state_warned is True
|
||||||
|
|
||||||
|
|
||||||
class _NoopPerf:
|
|
||||||
def start(self, name: str) -> None:
|
|
||||||
pass
|
|
||||||
|
|
||||||
def stop(self, name: str) -> None:
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class _FakeTTS:
|
|
||||||
feat_decoder = SimpleNamespace(estimator=SimpleNamespace(_compiled=False))
|
|
||||||
|
|
||||||
def fsq_layer(self, x: torch.Tensor) -> torch.Tensor:
|
|
||||||
return x + 10
|
|
||||||
|
|
||||||
def fusion_concat_proj(self, x: torch.Tensor) -> torch.Tensor:
|
|
||||||
half = x.shape[-1] // 2
|
|
||||||
return x[:, :half] * 2 + x[:, half:] * 3
|
|
||||||
|
|
||||||
def feat_encoder(self, x: torch.Tensor) -> torch.Tensor:
|
|
||||||
return x
|
|
||||||
|
|
||||||
def enc_to_lm_proj(self, x: torch.Tensor) -> torch.Tensor:
|
|
||||||
return x.squeeze(1).mean(dim=1).repeat(1, 4)
|
|
||||||
|
|
||||||
|
|
||||||
class TestDecodeBatchContract:
|
|
||||||
def test_forward_skip_pending_prefill_keeps_audio_queue_aligned(self) -> None:
|
|
||||||
class FakeScaffold:
|
|
||||||
def __call__(self, input_ids, positions, intermediate_tensors, inputs_embeds):
|
|
||||||
return torch.zeros(input_ids.shape[0], 4)
|
|
||||||
|
|
||||||
talker = _make_bare_talker()
|
|
||||||
talker._perf = _NoopPerf()
|
|
||||||
talker._tts = _FakeTTS()
|
|
||||||
talker.model = FakeScaffold()
|
|
||||||
talker._enable_cuda_graph = False
|
|
||||||
talker._cuda_graph_warmup_steps = 0
|
|
||||||
talker._cuda_graph_warmup_threshold = 3
|
|
||||||
talker._max_cached_graphs = 4
|
|
||||||
talker._scaffold_graphs = {}
|
|
||||||
talker._residual_graphs = {}
|
|
||||||
talker._pending_requests = [("req", False, None, 1)]
|
|
||||||
|
|
||||||
out = talker.forward(
|
|
||||||
torch.tensor([1]),
|
|
||||||
torch.tensor([0]),
|
|
||||||
inputs_embeds=torch.zeros(1, 4),
|
|
||||||
)
|
|
||||||
|
|
||||||
torch.testing.assert_close(out, torch.zeros(1, 4))
|
|
||||||
assert talker._results_queue == [("req", None)]
|
|
||||||
assert talker._audio_queue == [("req", None)]
|
|
||||||
|
|
||||||
def _make_decode_talker(self) -> object:
|
|
||||||
talker = _make_bare_talker()
|
|
||||||
talker._perf = _NoopPerf()
|
|
||||||
talker._tts = _FakeTTS()
|
|
||||||
talker._side_dtype = torch.float32
|
|
||||||
talker._patch_size = 2
|
|
||||||
talker._feat_dim = 3
|
|
||||||
|
|
||||||
def dit_proj(lm_h: torch.Tensor, res_h: torch.Tensor) -> torch.Tensor:
|
|
||||||
return lm_h + 2 * res_h
|
|
||||||
|
|
||||||
def stop_fn(lm_h: torch.Tensor) -> torch.Tensor:
|
|
||||||
return torch.stack([lm_h.sum(dim=1), -lm_h.sum(dim=1)], dim=1)
|
|
||||||
|
|
||||||
def run_cfm(dit_h: torch.Tensor, cond: torch.Tensor) -> torch.Tensor:
|
|
||||||
return dit_h[:, :1].view(-1, 1, 1) + cond.transpose(1, 2) * 0.1
|
|
||||||
|
|
||||||
talker._dit_proj_fn = dit_proj
|
|
||||||
talker._stop_fn = stop_fn
|
|
||||||
talker._run_cfm = run_cfm
|
|
||||||
talker._run_cfm_for_state = lambda _state, dit_h, cond: talker._run_cfm(dit_h, cond)
|
|
||||||
return talker
|
|
||||||
|
|
||||||
def _make_decode_state(self, request_id: str, offset: float):
|
|
||||||
_, RState, _ = _voxcpm2_talker_mod()
|
|
||||||
state = RState(request_id=request_id)
|
|
||||||
state.curr_prefix_feat_cond = torch.arange(6, dtype=torch.float32).reshape(2, 3) + offset
|
|
||||||
return state
|
|
||||||
|
|
||||||
def test_batched_fsq_fusion_matches_per_request_decode_prepare(self) -> None:
|
|
||||||
seq_talker = self._make_decode_talker()
|
|
||||||
batch_talker = self._make_decode_talker()
|
|
||||||
seq_talker._max_decode_steps = 2000
|
|
||||||
batch_talker._max_decode_steps = 2000
|
|
||||||
|
|
||||||
seq_states = [self._make_decode_state("req-0", 0.0), self._make_decode_state("req-1", 10.0)]
|
|
||||||
batch_states = [self._make_decode_state("req-0", 0.0), self._make_decode_state("req-1", 10.0)]
|
|
||||||
for idx, (seq_state, batch_state) in enumerate(zip(seq_states, batch_states, strict=True)):
|
|
||||||
prev = torch.full((1, 4), float(idx + 1))
|
|
||||||
seq_state.prev_feat_embed = prev.clone()
|
|
||||||
batch_state.prev_feat_embed = prev.clone()
|
|
||||||
|
|
||||||
hiddens = [
|
|
||||||
torch.tensor([[1.0, 2.0, 3.0, 4.0]]),
|
|
||||||
torch.tensor([[5.0, 6.0, 7.0, 8.0]]),
|
|
||||||
]
|
|
||||||
|
|
||||||
expected = [
|
|
||||||
seq_talker._prepare_residual_decode(state, hidden, torch.device("cpu"))
|
|
||||||
for state, hidden in zip(seq_states, hiddens, strict=True)
|
|
||||||
]
|
|
||||||
actual = batch_talker._prepare_residual_decode_batch(batch_states, hiddens, torch.device("cpu"))
|
|
||||||
|
|
||||||
for (expected_res, expected_meta), (actual_res, actual_meta) in zip(expected, actual, strict=True):
|
|
||||||
torch.testing.assert_close(actual_res, expected_res)
|
|
||||||
torch.testing.assert_close(actual_meta["new_lm_hidden"], expected_meta["new_lm_hidden"])
|
|
||||||
assert [state.decode_step_count for state in batch_states] == [1, 1]
|
|
||||||
|
|
||||||
def test_batch_decode_matches_per_request_state_updates(self) -> None:
|
|
||||||
seq_talker = self._make_decode_talker()
|
|
||||||
batch_talker = self._make_decode_talker()
|
|
||||||
|
|
||||||
seq_states = [self._make_decode_state("req-0", 0.0), self._make_decode_state("req-1", 10.0)]
|
|
||||||
batch_states = [self._make_decode_state("req-0", 0.0), self._make_decode_state("req-1", 10.0)]
|
|
||||||
metas = [
|
|
||||||
{"new_lm_hidden": torch.tensor([[1.0, 2.0, 3.0, 4.0]])},
|
|
||||||
{"new_lm_hidden": torch.tensor([[5.0, 6.0, 7.0, 8.0]])},
|
|
||||||
]
|
|
||||||
batch_out = torch.tensor([[0.5, 1.0, 1.5, 2.0], [2.5, 3.0, 3.5, 4.0]])
|
|
||||||
|
|
||||||
for state, meta, res_out in zip(seq_states, metas, batch_out, strict=True):
|
|
||||||
seq_talker._finish_decode(state, meta, res_out.unsqueeze(0), torch.device("cpu"))
|
|
||||||
|
|
||||||
batch_talker._finish_decode_batch(
|
|
||||||
[(state, False, meta) for state, meta in zip(batch_states, metas, strict=True)],
|
|
||||||
batch_out,
|
|
||||||
)
|
|
||||||
|
|
||||||
for expected, actual in zip(seq_states, batch_states, strict=True):
|
|
||||||
torch.testing.assert_close(actual.precomputed_stop_logits, expected.precomputed_stop_logits)
|
|
||||||
torch.testing.assert_close(actual.curr_embed_for_next, expected.curr_embed_for_next)
|
|
||||||
torch.testing.assert_close(actual.prev_feat_embed, expected.prev_feat_embed)
|
|
||||||
torch.testing.assert_close(actual.curr_prefix_feat_cond, expected.curr_prefix_feat_cond)
|
|
||||||
torch.testing.assert_close(actual.last_audio_patch_gpu, expected.last_audio_patch_gpu)
|
|
||||||
|
|
||||||
def test_batch_decode_preserves_per_request_cfm_rng_order(self) -> None:
|
|
||||||
talker = self._make_decode_talker()
|
|
||||||
talker._enable_batched_cfm = False
|
|
||||||
states = [self._make_decode_state("req-0", 0.0), self._make_decode_state("req-1", 10.0)]
|
|
||||||
metas = [
|
|
||||||
{"new_lm_hidden": torch.tensor([[1.0, 2.0, 3.0, 4.0]])},
|
|
||||||
{"new_lm_hidden": torch.tensor([[5.0, 6.0, 7.0, 8.0]])},
|
|
||||||
]
|
|
||||||
batch_out = torch.tensor([[0.5, 1.0, 1.5, 2.0], [2.5, 3.0, 3.5, 4.0]])
|
|
||||||
|
|
||||||
captured_noise: list[torch.Tensor] = []
|
|
||||||
torch.manual_seed(1234)
|
|
||||||
expected_noise = [torch.empty(1, 3, 2).normal_().detach().clone() for _ in states]
|
|
||||||
|
|
||||||
def run_cfm(
|
|
||||||
dit_h: torch.Tensor,
|
|
||||||
cond: torch.Tensor,
|
|
||||||
) -> torch.Tensor:
|
|
||||||
captured_noise.append(torch.empty(1, 3, 2).normal_().detach().clone())
|
|
||||||
return dit_h[:, :1].view(-1, 1, 1) + cond.transpose(1, 2) * 0.1
|
|
||||||
|
|
||||||
talker._run_cfm = run_cfm
|
|
||||||
|
|
||||||
torch.manual_seed(1234)
|
|
||||||
talker._finish_decode_batch(
|
|
||||||
[(state, False, meta) for state, meta in zip(states, metas, strict=True)],
|
|
||||||
batch_out,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert len(captured_noise) == len(states)
|
|
||||||
for actual, expected in zip(captured_noise, expected_noise, strict=True):
|
|
||||||
torch.testing.assert_close(actual, expected)
|
|
||||||
|
|
||||||
|
|
||||||
class _FakeAudioVAE:
|
|
||||||
decode_chunk_size = 2
|
|
||||||
|
|
||||||
|
|
||||||
class _FakeAudioTTS:
|
|
||||||
audio_vae = _FakeAudioVAE()
|
|
||||||
|
|
||||||
|
|
||||||
class TestAudioEmitCoalescing:
|
|
||||||
def test_audio_emit_every_accumulates_gpu_chunks_without_changing_order(self) -> None:
|
|
||||||
_, RState, _ = _voxcpm2_talker_mod()
|
|
||||||
talker = _make_bare_talker()
|
|
||||||
talker._audio_emit_every = 2
|
|
||||||
talker._feat_dim = 1
|
|
||||||
talker._n_decode_pad_frames = 1
|
|
||||||
talker._device = torch.device("cpu")
|
|
||||||
talker._tts = _FakeAudioTTS()
|
|
||||||
talker._perf = _NoopPerf()
|
|
||||||
|
|
||||||
values = [torch.tensor([10.0, 11.0]), torch.tensor([99.0, 99.0, 20.0, 21.0])]
|
|
||||||
|
|
||||||
def run_vae_decode(feat: torch.Tensor) -> torch.Tensor:
|
|
||||||
return values.pop(0)
|
|
||||||
|
|
||||||
talker._run_vae_decode = run_vae_decode
|
|
||||||
state = RState(request_id="req")
|
|
||||||
|
|
||||||
state.last_audio_patch_gpu = torch.tensor([[1.0]])
|
|
||||||
assert talker._collect_audio(state) is None
|
|
||||||
assert len(state.pending_audio_chunks_gpu) == 1
|
|
||||||
|
|
||||||
state.last_audio_patch_gpu = torch.tensor([[2.0]])
|
|
||||||
audio = talker._collect_audio(state)
|
|
||||||
|
|
||||||
torch.testing.assert_close(audio, torch.tensor([10.0, 11.0, 20.0, 21.0]))
|
|
||||||
assert state.pending_audio_chunks_gpu == []
|
|
||||||
|
|
||||||
def test_audio_emit_every_flushes_on_stop(self) -> None:
|
|
||||||
_, RState, _ = _voxcpm2_talker_mod()
|
|
||||||
talker = _make_bare_talker()
|
|
||||||
talker._audio_emit_every = 8
|
|
||||||
talker._feat_dim = 1
|
|
||||||
talker._n_decode_pad_frames = 1
|
|
||||||
talker._device = torch.device("cpu")
|
|
||||||
talker._tts = _FakeAudioTTS()
|
|
||||||
talker._perf = _NoopPerf()
|
|
||||||
talker._run_vae_decode = lambda feat: torch.tensor([3.0, 4.0])
|
|
||||||
state = RState(request_id="req", precomputed_stop_logits=torch.tensor([[0.0, 1.0]]))
|
|
||||||
|
|
||||||
state.last_audio_patch_gpu = torch.tensor([[1.0]])
|
|
||||||
audio = talker._collect_audio(state)
|
|
||||||
|
|
||||||
torch.testing.assert_close(audio, torch.tensor([3.0, 4.0]))
|
|
||||||
assert state.pending_audio_chunks_gpu == []
|
|
||||||
|
|
||||||
def test_vae_decode_every_accumulates_latents_before_decode(self) -> None:
|
|
||||||
_, RState, _ = _voxcpm2_talker_mod()
|
|
||||||
talker = _make_bare_talker()
|
|
||||||
talker._vae_decode_every = 2
|
|
||||||
talker._feat_dim = 1
|
|
||||||
talker._n_decode_pad_frames = 1
|
|
||||||
talker._device = torch.device("cpu")
|
|
||||||
talker._tts = _FakeAudioTTS()
|
|
||||||
talker._perf = _NoopPerf()
|
|
||||||
|
|
||||||
seen_feats = []
|
|
||||||
values = [torch.tensor([10.0, 11.0, 20.0, 21.0])]
|
|
||||||
|
|
||||||
def run_vae_decode(feat: torch.Tensor) -> torch.Tensor:
|
|
||||||
seen_feats.append(feat.detach().clone())
|
|
||||||
return values.pop(0)
|
|
||||||
|
|
||||||
talker._run_vae_decode = run_vae_decode
|
|
||||||
state = RState(request_id="req")
|
|
||||||
|
|
||||||
state.last_audio_patch_gpu = torch.tensor([[1.0]])
|
|
||||||
assert talker._collect_audio(state) is None
|
|
||||||
assert len(state.pending_vae_latents_gpu) == 1
|
|
||||||
assert seen_feats == []
|
|
||||||
|
|
||||||
state.last_audio_patch_gpu = torch.tensor([[2.0]])
|
|
||||||
audio = talker._collect_audio(state)
|
|
||||||
|
|
||||||
torch.testing.assert_close(audio, torch.tensor([10.0, 11.0, 20.0, 21.0]))
|
|
||||||
assert state.pending_vae_latents_gpu == []
|
|
||||||
torch.testing.assert_close(seen_feats[0].reshape(-1), torch.tensor([1.0, 2.0]))
|
|
||||||
|
|
||||||
def test_vae_decode_every_flushes_pending_latent_on_stop(self) -> None:
|
|
||||||
_, RState, _ = _voxcpm2_talker_mod()
|
|
||||||
talker = _make_bare_talker()
|
|
||||||
talker._vae_decode_every = 4
|
|
||||||
talker._feat_dim = 1
|
|
||||||
talker._n_decode_pad_frames = 1
|
|
||||||
talker._device = torch.device("cpu")
|
|
||||||
talker._tts = _FakeAudioTTS()
|
|
||||||
talker._perf = _NoopPerf()
|
|
||||||
talker._run_vae_decode = lambda feat: torch.tensor([7.0, 8.0])
|
|
||||||
state = RState(request_id="req", precomputed_stop_logits=torch.tensor([[0.0, 1.0]]))
|
|
||||||
|
|
||||||
state.last_audio_patch_gpu = torch.tensor([[1.0]])
|
|
||||||
audio = talker._collect_audio(state)
|
|
||||||
|
|
||||||
torch.testing.assert_close(audio, torch.tensor([7.0, 8.0]))
|
|
||||||
assert state.pending_vae_latents_gpu == []
|
|
||||||
assert state.is_stopping is True
|
|
||||||
|
|
||||||
def test_batched_vae_decode_collects_ready_requests_in_order(self) -> None:
|
|
||||||
_, RState, _ = _voxcpm2_talker_mod()
|
|
||||||
talker = _make_bare_talker()
|
|
||||||
talker._enable_batched_vae_decode = True
|
|
||||||
talker._coalesce_audio_d2h = True
|
|
||||||
talker._vae_decode_every = 2
|
|
||||||
talker._feat_dim = 1
|
|
||||||
talker._n_decode_pad_frames = 1
|
|
||||||
talker._device = torch.device("cpu")
|
|
||||||
talker._tts = _FakeAudioTTS()
|
|
||||||
talker._perf = _NoopPerf()
|
|
||||||
|
|
||||||
seen_feats: list[torch.Tensor] = []
|
|
||||||
|
|
||||||
def run_vae_decode(feat: torch.Tensor) -> torch.Tensor:
|
|
||||||
seen_feats.append(feat.detach().clone())
|
|
||||||
# One output row per request. decode_chunk_size=2, and each ready
|
|
||||||
# request has two new latent frames, so the full decoded length is 4.
|
|
||||||
return torch.tensor(
|
|
||||||
[
|
|
||||||
[[10.0, 11.0, 12.0, 13.0]],
|
|
||||||
[[20.0, 21.0, 22.0, 23.0]],
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
talker._run_vae_decode = run_vae_decode
|
|
||||||
|
|
||||||
ready0 = RState(request_id="ready-0")
|
|
||||||
ready0.pending_vae_latents_gpu.append(torch.tensor([[1.0]]))
|
|
||||||
ready0.last_audio_patch_gpu = torch.tensor([[2.0]])
|
|
||||||
|
|
||||||
pending = RState(request_id="pending")
|
|
||||||
pending.last_audio_patch_gpu = torch.tensor([[99.0]])
|
|
||||||
|
|
||||||
ready1 = RState(request_id="ready-1")
|
|
||||||
ready1.pending_vae_latents_gpu.append(torch.tensor([[3.0]]))
|
|
||||||
ready1.last_audio_patch_gpu = torch.tensor([[4.0]])
|
|
||||||
|
|
||||||
outputs = talker._collect_audio_batch([ready0, pending, ready1])
|
|
||||||
|
|
||||||
assert list(outputs) == ["ready-0", "pending", "ready-1"]
|
|
||||||
torch.testing.assert_close(outputs["ready-0"], torch.tensor([10.0, 11.0, 12.0, 13.0]))
|
|
||||||
assert outputs["pending"] is None
|
|
||||||
torch.testing.assert_close(outputs["ready-1"], torch.tensor([20.0, 21.0, 22.0, 23.0]))
|
|
||||||
assert len(seen_feats) == 1
|
|
||||||
torch.testing.assert_close(seen_feats[0].reshape(2, -1), torch.tensor([[1.0, 2.0], [3.0, 4.0]]))
|
|
||||||
assert pending.pending_vae_latents_gpu
|
|
||||||
assert ready0.pending_vae_latents_gpu == []
|
|
||||||
assert ready1.pending_vae_latents_gpu == []
|
|
||||||
|
|
||||||
def test_audio_emit_every_keeps_private_chunk_when_vae_output_storage_is_reused(self) -> None:
|
|
||||||
_, RState, _ = _voxcpm2_talker_mod()
|
|
||||||
talker = _make_bare_talker()
|
|
||||||
talker._audio_emit_every = 2
|
|
||||||
talker._enable_vae_cuda_graph = True
|
|
||||||
talker._feat_dim = 1
|
|
||||||
talker._n_decode_pad_frames = 1
|
|
||||||
talker._device = torch.device("cpu")
|
|
||||||
talker._tts = _FakeAudioTTS()
|
|
||||||
talker._perf = _NoopPerf()
|
|
||||||
|
|
||||||
shared_output = torch.empty(4)
|
|
||||||
values = [torch.tensor([10.0, 11.0, -1.0, -1.0]), torch.tensor([99.0, 99.0, 20.0, 21.0])]
|
|
||||||
|
|
||||||
def run_vae_decode(feat: torch.Tensor) -> torch.Tensor:
|
|
||||||
shared_output.copy_(values.pop(0))
|
|
||||||
return shared_output
|
|
||||||
|
|
||||||
talker._run_vae_decode = run_vae_decode
|
|
||||||
state = RState(request_id="req")
|
|
||||||
|
|
||||||
state.last_audio_patch_gpu = torch.tensor([[1.0]])
|
|
||||||
assert talker._collect_audio(state) is None
|
|
||||||
|
|
||||||
state.last_audio_patch_gpu = torch.tensor([[2.0]])
|
|
||||||
audio = talker._collect_audio(state)
|
|
||||||
|
|
||||||
torch.testing.assert_close(audio, torch.tensor([10.0, 11.0, 20.0, 21.0]))
|
|
||||||
|
|
||||||
def test_audio_emit_every_keeps_private_chunk_when_compiled_vae_reuses_storage(self) -> None:
|
|
||||||
_, RState, _ = _voxcpm2_talker_mod()
|
|
||||||
talker = _make_bare_talker()
|
|
||||||
talker._audio_emit_every = 2
|
|
||||||
talker._feat_dim = 1
|
|
||||||
talker._n_decode_pad_frames = 1
|
|
||||||
talker._device = torch.device("cpu")
|
|
||||||
talker._tts = _FakeAudioTTS()
|
|
||||||
talker._tts.audio_vae._compiled = True
|
|
||||||
talker._perf = _NoopPerf()
|
|
||||||
|
|
||||||
shared_output = torch.empty(4)
|
|
||||||
values = [torch.tensor([10.0, 11.0, -1.0, -1.0]), torch.tensor([99.0, 99.0, 20.0, 21.0])]
|
|
||||||
|
|
||||||
def run_vae_decode(feat: torch.Tensor) -> torch.Tensor:
|
|
||||||
shared_output.copy_(values.pop(0))
|
|
||||||
return shared_output
|
|
||||||
|
|
||||||
talker._run_vae_decode = run_vae_decode
|
|
||||||
state = RState(request_id="req")
|
|
||||||
|
|
||||||
state.last_audio_patch_gpu = torch.tensor([[1.0]])
|
|
||||||
assert talker._collect_audio(state) is None
|
|
||||||
|
|
||||||
state.last_audio_patch_gpu = torch.tensor([[2.0]])
|
|
||||||
audio = talker._collect_audio(state)
|
|
||||||
|
|
||||||
torch.testing.assert_close(audio, torch.tensor([10.0, 11.0, 20.0, 21.0]))
|
|
||||||
|
|
||||||
def test_delayed_audio_copy_emits_previous_chunk_and_flushes_on_stop(self) -> None:
|
|
||||||
_, RState, _ = _voxcpm2_talker_mod()
|
|
||||||
talker = _make_bare_talker()
|
|
||||||
talker._enable_delayed_audio_copy = True
|
|
||||||
talker._audio_emit_every = 1
|
|
||||||
talker._feat_dim = 1
|
|
||||||
talker._n_decode_pad_frames = 1
|
|
||||||
talker._device = torch.device("cpu")
|
|
||||||
talker._tts = _FakeAudioTTS()
|
|
||||||
talker._perf = _NoopPerf()
|
|
||||||
|
|
||||||
values = [
|
|
||||||
torch.tensor([10.0, 11.0]),
|
|
||||||
torch.tensor([99.0, 99.0, 20.0, 21.0]),
|
|
||||||
torch.tensor([88.0, 88.0, 30.0, 31.0]),
|
|
||||||
]
|
|
||||||
|
|
||||||
def run_vae_decode(feat: torch.Tensor) -> torch.Tensor:
|
|
||||||
return values.pop(0)
|
|
||||||
|
|
||||||
talker._run_vae_decode = run_vae_decode
|
|
||||||
state = RState(request_id="req")
|
|
||||||
|
|
||||||
state.last_audio_patch_gpu = torch.tensor([[1.0]])
|
|
||||||
assert talker._collect_audio(state) is None
|
|
||||||
assert len(state.pending_audio_copies) == 1
|
|
||||||
|
|
||||||
state.last_audio_patch_gpu = torch.tensor([[2.0]])
|
|
||||||
audio = talker._collect_audio(state)
|
|
||||||
torch.testing.assert_close(audio, torch.tensor([10.0, 11.0]))
|
|
||||||
assert len(state.pending_audio_copies) == 1
|
|
||||||
|
|
||||||
state.precomputed_stop_logits = torch.tensor([[0.0, 1.0]])
|
|
||||||
state.last_audio_patch_gpu = torch.tensor([[3.0]])
|
|
||||||
audio = talker._collect_audio(state)
|
|
||||||
torch.testing.assert_close(audio, torch.tensor([20.0, 21.0, 30.0, 31.0]))
|
|
||||||
assert state.pending_audio_copies == []
|
|
||||||
assert state.is_stopping is True
|
|
||||||
|
|
||||||
def test_delayed_audio_copy_flushes_on_forced_stop(self) -> None:
|
|
||||||
_, RState, _ = _voxcpm2_talker_mod()
|
|
||||||
talker = _make_bare_talker()
|
|
||||||
talker._enable_delayed_audio_copy = True
|
|
||||||
talker._audio_emit_every = 1
|
|
||||||
talker._feat_dim = 1
|
|
||||||
talker._n_decode_pad_frames = 1
|
|
||||||
talker._device = torch.device("cpu")
|
|
||||||
talker._tts = _FakeAudioTTS()
|
|
||||||
talker._perf = _NoopPerf()
|
|
||||||
|
|
||||||
values = [
|
|
||||||
torch.tensor([10.0, 11.0]),
|
|
||||||
torch.tensor([99.0, 99.0, 20.0, 21.0]),
|
|
||||||
]
|
|
||||||
|
|
||||||
def run_vae_decode(feat: torch.Tensor) -> torch.Tensor:
|
|
||||||
return values.pop(0)
|
|
||||||
|
|
||||||
talker._run_vae_decode = run_vae_decode
|
|
||||||
state = RState(request_id="req")
|
|
||||||
|
|
||||||
state.last_audio_patch_gpu = torch.tensor([[1.0]])
|
|
||||||
assert talker._collect_audio(state) is None
|
|
||||||
|
|
||||||
state.is_stopping = True
|
|
||||||
state.last_audio_patch_gpu = torch.tensor([[2.0]])
|
|
||||||
audio = talker._collect_audio(state)
|
|
||||||
|
|
||||||
torch.testing.assert_close(audio, torch.tensor([10.0, 11.0, 20.0, 21.0]))
|
|
||||||
assert state.pending_audio_copies == []
|
|
||||||
|
|
||||||
def test_make_omni_output_uses_sparse_req_ids_for_deferred_chunks(self) -> None:
|
|
||||||
talker = _make_bare_talker()
|
|
||||||
talker._audio_emit_every = 4
|
|
||||||
talker._sample_rate = 16000
|
|
||||||
talker._last_audio_output_req_ids = ["req-0", "req-1", "req-2"]
|
|
||||||
talker._audio_queue = [
|
|
||||||
("req-0", None),
|
|
||||||
("req-1", torch.tensor([1.0, 2.0])),
|
|
||||||
("req-2", None),
|
|
||||||
]
|
|
||||||
|
|
||||||
out = talker.make_omni_output(torch.zeros(3, 1))
|
|
||||||
|
|
||||||
audio = out.multimodal_outputs["model_outputs"]
|
|
||||||
sr = out.multimodal_outputs["sr"]
|
|
||||||
assert out.multimodal_outputs["meta"]["req_id"] == ["req-1"]
|
|
||||||
assert out.multimodal_outputs["meta"]["sparse_audio"] == ["1"]
|
|
||||||
assert len(audio) == 1
|
|
||||||
torch.testing.assert_close(audio[0], torch.tensor([1.0, 2.0]))
|
|
||||||
assert int(sr[0]) == 16000
|
|
||||||
|
|
||||||
def test_make_omni_output_emits_empty_sparse_marker_for_deferred_chunks(self) -> None:
|
|
||||||
talker = _make_bare_talker()
|
|
||||||
talker._audio_emit_every = 4
|
|
||||||
talker._sample_rate = 16000
|
|
||||||
talker._audio_queue = [("req-0", None), ("req-1", None)]
|
|
||||||
|
|
||||||
out = talker.make_omni_output(torch.zeros(2, 1))
|
|
||||||
|
|
||||||
assert out.multimodal_outputs["model_outputs"] == []
|
|
||||||
assert out.multimodal_outputs["sr"] == []
|
|
||||||
assert out.multimodal_outputs["meta"]["req_id"] == []
|
|
||||||
assert out.multimodal_outputs["meta"]["sparse_audio"] == ["1"]
|
|
||||||
|
|
||||||
def test_make_omni_output_uses_sparse_req_ids_for_delayed_audio_copy(self) -> None:
|
|
||||||
talker = _make_bare_talker()
|
|
||||||
talker._enable_delayed_audio_copy = True
|
|
||||||
talker._sample_rate = 16000
|
|
||||||
talker._audio_queue = [
|
|
||||||
("req-0", None),
|
|
||||||
("req-1", torch.tensor([1.0, 2.0])),
|
|
||||||
("req-2", None),
|
|
||||||
]
|
|
||||||
|
|
||||||
out = talker.make_omni_output(torch.zeros(3, 1))
|
|
||||||
|
|
||||||
assert out.multimodal_outputs["meta"]["req_id"] == ["req-1"]
|
|
||||||
assert out.multimodal_outputs["meta"]["sparse_audio"] == ["1"]
|
|
||||||
assert len(out.multimodal_outputs["model_outputs"]) == 1
|
|
||||||
torch.testing.assert_close(out.multimodal_outputs["model_outputs"][0], torch.tensor([1.0, 2.0]))
|
|
||||||
|
|
||||||
def test_make_omni_output_uses_sparse_req_ids_for_deferred_vae_decode(self) -> None:
|
|
||||||
talker = _make_bare_talker()
|
|
||||||
talker._vae_decode_every = 2
|
|
||||||
talker._sample_rate = 16000
|
|
||||||
talker._audio_queue = [
|
|
||||||
("req-0", None),
|
|
||||||
("req-1", torch.tensor([1.0, 2.0])),
|
|
||||||
("req-2", None),
|
|
||||||
]
|
|
||||||
|
|
||||||
out = talker.make_omni_output(torch.zeros(3, 1))
|
|
||||||
|
|
||||||
assert out.multimodal_outputs["meta"]["req_id"] == ["req-1"]
|
|
||||||
assert out.multimodal_outputs["meta"]["sparse_audio"] == ["1"]
|
|
||||||
assert len(out.multimodal_outputs["model_outputs"]) == 1
|
|
||||||
torch.testing.assert_close(out.multimodal_outputs["model_outputs"][0], torch.tensor([1.0, 2.0]))
|
|
||||||
|
|
||||||
def test_make_omni_output_coalesces_gpu_audio_to_cpu(self) -> None:
|
|
||||||
if not torch.cuda.is_available():
|
|
||||||
pytest.skip("CUDA is required to exercise D2H coalescing")
|
|
||||||
talker = _make_bare_talker()
|
|
||||||
talker._coalesce_audio_d2h = True
|
|
||||||
talker._sample_rate = 16000
|
|
||||||
talker._audio_queue = [
|
|
||||||
("req-0", torch.tensor([1.0, 2.0], device="cuda")),
|
|
||||||
("req-1", torch.tensor([3.0], device="cuda")),
|
|
||||||
]
|
|
||||||
|
|
||||||
out = talker.make_omni_output(torch.zeros(2, 1))
|
|
||||||
|
|
||||||
audio = out.multimodal_outputs["model_outputs"]
|
|
||||||
assert len(audio) == 2
|
|
||||||
assert all(not chunk.is_cuda for chunk in audio)
|
|
||||||
torch.testing.assert_close(audio[0], torch.tensor([1.0, 2.0]))
|
|
||||||
torch.testing.assert_close(audio[1], torch.tensor([3.0]))
|
|
||||||
@@ -1638,6 +1638,9 @@ class TestBaseConfigInheritance:
|
|||||||
s0 = deploy.stages[0].default_sampling_params
|
s0 = deploy.stages[0].default_sampling_params
|
||||||
# CI overrides max_tokens
|
# CI overrides max_tokens
|
||||||
assert s0["max_tokens"] == 150
|
assert s0["max_tokens"] == 150
|
||||||
|
# Inherited from base
|
||||||
|
assert s0["temperature"] == 0.4
|
||||||
|
assert s0["seed"] == 42
|
||||||
|
|
||||||
def test_pure_inheritance_overlay(self, tmp_path):
|
def test_pure_inheritance_overlay(self, tmp_path):
|
||||||
"""An overlay with only ``base_config`` inherits everything."""
|
"""An overlay with only ``base_config`` inherits everything."""
|
||||||
@@ -1957,3 +1960,5 @@ class TestSamplingConstraintsPrecedence:
|
|||||||
assert stages[0].yaml_extras["default_sampling_params"]["detokenize"] is True
|
assert stages[0].yaml_extras["default_sampling_params"]["detokenize"] is True
|
||||||
# Pipeline says stop_token_ids=[2150] for talker
|
# Pipeline says stop_token_ids=[2150] for talker
|
||||||
assert stages[1].yaml_extras["default_sampling_params"]["stop_token_ids"] == [2150]
|
assert stages[1].yaml_extras["default_sampling_params"]["stop_token_ids"] == [2150]
|
||||||
|
# Deploy temperature still flows through
|
||||||
|
assert stages[0].yaml_extras["default_sampling_params"]["temperature"] == 0.4
|
||||||
Whitespace-only changes.
@@ -1,910 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
"""Deterministic unit tests for higgs-audio v3.
|
|
||||||
|
|
||||||
These tests verify AC-1 (config), AC-3 (prompt), AC-4 (fused modules),
|
|
||||||
AC-5 (delay pattern), AC-7 (stage processor), and AC-10 (registry)
|
|
||||||
without requiring the actual checkpoint or GPU.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
import torch
|
|
||||||
|
|
||||||
# ---- AC-1: Configuration ----
|
|
||||||
|
|
||||||
|
|
||||||
class TestHiggsAudioV3Config:
|
|
||||||
def test_default_config_loads(self):
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
|
||||||
HiggsAudioV3Config,
|
|
||||||
)
|
|
||||||
|
|
||||||
config = HiggsAudioV3Config()
|
|
||||||
assert config.num_codebooks == 8
|
|
||||||
assert config.codebook_size == 1026
|
|
||||||
assert config.audio_stream_bos_id == 1024
|
|
||||||
assert config.audio_stream_eos_id == 1025
|
|
||||||
assert config.sample_rate == 24000
|
|
||||||
assert config.frame_rate == 25
|
|
||||||
assert config.num_real_codes == 1024
|
|
||||||
assert config.tie_modality_embeddings is True
|
|
||||||
|
|
||||||
def test_custom_audio_encoder_config(self):
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
|
||||||
HiggsAudioV3Config,
|
|
||||||
)
|
|
||||||
|
|
||||||
config = HiggsAudioV3Config(
|
|
||||||
audio_encoder_config={
|
|
||||||
"encoder_type": "discrete",
|
|
||||||
"num_codebooks": 4,
|
|
||||||
"vocab_size": 512,
|
|
||||||
"tie_word_embeddings": False,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
assert config.num_codebooks == 4
|
|
||||||
assert config.codebook_size == 512
|
|
||||||
assert config.tie_modality_embeddings is False
|
|
||||||
|
|
||||||
def test_invalid_num_codebooks_rejected(self):
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
|
||||||
HiggsAudioV3Config,
|
|
||||||
)
|
|
||||||
|
|
||||||
with pytest.raises(ValueError, match="num_codebooks must be > 0"):
|
|
||||||
HiggsAudioV3Config(audio_encoder_config={"num_codebooks": 0, "vocab_size": 1026})
|
|
||||||
|
|
||||||
def test_negative_num_codebooks_rejected(self):
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
|
||||||
HiggsAudioV3Config,
|
|
||||||
)
|
|
||||||
|
|
||||||
with pytest.raises(ValueError, match="num_codebooks must be > 0"):
|
|
||||||
HiggsAudioV3Config(audio_encoder_config={"num_codebooks": -1, "vocab_size": 1026})
|
|
||||||
|
|
||||||
def test_special_token_ids_initially_none(self):
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
|
||||||
HiggsAudioV3Config,
|
|
||||||
)
|
|
||||||
|
|
||||||
config = HiggsAudioV3Config()
|
|
||||||
assert config.tts_token_id is None
|
|
||||||
assert config.text_token_id is None
|
|
||||||
assert config.audio_continuation_id is None
|
|
||||||
|
|
||||||
def test_hidden_size_from_text_config(self):
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
|
||||||
HiggsAudioV3Config,
|
|
||||||
)
|
|
||||||
|
|
||||||
config = HiggsAudioV3Config(text_config={"model_type": "qwen3", "hidden_size": 2560})
|
|
||||||
assert config.hidden_size == 2560
|
|
||||||
assert config.audio_hidden_size == 2560
|
|
||||||
|
|
||||||
|
|
||||||
# ---- AC-4: Fused Multi-Codebook Modules ----
|
|
||||||
|
|
||||||
|
|
||||||
class TestFusedModules:
|
|
||||||
def test_embedding_shape(self):
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
|
||||||
HiggsFusedMultiTextEmbedding,
|
|
||||||
)
|
|
||||||
|
|
||||||
embed = HiggsFusedMultiTextEmbedding(num_codebooks=8, vocab_size=1026, hidden_size=256)
|
|
||||||
assert embed.weight.shape == (8 * 1026, 256)
|
|
||||||
|
|
||||||
codes = torch.randint(0, 1024, (5, 8))
|
|
||||||
out = embed(codes)
|
|
||||||
assert out.shape == (5, 256)
|
|
||||||
|
|
||||||
def test_embedding_offset_indexing(self):
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
|
||||||
HiggsFusedMultiTextEmbedding,
|
|
||||||
)
|
|
||||||
|
|
||||||
embed = HiggsFusedMultiTextEmbedding(num_codebooks=2, vocab_size=4, hidden_size=3)
|
|
||||||
# Set weights to identity-like pattern so we can verify offsets
|
|
||||||
with torch.no_grad():
|
|
||||||
embed.weight.zero_()
|
|
||||||
# Codebook 0, vocab [0..3] -> row 0..3
|
|
||||||
# Codebook 1, vocab [0..3] -> row 4..7
|
|
||||||
for i in range(8):
|
|
||||||
embed.weight[i, i % 3] = float(i)
|
|
||||||
|
|
||||||
# codes[0] = [1, 2] -> embed[1] + embed[4+2=6]
|
|
||||||
codes = torch.tensor([[1, 2]])
|
|
||||||
out = embed(codes)
|
|
||||||
expected = embed.weight[1] + embed.weight[6]
|
|
||||||
assert torch.allclose(out[0], expected)
|
|
||||||
|
|
||||||
def test_head_shape(self):
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
|
||||||
HiggsFusedMultiTextHead,
|
|
||||||
)
|
|
||||||
|
|
||||||
head = HiggsFusedMultiTextHead(num_codebooks=8, vocab_size=1026, hidden_size=256)
|
|
||||||
assert head.weight.shape == (8 * 1026, 256)
|
|
||||||
|
|
||||||
hidden = torch.randn(3, 256)
|
|
||||||
out = head.generate(hidden)
|
|
||||||
assert out.shape == (3, 8, 1026)
|
|
||||||
|
|
||||||
def test_tying(self):
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
|
||||||
HiggsFusedMultiTextEmbedding,
|
|
||||||
HiggsFusedMultiTextHead,
|
|
||||||
)
|
|
||||||
|
|
||||||
embed = HiggsFusedMultiTextEmbedding(num_codebooks=8, vocab_size=1026, hidden_size=64)
|
|
||||||
head = HiggsFusedMultiTextHead(num_codebooks=8, vocab_size=1026, hidden_size=64)
|
|
||||||
head.weight = embed.weight
|
|
||||||
assert head.weight is embed.weight
|
|
||||||
# Modification to one should reflect in the other
|
|
||||||
with torch.no_grad():
|
|
||||||
embed.weight[0, 0] = 42.0
|
|
||||||
assert head.weight[0, 0] == 42.0
|
|
||||||
|
|
||||||
def test_codes_out_of_range(self):
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
|
||||||
HiggsFusedMultiTextEmbedding,
|
|
||||||
)
|
|
||||||
|
|
||||||
embed = HiggsFusedMultiTextEmbedding(num_codebooks=8, vocab_size=1026, hidden_size=16)
|
|
||||||
# Code 1026 + offset for codebook 7 = 1026 + 7*1026 = 8208
|
|
||||||
# But weight only has 8*1026 = 8208 rows, so index 8208 is out of bounds
|
|
||||||
codes = torch.tensor([[0, 0, 0, 0, 0, 0, 0, 1026]]) # Last cb: 1026 is OOB
|
|
||||||
with pytest.raises(IndexError):
|
|
||||||
embed(codes)
|
|
||||||
|
|
||||||
|
|
||||||
# ---- AC-5: Delay Pattern Behavior ----
|
|
||||||
|
|
||||||
|
|
||||||
class TestDelayPatternBehavior:
|
|
||||||
"""Test the delay pattern masking logic extracted from the talker."""
|
|
||||||
|
|
||||||
def test_boc_eoc_ids(self):
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
|
||||||
BOC_ID,
|
|
||||||
EOC_ID,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert BOC_ID == 1024
|
|
||||||
assert EOC_ID == 1025
|
|
||||||
|
|
||||||
def test_delay_phase_boc_masking(self):
|
|
||||||
"""During delay phase, codebooks beyond delay_count are forced to BOC."""
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import BOC_ID
|
|
||||||
|
|
||||||
num_codebooks = 8
|
|
||||||
# Simulate delay phase at step 3 (delay_count=2, so CBs 3-7 should be BOC)
|
|
||||||
codes = torch.randint(0, 1024, (num_codebooks,))
|
|
||||||
delay_count = 2
|
|
||||||
next_cb = delay_count + 1
|
|
||||||
if next_cb < num_codebooks:
|
|
||||||
codes[next_cb:] = BOC_ID
|
|
||||||
# CBs 0-2 should have original codes, CBs 3-7 should be BOC
|
|
||||||
assert all(codes[i] != BOC_ID or i >= 3 for i in range(num_codebooks))
|
|
||||||
assert all(codes[i] == BOC_ID for i in range(3, num_codebooks))
|
|
||||||
|
|
||||||
def test_cb0_eoc_triggers_rampdown(self):
|
|
||||||
"""EOC on codebook 0 starts ramp-down; EOC on other codebooks does not."""
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
|
||||||
EOC_ID,
|
|
||||||
)
|
|
||||||
|
|
||||||
num_codebooks = 8
|
|
||||||
codes = torch.randint(0, 1024, (num_codebooks,))
|
|
||||||
|
|
||||||
# Simulate: cb0 emits EOC
|
|
||||||
codes[0] = EOC_ID
|
|
||||||
assert int(codes[0].item()) == EOC_ID
|
|
||||||
# Ramp-down should start with countdown = N-2 = 6
|
|
||||||
eoc_countdown = num_codebooks - 2
|
|
||||||
assert eoc_countdown == 6
|
|
||||||
|
|
||||||
# If cb3 emits EOC but cb0 doesn't, no ramp-down
|
|
||||||
codes2 = torch.randint(0, 1024, (num_codebooks,))
|
|
||||||
codes2[3] = EOC_ID
|
|
||||||
assert int(codes2[0].item()) != EOC_ID # cb0 is not EOC
|
|
||||||
|
|
||||||
def test_rampdown_termination(self):
|
|
||||||
"""After N-2 steps of ramp-down, generation_done becomes True."""
|
|
||||||
num_codebooks = 8
|
|
||||||
eoc_countdown = num_codebooks - 2 # = 6
|
|
||||||
generation_done = False
|
|
||||||
for _ in range(6):
|
|
||||||
eoc_countdown -= 1
|
|
||||||
if eoc_countdown <= 0:
|
|
||||||
generation_done = True
|
|
||||||
assert generation_done is True
|
|
||||||
|
|
||||||
def test_all_boc_seed(self):
|
|
||||||
"""First audio step should seed last_codes with all-BOC."""
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import BOC_ID
|
|
||||||
|
|
||||||
seeded = torch.full((8,), BOC_ID, dtype=torch.long)
|
|
||||||
assert seeded.shape == (8,)
|
|
||||||
assert all(seeded[i] == BOC_ID for i in range(8))
|
|
||||||
|
|
||||||
|
|
||||||
# ---- AC-5: Real Sampler Method Tests ----
|
|
||||||
|
|
||||||
|
|
||||||
class TestSamplerMethods:
|
|
||||||
"""Test the actual _sample_audio_codes and _apply_delay_pattern_masking methods."""
|
|
||||||
|
|
||||||
def _make_minimal_talker(self):
|
|
||||||
"""Create a minimal talker-like object with sampler/masking methods."""
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod
|
|
||||||
|
|
||||||
class FakeTalker:
|
|
||||||
num_codebooks = 8
|
|
||||||
codebook_size = 1026
|
|
||||||
_audio_state = {}
|
|
||||||
|
|
||||||
t = FakeTalker()
|
|
||||||
t._sample_audio_codes = mod.HiggsAudioV3TalkerForConditionalGeneration._sample_audio_codes.__get__(t)
|
|
||||||
t._apply_delay_pattern_masking = (
|
|
||||||
mod.HiggsAudioV3TalkerForConditionalGeneration._apply_delay_pattern_masking.__get__(t)
|
|
||||||
)
|
|
||||||
return t
|
|
||||||
|
|
||||||
def test_sample_respects_mask(self):
|
|
||||||
"""Tokens masked to -inf must never be sampled."""
|
|
||||||
t = self._make_minimal_talker()
|
|
||||||
logits = torch.full((10, 1026), float("-inf"))
|
|
||||||
# Only token 500 is allowed for each row
|
|
||||||
logits[:, 500] = 0.0
|
|
||||||
result = t._sample_audio_codes(logits)
|
|
||||||
assert result.shape == (10,)
|
|
||||||
assert (result == 500).all(), f"Expected all 500, got {result.tolist()}"
|
|
||||||
|
|
||||||
def test_sample_all_masked_falls_back_to_argmax(self):
|
|
||||||
"""All-masked row should fall back to argmax (least-negative logit)."""
|
|
||||||
t = self._make_minimal_talker()
|
|
||||||
logits = torch.full((2, 1026), float("-inf"))
|
|
||||||
# Row 0: all masked
|
|
||||||
# Row 1: only token 42 allowed
|
|
||||||
logits[1, 42] = 0.0
|
|
||||||
result = t._sample_audio_codes(logits)
|
|
||||||
assert result.shape == (2,)
|
|
||||||
assert result[1].item() == 42
|
|
||||||
|
|
||||||
def test_delay_masking_forces_boc_during_delay(self):
|
|
||||||
"""During delay phase, codebooks beyond delay_count must have only BOC allowed."""
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import BOC_ID
|
|
||||||
|
|
||||||
t = self._make_minimal_talker()
|
|
||||||
# Simulate delay_count=2 for batch row 0
|
|
||||||
t._audio_state[0] = {"num_delay": 2, "num_remaining_delays": None}
|
|
||||||
cb_logits = torch.zeros(1, 8, 1026) # [1 audio row, 8 codebooks, 1026 vocab]
|
|
||||||
t._apply_delay_pattern_masking(cb_logits, [0])
|
|
||||||
# CBs 3-7 should have only BOC allowed (everything else -inf)
|
|
||||||
for q in range(3, 8):
|
|
||||||
row = cb_logits[0, q]
|
|
||||||
assert row[BOC_ID].item() == 0.0 # BOC kept
|
|
||||||
# All non-BOC should be -inf
|
|
||||||
non_boc = torch.cat([row[:BOC_ID], row[BOC_ID + 1 :]])
|
|
||||||
assert (non_boc == float("-inf")).all(), f"CB{q} has non-inf non-BOC values"
|
|
||||||
|
|
||||||
def test_delay_masking_disallows_boc_for_active_codebooks(self):
|
|
||||||
"""Active codebooks during delay should have BOC disallowed."""
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import BOC_ID
|
|
||||||
|
|
||||||
t = self._make_minimal_talker()
|
|
||||||
t._audio_state[0] = {"num_delay": 3, "num_remaining_delays": None}
|
|
||||||
cb_logits = torch.zeros(1, 8, 1026)
|
|
||||||
t._apply_delay_pattern_masking(cb_logits, [0])
|
|
||||||
# CBs 0-3 are active: BOC should be -inf
|
|
||||||
for q in range(4):
|
|
||||||
assert cb_logits[0, q, BOC_ID].item() == float("-inf")
|
|
||||||
|
|
||||||
def test_delay_masking_only_cb0_allows_eoc(self):
|
|
||||||
"""Only codebook 0 should allow EOC during normal generation."""
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import EOC_ID
|
|
||||||
|
|
||||||
t = self._make_minimal_talker()
|
|
||||||
t._audio_state[0] = {"num_delay": 8, "num_remaining_delays": None}
|
|
||||||
cb_logits = torch.zeros(1, 8, 1026)
|
|
||||||
t._apply_delay_pattern_masking(cb_logits, [0])
|
|
||||||
# CB0 should keep EOC
|
|
||||||
assert cb_logits[0, 0, EOC_ID].item() != float("-inf")
|
|
||||||
# CB1-7 should have EOC masked
|
|
||||||
for q in range(1, 8):
|
|
||||||
assert cb_logits[0, q, EOC_ID].item() == float("-inf"), f"CB{q} EOC not masked"
|
|
||||||
|
|
||||||
def test_rampdown_masking_locks_to_eoc(self):
|
|
||||||
"""During ramp-down, locked codebooks should only allow EOC."""
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
|
||||||
BOC_ID,
|
|
||||||
EOC_ID,
|
|
||||||
)
|
|
||||||
|
|
||||||
t = self._make_minimal_talker()
|
|
||||||
# Ramp-down with 4 remaining delays: lock CBs 0-3 to EOC
|
|
||||||
t._audio_state[0] = {"num_delay": 8, "num_remaining_delays": 4}
|
|
||||||
cb_logits = torch.zeros(1, 8, 1026)
|
|
||||||
t._apply_delay_pattern_masking(cb_logits, [0])
|
|
||||||
# CBs 0-3 locked: only EOC allowed
|
|
||||||
for q in range(4):
|
|
||||||
row = cb_logits[0, q]
|
|
||||||
assert row[EOC_ID].item() == 0.0
|
|
||||||
non_eoc = torch.cat([row[:EOC_ID], row[EOC_ID + 1 :]])
|
|
||||||
assert (non_eoc == float("-inf")).all(), f"Locked CB{q} has non-inf non-EOC"
|
|
||||||
# CBs 4-7 active: BOC and EOC disallowed
|
|
||||||
for q in range(4, 8):
|
|
||||||
assert cb_logits[0, q, BOC_ID].item() == float("-inf")
|
|
||||||
assert cb_logits[0, q, EOC_ID].item() == float("-inf")
|
|
||||||
|
|
||||||
|
|
||||||
# ---- AC-6: Feedback Method Tests ----
|
|
||||||
|
|
||||||
|
|
||||||
class TestFeedbackMethods:
|
|
||||||
def test_postprocess_emits_audio_codes(self):
|
|
||||||
"""postprocess() should return codes from _last_audio_codes."""
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod
|
|
||||||
|
|
||||||
class FakeTalker:
|
|
||||||
_last_audio_codes = torch.tensor([[100, 200, 300, 400, 500, 600, 700, 800]])
|
|
||||||
_postprocess_cursor = 0
|
|
||||||
|
|
||||||
t = FakeTalker()
|
|
||||||
t.postprocess = mod.HiggsAudioV3TalkerForConditionalGeneration.postprocess.__get__(t)
|
|
||||||
result = t.postprocess(torch.zeros(1, 64))
|
|
||||||
assert "codes" in result
|
|
||||||
assert "audio" in result["codes"]
|
|
||||||
assert result["codes"]["audio"].shape == (1, 8)
|
|
||||||
|
|
||||||
def test_postprocess_skips_negative_rows(self):
|
|
||||||
"""postprocess() should skip rows with -1 (no audio)."""
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod
|
|
||||||
|
|
||||||
class FakeTalker:
|
|
||||||
_last_audio_codes = torch.tensor([[-1, -1, -1, -1, -1, -1, -1, -1]])
|
|
||||||
_postprocess_cursor = 0
|
|
||||||
|
|
||||||
t = FakeTalker()
|
|
||||||
t.postprocess = mod.HiggsAudioV3TalkerForConditionalGeneration.postprocess.__get__(t)
|
|
||||||
result = t.postprocess(torch.zeros(1, 64))
|
|
||||||
assert result == {}
|
|
||||||
|
|
||||||
def test_postprocess_advances_cursor(self):
|
|
||||||
"""postprocess() should advance cursor by 1 per call."""
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod
|
|
||||||
|
|
||||||
class FakeTalker:
|
|
||||||
_last_audio_codes = torch.tensor(
|
|
||||||
[[100, 200, 300, 400, 500, 600, 700, 800], [-1, -1, -1, -1, -1, -1, -1, -1]]
|
|
||||||
)
|
|
||||||
_postprocess_cursor = 0
|
|
||||||
|
|
||||||
t = FakeTalker()
|
|
||||||
t.postprocess = mod.HiggsAudioV3TalkerForConditionalGeneration.postprocess.__get__(t)
|
|
||||||
r1 = t.postprocess(torch.zeros(1, 64))
|
|
||||||
assert "codes" in r1
|
|
||||||
r2 = t.postprocess(torch.zeros(1, 64))
|
|
||||||
assert r2 == {} # Second row is all -1
|
|
||||||
|
|
||||||
|
|
||||||
# ---- AC-6: Audio Feedback Method Tests ----
|
|
||||||
|
|
||||||
|
|
||||||
class TestAudioFeedback:
|
|
||||||
"""Test _apply_audio_feedback with minimal fake talker."""
|
|
||||||
|
|
||||||
def _make_feedback_talker(self):
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod
|
|
||||||
|
|
||||||
embed = mod.HiggsFusedMultiTextEmbedding(num_codebooks=8, vocab_size=1026, hidden_size=16)
|
|
||||||
torch.nn.init.ones_(embed.weight)
|
|
||||||
|
|
||||||
class FakeTalker:
|
|
||||||
num_codebooks = 8
|
|
||||||
codebook_size = 1026
|
|
||||||
_audio_continuation_id = 99999 # fake audio token
|
|
||||||
_last_step_query_start_loc = None
|
|
||||||
_audio_state = {}
|
|
||||||
multimodal_embedding = embed
|
|
||||||
model = type("M", (), {"embed_tokens": lambda self, ids: torch.zeros(ids.shape[0], 16)})()
|
|
||||||
|
|
||||||
t = FakeTalker()
|
|
||||||
t._apply_audio_feedback = mod.HiggsAudioV3TalkerForConditionalGeneration._apply_audio_feedback.__get__(t)
|
|
||||||
return t
|
|
||||||
|
|
||||||
def test_text_positions_unchanged(self):
|
|
||||||
"""Non-audio positions should not be modified."""
|
|
||||||
t = self._make_feedback_talker()
|
|
||||||
# No audio state → all positions unchanged
|
|
||||||
input_ids = torch.tensor([1, 2, 3, 4])
|
|
||||||
hidden = torch.randn(4, 16)
|
|
||||||
result = t._apply_audio_feedback(hidden, input_ids)
|
|
||||||
assert torch.equal(result, hidden)
|
|
||||||
|
|
||||||
def test_audio_position_replaced_with_state(self):
|
|
||||||
"""Audio position with state should have its embedding replaced."""
|
|
||||||
t = self._make_feedback_talker()
|
|
||||||
audio_id = t._audio_continuation_id
|
|
||||||
t._audio_state[1] = {"last_codes": torch.zeros(8, dtype=torch.long)}
|
|
||||||
input_ids = torch.tensor([1, audio_id, 3])
|
|
||||||
hidden = torch.zeros(3, 16)
|
|
||||||
result = t._apply_audio_feedback(hidden, input_ids)
|
|
||||||
# Position 1 (audio) should be non-zero (embedding of all-0 codes)
|
|
||||||
assert result[1].abs().sum() > 0
|
|
||||||
# Positions 0 and 2 should remain zero
|
|
||||||
assert result[0].abs().sum() == 0
|
|
||||||
assert result[2].abs().sum() == 0
|
|
||||||
|
|
||||||
def test_no_state_no_replacement(self):
|
|
||||||
"""Audio position without state should not be replaced."""
|
|
||||||
t = self._make_feedback_talker()
|
|
||||||
audio_id = t._audio_continuation_id
|
|
||||||
# No state for row 0
|
|
||||||
input_ids = torch.tensor([audio_id])
|
|
||||||
hidden = torch.zeros(1, 16)
|
|
||||||
result = t._apply_audio_feedback(hidden, input_ids)
|
|
||||||
assert torch.equal(result, hidden)
|
|
||||||
|
|
||||||
|
|
||||||
# ---- AC-8: Codec Strictness ----
|
|
||||||
|
|
||||||
|
|
||||||
class TestCodecStrictness:
|
|
||||||
def test_bundled_missing_quantizer_key_raises(self):
|
|
||||||
"""Bundled codec load must fail when a quantizer codebook key is missing."""
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
|
||||||
HiggsAudioV3Config,
|
|
||||||
)
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_code2wav import (
|
|
||||||
HiggsAudioV3Code2Wav,
|
|
||||||
)
|
|
||||||
|
|
||||||
config = HiggsAudioV3Config()
|
|
||||||
c2w = HiggsAudioV3Code2Wav(config=config)
|
|
||||||
|
|
||||||
# State with only 7 quantizers (missing quantizer.quantizers.7.*)
|
|
||||||
codec_state = {}
|
|
||||||
for i in range(7):
|
|
||||||
codec_state[f"quantizer.quantizers.{i}.codebook.embed"] = torch.randn(1024, 64)
|
|
||||||
codec_state[f"quantizer.quantizers.{i}.project_out.weight"] = torch.randn(1024, 64)
|
|
||||||
codec_state[f"quantizer.quantizers.{i}.project_out.bias"] = torch.randn(1024)
|
|
||||||
codec_state["fc2.weight"] = torch.randn(256, 1024)
|
|
||||||
codec_state["fc2.bias"] = torch.randn(256)
|
|
||||||
|
|
||||||
with pytest.raises(KeyError, match="quantizer 7"):
|
|
||||||
c2w._load_from_bundled_state(codec_state, device=torch.device("cpu"))
|
|
||||||
|
|
||||||
def test_bundled_missing_fc2_raises(self):
|
|
||||||
"""Bundled codec load must fail when fc2 keys are missing."""
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
|
||||||
HiggsAudioV3Config,
|
|
||||||
)
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_code2wav import (
|
|
||||||
HiggsAudioV3Code2Wav,
|
|
||||||
)
|
|
||||||
|
|
||||||
config = HiggsAudioV3Config()
|
|
||||||
c2w = HiggsAudioV3Code2Wav(config=config)
|
|
||||||
|
|
||||||
codec_state = {}
|
|
||||||
for i in range(8):
|
|
||||||
codec_state[f"quantizer.quantizers.{i}.codebook.embed"] = torch.randn(1024, 64)
|
|
||||||
codec_state[f"quantizer.quantizers.{i}.project_out.weight"] = torch.randn(1024, 64)
|
|
||||||
codec_state[f"quantizer.quantizers.{i}.project_out.bias"] = torch.randn(1024)
|
|
||||||
# No fc2 keys
|
|
||||||
|
|
||||||
with pytest.raises(KeyError, match="fc2"):
|
|
||||||
c2w._load_from_bundled_state(codec_state, device=torch.device("cpu"))
|
|
||||||
|
|
||||||
def test_bundled_unknown_decoder_key_rejected(self, monkeypatch):
|
|
||||||
"""Unknown decoder-side key should be rejected as unconsumed."""
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_code2wav as c2w_mod
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
|
||||||
HiggsAudioV3Config,
|
|
||||||
)
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_code2wav import (
|
|
||||||
HiggsAudioV3Code2Wav,
|
|
||||||
)
|
|
||||||
|
|
||||||
class FakeDAC(torch.nn.Module):
|
|
||||||
def __init__(self):
|
|
||||||
super().__init__()
|
|
||||||
self.dummy = torch.nn.Parameter(torch.zeros(1))
|
|
||||||
|
|
||||||
monkeypatch.setattr(c2w_mod, "build_boson_dac_decoder", lambda device: FakeDAC().to(device))
|
|
||||||
monkeypatch.setattr(c2w_mod, "build_higgs_audio_acoustic_decoder", lambda cfg, device: FakeDAC().to(device))
|
|
||||||
|
|
||||||
config = HiggsAudioV3Config()
|
|
||||||
c2w = HiggsAudioV3Code2Wav(config=config)
|
|
||||||
|
|
||||||
codec_state = _build_full_codec_state()
|
|
||||||
codec_state["unknown_decoder_module.weight"] = torch.randn(10)
|
|
||||||
|
|
||||||
with pytest.raises(RuntimeError, match="unexpected decoder-side keys"):
|
|
||||||
c2w._load_from_bundled_state(codec_state, device=torch.device("cpu"))
|
|
||||||
|
|
||||||
def test_bundled_encoder_side_keys_accepted(self, monkeypatch):
|
|
||||||
"""Known encoder-side keys should not trigger unconsumed-key rejection."""
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_code2wav as c2w_mod
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
|
||||||
HiggsAudioV3Config,
|
|
||||||
)
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_code2wav import (
|
|
||||||
HiggsAudioV3Code2Wav,
|
|
||||||
)
|
|
||||||
|
|
||||||
class FakeDAC(torch.nn.Module):
|
|
||||||
def __init__(self):
|
|
||||||
super().__init__()
|
|
||||||
self.dummy = torch.nn.Parameter(torch.zeros(1))
|
|
||||||
|
|
||||||
monkeypatch.setattr(c2w_mod, "build_boson_dac_decoder", lambda device: FakeDAC().to(device))
|
|
||||||
monkeypatch.setattr(c2w_mod, "build_higgs_audio_acoustic_decoder", lambda cfg, device: FakeDAC().to(device))
|
|
||||||
|
|
||||||
config = HiggsAudioV3Config()
|
|
||||||
c2w = HiggsAudioV3Code2Wav(config=config)
|
|
||||||
|
|
||||||
codec_state = _build_full_codec_state()
|
|
||||||
codec_state["acoustic_encoder.block.0.conv_t1.weight"] = torch.randn(10)
|
|
||||||
codec_state["semantic_model.encoder.layers.0.weight"] = torch.randn(10)
|
|
||||||
codec_state["fc.weight"] = torch.randn(10)
|
|
||||||
|
|
||||||
# Should NOT raise — encoder-side keys are allowed
|
|
||||||
c2w._load_from_bundled_state(codec_state, device=torch.device("cpu"))
|
|
||||||
assert c2w._loaded
|
|
||||||
|
|
||||||
|
|
||||||
def _build_minimal_codec_state() -> dict[str, torch.Tensor]:
|
|
||||||
"""Build a minimal codec state dict (quantizer + fc2, no DAC)."""
|
|
||||||
state: dict[str, torch.Tensor] = {}
|
|
||||||
for i in range(8):
|
|
||||||
state[f"quantizer.quantizers.{i}.codebook.embed"] = torch.randn(1024, 64)
|
|
||||||
state[f"quantizer.quantizers.{i}.project_out.weight"] = torch.randn(1024, 64)
|
|
||||||
state[f"quantizer.quantizers.{i}.project_out.bias"] = torch.randn(1024)
|
|
||||||
state["fc2.weight"] = torch.randn(256, 1024)
|
|
||||||
state["fc2.bias"] = torch.randn(256)
|
|
||||||
return state
|
|
||||||
|
|
||||||
|
|
||||||
def _build_full_codec_state() -> dict[str, torch.Tensor]:
|
|
||||||
"""Build a full codec state dict including acoustic_decoder keys.
|
|
||||||
|
|
||||||
The acoustic_decoder keys here match what a FakeDAC module expects
|
|
||||||
(just 'dummy'). Real DAC has many more keys.
|
|
||||||
"""
|
|
||||||
state = _build_minimal_codec_state()
|
|
||||||
state["acoustic_decoder.dummy"] = torch.zeros(1)
|
|
||||||
return state
|
|
||||||
|
|
||||||
|
|
||||||
# ---- AC-2: Loader Required-Key Validation ----
|
|
||||||
|
|
||||||
|
|
||||||
class TestLoaderRequiredKeys:
|
|
||||||
"""Test exact required-key validation in load_weights."""
|
|
||||||
|
|
||||||
def test_build_required_keys_count(self):
|
|
||||||
"""Required key set should have 1 text_emb + 36*11 layers + 1 norm + 1 modality = 399."""
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
|
||||||
HiggsAudioV3TalkerForConditionalGeneration,
|
|
||||||
)
|
|
||||||
|
|
||||||
keys = HiggsAudioV3TalkerForConditionalGeneration._build_required_keys(36)
|
|
||||||
assert len(keys) == 1 + 36 * 11 + 1 + 1 # text + layers + norm + modality = 399
|
|
||||||
|
|
||||||
def test_build_required_keys_contains_norm(self):
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
|
||||||
HiggsAudioV3TalkerForConditionalGeneration,
|
|
||||||
)
|
|
||||||
|
|
||||||
keys = HiggsAudioV3TalkerForConditionalGeneration._build_required_keys(2)
|
|
||||||
assert "body.norm.weight" in keys
|
|
||||||
|
|
||||||
def test_build_required_keys_contains_layer_subkeys(self):
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
|
||||||
HiggsAudioV3TalkerForConditionalGeneration,
|
|
||||||
)
|
|
||||||
|
|
||||||
keys = HiggsAudioV3TalkerForConditionalGeneration._build_required_keys(2)
|
|
||||||
assert "body.layers.0.self_attn.k_proj.weight" in keys
|
|
||||||
assert "body.layers.1.mlp.gate_proj.weight" in keys
|
|
||||||
|
|
||||||
def test_build_required_keys_contains_modality_embedding(self):
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
|
||||||
HiggsAudioV3TalkerForConditionalGeneration,
|
|
||||||
)
|
|
||||||
|
|
||||||
keys = HiggsAudioV3TalkerForConditionalGeneration._build_required_keys(1)
|
|
||||||
assert "tied.embedding.modality_embeddings.0.embedding.weight" in keys
|
|
||||||
|
|
||||||
def test_build_required_keys_contains_text_embedding(self):
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
|
||||||
HiggsAudioV3TalkerForConditionalGeneration,
|
|
||||||
)
|
|
||||||
|
|
||||||
keys = HiggsAudioV3TalkerForConditionalGeneration._build_required_keys(1)
|
|
||||||
assert "tied.embedding.text_embedding.weight" in keys
|
|
||||||
|
|
||||||
|
|
||||||
class TestLoadWeightsEnforcement:
|
|
||||||
"""Test load_weights() itself with synthetic key sets via a minimal fake talker."""
|
|
||||||
|
|
||||||
def _make_fake_talker_and_keys(self):
|
|
||||||
"""Build a minimal talker-like object and the full 1-layer key set."""
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod
|
|
||||||
|
|
||||||
# Minimal fake talker with just enough to run load_weights
|
|
||||||
talker = object.__new__(mod.HiggsAudioV3TalkerForConditionalGeneration)
|
|
||||||
torch.nn.Module.__init__(talker)
|
|
||||||
|
|
||||||
# Attach minimal attributes load_weights needs
|
|
||||||
talker.multimodal_embedding = mod.HiggsFusedMultiTextEmbedding(8, 1026, 16)
|
|
||||||
talker.modality_head = mod.HiggsFusedMultiTextHead(8, 1026, 16)
|
|
||||||
talker.tie_modality = True
|
|
||||||
talker.modality_head.weight = talker.multimodal_embedding.weight
|
|
||||||
|
|
||||||
# Fake backbone config with 1 layer
|
|
||||||
class FakeConfig:
|
|
||||||
num_hidden_layers = 1
|
|
||||||
tie_word_embeddings = True
|
|
||||||
|
|
||||||
talker._backbone_config = FakeConfig()
|
|
||||||
|
|
||||||
# Minimal model and lm_head so _BackboneWrapper can be constructed
|
|
||||||
talker.model = torch.nn.Module()
|
|
||||||
talker.lm_head = torch.nn.Module()
|
|
||||||
|
|
||||||
# Fake vllm_config for resolve_special_tokens (no-op)
|
|
||||||
class FakeModelConfig:
|
|
||||||
model = None
|
|
||||||
|
|
||||||
class FakeVllmConfig:
|
|
||||||
model_config = FakeModelConfig()
|
|
||||||
|
|
||||||
talker.vllm_config = FakeVllmConfig()
|
|
||||||
talker.config = type("C", (), {"resolve_special_tokens": lambda self, p: None})()
|
|
||||||
talker._resolved_tokens = True
|
|
||||||
talker._audio_continuation_id = None
|
|
||||||
talker._eos_token_id = None
|
|
||||||
|
|
||||||
# Build the full required key set for 1 layer
|
|
||||||
required = mod.HiggsAudioV3TalkerForConditionalGeneration._build_required_keys(1)
|
|
||||||
|
|
||||||
# Build synthetic weights: tiny tensors for every required key
|
|
||||||
weights = {}
|
|
||||||
for key in required:
|
|
||||||
if "modality_embeddings.0.embedding" in key:
|
|
||||||
weights[key] = torch.randn(8 * 1026, 16)
|
|
||||||
else:
|
|
||||||
weights[key] = torch.randn(4) # tiny placeholder
|
|
||||||
|
|
||||||
# Monkeypatch _BackboneWrapper.load_weights to accept and return keys
|
|
||||||
def fake_backbone_load(self_wrapper, ws):
|
|
||||||
consumed = set()
|
|
||||||
for name, _ in ws:
|
|
||||||
consumed.add(name)
|
|
||||||
return consumed
|
|
||||||
|
|
||||||
talker._fake_backbone_load = fake_backbone_load
|
|
||||||
return talker, weights, mod, fake_backbone_load
|
|
||||||
|
|
||||||
def test_load_weights_succeeds_with_all_keys(self, monkeypatch):
|
|
||||||
"""load_weights succeeds when all required keys are present."""
|
|
||||||
talker, weights, mod, fake_load = self._make_fake_talker_and_keys()
|
|
||||||
monkeypatch.setattr(mod._BackboneWrapper, "load_weights", fake_load)
|
|
||||||
result = talker.load_weights(iter(weights.items()))
|
|
||||||
assert isinstance(result, set)
|
|
||||||
assert "multimodal_embedding.weight" in result
|
|
||||||
|
|
||||||
def test_load_weights_fails_missing_norm(self, monkeypatch):
|
|
||||||
"""load_weights raises when body.norm.weight is missing."""
|
|
||||||
talker, weights, mod, fake_load = self._make_fake_talker_and_keys()
|
|
||||||
monkeypatch.setattr(mod._BackboneWrapper, "load_weights", fake_load)
|
|
||||||
del weights["body.norm.weight"]
|
|
||||||
with pytest.raises(RuntimeError, match="body.norm.weight"):
|
|
||||||
talker.load_weights(iter(weights.items()))
|
|
||||||
|
|
||||||
def test_load_weights_fails_missing_layer_subkey(self, monkeypatch):
|
|
||||||
"""load_weights raises when a specific layer subkey is missing."""
|
|
||||||
talker, weights, mod, fake_load = self._make_fake_talker_and_keys()
|
|
||||||
monkeypatch.setattr(mod._BackboneWrapper, "load_weights", fake_load)
|
|
||||||
del weights["body.layers.0.self_attn.k_proj.weight"]
|
|
||||||
with pytest.raises(RuntimeError, match="body.layers.0.self_attn.k_proj.weight"):
|
|
||||||
talker.load_weights(iter(weights.items()))
|
|
||||||
|
|
||||||
def test_load_weights_fails_missing_modality_embedding(self, monkeypatch):
|
|
||||||
"""load_weights raises when modality embedding is missing."""
|
|
||||||
talker, weights, mod, fake_load = self._make_fake_talker_and_keys()
|
|
||||||
monkeypatch.setattr(mod._BackboneWrapper, "load_weights", fake_load)
|
|
||||||
del weights["tied.embedding.modality_embeddings.0.embedding.weight"]
|
|
||||||
with pytest.raises(RuntimeError, match="modality_embeddings"):
|
|
||||||
talker.load_weights(iter(weights.items()))
|
|
||||||
|
|
||||||
def test_load_weights_fails_unknown_prefix(self, monkeypatch):
|
|
||||||
"""load_weights raises on unknown non-codec checkpoint prefix."""
|
|
||||||
talker, weights, mod, fake_load = self._make_fake_talker_and_keys()
|
|
||||||
monkeypatch.setattr(mod._BackboneWrapper, "load_weights", fake_load)
|
|
||||||
weights["unknown.prefix.weight"] = torch.randn(4)
|
|
||||||
with pytest.raises(ValueError, match="Unexpected checkpoint key"):
|
|
||||||
talker.load_weights(iter(weights.items()))
|
|
||||||
|
|
||||||
|
|
||||||
# ---- AC-7: Stage Input Processor ----
|
|
||||||
|
|
||||||
|
|
||||||
class TestStageInputProcessor:
|
|
||||||
def test_revert_delay_pattern(self):
|
|
||||||
from vllm_omni.model_executor.stage_input_processors.higgs_audio_v3 import (
|
|
||||||
_revert_delay_pattern,
|
|
||||||
)
|
|
||||||
|
|
||||||
# 8 codebooks, 3 real frames -> delayed shape [8, 3+8-1=10]
|
|
||||||
Q, T = 8, 3
|
|
||||||
delayed = torch.full((Q, T + Q - 1), 1024) # Fill with BOC
|
|
||||||
for i in range(Q):
|
|
||||||
for t in range(T):
|
|
||||||
delayed[i, i + t] = i * 100 + t # Real codes at shifted positions
|
|
||||||
result = _revert_delay_pattern(delayed)
|
|
||||||
assert result.shape == (Q, T)
|
|
||||||
for i in range(Q):
|
|
||||||
for t in range(T):
|
|
||||||
assert result[i, t].item() == i * 100 + t
|
|
||||||
|
|
||||||
def test_revert_delay_pattern_rejects_wrong_codebooks(self):
|
|
||||||
from vllm_omni.model_executor.stage_input_processors.higgs_audio_v3 import (
|
|
||||||
_revert_delay_pattern,
|
|
||||||
)
|
|
||||||
|
|
||||||
# 7 codebooks should be rejected
|
|
||||||
codes = torch.zeros(7, 20)
|
|
||||||
with pytest.raises(ValueError, match="Expected exactly 8 codebook rows"):
|
|
||||||
_revert_delay_pattern(codes)
|
|
||||||
|
|
||||||
def test_revert_delay_pattern_rejects_too_few_frames(self):
|
|
||||||
from vllm_omni.model_executor.stage_input_processors.higgs_audio_v3 import (
|
|
||||||
_revert_delay_pattern,
|
|
||||||
)
|
|
||||||
|
|
||||||
# 8 codebooks but only 5 frames (need at least 8)
|
|
||||||
codes = torch.zeros(8, 5)
|
|
||||||
with pytest.raises(ValueError, match="Not enough frames"):
|
|
||||||
_revert_delay_pattern(codes)
|
|
||||||
|
|
||||||
def test_filter_real_code_frames(self):
|
|
||||||
from vllm_omni.model_executor.stage_input_processors.higgs_audio_v3 import (
|
|
||||||
_filter_real_code_frames,
|
|
||||||
)
|
|
||||||
|
|
||||||
# 8 codebooks, 4 frames
|
|
||||||
codes = torch.tensor(
|
|
||||||
[
|
|
||||||
[100, 200, 1024, 300], # cb0: frame 2 has BOC
|
|
||||||
[101, 201, 1024, 301],
|
|
||||||
[102, 202, 1024, 302],
|
|
||||||
[103, 203, 1024, 303],
|
|
||||||
[104, 204, 1024, 304],
|
|
||||||
[105, 205, 1024, 305],
|
|
||||||
[106, 206, 1024, 306],
|
|
||||||
[107, 207, 1024, 307],
|
|
||||||
]
|
|
||||||
)
|
|
||||||
result = _filter_real_code_frames(codes)
|
|
||||||
# Frame 2 (column 2) has BOC in all codebooks -> filtered out
|
|
||||||
assert result.shape == (8, 3)
|
|
||||||
assert result[0, 0].item() == 100
|
|
||||||
assert result[0, 1].item() == 200
|
|
||||||
assert result[0, 2].item() == 300
|
|
||||||
|
|
||||||
|
|
||||||
# ---- AC-10: Registry ----
|
|
||||||
|
|
||||||
|
|
||||||
class TestRegistry:
|
|
||||||
def test_talker_registered(self):
|
|
||||||
from vllm_omni.model_executor.models.registry import _OMNI_MODELS
|
|
||||||
|
|
||||||
assert "HiggsMultimodalQwen3ForConditionalGeneration" in _OMNI_MODELS
|
|
||||||
assert "HiggsAudioV3TalkerForConditionalGeneration" in _OMNI_MODELS
|
|
||||||
|
|
||||||
def test_code2wav_registered(self):
|
|
||||||
from vllm_omni.model_executor.models.registry import _OMNI_MODELS
|
|
||||||
|
|
||||||
assert "HiggsAudioV3Code2WavForConditionalGeneration" in _OMNI_MODELS
|
|
||||||
|
|
||||||
def test_pipeline_registered(self):
|
|
||||||
from vllm_omni.config.pipeline_registry import _OMNI_PIPELINES
|
|
||||||
|
|
||||||
assert "higgs_multimodal_qwen3" in _OMNI_PIPELINES
|
|
||||||
|
|
||||||
def test_deploy_yaml_exists(self):
|
|
||||||
import os
|
|
||||||
|
|
||||||
# __file__ = tests/unit/higgs_audio_v3/test_*.py → 4 dirnames to repo root
|
|
||||||
repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
|
|
||||||
yaml_path = os.path.join(repo_root, "vllm_omni", "deploy", "higgs_multimodal_qwen3.yaml")
|
|
||||||
assert os.path.isfile(yaml_path), f"Deploy YAML not found at {yaml_path}"
|
|
||||||
|
|
||||||
|
|
||||||
# ---- AC-3: Prompt Builder ----
|
|
||||||
|
|
||||||
|
|
||||||
class TestPromptBuilder:
|
|
||||||
def _make_mock_tokenizer(self):
|
|
||||||
"""Create a mock tokenizer with the required special tokens."""
|
|
||||||
|
|
||||||
class MockTokenizer:
|
|
||||||
def __init__(self):
|
|
||||||
self._added_vocab = {
|
|
||||||
"<|tts|>": 151700,
|
|
||||||
"<|text|>": 151701,
|
|
||||||
"<|audio|>": 151702,
|
|
||||||
"<|ref_audio|>": 151703,
|
|
||||||
"<|ref_text|>": 151704,
|
|
||||||
}
|
|
||||||
|
|
||||||
def get_added_vocab(self):
|
|
||||||
return self._added_vocab
|
|
||||||
|
|
||||||
def encode(self, text, add_special_tokens=True):
|
|
||||||
# Simple word-level tokenization for testing
|
|
||||||
return list(range(100, 100 + len(text.split())))
|
|
||||||
|
|
||||||
return MockTokenizer()
|
|
||||||
|
|
||||||
def test_plain_tts_prompt(self):
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_tokenizer import (
|
|
||||||
HiggsAudioV3TokenizerAdapter,
|
|
||||||
)
|
|
||||||
|
|
||||||
tok = self._make_mock_tokenizer()
|
|
||||||
adapter = HiggsAudioV3TokenizerAdapter(tok)
|
|
||||||
ids = adapter.build_prompt("Hello world")
|
|
||||||
assert ids[0] == 151700 # <|tts|>
|
|
||||||
assert ids[1] == 151701 # <|text|>
|
|
||||||
assert ids[-1] == 151702 # <|audio|>
|
|
||||||
assert len(ids) == 2 + 2 + 1 # tts + text + 2 word tokens + audio
|
|
||||||
|
|
||||||
def test_empty_text_rejected(self):
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_tokenizer import (
|
|
||||||
HiggsAudioV3TokenizerAdapter,
|
|
||||||
)
|
|
||||||
|
|
||||||
tok = self._make_mock_tokenizer()
|
|
||||||
adapter = HiggsAudioV3TokenizerAdapter(tok)
|
|
||||||
with pytest.raises(ValueError, match="non-empty"):
|
|
||||||
adapter.build_prompt("")
|
|
||||||
|
|
||||||
def test_whitespace_only_rejected(self):
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_tokenizer import (
|
|
||||||
HiggsAudioV3TokenizerAdapter,
|
|
||||||
)
|
|
||||||
|
|
||||||
tok = self._make_mock_tokenizer()
|
|
||||||
adapter = HiggsAudioV3TokenizerAdapter(tok)
|
|
||||||
with pytest.raises(ValueError, match="non-empty"):
|
|
||||||
adapter.build_prompt(" ")
|
|
||||||
|
|
||||||
def test_missing_specials_rejected(self):
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_tokenizer import (
|
|
||||||
HiggsAudioV3TokenizerAdapter,
|
|
||||||
)
|
|
||||||
|
|
||||||
class BadTokenizer:
|
|
||||||
def get_added_vocab(self):
|
|
||||||
return {"<|tts|>": 1} # Missing <|text|> and <|audio|>
|
|
||||||
|
|
||||||
with pytest.raises(ValueError, match="missing"):
|
|
||||||
HiggsAudioV3TokenizerAdapter(BadTokenizer())
|
|
||||||
|
|
||||||
def test_no_voice_clone_tokens_in_plain_tts(self):
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_tokenizer import (
|
|
||||||
HiggsAudioV3TokenizerAdapter,
|
|
||||||
)
|
|
||||||
|
|
||||||
tok = self._make_mock_tokenizer()
|
|
||||||
adapter = HiggsAudioV3TokenizerAdapter(tok)
|
|
||||||
ids = adapter.build_prompt("Hello")
|
|
||||||
# Should not contain ref_audio or ref_text token IDs
|
|
||||||
assert 151703 not in ids # <|ref_audio|>
|
|
||||||
assert 151704 not in ids # <|ref_text|>
|
|
||||||
@@ -44,14 +44,6 @@ def test_resolve_pooler_payload_req_ids_downstream_stage_uses_filtered_requests(
|
|||||||
assert payload_req_ids == ["r2"]
|
assert payload_req_ids == ["r2"]
|
||||||
|
|
||||||
|
|
||||||
def test_sparse_mm_req_ids_requires_sparse_audio_marker():
|
|
||||||
assert GPUARModelRunner._sparse_mm_req_ids({"meta": {"req_id": ["r1"]}}) is None
|
|
||||||
assert GPUARModelRunner._sparse_mm_req_ids({"meta.req_id": ["r1"]}) is None
|
|
||||||
|
|
||||||
assert GPUARModelRunner._sparse_mm_req_ids({"meta": {"req_id": ["r1"], "sparse_audio": ["1"]}}) == ["r1"]
|
|
||||||
assert GPUARModelRunner._sparse_mm_req_ids({"meta.req_id": ["r1"], "meta.sparse_audio": ["1"]}) == ["r1"]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("query_start_loc_attr", ["method", "tensor_attr"])
|
@pytest.mark.parametrize("query_start_loc_attr", ["method", "tensor_attr"])
|
||||||
def test_sample_tokens_tail_only_prefix_cache_uses_staged_cpu_hidden_states(monkeypatch, query_start_loc_attr):
|
def test_sample_tokens_tail_only_prefix_cache_uses_staged_cpu_hidden_states(monkeypatch, query_start_loc_attr):
|
||||||
runner = object.__new__(GPUARModelRunner)
|
runner = object.__new__(GPUARModelRunner)
|
||||||
|
|||||||
@@ -0,0 +1,16 @@
|
|||||||
|
import numpy as np
|
||||||
|
from vllm.assets.video import VideoAsset
|
||||||
|
from vllm.multimodal.media.audio import load_audio
|
||||||
|
|
||||||
|
|
||||||
|
def extract_video_audio(path: str = None, sampling_rate: int = 16000) -> np.ndarray:
|
||||||
|
"""This function extracts the audio from a video file path and returns the audio as a numpy array.
|
||||||
|
Args:
|
||||||
|
path: The path to the video file.
|
||||||
|
Returns:
|
||||||
|
The audio as a numpy array.
|
||||||
|
"""
|
||||||
|
if not path:
|
||||||
|
path = VideoAsset(name="baby_reading").video_path
|
||||||
|
audio_signal, sr = load_audio(path, sr=sampling_rate)
|
||||||
|
return audio_signal
|
||||||
@@ -1,6 +1,5 @@
|
|||||||
import warnings
|
import warnings
|
||||||
from collections import defaultdict
|
from dataclasses import dataclass
|
||||||
from dataclasses import field, make_dataclass
|
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
from vllm.benchmarks.datasets import SampleRequest
|
from vllm.benchmarks.datasets import SampleRequest
|
||||||
@@ -10,172 +9,30 @@ from vllm.tokenizers import TokenizerLike
|
|||||||
|
|
||||||
from vllm_omni.metrics import definitions as defs
|
from vllm_omni.metrics import definitions as defs
|
||||||
|
|
||||||
_PERCENTILE_ROWS_TYPE = list[tuple[float, float]] | None
|
|
||||||
_FLOAT_LIST_TYPE = list[float]
|
|
||||||
_INT_LIST_TYPE = list[int]
|
|
||||||
|
|
||||||
_MULTIMODAL_BENCHMARK_FIELDS = [
|
@dataclass
|
||||||
(defs.MEAN_AUDIO_TTFP_MS, float, field(default=0.0)),
|
class MultiModalsBenchmarkMetrics(BenchmarkMetrics):
|
||||||
(defs.MEDIAN_AUDIO_TTFP_MS, float, field(default=0.0)),
|
mean_audio_ttfp_ms: float = 0.0
|
||||||
(defs.STD_AUDIO_TTFP_MS, float, field(default=0.0)),
|
median_audio_ttfp_ms: float = 0.0
|
||||||
(defs.PERCENTILES_AUDIO_TTFP_MS, _PERCENTILE_ROWS_TYPE, field(default=None)),
|
std_audio_ttfp_ms: float = 0.0
|
||||||
(defs.TOTAL_AUDIO_DURATION_S, float, field(default=0.0)),
|
percentiles_audio_ttfp_ms: list[tuple[float, float]] = None
|
||||||
(defs.TOTAL_AUDIO_FRAMES, int, field(default=0)),
|
total_audio_duration_s: float = 0.0
|
||||||
(defs.AUDIO_THROUGHPUT, float, field(default=0.0)),
|
total_audio_frames: int = 0
|
||||||
(defs.MEAN_AUDIO_RTF, float, field(default=0.0)),
|
audio_throughput: float = 0.0
|
||||||
(defs.MEDIAN_AUDIO_RTF, float, field(default=0.0)),
|
mean_audio_rtf: float = 0.0
|
||||||
(defs.STD_AUDIO_RTF, float, field(default=0.0)),
|
median_audio_rtf: float = 0.0
|
||||||
(defs.PERCENTILES_AUDIO_RTF, _PERCENTILE_ROWS_TYPE, field(default=None)),
|
std_audio_rtf: float = 0.0
|
||||||
(defs.MEAN_AUDIO_DURATION_S, float, field(default=0.0)),
|
percentiles_audio_rtf: list[tuple[float, float]] = None
|
||||||
(defs.MEDIAN_AUDIO_DURATION_S, float, field(default=0.0)),
|
mean_audio_duration_s: float = 0.0
|
||||||
(defs.STD_AUDIO_DURATION_S, float, field(default=0.0)),
|
median_audio_duration_s: float = 0.0
|
||||||
(defs.PERCENTILES_AUDIO_DURATION_S, _PERCENTILE_ROWS_TYPE, field(default=None)),
|
std_audio_duration_s: float = 0.0
|
||||||
(defs.MEAN_AUDIO_UNDERRUN_S, float, field(default=0.0)),
|
percentiles_audio_duration_s: list[tuple[float, float]] = None
|
||||||
(defs.MEDIAN_AUDIO_UNDERRUN_S, float, field(default=0.0)),
|
# Streaming-audio continuity (see vllm_omni/benchmarks/audio_continuity.py).
|
||||||
(defs.STD_AUDIO_UNDERRUN_S, float, field(default=0.0)),
|
mean_audio_underrun_s: float = 0.0
|
||||||
(defs.PERCENTILES_AUDIO_UNDERRUN_S, _PERCENTILE_ROWS_TYPE, field(default=None)),
|
median_audio_underrun_s: float = 0.0
|
||||||
(defs.AUDIO_CONTINUITY_OK_RATE, float, field(default=1.0)),
|
std_audio_underrun_s: float = 0.0
|
||||||
(defs.TOTAL_IMAGES, int, field(default=0)),
|
percentiles_audio_underrun_s: list[tuple[float, float]] = None
|
||||||
(defs.IMAGE_THROUGHPUT, float, field(default=0.0)),
|
audio_continuity_ok_rate: float = 1.0
|
||||||
(defs.AVERAGE_PIXELS_PER_IMAGE, float, field(default=0.0)),
|
|
||||||
(defs.MEAN_DENOISE_STEP_LATENCY_MS, float, field(default=0.0)),
|
|
||||||
(defs.MEAN_IMAGE_GENERATION_MS, float, field(default=0.0)),
|
|
||||||
(defs.MEDIAN_IMAGE_GENERATION_MS, float, field(default=0.0)),
|
|
||||||
(defs.STD_IMAGE_GENERATION_MS, float, field(default=0.0)),
|
|
||||||
(defs.PERCENTILES_IMAGE_GENERATION_MS, _PERCENTILE_ROWS_TYPE, field(default=None)),
|
|
||||||
]
|
|
||||||
|
|
||||||
MultiModalsBenchmarkMetrics = make_dataclass(
|
|
||||||
"MultiModalsBenchmarkMetrics",
|
|
||||||
_MULTIMODAL_BENCHMARK_FIELDS,
|
|
||||||
bases=(BenchmarkMetrics,),
|
|
||||||
)
|
|
||||||
|
|
||||||
_STAGE_BENCHMARK_FIELDS = [
|
|
||||||
("stage_id", int, field(default=0)),
|
|
||||||
("stage_name", str, field(default="")),
|
|
||||||
("final_output_type", str, field(default="")),
|
|
||||||
(defs.TOTAL_OUTPUT, int, field(default=0)),
|
|
||||||
(defs.TTFTS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
||||||
(defs.TPOTS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
||||||
(defs.ITLS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
||||||
(defs.VLLM_TTFTS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
||||||
(defs.VLLM_TPOTS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
||||||
(defs.VLLM_ITLS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
||||||
(defs.AUDIO_TTFPS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
||||||
(defs.AUDIO_RTFS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
||||||
(defs.AUDIO_DURATIONS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
||||||
(defs.AUDIO_FRAMES, _INT_LIST_TYPE, field(default_factory=list)),
|
|
||||||
(defs.MISSING_AUDIO_DURATION_COUNT, int, field(default=0)),
|
|
||||||
(defs.MISSING_AUDIO_RTF_COUNT, int, field(default=0)),
|
|
||||||
(defs.STAGE_GEN_TIMES_MS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
||||||
(defs.POSTPROCESS_TIMES_MS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
||||||
("output_unit_type", str, field(default="")),
|
|
||||||
(defs.OUTPUT_UNIT_COUNT, int, field(default=0)),
|
|
||||||
(defs.SERVING_TIME_TO_FIRST_OUTPUTS_MS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
||||||
(defs.TIME_PER_OUTPUT_UNITS_MS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
||||||
(defs.INTER_OUTPUT_LATENCIES_MS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
||||||
]
|
|
||||||
|
|
||||||
StageBenchmarkMetrics = make_dataclass(
|
|
||||||
"StageBenchmarkMetrics",
|
|
||||||
_STAGE_BENCHMARK_FIELDS,
|
|
||||||
namespace={"__doc__": "Aggregated metrics for one pipeline stage (for printing only)."},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _percentile_rows_seconds(
|
|
||||||
values: list[float], selected_percentiles: list[float], *, to_ms: bool
|
|
||||||
) -> tuple[float, float, list[tuple[float, float]]]:
|
|
||||||
arr = np.asarray(values, dtype=float)
|
|
||||||
scale = 1000.0 if to_ms else 1.0
|
|
||||||
mean_v = float(np.mean(arr)) * scale
|
|
||||||
median_v = float(np.median(arr)) * scale
|
|
||||||
rows = [(float(p), float(np.percentile(arr, p)) * scale) for p in selected_percentiles]
|
|
||||||
return mean_v, median_v, rows
|
|
||||||
|
|
||||||
|
|
||||||
def _p_label(p: float) -> str:
|
|
||||||
pf = float(p)
|
|
||||||
if abs(pf - round(pf)) < 1e-9:
|
|
||||||
return str(int(round(pf)))
|
|
||||||
return str(pf)
|
|
||||||
|
|
||||||
|
|
||||||
_STREAMING_OUTPUT_UNIT_TYPES = frozenset(
|
|
||||||
{
|
|
||||||
"text",
|
|
||||||
"stream",
|
|
||||||
"audio",
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
_AGGREGATE_PERCENTILE_FIELD_NAMES = {
|
|
||||||
defs.AUDIO_TTFP: (
|
|
||||||
defs.MEAN_AUDIO_TTFP_MS,
|
|
||||||
defs.MEDIAN_AUDIO_TTFP_MS,
|
|
||||||
defs.PERCENTILES_AUDIO_TTFP_MS,
|
|
||||||
),
|
|
||||||
defs.AUDIO_RTF: (
|
|
||||||
defs.MEAN_AUDIO_RTF,
|
|
||||||
defs.MEDIAN_AUDIO_RTF,
|
|
||||||
defs.PERCENTILES_AUDIO_RTF,
|
|
||||||
),
|
|
||||||
defs.AUDIO_DURATION: (
|
|
||||||
defs.MEAN_AUDIO_DURATION_S,
|
|
||||||
defs.MEDIAN_AUDIO_DURATION_S,
|
|
||||||
defs.PERCENTILES_AUDIO_DURATION_S,
|
|
||||||
),
|
|
||||||
defs.AUDIO_UNDERRUN: (
|
|
||||||
defs.MEAN_AUDIO_UNDERRUN_S,
|
|
||||||
defs.MEDIAN_AUDIO_UNDERRUN_S,
|
|
||||||
defs.PERCENTILES_AUDIO_UNDERRUN_S,
|
|
||||||
),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _wants_text_ttft(metrics: list[str]) -> bool:
|
|
||||||
return "ttft" in metrics
|
|
||||||
|
|
||||||
|
|
||||||
def _wants_text_tpot(metrics: list[str]) -> bool:
|
|
||||||
return "tpot" in metrics or "tpop" in metrics
|
|
||||||
|
|
||||||
|
|
||||||
def _wants_text_itl(metrics: list[str]) -> bool:
|
|
||||||
return "itl" in metrics
|
|
||||||
|
|
||||||
|
|
||||||
def _wants_stream_ttfc(metrics: list[str]) -> bool:
|
|
||||||
return "ttfc" in metrics
|
|
||||||
|
|
||||||
|
|
||||||
def _wants_stream_tpoc(metrics: list[str]) -> bool:
|
|
||||||
return "tpoc" in metrics or "tpop" in metrics
|
|
||||||
|
|
||||||
|
|
||||||
def _wants_stream_icl(metrics: list[str]) -> bool:
|
|
||||||
return "icl" in metrics
|
|
||||||
|
|
||||||
|
|
||||||
def _stage_modality_flags(
|
|
||||||
final_output_type: str,
|
|
||||||
output_unit_type: str,
|
|
||||||
) -> tuple[bool, bool, bool, bool, bool]:
|
|
||||||
is_text_stage = final_output_type == "text" or output_unit_type == "text"
|
|
||||||
is_audio_stage = final_output_type == "audio" or output_unit_type == "audio"
|
|
||||||
is_image_stage = final_output_type in {"image", "images"} or output_unit_type == "image"
|
|
||||||
is_video_stage = final_output_type in {"video", "videos"} or output_unit_type == "video"
|
|
||||||
is_internal_stream_stage = (
|
|
||||||
output_unit_type in _STREAMING_OUTPUT_UNIT_TYPES and not is_text_stage and not is_audio_stage
|
|
||||||
)
|
|
||||||
return (
|
|
||||||
is_text_stage,
|
|
||||||
is_audio_stage,
|
|
||||||
is_image_stage,
|
|
||||||
is_video_stage,
|
|
||||||
is_internal_stream_stage,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def print_metrics(
|
def print_metrics(
|
||||||
@@ -186,10 +43,6 @@ def print_metrics(
|
|||||||
benchmark_duration,
|
benchmark_duration,
|
||||||
goodput_config_dict,
|
goodput_config_dict,
|
||||||
metrics: MultiModalsBenchmarkMetrics,
|
metrics: MultiModalsBenchmarkMetrics,
|
||||||
*,
|
|
||||||
outputs: list[RequestFuncOutput] | None = None,
|
|
||||||
selected_percentiles: list[float] | None = None,
|
|
||||||
print_stage: bool = False,
|
|
||||||
):
|
):
|
||||||
print("{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="="))
|
print("{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="="))
|
||||||
print("{:<40} {:<10}".format("Successful requests:", metrics.completed))
|
print("{:<40} {:<10}".format("Successful requests:", metrics.completed))
|
||||||
@@ -208,19 +61,7 @@ def print_metrics(
|
|||||||
process_one_metric("e2el", metrics)
|
process_one_metric("e2el", metrics)
|
||||||
print_text_metrics(task_type, selected_percentile_metrics, metrics)
|
print_text_metrics(task_type, selected_percentile_metrics, metrics)
|
||||||
if task_type == TaskType.GENERATION:
|
if task_type == TaskType.GENERATION:
|
||||||
if _has_audio_output(metrics):
|
print_audio_metrics(selected_percentile_metrics, metrics)
|
||||||
print_audio_metrics(selected_percentile_metrics, metrics)
|
|
||||||
if _has_image_output(metrics):
|
|
||||||
print_image_metrics(selected_percentiles or [], metrics)
|
|
||||||
if print_stage and outputs and selected_percentiles is not None:
|
|
||||||
print("\n{s:{c}^{n}}".format(s=" Stage Benchmark Result ", n=50, c="="))
|
|
||||||
for sm in _build_stage_metrics_from_outputs(outputs):
|
|
||||||
print_stage_metrics(
|
|
||||||
task_type,
|
|
||||||
selected_percentile_metrics,
|
|
||||||
selected_percentiles,
|
|
||||||
sm,
|
|
||||||
)
|
|
||||||
print("=" * 50)
|
print("=" * 50)
|
||||||
|
|
||||||
|
|
||||||
@@ -238,36 +79,22 @@ def print_text_metrics(task_type, selected_percentile_metrics, metrics: MultiMod
|
|||||||
# No text tokens generated (e.g. pure TTS speech endpoint): per-token
|
# No text tokens generated (e.g. pure TTS speech endpoint): per-token
|
||||||
# latency metrics (ttft/tpot/itl) are undefined, so skip them.
|
# latency metrics (ttft/tpot/itl) are undefined, so skip them.
|
||||||
has_text_output = metrics.total_output > 0
|
has_text_output = metrics.total_output > 0
|
||||||
if has_text_output:
|
for metric in selected_percentile_metrics:
|
||||||
if _wants_text_ttft(selected_percentile_metrics):
|
if metric == "e2el":
|
||||||
process_one_metric("ttft", metrics)
|
continue
|
||||||
if _wants_text_tpot(selected_percentile_metrics):
|
if not metric.startswith("audio") and has_text_output:
|
||||||
process_one_metric("tpot", metrics)
|
process_one_metric(metric, metrics)
|
||||||
if _wants_text_itl(selected_percentile_metrics):
|
|
||||||
process_one_metric("itl", metrics)
|
|
||||||
|
|
||||||
|
|
||||||
def _has_audio_output(metrics: MultiModalsBenchmarkMetrics) -> bool:
|
|
||||||
return bool(
|
|
||||||
getattr(metrics, defs.TOTAL_AUDIO_DURATION_S, 0.0) > 0 or getattr(metrics, defs.TOTAL_AUDIO_FRAMES, 0) > 0
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _has_image_output(metrics: MultiModalsBenchmarkMetrics) -> bool:
|
|
||||||
return int(getattr(metrics, defs.TOTAL_IMAGES, 0) or 0) > 0
|
|
||||||
|
|
||||||
|
|
||||||
def print_audio_metrics(selected_percentile_metrics, metrics: MultiModalsBenchmarkMetrics):
|
def print_audio_metrics(selected_percentile_metrics, metrics: MultiModalsBenchmarkMetrics):
|
||||||
print("{s:{c}^{n}}".format(s=" Audio Result ", n=50, c="="))
|
print("{s:{c}^{n}}".format(s=" Audio Result ", n=50, c="="))
|
||||||
print(
|
print("{:<40} {:<10.2f}".format("Total audio duration generated(s):", metrics.total_audio_duration_s))
|
||||||
"{:<40} {:<10.2f}".format("Total audio duration generated(s):", getattr(metrics, defs.TOTAL_AUDIO_DURATION_S))
|
print("{:<40} {:<10}".format("Total audio frames generated:", metrics.total_audio_frames))
|
||||||
)
|
print("{:<40} {:<10.2f}".format("Audio throughput(audio duration/s):", metrics.audio_throughput))
|
||||||
print("{:<40} {:<10}".format("Total audio frames generated:", getattr(metrics, defs.TOTAL_AUDIO_FRAMES)))
|
|
||||||
print("{:<40} {:<10.2f}".format("Audio throughput(audio duration/s):", getattr(metrics, defs.AUDIO_THROUGHPUT)))
|
|
||||||
print(
|
print(
|
||||||
"{:<40} {:<10.2%}".format(
|
"{:<40} {:<10.2%}".format(
|
||||||
"Streaming continuity OK rate:",
|
"Streaming continuity OK rate:",
|
||||||
getattr(metrics, defs.AUDIO_CONTINUITY_OK_RATE),
|
metrics.audio_continuity_ok_rate,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
for metric in selected_percentile_metrics:
|
for metric in selected_percentile_metrics:
|
||||||
@@ -275,37 +102,6 @@ def print_audio_metrics(selected_percentile_metrics, metrics: MultiModalsBenchma
|
|||||||
process_one_metric(metric, metrics)
|
process_one_metric(metric, metrics)
|
||||||
|
|
||||||
|
|
||||||
def print_image_metrics(selected_percentiles: list[float], metrics: MultiModalsBenchmarkMetrics):
|
|
||||||
print("{s:{c}^{n}}".format(s=" Image Result ", n=50, c="="))
|
|
||||||
print("{:<40} {:<10}".format("Total images generated:", getattr(metrics, defs.TOTAL_IMAGES)))
|
|
||||||
print("{:<40} {:<10.2f}".format("Image throughput (img/s):", getattr(metrics, defs.IMAGE_THROUGHPUT)))
|
|
||||||
print(
|
|
||||||
"{:<40} {:<10.2f}".format(
|
|
||||||
"Average pixels per image:",
|
|
||||||
getattr(metrics, defs.AVERAGE_PIXELS_PER_IMAGE),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
if getattr(metrics, defs.MEAN_DENOISE_STEP_LATENCY_MS) > 0:
|
|
||||||
print(
|
|
||||||
"{:<40} {:<10.2f}".format(
|
|
||||||
"Mean denoise step latency (ms):",
|
|
||||||
getattr(metrics, defs.MEAN_DENOISE_STEP_LATENCY_MS),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
if getattr(metrics, defs.MEAN_IMAGE_GENERATION_MS) > 0:
|
|
||||||
print("-----------------Image Generation-----------------")
|
|
||||||
print("{:<40} {:<10.2f}".format("Mean IMAGE_GENERATION (ms):", getattr(metrics, defs.MEAN_IMAGE_GENERATION_MS)))
|
|
||||||
print(
|
|
||||||
"{:<40} {:<10.2f}".format(
|
|
||||||
"Median IMAGE_GENERATION (ms):",
|
|
||||||
getattr(metrics, defs.MEDIAN_IMAGE_GENERATION_MS),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
for p, value in getattr(metrics, defs.PERCENTILES_IMAGE_GENERATION_MS) or []:
|
|
||||||
if p in selected_percentiles:
|
|
||||||
print("{:<40} {:<10.2f}".format(f"P{_p_label(p)} IMAGE_GENERATION (ms):", value))
|
|
||||||
|
|
||||||
|
|
||||||
def process_one_metric(
|
def process_one_metric(
|
||||||
metric_attribute_name: str,
|
metric_attribute_name: str,
|
||||||
metrics: MultiModalsBenchmarkMetrics,
|
metrics: MultiModalsBenchmarkMetrics,
|
||||||
@@ -335,16 +131,15 @@ def process_one_metric(
|
|||||||
elif is_audio_rtf:
|
elif is_audio_rtf:
|
||||||
suffix = ""
|
suffix = ""
|
||||||
unit_suffix = ""
|
unit_suffix = ""
|
||||||
field_names = _AGGREGATE_PERCENTILE_FIELD_NAMES.get(metric_attribute_name)
|
mean_attr_name = f"mean_{metric_attribute_name}{suffix}"
|
||||||
mean_attr_name = field_names[0] if field_names else f"mean_{metric_attribute_name}{suffix}"
|
|
||||||
mean_value = getattr(metrics, mean_attr_name, 0.0)
|
mean_value = getattr(metrics, mean_attr_name, 0.0)
|
||||||
print(f"{f'Mean {metric_attribute_name.upper()}{unit_suffix}:':<40} {mean_value:<10.2f}")
|
print(f"{f'Mean {metric_attribute_name.upper()}{unit_suffix}:':<40} {mean_value:<10.2f}")
|
||||||
|
|
||||||
median_attr_name = field_names[1] if field_names else f"median_{metric_attribute_name}{suffix}"
|
median_attr_name = f"median_{metric_attribute_name}{suffix}"
|
||||||
median_value = getattr(metrics, median_attr_name, 0.0)
|
median_value = getattr(metrics, median_attr_name, 0.0)
|
||||||
print(f"{f'Median {metric_attribute_name.upper()}{unit_suffix}:':<40} {median_value:<10.2f}")
|
print(f"{f'Median {metric_attribute_name.upper()}{unit_suffix}:':<40} {median_value:<10.2f}")
|
||||||
|
|
||||||
percentiles_attr_name = field_names[2] if field_names else f"percentiles_{metric_attribute_name}{suffix}"
|
percentiles_attr_name = f"percentiles_{metric_attribute_name}{suffix}"
|
||||||
percentiles = getattr(metrics, percentiles_attr_name, [])
|
percentiles = getattr(metrics, percentiles_attr_name, [])
|
||||||
|
|
||||||
for percentile, value in percentiles:
|
for percentile, value in percentiles:
|
||||||
@@ -353,356 +148,6 @@ def process_one_metric(
|
|||||||
print(f"{label:<40} {value:<10.2f}")
|
print(f"{label:<40} {value:<10.2f}")
|
||||||
|
|
||||||
|
|
||||||
def _print_percentile_metric(
|
|
||||||
header: str,
|
|
||||||
label: str,
|
|
||||||
values: list[float],
|
|
||||||
selected_percentiles: list[float],
|
|
||||||
*,
|
|
||||||
values_are_ms: bool = False,
|
|
||||||
to_ms: bool = True,
|
|
||||||
) -> None:
|
|
||||||
if not values:
|
|
||||||
return
|
|
||||||
values_s = [v / 1000.0 for v in values] if values_are_ms else values
|
|
||||||
mean_v, med_v, pcts = _percentile_rows_seconds(values_s, selected_percentiles, to_ms=to_ms)
|
|
||||||
unit_suffix = " (ms)" if to_ms else ""
|
|
||||||
print("{s:{c}^{n}}".format(s=header, n=50, c="-"))
|
|
||||||
print(f"{f'Mean {label}{unit_suffix}:':<40} {mean_v:<10.2f}")
|
|
||||||
print(f"{f'Median {label}{unit_suffix}:':<40} {med_v:<10.2f}")
|
|
||||||
for p, v in pcts:
|
|
||||||
p_str = _p_label(p)
|
|
||||||
print(f"{f'P{p_str} {label}{unit_suffix}:':<40} {v:<10.2f}")
|
|
||||||
|
|
||||||
|
|
||||||
def _print_stage_timing(sm: StageBenchmarkMetrics, selected_percentiles: list[float]) -> None:
|
|
||||||
_print_percentile_metric(
|
|
||||||
"Stage Timing",
|
|
||||||
"stage_gen_time",
|
|
||||||
getattr(sm, defs.STAGE_GEN_TIMES_MS),
|
|
||||||
selected_percentiles,
|
|
||||||
values_are_ms=True,
|
|
||||||
to_ms=True,
|
|
||||||
)
|
|
||||||
postprocess_times_ms = getattr(sm, defs.POSTPROCESS_TIMES_MS)
|
|
||||||
if postprocess_times_ms and any(v > 0 for v in postprocess_times_ms):
|
|
||||||
_print_percentile_metric(
|
|
||||||
"Stage Postprocess",
|
|
||||||
"postprocess",
|
|
||||||
postprocess_times_ms,
|
|
||||||
selected_percentiles,
|
|
||||||
values_are_ms=True,
|
|
||||||
to_ms=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _print_text_stage_metrics(
|
|
||||||
task_type: TaskType,
|
|
||||||
selected_percentile_metrics: list[str],
|
|
||||||
selected_percentiles: list[float],
|
|
||||||
sm: StageBenchmarkMetrics,
|
|
||||||
) -> None:
|
|
||||||
print("{s:{c}^{n}}".format(s=" Text Result ", n=50, c="="))
|
|
||||||
print("{:<40} {:<10}".format("Stage generated tokens:", getattr(sm, defs.TOTAL_OUTPUT)))
|
|
||||||
|
|
||||||
if task_type != TaskType.GENERATION:
|
|
||||||
return
|
|
||||||
if _wants_text_ttft(selected_percentile_metrics):
|
|
||||||
_print_percentile_metric(
|
|
||||||
"Serving Time to First Token",
|
|
||||||
"Serving TTFT",
|
|
||||||
getattr(sm, defs.TTFTS),
|
|
||||||
selected_percentiles,
|
|
||||||
)
|
|
||||||
_print_percentile_metric("Time to First Token", "TTFT", getattr(sm, defs.VLLM_TTFTS), selected_percentiles)
|
|
||||||
if _wants_text_tpot(selected_percentile_metrics):
|
|
||||||
_print_percentile_metric(
|
|
||||||
"Time per Output Token (excl. 1st token)",
|
|
||||||
"TPOT",
|
|
||||||
getattr(sm, defs.VLLM_TPOTS),
|
|
||||||
selected_percentiles,
|
|
||||||
)
|
|
||||||
if _wants_text_itl(selected_percentile_metrics):
|
|
||||||
_print_percentile_metric("Inter-token Latency", "ITL", getattr(sm, defs.VLLM_ITLS), selected_percentiles)
|
|
||||||
|
|
||||||
|
|
||||||
def _print_audio_stage_metrics(
|
|
||||||
selected_percentile_metrics: list[str],
|
|
||||||
selected_percentiles: list[float],
|
|
||||||
sm: StageBenchmarkMetrics,
|
|
||||||
) -> None:
|
|
||||||
total_duration = float(sum(getattr(sm, defs.AUDIO_DURATIONS)))
|
|
||||||
total_frames = int(sum(getattr(sm, defs.AUDIO_FRAMES)))
|
|
||||||
print("{s:{c}^{n}}".format(s=" Audio Result ", n=50, c="="))
|
|
||||||
print("{:<40} {:<10.2f}".format("Stage audio duration generated(s):", total_duration))
|
|
||||||
print("{:<40} {:<10}".format("Stage audio frames generated:", total_frames))
|
|
||||||
if defs.AUDIO_TTFP in selected_percentile_metrics:
|
|
||||||
_print_percentile_metric(
|
|
||||||
"Serving Time to First Packet",
|
|
||||||
"Serving AUDIO_TTFP",
|
|
||||||
getattr(sm, defs.AUDIO_TTFPS),
|
|
||||||
selected_percentiles,
|
|
||||||
)
|
|
||||||
if defs.AUDIO_RTF in selected_percentile_metrics:
|
|
||||||
audio_rtfs = getattr(sm, defs.AUDIO_RTFS)
|
|
||||||
if audio_rtfs:
|
|
||||||
_print_percentile_metric("Real Time Factor", "AUDIO_RTF", audio_rtfs, selected_percentiles, to_ms=False)
|
|
||||||
elif getattr(sm, defs.MISSING_AUDIO_RTF_COUNT) > 0:
|
|
||||||
print("{:<40} {:<10}".format("AUDIO_RTF skipped:", "missing stage-local audio duration/sample rate"))
|
|
||||||
if defs.AUDIO_DURATION in selected_percentile_metrics:
|
|
||||||
audio_durations = getattr(sm, defs.AUDIO_DURATIONS)
|
|
||||||
if audio_durations:
|
|
||||||
_print_percentile_metric(
|
|
||||||
"Audio Duration",
|
|
||||||
"AUDIO_DURATION",
|
|
||||||
audio_durations,
|
|
||||||
selected_percentiles,
|
|
||||||
to_ms=False,
|
|
||||||
)
|
|
||||||
elif getattr(sm, defs.MISSING_AUDIO_DURATION_COUNT) > 0:
|
|
||||||
print("{:<40} {:<10}".format("AUDIO_DURATION skipped:", "missing stage-local audio duration/sample rate"))
|
|
||||||
|
|
||||||
|
|
||||||
def _print_image_stage_metrics(
|
|
||||||
selected_percentiles: list[float],
|
|
||||||
sm: StageBenchmarkMetrics,
|
|
||||||
) -> None:
|
|
||||||
image_num = getattr(sm, defs.OUTPUT_UNIT_COUNT) if getattr(sm, "output_unit_type") == "image" else 0
|
|
||||||
print("{s:{c}^{n}}".format(s=" Image Result ", n=50, c="="))
|
|
||||||
print("{:<40} {:<10}".format("Total images generated:", image_num))
|
|
||||||
_print_percentile_metric(
|
|
||||||
"Image Generation",
|
|
||||||
"IMAGE_GENERATION",
|
|
||||||
getattr(sm, defs.STAGE_GEN_TIMES_MS),
|
|
||||||
selected_percentiles,
|
|
||||||
values_are_ms=True,
|
|
||||||
to_ms=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _print_internal_stream_stage_metrics(
|
|
||||||
selected_percentile_metrics: list[str],
|
|
||||||
selected_percentiles: list[float],
|
|
||||||
sm: StageBenchmarkMetrics,
|
|
||||||
) -> None:
|
|
||||||
print("{s:{c}^{n}}".format(s=" Internal Stream Result ", n=50, c="="))
|
|
||||||
serving_time_to_first_outputs_ms = getattr(sm, defs.SERVING_TIME_TO_FIRST_OUTPUTS_MS)
|
|
||||||
if _wants_stream_ttfc(selected_percentile_metrics) and serving_time_to_first_outputs_ms:
|
|
||||||
_print_percentile_metric(
|
|
||||||
"Serving Time to First Chunk",
|
|
||||||
"Serving TTFC",
|
|
||||||
serving_time_to_first_outputs_ms,
|
|
||||||
selected_percentiles,
|
|
||||||
values_are_ms=True,
|
|
||||||
to_ms=True,
|
|
||||||
)
|
|
||||||
time_per_output_units_ms = getattr(sm, defs.TIME_PER_OUTPUT_UNITS_MS)
|
|
||||||
if _wants_stream_tpoc(selected_percentile_metrics) and time_per_output_units_ms:
|
|
||||||
_print_percentile_metric(
|
|
||||||
"Time per Output Chunk (excl. 1st chunk)",
|
|
||||||
"TPOP",
|
|
||||||
time_per_output_units_ms,
|
|
||||||
selected_percentiles,
|
|
||||||
values_are_ms=True,
|
|
||||||
to_ms=True,
|
|
||||||
)
|
|
||||||
inter_output_latencies_ms = getattr(sm, defs.INTER_OUTPUT_LATENCIES_MS)
|
|
||||||
if _wants_stream_icl(selected_percentile_metrics) and inter_output_latencies_ms:
|
|
||||||
_print_percentile_metric(
|
|
||||||
"Inter-chunk Latency",
|
|
||||||
"ICL",
|
|
||||||
inter_output_latencies_ms,
|
|
||||||
selected_percentiles,
|
|
||||||
values_are_ms=True,
|
|
||||||
to_ms=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def print_stage_metrics(
|
|
||||||
task_type: TaskType,
|
|
||||||
selected_percentile_metrics: list[str],
|
|
||||||
selected_percentiles: list[float],
|
|
||||||
sm: StageBenchmarkMetrics,
|
|
||||||
):
|
|
||||||
title = f" Stage {getattr(sm, 'stage_id')} ({getattr(sm, 'stage_name')}) "
|
|
||||||
(
|
|
||||||
is_text_stage,
|
|
||||||
is_audio_stage,
|
|
||||||
is_image_stage,
|
|
||||||
is_video_stage,
|
|
||||||
is_internal_stream_stage,
|
|
||||||
) = _stage_modality_flags(getattr(sm, "final_output_type"), getattr(sm, "output_unit_type"))
|
|
||||||
|
|
||||||
print("{s:{c}^{n}}".format(s=title, n=50, c="="))
|
|
||||||
if is_image_stage:
|
|
||||||
_print_image_stage_metrics(selected_percentiles, sm)
|
|
||||||
return
|
|
||||||
|
|
||||||
if is_video_stage:
|
|
||||||
return
|
|
||||||
|
|
||||||
_print_stage_timing(sm, selected_percentiles)
|
|
||||||
if is_text_stage:
|
|
||||||
_print_text_stage_metrics(task_type, selected_percentile_metrics, selected_percentiles, sm)
|
|
||||||
elif is_audio_stage:
|
|
||||||
_print_audio_stage_metrics(
|
|
||||||
selected_percentile_metrics,
|
|
||||||
selected_percentiles,
|
|
||||||
sm,
|
|
||||||
)
|
|
||||||
elif is_internal_stream_stage:
|
|
||||||
_print_internal_stream_stage_metrics(selected_percentile_metrics, selected_percentiles, sm)
|
|
||||||
|
|
||||||
|
|
||||||
def _build_stage_metrics_from_outputs(
|
|
||||||
outputs: list[RequestFuncOutput],
|
|
||||||
) -> list[StageBenchmarkMetrics]:
|
|
||||||
"""Aggregate per ``stage_id`` using ``stage_metrics`` snapshots from the client."""
|
|
||||||
buckets: dict[str, list[tuple[RequestFuncOutput, dict]]] = defaultdict(list)
|
|
||||||
for out in outputs:
|
|
||||||
if not getattr(out, "success", False):
|
|
||||||
continue
|
|
||||||
smap = getattr(out, "stage_metrics", None) or {}
|
|
||||||
if not isinstance(smap, dict) or not smap:
|
|
||||||
continue
|
|
||||||
for sid, info in smap.items():
|
|
||||||
if isinstance(info, dict):
|
|
||||||
buckets[str(sid)].append((out, info))
|
|
||||||
|
|
||||||
result: list[StageBenchmarkMetrics] = []
|
|
||||||
for sid in sorted(buckets.keys(), key=lambda x: int(x)):
|
|
||||||
rows = buckets[sid]
|
|
||||||
stage_id_int = int(sid)
|
|
||||||
stage_name = str((rows[0][1] or {}).get("stage_name") or f"stage_{stage_id_int}")
|
|
||||||
final_output_type = str((rows[0][1] or {}).get("final_output_type") or "unknown")
|
|
||||||
output_unit_types = [
|
|
||||||
str((info or {}).get("output_unit_type") or "") for _, info in rows if (info or {}).get("output_unit_type")
|
|
||||||
]
|
|
||||||
output_unit_type = output_unit_types[0] if output_unit_types else "other"
|
|
||||||
|
|
||||||
is_text_stage, is_audio_stage, _, _, is_internal_stream_stage = _stage_modality_flags(
|
|
||||||
final_output_type, output_unit_type
|
|
||||||
)
|
|
||||||
total_output = 0
|
|
||||||
|
|
||||||
ttfts: list[float] = []
|
|
||||||
tpots: list[float] = []
|
|
||||||
itls: list[float] = []
|
|
||||||
vllm_ttfts: list[float] = []
|
|
||||||
vllm_tpots: list[float] = []
|
|
||||||
vllm_itls: list[float] = []
|
|
||||||
stage_gen_times_ms = [float((info or {}).get(defs.STAGE_GEN_TIME_MS) or 0.0) for _, info in rows]
|
|
||||||
postprocess_times_ms = [float((info or {}).get(defs.POSTPROCESS_TIME_MS) or 0.0) for _, info in rows]
|
|
||||||
output_unit_count = sum(int((info or {}).get(defs.OUTPUT_UNIT_COUNT) or 0) for _, info in rows)
|
|
||||||
inter_output_latencies_ms: list[float] = []
|
|
||||||
serving_time_to_first_outputs_ms: list[float] = []
|
|
||||||
time_per_output_units_ms: list[float] = []
|
|
||||||
if is_text_stage or is_internal_stream_stage:
|
|
||||||
serving_time_to_first_outputs_ms = [
|
|
||||||
float((info or {}).get(defs.SERVING_TIME_TO_FIRST_OUTPUT_MS) or 0.0)
|
|
||||||
for _, info in rows
|
|
||||||
if (info or {}).get(defs.SERVING_TIME_TO_FIRST_OUTPUT_MS) is not None
|
|
||||||
]
|
|
||||||
time_per_output_units_ms = [
|
|
||||||
float((info or {}).get(defs.TIME_PER_OUTPUT_UNIT_MS) or 0.0)
|
|
||||||
for _, info in rows
|
|
||||||
if int((info or {}).get(defs.OUTPUT_UNIT_COUNT) or 0) > 1
|
|
||||||
and (info or {}).get(defs.TIME_PER_OUTPUT_UNIT_MS) is not None
|
|
||||||
]
|
|
||||||
for _, info in rows:
|
|
||||||
values = (info or {}).get(defs.INTER_OUTPUT_LATENCIES_MS)
|
|
||||||
if isinstance(values, list):
|
|
||||||
inter_output_latencies_ms.extend(float(v or 0.0) for v in values)
|
|
||||||
elif (info or {}).get(defs.INTER_OUTPUT_LATENCY_MS) is not None:
|
|
||||||
inter_output_latencies_ms.append(float((info or {}).get(defs.INTER_OUTPUT_LATENCY_MS) or 0.0))
|
|
||||||
|
|
||||||
if is_text_stage:
|
|
||||||
for _, info in rows:
|
|
||||||
ttft_ms = float((info or {}).get(defs.SERVING_TIME_TO_FIRST_OUTPUT_MS) or 0.0)
|
|
||||||
if ttft_ms > 0:
|
|
||||||
ttfts.append(ttft_ms / 1000.0)
|
|
||||||
tpot_ms = float((info or {}).get(defs.TIME_PER_OUTPUT_UNIT_MS) or 0.0)
|
|
||||||
if int((info or {}).get(defs.OUTPUT_UNIT_COUNT) or 0) > 1 and tpot_ms > 0:
|
|
||||||
tpots.append(tpot_ms / 1000.0)
|
|
||||||
values = (info or {}).get(defs.INTER_OUTPUT_LATENCIES_MS)
|
|
||||||
if isinstance(values, list):
|
|
||||||
itls.extend(float(v or 0.0) / 1000.0 for v in values)
|
|
||||||
elif (info or {}).get(defs.INTER_OUTPUT_LATENCY_MS) is not None:
|
|
||||||
itls.append(float((info or {}).get(defs.INTER_OUTPUT_LATENCY_MS) or 0.0) / 1000.0)
|
|
||||||
vllm_ttft_ms = float((info or {}).get(defs.VLLM_TTFT_MS) or 0.0)
|
|
||||||
if vllm_ttft_ms > 0:
|
|
||||||
vllm_ttfts.append(vllm_ttft_ms / 1000.0)
|
|
||||||
vllm_tpot_ms = float((info or {}).get(defs.VLLM_TPOT_MS) or 0.0)
|
|
||||||
if vllm_tpot_ms > 0:
|
|
||||||
vllm_tpots.append(vllm_tpot_ms / 1000.0)
|
|
||||||
vllm_values = (info or {}).get(defs.VLLM_ITLS_MS)
|
|
||||||
if isinstance(vllm_values, list):
|
|
||||||
vllm_itls.extend(float(v or 0.0) / 1000.0 for v in vllm_values)
|
|
||||||
elif (info or {}).get(defs.VLLM_ITL_MS) is not None:
|
|
||||||
vllm_itls.append(float((info or {}).get(defs.VLLM_ITL_MS) or 0.0) / 1000.0)
|
|
||||||
total_output = sum(int((info or {}).get(defs.NUM_TOKENS_OUT) or 0) for _, info in rows)
|
|
||||||
|
|
||||||
audio_ttfps: list[float] = []
|
|
||||||
audio_rtfs: list[float] = []
|
|
||||||
audio_durations: list[float] = []
|
|
||||||
audio_frames: list[int] = []
|
|
||||||
missing_audio_duration_count = 0
|
|
||||||
missing_audio_rtf_count = 0
|
|
||||||
if is_audio_stage:
|
|
||||||
for _, info in rows:
|
|
||||||
audio_ttfp_ms = float((info or {}).get(defs.SERVING_TIME_TO_FIRST_OUTPUT_MS) or 0.0)
|
|
||||||
if audio_ttfp_ms > 0:
|
|
||||||
audio_ttfps.append(audio_ttfp_ms / 1000.0)
|
|
||||||
|
|
||||||
frame_count = int((info or {}).get(defs.AUDIO_FRAMES) or 0)
|
|
||||||
if frame_count <= 0 and (info or {}).get("output_unit_type") == "audio":
|
|
||||||
frame_count = int((info or {}).get(defs.OUTPUT_UNIT_COUNT) or 0)
|
|
||||||
audio_frames.append(frame_count)
|
|
||||||
|
|
||||||
duration_s = float((info or {}).get(f"{defs.AUDIO_DURATION}_s") or 0.0)
|
|
||||||
if duration_s > 0:
|
|
||||||
audio_durations.append(duration_s)
|
|
||||||
else:
|
|
||||||
missing_audio_duration_count += 1
|
|
||||||
|
|
||||||
rtf = float((info or {}).get(defs.AUDIO_RTF) or 0.0)
|
|
||||||
if rtf > 0:
|
|
||||||
audio_rtfs.append(rtf)
|
|
||||||
else:
|
|
||||||
missing_audio_rtf_count += 1
|
|
||||||
|
|
||||||
result.append(
|
|
||||||
StageBenchmarkMetrics(
|
|
||||||
**{
|
|
||||||
"stage_id": stage_id_int,
|
|
||||||
"stage_name": stage_name,
|
|
||||||
"final_output_type": final_output_type,
|
|
||||||
defs.TOTAL_OUTPUT: total_output,
|
|
||||||
defs.TTFTS: ttfts,
|
|
||||||
defs.TPOTS: tpots,
|
|
||||||
defs.ITLS: itls,
|
|
||||||
defs.VLLM_TTFTS: vllm_ttfts,
|
|
||||||
defs.VLLM_TPOTS: vllm_tpots,
|
|
||||||
defs.VLLM_ITLS: vllm_itls,
|
|
||||||
defs.AUDIO_TTFPS: audio_ttfps,
|
|
||||||
defs.AUDIO_RTFS: audio_rtfs,
|
|
||||||
defs.AUDIO_DURATIONS: audio_durations,
|
|
||||||
defs.AUDIO_FRAMES: audio_frames,
|
|
||||||
defs.MISSING_AUDIO_DURATION_COUNT: missing_audio_duration_count,
|
|
||||||
defs.MISSING_AUDIO_RTF_COUNT: missing_audio_rtf_count,
|
|
||||||
defs.STAGE_GEN_TIMES_MS: stage_gen_times_ms,
|
|
||||||
defs.POSTPROCESS_TIMES_MS: postprocess_times_ms,
|
|
||||||
"output_unit_type": output_unit_type,
|
|
||||||
defs.OUTPUT_UNIT_COUNT: output_unit_count,
|
|
||||||
defs.SERVING_TIME_TO_FIRST_OUTPUTS_MS: serving_time_to_first_outputs_ms,
|
|
||||||
defs.TIME_PER_OUTPUT_UNITS_MS: time_per_output_units_ms,
|
|
||||||
defs.INTER_OUTPUT_LATENCIES_MS: inter_output_latencies_ms,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def calculate_metrics(
|
def calculate_metrics(
|
||||||
input_requests: list[SampleRequest],
|
input_requests: list[SampleRequest],
|
||||||
outputs: list[RequestFuncOutput],
|
outputs: list[RequestFuncOutput],
|
||||||
@@ -715,7 +160,6 @@ def calculate_metrics(
|
|||||||
max_concurrency,
|
max_concurrency,
|
||||||
request_rate,
|
request_rate,
|
||||||
benchmark_duration,
|
benchmark_duration,
|
||||||
print_stage: bool = False,
|
|
||||||
) -> tuple[BenchmarkMetrics, list[int]]:
|
) -> tuple[BenchmarkMetrics, list[int]]:
|
||||||
"""Calculate the metrics for the benchmark.
|
"""Calculate the metrics for the benchmark.
|
||||||
|
|
||||||
@@ -743,10 +187,6 @@ def calculate_metrics(
|
|||||||
audio_rtfs: list[float] = []
|
audio_rtfs: list[float] = []
|
||||||
audio_duration: list[float] = []
|
audio_duration: list[float] = []
|
||||||
audio_frames: list[int] = []
|
audio_frames: list[int] = []
|
||||||
image_generation_times_ms: list[float] = []
|
|
||||||
denoise_step_latencies_ms: list[float] = []
|
|
||||||
total_images = 0
|
|
||||||
total_image_pixels = 0
|
|
||||||
audio_underruns: list[float] = []
|
audio_underruns: list[float] = []
|
||||||
audio_continuity_ok: list[bool] = []
|
audio_continuity_ok: list[bool] = []
|
||||||
input_audio_duration = 0.0
|
input_audio_duration = 0.0
|
||||||
@@ -782,15 +222,6 @@ def calculate_metrics(
|
|||||||
audio_rtfs.append(getattr(outputs[i], defs.AUDIO_RTF, 0.0))
|
audio_rtfs.append(getattr(outputs[i], defs.AUDIO_RTF, 0.0))
|
||||||
audio_duration.append(getattr(outputs[i], defs.AUDIO_DURATION, 0.0))
|
audio_duration.append(getattr(outputs[i], defs.AUDIO_DURATION, 0.0))
|
||||||
audio_frames.append(getattr(outputs[i], defs.AUDIO_FRAMES, 0.0))
|
audio_frames.append(getattr(outputs[i], defs.AUDIO_FRAMES, 0.0))
|
||||||
image_count = int(getattr(outputs[i], defs.IMAGE_COUNT, 0) or 0)
|
|
||||||
total_images += image_count
|
|
||||||
image_generation_time_ms = float(getattr(outputs[i], defs.IMAGE_GENERATION_TIME_MS, 0.0) or 0.0)
|
|
||||||
if image_generation_time_ms > 0:
|
|
||||||
image_generation_times_ms.append(image_generation_time_ms)
|
|
||||||
total_image_pixels += int(getattr(outputs[i], defs.IMAGE_PIXELS, 0) or 0)
|
|
||||||
denoise_step_latency_ms = float(getattr(outputs[i], defs.DENOISE_STEP_LATENCY_MS, 0.0) or 0.0)
|
|
||||||
if denoise_step_latency_ms > 0:
|
|
||||||
denoise_step_latencies_ms.append(denoise_step_latency_ms)
|
|
||||||
audio_underruns.append(getattr(outputs[i], f"{defs.AUDIO_UNDERRUN}_s", 0.0))
|
audio_underruns.append(getattr(outputs[i], f"{defs.AUDIO_UNDERRUN}_s", 0.0))
|
||||||
audio_continuity_ok.append(bool(getattr(outputs[i], defs.AUDIO_CONTINUITY_OK, True)))
|
audio_continuity_ok.append(bool(getattr(outputs[i], defs.AUDIO_CONTINUITY_OK, True)))
|
||||||
e2els.append(outputs[i].latency)
|
e2els.append(outputs[i].latency)
|
||||||
@@ -905,6 +336,26 @@ def calculate_metrics(
|
|||||||
std_ttft_ms=np.std(ttfts or 0) * 1000,
|
std_ttft_ms=np.std(ttfts or 0) * 1000,
|
||||||
median_ttft_ms=np.median(ttfts or 0) * 1000,
|
median_ttft_ms=np.median(ttfts or 0) * 1000,
|
||||||
percentiles_ttft_ms=[(p, np.percentile(ttfts or 0, p) * 1000) for p in selected_percentiles],
|
percentiles_ttft_ms=[(p, np.percentile(ttfts or 0, p) * 1000) for p in selected_percentiles],
|
||||||
|
mean_audio_ttfp_ms=np.mean(audio_ttfps or 0) * 1000,
|
||||||
|
std_audio_ttfp_ms=np.std(audio_ttfps or 0) * 1000,
|
||||||
|
median_audio_ttfp_ms=np.median(audio_ttfps or 0) * 1000,
|
||||||
|
percentiles_audio_ttfp_ms=[(p, np.percentile(audio_ttfps or 0, p) * 1000) for p in selected_percentiles],
|
||||||
|
mean_audio_duration_s=np.mean(audio_duration or 0),
|
||||||
|
std_audio_duration_s=np.std(audio_duration or 0),
|
||||||
|
median_audio_duration_s=np.median(audio_duration or 0),
|
||||||
|
percentiles_audio_duration_s=[(p, np.percentile(audio_duration or 0, p)) for p in selected_percentiles],
|
||||||
|
total_audio_duration_s=sum(audio_duration),
|
||||||
|
total_audio_frames=sum(audio_frames),
|
||||||
|
audio_throughput=sum(audio_duration) / dur_s,
|
||||||
|
mean_audio_rtf=np.mean(audio_rtfs or 0),
|
||||||
|
std_audio_rtf=np.std(audio_rtfs or 0),
|
||||||
|
median_audio_rtf=np.median(audio_rtfs or 0),
|
||||||
|
percentiles_audio_rtf=[(p, np.percentile(audio_rtfs or 0, p)) for p in selected_percentiles],
|
||||||
|
mean_audio_underrun_s=np.mean(audio_underruns or 0),
|
||||||
|
std_audio_underrun_s=np.std(audio_underruns or 0),
|
||||||
|
median_audio_underrun_s=np.median(audio_underruns or 0),
|
||||||
|
percentiles_audio_underrun_s=[(p, np.percentile(audio_underruns or 0, p)) for p in selected_percentiles],
|
||||||
|
audio_continuity_ok_rate=(sum(audio_continuity_ok) / len(audio_continuity_ok)) if audio_continuity_ok else 1.0,
|
||||||
mean_tpot_ms=np.mean(tpots or 0) * 1000,
|
mean_tpot_ms=np.mean(tpots or 0) * 1000,
|
||||||
std_tpot_ms=np.std(tpots or 0) * 1000,
|
std_tpot_ms=np.std(tpots or 0) * 1000,
|
||||||
median_tpot_ms=np.median(tpots or 0) * 1000,
|
median_tpot_ms=np.median(tpots or 0) * 1000,
|
||||||
@@ -920,46 +371,6 @@ def calculate_metrics(
|
|||||||
max_output_tokens_per_s=max_output_tokens_per_s,
|
max_output_tokens_per_s=max_output_tokens_per_s,
|
||||||
max_concurrent_requests=max_concurrent_requests,
|
max_concurrent_requests=max_concurrent_requests,
|
||||||
rtfx=input_audio_duration / dur_s,
|
rtfx=input_audio_duration / dur_s,
|
||||||
**{
|
|
||||||
defs.MEAN_AUDIO_TTFP_MS: np.mean(audio_ttfps or 0) * 1000,
|
|
||||||
defs.STD_AUDIO_TTFP_MS: np.std(audio_ttfps or 0) * 1000,
|
|
||||||
defs.MEDIAN_AUDIO_TTFP_MS: np.median(audio_ttfps or 0) * 1000,
|
|
||||||
defs.PERCENTILES_AUDIO_TTFP_MS: [
|
|
||||||
(p, np.percentile(audio_ttfps or 0, p) * 1000) for p in selected_percentiles
|
|
||||||
],
|
|
||||||
defs.MEAN_AUDIO_DURATION_S: np.mean(audio_duration or 0),
|
|
||||||
defs.STD_AUDIO_DURATION_S: np.std(audio_duration or 0),
|
|
||||||
defs.MEDIAN_AUDIO_DURATION_S: np.median(audio_duration or 0),
|
|
||||||
defs.PERCENTILES_AUDIO_DURATION_S: [
|
|
||||||
(p, np.percentile(audio_duration or 0, p)) for p in selected_percentiles
|
|
||||||
],
|
|
||||||
defs.TOTAL_AUDIO_DURATION_S: sum(audio_duration),
|
|
||||||
defs.TOTAL_AUDIO_FRAMES: sum(audio_frames),
|
|
||||||
defs.AUDIO_THROUGHPUT: sum(audio_duration) / dur_s,
|
|
||||||
defs.MEAN_AUDIO_RTF: np.mean(audio_rtfs or 0),
|
|
||||||
defs.STD_AUDIO_RTF: np.std(audio_rtfs or 0),
|
|
||||||
defs.MEDIAN_AUDIO_RTF: np.median(audio_rtfs or 0),
|
|
||||||
defs.PERCENTILES_AUDIO_RTF: [(p, np.percentile(audio_rtfs or 0, p)) for p in selected_percentiles],
|
|
||||||
defs.TOTAL_IMAGES: total_images,
|
|
||||||
defs.IMAGE_THROUGHPUT: total_images / dur_s,
|
|
||||||
defs.AVERAGE_PIXELS_PER_IMAGE: (total_image_pixels / total_images) if total_images > 0 else 0.0,
|
|
||||||
defs.MEAN_DENOISE_STEP_LATENCY_MS: np.mean(denoise_step_latencies_ms or 0),
|
|
||||||
defs.MEAN_IMAGE_GENERATION_MS: np.mean(image_generation_times_ms or 0),
|
|
||||||
defs.STD_IMAGE_GENERATION_MS: np.std(image_generation_times_ms or 0),
|
|
||||||
defs.MEDIAN_IMAGE_GENERATION_MS: np.median(image_generation_times_ms or 0),
|
|
||||||
defs.PERCENTILES_IMAGE_GENERATION_MS: [
|
|
||||||
(p, np.percentile(image_generation_times_ms or 0, p)) for p in selected_percentiles
|
|
||||||
],
|
|
||||||
defs.MEAN_AUDIO_UNDERRUN_S: np.mean(audio_underruns or 0),
|
|
||||||
defs.STD_AUDIO_UNDERRUN_S: np.std(audio_underruns or 0),
|
|
||||||
defs.MEDIAN_AUDIO_UNDERRUN_S: np.median(audio_underruns or 0),
|
|
||||||
defs.PERCENTILES_AUDIO_UNDERRUN_S: [
|
|
||||||
(p, np.percentile(audio_underruns or 0, p)) for p in selected_percentiles
|
|
||||||
],
|
|
||||||
defs.AUDIO_CONTINUITY_OK_RATE: (
|
|
||||||
(sum(audio_continuity_ok) / len(audio_continuity_ok)) if audio_continuity_ok else 1.0
|
|
||||||
),
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
print_metrics(
|
print_metrics(
|
||||||
task_type,
|
task_type,
|
||||||
@@ -969,8 +380,5 @@ def calculate_metrics(
|
|||||||
benchmark_duration,
|
benchmark_duration,
|
||||||
goodput_config_dict,
|
goodput_config_dict,
|
||||||
metrics,
|
metrics,
|
||||||
outputs=outputs,
|
|
||||||
selected_percentiles=selected_percentiles,
|
|
||||||
print_stage=print_stage,
|
|
||||||
)
|
)
|
||||||
return metrics, actual_output_lens
|
return metrics, actual_output_lens
|
||||||
@@ -2,7 +2,6 @@ import asyncio
|
|||||||
import contextlib
|
import contextlib
|
||||||
import io
|
import io
|
||||||
import json
|
import json
|
||||||
import mimetypes
|
|
||||||
import os
|
import os
|
||||||
import random
|
import random
|
||||||
import ssl
|
import ssl
|
||||||
@@ -13,7 +12,7 @@ import wave
|
|||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Any, Literal
|
from typing import Literal
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
import pybase64 as base64
|
import pybase64 as base64
|
||||||
@@ -49,41 +48,9 @@ from vllm_omni.benchmarks.data_modules.seed_tts_dataset import (
|
|||||||
)
|
)
|
||||||
from vllm_omni.benchmarks.data_modules.sound_effect_dataset import SoundEffectDataset
|
from vllm_omni.benchmarks.data_modules.sound_effect_dataset import SoundEffectDataset
|
||||||
from vllm_omni.benchmarks.data_modules.ttsd_dataset import TTSDDataset
|
from vllm_omni.benchmarks.data_modules.ttsd_dataset import TTSDDataset
|
||||||
from vllm_omni.metrics import definitions as defs
|
|
||||||
|
|
||||||
_AUDIO_CONTINUITY_THRESHOLD_ENV = "VLLM_OMNI_BENCH_AUDIO_CONTINUITY_THRESHOLD_S"
|
_AUDIO_CONTINUITY_THRESHOLD_ENV = "VLLM_OMNI_BENCH_AUDIO_CONTINUITY_THRESHOLD_S"
|
||||||
RETURN_STAGE_METRICS_FIELD = "return_stage_metrics"
|
_DEFAULT_AUDIO_CONTINUITY_THRESHOLD_S = 0.1
|
||||||
_IMAGE_STAGE_METRICS_BACKENDS = frozenset({"openai-image-edits-omni"})
|
|
||||||
_PRINT_STAGE = False
|
|
||||||
|
|
||||||
|
|
||||||
def maybe_enable_stage_metrics(extra_body: dict[str, Any] | None, *, enabled: bool) -> dict[str, Any] | None:
|
|
||||||
"""Return extra_body with stage-metric opt-in when benchmark metrics need it."""
|
|
||||||
if not enabled:
|
|
||||||
return extra_body
|
|
||||||
body = dict(extra_body or {})
|
|
||||||
body.setdefault(RETURN_STAGE_METRICS_FIELD, True)
|
|
||||||
return body
|
|
||||||
|
|
||||||
|
|
||||||
def should_request_stage_metrics(args: Any) -> bool:
|
|
||||||
"""Whether this benchmark run needs server-side stage metrics in responses."""
|
|
||||||
if getattr(args, "print_stage", False):
|
|
||||||
return True
|
|
||||||
|
|
||||||
backend = getattr(args, "backend", None)
|
|
||||||
if backend in _IMAGE_STAGE_METRICS_BACKENDS:
|
|
||||||
return True
|
|
||||||
|
|
||||||
extra_body = getattr(args, "extra_body", None) or {}
|
|
||||||
modalities = extra_body.get("modalities") if isinstance(extra_body, dict) else None
|
|
||||||
return backend == "openai-chat-omni" and "image" in (modalities or [])
|
|
||||||
|
|
||||||
|
|
||||||
def set_print_stage(enabled: bool) -> None:
|
|
||||||
"""Set whether this benchmark run prints the stage benchmark section."""
|
|
||||||
global _PRINT_STAGE
|
|
||||||
_PRINT_STAGE = bool(enabled)
|
|
||||||
|
|
||||||
|
|
||||||
def _audio_continuity_threshold_s() -> float:
|
def _audio_continuity_threshold_s() -> float:
|
||||||
@@ -95,7 +62,7 @@ def _audio_continuity_threshold_s() -> float:
|
|||||||
"""
|
"""
|
||||||
raw = os.environ.get(_AUDIO_CONTINUITY_THRESHOLD_ENV)
|
raw = os.environ.get(_AUDIO_CONTINUITY_THRESHOLD_ENV)
|
||||||
if not raw:
|
if not raw:
|
||||||
return defs.AUDIO_CONTINUITY_DEFAULT_THRESHOLD_S
|
return _DEFAULT_AUDIO_CONTINUITY_THRESHOLD_S
|
||||||
try:
|
try:
|
||||||
value = float(raw)
|
value = float(raw)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
@@ -103,9 +70,9 @@ def _audio_continuity_threshold_s() -> float:
|
|||||||
"Invalid %s=%r; using default %.3fs",
|
"Invalid %s=%r; using default %.3fs",
|
||||||
_AUDIO_CONTINUITY_THRESHOLD_ENV,
|
_AUDIO_CONTINUITY_THRESHOLD_ENV,
|
||||||
raw,
|
raw,
|
||||||
defs.AUDIO_CONTINUITY_DEFAULT_THRESHOLD_S,
|
_DEFAULT_AUDIO_CONTINUITY_THRESHOLD_S,
|
||||||
)
|
)
|
||||||
return defs.AUDIO_CONTINUITY_DEFAULT_THRESHOLD_S
|
return _DEFAULT_AUDIO_CONTINUITY_THRESHOLD_S
|
||||||
return max(value, 0.0)
|
return max(value, 0.0)
|
||||||
|
|
||||||
|
|
||||||
@@ -385,10 +352,6 @@ class MixRequestFuncOutput(RequestFuncOutput):
|
|||||||
audio_duration: float = 0.0
|
audio_duration: float = 0.0
|
||||||
audio_frames: int = 0
|
audio_frames: int = 0
|
||||||
audio_rtf: float = 0.0
|
audio_rtf: float = 0.0
|
||||||
image_count: int = 0
|
|
||||||
image_generation_time_ms: float = 0.0
|
|
||||||
image_pixels: int = 0
|
|
||||||
denoise_step_latency_ms: float = 0.0
|
|
||||||
text_latency: float = 0.0
|
text_latency: float = 0.0
|
||||||
#: Worst-case streaming-audio underrun (wall-clock seconds the player
|
#: Worst-case streaming-audio underrun (wall-clock seconds the player
|
||||||
#: would have been starved). Populated by the audio-speech backend; ``0.0``
|
#: would have been starved). Populated by the audio-speech backend; ``0.0``
|
||||||
@@ -403,200 +366,6 @@ class MixRequestFuncOutput(RequestFuncOutput):
|
|||||||
#: Raw PCM s16le mono at 24 kHz for Seed-TTS WER: from ``/v1/audio/speech`` stream or
|
#: Raw PCM s16le mono at 24 kHz for Seed-TTS WER: from ``/v1/audio/speech`` stream or
|
||||||
#: resampled export after ``openai-chat-omni`` audio deltas.
|
#: resampled export after ``openai-chat-omni`` audio deltas.
|
||||||
tts_output_pcm_bytes: bytes | None = None
|
tts_output_pcm_bytes: bytes | None = None
|
||||||
#: Per-stage snapshot from orchestrator ``metrics["stage_metrics"]`` (merged across SSE chunks).
|
|
||||||
stage_metrics: dict[str, dict] | None = None
|
|
||||||
stage_id: int | None = None
|
|
||||||
final_output_type: str | None = None
|
|
||||||
|
|
||||||
|
|
||||||
_IMAGE_EDITS_EXTRA_BODY_FORM_FIELDS = (
|
|
||||||
"negative_prompt",
|
|
||||||
"num_inference_steps",
|
|
||||||
"guidance_scale",
|
|
||||||
"strength",
|
|
||||||
"true_cfg_scale",
|
|
||||||
"seed",
|
|
||||||
"generator_device",
|
|
||||||
"lora",
|
|
||||||
"layers",
|
|
||||||
"resolution",
|
|
||||||
"bot_task",
|
|
||||||
"sys_type",
|
|
||||||
"system_prompt",
|
|
||||||
RETURN_STAGE_METRICS_FIELD,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _guess_mime_type(path: str) -> str:
|
|
||||||
mime, _ = mimetypes.guess_type(path)
|
|
||||||
return mime or "application/octet-stream"
|
|
||||||
|
|
||||||
|
|
||||||
def _iter_image_edit_inputs(value: Any) -> Iterable[Any]:
|
|
||||||
"""Yield image references from benchmark multimodal content."""
|
|
||||||
if value is None:
|
|
||||||
return
|
|
||||||
if isinstance(value, list):
|
|
||||||
for item in value:
|
|
||||||
yield from _iter_image_edit_inputs(item)
|
|
||||||
return
|
|
||||||
if not isinstance(value, dict):
|
|
||||||
yield value
|
|
||||||
return
|
|
||||||
|
|
||||||
content_type = value.get("type")
|
|
||||||
if content_type == "image_url":
|
|
||||||
image_url = value.get("image_url")
|
|
||||||
if isinstance(image_url, dict):
|
|
||||||
url = image_url.get("url")
|
|
||||||
if url:
|
|
||||||
yield url
|
|
||||||
elif image_url:
|
|
||||||
yield image_url
|
|
||||||
return
|
|
||||||
|
|
||||||
for key in ("image", "images"):
|
|
||||||
if key in value:
|
|
||||||
yield from _iter_image_edit_inputs(value[key])
|
|
||||||
|
|
||||||
|
|
||||||
def _add_image_edit_input_to_form(form: aiohttp.FormData, image_input: Any) -> None:
|
|
||||||
if isinstance(image_input, dict) and "bytes" in image_input:
|
|
||||||
form.add_field(
|
|
||||||
"image",
|
|
||||||
image_input["bytes"],
|
|
||||||
filename="benchmark.png",
|
|
||||||
content_type="image/png",
|
|
||||||
)
|
|
||||||
return
|
|
||||||
|
|
||||||
if isinstance(image_input, str):
|
|
||||||
if image_input.startswith(("data:image", "http://", "https://")):
|
|
||||||
form.add_field("url", image_input)
|
|
||||||
return
|
|
||||||
local_path = image_input.removeprefix("file://")
|
|
||||||
if os.path.exists(local_path):
|
|
||||||
with open(local_path, "rb") as f:
|
|
||||||
image_bytes = f.read()
|
|
||||||
form.add_field(
|
|
||||||
"image",
|
|
||||||
image_bytes,
|
|
||||||
filename=os.path.basename(local_path),
|
|
||||||
content_type=_guess_mime_type(local_path),
|
|
||||||
)
|
|
||||||
return
|
|
||||||
|
|
||||||
raise ValueError(f"Unsupported image edit input: {type(image_input).__name__}")
|
|
||||||
|
|
||||||
|
|
||||||
def _add_image_edit_extra_body_to_form(form: aiohttp.FormData, extra_body: dict[str, Any]) -> None:
|
|
||||||
for key in _IMAGE_EDITS_EXTRA_BODY_FORM_FIELDS:
|
|
||||||
value = extra_body.get(key)
|
|
||||||
if value is None:
|
|
||||||
continue
|
|
||||||
if isinstance(value, (dict, list)):
|
|
||||||
form.add_field(key, json.dumps(value))
|
|
||||||
else:
|
|
||||||
form.add_field(key, str(value))
|
|
||||||
|
|
||||||
|
|
||||||
def _coerce_positive_int(value: Any) -> int | None:
|
|
||||||
try:
|
|
||||||
coerced = int(value)
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
return None
|
|
||||||
return coerced if coerced > 0 else None
|
|
||||||
|
|
||||||
|
|
||||||
def _extract_output_tokens_from_metrics(metrics: dict[str, Any]) -> int | None:
|
|
||||||
top_level_tokens = _coerce_positive_int(metrics.get(defs.NUM_TOKENS_OUT))
|
|
||||||
if top_level_tokens is not None:
|
|
||||||
return top_level_tokens
|
|
||||||
|
|
||||||
stage_snapshot = metrics.get("stage_metrics")
|
|
||||||
if not isinstance(stage_snapshot, dict):
|
|
||||||
return None
|
|
||||||
|
|
||||||
fallback_tokens: list[int] = []
|
|
||||||
for info in stage_snapshot.values():
|
|
||||||
if not isinstance(info, dict):
|
|
||||||
continue
|
|
||||||
num_tokens_out = _coerce_positive_int(info.get(defs.NUM_TOKENS_OUT))
|
|
||||||
if num_tokens_out is None:
|
|
||||||
continue
|
|
||||||
if info.get("final_output_type") == "text" or info.get("output_unit_type") == "token":
|
|
||||||
return num_tokens_out
|
|
||||||
fallback_tokens.append(num_tokens_out)
|
|
||||||
return max(fallback_tokens, default=None)
|
|
||||||
|
|
||||||
|
|
||||||
def _update_output_stage_metrics_from_payload(
|
|
||||||
output: MixRequestFuncOutput,
|
|
||||||
data: dict[str, Any],
|
|
||||||
*,
|
|
||||||
update_output_tokens: bool = True,
|
|
||||||
) -> None:
|
|
||||||
metrics = data.get("metrics")
|
|
||||||
if not isinstance(metrics, dict):
|
|
||||||
return
|
|
||||||
if update_output_tokens:
|
|
||||||
if (num_tokens_out := _extract_output_tokens_from_metrics(metrics)) is not None:
|
|
||||||
output.output_tokens = num_tokens_out
|
|
||||||
if isinstance(sid := metrics.get("stage_id"), int):
|
|
||||||
output.stage_id = sid
|
|
||||||
if isinstance(final_output_type := metrics.get("final_output_type"), str):
|
|
||||||
output.final_output_type = final_output_type
|
|
||||||
stage_snapshot = metrics.get("stage_metrics")
|
|
||||||
if isinstance(stage_snapshot, dict):
|
|
||||||
if output.stage_metrics is None:
|
|
||||||
output.stage_metrics = {}
|
|
||||||
output.stage_metrics.update(stage_snapshot)
|
|
||||||
|
|
||||||
|
|
||||||
def _image_metrics_from_stage_metrics(metrics: dict[str, Any] | None) -> tuple[int, float, int, float]:
|
|
||||||
if not isinstance(metrics, dict):
|
|
||||||
return 0, 0.0, 0, 0.0
|
|
||||||
stage_snapshot = metrics.get("stage_metrics")
|
|
||||||
if not isinstance(stage_snapshot, dict):
|
|
||||||
return 0, 0.0, 0, 0.0
|
|
||||||
image_count = 0
|
|
||||||
image_generation_ms = 0.0
|
|
||||||
image_pixels = 0
|
|
||||||
denoise_step_latency_ms = 0.0
|
|
||||||
for info in stage_snapshot.values():
|
|
||||||
if not isinstance(info, dict):
|
|
||||||
continue
|
|
||||||
final_output_type = info.get("final_output_type")
|
|
||||||
output_unit_type = info.get("output_unit_type")
|
|
||||||
if final_output_type not in {"image", "images"} and output_unit_type != "image":
|
|
||||||
continue
|
|
||||||
image_count += int(info.get(defs.OUTPUT_UNIT_COUNT) or 0)
|
|
||||||
image_generation_ms += float(info.get(defs.STAGE_GEN_TIME_MS) or 0.0)
|
|
||||||
image_pixels += int(info.get(defs.IMAGE_PIXELS) or 0)
|
|
||||||
denoise_step_latency_ms = max(
|
|
||||||
denoise_step_latency_ms,
|
|
||||||
float(info.get(defs.DENOISE_STEP_LATENCY_MS) or 0.0),
|
|
||||||
)
|
|
||||||
return image_count, image_generation_ms, image_pixels, denoise_step_latency_ms
|
|
||||||
|
|
||||||
|
|
||||||
def _image_generation_ms_from_content(content: Any) -> float:
|
|
||||||
if not isinstance(content, list):
|
|
||||||
return 0.0
|
|
||||||
for item in content:
|
|
||||||
if not isinstance(item, dict):
|
|
||||||
continue
|
|
||||||
stage_durations = item.get("stage_durations")
|
|
||||||
if not isinstance(stage_durations, dict):
|
|
||||||
continue
|
|
||||||
gen_values = [
|
|
||||||
float(value)
|
|
||||||
for key, value in stage_durations.items()
|
|
||||||
if str(key).endswith("_gen_ms") and isinstance(value, (int, float))
|
|
||||||
]
|
|
||||||
if gen_values:
|
|
||||||
return max(gen_values)
|
|
||||||
return 0.0
|
|
||||||
|
|
||||||
|
|
||||||
async def async_request_openai_chat_omni_completions(
|
async def async_request_openai_chat_omni_completions(
|
||||||
@@ -679,13 +448,6 @@ async def async_request_openai_chat_omni_completions(
|
|||||||
output.output_tokens = 0
|
output.output_tokens = 0
|
||||||
output.error = ""
|
output.error = ""
|
||||||
output.success = False
|
output.success = False
|
||||||
output.stage_metrics = {}
|
|
||||||
output.stage_id = None
|
|
||||||
output.final_output_type = None
|
|
||||||
output.image_count = 0
|
|
||||||
output.image_generation_time_ms = 0.0
|
|
||||||
output.image_pixels = 0
|
|
||||||
output.denoise_step_latency_ms = 0.0
|
|
||||||
try:
|
try:
|
||||||
async with session.post(url=api_url, json=payload, headers=headers) as response:
|
async with session.post(url=api_url, json=payload, headers=headers) as response:
|
||||||
if response.status == 200:
|
if response.status == 200:
|
||||||
@@ -757,27 +519,9 @@ async def async_request_openai_chat_omni_completions(
|
|||||||
logger.warning("Failed to parse wav audio chunk: %s", ex)
|
logger.warning("Failed to parse wav audio chunk: %s", ex)
|
||||||
else:
|
else:
|
||||||
audio_bytes_buffer.extend(audio_bytes)
|
audio_bytes_buffer.extend(audio_bytes)
|
||||||
elif modality == "image":
|
|
||||||
output.image_count += 1
|
|
||||||
content_image_ms = _image_generation_ms_from_content(content)
|
|
||||||
if content_image_ms > 0:
|
|
||||||
output.image_generation_time_ms += content_image_ms
|
|
||||||
|
|
||||||
_update_output_stage_metrics_from_payload(output, data)
|
if metrics := data.get("metrics"):
|
||||||
(
|
output.output_tokens = metrics.get("num_tokens_out", 0)
|
||||||
metrics_image_count,
|
|
||||||
metrics_image_ms,
|
|
||||||
metrics_image_pixels,
|
|
||||||
metrics_denoise_step_ms,
|
|
||||||
) = _image_metrics_from_stage_metrics(data.get("metrics"))
|
|
||||||
if metrics_image_count > output.image_count:
|
|
||||||
output.image_count = metrics_image_count
|
|
||||||
if metrics_image_ms > output.image_generation_time_ms:
|
|
||||||
output.image_generation_time_ms = metrics_image_ms
|
|
||||||
if metrics_image_pixels > output.image_pixels:
|
|
||||||
output.image_pixels = metrics_image_pixels
|
|
||||||
if metrics_denoise_step_ms > output.denoise_step_latency_ms:
|
|
||||||
output.denoise_step_latency_ms = metrics_denoise_step_ms
|
|
||||||
|
|
||||||
if usage := data.get("usage"):
|
if usage := data.get("usage"):
|
||||||
if (pt := usage.get("prompt_tokens")) is not None:
|
if (pt := usage.get("prompt_tokens")) is not None:
|
||||||
@@ -866,138 +610,6 @@ async def async_request_openai_chat_omni_completions(
|
|||||||
return output
|
return output
|
||||||
|
|
||||||
|
|
||||||
async def async_request_openai_image_edits_omni(
|
|
||||||
request_func_input: RequestFuncInput,
|
|
||||||
session: aiohttp.ClientSession,
|
|
||||||
pbar: tqdm | None = None,
|
|
||||||
) -> MixRequestFuncOutput:
|
|
||||||
"""Streaming request to /v1/images/edits for multi-stage image-edit benchmarks."""
|
|
||||||
api_url = request_func_input.api_url
|
|
||||||
_validate_api_url(api_url, "OpenAI Image Edits API", "images/edits")
|
|
||||||
|
|
||||||
extra_body = dict(request_func_input.extra_body or {})
|
|
||||||
model = request_func_input.model_name if request_func_input.model_name else request_func_input.model
|
|
||||||
output = MixRequestFuncOutput()
|
|
||||||
output.prompt_len = request_func_input.prompt_len
|
|
||||||
output.itl = []
|
|
||||||
output.stage_metrics = {}
|
|
||||||
output.output_tokens = 0
|
|
||||||
output.image_count = 0
|
|
||||||
output.image_generation_time_ms = 0.0
|
|
||||||
output.image_pixels = 0
|
|
||||||
output.denoise_step_latency_ms = 0.0
|
|
||||||
|
|
||||||
form = aiohttp.FormData()
|
|
||||||
form.add_field("model", model)
|
|
||||||
form.add_field("prompt", request_func_input.prompt)
|
|
||||||
form.add_field("response_format", "b64_json")
|
|
||||||
form.add_field("output_format", str(extra_body.get("output_format", "png")))
|
|
||||||
form.add_field("stream", "true")
|
|
||||||
|
|
||||||
size = extra_body.get("size")
|
|
||||||
if size is None:
|
|
||||||
width, height = extra_body.get("width"), extra_body.get("height")
|
|
||||||
size = f"{width}x{height}" if width is not None and height is not None else "auto"
|
|
||||||
form.add_field("size", str(size))
|
|
||||||
|
|
||||||
_add_image_edit_extra_body_to_form(form, extra_body)
|
|
||||||
|
|
||||||
try:
|
|
||||||
image_inputs = list(_iter_image_edit_inputs(request_func_input.multi_modal_content))
|
|
||||||
if not image_inputs:
|
|
||||||
raise ValueError(
|
|
||||||
"openai-image-edits-omni requires image multimodal content. "
|
|
||||||
"For synthetic inputs, use --dataset-name random-mm with an image bucket."
|
|
||||||
)
|
|
||||||
for image_input in image_inputs:
|
|
||||||
_add_image_edit_input_to_form(form, image_input)
|
|
||||||
except Exception:
|
|
||||||
output.success = False
|
|
||||||
output.error = traceback.format_exc()
|
|
||||||
if pbar:
|
|
||||||
pbar.update(1)
|
|
||||||
return output
|
|
||||||
|
|
||||||
headers = {
|
|
||||||
"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}",
|
|
||||||
}
|
|
||||||
_update_headers_common(headers, request_func_input)
|
|
||||||
|
|
||||||
st = time.perf_counter()
|
|
||||||
output.start_time = st
|
|
||||||
timestamp = st
|
|
||||||
most_recent_text_timestamp = st
|
|
||||||
generated_text = ""
|
|
||||||
try:
|
|
||||||
async with session.post(url=api_url, data=form, headers=headers) as response:
|
|
||||||
if response.status == 200:
|
|
||||||
handler = StreamedResponseHandler()
|
|
||||||
async for chunk_bytes in response.content.iter_any():
|
|
||||||
if not chunk_bytes:
|
|
||||||
continue
|
|
||||||
for message in handler.add_chunk(chunk_bytes):
|
|
||||||
if type(message) is bytes:
|
|
||||||
message = message.decode("utf-8")
|
|
||||||
if message.startswith(":"):
|
|
||||||
continue
|
|
||||||
chunk = message.removeprefix("data: ")
|
|
||||||
if chunk == "[DONE]":
|
|
||||||
continue
|
|
||||||
|
|
||||||
timestamp = time.perf_counter()
|
|
||||||
data = json.loads(chunk)
|
|
||||||
_update_output_stage_metrics_from_payload(
|
|
||||||
output,
|
|
||||||
data,
|
|
||||||
update_output_tokens=(data.get("type") == "ar_delta"),
|
|
||||||
)
|
|
||||||
|
|
||||||
chunk_type = data.get("type")
|
|
||||||
if chunk_type == "ar_delta":
|
|
||||||
if output.ttft == 0.0:
|
|
||||||
output.ttft = timestamp - st
|
|
||||||
else:
|
|
||||||
output.itl.append(timestamp - most_recent_text_timestamp)
|
|
||||||
delta = data.get("delta") or ""
|
|
||||||
generated_text += delta
|
|
||||||
most_recent_text_timestamp = timestamp
|
|
||||||
output.text_latency = timestamp - st
|
|
||||||
elif chunk_type == "image":
|
|
||||||
image_data = data.get("data")
|
|
||||||
output.image_count += len(image_data) if isinstance(image_data, list) else 1
|
|
||||||
content_image_ms = _image_generation_ms_from_content(data.get("data"))
|
|
||||||
if content_image_ms > 0:
|
|
||||||
output.image_generation_time_ms += content_image_ms
|
|
||||||
(
|
|
||||||
metrics_image_count,
|
|
||||||
metrics_image_ms,
|
|
||||||
metrics_image_pixels,
|
|
||||||
metrics_denoise_step_ms,
|
|
||||||
) = _image_metrics_from_stage_metrics(data.get("metrics"))
|
|
||||||
if metrics_image_count > output.image_count:
|
|
||||||
output.image_count = metrics_image_count
|
|
||||||
if metrics_image_ms > output.image_generation_time_ms:
|
|
||||||
output.image_generation_time_ms = metrics_image_ms
|
|
||||||
if metrics_image_pixels > output.image_pixels:
|
|
||||||
output.image_pixels = metrics_image_pixels
|
|
||||||
if metrics_denoise_step_ms > output.denoise_step_latency_ms:
|
|
||||||
output.denoise_step_latency_ms = metrics_denoise_step_ms
|
|
||||||
output.latency = timestamp - st
|
|
||||||
output.generated_text = generated_text
|
|
||||||
output.success = True
|
|
||||||
else:
|
|
||||||
output.error = f"HTTP {response.status}: {await response.text()}"
|
|
||||||
output.success = False
|
|
||||||
except Exception:
|
|
||||||
output.success = False
|
|
||||||
output.error = traceback.format_exc()
|
|
||||||
logger.error(f"ERROR: send image edit request failed, reason is: {output.error}")
|
|
||||||
|
|
||||||
if pbar:
|
|
||||||
pbar.update(1)
|
|
||||||
return output
|
|
||||||
|
|
||||||
|
|
||||||
async def async_request_openai_audio_speech(
|
async def async_request_openai_audio_speech(
|
||||||
request_func_input: RequestFuncInput, session: aiohttp.ClientSession, pbar: tqdm | None = None
|
request_func_input: RequestFuncInput, session: aiohttp.ClientSession, pbar: tqdm | None = None
|
||||||
) -> MixRequestFuncOutput:
|
) -> MixRequestFuncOutput:
|
||||||
@@ -1115,10 +727,6 @@ ASYNC_REQUEST_FUNCS["openai-audio-speech"] = async_request_openai_audio_speech
|
|||||||
if "openai-audio-speech" not in OPENAI_COMPATIBLE_BACKENDS:
|
if "openai-audio-speech" not in OPENAI_COMPATIBLE_BACKENDS:
|
||||||
OPENAI_COMPATIBLE_BACKENDS.append("openai-audio-speech")
|
OPENAI_COMPATIBLE_BACKENDS.append("openai-audio-speech")
|
||||||
|
|
||||||
ASYNC_REQUEST_FUNCS["openai-image-edits-omni"] = async_request_openai_image_edits_omni
|
|
||||||
if "openai-image-edits-omni" not in OPENAI_COMPATIBLE_BACKENDS:
|
|
||||||
OPENAI_COMPATIBLE_BACKENDS.append("openai-image-edits-omni")
|
|
||||||
|
|
||||||
# Daily-Omni backend for audio-visual reasoning benchmark
|
# Daily-Omni backend for audio-visual reasoning benchmark
|
||||||
# Reuses openai-chat-omni completions for video+text understanding
|
# Reuses openai-chat-omni completions for video+text understanding
|
||||||
ASYNC_REQUEST_FUNCS["daily-omni"] = async_request_openai_chat_omni_completions
|
ASYNC_REQUEST_FUNCS["daily-omni"] = async_request_openai_chat_omni_completions
|
||||||
@@ -1131,10 +739,7 @@ from vllm.benchmarks import serve
|
|||||||
from vllm.benchmarks.lib.ready_checker import wait_for_endpoint
|
from vllm.benchmarks.lib.ready_checker import wait_for_endpoint
|
||||||
from vllm.benchmarks.serve import TaskType, calculate_metrics_for_embeddings, get_request
|
from vllm.benchmarks.serve import TaskType, calculate_metrics_for_embeddings, get_request
|
||||||
|
|
||||||
from vllm_omni.benchmarks.metrics.metrics import (
|
from vllm_omni.benchmarks.metrics.metrics import MultiModalsBenchmarkMetrics, calculate_metrics
|
||||||
MultiModalsBenchmarkMetrics,
|
|
||||||
calculate_metrics,
|
|
||||||
)
|
|
||||||
|
|
||||||
# ruff: noqa: E402
|
# ruff: noqa: E402
|
||||||
|
|
||||||
@@ -1391,7 +996,6 @@ async def benchmark(
|
|||||||
max_concurrency=max_concurrency,
|
max_concurrency=max_concurrency,
|
||||||
request_rate=request_rate,
|
request_rate=request_rate,
|
||||||
benchmark_duration=benchmark_duration,
|
benchmark_duration=benchmark_duration,
|
||||||
print_stage=_PRINT_STAGE,
|
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
metrics = calculate_metrics_for_embeddings(
|
metrics = calculate_metrics_for_embeddings(
|
||||||
@@ -1412,13 +1016,9 @@ async def benchmark(
|
|||||||
"request_goodput": metrics.request_goodput if goodput_config_dict else None,
|
"request_goodput": metrics.request_goodput if goodput_config_dict else None,
|
||||||
"output_throughput": metrics.output_throughput,
|
"output_throughput": metrics.output_throughput,
|
||||||
"total_token_throughput": metrics.total_token_throughput,
|
"total_token_throughput": metrics.total_token_throughput,
|
||||||
defs.TOTAL_AUDIO_DURATION_S: getattr(metrics, defs.TOTAL_AUDIO_DURATION_S),
|
"total_audio_duration_s": metrics.total_audio_duration_s,
|
||||||
defs.TOTAL_AUDIO_FRAMES: getattr(metrics, defs.TOTAL_AUDIO_FRAMES),
|
"total_audio_frames": metrics.total_audio_frames,
|
||||||
defs.AUDIO_THROUGHPUT: getattr(metrics, defs.AUDIO_THROUGHPUT),
|
"audio_throughput": metrics.audio_throughput,
|
||||||
defs.TOTAL_IMAGES: getattr(metrics, defs.TOTAL_IMAGES),
|
|
||||||
defs.IMAGE_THROUGHPUT: getattr(metrics, defs.IMAGE_THROUGHPUT),
|
|
||||||
defs.AVERAGE_PIXELS_PER_IMAGE: getattr(metrics, defs.AVERAGE_PIXELS_PER_IMAGE),
|
|
||||||
defs.MEAN_DENOISE_STEP_LATENCY_MS: getattr(metrics, defs.MEAN_DENOISE_STEP_LATENCY_MS),
|
|
||||||
"input_lens": [output.prompt_len for output in outputs],
|
"input_lens": [output.prompt_len for output in outputs],
|
||||||
"output_lens": actual_output_lens,
|
"output_lens": actual_output_lens,
|
||||||
"ttfts": [output.ttft for output in outputs],
|
"ttfts": [output.ttft for output in outputs],
|
||||||
@@ -1474,34 +1074,21 @@ async def benchmark(
|
|||||||
if rps_change_events:
|
if rps_change_events:
|
||||||
result["rps_change_events"] = rps_change_events
|
result["rps_change_events"] = rps_change_events
|
||||||
|
|
||||||
result_percentile_metrics: list[str] = []
|
|
||||||
if "ttft" in selected_percentile_metrics:
|
|
||||||
result_percentile_metrics.append("ttft")
|
|
||||||
if "tpot" in selected_percentile_metrics or "tpop" in selected_percentile_metrics:
|
|
||||||
result_percentile_metrics.append("tpot")
|
|
||||||
if "itl" in selected_percentile_metrics:
|
|
||||||
result_percentile_metrics.append("itl")
|
|
||||||
if "e2el" in selected_percentile_metrics:
|
|
||||||
result_percentile_metrics.append("e2el")
|
|
||||||
for metric in selected_percentile_metrics:
|
|
||||||
if metric.startswith("audio") and metric not in result_percentile_metrics:
|
|
||||||
result_percentile_metrics.append(metric)
|
|
||||||
|
|
||||||
def process_one_metric(
|
def process_one_metric(
|
||||||
# E.g., "ttft"
|
# E.g., "ttft"
|
||||||
metric_attribute_name: str,
|
metric_attribute_name: str,
|
||||||
):
|
):
|
||||||
# This function prints and adds statistics of the specified
|
# This function prints and adds statistics of the specified
|
||||||
# metric.
|
# metric.
|
||||||
if metric_attribute_name not in result_percentile_metrics:
|
if metric_attribute_name not in selected_percentile_metrics:
|
||||||
return
|
return
|
||||||
# No text tokens generated (e.g. pure TTS speech endpoint): per-token
|
# No text tokens generated (e.g. pure TTS speech endpoint): per-token
|
||||||
# latency metrics (ttft/tpot/itl) are undefined, so skip them.
|
# latency metrics (ttft/tpot/itl) are undefined, so skip them.
|
||||||
is_text_token_metric = not (metric_attribute_name == "e2el" or metric_attribute_name.startswith("audio"))
|
is_text_token_metric = not (metric_attribute_name == "e2el" or metric_attribute_name.startswith("audio"))
|
||||||
if is_text_token_metric and getattr(metrics, "total_output", 0) == 0:
|
if is_text_token_metric and getattr(metrics, "total_output", 0) == 0:
|
||||||
return
|
return
|
||||||
is_audio_rtf = metric_attribute_name == defs.AUDIO_RTF
|
is_audio_rtf = metric_attribute_name == "audio_rtf"
|
||||||
is_audio_duration_or_underrun = metric_attribute_name in (defs.AUDIO_DURATION, defs.AUDIO_UNDERRUN)
|
is_audio_duration_or_underrun = metric_attribute_name in ("audio_duration", "audio_underrun")
|
||||||
|
|
||||||
suffix = "_ms"
|
suffix = "_ms"
|
||||||
if is_audio_duration_or_underrun:
|
if is_audio_duration_or_underrun:
|
||||||
@@ -1520,10 +1107,9 @@ async def benchmark(
|
|||||||
result[f"p{p_word}_{metric_attribute_name}{suffix}"] = value
|
result[f"p{p_word}_{metric_attribute_name}{suffix}"] = value
|
||||||
|
|
||||||
if task_type == TaskType.GENERATION:
|
if task_type == TaskType.GENERATION:
|
||||||
for metric in result_percentile_metrics:
|
for metric in selected_percentile_metrics:
|
||||||
process_one_metric(metric)
|
process_one_metric(metric)
|
||||||
else:
|
else:
|
||||||
result_percentile_metrics.append("e2el")
|
|
||||||
process_one_metric("e2el")
|
process_one_metric("e2el")
|
||||||
|
|
||||||
if profile:
|
if profile:
|
||||||
|
|||||||
@@ -9,11 +9,6 @@ from vllm.benchmarks.serve import main_async
|
|||||||
# This monkey-patches vllm.benchmarks.datasets.get_samples before it's used
|
# This monkey-patches vllm.benchmarks.datasets.get_samples before it's used
|
||||||
# Must be imported before any vllm.benchmarks module usage
|
# Must be imported before any vllm.benchmarks module usage
|
||||||
import vllm_omni.benchmarks.patch.patch # noqa: F401
|
import vllm_omni.benchmarks.patch.patch # noqa: F401
|
||||||
from vllm_omni.benchmarks.patch.patch import (
|
|
||||||
maybe_enable_stage_metrics,
|
|
||||||
set_print_stage,
|
|
||||||
should_request_stage_metrics,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def main(args: argparse.Namespace) -> dict[str, Any]:
|
def main(args: argparse.Namespace) -> dict[str, Any]:
|
||||||
@@ -23,9 +18,4 @@ def main(args: argparse.Namespace) -> dict[str, Any]:
|
|||||||
os.environ["SEED_TTS_WER_SAVE_ITEMS"] = "1"
|
os.environ["SEED_TTS_WER_SAVE_ITEMS"] = "1"
|
||||||
if getattr(args, "daily_omni_save_eval_items", False):
|
if getattr(args, "daily_omni_save_eval_items", False):
|
||||||
os.environ["DAILY_OMNI_SAVE_EVAL_ITEMS"] = "1"
|
os.environ["DAILY_OMNI_SAVE_EVAL_ITEMS"] = "1"
|
||||||
set_print_stage(getattr(args, "print_stage", False))
|
|
||||||
args.extra_body = maybe_enable_stage_metrics(
|
|
||||||
getattr(args, "extra_body", None),
|
|
||||||
enabled=should_request_stage_metrics(args),
|
|
||||||
)
|
|
||||||
return asyncio.run(main_async(args))
|
return asyncio.run(main_async(args))
|
||||||
@@ -17,10 +17,11 @@ Adding a new pipeline:
|
|||||||
``vllm_omni/.../pipeline.py``.
|
``vllm_omni/.../pipeline.py``.
|
||||||
2. Add one line to ``_OMNI_PIPELINES`` below.
|
2. Add one line to ``_OMNI_PIPELINES`` below.
|
||||||
|
|
||||||
Plain single-stage diffusion models continue to use the
|
Single-stage diffusion models continue to use the
|
||||||
``_create_default_diffusion_stage_cfg`` fallback in ``async_omni_engine.py``.
|
``_create_default_diffusion_stage_cfg`` fallback in
|
||||||
The empty ``_DIFFUSION_PIPELINES`` placeholder previously here (#2915) was
|
``async_omni_engine.py`` — they don't need a registry entry. The empty
|
||||||
removed once #2987 (which would have populated it) was deferred.
|
``_DIFFUSION_PIPELINES`` placeholder previously here (#2915) was removed
|
||||||
|
once #2987 (which would have populated it) was deferred.
|
||||||
|
|
||||||
``register_pipeline(config)`` in ``stage_config`` is still supported for
|
``register_pipeline(config)`` in ``stage_config`` is still supported for
|
||||||
out-of-tree plugins and tests that create pipelines at runtime; those override
|
out-of-tree plugins and tests that create pipelines at runtime; those override
|
||||||
@@ -69,10 +70,6 @@ _OMNI_PIPELINES: dict[str, tuple[str, str]] = {
|
|||||||
"vllm_omni.model_executor.models.lance.pipeline",
|
"vllm_omni.model_executor.models.lance.pipeline",
|
||||||
"LANCE_PIPELINE",
|
"LANCE_PIPELINE",
|
||||||
),
|
),
|
||||||
"dreamzero": (
|
|
||||||
"vllm_omni.model_executor.models.dreamzero.pipeline",
|
|
||||||
"DREAMZERO_PIPELINE",
|
|
||||||
),
|
|
||||||
"glm_image": (
|
"glm_image": (
|
||||||
"vllm_omni.model_executor.models.glm_image.pipeline",
|
"vllm_omni.model_executor.models.glm_image.pipeline",
|
||||||
"GLM_IMAGE_PIPELINE",
|
"GLM_IMAGE_PIPELINE",
|
||||||
@@ -149,12 +146,4 @@ _OMNI_PIPELINES: dict[str, tuple[str, str]] = {
|
|||||||
"vllm_omni.model_executor.models.higgs_audio_v2.pipeline",
|
"vllm_omni.model_executor.models.higgs_audio_v2.pipeline",
|
||||||
"HIGGS_AUDIO_V2_PIPELINE",
|
"HIGGS_AUDIO_V2_PIPELINE",
|
||||||
),
|
),
|
||||||
"higgs_multimodal_qwen3": (
|
|
||||||
"vllm_omni.model_executor.models.higgs_audio_v3.pipeline",
|
|
||||||
"HIGGS_AUDIO_V3_PIPELINE",
|
|
||||||
),
|
|
||||||
"dynin_omni": (
|
|
||||||
"vllm_omni.model_executor.models.dynin_omni.pipeline",
|
|
||||||
"DYNIN_OMNI_PIPELINE",
|
|
||||||
),
|
|
||||||
}
|
}
|
||||||
@@ -1165,11 +1165,6 @@ class StageConfigFactory:
|
|||||||
|
|
||||||
# --- New path: check pipeline registry by model_type first ---
|
# --- New path: check pipeline registry by model_type first ---
|
||||||
model_type, hf_config = cls._auto_detect_model_type(model, trust_remote_code=trust_remote_code)
|
model_type, hf_config = cls._auto_detect_model_type(model, trust_remote_code=trust_remote_code)
|
||||||
if model_type == "vla":
|
|
||||||
from vllm_omni.diffusion.utils.hf_utils import _looks_like_dreamzero
|
|
||||||
|
|
||||||
if _looks_like_dreamzero(model):
|
|
||||||
model_type = "dreamzero"
|
|
||||||
if model_type and model_type in _PIPELINE_REGISTRY:
|
if model_type and model_type in _PIPELINE_REGISTRY:
|
||||||
return cls._create_from_registry(model_type, cli_overrides, deploy_config_path)
|
return cls._create_from_registry(model_type, cli_overrides, deploy_config_path)
|
||||||
|
|
||||||
|
|||||||
@@ -391,33 +391,27 @@ class OmniTensorPrefixCache:
|
|||||||
self,
|
self,
|
||||||
query_start_loc: torch.Tensor,
|
query_start_loc: torch.Tensor,
|
||||||
input_batch: InputBatch,
|
input_batch: InputBatch,
|
||||||
multimodal_outputs: dict | None,
|
multimodal_outputs: dict,
|
||||||
num_scheduled_tokens: dict[str, int],
|
num_scheduled_tokens: dict[str, int],
|
||||||
):
|
):
|
||||||
"""Get the merged multimodal states if hidden state prefix caching is enabled."""
|
"""Get the merged multimodal states if hidden state prefix caching is enabled."""
|
||||||
combined_multimodal_outputs = {}
|
combined_multimodal_outputs = {}
|
||||||
# Talkers that produce multimodal outputs only at decode (e.g. Higgs
|
|
||||||
# Audio v3's audio codes) have ``multimodal_outputs is None`` at the
|
|
||||||
# initial prefill step. Treat it as an empty mapping so the merger
|
|
||||||
# short-circuits cleanly and the cache merge still runs once codes
|
|
||||||
# start arriving.
|
|
||||||
mm_outputs = multimodal_outputs if multimodal_outputs is not None else {}
|
|
||||||
# First get the prefix cached tensors that are present in the mm data
|
# First get the prefix cached tensors that are present in the mm data
|
||||||
for mm_key in self.mm_cache_keys:
|
for mm_key in self.mm_cache_keys:
|
||||||
if mm_key in mm_outputs:
|
if mm_key in multimodal_outputs:
|
||||||
combined_multimodal_outputs[mm_key] = self._get_merged_tensors(
|
combined_multimodal_outputs[mm_key] = self._get_merged_tensors(
|
||||||
query_start_loc=query_start_loc,
|
query_start_loc=query_start_loc,
|
||||||
input_batch=input_batch,
|
input_batch=input_batch,
|
||||||
cache=self.mm_outputs_cache[mm_key],
|
cache=self.mm_outputs_cache[mm_key],
|
||||||
hidden_states=mm_outputs[mm_key],
|
hidden_states=multimodal_outputs[mm_key],
|
||||||
num_scheduled_tokens=num_scheduled_tokens,
|
num_scheduled_tokens=num_scheduled_tokens,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Then, get everything else (passthrough data); first, convert to CPU
|
# Then, get everything else (passthrough data); first, convert to CPU
|
||||||
# tensors similarly to the non prefix cached path, and then populate
|
# tensors similarly to the non prefix cached path, and then populate
|
||||||
# the subdicts mapping request IDs -> payload objects
|
# the subdicts mapping request IDs -> payload objects
|
||||||
passthrough_keys = set(mm_outputs.keys()) - self.mm_cache_keys
|
passthrough_keys = set(multimodal_outputs.keys()) - self.mm_cache_keys
|
||||||
passthrough_mm_data = {k: v for k, v in mm_outputs.items() if k in passthrough_keys}
|
passthrough_mm_data = {k: v for k, v in multimodal_outputs.items() if k in passthrough_keys}
|
||||||
mm_cpu = build_mm_cpu(multimodal_outputs=passthrough_mm_data)
|
mm_cpu = build_mm_cpu(multimodal_outputs=passthrough_mm_data)
|
||||||
|
|
||||||
for mm_key, mm_val in mm_cpu.items():
|
for mm_key, mm_val in mm_cpu.items():
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ from vllm.v1.core.sched.async_scheduler import AsyncScheduler as AsyncVLLMSchedu
|
|||||||
from vllm.v1.core.sched.output import SchedulerOutput
|
from vllm.v1.core.sched.output import SchedulerOutput
|
||||||
from vllm.v1.core.sched.scheduler import Scheduler as VLLMScheduler
|
from vllm.v1.core.sched.scheduler import Scheduler as VLLMScheduler
|
||||||
from vllm.v1.core.sched.utils import remove_all
|
from vllm.v1.core.sched.utils import remove_all
|
||||||
from vllm.v1.engine import EngineCoreEventType, EngineCoreOutput, EngineCoreOutputs, FinishReason
|
from vllm.v1.engine import EngineCoreEventType, EngineCoreOutput, EngineCoreOutputs
|
||||||
from vllm.v1.metrics.perf import PerfStats
|
from vllm.v1.metrics.perf import PerfStats
|
||||||
from vllm.v1.outputs import ModelRunnerOutput
|
from vllm.v1.outputs import ModelRunnerOutput
|
||||||
from vllm.v1.request import Request, RequestStatus, StreamingUpdate
|
from vllm.v1.request import Request, RequestStatus, StreamingUpdate
|
||||||
@@ -69,8 +69,6 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler):
|
|||||||
# alive. Stopped on the first scheduler step after extraction ack.
|
# alive. Stopped on the first scheduler step after extraction ack.
|
||||||
self.pending_stop_after_extraction: set[str] = set()
|
self.pending_stop_after_extraction: set[str] = set()
|
||||||
|
|
||||||
self.finished_req_ids_dict = defaultdict(set)
|
|
||||||
|
|
||||||
# [Omni] Pre-parse KV transfer criteria
|
# [Omni] Pre-parse KV transfer criteria
|
||||||
self.kv_transfer_criteria = self._get_kv_transfer_criteria()
|
self.kv_transfer_criteria = self._get_kv_transfer_criteria()
|
||||||
|
|
||||||
@@ -436,7 +434,7 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler):
|
|||||||
# Downstream async-chunk stages receive real payloads from the
|
# Downstream async-chunk stages receive real payloads from the
|
||||||
# connector. This update only resumes polling for the next segment.
|
# connector. This update only resumes polling for the next segment.
|
||||||
self.chunk_transfer_adapter.segment_finished_requests.discard(request.request_id)
|
self.chunk_transfer_adapter.segment_finished_requests.discard(request.request_id)
|
||||||
request.async_tokens_to_discard = 1
|
request.discard_latest_async_tokens = True
|
||||||
request.num_output_placeholders = 0
|
request.num_output_placeholders = 0
|
||||||
request.spec_token_ids = []
|
request.spec_token_ids = []
|
||||||
request._output_token_ids.clear()
|
request._output_token_ids.clear()
|
||||||
@@ -558,24 +556,16 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler):
|
|||||||
# outputs in this step.
|
# outputs in this step.
|
||||||
engine_core_outputs = {client_index: EngineCoreOutputs(outputs=outs) for client_index, outs in outputs.items()}
|
engine_core_outputs = {client_index: EngineCoreOutputs(outputs=outs) for client_index, outs in outputs.items()}
|
||||||
|
|
||||||
# FIXME: finished_req_ids_dict is unconditionally initialized as
|
|
||||||
# defaultdict(set) in __init__ (not gated by include_finished_set).
|
|
||||||
# This branch is therefore always eligible once any client_index is
|
|
||||||
# populated; revisit when wiring streaming-only / upstream semantics.
|
|
||||||
finished_req_ids = self.finished_req_ids_dict
|
finished_req_ids = self.finished_req_ids_dict
|
||||||
if finished_req_ids:
|
if finished_req_ids:
|
||||||
# Include ids of requests that finished since last outputs
|
# Include ids of requests that finished since last outputs
|
||||||
# were sent.
|
# were sent.
|
||||||
for client_index, finished_set in finished_req_ids.items():
|
for client_index, finished_set in finished_req_ids.items():
|
||||||
eco = engine_core_outputs.get(client_index)
|
# Set finished request set in EngineCoreOutputs for this client.
|
||||||
if eco is None:
|
if (eco := engine_core_outputs.get(client_index)) is not None:
|
||||||
eco = EngineCoreOutputs()
|
eco.finished_requests = finished_set
|
||||||
engine_core_outputs[client_index] = eco
|
else:
|
||||||
emitted = {o.request_id for o in eco.outputs}
|
engine_core_outputs[client_index] = EngineCoreOutputs(finished_requests=finished_set)
|
||||||
for req_id in finished_set:
|
|
||||||
if req_id not in emitted:
|
|
||||||
eco.outputs.append(EngineCoreOutput(req_id, [], finish_reason=FinishReason.ABORT))
|
|
||||||
eco.finished_requests = finished_set
|
|
||||||
finished_req_ids.clear()
|
finished_req_ids.clear()
|
||||||
|
|
||||||
if (stats := self.make_stats(spec_decoding_stats, kv_connector_stats, cudagraph_stats, perf_stats)) is not None:
|
if (stats := self.make_stats(spec_decoding_stats, kv_connector_stats, cudagraph_stats, perf_stats)) is not None:
|
||||||
@@ -642,12 +632,13 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler):
|
|||||||
"""
|
"""
|
||||||
req_id = session.request_id
|
req_id = session.request_id
|
||||||
self._new_prompt_len_snapshot[req_id] = len(update.prompt_token_ids)
|
self._new_prompt_len_snapshot[req_id] = len(update.prompt_token_ids)
|
||||||
|
session.discard_latest_async_tokens = False
|
||||||
outstanding_async_tokens = getattr(session, "num_output_placeholders", 0)
|
outstanding_async_tokens = getattr(session, "num_output_placeholders", 0)
|
||||||
if outstanding_async_tokens > 0:
|
if outstanding_async_tokens > 0:
|
||||||
# Async scheduling may already have sampled the previous
|
# Async scheduling may already have sampled the previous
|
||||||
# segment's next token. Drop that late token instead of
|
# segment's next token. Drop that late token instead of
|
||||||
# appending it to the new streaming segment.
|
# appending it to the new streaming segment.
|
||||||
session.async_tokens_to_discard = 1
|
session.discard_latest_async_tokens = True
|
||||||
session.num_computed_tokens -= session.num_output_placeholders
|
session.num_computed_tokens -= session.num_output_placeholders
|
||||||
session.num_output_placeholders = 0
|
session.num_output_placeholders = 0
|
||||||
session.spec_token_ids = []
|
session.spec_token_ids = []
|
||||||
|
|||||||
@@ -63,7 +63,6 @@ class OmniPayloadMeta(TypedDict, total=False):
|
|||||||
stream_finished: torch.Tensor
|
stream_finished: torch.Tensor
|
||||||
req_id: list[str]
|
req_id: list[str]
|
||||||
left_context_size: int
|
left_context_size: int
|
||||||
right_holdback_size: int
|
|
||||||
override_keys: list[tuple[str, str]]
|
override_keys: list[tuple[str, str]]
|
||||||
num_processed_tokens: int
|
num_processed_tokens: int
|
||||||
next_stage_prompt_len: int
|
next_stage_prompt_len: int
|
||||||
@@ -151,7 +150,6 @@ class MetaStruct(_StructBase):
|
|||||||
stream_finished: torch.Tensor | None = None
|
stream_finished: torch.Tensor | None = None
|
||||||
req_id: list[str] | None = None
|
req_id: list[str] | None = None
|
||||||
left_context_size: int | None = None
|
left_context_size: int | None = None
|
||||||
right_holdback_size: int | None = None
|
|
||||||
override_keys: list[tuple[str, str]] | None = None
|
override_keys: list[tuple[str, str]] | None = None
|
||||||
num_processed_tokens: int | None = None
|
num_processed_tokens: int | None = None
|
||||||
next_stage_prompt_len: int | None = None
|
next_stage_prompt_len: int | None = None
|
||||||
|
|||||||
@@ -1,25 +0,0 @@
|
|||||||
# DreamZero-DROID deploy: single diffusion stage.
|
|
||||||
#
|
|
||||||
# Topology is declared in vllm_omni/model_executor/models/dreamzero/pipeline.py.
|
|
||||||
# This default uses one GPU with TP=1 and CFG parallel disabled.
|
|
||||||
|
|
||||||
pipeline: dreamzero
|
|
||||||
async_chunk: false
|
|
||||||
distributed_executor_backend: mp
|
|
||||||
dtype: bfloat16
|
|
||||||
|
|
||||||
stages:
|
|
||||||
- stage_id: 0
|
|
||||||
devices: "0"
|
|
||||||
max_num_seqs: 1
|
|
||||||
enforce_eager: true
|
|
||||||
model_class_name: DreamZeroPipeline
|
|
||||||
model_config:
|
|
||||||
default_robot_embodiment: roboarena
|
|
||||||
policy_server_config:
|
|
||||||
image_resolution: [180, 320]
|
|
||||||
n_external_cameras: 2
|
|
||||||
needs_wrist_camera: true
|
|
||||||
needs_stereo_camera: false
|
|
||||||
needs_session_id: true
|
|
||||||
action_space: joint_position
|
|
||||||
@@ -1,25 +0,0 @@
|
|||||||
# DreamZero-DROID deploy: TP=1, CFG parallel size=2.
|
|
||||||
|
|
||||||
pipeline: dreamzero
|
|
||||||
async_chunk: false
|
|
||||||
distributed_executor_backend: mp
|
|
||||||
dtype: bfloat16
|
|
||||||
|
|
||||||
stages:
|
|
||||||
- stage_id: 0
|
|
||||||
devices: "0,1"
|
|
||||||
max_num_seqs: 1
|
|
||||||
enforce_eager: true
|
|
||||||
model_class_name: DreamZeroPipeline
|
|
||||||
parallel_config:
|
|
||||||
tensor_parallel_size: 1
|
|
||||||
cfg_parallel_size: 2
|
|
||||||
model_config:
|
|
||||||
default_robot_embodiment: roboarena
|
|
||||||
policy_server_config:
|
|
||||||
image_resolution: [180, 320]
|
|
||||||
n_external_cameras: 2
|
|
||||||
needs_wrist_camera: true
|
|
||||||
needs_stereo_camera: false
|
|
||||||
needs_session_id: true
|
|
||||||
action_space: joint_position
|
|
||||||
@@ -1,38 +0,0 @@
|
|||||||
# Dynin-Omni 3-stage deploy: token2text → token2image → token2audio.
|
|
||||||
#
|
|
||||||
# Topology (stages, processor functions, model_arch) is frozen in
|
|
||||||
# vllm_omni/model_executor/models/dynin_omni/pipeline.py. This file carries
|
|
||||||
# only the deploy-time knobs (devices, GPU memory, batched tokens).
|
|
||||||
#
|
|
||||||
# async_chunk=false: dynin hands the full token payload across stages through
|
|
||||||
# the worker-connector data plane (``*_full_payload`` producers), not chunked
|
|
||||||
# streaming. Pipeline-wide ``trust_remote_code`` is required by the custom
|
|
||||||
# Dynin HF modeling code.
|
|
||||||
async_chunk: false
|
|
||||||
trust_remote_code: true
|
|
||||||
enable_prefix_caching: false
|
|
||||||
|
|
||||||
stages:
|
|
||||||
- stage_id: 0
|
|
||||||
devices: "0"
|
|
||||||
max_num_seqs: 1
|
|
||||||
gpu_memory_utilization: 0.5
|
|
||||||
max_num_batched_tokens: 32768
|
|
||||||
enforce_eager: true
|
|
||||||
async_scheduling: false
|
|
||||||
|
|
||||||
- stage_id: 1
|
|
||||||
devices: "0"
|
|
||||||
max_num_seqs: 1
|
|
||||||
gpu_memory_utilization: 0.1
|
|
||||||
max_num_batched_tokens: 32768
|
|
||||||
enforce_eager: true
|
|
||||||
async_scheduling: false
|
|
||||||
|
|
||||||
- stage_id: 2
|
|
||||||
devices: "0"
|
|
||||||
max_num_seqs: 1
|
|
||||||
gpu_memory_utilization: 0.1
|
|
||||||
max_num_batched_tokens: 32768
|
|
||||||
enforce_eager: true
|
|
||||||
async_scheduling: false
|
|
||||||
@@ -1,35 +1,81 @@
|
|||||||
# Dynin-Omni deploy for e2e smoke tests (3-stage, one GPU).
|
# stage config for running dynin_omni with a 3-stage architecture.
|
||||||
#
|
# this config is intended for e2e smoke tests.
|
||||||
# Topology is frozen in models/dynin_omni/pipeline.py; ``pipeline: dynin_omni``
|
|
||||||
# pins it explicitly so this CI YAML resolves regardless of model_type
|
|
||||||
# auto-detection. Differs from the default deploy only in reduced GPU memory
|
|
||||||
# and a smaller max_num_batched_tokens so the pipeline fits L4-class CI GPUs.
|
|
||||||
pipeline: dynin_omni
|
|
||||||
async_chunk: false
|
|
||||||
trust_remote_code: true
|
|
||||||
enable_prefix_caching: false
|
|
||||||
|
|
||||||
stages:
|
stage_args:
|
||||||
- stage_id: 0
|
- stage_id: 0
|
||||||
devices: "0"
|
stage_type: llm
|
||||||
max_num_seqs: 1
|
runtime:
|
||||||
gpu_memory_utilization: 0.5
|
process: true
|
||||||
max_num_batched_tokens: 4096
|
devices: "0"
|
||||||
enforce_eager: true
|
max_batch_size: 1
|
||||||
async_scheduling: false
|
engine_args:
|
||||||
|
model_stage: token2text
|
||||||
|
model_arch: DyninOmniForConditionalGeneration
|
||||||
|
worker_type: generation
|
||||||
|
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||||
|
engine_output_type: latent
|
||||||
|
custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.dynin_omni.token2text_to_token2image_full_payload
|
||||||
|
trust_remote_code: true
|
||||||
|
gpu_memory_utilization: 0.5
|
||||||
|
enforce_eager: true
|
||||||
|
enable_prefix_caching: false
|
||||||
|
async_scheduling: false
|
||||||
|
max_num_batched_tokens: 4096
|
||||||
|
is_comprehension: true
|
||||||
|
final_output: true
|
||||||
|
final_output_type: text
|
||||||
|
|
||||||
- stage_id: 1
|
- stage_id: 1
|
||||||
devices: "0"
|
stage_type: llm
|
||||||
max_num_seqs: 1
|
runtime:
|
||||||
gpu_memory_utilization: 0.2
|
process: true
|
||||||
max_num_batched_tokens: 4096
|
devices: "0"
|
||||||
enforce_eager: true
|
max_batch_size: 1
|
||||||
async_scheduling: false
|
engine_args:
|
||||||
|
model_stage: token2image
|
||||||
|
model_arch: DyninOmniForConditionalGeneration
|
||||||
|
worker_type: generation
|
||||||
|
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||||
|
engine_output_type: latent
|
||||||
|
custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.dynin_omni.token2image_to_token2audio_full_payload
|
||||||
|
trust_remote_code: true
|
||||||
|
gpu_memory_utilization: 0.2
|
||||||
|
enforce_eager: true
|
||||||
|
enable_prefix_caching: false
|
||||||
|
async_scheduling: false
|
||||||
|
max_num_batched_tokens: 4096
|
||||||
|
engine_input_source: [0]
|
||||||
|
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.dynin_omni.token2text_to_token2image
|
||||||
|
final_output: true
|
||||||
|
final_output_type: image
|
||||||
|
|
||||||
- stage_id: 2
|
- stage_id: 2
|
||||||
devices: "0"
|
stage_type: llm
|
||||||
max_num_seqs: 1
|
runtime:
|
||||||
gpu_memory_utilization: 0.2
|
process: true
|
||||||
max_num_batched_tokens: 4096
|
devices: "0"
|
||||||
enforce_eager: true
|
max_batch_size: 1
|
||||||
async_scheduling: false
|
engine_args:
|
||||||
|
model_stage: token2audio
|
||||||
|
model_arch: DyninOmniForConditionalGeneration
|
||||||
|
worker_type: generation
|
||||||
|
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||||
|
engine_output_type: latent
|
||||||
|
trust_remote_code: true
|
||||||
|
gpu_memory_utilization: 0.2
|
||||||
|
enforce_eager: true
|
||||||
|
enable_prefix_caching: false
|
||||||
|
async_scheduling: false
|
||||||
|
max_num_batched_tokens: 4096
|
||||||
|
engine_input_source: [1]
|
||||||
|
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.dynin_omni.token2image_to_token2audio
|
||||||
|
final_output: true
|
||||||
|
final_output_type: audio
|
||||||
|
|
||||||
|
runtime:
|
||||||
|
enabled: true
|
||||||
|
edges:
|
||||||
|
- from: 0
|
||||||
|
to: 1
|
||||||
|
- from: 1
|
||||||
|
to: 2
|
||||||
@@ -1,65 +0,0 @@
|
|||||||
# Dynin-Omni 3-stage deploy with cross-node connectors (Mooncake demo).
|
|
||||||
#
|
|
||||||
# Topology is frozen in models/dynin_omni/pipeline.py (``pipeline: dynin_omni``).
|
|
||||||
# This variant wires a MooncakeConnector for each stage hand-off in place of
|
|
||||||
# the default SharedMemoryConnector. Update the ``extra`` host/master/metadata
|
|
||||||
# fields to match your Mooncake deployment before use.
|
|
||||||
pipeline: dynin_omni
|
|
||||||
async_chunk: false
|
|
||||||
trust_remote_code: false
|
|
||||||
enable_prefix_caching: false
|
|
||||||
|
|
||||||
connectors:
|
|
||||||
# Mooncake connector for cross-node/intra-node communication.
|
|
||||||
mooncake_connector:
|
|
||||||
name: MooncakeConnector
|
|
||||||
extra:
|
|
||||||
host: "127.0.0.1"
|
|
||||||
metadata_server: "http://10.90.67.86:8080/metadata"
|
|
||||||
master: "10.90.67.86:50051"
|
|
||||||
segment: 512000000 # 512MB
|
|
||||||
localbuf: 64000000 # 64MB
|
|
||||||
proto: "tcp"
|
|
||||||
|
|
||||||
# Yuanrong connector for cross-node/intra-node communication.
|
|
||||||
yuanrong_connector:
|
|
||||||
name: YuanrongConnector
|
|
||||||
extra:
|
|
||||||
host: "127.0.0.1"
|
|
||||||
port: "35000"
|
|
||||||
|
|
||||||
# Alternative SHM connector with an explicit threshold.
|
|
||||||
shared_memory_connector:
|
|
||||||
name: SharedMemoryConnector
|
|
||||||
extra:
|
|
||||||
shm_threshold_bytes: 65536 # 64KB threshold
|
|
||||||
|
|
||||||
stages:
|
|
||||||
- stage_id: 0
|
|
||||||
devices: "0"
|
|
||||||
max_num_seqs: 1
|
|
||||||
max_num_batched_tokens: 32768
|
|
||||||
enforce_eager: true
|
|
||||||
async_scheduling: false
|
|
||||||
output_connectors:
|
|
||||||
to_stage_1: mooncake_connector
|
|
||||||
|
|
||||||
- stage_id: 1
|
|
||||||
devices: "0"
|
|
||||||
max_num_seqs: 1
|
|
||||||
max_num_batched_tokens: 32768
|
|
||||||
enforce_eager: true
|
|
||||||
async_scheduling: false
|
|
||||||
input_connectors:
|
|
||||||
from_stage_0: mooncake_connector
|
|
||||||
output_connectors:
|
|
||||||
to_stage_2: mooncake_connector
|
|
||||||
|
|
||||||
- stage_id: 2
|
|
||||||
devices: "0"
|
|
||||||
max_num_seqs: 1
|
|
||||||
max_num_batched_tokens: 32768
|
|
||||||
enforce_eager: true
|
|
||||||
async_scheduling: false
|
|
||||||
input_connectors:
|
|
||||||
from_stage_1: mooncake_connector
|
|
||||||
@@ -1,74 +0,0 @@
|
|||||||
# Higgs-audio v3 deploy: Stage 0 (talker) -> Stage 1 (code2wav) via shared memory.
|
|
||||||
#
|
|
||||||
# Plain TTS only (no voice cloning). async_chunk streams codec chunks from
|
|
||||||
# Stage 0 to Stage 1 as they are produced (TTFA approaches one AR loop's
|
|
||||||
# worth of work). Stage 1 honors meta.left_context_size + meta.right_holdback_size
|
|
||||||
# so the per-chunk codec windows stitch into a coherent PCM stream.
|
|
||||||
async_chunk: true
|
|
||||||
|
|
||||||
connectors:
|
|
||||||
connector_of_shared_memory:
|
|
||||||
name: SharedMemoryConnector
|
|
||||||
extra:
|
|
||||||
shm_threshold_bytes: 65536
|
|
||||||
# Stream PCM bytes back to the client as Stage 1 emits each chunk.
|
|
||||||
codec_streaming: true
|
|
||||||
# Streaming chunk shape for talker2code2wav_async_chunk:
|
|
||||||
# codec_chunk_frames steady-state new frames per emit
|
|
||||||
# codec_left_context_frames sliding past context to avoid leading-edge artifacts
|
|
||||||
# codec_right_holdback_frames trailing frames held back so the codec
|
|
||||||
# has future context for emit-region tail samples
|
|
||||||
# initial_codec_chunk_frames smaller first chunk to minimize TTFA
|
|
||||||
codec_chunk_frames: 25
|
|
||||||
codec_left_context_frames: 25
|
|
||||||
codec_right_holdback_frames: 4
|
|
||||||
initial_codec_chunk_frames: 1
|
|
||||||
connector_get_sleep_s: 0.01
|
|
||||||
connector_get_max_wait_first_chunk: 5000
|
|
||||||
connector_get_max_wait: 500
|
|
||||||
|
|
||||||
stages:
|
|
||||||
- stage_id: 0
|
|
||||||
max_num_seqs: 16
|
|
||||||
gpu_memory_utilization: 0.6
|
|
||||||
enforce_eager: true
|
|
||||||
trust_remote_code: true
|
|
||||||
# Stage-0 prefix caching is on by default. The talker's HS opt-out and
|
|
||||||
# deferred ``codes.audio`` mm-output write (see higgs_audio_v3_talker.py)
|
|
||||||
# keep per-step bookkeeping negligible; any shared text prefix across
|
|
||||||
# concurrent requests is a net throughput win. Stage 1 (codec decoder)
|
|
||||||
# has no KV cache and stays disabled.
|
|
||||||
enable_prefix_caching: true
|
|
||||||
async_scheduling: false
|
|
||||||
max_num_batched_tokens: 4096
|
|
||||||
max_model_len: 8192
|
|
||||||
devices: "0"
|
|
||||||
output_connectors:
|
|
||||||
to_stage_1: connector_of_shared_memory
|
|
||||||
default_sampling_params:
|
|
||||||
temperature: 1.0
|
|
||||||
top_p: 0.95
|
|
||||||
top_k: 50
|
|
||||||
max_tokens: 2048
|
|
||||||
seed: 42
|
|
||||||
repetition_penalty: 1.0
|
|
||||||
|
|
||||||
- stage_id: 1
|
|
||||||
max_num_seqs: 16
|
|
||||||
gpu_memory_utilization: 0.25
|
|
||||||
enforce_eager: true
|
|
||||||
trust_remote_code: true
|
|
||||||
enable_prefix_caching: false
|
|
||||||
async_scheduling: false
|
|
||||||
max_num_batched_tokens: 65536
|
|
||||||
max_model_len: 65536
|
|
||||||
devices: "0"
|
|
||||||
input_connectors:
|
|
||||||
from_stage_0: connector_of_shared_memory
|
|
||||||
default_sampling_params:
|
|
||||||
temperature: 0.0
|
|
||||||
top_p: 1.0
|
|
||||||
top_k: -1
|
|
||||||
max_tokens: 65536
|
|
||||||
seed: 42
|
|
||||||
repetition_penalty: 1.0
|
|
||||||
@@ -18,7 +18,6 @@ connectors:
|
|||||||
connector_of_shared_memory:
|
connector_of_shared_memory:
|
||||||
name: SharedMemoryConnector
|
name: SharedMemoryConnector
|
||||||
extra:
|
extra:
|
||||||
initial_codec_chunk_frames: 4
|
|
||||||
codec_chunk_frames: 25
|
codec_chunk_frames: 25
|
||||||
codec_left_context_frames: 25
|
codec_left_context_frames: 25
|
||||||
|
|
||||||
@@ -31,8 +30,12 @@ stages:
|
|||||||
enable_prefix_caching: false
|
enable_prefix_caching: false
|
||||||
devices: "0"
|
devices: "0"
|
||||||
default_sampling_params:
|
default_sampling_params:
|
||||||
temperature: 0.0
|
temperature: 0.4
|
||||||
|
top_p: 0.9
|
||||||
|
top_k: 1
|
||||||
max_tokens: 2048
|
max_tokens: 2048
|
||||||
|
seed: 42
|
||||||
|
repetition_penalty: 1.05
|
||||||
|
|
||||||
- stage_id: 1
|
- stage_id: 1
|
||||||
max_num_batched_tokens: 32768
|
max_num_batched_tokens: 32768
|
||||||
|
|||||||
@@ -18,7 +18,6 @@ connectors:
|
|||||||
connector_of_shared_memory:
|
connector_of_shared_memory:
|
||||||
name: SharedMemoryConnector
|
name: SharedMemoryConnector
|
||||||
extra:
|
extra:
|
||||||
initial_codec_chunk_frames: 4
|
|
||||||
codec_chunk_frames: 25
|
codec_chunk_frames: 25
|
||||||
codec_left_context_frames: 25
|
codec_left_context_frames: 25
|
||||||
|
|
||||||
|
|||||||
@@ -1,13 +1,19 @@
|
|||||||
# VoxCPM2 deploy: single-stage AR pipeline with per-request state batching.
|
# VoxCPM2 deploy: single-stage AR pipeline with per-request state batching.
|
||||||
# Verified on 1x H20 141GB.
|
# Verified on 1x H20 141GB.
|
||||||
#
|
#
|
||||||
# VoxCPM2-local runtime knobs are passed through hf_overrides.
|
# Fields omitted from a stage fall back to StageDeployConfig defaults (see
|
||||||
|
# vllm_omni/config/stage_config.py). Notable choices:
|
||||||
|
# * enforce_eager=true because the talker's KV-cache save/restore loop is
|
||||||
|
# not cudagraph-compatible (captured separately via voxcpm2_talker's own
|
||||||
|
# _CapturedGraph path).
|
||||||
|
# * max_num_seqs=4 matches the legacy max_batch_size=4 contract: concurrent
|
||||||
|
# requests are supported via per-request StaticKVCache save/restore.
|
||||||
async_chunk: false
|
async_chunk: false
|
||||||
dtype: bfloat16
|
dtype: bfloat16
|
||||||
|
|
||||||
stages:
|
stages:
|
||||||
- stage_id: 0
|
- stage_id: 0
|
||||||
max_num_seqs: 32
|
max_num_seqs: 4
|
||||||
gpu_memory_utilization: 0.9
|
gpu_memory_utilization: 0.9
|
||||||
enforce_eager: true
|
enforce_eager: true
|
||||||
trust_remote_code: true
|
trust_remote_code: true
|
||||||
@@ -16,18 +22,6 @@ stages:
|
|||||||
max_num_batched_tokens: 4096
|
max_num_batched_tokens: 4096
|
||||||
max_model_len: 4096
|
max_model_len: 4096
|
||||||
devices: "0"
|
devices: "0"
|
||||||
engine_extras:
|
|
||||||
hf_overrides:
|
|
||||||
voxcpm2_runtime_config:
|
|
||||||
audio_emit_every: 1
|
|
||||||
vae_decode_every: 3
|
|
||||||
enable_batched_cfm: true
|
|
||||||
enable_batched_vae_decode: true
|
|
||||||
coalesce_audio_d2h: true
|
|
||||||
enable_loc_dit_fused_qkv: true
|
|
||||||
enable_loc_dit_fused_mlp: true
|
|
||||||
enable_loc_dit_zero_dt_cache: true
|
|
||||||
enable_loc_dit_skip_qkv_contig: true
|
|
||||||
default_sampling_params:
|
default_sampling_params:
|
||||||
temperature: 0.0
|
temperature: 0.0
|
||||||
top_p: 1.0
|
top_p: 1.0
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ from vllm_omni.diffusion.attention.backends.abstract import (
|
|||||||
AttentionImpl,
|
AttentionImpl,
|
||||||
AttentionMetadata,
|
AttentionMetadata,
|
||||||
)
|
)
|
||||||
from vllm_omni.diffusion.attention.backends.sdpa import _maybe_reshape_attn_mask
|
|
||||||
from vllm_omni.diffusion.attention.backends.utils.piecewise_attn import (
|
from vllm_omni.diffusion.attention.backends.utils.piecewise_attn import (
|
||||||
piecewise_attn,
|
piecewise_attn,
|
||||||
)
|
)
|
||||||
@@ -310,13 +309,6 @@ class FlashAttentionImpl(AttentionImpl):
|
|||||||
"Otherwise, use SDPA backend by setting DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA"
|
"Otherwise, use SDPA backend by setting DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA"
|
||||||
)
|
)
|
||||||
attention_mask = attn_metadata.attn_mask if attn_metadata else None
|
attention_mask = attn_metadata.attn_mask if attn_metadata else None
|
||||||
|
|
||||||
# NPU aclnnFlashAttentionScore requires mask shape to be one of:
|
|
||||||
# [B, N, Sq, Skv], [B, 1, Sq, Skv], [1, 1, Sq, Skv], or [Sq, Skv]
|
|
||||||
# But the incoming mask is 2D [B, S] — reshape to [B, 1, 1, S]
|
|
||||||
# So reuse SDPA's mask reshape logic: [B, S] -> [B, 1, Sq, Skv]
|
|
||||||
attention_mask = _maybe_reshape_attn_mask(query, key, attention_mask, mask_mode="full_qk")
|
|
||||||
|
|
||||||
layout = self.qkv_layout or "BNSD"
|
layout = self.qkv_layout or "BNSD"
|
||||||
return attention_forward(
|
return attention_forward(
|
||||||
query,
|
query,
|
||||||
|
|||||||
Loaded 100 of 192 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user