Compare commits
44
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
963ba1ab7a | ||
|
|
e25239c2ec | ||
|
|
2414b6f94d | ||
|
|
b3dd0a4baf | ||
|
|
6cefe91370 | ||
|
|
84569280b1 | ||
|
|
1e43656ab1 | ||
|
|
3875b81c18 | ||
|
|
b63b36ded3 | ||
|
|
95d56cf062 | ||
|
|
6e8ef213d8 | ||
|
|
31f882935d | ||
|
|
fccd95b12b | ||
|
|
469c1dc87e | ||
|
|
519cf3ecf1 | ||
|
|
bfc7cedcf1 | ||
|
|
e10aca30ec | ||
|
|
c6e8d78eda | ||
|
|
e5030c7524 | ||
|
|
a1cc583665 | ||
|
|
a57b8f858b | ||
|
|
9a9c8f7503 | ||
|
|
af02713952 | ||
|
|
d8dc1defa7 | ||
|
|
800c2b8251 | ||
|
|
221584a489 | ||
|
|
4e1c13236e | ||
|
|
706bad253d | ||
|
|
8949fdc440 | ||
|
|
3818ba4c65 | ||
|
|
c33ca60208 | ||
|
|
0fc4032a71 | ||
|
|
2e108a4109 | ||
|
|
ce69eb35e9 | ||
|
|
07737e6c61 | ||
|
|
d2e65a92f1 | ||
|
|
3842970a82 | ||
|
|
9bb4974198 | ||
|
|
0e2d55f46d | ||
|
|
40b2959183 | ||
|
|
3329f1b674 | ||
|
|
23484d8a11 | ||
|
|
bdfc77147d | ||
|
|
e35cd213d7 |
No files matched your search
@@ -389,6 +389,29 @@ steps:
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "TTS · VoxCPM2 Online Test"
|
||||
timeout_in_minutes: 30
|
||||
commands:
|
||||
- |
|
||||
timeout 30m bash -c "
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
"
|
||||
agents:
|
||||
queue: "gpu_1_queue"
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
shm-size: "8gb"
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
- "HF_TOKEN"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "Omni · Higgs-Audio V2 Test"
|
||||
timeout_in_minutes: 30
|
||||
commands:
|
||||
|
||||
+188
-4
@@ -394,11 +394,11 @@ steps:
|
||||
depends_on: upload-nightly-pipeline
|
||||
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
|
||||
steps:
|
||||
- label: ":full_moon: TTS · Function Test"
|
||||
- label: ":full_moon: TTS · Function Test with L4"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy
|
||||
- pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_voxcpm2_tts_expansion.py
|
||||
agents:
|
||||
queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
|
||||
plugins:
|
||||
@@ -413,6 +413,69 @@ steps:
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
# - label: ":full_moon: TTS · Function Test with H100"
|
||||
# timeout_in_minutes: 90
|
||||
# commands:
|
||||
# - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
# - pytest -s -v tests/e2e/ -m "full_model and H100 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy
|
||||
# agents:
|
||||
# queue: "mithril-h100-pool"
|
||||
# plugins:
|
||||
# - kubernetes:
|
||||
# podSpec:
|
||||
# containers:
|
||||
# - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
# resources:
|
||||
# limits:
|
||||
# nvidia.com/gpu: 1
|
||||
# volumeMounts:
|
||||
# - name: devshm
|
||||
# mountPath: /dev/shm
|
||||
# - name: hf-cache
|
||||
# mountPath: /root/.cache/huggingface
|
||||
# env:
|
||||
# - name: HF_HOME
|
||||
# value: /root/.cache/huggingface
|
||||
# - name: HF_TOKEN
|
||||
# valueFrom:
|
||||
# secretKeyRef:
|
||||
# name: hf-token-secret
|
||||
# key: token
|
||||
# nodeSelector:
|
||||
# node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
# volumes:
|
||||
# - name: devshm
|
||||
# emptyDir:
|
||||
# medium: Memory
|
||||
# - name: hf-cache
|
||||
# hostPath:
|
||||
# path: /mnt/hf-cache
|
||||
# type: DirectoryOrCreate
|
||||
|
||||
- label: ":full_moon: TTS · Accuracy Test · VoxCPM2"
|
||||
timeout_in_minutes: 30
|
||||
commands:
|
||||
- |
|
||||
timeout 30m bash -c "
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
||||
pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts_expansion.py -m 'full_model and cuda and tts' --run-level 'full_model'
|
||||
"
|
||||
agents:
|
||||
queue: "gpu_1_queue"
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
shm-size: "8gb"
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
- "HF_TOKEN"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: ":full_moon: TTS · Perf Test"
|
||||
key: nightly-tts-performance
|
||||
timeout_in_minutes: 180
|
||||
@@ -459,8 +522,8 @@ steps:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: ":full_moon: VoxCPM2 · Perf Test"
|
||||
key: nightly-voxcpm2-performance
|
||||
- label: ":full_moon: TTS · VoxCPM2 · Perf Test"
|
||||
key: nightly-tts-performance-voxcpm2
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
@@ -695,6 +758,87 @@ steps:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU DreamZero"
|
||||
timeout_in_minutes: 120
|
||||
commands:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/online_serving/test_dreamzero_expansion.py
|
||||
-m "full_model and diffusion and H100 and distributed_cuda"
|
||||
--run-level "full_model"
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 2
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · GLM-Image Function Test with H100"
|
||||
timeout_in_minutes: 120
|
||||
commands:
|
||||
- pytest -sv tests/e2e/online_serving/test_glm_image_expansion.py -m "full_model and diffusion and H100" --run-level "full_model"
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 2
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4"
|
||||
timeout_in_minutes: 60
|
||||
@@ -793,6 +937,46 @@ steps:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: ":full_moon: HunyuanImage3-DIT · Accuracy Test"
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export HUNYUAN_IMAGE3_MODEL="tencent/HunyuanImage-3.0-Instruct"
|
||||
- export HUNYUAN_IMAGE3_DEVICES="0,1,2,3"
|
||||
- pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m full_model --run-level full_model
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 4
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image"
|
||||
key: nightly-diffusion-x2iat-performance-qwen-image
|
||||
timeout_in_minutes: 180
|
||||
|
||||
@@ -273,6 +273,42 @@ steps:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "Omni · Qwen3-Omni Realtime WebSocket"
|
||||
commands:
|
||||
- |
|
||||
timeout 30m bash -c "
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
pytest -s -v tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py::TestQwen3OmniRealtimeWebSocket::test_streaming_audio_input_pcm_output_async_chunk_smoke -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 2
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "Omni · VoxCPM2 Test"
|
||||
timeout_in_minutes: 20
|
||||
commands:
|
||||
@@ -296,6 +332,29 @@ steps:
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "TTS · VoxCPM2 Online Test"
|
||||
timeout_in_minutes: 30
|
||||
commands:
|
||||
- |
|
||||
timeout 30m bash -c "
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts.py -m 'core_model and tts' --run-level 'core_model'
|
||||
"
|
||||
agents:
|
||||
queue: "gpu_1_queue"
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
shm-size: "8gb"
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
- "HF_TOKEN"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "TTS · Qwen3-TTS CustomVoice Test"
|
||||
commands:
|
||||
- |
|
||||
|
||||
@@ -151,6 +151,9 @@ class VllmOmniImageClient:
|
||||
seed: int | None = None,
|
||||
negative_prompt: str | None = None,
|
||||
output_compression: int | None = None,
|
||||
bot_task: str | None = None,
|
||||
sys_type: str | None = None,
|
||||
system_prompt: str | None = None,
|
||||
) -> Image.Image:
|
||||
if not isinstance(images, list):
|
||||
images = [images]
|
||||
@@ -170,6 +173,12 @@ class VllmOmniImageClient:
|
||||
data["negative_prompt"] = negative_prompt
|
||||
if output_compression is not None:
|
||||
data["output_compression"] = str(output_compression)
|
||||
if bot_task is not None:
|
||||
data["bot_task"] = bot_task
|
||||
if sys_type is not None:
|
||||
data["sys_type"] = sys_type
|
||||
if system_prompt is not None:
|
||||
data["system_prompt"] = system_prompt
|
||||
|
||||
files = [
|
||||
(
|
||||
|
||||
@@ -64,6 +64,47 @@ def resolve_model_name(*, model_name: str | None, model: str | None = None, outp
|
||||
raise ValueError("model-name is required when it cannot be inferred from model or output-root")
|
||||
|
||||
|
||||
def _coerce_score_int(value: Any) -> int:
|
||||
if isinstance(value, bool):
|
||||
raise TypeError(f"Expected numeric score, got bool: {value!r}")
|
||||
if isinstance(value, (int, float)):
|
||||
return int(value)
|
||||
if isinstance(value, str):
|
||||
stripped = value.strip()
|
||||
if stripped.lstrip("-").isdigit():
|
||||
return int(stripped)
|
||||
raise TypeError(f"Expected numeric score, got {type(value).__name__}: {value!r}")
|
||||
|
||||
|
||||
def _expand_score_entry(entry: Any) -> list[int]:
|
||||
if isinstance(entry, dict):
|
||||
if "score" in entry:
|
||||
return [_coerce_score_int(entry["score"])]
|
||||
if "value" in entry:
|
||||
return [_coerce_score_int(entry["value"])]
|
||||
numeric_values = [_coerce_score_int(value) for value in entry.values()]
|
||||
if numeric_values:
|
||||
return numeric_values
|
||||
raise TypeError(f"Score dict has no numeric values: {entry!r}")
|
||||
return [_coerce_score_int(entry)]
|
||||
|
||||
|
||||
def _normalize_score_list(score: Any) -> list[int]:
|
||||
if isinstance(score, dict):
|
||||
entries: list[Any] = [score]
|
||||
elif isinstance(score, list):
|
||||
entries = score
|
||||
else:
|
||||
entries = [score]
|
||||
|
||||
normalized: list[int] = []
|
||||
for entry in entries:
|
||||
normalized.extend(_expand_score_entry(entry))
|
||||
if not normalized:
|
||||
raise ValueError(f"Empty score list after normalization: {score!r}")
|
||||
return normalized
|
||||
|
||||
|
||||
def parse_score_payload(raw_text: str) -> dict[str, Any]:
|
||||
try:
|
||||
parsed = extract_json_object(raw_text)
|
||||
@@ -73,10 +114,7 @@ def parse_score_payload(raw_text: str) -> dict[str, Any]:
|
||||
parsed = {"score": json.loads(stripped), "reasoning": ""}
|
||||
else:
|
||||
raise
|
||||
score = parsed.get("score", [])
|
||||
if not isinstance(score, list):
|
||||
score = [score]
|
||||
parsed["score"] = [int(value) for value in score]
|
||||
parsed["score"] = _normalize_score_list(parsed.get("score", []))
|
||||
return parsed
|
||||
|
||||
|
||||
@@ -391,6 +429,9 @@ class GEditBenchRunner:
|
||||
num_inference_steps: int = 20,
|
||||
guidance_scale: float | None = None,
|
||||
seed: int | None = 42,
|
||||
bot_task: str | None = None,
|
||||
sys_type: str | None = None,
|
||||
system_prompt: str | None = None,
|
||||
):
|
||||
self.dataset_ref = dataset_ref
|
||||
self.output_root = output_root
|
||||
@@ -400,6 +441,9 @@ class GEditBenchRunner:
|
||||
self.num_inference_steps = num_inference_steps
|
||||
self.guidance_scale = guidance_scale
|
||||
self.seed = seed
|
||||
self.bot_task = bot_task
|
||||
self.sys_type = sys_type
|
||||
self.system_prompt = system_prompt
|
||||
self.client = VllmOmniImageClient(base_url=base_url, api_key=api_key)
|
||||
|
||||
def generate(
|
||||
@@ -478,6 +522,9 @@ class GEditBenchRunner:
|
||||
num_inference_steps=self.num_inference_steps,
|
||||
guidance_scale=self.guidance_scale,
|
||||
seed=self.seed,
|
||||
bot_task=self.bot_task,
|
||||
sys_type=self.sys_type,
|
||||
system_prompt=self.system_prompt,
|
||||
)
|
||||
edited_image.save(output_path)
|
||||
return {
|
||||
@@ -633,6 +680,25 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
generate.add_argument("--num-inference-steps", type=int, default=20)
|
||||
generate.add_argument("--guidance-scale", type=float, default=None)
|
||||
generate.add_argument("--seed", type=int, default=42)
|
||||
generate.add_argument(
|
||||
"--bot-task",
|
||||
type=str,
|
||||
default=None,
|
||||
choices=["think", "recaption", "think_recaption", "vanilla"],
|
||||
help="HunyuanImage-3 prompting mode for /v1/images/edits (optional).",
|
||||
)
|
||||
generate.add_argument(
|
||||
"--sys-type",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Hunyuan system-prompt type (e.g. en_unified). Defaults from --bot-task when omitted.",
|
||||
)
|
||||
generate.add_argument(
|
||||
"--system-prompt",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Custom Hunyuan system prompt body (sys_type=custom).",
|
||||
)
|
||||
generate.add_argument("--workers", type=int, default=1)
|
||||
generate.add_argument("--max-samples", type=int, default=None)
|
||||
generate.add_argument("--samples-per-group", type=int, default=None)
|
||||
@@ -675,6 +741,9 @@ def main(argv: list[str] | None = None) -> int:
|
||||
num_inference_steps=args.num_inference_steps,
|
||||
guidance_scale=args.guidance_scale,
|
||||
seed=args.seed,
|
||||
bot_task=args.bot_task,
|
||||
sys_type=args.sys_type,
|
||||
system_prompt=args.system_prompt,
|
||||
)
|
||||
records = runner.generate(
|
||||
model_name=model_name,
|
||||
|
||||
+134
-26
@@ -2,29 +2,29 @@
|
||||
The vllm bench command launches the vLLM-Omni benchmark to evaluate the performance of multimodal models.
|
||||
|
||||
## Notes
|
||||
We currently only support using the "openai-chat-omni" backend.
|
||||
We currently only support using the "openai-chat-omni" and "openai-image-edits-omni" backend.
|
||||
|
||||
## Basic Parameter Description
|
||||
You can use `vllm bench serve --omni --help=all` to get descriptions of all parameters. The commonly used parameters are described below:
|
||||
- `--omni`
|
||||
- `--omni`
|
||||
Enable Omni (multimodal) mode, supporting multimodal inputs and outputs such as images, videos, and audio.
|
||||
|
||||
- `--backend`
|
||||
- `--backend`
|
||||
Specify the backend adapter as openai-chat-omni, using OpenAI Chat compatible API behavior as the protocol. Currently only openai-chat-omni is supported.
|
||||
|
||||
- `--model`
|
||||
- `--model`
|
||||
The model identifier to load, filled according to the models supported by vLLM-Omni.
|
||||
|
||||
- `--endpoint`
|
||||
- `--endpoint`
|
||||
The API endpoint exposed externally, to which clients send their requests.
|
||||
|
||||
- `--dataset-name`
|
||||
- `--dataset-name`
|
||||
The name of the dataset used; random-mm indicates generating random multimodal inputs (images, videos, audio).
|
||||
|
||||
- `--num-prompts`
|
||||
- `--num-prompts`
|
||||
The total number of requests to send, an integer.
|
||||
|
||||
- `--max-concurrency`
|
||||
- `--max-concurrency`
|
||||
"Maximum number of concurrent requests. This can be used "
|
||||
"to help simulate an environment where a higher level component "
|
||||
"is enforcing a maximum number of concurrent requests. While the "
|
||||
@@ -34,69 +34,73 @@ You can use `vllm bench serve --omni --help=all` to get descriptions of all para
|
||||
"actual request rate may be lower than specified with --request-rate, "
|
||||
"if the server is not processing requests fast enough to keep up."
|
||||
|
||||
- `--request-rate`
|
||||
- `--request-rate`
|
||||
"Number of requests per second. If this is inf, "
|
||||
"then all the requests are sent at time 0. "
|
||||
"Otherwise, we use Poisson process or gamma distribution "
|
||||
"to synthesize the request arrival times."
|
||||
|
||||
- `--ignore-eos`
|
||||
- `--ignore-eos`
|
||||
"Set ignore_eos flag when sending the benchmark request."
|
||||
|
||||
- `--metric-percentiles`
|
||||
- `--metric-percentiles`
|
||||
Comma-separated list of percentiles for selected metrics. "
|
||||
"To report 25-th, 50-th, and 75-th percentiles, use \"25,50,75\". "
|
||||
"Default value is \"99\"."
|
||||
"Use \"--percentile-metrics\" to select metrics.
|
||||
|
||||
- `--percentile-metrics`
|
||||
- `--percentile-metrics`
|
||||
"Comma-separated list of selected metrics to report percentiles."
|
||||
"This argument specifies the metrics to report percentiles."
|
||||
'Allowed metric names are "ttft", "tpot", "itl", "e2el", "audio_ttfp", "audio_rtf", "audio_duration". '
|
||||
'Allowed metric names are "ttft", "tpot", "itl", "ttfc", "tpoc", "icl", '
|
||||
'"tpop", "e2el", "audio_ttfp", "audio_rtf", "audio_duration". '
|
||||
|
||||
- `--save-result`
|
||||
- `-print-stage`
|
||||
Print per-stage benchmark metrics for --omni serving when stage metrics are returned by the server. Disabled by default.
|
||||
|
||||
- `--save-result`
|
||||
Specify to save benchmark results to a json file
|
||||
|
||||
- `--save-detailed`
|
||||
- `--save-detailed`
|
||||
"When saving the results, whether to include per request "
|
||||
"information such as response, error, ttfs, tpots, etc."
|
||||
|
||||
- `--result-dir`
|
||||
- `--result-dir`
|
||||
"Specify directory to save benchmark json results."
|
||||
"If not specified, results are saved in the current directory."
|
||||
|
||||
- `--result-filename`
|
||||
- `--result-filename`
|
||||
"Specify the filename to save benchmark json results."
|
||||
"If not specified, results will be saved in "
|
||||
"{label}-{args.request_rate}qps-{base_model_id}-{current_dt}.json"
|
||||
|
||||
- `--random-prefix-len`
|
||||
- `--random-prefix-len`
|
||||
Number of fixed prefix tokens before the random context in a request.
|
||||
The total input length is the sum of random-prefix-len and a random
|
||||
context length sampled from [input_len * (1 - range_ratio),
|
||||
input_len * (1 + range_ratio)].Only the random and random-mm modes
|
||||
support this parameter.
|
||||
|
||||
- `--random-input-len`
|
||||
- `--random-input-len`
|
||||
Number of input tokens per request.Only the random and random-mm modes support this parameter.
|
||||
|
||||
- `--random-output-len`
|
||||
- `--random-output-len`
|
||||
Number of output tokens per request.Only the random and random-mm modes support this parameter.
|
||||
|
||||
- `--random-range-ratio`
|
||||
- `--random-range-ratio`
|
||||
Range ratio for sampling input/output length,
|
||||
used only for random sampling. Must be in the range [0, 1) to define
|
||||
a symmetric sampling range
|
||||
[length * (1 - range_ratio), length * (1 + range_ratio)].
|
||||
Only the random and random-mm modes support this parameter.
|
||||
|
||||
- `--random-mm-base-items-per-request`
|
||||
- `--random-mm-base-items-per-request`
|
||||
Base number of multimodal items per request for random-mm.
|
||||
Actual per-request count is sampled around this base using
|
||||
--random-mm-num-mm-items-range-ratio.
|
||||
Only the random-mm mode supports this parameter.
|
||||
|
||||
- `--random-mm-limit-mm-per-prompt`
|
||||
- `--random-mm-limit-mm-per-prompt`
|
||||
Per-modality hard caps for items attached per request, e.g.
|
||||
'{"image": 3, "video": 1, "audio": 1}'. The sampled per-request item
|
||||
count is clamped to the sum of these limits. When a modality
|
||||
@@ -104,7 +108,7 @@ Specify to save benchmark results to a json file
|
||||
renormalized.
|
||||
Only the random-mm mode supports this parameter.
|
||||
|
||||
- `--random-mm-num-mm-items-range-ratio`
|
||||
- `--random-mm-num-mm-items-range-ratio`
|
||||
Range ratio r in [0, 1] for sampling items per request.
|
||||
We sample uniformly from the closed integer range
|
||||
[floor(n*(1-r)), ceil(n*(1+r))]
|
||||
@@ -115,7 +119,7 @@ Specify to save benchmark results to a json file
|
||||
An error is raised if the computed min exceeds the max.
|
||||
Only the random-mm mode supports this parameter.
|
||||
|
||||
- `--random-mm-bucket-config`
|
||||
- `--random-mm-bucket-config`
|
||||
The bucket config is a dictionary mapping a multimodal item
|
||||
sampling configuration to a probability.
|
||||
Currently allows for 3 modalities: audio, images and videos.
|
||||
@@ -276,7 +280,9 @@ Generate synthetic image、video、audio inputs alongside random text prompts to
|
||||
|
||||
Notes:
|
||||
|
||||
- Works only with online benchmark via the OpenAI backend (`--backend openai-chat-omni`) and endpoint `/v1/chat/completions`.
|
||||
- Works only with online benchmark via:
|
||||
- the OpenAI chat backend (`--backend openai-chat-omni`) and endpoint `/v1/chat/completions`.
|
||||
- the OpenAI edit image backend (`--backend openai-image-edits-omni`) and endpoint `/v1/images/edits`.
|
||||
|
||||
Start the server (example):
|
||||
|
||||
@@ -357,3 +363,105 @@ How sampling works:
|
||||
This should be seen as an edge case, and if this behavior can be avoided by setting `--random-mm-limit-mm-per-prompt` to a large number. Note that this might result in errors due to engine config `--limit-mm-per-prompt`.
|
||||
- The resulting request contains synthetic image data in `multi_modal_data` (OpenAI Chat format). When `random-mm` is used with the OpenAI Chat backend, prompts remain text and MM content is attached via `multi_modal_data`.
|
||||
</details>
|
||||
|
||||
### Multi-Stage Benchmark
|
||||
|
||||
<details class="admonition abstract" markdown="1">
|
||||
<summary>Show more</summary>
|
||||
|
||||
Start the server (change the port and model):
|
||||
|
||||
```bash
|
||||
vllm serve --omni --port 29999 --model ~/Qwen3-Omni-30B-A3B-Instruct
|
||||
```
|
||||
|
||||
Then run the benchmarking script (remember to add the flag `--print-stage`):
|
||||
```bash
|
||||
vllm bench serve --omni \
|
||||
--dataset-name random \
|
||||
--port 29999 \
|
||||
--max-concurrency 1 \
|
||||
--num-warmups 2 \
|
||||
--model ~/Qwen3-Omni-30B-A3B-Instruct \
|
||||
--endpoint /v1/chat/completions \
|
||||
--backend openai-chat-omni \
|
||||
--num-prompts 3 \
|
||||
--random-input-len 2500 \
|
||||
--ignore-eos \
|
||||
--percentile-metrics ttft,tpot,itl,e2el,audio_ttfp,audio_rtf,ttfc,tpoc,icl \
|
||||
--random-output-len 900 \
|
||||
--extra_body '{"modalities": ["text","audio"]}' \
|
||||
--print-stage
|
||||
```
|
||||
|
||||
Besides the "Serving Benchmark Result", there will be "Stage Benchmark Result" below:
|
||||
```text
|
||||
============= Stage Benchmark Result =============
|
||||
=============== Stage 0 (thinker) ================
|
||||
-------------------Stage Timing-------------------
|
||||
Mean stage_gen_time (ms): 18243.16
|
||||
Median stage_gen_time (ms): 18220.83
|
||||
P99 stage_gen_time (ms): 18296.75
|
||||
================== Text Result ===================
|
||||
Stage generated tokens: 2700
|
||||
-----------Serving Time to First Token------------
|
||||
Mean Serving TTFT (ms): 816.31
|
||||
Median Serving TTFT (ms): 818.94
|
||||
P99 Serving TTFT (ms): 829.76
|
||||
-----Time per Output Token (excl. 1st token)------
|
||||
Mean TPOP (ms): 19.41
|
||||
Median TPOP (ms): 19.40
|
||||
P99 TPOP (ms): 19.47
|
||||
---------------Inter-token Latency----------------
|
||||
Mean ITL (ms): 19.41
|
||||
Median ITL (ms): 19.12
|
||||
P99 ITL (ms): 23.58
|
||||
================ Stage 1 (talker) ================
|
||||
-------------------Stage Timing-------------------
|
||||
Mean stage_gen_time (ms): 18849.64
|
||||
Median stage_gen_time (ms): 13641.15
|
||||
P99 stage_gen_time (ms): 33655.58
|
||||
============= Internal Stream Result =============
|
||||
-----------Serving Time to First Chunk------------
|
||||
Mean Serving TTFC (ms): 1029.14
|
||||
Median Serving TTFC (ms): 1027.98
|
||||
P99 Serving TTFC (ms): 1032.59
|
||||
-----Time per Output Chunk (excl. 1st chunk)------
|
||||
Mean TPOP (ms): 17.53
|
||||
Median TPOP (ms): 18.70
|
||||
P99 TPOP (ms): 18.88
|
||||
---------------Inter-chunk Latency----------------
|
||||
Mean ICL (ms): 16.27
|
||||
Median ICL (ms): 18.19
|
||||
P99 ICL (ms): 25.86
|
||||
=============== Stage 2 (code2wav) ===============
|
||||
-------------------Stage Timing-------------------
|
||||
Mean stage_gen_time (ms): 18860.74
|
||||
Median stage_gen_time (ms): 13655.84
|
||||
P99 stage_gen_time (ms): 33659.79
|
||||
================== Audio Result ==================
|
||||
Stage audio duration generated(s): 260.11
|
||||
Stage audio frames generated: 6242730
|
||||
-----------Serving Time to First Packet-----------
|
||||
Mean Serving AUDIO_TTFP (ms): 1366.16
|
||||
Median Serving AUDIO_TTFP (ms): 1367.20
|
||||
P99 Serving AUDIO_TTFP (ms): 1368.26
|
||||
-----------------Real Time Factor-----------------
|
||||
Mean AUDIO_RTF: 0.24
|
||||
Median AUDIO_RTF: 0.26
|
||||
P99 AUDIO_RTF: 0.27
|
||||
```
|
||||
Explanation:
|
||||
- stage_gen_time: Time from submitting a request to a specific stage to that stage finishing generation.
|
||||
|
||||
- Serving TTFC (Time to First Chunk): Time from the HTTP request being accepted by the serving frontend to
|
||||
the stage producing its first non-empty output chunk. Keep the name TTFT in text stage, TTFP in audio stage
|
||||
for easier mapping.
|
||||
|
||||
- TPOP (Time per Output Chunk): Average time from the first output chunk to stage completion, divided by the
|
||||
number of remaining output chunks. The TPOP abbreviation follows Qwen3.5-Omni Technical Report.
|
||||
|
||||
- ICL (Inter-Chunk Latency): Time between two consecutive output chunks produced by the same stage. Keep the
|
||||
name ITL in text stage for easier mapping.
|
||||
|
||||
</details>
|
||||
@@ -33,7 +33,7 @@ th {
|
||||
| `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `Cosmos3OmniDiffusersPipeline` | Cosmos3 T2I, T2V, I2V | `nvidia/Cosmos3-Nano` | ✅︎ | | | |
|
||||
| `Cosmos3OmniDiffusersPipeline` | Cosmos3 T2I, T2V, I2V, T2V with sound, action policy | `nvidia/Cosmos3-Nano` | ✅︎ | | | |
|
||||
| `WanSpeechToVideoPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `LTX2Pipeline` | LTX-2-T2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | |
|
||||
@@ -42,6 +42,7 @@ th {
|
||||
| `LTX2ImageToVideoTwoStagesPipeline` | LTX-2-I2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | |
|
||||
| `LTX23Pipeline` | LTX-2.3-T2V | `dg845/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | |
|
||||
| `LTX23ImageToVideoPipeline` | LTX-2.3-I2V | `dg845/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | |
|
||||
| `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | | | |
|
||||
| `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | |
|
||||
| `MagiHumanPipeline` | MagiHuman | `SII-GAIR/daVinci-MagiHuman-Base-1080p` | ✅︎ | ✅︎ | | |
|
||||
| `OvisImagePipeline` | Ovis-Image | `OvisAI/Ovis-Image` | ✅︎ | ✅︎ | | ✅︎ |
|
||||
|
||||
@@ -145,6 +145,7 @@ The following tables show which models support each feature:
|
||||
| Model | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | Pipeline-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
|
||||
|------------------------------|:---------:|:----------:|:---------------------:|:--------------:|:-----------------:|:-----------------:|:------:|:-------------------------:|:--------------------:|:--------------:|:----------------:|
|
||||
| **Wan2.2** | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (encode/decode) | ❌ | ❌ |
|
||||
| **Wan2.2-S2V** | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ (encode/decode) | ❌ | ❌ |
|
||||
| **Wan2.1-VACE** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
|
||||
| **LTX-2** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| **LTX-2.3** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||
|
||||
@@ -0,0 +1,304 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from vllm_omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
WORKER_EXTENSION = "vllm_omni.diffusion.models.dreamzero.video_export_worker.DreamZeroVideoExportWorkerExtension"
|
||||
ASSET_REPO_ID = "YangshenDeng/vllm-omni-dreamzero-assets"
|
||||
DEFAULT_SESSION_PREFIX = "dreamzero-export"
|
||||
RELATIVE_OFFSETS = [-23, -16, -8, 0]
|
||||
ACTION_HORIZON = 24
|
||||
CAMERA_FILES = {
|
||||
"observation/exterior_image_0_left": "exterior_image_1_left.mp4",
|
||||
"observation/exterior_image_1_left": "exterior_image_2_left.mp4",
|
||||
"observation/wrist_image_left": "wrist_image_left.mp4",
|
||||
}
|
||||
|
||||
|
||||
def _parse_args() -> argparse.Namespace:
|
||||
repo_root = Path(__file__).resolve().parents[3]
|
||||
parser = argparse.ArgumentParser(description="Export DreamZero prediction video from downloaded example inputs.")
|
||||
parser.add_argument("--model", default="GEAR-Dreams/DreamZero-DROID")
|
||||
parser.add_argument("--deploy-config", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
"--video-dir",
|
||||
type=Path,
|
||||
default=repo_root / "outputs" / "dreamzero" / "assets",
|
||||
help="Directory containing the three camera MP4 files.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir", type=Path, default=repo_root / "outputs" / "dreamzero" / "generated_predictions"
|
||||
)
|
||||
parser.add_argument("--output-stem", default="dreamzero_prediction")
|
||||
parser.add_argument(
|
||||
"--prompt",
|
||||
default="Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan",
|
||||
)
|
||||
parser.add_argument("--session-id", default=None)
|
||||
parser.add_argument("--save-input-video", action="store_true")
|
||||
parser.add_argument("--save-gif", action="store_true")
|
||||
parser.add_argument("--save-actions", action="store_true")
|
||||
parser.add_argument("--fps", type=int, default=5)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _load_all_frames(video_path: Path) -> np.ndarray:
|
||||
cap = cv2.VideoCapture(str(video_path))
|
||||
frames = []
|
||||
while True:
|
||||
ok, frame = cap.read()
|
||||
if not ok:
|
||||
break
|
||||
frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
|
||||
cap.release()
|
||||
if not frames:
|
||||
raise RuntimeError(f"No frames loaded from {video_path}")
|
||||
return np.stack(frames, axis=0)
|
||||
|
||||
|
||||
def _load_camera_frames(video_dir: Path) -> dict[str, np.ndarray]:
|
||||
camera_frames: dict[str, np.ndarray] = {}
|
||||
for camera_key, file_name in CAMERA_FILES.items():
|
||||
video_path = video_dir / file_name
|
||||
if not video_path.exists():
|
||||
raise FileNotFoundError(
|
||||
f"Missing DreamZero example asset: {video_path}. "
|
||||
"Download the example videos with: "
|
||||
f"`hf download {ASSET_REPO_ID} --repo-type dataset --local-dir {video_dir}`"
|
||||
)
|
||||
camera_frames[camera_key] = _load_all_frames(video_path)
|
||||
return camera_frames
|
||||
|
||||
|
||||
def _build_frame_schedule(total_frames: int, num_chunks: int) -> list[list[int]]:
|
||||
chunks: list[list[int]] = []
|
||||
current_frame = 23
|
||||
for _ in range(num_chunks):
|
||||
indices = [max(current_frame + offset, 0) for offset in RELATIVE_OFFSETS]
|
||||
if indices[-1] >= total_frames:
|
||||
break
|
||||
chunks.append(indices)
|
||||
current_frame += ACTION_HORIZON
|
||||
return chunks
|
||||
|
||||
|
||||
def _make_obs_from_video(
|
||||
camera_frames: dict[str, np.ndarray],
|
||||
frame_indices: list[int],
|
||||
*,
|
||||
prompt: str,
|
||||
session_id: str,
|
||||
) -> dict:
|
||||
obs: dict = {}
|
||||
for camera_key, all_frames in camera_frames.items():
|
||||
selected = all_frames[frame_indices]
|
||||
obs[camera_key] = selected[0] if len(frame_indices) == 1 else selected
|
||||
|
||||
obs["observation/joint_position"] = np.zeros(7, dtype=np.float32)
|
||||
obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32)
|
||||
obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32)
|
||||
obs["prompt"] = prompt
|
||||
obs["session_id"] = session_id
|
||||
return obs
|
||||
|
||||
|
||||
def _build_observations(video_dir: Path, prompt: str, session_id: str) -> tuple[dict[str, np.ndarray], list[dict]]:
|
||||
camera_frames = _load_camera_frames(video_dir)
|
||||
total_frames = min(frames.shape[0] for frames in camera_frames.values())
|
||||
chunks = _build_frame_schedule(total_frames, 1)
|
||||
observations = [
|
||||
_make_obs_from_video(camera_frames, [0], prompt=prompt, session_id=session_id),
|
||||
]
|
||||
if chunks:
|
||||
observations.append(
|
||||
_make_obs_from_video(
|
||||
camera_frames,
|
||||
chunks[0],
|
||||
prompt=prompt,
|
||||
session_id=session_id,
|
||||
)
|
||||
)
|
||||
if len(observations) < 2:
|
||||
raise RuntimeError("Need at least two DreamZero example observations to export a prediction video.")
|
||||
return camera_frames, observations[:2]
|
||||
|
||||
|
||||
def _extract_latents(output: OmniRequestOutput) -> torch.Tensor:
|
||||
if not isinstance(output, OmniRequestOutput):
|
||||
raise TypeError(f"Expected OmniRequestOutput, got {type(output)!r}")
|
||||
if not output.images:
|
||||
raise RuntimeError("DreamZero output does not contain video latents in `images`.")
|
||||
|
||||
latents = output.images[0]
|
||||
if not isinstance(latents, torch.Tensor):
|
||||
raise TypeError(f"Expected tensor latents, got {type(latents)!r}")
|
||||
|
||||
latents = latents.detach().cpu()
|
||||
if latents.dim() == 4:
|
||||
latents = latents.unsqueeze(0)
|
||||
if latents.dim() != 5:
|
||||
raise ValueError(f"Unexpected latent shape: {tuple(latents.shape)}")
|
||||
|
||||
if latents.shape[1] < latents.shape[2]:
|
||||
latents = latents.transpose(1, 2).contiguous()
|
||||
return latents
|
||||
|
||||
|
||||
def _write_mp4(path: Path, frames: np.ndarray, fps: int) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
height, width = frames.shape[1:3]
|
||||
writer = cv2.VideoWriter(
|
||||
str(path),
|
||||
cv2.VideoWriter_fourcc(*"mp4v"),
|
||||
float(fps),
|
||||
(width, height),
|
||||
)
|
||||
if not writer.isOpened():
|
||||
raise RuntimeError(f"Failed to open video writer for {path}")
|
||||
try:
|
||||
for frame in frames:
|
||||
writer.write(cv2.cvtColor(frame, cv2.COLOR_RGB2BGR))
|
||||
finally:
|
||||
writer.release()
|
||||
|
||||
|
||||
def _write_gif(path: Path, frames: np.ndarray, fps: int) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
images = [Image.fromarray(frame) for frame in frames]
|
||||
duration_ms = max(int(round(1000 / max(fps, 1))), 1)
|
||||
images[0].save(
|
||||
path,
|
||||
save_all=True,
|
||||
append_images=images[1:],
|
||||
duration=duration_ms,
|
||||
loop=0,
|
||||
)
|
||||
|
||||
|
||||
def _stitch_input_frames(camera_frames: dict[str, np.ndarray]) -> np.ndarray:
|
||||
total_frames = min(frames.shape[0] for frames in camera_frames.values())
|
||||
stitched = []
|
||||
for frame_index in range(total_frames):
|
||||
left = camera_frames["observation/exterior_image_0_left"][frame_index]
|
||||
right = camera_frames["observation/exterior_image_1_left"][frame_index]
|
||||
wrist = camera_frames["observation/wrist_image_left"][frame_index]
|
||||
pad = np.zeros((left.shape[0], left.shape[1], 3), dtype=np.uint8)
|
||||
canvas = np.concatenate([left, right], axis=1)
|
||||
bottom = np.concatenate([wrist, pad], axis=1)
|
||||
stitched.append(np.concatenate([canvas, bottom], axis=0))
|
||||
return np.stack(stitched, axis=0)
|
||||
|
||||
|
||||
def _run_generation(
|
||||
model: str, deploy_config_path: Path, observations: list[dict]
|
||||
) -> tuple[Omni, list[OmniRequestOutput]]:
|
||||
omni = Omni(
|
||||
model=model,
|
||||
deploy_config=str(deploy_config_path),
|
||||
enforce_eager=True,
|
||||
worker_extension_cls=WORKER_EXTENSION,
|
||||
)
|
||||
|
||||
outputs: list[OmniRequestOutput] = []
|
||||
for index, obs in enumerate(observations):
|
||||
sampling_params = OmniDiffusionSamplingParams(
|
||||
extra_args={
|
||||
"reset": index == 0,
|
||||
"session_id": obs["session_id"],
|
||||
"robot_obs": obs,
|
||||
}
|
||||
)
|
||||
result = omni.generate(obs["prompt"], sampling_params_list=[sampling_params])
|
||||
if not result:
|
||||
raise RuntimeError(f"No output returned for DreamZero request {index}")
|
||||
outputs.append(result[0])
|
||||
return omni, outputs
|
||||
|
||||
|
||||
def _decode_with_worker(omni: Omni, full_latents: torch.Tensor) -> np.ndarray:
|
||||
stage_client = omni.engine.stage_clients[0]
|
||||
engine = getattr(stage_client, "_engine", None)
|
||||
if engine is None:
|
||||
raise RuntimeError("DreamZero export requires inline diffusion stage access.")
|
||||
|
||||
decoded = engine.executor.collective_rpc(
|
||||
"decode_video_latents_to_uint8",
|
||||
args=(full_latents,),
|
||||
unique_reply_rank=0,
|
||||
exec_all_ranks=True,
|
||||
)
|
||||
if isinstance(decoded, torch.Tensor):
|
||||
decoded = decoded.numpy()
|
||||
if not isinstance(decoded, np.ndarray):
|
||||
raise TypeError(f"Unexpected decoded output type: {type(decoded)!r}")
|
||||
return decoded
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = _parse_args()
|
||||
session_id = args.session_id or f"{DEFAULT_SESSION_PREFIX}-{uuid.uuid4()}"
|
||||
|
||||
camera_frames, observations = _build_observations(
|
||||
video_dir=args.video_dir,
|
||||
prompt=args.prompt,
|
||||
session_id=session_id,
|
||||
)
|
||||
|
||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if args.save_input_video:
|
||||
input_frames = _stitch_input_frames(camera_frames)
|
||||
_write_mp4(args.output_dir / f"{args.output_stem}_input.mp4", input_frames, fps=15)
|
||||
if args.save_gif:
|
||||
_write_gif(args.output_dir / f"{args.output_stem}_input.gif", input_frames[::3], fps=5)
|
||||
|
||||
omni = None
|
||||
try:
|
||||
omni, outputs = _run_generation(
|
||||
model=args.model,
|
||||
deploy_config_path=args.deploy_config,
|
||||
observations=observations,
|
||||
)
|
||||
latent_steps = [_extract_latents(output) for output in outputs]
|
||||
full_latents = torch.cat(latent_steps, dim=2)
|
||||
frames = _decode_with_worker(omni, full_latents)
|
||||
finally:
|
||||
if omni is not None:
|
||||
omni.close()
|
||||
|
||||
mp4_path = args.output_dir / f"{args.output_stem}.mp4"
|
||||
|
||||
_write_mp4(mp4_path, frames, fps=args.fps)
|
||||
print(f"SAVED_MP4={mp4_path}")
|
||||
|
||||
if args.save_gif:
|
||||
gif_path = args.output_dir / f"{args.output_stem}.gif"
|
||||
_write_gif(gif_path, frames, fps=args.fps)
|
||||
print(f"SAVED_GIF={gif_path}")
|
||||
|
||||
if args.save_actions:
|
||||
npz_path = args.output_dir / f"{args.output_stem}_actions.npz"
|
||||
np.savez(
|
||||
npz_path,
|
||||
step0=np.asarray(outputs[0].multimodal_output.get("actions")),
|
||||
step1=np.asarray(outputs[1].multimodal_output.get("actions")),
|
||||
)
|
||||
print(f"SAVED_ACTIONS={npz_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -42,7 +42,7 @@ pip install \
|
||||
## 3. Hardware and VRAM Requirements
|
||||
|
||||
This example uses a 3-stage pipeline on one GPU by default
|
||||
([`dynin_omni.yaml`](../../../vllm_omni/model_executor/stage_configs/dynin_omni.yaml)):
|
||||
([`dynin_omni.yaml`](../../../vllm_omni/deploy/dynin_omni.yaml)):
|
||||
|
||||
- Stage-0 (`token2text`): `gpu_memory_utilization: 0.5`
|
||||
- Stage-1 (`token2image`): `gpu_memory_utilization: 0.1`
|
||||
|
||||
@@ -878,8 +878,8 @@ def parse_args(repo_root: Path) -> argparse.Namespace:
|
||||
parser.add_argument(
|
||||
"--stage-config-path",
|
||||
type=str,
|
||||
default=str(repo_root / "vllm_omni/model_executor/stage_configs/dynin_omni.yaml"),
|
||||
help="Path to stage config yaml.",
|
||||
default=str(repo_root / "vllm_omni/deploy/dynin_omni.yaml"),
|
||||
help="Path to deploy config yaml.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dynin-config-path",
|
||||
|
||||
@@ -16,9 +16,10 @@ Usage:
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
@@ -29,6 +30,16 @@ from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
|
||||
def parse_profiler_config(value: str) -> dict[str, Any]:
|
||||
try:
|
||||
config = json.loads(value)
|
||||
except json.JSONDecodeError as e:
|
||||
raise argparse.ArgumentTypeError(f"--profiler-config must be valid JSON: {e}") from e
|
||||
if not isinstance(config, dict):
|
||||
raise argparse.ArgumentTypeError("--profiler-config must be a JSON object")
|
||||
return config
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Generate a talking-head video from a reference image and audio (Wan2.2 S2V)."
|
||||
@@ -87,6 +98,12 @@ def parse_args() -> argparse.Namespace:
|
||||
default=3.0,
|
||||
help="Scheduler flow shift (default: 3.0).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--boundary-ratio",
|
||||
type=float,
|
||||
default=None,
|
||||
help="Boundary split ratio for low/high DiT stages (S2V). Default varies by model.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=str,
|
||||
@@ -178,30 +195,10 @@ def parse_args() -> argparse.Namespace:
|
||||
help="Enable cache-dit summary logging after diffusion forward passes.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--profile-dir",
|
||||
type=str,
|
||||
"--profiler-config",
|
||||
type=parse_profiler_config,
|
||||
default=None,
|
||||
help="Enable torch profiler and save traces to this directory.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--profile-record-shapes",
|
||||
action="store_true",
|
||||
help="Record tensor shapes in profiler (increases trace size).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--profile-with-stack",
|
||||
action="store_true",
|
||||
help="Record stack traces in profiler (increases overhead).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--profile-with-memory",
|
||||
action="store_true",
|
||||
help="Profile memory usage.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--profile-with-flops",
|
||||
action="store_true",
|
||||
help="Estimate FLOPs for operations.",
|
||||
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
@@ -239,23 +236,9 @@ def main():
|
||||
ulysses_degree=args.ulysses_degree,
|
||||
)
|
||||
|
||||
# Check if profiling is requested via CLI args or environment variable
|
||||
profile_dir = args.profile_dir or os.getenv("VLLM_TORCH_PROFILER_DIR")
|
||||
profiler_enabled = bool(profile_dir)
|
||||
profiler_config = None
|
||||
if profiler_enabled:
|
||||
from vllm.config import ProfilerConfig
|
||||
profiler_enabled = args.profiler_config is not None
|
||||
|
||||
profiler_config = ProfilerConfig(
|
||||
profiler="torch",
|
||||
torch_profiler_dir=profile_dir,
|
||||
torch_profiler_record_shapes=args.profile_record_shapes,
|
||||
torch_profiler_with_stack=args.profile_with_stack,
|
||||
torch_profiler_with_memory=args.profile_with_memory,
|
||||
torch_profiler_with_flops=args.profile_with_flops,
|
||||
)
|
||||
|
||||
omni = Omni(
|
||||
omni_kwargs = dict(
|
||||
model=args.model,
|
||||
model_class_name="WanS2VPipeline",
|
||||
flow_shift=args.flow_shift,
|
||||
@@ -269,8 +252,12 @@ def main():
|
||||
cache_backend=args.cache_backend,
|
||||
cache_config=cache_config,
|
||||
enable_cache_dit_summary=args.enable_cache_dit_summary,
|
||||
profiler_config=profiler_config,
|
||||
profiler_config=args.profiler_config,
|
||||
)
|
||||
if args.boundary_ratio is not None:
|
||||
omni_kwargs["boundary_ratio"] = args.boundary_ratio
|
||||
|
||||
omni = Omni(**omni_kwargs)
|
||||
|
||||
# Print generation configuration
|
||||
print(f"\n{'=' * 60}")
|
||||
@@ -325,13 +312,16 @@ def main():
|
||||
if profiler_enabled:
|
||||
print("\n[Profiler] Stopping profiler and collecting results...")
|
||||
profile_results = omni.stop_profile()
|
||||
if profile_results and isinstance(profile_results, list):
|
||||
if profile_results and isinstance(profile_results, dict):
|
||||
traces = profile_results.get("traces", [])
|
||||
print("\n" + "=" * 60)
|
||||
print("PROFILING RESULTS:")
|
||||
for i, result in enumerate(profile_results):
|
||||
print(f"\nStage {i}:")
|
||||
if result:
|
||||
print(f" {result}")
|
||||
for rank, trace in enumerate(traces):
|
||||
print(f"\nRank {rank}:")
|
||||
if trace:
|
||||
print(f" • Trace: {trace}")
|
||||
if not traces:
|
||||
print(" No traces collected.")
|
||||
print("=" * 60)
|
||||
else:
|
||||
print("[Profiler] No valid profiling data returned.")
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
# DreamZero
|
||||
|
||||
DreamZero is the robot-policy serving example for the OpenPI-compatible `/v1/realtime/robot/openpi` endpoint.
|
||||
|
||||
## Supported checkpoints
|
||||
|
||||
| Model | HuggingFace repo |
|
||||
|---|---|
|
||||
| DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` |
|
||||
|
||||
## Quick start
|
||||
|
||||
### Start the server
|
||||
|
||||
```bash
|
||||
bash examples/online_serving/dreamzero/run_server.sh
|
||||
```
|
||||
|
||||
By default this launches:
|
||||
|
||||
```bash
|
||||
vllm serve GEAR-Dreams/DreamZero-DROID --omni --port 8000 \
|
||||
--served-model-name dreamzero-droid \
|
||||
--deploy-config vllm_omni/deploy/dreamzero_tp1_cfg2.yaml \
|
||||
--enforce-eager --disable-log-stats
|
||||
```
|
||||
|
||||
Override `MODEL`, `PORT`, `HOST`, `DEPLOY_CONFIG`, or `SERVED_MODEL_NAME` through the script environment if needed.
|
||||
|
||||
### Install example dependencies
|
||||
|
||||
The core `pip install -e .` setup does not include the extra packages used by the DreamZero example scripts.
|
||||
|
||||
- `openpi_client.py`:
|
||||
`openpi-client`, `websockets`, `opencv-python`
|
||||
- `droid_sim_eval_client.py`:
|
||||
`mediapy`, `websockets`, `openpi-client`
|
||||
|
||||
The DROID sim-eval script also needs an Isaac Lab environment that provides `isaaclab`, `isaaclab_tasks`, `sim_evals`, and `gymnasium`.
|
||||
|
||||
If you run the DROID client on Python < 3.12, also install `typing-extensions`.
|
||||
|
||||
### Configure TP and CFG parallelism
|
||||
|
||||
The bundled DreamZero configs intentionally keep only:
|
||||
|
||||
| Config | Purpose |
|
||||
|---|---|
|
||||
| `vllm_omni/deploy/dreamzero.yaml` | Default TP=1, CFG parallel disabled |
|
||||
| `vllm_omni/deploy/dreamzero_tp1_cfg2.yaml` | TP=1, CFG parallel size=2 |
|
||||
|
||||
For other topologies, use CLI parallelism flags and update stage 0 `devices` with `--stage-overrides`. The number of listed devices must match `tensor_parallel_size * cfg_parallel_size`.
|
||||
|
||||
TP=2 with CFG parallel disabled:
|
||||
|
||||
```bash
|
||||
vllm serve GEAR-Dreams/DreamZero-DROID --omni --port 8000 \
|
||||
--served-model-name dreamzero-droid \
|
||||
--deploy-config vllm_omni/deploy/dreamzero.yaml \
|
||||
--tensor-parallel-size 2 \
|
||||
--stage-overrides '{"0": {"devices": "0,1"}}' \
|
||||
--enforce-eager --disable-log-stats
|
||||
```
|
||||
|
||||
TP=2 with CFG parallel size=2:
|
||||
|
||||
```bash
|
||||
vllm serve GEAR-Dreams/DreamZero-DROID --omni --port 8000 \
|
||||
--served-model-name dreamzero-droid \
|
||||
--deploy-config vllm_omni/deploy/dreamzero.yaml \
|
||||
--tensor-parallel-size 2 \
|
||||
--cfg-parallel-size 2 \
|
||||
--stage-overrides '{"0": {"devices": "0,1,2,3"}}' \
|
||||
--enforce-eager --disable-log-stats
|
||||
```
|
||||
|
||||
### Download example assets
|
||||
|
||||
The OpenPI client and DROID sim-eval example expect the three camera MP4 files in `outputs/dreamzero/assets`.
|
||||
|
||||
```bash
|
||||
hf download YangshenDeng/vllm-omni-dreamzero-assets --repo-type dataset --local-dir outputs/dreamzero/assets
|
||||
```
|
||||
|
||||
### Run the OpenPI client
|
||||
|
||||
```bash
|
||||
python examples/online_serving/dreamzero/openpi_client.py \
|
||||
--host 127.0.0.1 \
|
||||
--port 8000 \
|
||||
--video-dir outputs/dreamzero/assets
|
||||
```
|
||||
|
||||
This client uses downloaded example videos and talks to the OpenPI websocket server.
|
||||
|
||||
### Run DROID sim eval
|
||||
|
||||
```bash
|
||||
${ISAACLAB_LAUNCHER} -p examples/online_serving/dreamzero/droid_sim_eval_client.py \
|
||||
--host 127.0.0.1 \
|
||||
--port 8000 \
|
||||
--scene 1 \
|
||||
--episodes 1 \
|
||||
--headless \
|
||||
--device cuda:0
|
||||
```
|
||||
|
||||
Set `ISAACLAB_LAUNCHER=path/to/isaaclab.sh` from the vLLM-Omni repository root before running the command.
|
||||
This launches Isaac Lab / sim-evals and runs the DROID benchmark loop against the same websocket endpoint.
|
||||
|
||||
### Export comparison videos offline
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/dreamzero/export_prediction_video.py \
|
||||
--deploy-config vllm_omni/deploy/dreamzero_tp1_cfg2.yaml \
|
||||
--save-input-video \
|
||||
--save-gif
|
||||
```
|
||||
|
||||
The export script uses local `Omni` inference, not the websocket server. It writes the input rollout video and the predicted output video artifacts for side-by-side inspection.
|
||||
|
||||
### MolmoSpace demo
|
||||
|
||||
```bash
|
||||
python examples/online_serving/dreamzero/molmospace_dreamzero_eval_demo.py \
|
||||
--host 127.0.0.1 \
|
||||
--port 8000 \
|
||||
--benchmark_dir /path/to/benchmark \
|
||||
--output_dir /path/to/output
|
||||
```
|
||||
|
||||
This demo adapts DreamZero to the MolmoSpace-style remote policy eval loop.
|
||||
@@ -0,0 +1,819 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Run one or more DROID sim-eval rollouts against the vLLM DreamZero server.
|
||||
|
||||
This script is the vLLM/OpenPI adaptation of the upstream DreamZero
|
||||
`eval_utils/run_sim_eval.py` sim-eval client.
|
||||
|
||||
Behavior intentionally kept close to upstream:
|
||||
- same DROID observation extraction (`external_cam`, `external_cam_2`,
|
||||
`wrist_cam`, joint position, gripper position)
|
||||
- same resize-with-pad preprocessing to `(180, 320)`
|
||||
- same `open_loop_horizon=8`
|
||||
- same gripper binarization rule (`> 0.5 -> 1`, else `0`)
|
||||
- same per-scene language prompts
|
||||
|
||||
Unlike upstream DreamZero, vLLM serves the compatible websocket policy endpoint
|
||||
at `/v1/realtime/robot/openpi`, so this script includes the path suffix in the
|
||||
client URI.
|
||||
|
||||
Run this script through Isaac Lab's launcher from the vLLM-Omni repository
|
||||
root, for example:
|
||||
|
||||
"${ISAACLAB_LAUNCHER}" -p \
|
||||
examples/online_serving/dreamzero/droid_sim_eval_client.py \
|
||||
--host 127.0.0.1 \
|
||||
--port 8000 \
|
||||
--scene 1 \
|
||||
--episodes 1 \
|
||||
--headless \
|
||||
--device cuda:1
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
try:
|
||||
import mediapy
|
||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
||||
raise ImportError("Optional DROID sim-eval client requires `mediapy`.") from exc
|
||||
|
||||
try:
|
||||
from typing import override
|
||||
except ImportError:
|
||||
try:
|
||||
from typing_extensions import override
|
||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
||||
raise ImportError("Optional DROID sim-eval client requires `typing-extensions` on Python < 3.12.") from exc
|
||||
|
||||
try:
|
||||
import websockets.sync.client
|
||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
||||
raise ImportError("Optional DROID sim-eval client requires `websockets`.") from exc
|
||||
|
||||
# NOTE:
|
||||
# This directory already contains a local file named `openpi_client.py`.
|
||||
# However, what we want here is the *installed* `openpi_client` package from
|
||||
# upstream OpenPI, not the sibling example file. When a script is executed
|
||||
# directly, Python often puts the script directory at `sys.path[0]`, which
|
||||
# would cause `import openpi_client` to resolve to the local example file and
|
||||
# create a circular import.
|
||||
#
|
||||
# To avoid that ambiguity, temporarily remove the current example directory
|
||||
# from the front of `sys.path`, import the real package, and then restore the
|
||||
# path afterwards.
|
||||
example_dir = str(Path(__file__).resolve().parent)
|
||||
removed_path = False
|
||||
if sys.path and sys.path[0] == example_dir:
|
||||
sys.path.pop(0)
|
||||
removed_path = True
|
||||
try:
|
||||
from openpi_client import image_tools, msgpack_numpy
|
||||
from openpi_client.base_policy import BasePolicy
|
||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
||||
raise ImportError("Optional DROID sim-eval client requires the `openpi-client` package.") from exc
|
||||
finally:
|
||||
if removed_path:
|
||||
sys.path.insert(0, example_dir)
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Constants
|
||||
# -----------------------------------------------------------------------------
|
||||
#
|
||||
# These values intentionally mirror the upstream DreamZero sim-eval client
|
||||
# where possible. The important distinction is:
|
||||
#
|
||||
# - ACTION_HORIZON = 24
|
||||
# The model returns 24 future actions per inference call.
|
||||
# - DEFAULT_OPEN_LOOP_HORIZON = 8
|
||||
# The sim client only executes the first 8 actions locally before asking
|
||||
# the server to replan from a fresh observation.
|
||||
#
|
||||
# So a single server call predicts 24x8 actions, but the rollout consumes only
|
||||
# 8 of them before replanning.
|
||||
PING_INTERVAL_SECS = 60
|
||||
PING_TIMEOUT_SECS = 600
|
||||
DEFAULT_PATH = "/v1/realtime/robot/openpi"
|
||||
DEFAULT_OPEN_LOOP_HORIZON = 8
|
||||
ACTION_HORIZON = 24
|
||||
ACTION_DIM = 8
|
||||
DEFAULT_OUTPUT_ROOT = Path("runs") / "dreamzero_sim_eval"
|
||||
SCENE_PROMPTS = {
|
||||
1: "put the cube in the bowl",
|
||||
2: "pick up the can and put it in the mug",
|
||||
3: "put the banana in the bin",
|
||||
}
|
||||
|
||||
|
||||
def _decode_action_response(response: bytes | str) -> np.ndarray:
|
||||
if isinstance(response, str):
|
||||
raise RuntimeError(f"Error in inference server:\n{response}")
|
||||
decoded = msgpack_numpy.unpackb(response)
|
||||
if isinstance(decoded, dict) and decoded.get("type") == "error":
|
||||
message = decoded.get("message", decoded)
|
||||
raise RuntimeError(f"Error in inference server:\n{message}")
|
||||
return np.asarray(decoded, dtype=np.float32)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StepRecord:
|
||||
"""One fully materialized rollout step for later JSON export.
|
||||
|
||||
The `episode_00.json` artifact is intended to be human-readable and
|
||||
post-process-friendly. Instead of keeping raw tensors around, each step is
|
||||
flattened into plain Python types so it can be serialized directly.
|
||||
"""
|
||||
|
||||
# Index of this control step within the episode.
|
||||
step_index: int
|
||||
# Whether this step triggered a fresh model call (as opposed to reusing the
|
||||
# cached open-loop chunk from the previous server response).
|
||||
used_server_call: bool
|
||||
# End-to-end latency of the server call that produced the current chunk.
|
||||
# This is `None` on steps that only reuse cached actions.
|
||||
chunk_latency_s: float | None
|
||||
# The concrete 8-D action sent into the simulator at this step.
|
||||
action: list[float]
|
||||
# Observed 7-DoF arm joint positions before the next environment step.
|
||||
joint_position: list[float]
|
||||
# Observed gripper scalar before the next environment step.
|
||||
gripper_position: list[float]
|
||||
# Reward and termination signals directly returned by the simulator.
|
||||
reward: float
|
||||
terminated: bool
|
||||
truncated: bool
|
||||
# Optional scene object positions for downstream debugging / success
|
||||
# heuristics. This may be empty if the environment does not expose them.
|
||||
scene_objects: dict[str, list[float]]
|
||||
|
||||
|
||||
class OpenPIWebsocketClientPolicy(BasePolicy):
|
||||
"""Minimal websocket client for the DreamZero/OpenPI policy protocol.
|
||||
|
||||
Protocol shape:
|
||||
- connect -> server immediately sends a metadata dict
|
||||
- infer -> send msgpack observation, receive action chunk
|
||||
- reset -> send msgpack reset command, receive confirmation string
|
||||
|
||||
This class intentionally stays very small because the more interesting
|
||||
DreamZero-specific behavior lives one layer above, in
|
||||
`DreamZeroJointPosClient`.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
host: str = "127.0.0.1",
|
||||
port: int = 8000,
|
||||
path: str = DEFAULT_PATH,
|
||||
) -> None:
|
||||
# vLLM serves the robot endpoint under `/v1/realtime/robot/openpi`.
|
||||
self._uri = f"ws://{host}:{port}{path}"
|
||||
# Upstream protocol uses msgpack with numpy support, not JSON.
|
||||
self._packer = msgpack_numpy.Packer()
|
||||
# Connect immediately and cache the server handshake metadata.
|
||||
self._ws, self._server_metadata = self._wait_for_server()
|
||||
|
||||
def get_server_metadata(self) -> dict[str, Any]:
|
||||
"""Return a copy of the server handshake metadata."""
|
||||
|
||||
return dict(self._server_metadata)
|
||||
|
||||
def _wait_for_server(self):
|
||||
"""Connect to the websocket server and read the initial metadata frame."""
|
||||
|
||||
logging.info("Connecting to %s", self._uri)
|
||||
conn = websockets.sync.client.connect(
|
||||
self._uri,
|
||||
compression=None,
|
||||
max_size=None,
|
||||
ping_interval=PING_INTERVAL_SECS,
|
||||
ping_timeout=PING_TIMEOUT_SECS,
|
||||
)
|
||||
metadata = msgpack_numpy.unpackb(conn.recv())
|
||||
if not isinstance(metadata, dict):
|
||||
raise TypeError(f"Expected dict metadata from server, got {type(metadata)!r}")
|
||||
return conn, metadata
|
||||
|
||||
@override
|
||||
def infer(self, obs: dict[str, Any]) -> np.ndarray:
|
||||
"""Send an inference request and return the decoded action chunk."""
|
||||
|
||||
# Keep the upstream DreamZero/OpenPI convention that the request itself
|
||||
# tells the server which logical endpoint is being called.
|
||||
payload = dict(obs)
|
||||
payload["endpoint"] = "infer"
|
||||
self._ws.send(self._packer.pack(payload))
|
||||
response = self._ws.recv()
|
||||
return _decode_action_response(response)
|
||||
|
||||
@override
|
||||
def reset(self, reset_info: dict[str, Any] | None = None) -> str:
|
||||
"""Tell the server to reset its session-side state."""
|
||||
|
||||
payload = dict(reset_info or {})
|
||||
payload["endpoint"] = "reset"
|
||||
self._ws.send(self._packer.pack(payload))
|
||||
response = self._ws.recv()
|
||||
if isinstance(response, str):
|
||||
return response
|
||||
decoded = msgpack_numpy.unpackb(response)
|
||||
if not isinstance(decoded, dict) or decoded.get("status") != "reset successful":
|
||||
raise RuntimeError(f"Unexpected reset response: {decoded!r}")
|
||||
return str(decoded["status"])
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the websocket connection explicitly."""
|
||||
|
||||
self._ws.close()
|
||||
|
||||
|
||||
class DreamZeroJointPosClient:
|
||||
"""DROID sim-eval client that talks to the vLLM OpenPI websocket server.
|
||||
|
||||
This is the main compatibility layer between:
|
||||
- Isaac Lab DROID observations (`obs["policy"][...]`)
|
||||
- DreamZero/OpenPI websocket payloads
|
||||
- local open-loop action reuse across several simulator steps
|
||||
|
||||
In other words:
|
||||
simulator obs -> websocket request -> action chunk -> one action per step
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
remote_host: str = "127.0.0.1",
|
||||
remote_port: int = 8000,
|
||||
path: str = DEFAULT_PATH,
|
||||
open_loop_horizon: int = DEFAULT_OPEN_LOOP_HORIZON,
|
||||
) -> None:
|
||||
# Low-level transport client.
|
||||
self.client = OpenPIWebsocketClientPolicy(remote_host, remote_port, path=path)
|
||||
# Number of actions to execute locally before replanning.
|
||||
self.open_loop_horizon = open_loop_horizon
|
||||
# Cursor into the currently cached action chunk.
|
||||
self.actions_from_chunk_completed = 0
|
||||
# Most recent `(ACTION_HORIZON, ACTION_DIM)` server response.
|
||||
self.pred_action_chunk: np.ndarray | None = None
|
||||
# Session id is part of the DreamZero serving contract. Changing it
|
||||
# causes the server side to treat the rollout as a fresh episode.
|
||||
self.session_id = str(uuid.uuid4())
|
||||
# Simple runtime stats for reporting.
|
||||
self.server_calls = 0
|
||||
self.last_chunk_latency_s: float | None = None
|
||||
self.last_used_server_call = False
|
||||
|
||||
def metadata(self) -> dict[str, Any]:
|
||||
"""Expose the server metadata to callers / logs."""
|
||||
|
||||
return self.client.get_server_metadata()
|
||||
|
||||
def reset(self) -> str:
|
||||
"""Reset local chunk state and remote session state.
|
||||
|
||||
Local reset:
|
||||
- drop cached action chunk
|
||||
- rewind chunk cursor
|
||||
- allocate a fresh session id
|
||||
|
||||
Remote reset:
|
||||
- send a websocket `reset` message so the server can clear any
|
||||
request/session-side state it associates with this client
|
||||
"""
|
||||
|
||||
self.actions_from_chunk_completed = 0
|
||||
self.pred_action_chunk = None
|
||||
self.session_id = str(uuid.uuid4())
|
||||
self.last_chunk_latency_s = None
|
||||
self.last_used_server_call = False
|
||||
return self.client.reset({})
|
||||
|
||||
def infer(self, obs: dict[str, Any], instruction: str) -> dict[str, Any]:
|
||||
"""Turn one simulator observation into one executable 8-D action.
|
||||
|
||||
Key behavior:
|
||||
- call the server only when the local chunk cache is empty/exhausted
|
||||
- otherwise, keep consuming the cached chunk open-loop
|
||||
- always return exactly one 8-D action for the current simulator step
|
||||
"""
|
||||
|
||||
# Convert Isaac Lab observation structure into a plain numpy-friendly
|
||||
# record that is easier to serialize and visualize.
|
||||
curr_obs = self._extract_observation(obs)
|
||||
self.last_used_server_call = False
|
||||
|
||||
# Replan if:
|
||||
# 1. this is the first step of a rollout / chunk
|
||||
# 2. we already consumed `open_loop_horizon` actions from the current chunk
|
||||
# 3. no cached chunk is currently available
|
||||
if (
|
||||
self.actions_from_chunk_completed == 0
|
||||
or self.actions_from_chunk_completed >= self.open_loop_horizon
|
||||
or self.pred_action_chunk is None
|
||||
):
|
||||
self.actions_from_chunk_completed = 0
|
||||
# Build the exact DreamZero/OpenPI payload expected by the server.
|
||||
#
|
||||
# Notes:
|
||||
# - images are resized/padded to the serving contract's 180x320
|
||||
# - proprio is cast to float64 to match upstream client behavior
|
||||
# - cartesian_position is currently unused by DreamZero DROID, so
|
||||
# a dummy zero vector is sent for protocol completeness
|
||||
request_data = {
|
||||
"observation/exterior_image_0_left": image_tools.resize_with_pad(curr_obs["right_image"], 180, 320),
|
||||
"observation/exterior_image_1_left": image_tools.resize_with_pad(curr_obs["left_image"], 180, 320),
|
||||
"observation/wrist_image_left": image_tools.resize_with_pad(curr_obs["wrist_image"], 180, 320),
|
||||
"observation/joint_position": curr_obs["joint_position"].astype(np.float64),
|
||||
"observation/cartesian_position": np.zeros((6,), dtype=np.float64),
|
||||
"observation/gripper_position": curr_obs["gripper_position"].astype(np.float64),
|
||||
"prompt": instruction,
|
||||
"session_id": self.session_id,
|
||||
}
|
||||
|
||||
# Measure end-to-end server latency for this chunk request.
|
||||
start = time.perf_counter()
|
||||
actions = self.client.infer(request_data)
|
||||
self.last_chunk_latency_s = time.perf_counter() - start
|
||||
self.last_used_server_call = True
|
||||
self.server_calls += 1
|
||||
|
||||
# DreamZero DROID serving is expected to return an action chunk with
|
||||
# 24 future actions, each action being 8-D.
|
||||
if actions.ndim != 2:
|
||||
raise AssertionError(f"Expected 2D action array, got shape {actions.shape}")
|
||||
if actions.shape != (ACTION_HORIZON, ACTION_DIM):
|
||||
raise AssertionError(f"Expected action shape {(ACTION_HORIZON, ACTION_DIM)}, got {actions.shape}")
|
||||
self.pred_action_chunk = actions
|
||||
|
||||
# Consume exactly one action row from the cached chunk for this
|
||||
# simulator step.
|
||||
action = np.array(self.pred_action_chunk[self.actions_from_chunk_completed], copy=True)
|
||||
self.actions_from_chunk_completed += 1
|
||||
|
||||
# Upstream DreamZero sim-eval binarizes the gripper command.
|
||||
action[-1] = 1.0 if action[-1].item() > 0.5 else 0.0
|
||||
|
||||
# Produce a human-friendly visualization strip for videos:
|
||||
# right external | wrist | left external
|
||||
img1 = image_tools.resize_with_pad(curr_obs["right_image"], 224, 224)
|
||||
img2 = image_tools.resize_with_pad(curr_obs["wrist_image"], 224, 224)
|
||||
img3 = image_tools.resize_with_pad(curr_obs["left_image"], 224, 224)
|
||||
viz = np.concatenate([img1, img2, img3], axis=1)
|
||||
|
||||
# Return both the executable action and auxiliary debug info.
|
||||
return {
|
||||
"action": action,
|
||||
"viz": viz,
|
||||
"joint_position": curr_obs["joint_position"],
|
||||
"gripper_position": curr_obs["gripper_position"],
|
||||
"used_server_call": self.last_used_server_call,
|
||||
"chunk_latency_s": self.last_chunk_latency_s if self.last_used_server_call else None,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _extract_observation(obs_dict: dict[str, Any]) -> dict[str, np.ndarray]:
|
||||
"""Extract the pieces DreamZero cares about from Isaac Lab observations.
|
||||
|
||||
`sim-evals` exposes camera frames and robot state inside the
|
||||
`obs["policy"]` group. This helper converts those tensors into numpy
|
||||
arrays so they can be fed into image preprocessing / websocket packing.
|
||||
"""
|
||||
|
||||
policy = obs_dict["policy"]
|
||||
# Isaac Lab stores camera observations as batched tensors; use env 0.
|
||||
right_image = policy["external_cam"][0].clone().detach().cpu().numpy()
|
||||
left_image = policy["external_cam_2"][0].clone().detach().cpu().numpy()
|
||||
wrist_image = policy["wrist_cam"][0].clone().detach().cpu().numpy()
|
||||
# Robot proprioception.
|
||||
joint_position = policy["arm_joint_pos"].clone().detach().cpu().numpy()
|
||||
gripper_position = policy["gripper_pos"].clone().detach().cpu().numpy()
|
||||
|
||||
return {
|
||||
"right_image": right_image,
|
||||
"left_image": left_image,
|
||||
"wrist_image": wrist_image,
|
||||
"joint_position": joint_position,
|
||||
"gripper_position": gripper_position,
|
||||
}
|
||||
|
||||
|
||||
def _scene_instruction(scene: int) -> str:
|
||||
"""Map a numeric scene id onto the fixed language prompt used for rollout."""
|
||||
|
||||
try:
|
||||
return SCENE_PROMPTS[scene]
|
||||
except KeyError as exc:
|
||||
raise ValueError(f"Unsupported scene {scene}. Available scenes: {sorted(SCENE_PROMPTS)}") from exc
|
||||
|
||||
|
||||
def _capture_scene_objects(env: Any) -> dict[str, list[float]]:
|
||||
"""Best-effort extraction of scene object root positions.
|
||||
|
||||
This is only for debugging / reporting. The rollout logic itself does not
|
||||
depend on these positions.
|
||||
"""
|
||||
|
||||
objects: dict[str, list[float]] = {}
|
||||
scene = getattr(env, "scene", None)
|
||||
if scene is None:
|
||||
return objects
|
||||
|
||||
# Skip non-task entities such as cameras, the robot, and lighting.
|
||||
for name in scene.keys():
|
||||
if name in {"robot", "external_cam", "external_cam_2", "wrist_cam", "sphere_light", "scene"}:
|
||||
continue
|
||||
entity = scene[name]
|
||||
data = getattr(entity, "data", None)
|
||||
root_pos_w = getattr(data, "root_pos_w", None)
|
||||
if root_pos_w is None:
|
||||
continue
|
||||
# Convert tensors to plain lists for JSON serialization.
|
||||
value = root_pos_w[0].detach().cpu().to(torch.float32).tolist()
|
||||
objects[str(name)] = [float(x) for x in value]
|
||||
return objects
|
||||
|
||||
|
||||
def _maybe_infer_success(scene: int, final_objects: dict[str, list[float]]) -> dict[str, Any]:
|
||||
"""Best-effort geometric heuristic.
|
||||
|
||||
The simulator itself does not expose a built-in success term; this function
|
||||
provides a transparent fallback for human-readable reporting only.
|
||||
"""
|
||||
|
||||
task_pairs = {
|
||||
1: ("cube", "bowl"),
|
||||
2: ("can", "mug"),
|
||||
3: ("banana", "bin"),
|
||||
}
|
||||
source_name, target_name = task_pairs.get(scene, (None, None))
|
||||
if source_name not in final_objects or target_name not in final_objects:
|
||||
return {
|
||||
"has_builtin_success": False,
|
||||
"heuristic_success": None,
|
||||
"reason": "scene object names unavailable for heuristic",
|
||||
}
|
||||
|
||||
source = np.asarray(final_objects[source_name], dtype=np.float32)
|
||||
target = np.asarray(final_objects[target_name], dtype=np.float32)
|
||||
xy_distance = float(np.linalg.norm(source[:2] - target[:2]))
|
||||
z_delta = float(source[2] - target[2])
|
||||
heuristic_success = bool(xy_distance < 0.12 and z_delta > -0.08)
|
||||
return {
|
||||
"has_builtin_success": False,
|
||||
"heuristic_success": heuristic_success,
|
||||
"xy_distance": xy_distance,
|
||||
"z_delta": z_delta,
|
||||
"source_object": source_name,
|
||||
"target_object": target_name,
|
||||
"reason": "no built-in env success flag; using final object pose heuristic",
|
||||
}
|
||||
|
||||
|
||||
def _dump_json(path: Path, payload: dict[str, Any]) -> None:
|
||||
"""Write a JSON file with stable UTF-8 formatting."""
|
||||
|
||||
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
|
||||
def _scalar_from_env_value(value: Any) -> float:
|
||||
"""Normalize simulator scalar outputs into a plain Python float.
|
||||
|
||||
Isaac Lab / Gym values may come back as tensors, numpy arrays, tuples, or
|
||||
direct Python scalars depending on the wrapper stack. Centralizing the
|
||||
conversion here makes the rollout loop cleaner and more robust.
|
||||
"""
|
||||
|
||||
if isinstance(value, torch.Tensor):
|
||||
return float(value.reshape(-1)[0].detach().cpu().item())
|
||||
if isinstance(value, np.ndarray):
|
||||
return float(value.reshape(-1)[0])
|
||||
if isinstance(value, (list, tuple)):
|
||||
return float(np.asarray(value).reshape(-1)[0])
|
||||
return float(value)
|
||||
|
||||
|
||||
def _bool_from_env_value(value: Any) -> bool:
|
||||
"""Normalize simulator boolean-like outputs into a plain Python bool."""
|
||||
|
||||
if isinstance(value, torch.Tensor):
|
||||
return bool(value.reshape(-1)[0].detach().cpu().item())
|
||||
if isinstance(value, np.ndarray):
|
||||
return bool(value.reshape(-1)[0])
|
||||
if isinstance(value, (list, tuple)):
|
||||
return bool(np.asarray(value).reshape(-1)[0])
|
||||
return bool(value)
|
||||
|
||||
|
||||
def _make_output_dir(output_root: Path, scene: int) -> Path:
|
||||
"""Create a timestamped output directory for one sim-eval run."""
|
||||
|
||||
timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
||||
output_dir = output_root / f"scene{scene}_{timestamp}"
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
return output_dir
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Entry point for one or more DROID simulation rollouts.
|
||||
|
||||
High-level flow:
|
||||
1. parse command-line flags
|
||||
2. bootstrap Isaac Lab / sim-evals imports
|
||||
3. create the DROID environment
|
||||
4. connect the DreamZero websocket client
|
||||
5. run `episodes` rollouts
|
||||
6. export videos + JSON summaries
|
||||
"""
|
||||
|
||||
# Script-level arguments.
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--episodes", type=int, default=1, help="Number of episodes to run.")
|
||||
parser.add_argument("--scene", type=int, default=1, help="DROID scene id (1/2/3).")
|
||||
parser.add_argument("--host", type=str, default="127.0.0.1", help="vLLM DreamZero server host.")
|
||||
parser.add_argument("--port", type=int, default=8000, help="vLLM DreamZero server port.")
|
||||
parser.add_argument("--path", type=str, default=DEFAULT_PATH, help="Websocket path suffix.")
|
||||
parser.add_argument(
|
||||
"--open-loop-horizon",
|
||||
type=int,
|
||||
default=DEFAULT_OPEN_LOOP_HORIZON,
|
||||
help="How many actions to consume locally before requesting the next chunk.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-root",
|
||||
type=Path,
|
||||
default=DEFAULT_OUTPUT_ROOT,
|
||||
help="Directory where videos and trajectory logs are stored.",
|
||||
)
|
||||
|
||||
try:
|
||||
from isaaclab.app import AppLauncher
|
||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
||||
raise ImportError(
|
||||
"Optional DROID sim-eval client requires Isaac Lab (`isaaclab`). "
|
||||
"Launch it from an Isaac Lab environment, e.g. via `isaaclab.sh -p`."
|
||||
) from exc
|
||||
|
||||
# Let Isaac Lab inject its own runtime flags (e.g. headless, device).
|
||||
AppLauncher.add_app_launcher_args(parser)
|
||||
args = parser.parse_args()
|
||||
|
||||
# DreamZero sim-eval always needs camera observations enabled.
|
||||
args.enable_cameras = True
|
||||
# Boot Isaac Sim / Isaac Lab.
|
||||
app_launcher = AppLauncher(args)
|
||||
simulation_app = app_launcher.app
|
||||
# Set defaults so the `finally` block can clean up safely even if an
|
||||
# earlier step fails.
|
||||
env = None
|
||||
client = None
|
||||
|
||||
# Import simulator modules only *after* the app is launched. This matches
|
||||
# Isaac Lab's required import ordering.
|
||||
try:
|
||||
import gymnasium as gym
|
||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
||||
raise ImportError("Optional DROID sim-eval client requires `gymnasium`.") from exc
|
||||
|
||||
try:
|
||||
import sim_evals.environments # noqa: F401
|
||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
||||
raise ImportError(
|
||||
"Optional DROID sim-eval client requires the external `sim-evals` package or checkout to be importable."
|
||||
) from exc
|
||||
|
||||
try:
|
||||
from isaaclab_tasks.utils import parse_env_cfg
|
||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
||||
raise ImportError("Optional DROID sim-eval client requires `isaaclab_tasks`.") from exc
|
||||
|
||||
# Resolve output location and scene prompt.
|
||||
output_dir = _make_output_dir(args.output_root.expanduser().resolve(), args.scene)
|
||||
instruction = _scene_instruction(args.scene)
|
||||
|
||||
# Build the DROID environment configuration from `sim-evals`.
|
||||
env_cfg = parse_env_cfg(
|
||||
"DROID",
|
||||
device=args.device,
|
||||
num_envs=1,
|
||||
use_fabric=True,
|
||||
)
|
||||
# Select one of the pre-authored scenes/tasks.
|
||||
env_cfg.set_scene(args.scene)
|
||||
env = gym.make("DROID", cfg=env_cfg)
|
||||
|
||||
# Upstream sim-evals resets twice so materials / cameras are fully ready.
|
||||
obs, _ = env.reset()
|
||||
obs, _ = env.reset()
|
||||
|
||||
# Connect the websocket policy client.
|
||||
client = DreamZeroJointPosClient(
|
||||
remote_host=args.host,
|
||||
remote_port=args.port,
|
||||
path=args.path,
|
||||
open_loop_horizon=args.open_loop_horizon,
|
||||
)
|
||||
|
||||
# Aggregated per-run results.
|
||||
all_episode_summaries: list[dict[str, Any]] = []
|
||||
max_steps = int(env.env.max_episode_length)
|
||||
logging.info("DreamZero metadata: %s", client.metadata())
|
||||
logging.info("Scene %s prompt: %s", args.scene, instruction)
|
||||
logging.info("Writing outputs to %s", output_dir)
|
||||
|
||||
try:
|
||||
# No gradients are needed in inference-only rollout mode.
|
||||
with torch.no_grad():
|
||||
for episode_index in range(args.episodes):
|
||||
# Per-episode collectors.
|
||||
frames: list[np.ndarray] = []
|
||||
step_records: list[StepRecord] = []
|
||||
episode_start = time.perf_counter()
|
||||
server_time_s = 0.0
|
||||
final_reward = 0.0
|
||||
terminated = False
|
||||
truncated = False
|
||||
|
||||
for step_index in range(max_steps):
|
||||
# Ask the policy for the next action. Internally this may or
|
||||
# may not trigger a real server request depending on whether
|
||||
# the local chunk cache has been exhausted.
|
||||
logging.debug("Episode %d step %d: requesting action", episode_index, step_index)
|
||||
result = client.infer(obs, instruction)
|
||||
logging.debug(
|
||||
"Episode %d step %d: got action (server_call=%s latency=%s)",
|
||||
episode_index,
|
||||
step_index,
|
||||
result["used_server_call"],
|
||||
result["chunk_latency_s"],
|
||||
)
|
||||
# Save one visualization frame per simulator step.
|
||||
frames.append(result["viz"])
|
||||
|
||||
# Isaac Lab expects batched actions, hence `[None]`.
|
||||
action_tensor = torch.tensor(result["action"], dtype=torch.float32)[None]
|
||||
logging.debug("Episode %d step %d: stepping env", episode_index, step_index)
|
||||
obs, reward, term, trunc, info = env.step(action_tensor)
|
||||
logging.debug("Episode %d step %d: env.step returned", episode_index, step_index)
|
||||
logging.debug("Episode %d step %d: parsing reward/flags", episode_index, step_index)
|
||||
logging.debug(
|
||||
"Episode %d step %d: raw types reward=%s term=%s trunc=%s",
|
||||
episode_index,
|
||||
step_index,
|
||||
type(reward).__name__,
|
||||
type(term).__name__,
|
||||
type(trunc).__name__,
|
||||
)
|
||||
# Normalize environment outputs into plain Python scalars so
|
||||
# the rest of the code does not depend on wrapper-specific types.
|
||||
reward_value = _scalar_from_env_value(reward)
|
||||
term_value = _bool_from_env_value(term)
|
||||
trunc_value = _bool_from_env_value(trunc)
|
||||
logging.debug(
|
||||
"Episode %d step %d: parsed reward=%s term=%s trunc=%s",
|
||||
episode_index,
|
||||
step_index,
|
||||
reward_value,
|
||||
term_value,
|
||||
trunc_value,
|
||||
)
|
||||
# Keep scene-object capture optional. It is useful for
|
||||
# debugging / success heuristics, but the rollout should not
|
||||
# fail if the environment does not expose object roots.
|
||||
scene_objects = _capture_scene_objects(env)
|
||||
|
||||
# Accumulate total server-side time only on steps that
|
||||
# triggered a fresh chunk inference.
|
||||
if result["chunk_latency_s"] is not None:
|
||||
server_time_s += float(result["chunk_latency_s"])
|
||||
|
||||
# Materialize one JSON-serializable trajectory record.
|
||||
logging.debug("Episode %d step %d: appending trajectory", episode_index, step_index)
|
||||
step_records.append(
|
||||
StepRecord(
|
||||
step_index=step_index,
|
||||
used_server_call=bool(result["used_server_call"]),
|
||||
chunk_latency_s=(
|
||||
float(result["chunk_latency_s"]) if result["chunk_latency_s"] is not None else None
|
||||
),
|
||||
action=[float(x) for x in np.asarray(result["action"], dtype=np.float32).tolist()],
|
||||
joint_position=[
|
||||
float(x) for x in np.asarray(result["joint_position"], dtype=np.float32).tolist()
|
||||
],
|
||||
gripper_position=[
|
||||
float(x) for x in np.asarray(result["gripper_position"], dtype=np.float32).tolist()
|
||||
],
|
||||
reward=reward_value,
|
||||
terminated=term_value,
|
||||
truncated=trunc_value,
|
||||
scene_objects=scene_objects,
|
||||
)
|
||||
)
|
||||
logging.debug("Episode %d step %d: trajectory appended", episode_index, step_index)
|
||||
|
||||
# Track final status for summary export.
|
||||
final_reward = reward_value
|
||||
terminated = term_value
|
||||
truncated = trunc_value
|
||||
if term_value or trunc_value:
|
||||
# End the rollout early if the environment terminates.
|
||||
break
|
||||
|
||||
# Episode-level timing and video export.
|
||||
episode_wall_time_s = time.perf_counter() - episode_start
|
||||
video_path = output_dir / f"episode_{episode_index:02d}.mp4"
|
||||
logging.info("Episode %d: writing video to %s", episode_index, video_path)
|
||||
mediapy.write_video(video_path, frames, fps=15)
|
||||
|
||||
# Reset the policy server between episodes.
|
||||
logging.info("Episode %d: sending reset", episode_index)
|
||||
reset_response = client.reset()
|
||||
final_objects = step_records[-1].scene_objects if step_records else {}
|
||||
success_report = _maybe_infer_success(args.scene, final_objects)
|
||||
|
||||
# Assemble the per-episode summary that is written to
|
||||
# `episode_XX.json`.
|
||||
episode_summary = {
|
||||
"episode_index": episode_index,
|
||||
"prompt": instruction,
|
||||
"video_path": str(video_path),
|
||||
"steps_executed": len(step_records),
|
||||
"max_steps": max_steps,
|
||||
"terminated": terminated,
|
||||
"truncated": truncated,
|
||||
"final_reward": final_reward,
|
||||
"server_calls": client.server_calls,
|
||||
"server_time_s": server_time_s,
|
||||
"episode_wall_time_s": episode_wall_time_s,
|
||||
"avg_server_time_per_call_s": (
|
||||
server_time_s / client.server_calls if client.server_calls else None
|
||||
),
|
||||
"reset_response": reset_response,
|
||||
"success_report": success_report,
|
||||
"server_metadata": client.metadata(),
|
||||
"trajectory": [record.__dict__ for record in step_records],
|
||||
}
|
||||
_dump_json(output_dir / f"episode_{episode_index:02d}.json", episode_summary)
|
||||
all_episode_summaries.append(episode_summary)
|
||||
|
||||
logging.info(
|
||||
"Episode %d done: steps=%d wall=%.2fs server_calls=%d heuristic_success=%s",
|
||||
episode_index,
|
||||
len(step_records),
|
||||
episode_wall_time_s,
|
||||
client.server_calls,
|
||||
success_report.get("heuristic_success"),
|
||||
)
|
||||
|
||||
# Reset per-episode counters while keeping the client alive.
|
||||
client.server_calls = 0
|
||||
|
||||
# Top-level run summary across all episodes.
|
||||
summary = {
|
||||
"scene": args.scene,
|
||||
"prompt": instruction,
|
||||
"episodes": args.episodes,
|
||||
"host": args.host,
|
||||
"port": args.port,
|
||||
"path": args.path,
|
||||
"device": args.device,
|
||||
"output_dir": str(output_dir),
|
||||
"server_metadata": client.metadata(),
|
||||
"episodes_summary": all_episode_summaries,
|
||||
}
|
||||
_dump_json(output_dir / "summary.json", summary)
|
||||
# Also print the summary to stdout so the caller can capture it in logs.
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
finally:
|
||||
# Best-effort cleanup. Avoid masking the main error if cleanup fails.
|
||||
try:
|
||||
if client is not None:
|
||||
client.client.close()
|
||||
except Exception:
|
||||
pass
|
||||
if env is not None:
|
||||
env.close()
|
||||
simulation_app.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Keep the script-level logs readable. Per-step rollout details are still
|
||||
# available via `DEBUG` if needed, but websocket / asyncio internals are
|
||||
# usually too noisy for normal usage.
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logging.getLogger("websockets").setLevel(logging.WARNING)
|
||||
logging.getLogger("asyncio").setLevel(logging.WARNING)
|
||||
main()
|
||||
@@ -0,0 +1,86 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
os.environ.setdefault("MUJOCO_GL", "egl")
|
||||
os.environ.setdefault("PYOPENGL_PLATFORM", "egl")
|
||||
|
||||
# Import base configs at module top level so the subclasses below are pickle-
|
||||
# resolvable (worker processes import this module fresh via __main__).
|
||||
from molmo_spaces.configs.policy_configs_baselines import ( # noqa: E402
|
||||
DreamZeroPolicyConfig,
|
||||
)
|
||||
from molmo_spaces.evaluation.configs.evaluation_configs import ( # noqa: E402
|
||||
DreamZeroPolicyEvalConfig,
|
||||
)
|
||||
|
||||
|
||||
# We only need to change the backend host and port to the vllm-host!
|
||||
class DreamZeroVllmOmniPolicyConfig(DreamZeroPolicyConfig):
|
||||
remote_config: dict = dict(host="127.0.0.1", port=8000)
|
||||
|
||||
|
||||
class DreamZeroVllmOmniEvalConfig(DreamZeroPolicyEvalConfig):
|
||||
policy_config: DreamZeroVllmOmniPolicyConfig = DreamZeroVllmOmniPolicyConfig()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--host", default="127.0.0.1")
|
||||
parser.add_argument("--port", type=int, default=8000)
|
||||
parser.add_argument(
|
||||
"--benchmark_dir",
|
||||
required=True,
|
||||
help=(
|
||||
"Path to a MolmoSpaces benchmark directory, for example "
|
||||
"$MOLMOSPACES_BENCHMARK_DIR/20260327/ithor/FrankaCloseHardBench/"
|
||||
"FrankaCloseHardBench_20260206_json_benchmark"
|
||||
),
|
||||
)
|
||||
parser.add_argument("--max_episodes", type=int, default=1)
|
||||
parser.add_argument("--task_horizon_steps", type=int, default=80)
|
||||
parser.add_argument(
|
||||
"--output_dir",
|
||||
required=True,
|
||||
help="Directory to write evaluation outputs (created if missing).",
|
||||
)
|
||||
parser.add_argument("--episode_idx", type=int, default=None)
|
||||
args = parser.parse_args()
|
||||
|
||||
policy_config = DreamZeroVllmOmniPolicyConfig(remote_config=dict(host=args.host, port=args.port))
|
||||
DreamZeroVllmOmniPolicyConfig.model_fields["remote_config"].default = policy_config.remote_config
|
||||
DreamZeroVllmOmniEvalConfig.model_fields["policy_config"].default = policy_config
|
||||
|
||||
# Import after env vars are set so MuJoCo picks EGL.
|
||||
from molmo_spaces.evaluation import run_evaluation
|
||||
|
||||
cfg_cls = DreamZeroVllmOmniEvalConfig
|
||||
|
||||
output_dir = args.output_dir
|
||||
Path(output_dir).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
print(f"[eval] benchmark_dir={args.benchmark_dir}")
|
||||
print(f"[eval] max_episodes={args.max_episodes} task_horizon_steps={args.task_horizon_steps}")
|
||||
print(f"[eval] remote policy: ws://{args.host}:{args.port}/v1/realtime/robot/openpi")
|
||||
|
||||
results = run_evaluation(
|
||||
eval_config_cls=cfg_cls,
|
||||
benchmark_dir=Path(args.benchmark_dir),
|
||||
max_episodes=args.max_episodes,
|
||||
task_horizon_steps=args.task_horizon_steps,
|
||||
num_workers=1,
|
||||
use_wandb=False,
|
||||
output_dir=output_dir,
|
||||
episode_idx=args.episode_idx,
|
||||
)
|
||||
|
||||
print(f"[eval] success={results.success_count}/{results.total_count} ({results.success_rate:.1%})")
|
||||
print(f"[eval] output_dir={results.output_dir}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+311
@@ -0,0 +1,311 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
try:
|
||||
import cv2
|
||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
||||
raise ImportError("DreamZero OpenPI example requires `opencv-python`.") from exc
|
||||
|
||||
try:
|
||||
example_dir = str(Path(__file__).resolve().parent)
|
||||
removed_path = False
|
||||
if sys.path and sys.path[0] == example_dir:
|
||||
sys.path.pop(0)
|
||||
removed_path = True
|
||||
try:
|
||||
from openpi_client.websocket_client_policy import WebsocketClientPolicy
|
||||
finally:
|
||||
if removed_path:
|
||||
sys.path.insert(0, example_dir)
|
||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
||||
raise ImportError("DreamZero OpenPI example requires `openpi-client`.") from exc
|
||||
|
||||
DEFAULT_HOST = "127.0.0.1"
|
||||
DEFAULT_PORT = 8000
|
||||
DEFAULT_PATH = "/v1/realtime/robot/openpi"
|
||||
DEFAULT_PROMPT = "Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan"
|
||||
ACTION_HORIZON = 24
|
||||
DEFAULT_ACTION_DIM = 8
|
||||
RELATIVE_OFFSETS = [-23, -16, -8, 0]
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
ASSET_REPO_ID = "YangshenDeng/vllm-omni-dreamzero-assets"
|
||||
DEFAULT_VIDEO_DIR = REPO_ROOT / "outputs" / "dreamzero" / "assets"
|
||||
CAMERA_FILES = {
|
||||
"observation/exterior_image_0_left": "exterior_image_1_left.mp4",
|
||||
"observation/exterior_image_1_left": "exterior_image_2_left.mp4",
|
||||
"observation/wrist_image_left": "wrist_image_left.mp4",
|
||||
}
|
||||
|
||||
|
||||
def _as_action_array(response: Any) -> np.ndarray:
|
||||
if isinstance(response, dict) and response.get("type") == "error":
|
||||
message = response.get("message", response)
|
||||
raise RuntimeError(f"Inference failed: {message}")
|
||||
return np.asarray(response, dtype=np.float32)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DreamZeroServerMetadata:
|
||||
image_resolution: tuple[int, int]
|
||||
n_external_cameras: int
|
||||
needs_wrist_camera: bool
|
||||
needs_stereo_camera: bool
|
||||
needs_session_id: bool
|
||||
action_space: str
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, payload: dict[str, Any]) -> DreamZeroServerMetadata:
|
||||
required_keys = (
|
||||
"image_resolution",
|
||||
"n_external_cameras",
|
||||
"needs_wrist_camera",
|
||||
"needs_stereo_camera",
|
||||
"needs_session_id",
|
||||
"action_space",
|
||||
)
|
||||
missing_keys = [key for key in required_keys if key not in payload]
|
||||
if missing_keys:
|
||||
raise ValueError(f"Missing DreamZero metadata keys: {missing_keys}")
|
||||
|
||||
image_resolution = payload["image_resolution"]
|
||||
if not isinstance(image_resolution, (list, tuple)) or len(image_resolution) != 2:
|
||||
raise ValueError(f"Invalid image_resolution: {image_resolution!r}")
|
||||
|
||||
return cls(
|
||||
image_resolution=(int(image_resolution[0]), int(image_resolution[1])),
|
||||
n_external_cameras=int(payload["n_external_cameras"]),
|
||||
needs_wrist_camera=bool(payload["needs_wrist_camera"]),
|
||||
needs_stereo_camera=bool(payload["needs_stereo_camera"]),
|
||||
needs_session_id=bool(payload["needs_session_id"]),
|
||||
action_space=str(payload["action_space"]),
|
||||
)
|
||||
|
||||
|
||||
def _openpi_uri(host: str, port: int, path: str) -> str:
|
||||
if host.startswith(("ws://", "wss://")):
|
||||
return host
|
||||
return f"ws://{host}:{port}{path}"
|
||||
|
||||
|
||||
def _close_policy_client(client: WebsocketClientPolicy) -> None:
|
||||
ws = getattr(client, "_ws", None)
|
||||
if ws is not None:
|
||||
ws.close()
|
||||
|
||||
|
||||
def load_all_frames(video_path: Path) -> np.ndarray:
|
||||
cap = cv2.VideoCapture(str(video_path))
|
||||
frames = []
|
||||
while True:
|
||||
ok, frame = cap.read()
|
||||
if not ok:
|
||||
break
|
||||
frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
|
||||
cap.release()
|
||||
if not frames:
|
||||
raise RuntimeError(f"No frames loaded from {video_path}")
|
||||
return np.stack(frames, axis=0)
|
||||
|
||||
|
||||
def load_camera_frames(video_dir: Path) -> dict[str, np.ndarray]:
|
||||
camera_frames: dict[str, np.ndarray] = {}
|
||||
for camera_key, file_name in CAMERA_FILES.items():
|
||||
video_path = video_dir / file_name
|
||||
if not video_path.exists():
|
||||
raise FileNotFoundError(
|
||||
f"Missing DreamZero example asset: {video_path}. "
|
||||
"Download the example videos with: "
|
||||
f"`hf download {ASSET_REPO_ID} --repo-type dataset --local-dir {video_dir}`"
|
||||
)
|
||||
camera_frames[camera_key] = load_all_frames(video_path)
|
||||
return camera_frames
|
||||
|
||||
|
||||
def build_frame_schedule(total_frames: int, num_chunks: int) -> list[list[int]]:
|
||||
chunks: list[list[int]] = []
|
||||
current_frame = 23
|
||||
for _ in range(num_chunks):
|
||||
indices = [max(current_frame + offset, 0) for offset in RELATIVE_OFFSETS]
|
||||
if indices[-1] >= total_frames:
|
||||
break
|
||||
chunks.append(indices)
|
||||
current_frame += ACTION_HORIZON
|
||||
return chunks
|
||||
|
||||
|
||||
def make_obs_from_video(
|
||||
camera_frames: dict[str, np.ndarray],
|
||||
frame_indices: list[int],
|
||||
*,
|
||||
prompt: str,
|
||||
session_id: str,
|
||||
) -> dict[str, Any]:
|
||||
obs: dict[str, Any] = {}
|
||||
for camera_key, all_frames in camera_frames.items():
|
||||
selected = all_frames[frame_indices]
|
||||
obs[camera_key] = selected[0] if len(frame_indices) == 1 else selected
|
||||
|
||||
obs["observation/joint_position"] = np.zeros(7, dtype=np.float32)
|
||||
obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32)
|
||||
obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32)
|
||||
obs["prompt"] = prompt
|
||||
obs["session_id"] = session_id
|
||||
return obs
|
||||
|
||||
|
||||
def build_demo_observations(
|
||||
camera_frames: dict[str, np.ndarray],
|
||||
*,
|
||||
prompt: str,
|
||||
session_id: str,
|
||||
num_chunks: int = 2,
|
||||
) -> list[dict[str, Any]]:
|
||||
if num_chunks < 1:
|
||||
raise ValueError("num_chunks must be at least 1")
|
||||
|
||||
total_frames = min(frames.shape[0] for frames in camera_frames.values())
|
||||
observations = [
|
||||
make_obs_from_video(
|
||||
camera_frames,
|
||||
[0],
|
||||
prompt=prompt,
|
||||
session_id=session_id,
|
||||
)
|
||||
]
|
||||
for indices in build_frame_schedule(total_frames, num_chunks - 1):
|
||||
observations.append(
|
||||
make_obs_from_video(
|
||||
camera_frames,
|
||||
indices,
|
||||
prompt=prompt,
|
||||
session_id=session_id,
|
||||
)
|
||||
)
|
||||
return observations
|
||||
|
||||
|
||||
def validate_session_result(
|
||||
result: dict[str, Any],
|
||||
*,
|
||||
expected_action_horizon: int = ACTION_HORIZON,
|
||||
expected_action_dim: int = DEFAULT_ACTION_DIM,
|
||||
expected_action_count: int | None = None,
|
||||
) -> None:
|
||||
metadata = DreamZeroServerMetadata.from_dict(result["metadata"])
|
||||
if metadata.image_resolution != (180, 320):
|
||||
raise AssertionError(f"Unexpected image_resolution: {metadata.image_resolution}")
|
||||
if metadata.n_external_cameras != 2:
|
||||
raise AssertionError(f"Unexpected n_external_cameras: {metadata.n_external_cameras}")
|
||||
if not metadata.needs_wrist_camera:
|
||||
raise AssertionError("DreamZero example expects wrist camera metadata")
|
||||
if metadata.action_space != "joint_position":
|
||||
raise AssertionError(f"Unexpected action_space: {metadata.action_space}")
|
||||
|
||||
actions = result["actions"]
|
||||
if expected_action_count is not None and len(actions) != expected_action_count:
|
||||
raise AssertionError(f"Expected {expected_action_count} action tensors, got {len(actions)}")
|
||||
if not actions:
|
||||
raise AssertionError("Expected at least one action tensor")
|
||||
for index, action in enumerate(actions):
|
||||
if action.shape != (expected_action_horizon, expected_action_dim):
|
||||
raise AssertionError(
|
||||
f"Action {index} shape mismatch: expected "
|
||||
f"{(expected_action_horizon, expected_action_dim)}, got {action.shape}"
|
||||
)
|
||||
if not np.isfinite(action).all():
|
||||
raise AssertionError(f"Action {index} contains non-finite values")
|
||||
|
||||
|
||||
def run_policy_session(
|
||||
*,
|
||||
host: str = DEFAULT_HOST,
|
||||
port: int = DEFAULT_PORT,
|
||||
path: str = DEFAULT_PATH,
|
||||
video_dir: Path = DEFAULT_VIDEO_DIR,
|
||||
prompt: str = DEFAULT_PROMPT,
|
||||
session_id: str | None = None,
|
||||
num_chunks: int = 2,
|
||||
) -> dict[str, Any]:
|
||||
session_id = session_id or str(uuid.uuid4())
|
||||
camera_frames = load_camera_frames(video_dir)
|
||||
observations = build_demo_observations(
|
||||
camera_frames,
|
||||
prompt=prompt,
|
||||
session_id=session_id,
|
||||
num_chunks=num_chunks,
|
||||
)
|
||||
|
||||
uri = _openpi_uri(host, port, path)
|
||||
logging.info("Connecting to %s", uri)
|
||||
client = WebsocketClientPolicy(host=uri)
|
||||
try:
|
||||
metadata = dict(client.get_server_metadata())
|
||||
actions = [_as_action_array(client.infer(obs)) for obs in observations]
|
||||
return {
|
||||
"metadata": metadata,
|
||||
"actions": actions,
|
||||
"session_id": session_id,
|
||||
}
|
||||
finally:
|
||||
_close_policy_client(client)
|
||||
|
||||
|
||||
def format_action_summary(index: int, action: np.ndarray) -> str:
|
||||
return (
|
||||
f"Action {index}: shape={tuple(action.shape)} dtype={action.dtype} "
|
||||
f"min={action.min():.6f} max={action.max():.6f}"
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="DreamZero OpenPI client example with downloaded real videos.")
|
||||
parser.add_argument("--host", default=DEFAULT_HOST)
|
||||
parser.add_argument("--port", type=int, default=DEFAULT_PORT)
|
||||
parser.add_argument("--path", default=DEFAULT_PATH)
|
||||
parser.add_argument(
|
||||
"--video-dir", type=Path, default=DEFAULT_VIDEO_DIR, help="Directory containing the three camera MP4 files."
|
||||
)
|
||||
parser.add_argument("--prompt", default=DEFAULT_PROMPT)
|
||||
parser.add_argument("--session-id", default=None)
|
||||
parser.add_argument("--num-chunks", type=int, default=2)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
result = run_policy_session(
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
path=args.path,
|
||||
video_dir=args.video_dir,
|
||||
prompt=args.prompt,
|
||||
session_id=args.session_id,
|
||||
num_chunks=args.num_chunks,
|
||||
)
|
||||
validate_session_result(result, expected_action_count=args.num_chunks)
|
||||
|
||||
print("Server metadata:", json.dumps(result["metadata"], sort_keys=True))
|
||||
for index, action in enumerate(result["actions"]):
|
||||
print(format_action_summary(index, action))
|
||||
print("Session ID:", result["session_id"])
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
MODEL="${MODEL:-GEAR-Dreams/DreamZero-DROID}"
|
||||
HOST="${HOST:-127.0.0.1}"
|
||||
PORT="${PORT:-8000}"
|
||||
DEPLOY_CONFIG="${DEPLOY_CONFIG:-vllm_omni/deploy/dreamzero_tp1_cfg2.yaml}"
|
||||
SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-dreamzero-droid}"
|
||||
|
||||
args=(
|
||||
serve
|
||||
"$MODEL"
|
||||
--omni
|
||||
--host "$HOST"
|
||||
--port "$PORT"
|
||||
--served-model-name "$SERVED_MODEL_NAME"
|
||||
--deploy-config "$DEPLOY_CONFIG"
|
||||
--enforce-eager
|
||||
--disable-log-stats
|
||||
)
|
||||
|
||||
ATTENTION_BACKEND="${ATTENTION_BACKEND:-torch}" \
|
||||
DIFFUSION_ATTENTION_BACKEND="${DIFFUSION_ATTENTION_BACKEND:-TORCH_SDPA}" \
|
||||
vllm "${args[@]}"
|
||||
@@ -22,13 +22,14 @@ export PYTHONPATH=<transformers_modules_path>:$PYTHONPATH
|
||||
export HF_MODULES_CACHE=<transformers_modules_path>
|
||||
```
|
||||
|
||||
Run from repository root:
|
||||
Run from repository root. The 3-stage pipeline is resolved automatically from
|
||||
the model type (`vllm_omni/deploy/dynin_omni.yaml`), so no config path is
|
||||
required. Pass `--deploy-config` only to override the defaults:
|
||||
|
||||
```bash
|
||||
vllm-omni serve snu-aidas/Dynin-Omni \
|
||||
--omni \
|
||||
--port 8091 \
|
||||
--stage-configs-path "$(pwd)/vllm_omni/model_executor/stage_configs/dynin_omni.yaml"
|
||||
--port 8091
|
||||
```
|
||||
|
||||
If `vllm-omni` is not in PATH, run:
|
||||
@@ -36,8 +37,7 @@ If `vllm-omni` is not in PATH, run:
|
||||
```bash
|
||||
PYTHONPATH="$(pwd)" python -m vllm_omni.entrypoints.cli.main serve snu-aidas/Dynin-Omni \
|
||||
--omni \
|
||||
--port 8091 \
|
||||
--stage-configs-path "$(pwd)/vllm_omni/model_executor/stage_configs/dynin_omni.yaml"
|
||||
--port 8091
|
||||
```
|
||||
|
||||
Wait until the server logs show both `All stages initialized successfully` and
|
||||
|
||||
@@ -198,7 +198,14 @@ def _build_prompt_for_query_type(
|
||||
return query_func(custom_prompt=custom_prompt)
|
||||
if query_type == "use_audio_in_video":
|
||||
return query_func(video_path=video_path, custom_prompt=custom_prompt)
|
||||
# use_mixed_modalities / use_multi_audios
|
||||
if query_type == "use_mixed_modalities":
|
||||
return query_func(
|
||||
video_path=video_path,
|
||||
image_path=image_path,
|
||||
audio_path=audio_path,
|
||||
custom_prompt=custom_prompt,
|
||||
)
|
||||
# use_multi_audios
|
||||
return query_func(custom_prompt=custom_prompt)
|
||||
|
||||
|
||||
|
||||
+3
-1
@@ -64,7 +64,9 @@ dev = [
|
||||
"lpips>=0.1.4", # for quantization fp8
|
||||
"orjson>=3.11.7", # to make v1/embeddings API fast
|
||||
"voxcpm>=2.0.2", # for VoxCPM2 path-locating in tests
|
||||
"torchdiffeq>=0.2.5" # for covo_audio Test
|
||||
"torchdiffeq>=0.2.5", # for covo_audio Test
|
||||
"FlagEmbedding", # for hunyuan_image3 tests
|
||||
"kernels<0.15.0" # for accuracy tests comparison with diffusers output (0.15.0 incompatible with diffusers 0.38.0)
|
||||
]
|
||||
|
||||
demo = [
|
||||
|
||||
+2
-1
@@ -36,7 +36,8 @@ recipes/
|
||||
| [`LTX/LTX-2.md`](./LTX/LTX-2.md) | Text-to-video and image-to-video serving | 1x H200 141GB |
|
||||
| [`LTX/LTX-2.3.md`](./LTX/LTX-2.3.md) | Text-to-video with audio generation (22B) | 1x GPU (96GB VRAM) |
|
||||
| [`mistralai/Voxtral-TTS.md`](./mistralai/Voxtral-TTS.md) | Online serving for TTS | 1x RTX 4090 24GB |
|
||||
| [`nvidia/Cosmos3-Nano.md`](./nvidia/Cosmos3-Nano.md) | Text-to-image, text-to-video, and image-to-video generation | 1x H200 141GB / B300 |
|
||||
| [`cosmos3/Cosmos3-Nano.md`](./cosmos3/Cosmos3-Nano.md) | Text-to-image, text-to-video, image-to-video generation, text to video with sound, action policy | 1x H200 141GB / B300 |
|
||||
| [`cosmos3/Cosmos3-Super.md`](./cosmos3/Cosmos3-Super.md) | 64B T2I / T2V / I2V generation (+ optional audio) / Action policy | 8x H200/H100/A100 / 2x H200 / B300 |
|
||||
| [`OpenBMB/MiniCPM-o-4_5.md`](./OpenBMB/MiniCPM-o-4_5.md) | Online serving for omni multimodal chat (text / image / audio / video → text + 24 kHz speech) | 2x A100/H100 80GB / 3x mid-tier GPU / 8x RTX 4090 24GB |
|
||||
| [`OpenBMB/VoxCPM2.md`](./OpenBMB/VoxCPM2.md) | Online + offline TTS with native AR pipeline (48 kHz, 30+ languages) | 1x RTX 4090 24GB |
|
||||
| [`Qwen/Qwen-Image.md`](./Qwen/Qwen-Image.md) | Text-to-image serving with step-wise continuous batching replay and ModelOpt mixed FP8/NVFP4 | 1x A100 80GB / 2x B200 |
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
- Vendor: NVIDIA
|
||||
- Model: `nvidia/Cosmos3-Nano`
|
||||
- Task: Text-to-image (T2I), text-to-video (T2V), and image-to-video (I2V) generation
|
||||
- Task: Text-to-image (T2I), text-to-video (T2V), and image-to-video (I2V) generation, with optional synchronized audio (video + sound), action policy
|
||||
- Mode: Online serving with the OpenAI-compatible image/video APIs, plus offline generation via the `Omni` API
|
||||
- Maintainer: Community
|
||||
|
||||
@@ -20,12 +20,34 @@ the mode is selected per request:
|
||||
- **T2V** — `POST /v1/videos/sync` with `num_frames > 1` and no reference image.
|
||||
- **I2V** — `POST /v1/videos/sync` with a reference image (`input_reference` file
|
||||
upload, or `image_reference` JSON).
|
||||
- **T2VS / I2VS** — add `generate_sound=true` (and optional `sound_duration`) to a
|
||||
T2V/I2V `/v1/videos/sync` request to also generate synchronized audio, muxed into
|
||||
the mp4 as AAC 48 kHz stereo. See the official model card's "Video + Audio" examples.
|
||||
- **Action** — pass `extra_params={"action_mode": ...}` to drive Physical-AI tasks:
|
||||
- `forward_dynamics` — given a first frame **and** an action trajectory, roll out
|
||||
the resulting video. Synchronous: `POST /v1/videos/sync`.
|
||||
- `policy` — given a first frame and a language instruction, **predict** the action
|
||||
trajectory (and a rollout video). Use the async `POST /v1/videos` endpoint and
|
||||
read the predicted action from the top-level `action` field
|
||||
(`{data, shape, dtype, raw_action_dim, domain_id}`).
|
||||
|
||||
Action requests also take `domain_name` (e.g. `av`, `bridge_orig_lerobot`,
|
||||
`droid_lerobot`, `agibotworld`, …; or a numeric `domain_id`), `raw_action_dim`,
|
||||
and `action_chunk_size` (must equal `num_frames` or `num_frames - 1`). For
|
||||
`forward_dynamics` also pass the `action` array. The dedicated policy checkpoint
|
||||
**`nvidia/Cosmos3-Nano-Policy-DROID`** is served the same way
|
||||
(`domain_name=droid_lerobot`).
|
||||
|
||||
`inverse_dynamics` (recover the action from a given video) is supported by the
|
||||
pipeline; **online inference of inverse dynamics will be added in a follow-up MR.**
|
||||
|
||||
## References
|
||||
|
||||
- Model card (authoritative usage + example assets): <https://huggingface.co/nvidia/Cosmos3-Nano>
|
||||
- Example inputs/outputs live in the repo's `assets/` (`example_t2v_prompt.json`,
|
||||
`example_i2v_prompt.json`, `example_i2v_input.jpg`, `negative_prompt.json`).
|
||||
`example_i2v_prompt.json`, `example_i2v_input.jpg`, `negative_prompt.json`;
|
||||
audio examples: `example_t2vs_prompt.json`, `example_t2vs_output.mp4`,
|
||||
`example_i2vs_output.mp4`).
|
||||
- Prompt upsampling (recommended for quality): the model expects JSON-upsampled
|
||||
structured prompts; see NVIDIA's `cosmos-framework` prompt-upsampling docs.
|
||||
- Pipeline: [`vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py`](../../vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py)
|
||||
@@ -47,6 +69,9 @@ the mode is selected per request:
|
||||
|
||||
#### Command
|
||||
|
||||
Requires the `vllm-omni` package (or the `vllm/vllm-omni:cosmos3` container),
|
||||
which provides the `vllm serve … --omni` entrypoint used below.
|
||||
|
||||
Safety guardrails are **on by default** (NVIDIA Open Model License). They load
|
||||
the **gated** `nvidia/Cosmos-1.0-Guardrail` model, so to keep them on you must:
|
||||
|
||||
@@ -116,6 +141,55 @@ curl -sS -X POST http://localhost:8000/v1/videos/sync \
|
||||
-F "seed=1111" \
|
||||
-F "input_reference=@/path/to/reference.jpg;type=image/jpeg" \
|
||||
-o cosmos3_i2v.mp4
|
||||
|
||||
|
||||
# Text-to-video-with-sound
|
||||
curl -sS -X POST http://localhost:8000/v1/videos/sync \
|
||||
-H "Accept: video/mp4" \
|
||||
-F "model=nvidia/Cosmos3-Nano" \
|
||||
-F "prompt=The video opens with a view of a well-lit indoor fruit display. A robotic arm picks up a pear, an orange, and a carambola one by one, placing each into a plastic bag in a shopping cart with red handles. The video is 7.875 seconds long, 24 FPS, and 1280x720. Audio description: soft servo whirs, gentle fruit thuds, plastic bag rustling, and a faint refrigeration hum." \
|
||||
-F "negative_prompt=blurry, distorted, low quality" \
|
||||
-F "size=1280x720" \
|
||||
-F "num_frames=189" \
|
||||
-F "fps=24" \
|
||||
-F "num_inference_steps=35" \
|
||||
-F "guidance_scale=6.0" \
|
||||
-F "max_sequence_length=4096" \
|
||||
-F "flow_shift=10.0" \
|
||||
-F "seed=0" \
|
||||
-F "generate_sound=true" \
|
||||
-F "sound_duration=7.875" \
|
||||
-F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":true}' \
|
||||
-o cosmos3_t2v_with_sound.mp4
|
||||
|
||||
# Action — forward dynamics (first frame + action trajectory -> rollout video).
|
||||
# Synchronous; `action` is a JSON array shaped [action_chunk_size, raw_action_dim].
|
||||
curl -sS -X POST http://localhost:8000/v1/videos/sync \
|
||||
-H "Accept: video/mp4" \
|
||||
--form-string "model=nvidia/Cosmos3-Nano" \
|
||||
--form-string "prompt=You are an autonomous vehicle. This video is captured from a first-person perspective." \
|
||||
-F "input_reference=@first_frame.jpg;type=image/jpeg" \
|
||||
-F "size=640x480" -F "num_frames=61" -F "fps=10" \
|
||||
-F "num_inference_steps=30" -F "guidance_scale=1.0" -F "flow_shift=5.0" \
|
||||
--form-string "extra_params={\"action_mode\":\"forward_dynamics\",\"domain_name\":\"av\",\"raw_action_dim\":9,\"action_chunk_size\":60,\"action\":$(cat action.json)}" \
|
||||
-F "seed=0" \
|
||||
-o cosmos3_forward_dynamics.mp4
|
||||
|
||||
# Action — policy (first frame + instruction -> predicted action trajectory + video).
|
||||
# Asynchronous: POST returns a job id; poll, then read the predicted action from
|
||||
# the top-level `action` field ({data, shape, dtype, raw_action_dim, domain_id}).
|
||||
VIDEO_ID=$(curl -sS -X POST http://localhost:8000/v1/videos \
|
||||
-H "Accept: application/json" \
|
||||
--form-string "model=nvidia/Cosmos3-Nano" \
|
||||
--form-string "prompt=Pick up the banana and place it in the bowl." \
|
||||
-F "input_reference=@first_frame.jpg;type=image/jpeg" \
|
||||
-F "size=640x480" -F "num_frames=17" -F "fps=5" \
|
||||
-F "num_inference_steps=30" -F "guidance_scale=1.0" -F "flow_shift=5.0" \
|
||||
--form-string 'extra_params={"action_mode":"policy","domain_name":"bridge_orig_lerobot","raw_action_dim":10,"action_chunk_size":16}' \
|
||||
-F "seed=0" | jq -r '.id')
|
||||
# poll until status == completed, then:
|
||||
curl -sS "http://localhost:8000/v1/videos/$VIDEO_ID" | jq '.action | {shape, dtype, raw_action_dim, domain_id}'
|
||||
curl -sS -L "http://localhost:8000/v1/videos/$VIDEO_ID/content" -o cosmos3_policy.mp4
|
||||
```
|
||||
|
||||
#### Notes
|
||||
@@ -124,6 +198,7 @@ curl -sS -X POST http://localhost:8000/v1/videos/sync \
|
||||
- T2I 1024² — 10 / 25 / 50 steps → ~0.4 / 0.7 / **1.3 s**
|
||||
- T2V 1280×720 @ 35 steps — 25 / 49 / 93 / **189** frames → ~7 / 15 / 33 / **~93 s**
|
||||
- I2V 1280×720, 189 frames @ 35 steps → ~**99 s**
|
||||
- Action 640×480 @ 30 steps — forward-dynamics 61f ~**4 s**, policy 17f ~**1–3 s**.
|
||||
- Guardrails-on overhead: ~8% on T2I, negligible on video.
|
||||
- **Memory:** transformer ~17 GiB (bf16); peak ~46 GiB for 720p video on 1 GPU;
|
||||
full repo (transformer + Wan VAE + Qwen3-VL vision encoder + audio tokenizer)
|
||||
@@ -134,7 +209,9 @@ curl -sS -X POST http://localhost:8000/v1/videos/sync \
|
||||
3:4, 9:16. Defaults: T2I 1024², 50 steps, guidance 7.0; T2V/I2V 1280×720,
|
||||
189 frames, 35 steps, guidance 6.0, `flow_shift=10.0`.
|
||||
- **Key flags / params:** `--no-guardrails` (server) or
|
||||
`extra_params={"guardrails":false}` (per request) toggles safety;
|
||||
`extra_params={"guardrails":false}` (per request) toggles safety. The
|
||||
per-request flag only takes effect when the server was launched **with**
|
||||
guardrails enabled (it cannot re-enable them on a `--no-guardrails` server).
|
||||
`use_resolution_template` / `use_duration_template` are off by default and only
|
||||
needed when not using upsampled prompts that already encode resolution/duration.
|
||||
- **Known limitations:**
|
||||
@@ -143,8 +220,10 @@ curl -sS -X POST http://localhost:8000/v1/videos/sync \
|
||||
the server fails at pipeline build with a gated-repo / safety-checker error.
|
||||
- A guardrail-blocked prompt currently returns HTTP 500
|
||||
(`"Guardrail blocked prompt"`).
|
||||
- Video + audio, and action (policy / forward- / inverse-dynamics) modalities
|
||||
are not part of this integration yet.
|
||||
- Action `forward_dynamics` (sync `/v1/videos/sync`) and `policy` (async
|
||||
`/v1/videos`, returns the predicted action under the top-level `action`
|
||||
field) are supported online. **Online inference of inverse dynamics will be
|
||||
added in a follow-up MR.**
|
||||
|
||||
### 1x GPU (Offline generation)
|
||||
|
||||
@@ -170,8 +249,8 @@ def main():
|
||||
model_class_name="Cosmos3OmniDiffusersPipeline",
|
||||
trust_remote_code=True,
|
||||
enforce_eager=True,
|
||||
# Keep guardrails on by installing cosmos-guardrail + gated-repo access;
|
||||
# this disables them for a quick local run.
|
||||
# Guardrails are disabled here for a quick local run; install
|
||||
# cosmos-guardrail + gated-repo access and drop this to enable them.
|
||||
model_config={"guardrails": False},
|
||||
)
|
||||
gen = torch.Generator(device="cpu").manual_seed(42)
|
||||
@@ -0,0 +1,112 @@
|
||||
# Cosmos3-Super
|
||||
|
||||
> Frontier 64B world model: text-to-image, text-to-video, image-to-video (+ optional audio)
|
||||
|
||||
## Summary
|
||||
|
||||
- Vendor: NVIDIA
|
||||
- Model: `nvidia/Cosmos3-Super` (64B; also `Cosmos3-Super-Text2Image`, `Cosmos3-Super-Image2Video`)
|
||||
- Task: T2I, T2V, I2V generation, with optional synchronized audio (video + sound)
|
||||
- Mode: Online serving with the OpenAI-compatible image/video APIs
|
||||
- Maintainer: Community
|
||||
|
||||
## When to use this recipe
|
||||
|
||||
Use this recipe to deploy the 64B `nvidia/Cosmos3-Super` for the highest-quality
|
||||
Cosmos3 generation. It shares the same `Cosmos3OmniDiffusersPipeline` and request
|
||||
formats as [Cosmos3-Nano](./Cosmos3-Nano.md) — only the checkpoint size and the
|
||||
recommended parallelism differ. Mode is selected per request (T2I →
|
||||
`/v1/images/generations`; T2V/I2V → `/v1/videos/sync`; add `generate_sound=true`
|
||||
for audio).
|
||||
|
||||
## References
|
||||
|
||||
- Model card (authoritative usage + example assets): <https://huggingface.co/nvidia/Cosmos3-Super>
|
||||
- Nano recipe (same APIs/params): [`Cosmos3-Nano.md`](./Cosmos3-Nano.md)
|
||||
- Pipeline: [`vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py`](../../vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py)
|
||||
|
||||
## Hardware Support
|
||||
|
||||
## GPU
|
||||
|
||||
Requires the `vllm-omni` package (or the `vllm/vllm-omni:cosmos3` container),
|
||||
which provides the `vllm serve … --omni` entrypoint used below.
|
||||
|
||||
### 8x H200/H100/A100 (recommended, per model card)
|
||||
|
||||
```bash
|
||||
vllm serve nvidia/Cosmos3-Super \
|
||||
--omni \
|
||||
--host 0.0.0.0 --port 8000 \
|
||||
--cfg-parallel-size 2 \
|
||||
--ulysses-degree 4 \
|
||||
--use-hsdp --hsdp-shard-size 8 \
|
||||
--init-timeout 1800
|
||||
```
|
||||
|
||||
### 2x H200 / B300 (minimum)
|
||||
|
||||
```bash
|
||||
vllm serve nvidia/Cosmos3-Super \
|
||||
--omni \
|
||||
--host 0.0.0.0 --port 8000 \
|
||||
--cfg-parallel-size 2 \
|
||||
--use-hsdp --hsdp-shard-size 2 \
|
||||
--init-timeout 1800
|
||||
```
|
||||
|
||||
Guardrails are on by default (gated `nvidia/Cosmos-1.0-Guardrail` — `pip install
|
||||
cosmos-guardrail`, accept the license, set `HF_TOKEN`); add `--no-guardrails` to
|
||||
disable. `--enable-layerwise-offload` reduces VRAM on smaller GPUs.
|
||||
|
||||
#### Verification
|
||||
|
||||
Requests are identical to Nano (see [`Cosmos3-Nano.md`](./Cosmos3-Nano.md) for full
|
||||
T2I/T2V/I2V/T2VS curls); official params: `size=1280x720, num_frames=189, fps=24,
|
||||
num_inference_steps=35, guidance_scale=6.0, flow_shift=10.0, max_sequence_length=4096`.
|
||||
|
||||
```bash
|
||||
curl http://localhost:8000/v1/models
|
||||
# T2V (official prompt assets give best quality)
|
||||
curl -sS -X POST http://localhost:8000/v1/videos/sync -H "Accept: video/mp4" \
|
||||
-F "model=nvidia/Cosmos3-Super" -F "prompt=A robot arm is cleaning a plate in the kitchen" \
|
||||
-F "size=1280x720" -F "num_frames=189" -F "fps=24" -F "num_inference_steps=35" \
|
||||
-F "guidance_scale=6.0" -F "max_sequence_length=4096" -F "flow_shift=10.0" \
|
||||
-F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":true}' \
|
||||
-F "seed=17" -o cosmos3_super_t2v.mp4
|
||||
|
||||
# I2V — add an uploaded reference image
|
||||
curl -sS -X POST http://localhost:8000/v1/videos/sync -H "Accept: video/mp4" \
|
||||
-F "model=nvidia/Cosmos3-Super" -F "prompt=The scene comes to life with smooth, natural motion." \
|
||||
-F "size=1280x720" -F "num_frames=189" -F "fps=24" -F "num_inference_steps=35" \
|
||||
-F "guidance_scale=6.0" -F "max_sequence_length=4096" -F "flow_shift=10.0" \
|
||||
-F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":true}' \
|
||||
-F "seed=1111" -F "input_reference=@/path/to/reference.jpg;type=image/jpeg" \
|
||||
-o cosmos3_super_i2v.mp4
|
||||
|
||||
# T2V + sound — add generate_sound/sound_duration (output muxes AAC 48 kHz stereo)
|
||||
curl -sS -X POST http://localhost:8000/v1/videos/sync -H "Accept: video/mp4" \
|
||||
-F "model=nvidia/Cosmos3-Super" -F "prompt=A robot arm is cleaning a plate in the kitchen" \
|
||||
-F "size=1280x720" -F "num_frames=189" -F "fps=24" -F "num_inference_steps=35" \
|
||||
-F "guidance_scale=6.0" -F "max_sequence_length=4096" -F "flow_shift=10.0" \
|
||||
-F "generate_sound=true" -F "sound_duration=7.875" \
|
||||
-F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":true}' \
|
||||
-F "seed=17" -o cosmos3_super_t2vs.mp4
|
||||
```
|
||||
|
||||
#### Notes
|
||||
|
||||
- **Measured (2x B300, bf16, guardrails off, official 2-GPU config above):**
|
||||
- T2I 1024², 50 steps → **~6 s**
|
||||
- T2V 1280×720, 189 frames, 35 steps → **~197 s**
|
||||
- I2V 1280×720, 189 frames, 35 steps → **~200 s**
|
||||
- T2V + sound (189 frames, 35 steps) → **~198 s**, output muxes **AAC 48 kHz stereo**
|
||||
- (NVIDIA's reference: 8×H200 @ 50 steps ≈ 55 s/video; 2×H200 @ 35 steps ≈ 3 min/video.)
|
||||
- **Memory:** ~61.5 GiB per GPU when sharded across 2 GPUs (HSDP shard 2); repo ~135 GB on disk.
|
||||
- Same generation defaults, supported sizes, and `generate_sound`/`sound_duration`
|
||||
semantics as Nano, including the **action** modality: `forward_dynamics`
|
||||
(sync `/v1/videos/sync`) and `policy` (async `/v1/videos`, predicted action under
|
||||
the top-level `action` field) — see the Cosmos3-Nano recipe for the request shape.
|
||||
Online inference of inverse dynamics will be added in a follow-up MR. Verified on
|
||||
the 64B Super under `--cfg-parallel-size 2`: async `policy` returns the predicted
|
||||
action (`[16, 10]`) and the rollout video reliably.
|
||||
@@ -57,12 +57,6 @@ Adjust `devices` in the YAML to match your hardware.
|
||||
|
||||
#### Command
|
||||
|
||||
Thinker only (text output):
|
||||
|
||||
```bash
|
||||
vllm serve Jonathan1909/Ming-flash-omni-2.0 --omni --port 8091
|
||||
```
|
||||
|
||||
Thinker + talker (text and/or audio output):
|
||||
|
||||
```bash
|
||||
@@ -72,6 +66,15 @@ vllm serve Jonathan1909/Ming-flash-omni-2.0 \
|
||||
--log-stats
|
||||
```
|
||||
|
||||
Thinker only (text-only output):
|
||||
|
||||
```bash
|
||||
vllm serve Jonathan1909/Ming-flash-omni-2.0 \
|
||||
--omni \
|
||||
--deploy-config vllm_omni/deploy/ming_flash_omni_thinker_only.yaml \
|
||||
--port 8091
|
||||
```
|
||||
|
||||
`--log-stats` is optional but recommended while validating the deployment.
|
||||
|
||||
#### Verification
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.1 MiB |
@@ -345,6 +345,14 @@ def test_parse_score_payload_handles_raw_json_and_delimited_json():
|
||||
assert parse_score_payload(wrapped)["score"] == [6]
|
||||
|
||||
|
||||
def test_parse_score_payload_handles_qwen_vl_nested_score_dicts():
|
||||
qwen_style = '{"score": [{"naturalness": 8, "artifact_free": 7}], "reasoning": "good quality"}'
|
||||
assert parse_score_payload(qwen_style)["score"] == [8, 7]
|
||||
|
||||
nested_values = '{"score": [{"score": 9}, {"score": 6}], "reasoning": "ok"}'
|
||||
assert parse_score_payload(nested_values)["score"] == [9, 6]
|
||||
|
||||
|
||||
def test_summarize_gedit_generated_records_groups_by_task_and_language():
|
||||
records = []
|
||||
for group in GEDIT_GROUPS[:2]:
|
||||
|
||||
@@ -61,7 +61,7 @@ def test_stage0_streaming_update_discards_outstanding_async_placeholder_token()
|
||||
|
||||
sched._update_request_as_session(session, _make_update([10, 20]))
|
||||
|
||||
assert session.discard_latest_async_tokens is True
|
||||
assert session.async_tokens_to_discard == 1
|
||||
assert session.num_output_placeholders == 0
|
||||
assert session.spec_token_ids == []
|
||||
# The async placeholder makes token 9 unconfirmed, so only 7 and 8 are
|
||||
@@ -83,7 +83,7 @@ def test_stage0_streaming_update_keeps_all_computed_tokens_without_placeholder()
|
||||
|
||||
sched._update_request_as_session(session, _make_update([10, 20]))
|
||||
|
||||
assert session.discard_latest_async_tokens is False
|
||||
assert getattr(session, "async_tokens_to_discard", 0) == 0
|
||||
assert session.num_output_placeholders == 0
|
||||
assert session.prompt_token_ids == [1, 2, 3, 7, 8, 9, 10, 20]
|
||||
assert list(session._all_token_ids) == [1, 2, 3, 7, 8, 9, 10, 20]
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_hunyuan_image3_ti2i_tp4_tp4_inline",
|
||||
"description": "AR (TP=4) + DiT (TP=4) - using inline dataset and deploy-config",
|
||||
"test_name": "test_hunyuan_image3_ti2i_inline",
|
||||
"description": "AR (TP=2) + 2 * DiT (TP=2) - using inline dataset and deploy-config",
|
||||
"server_type": "vllm-omni",
|
||||
"benchmark_endpoint": "/v1/images/edits",
|
||||
"server_params": {
|
||||
@@ -21,7 +21,7 @@
|
||||
"zmq_port": 50051,
|
||||
"protocol": "rdma",
|
||||
"device_name": "",
|
||||
"memory_pool_size": 4294967296,
|
||||
"memory_pool_size": 17179869184,
|
||||
"memory_pool_device": "cpu"
|
||||
}
|
||||
}
|
||||
@@ -32,13 +32,13 @@
|
||||
"is_comprehension": true,
|
||||
"final_output": true,
|
||||
"final_output_type": "text",
|
||||
"max_num_seqs": 4,
|
||||
"max_num_seqs": 16,
|
||||
"gpu_memory_utilization": 0.95,
|
||||
"enforce_eager": false,
|
||||
"enable_prefix_caching": false,
|
||||
"max_num_batched_tokens": 32768,
|
||||
"devices": "0,1,2,3",
|
||||
"tensor_parallel_size": 4,
|
||||
"devices": "0,1",
|
||||
"tensor_parallel_size": 2,
|
||||
"hf_overrides": {
|
||||
"rope_parameters": {
|
||||
"mrope_section": [0, 32, 32],
|
||||
@@ -65,13 +65,14 @@
|
||||
"stage_id": 1,
|
||||
"max_num_seqs": 1,
|
||||
"enforce_eager": true,
|
||||
"devices": "4,5,6,7",
|
||||
"devices": "2,3",
|
||||
"distributed_executor_backend": "mp",
|
||||
"omni_kv_config": {
|
||||
"need_recv_cache": true
|
||||
},
|
||||
"parallel_config": {
|
||||
"tensor_parallel_size": 4,
|
||||
"tensor_parallel_size": 2,
|
||||
"vae_patch_parallel_size": 2,
|
||||
"enable_expert_parallel": true
|
||||
},
|
||||
"input_connectors": {
|
||||
@@ -114,8 +115,8 @@
|
||||
},
|
||||
"task": "ti2i",
|
||||
"bot-task": "think_recaption",
|
||||
"max-concurrency": 4,
|
||||
"num-prompts": 16,
|
||||
"max-concurrency": 16,
|
||||
"num-prompts": 96,
|
||||
"skip-performance-assertion": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.1,
|
||||
|
||||
@@ -23,10 +23,10 @@
|
||||
"ignore_eos": true,
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [1000, 2000, 3000, 4000],
|
||||
"mean_tpot_ms": [200, 200, 200, 200],
|
||||
"mean_audio_ttfp_ms": [10000, 15000, 20000, 25000],
|
||||
"mean_audio_rtf": [1.5, 1.5, 1.5, 1.5]
|
||||
"mean_ttft_ms": [203, 283, 470, 869],
|
||||
"mean_tpot_ms": [28, 36, 41, 45],
|
||||
"mean_audio_ttfp_ms": [861, 1200, 1528, 2100],
|
||||
"mean_audio_rtf": [0.24, 0.30, 0.35, 0.39]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -49,9 +49,9 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [2000],
|
||||
"mean_audio_ttfp_ms": [2000],
|
||||
"mean_audio_rtf": [0.25]
|
||||
"mean_ttft_ms": [377],
|
||||
"mean_audio_ttfp_ms": [684],
|
||||
"mean_audio_rtf": [0.14]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -76,9 +76,9 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [6000],
|
||||
"mean_audio_ttfp_ms": [6000],
|
||||
"mean_audio_rtf": [0.7]
|
||||
"mean_ttft_ms": [228],
|
||||
"mean_audio_ttfp_ms": [686],
|
||||
"mean_audio_rtf": [0.16]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -105,9 +105,9 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [12000],
|
||||
"mean_audio_ttfp_ms": [12000],
|
||||
"mean_audio_rtf": [1.0]
|
||||
"mean_ttft_ms": [522],
|
||||
"mean_audio_ttfp_ms": [1119],
|
||||
"mean_audio_rtf": [0.21]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -31,11 +31,6 @@
|
||||
],
|
||||
"random_input_len": 2500,
|
||||
"random_output_len": 900,
|
||||
"temperature": 0.4,
|
||||
"top_p": 0.9,
|
||||
"top_k": 1,
|
||||
"seed": 42,
|
||||
"repetition_penalty": 1.05,
|
||||
"ignore_eos": true,
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
@@ -74,11 +69,6 @@
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"temperature": 0.4,
|
||||
"top_p": 0.9,
|
||||
"top_k": 1,
|
||||
"seed": 42,
|
||||
"repetition_penalty": 1.05,
|
||||
"ignore_eos": true,
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
@@ -117,11 +107,6 @@
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"temperature": 0.4,
|
||||
"top_p": 0.9,
|
||||
"top_k": 1,
|
||||
"seed": 42,
|
||||
"repetition_penalty": 1.05,
|
||||
"ignore_eos": true,
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
@@ -162,11 +147,6 @@
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"temperature": 0.4,
|
||||
"top_p": 0.9,
|
||||
"top_k": 1,
|
||||
"seed": 42,
|
||||
"repetition_penalty": 1.05,
|
||||
"ignore_eos": true,
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
|
||||
@@ -73,14 +73,42 @@ class StubCosmos3VAE:
|
||||
return (latents,)
|
||||
|
||||
|
||||
class StubCosmos3AVAE:
|
||||
def __init__(self, **kwargs: Any) -> None:
|
||||
self.kwargs = kwargs
|
||||
self.sample_rate = int(kwargs["sample_rate"])
|
||||
self.audio_channels = int(kwargs["audio_channels"])
|
||||
self.latent_ch = int(kwargs["io_channels"])
|
||||
self.temporal_compression_factor = int(kwargs["hop_size"])
|
||||
|
||||
def get_latent_num_samples(self, num_audio_samples: int) -> int:
|
||||
return int(num_audio_samples) // self.temporal_compression_factor
|
||||
|
||||
def get_audio_num_samples(self, num_latent_samples: int) -> int:
|
||||
return int(num_latent_samples) * self.temporal_compression_factor
|
||||
|
||||
def decode(self, latents: torch.Tensor) -> torch.Tensor:
|
||||
return torch.zeros(latents.shape[0], self.audio_channels, 8)
|
||||
|
||||
|
||||
class StubCosmos3Transformer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
latent_channel_size: int = 2,
|
||||
sound_gen: bool = False,
|
||||
sound_dim: int = 3,
|
||||
sound_latent_fps: float = 25.0,
|
||||
action_gen: bool = False,
|
||||
action_dim: int = 4,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.latent_channel_size = latent_channel_size
|
||||
self.sound_gen = sound_gen
|
||||
self.sound_dim = sound_dim
|
||||
self.sound_latent_fps = sound_latent_fps
|
||||
self.action_gen = action_gen
|
||||
self.action_dim = action_dim
|
||||
self.cached_kv: Any | None = None
|
||||
self.cached_freqs_gen: Any | None = None
|
||||
self.calls: list[dict[str, Any]] = []
|
||||
@@ -99,8 +127,9 @@ class StubCosmos3Transformer(nn.Module):
|
||||
text_ids: torch.Tensor,
|
||||
text_mask: torch.Tensor,
|
||||
**kwargs: Any,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
token = int(text_ids.reshape(-1)[0].item()) if text_ids.numel() else 0
|
||||
sound_latents = kwargs.get("sound_latents")
|
||||
self.calls.append(
|
||||
{
|
||||
"token": token,
|
||||
@@ -114,7 +143,13 @@ class StubCosmos3Transformer(nn.Module):
|
||||
marker = torch.tensor([token], dtype=torch.float32)
|
||||
self.cached_kv = [(marker, marker + 100)]
|
||||
self.cached_freqs_gen = (marker + 200, marker + 300)
|
||||
return torch.full_like(hidden_states, float(token))
|
||||
action_latents = kwargs.get("action_latents")
|
||||
outputs: list[torch.Tensor] = [torch.full_like(hidden_states, float(token))]
|
||||
if action_latents is not None:
|
||||
outputs.append(torch.full_like(action_latents, float(token + 20)))
|
||||
if sound_latents is not None:
|
||||
outputs.append(torch.full_like(sound_latents, float(token + 10)))
|
||||
return outputs[0] if len(outputs) == 1 else tuple(outputs)
|
||||
|
||||
|
||||
def passthrough_progress_bar(iterable):
|
||||
@@ -155,6 +190,7 @@ def make_cosmos3_pipeline():
|
||||
pipeline._guidance_scale = None
|
||||
pipeline._num_timesteps = None
|
||||
pipeline._cache_dit_requires_paired_cfg = False
|
||||
pipeline._sound_tokenizer = None
|
||||
pipeline.progress_bar = passthrough_progress_bar
|
||||
return pipeline
|
||||
|
||||
@@ -213,7 +249,109 @@ def test_pipeline_registered_and_exported() -> None:
|
||||
assert "Cosmos3OmniDiffusersPipeline" in cosmos3.__all__
|
||||
|
||||
|
||||
def test_preprocess_i2v_image_input() -> None:
|
||||
@pytest.fixture
|
||||
def stub_real_pipeline_init(monkeypatch: pytest.MonkeyPatch):
|
||||
from vllm_omni.diffusion.models.cosmos3 import pipeline_cosmos3
|
||||
|
||||
class _StubAutoTokenizer:
|
||||
@classmethod
|
||||
def from_pretrained(cls, *args, **kwargs):
|
||||
return SimpleNamespace()
|
||||
|
||||
class _StubDiffusersVAE:
|
||||
config = SimpleNamespace(scale_factor_temporal=4, scale_factor_spatial=8)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(cls, *args, **kwargs):
|
||||
return cls()
|
||||
|
||||
def to(self, _device):
|
||||
return self
|
||||
|
||||
class _StubDiffusersScheduler:
|
||||
config = SimpleNamespace(flow_shift=1.0)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(cls, *args, **kwargs):
|
||||
return cls()
|
||||
|
||||
class _StubVideoProcessor:
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(pipeline_cosmos3, "AutoTokenizer", _StubAutoTokenizer)
|
||||
monkeypatch.setattr(pipeline_cosmos3, "DistributedAutoencoderKLWan", _StubDiffusersVAE)
|
||||
monkeypatch.setattr(pipeline_cosmos3, "UniPCMultistepScheduler", _StubDiffusersScheduler)
|
||||
monkeypatch.setattr(pipeline_cosmos3, "VideoProcessor", _StubVideoProcessor)
|
||||
monkeypatch.setattr(pipeline_cosmos3, "get_local_device", lambda: torch.device("cpu"))
|
||||
|
||||
|
||||
def _make_od_config(*, sound_gen: bool) -> SimpleNamespace:
|
||||
tf_model_config = {
|
||||
"hidden_size": 8,
|
||||
"num_hidden_layers": 0,
|
||||
"num_attention_heads": 2,
|
||||
"num_key_value_heads": 2,
|
||||
"head_dim": 4,
|
||||
"intermediate_size": 16,
|
||||
"vocab_size": 32,
|
||||
"latent_patch_size": 1,
|
||||
"latent_channel": 2,
|
||||
"rope_scaling": {"mrope_section": [1, 1, 0]},
|
||||
}
|
||||
if sound_gen:
|
||||
tf_model_config["sound_gen"] = True
|
||||
return SimpleNamespace(
|
||||
enable_cpu_offload=False,
|
||||
enable_diffusion_pipeline_profiler=False,
|
||||
model="/nonexistent/model/path",
|
||||
dtype=torch.float32,
|
||||
flow_shift=None,
|
||||
quantization_config=None,
|
||||
custom_pipeline_args={},
|
||||
model_config={},
|
||||
tf_model_config=tf_model_config,
|
||||
)
|
||||
|
||||
|
||||
def test_pipeline_init_skips_tokenizer_when_sound_disabled(stub_real_pipeline_init) -> None:
|
||||
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import Cosmos3OmniDiffusersPipeline
|
||||
|
||||
pipeline = Cosmos3OmniDiffusersPipeline(od_config=_make_od_config(sound_gen=False))
|
||||
|
||||
assert pipeline._sound_tokenizer is None
|
||||
assert pipeline.transformer.sound_gen is False
|
||||
assert not hasattr(pipeline.transformer, "audio_proj_in")
|
||||
assert not hasattr(pipeline.transformer, "audio_proj_out")
|
||||
|
||||
|
||||
def test_pipeline_init_passes_tokenizer_attrs_into_transformer(
|
||||
stub_real_pipeline_init,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from vllm_omni.diffusion.models.cosmos3 import sound_tokenizer
|
||||
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import Cosmos3OmniDiffusersPipeline
|
||||
|
||||
stub_tokenizer = sound_tokenizer.Cosmos3SoundTokenizer(
|
||||
StubCosmos3AVAE(sample_rate=32000, audio_channels=2, io_channels=5, hop_size=800)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sound_tokenizer.Cosmos3SoundTokenizer,
|
||||
"from_config",
|
||||
classmethod(lambda cls, od_config: stub_tokenizer),
|
||||
)
|
||||
|
||||
pipeline = Cosmos3OmniDiffusersPipeline(od_config=_make_od_config(sound_gen=True))
|
||||
|
||||
assert pipeline._sound_tokenizer is stub_tokenizer
|
||||
assert pipeline.transformer.sound_gen is True
|
||||
assert pipeline.transformer.sound_dim == pipeline._sound_tokenizer.latent_ch == 5
|
||||
assert pipeline.transformer.sound_latent_fps == pipeline._sound_tokenizer.latent_fps == 40.0
|
||||
assert pipeline.transformer.audio_proj_in.in_features == 5
|
||||
assert pipeline.transformer.audio_proj_out.out_features == 5
|
||||
|
||||
|
||||
def test_preprocess_i2v_image_and_action_video_inputs() -> None:
|
||||
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import get_cosmos3_pre_process_func
|
||||
|
||||
preprocess = get_cosmos3_pre_process_func(SimpleNamespace())
|
||||
@@ -226,8 +364,18 @@ def test_preprocess_i2v_image_input() -> None:
|
||||
assert (result.sampling_params.height, result.sampling_params.width) == (672, 1344)
|
||||
assert tuple(result.prompts[0]["additional_information"]["preprocessed_image"].shape[-2:]) == (672, 1344)
|
||||
|
||||
frames = [Image.new("RGB", (8, 4), color) for color in ("red", "green", "blue")]
|
||||
action = SimpleNamespace(
|
||||
prompts=[{"prompt": "Move.", "multi_modal_data": {"video": frames}}],
|
||||
sampling_params=SimpleNamespace(height=16, width=32, extra_args={"action_mode": "forward_dynamics"}),
|
||||
)
|
||||
|
||||
def test_postprocess_handles_image_video_and_validation() -> None:
|
||||
additional = preprocess(action).prompts[0]["additional_information"]
|
||||
assert tuple(additional["preprocessed_image"].shape) == (1, 3, 16, 32)
|
||||
assert tuple(additional["preprocessed_video"].shape) == (1, 3, 3, 16, 32)
|
||||
|
||||
|
||||
def test_postprocess_handles_image_video_audio_and_validation() -> None:
|
||||
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import get_cosmos3_post_process_func
|
||||
|
||||
func = get_cosmos3_post_process_func(SimpleNamespace())
|
||||
@@ -235,6 +383,16 @@ def test_postprocess_handles_image_video_and_validation() -> None:
|
||||
|
||||
assert func(video, output_type="latent") is video
|
||||
assert func({"image": video})[0].size == (4, 4)
|
||||
# Video-only postprocess returns the bare processed video (not a dict),
|
||||
# matching the image/latent branches and peer audio-capable pipelines.
|
||||
assert not isinstance(func({"video": video}), dict)
|
||||
assert (
|
||||
func(
|
||||
{"video": video, "audio": torch.ones(1, 2, 16), "audio_sample_rate": 48000},
|
||||
sampling_params=SimpleNamespace(extra_args={"resolved_frame_rate": 12}),
|
||||
)["audio_sample_rate"]
|
||||
== 48000
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="text-to-image postprocess expects"):
|
||||
func({"image": torch.zeros(1, 3, 2, 4, 4)})
|
||||
@@ -293,7 +451,7 @@ def test_prompt_formatting_and_checkpoint_key_remap(make_cosmos3_pipeline) -> No
|
||||
assert {key: Cosmos3OmniDiffusersPipeline._remap_ckpt_key(key) for key in remaps} == remaps
|
||||
|
||||
|
||||
def test_prepare_latents_for_video_and_image(make_cosmos3_pipeline) -> None:
|
||||
def test_prepare_latents_for_video_image_sound_and_action(make_cosmos3_pipeline) -> None:
|
||||
pipeline = make_cosmos3_pipeline()
|
||||
latents = pipeline._prepare_latents(16, 24, 5, torch.Generator(device="cpu").manual_seed(0))
|
||||
assert latents.shape == (1, 2, 2, 2, 3)
|
||||
@@ -306,8 +464,36 @@ def test_prepare_latents_for_video_and_image(make_cosmos3_pipeline) -> None:
|
||||
assert velocity_mask.tolist() == [[[[[0.0]], [[1.0]]]]]
|
||||
assert image_latent.shape == (1, 2, 1, 2, 3)
|
||||
|
||||
pipeline.transformer = pipeline.transformer.__class__(latent_channel_size=2, sound_gen=True, sound_dim=3)
|
||||
pipeline._sound_tokenizer = SimpleNamespace(
|
||||
sample_rate=10,
|
||||
latent_ch=3,
|
||||
hop_size=4,
|
||||
decode=lambda x: torch.ones(x.shape[0], 2, 24),
|
||||
)
|
||||
assert pipeline._resolve_sound_target_samples(SimpleNamespace(extra_args={"sound_duration": 2.0}), 9, 3.0) == (
|
||||
20,
|
||||
2.0,
|
||||
10,
|
||||
)
|
||||
sound_latents, latent_frames = pipeline._prepare_sound_latents(21, torch.Generator(device="cpu").manual_seed(0))
|
||||
assert (sound_latents.shape, latent_frames) == (torch.Size([1, 3, 6]), 6)
|
||||
assert pipeline._decode_sound_latents(torch.zeros(1, 3, 6), target_audio_samples=21).shape == (1, 2, 21)
|
||||
|
||||
def test_diffuse_covers_cfg_and_i2v_steps(make_cosmos3_pipeline) -> None:
|
||||
pipeline.transformer = pipeline.transformer.__class__(action_gen=True, action_dim=4)
|
||||
action, action_mask, clean, raw_dim = pipeline._prepare_action_latents(
|
||||
mode="forward_dynamics",
|
||||
action_chunk_size=2,
|
||||
raw_action_dim=None,
|
||||
generator=torch.Generator(device="cpu").manual_seed(0),
|
||||
sp=SimpleNamespace(extra_args={"action": [[1.0, 2.0], [3.0, 4.0]]}),
|
||||
)
|
||||
assert raw_dim == 2
|
||||
assert action_mask.tolist() == [[[0.0], [0.0]]]
|
||||
torch.testing.assert_close(action, clean)
|
||||
|
||||
|
||||
def test_diffuse_covers_cfg_i2v_and_multimodal_steps(make_cosmos3_pipeline) -> None:
|
||||
pipeline = make_cosmos3_pipeline()
|
||||
latents = torch.zeros(1, 2, 1, 1, 1)
|
||||
|
||||
@@ -339,6 +525,23 @@ def test_diffuse_covers_cfg_and_i2v_steps(make_cosmos3_pipeline) -> None:
|
||||
)
|
||||
torch.testing.assert_close(i2v[:, :, 0:1], torch.full((1, 2, 1, 1, 1), 7.0))
|
||||
|
||||
pipeline.transformer = pipeline.transformer.__class__(latent_channel_size=2, action_gen=True, action_dim=4)
|
||||
video_result, action_result = pipeline.diffuse(
|
||||
latents=latents,
|
||||
action_latents=torch.zeros(1, 3, 4),
|
||||
action_velocity_mask=torch.ones(1, 3, 1),
|
||||
action_condition_latents=torch.zeros(1, 3, 4),
|
||||
timesteps=torch.tensor([7, 3]),
|
||||
cond_ids=_ids(2),
|
||||
cond_mask=_mask(),
|
||||
uncond_ids=_ids(1),
|
||||
uncond_mask=_mask(),
|
||||
guidance_scale=1.0,
|
||||
shared_kwargs={"video_shape": (1, 1, 1), "fps": 24.0, "action_domain_ids": torch.tensor([0])},
|
||||
)
|
||||
torch.testing.assert_close(video_result, torch.full_like(latents, 4.0))
|
||||
torch.testing.assert_close(action_result, torch.full((), 44.0).expand_as(action_result))
|
||||
|
||||
|
||||
def test_diffuse_keeps_paired_cfg_when_cache_dit_active(make_cosmos3_pipeline) -> None:
|
||||
"""With cache-dit active the uncond pass is kept even outside the guidance
|
||||
@@ -395,7 +598,12 @@ class TestForwardRouting:
|
||||
|
||||
def fake_diffuse(**kwargs):
|
||||
captured["diffuse_calls"].append(kwargs)
|
||||
return kwargs["latents"] + len(captured["diffuse_calls"])
|
||||
outputs = [kwargs["latents"] + len(captured["diffuse_calls"])]
|
||||
if kwargs.get("action_latents") is not None:
|
||||
outputs.append(kwargs["action_latents"] + 3.0)
|
||||
if kwargs.get("sound_latents") is not None:
|
||||
outputs.append(kwargs["sound_latents"] + 2.0)
|
||||
return outputs[0] if len(outputs) == 1 else tuple(outputs)
|
||||
|
||||
pipeline._format_and_tokenize_prompts = fake_format
|
||||
pipeline._prepare_latents = fake_prepare
|
||||
@@ -437,7 +645,7 @@ class TestForwardRouting:
|
||||
assert captured["flow_shifts"] == expected["flow"]
|
||||
assert [call[0] for call in pipeline.scheduler.set_timesteps_calls] == expected["steps"]
|
||||
|
||||
def test_forward_i2v_route(self, make_cosmos3_pipeline) -> None:
|
||||
def test_forward_i2v_sound_and_action_routes(self, make_cosmos3_pipeline) -> None:
|
||||
pipeline = make_cosmos3_pipeline()
|
||||
captured = self._install_forward_stubs(pipeline)
|
||||
image_tensor = torch.zeros(1, 3, 16, 16)
|
||||
@@ -462,11 +670,55 @@ class TestForwardRouting:
|
||||
)
|
||||
assert captured["diffuse_calls"][-1]["shared_kwargs"]["noisy_frame_mask"] is velocity_mask
|
||||
|
||||
pipeline.transformer = pipeline.transformer.__class__(latent_channel_size=2, sound_gen=True, sound_dim=3)
|
||||
sound_latents = torch.zeros(1, 3, 4)
|
||||
pipeline._resolve_sound_target_samples = lambda *args: (20, 2.0, 10)
|
||||
pipeline._prepare_sound_latents = lambda *args: (sound_latents, 4)
|
||||
pipeline._decode_sound_latents = lambda *args: torch.ones(1, 2, 20)
|
||||
output = pipeline.forward(
|
||||
SimpleNamespace(
|
||||
prompts=[{"prompt": "A robot", "modalities": ["video"], "generate_sound": True}],
|
||||
sampling_params=make_sampling_params(num_frames=9, frame_rate=3.0),
|
||||
)
|
||||
)
|
||||
assert captured["diffuse_calls"][-1]["sound_latents"] is sound_latents
|
||||
assert output.output["audio_sample_rate"] == 10
|
||||
|
||||
pipeline.transformer = pipeline.transformer.__class__(latent_channel_size=2, action_gen=True, action_dim=4)
|
||||
output = pipeline.forward(
|
||||
SimpleNamespace(
|
||||
prompts=[
|
||||
{
|
||||
"prompt": "Pick the block.",
|
||||
"modalities": ["video"],
|
||||
"additional_information": {"preprocessed_image": image_tensor},
|
||||
}
|
||||
],
|
||||
sampling_params=make_sampling_params(
|
||||
height=16,
|
||||
width=16,
|
||||
extra_args={
|
||||
"action_mode": "policy",
|
||||
"action_chunk_size": 2,
|
||||
"raw_action_dim": 2,
|
||||
"domain_name": "bridge_orig_lerobot",
|
||||
},
|
||||
),
|
||||
)
|
||||
)
|
||||
assert captured["diffuse_calls"][-1]["shared_kwargs"]["action_domain_ids"].tolist() == [7]
|
||||
assert output.custom_output["action"].shape == (1, 2, 2)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("prompt", "sampling_params", "message"),
|
||||
[
|
||||
(["one", "two"], make_sampling_params(), "single prompt"),
|
||||
([{"prompt": "one", "modalities": ["image", "video"]}], make_sampling_params(), "both image and video"),
|
||||
(
|
||||
[{"prompt": "x", "modalities": ["image"], "generate_sound": True}],
|
||||
make_sampling_params(),
|
||||
"only for video",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_forward_rejects_invalid_public_requests(
|
||||
@@ -477,6 +729,7 @@ class TestForwardRouting:
|
||||
message,
|
||||
) -> None:
|
||||
pipeline = make_cosmos3_pipeline()
|
||||
pipeline.transformer = pipeline.transformer.__class__(latent_channel_size=2, sound_gen=True, sound_dim=3)
|
||||
|
||||
with pytest.raises(ValueError, match=message):
|
||||
pipeline.forward(SimpleNamespace(prompts=prompt, sampling_params=sampling_params))
|
||||
@@ -0,0 +1,236 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME = "diffusion_pytorch_model.safetensors"
|
||||
|
||||
|
||||
class _FakeAVAEAudioTokenizer:
|
||||
def __init__(self, **kwargs) -> None:
|
||||
self.kwargs = kwargs
|
||||
self.sample_rate = int(kwargs["sample_rate"])
|
||||
self.audio_channels = int(kwargs["audio_channels"])
|
||||
self.latent_ch = int(kwargs["io_channels"])
|
||||
self.temporal_compression_factor = int(kwargs["hop_size"])
|
||||
|
||||
def get_latent_num_samples(self, num_audio_samples: int) -> int:
|
||||
return int(num_audio_samples) // self.temporal_compression_factor
|
||||
|
||||
def get_audio_num_samples(self, num_latent_samples: int) -> int:
|
||||
return int(num_latent_samples) * self.temporal_compression_factor
|
||||
|
||||
def decode(self, latents: torch.Tensor) -> torch.Tensor:
|
||||
return torch.zeros(latents.shape[0], self.audio_channels, 8)
|
||||
|
||||
|
||||
def _write_component(root: Path, config: dict | None = None, checkpoint_name: str | None = None) -> Path:
|
||||
tokenizer_dir = root / "sound_tokenizer"
|
||||
tokenizer_dir.mkdir(parents=True)
|
||||
if checkpoint_name:
|
||||
(tokenizer_dir / checkpoint_name).write_bytes(b"stub")
|
||||
(tokenizer_dir / "config.json").write_text(json.dumps(config or {}), encoding="utf-8")
|
||||
return tokenizer_dir
|
||||
|
||||
|
||||
def _patch_fake_avae(monkeypatch: pytest.MonkeyPatch, created: dict) -> None:
|
||||
from vllm_omni.diffusion.models.cosmos3 import sound_tokenizer
|
||||
|
||||
class FakeAVAE(_FakeAVAEAudioTokenizer):
|
||||
def __init__(self, **kwargs) -> None:
|
||||
created.update(kwargs)
|
||||
super().__init__(**kwargs)
|
||||
|
||||
monkeypatch.setattr(sound_tokenizer, "Cosmos3AVAEAudioTokenizer", FakeAVAE)
|
||||
monkeypatch.setattr(sound_tokenizer, "get_local_device", lambda: torch.device("cpu"))
|
||||
|
||||
|
||||
def test_from_config_loads_local_diffusers_component(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from vllm_omni.diffusion.models.cosmos3 import sound_tokenizer
|
||||
|
||||
model_dir = tmp_path / "model"
|
||||
tokenizer_dir = _write_component(model_dir, checkpoint_name=DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME)
|
||||
created = {}
|
||||
_patch_fake_avae(monkeypatch, created)
|
||||
|
||||
tokenizer = sound_tokenizer.Cosmos3SoundTokenizer.from_config(
|
||||
SimpleNamespace(
|
||||
model=str(model_dir),
|
||||
custom_pipeline_args={"sound_sample_rate": 32000, "sound_hop_size": 800, "sound_dim": 3},
|
||||
dtype=torch.float32,
|
||||
)
|
||||
)
|
||||
|
||||
assert created["checkpoint_path"] == str(tokenizer_dir / DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME)
|
||||
assert created["config_path"] == str(tokenizer_dir / "config.json")
|
||||
assert (tokenizer.sample_rate, tokenizer.latent_ch, tokenizer.hop_size, tokenizer.latent_fps) == (
|
||||
32000,
|
||||
3,
|
||||
800,
|
||||
40.0,
|
||||
)
|
||||
|
||||
|
||||
def test_from_config_downloads_component_from_hf_repo(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import huggingface_hub
|
||||
|
||||
from vllm_omni.diffusion.models.cosmos3 import sound_tokenizer
|
||||
|
||||
cache_dir = tmp_path / "hf"
|
||||
_write_component(cache_dir, checkpoint_name=DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME)
|
||||
calls = []
|
||||
created = {}
|
||||
_patch_fake_avae(monkeypatch, created)
|
||||
|
||||
def fake_snapshot_download(repo_id: str, *, revision: str | None, allow_patterns: list[str]) -> str:
|
||||
calls.append((repo_id, revision, allow_patterns))
|
||||
return str(cache_dir)
|
||||
|
||||
monkeypatch.setattr(huggingface_hub, "snapshot_download", fake_snapshot_download)
|
||||
|
||||
sound_tokenizer.Cosmos3SoundTokenizer.from_config(
|
||||
SimpleNamespace(
|
||||
model="nvidia/cosmos3",
|
||||
revision="test-rev",
|
||||
custom_pipeline_args={"sound_sample_rate": 32000, "sound_hop_size": 800, "sound_dim": 3},
|
||||
dtype=torch.float32,
|
||||
)
|
||||
)
|
||||
|
||||
assert created["checkpoint_path"].endswith(DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME)
|
||||
assert calls == [
|
||||
(
|
||||
"nvidia/cosmos3",
|
||||
"test-rev",
|
||||
["sound_tokenizer/config.json", f"sound_tokenizer/{DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME}"],
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("checkpoint_name", "message"),
|
||||
[
|
||||
(None, "no AVAE sound tokenizer checkpoint"),
|
||||
("model.safetensors", DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME),
|
||||
],
|
||||
)
|
||||
def test_default_component_requires_diffusers_checkpoint_name(tmp_path, checkpoint_name, message) -> None:
|
||||
from vllm_omni.diffusion.models.cosmos3 import sound_tokenizer
|
||||
|
||||
model_dir = tmp_path / "model"
|
||||
_write_component(model_dir, checkpoint_name=checkpoint_name)
|
||||
|
||||
with pytest.raises(ValueError, match=message):
|
||||
sound_tokenizer.Cosmos3SoundTokenizer.from_config(
|
||||
SimpleNamespace(model=str(model_dir), custom_pipeline_args={}, dtype=torch.float32)
|
||||
)
|
||||
|
||||
|
||||
def test_component_config_precedence_and_conflict_detection(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from vllm_omni.diffusion.models.cosmos3 import sound_tokenizer
|
||||
|
||||
component_config = {
|
||||
"sampling_rate": 48000,
|
||||
"dec_out_channels": 2,
|
||||
"vocoder_input_dim": 64,
|
||||
"hop_size": 1920,
|
||||
}
|
||||
model_dir = tmp_path / "model"
|
||||
_write_component(model_dir, component_config, DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME)
|
||||
created = {}
|
||||
_patch_fake_avae(monkeypatch, created)
|
||||
|
||||
tokenizer = sound_tokenizer.Cosmos3SoundTokenizer.from_config(
|
||||
SimpleNamespace(
|
||||
model=str(model_dir),
|
||||
custom_pipeline_args={
|
||||
"sound_normalize_latents": True,
|
||||
"sound_normalization_type": "tanh",
|
||||
"sound_tanh_input_scale": 2.0,
|
||||
},
|
||||
model_config={
|
||||
"sound_tokenizer": {
|
||||
"sample_rate": 32000,
|
||||
"audio_channels": 1,
|
||||
"io_channels": 3,
|
||||
"hop_size": 800,
|
||||
"normalize_latents": False,
|
||||
"normalization_type": "none",
|
||||
}
|
||||
},
|
||||
dtype=torch.float32,
|
||||
)
|
||||
)
|
||||
|
||||
assert (created["sample_rate"], created["audio_channels"], created["io_channels"], created["hop_size"]) == (
|
||||
48000,
|
||||
2,
|
||||
64,
|
||||
1920,
|
||||
)
|
||||
assert (created["normalize_latents"], created["normalization_type"], created["tanh_input_scale"]) == (
|
||||
True,
|
||||
"tanh",
|
||||
2.0,
|
||||
)
|
||||
assert (tokenizer.sample_rate, tokenizer.latent_ch, tokenizer.hop_size, tokenizer.latent_fps) == (
|
||||
48000,
|
||||
64,
|
||||
1920,
|
||||
25.0,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match=r"sample_rate.*48000.*32000"):
|
||||
sound_tokenizer.Cosmos3SoundTokenizer.from_config(
|
||||
SimpleNamespace(
|
||||
model=str(model_dir),
|
||||
custom_pipeline_args={"sound_sample_rate": 32000},
|
||||
dtype=torch.float32,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_avae_uses_diffusers_decoder_state_dict_layout(tmp_path) -> None:
|
||||
from safetensors.torch import save_file
|
||||
|
||||
from vllm_omni.diffusion.models.cosmos3.audio_tokenizer import avae
|
||||
|
||||
config = {
|
||||
"sampling_rate": 8000,
|
||||
"hop_size": 2,
|
||||
"dec_dim": 4,
|
||||
"dec_c_mults": [1],
|
||||
"dec_strides": [2],
|
||||
"dec_out_channels": 1,
|
||||
"vocoder_input_dim": 2,
|
||||
"normalization_type": "none",
|
||||
}
|
||||
checkpoint_path = tmp_path / DIFFUSERS_SOUND_TOKENIZER_CHECKPOINT_NAME
|
||||
config_path = tmp_path / "config.json"
|
||||
config_path.write_text(json.dumps(config), encoding="utf-8")
|
||||
|
||||
decoder = avae.OobleckDecoder(4, 2, 1, [2], [1])
|
||||
save_file({f"decoder.{key}": value for key, value in decoder.state_dict().items()}, str(checkpoint_path))
|
||||
|
||||
tokenizer = avae.Cosmos3AVAEAudioTokenizer(
|
||||
checkpoint_path=checkpoint_path,
|
||||
config_path=config_path,
|
||||
dtype=torch.float32,
|
||||
device="cpu",
|
||||
)
|
||||
|
||||
keys = set(tokenizer.state_dict())
|
||||
assert {"decoder.conv1.weight_g", "decoder.block.0.conv_t1.weight_g", "decoder.conv2.weight_g"} <= keys
|
||||
assert not any(key.startswith(("decoder.layers.", "model.decoder.")) for key in keys)
|
||||
assert tokenizer.decode(torch.zeros(1, 2, 3)).shape == (1, 1, 6)
|
||||
with pytest.raises(NotImplementedError, match="decoder-only"):
|
||||
tokenizer.encode(torch.zeros(1, 1, 6))
|
||||
@@ -4,6 +4,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -29,8 +30,10 @@ def _tiny_cosmos3_config(**overrides):
|
||||
return config
|
||||
|
||||
|
||||
def test_mrope_position_ids_cover_text_and_video() -> None:
|
||||
def test_mrope_position_ids_cover_text_video_sound_and_action() -> None:
|
||||
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import (
|
||||
compute_mrope_position_ids_action,
|
||||
compute_mrope_position_ids_sound,
|
||||
compute_mrope_position_ids_text,
|
||||
compute_mrope_position_ids_vision,
|
||||
)
|
||||
@@ -56,6 +59,14 @@ def test_mrope_position_ids_cover_text_and_video() -> None:
|
||||
torch.testing.assert_close(modulated_ids[0], torch.tensor([10.0, 12.0]))
|
||||
assert modulated_offset == 13
|
||||
|
||||
sound_ids, sound_offset = compute_mrope_position_ids_sound(3, temporal_offset=10, sound_latent_fps=25.0)
|
||||
torch.testing.assert_close(sound_ids[0], torch.tensor([10.0, 10.96, 11.92]))
|
||||
assert sound_offset == 12
|
||||
|
||||
action_ids, action_offset = compute_mrope_position_ids_action(3, temporal_offset=10, action_fps=None)
|
||||
assert action_ids.tolist() == [[11, 12, 13], [0, 0, 0], [0, 0, 0]]
|
||||
assert action_offset == 14
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("key", "value"),
|
||||
@@ -120,7 +131,142 @@ def test_forward_returns_video_prediction(monkeypatch: pytest.MonkeyPatch) -> No
|
||||
assert tuple(output.shape) == (1, 2, 1, 2, 2)
|
||||
|
||||
|
||||
def test_compute_rope_freqs_places_text_and_video_positions() -> None:
|
||||
def test_sound_and_action_modules_follow_config() -> None:
|
||||
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3VFMTransformer
|
||||
|
||||
tiny = _tiny_cosmos3_config()
|
||||
no_modal = Cosmos3VFMTransformer(SimpleNamespace(tf_model_config=tiny, dtype=torch.float32))
|
||||
with_sound = Cosmos3VFMTransformer(
|
||||
SimpleNamespace(tf_model_config=tiny, dtype=torch.float32),
|
||||
sound_gen=True,
|
||||
sound_dim=5,
|
||||
sound_latent_fps=40.0,
|
||||
)
|
||||
with_action = Cosmos3VFMTransformer(
|
||||
SimpleNamespace(
|
||||
tf_model_config={**tiny, "action_gen": True, "max_action_dim": 6, "num_embodiment_domains": 9},
|
||||
dtype=torch.float32,
|
||||
)
|
||||
)
|
||||
|
||||
assert no_modal.sound_gen is False
|
||||
assert no_modal.action_gen is False
|
||||
assert not hasattr(no_modal, "audio_proj_in")
|
||||
assert not hasattr(no_modal, "action_proj_in")
|
||||
assert with_sound.sound_dim == 5
|
||||
assert with_sound.sound_latent_fps == 40.0
|
||||
assert with_sound.audio_proj_in.in_features == 5
|
||||
assert with_action.action_dim == 6
|
||||
assert with_action.action_proj_in.num_domains == 9
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"kwargs",
|
||||
[
|
||||
{"sound_gen": True},
|
||||
{"sound_gen": True, "sound_dim": 5},
|
||||
{"sound_gen": True, "sound_latent_fps": 40.0},
|
||||
],
|
||||
)
|
||||
def test_transformer_requires_sound_dim_and_fps_when_sound_gen_true(kwargs: dict[str, Any]) -> None:
|
||||
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3VFMTransformer
|
||||
|
||||
with pytest.raises(ValueError, match=r"requires an explicit sound_dim and sound_latent_fps"):
|
||||
Cosmos3VFMTransformer(
|
||||
SimpleNamespace(tf_model_config=_tiny_cosmos3_config(), dtype=torch.float32),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def test_sound_and_action_pack_unpack_validate_shapes() -> None:
|
||||
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3VFMTransformer
|
||||
|
||||
model = object.__new__(Cosmos3VFMTransformer)
|
||||
nn.Module.__init__(model)
|
||||
model.sound_dim = 3
|
||||
model.action_dim = 3
|
||||
|
||||
sound = torch.arange(2 * 3 * 4, dtype=torch.float32).reshape(2, 3, 4)
|
||||
action = torch.arange(2 * 4 * 3, dtype=torch.float32).reshape(2, 4, 3)
|
||||
torch.testing.assert_close(model.unpack_sound(model.pack_sound(sound)), sound)
|
||||
torch.testing.assert_close(model.unpack_action(model.pack_action(action)), action)
|
||||
|
||||
with pytest.raises(ValueError, match="channel mismatch"):
|
||||
model.pack_sound(torch.zeros(1, 4, 2))
|
||||
with pytest.raises(ValueError, match="dimension mismatch"):
|
||||
model.pack_action(torch.zeros(1, 2, 4))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("config", "transformer_kwargs", "extra_kwargs", "expected_shapes"),
|
||||
[
|
||||
(
|
||||
_tiny_cosmos3_config(),
|
||||
{"sound_gen": True, "sound_dim": 3, "sound_latent_fps": 24.0},
|
||||
{"sound_latents": torch.zeros(1, 3, 4)},
|
||||
[(1, 2, 1, 2, 2), (1, 3, 4)],
|
||||
),
|
||||
(
|
||||
_tiny_cosmos3_config(action_gen=True, max_action_dim=3, num_embodiment_domains=4),
|
||||
{},
|
||||
{"action_latents": torch.zeros(1, 5, 3), "action_domain_ids": torch.tensor([2])},
|
||||
[(1, 2, 1, 2, 2), (1, 5, 3)],
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_forward_returns_video_plus_optional_modality_predictions(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
config,
|
||||
transformer_kwargs,
|
||||
extra_kwargs,
|
||||
expected_shapes,
|
||||
) -> None:
|
||||
from vllm_omni.diffusion.models.cosmos3 import transformer_cosmos3
|
||||
|
||||
monkeypatch.setattr(transformer_cosmos3, "_get_ulysses_state", lambda: (1, 0, None))
|
||||
|
||||
output = transformer_cosmos3.Cosmos3VFMTransformer(
|
||||
SimpleNamespace(tf_model_config=config, dtype=torch.float32),
|
||||
**transformer_kwargs,
|
||||
)(
|
||||
hidden_states=torch.zeros(1, 2, 1, 2, 2),
|
||||
timestep=torch.tensor([1.0]),
|
||||
text_ids=torch.tensor([[1, 2]], dtype=torch.long),
|
||||
text_mask=torch.ones(1, 2, dtype=torch.long),
|
||||
video_shape=(1, 2, 2),
|
||||
fps=24.0,
|
||||
action_noisy_mask=torch.ones(1, 5, 1),
|
||||
**extra_kwargs,
|
||||
)
|
||||
|
||||
assert isinstance(output, tuple)
|
||||
assert [tuple(tensor.shape) for tensor in output] == expected_shapes
|
||||
|
||||
|
||||
def test_forward_with_sound_ulysses_error_mentions_combined_sequence(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 as cosmos3_module
|
||||
|
||||
model = cosmos3_module.Cosmos3VFMTransformer(
|
||||
SimpleNamespace(tf_model_config=_tiny_cosmos3_config(), dtype=torch.float32),
|
||||
sound_gen=True,
|
||||
sound_dim=3,
|
||||
sound_latent_fps=40.0,
|
||||
)
|
||||
monkeypatch.setattr(cosmos3_module, "_get_ulysses_state", lambda: (2, 0, None))
|
||||
|
||||
with pytest.raises(ValueError, match=r"GEN sequence length \(3 = video tokens 2 \+ sound tokens 1\)"):
|
||||
model(
|
||||
hidden_states=torch.zeros(1, 2, 1, 1, 2),
|
||||
timestep=torch.tensor([1.0]),
|
||||
text_ids=torch.tensor([[1, 2]], dtype=torch.long),
|
||||
text_mask=torch.ones(1, 2, dtype=torch.long),
|
||||
video_shape=(1, 1, 2),
|
||||
fps=24.0,
|
||||
sound_latents=torch.zeros(1, 3, 1),
|
||||
)
|
||||
|
||||
|
||||
def test_compute_rope_freqs_places_text_video_action_and_sound_positions() -> None:
|
||||
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3VFMTransformer
|
||||
|
||||
class FakeRotary:
|
||||
@@ -140,6 +286,8 @@ def test_compute_rope_freqs_places_text_and_video_positions() -> None:
|
||||
model.temporal_modality_margin = 100
|
||||
model.base_fps = 24.0
|
||||
model.temporal_compression_factor = 4
|
||||
model.temporal_compression_factor_sound = 1
|
||||
model.sound_latent_fps = 25.0
|
||||
model.enable_fps_modulation = False
|
||||
|
||||
freqs_und, freqs_gen = model._compute_rope_freqs(
|
||||
@@ -156,3 +304,21 @@ def test_compute_rope_freqs_places_text_and_video_positions() -> None:
|
||||
assert vision_pos[0, 0].tolist() == [102, 103]
|
||||
assert freqs_und[0].shape == (2, 3, 1, 4)
|
||||
assert freqs_gen[0].shape == (2, 2, 1, 4)
|
||||
|
||||
rotary.position_ids.clear()
|
||||
model._compute_rope_freqs(
|
||||
text_mask=torch.tensor([[1, 1]], dtype=torch.long),
|
||||
t=2,
|
||||
hp=1,
|
||||
wp=1,
|
||||
fps=24.0,
|
||||
device=torch.device("cpu"),
|
||||
dtype=torch.float32,
|
||||
t_action=2,
|
||||
action_start_frame_offset=1,
|
||||
t_sound=1,
|
||||
)
|
||||
|
||||
_, gen_pos = rotary.position_ids
|
||||
assert gen_pos.shape == (3, 1, 5)
|
||||
assert gen_pos[0, 0].tolist() == [102, 103, 103, 104, 102]
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import queue
|
||||
import threading
|
||||
from types import SimpleNamespace
|
||||
@@ -447,11 +446,7 @@ class TestDiffusionEngine:
|
||||
@pytest.mark.asyncio
|
||||
async def test_step_raises_aborted_error(self, mocker: MockerFixture) -> None:
|
||||
engine = DiffusionEngine.__new__(DiffusionEngine)
|
||||
engine._closed = False
|
||||
engine._loop_started = True
|
||||
engine._init_lock = asyncio.Lock()
|
||||
engine.main_loop = asyncio.get_running_loop()
|
||||
engine.stop_event = threading.Event()
|
||||
engine._check_and_start_background_loop = mocker.AsyncMock()
|
||||
engine.pre_process_func = None
|
||||
engine.async_add_req_and_wait_for_response = mocker.AsyncMock(
|
||||
return_value=DiffusionOutput(aborted=True, abort_message="Request req-abort aborted.")
|
||||
@@ -526,6 +521,52 @@ class TestDiffusionEngine:
|
||||
with pytest.raises(RuntimeError, match="Dummy run failed: boom"):
|
||||
engine._dummy_run()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_step_multi_request_reuses_multimodal_slice_logic(self, mocker: MockerFixture) -> None:
|
||||
engine = DiffusionEngine.__new__(DiffusionEngine)
|
||||
engine.od_config = SimpleNamespace(
|
||||
model_class_name="mock_model",
|
||||
enable_cpu_offload=False,
|
||||
)
|
||||
engine.pre_process_func = None
|
||||
engine.post_process_func = None
|
||||
engine._check_and_start_background_loop = mocker.AsyncMock()
|
||||
engine.async_add_req_and_wait_for_response = mocker.AsyncMock(
|
||||
return_value=DiffusionOutput(
|
||||
output={
|
||||
"video": ["frame-0", "frame-1"],
|
||||
"audio": ["audio-0", "audio-1"],
|
||||
"actions": torch.tensor([[1.0, 2.0], [3.0, 4.0]]),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
request = OmniDiffusionRequest(
|
||||
prompts=["prompt-0", "prompt-1"],
|
||||
sampling_params=OmniDiffusionSamplingParams(
|
||||
num_inference_steps=1,
|
||||
num_outputs_per_prompt=1,
|
||||
),
|
||||
request_id="req-batch",
|
||||
)
|
||||
|
||||
mocker.patch("vllm_omni.diffusion.diffusion_engine.supports_audio_output", return_value=False)
|
||||
outputs = await engine.step(request)
|
||||
|
||||
assert len(outputs) == 2
|
||||
assert outputs[0].images == ["frame-0"]
|
||||
assert outputs[1].images == ["frame-1"]
|
||||
assert outputs[0].multimodal_output["audio"] == "audio-0"
|
||||
assert outputs[1].multimodal_output["audio"] == "audio-1"
|
||||
torch.testing.assert_close(
|
||||
outputs[0].multimodal_output["actions"],
|
||||
torch.tensor([1.0, 2.0]),
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
outputs[1].multimodal_output["actions"],
|
||||
torch.tensor([3.0, 4.0]),
|
||||
)
|
||||
|
||||
|
||||
class TestStepScheduler:
|
||||
def setup_method(self) -> None:
|
||||
|
||||
@@ -32,8 +32,8 @@ BAGEL_MOONCAKE_CI_DEPLOY = get_deploy_config_path("ci/bagel_mooncake.yaml")
|
||||
|
||||
# Reference pixel data extracted from the known-good output image
|
||||
# Each entry contains (x, y) position and expected (R, G, B) values
|
||||
# "Generated with seed=52, num_inference_steps=15,
|
||||
# prompt='A futuristic city skyline at twilight, cyberpunk style'"
|
||||
# "Generated with seed=52, num_inference_steps=14,
|
||||
# prompt='A cute cat'"
|
||||
REFERENCE_PIXELS = [
|
||||
{"position": (100, 100), "rgb": (115, 113, 94)},
|
||||
{"position": (400, 50), "rgb": (159, 160, 144)},
|
||||
@@ -63,7 +63,7 @@ def _find_free_port() -> int:
|
||||
return port
|
||||
|
||||
|
||||
def _configure_sampling_params(omni: Omni, num_inference_steps: int = 15) -> list:
|
||||
def _configure_sampling_params(omni: Omni, num_inference_steps: int = 14) -> list:
|
||||
"""Configure sampling parameters for Bagel text2img generation.
|
||||
|
||||
Args:
|
||||
@@ -269,7 +269,6 @@ def _load_mooncake_config(host: str, rpc_port: int, http_port: int) -> str:
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.skip(reason="Skip failed CI issue 3977: https://github.com/vllm-project/vllm-omni/issues/3977")
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_bagel_text2img_mooncake_connector(run_level):
|
||||
"""Test Bagel text2img with Mooncake connector for inter-stage communication."""
|
||||
|
||||
@@ -24,14 +24,13 @@ from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
pytestmark = [pytest.mark.usefixtures("clean_gpu_memory_between_tests")]
|
||||
|
||||
BAGEL_CI_DEPLOY = get_deploy_config_path("ci/bagel.yaml")
|
||||
|
||||
# Reference pixel data extracted from the known-good output image
|
||||
# Generated with seed=52, num_inference_steps=15,
|
||||
# Generated with seed=52, num_inference_steps=14,
|
||||
# prompt='Change the grass color to red',
|
||||
# input image: 2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg
|
||||
REFERENCE_PIXELS = [
|
||||
@@ -47,23 +46,10 @@ REFERENCE_PIXELS = [
|
||||
{"position": (256, 256), "rgb": (181, 201, 221)},
|
||||
]
|
||||
|
||||
if current_omni_platform.is_rocm():
|
||||
REFERENCE_PIXELS = [
|
||||
{"position": (100, 100), "rgb": (156, 172, 217)},
|
||||
{"position": (400, 50), "rgb": (105, 144, 217)},
|
||||
{"position": (700, 100), "rgb": (118, 159, 232)},
|
||||
{"position": (150, 400), "rgb": (180, 22, 52)},
|
||||
{"position": (512, 336), "rgb": (221, 211, 194)},
|
||||
{"position": (700, 400), "rgb": (192, 10, 46)},
|
||||
{"position": (100, 600), "rgb": (102, 12, 22)},
|
||||
{"position": (400, 600), "rgb": (161, 28, 47)},
|
||||
{"position": (700, 600), "rgb": (100, 87, 94)},
|
||||
{"position": (256, 256), "rgb": (181, 201, 221)},
|
||||
]
|
||||
|
||||
# text2img reference pixels (aligned with offline `bagel/end2end.py` text2img, 15 steps)
|
||||
# "Generated with seed=52, num_inference_steps=15,
|
||||
# prompt='A futuristic city skyline at twilight, cyberpunk style'"
|
||||
# "Generated with seed=52, num_inference_steps=14,
|
||||
# prompt='A cute cat'"
|
||||
TEXT2IMG_REFERENCE_PIXELS = [
|
||||
{"position": (100, 100), "rgb": (115, 113, 94)},
|
||||
{"position": (400, 50), "rgb": (159, 160, 144)},
|
||||
@@ -77,19 +63,6 @@ TEXT2IMG_REFERENCE_PIXELS = [
|
||||
{"position": (256, 256), "rgb": (167, 156, 150)},
|
||||
]
|
||||
|
||||
if current_omni_platform.is_rocm():
|
||||
TEXT2IMG_REFERENCE_PIXELS = [
|
||||
{"position": (100, 100), "rgb": (115, 113, 94)},
|
||||
{"position": (400, 50), "rgb": (159, 160, 144)},
|
||||
{"position": (700, 100), "rgb": (164, 151, 123)},
|
||||
{"position": (150, 400), "rgb": (120, 121, 107)},
|
||||
{"position": (512, 512), "rgb": (165, 133, 127)},
|
||||
{"position": (700, 400), "rgb": (217, 130, 66)},
|
||||
{"position": (100, 700), "rgb": (191, 168, 152)},
|
||||
{"position": (400, 700), "rgb": (130, 96, 77)},
|
||||
{"position": (700, 700), "rgb": (247, 203, 140)},
|
||||
{"position": (256, 256), "rgb": (167, 156, 150)},
|
||||
]
|
||||
|
||||
PIXEL_TOLERANCE = 10
|
||||
TEXT2IMG_PIXEL_TOLERANCE = 5
|
||||
@@ -115,7 +88,7 @@ def _find_free_port() -> int:
|
||||
return port
|
||||
|
||||
|
||||
def _configure_sampling_params(omni: Omni, num_inference_steps: int = 15) -> list:
|
||||
def _configure_sampling_params(omni: Omni, num_inference_steps: int = 14) -> list:
|
||||
"""Configure sampling parameters for Bagel img2img generation.
|
||||
|
||||
Args:
|
||||
@@ -258,7 +231,6 @@ def _resolve_deploy_config(config_path: str, run_level: str) -> str:
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.skip(reason="Skip failed CI issue 3977: https://github.com/vllm-project/vllm-omni/issues/3977")
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
|
||||
def test_bagel_img2img_shared_memory_connector(run_level):
|
||||
"""Test Bagel img2img with shared memory connector."""
|
||||
@@ -276,7 +248,6 @@ def test_bagel_img2img_shared_memory_connector(run_level):
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.skip(reason="Skip failed CI issue 3977: https://github.com/vllm-project/vllm-omni/issues/3977")
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
|
||||
def test_bagel_text2img_shared_memory_connector(run_level):
|
||||
"""Test Bagel text2img with shared memory connector."""
|
||||
|
||||
@@ -0,0 +1,307 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
try:
|
||||
import cv2
|
||||
except ImportError: # pragma: no cover - optional e2e dependency
|
||||
cv2 = None
|
||||
|
||||
try:
|
||||
import websockets.sync.client as websockets_client
|
||||
except ImportError: # pragma: no cover - optional e2e dependency
|
||||
websockets_client = None
|
||||
|
||||
try:
|
||||
from openpi_client import msgpack_numpy
|
||||
except ImportError: # pragma: no cover - optional e2e dependency
|
||||
msgpack_numpy = None
|
||||
|
||||
PING_INTERVAL_SECS = 300
|
||||
PING_TIMEOUT_SECS = 3600
|
||||
DEFAULT_HOST = "127.0.0.1"
|
||||
DEFAULT_PORT = 8000
|
||||
DEFAULT_PATH = "/v1/realtime/robot/openpi"
|
||||
DEFAULT_PROMPT = "Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan"
|
||||
ACTION_HORIZON = 24
|
||||
DEFAULT_ACTION_DIM = 8
|
||||
RELATIVE_OFFSETS = [-23, -16, -8, 0]
|
||||
CAMERA_FILES = {
|
||||
"observation/exterior_image_0_left": "exterior_image_1_left.mp4",
|
||||
"observation/exterior_image_1_left": "exterior_image_2_left.mp4",
|
||||
"observation/wrist_image_left": "wrist_image_left.mp4",
|
||||
}
|
||||
|
||||
|
||||
def _decode_action_response(response: bytes | str) -> np.ndarray:
|
||||
if isinstance(response, str):
|
||||
raise RuntimeError(f"Inference failed: {response}")
|
||||
decoded = msgpack_numpy.unpackb(response)
|
||||
if isinstance(decoded, dict) and decoded.get("type") == "error":
|
||||
message = decoded.get("message", decoded)
|
||||
raise RuntimeError(f"Inference failed: {message}")
|
||||
return np.asarray(decoded, dtype=np.float32)
|
||||
|
||||
|
||||
def require_dependencies() -> None:
|
||||
missing = []
|
||||
if cv2 is None:
|
||||
missing.append("opencv-python")
|
||||
if websockets_client is None:
|
||||
missing.append("websockets")
|
||||
if msgpack_numpy is None:
|
||||
missing.append("openpi-client")
|
||||
if missing:
|
||||
raise ModuleNotFoundError(f"DreamZero OpenPI test dependencies are missing: {', '.join(missing)}")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DreamZeroServerMetadata:
|
||||
image_resolution: tuple[int, int]
|
||||
n_external_cameras: int
|
||||
needs_wrist_camera: bool
|
||||
needs_stereo_camera: bool
|
||||
needs_session_id: bool
|
||||
action_space: str
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, payload: dict[str, Any]) -> DreamZeroServerMetadata:
|
||||
required_keys = (
|
||||
"image_resolution",
|
||||
"n_external_cameras",
|
||||
"needs_wrist_camera",
|
||||
"needs_stereo_camera",
|
||||
"needs_session_id",
|
||||
"action_space",
|
||||
)
|
||||
missing_keys = [key for key in required_keys if key not in payload]
|
||||
if missing_keys:
|
||||
raise ValueError(f"Missing DreamZero metadata keys: {missing_keys}")
|
||||
|
||||
image_resolution = payload["image_resolution"]
|
||||
if not isinstance(image_resolution, (list, tuple)) or len(image_resolution) != 2:
|
||||
raise ValueError(f"Invalid image_resolution: {image_resolution!r}")
|
||||
|
||||
return cls(
|
||||
image_resolution=(int(image_resolution[0]), int(image_resolution[1])),
|
||||
n_external_cameras=int(payload["n_external_cameras"]),
|
||||
needs_wrist_camera=bool(payload["needs_wrist_camera"]),
|
||||
needs_stereo_camera=bool(payload["needs_stereo_camera"]),
|
||||
needs_session_id=bool(payload["needs_session_id"]),
|
||||
action_space=str(payload["action_space"]),
|
||||
)
|
||||
|
||||
|
||||
class OpenPIWebsocketClient:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
host: str = DEFAULT_HOST,
|
||||
port: int = DEFAULT_PORT,
|
||||
path: str = DEFAULT_PATH,
|
||||
) -> None:
|
||||
require_dependencies()
|
||||
self._uri = f"ws://{host}:{port}{path}"
|
||||
self._packer = msgpack_numpy.Packer()
|
||||
self._ws, self._server_metadata = self._connect()
|
||||
|
||||
def _connect(self):
|
||||
conn = websockets_client.connect(
|
||||
self._uri,
|
||||
compression=None,
|
||||
max_size=None,
|
||||
ping_interval=PING_INTERVAL_SECS,
|
||||
ping_timeout=PING_TIMEOUT_SECS,
|
||||
)
|
||||
metadata = msgpack_numpy.unpackb(conn.recv())
|
||||
if not isinstance(metadata, dict):
|
||||
raise TypeError(f"Expected dict metadata from server, got {type(metadata)!r}")
|
||||
return conn, metadata
|
||||
|
||||
def get_server_metadata(self) -> dict[str, Any]:
|
||||
return dict(self._server_metadata)
|
||||
|
||||
def infer(self, obs: dict[str, Any]) -> np.ndarray:
|
||||
payload = dict(obs)
|
||||
payload["endpoint"] = "infer"
|
||||
self._ws.send(self._packer.pack(payload))
|
||||
response = self._ws.recv()
|
||||
return _decode_action_response(response)
|
||||
|
||||
def reset(self, reset_info: dict[str, Any] | None = None) -> str:
|
||||
payload = dict(reset_info or {})
|
||||
payload["endpoint"] = "reset"
|
||||
self._ws.send(self._packer.pack(payload))
|
||||
response = self._ws.recv()
|
||||
if isinstance(response, str):
|
||||
return response
|
||||
decoded = msgpack_numpy.unpackb(response)
|
||||
if not isinstance(decoded, dict) or decoded.get("status") != "reset successful":
|
||||
raise RuntimeError(f"Unexpected reset response: {decoded!r}")
|
||||
return str(decoded["status"])
|
||||
|
||||
def close(self) -> None:
|
||||
self._ws.close()
|
||||
|
||||
|
||||
def load_all_frames(video_path: Path) -> np.ndarray:
|
||||
require_dependencies()
|
||||
cap = cv2.VideoCapture(str(video_path))
|
||||
frames = []
|
||||
while True:
|
||||
ok, frame = cap.read()
|
||||
if not ok:
|
||||
break
|
||||
frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
|
||||
cap.release()
|
||||
if not frames:
|
||||
raise RuntimeError(f"No frames loaded from {video_path}")
|
||||
return np.stack(frames, axis=0)
|
||||
|
||||
|
||||
def load_camera_frames(video_dir: Path) -> dict[str, np.ndarray]:
|
||||
camera_frames: dict[str, np.ndarray] = {}
|
||||
for camera_key, file_name in CAMERA_FILES.items():
|
||||
video_path = video_dir / file_name
|
||||
if not video_path.exists():
|
||||
raise FileNotFoundError(f"Missing DreamZero test asset: {video_path}")
|
||||
camera_frames[camera_key] = load_all_frames(video_path)
|
||||
return camera_frames
|
||||
|
||||
|
||||
def build_frame_schedule(total_frames: int, num_chunks: int) -> list[list[int]]:
|
||||
chunks: list[list[int]] = []
|
||||
current_frame = 23
|
||||
for _ in range(num_chunks):
|
||||
indices = [max(current_frame + offset, 0) for offset in RELATIVE_OFFSETS]
|
||||
if indices[-1] >= total_frames:
|
||||
break
|
||||
chunks.append(indices)
|
||||
current_frame += ACTION_HORIZON
|
||||
return chunks
|
||||
|
||||
|
||||
def make_obs_from_video(
|
||||
camera_frames: dict[str, np.ndarray],
|
||||
frame_indices: list[int],
|
||||
*,
|
||||
prompt: str,
|
||||
session_id: str,
|
||||
) -> dict[str, Any]:
|
||||
obs: dict[str, Any] = {}
|
||||
for camera_key, all_frames in camera_frames.items():
|
||||
selected = all_frames[frame_indices]
|
||||
obs[camera_key] = selected[0] if len(frame_indices) == 1 else selected
|
||||
|
||||
obs["observation/joint_position"] = np.zeros(7, dtype=np.float32)
|
||||
obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32)
|
||||
obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32)
|
||||
obs["prompt"] = prompt
|
||||
obs["session_id"] = session_id
|
||||
return obs
|
||||
|
||||
|
||||
def build_demo_observations(
|
||||
camera_frames: dict[str, np.ndarray],
|
||||
*,
|
||||
prompt: str,
|
||||
session_id: str,
|
||||
num_chunks: int = 2,
|
||||
) -> list[dict[str, Any]]:
|
||||
if num_chunks < 1:
|
||||
raise ValueError("num_chunks must be at least 1")
|
||||
|
||||
total_frames = min(frames.shape[0] for frames in camera_frames.values())
|
||||
observations = [
|
||||
make_obs_from_video(
|
||||
camera_frames,
|
||||
[0],
|
||||
prompt=prompt,
|
||||
session_id=session_id,
|
||||
)
|
||||
]
|
||||
for indices in build_frame_schedule(total_frames, num_chunks - 1):
|
||||
observations.append(
|
||||
make_obs_from_video(
|
||||
camera_frames,
|
||||
indices,
|
||||
prompt=prompt,
|
||||
session_id=session_id,
|
||||
)
|
||||
)
|
||||
return observations
|
||||
|
||||
|
||||
def validate_session_result(
|
||||
result: dict[str, Any],
|
||||
*,
|
||||
expected_action_horizon: int = ACTION_HORIZON,
|
||||
expected_action_dim: int = DEFAULT_ACTION_DIM,
|
||||
) -> None:
|
||||
metadata = DreamZeroServerMetadata.from_dict(result["metadata"])
|
||||
if metadata.image_resolution != (180, 320):
|
||||
raise AssertionError(f"Unexpected image_resolution: {metadata.image_resolution}")
|
||||
if metadata.n_external_cameras != 2:
|
||||
raise AssertionError(f"Unexpected n_external_cameras: {metadata.n_external_cameras}")
|
||||
if not metadata.needs_wrist_camera:
|
||||
raise AssertionError("DreamZero test expects wrist camera metadata")
|
||||
if metadata.action_space != "joint_position":
|
||||
raise AssertionError(f"Unexpected action_space: {metadata.action_space}")
|
||||
|
||||
actions = result["actions"]
|
||||
if len(actions) != 3:
|
||||
raise AssertionError(f"Expected 3 action tensors, got {len(actions)}")
|
||||
for index, action in enumerate(actions):
|
||||
if action.shape != (expected_action_horizon, expected_action_dim):
|
||||
raise AssertionError(
|
||||
f"Action {index} shape mismatch: expected "
|
||||
f"{(expected_action_horizon, expected_action_dim)}, got {action.shape}"
|
||||
)
|
||||
if not np.isfinite(action).all():
|
||||
raise AssertionError(f"Action {index} contains non-finite values")
|
||||
|
||||
if result["reset_status"] != "reset successful":
|
||||
raise AssertionError(f"Unexpected reset status: {result['reset_status']!r}")
|
||||
|
||||
|
||||
def run_policy_session(
|
||||
*,
|
||||
host: str = DEFAULT_HOST,
|
||||
port: int = DEFAULT_PORT,
|
||||
path: str = DEFAULT_PATH,
|
||||
video_dir: Path,
|
||||
prompt: str = DEFAULT_PROMPT,
|
||||
session_id: str | None = None,
|
||||
num_chunks: int = 2,
|
||||
) -> dict[str, Any]:
|
||||
session_id = session_id or str(uuid.uuid4())
|
||||
camera_frames = load_camera_frames(video_dir)
|
||||
observations = build_demo_observations(
|
||||
camera_frames,
|
||||
prompt=prompt,
|
||||
session_id=session_id,
|
||||
num_chunks=num_chunks,
|
||||
)
|
||||
|
||||
client = OpenPIWebsocketClient(host=host, port=port, path=path)
|
||||
try:
|
||||
metadata = client.get_server_metadata()
|
||||
actions = [client.infer(obs) for obs in observations]
|
||||
reset_status = client.reset({})
|
||||
actions.append(client.infer(observations[0]))
|
||||
return {
|
||||
"metadata": metadata,
|
||||
"actions": actions,
|
||||
"reset_status": reset_status,
|
||||
"session_id": session_id,
|
||||
}
|
||||
finally:
|
||||
client.close()
|
||||
@@ -0,0 +1,24 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.diffusion.models.dreamzero.utils import (
|
||||
DEFAULT_CFG_SCALE,
|
||||
DEFAULT_EMBODIMENT_NAME_TO_ID,
|
||||
DEFAULT_NEGATIVE_PROMPT,
|
||||
DEFAULT_NUM_INFERENCE_STEPS,
|
||||
DEFAULT_SEED,
|
||||
DEFAULT_SIGMA_SHIFT,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_dreamzero_default_constants_match_source_baseline():
|
||||
assert DEFAULT_NUM_INFERENCE_STEPS == 16
|
||||
assert DEFAULT_CFG_SCALE == 5.0
|
||||
assert DEFAULT_SIGMA_SHIFT == 5.0
|
||||
assert DEFAULT_SEED == 1140
|
||||
assert "worst quality" in DEFAULT_NEGATIVE_PROMPT
|
||||
assert DEFAULT_EMBODIMENT_NAME_TO_ID["oxe_droid"] == 17
|
||||
@@ -0,0 +1,46 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from tests.dreamzero import openpi_client_helper
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class FakeMsgpackNumpy:
|
||||
payloads = {}
|
||||
|
||||
@staticmethod
|
||||
def packb(obj):
|
||||
key = str(len(FakeMsgpackNumpy.payloads)).encode()
|
||||
FakeMsgpackNumpy.payloads[key] = obj
|
||||
return key
|
||||
|
||||
@staticmethod
|
||||
def unpackb(data):
|
||||
return FakeMsgpackNumpy.payloads[data]
|
||||
|
||||
|
||||
def test_decode_action_response_surfaces_structured_error(monkeypatch):
|
||||
monkeypatch.setattr(openpi_client_helper, "msgpack_numpy", FakeMsgpackNumpy)
|
||||
payload = FakeMsgpackNumpy.packb(
|
||||
{
|
||||
"type": "error",
|
||||
"message": "Internal inference error",
|
||||
}
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="Internal inference error"):
|
||||
openpi_client_helper._decode_action_response(payload)
|
||||
|
||||
|
||||
def test_decode_action_response_converts_action_payload_to_float32(monkeypatch):
|
||||
monkeypatch.setattr(openpi_client_helper, "msgpack_numpy", FakeMsgpackNumpy)
|
||||
payload = FakeMsgpackNumpy.packb(np.asarray([[1.0, 2.0]], dtype=np.float64))
|
||||
|
||||
actions = openpi_client_helper._decode_action_response(payload)
|
||||
|
||||
assert actions.dtype == np.float32
|
||||
np.testing.assert_array_equal(actions, np.asarray([[1.0, 2.0]], dtype=np.float32))
|
||||
@@ -0,0 +1,59 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections import OrderedDict
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.models.dreamzero.pipeline_dreamzero import DreamZeroPipeline
|
||||
from vllm_omni.diffusion.models.dreamzero.state_dreamzero import DreamZeroState
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _empty_pipeline() -> DreamZeroPipeline:
|
||||
pipeline = DreamZeroPipeline.__new__(DreamZeroPipeline)
|
||||
pipeline._states = OrderedDict()
|
||||
pipeline._max_session_states = 2
|
||||
return pipeline
|
||||
|
||||
|
||||
def test_dreamzero_pipeline_state_is_session_keyed() -> None:
|
||||
pipeline = _empty_pipeline()
|
||||
|
||||
session_a = pipeline._get_or_create_state("session-a")
|
||||
session_b = pipeline._get_or_create_state("session-b")
|
||||
session_a.call_count = 7
|
||||
session_b.call_count = 3
|
||||
|
||||
assert pipeline._get_or_create_state("session-a") is session_a
|
||||
assert pipeline._get_or_create_state("session-b") is session_b
|
||||
assert session_a.call_count == 7
|
||||
assert session_b.call_count == 3
|
||||
|
||||
|
||||
def test_dreamzero_pipeline_state_lru_caps_retained_sessions() -> None:
|
||||
pipeline = _empty_pipeline()
|
||||
|
||||
session_a = pipeline._get_or_create_state("session-a")
|
||||
pipeline._get_or_create_state("session-b")
|
||||
assert pipeline._get_or_create_state("session-a") is session_a
|
||||
|
||||
pipeline._get_or_create_state("session-c")
|
||||
|
||||
assert list(pipeline._states) == ["session-a", "session-c"]
|
||||
assert "session-b" not in pipeline._states
|
||||
|
||||
|
||||
def test_dreamzero_state_cache_access_requires_initialization() -> None:
|
||||
state = DreamZeroState()
|
||||
|
||||
with pytest.raises(RuntimeError, match="KV caches not initialized"):
|
||||
state.get_kv_caches()
|
||||
|
||||
with pytest.raises(RuntimeError, match="Cross-attn caches not initialized"):
|
||||
state.get_crossattn_caches()
|
||||
|
||||
with pytest.raises(RuntimeError, match="create_kv_caches first"):
|
||||
state.update_kv_cache(0, torch.empty(0))
|
||||
@@ -0,0 +1,381 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Copied from the DreamZero repository's `test_client_AR.py`:
|
||||
https://github.com/dreamzero0/dreamzero/blob/main/test_client_AR.py
|
||||
|
||||
Kept here for end-to-end compatibility / parity testing against the vLLM
|
||||
OpenPI server.
|
||||
|
||||
Test client for AR_droid policy server using roboarena interface.
|
||||
|
||||
Sends real video frames from debug_image/ directory instead of zero dummy images.
|
||||
|
||||
Frame schedule (matching debug_inference.py):
|
||||
- Step 0 (initial): send frame [0] (1 frame, H W 3)
|
||||
- Step 1: send frames [0, 7, 15, 23] (4 frames, 4 H W 3)
|
||||
- Step 2: send frames [24, 31, 39, 47] (4 frames)
|
||||
- Step 3: send frames [48, 55, 63, 71] (4 frames)
|
||||
- ...
|
||||
|
||||
Expected server configuration:
|
||||
- image_resolution: (180, 320)
|
||||
- n_external_cameras: 2
|
||||
- needs_wrist_camera: True
|
||||
- action_space: "joint_position"
|
||||
|
||||
Usage:
|
||||
# Start server with roboarena interface:
|
||||
torchrun --nproc_per_node=8 socket_test_optimized_AR.py --port 8000
|
||||
|
||||
# Run against the original DreamZero websocket server (default path):
|
||||
python test_client_AR.py --host <server_host> --port 8000
|
||||
|
||||
# Run against the vLLM OpenPI server:
|
||||
python test_client_AR.py --host <server_host> --port 8000 --path /v1/realtime/robot/openpi
|
||||
|
||||
# Use zero images instead of real video (old behavior):
|
||||
python test_client_AR.py --host <server_host> --port 8000 --use-zero-images
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
from openpi_client import msgpack_numpy
|
||||
|
||||
_DREAMZERO_REPO_ENV = os.environ.get("DREAMZERO_REPO")
|
||||
DREAMZERO_REPO = Path(_DREAMZERO_REPO_ENV).expanduser() if _DREAMZERO_REPO_ENV else None
|
||||
|
||||
|
||||
def _import_upstream_policy_modules():
|
||||
if DREAMZERO_REPO is None:
|
||||
raise ImportError("Set DREAMZERO_REPO to an upstream DreamZero checkout before using this helper.")
|
||||
if DREAMZERO_REPO.exists() and str(DREAMZERO_REPO) not in sys.path:
|
||||
sys.path.insert(0, str(DREAMZERO_REPO))
|
||||
|
||||
import eval_utils.policy_server as policy_server
|
||||
from eval_utils.policy_client import WebsocketClientPolicy
|
||||
|
||||
return policy_server, WebsocketClientPolicy
|
||||
|
||||
|
||||
policy_server, WebsocketClientPolicy = _import_upstream_policy_modules()
|
||||
|
||||
VIDEO_DIR = os.environ.get(
|
||||
"DREAMZERO_VIDEO_DIR",
|
||||
str(DREAMZERO_REPO / "debug_image") if DREAMZERO_REPO is not None else "debug_image",
|
||||
)
|
||||
|
||||
# roboarena key -> video filename
|
||||
CAMERA_FILES = {
|
||||
"observation/exterior_image_0_left": "exterior_image_1_left.mp4",
|
||||
"observation/exterior_image_1_left": "exterior_image_2_left.mp4",
|
||||
"observation/wrist_image_left": "wrist_image_left.mp4",
|
||||
}
|
||||
|
||||
# Frame schedule constants (matching debug_inference.py)
|
||||
RELATIVE_OFFSETS = [-23, -16, -8, 0]
|
||||
ACTION_HORIZON = 24
|
||||
DEFAULT_WEBSOCKET_PATH = ""
|
||||
|
||||
|
||||
class OpenPIWebsocketClientPolicy(WebsocketClientPolicy):
|
||||
"""DreamZero websocket client with a configurable path suffix.
|
||||
|
||||
The original DreamZero client connects to ``ws://host:port``.
|
||||
vLLM serves the compatible robot policy endpoint at
|
||||
``/v1/realtime/robot/openpi`` when ``path`` is set accordingly.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
host: str = "0.0.0.0",
|
||||
port: int = 8000,
|
||||
path: str = DEFAULT_WEBSOCKET_PATH,
|
||||
) -> None:
|
||||
self._uri = f"ws://{host}:{port}{path}"
|
||||
self._packer = msgpack_numpy.Packer()
|
||||
self._ws, self._server_metadata = self._wait_for_server()
|
||||
|
||||
|
||||
def load_all_frames(video_path: str) -> np.ndarray:
|
||||
"""Load all frames from a video file. Returns (N, H, W, 3) uint8 array (RGB)."""
|
||||
cap = cv2.VideoCapture(video_path)
|
||||
frames = []
|
||||
while True:
|
||||
ret, frame = cap.read()
|
||||
if not ret:
|
||||
break
|
||||
frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
|
||||
cap.release()
|
||||
if not frames:
|
||||
raise RuntimeError(f"No frames loaded from {video_path}")
|
||||
return np.stack(frames, axis=0)
|
||||
|
||||
|
||||
def load_camera_frames() -> dict[str, np.ndarray]:
|
||||
"""Load all video frames for each camera from the debug_image/ directory.
|
||||
|
||||
Returns:
|
||||
Dict mapping roboarena camera keys to (N, H, W, 3) uint8 arrays.
|
||||
"""
|
||||
camera_frames: dict[str, np.ndarray] = {}
|
||||
for cam_key, fname in CAMERA_FILES.items():
|
||||
path = os.path.join(VIDEO_DIR, fname)
|
||||
camera_frames[cam_key] = load_all_frames(path)
|
||||
logging.info(f"Loaded {cam_key}: {camera_frames[cam_key].shape}")
|
||||
return camera_frames
|
||||
|
||||
|
||||
def build_frame_schedule(total_frames: int, num_chunks: int) -> list[list[int]]:
|
||||
"""Build the frame index schedule for multi-frame chunks.
|
||||
|
||||
Returns a list of frame-index lists. Each inner list has 4 indices.
|
||||
"""
|
||||
chunks: list[list[int]] = []
|
||||
current_frame = 23 # first anchor frame
|
||||
for _ in range(num_chunks):
|
||||
indices = [max(current_frame + off, 0) for off in RELATIVE_OFFSETS]
|
||||
if indices[-1] >= total_frames:
|
||||
logging.info(f"Frame {indices[-1]} >= {total_frames}, stopping at {len(chunks)} chunks")
|
||||
break
|
||||
chunks.append(indices)
|
||||
current_frame += ACTION_HORIZON
|
||||
return chunks
|
||||
|
||||
|
||||
def _make_obs_from_video(
|
||||
camera_frames: dict[str, np.ndarray],
|
||||
frame_indices: list[int],
|
||||
prompt: str,
|
||||
session_id: str,
|
||||
) -> dict:
|
||||
"""Build an observation dict from real video frames.
|
||||
|
||||
For 1 frame: each image key is (H, W, 3).
|
||||
For 4 frames: each image key is (4, H, W, 3).
|
||||
"""
|
||||
obs: dict = {}
|
||||
for cam_key, all_frames in camera_frames.items():
|
||||
selected = all_frames[frame_indices] # (T, H, W, 3)
|
||||
if len(frame_indices) == 1:
|
||||
selected = selected[0] # (H, W, 3)
|
||||
obs[cam_key] = selected
|
||||
|
||||
obs["observation/joint_position"] = np.zeros(7, dtype=np.float32)
|
||||
obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32)
|
||||
obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32)
|
||||
obs["prompt"] = prompt
|
||||
obs["session_id"] = session_id
|
||||
return obs
|
||||
|
||||
|
||||
def _make_zero_observation(
|
||||
server_config: policy_server.PolicyServerConfig,
|
||||
prompt: str = "pick up the object",
|
||||
session_id: str | None = None,
|
||||
) -> dict:
|
||||
"""Create a dummy observation matching AR_droid expectations.
|
||||
|
||||
AR_droid expects:
|
||||
- 2 external cameras (exterior_image_0_left, exterior_image_1_left)
|
||||
- 1 wrist camera (wrist_image_left)
|
||||
- Image resolution: 180x320 (H x W)
|
||||
- joint_position: 7 DoF
|
||||
- gripper_position: 1 DoF
|
||||
"""
|
||||
obs = {}
|
||||
|
||||
# Determine image resolution
|
||||
if server_config.image_resolution is not None:
|
||||
h, w = server_config.image_resolution
|
||||
else:
|
||||
# Default for AR_droid
|
||||
h, w = 180, 320
|
||||
|
||||
# External cameras (0-indexed in roboarena)
|
||||
for i in range(server_config.n_external_cameras):
|
||||
obs[f"observation/exterior_image_{i}_left"] = np.zeros((h, w, 3), dtype=np.uint8)
|
||||
if server_config.needs_stereo_camera:
|
||||
obs[f"observation/exterior_image_{i}_right"] = np.zeros((h, w, 3), dtype=np.uint8)
|
||||
|
||||
# Wrist camera
|
||||
if server_config.needs_wrist_camera:
|
||||
obs["observation/wrist_image_left"] = np.zeros((h, w, 3), dtype=np.uint8)
|
||||
if server_config.needs_stereo_camera:
|
||||
obs["observation/wrist_image_right"] = np.zeros((h, w, 3), dtype=np.uint8)
|
||||
|
||||
# Session ID - should be passed in to ensure consistency within a session
|
||||
if server_config.needs_session_id:
|
||||
import uuid
|
||||
|
||||
# Generate unique session ID if not provided
|
||||
obs["session_id"] = session_id if session_id else str(uuid.uuid4())
|
||||
|
||||
# State observations (AR_droid: 7 DoF arm + 1 gripper)
|
||||
obs["observation/joint_position"] = np.zeros(7, dtype=np.float32)
|
||||
obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32)
|
||||
obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32)
|
||||
|
||||
# Language prompt
|
||||
obs["prompt"] = prompt
|
||||
|
||||
return obs
|
||||
|
||||
|
||||
def test_ar_droid_policy_server(
|
||||
host: str = "localhost",
|
||||
port: int = 8000,
|
||||
path: str = DEFAULT_WEBSOCKET_PATH,
|
||||
num_chunks: int = 15,
|
||||
prompt: str = "Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan",
|
||||
use_zero_images: bool = False,
|
||||
):
|
||||
"""Test the AR_droid policy server with roboarena interface.
|
||||
|
||||
When use_zero_images is False (default), loads real video frames from
|
||||
debug_image/ and follows the frame schedule from debug_inference.py.
|
||||
"""
|
||||
logging.info(f"Connecting to AR_droid server at ws://{host}:{port}{path} ...")
|
||||
|
||||
client = OpenPIWebsocketClientPolicy(host=host, port=port, path=path)
|
||||
|
||||
# Validate server metadata
|
||||
metadata = client.get_server_metadata()
|
||||
logging.info(f"Server metadata: {metadata}")
|
||||
assert isinstance(metadata, dict), "Metadata should be a dict"
|
||||
|
||||
try:
|
||||
server_config = policy_server.PolicyServerConfig(**metadata)
|
||||
except Exception as e:
|
||||
logging.error(f"Error parsing metadata: {e}")
|
||||
raise e
|
||||
|
||||
# Validate expected AR_droid configuration
|
||||
logging.info(f"Server config: {server_config}")
|
||||
assert server_config.n_external_cameras == 2, f"Expected 2 external cameras, got {server_config.n_external_cameras}"
|
||||
assert server_config.needs_wrist_camera, "Expected wrist camera to be enabled"
|
||||
assert server_config.action_space == "joint_position", (
|
||||
f"Expected joint_position action space, got {server_config.action_space}"
|
||||
)
|
||||
|
||||
logging.info("Server configuration validated for AR_droid")
|
||||
|
||||
# Generate unique session ID for this test run
|
||||
session_id = str(uuid.uuid4())
|
||||
logging.info(f"Session ID: {session_id}")
|
||||
|
||||
# ── Zero-image fallback mode ──────────────────────────────────────
|
||||
if use_zero_images:
|
||||
logging.info("Using ZERO dummy images (legacy mode)")
|
||||
for i in range(num_chunks):
|
||||
obs = _make_zero_observation(server_config, prompt=prompt, session_id=session_id)
|
||||
logging.info(f"Inference {i + 1}/{num_chunks}: prompt='{prompt}'")
|
||||
t0 = time.time()
|
||||
actions = client.infer(obs)
|
||||
dt = time.time() - t0
|
||||
_log_action(actions, dt)
|
||||
|
||||
logging.info("Sending reset...")
|
||||
client.reset({})
|
||||
logging.info("Done (zero-image mode).")
|
||||
return
|
||||
|
||||
# ── Real video frame mode ─────────────────────────────────────────
|
||||
logging.info("Loading real video frames from debug_image/ directory")
|
||||
camera_frames = load_camera_frames()
|
||||
|
||||
total_frames = min(v.shape[0] for v in camera_frames.values())
|
||||
logging.info(f"Total frames available: {total_frames}")
|
||||
|
||||
# Build frame schedule
|
||||
chunks = build_frame_schedule(total_frames, num_chunks)
|
||||
|
||||
logging.info("Frame schedule:")
|
||||
logging.info(" Initial: [0]")
|
||||
for i, indices in enumerate(chunks):
|
||||
logging.info(f" Chunk {i}: {indices}")
|
||||
|
||||
# Step 0: initial single frame
|
||||
logging.info("=== Initial: frame [0] ===")
|
||||
obs = _make_obs_from_video(camera_frames, [0], prompt, session_id)
|
||||
t0 = time.time()
|
||||
actions = client.infer(obs)
|
||||
dt = time.time() - t0
|
||||
_log_action(actions, dt)
|
||||
|
||||
# Subsequent chunks: send 4 frames at a time
|
||||
for chunk_idx, frame_indices in enumerate(chunks):
|
||||
logging.info(f"=== Chunk {chunk_idx}: frames {frame_indices} ===")
|
||||
obs = _make_obs_from_video(camera_frames, frame_indices, prompt, session_id)
|
||||
t0 = time.time()
|
||||
actions = client.infer(obs)
|
||||
dt = time.time() - t0
|
||||
_log_action(actions, dt)
|
||||
|
||||
# Reset triggers video save on the server
|
||||
logging.info("Sending reset to save video...")
|
||||
client.reset({})
|
||||
|
||||
logging.info("Done.")
|
||||
|
||||
|
||||
def _log_action(actions: np.ndarray, dt: float) -> None:
|
||||
"""Pretty-print action shape, range, and timing."""
|
||||
assert isinstance(actions, np.ndarray), f"Expected numpy array, got {type(actions)}"
|
||||
assert actions.ndim == 2, f"Expected 2D array, got shape {actions.shape}"
|
||||
assert actions.shape[-1] == 8, f"Expected 8 action dims (7 joints + 1 gripper), got {actions.shape[-1]}"
|
||||
logging.info(f" Action shape: {actions.shape}, range: [{actions.min():.4f}, {actions.max():.4f}], time: {dt:.2f}s")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Test AR_droid policy server with real video frames from debug_image/")
|
||||
parser.add_argument("--host", default="localhost", help="Server hostname")
|
||||
parser.add_argument("--port", type=int, default=8000, help="Server port")
|
||||
parser.add_argument(
|
||||
"--path",
|
||||
default=DEFAULT_WEBSOCKET_PATH,
|
||||
help="WebSocket path suffix (default: empty string for the original DreamZero server)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-chunks",
|
||||
type=int,
|
||||
default=15,
|
||||
help="Number of 4-frame chunks to send after the initial frame (default: 15)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prompt",
|
||||
default="Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan",
|
||||
help="Language prompt for the policy",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--use-zero-images",
|
||||
action="store_true",
|
||||
help="Use zero dummy images instead of real video frames (legacy mode)",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||
)
|
||||
|
||||
test_ar_droid_policy_server(
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
path=args.path,
|
||||
num_chunks=args.num_chunks,
|
||||
prompt=args.prompt,
|
||||
use_zero_images=args.use_zero_images,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,358 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Formal OpenPI end-to-end parity: upstream DreamZero server vs `vllm serve`.
|
||||
|
||||
This test uses DreamZero's own client-side observation builders from
|
||||
`${DREAMZERO_REPO}/test_client_AR.py`, and client-side websocket protocol from
|
||||
`${DREAMZERO_REPO}/eval_utils/policy_client.py`.
|
||||
|
||||
The only client-side adaptation for vLLM is the websocket path:
|
||||
DreamZero's upstream server serves at `/`, while vLLM serves OpenPI at
|
||||
`/v1/realtime/robot/openpi`.
|
||||
|
||||
Current scope for this test:
|
||||
- default two-GPU run (`nproc_per_node=2` on upstream, `--cfg-parallel-size 2` on `vllm serve`)
|
||||
- non-`torch.compile` (upstream launched through
|
||||
`upstream_socket_server_no_compile.py`, vLLM with `--enforce-eager`)
|
||||
- non-DiT-cache / non-skip-schedule (`NUM_DIT_STEPS=16`)
|
||||
|
||||
Serving contract locked by this test:
|
||||
- upstream DreamZero still boots from the local checkpoint directory
|
||||
- vLLM boots from the official DreamZero HF repo name (`GEAR-Dreams/DreamZero-DROID`)
|
||||
rather than a prepared local bundle path
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.helpers.runtime import get_open_port
|
||||
|
||||
msgpack_numpy = pytest.importorskip("openpi_client.msgpack_numpy")
|
||||
|
||||
_DREAMZERO_REPO_ENV = os.environ.get("DREAMZERO_REPO")
|
||||
DREAMZERO_REPO = Path(_DREAMZERO_REPO_ENV).expanduser() if _DREAMZERO_REPO_ENV else None
|
||||
if DREAMZERO_REPO is not None and str(DREAMZERO_REPO) not in sys.path:
|
||||
sys.path.insert(0, str(DREAMZERO_REPO))
|
||||
|
||||
try:
|
||||
import test_client_AR as dreamzero_client
|
||||
from eval_utils.policy_client import WebsocketClientPolicy
|
||||
except Exception: # pragma: no cover - guarded by pytest skip below
|
||||
dreamzero_client = None
|
||||
WebsocketClientPolicy = None
|
||||
_BaseWebsocketClientPolicy = WebsocketClientPolicy if WebsocketClientPolicy is not None else object
|
||||
|
||||
CHECKPOINT_DIR = DREAMZERO_REPO / "checkpoints" / "dreamzero" if DREAMZERO_REPO is not None else None
|
||||
VLLM_MODEL = os.environ.get("VLLM_DREAMZERO_MODEL", "GEAR-Dreams/DreamZero-DROID")
|
||||
SERVICE_READY_TIMEOUT_S = int(os.environ.get("OPENPI_SERVICE_READY_TIMEOUT_S", "900"))
|
||||
PROMPT = "Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan"
|
||||
SESSION_ID = "openpi-e2e-parity-session"
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.skipif(not torch.cuda.is_available(), reason="GPU required"),
|
||||
pytest.mark.skipif(
|
||||
dreamzero_client is None or WebsocketClientPolicy is None,
|
||||
reason="DreamZero client modules are required on PYTHONPATH",
|
||||
),
|
||||
pytest.mark.skipif(
|
||||
DREAMZERO_REPO is None or not DREAMZERO_REPO.exists(),
|
||||
reason="DreamZero source repo is required at DREAMZERO_REPO",
|
||||
),
|
||||
pytest.mark.skipif(
|
||||
CHECKPOINT_DIR is None or not CHECKPOINT_DIR.exists(), reason="DreamZero local checkpoint is required"
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class OpenPIWebsocketClientPolicy(_BaseWebsocketClientPolicy):
|
||||
"""DreamZero client protocol with an OpenPI websocket path suffix."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
host: str = "127.0.0.1",
|
||||
port: int = 8000,
|
||||
path: str = "/v1/realtime/robot/openpi",
|
||||
) -> None:
|
||||
self._uri = f"ws://{host}:{port}{path}"
|
||||
self._packer = msgpack_numpy.Packer()
|
||||
self._ws, self._server_metadata = self._wait_for_server()
|
||||
|
||||
|
||||
def _vllm_executable() -> str:
|
||||
fallback = Path(sys.executable).with_name("vllm")
|
||||
if fallback.exists():
|
||||
return str(fallback)
|
||||
exe = shutil.which("vllm")
|
||||
if exe:
|
||||
return exe
|
||||
raise FileNotFoundError("Unable to locate `vllm` executable in current environment.")
|
||||
|
||||
|
||||
def _cfg_parallel_size() -> int:
|
||||
return int(os.environ.get("OPENPI_E2E_CFG_PARALLEL_SIZE", "2"))
|
||||
|
||||
|
||||
def _pick_test_gpus() -> list[str]:
|
||||
cfg_parallel_size = _cfg_parallel_size()
|
||||
override = os.environ.get("OPENPI_E2E_GPUS") or os.environ.get("OPENPI_E2E_GPU")
|
||||
if override is not None:
|
||||
gpus = [part.strip() for part in override.split(",") if part.strip()]
|
||||
if not gpus:
|
||||
raise ValueError("OPENPI_E2E_GPUS is set but empty.")
|
||||
if len(gpus) < cfg_parallel_size:
|
||||
raise RuntimeError(f"Need {cfg_parallel_size} GPUs, but OPENPI_E2E_GPUS only provided {gpus}.")
|
||||
return gpus
|
||||
|
||||
query = subprocess.check_output(
|
||||
[
|
||||
"nvidia-smi",
|
||||
"--query-gpu=index,memory.used",
|
||||
"--format=csv,noheader,nounits",
|
||||
],
|
||||
text=True,
|
||||
)
|
||||
gpu_rows = []
|
||||
for line in query.strip().splitlines():
|
||||
gpu_index, used_mb = [part.strip() for part in line.split(",", maxsplit=1)]
|
||||
gpu_rows.append((int(used_mb), gpu_index))
|
||||
gpu_rows.sort()
|
||||
gpus = [gpu_index for _, gpu_index in gpu_rows[: max(cfg_parallel_size, 1)]]
|
||||
if len(gpus) < cfg_parallel_size:
|
||||
raise RuntimeError(
|
||||
f"Need {cfg_parallel_size} GPUs for cfg_parallel_size={cfg_parallel_size}, "
|
||||
f"but found only {len(gpus)} candidates."
|
||||
)
|
||||
return gpus
|
||||
|
||||
|
||||
def _torchrun_argv(script: str, port: int) -> list[str]:
|
||||
return [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"torch.distributed.run",
|
||||
"--standalone",
|
||||
"--nproc_per_node",
|
||||
str(_cfg_parallel_size()),
|
||||
script,
|
||||
"--port",
|
||||
str(port),
|
||||
"--model_path",
|
||||
str(CHECKPOINT_DIR),
|
||||
]
|
||||
|
||||
|
||||
def _run_upstream_service(port: int, log_path: Path) -> subprocess.Popen[str]:
|
||||
env = os.environ.copy()
|
||||
env.setdefault("PYTHONPATH", "")
|
||||
env["PYTHONPATH"] = f"{Path.cwd()}:{DREAMZERO_REPO}:{env['PYTHONPATH']}".rstrip(":")
|
||||
env["CUDA_VISIBLE_DEVICES"] = ",".join(_pick_test_gpus())
|
||||
env.setdefault("NO_ALBUMENTATIONS_UPDATE", "1")
|
||||
env["ATTENTION_BACKEND"] = "torch"
|
||||
env.setdefault("ENABLE_TENSORRT", "false")
|
||||
env["ENABLE_DIT_CACHE"] = "false"
|
||||
env["NUM_DIT_STEPS"] = "16"
|
||||
env["DYNAMIC_CACHE_SCHEDULE"] = "false"
|
||||
argv = _torchrun_argv(
|
||||
str(Path("tests/dreamzero/upstream/upstream_socket_server_no_compile.py")),
|
||||
port,
|
||||
)
|
||||
log_file = log_path.open("w")
|
||||
proc = subprocess.Popen(
|
||||
argv,
|
||||
stdout=log_file,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
cwd=str(Path.cwd()),
|
||||
env=env,
|
||||
)
|
||||
proc._codex_log_file = log_file # type: ignore[attr-defined]
|
||||
return proc
|
||||
|
||||
|
||||
def _run_vllm_service(port: int, log_path: Path) -> subprocess.Popen[str]:
|
||||
env = os.environ.copy()
|
||||
gpus = _pick_test_gpus()
|
||||
cfg_parallel_size = _cfg_parallel_size()
|
||||
if cfg_parallel_size > len(gpus):
|
||||
raise RuntimeError(
|
||||
f"cfg_parallel_size={cfg_parallel_size} requires at least {cfg_parallel_size} GPUs, but only got {gpus}."
|
||||
)
|
||||
env["CUDA_VISIBLE_DEVICES"] = ",".join(gpus[:cfg_parallel_size])
|
||||
env.setdefault("ATTENTION_BACKEND", "torch")
|
||||
env.setdefault("DIFFUSION_ATTENTION_BACKEND", "TORCH_SDPA")
|
||||
env.setdefault("MASTER_PORT", str(get_open_port()))
|
||||
argv = [
|
||||
_vllm_executable(),
|
||||
"serve",
|
||||
VLLM_MODEL,
|
||||
"--omni",
|
||||
"--host",
|
||||
"127.0.0.1",
|
||||
"--port",
|
||||
str(port),
|
||||
"--served-model-name",
|
||||
"dreamzero-droid",
|
||||
"--enforce-eager",
|
||||
]
|
||||
if cfg_parallel_size > 1:
|
||||
argv.extend(["--cfg-parallel-size", str(cfg_parallel_size)])
|
||||
log_file = log_path.open("w")
|
||||
proc = subprocess.Popen(
|
||||
argv,
|
||||
stdout=log_file,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
env=env,
|
||||
cwd=str(Path.cwd()),
|
||||
)
|
||||
proc._codex_log_file = log_file # type: ignore[attr-defined]
|
||||
return proc
|
||||
|
||||
|
||||
def _stop_process(proc: subprocess.Popen[str]) -> None:
|
||||
log_file = getattr(proc, "_codex_log_file", None)
|
||||
if proc.poll() is None:
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=30)
|
||||
except subprocess.TimeoutExpired: # pragma: no cover - cleanup path
|
||||
proc.kill()
|
||||
proc.wait(timeout=10)
|
||||
if log_file is not None:
|
||||
log_file.close()
|
||||
|
||||
|
||||
def _build_obs_sequence() -> tuple[dict, dict]:
|
||||
camera_frames = dreamzero_client.load_camera_frames()
|
||||
chunks = dreamzero_client.build_frame_schedule(
|
||||
min(v.shape[0] for v in camera_frames.values()),
|
||||
1,
|
||||
)
|
||||
obs0 = dreamzero_client._make_obs_from_video(camera_frames, [0], PROMPT, SESSION_ID)
|
||||
obs1 = dreamzero_client._make_obs_from_video(camera_frames, chunks[0], PROMPT, SESSION_ID)
|
||||
return obs0, obs1
|
||||
|
||||
|
||||
def _wait_for_client_ready(client_factory, timeout_s: float, proc=None, log_path: Path | None = None):
|
||||
deadline = time.time() + timeout_s
|
||||
last_err: Exception | None = None
|
||||
while time.time() < deadline:
|
||||
if proc is not None and proc.poll() is not None:
|
||||
details = ""
|
||||
if log_path is not None and log_path.exists():
|
||||
details = log_path.read_text(errors="replace")[-8000:]
|
||||
raise RuntimeError(f"Service exited before becoming ready with code {proc.returncode}.\n{details}")
|
||||
try:
|
||||
return client_factory()
|
||||
except Exception as exc: # pragma: no cover - retry path
|
||||
last_err = exc
|
||||
time.sleep(1)
|
||||
raise TimeoutError(f"Timed out waiting for websocket service: {last_err}")
|
||||
|
||||
|
||||
def _collect_outputs_with_client(client) -> tuple[dict, list[np.ndarray]]:
|
||||
metadata = client.get_server_metadata()
|
||||
obs0, obs1 = _build_obs_sequence()
|
||||
outputs = [
|
||||
client.infer(dict(obs0)),
|
||||
client.infer(dict(obs1)),
|
||||
]
|
||||
assert _normalize_reset_response(client.reset({})) == "reset successful"
|
||||
outputs.append(client.infer(dict(obs0)))
|
||||
client._ws.close()
|
||||
return metadata, outputs
|
||||
|
||||
|
||||
def _normalize_reset_response(response) -> str:
|
||||
if isinstance(response, str):
|
||||
return response
|
||||
decoded = msgpack_numpy.unpackb(response)
|
||||
if isinstance(decoded, dict):
|
||||
return str(decoded.get("status"))
|
||||
return str(decoded)
|
||||
|
||||
|
||||
def _normalize_metadata(metadata: dict) -> dict:
|
||||
normalized = dict(metadata)
|
||||
if isinstance(normalized.get("image_resolution"), tuple):
|
||||
normalized["image_resolution"] = list(normalized["image_resolution"])
|
||||
return normalized
|
||||
|
||||
|
||||
def _assert_logs_clean(log_path: Path) -> None:
|
||||
text = log_path.read_text(errors="replace")
|
||||
if "SignalException: Process" in text and "got signal: 15" in text:
|
||||
text = text.split("Traceback (most recent call last):", 1)[0]
|
||||
assert "Traceback" not in text, text
|
||||
assert "RuntimeError:" not in text, text
|
||||
|
||||
|
||||
def _assert_upstream_log_matches_vllm_baseline(log_path: Path) -> None:
|
||||
text = log_path.read_text(errors="replace")
|
||||
assert "DIT Compute Steps 8 steps" not in text, text
|
||||
assert "DIT Compute Steps 16 steps" in text, text
|
||||
|
||||
|
||||
def test_openpi_service_matches_upstream_server_noncompile(tmp_path: Path) -> None:
|
||||
expected_metadata = {
|
||||
"image_resolution": [180, 320],
|
||||
"n_external_cameras": 2,
|
||||
"needs_wrist_camera": True,
|
||||
"needs_stereo_camera": False,
|
||||
"needs_session_id": True,
|
||||
"action_space": "joint_position",
|
||||
}
|
||||
|
||||
upstream_port = get_open_port()
|
||||
upstream_log = tmp_path / "dreamzero_upstream.log"
|
||||
upstream_proc = _run_upstream_service(upstream_port, upstream_log)
|
||||
try:
|
||||
upstream_client = _wait_for_client_ready(
|
||||
lambda: WebsocketClientPolicy(host="127.0.0.1", port=upstream_port),
|
||||
timeout_s=SERVICE_READY_TIMEOUT_S,
|
||||
proc=upstream_proc,
|
||||
log_path=upstream_log,
|
||||
)
|
||||
upstream_metadata, upstream_outputs = _collect_outputs_with_client(upstream_client)
|
||||
finally:
|
||||
_stop_process(upstream_proc)
|
||||
_assert_logs_clean(upstream_log)
|
||||
_assert_upstream_log_matches_vllm_baseline(upstream_log)
|
||||
|
||||
vllm_port = get_open_port()
|
||||
vllm_log = tmp_path / "vllm_openpi.log"
|
||||
vllm_proc = _run_vllm_service(vllm_port, vllm_log)
|
||||
try:
|
||||
vllm_client = _wait_for_client_ready(
|
||||
lambda: OpenPIWebsocketClientPolicy(host="127.0.0.1", port=vllm_port),
|
||||
timeout_s=SERVICE_READY_TIMEOUT_S,
|
||||
proc=vllm_proc,
|
||||
log_path=vllm_log,
|
||||
)
|
||||
vllm_metadata, vllm_outputs = _collect_outputs_with_client(vllm_client)
|
||||
finally:
|
||||
_stop_process(vllm_proc)
|
||||
_assert_logs_clean(vllm_log)
|
||||
|
||||
assert _normalize_metadata(upstream_metadata) == expected_metadata
|
||||
assert _normalize_metadata(vllm_metadata) == expected_metadata
|
||||
|
||||
for idx, (actual, expected) in enumerate(zip(vllm_outputs, upstream_outputs, strict=True)):
|
||||
np.testing.assert_allclose(
|
||||
actual,
|
||||
expected,
|
||||
rtol=1e-2,
|
||||
atol=1e-3,
|
||||
err_msg=f"OpenPI step {idx} output mismatch",
|
||||
)
|
||||
@@ -0,0 +1,184 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Launch the upstream DreamZero websocket server with `torch.compile` disabled.
|
||||
|
||||
This wrapper is meant for formal parity tests against `vllm serve --omni`.
|
||||
It monkeypatches `torch.compile` before importing DreamZero modules so that
|
||||
all import-time decorators and `post_initialize()` compile calls become eager.
|
||||
|
||||
For the current DreamZero port baseline we also disable all upstream DiT cache
|
||||
and step-skipping behavior so the reference server matches vLLM's current
|
||||
eager/no-skip implementation:
|
||||
- `ENABLE_DIT_CACHE=false`
|
||||
- `NUM_DIT_STEPS=16`
|
||||
- `DYNAMIC_CACHE_SCHEDULE=false`
|
||||
|
||||
In our CI/dev environment we also do not install Transformer Engine or
|
||||
FlashAttention, so upstream's default `socket_test_optimized_AR.main()` path
|
||||
(`ATTENTION_BACKEND=TE` and then fallback to `FA2`) cannot execute. To keep the
|
||||
formal server-vs-server test runnable without pulling in those heavyweight
|
||||
optional deps, this wrapper reproduces upstream `main()` but pins attention to
|
||||
PyTorch SDPA (`ATTENTION_BACKEND=torch`) for this subprocess only.
|
||||
|
||||
Usage:
|
||||
PYTHONPATH="${DREAMZERO_REPO}" \\
|
||||
.venv/bin/python -m torch.distributed.run --standalone --nproc_per_node=2 \\
|
||||
tests/dreamzero/upstream/upstream_socket_server_no_compile.py --port 18081 \\
|
||||
--model_path "${DREAMZERO_REPO}/checkpoints/dreamzero"
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
|
||||
def _identity_compile(*args, **kwargs):
|
||||
if args and callable(args[0]) and len(args) == 1 and not kwargs:
|
||||
return args[0]
|
||||
|
||||
def deco(fn):
|
||||
return fn
|
||||
|
||||
return deco
|
||||
|
||||
|
||||
torch.compile = _identity_compile
|
||||
|
||||
os.environ.setdefault("NO_ALBUMENTATIONS_UPDATE", "1")
|
||||
|
||||
DREAMZERO_REPO_ENV = os.environ.get("DREAMZERO_REPO")
|
||||
if not DREAMZERO_REPO_ENV:
|
||||
raise RuntimeError("Set DREAMZERO_REPO to an upstream DreamZero checkout before launching this helper.")
|
||||
DREAMZERO_REPO = Path(DREAMZERO_REPO_ENV).expanduser()
|
||||
if str(DREAMZERO_REPO) not in sys.path:
|
||||
sys.path.insert(0, str(DREAMZERO_REPO))
|
||||
|
||||
import socket_test_optimized_AR as upstream # noqa: E402
|
||||
import tyro # noqa: E402
|
||||
from groot.vla.model.dreamzero.modules import attention as upstream_attention # noqa: E402
|
||||
from groot.vla.model.dreamzero.modules import wan2_1_submodule as upstream_submodule # noqa: E402
|
||||
from groot.vla.model.dreamzero.modules import wan_video_dit as upstream_wan_video_dit # noqa: E402
|
||||
|
||||
|
||||
def _torch_varlen_flash_attention(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
q_lens: torch.Tensor | None = None,
|
||||
k_lens: torch.Tensor | None = None,
|
||||
dropout_p: float = 0.0,
|
||||
causal: bool = False,
|
||||
dtype: torch.dtype = torch.bfloat16,
|
||||
**_: object,
|
||||
) -> torch.Tensor:
|
||||
if q_lens is not None or k_lens is not None:
|
||||
upstream_attention.warnings.warn(
|
||||
"Padding mask is disabled in the test-only SDPA fallback.",
|
||||
)
|
||||
out_dtype = q.dtype
|
||||
q = q.transpose(1, 2).to(dtype)
|
||||
k = k.transpose(1, 2).to(dtype)
|
||||
v = v.transpose(1, 2).to(dtype)
|
||||
out = F.scaled_dot_product_attention(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
attn_mask=None,
|
||||
is_causal=causal,
|
||||
dropout_p=dropout_p,
|
||||
)
|
||||
return out.transpose(1, 2).contiguous().to(out_dtype)
|
||||
|
||||
|
||||
upstream_attention.flash_attention = _torch_varlen_flash_attention
|
||||
upstream_submodule.flash_attention = _torch_varlen_flash_attention
|
||||
upstream_wan_video_dit.FLASH_ATTN_3_AVAILABLE = False
|
||||
upstream_wan_video_dit.FLASH_ATTN_2_AVAILABLE = False
|
||||
upstream_wan_video_dit.SAGE_ATTN_AVAILABLE = False
|
||||
|
||||
|
||||
def main(args: upstream.Args) -> None:
|
||||
os.environ["ENABLE_DIT_CACHE"] = "false"
|
||||
os.environ["ATTENTION_BACKEND"] = "torch"
|
||||
os.environ["NUM_DIT_STEPS"] = "16"
|
||||
os.environ["DYNAMIC_CACHE_SCHEDULE"] = "false"
|
||||
torch._dynamo.config.recompile_limit = 800
|
||||
|
||||
embodiment_tag = "oxe_droid"
|
||||
model_path = args.model_path
|
||||
|
||||
device_mesh = upstream.init_mesh()
|
||||
rank = upstream.dist.get_rank()
|
||||
|
||||
timeout_delta = upstream.datetime.timedelta(seconds=args.timeout_seconds)
|
||||
signal_group = upstream.dist.new_group(backend="gloo", timeout=timeout_delta)
|
||||
upstream.logger.info("Rank %s initialized signal_group (gloo)", rank)
|
||||
|
||||
policy = upstream.GrootSimPolicy(
|
||||
embodiment_tag=upstream.EmbodimentTag(embodiment_tag),
|
||||
model_path=model_path,
|
||||
device="cuda" if torch.cuda.is_available() else "cpu",
|
||||
device_mesh=device_mesh,
|
||||
)
|
||||
|
||||
hostname = upstream.socket.gethostname()
|
||||
local_ip = upstream.socket.gethostbyname(hostname)
|
||||
|
||||
if rank == 0:
|
||||
upstream.logging.info("Creating server (host: %s, ip: %s)", hostname, local_ip)
|
||||
parent_dir = os.path.dirname(model_path)
|
||||
date_suffix = upstream.datetime.datetime.now().strftime("%Y%m%d")
|
||||
checkpoint_name = os.path.basename(model_path)
|
||||
output_dir = os.path.join(
|
||||
parent_dir,
|
||||
f"real_world_eval_gen_{date_suffix}_{args.index}",
|
||||
checkpoint_name,
|
||||
)
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
upstream.logging.info("Videos will be saved to: %s", output_dir)
|
||||
else:
|
||||
output_dir = None
|
||||
upstream.logging.info("Rank %s starting as worker for distributed inference...", rank)
|
||||
|
||||
wrapper_policy = upstream.ARDroidRoboarenaPolicy(
|
||||
groot_policy=policy,
|
||||
signal_group=signal_group,
|
||||
output_dir=output_dir,
|
||||
)
|
||||
|
||||
server_config = upstream.PolicyServerConfig(
|
||||
image_resolution=(180, 320),
|
||||
needs_wrist_camera=True,
|
||||
n_external_cameras=2,
|
||||
needs_stereo_camera=False,
|
||||
needs_session_id=True,
|
||||
action_space="joint_position",
|
||||
)
|
||||
|
||||
if rank == 0:
|
||||
upstream.logging.info("Using roboarena policy server interface")
|
||||
upstream.logging.info("Server config: %s", server_config)
|
||||
roboarena_server = upstream.RoboarenaServer(
|
||||
policy=wrapper_policy,
|
||||
server_config=server_config,
|
||||
host="0.0.0.0",
|
||||
port=args.port,
|
||||
)
|
||||
roboarena_server.serve_forever()
|
||||
else:
|
||||
worker = upstream.DistributedWorker(
|
||||
policy=policy,
|
||||
signal_group=signal_group,
|
||||
)
|
||||
upstream.asyncio.run(worker.run())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
upstream.logging.basicConfig(level=upstream.logging.INFO, force=True)
|
||||
main(tyro.cli(upstream.Args))
|
||||
@@ -8,7 +8,7 @@ L4-style smoke runs against an already-running server. The pytest wrappers in
|
||||
H100 / MI325) because they launch the live Omni server inside the test.
|
||||
|
||||
1. **Daily-Omni** — MCQ accuracy fields in the saved JSON (``daily_omni_accuracy``, …); by default the
|
||||
run **fails** if accuracy is strictly below **0.67** (``--min-daily-omni-accuracy`` / ``ACC_BENCH_MIN_DAILY_OMNI_ACCURACY``).
|
||||
run **fails** if accuracy is strictly below **0.69** (``--min-daily-omni-accuracy`` / ``ACC_BENCH_MIN_DAILY_OMNI_ACCURACY``).
|
||||
2. **Seed-TTS** — ``seed-tts-eval``-style metrics when ``--seed-tts-wer-eval`` is used
|
||||
(WER / SIM / UTMOS keys from :func:`compute_seed_tts_wer_metrics`).
|
||||
|
||||
@@ -308,9 +308,9 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
||||
p.add_argument(
|
||||
"--min-daily-omni-accuracy",
|
||||
type=float,
|
||||
default=float((os.environ.get("ACC_BENCH_MIN_DAILY_OMNI_ACCURACY") or "0.67").strip() or "0.67"),
|
||||
default=float((os.environ.get("ACC_BENCH_MIN_DAILY_OMNI_ACCURACY") or "0.69").strip() or "0.69"),
|
||||
help="Fail when daily_omni_accuracy is strictly below this threshold (0–1). "
|
||||
"Default baseline 0.67; override with env ACC_BENCH_MIN_DAILY_OMNI_ACCURACY or pass 0 to disable the floor.",
|
||||
"Default baseline 0.69; override with env ACC_BENCH_MIN_DAILY_OMNI_ACCURACY or pass 0 to disable the floor.",
|
||||
)
|
||||
|
||||
p.add_argument(
|
||||
@@ -340,7 +340,7 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
||||
p.add_argument(
|
||||
"--max-seed-tts-mean-wer",
|
||||
type=float,
|
||||
default=0.5,
|
||||
default=0.35,
|
||||
help="If set, fail when seed_tts_content_error_mean is strictly above this value.",
|
||||
)
|
||||
p.add_argument(
|
||||
|
||||
@@ -218,7 +218,7 @@ def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> tuple[Image.I
|
||||
torch_dtype=torch.bfloat16,
|
||||
trust_remote_code=True,
|
||||
).to("cuda")
|
||||
|
||||
pipe.transformer.set_attention_backend("_flash_3_hub")
|
||||
_diffusers_dummy_run(pipe)
|
||||
|
||||
generator = torch.Generator(device="cuda").manual_seed(SEED)
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import copy
|
||||
import io
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
import yaml
|
||||
from PIL import Image
|
||||
|
||||
from tests.e2e.accuracy.helpers import assert_images_pixel_close, assert_similarity, model_output_dir
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniServer
|
||||
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]
|
||||
|
||||
MODEL_NAME = "tencent/HunyuanImage-3.0-Instruct"
|
||||
SEED = 42
|
||||
NUM_INFERENCE_STEPS = 50
|
||||
GUIDANCE_SCALE = 2.5
|
||||
HEIGHT = 1024
|
||||
WIDTH = 1024
|
||||
PROMPT = "A brown and white dog is running on the grass."
|
||||
MEAN_THRESHOLD = 3e-2
|
||||
P99_THRESHOLD = 3e-1
|
||||
SSIM_THRESHOLD = 0.97
|
||||
PSNR_THRESHOLD = 30.0
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent
|
||||
BASELINE_PATH = _REPO_ROOT / "tests" / "assets" / "hunyuan" / "hunyuan_baseline.png"
|
||||
_OFFLINE_SCRIPT = _REPO_ROOT / "examples" / "offline_inference" / "hunyuan_image3" / "end2end.py"
|
||||
|
||||
# DiT-only deploy config with trust_remote_code (based on hunyuan_image3_dit.yaml).
|
||||
_DEPLOY_CONFIG = {
|
||||
"pipeline": "hunyuan_image3_dit",
|
||||
"async_chunk": False,
|
||||
"trust_remote_code": True,
|
||||
"stages": [
|
||||
{
|
||||
"stage_id": 0,
|
||||
"max_num_seqs": 1,
|
||||
"gpu_memory_utilization": 0.9,
|
||||
"enforce_eager": True,
|
||||
"trust_remote_code": True,
|
||||
"devices": "0,1,2,3", # set dynamically by _write_deploy_config
|
||||
"vae_use_slicing": False,
|
||||
"vae_use_tiling": False,
|
||||
"parallel_config": {
|
||||
"pipeline_parallel_size": 1,
|
||||
"data_parallel_size": 1,
|
||||
"tensor_parallel_size": 4,
|
||||
"enable_expert_parallel": True,
|
||||
"sequence_parallel_size": 1,
|
||||
"ulysses_degree": 1,
|
||||
"ring_degree": 1,
|
||||
"cfg_parallel_size": 1,
|
||||
"vae_patch_parallel_size": 1,
|
||||
"use_hsdp": False,
|
||||
"hsdp_shard_size": -1,
|
||||
"hsdp_replicate_size": 1,
|
||||
},
|
||||
"default_sampling_params": {
|
||||
"seed": SEED,
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _model_name() -> str:
|
||||
return os.environ.get("HUNYUAN_IMAGE3_MODEL", MODEL_NAME)
|
||||
|
||||
|
||||
def _devices() -> str:
|
||||
return os.environ.get("HUNYUAN_IMAGE3_DEVICES", "0,1,2,3")
|
||||
|
||||
|
||||
def _write_deploy_config(path: Path) -> None:
|
||||
config = copy.deepcopy(_DEPLOY_CONFIG)
|
||||
devices = _devices()
|
||||
config["stages"][0]["devices"] = devices
|
||||
config["stages"][0]["parallel_config"]["tensor_parallel_size"] = len(devices.split(","))
|
||||
path.write_text(yaml.dump(config, default_flow_style=False, sort_keys=False))
|
||||
|
||||
|
||||
def _run_vllm_omni_hunyuan_image3_online(*, model: str, deploy_config: str, output_path: Path) -> Image.Image:
|
||||
server_args = [
|
||||
"--deploy-config",
|
||||
deploy_config,
|
||||
"--stage-init-timeout",
|
||||
"300",
|
||||
"--init-timeout",
|
||||
"900",
|
||||
"--enforce-eager",
|
||||
"--trust-remote-code",
|
||||
]
|
||||
with OmniServer(model, server_args, use_omni=True) as omni_server:
|
||||
response = requests.post(
|
||||
f"http://{omni_server.host}:{omni_server.port}/v1/images/generations",
|
||||
json={
|
||||
"model": omni_server.model,
|
||||
"prompt": PROMPT,
|
||||
"size": f"{WIDTH}x{HEIGHT}",
|
||||
"n": 1,
|
||||
"response_format": "b64_json",
|
||||
"num_inference_steps": NUM_INFERENCE_STEPS,
|
||||
"guidance_scale": GUIDANCE_SCALE,
|
||||
"seed": SEED,
|
||||
"bot_task": "none",
|
||||
"use_system_prompt": "en_unified",
|
||||
},
|
||||
timeout=600,
|
||||
)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
assert len(payload["data"]) == 1
|
||||
image_bytes = base64.b64decode(payload["data"][0]["b64_json"])
|
||||
image = Image.open(io.BytesIO(image_bytes)).convert("RGB")
|
||||
image.load()
|
||||
image.save(output_path)
|
||||
return image
|
||||
|
||||
|
||||
def _run_vllm_omni_hunyuan_image3_offline(*, model: str, deploy_config: str, output_path: Path) -> Image.Image:
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
output_dir = str(output_path.parent)
|
||||
subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(_OFFLINE_SCRIPT),
|
||||
"--modality",
|
||||
"text2img",
|
||||
"--deploy-config",
|
||||
deploy_config,
|
||||
"--prompts",
|
||||
PROMPT,
|
||||
"--output",
|
||||
output_dir,
|
||||
"--steps",
|
||||
str(NUM_INFERENCE_STEPS),
|
||||
"--guidance-scale",
|
||||
str(GUIDANCE_SCALE),
|
||||
"--seed",
|
||||
str(SEED),
|
||||
"--height",
|
||||
str(HEIGHT),
|
||||
"--width",
|
||||
str(WIDTH),
|
||||
"--bot-task",
|
||||
"none",
|
||||
"--sys-type",
|
||||
"en_unified",
|
||||
"--model",
|
||||
model,
|
||||
"--enforce-eager",
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
images = sorted(Path(output_dir).glob("output_*.png"))
|
||||
assert images, f"No output image found in {output_dir}"
|
||||
image = Image.open(images[0]).convert("RGB")
|
||||
image.load()
|
||||
image.save(output_path)
|
||||
return image
|
||||
|
||||
|
||||
def _assert_against_baseline(image: Image.Image, label: str) -> None:
|
||||
assert BASELINE_PATH.exists(), f"Baseline image not found at {BASELINE_PATH}"
|
||||
baseline_image = Image.open(BASELINE_PATH).convert("RGB")
|
||||
|
||||
assert_images_pixel_close(
|
||||
model_name=f"{MODEL_NAME} ({label} vs baseline)",
|
||||
vllm_image=image,
|
||||
diffusers_image=baseline_image,
|
||||
mean_threshold=MEAN_THRESHOLD,
|
||||
p99_threshold=P99_THRESHOLD,
|
||||
)
|
||||
assert_similarity(
|
||||
model_name=f"{MODEL_NAME} ({label} vs baseline)",
|
||||
vllm_image=image,
|
||||
diffusers_image=baseline_image,
|
||||
ssim_threshold=SSIM_THRESHOLD,
|
||||
psnr_threshold=PSNR_THRESHOLD,
|
||||
)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
||||
def test_hunyuan_image3_pixel_accuracy_online(accuracy_artifact_root: Path) -> None:
|
||||
model = _model_name()
|
||||
output_dir = model_output_dir(accuracy_artifact_root, MODEL_NAME)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
deploy_config_path = Path(tmpdir) / "deploy.yaml"
|
||||
_write_deploy_config(deploy_config_path)
|
||||
image = _run_vllm_omni_hunyuan_image3_online(
|
||||
model=model, deploy_config=str(deploy_config_path), output_path=output_dir / "vllm_omni_online.png"
|
||||
)
|
||||
_assert_against_baseline(image, "online")
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
||||
def test_hunyuan_image3_pixel_accuracy_offline(accuracy_artifact_root: Path) -> None:
|
||||
model = _model_name()
|
||||
output_dir = model_output_dir(accuracy_artifact_root, MODEL_NAME)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
deploy_config_path = Path(tmpdir) / "deploy.yaml"
|
||||
_write_deploy_config(deploy_config_path)
|
||||
image = _run_vllm_omni_hunyuan_image3_offline(
|
||||
model=model, deploy_config=str(deploy_config_path), output_path=output_dir / "vllm_omni_offline.png"
|
||||
)
|
||||
_assert_against_baseline(image, "offline")
|
||||
@@ -106,6 +106,7 @@ def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> Image.Image:
|
||||
trust_remote_code=True,
|
||||
local_files_only=_local_files_only(model),
|
||||
).to("cuda")
|
||||
pipe.transformer.set_attention_backend("_flash_3_hub")
|
||||
generator = torch.Generator(device="cuda").manual_seed(SEED)
|
||||
result = pipe( # pyright: ignore[reportCallIssue]
|
||||
prompt=PROMPT,
|
||||
@@ -167,6 +168,7 @@ def _run_diffusers_qwen_image_2512(*, model: str, output_path: Path) -> Image.Im
|
||||
trust_remote_code=True,
|
||||
local_files_only=_local_files_only(model),
|
||||
).to("cuda")
|
||||
pipe.transformer.set_attention_backend("_flash_3_hub")
|
||||
generator = torch.Generator(device="cuda").manual_seed(SEED_2512)
|
||||
result = pipe( # pyright: ignore[reportCallIssue]
|
||||
prompt=PROMPT_2512,
|
||||
|
||||
@@ -32,7 +32,8 @@ from vllm_omni import Omni
|
||||
|
||||
# Selected by the nightly "TTS · Function Test" (-m "full_model and L4 and tts").
|
||||
# Without the tts + run-level marks these tests are collected but never run.
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
||||
# TODO: Fix this test
|
||||
# pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
||||
|
||||
SAMPLE_RATE = 24_000 # All MOSS-TTS full variants output 24 kHz
|
||||
|
||||
@@ -128,7 +129,6 @@ def delay_engine():
|
||||
return Omni(model=_DELAY_MODEL, stage_init_timeout=300)
|
||||
|
||||
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_moss_tts_delay_english(delay_engine, ref_audio_path):
|
||||
"""MossTTSDelayModel: English voice_clone produces non-empty 24 kHz audio."""
|
||||
@@ -140,7 +140,6 @@ def test_moss_tts_delay_english(delay_engine, ref_audio_path):
|
||||
assert not torch.all(audio == 0), "Audio is silence"
|
||||
|
||||
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_moss_tts_delay_chinese(delay_engine, ref_audio_path):
|
||||
"""MossTTSDelayModel: Chinese input produces non-empty audio."""
|
||||
@@ -152,7 +151,6 @@ def test_moss_tts_delay_chinese(delay_engine, ref_audio_path):
|
||||
assert not torch.all(audio == 0)
|
||||
|
||||
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_moss_tts_delay_deterministic(delay_engine, ref_audio_path):
|
||||
"""MossTTSDelayModel: same seed yields identical waveforms."""
|
||||
@@ -164,7 +162,6 @@ def test_moss_tts_delay_deterministic(delay_engine, ref_audio_path):
|
||||
assert torch.allclose(audio1, audio2, atol=1e-4), "Waveforms differ across identical seeds"
|
||||
|
||||
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_moss_tts_delay_batch(delay_engine, ref_audio_path):
|
||||
"""MossTTSDelayModel: batch of two requests each returns non-empty audio."""
|
||||
@@ -206,7 +203,6 @@ def realtime_engine():
|
||||
return Omni(model=_REALTIME_MODEL, stage_init_timeout=300)
|
||||
|
||||
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_moss_tts_realtime_english(realtime_engine, ref_audio_path):
|
||||
"""MossTTSRealtime: English voice_clone produces non-empty 24 kHz audio."""
|
||||
@@ -218,7 +214,6 @@ def test_moss_tts_realtime_english(realtime_engine, ref_audio_path):
|
||||
assert not torch.all(audio == 0), "Audio is silence"
|
||||
|
||||
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_moss_tts_realtime_deterministic(realtime_engine, ref_audio_path):
|
||||
"""MossTTSRealtime: same seed yields identical waveforms."""
|
||||
|
||||
@@ -120,7 +120,6 @@ def _collect_audio(omni: Omni, request: dict) -> tuple[torch.Tensor, int]:
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_moss_tts_nano_english(omni_runner: OmniRunner, ref_audio_path) -> None:
|
||||
"""English TTS produces non-empty 48 kHz stereo audio."""
|
||||
@@ -133,7 +132,6 @@ def test_moss_tts_nano_english(omni_runner: OmniRunner, ref_audio_path) -> None:
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_moss_tts_nano_chinese(omni_runner: OmniRunner, ref_audio_path) -> None:
|
||||
"""Chinese TTS produces non-empty audio."""
|
||||
@@ -146,7 +144,6 @@ def test_moss_tts_nano_chinese(omni_runner: OmniRunner, ref_audio_path) -> None:
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_moss_tts_nano_deterministic(omni_runner: OmniRunner, ref_audio_path) -> None:
|
||||
"""Same seed produces identical waveforms."""
|
||||
@@ -159,7 +156,6 @@ def test_moss_tts_nano_deterministic(omni_runner: OmniRunner, ref_audio_path) ->
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_moss_tts_nano_batch(omni_runner: OmniRunner, ref_audio_path) -> None:
|
||||
"""Batch of two requests returns audio for each."""
|
||||
|
||||
@@ -24,10 +24,11 @@ from vllm import SamplingParams
|
||||
from tests.helpers.mark import hardware_test
|
||||
from vllm_omni import Omni
|
||||
|
||||
# H100-gated (8B). Picked up by the nightly H100 omni job
|
||||
# (-m "full_model and H100 and omni"); the tts mark keeps it consistent with
|
||||
# the other MOSS-TTS tests and any tts-scoped sweeps.
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
||||
# H100-gated (8B). Picked up by the nightly "TTS · Function Test with H100"
|
||||
# job (-m "full_model and H100 and tts"); the L4 MOSS-TTS tests land in the
|
||||
# sibling L4 tts lane instead.
|
||||
# TODO: Fix this test
|
||||
# pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
||||
|
||||
SAMPLE_RATE = 24_000
|
||||
REF_AUDIO_URL = "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS/main/assets/audio/reference_zh_1.wav"
|
||||
@@ -101,7 +102,6 @@ def _collect_audio(omni: Omni, request: dict) -> tuple[torch.Tensor, int]:
|
||||
raise AssertionError("No stage outputs received")
|
||||
|
||||
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"})
|
||||
def test_moss_tts_v15_voice_clone(v15_engine, ref_audio_path):
|
||||
"""MOSS-TTS-v1.5 (8B): voice_clone produces non-empty 24 kHz audio."""
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""E2E online serving test for DreamZero OpenPI websocket serving."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from tests.dreamzero import openpi_client_helper as openpi_client
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniServerParams, get_open_port
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
MODEL = "GEAR-Dreams/DreamZero-DROID"
|
||||
|
||||
|
||||
test_params = [
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
port=8091,
|
||||
server_args=[
|
||||
"--deploy-config",
|
||||
"vllm_omni/deploy/dreamzero_tp1_cfg2.yaml",
|
||||
"--enforce-eager",
|
||||
"--disable-log-stats",
|
||||
],
|
||||
env_dict={
|
||||
"ATTENTION_BACKEND": "torch",
|
||||
"DIFFUSION_ATTENTION_BACKEND": "TORCH_SDPA",
|
||||
"VLLM_DISABLE_COMPILE_CACHE": "1",
|
||||
"MASTER_PORT": str(get_open_port()),
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def _write_synthetic_video(path: Path, cv2_module, *, channel: int) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
height, width, num_frames = 180, 320, 24
|
||||
writer = cv2_module.VideoWriter(str(path), cv2_module.VideoWriter_fourcc(*"mp4v"), 15.0, (width, height))
|
||||
if not writer.isOpened():
|
||||
raise RuntimeError(f"Failed to open video writer for {path}")
|
||||
try:
|
||||
for frame_idx in range(num_frames):
|
||||
frame = np.zeros((height, width, 3), dtype=np.uint8)
|
||||
frame[..., channel] = (frame_idx * 7) % 255
|
||||
frame[..., (channel + 1) % 3] = 64
|
||||
writer.write(cv2_module.cvtColor(frame, cv2_module.COLOR_RGB2BGR))
|
||||
finally:
|
||||
writer.release()
|
||||
|
||||
|
||||
def _write_synthetic_dreamzero_videos(client_mod, video_dir: Path) -> None:
|
||||
for channel, file_name in enumerate(client_mod.CAMERA_FILES.values()):
|
||||
_write_synthetic_video(video_dir / file_name, client_mod.cv2, channel=channel)
|
||||
|
||||
|
||||
@pytest.mark.full_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.distributed_cuda
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_dreamzero_openpi_online(omni_server, tmp_path: Path) -> None:
|
||||
try:
|
||||
openpi_client.require_dependencies()
|
||||
except ModuleNotFoundError as exc:
|
||||
pytest.skip(str(exc))
|
||||
|
||||
video_dir = tmp_path / "dreamzero_videos"
|
||||
_write_synthetic_dreamzero_videos(openpi_client, video_dir)
|
||||
result = openpi_client.run_policy_session(
|
||||
host=omni_server.host,
|
||||
port=omni_server.port,
|
||||
video_dir=video_dir,
|
||||
session_id="dreamzero-online-e2e",
|
||||
)
|
||||
|
||||
openpi_client.validate_session_result(result)
|
||||
|
||||
metadata = result["metadata"]
|
||||
assert metadata["needs_session_id"] is True
|
||||
assert metadata["needs_stereo_camera"] is False
|
||||
assert tuple(metadata["image_resolution"]) == (180, 320)
|
||||
@@ -0,0 +1,183 @@
|
||||
"""
|
||||
Comprehensive tests of diffusion features that are available in online serving mode
|
||||
and are supported by the following model:
|
||||
- zai-org/GLM-Image (t2i and i2i)
|
||||
|
||||
Coverage (per mode):
|
||||
- Baseline
|
||||
- Cache-DiT
|
||||
- Tensor-Parallel
|
||||
- HSDP
|
||||
- Sequence parallel
|
||||
|
||||
Topology for multi-GPU cases uses runtime deploy YAML (except TP)
|
||||
so stage-1 parallel fields reach the diffusion worker without relying on CLI
|
||||
|
||||
assert_diffusion_response validates successful generation and the expected resolution.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.media import generate_synthetic_image
|
||||
from tests.helpers.runtime import (
|
||||
OmniServer,
|
||||
OmniServerParams,
|
||||
OpenAIClientHandler,
|
||||
dummy_messages_from_mix_data,
|
||||
)
|
||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
|
||||
|
||||
MODEL = os.environ.get("GLM_IMAGE_MODEL_PATH", "zai-org/GLM-Image")
|
||||
|
||||
T2I_PROMPT = "A Vincent van Gogh style impressionist painting."
|
||||
I2I_PROMPT = "Transform this modern, geometric image into a Vincent van Gogh style impressionist painting."
|
||||
NEGATIVE_PROMPT = "low quality, blurry, distorted, unnatural colors"
|
||||
|
||||
TWO_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}, num_cards=2)
|
||||
|
||||
GLM_DEPLOY = get_deploy_config_path("glm_image.yaml")
|
||||
GLM_CACHEDIT_DEPLOY = modify_stage_config(
|
||||
GLM_DEPLOY,
|
||||
updates={
|
||||
"stages": {
|
||||
1: {"cache_backend": "cache_dit"},
|
||||
},
|
||||
},
|
||||
)
|
||||
GLM_TP_2_DEPLOY = modify_stage_config(
|
||||
GLM_DEPLOY,
|
||||
updates={
|
||||
"stages": {
|
||||
0: {"devices": "0", "tensor_parallel_size": 1},
|
||||
1: {"devices": "0,1", "tensor_parallel_size": 2},
|
||||
},
|
||||
},
|
||||
)
|
||||
GLM_HSDP_2_DEPLOY = modify_stage_config(
|
||||
GLM_DEPLOY,
|
||||
updates={
|
||||
"stages": {
|
||||
0: {"devices": "0"},
|
||||
1: {
|
||||
"devices": "0,1",
|
||||
"parallel_config": {
|
||||
"use_hsdp": True,
|
||||
"hsdp_shard_size": 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
GLM_SP_2_DEPLOY = modify_stage_config(
|
||||
GLM_DEPLOY,
|
||||
updates={
|
||||
"stages": {
|
||||
0: {"devices": "0"},
|
||||
1: {
|
||||
"devices": "0,1",
|
||||
"parallel_config": {
|
||||
"ulysses_degree": 2,
|
||||
"ring_degree": 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _get_diffusion_feature_cases(model: str):
|
||||
return [
|
||||
# Baseline: default deploy (stage 0 -> GPU 0, stage 1 -> GPU 1)
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
stage_config_path=GLM_DEPLOY,
|
||||
),
|
||||
id="baseline",
|
||||
marks=TWO_CARD_FEATURE_MARKS,
|
||||
),
|
||||
# Cache-DiT via runtime deploy patch (cache_backend -> engine_extras on stage 1)
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
stage_config_path=GLM_CACHEDIT_DEPLOY,
|
||||
),
|
||||
id="cachedit",
|
||||
marks=TWO_CARD_FEATURE_MARKS,
|
||||
),
|
||||
# Tensor-Parallel (2 GPUs on diffusion stage)
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
stage_config_path=GLM_TP_2_DEPLOY,
|
||||
),
|
||||
id="tensor_parallel_2",
|
||||
marks=TWO_CARD_FEATURE_MARKS,
|
||||
),
|
||||
# HSDP (2 GPUs on diffusion stage)
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
stage_config_path=GLM_HSDP_2_DEPLOY,
|
||||
),
|
||||
id="hsdp_2",
|
||||
marks=TWO_CARD_FEATURE_MARKS,
|
||||
),
|
||||
# Ulysses sequence parallel degree=2 (2 GPUs)
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
stage_config_path=GLM_SP_2_DEPLOY,
|
||||
),
|
||||
id="sequence_parallel_2",
|
||||
marks=TWO_CARD_FEATURE_MARKS,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
MODES = ["t2i", "i2i"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", MODES)
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
_get_diffusion_feature_cases(MODEL),
|
||||
indirect=True,
|
||||
)
|
||||
def test_glm_image(
|
||||
omni_server: OmniServer,
|
||||
mode: str,
|
||||
openai_client: OpenAIClientHandler,
|
||||
):
|
||||
"""GLM-Image diffusion feature coverage for t2i and i2i on two H100s."""
|
||||
|
||||
if mode == "t2i":
|
||||
messages = dummy_messages_from_mix_data(content_text=T2I_PROMPT)
|
||||
else:
|
||||
image_size = 1024
|
||||
image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(image_size, image_size)['base64']}"
|
||||
messages = dummy_messages_from_mix_data(
|
||||
image_data_url=image_data_url,
|
||||
content_text=I2I_PROMPT,
|
||||
)
|
||||
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"extra_body": {
|
||||
"height": 1024,
|
||||
"width": 1024,
|
||||
"num_inference_steps": 2,
|
||||
"guidance_scale": 1.5,
|
||||
"true_cfg_scale": 4.0,
|
||||
"negative_prompt": NEGATIVE_PROMPT,
|
||||
"seed": 42,
|
||||
},
|
||||
}
|
||||
|
||||
openai_client.send_diffusion_request(request_config)
|
||||
@@ -30,7 +30,8 @@ from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
||||
# TODO: Fix this test
|
||||
# pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
||||
|
||||
MODEL = "OpenMOSS-Team/MOSS-TTS-Realtime"
|
||||
REF_AUDIO_URL = "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS/HEAD/assets/audio/reference_zh_1.wav"
|
||||
|
||||
@@ -9,63 +9,40 @@ import pytest
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.media import generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video
|
||||
from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data
|
||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
|
||||
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
|
||||
|
||||
# Set VLLM_TEST_PD_MODE=1 to test PD disaggregation (follow-up — deploy overlay not yet migrated).
|
||||
_USE_PD = os.environ.get("VLLM_TEST_PD_MODE", "0") == "1"
|
||||
|
||||
_MODEL = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
|
||||
_CI_DEPLOY = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
|
||||
|
||||
|
||||
def get_chunk_config(config_path: str | None = None):
|
||||
"""Load the qwen3_omni CI deploy yaml with async_chunk modifications for streaming mode."""
|
||||
if config_path is None:
|
||||
config_path = _CI_DEPLOY
|
||||
# TODO: remove this workaround once legacy `stage_args` path is deleted.
|
||||
# The pipeline (qwen3_omni/pipeline.py) already wires
|
||||
# thinker2talker_async_chunk / talker2code2wav_async_chunk on stage 0/1,
|
||||
# so only async_chunk needs flipping. Writing nested `engine_args:` into
|
||||
# the new-schema overlay trips _parse_stage_deploy's legacy branch and
|
||||
# drops flat fields (load_format, max_num_seqs, ...).
|
||||
return modify_stage_config(config_path, updates={"async_chunk": True})
|
||||
|
||||
|
||||
# Platform-specific overrides live inside the new deploy yaml's ``platforms:``
|
||||
# section, so a single ``_CI_DEPLOY`` path serves CUDA, ROCm, and XPU.
|
||||
# TODO: re-add VLLM_TEST_PD_MODE branch once the PD-disaggregation deploy
|
||||
# overlay has been migrated to the new schema (previously used the deleted
|
||||
# ``qwen3_omni_moe_pd_ci.yaml`` stage-configs file).
|
||||
if current_omni_platform.is_xpu():
|
||||
stage_configs = [_CI_DEPLOY]
|
||||
else: # CUDA + ROCm MI325 share the same deploy config
|
||||
stage_configs = [get_chunk_config()]
|
||||
|
||||
# Create parameter combinations for model and stage config
|
||||
test_params = [
|
||||
OmniServerParams(model=model, stage_config_path=stage_config) for model in models for stage_config in stage_configs
|
||||
]
|
||||
# For prefix caching, we enable it on the thinker (stage 0) via CLI override
|
||||
# For prefix caching checks against we enable it on the thinker and talker via CLI override
|
||||
# and enable prompt token details so that we can determine if any tokens were cached.
|
||||
# We also explicitly set block size so that we can make sure the cached token counts are a
|
||||
# multiple of the block size.
|
||||
BLOCK_SIZE = 16
|
||||
prefix_test_params = [
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
stage_config_path=_CI_DEPLOY,
|
||||
server_args=[
|
||||
"--block-size",
|
||||
str(BLOCK_SIZE),
|
||||
"--stage-overrides",
|
||||
'{"0": {"enable_prefix_caching": true}}',
|
||||
"--enable-prompt-tokens-details",
|
||||
],
|
||||
test_params = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=_MODEL,
|
||||
stage_config_path=_CI_DEPLOY,
|
||||
use_stage_cli=True,
|
||||
server_args=[
|
||||
"--no-async-chunk",
|
||||
"--block-size",
|
||||
str(BLOCK_SIZE),
|
||||
"--stage-overrides",
|
||||
'{"0": {"enable_prefix_caching": true}, "1": {"enable_prefix_caching": true}}',
|
||||
"--enable-prompt-tokens-details",
|
||||
],
|
||||
),
|
||||
id="default",
|
||||
)
|
||||
for model in models
|
||||
]
|
||||
|
||||
|
||||
@@ -166,45 +143,14 @@ def test_text_to_text_001(omni_server, openai_client) -> None:
|
||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", prefix_test_params, indirect=True)
|
||||
def test_thinker_prefix_caching(omni_server, openai_client, run_level) -> None:
|
||||
def _run_prefix_cache_check(openai_client, request_config: dict):
|
||||
"""Make two requests given a request config, and validate that:
|
||||
1. The second request actually had cached tokens
|
||||
2. The number of cached tokens is divisible by the block size used in
|
||||
test_params, because currently upstream vLLM does not cache partial
|
||||
blocks.
|
||||
"""
|
||||
Test thinker prefix caching by sending identical requests with an image (i.e.,
|
||||
a large shared prefix) and verifying that the second request uses cached tokens
|
||||
& produces the same output with greedy decoding.
|
||||
|
||||
NOTE: The reason that we check against logprobs instead of direct text here is that
|
||||
the outputs may still diverge a bit even though we set the seed and temperature.
|
||||
This is mostly because the GEMM algorithm may vary based on the input tensors dims.
|
||||
Because of this, we don't check the logprobs if it's a dummy load, since in that case
|
||||
the top logprobs will all be very close.
|
||||
"""
|
||||
seed = 10
|
||||
img_res = generate_synthetic_image(224, 224, seed=seed)
|
||||
image_data_url = f"data:image/jpeg;base64,{img_res['base64']}"
|
||||
messages = dummy_messages_from_mix_data(
|
||||
system_prompt=get_system_prompt(),
|
||||
image_data_url=image_data_url,
|
||||
content_text=get_prompt("text_image"),
|
||||
)
|
||||
|
||||
top_k = 10
|
||||
sampling_params = {"seed": seed, "temperature": 0, "max_tokens": 8, "logprobs": top_k}
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
"modalities": ["text"],
|
||||
"logprobs": True,
|
||||
"top_logprobs": top_k,
|
||||
"sampling_params_list": [sampling_params] * 3,
|
||||
}
|
||||
|
||||
uncached_response = openai_client.send_omni_request(request_config, request_num=1)[0]
|
||||
openai_client.send_omni_request(request_config, request_num=1)[0]
|
||||
cached_response = openai_client.send_omni_request(request_config, request_num=1)[0]
|
||||
|
||||
# Ensure that we have a prefix cache hit on the second request and that only the last
|
||||
@@ -217,16 +163,66 @@ def test_thinker_prefix_caching(omni_server, openai_client, run_level) -> None:
|
||||
assert num_cached_tokens % BLOCK_SIZE == 0
|
||||
assert (num_cached_tokens + num_uncached_tokens) == num_prompt_tokens
|
||||
|
||||
# Ensure that we have logprobs and tokens were generated for both requests
|
||||
assert uncached_response.logprobs is not None
|
||||
assert cached_response.logprobs is not None
|
||||
n_tokens = min(len(uncached_response.logprobs), len(cached_response.logprobs))
|
||||
assert n_tokens > 0
|
||||
|
||||
if run_level == "advanced_model":
|
||||
# For each token index where both responses have an output, ensure that the greedy token
|
||||
# predicted in the uncached case is in the top k logprobs for the cached case
|
||||
for idx in range(n_tokens):
|
||||
greedy_token = uncached_response.logprobs[idx].token
|
||||
cached_top_k = {lp.token for lp in cached_response.logprobs[idx].top_logprobs}
|
||||
assert greedy_token in cached_top_k
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_thinker_prefix_caching_text_output(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test thinker prefix caching by sending identical requests with an image (i.e.,
|
||||
a large shared prefix) and verifying that the second request uses cached tokens
|
||||
& produces the same output with greedy decoding.
|
||||
|
||||
NOTE: Checking the output of prefix caching directly can be a bit unstable
|
||||
due to slight numerical differences as a result of running different scheduled
|
||||
sequence lengths. As such, for E2E tests on prefix cache, we only check the cached
|
||||
token count and not the output, since the omni tensor cache has solid unit tests,
|
||||
and the core prefix cache algorithm is largely tested by upstream vLLM.
|
||||
"""
|
||||
img_res = generate_synthetic_image(224, 224)
|
||||
image_data_url = f"data:image/jpeg;base64,{img_res['base64']}"
|
||||
messages = dummy_messages_from_mix_data(
|
||||
system_prompt=get_system_prompt(),
|
||||
image_data_url=image_data_url,
|
||||
content_text=get_prompt("text_image"),
|
||||
)
|
||||
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
"modalities": ["text"],
|
||||
}
|
||||
_run_prefix_cache_check(openai_client, request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_thinker_prefix_caching_audio_output(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Verify that thinker prefix caching does not hang when the request
|
||||
produces audio output (text + audio modalities). Sends two identical
|
||||
requests so the second exercises the prefix-cached path through the
|
||||
full thinker -> talker -> code2wav pipeline.
|
||||
|
||||
Regression test for https://github.com/vllm-project/vllm-omni/issues/3510
|
||||
"""
|
||||
messages = dummy_messages_from_mix_data(
|
||||
system_prompt=get_system_prompt(),
|
||||
content_text=get_prompt(),
|
||||
)
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"stream": True,
|
||||
"stream_options": {
|
||||
"include_usage": True,
|
||||
},
|
||||
}
|
||||
|
||||
_run_prefix_cache_check(openai_client, request_config)
|
||||
@@ -0,0 +1,97 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
E2E Online tests for VoxCPM2 with text input and audio output.
|
||||
|
||||
These tests verify the /v1/audio/speech endpoint works correctly with
|
||||
actual model inference, not mocks. VoxCPM2 zero-shot needs only ``input``
|
||||
and a placeholder ``voice`` (``default``); no ``task_type`` / ``ref_text``.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
MODEL = "openbmb/VoxCPM2"
|
||||
DEFAULT_AUDIO_SPEECH_TIMEOUT_S = 300.0
|
||||
MAX_CONCURRENT = 4
|
||||
|
||||
# ~0.5 s of 48 kHz mono PCM_16 in WAV (~48k payload + header).
|
||||
_MIN_AUDIO_BYTES = 40_000
|
||||
|
||||
|
||||
def get_prompt(prompt_type="text"):
|
||||
"""English prompt for zero-shot TTS (Whisper-friendly for advanced_model accuracy)."""
|
||||
prompts = {
|
||||
"text": "The weather is nice today, perfect for a walk in the park.",
|
||||
}
|
||||
return prompts.get(prompt_type, prompts["text"])
|
||||
|
||||
|
||||
tts_server_params = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
stage_config_path=get_deploy_config_path("voxcpm2.yaml"),
|
||||
server_args=["--trust-remote-code", "--disable-log-stats"],
|
||||
),
|
||||
id="voxcpm2",
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
||||
def test_text_to_audio_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test zero-shot text-to-audio via OpenAI API.
|
||||
Deploy Setting: voxcpm2.yaml
|
||||
Input Modal: text
|
||||
Output Modal: audio
|
||||
Input Setting: stream=False
|
||||
Datasets: few requests (max_num_seqs=4)
|
||||
"""
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"input": get_prompt(),
|
||||
"stream": False,
|
||||
"timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S,
|
||||
"response_format": "wav",
|
||||
"voice": "default",
|
||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
||||
}
|
||||
openai_client.send_audio_speech_request(request_config, request_num=MAX_CONCURRENT)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
||||
def test_text_to_audio_002(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test zero-shot streaming text-to-audio via OpenAI API.
|
||||
Deploy Setting: voxcpm2.yaml
|
||||
Input Modal: text
|
||||
Output Modal: audio
|
||||
Input Setting: stream=True
|
||||
Datasets: single request
|
||||
"""
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"input": get_prompt(),
|
||||
"stream": True,
|
||||
"timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S,
|
||||
"response_format": "wav",
|
||||
"voice": "default",
|
||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
||||
}
|
||||
openai_client.send_audio_speech_request(request_config)
|
||||
@@ -0,0 +1,95 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
E2E Online expansion tests for VoxCPM2 (voice clone, PCM, concurrency).
|
||||
|
||||
Voice clone uses ``ref_audio`` only (no ``ref_text``). Reference clip is
|
||||
vendored under tests/assets/qwen3_tts/clone_2.wav — see test_qwen3_tts_base.py.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.media import load_test_audio_data_url
|
||||
from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
MODEL = "openbmb/VoxCPM2"
|
||||
DEFAULT_AUDIO_SPEECH_TIMEOUT_S = 300.0
|
||||
_MIN_AUDIO_BYTES = 40_000
|
||||
MAX_CONCURRENT = 4
|
||||
|
||||
REF_AUDIO_URL = load_test_audio_data_url("qwen3_tts/clone_2.wav")
|
||||
|
||||
|
||||
def get_prompt(prompt_type="text"):
|
||||
prompts = {
|
||||
"text": "The weather is nice today, perfect for a walk in the park.",
|
||||
}
|
||||
return prompts.get(prompt_type, prompts["text"])
|
||||
|
||||
|
||||
tts_server_params = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
stage_config_path=get_deploy_config_path("voxcpm2.yaml"),
|
||||
server_args=["--trust-remote-code", "--disable-log-stats"],
|
||||
),
|
||||
id="voxcpm2",
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
||||
def test_voice_clone_streaming_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test voice-clone text-to-audio via OpenAI API.
|
||||
Deploy Setting: voxcpm2.yaml
|
||||
Input Modal: text + ref_audio
|
||||
Output Modal: audio
|
||||
Input Setting: stream=True
|
||||
Datasets: few requests (max_num_seqs=4)
|
||||
"""
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"input": get_prompt(),
|
||||
"stream": True,
|
||||
"timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S,
|
||||
"response_format": "wav",
|
||||
"voice": "default",
|
||||
"ref_audio": REF_AUDIO_URL,
|
||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
||||
}
|
||||
openai_client.send_audio_speech_request(request_config, request_num=MAX_CONCURRENT)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
||||
def test_response_format_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test voice-clone non-stream PCM output via OpenAI API.
|
||||
Deploy Setting: voxcpm2.yaml
|
||||
Input Modal: text + ref_audio
|
||||
Output Modal: audio (pcm)
|
||||
Input Setting: stream=False
|
||||
Datasets: single request
|
||||
"""
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"input": get_prompt(),
|
||||
"response_format": "pcm",
|
||||
"stream": False,
|
||||
"timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S,
|
||||
"voice": "default",
|
||||
"ref_audio": REF_AUDIO_URL,
|
||||
"min_audio_bytes": _MIN_AUDIO_BYTES,
|
||||
}
|
||||
openai_client.send_audio_speech_request(request_config)
|
||||
@@ -427,6 +427,7 @@ async def _enqueue_add_request(
|
||||
sampling_params_list=sampling_params_list,
|
||||
final_stage_id=final_stage_id,
|
||||
preprocess_ms=0.0,
|
||||
request_timestamp=time.time(),
|
||||
enqueue_ts=time.perf_counter(),
|
||||
)
|
||||
)
|
||||
@@ -984,6 +985,7 @@ async def test_handle_streaming_update_passes_prompt_text_to_stage_pool() -> Non
|
||||
sampling_params_list=[_sampling_params()],
|
||||
final_stage_id=0,
|
||||
preprocess_ms=0.0,
|
||||
request_timestamp=time.time(),
|
||||
enqueue_ts=time.perf_counter(),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -156,7 +156,7 @@ class FakeAsyncOmni:
|
||||
self.captured_prompt = None
|
||||
self._images = images or [Image.new("RGB", (64, 64), color="green")]
|
||||
|
||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None):
|
||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
||||
if sampling_params_list is not None:
|
||||
self.captured_sampling_params_list = sampling_params_list
|
||||
else:
|
||||
@@ -254,7 +254,7 @@ def async_omni_test_client():
|
||||
self._images = [Image.new("RGB", (64, 64), color="green")]
|
||||
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
||||
|
||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None):
|
||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
||||
if sampling_params_list is not None:
|
||||
self.captured_sampling_params_list = sampling_params_list
|
||||
else:
|
||||
@@ -318,7 +318,7 @@ def async_omni_rgba_test_client():
|
||||
self._images = [Image.new("RGBA", (64, 64), color=(0, 255, 0, 128))]
|
||||
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
||||
|
||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None):
|
||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
||||
if sampling_params_list is not None:
|
||||
self.captured_sampling_params_list = sampling_params_list
|
||||
else:
|
||||
@@ -382,7 +382,7 @@ def async_omni_stage_configs_only_client():
|
||||
self._images = [Image.new("RGB", (64, 64), color="green")]
|
||||
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
||||
|
||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None):
|
||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
||||
if sampling_params_list is not None:
|
||||
self.captured_sampling_params_list = sampling_params_list
|
||||
else:
|
||||
@@ -442,7 +442,7 @@ def streaming_image_edit_client():
|
||||
self.captured_prompt = None
|
||||
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
||||
|
||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None):
|
||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
||||
self.captured_prompt = prompt
|
||||
self.captured_sampling_params_list = sampling_params_list or [sampling_params]
|
||||
assert self.captured_sampling_params_list[0].output_kind == RequestOutputKind.DELTA
|
||||
@@ -696,7 +696,7 @@ def test_generate_images_async_omni_glm_image_sets_stage0_max_tokens():
|
||||
self._images = [Image.new("RGB", (64, 64), color="green")]
|
||||
self.od_config = SimpleNamespace(supports_multimodal_inputs=True)
|
||||
|
||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None):
|
||||
async def generate(self, prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
||||
self.captured_sampling_params_list = (
|
||||
sampling_params_list if sampling_params_list is not None else [sampling_params]
|
||||
)
|
||||
@@ -815,7 +815,9 @@ def test_image_edits_streaming_returns_ar_delta_then_image(streaming_image_edit_
|
||||
|
||||
|
||||
def test_image_edits_streaming_ar_delta_chunks_include_index(streaming_image_edit_client):
|
||||
async def generate_multi_output_delta(prompt, request_id, sampling_params=None, sampling_params_list=None):
|
||||
async def generate_multi_output_delta(
|
||||
prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs
|
||||
):
|
||||
yield MockStageResult(stage_id=0, final_output_type="text", texts=["first", "second"])
|
||||
yield MockStageResult(
|
||||
stage_id=1,
|
||||
@@ -843,7 +845,7 @@ def test_image_edits_streaming_ar_delta_chunks_include_index(streaming_image_edi
|
||||
|
||||
|
||||
def test_image_edits_streaming_errors_without_final_image(streaming_image_edit_client):
|
||||
async def generate_without_image(prompt, request_id, sampling_params=None, sampling_params_list=None):
|
||||
async def generate_without_image(prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
||||
yield MockStageResult(stage_id=0, final_output_type="text", text="recap")
|
||||
|
||||
streaming_image_edit_client.app.state.engine_client.generate = generate_without_image
|
||||
@@ -866,7 +868,7 @@ def test_image_edits_streaming_errors_without_final_image(streaming_image_edit_c
|
||||
|
||||
|
||||
def test_image_edits_streaming_errors_on_empty_final_image(streaming_image_edit_client):
|
||||
async def generate_empty_image(prompt, request_id, sampling_params=None, sampling_params_list=None):
|
||||
async def generate_empty_image(prompt, request_id, sampling_params=None, sampling_params_list=None, **kwargs):
|
||||
yield MockStageResult(stage_id=0, final_output_type="text", text="recap")
|
||||
yield MockStageResult(stage_id=1, final_output_type="image", images=[])
|
||||
|
||||
|
||||
@@ -2,6 +2,11 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
E2E online tests for Qwen3-Omni /v1/realtime WebSocket (streaming PCM in, audio out).
|
||||
|
||||
Three scenarios:
|
||||
- Ready CI: async_chunk on, smoke only (no send delay, no accuracy check).
|
||||
- Merge CI: async_chunk on + send delay, full accuracy check.
|
||||
- Merge CI: async_chunk off, no send delay, full accuracy check.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -34,18 +39,32 @@ REALTIME_SYNTH_PHRASE_TEXT = (
|
||||
"Translate into Chinese: Beijing is the Capital of China. It is the center of culture and politics"
|
||||
)
|
||||
|
||||
# The new-schema CI overlay bakes in async_chunk: False and covers CUDA/ROCm/XPU
|
||||
# via its ``platforms:`` section, so one path serves all three.
|
||||
# Simulate realtime upload pacing (``openai_realtime_client.py --send-delay-ms``).
|
||||
SEND_DELAY_MS = 200
|
||||
|
||||
# CI overlay bakes in async_chunk: False and covers CUDA/ROCm/XPU via ``platforms:``.
|
||||
default_stage_config = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
|
||||
|
||||
realtime_server_params = [
|
||||
realtime_sync_server_params = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
stage_config_path=default_stage_config,
|
||||
use_stage_cli=True,
|
||||
),
|
||||
id="default",
|
||||
id="sync",
|
||||
),
|
||||
]
|
||||
|
||||
realtime_async_chunk_server_params = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
stage_config_path=default_stage_config,
|
||||
use_stage_cli=True,
|
||||
server_args=["--async-chunk"],
|
||||
),
|
||||
id="async_chunk",
|
||||
),
|
||||
]
|
||||
|
||||
@@ -80,6 +99,7 @@ async def _run_realtime_audio_roundtrip(
|
||||
pcm16: bytes,
|
||||
*,
|
||||
chunk_ms: int = 100,
|
||||
send_delay_ms: int = 0,
|
||||
) -> dict:
|
||||
uri = f"ws://{host}:{port}/v1/realtime"
|
||||
incremental: list[bytes] = []
|
||||
@@ -105,6 +125,8 @@ async def _run_realtime_audio_roundtrip(
|
||||
}
|
||||
)
|
||||
)
|
||||
if send_delay_ms > 0:
|
||||
await asyncio.sleep(send_delay_ms / 1000.0)
|
||||
|
||||
await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True}))
|
||||
|
||||
@@ -156,26 +178,49 @@ async def _run_realtime_audio_roundtrip(
|
||||
}
|
||||
|
||||
|
||||
def _synthetic_pcm16_input() -> bytes:
|
||||
syn = generate_synthetic_audio(
|
||||
10,
|
||||
1,
|
||||
sample_rate=16000,
|
||||
phrase_text=REALTIME_SYNTH_PHRASE_TEXT,
|
||||
)
|
||||
wav_bytes = base64.b64decode(syn["base64"])
|
||||
return _pcm16_mono_16k_from_wav_bytes(wav_bytes)
|
||||
|
||||
|
||||
def _assert_realtime_smoke(result: dict) -> None:
|
||||
out_pcm = result["output_pcm"]
|
||||
assert result["delta_events"] >= 1
|
||||
assert out_pcm, "No output PCM from response.audio.delta"
|
||||
assert len(out_pcm) % 2 == 0
|
||||
assert len(out_pcm) >= 4096, "Output audio unexpectedly small"
|
||||
assert result["output_sample_rate"] > 0
|
||||
|
||||
|
||||
def _assert_realtime_accuracy(result: dict) -> None:
|
||||
final_text = (result["transcription_text"] or "").strip()
|
||||
assert final_text, "Expected non-empty transcription (model text stream)"
|
||||
|
||||
wav_out = _wav_bytes_from_pcm16(result["output_pcm"], result["output_sample_rate"])
|
||||
whisper_text = convert_audio_bytes_to_text(wav_out).strip()
|
||||
assert whisper_text, "Whisper returned empty string for synthesized output audio"
|
||||
|
||||
sim = cosine_similarity_text(whisper_text.lower(), final_text.lower())
|
||||
assert sim > 0.8, (
|
||||
f"Output audio transcript should match model text (sim={sim:.3f}): "
|
||||
f"whisper={whisper_text!r}, model_text={final_text!r}"
|
||||
)
|
||||
|
||||
|
||||
class TestQwen3OmniRealtimeWebSocket:
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", realtime_server_params, indirect=True)
|
||||
def test_streaming_audio_input_pcm_output(self, omni_server) -> None:
|
||||
"""
|
||||
Short streamed 16 kHz mono PCM16 input; expect streamed PCM16 audio deltas and
|
||||
transcription. Verify Whisper(output audio) aligns with model text (same idea
|
||||
as multimodal omni e2e). Input speech is synthesized from
|
||||
``REALTIME_SYNTH_PHRASE_TEXT``.
|
||||
"""
|
||||
syn = generate_synthetic_audio(
|
||||
10,
|
||||
1,
|
||||
sample_rate=16000,
|
||||
phrase_text=REALTIME_SYNTH_PHRASE_TEXT,
|
||||
)
|
||||
wav_bytes = base64.b64decode(syn["base64"])
|
||||
pcm16 = _pcm16_mono_16k_from_wav_bytes(wav_bytes)
|
||||
@pytest.mark.parametrize("omni_server", realtime_async_chunk_server_params, indirect=True)
|
||||
def test_streaming_audio_input_pcm_output_async_chunk_smoke(self, omni_server) -> None:
|
||||
"""Ready CI: async_chunk on, no send delay, smoke only."""
|
||||
pcm16 = _synthetic_pcm16_input()
|
||||
|
||||
result = asyncio.run(
|
||||
_run_realtime_audio_roundtrip(
|
||||
@@ -184,25 +229,52 @@ class TestQwen3OmniRealtimeWebSocket:
|
||||
omni_server.model,
|
||||
pcm16,
|
||||
chunk_ms=100,
|
||||
send_delay_ms=0,
|
||||
)
|
||||
)
|
||||
|
||||
out_pcm = result["output_pcm"]
|
||||
assert result["delta_events"] >= 1
|
||||
assert out_pcm, "No output PCM from response.audio.delta"
|
||||
assert len(out_pcm) % 2 == 0
|
||||
assert len(out_pcm) >= 4096, "Output audio unexpectedly small"
|
||||
assert result["output_sample_rate"] > 0
|
||||
_assert_realtime_smoke(result)
|
||||
|
||||
final_text = (result["transcription_text"] or "").strip()
|
||||
assert final_text, "Expected non-empty transcription (model text stream)"
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", realtime_async_chunk_server_params, indirect=True)
|
||||
def test_streaming_audio_input_pcm_output_async_chunk(self, omni_server) -> None:
|
||||
"""Merge CI: async_chunk on, paced upload, full accuracy check."""
|
||||
pcm16 = _synthetic_pcm16_input()
|
||||
|
||||
wav_out = _wav_bytes_from_pcm16(out_pcm, result["output_sample_rate"])
|
||||
whisper_text = convert_audio_bytes_to_text(wav_out).strip()
|
||||
assert whisper_text, "Whisper returned empty string for synthesized output audio"
|
||||
|
||||
sim = cosine_similarity_text(whisper_text.lower(), final_text.lower())
|
||||
assert sim > 0.8, (
|
||||
f"Output audio transcript should match model text (sim={sim:.3f}): "
|
||||
f"whisper={whisper_text!r}, model_text={final_text!r}"
|
||||
result = asyncio.run(
|
||||
_run_realtime_audio_roundtrip(
|
||||
omni_server.host,
|
||||
omni_server.port,
|
||||
omni_server.model,
|
||||
pcm16,
|
||||
chunk_ms=100,
|
||||
send_delay_ms=SEND_DELAY_MS,
|
||||
)
|
||||
)
|
||||
|
||||
_assert_realtime_smoke(result)
|
||||
_assert_realtime_accuracy(result)
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", realtime_sync_server_params, indirect=True)
|
||||
def test_streaming_audio_input_pcm_output(self, omni_server) -> None:
|
||||
"""Merge CI: async_chunk off, no send delay, full accuracy check."""
|
||||
pcm16 = _synthetic_pcm16_input()
|
||||
|
||||
result = asyncio.run(
|
||||
_run_realtime_audio_roundtrip(
|
||||
omni_server.host,
|
||||
omni_server.port,
|
||||
omni_server.model,
|
||||
pcm16,
|
||||
chunk_ms=100,
|
||||
send_delay_ms=0,
|
||||
)
|
||||
)
|
||||
|
||||
_assert_realtime_smoke(result)
|
||||
_assert_realtime_accuracy(result)
|
||||
@@ -61,7 +61,7 @@ def test_build_multistage_generation_inputs_applies_stage_specific_overrides(ser
|
||||
assert engine_prompt["prompt"] == "draw a robot"
|
||||
assert engine_prompt["modalities"] == ["img2img"]
|
||||
assert engine_prompt["negative_prompt"] == "blurry"
|
||||
assert engine_prompt["mm_processor_kwargs"] == {"target_h": 768, "target_w": 1024}
|
||||
assert engine_prompt["mm_processor_kwargs"] == {"target_h": 768, "target_w": 1024, "vae_generator_seed": 0}
|
||||
assert engine_prompt["multi_modal_data"]["img2img"].size == (24, 24)
|
||||
|
||||
assert len(sampling_params_list) == 3
|
||||
|
||||
@@ -399,6 +399,8 @@ def test_sampling_params_pass_through(test_client, mocker: MockerFixture):
|
||||
"true_cfg_scale": "4.0",
|
||||
"boundary_ratio": "0.7",
|
||||
"flow_shift": "0.25",
|
||||
"generate_sound": "true",
|
||||
"sound_duration": "2.5",
|
||||
},
|
||||
)
|
||||
|
||||
@@ -413,6 +415,8 @@ def test_sampling_params_pass_through(test_client, mocker: MockerFixture):
|
||||
assert captured.true_cfg_scale == 4.0
|
||||
assert captured.boundary_ratio == 0.7
|
||||
assert captured.extra_args["flow_shift"] == 0.25
|
||||
assert captured.extra_args["generate_sound"] is True
|
||||
assert captured.extra_args["sound_duration"] == 2.5
|
||||
|
||||
|
||||
def test_frame_interpolation_params_pass_to_diffusion_sampling_params(test_client, mocker: MockerFixture):
|
||||
@@ -623,6 +627,115 @@ def test_video_job_persists_profiler_metadata(test_client, mocker: MockerFixture
|
||||
|
||||
assert completed["stage_durations"] == {"diffuse": 2.5, "vae.decode": 0.3}
|
||||
assert completed["peak_memory_mb"] == 4096.5
|
||||
assert completed["action"] is None
|
||||
|
||||
|
||||
def test_video_generation_response_exposes_action_payload(mocker: MockerFixture):
|
||||
engine = FakeAsyncOmni()
|
||||
handler = OmniOpenAIServingVideo.for_diffusion(
|
||||
diffusion_engine=engine,
|
||||
model_name="Cosmos3-8B-UVA",
|
||||
)
|
||||
|
||||
async def _generate(prompt, request_id, sampling_params_list):
|
||||
del prompt, request_id, sampling_params_list
|
||||
import numpy as np
|
||||
|
||||
yield MockVideoResult(
|
||||
[object()],
|
||||
custom_output={
|
||||
"action": np.array([[[1.5, 2.5], [3.5, 4.5]]], dtype=np.float32),
|
||||
"raw_action_dim": 2,
|
||||
"action_mode": "policy",
|
||||
"domain_id": 7,
|
||||
},
|
||||
)
|
||||
|
||||
engine.generate = _generate
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
|
||||
return_value="encoded-video",
|
||||
)
|
||||
|
||||
response = asyncio.run(
|
||||
handler.generate_videos(
|
||||
VideoGenerationRequest(prompt="predict actions"),
|
||||
"action-json",
|
||||
)
|
||||
)
|
||||
|
||||
action = response.data[0].action
|
||||
assert action is not None
|
||||
assert action.data == [[1.5, 2.5], [3.5, 4.5]]
|
||||
assert action.shape == [2, 2]
|
||||
assert action.dtype == "float32"
|
||||
assert action.raw_action_dim == 2
|
||||
assert action.action_mode == "policy"
|
||||
assert action.domain_id == 7
|
||||
assert response.model_dump(mode="json")["data"][0]["action"]["data"] == [[1.5, 2.5], [3.5, 4.5]]
|
||||
|
||||
|
||||
def test_video_job_persists_action_metadata(test_client, mocker: MockerFixture):
|
||||
engine = test_client.app.state.openai_serving_video._engine_client
|
||||
|
||||
async def _generate(prompt, request_id, sampling_params_list):
|
||||
import numpy as np
|
||||
|
||||
engine.captured_prompt = prompt
|
||||
engine.captured_sampling_params_list = sampling_params_list
|
||||
yield MockVideoResult(
|
||||
[object()],
|
||||
custom_output={
|
||||
"action": np.array([[[1.0, 2.0], [3.0, 4.0]]], dtype=np.float32),
|
||||
"raw_action_dim": 2,
|
||||
"action_mode": "policy",
|
||||
"domain_id": 7,
|
||||
},
|
||||
)
|
||||
|
||||
engine.generate = _generate
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
||||
return_value=b"fake-video",
|
||||
)
|
||||
|
||||
response = test_client.post("/v1/videos", data={"prompt": "profile me"})
|
||||
assert response.status_code == 200
|
||||
video_id = response.json()["id"]
|
||||
completed = _wait_for_status(test_client, video_id, VideoGenerationStatus.COMPLETED.value)
|
||||
|
||||
expected_action = {
|
||||
"data": [[1.0, 2.0], [3.0, 4.0]],
|
||||
"shape": [2, 2],
|
||||
"dtype": "float32",
|
||||
"raw_action_dim": 2,
|
||||
"action_mode": "policy",
|
||||
"domain_id": 7,
|
||||
}
|
||||
assert completed["action"] == expected_action
|
||||
|
||||
listed = test_client.get("/v1/videos").json()
|
||||
assert listed["data"][0]["action"] == expected_action
|
||||
|
||||
|
||||
def test_action_extraction_accepts_unbatched_action():
|
||||
import numpy as np
|
||||
|
||||
result = MockVideoResult(
|
||||
[object()],
|
||||
custom_output={
|
||||
"action": np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32),
|
||||
"raw_action_dim": 2,
|
||||
"action_mode": "policy",
|
||||
"domain_id": 7,
|
||||
},
|
||||
)
|
||||
|
||||
actions = OmniOpenAIServingVideo._extract_action_outputs(result, expected_count=1)
|
||||
|
||||
assert actions[0] is not None
|
||||
assert actions[0].data == [[1.0, 2.0], [3.0, 4.0]]
|
||||
assert actions[0].shape == [2, 2]
|
||||
|
||||
|
||||
def test_missing_handler_returns_503():
|
||||
@@ -756,6 +869,9 @@ def test_invalid_uploaded_input_reference_returns_400(test_client):
|
||||
def test_video_request_validation():
|
||||
req = VideoGenerationRequest(prompt="test")
|
||||
assert req.prompt == "test"
|
||||
assert req.generate_sound is False
|
||||
assert req.sound_duration is None
|
||||
assert VideoGenerationRequest(prompt="test", generate_sound=True, sound_duration=1.5).generate_sound is True
|
||||
with pytest.raises(ValueError):
|
||||
VideoGenerationRequest(prompt="test", size="invalid")
|
||||
|
||||
@@ -768,6 +884,8 @@ def test_video_request_validation():
|
||||
VideoGenerationRequest(prompt="test", frame_interpolation_exp=0)
|
||||
with pytest.raises(ValueError):
|
||||
VideoGenerationRequest(prompt="test", frame_interpolation_scale=0)
|
||||
with pytest.raises(ValueError):
|
||||
VideoGenerationRequest(prompt="test", sound_duration=0)
|
||||
|
||||
|
||||
def test_list_videos_supports_order_after_and_limit(test_client, mocker: MockerFixture):
|
||||
@@ -1029,6 +1147,31 @@ def test_sample_solver_forwarded_via_extra_params(test_client, mocker: MockerFix
|
||||
assert captured.extra_args["sample_solver"] == "euler"
|
||||
|
||||
|
||||
def test_extra_params_allows_inline_action(test_client, mocker: MockerFixture):
|
||||
"""Inline ``action`` data is accepted and forwarded verbatim to
|
||||
``extra_args`` (the supported way to pass forward-dynamics actions)."""
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
||||
return_value=b"fake-video",
|
||||
)
|
||||
action = [[0.1, 0.2], [0.3, 0.4]]
|
||||
response = test_client.post(
|
||||
"/v1/videos",
|
||||
data={
|
||||
"prompt": "forward dynamics inline",
|
||||
"extra_params": json.dumps({"action_mode": "forward_dynamics", "action": action}),
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
video_id = response.json()["id"]
|
||||
_wait_for_status(test_client, video_id, VideoGenerationStatus.COMPLETED.value)
|
||||
engine = test_client.app.state.openai_serving_video._engine_client
|
||||
captured = engine.captured_sampling_params_list[0]
|
||||
assert captured.extra_args["action"] == action
|
||||
assert captured.extra_args["action_mode"] == "forward_dynamics"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sync endpoint tests (POST /v1/videos/sync)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -16,7 +16,7 @@ OMNI_MODEL = "Qwen/Qwen2.5-Omni-7B"
|
||||
OMNI_STAGE_CONFIG = get_deploy_config_path("ci/qwen2_5_omni_thinker_only.yaml")
|
||||
|
||||
|
||||
async def _noop(**kw):
|
||||
async def _noop(*args, **kw):
|
||||
pass
|
||||
|
||||
|
||||
@@ -61,6 +61,7 @@ def get_async_omni_instance(fake_add_request=_noop, fake_abort_request=_noop) ->
|
||||
omni._final_output_handler = lambda: None
|
||||
omni.resolve_sampling_params_list = lambda params, allow_delta_coercion: params
|
||||
omni._compute_final_stage_id = lambda output_modalities: 0
|
||||
omni._compute_final_output_stage_ids = lambda output_modalities: [0]
|
||||
omni._process_orchestrator_results = fake_process_results
|
||||
omni._log_summary_and_cleanup = lambda request_id: omni.request_states.pop(request_id, None)
|
||||
return omni
|
||||
|
||||
@@ -413,6 +413,19 @@ def test_openai_serving_models_can_consume_async_omni_compat_attrs():
|
||||
|
||||
|
||||
def test_get_diffusion_od_config_returns_diffusion_stage_config():
|
||||
diffusion_od_config = object()
|
||||
omni = object.__new__(AsyncOmni)
|
||||
omni.engine = SimpleNamespace(
|
||||
stage_clients=[
|
||||
SimpleNamespace(stage_type="llm"),
|
||||
SimpleNamespace(stage_type="diffusion", od_config=diffusion_od_config),
|
||||
]
|
||||
)
|
||||
|
||||
assert omni.get_diffusion_od_config() is diffusion_od_config
|
||||
|
||||
|
||||
def test_get_diffusion_od_config_falls_back_to_inner_engine():
|
||||
diffusion_od_config = object()
|
||||
omni = object.__new__(AsyncOmni)
|
||||
omni.engine = SimpleNamespace(
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
import pytest
|
||||
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.stage_diffusion_proc import StageDiffusionProc
|
||||
from vllm_omni.entrypoints.utils import load_stage_configs_from_model, resolve_model_config_path
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_dreamzero_vla_resolves_to_dreamzero_config(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.entrypoints.utils.get_config",
|
||||
lambda _model, trust_remote_code=True: type("Cfg", (), {"model_type": "vla"})(),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.entrypoints.utils._looks_like_dreamzero",
|
||||
lambda _model: True,
|
||||
)
|
||||
result = resolve_model_config_path("GEAR-Dreams/DreamZero-DROID")
|
||||
|
||||
assert result is not None
|
||||
assert result.endswith("vllm_omni/deploy/dreamzero.yaml")
|
||||
|
||||
|
||||
def test_dreamzero_config_sets_model_class_and_policy_config(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.config.stage_config.StageConfigFactory._auto_detect_model_type",
|
||||
classmethod(lambda _cls, _model, trust_remote_code=True: ("vla", None)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.diffusion.utils.hf_utils._looks_like_dreamzero",
|
||||
lambda _model: True,
|
||||
)
|
||||
|
||||
stage_configs = load_stage_configs_from_model("GEAR-Dreams/DreamZero-DROID")
|
||||
engine_args = stage_configs[0].engine_args
|
||||
|
||||
assert engine_args.model_class_name == "DreamZeroPipeline"
|
||||
assert engine_args.model_config.policy_server_config.action_space == "joint_position"
|
||||
|
||||
|
||||
def test_dreamzero_enrich_config_preserves_explicit_model_class_name(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"vllm.transformers_utils.config.get_hf_file_to_dict",
|
||||
lambda path, _model: None if path == "model_index.json" else {"model_type": "vla", "architectures": ["VLA"]},
|
||||
)
|
||||
|
||||
od_config = OmniDiffusionConfig(
|
||||
model="GEAR-Dreams/DreamZero-DROID",
|
||||
model_class_name="DreamZeroPipeline",
|
||||
)
|
||||
proc = StageDiffusionProc(od_config.model, od_config)
|
||||
|
||||
proc._enrich_config()
|
||||
|
||||
assert od_config.model_class_name == "DreamZeroPipeline"
|
||||
@@ -135,22 +135,6 @@ def test_set_stage_devices_npu_platform(mocker: MockerFixture, monkeypatch: pyte
|
||||
# ---- _map_device_list unit tests ----
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
def test_map_device_list_idempotency():
|
||||
"""Device IDs already in the visible set are returned as-is (idempotency)."""
|
||||
result = _map_device_list(0, ["0", "1"], ["0", "1", "2", "3"])
|
||||
assert result == ["0", "1"]
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
def test_map_device_list_idempotency_single():
|
||||
"""Single device ID in visible set passes through."""
|
||||
result = _map_device_list(1, ["1"], ["0", "1"])
|
||||
assert result == ["1"]
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
def test_map_device_list_index_mapping():
|
||||
@@ -211,6 +195,18 @@ def test_map_device_list_partial_mapping():
|
||||
assert result == ["5", "6"]
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
def test_map_device_list_multi_replica_offset_cvd():
|
||||
"""Logical indices from split_devices_for_replicas must map through an offset
|
||||
CUDA_VISIBLE_DEVICES, not be treated as physical IDs.
|
||||
"""
|
||||
visible = ["2", "3", "4", "5"]
|
||||
assert _map_device_list(1, ["1"], visible) == ["3"]
|
||||
assert _map_device_list(1, ["2"], visible) == ["4"]
|
||||
assert _map_device_list(1, ["2", "3"], visible) == ["4", "5"]
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
def test_map_device_list_raises_on_non_numeric():
|
||||
|
||||
@@ -888,6 +888,12 @@ class OpenAIClientHandler:
|
||||
audio_data.append(content)
|
||||
elif modality == "text" and content:
|
||||
text_content += content
|
||||
# Usage is yielded after the last token
|
||||
if chunk.usage:
|
||||
result.prompt_tokens = chunk.usage.prompt_tokens
|
||||
if details := getattr(chunk.usage, "prompt_tokens_details", None):
|
||||
result.cached_tokens = details.cached_tokens
|
||||
|
||||
audio_content = None
|
||||
if audio_data:
|
||||
merged_seg = _merge_base64_audio_to_segment(audio_data)
|
||||
@@ -1582,6 +1588,8 @@ class OpenAIClientHandler:
|
||||
create_kwargs["logprobs"] = request_config["logprobs"]
|
||||
if "top_logprobs" in request_config:
|
||||
create_kwargs["top_logprobs"] = request_config["top_logprobs"]
|
||||
if "stream_options" in request_config:
|
||||
create_kwargs["stream_options"] = request_config["stream_options"]
|
||||
if extra_body:
|
||||
create_kwargs["extra_body"] = extra_body
|
||||
|
||||
|
||||
@@ -154,6 +154,7 @@ class TestRequestLifecycleGauges:
|
||||
obj.engine = SimpleNamespace(_running_counter=OmniRequestCounter())
|
||||
obj.prom_metrics = OmniPrometheusMetrics(model_name="lifecycle-test")
|
||||
obj.request_states = {}
|
||||
obj._consumed_metric_messages = {}
|
||||
obj.log_stats = False
|
||||
|
||||
# Simulate request lifecycle: start (counter 0→1, dict {} → {req}),
|
||||
@@ -182,6 +183,7 @@ class TestRequestLifecycleGauges:
|
||||
obj.engine = SimpleNamespace(_running_counter=OmniRequestCounter())
|
||||
obj.prom_metrics = OmniPrometheusMetrics(model_name="lifecycle-test-2")
|
||||
obj.request_states = {}
|
||||
obj._consumed_metric_messages = {}
|
||||
obj.log_stats = False
|
||||
|
||||
# Two in flight, one finalizes — running should report 1, waiting 0.
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -17,13 +18,14 @@ def _voxcpm2_talker_mod():
|
||||
from vllm_omni.model_executor.models.voxcpm2.voxcpm2_talker import (
|
||||
VoxCPM2TalkerForConditionalGeneration,
|
||||
_RequestState,
|
||||
_VoxCPM2RuntimeConfig,
|
||||
)
|
||||
|
||||
return VoxCPM2TalkerForConditionalGeneration, _RequestState
|
||||
return VoxCPM2TalkerForConditionalGeneration, _RequestState, _VoxCPM2RuntimeConfig
|
||||
|
||||
|
||||
def _make_bare_talker():
|
||||
VoxCPM2TalkerForConditionalGeneration, _ = _voxcpm2_talker_mod()
|
||||
VoxCPM2TalkerForConditionalGeneration, _, _ = _voxcpm2_talker_mod()
|
||||
talker = VoxCPM2TalkerForConditionalGeneration.__new__(VoxCPM2TalkerForConditionalGeneration)
|
||||
talker._active_states = {}
|
||||
talker._current_request_id = None
|
||||
@@ -34,11 +36,27 @@ def _make_bare_talker():
|
||||
talker._max_batch_size = 4
|
||||
talker._active_state_warn_threshold = 512
|
||||
talker._active_state_warned = False
|
||||
talker._enable_profiling = False
|
||||
talker._audio_emit_every = 1
|
||||
talker._vae_decode_every = 1
|
||||
talker._enable_delayed_audio_copy = False
|
||||
talker._delayed_audio_copy_use_events = False
|
||||
talker._coalesce_audio_d2h = False
|
||||
talker._enable_batched_vae_decode = False
|
||||
talker._audio_copy_stream = None
|
||||
talker._enable_vae_cuda_graph = False
|
||||
talker._enable_cfm_cuda_graph = False
|
||||
talker._enable_cfm_prealloc_output = False
|
||||
talker._enable_batched_cfm = True
|
||||
talker._deterministic_cfm_noise = False
|
||||
talker._cfm_buffers = None
|
||||
talker._last_audio_output_req_ids = []
|
||||
talker._batched_fsq_fusion_max_batch = 32
|
||||
return talker
|
||||
|
||||
|
||||
def _seed_cached_decode(talker, req_id: str):
|
||||
_, _RequestState = _voxcpm2_talker_mod()
|
||||
_, _RequestState, _ = _voxcpm2_talker_mod()
|
||||
state = _RequestState(request_id=req_id)
|
||||
state.prefill_completed = True
|
||||
state.decode_step_count = 5
|
||||
@@ -47,6 +65,23 @@ def _seed_cached_decode(talker, req_id: str):
|
||||
|
||||
|
||||
class TestStateEvictionContract:
|
||||
def test_runtime_config_normalizes_mutually_exclusive_paths(self) -> None:
|
||||
_, _, RuntimeConfig = _voxcpm2_talker_mod()
|
||||
|
||||
cfg = RuntimeConfig(
|
||||
enable_batched_cfm=True,
|
||||
enable_cfm_cuda_graph=True,
|
||||
enable_cfm_prealloc_output=True,
|
||||
enable_batched_vae_decode=True,
|
||||
enable_delayed_audio_copy=True,
|
||||
)._normalized()
|
||||
|
||||
assert cfg.enable_batched_cfm is True
|
||||
assert cfg.enable_cfm_cuda_graph is False
|
||||
assert cfg.enable_cfm_prealloc_output is False
|
||||
assert cfg.enable_batched_vae_decode is False
|
||||
assert cfg.enable_delayed_audio_copy is True
|
||||
|
||||
def test_pending_requests_is_not_used_for_eviction(self) -> None:
|
||||
talker = _make_bare_talker()
|
||||
|
||||
@@ -61,6 +96,61 @@ class TestStateEvictionContract:
|
||||
assert rid in talker._active_states
|
||||
assert talker._active_states[rid].prefill_completed is True
|
||||
|
||||
def test_compute_logits_does_not_sync_stop_bool_on_default_path(self) -> None:
|
||||
_, RState, _ = _voxcpm2_talker_mod()
|
||||
talker = _make_bare_talker()
|
||||
talker.config = SimpleNamespace(vocab_size=4)
|
||||
state = RState(request_id="req", precomputed_stop_logits=torch.tensor([[0.0, 1.0]]))
|
||||
talker._active_states["req"] = state
|
||||
talker._results_queue = [("req", state.precomputed_stop_logits)]
|
||||
|
||||
logits = talker.compute_logits(torch.zeros(1, 1))
|
||||
|
||||
assert logits[0, 0] == 0.0
|
||||
assert logits[0, 1] == 1.0
|
||||
assert state.is_stopping is False
|
||||
assert state.precomputed_stop_logits is None
|
||||
assert state.precomputed_is_stopping is None
|
||||
|
||||
def test_compute_logits_reuses_cached_stop_bool_from_sparse_audio_path(self) -> None:
|
||||
_, RState, _ = _voxcpm2_talker_mod()
|
||||
talker = _make_bare_talker()
|
||||
talker.config = SimpleNamespace(vocab_size=4)
|
||||
state = RState(
|
||||
request_id="req",
|
||||
precomputed_stop_logits=torch.tensor([[0.0, 1.0]]),
|
||||
precomputed_is_stopping=True,
|
||||
)
|
||||
talker._active_states["req"] = state
|
||||
talker._results_queue = [("req", state.precomputed_stop_logits)]
|
||||
|
||||
logits = talker.compute_logits(torch.zeros(1, 1))
|
||||
|
||||
assert logits[0, 0] == 0.0
|
||||
assert logits[0, 1] == 1.0
|
||||
assert state.is_stopping is True
|
||||
assert state.precomputed_stop_logits is None
|
||||
assert state.precomputed_is_stopping is None
|
||||
|
||||
def test_batched_stop_precompute_sets_audio_collect_cache(self) -> None:
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA is required to exercise batched stop D2H precompute")
|
||||
|
||||
_, RState, _ = _voxcpm2_talker_mod()
|
||||
talker = _make_bare_talker()
|
||||
talker._audio_emit_every = 2
|
||||
states = [
|
||||
RState(request_id="stop", precomputed_stop_logits=torch.tensor([[0.0, 1.0]], device="cuda")),
|
||||
RState(request_id="keep", precomputed_stop_logits=torch.tensor([[1.0, 0.0]], device="cuda")),
|
||||
]
|
||||
|
||||
talker._precompute_stop_flags_for_audio_collect(states)
|
||||
|
||||
assert states[0].precomputed_is_stopping is True
|
||||
assert states[0].is_stopping is True
|
||||
assert states[1].precomputed_is_stopping is False
|
||||
assert states[1].is_stopping is False
|
||||
|
||||
def test_on_requests_finished_defers_cleanup(self) -> None:
|
||||
talker = _make_bare_talker()
|
||||
_seed_cached_decode(talker, "req-A")
|
||||
@@ -119,7 +209,7 @@ class TestLeakWarnGuard:
|
||||
talker = _make_bare_talker()
|
||||
talker._active_state_warn_threshold = 3
|
||||
|
||||
_, RState = _voxcpm2_talker_mod()
|
||||
_, RState, _ = _voxcpm2_talker_mod()
|
||||
for i in range(4):
|
||||
talker._active_states[f"seed-{i}"] = RState(request_id=f"seed-{i}")
|
||||
|
||||
@@ -129,3 +219,548 @@ class TestLeakWarnGuard:
|
||||
leak_warnings = [m for m in calls if "cleanup path leak" in m]
|
||||
assert len(leak_warnings) == 1
|
||||
assert talker._active_state_warned is True
|
||||
|
||||
|
||||
class _NoopPerf:
|
||||
def start(self, name: str) -> None:
|
||||
pass
|
||||
|
||||
def stop(self, name: str) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class _FakeTTS:
|
||||
feat_decoder = SimpleNamespace(estimator=SimpleNamespace(_compiled=False))
|
||||
|
||||
def fsq_layer(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return x + 10
|
||||
|
||||
def fusion_concat_proj(self, x: torch.Tensor) -> torch.Tensor:
|
||||
half = x.shape[-1] // 2
|
||||
return x[:, :half] * 2 + x[:, half:] * 3
|
||||
|
||||
def feat_encoder(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return x
|
||||
|
||||
def enc_to_lm_proj(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return x.squeeze(1).mean(dim=1).repeat(1, 4)
|
||||
|
||||
|
||||
class TestDecodeBatchContract:
|
||||
def test_forward_skip_pending_prefill_keeps_audio_queue_aligned(self) -> None:
|
||||
class FakeScaffold:
|
||||
def __call__(self, input_ids, positions, intermediate_tensors, inputs_embeds):
|
||||
return torch.zeros(input_ids.shape[0], 4)
|
||||
|
||||
talker = _make_bare_talker()
|
||||
talker._perf = _NoopPerf()
|
||||
talker._tts = _FakeTTS()
|
||||
talker.model = FakeScaffold()
|
||||
talker._enable_cuda_graph = False
|
||||
talker._cuda_graph_warmup_steps = 0
|
||||
talker._cuda_graph_warmup_threshold = 3
|
||||
talker._max_cached_graphs = 4
|
||||
talker._scaffold_graphs = {}
|
||||
talker._residual_graphs = {}
|
||||
talker._pending_requests = [("req", False, None, 1)]
|
||||
|
||||
out = talker.forward(
|
||||
torch.tensor([1]),
|
||||
torch.tensor([0]),
|
||||
inputs_embeds=torch.zeros(1, 4),
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out, torch.zeros(1, 4))
|
||||
assert talker._results_queue == [("req", None)]
|
||||
assert talker._audio_queue == [("req", None)]
|
||||
|
||||
def _make_decode_talker(self) -> object:
|
||||
talker = _make_bare_talker()
|
||||
talker._perf = _NoopPerf()
|
||||
talker._tts = _FakeTTS()
|
||||
talker._side_dtype = torch.float32
|
||||
talker._patch_size = 2
|
||||
talker._feat_dim = 3
|
||||
|
||||
def dit_proj(lm_h: torch.Tensor, res_h: torch.Tensor) -> torch.Tensor:
|
||||
return lm_h + 2 * res_h
|
||||
|
||||
def stop_fn(lm_h: torch.Tensor) -> torch.Tensor:
|
||||
return torch.stack([lm_h.sum(dim=1), -lm_h.sum(dim=1)], dim=1)
|
||||
|
||||
def run_cfm(dit_h: torch.Tensor, cond: torch.Tensor) -> torch.Tensor:
|
||||
return dit_h[:, :1].view(-1, 1, 1) + cond.transpose(1, 2) * 0.1
|
||||
|
||||
talker._dit_proj_fn = dit_proj
|
||||
talker._stop_fn = stop_fn
|
||||
talker._run_cfm = run_cfm
|
||||
talker._run_cfm_for_state = lambda _state, dit_h, cond: talker._run_cfm(dit_h, cond)
|
||||
return talker
|
||||
|
||||
def _make_decode_state(self, request_id: str, offset: float):
|
||||
_, RState, _ = _voxcpm2_talker_mod()
|
||||
state = RState(request_id=request_id)
|
||||
state.curr_prefix_feat_cond = torch.arange(6, dtype=torch.float32).reshape(2, 3) + offset
|
||||
return state
|
||||
|
||||
def test_batched_fsq_fusion_matches_per_request_decode_prepare(self) -> None:
|
||||
seq_talker = self._make_decode_talker()
|
||||
batch_talker = self._make_decode_talker()
|
||||
seq_talker._max_decode_steps = 2000
|
||||
batch_talker._max_decode_steps = 2000
|
||||
|
||||
seq_states = [self._make_decode_state("req-0", 0.0), self._make_decode_state("req-1", 10.0)]
|
||||
batch_states = [self._make_decode_state("req-0", 0.0), self._make_decode_state("req-1", 10.0)]
|
||||
for idx, (seq_state, batch_state) in enumerate(zip(seq_states, batch_states, strict=True)):
|
||||
prev = torch.full((1, 4), float(idx + 1))
|
||||
seq_state.prev_feat_embed = prev.clone()
|
||||
batch_state.prev_feat_embed = prev.clone()
|
||||
|
||||
hiddens = [
|
||||
torch.tensor([[1.0, 2.0, 3.0, 4.0]]),
|
||||
torch.tensor([[5.0, 6.0, 7.0, 8.0]]),
|
||||
]
|
||||
|
||||
expected = [
|
||||
seq_talker._prepare_residual_decode(state, hidden, torch.device("cpu"))
|
||||
for state, hidden in zip(seq_states, hiddens, strict=True)
|
||||
]
|
||||
actual = batch_talker._prepare_residual_decode_batch(batch_states, hiddens, torch.device("cpu"))
|
||||
|
||||
for (expected_res, expected_meta), (actual_res, actual_meta) in zip(expected, actual, strict=True):
|
||||
torch.testing.assert_close(actual_res, expected_res)
|
||||
torch.testing.assert_close(actual_meta["new_lm_hidden"], expected_meta["new_lm_hidden"])
|
||||
assert [state.decode_step_count for state in batch_states] == [1, 1]
|
||||
|
||||
def test_batch_decode_matches_per_request_state_updates(self) -> None:
|
||||
seq_talker = self._make_decode_talker()
|
||||
batch_talker = self._make_decode_talker()
|
||||
|
||||
seq_states = [self._make_decode_state("req-0", 0.0), self._make_decode_state("req-1", 10.0)]
|
||||
batch_states = [self._make_decode_state("req-0", 0.0), self._make_decode_state("req-1", 10.0)]
|
||||
metas = [
|
||||
{"new_lm_hidden": torch.tensor([[1.0, 2.0, 3.0, 4.0]])},
|
||||
{"new_lm_hidden": torch.tensor([[5.0, 6.0, 7.0, 8.0]])},
|
||||
]
|
||||
batch_out = torch.tensor([[0.5, 1.0, 1.5, 2.0], [2.5, 3.0, 3.5, 4.0]])
|
||||
|
||||
for state, meta, res_out in zip(seq_states, metas, batch_out, strict=True):
|
||||
seq_talker._finish_decode(state, meta, res_out.unsqueeze(0), torch.device("cpu"))
|
||||
|
||||
batch_talker._finish_decode_batch(
|
||||
[(state, False, meta) for state, meta in zip(batch_states, metas, strict=True)],
|
||||
batch_out,
|
||||
)
|
||||
|
||||
for expected, actual in zip(seq_states, batch_states, strict=True):
|
||||
torch.testing.assert_close(actual.precomputed_stop_logits, expected.precomputed_stop_logits)
|
||||
torch.testing.assert_close(actual.curr_embed_for_next, expected.curr_embed_for_next)
|
||||
torch.testing.assert_close(actual.prev_feat_embed, expected.prev_feat_embed)
|
||||
torch.testing.assert_close(actual.curr_prefix_feat_cond, expected.curr_prefix_feat_cond)
|
||||
torch.testing.assert_close(actual.last_audio_patch_gpu, expected.last_audio_patch_gpu)
|
||||
|
||||
def test_batch_decode_preserves_per_request_cfm_rng_order(self) -> None:
|
||||
talker = self._make_decode_talker()
|
||||
talker._enable_batched_cfm = False
|
||||
states = [self._make_decode_state("req-0", 0.0), self._make_decode_state("req-1", 10.0)]
|
||||
metas = [
|
||||
{"new_lm_hidden": torch.tensor([[1.0, 2.0, 3.0, 4.0]])},
|
||||
{"new_lm_hidden": torch.tensor([[5.0, 6.0, 7.0, 8.0]])},
|
||||
]
|
||||
batch_out = torch.tensor([[0.5, 1.0, 1.5, 2.0], [2.5, 3.0, 3.5, 4.0]])
|
||||
|
||||
captured_noise: list[torch.Tensor] = []
|
||||
torch.manual_seed(1234)
|
||||
expected_noise = [torch.empty(1, 3, 2).normal_().detach().clone() for _ in states]
|
||||
|
||||
def run_cfm(
|
||||
dit_h: torch.Tensor,
|
||||
cond: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
captured_noise.append(torch.empty(1, 3, 2).normal_().detach().clone())
|
||||
return dit_h[:, :1].view(-1, 1, 1) + cond.transpose(1, 2) * 0.1
|
||||
|
||||
talker._run_cfm = run_cfm
|
||||
|
||||
torch.manual_seed(1234)
|
||||
talker._finish_decode_batch(
|
||||
[(state, False, meta) for state, meta in zip(states, metas, strict=True)],
|
||||
batch_out,
|
||||
)
|
||||
|
||||
assert len(captured_noise) == len(states)
|
||||
for actual, expected in zip(captured_noise, expected_noise, strict=True):
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
|
||||
class _FakeAudioVAE:
|
||||
decode_chunk_size = 2
|
||||
|
||||
|
||||
class _FakeAudioTTS:
|
||||
audio_vae = _FakeAudioVAE()
|
||||
|
||||
|
||||
class TestAudioEmitCoalescing:
|
||||
def test_audio_emit_every_accumulates_gpu_chunks_without_changing_order(self) -> None:
|
||||
_, RState, _ = _voxcpm2_talker_mod()
|
||||
talker = _make_bare_talker()
|
||||
talker._audio_emit_every = 2
|
||||
talker._feat_dim = 1
|
||||
talker._n_decode_pad_frames = 1
|
||||
talker._device = torch.device("cpu")
|
||||
talker._tts = _FakeAudioTTS()
|
||||
talker._perf = _NoopPerf()
|
||||
|
||||
values = [torch.tensor([10.0, 11.0]), torch.tensor([99.0, 99.0, 20.0, 21.0])]
|
||||
|
||||
def run_vae_decode(feat: torch.Tensor) -> torch.Tensor:
|
||||
return values.pop(0)
|
||||
|
||||
talker._run_vae_decode = run_vae_decode
|
||||
state = RState(request_id="req")
|
||||
|
||||
state.last_audio_patch_gpu = torch.tensor([[1.0]])
|
||||
assert talker._collect_audio(state) is None
|
||||
assert len(state.pending_audio_chunks_gpu) == 1
|
||||
|
||||
state.last_audio_patch_gpu = torch.tensor([[2.0]])
|
||||
audio = talker._collect_audio(state)
|
||||
|
||||
torch.testing.assert_close(audio, torch.tensor([10.0, 11.0, 20.0, 21.0]))
|
||||
assert state.pending_audio_chunks_gpu == []
|
||||
|
||||
def test_audio_emit_every_flushes_on_stop(self) -> None:
|
||||
_, RState, _ = _voxcpm2_talker_mod()
|
||||
talker = _make_bare_talker()
|
||||
talker._audio_emit_every = 8
|
||||
talker._feat_dim = 1
|
||||
talker._n_decode_pad_frames = 1
|
||||
talker._device = torch.device("cpu")
|
||||
talker._tts = _FakeAudioTTS()
|
||||
talker._perf = _NoopPerf()
|
||||
talker._run_vae_decode = lambda feat: torch.tensor([3.0, 4.0])
|
||||
state = RState(request_id="req", precomputed_stop_logits=torch.tensor([[0.0, 1.0]]))
|
||||
|
||||
state.last_audio_patch_gpu = torch.tensor([[1.0]])
|
||||
audio = talker._collect_audio(state)
|
||||
|
||||
torch.testing.assert_close(audio, torch.tensor([3.0, 4.0]))
|
||||
assert state.pending_audio_chunks_gpu == []
|
||||
|
||||
def test_vae_decode_every_accumulates_latents_before_decode(self) -> None:
|
||||
_, RState, _ = _voxcpm2_talker_mod()
|
||||
talker = _make_bare_talker()
|
||||
talker._vae_decode_every = 2
|
||||
talker._feat_dim = 1
|
||||
talker._n_decode_pad_frames = 1
|
||||
talker._device = torch.device("cpu")
|
||||
talker._tts = _FakeAudioTTS()
|
||||
talker._perf = _NoopPerf()
|
||||
|
||||
seen_feats = []
|
||||
values = [torch.tensor([10.0, 11.0, 20.0, 21.0])]
|
||||
|
||||
def run_vae_decode(feat: torch.Tensor) -> torch.Tensor:
|
||||
seen_feats.append(feat.detach().clone())
|
||||
return values.pop(0)
|
||||
|
||||
talker._run_vae_decode = run_vae_decode
|
||||
state = RState(request_id="req")
|
||||
|
||||
state.last_audio_patch_gpu = torch.tensor([[1.0]])
|
||||
assert talker._collect_audio(state) is None
|
||||
assert len(state.pending_vae_latents_gpu) == 1
|
||||
assert seen_feats == []
|
||||
|
||||
state.last_audio_patch_gpu = torch.tensor([[2.0]])
|
||||
audio = talker._collect_audio(state)
|
||||
|
||||
torch.testing.assert_close(audio, torch.tensor([10.0, 11.0, 20.0, 21.0]))
|
||||
assert state.pending_vae_latents_gpu == []
|
||||
torch.testing.assert_close(seen_feats[0].reshape(-1), torch.tensor([1.0, 2.0]))
|
||||
|
||||
def test_vae_decode_every_flushes_pending_latent_on_stop(self) -> None:
|
||||
_, RState, _ = _voxcpm2_talker_mod()
|
||||
talker = _make_bare_talker()
|
||||
talker._vae_decode_every = 4
|
||||
talker._feat_dim = 1
|
||||
talker._n_decode_pad_frames = 1
|
||||
talker._device = torch.device("cpu")
|
||||
talker._tts = _FakeAudioTTS()
|
||||
talker._perf = _NoopPerf()
|
||||
talker._run_vae_decode = lambda feat: torch.tensor([7.0, 8.0])
|
||||
state = RState(request_id="req", precomputed_stop_logits=torch.tensor([[0.0, 1.0]]))
|
||||
|
||||
state.last_audio_patch_gpu = torch.tensor([[1.0]])
|
||||
audio = talker._collect_audio(state)
|
||||
|
||||
torch.testing.assert_close(audio, torch.tensor([7.0, 8.0]))
|
||||
assert state.pending_vae_latents_gpu == []
|
||||
assert state.is_stopping is True
|
||||
|
||||
def test_batched_vae_decode_collects_ready_requests_in_order(self) -> None:
|
||||
_, RState, _ = _voxcpm2_talker_mod()
|
||||
talker = _make_bare_talker()
|
||||
talker._enable_batched_vae_decode = True
|
||||
talker._coalesce_audio_d2h = True
|
||||
talker._vae_decode_every = 2
|
||||
talker._feat_dim = 1
|
||||
talker._n_decode_pad_frames = 1
|
||||
talker._device = torch.device("cpu")
|
||||
talker._tts = _FakeAudioTTS()
|
||||
talker._perf = _NoopPerf()
|
||||
|
||||
seen_feats: list[torch.Tensor] = []
|
||||
|
||||
def run_vae_decode(feat: torch.Tensor) -> torch.Tensor:
|
||||
seen_feats.append(feat.detach().clone())
|
||||
# One output row per request. decode_chunk_size=2, and each ready
|
||||
# request has two new latent frames, so the full decoded length is 4.
|
||||
return torch.tensor(
|
||||
[
|
||||
[[10.0, 11.0, 12.0, 13.0]],
|
||||
[[20.0, 21.0, 22.0, 23.0]],
|
||||
]
|
||||
)
|
||||
|
||||
talker._run_vae_decode = run_vae_decode
|
||||
|
||||
ready0 = RState(request_id="ready-0")
|
||||
ready0.pending_vae_latents_gpu.append(torch.tensor([[1.0]]))
|
||||
ready0.last_audio_patch_gpu = torch.tensor([[2.0]])
|
||||
|
||||
pending = RState(request_id="pending")
|
||||
pending.last_audio_patch_gpu = torch.tensor([[99.0]])
|
||||
|
||||
ready1 = RState(request_id="ready-1")
|
||||
ready1.pending_vae_latents_gpu.append(torch.tensor([[3.0]]))
|
||||
ready1.last_audio_patch_gpu = torch.tensor([[4.0]])
|
||||
|
||||
outputs = talker._collect_audio_batch([ready0, pending, ready1])
|
||||
|
||||
assert list(outputs) == ["ready-0", "pending", "ready-1"]
|
||||
torch.testing.assert_close(outputs["ready-0"], torch.tensor([10.0, 11.0, 12.0, 13.0]))
|
||||
assert outputs["pending"] is None
|
||||
torch.testing.assert_close(outputs["ready-1"], torch.tensor([20.0, 21.0, 22.0, 23.0]))
|
||||
assert len(seen_feats) == 1
|
||||
torch.testing.assert_close(seen_feats[0].reshape(2, -1), torch.tensor([[1.0, 2.0], [3.0, 4.0]]))
|
||||
assert pending.pending_vae_latents_gpu
|
||||
assert ready0.pending_vae_latents_gpu == []
|
||||
assert ready1.pending_vae_latents_gpu == []
|
||||
|
||||
def test_audio_emit_every_keeps_private_chunk_when_vae_output_storage_is_reused(self) -> None:
|
||||
_, RState, _ = _voxcpm2_talker_mod()
|
||||
talker = _make_bare_talker()
|
||||
talker._audio_emit_every = 2
|
||||
talker._enable_vae_cuda_graph = True
|
||||
talker._feat_dim = 1
|
||||
talker._n_decode_pad_frames = 1
|
||||
talker._device = torch.device("cpu")
|
||||
talker._tts = _FakeAudioTTS()
|
||||
talker._perf = _NoopPerf()
|
||||
|
||||
shared_output = torch.empty(4)
|
||||
values = [torch.tensor([10.0, 11.0, -1.0, -1.0]), torch.tensor([99.0, 99.0, 20.0, 21.0])]
|
||||
|
||||
def run_vae_decode(feat: torch.Tensor) -> torch.Tensor:
|
||||
shared_output.copy_(values.pop(0))
|
||||
return shared_output
|
||||
|
||||
talker._run_vae_decode = run_vae_decode
|
||||
state = RState(request_id="req")
|
||||
|
||||
state.last_audio_patch_gpu = torch.tensor([[1.0]])
|
||||
assert talker._collect_audio(state) is None
|
||||
|
||||
state.last_audio_patch_gpu = torch.tensor([[2.0]])
|
||||
audio = talker._collect_audio(state)
|
||||
|
||||
torch.testing.assert_close(audio, torch.tensor([10.0, 11.0, 20.0, 21.0]))
|
||||
|
||||
def test_audio_emit_every_keeps_private_chunk_when_compiled_vae_reuses_storage(self) -> None:
|
||||
_, RState, _ = _voxcpm2_talker_mod()
|
||||
talker = _make_bare_talker()
|
||||
talker._audio_emit_every = 2
|
||||
talker._feat_dim = 1
|
||||
talker._n_decode_pad_frames = 1
|
||||
talker._device = torch.device("cpu")
|
||||
talker._tts = _FakeAudioTTS()
|
||||
talker._tts.audio_vae._compiled = True
|
||||
talker._perf = _NoopPerf()
|
||||
|
||||
shared_output = torch.empty(4)
|
||||
values = [torch.tensor([10.0, 11.0, -1.0, -1.0]), torch.tensor([99.0, 99.0, 20.0, 21.0])]
|
||||
|
||||
def run_vae_decode(feat: torch.Tensor) -> torch.Tensor:
|
||||
shared_output.copy_(values.pop(0))
|
||||
return shared_output
|
||||
|
||||
talker._run_vae_decode = run_vae_decode
|
||||
state = RState(request_id="req")
|
||||
|
||||
state.last_audio_patch_gpu = torch.tensor([[1.0]])
|
||||
assert talker._collect_audio(state) is None
|
||||
|
||||
state.last_audio_patch_gpu = torch.tensor([[2.0]])
|
||||
audio = talker._collect_audio(state)
|
||||
|
||||
torch.testing.assert_close(audio, torch.tensor([10.0, 11.0, 20.0, 21.0]))
|
||||
|
||||
def test_delayed_audio_copy_emits_previous_chunk_and_flushes_on_stop(self) -> None:
|
||||
_, RState, _ = _voxcpm2_talker_mod()
|
||||
talker = _make_bare_talker()
|
||||
talker._enable_delayed_audio_copy = True
|
||||
talker._audio_emit_every = 1
|
||||
talker._feat_dim = 1
|
||||
talker._n_decode_pad_frames = 1
|
||||
talker._device = torch.device("cpu")
|
||||
talker._tts = _FakeAudioTTS()
|
||||
talker._perf = _NoopPerf()
|
||||
|
||||
values = [
|
||||
torch.tensor([10.0, 11.0]),
|
||||
torch.tensor([99.0, 99.0, 20.0, 21.0]),
|
||||
torch.tensor([88.0, 88.0, 30.0, 31.0]),
|
||||
]
|
||||
|
||||
def run_vae_decode(feat: torch.Tensor) -> torch.Tensor:
|
||||
return values.pop(0)
|
||||
|
||||
talker._run_vae_decode = run_vae_decode
|
||||
state = RState(request_id="req")
|
||||
|
||||
state.last_audio_patch_gpu = torch.tensor([[1.0]])
|
||||
assert talker._collect_audio(state) is None
|
||||
assert len(state.pending_audio_copies) == 1
|
||||
|
||||
state.last_audio_patch_gpu = torch.tensor([[2.0]])
|
||||
audio = talker._collect_audio(state)
|
||||
torch.testing.assert_close(audio, torch.tensor([10.0, 11.0]))
|
||||
assert len(state.pending_audio_copies) == 1
|
||||
|
||||
state.precomputed_stop_logits = torch.tensor([[0.0, 1.0]])
|
||||
state.last_audio_patch_gpu = torch.tensor([[3.0]])
|
||||
audio = talker._collect_audio(state)
|
||||
torch.testing.assert_close(audio, torch.tensor([20.0, 21.0, 30.0, 31.0]))
|
||||
assert state.pending_audio_copies == []
|
||||
assert state.is_stopping is True
|
||||
|
||||
def test_delayed_audio_copy_flushes_on_forced_stop(self) -> None:
|
||||
_, RState, _ = _voxcpm2_talker_mod()
|
||||
talker = _make_bare_talker()
|
||||
talker._enable_delayed_audio_copy = True
|
||||
talker._audio_emit_every = 1
|
||||
talker._feat_dim = 1
|
||||
talker._n_decode_pad_frames = 1
|
||||
talker._device = torch.device("cpu")
|
||||
talker._tts = _FakeAudioTTS()
|
||||
talker._perf = _NoopPerf()
|
||||
|
||||
values = [
|
||||
torch.tensor([10.0, 11.0]),
|
||||
torch.tensor([99.0, 99.0, 20.0, 21.0]),
|
||||
]
|
||||
|
||||
def run_vae_decode(feat: torch.Tensor) -> torch.Tensor:
|
||||
return values.pop(0)
|
||||
|
||||
talker._run_vae_decode = run_vae_decode
|
||||
state = RState(request_id="req")
|
||||
|
||||
state.last_audio_patch_gpu = torch.tensor([[1.0]])
|
||||
assert talker._collect_audio(state) is None
|
||||
|
||||
state.is_stopping = True
|
||||
state.last_audio_patch_gpu = torch.tensor([[2.0]])
|
||||
audio = talker._collect_audio(state)
|
||||
|
||||
torch.testing.assert_close(audio, torch.tensor([10.0, 11.0, 20.0, 21.0]))
|
||||
assert state.pending_audio_copies == []
|
||||
|
||||
def test_make_omni_output_uses_sparse_req_ids_for_deferred_chunks(self) -> None:
|
||||
talker = _make_bare_talker()
|
||||
talker._audio_emit_every = 4
|
||||
talker._sample_rate = 16000
|
||||
talker._last_audio_output_req_ids = ["req-0", "req-1", "req-2"]
|
||||
talker._audio_queue = [
|
||||
("req-0", None),
|
||||
("req-1", torch.tensor([1.0, 2.0])),
|
||||
("req-2", None),
|
||||
]
|
||||
|
||||
out = talker.make_omni_output(torch.zeros(3, 1))
|
||||
|
||||
audio = out.multimodal_outputs["model_outputs"]
|
||||
sr = out.multimodal_outputs["sr"]
|
||||
assert out.multimodal_outputs["meta"]["req_id"] == ["req-1"]
|
||||
assert out.multimodal_outputs["meta"]["sparse_audio"] == ["1"]
|
||||
assert len(audio) == 1
|
||||
torch.testing.assert_close(audio[0], torch.tensor([1.0, 2.0]))
|
||||
assert int(sr[0]) == 16000
|
||||
|
||||
def test_make_omni_output_emits_empty_sparse_marker_for_deferred_chunks(self) -> None:
|
||||
talker = _make_bare_talker()
|
||||
talker._audio_emit_every = 4
|
||||
talker._sample_rate = 16000
|
||||
talker._audio_queue = [("req-0", None), ("req-1", None)]
|
||||
|
||||
out = talker.make_omni_output(torch.zeros(2, 1))
|
||||
|
||||
assert out.multimodal_outputs["model_outputs"] == []
|
||||
assert out.multimodal_outputs["sr"] == []
|
||||
assert out.multimodal_outputs["meta"]["req_id"] == []
|
||||
assert out.multimodal_outputs["meta"]["sparse_audio"] == ["1"]
|
||||
|
||||
def test_make_omni_output_uses_sparse_req_ids_for_delayed_audio_copy(self) -> None:
|
||||
talker = _make_bare_talker()
|
||||
talker._enable_delayed_audio_copy = True
|
||||
talker._sample_rate = 16000
|
||||
talker._audio_queue = [
|
||||
("req-0", None),
|
||||
("req-1", torch.tensor([1.0, 2.0])),
|
||||
("req-2", None),
|
||||
]
|
||||
|
||||
out = talker.make_omni_output(torch.zeros(3, 1))
|
||||
|
||||
assert out.multimodal_outputs["meta"]["req_id"] == ["req-1"]
|
||||
assert out.multimodal_outputs["meta"]["sparse_audio"] == ["1"]
|
||||
assert len(out.multimodal_outputs["model_outputs"]) == 1
|
||||
torch.testing.assert_close(out.multimodal_outputs["model_outputs"][0], torch.tensor([1.0, 2.0]))
|
||||
|
||||
def test_make_omni_output_uses_sparse_req_ids_for_deferred_vae_decode(self) -> None:
|
||||
talker = _make_bare_talker()
|
||||
talker._vae_decode_every = 2
|
||||
talker._sample_rate = 16000
|
||||
talker._audio_queue = [
|
||||
("req-0", None),
|
||||
("req-1", torch.tensor([1.0, 2.0])),
|
||||
("req-2", None),
|
||||
]
|
||||
|
||||
out = talker.make_omni_output(torch.zeros(3, 1))
|
||||
|
||||
assert out.multimodal_outputs["meta"]["req_id"] == ["req-1"]
|
||||
assert out.multimodal_outputs["meta"]["sparse_audio"] == ["1"]
|
||||
assert len(out.multimodal_outputs["model_outputs"]) == 1
|
||||
torch.testing.assert_close(out.multimodal_outputs["model_outputs"][0], torch.tensor([1.0, 2.0]))
|
||||
|
||||
def test_make_omni_output_coalesces_gpu_audio_to_cpu(self) -> None:
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA is required to exercise D2H coalescing")
|
||||
talker = _make_bare_talker()
|
||||
talker._coalesce_audio_d2h = True
|
||||
talker._sample_rate = 16000
|
||||
talker._audio_queue = [
|
||||
("req-0", torch.tensor([1.0, 2.0], device="cuda")),
|
||||
("req-1", torch.tensor([3.0], device="cuda")),
|
||||
]
|
||||
|
||||
out = talker.make_omni_output(torch.zeros(2, 1))
|
||||
|
||||
audio = out.multimodal_outputs["model_outputs"]
|
||||
assert len(audio) == 2
|
||||
assert all(not chunk.is_cuda for chunk in audio)
|
||||
torch.testing.assert_close(audio[0], torch.tensor([1.0, 2.0]))
|
||||
torch.testing.assert_close(audio[1], torch.tensor([3.0]))
|
||||
@@ -1638,9 +1638,6 @@ class TestBaseConfigInheritance:
|
||||
s0 = deploy.stages[0].default_sampling_params
|
||||
# CI overrides max_tokens
|
||||
assert s0["max_tokens"] == 150
|
||||
# Inherited from base
|
||||
assert s0["temperature"] == 0.4
|
||||
assert s0["seed"] == 42
|
||||
|
||||
def test_pure_inheritance_overlay(self, tmp_path):
|
||||
"""An overlay with only ``base_config`` inherits everything."""
|
||||
@@ -1960,5 +1957,3 @@ class TestSamplingConstraintsPrecedence:
|
||||
assert stages[0].yaml_extras["default_sampling_params"]["detokenize"] is True
|
||||
# Pipeline says stop_token_ids=[2150] for talker
|
||||
assert stages[1].yaml_extras["default_sampling_params"]["stop_token_ids"] == [2150]
|
||||
# Deploy temperature still flows through
|
||||
assert stages[0].yaml_extras["default_sampling_params"]["temperature"] == 0.4
|
||||
@@ -44,6 +44,14 @@ def test_resolve_pooler_payload_req_ids_downstream_stage_uses_filtered_requests(
|
||||
assert payload_req_ids == ["r2"]
|
||||
|
||||
|
||||
def test_sparse_mm_req_ids_requires_sparse_audio_marker():
|
||||
assert GPUARModelRunner._sparse_mm_req_ids({"meta": {"req_id": ["r1"]}}) is None
|
||||
assert GPUARModelRunner._sparse_mm_req_ids({"meta.req_id": ["r1"]}) is None
|
||||
|
||||
assert GPUARModelRunner._sparse_mm_req_ids({"meta": {"req_id": ["r1"], "sparse_audio": ["1"]}}) == ["r1"]
|
||||
assert GPUARModelRunner._sparse_mm_req_ids({"meta.req_id": ["r1"], "meta.sparse_audio": ["1"]}) == ["r1"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("query_start_loc_attr", ["method", "tensor_attr"])
|
||||
def test_sample_tokens_tail_only_prefix_cache_uses_staged_cpu_hidden_states(monkeypatch, query_start_loc_attr):
|
||||
runner = object.__new__(GPUARModelRunner)
|
||||
|
||||
@@ -1,16 +0,0 @@
|
||||
import numpy as np
|
||||
from vllm.assets.video import VideoAsset
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
|
||||
|
||||
def extract_video_audio(path: str = None, sampling_rate: int = 16000) -> np.ndarray:
|
||||
"""This function extracts the audio from a video file path and returns the audio as a numpy array.
|
||||
Args:
|
||||
path: The path to the video file.
|
||||
Returns:
|
||||
The audio as a numpy array.
|
||||
"""
|
||||
if not path:
|
||||
path = VideoAsset(name="baby_reading").video_path
|
||||
audio_signal, sr = load_audio(path, sr=sampling_rate)
|
||||
return audio_signal
|
||||
@@ -1,5 +1,6 @@
|
||||
import warnings
|
||||
from dataclasses import dataclass
|
||||
from collections import defaultdict
|
||||
from dataclasses import field, make_dataclass
|
||||
|
||||
import numpy as np
|
||||
from vllm.benchmarks.datasets import SampleRequest
|
||||
@@ -9,30 +10,172 @@ from vllm.tokenizers import TokenizerLike
|
||||
|
||||
from vllm_omni.metrics import definitions as defs
|
||||
|
||||
_PERCENTILE_ROWS_TYPE = list[tuple[float, float]] | None
|
||||
_FLOAT_LIST_TYPE = list[float]
|
||||
_INT_LIST_TYPE = list[int]
|
||||
|
||||
@dataclass
|
||||
class MultiModalsBenchmarkMetrics(BenchmarkMetrics):
|
||||
mean_audio_ttfp_ms: float = 0.0
|
||||
median_audio_ttfp_ms: float = 0.0
|
||||
std_audio_ttfp_ms: float = 0.0
|
||||
percentiles_audio_ttfp_ms: list[tuple[float, float]] = None
|
||||
total_audio_duration_s: float = 0.0
|
||||
total_audio_frames: int = 0
|
||||
audio_throughput: float = 0.0
|
||||
mean_audio_rtf: float = 0.0
|
||||
median_audio_rtf: float = 0.0
|
||||
std_audio_rtf: float = 0.0
|
||||
percentiles_audio_rtf: list[tuple[float, float]] = None
|
||||
mean_audio_duration_s: float = 0.0
|
||||
median_audio_duration_s: float = 0.0
|
||||
std_audio_duration_s: float = 0.0
|
||||
percentiles_audio_duration_s: list[tuple[float, float]] = None
|
||||
# Streaming-audio continuity (see vllm_omni/benchmarks/audio_continuity.py).
|
||||
mean_audio_underrun_s: float = 0.0
|
||||
median_audio_underrun_s: float = 0.0
|
||||
std_audio_underrun_s: float = 0.0
|
||||
percentiles_audio_underrun_s: list[tuple[float, float]] = None
|
||||
audio_continuity_ok_rate: float = 1.0
|
||||
_MULTIMODAL_BENCHMARK_FIELDS = [
|
||||
(defs.MEAN_AUDIO_TTFP_MS, float, field(default=0.0)),
|
||||
(defs.MEDIAN_AUDIO_TTFP_MS, float, field(default=0.0)),
|
||||
(defs.STD_AUDIO_TTFP_MS, float, field(default=0.0)),
|
||||
(defs.PERCENTILES_AUDIO_TTFP_MS, _PERCENTILE_ROWS_TYPE, field(default=None)),
|
||||
(defs.TOTAL_AUDIO_DURATION_S, float, field(default=0.0)),
|
||||
(defs.TOTAL_AUDIO_FRAMES, int, field(default=0)),
|
||||
(defs.AUDIO_THROUGHPUT, float, field(default=0.0)),
|
||||
(defs.MEAN_AUDIO_RTF, float, field(default=0.0)),
|
||||
(defs.MEDIAN_AUDIO_RTF, float, field(default=0.0)),
|
||||
(defs.STD_AUDIO_RTF, float, field(default=0.0)),
|
||||
(defs.PERCENTILES_AUDIO_RTF, _PERCENTILE_ROWS_TYPE, field(default=None)),
|
||||
(defs.MEAN_AUDIO_DURATION_S, float, field(default=0.0)),
|
||||
(defs.MEDIAN_AUDIO_DURATION_S, float, field(default=0.0)),
|
||||
(defs.STD_AUDIO_DURATION_S, float, field(default=0.0)),
|
||||
(defs.PERCENTILES_AUDIO_DURATION_S, _PERCENTILE_ROWS_TYPE, field(default=None)),
|
||||
(defs.MEAN_AUDIO_UNDERRUN_S, float, field(default=0.0)),
|
||||
(defs.MEDIAN_AUDIO_UNDERRUN_S, float, field(default=0.0)),
|
||||
(defs.STD_AUDIO_UNDERRUN_S, float, field(default=0.0)),
|
||||
(defs.PERCENTILES_AUDIO_UNDERRUN_S, _PERCENTILE_ROWS_TYPE, field(default=None)),
|
||||
(defs.AUDIO_CONTINUITY_OK_RATE, float, field(default=1.0)),
|
||||
(defs.TOTAL_IMAGES, int, field(default=0)),
|
||||
(defs.IMAGE_THROUGHPUT, float, field(default=0.0)),
|
||||
(defs.AVERAGE_PIXELS_PER_IMAGE, float, field(default=0.0)),
|
||||
(defs.MEAN_DENOISE_STEP_LATENCY_MS, float, field(default=0.0)),
|
||||
(defs.MEAN_IMAGE_GENERATION_MS, float, field(default=0.0)),
|
||||
(defs.MEDIAN_IMAGE_GENERATION_MS, float, field(default=0.0)),
|
||||
(defs.STD_IMAGE_GENERATION_MS, float, field(default=0.0)),
|
||||
(defs.PERCENTILES_IMAGE_GENERATION_MS, _PERCENTILE_ROWS_TYPE, field(default=None)),
|
||||
]
|
||||
|
||||
MultiModalsBenchmarkMetrics = make_dataclass(
|
||||
"MultiModalsBenchmarkMetrics",
|
||||
_MULTIMODAL_BENCHMARK_FIELDS,
|
||||
bases=(BenchmarkMetrics,),
|
||||
)
|
||||
|
||||
_STAGE_BENCHMARK_FIELDS = [
|
||||
("stage_id", int, field(default=0)),
|
||||
("stage_name", str, field(default="")),
|
||||
("final_output_type", str, field(default="")),
|
||||
(defs.TOTAL_OUTPUT, int, field(default=0)),
|
||||
(defs.TTFTS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
||||
(defs.TPOTS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
||||
(defs.ITLS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
||||
(defs.VLLM_TTFTS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
||||
(defs.VLLM_TPOTS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
||||
(defs.VLLM_ITLS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
||||
(defs.AUDIO_TTFPS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
||||
(defs.AUDIO_RTFS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
||||
(defs.AUDIO_DURATIONS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
||||
(defs.AUDIO_FRAMES, _INT_LIST_TYPE, field(default_factory=list)),
|
||||
(defs.MISSING_AUDIO_DURATION_COUNT, int, field(default=0)),
|
||||
(defs.MISSING_AUDIO_RTF_COUNT, int, field(default=0)),
|
||||
(defs.STAGE_GEN_TIMES_MS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
||||
(defs.POSTPROCESS_TIMES_MS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
||||
("output_unit_type", str, field(default="")),
|
||||
(defs.OUTPUT_UNIT_COUNT, int, field(default=0)),
|
||||
(defs.SERVING_TIME_TO_FIRST_OUTPUTS_MS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
||||
(defs.TIME_PER_OUTPUT_UNITS_MS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
||||
(defs.INTER_OUTPUT_LATENCIES_MS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
||||
]
|
||||
|
||||
StageBenchmarkMetrics = make_dataclass(
|
||||
"StageBenchmarkMetrics",
|
||||
_STAGE_BENCHMARK_FIELDS,
|
||||
namespace={"__doc__": "Aggregated metrics for one pipeline stage (for printing only)."},
|
||||
)
|
||||
|
||||
|
||||
def _percentile_rows_seconds(
|
||||
values: list[float], selected_percentiles: list[float], *, to_ms: bool
|
||||
) -> tuple[float, float, list[tuple[float, float]]]:
|
||||
arr = np.asarray(values, dtype=float)
|
||||
scale = 1000.0 if to_ms else 1.0
|
||||
mean_v = float(np.mean(arr)) * scale
|
||||
median_v = float(np.median(arr)) * scale
|
||||
rows = [(float(p), float(np.percentile(arr, p)) * scale) for p in selected_percentiles]
|
||||
return mean_v, median_v, rows
|
||||
|
||||
|
||||
def _p_label(p: float) -> str:
|
||||
pf = float(p)
|
||||
if abs(pf - round(pf)) < 1e-9:
|
||||
return str(int(round(pf)))
|
||||
return str(pf)
|
||||
|
||||
|
||||
_STREAMING_OUTPUT_UNIT_TYPES = frozenset(
|
||||
{
|
||||
"text",
|
||||
"stream",
|
||||
"audio",
|
||||
}
|
||||
)
|
||||
|
||||
_AGGREGATE_PERCENTILE_FIELD_NAMES = {
|
||||
defs.AUDIO_TTFP: (
|
||||
defs.MEAN_AUDIO_TTFP_MS,
|
||||
defs.MEDIAN_AUDIO_TTFP_MS,
|
||||
defs.PERCENTILES_AUDIO_TTFP_MS,
|
||||
),
|
||||
defs.AUDIO_RTF: (
|
||||
defs.MEAN_AUDIO_RTF,
|
||||
defs.MEDIAN_AUDIO_RTF,
|
||||
defs.PERCENTILES_AUDIO_RTF,
|
||||
),
|
||||
defs.AUDIO_DURATION: (
|
||||
defs.MEAN_AUDIO_DURATION_S,
|
||||
defs.MEDIAN_AUDIO_DURATION_S,
|
||||
defs.PERCENTILES_AUDIO_DURATION_S,
|
||||
),
|
||||
defs.AUDIO_UNDERRUN: (
|
||||
defs.MEAN_AUDIO_UNDERRUN_S,
|
||||
defs.MEDIAN_AUDIO_UNDERRUN_S,
|
||||
defs.PERCENTILES_AUDIO_UNDERRUN_S,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _wants_text_ttft(metrics: list[str]) -> bool:
|
||||
return "ttft" in metrics
|
||||
|
||||
|
||||
def _wants_text_tpot(metrics: list[str]) -> bool:
|
||||
return "tpot" in metrics or "tpop" in metrics
|
||||
|
||||
|
||||
def _wants_text_itl(metrics: list[str]) -> bool:
|
||||
return "itl" in metrics
|
||||
|
||||
|
||||
def _wants_stream_ttfc(metrics: list[str]) -> bool:
|
||||
return "ttfc" in metrics
|
||||
|
||||
|
||||
def _wants_stream_tpoc(metrics: list[str]) -> bool:
|
||||
return "tpoc" in metrics or "tpop" in metrics
|
||||
|
||||
|
||||
def _wants_stream_icl(metrics: list[str]) -> bool:
|
||||
return "icl" in metrics
|
||||
|
||||
|
||||
def _stage_modality_flags(
|
||||
final_output_type: str,
|
||||
output_unit_type: str,
|
||||
) -> tuple[bool, bool, bool, bool, bool]:
|
||||
is_text_stage = final_output_type == "text" or output_unit_type == "text"
|
||||
is_audio_stage = final_output_type == "audio" or output_unit_type == "audio"
|
||||
is_image_stage = final_output_type in {"image", "images"} or output_unit_type == "image"
|
||||
is_video_stage = final_output_type in {"video", "videos"} or output_unit_type == "video"
|
||||
is_internal_stream_stage = (
|
||||
output_unit_type in _STREAMING_OUTPUT_UNIT_TYPES and not is_text_stage and not is_audio_stage
|
||||
)
|
||||
return (
|
||||
is_text_stage,
|
||||
is_audio_stage,
|
||||
is_image_stage,
|
||||
is_video_stage,
|
||||
is_internal_stream_stage,
|
||||
)
|
||||
|
||||
|
||||
def print_metrics(
|
||||
@@ -43,6 +186,10 @@ def print_metrics(
|
||||
benchmark_duration,
|
||||
goodput_config_dict,
|
||||
metrics: MultiModalsBenchmarkMetrics,
|
||||
*,
|
||||
outputs: list[RequestFuncOutput] | None = None,
|
||||
selected_percentiles: list[float] | None = None,
|
||||
print_stage: bool = False,
|
||||
):
|
||||
print("{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="="))
|
||||
print("{:<40} {:<10}".format("Successful requests:", metrics.completed))
|
||||
@@ -61,7 +208,19 @@ def print_metrics(
|
||||
process_one_metric("e2el", metrics)
|
||||
print_text_metrics(task_type, selected_percentile_metrics, metrics)
|
||||
if task_type == TaskType.GENERATION:
|
||||
print_audio_metrics(selected_percentile_metrics, metrics)
|
||||
if _has_audio_output(metrics):
|
||||
print_audio_metrics(selected_percentile_metrics, metrics)
|
||||
if _has_image_output(metrics):
|
||||
print_image_metrics(selected_percentiles or [], metrics)
|
||||
if print_stage and outputs and selected_percentiles is not None:
|
||||
print("\n{s:{c}^{n}}".format(s=" Stage Benchmark Result ", n=50, c="="))
|
||||
for sm in _build_stage_metrics_from_outputs(outputs):
|
||||
print_stage_metrics(
|
||||
task_type,
|
||||
selected_percentile_metrics,
|
||||
selected_percentiles,
|
||||
sm,
|
||||
)
|
||||
print("=" * 50)
|
||||
|
||||
|
||||
@@ -79,22 +238,36 @@ def print_text_metrics(task_type, selected_percentile_metrics, metrics: MultiMod
|
||||
# No text tokens generated (e.g. pure TTS speech endpoint): per-token
|
||||
# latency metrics (ttft/tpot/itl) are undefined, so skip them.
|
||||
has_text_output = metrics.total_output > 0
|
||||
for metric in selected_percentile_metrics:
|
||||
if metric == "e2el":
|
||||
continue
|
||||
if not metric.startswith("audio") and has_text_output:
|
||||
process_one_metric(metric, metrics)
|
||||
if has_text_output:
|
||||
if _wants_text_ttft(selected_percentile_metrics):
|
||||
process_one_metric("ttft", metrics)
|
||||
if _wants_text_tpot(selected_percentile_metrics):
|
||||
process_one_metric("tpot", metrics)
|
||||
if _wants_text_itl(selected_percentile_metrics):
|
||||
process_one_metric("itl", metrics)
|
||||
|
||||
|
||||
def _has_audio_output(metrics: MultiModalsBenchmarkMetrics) -> bool:
|
||||
return bool(
|
||||
getattr(metrics, defs.TOTAL_AUDIO_DURATION_S, 0.0) > 0 or getattr(metrics, defs.TOTAL_AUDIO_FRAMES, 0) > 0
|
||||
)
|
||||
|
||||
|
||||
def _has_image_output(metrics: MultiModalsBenchmarkMetrics) -> bool:
|
||||
return int(getattr(metrics, defs.TOTAL_IMAGES, 0) or 0) > 0
|
||||
|
||||
|
||||
def print_audio_metrics(selected_percentile_metrics, metrics: MultiModalsBenchmarkMetrics):
|
||||
print("{s:{c}^{n}}".format(s=" Audio Result ", n=50, c="="))
|
||||
print("{:<40} {:<10.2f}".format("Total audio duration generated(s):", metrics.total_audio_duration_s))
|
||||
print("{:<40} {:<10}".format("Total audio frames generated:", metrics.total_audio_frames))
|
||||
print("{:<40} {:<10.2f}".format("Audio throughput(audio duration/s):", metrics.audio_throughput))
|
||||
print(
|
||||
"{:<40} {:<10.2f}".format("Total audio duration generated(s):", getattr(metrics, defs.TOTAL_AUDIO_DURATION_S))
|
||||
)
|
||||
print("{:<40} {:<10}".format("Total audio frames generated:", getattr(metrics, defs.TOTAL_AUDIO_FRAMES)))
|
||||
print("{:<40} {:<10.2f}".format("Audio throughput(audio duration/s):", getattr(metrics, defs.AUDIO_THROUGHPUT)))
|
||||
print(
|
||||
"{:<40} {:<10.2%}".format(
|
||||
"Streaming continuity OK rate:",
|
||||
metrics.audio_continuity_ok_rate,
|
||||
getattr(metrics, defs.AUDIO_CONTINUITY_OK_RATE),
|
||||
)
|
||||
)
|
||||
for metric in selected_percentile_metrics:
|
||||
@@ -102,6 +275,37 @@ def print_audio_metrics(selected_percentile_metrics, metrics: MultiModalsBenchma
|
||||
process_one_metric(metric, metrics)
|
||||
|
||||
|
||||
def print_image_metrics(selected_percentiles: list[float], metrics: MultiModalsBenchmarkMetrics):
|
||||
print("{s:{c}^{n}}".format(s=" Image Result ", n=50, c="="))
|
||||
print("{:<40} {:<10}".format("Total images generated:", getattr(metrics, defs.TOTAL_IMAGES)))
|
||||
print("{:<40} {:<10.2f}".format("Image throughput (img/s):", getattr(metrics, defs.IMAGE_THROUGHPUT)))
|
||||
print(
|
||||
"{:<40} {:<10.2f}".format(
|
||||
"Average pixels per image:",
|
||||
getattr(metrics, defs.AVERAGE_PIXELS_PER_IMAGE),
|
||||
)
|
||||
)
|
||||
if getattr(metrics, defs.MEAN_DENOISE_STEP_LATENCY_MS) > 0:
|
||||
print(
|
||||
"{:<40} {:<10.2f}".format(
|
||||
"Mean denoise step latency (ms):",
|
||||
getattr(metrics, defs.MEAN_DENOISE_STEP_LATENCY_MS),
|
||||
)
|
||||
)
|
||||
if getattr(metrics, defs.MEAN_IMAGE_GENERATION_MS) > 0:
|
||||
print("-----------------Image Generation-----------------")
|
||||
print("{:<40} {:<10.2f}".format("Mean IMAGE_GENERATION (ms):", getattr(metrics, defs.MEAN_IMAGE_GENERATION_MS)))
|
||||
print(
|
||||
"{:<40} {:<10.2f}".format(
|
||||
"Median IMAGE_GENERATION (ms):",
|
||||
getattr(metrics, defs.MEDIAN_IMAGE_GENERATION_MS),
|
||||
)
|
||||
)
|
||||
for p, value in getattr(metrics, defs.PERCENTILES_IMAGE_GENERATION_MS) or []:
|
||||
if p in selected_percentiles:
|
||||
print("{:<40} {:<10.2f}".format(f"P{_p_label(p)} IMAGE_GENERATION (ms):", value))
|
||||
|
||||
|
||||
def process_one_metric(
|
||||
metric_attribute_name: str,
|
||||
metrics: MultiModalsBenchmarkMetrics,
|
||||
@@ -131,15 +335,16 @@ def process_one_metric(
|
||||
elif is_audio_rtf:
|
||||
suffix = ""
|
||||
unit_suffix = ""
|
||||
mean_attr_name = f"mean_{metric_attribute_name}{suffix}"
|
||||
field_names = _AGGREGATE_PERCENTILE_FIELD_NAMES.get(metric_attribute_name)
|
||||
mean_attr_name = field_names[0] if field_names else f"mean_{metric_attribute_name}{suffix}"
|
||||
mean_value = getattr(metrics, mean_attr_name, 0.0)
|
||||
print(f"{f'Mean {metric_attribute_name.upper()}{unit_suffix}:':<40} {mean_value:<10.2f}")
|
||||
|
||||
median_attr_name = f"median_{metric_attribute_name}{suffix}"
|
||||
median_attr_name = field_names[1] if field_names else f"median_{metric_attribute_name}{suffix}"
|
||||
median_value = getattr(metrics, median_attr_name, 0.0)
|
||||
print(f"{f'Median {metric_attribute_name.upper()}{unit_suffix}:':<40} {median_value:<10.2f}")
|
||||
|
||||
percentiles_attr_name = f"percentiles_{metric_attribute_name}{suffix}"
|
||||
percentiles_attr_name = field_names[2] if field_names else f"percentiles_{metric_attribute_name}{suffix}"
|
||||
percentiles = getattr(metrics, percentiles_attr_name, [])
|
||||
|
||||
for percentile, value in percentiles:
|
||||
@@ -148,6 +353,356 @@ def process_one_metric(
|
||||
print(f"{label:<40} {value:<10.2f}")
|
||||
|
||||
|
||||
def _print_percentile_metric(
|
||||
header: str,
|
||||
label: str,
|
||||
values: list[float],
|
||||
selected_percentiles: list[float],
|
||||
*,
|
||||
values_are_ms: bool = False,
|
||||
to_ms: bool = True,
|
||||
) -> None:
|
||||
if not values:
|
||||
return
|
||||
values_s = [v / 1000.0 for v in values] if values_are_ms else values
|
||||
mean_v, med_v, pcts = _percentile_rows_seconds(values_s, selected_percentiles, to_ms=to_ms)
|
||||
unit_suffix = " (ms)" if to_ms else ""
|
||||
print("{s:{c}^{n}}".format(s=header, n=50, c="-"))
|
||||
print(f"{f'Mean {label}{unit_suffix}:':<40} {mean_v:<10.2f}")
|
||||
print(f"{f'Median {label}{unit_suffix}:':<40} {med_v:<10.2f}")
|
||||
for p, v in pcts:
|
||||
p_str = _p_label(p)
|
||||
print(f"{f'P{p_str} {label}{unit_suffix}:':<40} {v:<10.2f}")
|
||||
|
||||
|
||||
def _print_stage_timing(sm: StageBenchmarkMetrics, selected_percentiles: list[float]) -> None:
|
||||
_print_percentile_metric(
|
||||
"Stage Timing",
|
||||
"stage_gen_time",
|
||||
getattr(sm, defs.STAGE_GEN_TIMES_MS),
|
||||
selected_percentiles,
|
||||
values_are_ms=True,
|
||||
to_ms=True,
|
||||
)
|
||||
postprocess_times_ms = getattr(sm, defs.POSTPROCESS_TIMES_MS)
|
||||
if postprocess_times_ms and any(v > 0 for v in postprocess_times_ms):
|
||||
_print_percentile_metric(
|
||||
"Stage Postprocess",
|
||||
"postprocess",
|
||||
postprocess_times_ms,
|
||||
selected_percentiles,
|
||||
values_are_ms=True,
|
||||
to_ms=True,
|
||||
)
|
||||
|
||||
|
||||
def _print_text_stage_metrics(
|
||||
task_type: TaskType,
|
||||
selected_percentile_metrics: list[str],
|
||||
selected_percentiles: list[float],
|
||||
sm: StageBenchmarkMetrics,
|
||||
) -> None:
|
||||
print("{s:{c}^{n}}".format(s=" Text Result ", n=50, c="="))
|
||||
print("{:<40} {:<10}".format("Stage generated tokens:", getattr(sm, defs.TOTAL_OUTPUT)))
|
||||
|
||||
if task_type != TaskType.GENERATION:
|
||||
return
|
||||
if _wants_text_ttft(selected_percentile_metrics):
|
||||
_print_percentile_metric(
|
||||
"Serving Time to First Token",
|
||||
"Serving TTFT",
|
||||
getattr(sm, defs.TTFTS),
|
||||
selected_percentiles,
|
||||
)
|
||||
_print_percentile_metric("Time to First Token", "TTFT", getattr(sm, defs.VLLM_TTFTS), selected_percentiles)
|
||||
if _wants_text_tpot(selected_percentile_metrics):
|
||||
_print_percentile_metric(
|
||||
"Time per Output Token (excl. 1st token)",
|
||||
"TPOT",
|
||||
getattr(sm, defs.VLLM_TPOTS),
|
||||
selected_percentiles,
|
||||
)
|
||||
if _wants_text_itl(selected_percentile_metrics):
|
||||
_print_percentile_metric("Inter-token Latency", "ITL", getattr(sm, defs.VLLM_ITLS), selected_percentiles)
|
||||
|
||||
|
||||
def _print_audio_stage_metrics(
|
||||
selected_percentile_metrics: list[str],
|
||||
selected_percentiles: list[float],
|
||||
sm: StageBenchmarkMetrics,
|
||||
) -> None:
|
||||
total_duration = float(sum(getattr(sm, defs.AUDIO_DURATIONS)))
|
||||
total_frames = int(sum(getattr(sm, defs.AUDIO_FRAMES)))
|
||||
print("{s:{c}^{n}}".format(s=" Audio Result ", n=50, c="="))
|
||||
print("{:<40} {:<10.2f}".format("Stage audio duration generated(s):", total_duration))
|
||||
print("{:<40} {:<10}".format("Stage audio frames generated:", total_frames))
|
||||
if defs.AUDIO_TTFP in selected_percentile_metrics:
|
||||
_print_percentile_metric(
|
||||
"Serving Time to First Packet",
|
||||
"Serving AUDIO_TTFP",
|
||||
getattr(sm, defs.AUDIO_TTFPS),
|
||||
selected_percentiles,
|
||||
)
|
||||
if defs.AUDIO_RTF in selected_percentile_metrics:
|
||||
audio_rtfs = getattr(sm, defs.AUDIO_RTFS)
|
||||
if audio_rtfs:
|
||||
_print_percentile_metric("Real Time Factor", "AUDIO_RTF", audio_rtfs, selected_percentiles, to_ms=False)
|
||||
elif getattr(sm, defs.MISSING_AUDIO_RTF_COUNT) > 0:
|
||||
print("{:<40} {:<10}".format("AUDIO_RTF skipped:", "missing stage-local audio duration/sample rate"))
|
||||
if defs.AUDIO_DURATION in selected_percentile_metrics:
|
||||
audio_durations = getattr(sm, defs.AUDIO_DURATIONS)
|
||||
if audio_durations:
|
||||
_print_percentile_metric(
|
||||
"Audio Duration",
|
||||
"AUDIO_DURATION",
|
||||
audio_durations,
|
||||
selected_percentiles,
|
||||
to_ms=False,
|
||||
)
|
||||
elif getattr(sm, defs.MISSING_AUDIO_DURATION_COUNT) > 0:
|
||||
print("{:<40} {:<10}".format("AUDIO_DURATION skipped:", "missing stage-local audio duration/sample rate"))
|
||||
|
||||
|
||||
def _print_image_stage_metrics(
|
||||
selected_percentiles: list[float],
|
||||
sm: StageBenchmarkMetrics,
|
||||
) -> None:
|
||||
image_num = getattr(sm, defs.OUTPUT_UNIT_COUNT) if getattr(sm, "output_unit_type") == "image" else 0
|
||||
print("{s:{c}^{n}}".format(s=" Image Result ", n=50, c="="))
|
||||
print("{:<40} {:<10}".format("Total images generated:", image_num))
|
||||
_print_percentile_metric(
|
||||
"Image Generation",
|
||||
"IMAGE_GENERATION",
|
||||
getattr(sm, defs.STAGE_GEN_TIMES_MS),
|
||||
selected_percentiles,
|
||||
values_are_ms=True,
|
||||
to_ms=True,
|
||||
)
|
||||
|
||||
|
||||
def _print_internal_stream_stage_metrics(
|
||||
selected_percentile_metrics: list[str],
|
||||
selected_percentiles: list[float],
|
||||
sm: StageBenchmarkMetrics,
|
||||
) -> None:
|
||||
print("{s:{c}^{n}}".format(s=" Internal Stream Result ", n=50, c="="))
|
||||
serving_time_to_first_outputs_ms = getattr(sm, defs.SERVING_TIME_TO_FIRST_OUTPUTS_MS)
|
||||
if _wants_stream_ttfc(selected_percentile_metrics) and serving_time_to_first_outputs_ms:
|
||||
_print_percentile_metric(
|
||||
"Serving Time to First Chunk",
|
||||
"Serving TTFC",
|
||||
serving_time_to_first_outputs_ms,
|
||||
selected_percentiles,
|
||||
values_are_ms=True,
|
||||
to_ms=True,
|
||||
)
|
||||
time_per_output_units_ms = getattr(sm, defs.TIME_PER_OUTPUT_UNITS_MS)
|
||||
if _wants_stream_tpoc(selected_percentile_metrics) and time_per_output_units_ms:
|
||||
_print_percentile_metric(
|
||||
"Time per Output Chunk (excl. 1st chunk)",
|
||||
"TPOP",
|
||||
time_per_output_units_ms,
|
||||
selected_percentiles,
|
||||
values_are_ms=True,
|
||||
to_ms=True,
|
||||
)
|
||||
inter_output_latencies_ms = getattr(sm, defs.INTER_OUTPUT_LATENCIES_MS)
|
||||
if _wants_stream_icl(selected_percentile_metrics) and inter_output_latencies_ms:
|
||||
_print_percentile_metric(
|
||||
"Inter-chunk Latency",
|
||||
"ICL",
|
||||
inter_output_latencies_ms,
|
||||
selected_percentiles,
|
||||
values_are_ms=True,
|
||||
to_ms=True,
|
||||
)
|
||||
|
||||
|
||||
def print_stage_metrics(
|
||||
task_type: TaskType,
|
||||
selected_percentile_metrics: list[str],
|
||||
selected_percentiles: list[float],
|
||||
sm: StageBenchmarkMetrics,
|
||||
):
|
||||
title = f" Stage {getattr(sm, 'stage_id')} ({getattr(sm, 'stage_name')}) "
|
||||
(
|
||||
is_text_stage,
|
||||
is_audio_stage,
|
||||
is_image_stage,
|
||||
is_video_stage,
|
||||
is_internal_stream_stage,
|
||||
) = _stage_modality_flags(getattr(sm, "final_output_type"), getattr(sm, "output_unit_type"))
|
||||
|
||||
print("{s:{c}^{n}}".format(s=title, n=50, c="="))
|
||||
if is_image_stage:
|
||||
_print_image_stage_metrics(selected_percentiles, sm)
|
||||
return
|
||||
|
||||
if is_video_stage:
|
||||
return
|
||||
|
||||
_print_stage_timing(sm, selected_percentiles)
|
||||
if is_text_stage:
|
||||
_print_text_stage_metrics(task_type, selected_percentile_metrics, selected_percentiles, sm)
|
||||
elif is_audio_stage:
|
||||
_print_audio_stage_metrics(
|
||||
selected_percentile_metrics,
|
||||
selected_percentiles,
|
||||
sm,
|
||||
)
|
||||
elif is_internal_stream_stage:
|
||||
_print_internal_stream_stage_metrics(selected_percentile_metrics, selected_percentiles, sm)
|
||||
|
||||
|
||||
def _build_stage_metrics_from_outputs(
|
||||
outputs: list[RequestFuncOutput],
|
||||
) -> list[StageBenchmarkMetrics]:
|
||||
"""Aggregate per ``stage_id`` using ``stage_metrics`` snapshots from the client."""
|
||||
buckets: dict[str, list[tuple[RequestFuncOutput, dict]]] = defaultdict(list)
|
||||
for out in outputs:
|
||||
if not getattr(out, "success", False):
|
||||
continue
|
||||
smap = getattr(out, "stage_metrics", None) or {}
|
||||
if not isinstance(smap, dict) or not smap:
|
||||
continue
|
||||
for sid, info in smap.items():
|
||||
if isinstance(info, dict):
|
||||
buckets[str(sid)].append((out, info))
|
||||
|
||||
result: list[StageBenchmarkMetrics] = []
|
||||
for sid in sorted(buckets.keys(), key=lambda x: int(x)):
|
||||
rows = buckets[sid]
|
||||
stage_id_int = int(sid)
|
||||
stage_name = str((rows[0][1] or {}).get("stage_name") or f"stage_{stage_id_int}")
|
||||
final_output_type = str((rows[0][1] or {}).get("final_output_type") or "unknown")
|
||||
output_unit_types = [
|
||||
str((info or {}).get("output_unit_type") or "") for _, info in rows if (info or {}).get("output_unit_type")
|
||||
]
|
||||
output_unit_type = output_unit_types[0] if output_unit_types else "other"
|
||||
|
||||
is_text_stage, is_audio_stage, _, _, is_internal_stream_stage = _stage_modality_flags(
|
||||
final_output_type, output_unit_type
|
||||
)
|
||||
total_output = 0
|
||||
|
||||
ttfts: list[float] = []
|
||||
tpots: list[float] = []
|
||||
itls: list[float] = []
|
||||
vllm_ttfts: list[float] = []
|
||||
vllm_tpots: list[float] = []
|
||||
vllm_itls: list[float] = []
|
||||
stage_gen_times_ms = [float((info or {}).get(defs.STAGE_GEN_TIME_MS) or 0.0) for _, info in rows]
|
||||
postprocess_times_ms = [float((info or {}).get(defs.POSTPROCESS_TIME_MS) or 0.0) for _, info in rows]
|
||||
output_unit_count = sum(int((info or {}).get(defs.OUTPUT_UNIT_COUNT) or 0) for _, info in rows)
|
||||
inter_output_latencies_ms: list[float] = []
|
||||
serving_time_to_first_outputs_ms: list[float] = []
|
||||
time_per_output_units_ms: list[float] = []
|
||||
if is_text_stage or is_internal_stream_stage:
|
||||
serving_time_to_first_outputs_ms = [
|
||||
float((info or {}).get(defs.SERVING_TIME_TO_FIRST_OUTPUT_MS) or 0.0)
|
||||
for _, info in rows
|
||||
if (info or {}).get(defs.SERVING_TIME_TO_FIRST_OUTPUT_MS) is not None
|
||||
]
|
||||
time_per_output_units_ms = [
|
||||
float((info or {}).get(defs.TIME_PER_OUTPUT_UNIT_MS) or 0.0)
|
||||
for _, info in rows
|
||||
if int((info or {}).get(defs.OUTPUT_UNIT_COUNT) or 0) > 1
|
||||
and (info or {}).get(defs.TIME_PER_OUTPUT_UNIT_MS) is not None
|
||||
]
|
||||
for _, info in rows:
|
||||
values = (info or {}).get(defs.INTER_OUTPUT_LATENCIES_MS)
|
||||
if isinstance(values, list):
|
||||
inter_output_latencies_ms.extend(float(v or 0.0) for v in values)
|
||||
elif (info or {}).get(defs.INTER_OUTPUT_LATENCY_MS) is not None:
|
||||
inter_output_latencies_ms.append(float((info or {}).get(defs.INTER_OUTPUT_LATENCY_MS) or 0.0))
|
||||
|
||||
if is_text_stage:
|
||||
for _, info in rows:
|
||||
ttft_ms = float((info or {}).get(defs.SERVING_TIME_TO_FIRST_OUTPUT_MS) or 0.0)
|
||||
if ttft_ms > 0:
|
||||
ttfts.append(ttft_ms / 1000.0)
|
||||
tpot_ms = float((info or {}).get(defs.TIME_PER_OUTPUT_UNIT_MS) or 0.0)
|
||||
if int((info or {}).get(defs.OUTPUT_UNIT_COUNT) or 0) > 1 and tpot_ms > 0:
|
||||
tpots.append(tpot_ms / 1000.0)
|
||||
values = (info or {}).get(defs.INTER_OUTPUT_LATENCIES_MS)
|
||||
if isinstance(values, list):
|
||||
itls.extend(float(v or 0.0) / 1000.0 for v in values)
|
||||
elif (info or {}).get(defs.INTER_OUTPUT_LATENCY_MS) is not None:
|
||||
itls.append(float((info or {}).get(defs.INTER_OUTPUT_LATENCY_MS) or 0.0) / 1000.0)
|
||||
vllm_ttft_ms = float((info or {}).get(defs.VLLM_TTFT_MS) or 0.0)
|
||||
if vllm_ttft_ms > 0:
|
||||
vllm_ttfts.append(vllm_ttft_ms / 1000.0)
|
||||
vllm_tpot_ms = float((info or {}).get(defs.VLLM_TPOT_MS) or 0.0)
|
||||
if vllm_tpot_ms > 0:
|
||||
vllm_tpots.append(vllm_tpot_ms / 1000.0)
|
||||
vllm_values = (info or {}).get(defs.VLLM_ITLS_MS)
|
||||
if isinstance(vllm_values, list):
|
||||
vllm_itls.extend(float(v or 0.0) / 1000.0 for v in vllm_values)
|
||||
elif (info or {}).get(defs.VLLM_ITL_MS) is not None:
|
||||
vllm_itls.append(float((info or {}).get(defs.VLLM_ITL_MS) or 0.0) / 1000.0)
|
||||
total_output = sum(int((info or {}).get(defs.NUM_TOKENS_OUT) or 0) for _, info in rows)
|
||||
|
||||
audio_ttfps: list[float] = []
|
||||
audio_rtfs: list[float] = []
|
||||
audio_durations: list[float] = []
|
||||
audio_frames: list[int] = []
|
||||
missing_audio_duration_count = 0
|
||||
missing_audio_rtf_count = 0
|
||||
if is_audio_stage:
|
||||
for _, info in rows:
|
||||
audio_ttfp_ms = float((info or {}).get(defs.SERVING_TIME_TO_FIRST_OUTPUT_MS) or 0.0)
|
||||
if audio_ttfp_ms > 0:
|
||||
audio_ttfps.append(audio_ttfp_ms / 1000.0)
|
||||
|
||||
frame_count = int((info or {}).get(defs.AUDIO_FRAMES) or 0)
|
||||
if frame_count <= 0 and (info or {}).get("output_unit_type") == "audio":
|
||||
frame_count = int((info or {}).get(defs.OUTPUT_UNIT_COUNT) or 0)
|
||||
audio_frames.append(frame_count)
|
||||
|
||||
duration_s = float((info or {}).get(f"{defs.AUDIO_DURATION}_s") or 0.0)
|
||||
if duration_s > 0:
|
||||
audio_durations.append(duration_s)
|
||||
else:
|
||||
missing_audio_duration_count += 1
|
||||
|
||||
rtf = float((info or {}).get(defs.AUDIO_RTF) or 0.0)
|
||||
if rtf > 0:
|
||||
audio_rtfs.append(rtf)
|
||||
else:
|
||||
missing_audio_rtf_count += 1
|
||||
|
||||
result.append(
|
||||
StageBenchmarkMetrics(
|
||||
**{
|
||||
"stage_id": stage_id_int,
|
||||
"stage_name": stage_name,
|
||||
"final_output_type": final_output_type,
|
||||
defs.TOTAL_OUTPUT: total_output,
|
||||
defs.TTFTS: ttfts,
|
||||
defs.TPOTS: tpots,
|
||||
defs.ITLS: itls,
|
||||
defs.VLLM_TTFTS: vllm_ttfts,
|
||||
defs.VLLM_TPOTS: vllm_tpots,
|
||||
defs.VLLM_ITLS: vllm_itls,
|
||||
defs.AUDIO_TTFPS: audio_ttfps,
|
||||
defs.AUDIO_RTFS: audio_rtfs,
|
||||
defs.AUDIO_DURATIONS: audio_durations,
|
||||
defs.AUDIO_FRAMES: audio_frames,
|
||||
defs.MISSING_AUDIO_DURATION_COUNT: missing_audio_duration_count,
|
||||
defs.MISSING_AUDIO_RTF_COUNT: missing_audio_rtf_count,
|
||||
defs.STAGE_GEN_TIMES_MS: stage_gen_times_ms,
|
||||
defs.POSTPROCESS_TIMES_MS: postprocess_times_ms,
|
||||
"output_unit_type": output_unit_type,
|
||||
defs.OUTPUT_UNIT_COUNT: output_unit_count,
|
||||
defs.SERVING_TIME_TO_FIRST_OUTPUTS_MS: serving_time_to_first_outputs_ms,
|
||||
defs.TIME_PER_OUTPUT_UNITS_MS: time_per_output_units_ms,
|
||||
defs.INTER_OUTPUT_LATENCIES_MS: inter_output_latencies_ms,
|
||||
}
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def calculate_metrics(
|
||||
input_requests: list[SampleRequest],
|
||||
outputs: list[RequestFuncOutput],
|
||||
@@ -160,6 +715,7 @@ def calculate_metrics(
|
||||
max_concurrency,
|
||||
request_rate,
|
||||
benchmark_duration,
|
||||
print_stage: bool = False,
|
||||
) -> tuple[BenchmarkMetrics, list[int]]:
|
||||
"""Calculate the metrics for the benchmark.
|
||||
|
||||
@@ -187,6 +743,10 @@ def calculate_metrics(
|
||||
audio_rtfs: list[float] = []
|
||||
audio_duration: list[float] = []
|
||||
audio_frames: list[int] = []
|
||||
image_generation_times_ms: list[float] = []
|
||||
denoise_step_latencies_ms: list[float] = []
|
||||
total_images = 0
|
||||
total_image_pixels = 0
|
||||
audio_underruns: list[float] = []
|
||||
audio_continuity_ok: list[bool] = []
|
||||
input_audio_duration = 0.0
|
||||
@@ -222,6 +782,15 @@ def calculate_metrics(
|
||||
audio_rtfs.append(getattr(outputs[i], defs.AUDIO_RTF, 0.0))
|
||||
audio_duration.append(getattr(outputs[i], defs.AUDIO_DURATION, 0.0))
|
||||
audio_frames.append(getattr(outputs[i], defs.AUDIO_FRAMES, 0.0))
|
||||
image_count = int(getattr(outputs[i], defs.IMAGE_COUNT, 0) or 0)
|
||||
total_images += image_count
|
||||
image_generation_time_ms = float(getattr(outputs[i], defs.IMAGE_GENERATION_TIME_MS, 0.0) or 0.0)
|
||||
if image_generation_time_ms > 0:
|
||||
image_generation_times_ms.append(image_generation_time_ms)
|
||||
total_image_pixels += int(getattr(outputs[i], defs.IMAGE_PIXELS, 0) or 0)
|
||||
denoise_step_latency_ms = float(getattr(outputs[i], defs.DENOISE_STEP_LATENCY_MS, 0.0) or 0.0)
|
||||
if denoise_step_latency_ms > 0:
|
||||
denoise_step_latencies_ms.append(denoise_step_latency_ms)
|
||||
audio_underruns.append(getattr(outputs[i], f"{defs.AUDIO_UNDERRUN}_s", 0.0))
|
||||
audio_continuity_ok.append(bool(getattr(outputs[i], defs.AUDIO_CONTINUITY_OK, True)))
|
||||
e2els.append(outputs[i].latency)
|
||||
@@ -336,26 +905,6 @@ def calculate_metrics(
|
||||
std_ttft_ms=np.std(ttfts or 0) * 1000,
|
||||
median_ttft_ms=np.median(ttfts or 0) * 1000,
|
||||
percentiles_ttft_ms=[(p, np.percentile(ttfts or 0, p) * 1000) for p in selected_percentiles],
|
||||
mean_audio_ttfp_ms=np.mean(audio_ttfps or 0) * 1000,
|
||||
std_audio_ttfp_ms=np.std(audio_ttfps or 0) * 1000,
|
||||
median_audio_ttfp_ms=np.median(audio_ttfps or 0) * 1000,
|
||||
percentiles_audio_ttfp_ms=[(p, np.percentile(audio_ttfps or 0, p) * 1000) for p in selected_percentiles],
|
||||
mean_audio_duration_s=np.mean(audio_duration or 0),
|
||||
std_audio_duration_s=np.std(audio_duration or 0),
|
||||
median_audio_duration_s=np.median(audio_duration or 0),
|
||||
percentiles_audio_duration_s=[(p, np.percentile(audio_duration or 0, p)) for p in selected_percentiles],
|
||||
total_audio_duration_s=sum(audio_duration),
|
||||
total_audio_frames=sum(audio_frames),
|
||||
audio_throughput=sum(audio_duration) / dur_s,
|
||||
mean_audio_rtf=np.mean(audio_rtfs or 0),
|
||||
std_audio_rtf=np.std(audio_rtfs or 0),
|
||||
median_audio_rtf=np.median(audio_rtfs or 0),
|
||||
percentiles_audio_rtf=[(p, np.percentile(audio_rtfs or 0, p)) for p in selected_percentiles],
|
||||
mean_audio_underrun_s=np.mean(audio_underruns or 0),
|
||||
std_audio_underrun_s=np.std(audio_underruns or 0),
|
||||
median_audio_underrun_s=np.median(audio_underruns or 0),
|
||||
percentiles_audio_underrun_s=[(p, np.percentile(audio_underruns or 0, p)) for p in selected_percentiles],
|
||||
audio_continuity_ok_rate=(sum(audio_continuity_ok) / len(audio_continuity_ok)) if audio_continuity_ok else 1.0,
|
||||
mean_tpot_ms=np.mean(tpots or 0) * 1000,
|
||||
std_tpot_ms=np.std(tpots or 0) * 1000,
|
||||
median_tpot_ms=np.median(tpots or 0) * 1000,
|
||||
@@ -371,6 +920,46 @@ def calculate_metrics(
|
||||
max_output_tokens_per_s=max_output_tokens_per_s,
|
||||
max_concurrent_requests=max_concurrent_requests,
|
||||
rtfx=input_audio_duration / dur_s,
|
||||
**{
|
||||
defs.MEAN_AUDIO_TTFP_MS: np.mean(audio_ttfps or 0) * 1000,
|
||||
defs.STD_AUDIO_TTFP_MS: np.std(audio_ttfps or 0) * 1000,
|
||||
defs.MEDIAN_AUDIO_TTFP_MS: np.median(audio_ttfps or 0) * 1000,
|
||||
defs.PERCENTILES_AUDIO_TTFP_MS: [
|
||||
(p, np.percentile(audio_ttfps or 0, p) * 1000) for p in selected_percentiles
|
||||
],
|
||||
defs.MEAN_AUDIO_DURATION_S: np.mean(audio_duration or 0),
|
||||
defs.STD_AUDIO_DURATION_S: np.std(audio_duration or 0),
|
||||
defs.MEDIAN_AUDIO_DURATION_S: np.median(audio_duration or 0),
|
||||
defs.PERCENTILES_AUDIO_DURATION_S: [
|
||||
(p, np.percentile(audio_duration or 0, p)) for p in selected_percentiles
|
||||
],
|
||||
defs.TOTAL_AUDIO_DURATION_S: sum(audio_duration),
|
||||
defs.TOTAL_AUDIO_FRAMES: sum(audio_frames),
|
||||
defs.AUDIO_THROUGHPUT: sum(audio_duration) / dur_s,
|
||||
defs.MEAN_AUDIO_RTF: np.mean(audio_rtfs or 0),
|
||||
defs.STD_AUDIO_RTF: np.std(audio_rtfs or 0),
|
||||
defs.MEDIAN_AUDIO_RTF: np.median(audio_rtfs or 0),
|
||||
defs.PERCENTILES_AUDIO_RTF: [(p, np.percentile(audio_rtfs or 0, p)) for p in selected_percentiles],
|
||||
defs.TOTAL_IMAGES: total_images,
|
||||
defs.IMAGE_THROUGHPUT: total_images / dur_s,
|
||||
defs.AVERAGE_PIXELS_PER_IMAGE: (total_image_pixels / total_images) if total_images > 0 else 0.0,
|
||||
defs.MEAN_DENOISE_STEP_LATENCY_MS: np.mean(denoise_step_latencies_ms or 0),
|
||||
defs.MEAN_IMAGE_GENERATION_MS: np.mean(image_generation_times_ms or 0),
|
||||
defs.STD_IMAGE_GENERATION_MS: np.std(image_generation_times_ms or 0),
|
||||
defs.MEDIAN_IMAGE_GENERATION_MS: np.median(image_generation_times_ms or 0),
|
||||
defs.PERCENTILES_IMAGE_GENERATION_MS: [
|
||||
(p, np.percentile(image_generation_times_ms or 0, p)) for p in selected_percentiles
|
||||
],
|
||||
defs.MEAN_AUDIO_UNDERRUN_S: np.mean(audio_underruns or 0),
|
||||
defs.STD_AUDIO_UNDERRUN_S: np.std(audio_underruns or 0),
|
||||
defs.MEDIAN_AUDIO_UNDERRUN_S: np.median(audio_underruns or 0),
|
||||
defs.PERCENTILES_AUDIO_UNDERRUN_S: [
|
||||
(p, np.percentile(audio_underruns or 0, p)) for p in selected_percentiles
|
||||
],
|
||||
defs.AUDIO_CONTINUITY_OK_RATE: (
|
||||
(sum(audio_continuity_ok) / len(audio_continuity_ok)) if audio_continuity_ok else 1.0
|
||||
),
|
||||
},
|
||||
)
|
||||
print_metrics(
|
||||
task_type,
|
||||
@@ -380,5 +969,8 @@ def calculate_metrics(
|
||||
benchmark_duration,
|
||||
goodput_config_dict,
|
||||
metrics,
|
||||
outputs=outputs,
|
||||
selected_percentiles=selected_percentiles,
|
||||
print_stage=print_stage,
|
||||
)
|
||||
return metrics, actual_output_lens
|
||||
@@ -2,6 +2,7 @@ import asyncio
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
import random
|
||||
import ssl
|
||||
@@ -12,7 +13,7 @@ import wave
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
from typing import Any, Literal
|
||||
|
||||
import aiohttp
|
||||
import pybase64 as base64
|
||||
@@ -48,9 +49,41 @@ from vllm_omni.benchmarks.data_modules.seed_tts_dataset import (
|
||||
)
|
||||
from vllm_omni.benchmarks.data_modules.sound_effect_dataset import SoundEffectDataset
|
||||
from vllm_omni.benchmarks.data_modules.ttsd_dataset import TTSDDataset
|
||||
from vllm_omni.metrics import definitions as defs
|
||||
|
||||
_AUDIO_CONTINUITY_THRESHOLD_ENV = "VLLM_OMNI_BENCH_AUDIO_CONTINUITY_THRESHOLD_S"
|
||||
_DEFAULT_AUDIO_CONTINUITY_THRESHOLD_S = 0.1
|
||||
RETURN_STAGE_METRICS_FIELD = "return_stage_metrics"
|
||||
_IMAGE_STAGE_METRICS_BACKENDS = frozenset({"openai-image-edits-omni"})
|
||||
_PRINT_STAGE = False
|
||||
|
||||
|
||||
def maybe_enable_stage_metrics(extra_body: dict[str, Any] | None, *, enabled: bool) -> dict[str, Any] | None:
|
||||
"""Return extra_body with stage-metric opt-in when benchmark metrics need it."""
|
||||
if not enabled:
|
||||
return extra_body
|
||||
body = dict(extra_body or {})
|
||||
body.setdefault(RETURN_STAGE_METRICS_FIELD, True)
|
||||
return body
|
||||
|
||||
|
||||
def should_request_stage_metrics(args: Any) -> bool:
|
||||
"""Whether this benchmark run needs server-side stage metrics in responses."""
|
||||
if getattr(args, "print_stage", False):
|
||||
return True
|
||||
|
||||
backend = getattr(args, "backend", None)
|
||||
if backend in _IMAGE_STAGE_METRICS_BACKENDS:
|
||||
return True
|
||||
|
||||
extra_body = getattr(args, "extra_body", None) or {}
|
||||
modalities = extra_body.get("modalities") if isinstance(extra_body, dict) else None
|
||||
return backend == "openai-chat-omni" and "image" in (modalities or [])
|
||||
|
||||
|
||||
def set_print_stage(enabled: bool) -> None:
|
||||
"""Set whether this benchmark run prints the stage benchmark section."""
|
||||
global _PRINT_STAGE
|
||||
_PRINT_STAGE = bool(enabled)
|
||||
|
||||
|
||||
def _audio_continuity_threshold_s() -> float:
|
||||
@@ -62,7 +95,7 @@ def _audio_continuity_threshold_s() -> float:
|
||||
"""
|
||||
raw = os.environ.get(_AUDIO_CONTINUITY_THRESHOLD_ENV)
|
||||
if not raw:
|
||||
return _DEFAULT_AUDIO_CONTINUITY_THRESHOLD_S
|
||||
return defs.AUDIO_CONTINUITY_DEFAULT_THRESHOLD_S
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError:
|
||||
@@ -70,9 +103,9 @@ def _audio_continuity_threshold_s() -> float:
|
||||
"Invalid %s=%r; using default %.3fs",
|
||||
_AUDIO_CONTINUITY_THRESHOLD_ENV,
|
||||
raw,
|
||||
_DEFAULT_AUDIO_CONTINUITY_THRESHOLD_S,
|
||||
defs.AUDIO_CONTINUITY_DEFAULT_THRESHOLD_S,
|
||||
)
|
||||
return _DEFAULT_AUDIO_CONTINUITY_THRESHOLD_S
|
||||
return defs.AUDIO_CONTINUITY_DEFAULT_THRESHOLD_S
|
||||
return max(value, 0.0)
|
||||
|
||||
|
||||
@@ -352,6 +385,10 @@ class MixRequestFuncOutput(RequestFuncOutput):
|
||||
audio_duration: float = 0.0
|
||||
audio_frames: int = 0
|
||||
audio_rtf: float = 0.0
|
||||
image_count: int = 0
|
||||
image_generation_time_ms: float = 0.0
|
||||
image_pixels: int = 0
|
||||
denoise_step_latency_ms: float = 0.0
|
||||
text_latency: float = 0.0
|
||||
#: Worst-case streaming-audio underrun (wall-clock seconds the player
|
||||
#: would have been starved). Populated by the audio-speech backend; ``0.0``
|
||||
@@ -366,6 +403,200 @@ class MixRequestFuncOutput(RequestFuncOutput):
|
||||
#: Raw PCM s16le mono at 24 kHz for Seed-TTS WER: from ``/v1/audio/speech`` stream or
|
||||
#: resampled export after ``openai-chat-omni`` audio deltas.
|
||||
tts_output_pcm_bytes: bytes | None = None
|
||||
#: Per-stage snapshot from orchestrator ``metrics["stage_metrics"]`` (merged across SSE chunks).
|
||||
stage_metrics: dict[str, dict] | None = None
|
||||
stage_id: int | None = None
|
||||
final_output_type: str | None = None
|
||||
|
||||
|
||||
_IMAGE_EDITS_EXTRA_BODY_FORM_FIELDS = (
|
||||
"negative_prompt",
|
||||
"num_inference_steps",
|
||||
"guidance_scale",
|
||||
"strength",
|
||||
"true_cfg_scale",
|
||||
"seed",
|
||||
"generator_device",
|
||||
"lora",
|
||||
"layers",
|
||||
"resolution",
|
||||
"bot_task",
|
||||
"sys_type",
|
||||
"system_prompt",
|
||||
RETURN_STAGE_METRICS_FIELD,
|
||||
)
|
||||
|
||||
|
||||
def _guess_mime_type(path: str) -> str:
|
||||
mime, _ = mimetypes.guess_type(path)
|
||||
return mime or "application/octet-stream"
|
||||
|
||||
|
||||
def _iter_image_edit_inputs(value: Any) -> Iterable[Any]:
|
||||
"""Yield image references from benchmark multimodal content."""
|
||||
if value is None:
|
||||
return
|
||||
if isinstance(value, list):
|
||||
for item in value:
|
||||
yield from _iter_image_edit_inputs(item)
|
||||
return
|
||||
if not isinstance(value, dict):
|
||||
yield value
|
||||
return
|
||||
|
||||
content_type = value.get("type")
|
||||
if content_type == "image_url":
|
||||
image_url = value.get("image_url")
|
||||
if isinstance(image_url, dict):
|
||||
url = image_url.get("url")
|
||||
if url:
|
||||
yield url
|
||||
elif image_url:
|
||||
yield image_url
|
||||
return
|
||||
|
||||
for key in ("image", "images"):
|
||||
if key in value:
|
||||
yield from _iter_image_edit_inputs(value[key])
|
||||
|
||||
|
||||
def _add_image_edit_input_to_form(form: aiohttp.FormData, image_input: Any) -> None:
|
||||
if isinstance(image_input, dict) and "bytes" in image_input:
|
||||
form.add_field(
|
||||
"image",
|
||||
image_input["bytes"],
|
||||
filename="benchmark.png",
|
||||
content_type="image/png",
|
||||
)
|
||||
return
|
||||
|
||||
if isinstance(image_input, str):
|
||||
if image_input.startswith(("data:image", "http://", "https://")):
|
||||
form.add_field("url", image_input)
|
||||
return
|
||||
local_path = image_input.removeprefix("file://")
|
||||
if os.path.exists(local_path):
|
||||
with open(local_path, "rb") as f:
|
||||
image_bytes = f.read()
|
||||
form.add_field(
|
||||
"image",
|
||||
image_bytes,
|
||||
filename=os.path.basename(local_path),
|
||||
content_type=_guess_mime_type(local_path),
|
||||
)
|
||||
return
|
||||
|
||||
raise ValueError(f"Unsupported image edit input: {type(image_input).__name__}")
|
||||
|
||||
|
||||
def _add_image_edit_extra_body_to_form(form: aiohttp.FormData, extra_body: dict[str, Any]) -> None:
|
||||
for key in _IMAGE_EDITS_EXTRA_BODY_FORM_FIELDS:
|
||||
value = extra_body.get(key)
|
||||
if value is None:
|
||||
continue
|
||||
if isinstance(value, (dict, list)):
|
||||
form.add_field(key, json.dumps(value))
|
||||
else:
|
||||
form.add_field(key, str(value))
|
||||
|
||||
|
||||
def _coerce_positive_int(value: Any) -> int | None:
|
||||
try:
|
||||
coerced = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return coerced if coerced > 0 else None
|
||||
|
||||
|
||||
def _extract_output_tokens_from_metrics(metrics: dict[str, Any]) -> int | None:
|
||||
top_level_tokens = _coerce_positive_int(metrics.get(defs.NUM_TOKENS_OUT))
|
||||
if top_level_tokens is not None:
|
||||
return top_level_tokens
|
||||
|
||||
stage_snapshot = metrics.get("stage_metrics")
|
||||
if not isinstance(stage_snapshot, dict):
|
||||
return None
|
||||
|
||||
fallback_tokens: list[int] = []
|
||||
for info in stage_snapshot.values():
|
||||
if not isinstance(info, dict):
|
||||
continue
|
||||
num_tokens_out = _coerce_positive_int(info.get(defs.NUM_TOKENS_OUT))
|
||||
if num_tokens_out is None:
|
||||
continue
|
||||
if info.get("final_output_type") == "text" or info.get("output_unit_type") == "token":
|
||||
return num_tokens_out
|
||||
fallback_tokens.append(num_tokens_out)
|
||||
return max(fallback_tokens, default=None)
|
||||
|
||||
|
||||
def _update_output_stage_metrics_from_payload(
|
||||
output: MixRequestFuncOutput,
|
||||
data: dict[str, Any],
|
||||
*,
|
||||
update_output_tokens: bool = True,
|
||||
) -> None:
|
||||
metrics = data.get("metrics")
|
||||
if not isinstance(metrics, dict):
|
||||
return
|
||||
if update_output_tokens:
|
||||
if (num_tokens_out := _extract_output_tokens_from_metrics(metrics)) is not None:
|
||||
output.output_tokens = num_tokens_out
|
||||
if isinstance(sid := metrics.get("stage_id"), int):
|
||||
output.stage_id = sid
|
||||
if isinstance(final_output_type := metrics.get("final_output_type"), str):
|
||||
output.final_output_type = final_output_type
|
||||
stage_snapshot = metrics.get("stage_metrics")
|
||||
if isinstance(stage_snapshot, dict):
|
||||
if output.stage_metrics is None:
|
||||
output.stage_metrics = {}
|
||||
output.stage_metrics.update(stage_snapshot)
|
||||
|
||||
|
||||
def _image_metrics_from_stage_metrics(metrics: dict[str, Any] | None) -> tuple[int, float, int, float]:
|
||||
if not isinstance(metrics, dict):
|
||||
return 0, 0.0, 0, 0.0
|
||||
stage_snapshot = metrics.get("stage_metrics")
|
||||
if not isinstance(stage_snapshot, dict):
|
||||
return 0, 0.0, 0, 0.0
|
||||
image_count = 0
|
||||
image_generation_ms = 0.0
|
||||
image_pixels = 0
|
||||
denoise_step_latency_ms = 0.0
|
||||
for info in stage_snapshot.values():
|
||||
if not isinstance(info, dict):
|
||||
continue
|
||||
final_output_type = info.get("final_output_type")
|
||||
output_unit_type = info.get("output_unit_type")
|
||||
if final_output_type not in {"image", "images"} and output_unit_type != "image":
|
||||
continue
|
||||
image_count += int(info.get(defs.OUTPUT_UNIT_COUNT) or 0)
|
||||
image_generation_ms += float(info.get(defs.STAGE_GEN_TIME_MS) or 0.0)
|
||||
image_pixels += int(info.get(defs.IMAGE_PIXELS) or 0)
|
||||
denoise_step_latency_ms = max(
|
||||
denoise_step_latency_ms,
|
||||
float(info.get(defs.DENOISE_STEP_LATENCY_MS) or 0.0),
|
||||
)
|
||||
return image_count, image_generation_ms, image_pixels, denoise_step_latency_ms
|
||||
|
||||
|
||||
def _image_generation_ms_from_content(content: Any) -> float:
|
||||
if not isinstance(content, list):
|
||||
return 0.0
|
||||
for item in content:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
stage_durations = item.get("stage_durations")
|
||||
if not isinstance(stage_durations, dict):
|
||||
continue
|
||||
gen_values = [
|
||||
float(value)
|
||||
for key, value in stage_durations.items()
|
||||
if str(key).endswith("_gen_ms") and isinstance(value, (int, float))
|
||||
]
|
||||
if gen_values:
|
||||
return max(gen_values)
|
||||
return 0.0
|
||||
|
||||
|
||||
async def async_request_openai_chat_omni_completions(
|
||||
@@ -448,6 +679,13 @@ async def async_request_openai_chat_omni_completions(
|
||||
output.output_tokens = 0
|
||||
output.error = ""
|
||||
output.success = False
|
||||
output.stage_metrics = {}
|
||||
output.stage_id = None
|
||||
output.final_output_type = None
|
||||
output.image_count = 0
|
||||
output.image_generation_time_ms = 0.0
|
||||
output.image_pixels = 0
|
||||
output.denoise_step_latency_ms = 0.0
|
||||
try:
|
||||
async with session.post(url=api_url, json=payload, headers=headers) as response:
|
||||
if response.status == 200:
|
||||
@@ -519,9 +757,27 @@ async def async_request_openai_chat_omni_completions(
|
||||
logger.warning("Failed to parse wav audio chunk: %s", ex)
|
||||
else:
|
||||
audio_bytes_buffer.extend(audio_bytes)
|
||||
elif modality == "image":
|
||||
output.image_count += 1
|
||||
content_image_ms = _image_generation_ms_from_content(content)
|
||||
if content_image_ms > 0:
|
||||
output.image_generation_time_ms += content_image_ms
|
||||
|
||||
if metrics := data.get("metrics"):
|
||||
output.output_tokens = metrics.get("num_tokens_out", 0)
|
||||
_update_output_stage_metrics_from_payload(output, data)
|
||||
(
|
||||
metrics_image_count,
|
||||
metrics_image_ms,
|
||||
metrics_image_pixels,
|
||||
metrics_denoise_step_ms,
|
||||
) = _image_metrics_from_stage_metrics(data.get("metrics"))
|
||||
if metrics_image_count > output.image_count:
|
||||
output.image_count = metrics_image_count
|
||||
if metrics_image_ms > output.image_generation_time_ms:
|
||||
output.image_generation_time_ms = metrics_image_ms
|
||||
if metrics_image_pixels > output.image_pixels:
|
||||
output.image_pixels = metrics_image_pixels
|
||||
if metrics_denoise_step_ms > output.denoise_step_latency_ms:
|
||||
output.denoise_step_latency_ms = metrics_denoise_step_ms
|
||||
|
||||
if usage := data.get("usage"):
|
||||
if (pt := usage.get("prompt_tokens")) is not None:
|
||||
@@ -610,6 +866,138 @@ async def async_request_openai_chat_omni_completions(
|
||||
return output
|
||||
|
||||
|
||||
async def async_request_openai_image_edits_omni(
|
||||
request_func_input: RequestFuncInput,
|
||||
session: aiohttp.ClientSession,
|
||||
pbar: tqdm | None = None,
|
||||
) -> MixRequestFuncOutput:
|
||||
"""Streaming request to /v1/images/edits for multi-stage image-edit benchmarks."""
|
||||
api_url = request_func_input.api_url
|
||||
_validate_api_url(api_url, "OpenAI Image Edits API", "images/edits")
|
||||
|
||||
extra_body = dict(request_func_input.extra_body or {})
|
||||
model = request_func_input.model_name if request_func_input.model_name else request_func_input.model
|
||||
output = MixRequestFuncOutput()
|
||||
output.prompt_len = request_func_input.prompt_len
|
||||
output.itl = []
|
||||
output.stage_metrics = {}
|
||||
output.output_tokens = 0
|
||||
output.image_count = 0
|
||||
output.image_generation_time_ms = 0.0
|
||||
output.image_pixels = 0
|
||||
output.denoise_step_latency_ms = 0.0
|
||||
|
||||
form = aiohttp.FormData()
|
||||
form.add_field("model", model)
|
||||
form.add_field("prompt", request_func_input.prompt)
|
||||
form.add_field("response_format", "b64_json")
|
||||
form.add_field("output_format", str(extra_body.get("output_format", "png")))
|
||||
form.add_field("stream", "true")
|
||||
|
||||
size = extra_body.get("size")
|
||||
if size is None:
|
||||
width, height = extra_body.get("width"), extra_body.get("height")
|
||||
size = f"{width}x{height}" if width is not None and height is not None else "auto"
|
||||
form.add_field("size", str(size))
|
||||
|
||||
_add_image_edit_extra_body_to_form(form, extra_body)
|
||||
|
||||
try:
|
||||
image_inputs = list(_iter_image_edit_inputs(request_func_input.multi_modal_content))
|
||||
if not image_inputs:
|
||||
raise ValueError(
|
||||
"openai-image-edits-omni requires image multimodal content. "
|
||||
"For synthetic inputs, use --dataset-name random-mm with an image bucket."
|
||||
)
|
||||
for image_input in image_inputs:
|
||||
_add_image_edit_input_to_form(form, image_input)
|
||||
except Exception:
|
||||
output.success = False
|
||||
output.error = traceback.format_exc()
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}",
|
||||
}
|
||||
_update_headers_common(headers, request_func_input)
|
||||
|
||||
st = time.perf_counter()
|
||||
output.start_time = st
|
||||
timestamp = st
|
||||
most_recent_text_timestamp = st
|
||||
generated_text = ""
|
||||
try:
|
||||
async with session.post(url=api_url, data=form, headers=headers) as response:
|
||||
if response.status == 200:
|
||||
handler = StreamedResponseHandler()
|
||||
async for chunk_bytes in response.content.iter_any():
|
||||
if not chunk_bytes:
|
||||
continue
|
||||
for message in handler.add_chunk(chunk_bytes):
|
||||
if type(message) is bytes:
|
||||
message = message.decode("utf-8")
|
||||
if message.startswith(":"):
|
||||
continue
|
||||
chunk = message.removeprefix("data: ")
|
||||
if chunk == "[DONE]":
|
||||
continue
|
||||
|
||||
timestamp = time.perf_counter()
|
||||
data = json.loads(chunk)
|
||||
_update_output_stage_metrics_from_payload(
|
||||
output,
|
||||
data,
|
||||
update_output_tokens=(data.get("type") == "ar_delta"),
|
||||
)
|
||||
|
||||
chunk_type = data.get("type")
|
||||
if chunk_type == "ar_delta":
|
||||
if output.ttft == 0.0:
|
||||
output.ttft = timestamp - st
|
||||
else:
|
||||
output.itl.append(timestamp - most_recent_text_timestamp)
|
||||
delta = data.get("delta") or ""
|
||||
generated_text += delta
|
||||
most_recent_text_timestamp = timestamp
|
||||
output.text_latency = timestamp - st
|
||||
elif chunk_type == "image":
|
||||
image_data = data.get("data")
|
||||
output.image_count += len(image_data) if isinstance(image_data, list) else 1
|
||||
content_image_ms = _image_generation_ms_from_content(data.get("data"))
|
||||
if content_image_ms > 0:
|
||||
output.image_generation_time_ms += content_image_ms
|
||||
(
|
||||
metrics_image_count,
|
||||
metrics_image_ms,
|
||||
metrics_image_pixels,
|
||||
metrics_denoise_step_ms,
|
||||
) = _image_metrics_from_stage_metrics(data.get("metrics"))
|
||||
if metrics_image_count > output.image_count:
|
||||
output.image_count = metrics_image_count
|
||||
if metrics_image_ms > output.image_generation_time_ms:
|
||||
output.image_generation_time_ms = metrics_image_ms
|
||||
if metrics_image_pixels > output.image_pixels:
|
||||
output.image_pixels = metrics_image_pixels
|
||||
if metrics_denoise_step_ms > output.denoise_step_latency_ms:
|
||||
output.denoise_step_latency_ms = metrics_denoise_step_ms
|
||||
output.latency = timestamp - st
|
||||
output.generated_text = generated_text
|
||||
output.success = True
|
||||
else:
|
||||
output.error = f"HTTP {response.status}: {await response.text()}"
|
||||
output.success = False
|
||||
except Exception:
|
||||
output.success = False
|
||||
output.error = traceback.format_exc()
|
||||
logger.error(f"ERROR: send image edit request failed, reason is: {output.error}")
|
||||
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
|
||||
|
||||
async def async_request_openai_audio_speech(
|
||||
request_func_input: RequestFuncInput, session: aiohttp.ClientSession, pbar: tqdm | None = None
|
||||
) -> MixRequestFuncOutput:
|
||||
@@ -727,6 +1115,10 @@ ASYNC_REQUEST_FUNCS["openai-audio-speech"] = async_request_openai_audio_speech
|
||||
if "openai-audio-speech" not in OPENAI_COMPATIBLE_BACKENDS:
|
||||
OPENAI_COMPATIBLE_BACKENDS.append("openai-audio-speech")
|
||||
|
||||
ASYNC_REQUEST_FUNCS["openai-image-edits-omni"] = async_request_openai_image_edits_omni
|
||||
if "openai-image-edits-omni" not in OPENAI_COMPATIBLE_BACKENDS:
|
||||
OPENAI_COMPATIBLE_BACKENDS.append("openai-image-edits-omni")
|
||||
|
||||
# Daily-Omni backend for audio-visual reasoning benchmark
|
||||
# Reuses openai-chat-omni completions for video+text understanding
|
||||
ASYNC_REQUEST_FUNCS["daily-omni"] = async_request_openai_chat_omni_completions
|
||||
@@ -739,7 +1131,10 @@ from vllm.benchmarks import serve
|
||||
from vllm.benchmarks.lib.ready_checker import wait_for_endpoint
|
||||
from vllm.benchmarks.serve import TaskType, calculate_metrics_for_embeddings, get_request
|
||||
|
||||
from vllm_omni.benchmarks.metrics.metrics import MultiModalsBenchmarkMetrics, calculate_metrics
|
||||
from vllm_omni.benchmarks.metrics.metrics import (
|
||||
MultiModalsBenchmarkMetrics,
|
||||
calculate_metrics,
|
||||
)
|
||||
|
||||
# ruff: noqa: E402
|
||||
|
||||
@@ -996,6 +1391,7 @@ async def benchmark(
|
||||
max_concurrency=max_concurrency,
|
||||
request_rate=request_rate,
|
||||
benchmark_duration=benchmark_duration,
|
||||
print_stage=_PRINT_STAGE,
|
||||
)
|
||||
else:
|
||||
metrics = calculate_metrics_for_embeddings(
|
||||
@@ -1016,9 +1412,13 @@ async def benchmark(
|
||||
"request_goodput": metrics.request_goodput if goodput_config_dict else None,
|
||||
"output_throughput": metrics.output_throughput,
|
||||
"total_token_throughput": metrics.total_token_throughput,
|
||||
"total_audio_duration_s": metrics.total_audio_duration_s,
|
||||
"total_audio_frames": metrics.total_audio_frames,
|
||||
"audio_throughput": metrics.audio_throughput,
|
||||
defs.TOTAL_AUDIO_DURATION_S: getattr(metrics, defs.TOTAL_AUDIO_DURATION_S),
|
||||
defs.TOTAL_AUDIO_FRAMES: getattr(metrics, defs.TOTAL_AUDIO_FRAMES),
|
||||
defs.AUDIO_THROUGHPUT: getattr(metrics, defs.AUDIO_THROUGHPUT),
|
||||
defs.TOTAL_IMAGES: getattr(metrics, defs.TOTAL_IMAGES),
|
||||
defs.IMAGE_THROUGHPUT: getattr(metrics, defs.IMAGE_THROUGHPUT),
|
||||
defs.AVERAGE_PIXELS_PER_IMAGE: getattr(metrics, defs.AVERAGE_PIXELS_PER_IMAGE),
|
||||
defs.MEAN_DENOISE_STEP_LATENCY_MS: getattr(metrics, defs.MEAN_DENOISE_STEP_LATENCY_MS),
|
||||
"input_lens": [output.prompt_len for output in outputs],
|
||||
"output_lens": actual_output_lens,
|
||||
"ttfts": [output.ttft for output in outputs],
|
||||
@@ -1074,21 +1474,34 @@ async def benchmark(
|
||||
if rps_change_events:
|
||||
result["rps_change_events"] = rps_change_events
|
||||
|
||||
result_percentile_metrics: list[str] = []
|
||||
if "ttft" in selected_percentile_metrics:
|
||||
result_percentile_metrics.append("ttft")
|
||||
if "tpot" in selected_percentile_metrics or "tpop" in selected_percentile_metrics:
|
||||
result_percentile_metrics.append("tpot")
|
||||
if "itl" in selected_percentile_metrics:
|
||||
result_percentile_metrics.append("itl")
|
||||
if "e2el" in selected_percentile_metrics:
|
||||
result_percentile_metrics.append("e2el")
|
||||
for metric in selected_percentile_metrics:
|
||||
if metric.startswith("audio") and metric not in result_percentile_metrics:
|
||||
result_percentile_metrics.append(metric)
|
||||
|
||||
def process_one_metric(
|
||||
# E.g., "ttft"
|
||||
metric_attribute_name: str,
|
||||
):
|
||||
# This function prints and adds statistics of the specified
|
||||
# metric.
|
||||
if metric_attribute_name not in selected_percentile_metrics:
|
||||
if metric_attribute_name not in result_percentile_metrics:
|
||||
return
|
||||
# No text tokens generated (e.g. pure TTS speech endpoint): per-token
|
||||
# latency metrics (ttft/tpot/itl) are undefined, so skip them.
|
||||
is_text_token_metric = not (metric_attribute_name == "e2el" or metric_attribute_name.startswith("audio"))
|
||||
if is_text_token_metric and getattr(metrics, "total_output", 0) == 0:
|
||||
return
|
||||
is_audio_rtf = metric_attribute_name == "audio_rtf"
|
||||
is_audio_duration_or_underrun = metric_attribute_name in ("audio_duration", "audio_underrun")
|
||||
is_audio_rtf = metric_attribute_name == defs.AUDIO_RTF
|
||||
is_audio_duration_or_underrun = metric_attribute_name in (defs.AUDIO_DURATION, defs.AUDIO_UNDERRUN)
|
||||
|
||||
suffix = "_ms"
|
||||
if is_audio_duration_or_underrun:
|
||||
@@ -1107,9 +1520,10 @@ async def benchmark(
|
||||
result[f"p{p_word}_{metric_attribute_name}{suffix}"] = value
|
||||
|
||||
if task_type == TaskType.GENERATION:
|
||||
for metric in selected_percentile_metrics:
|
||||
for metric in result_percentile_metrics:
|
||||
process_one_metric(metric)
|
||||
else:
|
||||
result_percentile_metrics.append("e2el")
|
||||
process_one_metric("e2el")
|
||||
|
||||
if profile:
|
||||
|
||||
@@ -9,6 +9,11 @@ from vllm.benchmarks.serve import main_async
|
||||
# This monkey-patches vllm.benchmarks.datasets.get_samples before it's used
|
||||
# Must be imported before any vllm.benchmarks module usage
|
||||
import vllm_omni.benchmarks.patch.patch # noqa: F401
|
||||
from vllm_omni.benchmarks.patch.patch import (
|
||||
maybe_enable_stage_metrics,
|
||||
set_print_stage,
|
||||
should_request_stage_metrics,
|
||||
)
|
||||
|
||||
|
||||
def main(args: argparse.Namespace) -> dict[str, Any]:
|
||||
@@ -18,4 +23,9 @@ def main(args: argparse.Namespace) -> dict[str, Any]:
|
||||
os.environ["SEED_TTS_WER_SAVE_ITEMS"] = "1"
|
||||
if getattr(args, "daily_omni_save_eval_items", False):
|
||||
os.environ["DAILY_OMNI_SAVE_EVAL_ITEMS"] = "1"
|
||||
set_print_stage(getattr(args, "print_stage", False))
|
||||
args.extra_body = maybe_enable_stage_metrics(
|
||||
getattr(args, "extra_body", None),
|
||||
enabled=should_request_stage_metrics(args),
|
||||
)
|
||||
return asyncio.run(main_async(args))
|
||||
@@ -17,11 +17,10 @@ Adding a new pipeline:
|
||||
``vllm_omni/.../pipeline.py``.
|
||||
2. Add one line to ``_OMNI_PIPELINES`` below.
|
||||
|
||||
Single-stage diffusion models continue to use the
|
||||
``_create_default_diffusion_stage_cfg`` fallback in
|
||||
``async_omni_engine.py`` — they don't need a registry entry. The empty
|
||||
``_DIFFUSION_PIPELINES`` placeholder previously here (#2915) was removed
|
||||
once #2987 (which would have populated it) was deferred.
|
||||
Plain single-stage diffusion models continue to use the
|
||||
``_create_default_diffusion_stage_cfg`` fallback in ``async_omni_engine.py``.
|
||||
The empty ``_DIFFUSION_PIPELINES`` placeholder previously here (#2915) was
|
||||
removed once #2987 (which would have populated it) was deferred.
|
||||
|
||||
``register_pipeline(config)`` in ``stage_config`` is still supported for
|
||||
out-of-tree plugins and tests that create pipelines at runtime; those override
|
||||
@@ -70,6 +69,10 @@ _OMNI_PIPELINES: dict[str, tuple[str, str]] = {
|
||||
"vllm_omni.model_executor.models.lance.pipeline",
|
||||
"LANCE_PIPELINE",
|
||||
),
|
||||
"dreamzero": (
|
||||
"vllm_omni.model_executor.models.dreamzero.pipeline",
|
||||
"DREAMZERO_PIPELINE",
|
||||
),
|
||||
"glm_image": (
|
||||
"vllm_omni.model_executor.models.glm_image.pipeline",
|
||||
"GLM_IMAGE_PIPELINE",
|
||||
@@ -146,4 +149,8 @@ _OMNI_PIPELINES: dict[str, tuple[str, str]] = {
|
||||
"vllm_omni.model_executor.models.higgs_audio_v2.pipeline",
|
||||
"HIGGS_AUDIO_V2_PIPELINE",
|
||||
),
|
||||
"dynin_omni": (
|
||||
"vllm_omni.model_executor.models.dynin_omni.pipeline",
|
||||
"DYNIN_OMNI_PIPELINE",
|
||||
),
|
||||
}
|
||||
@@ -1165,6 +1165,11 @@ class StageConfigFactory:
|
||||
|
||||
# --- New path: check pipeline registry by model_type first ---
|
||||
model_type, hf_config = cls._auto_detect_model_type(model, trust_remote_code=trust_remote_code)
|
||||
if model_type == "vla":
|
||||
from vllm_omni.diffusion.utils.hf_utils import _looks_like_dreamzero
|
||||
|
||||
if _looks_like_dreamzero(model):
|
||||
model_type = "dreamzero"
|
||||
if model_type and model_type in _PIPELINE_REGISTRY:
|
||||
return cls._create_from_registry(model_type, cli_overrides, deploy_config_path)
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ from vllm.v1.core.sched.async_scheduler import AsyncScheduler as AsyncVLLMSchedu
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.core.sched.scheduler import Scheduler as VLLMScheduler
|
||||
from vllm.v1.core.sched.utils import remove_all
|
||||
from vllm.v1.engine import EngineCoreEventType, EngineCoreOutput, EngineCoreOutputs
|
||||
from vllm.v1.engine import EngineCoreEventType, EngineCoreOutput, EngineCoreOutputs, FinishReason
|
||||
from vllm.v1.metrics.perf import PerfStats
|
||||
from vllm.v1.outputs import ModelRunnerOutput
|
||||
from vllm.v1.request import Request, RequestStatus, StreamingUpdate
|
||||
@@ -69,6 +69,8 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler):
|
||||
# alive. Stopped on the first scheduler step after extraction ack.
|
||||
self.pending_stop_after_extraction: set[str] = set()
|
||||
|
||||
self.finished_req_ids_dict = defaultdict(set)
|
||||
|
||||
# [Omni] Pre-parse KV transfer criteria
|
||||
self.kv_transfer_criteria = self._get_kv_transfer_criteria()
|
||||
|
||||
@@ -434,7 +436,7 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler):
|
||||
# Downstream async-chunk stages receive real payloads from the
|
||||
# connector. This update only resumes polling for the next segment.
|
||||
self.chunk_transfer_adapter.segment_finished_requests.discard(request.request_id)
|
||||
request.discard_latest_async_tokens = True
|
||||
request.async_tokens_to_discard = 1
|
||||
request.num_output_placeholders = 0
|
||||
request.spec_token_ids = []
|
||||
request._output_token_ids.clear()
|
||||
@@ -556,16 +558,24 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler):
|
||||
# outputs in this step.
|
||||
engine_core_outputs = {client_index: EngineCoreOutputs(outputs=outs) for client_index, outs in outputs.items()}
|
||||
|
||||
# FIXME: finished_req_ids_dict is unconditionally initialized as
|
||||
# defaultdict(set) in __init__ (not gated by include_finished_set).
|
||||
# This branch is therefore always eligible once any client_index is
|
||||
# populated; revisit when wiring streaming-only / upstream semantics.
|
||||
finished_req_ids = self.finished_req_ids_dict
|
||||
if finished_req_ids:
|
||||
# Include ids of requests that finished since last outputs
|
||||
# were sent.
|
||||
for client_index, finished_set in finished_req_ids.items():
|
||||
# Set finished request set in EngineCoreOutputs for this client.
|
||||
if (eco := engine_core_outputs.get(client_index)) is not None:
|
||||
eco.finished_requests = finished_set
|
||||
else:
|
||||
engine_core_outputs[client_index] = EngineCoreOutputs(finished_requests=finished_set)
|
||||
eco = engine_core_outputs.get(client_index)
|
||||
if eco is None:
|
||||
eco = EngineCoreOutputs()
|
||||
engine_core_outputs[client_index] = eco
|
||||
emitted = {o.request_id for o in eco.outputs}
|
||||
for req_id in finished_set:
|
||||
if req_id not in emitted:
|
||||
eco.outputs.append(EngineCoreOutput(req_id, [], finish_reason=FinishReason.ABORT))
|
||||
eco.finished_requests = finished_set
|
||||
finished_req_ids.clear()
|
||||
|
||||
if (stats := self.make_stats(spec_decoding_stats, kv_connector_stats, cudagraph_stats, perf_stats)) is not None:
|
||||
@@ -632,13 +642,12 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler):
|
||||
"""
|
||||
req_id = session.request_id
|
||||
self._new_prompt_len_snapshot[req_id] = len(update.prompt_token_ids)
|
||||
session.discard_latest_async_tokens = False
|
||||
outstanding_async_tokens = getattr(session, "num_output_placeholders", 0)
|
||||
if outstanding_async_tokens > 0:
|
||||
# Async scheduling may already have sampled the previous
|
||||
# segment's next token. Drop that late token instead of
|
||||
# appending it to the new streaming segment.
|
||||
session.discard_latest_async_tokens = True
|
||||
session.async_tokens_to_discard = 1
|
||||
session.num_computed_tokens -= session.num_output_placeholders
|
||||
session.num_output_placeholders = 0
|
||||
session.spec_token_ids = []
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
# DreamZero-DROID deploy: single diffusion stage.
|
||||
#
|
||||
# Topology is declared in vllm_omni/model_executor/models/dreamzero/pipeline.py.
|
||||
# This default uses one GPU with TP=1 and CFG parallel disabled.
|
||||
|
||||
pipeline: dreamzero
|
||||
async_chunk: false
|
||||
distributed_executor_backend: mp
|
||||
dtype: bfloat16
|
||||
|
||||
stages:
|
||||
- stage_id: 0
|
||||
devices: "0"
|
||||
max_num_seqs: 1
|
||||
enforce_eager: true
|
||||
model_class_name: DreamZeroPipeline
|
||||
model_config:
|
||||
default_robot_embodiment: roboarena
|
||||
policy_server_config:
|
||||
image_resolution: [180, 320]
|
||||
n_external_cameras: 2
|
||||
needs_wrist_camera: true
|
||||
needs_stereo_camera: false
|
||||
needs_session_id: true
|
||||
action_space: joint_position
|
||||
@@ -0,0 +1,25 @@
|
||||
# DreamZero-DROID deploy: TP=1, CFG parallel size=2.
|
||||
|
||||
pipeline: dreamzero
|
||||
async_chunk: false
|
||||
distributed_executor_backend: mp
|
||||
dtype: bfloat16
|
||||
|
||||
stages:
|
||||
- stage_id: 0
|
||||
devices: "0,1"
|
||||
max_num_seqs: 1
|
||||
enforce_eager: true
|
||||
model_class_name: DreamZeroPipeline
|
||||
parallel_config:
|
||||
tensor_parallel_size: 1
|
||||
cfg_parallel_size: 2
|
||||
model_config:
|
||||
default_robot_embodiment: roboarena
|
||||
policy_server_config:
|
||||
image_resolution: [180, 320]
|
||||
n_external_cameras: 2
|
||||
needs_wrist_camera: true
|
||||
needs_stereo_camera: false
|
||||
needs_session_id: true
|
||||
action_space: joint_position
|
||||
@@ -0,0 +1,38 @@
|
||||
# Dynin-Omni 3-stage deploy: token2text → token2image → token2audio.
|
||||
#
|
||||
# Topology (stages, processor functions, model_arch) is frozen in
|
||||
# vllm_omni/model_executor/models/dynin_omni/pipeline.py. This file carries
|
||||
# only the deploy-time knobs (devices, GPU memory, batched tokens).
|
||||
#
|
||||
# async_chunk=false: dynin hands the full token payload across stages through
|
||||
# the worker-connector data plane (``*_full_payload`` producers), not chunked
|
||||
# streaming. Pipeline-wide ``trust_remote_code`` is required by the custom
|
||||
# Dynin HF modeling code.
|
||||
async_chunk: false
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
|
||||
stages:
|
||||
- stage_id: 0
|
||||
devices: "0"
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.5
|
||||
max_num_batched_tokens: 32768
|
||||
enforce_eager: true
|
||||
async_scheduling: false
|
||||
|
||||
- stage_id: 1
|
||||
devices: "0"
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.1
|
||||
max_num_batched_tokens: 32768
|
||||
enforce_eager: true
|
||||
async_scheduling: false
|
||||
|
||||
- stage_id: 2
|
||||
devices: "0"
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.1
|
||||
max_num_batched_tokens: 32768
|
||||
enforce_eager: true
|
||||
async_scheduling: false
|
||||
@@ -1,81 +1,35 @@
|
||||
# stage config for running dynin_omni with a 3-stage architecture.
|
||||
# this config is intended for e2e smoke tests.
|
||||
# Dynin-Omni deploy for e2e smoke tests (3-stage, one GPU).
|
||||
#
|
||||
# Topology is frozen in models/dynin_omni/pipeline.py; ``pipeline: dynin_omni``
|
||||
# pins it explicitly so this CI YAML resolves regardless of model_type
|
||||
# auto-detection. Differs from the default deploy only in reduced GPU memory
|
||||
# and a smaller max_num_batched_tokens so the pipeline fits L4-class CI GPUs.
|
||||
pipeline: dynin_omni
|
||||
async_chunk: false
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
|
||||
stage_args:
|
||||
stages:
|
||||
- stage_id: 0
|
||||
stage_type: llm
|
||||
runtime:
|
||||
process: true
|
||||
devices: "0"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: token2text
|
||||
model_arch: DyninOmniForConditionalGeneration
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
engine_output_type: latent
|
||||
custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.dynin_omni.token2text_to_token2image_full_payload
|
||||
trust_remote_code: true
|
||||
gpu_memory_utilization: 0.5
|
||||
enforce_eager: true
|
||||
enable_prefix_caching: false
|
||||
async_scheduling: false
|
||||
max_num_batched_tokens: 4096
|
||||
is_comprehension: true
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
devices: "0"
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.5
|
||||
max_num_batched_tokens: 4096
|
||||
enforce_eager: true
|
||||
async_scheduling: false
|
||||
|
||||
- stage_id: 1
|
||||
stage_type: llm
|
||||
runtime:
|
||||
process: true
|
||||
devices: "0"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: token2image
|
||||
model_arch: DyninOmniForConditionalGeneration
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
engine_output_type: latent
|
||||
custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.dynin_omni.token2image_to_token2audio_full_payload
|
||||
trust_remote_code: true
|
||||
gpu_memory_utilization: 0.2
|
||||
enforce_eager: true
|
||||
enable_prefix_caching: false
|
||||
async_scheduling: false
|
||||
max_num_batched_tokens: 4096
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.dynin_omni.token2text_to_token2image
|
||||
final_output: true
|
||||
final_output_type: image
|
||||
devices: "0"
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.2
|
||||
max_num_batched_tokens: 4096
|
||||
enforce_eager: true
|
||||
async_scheduling: false
|
||||
|
||||
- stage_id: 2
|
||||
stage_type: llm
|
||||
runtime:
|
||||
process: true
|
||||
devices: "0"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: token2audio
|
||||
model_arch: DyninOmniForConditionalGeneration
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
engine_output_type: latent
|
||||
trust_remote_code: true
|
||||
gpu_memory_utilization: 0.2
|
||||
enforce_eager: true
|
||||
enable_prefix_caching: false
|
||||
async_scheduling: false
|
||||
max_num_batched_tokens: 4096
|
||||
engine_input_source: [1]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.dynin_omni.token2image_to_token2audio
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
|
||||
runtime:
|
||||
enabled: true
|
||||
edges:
|
||||
- from: 0
|
||||
to: 1
|
||||
- from: 1
|
||||
to: 2
|
||||
devices: "0"
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.2
|
||||
max_num_batched_tokens: 4096
|
||||
enforce_eager: true
|
||||
async_scheduling: false
|
||||
@@ -0,0 +1,65 @@
|
||||
# Dynin-Omni 3-stage deploy with cross-node connectors (Mooncake demo).
|
||||
#
|
||||
# Topology is frozen in models/dynin_omni/pipeline.py (``pipeline: dynin_omni``).
|
||||
# This variant wires a MooncakeConnector for each stage hand-off in place of
|
||||
# the default SharedMemoryConnector. Update the ``extra`` host/master/metadata
|
||||
# fields to match your Mooncake deployment before use.
|
||||
pipeline: dynin_omni
|
||||
async_chunk: false
|
||||
trust_remote_code: false
|
||||
enable_prefix_caching: false
|
||||
|
||||
connectors:
|
||||
# Mooncake connector for cross-node/intra-node communication.
|
||||
mooncake_connector:
|
||||
name: MooncakeConnector
|
||||
extra:
|
||||
host: "127.0.0.1"
|
||||
metadata_server: "http://10.90.67.86:8080/metadata"
|
||||
master: "10.90.67.86:50051"
|
||||
segment: 512000000 # 512MB
|
||||
localbuf: 64000000 # 64MB
|
||||
proto: "tcp"
|
||||
|
||||
# Yuanrong connector for cross-node/intra-node communication.
|
||||
yuanrong_connector:
|
||||
name: YuanrongConnector
|
||||
extra:
|
||||
host: "127.0.0.1"
|
||||
port: "35000"
|
||||
|
||||
# Alternative SHM connector with an explicit threshold.
|
||||
shared_memory_connector:
|
||||
name: SharedMemoryConnector
|
||||
extra:
|
||||
shm_threshold_bytes: 65536 # 64KB threshold
|
||||
|
||||
stages:
|
||||
- stage_id: 0
|
||||
devices: "0"
|
||||
max_num_seqs: 1
|
||||
max_num_batched_tokens: 32768
|
||||
enforce_eager: true
|
||||
async_scheduling: false
|
||||
output_connectors:
|
||||
to_stage_1: mooncake_connector
|
||||
|
||||
- stage_id: 1
|
||||
devices: "0"
|
||||
max_num_seqs: 1
|
||||
max_num_batched_tokens: 32768
|
||||
enforce_eager: true
|
||||
async_scheduling: false
|
||||
input_connectors:
|
||||
from_stage_0: mooncake_connector
|
||||
output_connectors:
|
||||
to_stage_2: mooncake_connector
|
||||
|
||||
- stage_id: 2
|
||||
devices: "0"
|
||||
max_num_seqs: 1
|
||||
max_num_batched_tokens: 32768
|
||||
enforce_eager: true
|
||||
async_scheduling: false
|
||||
input_connectors:
|
||||
from_stage_1: mooncake_connector
|
||||
@@ -18,6 +18,7 @@ connectors:
|
||||
connector_of_shared_memory:
|
||||
name: SharedMemoryConnector
|
||||
extra:
|
||||
initial_codec_chunk_frames: 4
|
||||
codec_chunk_frames: 25
|
||||
codec_left_context_frames: 25
|
||||
|
||||
@@ -30,12 +31,8 @@ stages:
|
||||
enable_prefix_caching: false
|
||||
devices: "0"
|
||||
default_sampling_params:
|
||||
temperature: 0.4
|
||||
top_p: 0.9
|
||||
top_k: 1
|
||||
temperature: 0.0
|
||||
max_tokens: 2048
|
||||
seed: 42
|
||||
repetition_penalty: 1.05
|
||||
|
||||
- stage_id: 1
|
||||
max_num_batched_tokens: 32768
|
||||
|
||||
@@ -18,6 +18,7 @@ connectors:
|
||||
connector_of_shared_memory:
|
||||
name: SharedMemoryConnector
|
||||
extra:
|
||||
initial_codec_chunk_frames: 4
|
||||
codec_chunk_frames: 25
|
||||
codec_left_context_frames: 25
|
||||
|
||||
|
||||
@@ -1,19 +1,13 @@
|
||||
# VoxCPM2 deploy: single-stage AR pipeline with per-request state batching.
|
||||
# Verified on 1x H20 141GB.
|
||||
#
|
||||
# Fields omitted from a stage fall back to StageDeployConfig defaults (see
|
||||
# vllm_omni/config/stage_config.py). Notable choices:
|
||||
# * enforce_eager=true because the talker's KV-cache save/restore loop is
|
||||
# not cudagraph-compatible (captured separately via voxcpm2_talker's own
|
||||
# _CapturedGraph path).
|
||||
# * max_num_seqs=4 matches the legacy max_batch_size=4 contract: concurrent
|
||||
# requests are supported via per-request StaticKVCache save/restore.
|
||||
# VoxCPM2-local runtime knobs are passed through hf_overrides.
|
||||
async_chunk: false
|
||||
dtype: bfloat16
|
||||
|
||||
stages:
|
||||
- stage_id: 0
|
||||
max_num_seqs: 4
|
||||
max_num_seqs: 32
|
||||
gpu_memory_utilization: 0.9
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
@@ -22,6 +16,18 @@ stages:
|
||||
max_num_batched_tokens: 4096
|
||||
max_model_len: 4096
|
||||
devices: "0"
|
||||
engine_extras:
|
||||
hf_overrides:
|
||||
voxcpm2_runtime_config:
|
||||
audio_emit_every: 1
|
||||
vae_decode_every: 3
|
||||
enable_batched_cfm: true
|
||||
enable_batched_vae_decode: true
|
||||
coalesce_audio_d2h: true
|
||||
enable_loc_dit_fused_qkv: true
|
||||
enable_loc_dit_fused_mlp: true
|
||||
enable_loc_dit_zero_dt_cache: true
|
||||
enable_loc_dit_skip_qkv_contig: true
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
|
||||
@@ -11,6 +11,7 @@ from vllm_omni.diffusion.attention.backends.abstract import (
|
||||
AttentionImpl,
|
||||
AttentionMetadata,
|
||||
)
|
||||
from vllm_omni.diffusion.attention.backends.sdpa import _maybe_reshape_attn_mask
|
||||
from vllm_omni.diffusion.attention.backends.utils.piecewise_attn import (
|
||||
piecewise_attn,
|
||||
)
|
||||
@@ -309,6 +310,13 @@ class FlashAttentionImpl(AttentionImpl):
|
||||
"Otherwise, use SDPA backend by setting DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA"
|
||||
)
|
||||
attention_mask = attn_metadata.attn_mask if attn_metadata else None
|
||||
|
||||
# NPU aclnnFlashAttentionScore requires mask shape to be one of:
|
||||
# [B, N, Sq, Skv], [B, 1, Sq, Skv], [1, 1, Sq, Skv], or [Sq, Skv]
|
||||
# But the incoming mask is 2D [B, S] — reshape to [B, 1, 1, S]
|
||||
# So reuse SDPA's mask reshape logic: [B, S] -> [B, 1, Sq, Skv]
|
||||
attention_mask = _maybe_reshape_attn_mask(query, key, attention_mask, mask_mode="full_qk")
|
||||
|
||||
layout = self.qkv_layout or "BNSD"
|
||||
return attention_forward(
|
||||
query,
|
||||
|
||||
+220
@@ -242,6 +242,80 @@ def enable_cache_for_wan22(pipeline: Any, cache_config: Any) -> Callable[[int],
|
||||
return refresh_cache_context
|
||||
|
||||
|
||||
def enable_cache_for_wan22_s2v(pipeline: Any, cache_config: Any) -> Callable[[int], None]:
|
||||
"""Enable cache-dit for Wan2.2 S2V.
|
||||
|
||||
S2V uses a single transformer, but unlike the other Wan2.2 variants its
|
||||
block loop calls each block as ``block(hidden_states, **kwargs)`` and keeps
|
||||
the timestep modulation state in ``e`` rather than a second positional
|
||||
tensor. CacheDiT Pattern_3 matches that contract: cache hidden states only
|
||||
and pass the remaining conditioning through kwargs unchanged.
|
||||
|
||||
The S2V transformer has an ``after_transformer_block`` method that injects
|
||||
audio embeddings after specific layers. The cached blocks wrapper
|
||||
(Wan22S2VCachedBlocks._run_block) calls the original internally, so we
|
||||
permanently replace it with a no-op on the transformer to prevent double
|
||||
injection from the main forward loop.
|
||||
"""
|
||||
db_cache_config = _build_db_cache_config(cache_config)
|
||||
calibrator_config = None
|
||||
if cache_config.enable_taylorseer:
|
||||
taylorseer_order = cache_config.taylorseer_order
|
||||
calibrator_config = TaylorSeerCalibratorConfig(taylorseer_order=taylorseer_order)
|
||||
logger.info(f"TaylorSeer enabled with order={taylorseer_order}")
|
||||
|
||||
# Save the original after_transformer_block before cache-dit wrapping
|
||||
transformer = pipeline.transformer
|
||||
if hasattr(transformer, "after_transformer_block"):
|
||||
transformer._cache_dit_original_after_transformer_block = transformer.after_transformer_block
|
||||
|
||||
Wan22S2VCachedAdapter.apply(
|
||||
BlockAdapter(
|
||||
transformer=transformer,
|
||||
blocks=[transformer.blocks],
|
||||
forward_pattern=[ForwardPattern.Pattern_3],
|
||||
params_modifiers=[
|
||||
ParamsModifier(cache_config=db_cache_config, calibrator_config=calibrator_config),
|
||||
],
|
||||
has_separate_cfg=True,
|
||||
),
|
||||
cache_config=db_cache_config,
|
||||
calibrator_config=calibrator_config,
|
||||
)
|
||||
|
||||
# Permanently replace after_transformer_block with a no-op.
|
||||
# The cached blocks wrapper (Wan22S2VCachedBlocks._run_block) already calls
|
||||
# the original via _cache_dit_original_after_transformer_block.
|
||||
def _noop_after_transformer_block(block_idx: int, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return hidden_states
|
||||
|
||||
transformer.after_transformer_block = _noop_after_transformer_block
|
||||
|
||||
def refresh_cache_context(pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
|
||||
"""Refresh cache context for the S2V transformer."""
|
||||
if cache_config.scm_steps_mask_policy is None:
|
||||
cache_dit.refresh_context(
|
||||
pipeline.transformer,
|
||||
num_inference_steps=num_inference_steps,
|
||||
verbose=verbose,
|
||||
)
|
||||
else:
|
||||
cache_dit.refresh_context(
|
||||
pipeline.transformer,
|
||||
cache_config=DBCacheConfig().reset(
|
||||
num_inference_steps=num_inference_steps,
|
||||
steps_computation_mask=cache_dit.steps_mask(
|
||||
mask_policy=cache_config.scm_steps_mask_policy,
|
||||
total_steps=num_inference_steps,
|
||||
),
|
||||
steps_computation_policy=cache_config.scm_steps_policy,
|
||||
),
|
||||
verbose=verbose,
|
||||
)
|
||||
|
||||
return refresh_cache_context
|
||||
|
||||
|
||||
def enable_cache_for_longcat_image(pipeline: Any, cache_config: Any) -> Callable[[int], None]:
|
||||
"""Enable cache-dit for LongCatImage pipeline.
|
||||
|
||||
@@ -986,6 +1060,151 @@ class BagelCachedBlocks(CachedBlocks_Pattern_0_1_2):
|
||||
return hidden_states, encoder_hidden_states
|
||||
|
||||
|
||||
class Wan22S2VCachedBlocks(CachedBlocks_Pattern_3_4_5):
|
||||
"""CacheDiT blocks wrapper that preserves S2V per-layer audio injection."""
|
||||
|
||||
def _run_block(self, block_id: int, block: torch.nn.Module, hidden_states: torch.Tensor, *args, **kwargs):
|
||||
hidden_states = block(hidden_states, *args, **kwargs)
|
||||
hidden_states, new_encoder_hidden_states = self._process_block_outputs(hidden_states)
|
||||
original_after_transformer_block = getattr(
|
||||
self.transformer,
|
||||
"_cache_dit_original_after_transformer_block",
|
||||
getattr(self.transformer, "after_transformer_block", None),
|
||||
)
|
||||
if original_after_transformer_block is not None:
|
||||
hidden_states = original_after_transformer_block(block_id, hidden_states)
|
||||
return hidden_states, new_encoder_hidden_states
|
||||
|
||||
def call_blocks(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
*args,
|
||||
**kwargs,
|
||||
):
|
||||
new_encoder_hidden_states = None
|
||||
for block_id, block in enumerate(self.transformer_blocks):
|
||||
hidden_states, new_encoder_hidden_states = self._run_block(
|
||||
block_id,
|
||||
block,
|
||||
hidden_states,
|
||||
*args,
|
||||
**kwargs,
|
||||
)
|
||||
return hidden_states, new_encoder_hidden_states
|
||||
|
||||
def call_Fn_blocks(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
*args,
|
||||
**kwargs,
|
||||
):
|
||||
new_encoder_hidden_states = None
|
||||
for block_id, block in enumerate(self._Fn_blocks()):
|
||||
hidden_states, new_encoder_hidden_states = self._run_block(
|
||||
block_id,
|
||||
block,
|
||||
hidden_states,
|
||||
*args,
|
||||
**kwargs,
|
||||
)
|
||||
return hidden_states, new_encoder_hidden_states
|
||||
|
||||
def call_Mn_blocks(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
*args,
|
||||
**kwargs,
|
||||
):
|
||||
original_hidden_states = hidden_states
|
||||
new_encoder_hidden_states = None
|
||||
start_idx = self.context_manager.Fn_compute_blocks()
|
||||
for block_id, block in enumerate(self._Mn_blocks(), start=start_idx):
|
||||
hidden_states, new_encoder_hidden_states = self._run_block(
|
||||
block_id,
|
||||
block,
|
||||
hidden_states,
|
||||
*args,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
hidden_states = hidden_states.contiguous()
|
||||
hidden_states_residual = hidden_states - original_hidden_states.to(hidden_states.device)
|
||||
|
||||
return (
|
||||
hidden_states,
|
||||
new_encoder_hidden_states,
|
||||
hidden_states_residual,
|
||||
)
|
||||
|
||||
def call_Bn_blocks(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
*args,
|
||||
**kwargs,
|
||||
):
|
||||
new_encoder_hidden_states = None
|
||||
if self.context_manager.Bn_compute_blocks() == 0:
|
||||
return hidden_states, new_encoder_hidden_states
|
||||
|
||||
start_idx = len(self.transformer_blocks) - self.context_manager.Bn_compute_blocks()
|
||||
for block_id, block in enumerate(self._Bn_blocks(), start=start_idx):
|
||||
hidden_states, new_encoder_hidden_states = self._run_block(
|
||||
block_id,
|
||||
block,
|
||||
hidden_states,
|
||||
*args,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
return hidden_states, new_encoder_hidden_states
|
||||
|
||||
|
||||
class Wan22S2VCachedAdapter(CachedAdapter):
|
||||
"""CacheDiT adapter that uses Wan22S2VCachedBlocks for S2V audio injection.
|
||||
|
||||
Only overrides collect_unified_blocks to use Wan22S2VCachedBlocks (which
|
||||
calls after_transformer_block per-layer internally). The base class
|
||||
mock_transformer handles the forward wrapping — after_transformer_block is
|
||||
permanently replaced with a no-op in enable_cache_for_wan22_s2v() to prevent
|
||||
double injection.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def collect_unified_blocks(
|
||||
cls,
|
||||
block_adapter: BlockAdapter,
|
||||
contexts_kwargs: list[dict],
|
||||
) -> list[dict[str, torch.nn.ModuleList]]:
|
||||
BlockAdapter.assert_normalized(block_adapter)
|
||||
|
||||
total_cached_blocks: list[dict[str, torch.nn.ModuleList]] = []
|
||||
assert hasattr(block_adapter.pipe, "_context_manager")
|
||||
|
||||
for i in range(len(block_adapter.transformer)):
|
||||
unified_blocks_bind_context = {}
|
||||
for j in range(len(block_adapter.blocks[i])):
|
||||
cache_config: BasicCacheConfig = contexts_kwargs[i * len(block_adapter.blocks[i]) + j]["cache_config"]
|
||||
unified_blocks_bind_context[block_adapter.unique_blocks_name[i][j]] = torch.nn.ModuleList(
|
||||
[
|
||||
Wan22S2VCachedBlocks(
|
||||
block_adapter.blocks[i][j],
|
||||
transformer=block_adapter.transformer[i],
|
||||
forward_pattern=block_adapter.forward_pattern[i][j],
|
||||
check_forward_pattern=block_adapter.check_forward_pattern,
|
||||
check_num_outputs=block_adapter.check_num_outputs,
|
||||
cache_prefix=block_adapter.blocks_name[i][j],
|
||||
cache_context=block_adapter.unique_blocks_name[i][j],
|
||||
context_manager=block_adapter.pipe._context_manager,
|
||||
cache_type=cache_config.cache_type,
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
total_cached_blocks.append(unified_blocks_bind_context)
|
||||
|
||||
return total_cached_blocks
|
||||
|
||||
|
||||
class BagelCachedAdapter(CachedAdapter):
|
||||
"""
|
||||
Custom CachedAdapter for Bagel that uses BagelCachedContextManager and BagelCachedBlocks.
|
||||
@@ -1808,6 +2027,7 @@ CUSTOM_DIT_ENABLERS.update(
|
||||
{
|
||||
"Wan22Pipeline": enable_cache_for_wan22,
|
||||
"Wan22I2VPipeline": enable_cache_for_wan22,
|
||||
"Wan22S2VPipeline": enable_cache_for_wan22_s2v,
|
||||
"HunyuanImage3Pipeline": enable_cache_for_hunyuan_image3,
|
||||
"FluxPipeline": enable_cache_for_flux,
|
||||
"Flux2KleinPipeline": enable_cache_for_flux2_klein,
|
||||
|
||||
@@ -972,8 +972,24 @@ class OmniDiffusionConfig:
|
||||
self.model_class_name = "WanS2VPipeline"
|
||||
self.tf_model_config = TransformerConfig()
|
||||
self.update_multimodal_support()
|
||||
elif model_type == "vla":
|
||||
from vllm_omni.diffusion.utils.hf_utils import _looks_like_dreamzero
|
||||
|
||||
if _looks_like_dreamzero(self.model):
|
||||
self.model_class_name = "DreamZeroPipeline"
|
||||
self.set_tf_model_config(TransformerConfig())
|
||||
self.update_multimodal_support()
|
||||
else:
|
||||
raise
|
||||
elif architectures and len(architectures) == 1:
|
||||
self.model_class_name = architectures[0]
|
||||
architecture = architectures[0]
|
||||
from vllm_omni.diffusion.registry import DiffusionModelRegistry
|
||||
|
||||
if (
|
||||
self.model_class_name is None
|
||||
or DiffusionModelRegistry._try_load_model_cls(architecture) is not None
|
||||
):
|
||||
self.model_class_name = architecture
|
||||
else:
|
||||
raise
|
||||
|
||||
@@ -1043,10 +1059,10 @@ class DiffusionOutput:
|
||||
"""
|
||||
|
||||
# Fields may be replaced with SHM handle dicts by ipc.pack_diffusion_output_shm
|
||||
output: torch.Tensor | dict | None = None
|
||||
trajectory_timesteps: torch.Tensor | dict | None = None
|
||||
trajectory_latents: torch.Tensor | dict | None = None
|
||||
trajectory_log_probs: torch.Tensor | dict | None = None
|
||||
output: torch.Tensor | tuple[Any, ...] | dict[str, Any] | None = None
|
||||
trajectory_timesteps: torch.Tensor | dict[str, Any] | None = None
|
||||
trajectory_latents: torch.Tensor | dict[str, Any] | None = None
|
||||
trajectory_log_probs: torch.Tensor | dict[str, Any] | None = None
|
||||
trajectory_decoded: list[Image.Image] | None = None
|
||||
error: str | None = None
|
||||
aborted: bool = False
|
||||
|
||||
@@ -96,6 +96,8 @@ def _move_tensor_tree_to_cpu(value: object) -> object:
|
||||
|
||||
|
||||
def get_dummy_run_num_frames(model_class_name: str, supports_audio_input: bool) -> int:
|
||||
"""Get num_frames for the dummy warmup run. Returns 0 to skip warmup."""
|
||||
|
||||
model_cls = DiffusionModelRegistry._try_load_model_cls(model_class_name)
|
||||
if model_cls is not None and hasattr(model_cls, "dummy_run_num_frames"):
|
||||
return int(getattr(model_cls, "dummy_run_num_frames"))
|
||||
@@ -266,8 +268,10 @@ class DiffusionEngine:
|
||||
custom_output = output.custom_output or {}
|
||||
model_audio_sample_rate = None
|
||||
model_fps = None
|
||||
action_payload = None
|
||||
if isinstance(outputs, dict):
|
||||
audio_payload = outputs.get("audio")
|
||||
action_payload = outputs.get("actions")
|
||||
custom_output.update(outputs.get("custom_output") or {})
|
||||
model_audio_sample_rate = outputs.get("audio_sample_rate")
|
||||
model_fps = outputs.get("fps")
|
||||
@@ -361,6 +365,8 @@ class DiffusionEngine:
|
||||
mm_output["audio_sample_rate"] = model_audio_sample_rate
|
||||
if model_fps is not None:
|
||||
mm_output["fps"] = model_fps
|
||||
if action_payload is not None:
|
||||
mm_output["actions"] = action_payload
|
||||
return [
|
||||
OmniRequestOutput.from_diffusion(
|
||||
request_id=request_id,
|
||||
@@ -430,6 +436,18 @@ class DiffusionEngine:
|
||||
mm_output["audio_sample_rate"] = model_audio_sample_rate
|
||||
if model_fps is not None:
|
||||
mm_output["fps"] = model_fps
|
||||
if action_payload is not None:
|
||||
sliced_actions = action_payload
|
||||
if isinstance(action_payload, (list, tuple)):
|
||||
sliced_actions = action_payload[start_idx:end_idx]
|
||||
if len(sliced_actions) == 1:
|
||||
sliced_actions = sliced_actions[0]
|
||||
elif hasattr(action_payload, "shape") and getattr(action_payload, "shape", None) is not None:
|
||||
if len(action_payload.shape) > 0 and action_payload.shape[0] >= end_idx:
|
||||
sliced_actions = action_payload[start_idx:end_idx]
|
||||
if num_outputs == 1:
|
||||
sliced_actions = sliced_actions[0]
|
||||
mm_output["actions"] = sliced_actions
|
||||
results.append(
|
||||
OmniRequestOutput.from_diffusion(
|
||||
request_id=request_id,
|
||||
@@ -706,6 +724,9 @@ class DiffusionEngine:
|
||||
prompt.setdefault("multi_modal_data", {})["audio"] = dummy_audio
|
||||
|
||||
num_frames = get_dummy_run_num_frames(self.od_config.model_class_name, supports_audio_input)
|
||||
if num_frames <= 0:
|
||||
logger.info("Skipping dummy warmup run (num_frames=0)")
|
||||
return
|
||||
req = OmniDiffusionRequest(
|
||||
prompts=[prompt],
|
||||
request_id=DUMMY_DIFFUSION_REQUEST_ID,
|
||||
|
||||
@@ -52,6 +52,7 @@ class InlineStageDiffusionClient(StageClientBase):
|
||||
self.replica_id = metadata.replica_id
|
||||
self.final_output = metadata.final_output
|
||||
self.final_output_type = metadata.final_output_type
|
||||
self.model_stage = getattr(metadata, "model_stage", None)
|
||||
self.default_sampling_params = metadata.default_sampling_params
|
||||
self.requires_multimodal_data = metadata.requires_multimodal_data
|
||||
self.custom_process_input_func = metadata.custom_process_input_func
|
||||
|
||||
@@ -5,6 +5,7 @@ from einops import rearrange, repeat
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.layers.custom_op import CustomOp
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -300,3 +301,234 @@ def apply_rope_to_qk(
|
||||
query = rope(query, cos, sin)
|
||||
key = rope(key, cos, sin)
|
||||
return query, key
|
||||
|
||||
|
||||
class WanS2VRotaryPosEmbed(torch.nn.Module):
|
||||
"""Precompute complex-valued RoPE embeddings for S2V multi-grid positions.
|
||||
|
||||
Owns the base frequency buffer and provides forward() to compute position
|
||||
embeddings given hidden_states (for shape) and grid_sizes.
|
||||
"""
|
||||
|
||||
def __init__(self, num_heads: int, head_dim: int, max_seq_len: int = 1024):
|
||||
super().__init__()
|
||||
self.num_heads = num_heads
|
||||
self.head_dim = head_dim
|
||||
d = head_dim
|
||||
freqs = torch.cat(
|
||||
[
|
||||
self._rope_params(max_seq_len, d - 4 * (d // 6)),
|
||||
self._rope_params(max_seq_len, 2 * (d // 6)),
|
||||
self._rope_params(max_seq_len, 2 * (d // 6)),
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
self.register_buffer("freqs", freqs.to(torch.complex64), persistent=False)
|
||||
|
||||
@staticmethod
|
||||
@torch.amp.autocast(current_omni_platform.device_type, enabled=False)
|
||||
def _rope_params(max_seq_len, dim, theta=10000):
|
||||
if dim % 2 != 0:
|
||||
raise ValueError(f"dim ({dim}) must be even")
|
||||
freqs = torch.outer(
|
||||
torch.arange(max_seq_len), 1.0 / torch.pow(theta, torch.arange(0, dim, 2).to(torch.float64).div(dim))
|
||||
)
|
||||
return torch.polar(torch.ones_like(freqs), freqs)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
grid_sizes: list,
|
||||
trainable_freqs: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Precompute RoPE embeddings for the given grid layout.
|
||||
|
||||
Args:
|
||||
hidden_states: Tensor [B, S, ...] (used for batch/seq shape and device)
|
||||
grid_sizes: Grid specification (list of [offsets, sizes, totals])
|
||||
trainable_freqs: Optional trainable frequency overrides for t_f < 0
|
||||
|
||||
Returns:
|
||||
Complex tensor [B, S, 1, head_dim//2] of precomputed position embeddings
|
||||
"""
|
||||
b, s = hidden_states.shape[0], hidden_states.shape[1]
|
||||
c = self.head_dim // 2
|
||||
device = hidden_states.device
|
||||
|
||||
freqs = self.freqs.to(device)
|
||||
if trainable_freqs is not None:
|
||||
freqs_input = [freqs, trainable_freqs]
|
||||
else:
|
||||
freqs_input = freqs
|
||||
|
||||
if isinstance(freqs_input, list):
|
||||
trainable_f = freqs_input[1]
|
||||
freqs_split = freqs_input[0]
|
||||
else:
|
||||
trainable_f = None
|
||||
freqs_split = freqs_input
|
||||
freqs_split = freqs_split.split([c - 2 * (c // 3), c // 3, c // 3], dim=1)
|
||||
|
||||
output = torch.empty((b, s, 1, c), device=device, dtype=torch.complex64)
|
||||
seq_bucket = [0]
|
||||
if not isinstance(grid_sizes, list):
|
||||
grid_sizes = [grid_sizes]
|
||||
for g in grid_sizes:
|
||||
if not isinstance(g, list):
|
||||
g = [torch.zeros_like(g), g]
|
||||
batch_size = g[0].shape[0]
|
||||
for i in range(batch_size):
|
||||
f_o, h_o, w_o = g[0][i]
|
||||
f, h, w = g[1][i]
|
||||
t_f, t_h, t_w = g[2][i]
|
||||
seq_f, seq_h, seq_w = f - f_o, h - h_o, w - w_o
|
||||
seq_len = int(seq_f * seq_h * seq_w)
|
||||
if seq_len > 0:
|
||||
if t_f > 0:
|
||||
assert f_o * f >= 0 and h_o * h >= 0 and w_o * w >= 0
|
||||
seq_f_int = int(seq_f)
|
||||
seq_h_int = int(seq_h)
|
||||
seq_w_int = int(seq_w)
|
||||
|
||||
if f_o >= 0:
|
||||
f_sam = torch.linspace(int(f_o), int(t_f + f_o) - 1, seq_f_int, device=device).long()
|
||||
else:
|
||||
f_sam = torch.linspace(int(-f_o), int(-t_f - f_o) + 1, seq_f_int, device=device).long()
|
||||
h_sam = torch.linspace(int(h_o), int(t_h + h_o) - 1, seq_h_int, device=device).long()
|
||||
w_sam = torch.linspace(int(w_o), int(t_w + w_o) - 1, seq_w_int, device=device).long()
|
||||
|
||||
freqs_0 = freqs_split[0][f_sam] if f_o >= 0 else freqs_split[0][f_sam].conj()
|
||||
freqs_0 = freqs_0.view(seq_f_int, 1, 1, -1)
|
||||
|
||||
freqs_i = torch.cat(
|
||||
[
|
||||
freqs_0.expand(seq_f_int, seq_h_int, seq_w_int, -1),
|
||||
freqs_split[1][h_sam]
|
||||
.view(1, seq_h_int, 1, -1)
|
||||
.expand(seq_f_int, seq_h_int, seq_w_int, -1),
|
||||
freqs_split[2][w_sam]
|
||||
.view(1, 1, seq_w_int, -1)
|
||||
.expand(seq_f_int, seq_h_int, seq_w_int, -1),
|
||||
],
|
||||
dim=-1,
|
||||
).reshape(seq_len, 1, -1)
|
||||
elif t_f < 0:
|
||||
freqs_i = trainable_f.unsqueeze(1)
|
||||
output[i, seq_bucket[-1] : seq_bucket[-1] + seq_len] = freqs_i
|
||||
seq_bucket.append(seq_bucket[-1] + seq_len)
|
||||
return output
|
||||
|
||||
|
||||
class RotaryEmbeddingWanS2V(RotaryEmbeddingWan):
|
||||
"""Apply RoPE using precomputed complex freqs for Wan S2V main transformer.
|
||||
|
||||
Converts complex freqs (from WanS2VRotaryPosEmbed) to cos/sin and delegates
|
||||
to RotaryEmbeddingWan for platform-optimized application (float32 kernel).
|
||||
Under TP, freqs has 1 head — broadcasts automatically via cos/sin.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(is_neox_style=False, half_head_dim=True)
|
||||
|
||||
def forward(self, x: torch.Tensor, freqs: torch.Tensor) -> torch.Tensor:
|
||||
freqs_sliced = freqs[:, : x.size(1)]
|
||||
cos = freqs_sliced.real.to(x.dtype)
|
||||
sin = freqs_sliced.imag.to(x.dtype)
|
||||
return super().forward(x, cos, sin)
|
||||
|
||||
|
||||
class RotaryEmbeddingS2VGrid(torch.nn.Module):
|
||||
"""Grid-based RoPE for S2V motioner/init attention.
|
||||
|
||||
Applies complex-valued rotary embeddings using 3D grid sampling
|
||||
(frame, height, width). Used by SimpleSelfAttention, SwinSelfAttention,
|
||||
CausalSelfAttention in motioner blocks.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
@torch.amp.autocast(current_omni_platform.device_type, enabled=False)
|
||||
def precompute(
|
||||
seq_len: int, num_heads: int, head_dim: int, grid_sizes, freqs: torch.Tensor, device: torch.device, start=None
|
||||
) -> torch.Tensor:
|
||||
"""Precompute position frequency tensor from grid specification.
|
||||
|
||||
Returns a complex tensor that can be reused across layers via apply_precomputed().
|
||||
"""
|
||||
c = head_dim // 2
|
||||
|
||||
trainable_freqs = None
|
||||
if isinstance(freqs, list):
|
||||
trainable_freqs = freqs[1]
|
||||
freqs = freqs[0]
|
||||
freqs = freqs.to(device)
|
||||
freqs = freqs.split([c - 2 * (c // 3), c // 3, c // 3], dim=1)
|
||||
|
||||
if not isinstance(grid_sizes, list):
|
||||
grid_sizes = [grid_sizes]
|
||||
|
||||
batch_size = grid_sizes[0][0].shape[0] if isinstance(grid_sizes[0], list) else grid_sizes[0].shape[0]
|
||||
precomputed = torch.empty((batch_size, seq_len, 1, c), device=device, dtype=torch.complex64)
|
||||
seq_bucket = [0]
|
||||
|
||||
for g in grid_sizes:
|
||||
if not isinstance(g, list):
|
||||
g = [torch.zeros_like(g), g]
|
||||
g_batch_size = g[0].shape[0]
|
||||
for i in range(g_batch_size):
|
||||
if start is None:
|
||||
f_o, h_o, w_o = g[0][i]
|
||||
else:
|
||||
f_o, h_o, w_o = start[i]
|
||||
|
||||
f, h, w = g[1][i]
|
||||
t_f, t_h, t_w = g[2][i]
|
||||
seq_f, seq_h, seq_w = f - f_o, h - h_o, w - w_o
|
||||
seg_len = int(seq_f * seq_h * seq_w)
|
||||
if seg_len > 0:
|
||||
if t_f > 0:
|
||||
seq_f_int = int(seq_f)
|
||||
seq_h_int = int(seq_h)
|
||||
seq_w_int = int(seq_w)
|
||||
|
||||
if f_o >= 0:
|
||||
f_sam = torch.linspace(int(f_o), int(t_f + f_o) - 1, seq_f_int, device=device).long()
|
||||
else:
|
||||
f_sam = torch.linspace(int(-f_o), int(-t_f - f_o) + 1, seq_f_int, device=device).long()
|
||||
h_sam = torch.linspace(int(h_o), int(t_h + h_o) - 1, seq_h_int, device=device).long()
|
||||
w_sam = torch.linspace(int(w_o), int(t_w + w_o) - 1, seq_w_int, device=device).long()
|
||||
|
||||
freqs_0 = freqs[0][f_sam] if f_o >= 0 else freqs[0][f_sam].conj()
|
||||
freqs_0 = freqs_0.view(seq_f_int, 1, 1, -1)
|
||||
|
||||
freqs_i = torch.cat(
|
||||
[
|
||||
freqs_0.expand(seq_f_int, seq_h_int, seq_w_int, -1),
|
||||
freqs[1][h_sam].view(1, seq_h_int, 1, -1).expand(seq_f_int, seq_h_int, seq_w_int, -1),
|
||||
freqs[2][w_sam].view(1, 1, seq_w_int, -1).expand(seq_f_int, seq_h_int, seq_w_int, -1),
|
||||
],
|
||||
dim=-1,
|
||||
).reshape(seg_len, 1, -1)
|
||||
elif t_f < 0:
|
||||
freqs_i = trainable_freqs.unsqueeze(1)
|
||||
precomputed[i, seq_bucket[-1] : seq_bucket[-1] + seg_len] = freqs_i
|
||||
seq_bucket.append(seq_bucket[-1] + seg_len)
|
||||
return precomputed
|
||||
|
||||
@staticmethod
|
||||
@torch.amp.autocast(current_omni_platform.device_type, enabled=False)
|
||||
def apply_precomputed(x: torch.Tensor, precomputed_freqs: torch.Tensor) -> torch.Tensor:
|
||||
"""Apply precomputed position frequencies to input tensor."""
|
||||
n = x.size(2)
|
||||
input_dtype = x.dtype
|
||||
seq_len = x.size(1)
|
||||
precomputed_freqs = precomputed_freqs[:, :seq_len]
|
||||
x_c = torch.view_as_complex(x.to(torch.float64).reshape(x.size(0), seq_len, n, -1, 2))
|
||||
x_c = torch.view_as_real(x_c * precomputed_freqs).flatten(3)
|
||||
return x_c.reshape(x.size(0), seq_len, n, -1).to(input_dtype)
|
||||
|
||||
@staticmethod
|
||||
@torch.amp.autocast(current_omni_platform.device_type, enabled=False)
|
||||
def forward(x: torch.Tensor, grid_sizes, freqs: torch.Tensor, start=None) -> torch.Tensor:
|
||||
n, d = x.size(2), x.size(3)
|
||||
precomputed = RotaryEmbeddingS2VGrid.precompute(x.size(1), n, d, grid_sizes, freqs, x.device, start=start)
|
||||
return RotaryEmbeddingS2VGrid.apply_precomputed(x, precomputed)
|
||||
@@ -65,10 +65,22 @@ import errno
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Iterable, Iterator
|
||||
from collections.abc import Callable, Iterable, Iterator
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# A racy / partially-evicted HF cache (the exact failure this module defends
|
||||
# against) is transient: re-running ``snapshot_download`` blocks on the peer
|
||||
# writer's per-blob ``.lock`` and then returns a complete tree. So a bounded
|
||||
# retry with linear backoff is what actually closes the window that a single
|
||||
# best-effort attempt left open (Buildkite vllm-omni-rebase #1858: both the
|
||||
# ``cuda_ti2v_hsdp`` missing-shard ``OSError`` and the ``wan_2_1_vace`` default
|
||||
# ``UMT5Config`` size-mismatch were a swallowed prefetch followed by a
|
||||
# ``from_pretrained`` against the half-written cache).
|
||||
_PREFETCH_MAX_ATTEMPTS = 3
|
||||
_PREFETCH_BACKOFF_BASE_S = 1.0
|
||||
|
||||
|
||||
def _node_lock_dir() -> str:
|
||||
"""Return a node-local directory suitable for the prefetch lock files.
|
||||
@@ -276,7 +288,11 @@ def prefetch_subfolders(
|
||||
if local_files_only or not model or os.path.isdir(model):
|
||||
return
|
||||
|
||||
logger.info("Prefetching %s subfolders: %s", model, list(subfolders))
|
||||
# Materialise ``subfolders`` up-front: it may be a one-shot generator and
|
||||
# we reference it again in the retry / logging paths below.
|
||||
subfolders = list(subfolders)
|
||||
|
||||
logger.info("Prefetching %s subfolders: %s", model, subfolders)
|
||||
|
||||
try:
|
||||
from huggingface_hub import snapshot_download
|
||||
@@ -306,39 +322,65 @@ def prefetch_subfolders(
|
||||
# snapshot is now warm. This is what makes the prefetch race-free
|
||||
# even when many ``DiffusionWorker`` subprocesses (or multiple
|
||||
# OmniServer instances on the same node) hit this code in parallel.
|
||||
try:
|
||||
with _repo_prefetch_lock(model):
|
||||
snapshot_download(
|
||||
repo_id=model,
|
||||
allow_patterns=allow_patterns,
|
||||
)
|
||||
logger.info("Prefetch complete for %s", model)
|
||||
except Exception as exc:
|
||||
# Best-effort: propagate only via logging. The subsequent
|
||||
# ``from_pretrained`` call will raise a clearer, call-site-specific
|
||||
# error (auth, 404, disk full, ...) that we'd rather surface - EXCEPT
|
||||
# for auth/gating, which we escalate here with an explicit hint so
|
||||
# readers of CI logs don't have to correlate the generic "OSError:
|
||||
# <repo> does not appear to have a file named ..." that
|
||||
# ``from_pretrained`` would otherwise emit much later with an
|
||||
# unrelated-looking message.
|
||||
if _looks_like_auth_error(exc):
|
||||
logger.error(
|
||||
"Hub prefetch for '%s' failed with an authentication / gated "
|
||||
"repository error (%s: %s). The CI HF_TOKEN must (1) be set "
|
||||
"in the step env, (2) be valid, and (3) belong to an account "
|
||||
"that has accepted the model license on huggingface.co. See "
|
||||
"docs/contributing/ci/hf_credentials.md.",
|
||||
model,
|
||||
type(exc).__name__,
|
||||
exc,
|
||||
)
|
||||
else:
|
||||
# A single best-effort attempt is not enough: when several diffusion
|
||||
# workers race a cold cache (and the node-wide lock fails to serialise
|
||||
# them, as observed for HSDP / ring launches), ``snapshot_download`` can
|
||||
# raise on a half-written tree. Swallowing that and proceeding straight
|
||||
# to ``from_pretrained`` is exactly what turned a recoverable prefetch
|
||||
# hiccup into a hard server crash. Retry with backoff so the snapshot
|
||||
# actually completes before any loader reads the cache.
|
||||
for attempt in range(1, _PREFETCH_MAX_ATTEMPTS + 1):
|
||||
try:
|
||||
with _repo_prefetch_lock(model):
|
||||
snapshot_download(
|
||||
repo_id=model,
|
||||
allow_patterns=allow_patterns,
|
||||
)
|
||||
logger.info("Prefetch complete for %s", model)
|
||||
return
|
||||
except Exception as exc:
|
||||
# Auth / gating never heals on retry - escalate immediately with
|
||||
# an explicit hint so readers of CI logs don't have to correlate
|
||||
# the generic "OSError: <repo> does not appear to have a file
|
||||
# named ..." that ``from_pretrained`` would otherwise emit later.
|
||||
if _looks_like_auth_error(exc):
|
||||
logger.error(
|
||||
"Hub prefetch for '%s' failed with an authentication / gated "
|
||||
"repository error (%s: %s). The CI HF_TOKEN must (1) be set "
|
||||
"in the step env, (2) be valid, and (3) belong to an account "
|
||||
"that has accepted the model license on huggingface.co. See "
|
||||
"docs/contributing/ci/hf_credentials.md.",
|
||||
model,
|
||||
type(exc).__name__,
|
||||
exc,
|
||||
)
|
||||
return
|
||||
|
||||
if attempt < _PREFETCH_MAX_ATTEMPTS:
|
||||
backoff = _PREFETCH_BACKOFF_BASE_S * attempt
|
||||
logger.warning(
|
||||
"Hub prefetch for repo '%s' subfolders %s failed on attempt %d/%d (%s: %s); retrying in %.1fs",
|
||||
model,
|
||||
subfolders,
|
||||
attempt,
|
||||
_PREFETCH_MAX_ATTEMPTS,
|
||||
type(exc).__name__,
|
||||
exc,
|
||||
backoff,
|
||||
)
|
||||
time.sleep(backoff)
|
||||
continue
|
||||
|
||||
# Exhausted retries. Stay best-effort: propagate only via logging
|
||||
# so the subsequent ``from_pretrained`` call surfaces the real,
|
||||
# call-site-specific error (and ``from_pretrained_with_prefetch``
|
||||
# gets a final chance to heal the cache).
|
||||
logger.warning(
|
||||
"Hub prefetch for repo '%s' subfolders %s failed (%s: %s); "
|
||||
"falling back to on-demand download in from_pretrained",
|
||||
"Hub prefetch for repo '%s' subfolders %s failed after %d attempts "
|
||||
"(%s: %s); falling back to on-demand download in from_pretrained",
|
||||
model,
|
||||
list(subfolders),
|
||||
subfolders,
|
||||
_PREFETCH_MAX_ATTEMPTS,
|
||||
type(exc).__name__,
|
||||
exc,
|
||||
)
|
||||
@@ -376,6 +418,78 @@ def _looks_like_auth_error(exc: BaseException) -> bool:
|
||||
return "401 client error" in msg or "403 client error" in msg or "gatedrepo" in msg
|
||||
|
||||
|
||||
def from_pretrained_with_prefetch(
|
||||
factory: Callable[..., Any],
|
||||
model: str,
|
||||
*,
|
||||
subfolder: str,
|
||||
prefetch_list: Iterable[str],
|
||||
local_files_only: bool = False,
|
||||
max_attempts: int = _PREFETCH_MAX_ATTEMPTS,
|
||||
**from_pretrained_kwargs: Any,
|
||||
) -> Any:
|
||||
"""Call ``factory.from_pretrained`` healing a racy / partial HF cache.
|
||||
|
||||
``factory`` is a bound ``SomeModel.from_pretrained`` (or any callable with
|
||||
the same ``(model, *, subfolder, local_files_only, **kwargs)`` signature).
|
||||
|
||||
This is a stronger sibling of :func:`retry_on_missing_shard`: that helper
|
||||
only retries the missing-shard ``OSError`` and never re-prefetches, so it
|
||||
cannot recover the second face of the same race. Two shapes of partial
|
||||
-cache failure crash the diffusion server outright:
|
||||
|
||||
* ``OSError: <repo> does not appear to have a file named
|
||||
text_encoder/model-0000X-of-0000Y.safetensors`` - a shard is still under
|
||||
its ``.incomplete`` name.
|
||||
* ``RuntimeError: You set 'ignore_mismatched_sizes' to 'False' ...`` -
|
||||
``text_encoder/config.json`` was not present yet, so ``transformers`` v5
|
||||
silently fell back to the default (tiny) config and then could not load
|
||||
the real checkpoint into it.
|
||||
|
||||
Both heal once the cache is complete. So on those errors we re-run a
|
||||
*verified* prefetch (which blocks on the peer writer and retries the
|
||||
download) and reload, instead of letting the worker die. Local paths and
|
||||
``local_files_only`` loads cannot be healed by re-fetching, so they raise
|
||||
on the first failure exactly as before.
|
||||
"""
|
||||
prefetch_list = list(prefetch_list)
|
||||
can_heal = not local_files_only and bool(model) and not os.path.isdir(model)
|
||||
last_exc: BaseException | None = None
|
||||
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
try:
|
||||
return factory(
|
||||
model,
|
||||
subfolder=subfolder,
|
||||
local_files_only=local_files_only,
|
||||
**from_pretrained_kwargs,
|
||||
)
|
||||
except (OSError, RuntimeError, ValueError) as exc:
|
||||
last_exc = exc
|
||||
if not can_heal or attempt >= max_attempts:
|
||||
break
|
||||
backoff = _PREFETCH_BACKOFF_BASE_S * attempt
|
||||
logger.warning(
|
||||
"from_pretrained(%s, subfolder=%s) failed on attempt %d/%d "
|
||||
"(%s: %s); re-prefetching repo and retrying in %.1fs",
|
||||
model,
|
||||
subfolder,
|
||||
attempt,
|
||||
max_attempts,
|
||||
type(exc).__name__,
|
||||
exc,
|
||||
backoff,
|
||||
)
|
||||
time.sleep(backoff)
|
||||
# Force a fresh, verified snapshot of every component this pipeline
|
||||
# needs - not just ``subfolder`` - so a sibling component that was
|
||||
# also half-written gets repaired in the same pass.
|
||||
prefetch_subfolders(model, prefetch_list, local_files_only=False)
|
||||
|
||||
assert last_exc is not None # loop only exits via return or a caught exc
|
||||
raise last_exc
|
||||
|
||||
|
||||
def retry_on_missing_shard(load_fn, *, max_retries: int = 3, base_delay: float = 5.0):
|
||||
"""Call *load_fn* with retry on the transformers v5 shard-resolution race.
|
||||
|
||||
|
||||
@@ -777,7 +777,9 @@ class BagelPipeline(nn.Module, SupportsComponentDiscovery, DiffusionPipelineProf
|
||||
if torch.is_tensor(v):
|
||||
generation_input[k] = v.to(self.device)
|
||||
|
||||
self._regen_init_noise_on_device(generation_input, req.sampling_params.seed)
|
||||
# NOTE: For now we disable device specific noise regeneration so that e2e tests can run
|
||||
# on both CUDA and ROCm. Context: https://github.com/vllm-project/vllm-omni/pull/4081
|
||||
# self._regen_init_noise_on_device(generation_input, req.sampling_params.seed)
|
||||
|
||||
# text cfg
|
||||
generation_input_cfg_text = self.bagel.prepare_vae_latent_cfg(
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Action-token helpers for Cosmos3 UVA/action generation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
ACTION_MODE_POLICY = "policy"
|
||||
ACTION_MODE_FORWARD_DYNAMICS = "forward_dynamics"
|
||||
ACTION_MODE_INVERSE_DYNAMICS = "inverse_dynamics"
|
||||
ACTION_MODES = {
|
||||
ACTION_MODE_POLICY,
|
||||
ACTION_MODE_FORWARD_DYNAMICS,
|
||||
ACTION_MODE_INVERSE_DYNAMICS,
|
||||
}
|
||||
|
||||
|
||||
EMBODIMENT_TO_DOMAIN_ID: dict[str, int] = {
|
||||
"no_action": 0,
|
||||
"av": 1,
|
||||
"camera_pose": 2,
|
||||
"hand_pose": 3,
|
||||
"pusht": 4,
|
||||
"libero": 5,
|
||||
"umi": 6,
|
||||
"bridge_orig_lerobot": 7,
|
||||
"droid_lerobot": 8,
|
||||
"robomind-franka": 8,
|
||||
"galbot": 9,
|
||||
"robomind-franka-dual": 12,
|
||||
"robomind-ur": 13,
|
||||
"agibotworld": 15,
|
||||
"agibot_gear_gripper": 15,
|
||||
"agibot_gear_gripper_ext": 15,
|
||||
"fractal": 20,
|
||||
}
|
||||
|
||||
|
||||
VIDEO_RES_SIZE_INFO: dict[str, dict[str, tuple[int, int]]] = {
|
||||
"256": {
|
||||
"1,1": (256, 256),
|
||||
"4,3": (320, 256),
|
||||
"3,4": (256, 320),
|
||||
"16,9": (320, 192),
|
||||
"9,16": (192, 320),
|
||||
},
|
||||
"480": {
|
||||
"1,1": (640, 640),
|
||||
"4,3": (736, 544),
|
||||
"3,4": (544, 736),
|
||||
"16,9": (832, 480),
|
||||
"9,16": (480, 832),
|
||||
},
|
||||
"704": {
|
||||
"1,1": (960, 960),
|
||||
"4,3": (1088, 832),
|
||||
"3,4": (832, 1088),
|
||||
"16,9": (1280, 704),
|
||||
"9,16": (704, 1280),
|
||||
},
|
||||
"720": {
|
||||
"1,1": (960, 960),
|
||||
"4,3": (1104, 832),
|
||||
"3,4": (832, 1104),
|
||||
"16,9": (1280, 720),
|
||||
"9,16": (720, 1280),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def normalize_action_mode(mode: Any) -> str | None:
|
||||
if mode is None:
|
||||
return None
|
||||
normalized = str(mode).strip().lower()
|
||||
if not normalized:
|
||||
return None
|
||||
if normalized not in ACTION_MODES:
|
||||
raise ValueError(f"Unsupported Cosmos3 action_mode={mode!r}; expected one of {sorted(ACTION_MODES)}.")
|
||||
return normalized
|
||||
|
||||
|
||||
def resolve_domain_id(
|
||||
*,
|
||||
domain_id: Any = None,
|
||||
domain_name: Any = None,
|
||||
require_explicit: bool = False,
|
||||
) -> int:
|
||||
if domain_id is not None:
|
||||
resolved = int(domain_id)
|
||||
if resolved < 0:
|
||||
raise ValueError(f"Cosmos3 domain_id must be non-negative, got {resolved}.")
|
||||
return resolved
|
||||
|
||||
if domain_name is None or str(domain_name).strip() == "":
|
||||
if require_explicit:
|
||||
raise ValueError(
|
||||
"Cosmos3 action generation requires extra_args['domain_id'] or non-empty extra_args['domain_name']."
|
||||
)
|
||||
return 0
|
||||
|
||||
key = str(domain_name).strip().lower()
|
||||
if key not in EMBODIMENT_TO_DOMAIN_ID:
|
||||
raise ValueError(
|
||||
f"Unknown Cosmos3 action domain_name={domain_name!r}; "
|
||||
f"expected one of {sorted(EMBODIMENT_TO_DOMAIN_ID)} or pass domain_id directly."
|
||||
)
|
||||
return EMBODIMENT_TO_DOMAIN_ID[key]
|
||||
|
||||
|
||||
def action_condition_indexes(mode: str, action_length: int) -> list[int]:
|
||||
mode = normalize_action_mode(mode)
|
||||
if mode == ACTION_MODE_FORWARD_DYNAMICS:
|
||||
return list(range(action_length))
|
||||
if mode in {ACTION_MODE_POLICY, ACTION_MODE_INVERSE_DYNAMICS}:
|
||||
return []
|
||||
raise AssertionError(f"Unexpected action mode: {mode!r}")
|
||||
|
||||
|
||||
def vision_condition_indexes(mode: str, video_length: int, temporal_compression_factor: int) -> list[int]:
|
||||
mode = normalize_action_mode(mode)
|
||||
latent_frames = (video_length - 1) // temporal_compression_factor + 1
|
||||
if mode in {ACTION_MODE_POLICY, ACTION_MODE_FORWARD_DYNAMICS}:
|
||||
return [0]
|
||||
if mode == ACTION_MODE_INVERSE_DYNAMICS:
|
||||
return list(range(latent_frames))
|
||||
raise AssertionError(f"Unexpected action mode: {mode!r}")
|
||||
|
||||
|
||||
def action_start_frame_offset(mode: str, action_length: int, video_length: int) -> int:
|
||||
del mode
|
||||
if action_length == video_length - 1:
|
||||
return 1
|
||||
if action_length == video_length:
|
||||
return 0
|
||||
raise ValueError(
|
||||
"Cosmos3 action_chunk_size must equal num_frames - 1 or num_frames; "
|
||||
f"got action_chunk_size={action_length}, num_frames={video_length}."
|
||||
)
|
||||
|
||||
|
||||
def build_action_condition_mask(
|
||||
mode: str,
|
||||
action_length: int,
|
||||
*,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
mask = torch.zeros(1, action_length, 1, device=device, dtype=dtype)
|
||||
for idx in action_condition_indexes(mode, action_length):
|
||||
mask[:, idx, :] = 1.0
|
||||
return mask
|
||||
|
||||
|
||||
def build_vision_condition_mask(
|
||||
mode: str,
|
||||
video_length: int,
|
||||
temporal_compression_factor: int,
|
||||
*,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
latent_frames = (video_length - 1) // temporal_compression_factor + 1
|
||||
mask = torch.zeros(1, 1, latent_frames, 1, 1, device=device, dtype=dtype)
|
||||
for idx in vision_condition_indexes(mode, video_length, temporal_compression_factor):
|
||||
mask[:, :, idx, :, :] = 1.0
|
||||
return mask
|
||||
|
||||
|
||||
def pad_action_to_dim(action: torch.Tensor, action_dim: int) -> torch.Tensor:
|
||||
if action.shape[-1] > action_dim:
|
||||
raise ValueError(f"Cosmos3 action dimension {action.shape[-1]} exceeds model action_dim={action_dim}.")
|
||||
if action.shape[-1] == action_dim:
|
||||
return action
|
||||
padding = torch.zeros(*action.shape[:-1], action_dim - action.shape[-1], dtype=action.dtype, device=action.device)
|
||||
return torch.cat([action, padding], dim=-1)
|
||||
|
||||
|
||||
def load_action_tensor(action: Any = None) -> torch.Tensor:
|
||||
if action is None:
|
||||
raise ValueError(
|
||||
"Cosmos3 forward_dynamics action mode requires extra_args['action'] "
|
||||
"(a JSON array / nested list / tensor of shape [T, D])."
|
||||
)
|
||||
if isinstance(action, torch.Tensor):
|
||||
tensor = action.detach().to(dtype=torch.float32)
|
||||
else:
|
||||
tensor = torch.as_tensor(np.asarray(action), dtype=torch.float32)
|
||||
if tensor.ndim == 3 and tensor.shape[0] == 1:
|
||||
tensor = tensor.squeeze(0)
|
||||
if tensor.ndim != 2:
|
||||
raise ValueError(f"Cosmos3 action must have shape [T, D], got {tuple(tensor.shape)}.")
|
||||
return tensor
|
||||
|
||||
|
||||
def find_closest_target_size(h: int, w: int, resolution: str | int) -> tuple[int, int]:
|
||||
key = str(resolution)
|
||||
if key not in VIDEO_RES_SIZE_INFO:
|
||||
raise ValueError(
|
||||
f"Unknown Cosmos3 action resolution={resolution!r}; expected one of {sorted(VIDEO_RES_SIZE_INFO)}."
|
||||
)
|
||||
input_ratio = h / w
|
||||
best_size = None
|
||||
best_diff = float("inf")
|
||||
for cand_w, cand_h in VIDEO_RES_SIZE_INFO[key].values():
|
||||
diff = abs(input_ratio - cand_h / cand_w)
|
||||
if diff < best_diff:
|
||||
best_diff = diff
|
||||
best_size = (cand_w, cand_h)
|
||||
assert best_size is not None
|
||||
return best_size
|
||||
@@ -0,0 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from .avae import Cosmos3AVAEAudioTokenizer
|
||||
|
||||
__all__ = ["Cosmos3AVAEAudioTokenizer"]
|
||||
@@ -0,0 +1,323 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Diffusers-format AVAE audio tokenizer used by Cosmos3 sound generation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.nn.utils import weight_norm
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.models.progress_bar import _is_rank_zero
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _default_avae_config(
|
||||
*,
|
||||
sample_rate: int,
|
||||
audio_channels: int,
|
||||
io_channels: int,
|
||||
hop_size: int,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"sampling_rate": sample_rate,
|
||||
"hop_size": hop_size,
|
||||
"dec_dim": 320,
|
||||
"dec_c_mults": [1, 2, 4, 8, 16],
|
||||
"dec_strides": [2, 4, 5, 6, 8],
|
||||
"dec_out_channels": audio_channels,
|
||||
"vocoder_input_dim": io_channels,
|
||||
"normalization_type": "none",
|
||||
"normalize_latents": False,
|
||||
"tanh_input_scale": 1.5,
|
||||
"tanh_output_scale": 3.5,
|
||||
"tanh_clamp": 0.995,
|
||||
}
|
||||
|
||||
|
||||
def _config_get(config: dict[str, Any], *keys: str, default: Any = None) -> Any:
|
||||
for key in keys:
|
||||
value = config.get(key)
|
||||
if value is not None:
|
||||
return value
|
||||
return default
|
||||
|
||||
|
||||
def _load_config(
|
||||
config_path: str | Path | None,
|
||||
*,
|
||||
sample_rate: int,
|
||||
audio_channels: int,
|
||||
io_channels: int,
|
||||
hop_size: int,
|
||||
) -> dict[str, Any]:
|
||||
if config_path:
|
||||
with open(config_path, encoding="utf-8") as f:
|
||||
config = json.load(f)
|
||||
if not isinstance(config, dict):
|
||||
raise TypeError(f"Cosmos3 AVAE config must be a JSON object, got {type(config)!r}.")
|
||||
return config
|
||||
return _default_avae_config(
|
||||
sample_rate=sample_rate,
|
||||
audio_channels=audio_channels,
|
||||
io_channels=io_channels,
|
||||
hop_size=hop_size,
|
||||
)
|
||||
|
||||
|
||||
def _load_checkpoint(path: str | Path, map_location: torch.device | str) -> dict[str, torch.Tensor]:
|
||||
path = Path(path)
|
||||
if path.suffix == ".safetensors":
|
||||
try:
|
||||
from safetensors.torch import load_file
|
||||
except ImportError as exc:
|
||||
raise ImportError("Loading AVAE .safetensors checkpoints requires safetensors.") from exc
|
||||
checkpoint = load_file(str(path), device=str(map_location))
|
||||
else:
|
||||
checkpoint = torch.load(path, map_location=map_location)
|
||||
|
||||
if not isinstance(checkpoint, dict):
|
||||
raise TypeError(f"AVAE checkpoint must be a flat state dict, got {type(checkpoint)!r}.")
|
||||
if not all(isinstance(value, torch.Tensor) for value in checkpoint.values()):
|
||||
raise TypeError("AVAE checkpoint must be a flat tensor state dict.")
|
||||
return checkpoint
|
||||
|
||||
|
||||
def _validate_diffusers_state_dict(state_dict: dict[str, torch.Tensor]) -> None:
|
||||
if not state_dict:
|
||||
raise RuntimeError("AVAE checkpoint is empty.")
|
||||
|
||||
if not any(key.startswith("decoder.") for key in state_dict):
|
||||
raise RuntimeError("Cosmos3 AVAE checkpoint must contain diffusers-format decoder.* keys.")
|
||||
|
||||
|
||||
class Snake1d(nn.Module):
|
||||
"""One-dimensional Snake activation matching diffusers' Oobleck layout."""
|
||||
|
||||
def __init__(self, hidden_dim: int, logscale: bool = True) -> None:
|
||||
super().__init__()
|
||||
self.alpha = nn.Parameter(torch.zeros(1, hidden_dim, 1))
|
||||
self.beta = nn.Parameter(torch.zeros(1, hidden_dim, 1))
|
||||
self.logscale = logscale
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
shape = hidden_states.shape
|
||||
alpha = torch.exp(self.alpha) if self.logscale else self.alpha
|
||||
beta = torch.exp(self.beta) if self.logscale else self.beta
|
||||
hidden_states = hidden_states.reshape(shape[0], shape[1], -1)
|
||||
hidden_states = hidden_states + (beta + 1e-9).reciprocal() * torch.sin(alpha * hidden_states).pow(2)
|
||||
return hidden_states.reshape(shape)
|
||||
|
||||
|
||||
class OobleckResidualUnit(nn.Module):
|
||||
"""Residual unit used by the diffusers Oobleck decoder."""
|
||||
|
||||
def __init__(self, dimension: int = 16, dilation: int = 1) -> None:
|
||||
super().__init__()
|
||||
pad = ((7 - 1) * dilation) // 2
|
||||
self.snake1 = Snake1d(dimension)
|
||||
self.conv1 = weight_norm(nn.Conv1d(dimension, dimension, kernel_size=7, dilation=dilation, padding=pad))
|
||||
self.snake2 = Snake1d(dimension)
|
||||
self.conv2 = weight_norm(nn.Conv1d(dimension, dimension, kernel_size=1))
|
||||
|
||||
def forward(self, hidden_state: torch.Tensor) -> torch.Tensor:
|
||||
output_tensor = self.conv1(self.snake1(hidden_state))
|
||||
output_tensor = self.conv2(self.snake2(output_tensor))
|
||||
padding = (hidden_state.shape[-1] - output_tensor.shape[-1]) // 2
|
||||
if padding > 0:
|
||||
hidden_state = hidden_state[..., padding:-padding]
|
||||
return hidden_state + output_tensor
|
||||
|
||||
|
||||
class OobleckDecoderBlock(nn.Module):
|
||||
"""Decoder block used by the diffusers Oobleck decoder."""
|
||||
|
||||
def __init__(self, input_dim: int, output_dim: int, stride: int = 1, output_padding: int = 0) -> None:
|
||||
super().__init__()
|
||||
self.snake1 = Snake1d(input_dim)
|
||||
self.conv_t1 = weight_norm(
|
||||
nn.ConvTranspose1d(
|
||||
input_dim,
|
||||
output_dim,
|
||||
kernel_size=2 * stride,
|
||||
stride=stride,
|
||||
padding=math.ceil(stride / 2),
|
||||
output_padding=output_padding,
|
||||
)
|
||||
)
|
||||
self.res_unit1 = OobleckResidualUnit(output_dim, dilation=1)
|
||||
self.res_unit2 = OobleckResidualUnit(output_dim, dilation=3)
|
||||
self.res_unit3 = OobleckResidualUnit(output_dim, dilation=9)
|
||||
|
||||
def forward(self, hidden_state: torch.Tensor) -> torch.Tensor:
|
||||
hidden_state = self.snake1(hidden_state)
|
||||
hidden_state = self.conv_t1(hidden_state)
|
||||
hidden_state = self.res_unit1(hidden_state)
|
||||
hidden_state = self.res_unit2(hidden_state)
|
||||
return self.res_unit3(hidden_state)
|
||||
|
||||
|
||||
class OobleckDecoder(nn.Module):
|
||||
"""Diffusers-compatible Oobleck decoder for Cosmos3 AVAE latents."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
channels: int,
|
||||
input_channels: int,
|
||||
audio_channels: int,
|
||||
upsampling_ratios: list[int],
|
||||
channel_multiples: list[int],
|
||||
) -> None:
|
||||
super().__init__()
|
||||
strides = upsampling_ratios
|
||||
channel_multiples = [1] + channel_multiples
|
||||
|
||||
self.conv1 = weight_norm(nn.Conv1d(input_channels, channels * channel_multiples[-1], kernel_size=7, padding=3))
|
||||
|
||||
block = []
|
||||
for stride_index, stride in enumerate(strides):
|
||||
block.append(
|
||||
OobleckDecoderBlock(
|
||||
input_dim=channels * channel_multiples[len(strides) - stride_index],
|
||||
output_dim=channels * channel_multiples[len(strides) - stride_index - 1],
|
||||
stride=stride,
|
||||
output_padding=stride % 2,
|
||||
)
|
||||
)
|
||||
self.block = nn.ModuleList(block)
|
||||
self.snake1 = Snake1d(channels)
|
||||
self.conv2 = weight_norm(nn.Conv1d(channels, audio_channels, kernel_size=7, padding=3, bias=False))
|
||||
|
||||
def forward(self, hidden_state: torch.Tensor) -> torch.Tensor:
|
||||
hidden_state = self.conv1(hidden_state)
|
||||
for layer in self.block:
|
||||
hidden_state = layer(hidden_state)
|
||||
hidden_state = self.snake1(hidden_state)
|
||||
return self.conv2(hidden_state)
|
||||
|
||||
|
||||
class Cosmos3AVAEAudioTokenizer(nn.Module):
|
||||
"""Decoder-only AVAE tokenizer for Cosmos3 audio latents."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
checkpoint_path: str | Path,
|
||||
config_path: str | Path | None = None,
|
||||
sample_rate: int = 48000,
|
||||
audio_channels: int = 2,
|
||||
io_channels: int = 64,
|
||||
hop_size: int = 1920,
|
||||
normalize_latents: bool = False,
|
||||
normalization_type: str = "none",
|
||||
tanh_input_scale: float = 1.5,
|
||||
tanh_output_scale: float = 3.5,
|
||||
tanh_clamp: float = 0.995,
|
||||
dtype: torch.dtype = torch.bfloat16,
|
||||
device: torch.device | str = "cuda",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.dtype = dtype
|
||||
self.device = torch.device(device)
|
||||
|
||||
config = _load_config(
|
||||
config_path,
|
||||
sample_rate=sample_rate,
|
||||
audio_channels=audio_channels,
|
||||
io_channels=io_channels,
|
||||
hop_size=hop_size,
|
||||
)
|
||||
self.sample_rate = int(_config_get(config, "sampling_rate", "sample_rate", default=sample_rate))
|
||||
self.audio_channels = int(
|
||||
_config_get(
|
||||
config,
|
||||
"dec_out_channels",
|
||||
"audio_channels",
|
||||
default=2 if bool(config.get("stereo", audio_channels == 2)) else 1,
|
||||
)
|
||||
)
|
||||
self.latent_ch = int(_config_get(config, "vocoder_input_dim", "io_channels", "latent_ch", default=io_channels))
|
||||
dec_strides = [int(stride) for stride in _config_get(config, "dec_strides", default=[2, 4, 5, 6, 8])]
|
||||
self.hop_size = int(
|
||||
_config_get(config, "hop_size", default=math.prod(dec_strides) if dec_strides else hop_size)
|
||||
)
|
||||
dec_stride_product = math.prod(dec_strides)
|
||||
if dec_stride_product != self.hop_size:
|
||||
raise ValueError(
|
||||
"Cosmos3 AVAE config dec_strides product must equal hop_size "
|
||||
f"for correct latent/audio duration math: product={dec_stride_product}, hop_size={self.hop_size}."
|
||||
)
|
||||
|
||||
normalization_type = str(_config_get(config, "normalization_type", default=normalization_type))
|
||||
normalize_latents = bool(_config_get(config, "normalize_latents", default=normalize_latents))
|
||||
if normalization_type == "none" and normalize_latents:
|
||||
normalization_type = "tanh"
|
||||
self.normalization_type = normalization_type
|
||||
self.tanh_input_scale = float(_config_get(config, "tanh_input_scale", default=tanh_input_scale))
|
||||
self.tanh_output_scale = float(_config_get(config, "tanh_output_scale", default=tanh_output_scale))
|
||||
self.tanh_clamp = float(_config_get(config, "tanh_clamp", default=tanh_clamp))
|
||||
|
||||
self.decoder = OobleckDecoder(
|
||||
channels=int(_config_get(config, "dec_dim", default=320)),
|
||||
input_channels=self.latent_ch,
|
||||
audio_channels=self.audio_channels,
|
||||
upsampling_ratios=list(reversed(dec_strides)),
|
||||
channel_multiples=list(_config_get(config, "dec_c_mults", default=[1, 2, 4, 8, 16])),
|
||||
)
|
||||
state_dict = _load_checkpoint(checkpoint_path, self.device)
|
||||
_validate_diffusers_state_dict(state_dict)
|
||||
|
||||
# The checkpoint also contains encoder weights, which we do not support here, hence strict=False
|
||||
self.load_state_dict(state_dict, strict=False)
|
||||
|
||||
self.eval()
|
||||
for param in self.parameters():
|
||||
param.requires_grad = False
|
||||
self.to(device=self.device, dtype=self.dtype)
|
||||
if _is_rank_zero():
|
||||
logger.info("Loaded diffusers-format Cosmos3 AVAE checkpoint from %s", checkpoint_path)
|
||||
|
||||
@property
|
||||
def temporal_compression_factor(self) -> int:
|
||||
return self.hop_size
|
||||
|
||||
def get_latent_num_samples(self, num_audio_samples: int) -> int:
|
||||
return int(num_audio_samples) // self.temporal_compression_factor
|
||||
|
||||
def get_audio_num_samples(self, num_latent_samples: int) -> int:
|
||||
return int(num_latent_samples) * self.temporal_compression_factor
|
||||
|
||||
def _denormalize_latent(self, latent: torch.Tensor) -> torch.Tensor:
|
||||
if self.normalization_type == "tanh":
|
||||
in_dtype = latent.dtype
|
||||
latent = torch.clamp(
|
||||
latent.float() / self.tanh_output_scale,
|
||||
-self.tanh_clamp,
|
||||
self.tanh_clamp,
|
||||
)
|
||||
return (torch.atanh(latent) * self.tanh_input_scale).to(in_dtype)
|
||||
if self.normalization_type != "none":
|
||||
raise ValueError(f"Unsupported AVAE normalization_type={self.normalization_type!r}.")
|
||||
return latent
|
||||
|
||||
@torch.no_grad()
|
||||
def encode(self, audio: torch.Tensor, force_pad: bool = False) -> torch.Tensor:
|
||||
del audio, force_pad
|
||||
raise NotImplementedError("Cosmos3AVAEAudioTokenizer is decoder-only for diffusers-format sound_tokenizer/.")
|
||||
|
||||
@torch.no_grad()
|
||||
def decode(self, latent: torch.Tensor) -> torch.Tensor:
|
||||
in_dtype = latent.dtype
|
||||
squeeze = latent.ndim == 2
|
||||
if squeeze:
|
||||
latent = latent.unsqueeze(0)
|
||||
z = self._denormalize_latent(latent.to(self.device)).to(self.dtype)
|
||||
audio = self.decoder(z).clamp(-1.0, 1.0).to(in_dtype)
|
||||
return audio.squeeze(0) if squeeze else audio
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,504 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Cosmos3 sound tokenizer integration."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.models.progress_bar import _is_rank_zero
|
||||
|
||||
from .audio_tokenizer import Cosmos3AVAEAudioTokenizer
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
DEFAULT_SOUND_SAMPLE_RATE = 48000
|
||||
DEFAULT_SOUND_CHANNELS = 2
|
||||
DEFAULT_SOUND_DIM = 64
|
||||
DEFAULT_SOUND_HOP_SIZE = 1920
|
||||
DEFAULT_SOUND_LATENT_FPS = DEFAULT_SOUND_SAMPLE_RATE / DEFAULT_SOUND_HOP_SIZE
|
||||
DEFAULT_SOUND_NORMALIZE_LATENTS = False
|
||||
DEFAULT_SOUND_NORMALIZATION_TYPE = "none"
|
||||
DEFAULT_SOUND_TANH_INPUT_SCALE = 1.5
|
||||
DEFAULT_SOUND_TANH_OUTPUT_SCALE = 3.5
|
||||
DEFAULT_SOUND_TANH_CLAMP = 0.995
|
||||
SOUND_TOKENIZER_COMPONENT_NAME = "sound_tokenizer"
|
||||
SOUND_TOKENIZER_CHECKPOINT_NAME = "diffusion_pytorch_model.safetensors"
|
||||
|
||||
|
||||
def _pipeline_args(od_config: OmniDiffusionConfig) -> dict[str, Any]:
|
||||
return dict(getattr(od_config, "custom_pipeline_args", None) or {})
|
||||
|
||||
|
||||
def _config_get(config: Any, key: str, default: Any = None) -> Any:
|
||||
if config is None:
|
||||
return default
|
||||
if isinstance(config, dict):
|
||||
return config.get(key, default)
|
||||
if hasattr(config, "get"):
|
||||
value = config.get(key, None)
|
||||
return default if value is None else value
|
||||
return getattr(config, key, default)
|
||||
|
||||
|
||||
def _config_path_get(config: Any, *keys: str) -> Any:
|
||||
value = config
|
||||
for key in keys:
|
||||
value = _config_get(value, key, None)
|
||||
if value is None:
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
def _sound_tokenizer_config_from(config: Any) -> Any:
|
||||
"""Return nested ``sound_tokenizer`` config from Cosmos3 config shapes."""
|
||||
for path in (
|
||||
("sound_tokenizer",),
|
||||
("model", "config", "sound_tokenizer"),
|
||||
("config", "sound_tokenizer"),
|
||||
("model_config", "sound_tokenizer"),
|
||||
):
|
||||
value = _config_path_get(config, *path)
|
||||
if value is not None:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _nested_sound_tokenizer_configs(od_config: OmniDiffusionConfig | None) -> tuple[Any, ...]:
|
||||
if od_config is None:
|
||||
return ()
|
||||
configs = []
|
||||
for source in (
|
||||
getattr(od_config, "model_config", None),
|
||||
getattr(od_config, "tf_model_config", None),
|
||||
):
|
||||
config = _sound_tokenizer_config_from(source)
|
||||
if config is not None:
|
||||
configs.append(config)
|
||||
return tuple(configs)
|
||||
|
||||
|
||||
def _first_value_from_configs(configs: tuple[Any, ...], keys: tuple[str, ...]) -> Any:
|
||||
for config in configs:
|
||||
for key in keys:
|
||||
value = _config_get(config, key, None)
|
||||
if value is not None:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _top_level_model_value(od_config: OmniDiffusionConfig | None, keys: tuple[str, ...]) -> Any:
|
||||
if od_config is None:
|
||||
return None
|
||||
for source in (
|
||||
getattr(od_config, "model_config", None),
|
||||
getattr(od_config, "tf_model_config", None),
|
||||
):
|
||||
for key in keys:
|
||||
for path in ((key,), ("model", "config", key), ("config", key), ("model_config", key)):
|
||||
value = _config_path_get(source, *path)
|
||||
if value is not None:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _custom_arg_value(args: dict[str, Any], keys: tuple[str, ...]) -> Any:
|
||||
for key in keys:
|
||||
value = args.get(key)
|
||||
if value is not None:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _as_bool(value: Any) -> bool:
|
||||
if isinstance(value, str):
|
||||
return value.strip().lower() in {"1", "true", "yes", "on"}
|
||||
return bool(value)
|
||||
|
||||
|
||||
def _as_audio_channels(value: Any) -> int:
|
||||
if isinstance(value, bool):
|
||||
return 2 if value else 1
|
||||
if isinstance(value, str) and value.strip().lower() in {
|
||||
"1",
|
||||
"0",
|
||||
"true",
|
||||
"false",
|
||||
"yes",
|
||||
"no",
|
||||
"on",
|
||||
"off",
|
||||
}:
|
||||
return 2 if _as_bool(value) else 1
|
||||
return int(value)
|
||||
|
||||
|
||||
def _resolve_model_file(path: Any, model_root: str | None) -> str | None:
|
||||
if not path:
|
||||
return None
|
||||
path = str(path)
|
||||
if "://" in path or os.path.isabs(path) or os.path.exists(path) or not model_root:
|
||||
return path
|
||||
return str(Path(model_root) / path)
|
||||
|
||||
|
||||
def _load_sound_tokenizer_component_config(config_path: str | None) -> dict[str, Any]:
|
||||
if not config_path:
|
||||
return {}
|
||||
with open(config_path, encoding="utf-8") as f:
|
||||
config = json.load(f)
|
||||
if not isinstance(config, dict):
|
||||
raise TypeError(f"Cosmos3 sound tokenizer config must be a JSON object, got {type(config)!r}.")
|
||||
return config
|
||||
|
||||
|
||||
def _component_audio_channels(config: dict[str, Any]) -> Any:
|
||||
if config.get("dec_out_channels") is not None:
|
||||
return config["dec_out_channels"]
|
||||
if config.get("audio_channels") is not None:
|
||||
return config["audio_channels"]
|
||||
if config.get("stereo") is not None:
|
||||
return 2 if _as_bool(config["stereo"]) else 1
|
||||
return None
|
||||
|
||||
|
||||
def _component_arch_values(config: dict[str, Any]) -> dict[str, Any]:
|
||||
values = {
|
||||
"sample_rate": config.get("sampling_rate", config.get("sample_rate")),
|
||||
"audio_channels": _component_audio_channels(config),
|
||||
"io_channels": config.get("vocoder_input_dim", config.get("io_channels", config.get("latent_ch"))),
|
||||
"hop_size": config.get("hop_size"),
|
||||
}
|
||||
return {key: value for key, value in values.items() if value is not None}
|
||||
|
||||
|
||||
def _resolve_arch_value(
|
||||
od_config: OmniDiffusionConfig,
|
||||
args: dict[str, Any],
|
||||
component_values: dict[str, Any],
|
||||
*,
|
||||
field: str,
|
||||
custom_keys: tuple[str, ...],
|
||||
nested_keys: tuple[str, ...],
|
||||
top_level_keys: tuple[str, ...],
|
||||
default: Any,
|
||||
cast,
|
||||
) -> Any:
|
||||
custom_value = _custom_arg_value(args, custom_keys)
|
||||
component_value = component_values.get(field)
|
||||
if component_value is not None:
|
||||
resolved = cast(component_value)
|
||||
if custom_value is not None and cast(custom_value) != resolved:
|
||||
raise ValueError(
|
||||
"Conflicting Cosmos3 sound tokenizer architecture override for "
|
||||
f"{field}: component config has {resolved!r}, custom args have {cast(custom_value)!r}."
|
||||
)
|
||||
return resolved
|
||||
|
||||
if custom_value is not None:
|
||||
return cast(custom_value)
|
||||
|
||||
nested_value = _first_value_from_configs(_nested_sound_tokenizer_configs(od_config), nested_keys)
|
||||
if nested_value is not None:
|
||||
return cast(nested_value)
|
||||
|
||||
top_value = _top_level_model_value(od_config, top_level_keys)
|
||||
if top_value is not None:
|
||||
return cast(top_value)
|
||||
|
||||
return cast(default)
|
||||
|
||||
|
||||
def _resolve_normalization_value(
|
||||
od_config: OmniDiffusionConfig,
|
||||
args: dict[str, Any],
|
||||
*,
|
||||
name: str,
|
||||
default: Any,
|
||||
aliases: tuple[str, ...] = (),
|
||||
) -> Any:
|
||||
keys = (f"sound_{name}", name, *aliases)
|
||||
custom_value = _custom_arg_value(args, keys)
|
||||
if custom_value is not None:
|
||||
return custom_value
|
||||
nested_value = _first_value_from_configs(_nested_sound_tokenizer_configs(od_config), (name, *aliases))
|
||||
return default if nested_value is None else nested_value
|
||||
|
||||
|
||||
def get_sound_config_value(
|
||||
od_config: OmniDiffusionConfig,
|
||||
name: str,
|
||||
default: Any,
|
||||
aliases: tuple[str, ...] = (),
|
||||
) -> Any:
|
||||
# Backward-compatible generic accessor. Prefer the more specific helpers
|
||||
# below for Cosmos3 sound tokenizer fields so precedence stays explicit.
|
||||
keys = (name, *aliases)
|
||||
for config in (
|
||||
_pipeline_args(od_config),
|
||||
getattr(od_config, "model_config", None),
|
||||
getattr(od_config, "tf_model_config", None),
|
||||
):
|
||||
if config is None:
|
||||
continue
|
||||
for key in keys:
|
||||
if hasattr(config, "get"):
|
||||
value = config.get(key, None)
|
||||
else:
|
||||
value = getattr(config, key, None)
|
||||
if value is not None:
|
||||
return value
|
||||
return default
|
||||
|
||||
|
||||
def get_sound_sample_rate(od_config: OmniDiffusionConfig) -> int:
|
||||
args = _pipeline_args(od_config)
|
||||
return _resolve_arch_value(
|
||||
od_config,
|
||||
args,
|
||||
{},
|
||||
field="sample_rate",
|
||||
custom_keys=("sound_sample_rate", "sample_rate"),
|
||||
nested_keys=("sample_rate", "sampling_rate"),
|
||||
top_level_keys=("sound_sample_rate", "sample_rate"),
|
||||
default=DEFAULT_SOUND_SAMPLE_RATE,
|
||||
cast=int,
|
||||
)
|
||||
|
||||
|
||||
def get_sound_channels(od_config: OmniDiffusionConfig) -> int:
|
||||
args = _pipeline_args(od_config)
|
||||
return _resolve_arch_value(
|
||||
od_config,
|
||||
args,
|
||||
{},
|
||||
field="audio_channels",
|
||||
custom_keys=("sound_audio_channels", "audio_channels", "stereo"),
|
||||
nested_keys=("audio_channels", "dec_out_channels", "stereo"),
|
||||
top_level_keys=("sound_audio_channels", "audio_channels", "stereo"),
|
||||
default=DEFAULT_SOUND_CHANNELS,
|
||||
cast=_as_audio_channels,
|
||||
)
|
||||
|
||||
|
||||
def get_sound_hop_size(od_config: OmniDiffusionConfig) -> int:
|
||||
args = _pipeline_args(od_config)
|
||||
return _resolve_arch_value(
|
||||
od_config,
|
||||
args,
|
||||
{},
|
||||
field="hop_size",
|
||||
custom_keys=("sound_hop_size", "hop_size"),
|
||||
nested_keys=("hop_size",),
|
||||
top_level_keys=("sound_hop_size", "hop_size"),
|
||||
default=DEFAULT_SOUND_HOP_SIZE,
|
||||
cast=int,
|
||||
)
|
||||
|
||||
|
||||
class Cosmos3SoundTokenizer:
|
||||
"""Thin adapter around the local AVAE tokenizer implementation."""
|
||||
|
||||
def __init__(self, tokenizer: Any) -> None:
|
||||
self.tokenizer = tokenizer
|
||||
self.sample_rate = int(getattr(tokenizer, "sample_rate", DEFAULT_SOUND_SAMPLE_RATE))
|
||||
self.audio_channels = int(getattr(tokenizer, "audio_channels", DEFAULT_SOUND_CHANNELS))
|
||||
self.latent_ch = int(getattr(tokenizer, "latent_ch", DEFAULT_SOUND_DIM))
|
||||
self.hop_size = int(getattr(tokenizer, "temporal_compression_factor", DEFAULT_SOUND_HOP_SIZE))
|
||||
if self.hop_size <= 0:
|
||||
raise ValueError(f"Cosmos3 sound tokenizer hop_size must be positive, got {self.hop_size}.")
|
||||
self.latent_fps = float(self.sample_rate) / float(self.hop_size)
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, od_config: OmniDiffusionConfig) -> Cosmos3SoundTokenizer:
|
||||
args = _pipeline_args(od_config)
|
||||
model_path = getattr(od_config, "model", None)
|
||||
explicit_avae_path = (
|
||||
args.get("sound_tokenizer_path")
|
||||
or args.get("avae_path")
|
||||
or args.get("cosmos3_avae_path")
|
||||
or os.environ.get("COSMOS3_SOUND_TOKENIZER_PATH")
|
||||
)
|
||||
explicit_config_path = args.get("sound_tokenizer_config_path") or os.environ.get(
|
||||
"COSMOS3_SOUND_TOKENIZER_CONFIG_PATH"
|
||||
)
|
||||
|
||||
model_root = str(model_path) if model_path and os.path.isdir(model_path) else None
|
||||
if model_root is None and model_path and not explicit_avae_path:
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
model_root = snapshot_download(
|
||||
repo_id=str(model_path),
|
||||
revision=getattr(od_config, "revision", None),
|
||||
allow_patterns=[
|
||||
f"{SOUND_TOKENIZER_COMPONENT_NAME}/config.json",
|
||||
f"{SOUND_TOKENIZER_COMPONENT_NAME}/{SOUND_TOKENIZER_CHECKPOINT_NAME}",
|
||||
],
|
||||
)
|
||||
|
||||
if explicit_avae_path:
|
||||
avae_path = _resolve_model_file(explicit_avae_path, model_root)
|
||||
else:
|
||||
tokenizer_dir = Path(model_root) / SOUND_TOKENIZER_COMPONENT_NAME if model_root else None
|
||||
candidate = tokenizer_dir / SOUND_TOKENIZER_CHECKPOINT_NAME if tokenizer_dir else None
|
||||
avae_path = str(candidate) if candidate and candidate.exists() else None
|
||||
|
||||
if not avae_path:
|
||||
raise ValueError(
|
||||
"Cosmos3 sound generation was requested, but no AVAE sound "
|
||||
"tokenizer checkpoint was provided. Set "
|
||||
"custom_pipeline_args['sound_tokenizer_path'] or "
|
||||
"COSMOS3_SOUND_TOKENIZER_PATH, or include "
|
||||
f"{SOUND_TOKENIZER_COMPONENT_NAME}/{SOUND_TOKENIZER_CHECKPOINT_NAME} under the model path."
|
||||
)
|
||||
|
||||
config_path = _resolve_model_file(explicit_config_path, model_root)
|
||||
if config_path is None and model_root:
|
||||
candidate = Path(model_root) / SOUND_TOKENIZER_COMPONENT_NAME / "config.json"
|
||||
config_path = str(candidate) if candidate.exists() else None
|
||||
component_config = _load_sound_tokenizer_component_config(config_path)
|
||||
component_values = _component_arch_values(component_config)
|
||||
|
||||
sample_rate = _resolve_arch_value(
|
||||
od_config,
|
||||
args,
|
||||
component_values,
|
||||
field="sample_rate",
|
||||
custom_keys=("sound_sample_rate", "sample_rate"),
|
||||
nested_keys=("sample_rate", "sampling_rate"),
|
||||
top_level_keys=("sound_sample_rate", "sample_rate"),
|
||||
default=DEFAULT_SOUND_SAMPLE_RATE,
|
||||
cast=int,
|
||||
)
|
||||
audio_channels = _resolve_arch_value(
|
||||
od_config,
|
||||
args,
|
||||
component_values,
|
||||
field="audio_channels",
|
||||
custom_keys=("sound_audio_channels", "audio_channels", "stereo"),
|
||||
nested_keys=("audio_channels", "dec_out_channels", "stereo"),
|
||||
top_level_keys=("sound_audio_channels", "audio_channels", "stereo"),
|
||||
default=DEFAULT_SOUND_CHANNELS,
|
||||
cast=_as_audio_channels,
|
||||
)
|
||||
sound_dim = _resolve_arch_value(
|
||||
od_config,
|
||||
args,
|
||||
component_values,
|
||||
field="io_channels",
|
||||
custom_keys=("sound_dim", "io_channels", "latent_ch"),
|
||||
nested_keys=("io_channels", "vocoder_input_dim", "latent_ch"),
|
||||
top_level_keys=("sound_dim",),
|
||||
default=DEFAULT_SOUND_DIM,
|
||||
cast=int,
|
||||
)
|
||||
hop_size = _resolve_arch_value(
|
||||
od_config,
|
||||
args,
|
||||
component_values,
|
||||
field="hop_size",
|
||||
custom_keys=("sound_hop_size", "hop_size"),
|
||||
nested_keys=("hop_size",),
|
||||
top_level_keys=("sound_hop_size", "hop_size"),
|
||||
default=DEFAULT_SOUND_HOP_SIZE,
|
||||
cast=int,
|
||||
)
|
||||
normalize_latents = _as_bool(
|
||||
_resolve_normalization_value(
|
||||
od_config,
|
||||
args,
|
||||
name="normalize_latents",
|
||||
default=DEFAULT_SOUND_NORMALIZE_LATENTS,
|
||||
)
|
||||
)
|
||||
normalization_type = str(
|
||||
_resolve_normalization_value(
|
||||
od_config,
|
||||
args,
|
||||
name="normalization_type",
|
||||
default=DEFAULT_SOUND_NORMALIZATION_TYPE,
|
||||
)
|
||||
)
|
||||
tanh_input_scale = float(
|
||||
_resolve_normalization_value(
|
||||
od_config,
|
||||
args,
|
||||
name="tanh_input_scale",
|
||||
default=DEFAULT_SOUND_TANH_INPUT_SCALE,
|
||||
)
|
||||
)
|
||||
tanh_output_scale = float(
|
||||
_resolve_normalization_value(
|
||||
od_config,
|
||||
args,
|
||||
name="tanh_output_scale",
|
||||
default=DEFAULT_SOUND_TANH_OUTPUT_SCALE,
|
||||
)
|
||||
)
|
||||
tanh_clamp = float(
|
||||
_resolve_normalization_value(
|
||||
od_config,
|
||||
args,
|
||||
name="tanh_clamp",
|
||||
default=DEFAULT_SOUND_TANH_CLAMP,
|
||||
)
|
||||
)
|
||||
tokenizer = Cosmos3AVAEAudioTokenizer(
|
||||
checkpoint_path=str(avae_path),
|
||||
config_path=config_path,
|
||||
sample_rate=sample_rate,
|
||||
audio_channels=audio_channels,
|
||||
io_channels=sound_dim,
|
||||
hop_size=hop_size,
|
||||
normalize_latents=normalize_latents,
|
||||
normalization_type=normalization_type,
|
||||
tanh_input_scale=tanh_input_scale,
|
||||
tanh_output_scale=tanh_output_scale,
|
||||
tanh_clamp=tanh_clamp,
|
||||
dtype=getattr(od_config, "dtype", torch.bfloat16),
|
||||
device=get_local_device(),
|
||||
)
|
||||
if _is_rank_zero():
|
||||
logger.info(
|
||||
"Loaded Cosmos3 AVAE sound tokenizer from %s "
|
||||
"(sr=%d, channels=%d, latent_ch=%d, hop=%d, latent_fps=%.3f)",
|
||||
avae_path,
|
||||
sample_rate,
|
||||
audio_channels,
|
||||
sound_dim,
|
||||
hop_size,
|
||||
float(sample_rate) / float(hop_size),
|
||||
)
|
||||
return cls(tokenizer)
|
||||
|
||||
def get_latent_num_samples(self, num_audio_samples: int) -> int:
|
||||
return int(self.tokenizer.get_latent_num_samples(num_audio_samples))
|
||||
|
||||
def get_audio_num_samples(self, num_latent_samples: int) -> int:
|
||||
return int(self.tokenizer.get_audio_num_samples(num_latent_samples))
|
||||
|
||||
@torch.no_grad()
|
||||
def decode(self, latents: torch.Tensor) -> torch.Tensor:
|
||||
"""Decode sound latents.
|
||||
|
||||
Args:
|
||||
latents: ``[B, C, T]`` or ``[C, T]`` tensor.
|
||||
|
||||
Returns:
|
||||
``[B, audio_channels, N]`` tensor for batched input, or
|
||||
``[audio_channels, N]`` for unbatched input.
|
||||
"""
|
||||
squeeze = latents.ndim == 2
|
||||
if squeeze:
|
||||
latents = latents.unsqueeze(0)
|
||||
audio = self.tokenizer.decode(latents)
|
||||
audio = audio.clamp(-1.0, 1.0)
|
||||
return audio.squeeze(0) if squeeze else audio
|
||||
@@ -76,6 +76,107 @@ def _tf_config_get(config: Any, key: str, default: Any) -> Any:
|
||||
return getattr(config, key, default)
|
||||
|
||||
|
||||
def _nested_get(value: Any, key: str) -> Any:
|
||||
if isinstance(value, dict):
|
||||
if key in value:
|
||||
return value[key]
|
||||
for child in value.values():
|
||||
found = _nested_get(child, key)
|
||||
if found is not None:
|
||||
return found
|
||||
elif isinstance(value, list | tuple):
|
||||
for child in value:
|
||||
found = _nested_get(child, key)
|
||||
if found is not None:
|
||||
return found
|
||||
return None
|
||||
|
||||
|
||||
def _od_config_get(od_config: Any, key: str, default: Any = None) -> Any:
|
||||
"""Read Cosmos3 options from runtime, model, or transformer config."""
|
||||
if od_config is None:
|
||||
return default
|
||||
for attr in ("custom_pipeline_args", "model_config"):
|
||||
source = getattr(od_config, attr, None) or {}
|
||||
if isinstance(source, dict):
|
||||
if key in source:
|
||||
return source[key]
|
||||
found = _nested_get(source, key)
|
||||
if found is not None:
|
||||
return found
|
||||
tf_model_config = getattr(od_config, "tf_model_config", None)
|
||||
if isinstance(tf_model_config, dict):
|
||||
if key in tf_model_config:
|
||||
return tf_model_config[key]
|
||||
found = _nested_get(tf_model_config, key)
|
||||
if found is not None:
|
||||
return found
|
||||
value = _tf_config_get(tf_model_config, key, None)
|
||||
return default if value is None else value
|
||||
|
||||
|
||||
def _as_bool(value: Any) -> bool:
|
||||
if isinstance(value, str):
|
||||
return value.strip().lower() in {"1", "true", "yes", "on"}
|
||||
return bool(value)
|
||||
|
||||
|
||||
def resolve_sound_gen(od_config: Any) -> bool:
|
||||
"""Capability gate shared by the pipeline and transformer.
|
||||
|
||||
Explicit ``sound_gen`` flag wins (including an explicit False);
|
||||
otherwise infer from the presence of any sound-width key in od_config.
|
||||
"""
|
||||
sound_gen_value = _od_config_get(od_config, "sound_gen", None)
|
||||
if sound_gen_value is not None:
|
||||
return _as_bool(sound_gen_value)
|
||||
for key in ("sound_dim", "io_channels", "vocoder_input_dim", "latent_ch"):
|
||||
if _od_config_get(od_config, key, None) is not None:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class DomainAwareLinear(nn.Module):
|
||||
"""Linear projection with one weight/bias pair per action embodiment domain."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
num_domains: int,
|
||||
*,
|
||||
dtype: torch.dtype = torch.bfloat16,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.input_size = int(input_size)
|
||||
self.output_size = int(output_size)
|
||||
self.num_domains = int(num_domains)
|
||||
self.fc = nn.Embedding(self.num_domains, self.output_size * self.input_size, dtype=dtype)
|
||||
self.bias = nn.Embedding(self.num_domains, self.output_size, dtype=dtype)
|
||||
nn.init.xavier_uniform_(self.fc.weight)
|
||||
nn.init.zeros_(self.bias.weight)
|
||||
|
||||
def forward(self, x: torch.Tensor, domain_id: torch.Tensor) -> torch.Tensor:
|
||||
if domain_id.ndim == 0:
|
||||
domain_id = domain_id.unsqueeze(0)
|
||||
domain_id = domain_id.to(device=x.device, dtype=torch.long).reshape(-1)
|
||||
if x.shape[0] != domain_id.shape[0]:
|
||||
raise ValueError(
|
||||
"Cosmos3 action domain_id batch size must match action tokens: "
|
||||
f"tokens={x.shape[0]}, domain_id={domain_id.shape[0]}."
|
||||
)
|
||||
if torch.any((domain_id < 0) | (domain_id >= self.num_domains)):
|
||||
raise ValueError(f"Cosmos3 action domain_id must be in [0, {self.num_domains}), got {domain_id.tolist()}.")
|
||||
|
||||
weight = self.fc(domain_id).view(domain_id.shape[0], self.input_size, self.output_size)
|
||||
bias = self.bias(domain_id).view(domain_id.shape[0], self.output_size)
|
||||
if x.ndim == 2:
|
||||
return torch.bmm(x.unsqueeze(1), weight).squeeze(1) + bias
|
||||
if x.ndim == 3:
|
||||
return torch.bmm(x, weight) + bias.unsqueeze(1)
|
||||
raise ValueError(f"Cosmos3 DomainAwareLinear expected rank-2 or rank-3 input, got {tuple(x.shape)}.")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Rotary Position Embeddings (mRoPE)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -103,6 +204,7 @@ def compute_mrope_position_ids_vision(
|
||||
temporal_compression_factor: int = 4,
|
||||
base_temporal_compression_factor: int | None = None,
|
||||
enable_fps_modulation: bool = True,
|
||||
start_frame_offset: int = 0,
|
||||
) -> tuple[torch.Tensor, int | float]:
|
||||
"""Generate 3D mRoPE position IDs for vision tokens.
|
||||
|
||||
@@ -120,10 +222,17 @@ def compute_mrope_position_ids_vision(
|
||||
)
|
||||
base_tps = base_fps / effective_base_tcf
|
||||
frame_indices = torch.arange(grid_t, dtype=torch.float32)
|
||||
t_index = (frame_indices / tps * base_tps + temporal_offset).view(-1, 1).expand(-1, grid_h * grid_w).flatten()
|
||||
t_index = (
|
||||
((frame_indices + start_frame_offset) / tps * base_tps + temporal_offset)
|
||||
.view(-1, 1)
|
||||
.expand(-1, grid_h * grid_w)
|
||||
.flatten()
|
||||
)
|
||||
else:
|
||||
t_index = torch.arange(grid_t, dtype=torch.long).view(-1, 1).expand(-1, grid_h * grid_w).flatten() + int(
|
||||
temporal_offset
|
||||
t_index = (
|
||||
torch.arange(grid_t, dtype=torch.long).view(-1, 1).expand(-1, grid_h * grid_w).flatten()
|
||||
+ int(temporal_offset)
|
||||
+ start_frame_offset
|
||||
)
|
||||
|
||||
h_index = torch.arange(grid_h, dtype=torch.long).view(1, -1, 1).expand(grid_t, -1, grid_w).flatten()
|
||||
@@ -138,6 +247,52 @@ def compute_mrope_position_ids_vision(
|
||||
return mrope_ids, next_offset
|
||||
|
||||
|
||||
def compute_mrope_position_ids_sound(
|
||||
grid_t: int,
|
||||
temporal_offset: int | float,
|
||||
sound_latent_fps: float,
|
||||
base_fps: float = 24.0,
|
||||
temporal_compression_factor_sound: int = 1,
|
||||
enable_fps_modulation: bool = True,
|
||||
) -> tuple[torch.Tensor, int | float]:
|
||||
"""Generate mRoPE IDs for sound tokens as a (T, 1, 1) grid."""
|
||||
return compute_mrope_position_ids_vision(
|
||||
grid_t=grid_t,
|
||||
grid_h=1,
|
||||
grid_w=1,
|
||||
temporal_offset=temporal_offset,
|
||||
fps=sound_latent_fps,
|
||||
base_fps=base_fps,
|
||||
temporal_compression_factor=temporal_compression_factor_sound,
|
||||
base_temporal_compression_factor=temporal_compression_factor_sound,
|
||||
enable_fps_modulation=enable_fps_modulation,
|
||||
)
|
||||
|
||||
|
||||
def compute_mrope_position_ids_action(
|
||||
grid_t: int,
|
||||
temporal_offset: int | float,
|
||||
action_fps: float | None,
|
||||
base_fps: float = 24.0,
|
||||
base_temporal_compression_factor: int = 4,
|
||||
enable_fps_modulation: bool = True,
|
||||
start_frame_offset: int = 1,
|
||||
) -> tuple[torch.Tensor, int | float]:
|
||||
"""Generate mRoPE IDs for action tokens as a frame-rate (T, 1, 1) grid."""
|
||||
return compute_mrope_position_ids_vision(
|
||||
grid_t=grid_t,
|
||||
grid_h=1,
|
||||
grid_w=1,
|
||||
temporal_offset=temporal_offset,
|
||||
fps=action_fps,
|
||||
base_fps=base_fps,
|
||||
temporal_compression_factor=1,
|
||||
base_temporal_compression_factor=base_temporal_compression_factor,
|
||||
enable_fps_modulation=enable_fps_modulation,
|
||||
start_frame_offset=start_frame_offset,
|
||||
)
|
||||
|
||||
|
||||
class Qwen3VLTextRotaryEmbedding(nn.Module):
|
||||
"""Multi-dimensional rotary position embedding for Qwen3-VL."""
|
||||
|
||||
@@ -839,6 +994,9 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
od_config: OmniDiffusionConfig,
|
||||
*,
|
||||
temporal_compression_factor: int | None = None,
|
||||
sound_gen: bool = False,
|
||||
sound_dim: int | None = None,
|
||||
sound_latent_fps: float | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
model_config = od_config.tf_model_config
|
||||
@@ -859,9 +1017,31 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
self.latent_channel_size = int(_tf_config_get(model_config, "latent_channel", 48))
|
||||
self.timestep_scale = float(_tf_config_get(model_config, "timestep_scale", 0.001))
|
||||
self.base_fps = float(_tf_config_get(model_config, "base_fps", 24.0))
|
||||
self.sound_gen = sound_gen
|
||||
self.sound_dim = sound_dim
|
||||
self.sound_latent_fps = sound_latent_fps
|
||||
if self.sound_gen and (sound_dim is None or sound_latent_fps is None):
|
||||
raise ValueError(
|
||||
"Cosmos3VFMTransformer requires an explicit sound_dim and sound_latent_fps when sound_gen is True; "
|
||||
"the pipeline must pass Cosmos3SoundTokenizer.latent_ch so the audio projection "
|
||||
"layers are sized from the authoritative AVAE latent width."
|
||||
)
|
||||
action_gen_value = _od_config_get(od_config, "action_gen", None)
|
||||
action_dim_value = _od_config_get(od_config, "action_dim", None)
|
||||
if action_dim_value is None:
|
||||
action_dim_value = _od_config_get(od_config, "max_action_dim", None)
|
||||
self.action_gen = _as_bool(action_gen_value) if action_gen_value is not None else False
|
||||
self.action_dim = int(action_dim_value if action_dim_value is not None else 64)
|
||||
self.num_embodiment_domains = int(_od_config_get(od_config, "num_embodiment_domains", 32))
|
||||
if temporal_compression_factor is None:
|
||||
temporal_compression_factor = _tf_config_get(model_config, "temporal_compression_factor", 4)
|
||||
self.temporal_compression_factor = int(temporal_compression_factor)
|
||||
self.temporal_compression_factor_sound = int(
|
||||
_tf_config_get(model_config, "temporal_compression_factor_sound", 1)
|
||||
)
|
||||
self.temporal_compression_factor_sound = int(
|
||||
_tf_config_get(model_config, "temporal_compression_factor_sound", 1)
|
||||
)
|
||||
self.enable_fps_modulation = bool(_tf_config_get(model_config, "enable_fps_modulation", True))
|
||||
self.temporal_modality_margin = int(
|
||||
_tf_config_get(
|
||||
@@ -894,6 +1074,24 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
self.proj_in = nn.Linear(self.patch_latent_dim, self.hidden_size)
|
||||
self.proj_out = nn.Linear(self.hidden_size, self.patch_latent_dim)
|
||||
self.time_embedder = TimestepEmbedder(self.hidden_size, target_dtype=dtype)
|
||||
if self.action_gen:
|
||||
self.action_proj_in = DomainAwareLinear(
|
||||
self.action_dim,
|
||||
self.hidden_size,
|
||||
self.num_embodiment_domains,
|
||||
dtype=dtype,
|
||||
)
|
||||
self.action_proj_out = DomainAwareLinear(
|
||||
self.hidden_size,
|
||||
self.action_dim,
|
||||
self.num_embodiment_domains,
|
||||
dtype=dtype,
|
||||
)
|
||||
self.action_modality_embed = nn.Parameter(torch.zeros(self.hidden_size, dtype=dtype))
|
||||
if self.sound_gen:
|
||||
self.audio_proj_in = nn.Linear(self.sound_dim, self.hidden_size)
|
||||
self.audio_proj_out = nn.Linear(self.hidden_size, self.sound_dim)
|
||||
self.audio_modality_embed = nn.Parameter(torch.zeros(self.hidden_size))
|
||||
|
||||
self.gen_layers = nn.ModuleList(
|
||||
[
|
||||
@@ -962,6 +1160,36 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
x = x[:, :, :, :h, :w]
|
||||
return x
|
||||
|
||||
def pack_sound(self, sound_latents: torch.Tensor) -> torch.Tensor:
|
||||
"""[B, C_sound, T_sound] -> [B, T_sound, C_sound]."""
|
||||
if sound_latents.ndim != 3:
|
||||
raise ValueError(f"Cosmos3 sound latents must have shape [B, C, T], got {tuple(sound_latents.shape)}.")
|
||||
if sound_latents.shape[1] != self.sound_dim:
|
||||
raise ValueError(
|
||||
f"Cosmos3 sound latent channel mismatch: expected {self.sound_dim}, got {sound_latents.shape[1]}."
|
||||
)
|
||||
return sound_latents.permute(0, 2, 1).contiguous()
|
||||
|
||||
@staticmethod
|
||||
def unpack_sound(tokens: torch.Tensor) -> torch.Tensor:
|
||||
"""[B, T_sound, C_sound] -> [B, C_sound, T_sound]."""
|
||||
return tokens.permute(0, 2, 1).contiguous()
|
||||
|
||||
def pack_action(self, action_latents: torch.Tensor) -> torch.Tensor:
|
||||
"""Validate and return action latents as [B, T_action, D_action] tokens."""
|
||||
if action_latents.ndim != 3:
|
||||
raise ValueError(f"Cosmos3 action latents must have shape [B, T, D], got {tuple(action_latents.shape)}.")
|
||||
if action_latents.shape[-1] != self.action_dim:
|
||||
raise ValueError(
|
||||
f"Cosmos3 action latent dimension mismatch: expected {self.action_dim}, got {action_latents.shape[-1]}."
|
||||
)
|
||||
return action_latents.contiguous()
|
||||
|
||||
@staticmethod
|
||||
def unpack_action(tokens: torch.Tensor) -> torch.Tensor:
|
||||
"""Return [B, T_action, D_action] action predictions."""
|
||||
return tokens.contiguous()
|
||||
|
||||
# -- RoPE computation ----------------------------------------------------
|
||||
|
||||
def _compute_rope_freqs(
|
||||
@@ -973,12 +1201,18 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
fps: float | None,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
t_action: int | None = None,
|
||||
action_start_frame_offset: int = 1,
|
||||
action_fps: float | None = None,
|
||||
t_sound: int | None = None,
|
||||
) -> tuple[tuple[torch.Tensor, torch.Tensor], tuple[torch.Tensor, torch.Tensor]]:
|
||||
"""Compute mRoPE cos/sin for UND text and GEN media pathways."""
|
||||
B = text_mask.shape[0]
|
||||
S_text = text_mask.shape[1]
|
||||
text_lengths = text_mask.sum(dim=1).long()
|
||||
effective_fps = fps if fps is not None and t > 1 else None
|
||||
action_frames = int(t_action or 0)
|
||||
sound_frames = int(t_sound or 0)
|
||||
|
||||
text_pos_list = []
|
||||
gen_pos_list = []
|
||||
@@ -996,6 +1230,32 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
temporal_compression_factor=self.temporal_compression_factor,
|
||||
enable_fps_modulation=self.enable_fps_modulation,
|
||||
)
|
||||
gen_positions = [v_pos]
|
||||
if action_frames > 0:
|
||||
a_pos, _ = compute_mrope_position_ids_action(
|
||||
action_frames,
|
||||
temporal_offset=media_temporal_offset,
|
||||
action_fps=action_fps if action_fps is not None else fps,
|
||||
base_fps=self.base_fps,
|
||||
base_temporal_compression_factor=self.temporal_compression_factor,
|
||||
enable_fps_modulation=self.enable_fps_modulation,
|
||||
start_frame_offset=action_start_frame_offset,
|
||||
)
|
||||
gen_positions.append(a_pos)
|
||||
if sound_frames > 0:
|
||||
s_pos, _ = compute_mrope_position_ids_sound(
|
||||
sound_frames,
|
||||
temporal_offset=media_temporal_offset,
|
||||
sound_latent_fps=self.sound_latent_fps,
|
||||
base_fps=self.base_fps,
|
||||
temporal_compression_factor_sound=getattr(self, "temporal_compression_factor_sound", 1),
|
||||
enable_fps_modulation=self.enable_fps_modulation,
|
||||
)
|
||||
gen_positions.append(s_pos)
|
||||
pos_dtype = gen_positions[0].dtype
|
||||
for pos in gen_positions[1:]:
|
||||
pos_dtype = torch.promote_types(pos_dtype, pos.dtype)
|
||||
v_pos = torch.cat([pos.to(pos_dtype) for pos in gen_positions], dim=1)
|
||||
if real_len < S_text:
|
||||
t_pos = torch.cat(
|
||||
[t_pos, torch.zeros(3, S_text - real_len, dtype=t_pos.dtype)],
|
||||
@@ -1026,16 +1286,35 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
def _validate_gen_sequence_parallel(
|
||||
*,
|
||||
s_gen: int,
|
||||
s_video: int,
|
||||
s_action: int,
|
||||
s_sound: int,
|
||||
has_action: bool,
|
||||
has_sound: bool,
|
||||
ulysses_size: int,
|
||||
) -> None:
|
||||
if ulysses_size <= 1 or s_gen % ulysses_size == 0:
|
||||
return
|
||||
|
||||
detail_parts = [f"video tokens {s_video}"]
|
||||
if has_action:
|
||||
detail_parts.append(f"action tokens {s_action}")
|
||||
if has_sound:
|
||||
detail_parts.append(f"sound tokens {s_sound}")
|
||||
detail = " = " + " + ".join(detail_parts) if len(detail_parts) > 1 else ""
|
||||
adjust_detail = (
|
||||
"Adjust the spatial resolution so that t * ceil(h/patch) * ceil(w/patch) is a multiple of ulysses_degree."
|
||||
"Adjust the spatial resolution, frame count, action chunk size, "
|
||||
"sound duration, or sound latent FPS so the combined media sequence is a "
|
||||
"multiple of ulysses_degree."
|
||||
if has_action or has_sound
|
||||
else (
|
||||
"Adjust the spatial resolution so that "
|
||||
"t * ceil(h/patch) * ceil(w/patch) is a multiple "
|
||||
"of ulysses_degree."
|
||||
)
|
||||
)
|
||||
raise ValueError(
|
||||
f"GEN sequence length ({s_gen} video tokens) must be divisible by "
|
||||
f"GEN sequence length ({s_gen}{detail}) must be divisible by "
|
||||
f"ulysses_degree ({ulysses_size}). {adjust_detail}"
|
||||
)
|
||||
|
||||
@@ -1049,9 +1328,15 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
text_mask: torch.Tensor,
|
||||
video_shape: tuple[int, int, int],
|
||||
fps: float | None = None,
|
||||
action_latents: torch.Tensor | None = None,
|
||||
action_domain_ids: torch.Tensor | None = None,
|
||||
action_noisy_mask: torch.Tensor | None = None,
|
||||
action_start_frame_offset: int = 1,
|
||||
action_fps: float | None = None,
|
||||
sound_latents: torch.Tensor | None = None,
|
||||
noisy_frame_mask: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | tuple[torch.Tensor, ...]:
|
||||
"""
|
||||
Args:
|
||||
hidden_states: [B, C, t, h, w] noisy latents
|
||||
@@ -1060,13 +1345,19 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
text_mask: [B, S_text] attention mask (1=real, 0=pad)
|
||||
video_shape: (t, h, w) in latent space
|
||||
fps: video frame rate for temporal mRoPE modulation
|
||||
action_latents: Optional [B, T_action, D_action] noisy action latents.
|
||||
action_domain_ids: Optional [B] embodiment domain IDs for action projections.
|
||||
action_noisy_mask: Optional [B, T_action, 1] mask where 1=noisy
|
||||
action token and 0=clean conditioned token.
|
||||
sound_latents: Optional [B, C_sound, T_sound] noisy sound latents.
|
||||
noisy_frame_mask: Optional [B, 1, t, 1, 1] mask where 1=noisy (add
|
||||
timestep embedding, predict velocity) and 0=conditioned (clean
|
||||
context, skip timestep embedding). None means all frames noisy
|
||||
(T2V mode).
|
||||
|
||||
Returns:
|
||||
[B, C, t, h, w] velocity prediction.
|
||||
[B, C, t, h, w] velocity prediction, or
|
||||
tuple outputs in video, action, sound order when extra modalities are provided.
|
||||
"""
|
||||
t, h, w = video_shape
|
||||
hp, wp, _, _ = self._pad_to_patch_size(h, w)
|
||||
@@ -1078,12 +1369,51 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
f"Cosmos3 requires identical real text lengths within a batch "
|
||||
f"(got min={min_real_len}, max={max_real_len})."
|
||||
)
|
||||
has_action = action_latents is not None
|
||||
has_sound = sound_latents is not None
|
||||
if has_action and not self.action_gen:
|
||||
raise ValueError(
|
||||
"Cosmos3 action generation was requested, but this transformer "
|
||||
"was initialized without action modules. Check that the "
|
||||
"transformer config enables action_gen."
|
||||
)
|
||||
if has_sound and not self.sound_gen:
|
||||
raise ValueError(
|
||||
"Cosmos3 sound generation was requested, but this transformer "
|
||||
"was initialized without sound modules. Check that the "
|
||||
"transformer config enables sound_gen or defines sound_dim."
|
||||
)
|
||||
|
||||
# Query Ulysses state at runtime
|
||||
ulysses_size, _, _ = _get_ulysses_state()
|
||||
|
||||
# Patchify latents and project to hidden space
|
||||
hidden_video = self.proj_in(self.patchify(hidden_states, t, h, w))
|
||||
s_video = hidden_video.shape[1]
|
||||
s_action = 0
|
||||
hidden_action = None
|
||||
s_sound = 0
|
||||
hidden_sound = None
|
||||
if action_latents is not None:
|
||||
if action_latents.shape[0] != hidden_states.shape[0]:
|
||||
raise ValueError(
|
||||
"Cosmos3 action and video batch sizes must match: "
|
||||
f"video={hidden_states.shape[0]}, action={action_latents.shape[0]}."
|
||||
)
|
||||
if action_domain_ids is None:
|
||||
action_domain_ids = torch.zeros(action_latents.shape[0], dtype=torch.long, device=action_latents.device)
|
||||
hidden_action = self.action_proj_in(self.pack_action(action_latents), action_domain_ids)
|
||||
hidden_action = hidden_action + self.action_modality_embed.to(hidden_action.dtype)
|
||||
s_action = hidden_action.shape[1]
|
||||
if sound_latents is not None:
|
||||
if sound_latents.shape[0] != hidden_states.shape[0]:
|
||||
raise ValueError(
|
||||
"Cosmos3 sound and video batch sizes must match: "
|
||||
f"video={hidden_states.shape[0]}, sound={sound_latents.shape[0]}."
|
||||
)
|
||||
hidden_sound = self.audio_proj_in(self.pack_sound(sound_latents))
|
||||
hidden_sound = hidden_sound + self.audio_modality_embed.to(hidden_sound.dtype)
|
||||
s_sound = hidden_sound.shape[1]
|
||||
|
||||
# Timestep embedding (fp32 for precision).
|
||||
# For I2V: only add to noisy tokens, not conditioned ones.
|
||||
@@ -1106,7 +1436,25 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
else:
|
||||
hidden_video = hidden_video + time_embed.unsqueeze(1)
|
||||
|
||||
hidden_gen = hidden_video
|
||||
if hidden_action is not None:
|
||||
if action_noisy_mask is None:
|
||||
hidden_action = hidden_action + time_embed.unsqueeze(1)
|
||||
else:
|
||||
if action_noisy_mask.shape != (hidden_action.shape[0], hidden_action.shape[1], 1):
|
||||
raise ValueError(
|
||||
"Cosmos3 action_noisy_mask must have shape [B, T_action, 1], "
|
||||
f"got {tuple(action_noisy_mask.shape)}."
|
||||
)
|
||||
hidden_action = hidden_action + time_embed.unsqueeze(1) * action_noisy_mask.to(hidden_action.dtype)
|
||||
|
||||
if hidden_sound is not None:
|
||||
hidden_sound = hidden_sound + time_embed.unsqueeze(1)
|
||||
hidden_parts = [hidden_video]
|
||||
if hidden_action is not None:
|
||||
hidden_parts.append(hidden_action)
|
||||
if hidden_sound is not None:
|
||||
hidden_parts.append(hidden_sound)
|
||||
hidden_gen = torch.cat(hidden_parts, dim=1)
|
||||
|
||||
# Run UND pathway once and cache K/V (replicated across all ranks)
|
||||
if self.cached_kv is None:
|
||||
@@ -1118,6 +1466,10 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
fps,
|
||||
hidden_states.device,
|
||||
hidden_states.dtype,
|
||||
t_action=s_action,
|
||||
action_start_frame_offset=action_start_frame_offset,
|
||||
action_fps=action_fps,
|
||||
t_sound=s_sound,
|
||||
)
|
||||
cached_kv_full = self.language_model(text_ids, freqs_und)
|
||||
self.cached_freqs_gen = freqs_gen
|
||||
@@ -1133,6 +1485,11 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
raise RuntimeError("Cosmos3 GEN cache was not initialized before running GEN layers.")
|
||||
self._validate_gen_sequence_parallel(
|
||||
s_gen=hidden_gen.shape[1],
|
||||
s_video=s_video,
|
||||
s_action=s_action,
|
||||
s_sound=s_sound,
|
||||
has_action=has_action,
|
||||
has_sound=has_sound,
|
||||
ulysses_size=ulysses_size,
|
||||
)
|
||||
freqs_cos, freqs_sin = self.cached_freqs_gen
|
||||
@@ -1166,7 +1523,28 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
|
||||
# Final norm and project back to latent space
|
||||
hidden_gen = self.norm_moe_gen(hidden_gen)
|
||||
return self.unpatchify(self.proj_out(hidden_gen), t, h, w)
|
||||
if not has_action and not has_sound:
|
||||
return self.unpatchify(self.proj_out(hidden_gen), t, h, w)
|
||||
|
||||
split_sizes = [s_video]
|
||||
if has_action:
|
||||
split_sizes.append(s_action)
|
||||
if has_sound:
|
||||
split_sizes.append(s_sound)
|
||||
split_hidden = hidden_gen.split(split_sizes, dim=1)
|
||||
hidden_video = split_hidden[0]
|
||||
video_pred = self.unpatchify(self.proj_out(hidden_video), t, h, w)
|
||||
outputs: list[torch.Tensor] = [video_pred]
|
||||
split_idx = 1
|
||||
if has_action:
|
||||
hidden_action = split_hidden[split_idx]
|
||||
split_idx += 1
|
||||
assert action_domain_ids is not None
|
||||
outputs.append(self.unpack_action(self.action_proj_out(hidden_action, action_domain_ids)))
|
||||
if has_sound:
|
||||
hidden_sound = split_hidden[split_idx]
|
||||
outputs.append(self.unpack_sound(self.audio_proj_out(hidden_sound)))
|
||||
return tuple(outputs)
|
||||
|
||||
def post_load_weights(self) -> None:
|
||||
"""Post-load processing: ensure correct dtypes."""
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
Loaded 100 of 187 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user