[Refactor][CI] Reorganize test tree, and move slow e2e to weekly (#4951)

Signed-off-by: wangyu <410167048@qq.com>
This commit is contained in:
wangyu authored and GitHub committed 2026-07-29 23:53:03 +08:00
1 parent 925347332f
commit 7c17111ff5
158 files changed
+3479 -4130

No files matched your search

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.6 MiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 855 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.4 MiB

-225
View File
@@ -1,54 +1,15 @@
from __future__ import annotations
import os
import subprocess
from contextlib import contextmanager
from dataclasses import dataclass
from io import BytesIO
from pathlib import Path
import pytest
import requests
import torch
from PIL import Image
from tests.helpers.runtime import OmniServer, OmniServerParams
def pytest_addoption(parser):
group = parser.getgroup("accuracy-e2e")
group.addoption("--gebench-root", action="store", default=None, help="Local GEBench dataset root")
group.addoption("--gedit-root", action="store", default=None, help="Local GEdit-Bench dataset root")
group.addoption(
"--gebench-model", action="store", default="Qwen/Qwen-Image-2512", help="Generate model for GEBench smoke"
)
group.addoption(
"--gedit-model", action="store", default="Qwen/Qwen-Image-Edit", help="Generate model for GEdit-Bench smoke"
)
group.addoption(
"--accuracy-judge-model",
action="store",
default="QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ",
help="Judge model path",
)
group.addoption("--accuracy-gpu", action="store", default="0", help="Single GPU id used sequentially")
group.addoption("--gebench-port", action="store", type=int, default=8093, help="Generate port for GEBench")
group.addoption("--gedit-port", action="store", type=int, default=8093, help="Generate port for GEdit-Bench")
group.addoption(
"--gebench-samples-per-type",
action="store",
type=int,
default=10,
help="Balanced sample count per GEBench type",
)
group.addoption(
"--gedit-samples-per-group",
action="store",
type=int,
default=20,
help="Balanced sample count per GEdit task group",
)
group.addoption("--accuracy-workers", action="store", type=int, default=1, help="Worker count for accuracy benches")
group.addoption(
"--wan22-i2v-image-source",
action="store",
@@ -77,99 +38,6 @@ def pytest_addoption(parser):
)
def _hf_cache_root() -> Path:
return Path(os.environ.get("HF_HOME", "/root/.cache/huggingface"))
def _dataset_cache_dirs(dataset_id: str) -> list[Path]:
cache_root = _hf_cache_root() / "hub" / f"datasets--{dataset_id.replace('/', '--')}" / "snapshots"
if not cache_root.exists():
return []
return sorted(
(path for path in cache_root.iterdir() if path.is_dir()), key=lambda path: path.stat().st_mtime, reverse=True
)
def _ensure_dataset_snapshot(dataset_id: str) -> Path:
candidates = _dataset_cache_dirs(dataset_id)
if candidates:
return candidates[0]
subprocess.run(
["hf", "download", "--repo-type", "dataset", dataset_id],
check=True,
)
candidates = _dataset_cache_dirs(dataset_id)
if not candidates:
raise FileNotFoundError(
f"Dataset {dataset_id} was downloaded but no snapshot was found under {_hf_cache_root()}"
)
return candidates[0]
def _resolve_dataset_root(request: pytest.FixtureRequest, option_name: str, dataset_id: str) -> Path:
value = request.config.getoption(option_name)
if value:
path = Path(value)
if not path.exists():
pytest.skip(f"Dataset path does not exist: {path}")
return path
return _ensure_dataset_snapshot(dataset_id)
@dataclass
class AccuracyServerConfig:
generate_params: OmniServerParams
judge_params: OmniServerParams
run_level: str
model_prefix: str
@contextmanager
def generate_server(self):
params = self.generate_params
model = self.model_prefix + params.model
server_args = params.server_args or []
if params.use_omni and params.stage_init_timeout is not None:
server_args = ["--stage-init-timeout", str(params.stage_init_timeout), *server_args]
with OmniServer(
model,
server_args,
port=params.port,
env_dict=params.env_dict,
use_omni=params.use_omni,
) as server:
yield server
@contextmanager
def judge_server(self):
params = self.judge_params
model = self.model_prefix + params.model
server_args = params.server_args or []
with OmniServer(
model,
server_args,
port=params.port,
env_dict=params.env_dict,
use_omni=params.use_omni,
) as server:
yield server
@pytest.fixture(scope="session")
def gebench_dataset_root(request: pytest.FixtureRequest) -> Path:
return _resolve_dataset_root(request, "gebench_root", "stepfun-ai/GEBench")
@pytest.fixture(scope="session")
def gedit_dataset_root(request: pytest.FixtureRequest) -> Path:
return _resolve_dataset_root(request, "gedit_root", "stepfun-ai/GEdit-Bench")
@pytest.fixture(scope="session")
def accuracy_workers(request: pytest.FixtureRequest) -> int:
return int(request.config.getoption("accuracy_workers"))
@pytest.fixture(scope="session")
def wan22_i2v_image_source(request: pytest.FixtureRequest) -> str | None:
value = request.config.getoption("wan22_i2v_image_source")
@@ -192,16 +60,6 @@ def hunyuanvideo15_online_timeout_seconds(request: pytest.FixtureRequest) -> int
return int(request.config.getoption("hunyuanvideo15_online_timeout_seconds"))
@pytest.fixture(scope="session")
def gebench_samples_per_type(request: pytest.FixtureRequest) -> int:
return int(request.config.getoption("gebench_samples_per_type"))
@pytest.fixture(scope="session")
def gedit_samples_per_group(request: pytest.FixtureRequest) -> int:
return int(request.config.getoption("gedit_samples_per_group"))
@pytest.fixture(scope="session")
def accuracy_artifact_root() -> Path:
root = Path(__file__).resolve().parent / "artifacts"
@@ -209,12 +67,6 @@ def accuracy_artifact_root() -> Path:
return root
@pytest.fixture(scope="session")
def accuracy_assets_root() -> Path:
root = Path(__file__).resolve().parent / "assets"
return root
@pytest.fixture(scope="session")
def qwen_bear_image(accuracy_artifact_root: Path):
"""Download the Qwen bear image from the URL and save it to the accuracy artifact root."""
@@ -249,80 +101,3 @@ def rabbit_image(accuracy_artifact_root: Path):
image.save(image_path)
yield image
image.close()
def _build_accuracy_server_config(
*,
generate_model: str,
judge_model: str,
shared_gpu: str,
port: int,
run_level: str,
model_prefix: str,
) -> AccuracyServerConfig:
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 CUDA GPU for accuracy benchmark smoke tests.")
if not generate_model:
pytest.skip("No generate model configured for accuracy benchmark test.")
generate_server_args = ["--num-gpus", "1"]
judge_server_args = [
"--max-model-len",
"32768",
"--gpu-memory-utilization",
"0.8",
]
judge_env = {"CUDA_VISIBLE_DEVICES": shared_gpu}
return AccuracyServerConfig(
generate_params=OmniServerParams(
model=generate_model,
port=port,
server_args=generate_server_args,
env_dict={"CUDA_VISIBLE_DEVICES": shared_gpu},
use_omni=True,
stage_init_timeout=300,
),
judge_params=OmniServerParams(
model=judge_model,
port=port,
server_args=judge_server_args,
env_dict=judge_env,
use_omni=False,
),
run_level=run_level,
model_prefix=model_prefix,
)
@pytest.fixture
def gebench_accuracy_servers(
request: pytest.FixtureRequest,
run_level: str,
model_prefix: str,
) -> AccuracyServerConfig:
return _build_accuracy_server_config(
generate_model=request.config.getoption("gebench_model"),
judge_model=request.config.getoption("accuracy_judge_model"),
shared_gpu=str(request.config.getoption("accuracy_gpu")),
port=int(request.config.getoption("gebench_port")),
run_level=run_level,
model_prefix=model_prefix,
)
@pytest.fixture
def gedit_accuracy_servers(
request: pytest.FixtureRequest,
run_level: str,
model_prefix: str,
) -> AccuracyServerConfig:
return _build_accuracy_server_config(
generate_model=request.config.getoption("gedit_model"),
judge_model=request.config.getoption("accuracy_judge_model"),
shared_gpu=str(request.config.getoption("accuracy_gpu")),
port=int(request.config.getoption("gedit_port")),
run_level=run_level,
model_prefix=model_prefix,
)
+345 -2
View File
@@ -3,8 +3,12 @@ import os
import re
import shutil
import subprocess
import sys
import time
from abc import ABC, abstractmethod
from base64 import b64decode, b64encode
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from fractions import Fraction
from hashlib import sha1
from io import BytesIO
@@ -53,8 +57,6 @@ def env_to_apply_ftfy_mock_in_subproc(env: dict[str, str] | None = None) -> dict
def reset_artifact_dir(path: Path) -> Path:
import shutil
if path.exists():
shutil.rmtree(path)
path.mkdir(parents=True, exist_ok=True)
@@ -609,3 +611,344 @@ def download_images(urls: list[str]) -> list[Image.Image]:
images.append(image)
return images
_DREAMZERO_UPSTREAM_SERVER_SCRIPT = (
Path(__file__).resolve().parent / "upstream_mock" / "upstream_socket_server_no_compile.py"
)
_DREAMZERO_DEFAULT_MODEL = "GEAR-Dreams/DreamZero-DROID"
_DREAMZERO_DEFAULT_READY_TIMEOUT_S = int(os.environ.get("OPENPI_SERVICE_READY_TIMEOUT_S", "900"))
_DREAMZERO_DEFAULT_PROMPT = (
"Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan"
)
_DREAMZERO_DEFAULT_SESSION_ID = "openpi-e2e-parity-session"
_DREAMZERO_OPENPI_VLLM_PATH = "/v1/realtime/robot/openpi"
def require_dreamzero_parity_env() -> tuple[Path, Path]:
repo_env = os.environ.get("DREAMZERO_REPO")
if repo_env is None:
raise ValueError("DREAMZERO_REPO environment variable is required.")
repo = Path(repo_env).expanduser()
if not repo.is_dir():
raise FileNotFoundError(f"DREAMZERO_REPO does not exist: {repo}")
checkpoint_dir = repo / "checkpoints" / "dreamzero"
if not checkpoint_dir.is_dir():
raise FileNotFoundError(f"DreamZero checkpoint not found: {checkpoint_dir}")
if str(repo) not in sys.path:
sys.path.insert(0, str(repo))
return repo, checkpoint_dir
def load_dreamzero_upstream_modules():
require_dreamzero_parity_env()
import test_client_AR as dreamzero_upstream_client
from eval_utils.policy_client import WebsocketClientPolicy
return dreamzero_upstream_client, WebsocketClientPolicy
def dreamzero_cfg_parallel_size() -> int:
return int(os.environ.get("OPENPI_E2E_CFG_PARALLEL_SIZE", "2"))
def _dreamzero_parity_vllm_env() -> dict[str, str]:
from tests.helpers.env import pick_least_used_device_indices
from tests.helpers.runtime import get_open_port
parallel_size = dreamzero_cfg_parallel_size()
gpus = [str(index) for index in pick_least_used_device_indices(parallel_size)]
if parallel_size > len(gpus):
raise RuntimeError(
f"cfg_parallel_size={parallel_size} requires at least {parallel_size} GPUs, but only got {gpus}."
)
return {
"CUDA_VISIBLE_DEVICES": ",".join(gpus[:parallel_size]),
"ATTENTION_BACKEND": "torch",
"DIFFUSION_ATTENTION_BACKEND": "TORCH_SDPA",
"VLLM_DISABLE_COMPILE_CACHE": "1",
"MASTER_PORT": str(get_open_port()),
}
def dreamzero_parity_vllm_params(model: str | None = None):
from tests.helpers.runtime import OmniServerParams
from tests.helpers.stage_config import get_deploy_config_path
parallel_size = dreamzero_cfg_parallel_size()
server_args = [
"--served-model-name",
"dreamzero-droid",
"--enforce-eager",
]
if parallel_size > 1:
server_args.extend(["--cfg-parallel-size", str(parallel_size)])
return OmniServerParams(
model=model or os.environ.get("VLLM_DREAMZERO_MODEL", _DREAMZERO_DEFAULT_MODEL),
stage_config_path=get_deploy_config_path("dreamzero_tp1_cfg2.yaml"),
server_args=server_args,
env_dict=_dreamzero_parity_vllm_env(),
)
def _dreamzero_omni_server_from_params(params):
from tests.helpers.runtime import OmniServer
server_args = list(params.server_args or [])
if params.stage_config_path is not None:
server_args += ["--stage-configs-path", params.stage_config_path]
return OmniServer(
params.model,
server_args,
port=params.port,
env_dict=params.env_dict,
use_omni=params.use_omni,
)
@contextmanager
def dreamzero_parity_vllm_client(
params=None,
*,
ready_timeout_s: float = _DREAMZERO_DEFAULT_READY_TIMEOUT_S,
) -> Iterator["DreamZeroParityClient"]:
"""Start vLLM via ``OmniServer`` and return a ready OpenPI parity client."""
with _dreamzero_omni_server_from_params(params or dreamzero_parity_vllm_params()) as server:
client = _dreamzero_wait_for_client_ready(
lambda: _dreamzero_create_openpi_vllm_client(server.host, server.port),
timeout_s=ready_timeout_s,
proc=server.proc,
)
yield DreamZeroParityClient(client)
def _dreamzero_create_openpi_vllm_client(host: str, port: int) -> Any:
from openpi_client import msgpack_numpy
_, WebsocketClientPolicy = load_dreamzero_upstream_modules()
class _OpenPIWebsocketClientPolicy(WebsocketClientPolicy):
def __init__(
self,
host: str = "127.0.0.1",
port: int = 8000,
path: str = _DREAMZERO_OPENPI_VLLM_PATH,
) -> None:
self._uri = f"ws://{host}:{port}{path}"
self._packer = msgpack_numpy.Packer()
self._ws, self._server_metadata = self._wait_for_server()
return _OpenPIWebsocketClientPolicy(host=host, port=port)
def _dreamzero_stop_process(proc: subprocess.Popen[str]) -> None:
log_file = getattr(proc, "_codex_log_file", None)
if proc.poll() is None:
proc.terminate()
try:
proc.wait(timeout=30)
except subprocess.TimeoutExpired: # pragma: no cover - cleanup path
proc.kill()
proc.wait(timeout=10)
if log_file is not None:
log_file.close()
def _dreamzero_wait_for_client_ready(
client_factory: Callable[[], Any],
*,
timeout_s: float,
proc: subprocess.Popen[str] | None = None,
log_path: Path | None = None,
) -> Any:
deadline = time.time() + timeout_s
last_err: Exception | None = None
while time.time() < deadline:
if proc is not None and proc.poll() is not None:
details = ""
if log_path is not None and log_path.exists():
details = log_path.read_text(errors="replace")[-8000:]
raise RuntimeError(f"Service exited before becoming ready with code {proc.returncode}.\n{details}")
try:
return client_factory()
except Exception as exc: # pragma: no cover - retry path
last_err = exc
time.sleep(1)
raise TimeoutError(f"Timed out waiting for websocket service: {last_err}")
def _dreamzero_normalize_reset_response(response: Any) -> str:
from openpi_client import msgpack_numpy
if isinstance(response, str):
return response
decoded = msgpack_numpy.unpackb(response)
if isinstance(decoded, dict):
return str(decoded.get("status"))
return str(decoded)
def normalize_dreamzero_metadata(metadata: dict[str, Any]) -> dict[str, Any]:
normalized = dict(metadata)
if isinstance(normalized.get("image_resolution"), tuple):
normalized["image_resolution"] = list(normalized["image_resolution"])
return normalized
def assert_dreamzero_logs_clean(log_path: Path) -> None:
text = log_path.read_text(errors="replace")
if "SignalException: Process" in text and "got signal: 15" in text:
text = text.split("Traceback (most recent call last):", 1)[0]
assert "Traceback" not in text, text
assert "RuntimeError:" not in text, text
def assert_dreamzero_upstream_log_matches_vllm_baseline(log_path: Path) -> None:
text = log_path.read_text(errors="replace")
assert "DIT Compute Steps 8 steps" not in text, text
assert "DIT Compute Steps 16 steps" in text, text
class DreamZeroParityClient:
"""Connected OpenPI websocket client for DreamZero parity inference."""
def __init__(self, client: Any) -> None:
self._client = client
@property
def raw_client(self) -> Any:
return self._client
def build_obs_sequence(self) -> tuple[dict[str, Any], dict[str, Any]]:
dreamzero_upstream_client, _ = load_dreamzero_upstream_modules()
camera_frames = dreamzero_upstream_client.load_camera_frames()
chunks = dreamzero_upstream_client.build_frame_schedule(
min(v.shape[0] for v in camera_frames.values()),
1,
)
obs0 = dreamzero_upstream_client._make_obs_from_video(
camera_frames,
[0],
_DREAMZERO_DEFAULT_PROMPT,
_DREAMZERO_DEFAULT_SESSION_ID,
)
obs1 = dreamzero_upstream_client._make_obs_from_video(
camera_frames,
chunks[0],
_DREAMZERO_DEFAULT_PROMPT,
_DREAMZERO_DEFAULT_SESSION_ID,
)
return obs0, obs1
def collect_parity_outputs(self) -> tuple[dict[str, Any], list[np.ndarray]]:
metadata = self._client.get_server_metadata()
obs0, obs1 = self.build_obs_sequence()
outputs = [
self._client.infer(dict(obs0)),
self._client.infer(dict(obs1)),
]
assert _dreamzero_normalize_reset_response(self._client.reset({})) == "reset successful"
outputs.append(self._client.infer(dict(obs0)))
self._client._ws.close()
return metadata, outputs
class _DreamZeroParityServer(ABC):
host = "127.0.0.1"
def __init__(
self,
*,
log_path: Path,
port: int | None = None,
ready_timeout_s: float = _DREAMZERO_DEFAULT_READY_TIMEOUT_S,
) -> None:
from tests.helpers.runtime import get_open_port
self.log_path = log_path
self.port = get_open_port() if port is None else port
self.ready_timeout_s = ready_timeout_s
self.proc: subprocess.Popen[str] | None = None
self._log_file = None
@abstractmethod
def _build_argv(self) -> list[str]:
raise NotImplementedError
@abstractmethod
def _build_env(self) -> dict[str, str]:
raise NotImplementedError
@abstractmethod
def _create_client(self) -> Any:
raise NotImplementedError
def _start(self) -> None:
self.log_path.parent.mkdir(parents=True, exist_ok=True)
self._log_file = self.log_path.open("w")
self.proc = subprocess.Popen(
self._build_argv(),
stdout=self._log_file,
stderr=subprocess.STDOUT,
text=True,
cwd=str(Path.cwd()),
env=self._build_env(),
)
self.proc._codex_log_file = self._log_file # type: ignore[attr-defined]
def __enter__(self) -> DreamZeroParityClient:
self._start()
client = _dreamzero_wait_for_client_ready(
self._create_client,
timeout_s=self.ready_timeout_s,
proc=self.proc,
log_path=self.log_path,
)
return DreamZeroParityClient(client)
def __exit__(self, exc_type, exc_val, exc_tb) -> None:
if self.proc is not None:
_dreamzero_stop_process(self.proc)
self.proc = None
class DreamZeroUpstreamServer(_DreamZeroParityServer):
"""Official DreamZero upstream reference server launched via torchrun."""
def _build_argv(self) -> list[str]:
_, checkpoint_dir = require_dreamzero_parity_env()
return [
sys.executable,
"-m",
"torch.distributed.run",
"--standalone",
"--nproc_per_node",
str(dreamzero_cfg_parallel_size()),
str(_DREAMZERO_UPSTREAM_SERVER_SCRIPT),
"--port",
str(self.port),
"--model_path",
str(checkpoint_dir),
]
def _build_env(self) -> dict[str, str]:
from tests.helpers.env import pick_least_used_device_indices
dreamzero_repo, _ = require_dreamzero_parity_env()
env = os.environ.copy()
env.setdefault("PYTHONPATH", "")
env["PYTHONPATH"] = f"{Path.cwd()}:{dreamzero_repo}:{env['PYTHONPATH']}".rstrip(":")
env["CUDA_VISIBLE_DEVICES"] = ",".join(
str(index) for index in pick_least_used_device_indices(max(dreamzero_cfg_parallel_size(), 1))
)
env.setdefault("NO_ALBUMENTATIONS_UPDATE", "1")
env["ATTENTION_BACKEND"] = "torch"
env.setdefault("ENABLE_TENSORRT", "false")
env["ENABLE_DIT_CACHE"] = "false"
env["NUM_DIT_STEPS"] = "16"
env["DYNAMIC_CACHE_SCHEDULE"] = "false"
return env
def _create_client(self) -> Any:
_, WebsocketClientPolicy = load_dreamzero_upstream_modules()
return WebsocketClientPolicy(host=self.host, port=self.port)
+63
View File
@@ -0,0 +1,63 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DreamZero OpenPI upstream parity accuracy test."""
from __future__ import annotations
import os
from pathlib import Path
import numpy as np
import pytest
from tests.e2e.accuracy.helpers import (
DreamZeroUpstreamServer,
assert_dreamzero_logs_clean,
assert_dreamzero_upstream_log_matches_vllm_baseline,
dreamzero_parity_vllm_client,
load_dreamzero_upstream_modules,
normalize_dreamzero_metadata,
)
from tests.helpers.mark import hardware_test
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
EXPECTED_METADATA = {
"image_resolution": [180, 320],
"n_external_cameras": 2,
"needs_wrist_camera": True,
"needs_stereo_camera": False,
"needs_session_id": True,
"action_space": "joint_position",
}
@hardware_test(res={"cuda": "H100"}, num_cards=2)
def test_dreamzero_openpi_upstream_parity(tmp_path: Path) -> None:
"""Assert vLLM OpenPI actions match the upstream DreamZero reference server."""
load_dreamzero_upstream_modules()
upstream_log = tmp_path / "dreamzero_upstream.log"
with DreamZeroUpstreamServer(log_path=upstream_log) as upstream_client:
upstream_metadata, upstream_outputs = upstream_client.collect_parity_outputs()
assert_dreamzero_logs_clean(upstream_log)
assert_dreamzero_upstream_log_matches_vllm_baseline(upstream_log)
with dreamzero_parity_vllm_client() as vllm_client:
vllm_metadata, vllm_outputs = vllm_client.collect_parity_outputs()
assert normalize_dreamzero_metadata(upstream_metadata) == EXPECTED_METADATA
assert normalize_dreamzero_metadata(vllm_metadata) == EXPECTED_METADATA
for idx, (vllm_action, upstream_action) in enumerate(zip(vllm_outputs, upstream_outputs, strict=True)):
np.testing.assert_allclose(
vllm_action,
upstream_action,
rtol=1e-2,
atol=1e-3,
err_msg=f"OpenPI step {idx} output mismatch",
)
@@ -1,97 +0,0 @@
from __future__ import annotations
import json
from pathlib import Path
import pytest
from benchmarks.accuracy.text_to_image.gbench import main as gbench_main
from tests.e2e.accuracy.helpers import infer_model_label, reset_artifact_dir
from tests.helpers.mark import hardware_test
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
@pytest.mark.benchmark
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@pytest.mark.skip(reason="#3256")
def test_gebench_h100_smoke(
gebench_accuracy_servers,
accuracy_artifact_root: Path,
gebench_dataset_root: Path,
accuracy_workers: int,
) -> None:
model_label = infer_model_label(gebench_accuracy_servers.generate_params.model).lower()
output_root = reset_artifact_dir(accuracy_artifact_root / f"gebench_{model_label}")
with gebench_accuracy_servers.generate_server() as generate_server:
for data_type in ("type3", "type4"):
assert (
gbench_main(
[
"generate",
"--dataset-root",
str(gebench_dataset_root),
"--output-root",
str(output_root),
"--base-url",
f"http://{generate_server.host}:{generate_server.port}",
"--model",
generate_server.model,
"--data-type",
data_type,
"--width",
"768",
"--height",
"576",
"--output-compression",
"98",
"--num-inference-steps",
"8",
"--workers",
str(accuracy_workers),
]
)
== 0
)
with gebench_accuracy_servers.judge_server() as judge_server:
for data_type in ("type3", "type4"):
assert (
gbench_main(
[
"evaluate",
"--dataset-root",
str(gebench_dataset_root),
"--output-root",
str(output_root),
"--data-type",
data_type,
"--judge-base-url",
f"http://{judge_server.host}:{judge_server.port}",
"--judge-model",
judge_server.model,
"--judge-api-key",
"EMPTY",
"--workers",
str(accuracy_workers),
]
)
== 0
)
assert gbench_main(["summarize", "--output-root", str(output_root)]) == 0
summary = json.loads((output_root / "summary.json").read_text(encoding="utf-8"))
assert "generation" in summary
assert "evaluation" in summary
for data_type in ("type3", "type4"):
assert data_type in summary["generation"]["by_type"]
assert summary["generation"]["by_type"][data_type]["count"] > 0
assert data_type in summary["evaluation"]["by_type"]
assert summary["evaluation"]["by_type"][data_type]["count"] > 0
assert summary["evaluation"]["overall_mean"] >= 0.45
assert summary["evaluation"]["by_type"]["type3"]["overall_mean"] >= 0.45
assert summary["evaluation"]["by_type"]["type4"]["overall_mean"] >= 0.45
@@ -1,115 +0,0 @@
from __future__ import annotations
import json
from pathlib import Path
import pytest
from benchmarks.accuracy.image_to_image.gedit_bench import GROUPS
from benchmarks.accuracy.image_to_image.gedit_bench import main as gedit_main
from tests.e2e.accuracy.helpers import infer_model_label, reset_artifact_dir
from tests.helpers.mark import hardware_test
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
@pytest.mark.benchmark
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@pytest.mark.skip(reason="#3257")
def test_gedit_bench_h100_smoke(
gedit_accuracy_servers,
accuracy_artifact_root: Path,
gedit_dataset_root: Path,
gedit_samples_per_group: int,
accuracy_workers: int,
) -> None:
model_label = infer_model_label(gedit_accuracy_servers.generate_params.model).lower()
output_root = reset_artifact_dir(accuracy_artifact_root / f"gedit_results_{model_label}")
score_root = reset_artifact_dir(accuracy_artifact_root / f"gedit_scores_{model_label}")
model_name = model_label
with gedit_accuracy_servers.generate_server() as generate_server:
assert (
gedit_main(
[
"generate",
"--dataset-ref",
str(gedit_dataset_root),
"--output-root",
str(output_root),
"--base-url",
f"http://{generate_server.host}:{generate_server.port}",
"--model",
generate_server.model,
"--model-name",
model_name,
"--task-type",
"all",
"--instruction-language",
"all",
"--samples-per-group",
str(gedit_samples_per_group),
"--workers",
str(accuracy_workers),
]
)
== 0
)
with gedit_accuracy_servers.judge_server() as judge_server:
assert (
gedit_main(
[
"evaluate",
"--dataset-ref",
str(gedit_dataset_root),
"--output-root",
str(output_root),
"--model-name",
model_name,
"--save-dir",
str(score_root),
"--task-type",
"all",
"--instruction-language",
"all",
"--judge-base-url",
f"http://{judge_server.host}:{judge_server.port}",
"--judge-model",
judge_server.model,
"--judge-api-key",
"EMPTY",
"--samples-per-group",
str(gedit_samples_per_group),
"--workers",
str(accuracy_workers),
]
)
== 0
)
csv_path = score_root / f"{model_name}_all_all_vie_score.csv"
assert gedit_main(["summarize", "--csv-path", str(csv_path), "--language", "all"]) == 0
summary_path = score_root / f"{model_name}_all_all_summary.json"
summary = json.loads(summary_path.read_text(encoding="utf-8"))
assert set(summary["languages"]) == {"en", "cn"}
for language in ["en", "cn"]:
language_summary = summary["languages"][language]
assert language_summary["overall"]["count"] is not None
assert language_summary["intersection"]["count"] is not None
assert language_summary["overall"]["Q_SC"] is not None
assert language_summary["overall"]["Q_PQ"] is not None
assert language_summary["overall"]["Q_O"] is not None
for group in GROUPS:
group_summary = language_summary["by_group"][group]
assert set(group_summary) == {"count", "Q_SC", "Q_PQ", "Q_O"}
assert summary["languages"]["en"]["overall"]["Q_SC"] >= 6.95
assert summary["languages"]["en"]["overall"]["Q_PQ"] >= 5.8
assert summary["languages"]["en"]["overall"]["Q_O"] >= 6.15
assert summary["languages"]["cn"]["overall"]["Q_SC"] >= 6.9
assert summary["languages"]["cn"]["overall"]["Q_PQ"] >= 5.7
assert summary["languages"]["cn"]["overall"]["Q_O"] >= 6.1
+6 -4
View File
@@ -28,6 +28,7 @@ from tests.e2e.accuracy.helpers import (
model_output_dir,
)
from tests.helpers.mark import hardware_test
from tests.helpers.media import get_asset_path
from tests.helpers.runtime import OmniRunner, OmniServer
from vllm_omni.diffusion.models.hunyuan_image3.prompt_utils import (
build_prompt_tokens,
@@ -94,6 +95,7 @@ def _default_ar_dit_devices() -> tuple[str, str]:
AR_DEVICES, DIT_DEVICES = _default_ar_dit_devices()
MODEL_NAME = "tencent/HunyuanImage-3.0-Instruct"
HUNYUAN_IMAGE_REF_PATH = get_asset_path("hunyuan/hunyuan_image_ref.png")
NUM_INFERENCE_STEPS = 50
GUIDANCE_SCALE = 2.5
@@ -414,7 +416,7 @@ def _run_online(stage_configs_path: str, output_path: Path) -> tuple[Image.Image
torch.accelerator.device_count() < AR_TP_SIZE + DIT_TP_SIZE,
reason=f"Needs {AR_TP_SIZE + DIT_TP_SIZE}+ GPUs ({AR_TP_SIZE} AR + {DIT_TP_SIZE} DiT)",
)
def test_image_to_image_alignment_online(accuracy_artifact_root: Path, accuracy_assets_root: Path) -> None:
def test_image_to_image_alignment_online(accuracy_artifact_root: Path) -> None:
"""Online API test: same pipeline, same seed as offline → PSNR >= 10 dB."""
if importlib.util.find_spec("FlagEmbedding") is None:
raise ImportError("Missing dependency: FlagEmbedding\nInstall with: pip install FlagEmbedding")
@@ -431,7 +433,7 @@ def test_image_to_image_alignment_online(accuracy_artifact_root: Path, accuracy_
scorer = SemanticSimilarityScorer()
clip_scorer = CLIPScorer()
cot_results = scorer.text_similarity(online_cot, COT_REF)
image_ref = Image.open(str(accuracy_assets_root / "hunyuan_image_ref.png")).convert("RGB")
image_ref = Image.open(HUNYUAN_IMAGE_REF_PATH).convert("RGB")
image_clip_score = clip_scorer.image_image_score(online_image, image_ref)
ssim_value, psnr_value = compute_image_ssim_psnr(prediction=online_image, reference=image_ref, compare_mode="RGB")
@@ -529,7 +531,7 @@ def _run_dit_model(
torch.accelerator.device_count() < AR_TP_SIZE + DIT_TP_SIZE,
reason=f"Needs {AR_TP_SIZE + DIT_TP_SIZE}+ GPUs ({AR_TP_SIZE} AR + {DIT_TP_SIZE} DiT)",
)
def test_image_to_image_alignment(accuracy_artifact_root: Path, accuracy_assets_root: Path) -> None:
def test_image_to_image_alignment(accuracy_artifact_root: Path) -> None:
if importlib.util.find_spec("FlagEmbedding") is None:
raise ImportError("Missing dependency: FlagEmbedding\nInstall with: pip install FlagEmbedding")
from tabulate import tabulate # lazy import
@@ -545,7 +547,7 @@ def test_image_to_image_alignment(accuracy_artifact_root: Path, accuracy_assets_
scorer = SemanticSimilarityScorer()
clip_scorer = CLIPScorer()
cot_results = scorer.text_similarity(omni_cot, COT_REF)
image_ref = Image.open(str(accuracy_assets_root / "hunyuan_image_ref.png")).convert("RGB")
image_ref = Image.open(HUNYUAN_IMAGE_REF_PATH).convert("RGB")
image_clip_score = clip_scorer.image_image_score(omni_image, image_ref)
ssim_value, psnr_value = compute_image_ssim_psnr(prediction=omni_image, reference=image_ref, compare_mode="RGB")
@@ -15,6 +15,7 @@ from PIL import Image
from tests.e2e.accuracy.helpers import assert_images_pixel_close, assert_similarity, model_output_dir
from tests.helpers.mark import hardware_test
from tests.helpers.media import get_asset_path
from tests.helpers.runtime import OmniServer
pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]
@@ -33,8 +34,8 @@ P99_THRESHOLD = 3e-1
SSIM_THRESHOLD = 0.97
PSNR_THRESHOLD = 30.0
BASELINE_PATH = get_asset_path("hunyuan/hunyuan_baseline.png")
_REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent
BASELINE_PATH = _REPO_ROOT / "tests" / "assets" / "hunyuan" / "hunyuan_baseline.png"
_OFFLINE_SCRIPT = _REPO_ROOT / "examples" / "offline_inference" / "hunyuan_image3" / "end2end.py"
# DiT-only deploy config with trust_remote_code (based on hunyuan_image3_dit.yaml).
+3 -3
View File
@@ -14,6 +14,7 @@ from benchmarks.accuracy.common import decode_base64_image, pil_to_png_bytes
from tests.e2e.accuracy.helpers import assert_images_pixel_close, assert_similarity, model_output_dir
from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup
from tests.helpers.mark import hardware_test
from tests.helpers.media import get_asset_path
from tests.helpers.runtime import OmniServer
pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]
@@ -46,10 +47,9 @@ EDIT_2511_PROMPT = (
"(帽子紧贴头顶,未贴合部分可调整帽子摆放角度达到完全贴合), 头部与身体过渡必须自然,无拼接痕迹,"
"无额外元素出现在头部四周,纯白背景,不要出现绿色圆圈"
)
EDIT_2511_ASSET_DIR = Path(__file__).resolve().parent / "assets"
EDIT_2511_IMAGE_PATHS = [
EDIT_2511_ASSET_DIR / "qwen_image_edit_2511_test1.png",
EDIT_2511_ASSET_DIR / "qwen_image_edit_2511_test2.png",
get_asset_path("qwen_image_edit/qwen_image_edit_2511_test1.png"),
get_asset_path("qwen_image_edit/qwen_image_edit_2511_test2.png"),
]
PROMPT_SINGLE_IMAGE = "The input is a 2D cartoon bear mascot. Restyle it into a painterly oil artwork with warm colors while preserving the main structure."
@@ -0,0 +1,184 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Launch the upstream DreamZero websocket server with `torch.compile` disabled.
This wrapper is meant for formal parity tests against `vllm serve --omni`.
It monkeypatches `torch.compile` before importing DreamZero modules so that
all import-time decorators and `post_initialize()` compile calls become eager.
For the current DreamZero port baseline we also disable all upstream DiT cache
and step-skipping behavior so the reference server matches vLLM's current
eager/no-skip implementation:
- `ENABLE_DIT_CACHE=false`
- `NUM_DIT_STEPS=16`
- `DYNAMIC_CACHE_SCHEDULE=false`
In our CI/dev environment we also do not install Transformer Engine or
FlashAttention, so upstream's default `socket_test_optimized_AR.main()` path
(`ATTENTION_BACKEND=TE` and then fallback to `FA2`) cannot execute. To keep the
formal server-vs-server test runnable without pulling in those heavyweight
optional deps, this wrapper reproduces upstream `main()` but pins attention to
PyTorch SDPA (`ATTENTION_BACKEND=torch`) for this subprocess only.
Usage:
PYTHONPATH="${DREAMZERO_REPO}" \\
.venv/bin/python -m torch.distributed.run --standalone --nproc_per_node=2 \\
tests/e2e/accuracy/upstream_mock/upstream_socket_server_no_compile.py --port 18081 \\
--model_path "${DREAMZERO_REPO}/checkpoints/dreamzero"
"""
from __future__ import annotations
import os
import sys
from pathlib import Path
import torch
import torch.nn.functional as F
def _identity_compile(*args, **kwargs):
if args and callable(args[0]) and len(args) == 1 and not kwargs:
return args[0]
def deco(fn):
return fn
return deco
torch.compile = _identity_compile
os.environ.setdefault("NO_ALBUMENTATIONS_UPDATE", "1")
DREAMZERO_REPO_ENV = os.environ.get("DREAMZERO_REPO")
if not DREAMZERO_REPO_ENV:
raise RuntimeError("Set DREAMZERO_REPO to an upstream DreamZero checkout before launching this helper.")
DREAMZERO_REPO = Path(DREAMZERO_REPO_ENV).expanduser()
if str(DREAMZERO_REPO) not in sys.path:
sys.path.insert(0, str(DREAMZERO_REPO))
import socket_test_optimized_AR as upstream # noqa: E402
import tyro # noqa: E402
from groot.vla.model.dreamzero.modules import attention as upstream_attention # noqa: E402
from groot.vla.model.dreamzero.modules import wan2_1_submodule as upstream_submodule # noqa: E402
from groot.vla.model.dreamzero.modules import wan_video_dit as upstream_wan_video_dit # noqa: E402
def _torch_varlen_flash_attention(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
q_lens: torch.Tensor | None = None,
k_lens: torch.Tensor | None = None,
dropout_p: float = 0.0,
causal: bool = False,
dtype: torch.dtype = torch.bfloat16,
**_: object,
) -> torch.Tensor:
if q_lens is not None or k_lens is not None:
upstream_attention.warnings.warn(
"Padding mask is disabled in the test-only SDPA fallback.",
)
out_dtype = q.dtype
q = q.transpose(1, 2).to(dtype)
k = k.transpose(1, 2).to(dtype)
v = v.transpose(1, 2).to(dtype)
out = F.scaled_dot_product_attention(
q,
k,
v,
attn_mask=None,
is_causal=causal,
dropout_p=dropout_p,
)
return out.transpose(1, 2).contiguous().to(out_dtype)
upstream_attention.flash_attention = _torch_varlen_flash_attention
upstream_submodule.flash_attention = _torch_varlen_flash_attention
upstream_wan_video_dit.FLASH_ATTN_3_AVAILABLE = False
upstream_wan_video_dit.FLASH_ATTN_2_AVAILABLE = False
upstream_wan_video_dit.SAGE_ATTN_AVAILABLE = False
def main(args: upstream.Args) -> None:
os.environ["ENABLE_DIT_CACHE"] = "false"
os.environ["ATTENTION_BACKEND"] = "torch"
os.environ["NUM_DIT_STEPS"] = "16"
os.environ["DYNAMIC_CACHE_SCHEDULE"] = "false"
torch._dynamo.config.recompile_limit = 800
embodiment_tag = "oxe_droid"
model_path = args.model_path
device_mesh = upstream.init_mesh()
rank = upstream.dist.get_rank()
timeout_delta = upstream.datetime.timedelta(seconds=args.timeout_seconds)
signal_group = upstream.dist.new_group(backend="gloo", timeout=timeout_delta)
upstream.logger.info("Rank %s initialized signal_group (gloo)", rank)
policy = upstream.GrootSimPolicy(
embodiment_tag=upstream.EmbodimentTag(embodiment_tag),
model_path=model_path,
device="cuda" if torch.cuda.is_available() else "cpu",
device_mesh=device_mesh,
)
hostname = upstream.socket.gethostname()
local_ip = upstream.socket.gethostbyname(hostname)
if rank == 0:
upstream.logging.info("Creating server (host: %s, ip: %s)", hostname, local_ip)
parent_dir = os.path.dirname(model_path)
date_suffix = upstream.datetime.datetime.now().strftime("%Y%m%d")
checkpoint_name = os.path.basename(model_path)
output_dir = os.path.join(
parent_dir,
f"real_world_eval_gen_{date_suffix}_{args.index}",
checkpoint_name,
)
os.makedirs(output_dir, exist_ok=True)
upstream.logging.info("Videos will be saved to: %s", output_dir)
else:
output_dir = None
upstream.logging.info("Rank %s starting as worker for distributed inference...", rank)
wrapper_policy = upstream.ARDroidRoboarenaPolicy(
groot_policy=policy,
signal_group=signal_group,
output_dir=output_dir,
)
server_config = upstream.PolicyServerConfig(
image_resolution=(180, 320),
needs_wrist_camera=True,
n_external_cameras=2,
needs_stereo_camera=False,
needs_session_id=True,
action_space="joint_position",
)
if rank == 0:
upstream.logging.info("Using roboarena policy server interface")
upstream.logging.info("Server config: %s", server_config)
roboarena_server = upstream.RoboarenaServer(
policy=wrapper_policy,
server_config=server_config,
host="0.0.0.0",
port=args.port,
)
roboarena_server.serve_forever()
else:
worker = upstream.DistributedWorker(
policy=policy,
signal_group=signal_group,
)
upstream.asyncio.run(worker.run())
if __name__ == "__main__":
upstream.logging.basicConfig(level=upstream.logging.INFO, force=True)
main(tyro.cli(upstream.Args))
+89
View File
@@ -0,0 +1,89 @@
"""
Conftest for ComfyUI-vLLM-Omni tests.
This module sets up the test environment by:
1. Adding the ComfyUI plugin to Python path
2. Mocking comfy_api.input module (AudioInput, VideoInput) since comfyui is not installed
3. Mocking comfy_extras.nodes_audio module
"""
import os
import sys
from types import ModuleType, SimpleNamespace
from typing import BinaryIO, TypedDict
def pytest_configure(config):
"""
Called after command line options have been parsed and before test collection.
This is the right place to set up sys.path and mock modules.
"""
_setup_comfyui_test_environment()
def _setup_comfyui_test_environment():
"""Set up the test environment for ComfyUI plugin testing."""
# === Add ComfyUI plugin path to allow importing comfyui_vllm_omni ===
_COMFYUI_PLUGIN_PATH = os.path.abspath(
os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "apps", "ComfyUI-vLLM-Omni")
)
if not os.path.isdir(_COMFYUI_PLUGIN_PATH):
raise FileNotFoundError(
f"ComfyUI plugin not found at {_COMFYUI_PLUGIN_PATH}. "
"If it is moved elsewhere, please update the path in this conftest.py."
)
if _COMFYUI_PLUGIN_PATH not in sys.path:
sys.path.insert(0, _COMFYUI_PLUGIN_PATH)
# Import torch after changing import paths. (To be used later)
import torch
# === Mock ComfyUI internal modules (comfy_api & comfy_extras) and "import" them to sys.module ===
class AudioInput(TypedDict):
"""Mock AudioInput TypedDict from comfy_api.input"""
waveform: torch.Tensor # Shape: (B, C, T)
sample_rate: int
class VideoInput:
"""Mock VideoInput class from comfy_api.input"""
def __init__(self, data: bytes = b"mock_video_data"):
self._data = data
def save_to(self, file: str | BinaryIO):
"""Save video data to file or file-like object."""
if isinstance(file, str):
print("Called VideoInput.save_to with file path. Saving to a path is no-op in tests.")
else:
file.write(self._data)
mock_comfy_api = ModuleType("comfy_api")
mock_comfy_api_input = ModuleType("comfy_api.input")
mock_comfy_api_input.AudioInput = AudioInput
mock_comfy_api_input.VideoInput = VideoInput
mock_comfy_api.input = mock_comfy_api_input
mock_comfy_api_latest = ModuleType("comfy_api.latest")
mock_comfy_api_latest.Types = SimpleNamespace(VideoComponents=lambda **kwargs: kwargs)
mock_comfy_api_latest.InputImpl = SimpleNamespace(
VideoFromComponents=lambda _: VideoInput(b"mock_video_from_components")
)
mock_comfy_api.latest = mock_comfy_api_latest
def mock_load(_: str | BinaryIO):
"""Mock nodes_audio.load that returns a waveform tensor (channels, samples) and sample rate."""
waveform = torch.zeros((1, 24000), dtype=torch.float32)
sample_rate = 24000
return waveform, sample_rate
mock_comfy_extras = ModuleType("comfy_extras")
mock_nodes_audio = ModuleType("comfy_extras.nodes_audio")
mock_nodes_audio.load = mock_load
mock_comfy_extras.nodes_audio = mock_nodes_audio
# Install mock modules BEFORE importing any comfyui_vllm_omni code
sys.modules["comfy_api"] = mock_comfy_api
sys.modules["comfy_api.input"] = mock_comfy_api_input
sys.modules["comfy_api.latest"] = mock_comfy_api_latest
sys.modules["comfy_extras"] = mock_comfy_extras
sys.modules["comfy_extras.nodes_audio"] = mock_nodes_audio
@@ -0,0 +1,788 @@
"""
Integration tests for ComfyUI nodes that use the Omni API client, with a mocked AsyncOmni and a real API server running in a background process.
These tests cover the integration between ComfyUI node and the API server, without actual model inference logic.
It ensures that
1. Changes made to the API (e.g., request and response formats) do not break the ComfyUI frontend that use it.
2. The sampling parameters are correctly passed from the node to AsyncOmni through the API layer.
"""
from __future__ import annotations
import multiprocessing
import time
import traceback
from collections.abc import Iterable, Sequence
from enum import Enum
from types import SimpleNamespace
from typing import Any, NamedTuple
import pytest
import requests
import torch
from comfy_api.input import AudioInput, VideoInput
from comfyui_vllm_omni.nodes import (
VLLMOmniGenerateImage,
VLLMOmniGenerateVideo,
VLLMOmniTTS,
VLLMOmniUnderstanding,
VLLMOmniVoiceClone,
)
from comfyui_vllm_omni.utils.types import AutoregressionSamplingParams, DiffusionSamplingParams, WanModelSpecificParams
from PIL import Image
from pytest_mock import MockerFixture
from vllm import SamplingParams
from vllm.outputs import CompletionOutput, RequestOutput
from vllm_omni.entrypoints.async_omni import AsyncOmni as RealAsyncOmni
from vllm_omni.entrypoints.cli.serve import OmniServeCommand
from vllm_omni.inputs.data import OmniSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class ServerCase(NamedTuple):
"""Parametrizing the model to serve."""
served_model: str
stage_list: list
stage_configs: list[Any]
outputs: list[OmniRequestOutput]
class SamplingCase(NamedTuple):
"""Parametrizing the input sampling parameters."""
kind: SamplingKind
sampling_params: dict | list[dict] | None
lora: dict | None = None
class SamplingKind(str, Enum):
IMAGE_NONE = "image_none"
IMAGE_DIFFUSION_SINGLE = "image_diffusion_single"
UNDERSTANDING_NONE = "understanding_none"
UNDERSTANDING_AR_LIST = "understanding_ar_list"
TTS_NONE = "tts_none"
TTS_DIFFUSION_SINGLE = "tts_diffusion_single"
VIDEO_NONE = "video_none"
VIDEO_DIFFUSION_SINGLE = "video_diffusion_single"
# Pre-defined arguments to be used in function calls during the tests
IMAGE_WIDTH = 64
IMAGE_HEIGHT = 64
VIDEO_WIDTH = 32
VIDEO_HEIGHT = 32
VIDEO_FPS = 8
VIDEO_NUM_FRAMES = 5
DIFFUSION_SINGLE_SAMPLING_PARAMS = DiffusionSamplingParams(
{
"n": 2,
"num_inference_steps": 30,
"guidance_scale": 6.0,
"true_cfg_scale": 1.5,
}
)
DIFFUSION_VIDEO_SINGLE_SAMPLING_PARAMS = DiffusionSamplingParams(
{
"num_inference_steps": 30,
"guidance_scale": 6.0,
"true_cfg_scale": 1.5,
}
)
AR_LIST_SAMPLING_PARAMS = [
AutoregressionSamplingParams(
{
"max_tokens": 64,
"temperature": 0.6,
"top_p": 0.9,
"repetition_penalty": 1.0,
"seed": 21,
}
),
AutoregressionSamplingParams(
{
"max_tokens": 96,
"temperature": 0.75,
"top_p": 0.85,
"repetition_penalty": 1.05,
"seed": 22,
}
),
AutoregressionSamplingParams(
{
"max_tokens": 128,
"temperature": 0.8,
"top_p": 0.8,
"repetition_penalty": 1.1,
"seed": 23,
}
),
]
VIDEO_MODEL_PARAMS = WanModelSpecificParams({"guidance_scale_2": 5.0, "boundary_ratio": 0.98, "flow_shift": 12.0})
LORA_PARAMS = {"local_path": "test_lora_path", "name": "test_name", "scale": 0.7, "int_id": 10}
def _build_image_output(size: tuple[int, int] = (IMAGE_WIDTH, IMAGE_HEIGHT), color: str = "red") -> Image.Image:
return Image.new("RGB", size, color=color)
def _build_text_output(text: str = "This is a test response.") -> OmniRequestOutput:
completion_output = CompletionOutput(
index=0,
text=text,
token_ids=[1, 2, 3],
cumulative_logprob=0.0,
logprobs=None,
finish_reason="stop",
stop_reason=None,
)
request_output = RequestOutput(
request_id="test_req_text",
prompt="test prompt",
prompt_token_ids=[1, 2, 3],
prompt_logprobs=None,
outputs=[completion_output],
finished=True,
metrics=None,
lora_request=None,
)
return OmniRequestOutput(
request_id="test_req_text",
finished=True,
final_output_type="text",
request_output=request_output,
)
def _build_audio_chat_output(num_samples: int = 24000) -> OmniRequestOutput:
completion_output = CompletionOutput(
index=0,
text="",
token_ids=[],
cumulative_logprob=0.0,
logprobs=None,
finish_reason="stop",
stop_reason=None,
)
completion_output.multimodal_output = {"audio": [torch.zeros(1, num_samples)]}
request_output = RequestOutput(
request_id="test_req_audio_chat",
prompt="test prompt",
prompt_token_ids=[1, 2, 3],
prompt_logprobs=None,
outputs=[completion_output],
finished=True,
metrics=None,
lora_request=None,
)
return OmniRequestOutput(
request_id="test_req_audio_chat",
finished=True,
final_output_type="audio",
request_output=request_output,
)
def _build_audio_speech_output(num_samples: int = 24000) -> OmniRequestOutput:
return OmniRequestOutput.from_diffusion(
request_id="test_req_audio_speech",
images=[],
multimodal_output={"audio": torch.zeros(num_samples), "sr": 24000},
final_output_type="audio",
)
def _build_diffusion_image_output_for_images_endpoint() -> OmniRequestOutput:
return OmniRequestOutput.from_diffusion(
request_id="test_req_img_dalle",
images=[_build_image_output()],
final_output_type="image",
)
def _build_diffusion_video_output() -> OmniRequestOutput:
# Small video: VIDEO_NUM_FRAMES frames of (VIDEO_HEIGHT x VIDEO_WIDTH) RGB, shape (F, H, W, C)
video_frames = torch.zeros((VIDEO_NUM_FRAMES, VIDEO_HEIGHT, VIDEO_WIDTH, 3), dtype=torch.float32)
return OmniRequestOutput.from_diffusion(
request_id="test_req_video",
images=[video_frames],
final_output_type="video",
)
def _build_diffusion_image_output_for_chat_endpoint() -> OmniRequestOutput:
request_output = SimpleNamespace(
images=[_build_image_output(color="blue")],
finished=True,
)
return OmniRequestOutput(
request_id="test_req_img_chat",
finished=True,
final_output_type="image",
request_output=request_output,
)
def _assert_sampling_param_values(
received: OmniSamplingParams, expected: dict[str, Any], expected_lora: dict | None = None
):
for key, expected_value in expected.items():
actual_value = getattr(received, key)
assert actual_value == expected_value, (
f"Expected sampling param '{key}'={expected_value}, got {actual_value}. The received sampling params: {received}"
)
if expected_lora:
assert received.lora_request.lora_name == expected_lora["name"], (
f"Expected lora name={(expected_lora['name'])}, got {received.lora_request.lora_name}. The received sampling params: {received}"
)
assert received.lora_request.lora_int_id == expected_lora["int_id"], (
f"Expected lora int_id={expected_lora['int_id']}, got {received.lora_request.lora_int_id}. The received sampling params: {received}"
)
assert received.lora_request.lora_path == expected_lora["local_path"], (
f"Expected lora path={expected_lora['local_path']}, got {received.lora_request.lora_path}. The received sampling params: {received}"
)
assert received.lora_scale == expected_lora["scale"], (
f"Expected lora scale={expected_lora['scale']}, got {received.lora_scale}. The received sampling params: {received}"
)
def _assert_model_param_values(received: OmniSamplingParams, expected: dict):
for key, expected_value in expected.items():
try:
actual_value = getattr(received, key)
expected_param_name = key
except AttributeError:
actual_value = received.extra_args.get(key, None)
expected_param_name = f'extra_args["{key}"]'
assert actual_value == expected_value, (
f"Expected model param '{expected_param_name}'={expected_value}, got {actual_value}. The received sampling params: {received}"
)
def _make_stage_config(
stage_type: str,
*,
is_comprehension: bool = False,
model_stage: str | None = None,
):
engine_args = SimpleNamespace()
if model_stage is not None:
engine_args.model_stage = model_stage
return SimpleNamespace(
stage_type=stage_type,
is_comprehension=is_comprehension,
engine_args=engine_args,
)
def _stage_type(stage: Any) -> str | None:
return getattr(stage, "stage_type", None)
def _build_output_modalities(stage_configs: list[Any]) -> list[str]:
modalities: list[str] = []
for stage in stage_configs:
final_output = getattr(stage, "final_output", False)
final_output_type = getattr(stage, "final_output_type", None)
if final_output and isinstance(final_output_type, str):
modalities.append(final_output_type)
if modalities:
return modalities
if any(_stage_type(stage) == "diffusion" for stage in stage_configs):
return ["image"]
return ["text", "audio"]
def _build_mock_outputs(outputs: Iterable[OmniRequestOutput], sampling_case: SamplingCase, server_case: ServerCase):
async def _mock_generate(*args, **kwargs):
received_sampling_params_list: Sequence[OmniSamplingParams] | None = (
args[2] if len(args) > 2 else kwargs.get("sampling_params_list")
)
assert received_sampling_params_list is not None, (
"In the current codebase, the API layer always provides not-None sampling parameter list when calling `AsyncOmni.generate`"
"This test also uses this assumption for now."
"If this assertion fails, it means the API layer has changed and this test needs to be updated accordingly."
"It does not necessarily mean there is a bug, because `AsyncOmni.generate` does allow sampling_params_list to be None."
)
assert isinstance(received_sampling_params_list, Sequence), "sampling_params_list should be a Sequence"
if sampling_case.kind is SamplingKind.IMAGE_NONE:
assert len(received_sampling_params_list) == 1
_assert_sampling_param_values(
received_sampling_params_list[0],
{
"width": IMAGE_WIDTH,
"height": IMAGE_HEIGHT,
},
)
elif sampling_case.kind is SamplingKind.IMAGE_DIFFUSION_SINGLE:
assert len(received_sampling_params_list) == 1
expected = DIFFUSION_SINGLE_SAMPLING_PARAMS.copy()
expected["num_outputs_per_prompt"] = expected.pop("n") # convert from n to num_outputs_per_prompt
_assert_sampling_param_values(
received_sampling_params_list[0],
{
"width": IMAGE_WIDTH,
"height": IMAGE_HEIGHT,
**expected,
},
LORA_PARAMS,
)
elif sampling_case.kind is SamplingKind.UNDERSTANDING_NONE:
assert len(received_sampling_params_list) == 3
elif sampling_case.kind is SamplingKind.UNDERSTANDING_AR_LIST:
assert len(received_sampling_params_list) == 3
for i, expected in enumerate(AR_LIST_SAMPLING_PARAMS):
_assert_sampling_param_values(received_sampling_params_list[i], expected)
elif sampling_case.kind in {SamplingKind.TTS_NONE, SamplingKind.TTS_DIFFUSION_SINGLE}:
assert len(received_sampling_params_list) == 1
elif sampling_case.kind is SamplingKind.VIDEO_NONE:
assert len(received_sampling_params_list) == 1
_assert_sampling_param_values(
received_sampling_params_list[0],
{
"width": VIDEO_WIDTH,
"height": VIDEO_HEIGHT,
"num_frames": VIDEO_NUM_FRAMES,
"fps": VIDEO_FPS,
},
)
elif sampling_case.kind is SamplingKind.VIDEO_DIFFUSION_SINGLE:
assert len(received_sampling_params_list) == 1
expected = DIFFUSION_VIDEO_SINGLE_SAMPLING_PARAMS.copy()
# expected["num_outputs_per_prompt"] = expected.pop("n") # convert from n to num_outputs_per_prompt
_assert_sampling_param_values(
received_sampling_params_list[0],
{
"width": VIDEO_WIDTH,
"height": VIDEO_HEIGHT,
"num_frames": VIDEO_NUM_FRAMES,
"fps": VIDEO_FPS,
**expected,
},
LORA_PARAMS,
)
_assert_model_param_values(received_sampling_params_list[0], VIDEO_MODEL_PARAMS)
else:
raise AssertionError(f"Unknown sampling case: {sampling_case.kind}")
for output in outputs:
yield output
return _mock_generate
@pytest.fixture
def server_case(request) -> ServerCase:
return request.param
@pytest.fixture
def sampling_case(request) -> SamplingCase:
return request.param
@pytest.fixture
def mock_async_omni(
server_case: ServerCase,
sampling_case: SamplingCase,
monkeypatch: pytest.MonkeyPatch,
mocker: MockerFixture,
):
async def _mock_preprocess_chat(self, *args, **kwargs):
return ([{"role": "user", "content": "test"}], [{"prompt": "test prompt"}])
# Need to mock AsyncOmni itself (not only its generate method) because
# 1. The API layer uses its stage_list and stage_configs attributes
# 2. Its __init__ method has slow side effects (model & config loading).
mock_async_omni_cls = mocker.patch("vllm_omni.entrypoints.openai.api_server.AsyncOmni")
monkeypatch.setattr(
"vllm_omni.entrypoints.openai.serving_chat.OmniOpenAIServingChat._preprocess_chat",
_mock_preprocess_chat,
)
mock_instance = mocker.AsyncMock(spec=RealAsyncOmni)
mock_instance.generate = _build_mock_outputs(server_case.outputs, sampling_case, server_case)
mock_instance.stage_list = server_case.stage_list
mock_instance.stage_configs = server_case.stage_configs
mock_instance.output_modalities = _build_output_modalities(server_case.stage_configs)
mock_instance.default_sampling_params_list = [
SamplingParams() if _stage_type(stage) != "diffusion" else mocker.MagicMock()
for stage in server_case.stage_configs
]
mock_instance.errored = False
mock_instance.dead_error = RuntimeError("Mock engine error")
mock_instance.model_config = mocker.MagicMock(
max_model_len=4096,
io_processor_plugin=None,
allowed_local_media_path=None,
allowed_media_domains=None,
)
# Mimic Qwen3-TTS talker speaker config so CustomVoice validation passes.
mock_instance.model_config.hf_config = mocker.MagicMock()
mock_instance.model_config.hf_config.talker_config = mocker.MagicMock()
mock_instance.model_config.hf_config.talker_config.speaker_id = {"Vivian": 0}
mock_instance.io_processor = mocker.MagicMock()
mock_instance.input_processor = mocker.MagicMock()
mock_instance.shutdown = mocker.MagicMock()
mock_instance.get_vllm_config = mocker.AsyncMock(return_value=None)
mock_instance.get_supported_tasks = mocker.AsyncMock(return_value=["generate"])
mock_instance.get_tokenizer = mocker.AsyncMock(return_value=None)
mock_async_omni_cls.return_value = mock_instance
yield mock_async_omni_cls
@pytest.fixture
def api_server(unused_tcp_port_factory, server_case: ServerCase, mock_async_omni):
"""Set up a API server in background process from command line with parametrized model name and mocked AsyncOmni."""
parser = TrackingArgumentParser()
subparsers = parser.add_subparsers(dest="command")
cmd = OmniServeCommand()
cmd.subparser_init(subparsers)
port = unused_tcp_port_factory()
args = parser.parse_args(["serve", server_case.served_model, "--omni", "--port", str(port)])
def run_server():
try:
cmd.cmd(args)
except Exception:
traceback.print_exc()
server_process = multiprocessing.Process(target=run_server)
server_process.start()
# Wait for the server to be ready by polling the health endpoint.
wait_time = 30
wait_poll_interval = 1
for _ in range(wait_time // wait_poll_interval):
try:
response = requests.get(f"http://127.0.0.1:{port}/health")
if response.status_code == 200:
break
except requests.ConnectionError:
time.sleep(wait_poll_interval)
else:
if server_process.is_alive():
server_process.terminate()
server_process.join(timeout=5)
if server_process.is_alive():
server_process.kill()
server_process.join(timeout=5)
pytest.fail(f"API server failed to start within {wait_time} seconds")
yield f"http://127.0.0.1:{port}/v1"
if server_process.is_alive():
server_process.terminate()
server_process.join(timeout=10)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"server_case,model,image_input",
[
pytest.param(
ServerCase(
served_model="Tongyi-MAI/Z-Image-Turbo",
stage_list=["diffusion"],
stage_configs=[_make_stage_config("diffusion")],
outputs=[_build_diffusion_image_output_for_images_endpoint()],
),
"Tongyi-MAI/Z-Image-Turbo",
False,
id="text-to-image-dalle-endpoint",
),
pytest.param(
ServerCase(
served_model="ByteDance-Seed/BAGEL-7B-MoT",
stage_list=["diffusion"],
stage_configs=[_make_stage_config("diffusion")],
outputs=[_build_diffusion_image_output_for_chat_endpoint()],
),
"ByteDance-Seed/BAGEL-7B-MoT",
False,
id="text-to-image-bagel-chat-endpoint",
),
pytest.param(
ServerCase(
served_model="Qwen/Qwen-Image-Edit",
stage_list=["diffusion"],
stage_configs=[_make_stage_config("diffusion")],
outputs=[_build_diffusion_image_output_for_images_endpoint()],
),
"Qwen/Qwen-Image-Edit",
True,
id="image-to-image-dalle-endpoint",
),
pytest.param(
ServerCase(
served_model="ByteDance-Seed/BAGEL-7B-MoT",
stage_list=["diffusion"],
stage_configs=[_make_stage_config("diffusion")],
outputs=[_build_diffusion_image_output_for_chat_endpoint()],
),
"ByteDance-Seed/BAGEL-7B-MoT",
True,
id="image-to-image-bagel-chat-endpoint",
),
],
indirect=["server_case"],
)
@pytest.mark.parametrize(
"sampling_case",
[
pytest.param(
SamplingCase(kind=SamplingKind.IMAGE_NONE, sampling_params=None, lora=None), id="no-sampling-params"
),
pytest.param(
SamplingCase(
kind=SamplingKind.IMAGE_DIFFUSION_SINGLE,
sampling_params=DIFFUSION_SINGLE_SAMPLING_PARAMS,
lora=LORA_PARAMS,
),
id="single-diffusion-sampling-params",
),
],
indirect=["sampling_case"],
)
async def test_image_generation_node(api_server: str, model: str, image_input: bool, sampling_case: SamplingCase):
node = VLLMOmniGenerateImage()
kwargs = {
"url": api_server,
"model": model,
"prompt": "A beautiful sunset",
"width": IMAGE_WIDTH,
"height": IMAGE_HEIGHT,
}
if image_input:
kwargs["image"] = torch.zeros((1, IMAGE_WIDTH, IMAGE_HEIGHT, 3), dtype=torch.float32)
if sampling_case.sampling_params is not None:
kwargs["sampling_params"] = sampling_case.sampling_params
if sampling_case.lora:
kwargs["lora"] = sampling_case.lora
result = await node.generate(**kwargs)
assert isinstance(result, tuple)
assert len(result) == 1
assert isinstance(result[0], torch.Tensor)
assert result[0].shape == (1, IMAGE_WIDTH, IMAGE_HEIGHT, 3)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"server_case",
[
pytest.param(
ServerCase(
served_model="Qwen/Qwen2.5-Omni-7B",
stage_list=[
SimpleNamespace(is_comprehension=True, model_stage="llm"),
SimpleNamespace(is_comprehension=False, model_stage="llm"),
SimpleNamespace(is_comprehension=False, model_stage="llm"),
],
stage_configs=[
_make_stage_config("llm", is_comprehension=True, model_stage="thinker"),
_make_stage_config("llm", is_comprehension=False, model_stage="talker"),
_make_stage_config("llm", is_comprehension=False, model_stage="code2wav"),
],
outputs=[_build_audio_chat_output(), _build_text_output("Understanding response")],
),
id="multimodal-understanding",
)
],
indirect=["server_case"],
)
@pytest.mark.parametrize(
"sampling_case",
[
pytest.param(SamplingCase(kind=SamplingKind.UNDERSTANDING_NONE, sampling_params=None), id="no-sampling-params"),
pytest.param(
SamplingCase(kind=SamplingKind.UNDERSTANDING_AR_LIST, sampling_params=AR_LIST_SAMPLING_PARAMS),
id="ar-sampling-params-list",
),
],
indirect=["sampling_case"],
)
async def test_understanding_node(api_server: str, sampling_case: SamplingCase):
node = VLLMOmniUnderstanding()
image = torch.zeros((1, IMAGE_WIDTH, IMAGE_HEIGHT, 3), dtype=torch.float32)
video = VideoInput(b"mock_video_for_test") # type: ignore[reportAbstractUsage]
audio: AudioInput = {"waveform": torch.zeros((1, 1, 24000), dtype=torch.float32), "sample_rate": 24000}
text_response, audio_response = await node.generate(
url=api_server,
model="Qwen/Qwen2.5-Omni-7B",
prompt="Describe all modalities.",
image=image,
audio=audio,
video=video,
sampling_params=sampling_case.sampling_params,
output_text=True,
output_audio=True,
use_audio_in_video=True,
)
assert text_response == "Understanding response"
assert isinstance(audio_response, dict)
assert audio_response["sample_rate"] == 24000
assert isinstance(audio_response["waveform"], torch.Tensor)
assert audio_response["waveform"].shape == (1, 1, 24000)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"server_case,node_cls,call_kwargs",
[
pytest.param(
ServerCase(
served_model="Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
stage_list=["llm"],
stage_configs=[_make_stage_config("llm", model_stage="qwen3_tts")],
outputs=[_build_audio_speech_output()],
),
VLLMOmniTTS,
{
"model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
"input": "Hello from TTS test",
"voice": "Vivian",
"response_format": "wav",
"speed": 1.0,
"model_specific_params": None,
},
id="tts",
),
pytest.param(
ServerCase(
served_model="Qwen/Qwen3-TTS-12Hz-1.7B-Base",
stage_list=["llm"],
stage_configs=[_make_stage_config("llm", model_stage="qwen3_tts")],
outputs=[_build_audio_speech_output()],
),
VLLMOmniVoiceClone,
{
"model": "Qwen/Qwen3-TTS-12Hz-1.7B-Base",
"input": "Hello from voice clone test",
"voice": "Vivian",
"response_format": "wav",
"speed": 1.0,
"ref_audio": {"waveform": torch.zeros((1, 1, 24000), dtype=torch.float32), "sample_rate": 24000},
"ref_text": "Reference transcript",
"x_vector_only_mode": False,
"model_specific_params": None,
},
id="tts-voice-clone",
),
],
indirect=["server_case"],
)
@pytest.mark.parametrize(
"sampling_case",
[
pytest.param(SamplingCase(kind=SamplingKind.TTS_NONE, sampling_params=None), id="no-sampling-params"),
pytest.param(
SamplingCase(kind=SamplingKind.TTS_DIFFUSION_SINGLE, sampling_params=DIFFUSION_SINGLE_SAMPLING_PARAMS),
id="single-diffusion-sampling-params",
),
],
indirect=["sampling_case"],
)
async def test_tts_nodes(api_server: str, node_cls, call_kwargs: dict, sampling_case: SamplingCase):
node = node_cls()
actual_kwargs = dict(call_kwargs)
if sampling_case.sampling_params is not None:
actual_kwargs["model_specific_params"] = sampling_case.sampling_params
result = await node.generate(url=api_server, **actual_kwargs)
assert isinstance(result, tuple)
assert len(result) == 1
assert isinstance(result[0], dict)
assert result[0]["sample_rate"] == 24000
assert isinstance(result[0]["waveform"], torch.Tensor)
assert result[0]["waveform"].shape == (1, 1, 24000)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"server_case,model,image_input",
[
pytest.param(
ServerCase(
served_model="Wan-AI/Wan2.2-T2V-A14B-Diffusers",
stage_list=["diffusion"],
stage_configs=[{"stage_type": "diffusion"}],
outputs=[_build_diffusion_video_output()],
),
"Wan-AI/Wan2.2-T2V-A14B-Diffusers",
False,
id="text-to-video",
),
pytest.param(
ServerCase(
served_model="Wan-AI/Wan2.2-I2V-A14B-Diffusers",
stage_list=["diffusion"],
stage_configs=[{"stage_type": "diffusion"}],
outputs=[_build_diffusion_video_output()],
),
"Wan-AI/Wan2.2-I2V-A14B-Diffusers",
True,
id="image-to-video",
),
],
indirect=["server_case"],
)
@pytest.mark.parametrize(
"sampling_case",
[
pytest.param(SamplingCase(kind=SamplingKind.VIDEO_NONE, sampling_params=None), id="no-sampling-params"),
pytest.param(
SamplingCase(
kind=SamplingKind.VIDEO_DIFFUSION_SINGLE,
sampling_params=DIFFUSION_VIDEO_SINGLE_SAMPLING_PARAMS,
lora=LORA_PARAMS,
),
id="single-diffusion-sampling-params",
),
],
indirect=["sampling_case"],
)
async def test_video_generation_node(api_server: str, model: str, image_input: bool, sampling_case: SamplingCase):
node = VLLMOmniGenerateVideo()
kwargs = {
"url": api_server,
"model": model,
"prompt": "A beautiful sunset timelapse",
"negative_prompt": "",
"width": VIDEO_WIDTH,
"height": VIDEO_HEIGHT,
"fps": VIDEO_FPS,
"num_frames": VIDEO_NUM_FRAMES,
"model_params": VIDEO_MODEL_PARAMS,
}
if image_input:
kwargs["image"] = torch.zeros((1, VIDEO_HEIGHT, VIDEO_WIDTH, 3), dtype=torch.float32)
if sampling_case.sampling_params is not None:
kwargs["sampling_params"] = sampling_case.sampling_params
if sampling_case.lora:
kwargs["lora"] = sampling_case.lora
result = await node.generate(**kwargs)
assert isinstance(result, tuple)
assert len(result) == 1
assert isinstance(result[0], VideoInput)
@@ -16,7 +16,7 @@ exercises when it calls ``await self.engine.list_loras()`` before
dispatching a generation request.
Usage:
pytest tests/e2e/offline_inference/custom_pipeline/test_async_omni_collective_rpc.py -v -s
pytest tests/e2e/features/custom_pipeline/test_async_omni_collective_rpc.py -v -s
"""
from __future__ import annotations
@@ -34,12 +34,8 @@ from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
MODEL = "tiny-random/Qwen-Image"
CUSTOM_PIPELINE_CLASS = (
"tests.e2e.offline_inference.custom_pipeline.qwen_image_pipeline_with_logprob.QwenImagePipelineWithLogProbForTest"
)
WORKER_EXTENSION_CLASS = (
"tests.e2e.offline_inference.custom_pipeline.worker_extension.vLLMOmniColocateWorkerExtensionForTest"
)
CUSTOM_PIPELINE_CLASS = "tests.e2e.features.helpers.custom_pipeline.QwenImagePipelineWithLogProbForTest"
WORKER_EXTENSION_CLASS = "tests.e2e.features.helpers.custom_pipeline.vLLMOmniColocateWorkerExtensionForTest"
def _create_inline_engine() -> AsyncOmni:
@@ -24,12 +24,8 @@ from vllm_omni.entrypoints.async_omni import AsyncOmni
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
CUSTOM_PIPELINE_CLASS = (
"tests.e2e.offline_inference.custom_pipeline.qwen_image_pipeline_with_logprob.QwenImagePipelineWithLogProbForTest"
)
WORKER_EXTENSION_CLASS = (
"tests.e2e.offline_inference.custom_pipeline.worker_extension.vLLMOmniColocateWorkerExtensionForTest"
)
CUSTOM_PIPELINE_CLASS = "tests.e2e.features.helpers.custom_pipeline.QwenImagePipelineWithLogProbForTest"
WORKER_EXTENSION_CLASS = "tests.e2e.features.helpers.custom_pipeline.vLLMOmniColocateWorkerExtensionForTest"
# Use your specified HF repo name/path directly here
MODEL = "tiny-random/Qwen-Image"
@@ -7,37 +7,19 @@ from contextlib import ExitStack
import pytest
from tests.e2e.offline_inference.custom_pipeline.qwen_image_pipeline_with_logprob import (
from tests.e2e.features.helpers.custom_pipeline import (
QwenImagePipelineWithLogProbForTest,
)
from tests.e2e.offline_inference.custom_pipeline.worker_extension import (
vLLMOmniColocateWorkerExtensionForTest,
)
from tests.helpers.mark import hardware_test
from vllm_omni.diffusion.worker.diffusion_worker import CustomPipelineWorkerExtension
from vllm_omni.entrypoints.async_omni import AsyncOmni
CUSTOM_PIPELINE_CLASS = QwenImagePipelineWithLogProbForTest
WORKER_EXTENSION_CLASS = (
"tests.e2e.offline_inference.custom_pipeline.worker_extension.vLLMOmniColocateWorkerExtensionForTest"
)
WORKER_EXTENSION_CLASS = "tests.e2e.features.helpers.custom_pipeline.vLLMOmniColocateWorkerExtensionForTest"
MODEL = "tiny-random/Qwen-Image"
pytestmark = [pytest.mark.core_model]
@pytest.mark.cpu
def test_worker_extension_inheritance():
assert issubclass(vLLMOmniColocateWorkerExtensionForTest, CustomPipelineWorkerExtension)
@pytest.mark.cpu
def test_worker_extension_test_function():
assert (
vLLMOmniColocateWorkerExtensionForTest.test_extension_name() == "vllm-omni-colocate-worker-extension-for-test"
)
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.asyncio
@@ -0,0 +1,110 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import pytest
from vllm_omni.experimental.fullduplex.engine.duplex_control_client import (
DuplexControlClient,
DuplexControlRequestError,
)
from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseActivity
from vllm_omni.experimental.fullduplex.engine.messages import (
DuplexControlError,
DuplexControlResultMessage,
DuplexFence,
ResumeDuplexSessionMessage,
TouchDuplexSessionMessage,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class _Transport:
def __init__(self) -> None:
self.calls = []
def execute(self, key, message, **kwargs):
self.calls.append((key, message, kwargs))
return DuplexControlResultMessage(
control_id=message.control_id,
fence=message.fence,
operation="touch" if isinstance(message, TouchDuplexSessionMessage) else "resume",
session_id=message.session_id,
ok=True,
stage_results=[],
)
class _UnsupportedTransport:
def execute(self, key, message, **kwargs):
del key, kwargs
return DuplexControlResultMessage(
control_id=message.control_id,
fence=message.fence,
operation="touch",
session_id=message.session_id,
ok=False,
stage_results=[{"result": {"supported": False}}],
unsupported_count=1,
error=DuplexControlError(code="invalid_capability", message="unsupported", retryable=False),
accepted_fence=DuplexFence(message.session_id, epoch=2),
lease_generation=7,
)
def test_control_client_routes_touch_and_resume_by_control_id() -> None:
transport = _Transport()
control_ids = iter(("touch-id", "resume-id"))
client = DuplexControlClient(transport, control_id_factory=lambda: next(control_ids))
fence = DuplexFence("sid-client")
assert (
client.touch(
fence.session_id,
fence=fence,
activity=DuplexLeaseActivity.PLAYBACK_ACK,
timeout=2.0,
)["ok"]
is True
)
assert (
client.resume(
fence.session_id,
fence=fence,
expected_lease_generation=7,
timeout=3.0,
)["ok"]
is True
)
touch_key, touch_message, touch_kwargs = transport.calls[0]
assert touch_key == ("duplex", "touch-id")
assert isinstance(touch_message, TouchDuplexSessionMessage)
assert touch_message.activity == DuplexLeaseActivity.PLAYBACK_ACK.value
assert touch_kwargs["timeout"] == 2.0
resume_key, resume_message, resume_kwargs = transport.calls[1]
assert resume_key == ("duplex", "resume-id")
assert isinstance(resume_message, ResumeDuplexSessionMessage)
assert resume_message.expected_lease_generation == 7
assert resume_kwargs["timeout"] == 3.0
def test_control_client_treats_ok_false_as_authoritative_failure() -> None:
client = DuplexControlClient(_UnsupportedTransport(), control_id_factory=lambda: "unsupported")
fence = DuplexFence("sid-unsupported")
with pytest.raises(DuplexControlRequestError, match="duplex touch failed") as exc_info:
client.touch(
fence.session_id,
fence=fence,
activity=DuplexLeaseActivity.HEARTBEAT,
timeout=1.0,
)
assert exc_info.value.code == "invalid_capability"
assert exc_info.value.retryable is False
assert exc_info.value.accepted_fence == DuplexFence(fence.session_id, epoch=2)
assert exc_info.value.lease_generation == 7
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,68 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Focused tests for typed deployment-only runtime configuration."""
from __future__ import annotations
from dataclasses import FrozenInstanceError
import pytest
from vllm_omni.config.stage_config import (
DuplexSessionRuntimeConfig,
load_deploy_config,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_duplex_session_runtime_defaults_are_typed_and_immutable(tmp_path) -> None:
deploy_path = tmp_path / "duplex.yaml"
deploy_path.write_text("duplex_session: {}\nstages: []\n", encoding="utf-8")
deploy = load_deploy_config(deploy_path)
assert isinstance(deploy.duplex_session, DuplexSessionRuntimeConfig)
assert deploy.duplex_session.idle_ttl_s == 300.0
assert deploy.duplex_session.disconnect_grace_s == 30.0
assert deploy.duplex_session.reaper_interval_s == 5.0
assert deploy.duplex_session.resume_replay_ttl_s == 60.0
assert deploy.duplex_session.resume_replay_max_bytes_per_session == 8 * 1024 * 1024
assert deploy.duplex_session.max_pending_input_bytes_per_session == 16 * 1024 * 1024
assert deploy.duplex_session.max_pending_turns_per_session == 4
assert deploy.duplex_session.max_sessions == 1
assert deploy.duplex_session.completed_append_cache_size == 256
with pytest.raises(FrozenInstanceError):
deploy.duplex_session.idle_ttl_s = 1.0 # type: ignore[misc]
def test_duplex_session_runtime_accepts_disabled_idle_expiry(tmp_path) -> None:
deploy_path = tmp_path / "duplex.yaml"
deploy_path.write_text("duplex_session:\n idle_ttl_s: null\nstages: []\n", encoding="utf-8")
deploy = load_deploy_config(deploy_path)
assert deploy.duplex_session.idle_ttl_s is None
@pytest.mark.parametrize(
("name", "value"),
[
("idle_ttl_s", 0),
("idle_ttl_s", -1),
("disconnect_grace_s", 0),
("reaper_interval_s", -1),
("resume_replay_ttl_s", 0),
("resume_replay_max_bytes_per_session", -1),
("max_pending_input_bytes_per_session", 0),
("max_pending_turns_per_session", -1),
("max_sessions", 0),
("completed_append_cache_size", 0),
],
)
def test_duplex_session_runtime_rejects_non_positive_values(tmp_path, name: str, value: int) -> None:
deploy_path = tmp_path / "duplex.yaml"
deploy_path.write_text(f"duplex_session:\n {name}: {value}\nstages: []\n", encoding="utf-8")
with pytest.raises(ValueError, match=rf"duplex_session\.{name} must be positive"):
load_deploy_config(deploy_path)
@@ -0,0 +1,14 @@
import pytest
from vllm_omni.experimental.fullduplex.engine.intermediate import get_stream_request_key
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_get_stream_request_key_requires_stable_identifier():
with pytest.raises(ValueError, match="stable request id"):
get_stream_request_key({"ids": {"tts": [1, 2, 3]}})
def test_get_stream_request_key_accepts_global_request_id():
assert get_stream_request_key({"global_request_id": ["duplex-sid-stage0"]}) == "duplex-sid-stage0"
@@ -0,0 +1,224 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
from dataclasses import FrozenInstanceError
import pytest
from vllm_omni.experimental.fullduplex.engine.duplex_runtime import DuplexInputMode
from vllm_omni.experimental.fullduplex.engine.duplex_session import DuplexSessionRuntimeManager
from vllm_omni.experimental.fullduplex.engine.lease import (
DuplexLeaseActivity,
DuplexLeaseConfig,
)
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class FakeMonotonicClock:
def __init__(self, value: float) -> None:
self.value = value
def __call__(self) -> float:
return self.value
def advance(self, seconds: float) -> None:
self.value += seconds
def _lease_config(*, idle_ttl_s: float | None = 300.0) -> DuplexLeaseConfig:
return DuplexLeaseConfig(
idle_ttl_s=idle_ttl_s,
disconnect_grace_s=30.0,
)
def test_open_touch_detach_and_idle_expiry_use_monotonic_time() -> None:
clock = FakeMonotonicClock(100.0)
manager = DuplexSessionRuntimeManager(clock=clock)
fence = DuplexFence("sid-expiry")
session = manager.open_session(fence, lease_config=_lease_config())
assert session.lease.last_activity == 100.0
clock.advance(10.0)
session.touch(fence, DuplexLeaseActivity.HEARTBEAT)
session.detach(fence)
clock.advance(29.0)
assert session.lease.disconnect_grace_expired(clock()) is False
assert manager.collect_expired() == []
clock.advance(272.0)
expired = manager.collect_expired()
assert [item.session_id for item in expired] == ["sid-expiry"]
assert expired[0].reason == "disconnect_grace_expired"
assert manager.get("sid-expiry") is session
manager.finalize_close_session(session)
assert manager.get("sid-expiry") is None
def test_detach_grace_and_resume_advance_token_independent_lease_generation() -> None:
clock = FakeMonotonicClock(10.0)
manager = DuplexSessionRuntimeManager(clock=clock)
fence = DuplexFence("sid-resume")
session = manager.open_session(fence, lease_config=_lease_config())
session.detach(fence)
clock.advance(29.0)
assert session.lease.disconnect_grace_expired(clock()) is False
clock.advance(2.0)
assert session.lease.disconnect_grace_expired(clock()) is True
generation = session.resume(fence, expected_lease_generation=0)
assert generation == 1
assert session.lease.detached_at is None
assert session.lease.disconnect_grace_expired(clock()) is False
with pytest.raises(ValueError, match="lease generation mismatch"):
session.resume(fence, expected_lease_generation=0)
def test_active_operation_prevents_mid_transaction_expiry() -> None:
clock = FakeMonotonicClock(0.0)
manager = DuplexSessionRuntimeManager(clock=clock)
fence = DuplexFence("sid-operation")
session = manager.open_session(fence, lease_config=_lease_config(idle_ttl_s=5.0))
session.begin_operation(fence, "append-1")
clock.advance(10.0)
assert manager.collect_expired() == []
session.end_operation(fence, "append-1")
assert manager.collect_expired() == []
clock.advance(6.0)
assert [item.session_id for item in manager.collect_expired()] == ["sid-operation"]
def test_sessions_have_independent_activity_deadlines_and_resources() -> None:
clock = FakeMonotonicClock(0.0)
manager = DuplexSessionRuntimeManager(clock=clock)
fence_a = DuplexFence("sid-a")
fence_b = DuplexFence("sid-b")
session_a = manager.open_session(fence_a, lease_config=_lease_config(idle_ttl_s=10.0))
session_b = manager.open_session(fence_b, lease_config=_lease_config(idle_ttl_s=10.0))
session_a.reserve_stage_request(0, "req-a-reserved", fence=fence_a)
session_a.bind_stage_request(1, "req-a-submitted", fence=fence_a)
session_b.bind_stage_request(0, "req-b", fence=fence_b)
clock.advance(6.0)
session_b.touch(fence_b, DuplexLeaseActivity.MODEL_OUTPUT)
clock.advance(5.0)
expired = manager.collect_expired()
assert len(expired) == 1
assert expired[0].session_id == "sid-a"
assert expired[0].reserved_request_ids == ("req-a-reserved",)
assert expired[0].submitted_request_ids == ("req-a-submitted",)
assert manager.get("sid-b") is session_b
assert session_b.resource_request_ids() == ["req-b"]
def test_close_and_reaper_select_exactly_one_terminal_transition() -> None:
clock = FakeMonotonicClock(0.0)
manager = DuplexSessionRuntimeManager(clock=clock)
fence = DuplexFence("sid-race")
manager.open_session(fence, lease_config=_lease_config(idle_ttl_s=1.0))
clock.advance(2.0)
assert len(manager.collect_expired()) == 1
assert manager.close_session(fence, reason="explicit_close") is None
assert manager.collect_expired() == []
fence_2 = DuplexFence("sid-race-2")
manager.open_session(fence_2, lease_config=_lease_config(idle_ttl_s=1.0))
assert manager.close_session(fence_2, reason="explicit_close") is not None
clock.advance(2.0)
assert manager.collect_expired() == []
def test_stale_fence_cannot_touch_detach_or_resume_lease() -> None:
clock = FakeMonotonicClock(0.0)
manager = DuplexSessionRuntimeManager(clock=clock)
current = DuplexFence("sid-fence", epoch=1)
stale = DuplexFence("sid-fence", epoch=0)
session = manager.open_session(current, lease_config=_lease_config())
with pytest.raises(RuntimeError, match="fence mismatch"):
session.touch(stale, DuplexLeaseActivity.APPEND)
with pytest.raises(RuntimeError, match="fence mismatch"):
session.detach(stale)
with pytest.raises(RuntimeError, match="fence mismatch"):
session.resume(stale, expected_lease_generation=0)
def test_disabled_idle_expiry_never_collects_session() -> None:
clock = FakeMonotonicClock(0.0)
manager = DuplexSessionRuntimeManager(clock=clock)
manager.open_session(DuplexFence("sid-no-expiry"), lease_config=_lease_config(idle_ttl_s=None))
clock.advance(1_000_000.0)
assert manager.collect_expired() == []
def test_detached_session_expires_at_disconnect_grace_when_idle_ttl_is_disabled() -> None:
clock = FakeMonotonicClock(0.0)
manager = DuplexSessionRuntimeManager(clock=clock)
fence = DuplexFence("sid-disconnect-grace")
session = manager.open_session(fence, lease_config=_lease_config(idle_ttl_s=None))
session.detach(fence)
clock.advance(29.0)
assert manager.collect_expired() == []
clock.advance(1.0)
expired = manager.collect_expired()
assert [item.session_id for item in expired] == [fence.session_id]
assert expired[0].reason == "disconnect_grace_expired"
def test_runtime_manager_enforces_server_owned_session_admission() -> None:
manager = DuplexSessionRuntimeManager(max_sessions=2)
manager.open_session(DuplexFence("sid-admission-a"))
manager.open_session(DuplexFence("sid-admission-b"))
with pytest.raises(RuntimeError, match="duplex_session_capacity_exhausted"):
manager.open_session(DuplexFence("sid-admission-c"))
def test_completed_append_cache_is_bounded() -> None:
manager = DuplexSessionRuntimeManager(completed_append_limit=2)
fence = DuplexFence("sid-completed-cache")
session = manager.open_session(fence)
for index in range(3):
session.record_completed_append(
f"operation-{index}",
fence=fence,
mode=DuplexInputMode.TURN_COMMIT_ONLY,
final=True,
stage_results=[{"index": index}],
)
assert list(session.completed_appends) == ["operation-1", "operation-2"]
session.accept_fence(DuplexFence(fence.session_id, epoch=1))
assert session.completed_appends == {}
def test_lease_config_and_expiry_record_are_immutable() -> None:
config = _lease_config()
with pytest.raises(FrozenInstanceError):
config.idle_ttl_s = 1.0 # type: ignore[misc]
clock = FakeMonotonicClock(0.0)
manager = DuplexSessionRuntimeManager(clock=clock)
manager.open_session(DuplexFence("sid-record"), lease_config=_lease_config(idle_ttl_s=1.0))
clock.advance(2.0)
record = manager.collect_expired()[0]
with pytest.raises(FrozenInstanceError):
record.reason = "changed" # type: ignore[misc]
@@ -0,0 +1,480 @@
from dataclasses import FrozenInstanceError
from types import SimpleNamespace
import pytest
from vllm.sampling_params import SamplingParams
from vllm_omni.experimental.fullduplex.engine import duplex_runtime
from vllm_omni.experimental.fullduplex.engine.contracts import (
duplex_data_plane_request_info,
duplex_resource_request_belongs_to_session,
duplex_resource_request_id,
)
from vllm_omni.experimental.fullduplex.engine.duplex_runtime import (
DuplexInputMode,
DuplexOutputAction,
DuplexOutputDecision,
DuplexRuntimeCapabilities,
DuplexSessionRuntimeManager,
)
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
from vllm_omni.experimental.fullduplex.minicpmo45.runtime import (
MiniCPMO45DuplexRuntimeExtension,
build_duplex_data_plane_prompt,
duplex_scheduler_token_budget,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_duplex_fence_is_immutable():
fence = DuplexFence("session")
with pytest.raises(FrozenInstanceError):
fence.epoch = 1 # type: ignore[misc]
assert not hasattr(fence, "__dict__")
def test_duplex_runtime_tracks_stage_bindings_and_barge_in_epoch():
manager = DuplexSessionRuntimeManager()
session = manager.open_session(
DuplexFence("sid-1"),
capabilities=DuplexRuntimeCapabilities(
input_modes={DuplexInputMode.APPEND_TOKENS},
),
)
session.bind_stage_request(stage_id=0, request_id="req-stage0", fence=session.fence)
session.bind_stage_request(stage_id=1, request_id="req-stage1", fence=session.fence)
update = session.append_input(mode=DuplexInputMode.APPEND_TOKENS, fence=session.fence)
next_fence = DuplexFence("sid-1", epoch=1)
stale_request_ids = session.release_fence(session.fence)
session.accept_fence(next_fence)
assert update.seq == 1
assert session.fence == next_fence
assert stale_request_ids == ["req-stage0", "req-stage1"]
assert session.stage_bindings == {}
def test_duplex_runtime_tracks_same_request_id_for_each_pipeline_stage():
manager = DuplexSessionRuntimeManager()
fence = DuplexFence("sid-shared-pipeline-request")
session = manager.open_session(fence)
session.reserve_stage_request(0, "req-shared", fence=fence)
session.bind_stage_request(0, "req-shared", fence=fence)
session.bind_stage_request(1, "req-shared", fence=fence)
assert session.stage_bindings[0].request_id == "req-shared"
assert session.stage_bindings[1].request_id == "req-shared"
assert session.resource_request_ids() == ["req-shared"]
assert session.input_seq == 0
def test_duplex_runtime_extension_validation_rejects_missing_methods():
class IncompleteExtension:
def configure_sampling_params(self, *, runtime_config, defaults):
del runtime_config
return defaults
with pytest.raises(TypeError, match="plan_append"):
duplex_runtime.validate_duplex_runtime_extension(IncompleteExtension())
def test_duplex_runtime_extension_validation_rejects_stage_count_mismatch():
class WrongStageCountExtension(MiniCPMO45DuplexRuntimeExtension):
def configure_sampling_params(self, *, runtime_config, defaults):
del runtime_config
return defaults[:1]
with pytest.raises(ValueError, match="one sampling parameter per stage"):
duplex_runtime.validate_duplex_runtime_extension(
WrongStageCountExtension(),
sampling_defaults=(SamplingParams(), SamplingParams()),
)
def test_duplex_runtime_extension_validation_rejects_sampling_type_mismatch():
class WrongSamplingTypeExtension(MiniCPMO45DuplexRuntimeExtension):
def configure_sampling_params(self, *, runtime_config, defaults):
del runtime_config
return tuple(object() for _ in defaults)
with pytest.raises(TypeError, match="sampling parameter type"):
duplex_runtime.validate_duplex_runtime_extension(
WrongSamplingTypeExtension(),
sampling_defaults=(SamplingParams(), SamplingParams()),
)
def test_duplex_output_decision_metadata_is_immutable():
decision = DuplexOutputDecision(
action=DuplexOutputAction.DIRECT_RESPONSE,
metadata={"model_listen": True},
)
with pytest.raises(TypeError):
decision.metadata["model_listen"] = False
def _decide_minicpmo_output(
output: object,
*,
segment_token_ids: tuple[int, ...] = (),
segment_output_metadata: dict | None = None,
):
return MiniCPMO45DuplexRuntimeExtension().decide_output(
stage_id=0,
final_stage_id=1,
segment_finished=True,
segment_token_ids=segment_token_ids,
segment_output_metadata=segment_output_metadata or {},
output=output,
)
def test_minicpmo_extension_owns_stage_sampling_overrides():
defaults = (
SamplingParams(max_tokens=4),
SamplingParams(max_tokens=8),
)
configured = MiniCPMO45DuplexRuntimeExtension().configure_sampling_params(
runtime_config={
"duplex_stage_max_tokens": {"0": 20},
"duplex_stage_sampling_params": {"1": {"stop_token_ids": [151645]}},
},
defaults=defaults,
)
assert configured[0].max_tokens == 20
assert configured[1].stop_token_ids == [151645]
assert defaults[0].max_tokens == 4
assert 151645 not in (defaults[1].stop_token_ids or [])
def test_minicpmo_output_decision_uses_raw_streaming_token_snapshot():
decision = _decide_minicpmo_output(
SimpleNamespace(outputs=[SimpleNamespace()]),
segment_token_ids=(151705,),
segment_output_metadata={"special_token_ids": {"listen_token_id": 151705}},
)
assert decision is not None
assert decision.action is DuplexOutputAction.DIRECT_RESPONSE
assert decision.metadata["duplex_native_decision"] == "listen"
assert decision.metadata["model_listen"] is True
@pytest.mark.parametrize("attr", ["token_ids", "cumulative_token_ids"])
def test_minicpmo_output_decision_ignores_output_level_token_history(attr):
output = SimpleNamespace(
multimodal_output={"special_token_ids": {"listen_token_id": 151705}},
outputs=[SimpleNamespace()],
**{attr: [42, 151705]},
)
assert _decide_minicpmo_output(output) is None
@pytest.mark.parametrize("attr", ["token_ids", "cumulative_token_ids"])
def test_minicpmo_output_decision_uses_completion_token_ids(attr):
output = SimpleNamespace(
multimodal_output={"special_token_ids": {"listen_token_id": 151705}},
outputs=[SimpleNamespace(**{attr: [42, 151705]})],
)
assert _decide_minicpmo_output(output) is not None
def test_minicpmo_output_decision_uses_completion_stop_reason():
output = SimpleNamespace(
multimodal_output={"special_token_ids": {"listen_token_id": 151705}},
outputs=[SimpleNamespace(stop_reason=151705)],
)
assert _decide_minicpmo_output(output) is not None
def test_duplex_runtime_cancel_fence_rejects_late_append_and_accepts_next_epoch():
manager = DuplexSessionRuntimeManager()
cancelled_fence = DuplexFence("sid-cancel-race")
next_fence = DuplexFence("sid-cancel-race", epoch=1)
session = manager.open_session(
cancelled_fence,
capabilities=DuplexRuntimeCapabilities(
input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK},
),
)
session.bind_stage_request(
stage_id=0,
request_id="req-cancelled",
fence=cancelled_fence,
)
stale_request_ids = session.cancel_fence(cancelled_fence, next_fence)
assert stale_request_ids == ["req-cancelled"]
assert session.fence == next_fence
assert session.stage_bindings == {}
with pytest.raises(RuntimeError, match="fence mismatch"):
session.append_input(
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
fence=cancelled_fence,
)
update = session.append_input(
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
fence=next_fence,
)
assert update.seq == 1
def test_duplex_runtime_stale_close_preserves_live_session_and_bindings():
manager = DuplexSessionRuntimeManager()
current_fence = DuplexFence("sid-stale-close", epoch=1)
session = manager.open_session(
current_fence,
capabilities=DuplexRuntimeCapabilities(
input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK},
),
)
session.bind_stage_request(0, "req-live", fence=current_fence)
with pytest.raises(RuntimeError, match="fence mismatch"):
manager.close_session(DuplexFence("sid-stale-close"))
assert manager.get("sid-stale-close") is session
assert session.stage_request_ids() == ["req-live"]
def test_duplex_runtime_reopen_rejects_late_append_from_old_incarnation():
manager = DuplexSessionRuntimeManager()
old_fence = DuplexFence("sid-reopen", incarnation=0)
old_session = manager.open_session(
old_fence,
capabilities=DuplexRuntimeCapabilities(
input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK},
),
)
manager.close_session(old_fence)
new_fence = DuplexFence("sid-reopen", incarnation=1)
new_session = manager.open_session(
new_fence,
capabilities=old_session.capabilities,
)
with pytest.raises(RuntimeError, match="fence mismatch"):
new_session.append_input(
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
fence=old_fence,
)
assert (
new_session.append_input(
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
fence=new_fence,
).seq
== 1
)
def test_duplex_prompt_expands_incarnation_metadata():
fence = DuplexFence("sid-incarnation", incarnation=3)
prompt = build_duplex_data_plane_prompt(
request_id=duplex_resource_request_id(fence, "stage0"),
fence=fence,
session_config={},
runtime_config={},
seq=1,
turn_seq=1,
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
payload={"is_speech": True},
final=False,
)
assert prompt["model_intermediate_buffer"]["duplex"]["incarnation"] == 3
def test_duplex_runtime_tracks_turn_local_append_sequence():
manager = DuplexSessionRuntimeManager()
session = manager.open_session(
DuplexFence("sid-turn-seq"),
capabilities=DuplexRuntimeCapabilities(
input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK},
),
)
first = session.append_input(mode=DuplexInputMode.APPEND_AUDIO_CHUNK, fence=session.fence)
second = session.append_input(mode=DuplexInputMode.APPEND_AUDIO_CHUNK, fence=session.fence)
next_turn = DuplexFence("sid-turn-seq", turn_id=1, response_seq=1)
third = session.append_input(
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
fence=next_turn,
)
fourth = session.append_input(
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
fence=next_turn,
)
assert [first.seq, second.seq, third.seq, fourth.seq] == [1, 2, 3, 4]
assert [first.turn_seq, second.turn_seq, third.turn_seq, fourth.turn_seq] == [1, 2, 1, 2]
assert [first.turn_id, second.turn_id, third.turn_id, fourth.turn_id] == [0, 0, 1, 1]
def test_duplex_runtime_rejects_unsupported_append_mode():
manager = DuplexSessionRuntimeManager()
session = manager.open_session(
DuplexFence("sid-2"),
capabilities=DuplexRuntimeCapabilities(input_modes={DuplexInputMode.TURN_COMMIT_ONLY}),
)
with pytest.raises(ValueError, match="not supported"):
session.append_input(mode=DuplexInputMode.APPEND_TOKENS, fence=session.fence)
def test_duplex_data_plane_request_info_extracts_structured_stage_result():
request_id, response_stage_id = duplex_data_plane_request_info(
{
"stage_results": [
{"result": {"supported": True}},
{
"result": {
"data_plane_append": True,
"request_id": "duplex-sid-e0-stage0-s1",
"response_stage_id": 1,
}
},
]
}
)
assert request_id == "duplex-sid-e0-stage0-s1"
assert response_stage_id == 1
def test_duplex_data_plane_request_info_rejects_missing_request_id():
assert duplex_data_plane_request_info(
{
"stage_results": [
{
"result": {
"data_plane_append": True,
"request_id": "",
"response_stage_id": 1,
}
}
]
}
) == (None, None)
def test_duplex_scheduler_token_budget_estimates_pcm_slots():
assert (
duplex_scheduler_token_budget(
{
"audio": "AAAAAA==",
"format": "pcm_f32le",
"sample_rate_hz": 16000,
}
)
== 16
)
def test_duplex_scheduler_token_budget_ignores_client_budget_fields():
assert (
duplex_scheduler_token_budget(
{
"audio": "AAAAAA==",
"format": "pcm_f32le",
"duplex_num_input_tokens": 999,
"num_input_tokens": 999,
}
)
== 16
)
def test_resource_state_rejects_fence_regression_and_requires_explicit_fence():
current = DuplexFence("sid", epoch=2, turn_id=3, response_seq=4)
manager = DuplexSessionRuntimeManager()
session = manager.open_session(
current,
capabilities=DuplexRuntimeCapabilities(input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK}),
)
for stale in (
DuplexFence("sid", epoch=1, turn_id=99, response_seq=99),
DuplexFence("sid", epoch=2, turn_id=2, response_seq=4),
DuplexFence("sid", epoch=2, turn_id=3, response_seq=3),
):
with pytest.raises(RuntimeError, match="fence mismatch"):
session.accept_fence(stale)
assert session.fence == current
with pytest.raises(TypeError, match="fence"):
session.bind_stage_request(0, "request")
with pytest.raises(TypeError, match="fence"):
session.append_input(mode=DuplexInputMode.APPEND_AUDIO_CHUNK)
with pytest.raises(TypeError, match="DuplexFence"):
manager.open_session("legacy-session")
with pytest.raises(TypeError, match="DuplexFence"):
manager.close_session("legacy-session")
def test_resource_request_id_is_derived_from_fence_and_role():
fence = DuplexFence("sid-with-dashes", epoch=7, turn_id=11, response_seq=13)
assert duplex_resource_request_id(fence, "stage0") == "duplex-s.c2lkLXdpdGgtZGFzaGVz.i.0.e.7.r.stage0"
assert duplex_resource_request_id(fence, "stage1") == "duplex-s.c2lkLXdpdGgtZGFzaGVz.i.0.e.7.r.stage1"
def test_resource_request_id_codec_separates_session_id_from_incarnation():
embedded_incarnation = duplex_resource_request_id(
DuplexFence("foo-i1", incarnation=0),
"stage0",
)
actual_incarnation = duplex_resource_request_id(
DuplexFence("foo", incarnation=1),
"stage0",
)
assert embedded_incarnation != actual_incarnation
def test_resource_request_id_session_membership_uses_encoded_identity():
request_id = duplex_resource_request_id(
DuplexFence("sid-with-dashes", incarnation=2, epoch=7),
"stage0",
)
assert duplex_resource_request_belongs_to_session(request_id, "sid-with-dashes") is True
assert duplex_resource_request_belongs_to_session(request_id, "sid") is False
assert duplex_resource_request_belongs_to_session("duplex-s.invalid.i.x.e.7.r.stage0", "sid") is False
def test_placeholder_budget_is_planned_inside_omni_engine_boundary():
fence = DuplexFence("sid", turn_id=1, response_seq=1)
prompt = build_duplex_data_plane_prompt(
request_id=duplex_resource_request_id(fence, "stage0"),
fence=fence,
session_config={},
runtime_config={},
seq=2,
turn_seq=1,
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
payload={
"audio": "AAAAAA==",
"format": "pcm_f32le",
"duplex_num_input_tokens": 999,
"num_input_tokens": 999,
},
final=False,
)
assert len(prompt["prompt_token_ids"]) == 16
assert prompt["model_intermediate_buffer"]["duplex"]["fence"] == fence
assert prompt["model_intermediate_buffer"]["duplex"]["scheduler_token_budget"] == 16
@@ -0,0 +1,99 @@
from __future__ import annotations
import pytest
from vllm_omni.experimental.fullduplex.openai.commit_policy import (
CommitAction,
CommitSnapshot,
decide_commit_action,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@pytest.mark.parametrize(
("case", "snapshot", "expected"),
[
(
"first idle commit",
CommitSnapshot(
auto_responds=True,
speech_since_commit=True,
active_response_id=None,
overlap_speech_ms=0,
native_response_in_progress=False,
playback_active=False,
),
CommitAction.START_AUTO_RESPONSE,
),
(
"native auto-response remains model-owned during playback",
CommitSnapshot(
auto_responds=True,
speech_since_commit=True,
active_response_id=None,
overlap_speech_ms=800,
native_response_in_progress=True,
playback_active=True,
),
CommitAction.START_AUTO_RESPONSE,
),
(
"native auto-response commit without speech",
CommitSnapshot(
auto_responds=True,
speech_since_commit=False,
active_response_id=None,
overlap_speech_ms=0,
native_response_in_progress=True,
playback_active=False,
),
CommitAction.COMMIT_ONLY,
),
(
"non-auto response overlap",
CommitSnapshot(
auto_responds=False,
speech_since_commit=True,
active_response_id=None,
overlap_speech_ms=800,
native_response_in_progress=True,
playback_active=False,
),
CommitAction.DEFER_ACTIVE_RESPONSE,
),
(
"non-auto playback overlap",
CommitSnapshot(
auto_responds=False,
speech_since_commit=True,
active_response_id=None,
overlap_speech_ms=800,
native_response_in_progress=False,
playback_active=True,
),
CommitAction.DEFER_ACTIVE_RESPONSE,
),
(
"non-auto commit without overlap",
CommitSnapshot(
auto_responds=False,
speech_since_commit=True,
active_response_id=None,
overlap_speech_ms=0,
native_response_in_progress=False,
playback_active=False,
),
CommitAction.COMMIT_ONLY,
),
],
ids=lambda value: value if isinstance(value, str) else None,
)
def test_decide_commit_action_covers_realtime_lifecycle_sequences(
case: str,
snapshot: CommitSnapshot,
expected: CommitAction,
) -> None:
del case
assert decide_commit_action(snapshot) is expected
@@ -0,0 +1,179 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import base64
import numpy as np
import pytest
from vllm_omni.experimental.fullduplex.minicpmo45.input import MiniCPMO45PcmAppendBuffer
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def pcm_payload(samples: int, *, speech: bool = True) -> dict[str, object]:
audio = np.ones(samples, dtype=np.float32).tobytes()
return {
"type": "audio",
"audio": base64.b64encode(audio).decode("ascii"),
"format": "pcm_f32le",
"sample_rate_hz": 16_000,
"is_speech": speech,
}
def test_commit_does_not_add_silence_after_incremental_audio_was_drained():
buffer = MiniCPMO45PcmAppendBuffer()
emitted = buffer.append(pcm_payload(16_000), chunk_period_ms=1_000)
committed = buffer.commit(chunk_period_ms=1_000)
assert emitted is not None
assert not buffer.has_pending()
assert committed is None
def test_append_emits_one_model_unit_when_multiple_units_are_buffered():
buffer = MiniCPMO45PcmAppendBuffer()
emitted = buffer.append(pcm_payload(32_000), chunk_period_ms=1_000)
assert emitted is not None
assert len(base64.b64decode(emitted["audio"])) == 16_000 * 4
assert buffer.pending_byte_count == 16_000 * 4
def test_speech_marker_does_not_leak_across_irregular_chunk_boundaries():
buffer = MiniCPMO45PcmAppendBuffer()
assert buffer.append(pcm_payload(15_000), chunk_period_ms=1_000) is None
speech_unit = buffer.append(pcm_payload(2_000, speech=False), chunk_period_ms=1_000)
silence_unit = buffer.append(pcm_payload(15_000, speech=False), chunk_period_ms=1_000)
assert speech_unit is not None
assert speech_unit["is_speech"] is True
assert silence_unit is not None
assert silence_unit["is_speech"] is False
def test_force_listen_marker_does_not_leak_across_irregular_chunk_boundaries():
buffer = MiniCPMO45PcmAppendBuffer()
forced = pcm_payload(15_000, speech=False)
forced["force_listen"] = True
assert buffer.append(forced, chunk_period_ms=1_000) is None
forced_unit = buffer.append(pcm_payload(2_000, speech=False), chunk_period_ms=1_000)
unforced_unit = buffer.append(pcm_payload(15_000, speech=False), chunk_period_ms=1_000)
assert forced_unit is not None
assert forced_unit["force_listen"] is True
assert unforced_unit is not None
assert unforced_unit["force_listen"] is False
def test_pcm_append_rollback_restores_per_span_speech_markers():
buffer = MiniCPMO45PcmAppendBuffer()
assert buffer.append(pcm_payload(15_000), chunk_period_ms=1_000) is None
reservation = buffer.prepare_append(
pcm_payload(2_000, speech=False),
operation_id="mixed-unit",
chunk_period_ms=1_000,
)
assert reservation is not None
reservation.rollback()
first = buffer.append(pcm_payload(15_000, speech=False), chunk_period_ms=1_000)
second = buffer.flush(chunk_period_ms=1_000)
assert first is not None
assert first["is_speech"] is True
assert second is not None
assert second["is_speech"] is False
def test_commit_without_speech_does_not_synthesize_terminal_audio():
buffer = MiniCPMO45PcmAppendBuffer()
buffer.append(pcm_payload(8_000, speech=False), chunk_period_ms=1_000)
committed = buffer.commit(chunk_period_ms=1_000)
assert committed is None
def test_commit_resets_cumulative_turn_accounting():
buffer = MiniCPMO45PcmAppendBuffer()
buffer.append(pcm_payload(16_000), chunk_period_ms=1_000)
buffer.commit(chunk_period_ms=1_000)
empty = buffer.commit(chunk_period_ms=1_000)
assert empty is None
def test_pcm_append_reservation_rollback_restores_emitted_audio():
buffer = MiniCPMO45PcmAppendBuffer()
original = pcm_payload(16_000)
reservation = buffer.prepare_append(
original,
operation_id="append-1",
chunk_period_ms=1_000,
)
assert reservation is not None
assert not buffer.has_pending()
reservation.rollback()
assert buffer.has_pending()
retried = buffer.flush(chunk_period_ms=1_000)
assert retried is not None
assert base64.b64decode(retried["audio"]) == base64.b64decode(original["audio"])
def test_pcm_commit_keeps_prior_append_reservation_active():
buffer = MiniCPMO45PcmAppendBuffer()
append_reservation = buffer.prepare_append(
pcm_payload(16_000),
operation_id="append-before-commit",
chunk_period_ms=1_000,
)
commit_reservation = buffer.prepare_commit(
operation_id="commit-after-append",
chunk_period_ms=1_000,
)
assert append_reservation is not None
assert append_reservation.active
assert commit_reservation.payload is None
append_reservation.commit()
commit_reservation.commit()
def test_pcm_commit_reservation_rollback_restores_residual_audio():
buffer = MiniCPMO45PcmAppendBuffer()
original = pcm_payload(8_000)
assert (
buffer.prepare_append(
original,
operation_id="buffer-half-chunk",
chunk_period_ms=1_000,
)
is None
)
reservation = buffer.prepare_commit(
operation_id="final-half-chunk",
chunk_period_ms=1_000,
)
assert reservation.payload is not None
assert reservation.payload["final"] is True
reservation.rollback()
retried = buffer.prepare_commit(
operation_id="retry-final-half-chunk",
chunk_period_ms=1_000,
)
assert retried.payload is not None
assert base64.b64decode(retried.payload["audio"]) == (base64.b64decode(original["audio"]) + b"\x00" * (8_000 * 4))
@@ -0,0 +1,54 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import base64
import numpy as np
import pytest
from vllm_omni.experimental.fullduplex.openai.audio import convert_input_audio_with_rate
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_pcm16_input_conversion_is_pure_and_reports_target_rate():
pcm16 = np.array([-32768, 0, 32767], dtype="<i2").tobytes()
encoded = base64.b64encode(pcm16).decode("ascii")
converted, fmt, sample_rate = convert_input_audio_with_rate(
encoded,
"pcm16",
sample_rate_hz=16_000,
)
assert encoded == base64.b64encode(pcm16).decode("ascii")
assert fmt == "pcm_f32le"
assert sample_rate == 16_000
samples = np.frombuffer(base64.b64decode(converted), dtype="<f4")
assert np.allclose(samples, [-1.0, 0.0, 32767 / 32768])
@pytest.mark.parametrize("sample_rate_hz", [8_000, 192_000])
def test_pcm16_input_conversion_accepts_supported_sample_rate_boundaries(sample_rate_hz: int):
encoded = base64.b64encode(np.zeros(8, dtype="<i2").tobytes()).decode("ascii")
_, fmt, converted_rate = convert_input_audio_with_rate(
encoded,
"pcm16",
sample_rate_hz=sample_rate_hz,
)
assert fmt == "pcm_f32le"
assert converted_rate == 16_000
@pytest.mark.parametrize("sample_rate_hz", [7_999, 192_001, 10**1000])
def test_pcm16_input_conversion_rejects_sample_rates_outside_supported_range(sample_rate_hz: int):
encoded = base64.b64encode(np.zeros(8, dtype="<i2").tobytes()).decode("ascii")
with pytest.raises(ValueError, match="sample_rate_hz"):
convert_input_audio_with_rate(
encoded,
"pcm16",
sample_rate_hz=sample_rate_hz,
)
@@ -0,0 +1,139 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import subprocess
import sys
from pathlib import Path
from types import SimpleNamespace
import pytest
from vllm_omni.experimental.fullduplex.openai.runtime_adapter import (
validate_serving_runtime_adapter,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@pytest.mark.parametrize(
"module_name",
[
"vllm_omni.experimental.fullduplex.openai.session_runner",
"vllm_omni.experimental.fullduplex.openai.runtime_bridge",
"vllm_omni.experimental.fullduplex.openai.serving",
],
)
def test_generic_openai_runtime_import_does_not_load_minicpmo45(module_name: str) -> None:
repo_root = Path(__file__).resolve().parents[3]
script = f"""
import sys
import {module_name}
loaded = sorted(
name
for name in sys.modules
if name == "vllm_omni.experimental.fullduplex.minicpmo45"
or name.startswith("vllm_omni.experimental.fullduplex.minicpmo45.")
)
if loaded:
raise SystemExit("generic OpenAI runtime loaded MiniCPM-o: " + ", ".join(loaded))
"""
result = subprocess.run(
[sys.executable, "-c", script],
cwd=repo_root,
capture_output=True,
text=True,
timeout=60,
check=False,
)
assert result.returncode == 0, result.stderr
def test_generic_handler_requires_explicit_serving_runtime_adapter() -> None:
repo_root = Path(__file__).resolve().parents[3]
script = """
from types import SimpleNamespace
from vllm_omni.experimental.fullduplex.openai.serving import OmniDuplexSessionHandler
chat_service = SimpleNamespace(engine_client=SimpleNamespace())
try:
OmniDuplexSessionHandler(chat_service=chat_service)
except ValueError as exc:
if "serving runtime adapter" not in str(exc).lower():
raise
else:
raise SystemExit("generic handler silently selected a model serving adapter")
"""
result = subprocess.run(
[sys.executable, "-c", script],
cwd=repo_root,
capture_output=True,
text=True,
timeout=60,
check=False,
)
assert result.returncode == 0, result.stderr
def _valid_runtime_adapter() -> SimpleNamespace:
data_plane = SimpleNamespace(
begin_request=lambda request_id: None,
is_terminal=lambda request_id: False,
mark_terminal=lambda request_id: None,
close_stream=lambda request_id: None,
close_session=lambda session_id, **kwargs: None,
project=lambda result, **kwargs: (),
)
return SimpleNamespace(
adapter_id="test-adapter",
session_states={},
data_plane=data_plane,
clean_response_done_prefix="clean",
interrupted_tts_prefix="interrupted",
private_runtime_config_keys=frozenset({"private_key"}),
create_session_state=lambda: object(),
session_state=lambda session_id: object(),
remove_session_state=lambda session_id: None,
is_enabled=lambda config: True,
capabilities=lambda **kwargs: object(),
validate_client_extra_body=lambda extra_body: None,
prepare_runtime_config=lambda config, **kwargs: {},
runtime_config_for_update=lambda config, current: {},
data_plane_context=lambda **kwargs: object(),
)
@pytest.mark.parametrize(
("attribute", "invalid_value"),
[
("session_states", None),
("clean_response_done_prefix", None),
("interrupted_tts_prefix", None),
("private_runtime_config_keys", {"private_key"}),
],
)
def test_runtime_adapter_validator_rejects_invalid_protocol_attributes(
attribute: str,
invalid_value: object,
) -> None:
adapter = _valid_runtime_adapter()
setattr(adapter, attribute, invalid_value)
with pytest.raises(TypeError, match=attribute):
validate_serving_runtime_adapter(adapter)
def test_runtime_adapter_validator_rejects_incomplete_data_plane() -> None:
adapter = _valid_runtime_adapter()
adapter.data_plane.close_session = None
with pytest.raises(TypeError, match="data_plane.*close_session"):
validate_serving_runtime_adapter(adapter)
@@ -0,0 +1,145 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import pytest
from vllm_omni.experimental.fullduplex.openai.websocket import (
DuplexWebSocketActor,
normalize_duplex_input_event,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class FakeWebSocket:
def __init__(self) -> None:
self.sent: list[dict[str, object]] = []
async def send_json(self, payload: dict[str, object]) -> None:
self.sent.append(dict(payload))
@pytest.mark.parametrize(
("event", "expected"),
[
({"type": "signal_turn", "event": "barge_in"}, {"type": "turn.signal", "event": "barge_in"}),
({"type": "close_session"}, {"type": "session.close"}),
({"type": "audio.playback_ack", "played_ms": 1}, {"type": "playback.ack", "played_ms": 1}),
({"type": "input_text.append", "text": "a"}, {"type": "input.text.append", "text": "a"}),
({"type": "push_text", "text": "b"}, {"type": "input.text.append", "text": "b"}),
(
{"type": "input.audio.append", "audio": "wav"},
{"type": "input_audio_buffer.append", "audio": "wav", "format": "wav"},
),
(
{"type": "push_chunk", "audio": "wav", "format": "pcm_f32le"},
{"type": "input_audio_buffer.append", "audio": "wav", "format": "pcm_f32le"},
),
],
)
def test_input_aliases_normalize_once_at_mailbox_boundary(event, expected):
assert normalize_duplex_input_event(event) == expected
def test_actor_uses_one_fifo_mailbox_for_inbound_events():
actor = DuplexWebSocketActor(FakeWebSocket())
assert isinstance(actor.mailbox, asyncio.Queue)
assert not isinstance(actor.mailbox, asyncio.PriorityQueue)
assert not hasattr(actor, "control_queue")
assert not hasattr(actor, "input_queue")
assert not hasattr(actor, "event_queue")
assert not hasattr(actor, "lifecycle_state")
assert not hasattr(actor, "last_response_id")
assert not hasattr(actor, "overlap_speech_ms")
assert not hasattr(actor, "transition")
assert not hasattr(actor, "output_generation_in_flight")
assert not hasattr(actor, "session")
assert not hasattr(actor, "runtime_opened")
assert not hasattr(actor, "runtime_closed")
assert not hasattr(actor, "drain_input_queue")
assert not hasattr(actor, "control_events_seen")
assert not hasattr(actor, "input_events_seen")
assert not hasattr(actor, "cancel_count")
@pytest.mark.asyncio
async def test_control_event_does_not_overtake_earlier_audio_input():
actor = DuplexWebSocketActor(FakeWebSocket())
await actor.enqueue_event({"type": "input_audio_buffer.append", "audio": "pcm"})
await actor.enqueue_event({"type": "response.cancel"})
assert await actor.next_event() == {"type": "input_audio_buffer.append", "audio": "pcm"}
assert await actor.next_event() == {"type": "response.cancel"}
@pytest.mark.asyncio
@pytest.mark.parametrize("event_type", ["input.cancel", "session.close", "close_session"])
async def test_terminal_control_preserves_wire_order(event_type: str):
actor = DuplexWebSocketActor(FakeWebSocket())
await actor.enqueue_event({"type": "input_audio_buffer.append", "audio": "pcm"})
await actor.enqueue_event({"type": event_type})
assert await actor.next_event() == {"type": "input_audio_buffer.append", "audio": "pcm"}
assert await actor.next_event() == {"type": event_type}
@pytest.mark.asyncio
async def test_mailbox_delivers_every_enqueued_event_exactly_once():
actor = DuplexWebSocketActor(FakeWebSocket())
events = [
{"type": "input_audio_buffer.append", "audio": str(index)}
if index % 2
else {"type": "response.created", "response_id": f"resp-{index}"}
for index in range(20)
]
await asyncio.gather(*(actor.enqueue_event(dict(event)) for event in events))
received = [await actor.next_event() for _ in events]
assert sorted(received, key=repr) == sorted(events, key=repr)
@pytest.mark.asyncio
async def test_writer_is_single_owner_of_websocket_send():
websocket = FakeWebSocket()
actor = DuplexWebSocketActor(websocket)
writer = asyncio.create_task(actor.writer_loop())
await actor.send_json({"type": "one"})
await actor.send_json({"type": "two"})
await actor.close_writer()
await writer
assert websocket.sent == [{"type": "one"}, {"type": "two"}]
@pytest.mark.asyncio
async def test_stale_fence_payload_is_dropped_before_websocket_send():
websocket = FakeWebSocket()
actor = DuplexWebSocketActor(websocket, current_epoch=lambda: 2)
writer = asyncio.create_task(actor.writer_loop())
await actor.send_json({"type": "response.audio.delta", "epoch": 1})
await actor.send_json({"type": "response.audio.delta", "epoch": 2})
await actor.close_writer()
await writer
assert websocket.sent == [{"type": "response.audio.delta", "epoch": 2}]
assert actor.stale_output_dropped == 1
@pytest.mark.asyncio
async def test_writer_does_not_revoke_accepted_terminal_after_close_starts():
websocket = FakeWebSocket()
actor = DuplexWebSocketActor(websocket, current_epoch=lambda: 1)
await actor.send_json({"type": "response.done", "epoch": 1, "response_id": "resp-1"})
actor.closing = True
await actor.close_writer()
await actor.writer_loop()
assert websocket.sent == [{"type": "response.done", "epoch": 1, "response_id": "resp-1"}]
assert actor.stale_output_dropped == 0
@@ -0,0 +1,240 @@
import base64
import wave
from urllib.parse import parse_qs, urlsplit
import pytest
from vllm_omni.experimental.fullduplex.client import (
RealtimeDuplexClient,
RealtimeEventCollector,
build_realtime_url,
read_pcm16_wav,
write_pcm16_wav,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_realtime_client_builds_explicit_native_duplex_url():
url = build_realtime_url(
"ws://localhost:8099/v1/realtime?custom=1",
"openbmb/MiniCPM-o-4_5",
session_id="session-a",
)
query = parse_qs(urlsplit(url).query)
assert query == {
"custom": ["1"],
"duplex": ["1"],
"model": ["openbmb/MiniCPM-o-4_5"],
"minicpmo45_native_duplex": ["1"],
"session_id": ["session-a"],
}
def test_realtime_client_builds_resume_only_url_when_autostart_disabled():
url = build_realtime_url(
"ws://localhost:8099/v1/realtime?duplex=1",
"openbmb/MiniCPM-o-4_5",
autostart=False,
)
query = parse_qs(urlsplit(url).query)
assert query["autostart"] == ["0"]
assert query["minicpmo45_native_duplex"] == ["1"]
@pytest.mark.asyncio
async def test_realtime_client_configure_omits_ref_audio_by_default():
class Client(RealtimeDuplexClient):
def __init__(self):
super().__init__("ws://unused")
self.sent = []
async def send(self, event):
self.sent.append(event)
self.events.add({"type": "session.created"})
client = Client()
await client.configure("openbmb/MiniCPM-o-4_5", timeout_s=1)
session = client.sent[0]["session"]
assert "ref_audio" not in session
@pytest.mark.asyncio
async def test_realtime_client_configure_sends_explicit_ref_audio():
class Client(RealtimeDuplexClient):
def __init__(self):
super().__init__("ws://unused")
self.sent = []
async def send(self, event):
self.sent.append(event)
self.events.add({"type": "session.created"})
client = Client()
await client.configure(
"openbmb/MiniCPM-o-4_5",
ref_audio="data:audio/wav;base64,AAAA",
timeout_s=1,
)
session = client.sent[0]["session"]
assert session["ref_audio"] == "data:audio/wav;base64,AAAA"
def test_realtime_event_collector_partitions_audio_by_response():
collector = RealtimeEventCollector()
collector.add({"type": "response.created", "response": {"id": "resp-a"}})
collector.add(
{
"type": "response.audio.delta",
"response_id": "resp-a",
"delta": base64.b64encode(b"audio-a").decode("ascii"),
"sample_rate_hz": 16_000,
}
)
assert collector.response_ids == ["resp-a"]
assert collector.audio_bytes("resp-a") == b"audio-a"
assert collector.output_sample_rate_hz == 16_000
assert collector.first_received_at("response.created") is not None
assert collector.last_received_at("response.audio.delta") is not None
def test_realtime_event_collector_reports_engine_token_and_audio_intervals():
collector = RealtimeEventCollector()
collector.add(
{"type": "response.created", "response": {"id": "resp-a"}},
received_at_s=10.0,
)
stage_metrics = {
"0": {
"num_tokens_out": 4,
"vllm_ttft_ms": 120.0,
"vllm_tpot_ms": 15.0,
"vllm_itl_ms": 14.0,
"vllm_itls_ms": [10.0, 14.0, 18.0],
}
}
for received_at_s, cumulative_audio_ms in ((10.2, 80), (10.25, 160), (10.36, 240)):
collector.add(
{
"type": "response.audio.delta",
"response_id": "resp-a",
"delta": base64.b64encode(b"audio").decode("ascii"),
"sample_rate_hz": 16_000,
"metadata": {
"audio_duration_ms": cumulative_audio_ms,
"vllm_omni": {"stage_metrics": stage_metrics},
},
},
received_at_s=received_at_s,
)
timing = collector.timing_summary(
after_s=10.0,
input_committed_at_s=9.9,
response_id="resp-a",
)
assert timing["stage0_tokens"] == {
"source": "engine_stage_metrics",
"output_token_count": 4,
"ttft_ms": 120.0,
"tpot_ms": 15.0,
"inter_token_interval_ms": {
"count": 3,
"mean": 14.0,
"p50": 14.0,
"p95": 18.0,
"max": 18.0,
},
}
assert timing["audio_output"] == {
"source": "client_monotonic_receive",
"chunk_count": 3,
"response_created_to_first_audio_ms": 200.0,
"commit_to_first_audio_ms": 300.0,
"inter_chunk_interval_ms": {
"count": 2,
"mean": 80.0,
"p50": 50.0,
"p95": 110.0,
"max": 110.0,
},
"chunk_duration_ms": {
"count": 3,
"mean": 80.0,
"p50": 80.0,
"p95": 80.0,
"max": 80.0,
},
"max_chunk_gap_ms": 110.0,
}
def test_response_timing_ignores_unowned_session_level_metrics():
collector = RealtimeEventCollector()
collector.add(
{"type": "response.created", "response": {"id": "resp-a"}},
received_at_s=10.0,
)
collector.add(
{
"type": "response.audio.delta",
"response_id": "resp-a",
"delta": base64.b64encode(b"audio").decode("ascii"),
"metadata": {
"vllm_omni": {
"stage_metrics": {
"0": {
"num_tokens_out": 20,
"vllm_ttft_ms": 157.0,
"vllm_tpot_ms": 16.0,
"vllm_itls_ms": [15.0, 17.0],
}
}
}
},
},
received_at_s=10.2,
)
collector.add(
{
"type": "response.listen",
"metadata": {
"vllm_omni": {
"stage_metrics": {
"0": {
"num_tokens_out": 2,
"vllm_ttft_ms": 106.0,
"vllm_tpot_ms": 0.0,
"vllm_itls_ms": [],
}
}
}
},
},
received_at_s=10.3,
)
timing = collector.timing_summary(after_s=10.0, response_id="resp-a")
assert timing["stage0_tokens"]["output_token_count"] == 20
assert timing["stage0_tokens"]["ttft_ms"] == 157.0
def test_realtime_client_pcm16_wav_round_trip(tmp_path):
path = tmp_path / "audio.wav"
pcm16 = b"\x01\x00\x02\x00"
write_pcm16_wav(path, pcm16, sample_rate_hz=16_000)
with wave.open(str(path), "rb") as wav_file:
assert wav_file.getnchannels() == 1
assert wav_file.getframerate() == 16_000
assert read_pcm16_wav(path) == pcm16
@@ -0,0 +1,110 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm_omni.experimental.fullduplex.joyvl.decision.output_parser import parse_action
from vllm_omni.experimental.fullduplex.joyvl.memory.brain import InteractionBrain
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class _FakeSummarizer:
def __init__(self):
self.chunks = 0
async def summarize_chunk(self, chunk_index, frame_range, frames):
self.chunks += 1
return f"summary-of-chunk-{chunk_index}"
async def compress_to_long_term(self, mids):
return f"compressed({len(mids)})"
class _FakeDelegation:
def __init__(self):
self._ready = {}
async def submit(self, question, note, frames):
self._ready["t1"] = question
return "t1"
async def poll(self, task_id):
from vllm_omni.experimental.fullduplex.joyvl.bridges.delegation import DelegationResult
return DelegationResult(task_id, "ready", digest=f"answer to {self._ready[task_id]}")
def test_query_freshness_and_archive_on_change():
b = InteractionBrain()
assert b.update_query("count bottles") is True
assert b.update_query("count bottles") is False
b.record_response("1 bottle")
assert b.update_query("now describe") is True
assert b.memory.qa_history[-1].query == "count bottles"
assert b.memory.qa_history[-1].responses == [("0.0 seconds", "1 bottle")]
def test_query_in_frame_on_arrival_then_prefix_on_later_chunks():
b = InteractionBrain(frame_seconds=1.0)
b.tick(3)
b.update_query("alert me if a fire breaks out")
assert b.query_in_current_chunk is True
assert "alert me if a fire breaks out" not in b.build_prefix()
b.record_response("a fire is breaking out")
b.close_chunk()
assert b.query_in_current_chunk is False
prefix = b.build_prefix()
assert "alert me if a fire breaks out" in prefix
assert "a fire is breaking out" in prefix
@pytest.mark.asyncio
async def test_flush_archives_qa_and_summarizes():
summ = _FakeSummarizer()
b = InteractionBrain(summarizer=summ, chunk_frames=3, long_term_every_n_chunks=2, frame_seconds=1.0)
b.update_query("watch the room")
b.tick(3)
assert b.should_flush() is True
b.record_response("a person enters")
await b.flush([("0.0s", "u0"), ("2.0s", "u2")])
assert summ.chunks == 1
assert b.chunk_index == 2
assert b.should_flush() is False
assert b.memory.qa_history[-1].responses == [("2.0 seconds", "a person enters")]
assert len(b.memory.mid_term_summaries) == 1
@pytest.mark.asyncio
async def test_long_term_compression_after_n_chunks():
summ = _FakeSummarizer()
b = InteractionBrain(summarizer=summ, chunk_frames=1, long_term_every_n_chunks=2, frame_seconds=1.0)
for _ in range(2):
await b.flush([("0.0s", "u")])
assert b.memory.long_term_memory.startswith("compressed(")
assert b.memory.mid_term_summaries == []
@pytest.mark.asyncio
async def test_long_term_window_caps_blocks():
# Long-term memory must not grow unbounded over hours: oldest blocks are dropped
# once the sliding window is full (reference window = 15; here 2 for the test).
summ = _FakeSummarizer()
b = InteractionBrain(
summarizer=summ, chunk_frames=1, long_term_every_n_chunks=1, long_term_window=2, frame_seconds=1.0
)
for _ in range(4):
await b.flush([("0.0s", "u")])
assert len(b.memory.long_term_blocks) == 2 # capped, not 4
assert b.memory.long_term_memory.count("compressed(") == 2
@pytest.mark.asyncio
async def test_delegation_submit_and_fold():
b = InteractionBrain(delegation=_FakeDelegation())
action = parse_action("</response> hold on <delegation> solve this")
info = await b.submit_delegation(action, [])
assert info["status"] == "submitted"
folded = await b.fold_delegations()
assert folded["status"] == "ready"
assert b.memory.qa_history[-1].responses[0][1] == "answer to solve this"
@@ -0,0 +1,282 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import pytest
from vllm_omni.experimental.fullduplex.joyvl.bridges.delegation import (
DelegationResult,
ImageEditDelegationBridge,
ImageGenDelegationBridge,
OpenAIDelegationBridge,
)
from vllm_omni.experimental.fullduplex.joyvl.decision.policy import JoyVLPolicy, sample_frames
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_sample_frames_keeps_recent():
assert sample_frames(["a", "b"], 4) == ["a", "b"]
out = sample_frames([str(i) for i in range(10)], 4)
assert out[-1] == "9" and len(out) == 4
def test_build_messages_stable_head_and_query():
p = JoyVLPolicy(num_frames=4)
p.tick(2)
p.set_query("count the bottles")
msgs, user = p.build_messages([{"type": "image_url", "image_url": {"url": "x"}}])
assert msgs[0]["role"] == "system" and "</silence>" in msgs[0]["content"]
assert any("count the bottles" in part.get("text", "") for part in user["content"])
def test_commit_records_only_spoken():
p = JoyVLPolicy()
p.set_query("q")
assert p.commit("</silence>").action.value == "silence"
assert p.brain.response_records == []
assert p.commit("</response> hello").action.value == "response"
assert p.brain.response_records == [("0.0 seconds", "hello")]
def test_commit_dedup_default_is_exact_match_only():
p = JoyVLPolicy() # threshold 1.0 == reference behavior
assert p.commit("</response> a red ball appears").action.value == "response"
assert p.commit("</response> a red ball appears").action.value == "silence" # exact repeat dropped
assert p.commit("</response> a red ball is appearing").action.value == "response" # near-dup kept
def test_commit_dedup_threshold_drops_near_duplicates():
p = JoyVLPolicy(response_dedup_threshold=0.8)
assert p.commit("</response> a red ball appears").action.value == "response"
assert p.commit("</response> a red ball appears!").action.value == "silence" # near-dup dropped
assert p.commit("</response> the sky is now blue").action.value == "response" # distinct kept
def test_commit_dedup_rearms_after_silence():
p = JoyVLPolicy(response_dedup_threshold=0.85)
assert p.commit("</response> someone raises a hand").action.value == "response"
# no gap: an immediate repeat is still suppressed
assert p.commit("</response> someone raises a hand").action.value == "silence"
# a genuine silent tick (hand lowered) re-arms; the recurrence speaks again
assert p.commit("</silence>").action.value == "silence"
assert p.commit("</response> someone raises a hand").action.value == "response"
class _FakeBrain(OpenAIDelegationBridge):
"""Override only the network call so submit/poll plumbing is exercised for real."""
def __init__(self, answer, gate=None):
super().__init__("http://unused/v1", "background-model")
self._answer = answer
self._gate = gate
async def _complete(self, messages):
if self._gate is not None:
await self._gate.wait()
return self._answer
@pytest.mark.asyncio
async def test_openai_delegation_bridge_is_nonblocking_and_resolves():
gate = asyncio.Event()
bridge = _FakeBrain("the bottle is on the left shelf", gate=gate)
frames = [("0.0s", "data:image/jpeg;base64,AAA")]
task_id = await bridge.submit("where is the bottle?", "let me check", frames)
# poll must not block on the slow background brain
assert (await bridge.poll(task_id)).status == "pending"
gate.set()
await asyncio.sleep(0)
for _ in range(10):
result = await bridge.poll(task_id)
if result.status != "pending":
break
await asyncio.sleep(0)
assert result.is_ready
assert "bottle is on the left shelf" in result.digest
assert (await bridge.poll("missing")).status == "error"
class _FakeImageBrain(ImageGenDelegationBridge):
async def _call_images(self, payload):
# echo the prompt back so the test can assert it was passed through
return {"data": [{"b64_json": "QUJD"}]} # "ABC"
@pytest.mark.asyncio
async def test_image_delegation_bridge_returns_media_not_text():
bridge = _FakeImageBrain("http://unused/v1")
task_id = await bridge.submit("draw a red dragon", "", [])
for _ in range(10):
result = await bridge.poll(task_id)
if result.status != "pending":
break
await asyncio.sleep(0)
assert result.is_ready
# the image rides in media as a data URL; digest is only a short text placeholder
assert result.media == "data:image/png;base64,QUJD"
assert "QUJD" not in result.digest and "red dragon" in result.digest
class _FakeEditBrain(ImageEditDelegationBridge):
def __init__(self, base_url, model):
super().__init__(base_url, model)
self.seen_frame = None
async def _call_chat(self, frame, instruction):
self.seen_frame = frame
return {"choices": [{"message": {"content": [{"image_url": {"url": "data:image/png;base64,Q0FSVA=="}}]}}]}
@pytest.mark.asyncio
async def test_image_edit_delegation_conditions_on_frame_and_returns_media():
bridge = _FakeEditBrain("http://unused/v1", "Qwen-Image-Edit")
frames = [("0.0s", "data:image/jpeg;base64,OLD"), ("1.0s", "data:image/jpeg;base64,NOW")]
task_id = await bridge.submit("把画面变成卡通风格", "", frames)
for _ in range(10):
result = await bridge.poll(task_id)
if result.status != "pending":
break
await asyncio.sleep(0)
assert result.is_ready
assert bridge.seen_frame == "data:image/jpeg;base64,NOW" # latest frame, not the old one
assert result.media == "data:image/png;base64,Q0FSVA=="
assert result.media not in result.digest # image rides in media, not context
@pytest.mark.asyncio
async def test_image_edit_delegation_errors_without_frame():
bridge = _FakeEditBrain("http://unused/v1", "Qwen-Image-Edit")
task_id = await bridge.submit("cartoonify", "", [])
for _ in range(10):
result = await bridge.poll(task_id)
if result.status != "pending":
break
await asyncio.sleep(0)
assert result.status == "error"
class _NamedBridge:
def __init__(self, name):
self.name = name
async def submit(self, question, note, frames):
return f"{self.name}-1"
async def poll(self, task_id):
return DelegationResult(task_id, "ready", digest=self.name)
@pytest.mark.asyncio
async def test_routing_delegation_dispatches_by_request():
from vllm_omni.experimental.fullduplex.joyvl.bridges.delegation import RoutingDelegationBridge
chat, image, edit = _NamedBridge("chat"), _NamedBridge("image"), _NamedBridge("edit")
r = RoutingDelegationBridge(chat=chat, image=image, edit=edit)
async def route(q):
tid = await r.submit(q, "", [])
return (await r.poll(tid)).digest
assert await route("把你看到的画面变成卡通风格") == "edit" # restyle current view
assert await route("画一只红色的龙") == "image" # generate a new image
assert await route("这道数学题的答案是什么") == "chat" # hard question -> chat brain
class _Delegation:
async def submit(self, question, note, frames):
return "t1"
async def poll(self, task_id):
return DelegationResult(task_id, "ready", digest="background answer")
def test_build_delegation_off_unless_backend_configured():
from vllm_omni.experimental.fullduplex.joyvl.serving.config import InteractionConfig
from vllm_omni.experimental.fullduplex.joyvl.serving.server import SessionManager
# No backend URL -> delegation disabled (None), NOT a silent stub that folds fake answers.
assert SessionManager._build_delegation(InteractionConfig()) is None
assert SessionManager._build_delegation(InteractionConfig(delegation_kind="router")) is None
# Stub only on explicit opt-in.
assert SessionManager._build_delegation(InteractionConfig(delegation_kind="stub")) is not None
# A real chat backend -> a live bridge.
cfg = InteractionConfig(delegation_backend_url="http://x/v1", delegation_model="m")
assert SessionManager._build_delegation(cfg) is not None
@pytest.mark.asyncio
async def test_delegation_cancel_drops_task():
from vllm_omni.experimental.fullduplex.joyvl.bridges.delegation import StubDelegationBridge
b = StubDelegationBridge(ready_after_ticks=1)
task_id = await b.submit("q", "", [])
b.cancel(task_id)
# after cancel the task is gone -> poll reports it as unknown rather than leaking
assert (await b.poll(task_id)).status == "error"
@pytest.mark.asyncio
async def test_delegation_aclose_clears_pending():
from vllm_omni.experimental.fullduplex.joyvl.bridges.delegation import StubDelegationBridge
b = StubDelegationBridge(ready_after_ticks=5)
task_id = await b.submit("q", "", [])
await b.aclose()
assert (await b.poll(task_id)).status == "error"
@pytest.mark.asyncio
async def test_session_reset_cancels_and_awaits_consolidation():
from vllm_omni.experimental.fullduplex.joyvl.serving.config import InteractionConfig
from vllm_omni.experimental.fullduplex.joyvl.serving.session import InteractionSession
class _Backend:
async def generate(self, *a, **k):
return "</silence>", None
async def aclose(self):
pass
class _Summarizer:
async def summarize_chunk(self, *a, **k):
await asyncio.sleep(100) # hang so the consolidation task is in-flight
return "x"
async def compress_to_long_term(self, *a, **k):
return ""
async def aclose(self):
pass
sess = InteractionSession("s", InteractionConfig(), _Backend(), summarizer=_Summarizer())
sess._spawn_consolidation(1, [("0.0s", "u")])
assert len(sess._consolidating) == 1
await sess.reset() # must cancel AND await the in-flight task, not just clear the set
assert len(sess._consolidating) == 0
@pytest.mark.asyncio
async def test_manager_aclose_closes_without_error():
from vllm_omni.experimental.fullduplex.joyvl.serving.config import InteractionConfig
from vllm_omni.experimental.fullduplex.joyvl.serving.server import SessionManager
mgr = SessionManager(InteractionConfig(enable_memory=False, enable_delegation=False))
await mgr.aclose()
await mgr.aclose() # idempotent
@pytest.mark.asyncio
async def test_delegation_submit_and_fold():
p = JoyVLPolicy(delegation=_Delegation())
action = p.commit("</response> hold on <delegation> hard question")
assert action.action.value == "delegate"
info = await p.submit_if_delegate(action)
assert info["status"] == "submitted"
folded = await p.fold_delegations()
assert folded["status"] == "ready"
assert p.brain.memory.qa_history[-1].responses[0][1] == "background answer"
@@ -0,0 +1,172 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
from collections.abc import AsyncIterator
import pytest
from vllm_omni.experimental.fullduplex.core import protocol as ev
from vllm_omni.experimental.fullduplex.core.adapter import DuplexCapability, OutputChunk
from vllm_omni.experimental.fullduplex.core.runtime import DuplexRuntime
from vllm_omni.experimental.fullduplex.core.session import DuplexSession, DuplexSessionConfig
from vllm_omni.experimental.fullduplex.joyvl.adapter import JoyVLDuplexAdapter
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
async def _feed(events):
for e in events:
yield e
def _collector():
out: list[dict] = []
async def emit(event: dict) -> None:
out.append(event)
return out, emit
class _FakeAdapter:
def __init__(self, chunks, barge_after=None):
self._chunks = chunks
self._barge_after = barge_after
def capabilities(self):
return DuplexCapability(frozenset({"text"}), frozenset({"text"}), proactive=False)
async def on_input(self, session, modality, data):
pass
def should_respond(self, session):
return True
async def respond(self, session) -> AsyncIterator[OutputChunk]:
for i, c in enumerate(self._chunks):
if self._barge_after is not None and i == self._barge_after:
session.barge_in()
yield OutputChunk("text", c)
async def on_barge_in(self, session):
pass
async def on_playback_ack(self, session, cursor):
pass
@pytest.mark.asyncio
async def test_runtime_basic_response():
session = DuplexSession("s", DuplexSessionConfig(output_modalities=("text",)))
rt = DuplexRuntime(session, _FakeAdapter(["a", "b"]))
out, emit = _collector()
await rt.run(_feed([{"type": ev.INPUT_COMMIT}, {"type": ev.CLOSE}]), emit)
types = [e["type"] for e in out]
assert types == [ev.RESPONSE_CREATED, ev.RESPONSE_DELTA, ev.RESPONSE_DELTA, ev.RESPONSE_DONE]
assert [e["data"] for e in out if e["type"] == ev.RESPONSE_DELTA] == ["a", "b"]
@pytest.mark.asyncio
async def test_runtime_barge_in_drops_stale_output():
session = DuplexSession("s")
rt = DuplexRuntime(session, _FakeAdapter(["a", "b", "c"], barge_after=1))
out, emit = _collector()
await rt.run(_feed([{"type": ev.INPUT_COMMIT}, {"type": ev.CLOSE}]), emit)
data = [e["data"] for e in out if e["type"] == ev.RESPONSE_DELTA]
assert data == ["a"]
assert ev.RESPONSE_DONE not in [e["type"] for e in out]
class _SlowAdapter:
def capabilities(self):
return DuplexCapability(frozenset({"text"}), frozenset({"text"}), proactive=False)
async def on_input(self, session, modality, data):
pass
def should_respond(self, session):
return True
async def respond(self, session) -> AsyncIterator[OutputChunk]:
for c in ["a", "b", "c", "d", "e"]:
await asyncio.sleep(0.02)
yield OutputChunk("text", c)
async def on_barge_in(self, session):
pass
async def on_playback_ack(self, session, cursor):
pass
@pytest.mark.asyncio
async def test_runtime_cancel_event_interrupts_active_response():
session = DuplexSession("s")
rt = DuplexRuntime(session, _SlowAdapter())
out, emit = _collector()
async def feed():
yield {"type": ev.INPUT_COMMIT}
await asyncio.sleep(0.03)
yield {"type": ev.RESPONSE_CANCEL}
yield {"type": ev.CLOSE}
await rt.run(feed(), emit)
deltas = [e["data"] for e in out if e["type"] == ev.RESPONSE_DELTA]
types = [e["type"] for e in out]
assert len(deltas) < 5
assert ev.RESPONSE_CANCELLED in types
assert ev.RESPONSE_DONE not in types
@pytest.mark.asyncio
async def test_runtime_new_response_supersedes_inflight_without_blocking():
# A second response trigger arriving while the first is still streaming must
# supersede it (barge-in) rather than block the input loop until it finishes.
session = DuplexSession("s")
rt = DuplexRuntime(session, _SlowAdapter())
out, emit = _collector()
async def feed():
yield {"type": ev.INPUT_COMMIT}
await asyncio.sleep(0.03) # let the first response emit ~one chunk
yield {"type": ev.INPUT_COMMIT} # supersedes the in-flight first response
await asyncio.sleep(0.2) # let the second response finish
yield {"type": ev.CLOSE}
await rt.run(feed(), emit)
created = [e for e in out if e["type"] == ev.RESPONSE_CREATED]
done = [e for e in out if e["type"] == ev.RESPONSE_DONE]
# two responses created; only the latest one completes (the first was cancelled)
assert len(created) == 2
assert len(done) == 1
assert done[0]["response_index"] == created[-1]["response_index"]
@pytest.mark.asyncio
async def test_joyvl_adapter_speaks_then_silences():
replies = iter(["</response> a fire is breaking out", "</silence>"])
async def fake_generate(messages):
return next(replies)
adapter = JoyVLDuplexAdapter(fake_generate, num_frames=4)
cfg = DuplexSessionConfig(input_modalities=("video", "text"), output_modalities=("text",), proactive=True)
rt = DuplexRuntime(DuplexSession("s", cfg), adapter)
out, emit = _collector()
await rt.run(
_feed(
[
{"type": ev.INPUT_APPEND, "modality": "text", "data": "alert me if a fire breaks out"},
{"type": ev.INPUT_APPEND, "modality": "video", "data": "data:image/jpeg;base64,AAA"},
{"type": ev.INPUT_APPEND, "modality": "video", "data": "data:image/jpeg;base64,BBB"},
{"type": ev.CLOSE},
]
),
emit,
)
deltas = [e["data"] for e in out if e["type"] == ev.RESPONSE_DELTA]
assert "a fire is breaking out" in deltas
assert deltas.count("a fire is breaking out") == 1
@@ -1,27 +1,299 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Detailed custom Qwen-Image pipeline for E2E tests.
"""Test doubles for custom-pipeline E2E and RLHF integration.
This pipeline follows the structure of the user's reference implementation:
- supports pre-tokenized prompt IDs via OmniCustomPrompt-style dict input
- uses an SDE scheduler that can return step logprobs
- returns structured trajectory_* fields (latents, timesteps, log_probs)
consistent with the BAGEL trajectory recording design
Three layers:
* ``FlowMatchSDEDiscreteSchedulerForTest`` — SDE scheduler with per-step logprobs.
* ``vLLMOmniColocateWorkerExtensionForTest`` — verl-omni colocate worker extension mirror.
* ``QwenImagePipelineWithLogProbForTest`` — Qwen-Image pipeline returning trajectory_* fields.
"""
from __future__ import annotations
import ctypes
import logging
import math
import os
import platform
import signal
from dataclasses import dataclass
from typing import Any, Literal
import torch
from diffusers import FlowMatchEulerDiscreteScheduler
from diffusers.utils import BaseOutput
from diffusers.utils.torch_utils import randn_tensor
from msgspec import field
from tests.e2e.offline_inference.custom_pipeline.flow_match_sde_scheduler import FlowMatchSDEDiscreteSchedulerForTest
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.models.qwen_image import QwenImagePipeline
from vllm_omni.diffusion.worker.diffusion_worker import CustomPipelineWorkerExtension
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.lora.request import LoRARequest as OmniLoRARequest
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Scheduler layer
# ---------------------------------------------------------------------------
@dataclass
class FlowMatchSDEDiscreteSchedulerOutputForTest(BaseOutput):
"""
Output class for the scheduler's `step` function output.
Args:
prev_sample (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_channels)` for images):
Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the
denoising loop.
log_prob (`torch.FloatTensor` of shape `(batch_size,)`, *optional*):
The log probability of the previous sample.
prev_sample_mean (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_channels)` for images):
The mean of the computed sample of previous timestep.
std_dev_t (`torch.FloatTensor` of shape `(batch_size, 1, 1)`):
The standard deviation used to compute `prev_sample`.
"""
prev_sample: torch.FloatTensor
log_prob: torch.FloatTensor | None
prev_sample_mean: torch.FloatTensor
std_dev_t: torch.FloatTensor
class FlowMatchSDEDiscreteSchedulerForTest(FlowMatchEulerDiscreteScheduler):
"""SDE version of the FlowMatchEulerDiscreteScheduler.
The implementation is based on FlowGRPO paper (https://arxiv.org/abs/2505.05470)
and diffusers v0.37 branch.
"""
def step(
self,
model_output: torch.FloatTensor,
timestep: float | torch.FloatTensor,
sample: torch.FloatTensor,
s_churn: float = 0.0,
s_tmin: float = 0.0,
s_tmax: float = float("inf"),
s_noise: float = 1.0,
generator: torch.Generator | None = None,
per_token_timesteps: torch.Tensor | None = None,
return_dict: bool = True,
noise_level: float = 0.7,
prev_sample: torch.FloatTensor | None = None,
sde_type: Literal["sde", "cps"] = "sde",
return_logprobs: bool = True,
) -> FlowMatchSDEDiscreteSchedulerOutputForTest | tuple:
"""
Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion
process from the learned model outputs (most often the predicted noise).
Modified from https://github.com/yifan123/flow_grpo/blob/main/flow_grpo/diffusers_patch/sd3_sde_with_logprob.py
Args:
model_output (`torch.FloatTensor`):
The direct output from learned diffusion model.
timestep (`float`):
The current discrete timestep in the diffusion chain.
sample (`torch.FloatTensor`):
A current instance of a sample created by the diffusion process.
s_churn (`float`):
s_tmin (`float`):
s_tmax (`float`):
s_noise (`float`, defaults to 1.0):
Scaling factor for noise added to the sample.
generator (`torch.Generator`, *optional*):
A random number generator.
per_token_timesteps (`torch.Tensor`, *optional*):
The timesteps for each token in the sample.
return_dict (`bool`):
Whether or not to return a
[`~schedulers.scheduling_flow_match_euler_discrete.FlowMatchSDEDiscreteSchedulerOutput`] or tuple.
noise_level (`float`, *optional*, defaults to 0.7):
The noise level used in the SDE.
prev_sample (`torch.FloatTensor`, *optional*):
The sample from the previous timestep. If not provided, it will be sampled inside the function.
sde_type (`str`, *optional*, defaults to "sde"):
The type of SDE to use. Choose between "sde" and "cps".
return_logprobs (`bool`, *optional*, defaults to True):
Whether to return log probabilities of the previous sample.
"""
if isinstance(timestep, (int, torch.IntTensor, torch.LongTensor)):
raise ValueError(
(
"Passing integer indices (e.g. from `enumerate(timesteps)`) as timesteps to"
" `FlowMatchEulerDiscreteScheduler.step()` is not supported. Make sure to pass"
" one of the `scheduler.timesteps` as a timestep."
),
)
if self.step_index is None:
self._init_step_index(timestep)
# Upcast to avoid precision issues when computing prev_sample
sample = sample.to(torch.float32)
if prev_sample is not None:
prev_sample = prev_sample.to(torch.float32)
prev_sample, log_prob, prev_sample_mean, std_dev_t = self.sample_previous_step(
sample=sample,
model_output=model_output,
generator=generator,
per_token_timesteps=per_token_timesteps,
noise_level=noise_level,
prev_sample=prev_sample,
sde_type=sde_type,
return_logprobs=return_logprobs,
)
# upon completion increase step index by one
self._step_index += 1
if per_token_timesteps is None:
# Cast sample back to model compatible dtype
prev_sample = prev_sample.to(model_output.dtype)
if not return_dict:
return (prev_sample, log_prob, prev_sample_mean, std_dev_t)
return FlowMatchSDEDiscreteSchedulerOutputForTest(
prev_sample=prev_sample, log_prob=log_prob, prev_sample_mean=prev_sample_mean, std_dev_t=std_dev_t
)
def sample_previous_step(
self,
sample: torch.Tensor,
model_output: torch.Tensor,
timestep: torch.FloatTensor | None = None,
generator: torch.Generator | None = None,
per_token_timesteps: torch.Tensor | None = None,
noise_level: float = 0.7,
prev_sample: torch.Tensor | None = None,
sde_type: Literal["cps", "sde"] = "sde",
return_logprobs: bool = True,
):
assert sde_type in ["sde", "cps"]
assert sample.dtype == torch.float32
if prev_sample is not None:
assert prev_sample.dtype == torch.float32
if per_token_timesteps is not None:
raise NotImplementedError("per_token_timesteps is not supported yet for FlowMatchSDEDiscreteScheduler.")
else:
if timestep is None:
sigma_idx = self.step_index
sigma = self.sigmas[sigma_idx]
sigma_prev = self.sigmas[sigma_idx + 1]
else:
sigma_idx = torch.tensor([self.index_for_timestep(t) for t in timestep])
sigma = self.sigmas[sigma_idx].view(-1, *([1] * (len(sample.shape) - 1)))
sigma_prev = self.sigmas[sigma_idx + 1].view(-1, *([1] * (len(sample.shape) - 1)))
sigma_max = self.sigmas[1]
dt = sigma_prev - sigma
if sde_type == "sde":
std_dev_t = torch.sqrt(sigma / (1 - torch.where(sigma == 1, sigma_max, sigma))) * noise_level
prev_sample_mean = (
sample * (1 + std_dev_t**2 / (2 * sigma) * dt)
+ model_output * (1 + std_dev_t**2 * (1 - sigma) / (2 * sigma)) * dt
)
if prev_sample is None:
variance_noise = randn_tensor(
model_output.shape,
generator=generator,
device=model_output.device,
dtype=model_output.dtype,
)
prev_sample = prev_sample_mean + std_dev_t * torch.sqrt(-1 * dt) * variance_noise
if return_logprobs:
log_prob = (
-((prev_sample.detach() - prev_sample_mean) ** 2) / (2 * ((std_dev_t * torch.sqrt(-1 * dt)) ** 2))
- torch.log(std_dev_t * torch.sqrt(-1 * dt))
- torch.log(torch.sqrt(2 * torch.as_tensor(math.pi)))
)
else:
log_prob = None
elif sde_type == "cps":
std_dev_t = sigma_prev * math.sin(noise_level * math.pi / 2)
pred_original_sample = sample - sigma * model_output
noise_estimate = sample + model_output * (1 - sigma)
prev_sample_mean = pred_original_sample * (1 - sigma_prev) + noise_estimate * torch.sqrt(
sigma_prev**2 - std_dev_t**2
)
if prev_sample is None:
variance_noise = randn_tensor(
model_output.shape,
generator=generator,
device=model_output.device,
dtype=model_output.dtype,
)
prev_sample = prev_sample_mean + std_dev_t * variance_noise
if return_logprobs:
log_prob = -((prev_sample.detach() - prev_sample_mean) ** 2)
else:
log_prob = None
# mean along all but batch dimension
log_prob = log_prob.mean(dim=tuple(range(1, log_prob.ndim))) if log_prob is not None else None
return prev_sample, log_prob, prev_sample_mean, std_dev_t
# ---------------------------------------------------------------------------
# Worker extension layer
# ---------------------------------------------------------------------------
class OmniTensorLoRARequestForTest(OmniLoRARequest):
peft_config: dict = field(default=None)
lora_tensors: dict = field(default=None)
def set_death_signal() -> None:
"""Verbatim port of ``verl.workers.rollout.vllm_rollout.utils.set_death_signal``."""
if platform.system() != "Linux":
return
try:
libc = ctypes.CDLL("libc.so.6")
libc.prctl(1, signal.SIGKILL)
if os.getppid() == 1:
os.kill(os.getpid(), signal.SIGKILL)
except Exception: # noqa: BLE001
# libc.so.6 may not be present (musl, alpine) — best-effort only.
pass
class vLLMOmniColocateWorkerExtensionForTest(CustomPipelineWorkerExtension):
"""Mirror of ``vLLMOmniColocateWorkerExtension`` (verl-omni).
The production ``__new__`` runs ``set_death_signal`` + ``VLLMOmniHijack.hijack``
on every vLLM worker process. Replicating both ensures this test
reproduces the same monkey-patched environment.
"""
def __new__(cls, **kwargs):
set_death_signal()
return super().__new__(cls)
@staticmethod
def test_extension_name() -> str:
"""Return a stable identifier for assertions in unit tests."""
return "vllm-omni-colocate-worker-extension-for-test"
# ---------------------------------------------------------------------------
# Pipeline layer
# ---------------------------------------------------------------------------
def _maybe_to_cpu(v):
@@ -30,8 +302,6 @@ def _maybe_to_cpu(v):
return v
# Custom pipeline class for QwenImage that returns log probabilities during the diffusion process.
# This is for test
class QwenImagePipelineWithLogProbForTest(QwenImagePipeline):
supports_request_batch = False
@@ -1,30 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""E2E test that follows the EXACT same flow as
``verl-omni/tests/workers/rollout/rollout_vllm/test_vllm_omni_generate.py``
but inlines all the ``verl`` / ``verl_omni`` symbols the flow depends on
(stripped down to only what the diffusion-mode test actually exercises).
"""Ray-serializable verl-omni HTTP server test double for RLHF E2E tests.
Flow (identical to the verl-omni test):
1. Build two ``OmegaConf`` configs (``DiffusionRolloutConfig`` /
``DiffusionModelConfig`` shape).
2. ``ServerCls = ray.remote(vLLMOmniHttpServerLocal)`` then
``ServerCls.options(...).remote(config=..., model_config=...,
rollout_mode=RolloutMode.STANDALONE, workers=[], replica_rank=0,
node_rank=0, gpus_per_node=1, nnodes=1, cuda_visible_devices="0")``.
3. ``ray.get(server.launch_server.remote())``.
4. ``server.generate.remote(prompt_ids=..., sampling_params=..., request_id=...)``
and assert the returned ``DiffusionOutput``.
The inlined classes / helpers mirror their ``verl`` / ``verl_omni``
counterparts; bits not reached by this test (AR mode, LoRA,
quantization, run_uvicorn, headless, profiler, port wiring) are removed.
Usage:
pytest tests/e2e/offline_inference/rlhf_test/\
test_ray_async_omni_qwen_image_generate.py -v -s
The actor class must live in an importable module (not a pytest test file) so
Ray workers can unpickle ``vLLMOmniHttpServerLocal`` via
``tests.e2e.features.helpers.verl_omni_server``.
"""
from __future__ import annotations
@@ -35,23 +16,15 @@ import logging
import os
from dataclasses import asdict
from enum import Enum
from functools import lru_cache
from typing import Any
from uuid import uuid4
import pytest
import ray
import torch
import torchvision.transforms as T
from huggingface_hub import snapshot_download
from omegaconf import DictConfig, OmegaConf
from pydantic import BaseModel, ConfigDict
from transformers import AutoTokenizer
from vllm.entrypoints.openai.api_server import build_app
from vllm.utils.argparse_utils import FlexibleArgumentParser
import vllm_omni.entrypoints.cli.serve
from tests.helpers.mark import hardware_test
from vllm_omni.engine.arg_utils import OmniEngineArgs
from vllm_omni.entrypoints.async_omni import AsyncOmni
from vllm_omni.entrypoints.openai.api_server import omni_init_app_state
@@ -60,33 +33,8 @@ from vllm_omni.outputs import OmniRequestOutput
logger = logging.getLogger(__name__)
CUSTOM_PIPELINE_CLASS = (
"tests.e2e.offline_inference.custom_pipeline.qwen_image_pipeline_with_logprob.QwenImagePipelineWithLogProbForTest"
)
WORKER_EXTENSION_CLASS = (
"tests.e2e.offline_inference.custom_pipeline.worker_extension.vLLMOmniColocateWorkerExtensionForTest"
)
# Use your specified HF repo name/path directly here
MODEL = "tiny-random/Qwen-Image"
TOKENIZER_MODEL = "Qwen/Qwen2-1.5B-Instruct"
def _resolve_model_path(repo_id: str) -> str:
"""Resolve an HF repo ID to a local snapshot path, downloading if needed."""
# Allow overriding with a pre-existing local path (skips download).
if os.path.isdir(repo_id):
return repo_id
return snapshot_download(repo_id=repo_id)
_MIN_PROMPT_TOKENS = 35
# ---------------------------------------------------------------------
# Inlined: verl.workers.rollout.replica.RolloutMode
# ---------------------------------------------------------------------
CUSTOM_PIPELINE_CLASS = "tests.e2e.features.helpers.custom_pipeline.QwenImagePipelineWithLogProbForTest"
WORKER_EXTENSION_CLASS = "tests.e2e.features.helpers.custom_pipeline.vLLMOmniColocateWorkerExtensionForTest"
class RolloutMode(Enum):
@@ -95,11 +43,6 @@ class RolloutMode(Enum):
STANDALONE = "standalone"
# ---------------------------------------------------------------------
# Inlined: verl_omni.workers.rollout.replica.DiffusionOutput
# ---------------------------------------------------------------------
class DiffusionOutput(BaseModel):
"""Pydantic mirror of ``verl_omni.workers.rollout.replica.DiffusionOutput``."""
@@ -112,11 +55,6 @@ class DiffusionOutput(BaseModel):
extra_fields: dict[str, Any] = {}
# ---------------------------------------------------------------------
# Inlined: verl.utils.tokenizer.normalize_token_ids
# ---------------------------------------------------------------------
def normalize_token_ids(tokenized_output) -> list[int]:
token_ids = tokenized_output
if isinstance(tokenized_output, dict):
@@ -142,11 +80,6 @@ def normalize_token_ids(tokenized_output) -> list[int]:
return out
# ---------------------------------------------------------------------
# Inlined: verl.workers.rollout.vllm_rollout.utils.build_cli_args_from_config
# ---------------------------------------------------------------------
def build_cli_args_from_config(config: dict[str, Any]) -> list[str]:
cli_args: list[str] = []
for k, v in config.items():
@@ -166,34 +99,6 @@ def build_cli_args_from_config(config: dict[str, Any]) -> list[str]:
return cli_args
# ---------------------------------------------------------------------
# Tokenizer helper (test-side)
# ---------------------------------------------------------------------
@lru_cache(maxsize=1)
def _get_tokenizer():
return AutoTokenizer.from_pretrained(TOKENIZER_MODEL, trust_remote_code=True)
def _tokenize_prompt(text: str) -> list[int]:
tokenizer = _get_tokenizer()
messages = [{"role": "user", "content": text}]
token_ids = normalize_token_ids(tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=False))
assert len(token_ids) > _MIN_PROMPT_TOKENS, (
f"Prompt too short ({len(token_ids)} tokens, need >{_MIN_PROMPT_TOKENS}). "
"The pipeline drops the first 34 chat-template prefix tokens; "
"use a longer prompt so content tokens remain after the drop."
)
return token_ids
# ---------------------------------------------------------------------
# Inlined: vLLMOmniHttpServer (verl-omni) + relevant vLLMHttpServer (verl)
# bits, stripped to the diffusion-mode path the test exercises.
# ---------------------------------------------------------------------
def _cfg_get(cfg: Any, key: str, default: Any = None) -> Any:
"""Uniform getter that works for both DictConfig and dataclass-like objects."""
if isinstance(cfg, DictConfig):
@@ -202,13 +107,8 @@ def _cfg_get(cfg: Any, key: str, default: Any = None) -> Any:
class vLLMOmniHttpServerLocal:
"""Self-contained simulation of ``vLLMOmniHttpServer`` for diffusion mode.
"""Self-contained simulation of ``vLLMOmniHttpServer`` for diffusion mode."""
Constructor signature matches ``vLLMHttpServer.__init__`` exactly so the
Ray-actor call site in the test is identical to the verl-omni one.
"""
# ------------------------------------------------------------------ init
def __init__(
self,
config,
@@ -224,8 +124,6 @@ class vLLMOmniHttpServerLocal:
os.environ.setdefault("CUDA_VISIBLE_DEVICES", cuda_visible_devices)
os.environ["VERL_REPLICA_RANK"] = str(replica_rank)
# Resolve configs. verl uses omega_conf_to_dataclass; here we keep
# them as DictConfig — every read site below uses ``_cfg_get``.
self.config = OmegaConf.create(config) if not isinstance(config, DictConfig) else config
self.model_config = OmegaConf.create(model_config) if not isinstance(model_config, DictConfig) else model_config
@@ -237,8 +135,6 @@ class vLLMOmniHttpServerLocal:
self.nnodes = nnodes
self.global_steps = 0
self.engine: AsyncOmni | None = None
# Diffusion-mode parity with ``vLLMOmniHttpServer._post_init``.
self._to_tensor = T.PILToTensor()
logger.info(
@@ -249,11 +145,7 @@ class vLLMOmniHttpServerLocal:
cuda_visible_devices,
)
# ---------------------------------------------------------- launch_server
async def launch_server(self) -> None:
"""Diffusion-only mirror of ``vLLMHttpServer.launch_server``."""
# vLLMOmniHttpServer._get_engine_kwargs_key() returns "vllm_omni".
engine_kwargs_root = _cfg_get(self.config, "engine_kwargs", {}) or {}
if isinstance(engine_kwargs_root, (dict, DictConfig)):
engine_kwargs = engine_kwargs_root.get("vllm_omni", {}) or {}
@@ -261,10 +153,8 @@ class vLLMOmniHttpServerLocal:
engine_kwargs = {}
engine_kwargs = {k: v for k, v in dict(engine_kwargs).items() if v is not None}
# vLLMOmniHttpServer._preprocess_engine_kwargs (diffusion path).
engine_kwargs.pop("output_mode", None)
# ``compilation_config`` handling — verbatim from parent launch_server.
compilation_config = engine_kwargs.pop("compilation_config", None) or {}
if isinstance(compilation_config, str):
compilation_config = json.loads(compilation_config)
@@ -273,8 +163,6 @@ class vLLMOmniHttpServerLocal:
compilation_config.setdefault("cudagraph_mode", "FULL_AND_PIECEWISE")
compilation_config = json.dumps(compilation_config)
# Args dict (subset of parent's; the diffusion override drops AR-only
# entries via ``_get_override_generation_config`` returning ``{}``).
args = {
"dtype": _cfg_get(self.config, "dtype", "bfloat16"),
"load_format": _cfg_get(self.config, "load_format", "auto"),
@@ -303,8 +191,6 @@ class vLLMOmniHttpServerLocal:
model_path = str(_cfg_get(self.model_config, "local_path", None) or _cfg_get(self.model_config, "path"))
server_args_list = ["serve", model_path] + build_cli_args_from_config(args)
# Same parser pipeline as parent launch_server (with vllm-omni's
# cmd_init injected, matching ``_get_cli_modules`` + ``_get_cli_description``).
parser = FlexibleArgumentParser(description="vLLM-Omni CLI")
subparsers = parser.add_subparsers(required=False, dest="subparser")
cmds: dict[str, Any] = {}
@@ -316,19 +202,12 @@ class vLLMOmniHttpServerLocal:
if parsed.subparser in cmds:
cmds[parsed.subparser].validate(parsed)
# node_rank == 0 → run_server (the test never sets node_rank != 0).
await self.run_server(parsed)
# ------------------------------------------------------------- run_server
async def run_server(self, args: argparse.Namespace) -> None:
"""Diffusion branch of ``vLLMOmniHttpServer.run_server``."""
engine_args = OmniEngineArgs.from_cli_args(args)
engine_args = asdict(engine_args)
# Diffusion path: pin the in-repo test pipeline. The production code
# calls ``VllmOmniPipelineBase.get_pipeline_path(architecture)``; we
# bypass the registry and bind the test pipeline directly.
engine_args["enable_dummy_pipeline"] = True
engine_args["custom_pipeline_args"] = {"pipeline_class": CUSTOM_PIPELINE_CLASS}
@@ -337,10 +216,7 @@ class vLLMOmniHttpServerLocal:
await omni_init_app_state(engine_client, app.state, args)
self.engine = engine_client
# ``run_uvicorn`` intentionally skipped — the test drives the engine
# via Ray RPC, not HTTP.
# ---------------------------------------------------------------- generate
async def generate(
self,
prompt_ids: list[int],
@@ -351,8 +227,6 @@ class vLLMOmniHttpServerLocal:
negative_prompt_ids: list[int] | None = None,
priority: int = 0, # noqa: ARG002 (signature parity)
) -> DiffusionOutput:
# Verbatim copy of ``vLLMOmniHttpServer._generate_diffusion`` minus
# the LoRA branch (the test does not enable ``lora_as_adapter``).
prompt_ids = normalize_token_ids(prompt_ids)
multi_modal_data: dict[str, Any] = {}
@@ -444,149 +318,3 @@ class vLLMOmniHttpServerLocal:
num_preempted=num_preempted,
extra_fields=extra_fields,
)
# ---------------------------------------------------------------------
# Fixtures & tests
# ---------------------------------------------------------------------
@pytest.fixture
def init_server():
"""Create and launch a vLLMOmniHttpServerLocal Ray actor with Qwen-Image."""
model_path = _resolve_model_path(MODEL)
ray.init(
runtime_env={
"env_vars": {
"TOKENIZERS_PARALLELISM": "true",
"NCCL_DEBUG": "WARN",
"VLLM_LOGGING_LEVEL": "INFO",
}
},
ignore_reinit_error=True,
)
rollout_cfg = OmegaConf.create(
{
"name": "vllm_omni",
"mode": "async",
"tensor_model_parallel_size": 1,
"data_parallel_size": 1,
"pipeline_model_parallel_size": 1,
"gpu_memory_utilization": 0.8,
"max_num_batched_tokens": 8192,
# The custom log-prob pipeline only supports serial requests.
"max_num_seqs": 1,
"max_model_len": 1058,
"dtype": "bfloat16",
"load_format": "auto",
"enforce_eager": True,
"enable_chunked_prefill": False,
"enable_prefix_caching": False,
"enable_sleep_mode": False,
"free_cache_engine": True,
"disable_log_stats": True,
"logprobs_mode": "processed_logprobs",
"scheduling_policy": "fcfs",
"seed": 0,
"n": 4,
"pipeline": {
"height": 512,
"width": 512,
"num_inference_steps": 10,
},
}
)
model_cfg = OmegaConf.create(
{
"path": str(model_path),
"local_path": str(model_path),
"tokenizer_path": TOKENIZER_MODEL,
"trust_remote_code": True,
"load_tokenizer": True,
}
)
ServerCls = ray.remote(vLLMOmniHttpServerLocal)
server = ServerCls.options(
runtime_env={
"env_vars": {
"RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES": "1",
"RAY_EXPERIMENTAL_NOSET_ASCEND_RT_VISIBLE_DEVICES": "1",
"NCCL_CUMEM_ENABLE": "0",
}
},
max_concurrency=10,
num_gpus=1,
).remote(
config=rollout_cfg,
model_config=model_cfg,
rollout_mode=RolloutMode.STANDALONE,
workers=[],
replica_rank=0,
node_rank=0,
gpus_per_node=1,
nnodes=1,
cuda_visible_devices="0",
)
ray.get(server.launch_server.remote())
yield server
ray.shutdown()
@pytest.mark.core_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_generate(init_server):
"""Concurrent generate() calls covering basic output, logprobs, and multi-request correctness."""
server = init_server
prompts = [
"a beautiful sunset over the ocean with vibrant orange and purple clouds "
"reflecting on the calm water surface near a rocky coastline",
"a fluffy orange cat sitting on a wooden windowsill looking outside at "
"a garden full of colorful flowers on a bright sunny afternoon",
"a majestic mountain landscape covered with fresh white snow under a "
"clear blue sky with pine trees in the foreground and a frozen lake",
"a futuristic city at night with neon lights glowing on tall glass "
"skyscrapers and flying vehicles soaring between the buildings",
]
refs = []
for i, prompt in enumerate(prompts):
rid = f"test_{i}_{uuid4().hex[:8]}"
ref = server.generate.remote(
prompt_ids=_tokenize_prompt(prompt),
sampling_params={
"num_inference_steps": 10,
"true_cfg_scale": 4.0,
"height": 512,
"width": 512,
"logprobs": i == 0, # first request includes logprobs
},
request_id=rid,
)
refs.append(ref)
results = ray.get(refs, timeout=600)
for i, output in enumerate(results):
assert isinstance(output, DiffusionOutput), f"Request {i}: expected DiffusionOutput"
img = output.diffusion_output
assert isinstance(img, torch.Tensor) and img.ndim == 3, (
f"Request {i}: expected CHW torch.Tensor, got {type(img).__name__} ndim={getattr(img, 'ndim', None)}"
)
c, h, w = img.shape
assert c == 3, f"Request {i}: expected 3 channels (CHW), got {c}"
assert h > 0 and w > 0, f"Request {i}: image dimensions must be positive"
assert output.stop_reason in ("completed", "aborted", None), (
f"Request {i}: unexpected stop_reason {output.stop_reason!r}"
)
assert 0.0 <= float(img[0, 0, 0]) <= 1.0, f"Request {i}: pixel values must be in [0, 1]"
print(f"All {len(prompts)} concurrent requests returned valid DiffusionOutput")
@@ -0,0 +1,217 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""E2E test that follows the EXACT same flow as
``verl-omni/tests/workers/rollout/rollout_vllm/test_vllm_omni_generate.py``
but inlines all the ``verl`` / ``verl_omni`` symbols the flow depends on
(stripped down to only what the diffusion-mode test actually exercises).
Flow (identical to the verl-omni test):
1. Build two ``OmegaConf`` configs (``DiffusionRolloutConfig`` /
``DiffusionModelConfig`` shape).
2. ``ServerCls = ray.remote(vLLMOmniHttpServerLocal)`` then
``ServerCls.options(...).remote(config=..., model_config=...,
rollout_mode=RolloutMode.STANDALONE, workers=[], replica_rank=0,
node_rank=0, gpus_per_node=1, nnodes=1, cuda_visible_devices="0")``.
3. ``ray.get(server.launch_server.remote())``.
4. ``server.generate.remote(prompt_ids=..., sampling_params=..., request_id=...)``
and assert the returned ``DiffusionOutput``.
The Ray actor lives in ``tests.e2e.features.helpers.verl_omni_server`` so
workers can import it after directory migration (pytest test modules are not
importable as ``test_verl_omni_e2e`` in Ray child processes).
Usage:
pytest tests/e2e/features/rlhf_test/test_verl_omni_e2e.py -v -s
"""
from __future__ import annotations
import os
from functools import lru_cache
from uuid import uuid4
import pytest
import ray
import torch
from huggingface_hub import snapshot_download
from omegaconf import OmegaConf
from transformers import AutoTokenizer
from tests.e2e.features.helpers.verl_omni_server import (
DiffusionOutput,
RolloutMode,
normalize_token_ids,
vLLMOmniHttpServerLocal,
)
from tests.helpers.mark import hardware_test
MODEL = "tiny-random/Qwen-Image"
TOKENIZER_MODEL = "Qwen/Qwen2-1.5B-Instruct"
_MIN_PROMPT_TOKENS = 35
def _resolve_model_path(repo_id: str) -> str:
if os.path.isdir(repo_id):
return repo_id
return snapshot_download(repo_id=repo_id)
@lru_cache(maxsize=1)
def _get_tokenizer():
return AutoTokenizer.from_pretrained(TOKENIZER_MODEL, trust_remote_code=True)
def _tokenize_prompt(text: str) -> list[int]:
tokenizer = _get_tokenizer()
messages = [{"role": "user", "content": text}]
token_ids = normalize_token_ids(tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=False))
assert len(token_ids) > _MIN_PROMPT_TOKENS, (
f"Prompt too short ({len(token_ids)} tokens, need >{_MIN_PROMPT_TOKENS}). "
"The pipeline drops the first 34 chat-template prefix tokens; "
"use a longer prompt so content tokens remain after the drop."
)
return token_ids
@pytest.fixture
def init_server():
"""Create and launch a vLLMOmniHttpServerLocal Ray actor with Qwen-Image."""
model_path = _resolve_model_path(MODEL)
ray.init(
runtime_env={
"env_vars": {
"TOKENIZERS_PARALLELISM": "true",
"NCCL_DEBUG": "WARN",
"VLLM_LOGGING_LEVEL": "INFO",
}
},
ignore_reinit_error=True,
)
rollout_cfg = OmegaConf.create(
{
"name": "vllm_omni",
"mode": "async",
"tensor_model_parallel_size": 1,
"data_parallel_size": 1,
"pipeline_model_parallel_size": 1,
"gpu_memory_utilization": 0.8,
"max_num_batched_tokens": 8192,
# QwenImagePipeline (and the logprob custom subclass) only supports
# serial request execution; max_num_seqs>1 enables request-level batching.
"max_num_seqs": 1,
"max_model_len": 1058,
"dtype": "bfloat16",
"load_format": "auto",
"enforce_eager": True,
"enable_chunked_prefill": False,
"enable_prefix_caching": False,
"enable_sleep_mode": False,
"free_cache_engine": True,
"disable_log_stats": True,
"logprobs_mode": "processed_logprobs",
"scheduling_policy": "fcfs",
"seed": 0,
"n": 4,
"pipeline": {
"height": 512,
"width": 512,
"num_inference_steps": 10,
},
}
)
model_cfg = OmegaConf.create(
{
"path": str(model_path),
"local_path": str(model_path),
"tokenizer_path": TOKENIZER_MODEL,
"trust_remote_code": True,
"load_tokenizer": True,
}
)
ServerCls = ray.remote(vLLMOmniHttpServerLocal)
server = ServerCls.options(
runtime_env={
"env_vars": {
"RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES": "1",
"RAY_EXPERIMENTAL_NOSET_ASCEND_RT_VISIBLE_DEVICES": "1",
"NCCL_CUMEM_ENABLE": "0",
}
},
max_concurrency=10,
num_gpus=1,
).remote(
config=rollout_cfg,
model_config=model_cfg,
rollout_mode=RolloutMode.STANDALONE,
workers=[],
replica_rank=0,
node_rank=0,
gpus_per_node=1,
nnodes=1,
cuda_visible_devices="0",
)
ray.get(server.launch_server.remote())
yield server
ray.shutdown()
@pytest.mark.core_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_generate(init_server):
"""Concurrent generate() calls covering basic output, logprobs, and multi-request correctness."""
server = init_server
prompts = [
"a beautiful sunset over the ocean with vibrant orange and purple clouds "
"reflecting on the calm water surface near a rocky coastline",
"a fluffy orange cat sitting on a wooden windowsill looking outside at "
"a garden full of colorful flowers on a bright sunny afternoon",
"a majestic mountain landscape covered with fresh white snow under a "
"clear blue sky with pine trees in the foreground and a frozen lake",
"a futuristic city at night with neon lights glowing on tall glass "
"skyscrapers and flying vehicles soaring between the buildings",
]
refs = []
for i, prompt in enumerate(prompts):
rid = f"test_{i}_{uuid4().hex[:8]}"
ref = server.generate.remote(
prompt_ids=_tokenize_prompt(prompt),
sampling_params={
"num_inference_steps": 10,
"true_cfg_scale": 4.0,
"height": 512,
"width": 512,
"logprobs": i == 0,
},
request_id=rid,
)
refs.append(ref)
results = ray.get(refs, timeout=600)
for i, output in enumerate(results):
assert isinstance(output, DiffusionOutput), f"Request {i}: expected DiffusionOutput"
img = output.diffusion_output
assert isinstance(img, torch.Tensor) and img.ndim == 3, (
f"Request {i}: expected CHW torch.Tensor, got {type(img).__name__} ndim={getattr(img, 'ndim', None)}"
)
c, h, w = img.shape
assert c == 3, f"Request {i}: expected 3 channels (CHW), got {c}"
assert h > 0 and w > 0, f"Request {i}: image dimensions must be positive"
assert output.stop_reason in ("completed", "aborted", None), (
f"Request {i}: unexpected stop_reason {output.stop_reason!r}"
)
assert 0.0 <= float(img[0, 0, 0]) <= 1.0, f"Request {i}: pixel values must be in [0, 1]"
print(f"All {len(prompts)} concurrent requests returned valid DiffusionOutput")
@@ -1,14 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from tests.e2e.offline_inference.custom_pipeline.qwen_image_pipeline_with_logprob import (
QwenImagePipelineWithLogProbForTest,
)
from tests.e2e.offline_inference.custom_pipeline.worker_extension import (
vLLMOmniColocateWorkerExtensionForTest,
)
__all__ = [
"QwenImagePipelineWithLogProbForTest",
"vLLMOmniColocateWorkerExtensionForTest",
]
@@ -1,220 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import math
from dataclasses import dataclass
from typing import Literal
import torch
from diffusers import FlowMatchEulerDiscreteScheduler
from diffusers.utils import BaseOutput
from diffusers.utils.torch_utils import randn_tensor
@dataclass
class FlowMatchSDEDiscreteSchedulerOutputForTest(BaseOutput):
"""
Output class for the scheduler's `step` function output.
Args:
prev_sample (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_channels)` for images):
Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the
denoising loop.
log_prob (`torch.FloatTensor` of shape `(batch_size,)`, *optional*):
The log probability of the previous sample.
prev_sample_mean (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_channels)` for images):
The mean of the computed sample of previous timestep.
std_dev_t (`torch.FloatTensor` of shape `(batch_size, 1, 1)`):
The standard deviation used to compute `prev_sample`.
"""
prev_sample: torch.FloatTensor
log_prob: torch.FloatTensor | None
prev_sample_mean: torch.FloatTensor
std_dev_t: torch.FloatTensor
class FlowMatchSDEDiscreteSchedulerForTest(FlowMatchEulerDiscreteScheduler):
"""SDE version of the FlowMatchEulerDiscreteScheduler.
The implementation is based on FlowGRPO paper (https://arxiv.org/abs/2505.05470)
and diffusers v0.37 branch.
"""
def step(
self,
model_output: torch.FloatTensor,
timestep: float | torch.FloatTensor,
sample: torch.FloatTensor,
s_churn: float = 0.0,
s_tmin: float = 0.0,
s_tmax: float = float("inf"),
s_noise: float = 1.0,
generator: torch.Generator | None = None,
per_token_timesteps: torch.Tensor | None = None,
return_dict: bool = True,
noise_level: float = 0.7,
prev_sample: torch.FloatTensor | None = None,
sde_type: Literal["sde", "cps"] = "sde",
return_logprobs: bool = True,
) -> FlowMatchSDEDiscreteSchedulerOutputForTest | tuple:
"""
Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion
process from the learned model outputs (most often the predicted noise).
Modified from https://github.com/yifan123/flow_grpo/blob/main/flow_grpo/diffusers_patch/sd3_sde_with_logprob.py
Args:
model_output (`torch.FloatTensor`):
The direct output from learned diffusion model.
timestep (`float`):
The current discrete timestep in the diffusion chain.
sample (`torch.FloatTensor`):
A current instance of a sample created by the diffusion process.
s_churn (`float`):
s_tmin (`float`):
s_tmax (`float`):
s_noise (`float`, defaults to 1.0):
Scaling factor for noise added to the sample.
generator (`torch.Generator`, *optional*):
A random number generator.
per_token_timesteps (`torch.Tensor`, *optional*):
The timesteps for each token in the sample.
return_dict (`bool`):
Whether or not to return a
[`~schedulers.scheduling_flow_match_euler_discrete.FlowMatchSDEDiscreteSchedulerOutput`] or tuple.
noise_level (`float`, *optional*, defaults to 0.7):
The noise level used in the SDE.
prev_sample (`torch.FloatTensor`, *optional*):
The sample from the previous timestep. If not provided, it will be sampled inside the function.
sde_type (`str`, *optional*, defaults to "sde"):
The type of SDE to use. Choose between "sde" and "cps".
return_logprobs (`bool`, *optional*, defaults to True):
Whether to return log probabilities of the previous sample.
"""
if isinstance(timestep, (int, torch.IntTensor, torch.LongTensor)):
raise ValueError(
(
"Passing integer indices (e.g. from `enumerate(timesteps)`) as timesteps to"
" `FlowMatchEulerDiscreteScheduler.step()` is not supported. Make sure to pass"
" one of the `scheduler.timesteps` as a timestep."
),
)
if self.step_index is None:
self._init_step_index(timestep)
# Upcast to avoid precision issues when computing prev_sample
sample = sample.to(torch.float32)
if prev_sample is not None:
prev_sample = prev_sample.to(torch.float32)
prev_sample, log_prob, prev_sample_mean, std_dev_t = self.sample_previous_step(
sample=sample,
model_output=model_output,
generator=generator,
per_token_timesteps=per_token_timesteps,
noise_level=noise_level,
prev_sample=prev_sample,
sde_type=sde_type,
return_logprobs=return_logprobs,
)
# upon completion increase step index by one
self._step_index += 1
if per_token_timesteps is None:
# Cast sample back to model compatible dtype
prev_sample = prev_sample.to(model_output.dtype)
if not return_dict:
return (prev_sample, log_prob, prev_sample_mean, std_dev_t)
return FlowMatchSDEDiscreteSchedulerOutputForTest(
prev_sample=prev_sample, log_prob=log_prob, prev_sample_mean=prev_sample_mean, std_dev_t=std_dev_t
)
def sample_previous_step(
self,
sample: torch.Tensor,
model_output: torch.Tensor,
timestep: torch.FloatTensor | None = None,
generator: torch.Generator | None = None,
per_token_timesteps: torch.Tensor | None = None,
noise_level: float = 0.7,
prev_sample: torch.Tensor | None = None,
sde_type: Literal["cps", "sde"] = "sde",
return_logprobs: bool = True,
):
assert sde_type in ["sde", "cps"]
assert sample.dtype == torch.float32
if prev_sample is not None:
assert prev_sample.dtype == torch.float32
if per_token_timesteps is not None:
raise NotImplementedError("per_token_timesteps is not supported yet for FlowMatchSDEDiscreteScheduler.")
else:
if timestep is None:
sigma_idx = self.step_index
sigma = self.sigmas[sigma_idx]
sigma_prev = self.sigmas[sigma_idx + 1]
else:
sigma_idx = torch.tensor([self.index_for_timestep(t) for t in timestep])
sigma = self.sigmas[sigma_idx].view(-1, *([1] * (len(sample.shape) - 1)))
sigma_prev = self.sigmas[sigma_idx + 1].view(-1, *([1] * (len(sample.shape) - 1)))
sigma_max = self.sigmas[1]
dt = sigma_prev - sigma
if sde_type == "sde":
std_dev_t = torch.sqrt(sigma / (1 - torch.where(sigma == 1, sigma_max, sigma))) * noise_level
prev_sample_mean = (
sample * (1 + std_dev_t**2 / (2 * sigma) * dt)
+ model_output * (1 + std_dev_t**2 * (1 - sigma) / (2 * sigma)) * dt
)
if prev_sample is None:
variance_noise = randn_tensor(
model_output.shape,
generator=generator,
device=model_output.device,
dtype=model_output.dtype,
)
prev_sample = prev_sample_mean + std_dev_t * torch.sqrt(-1 * dt) * variance_noise
if return_logprobs:
log_prob = (
-((prev_sample.detach() - prev_sample_mean) ** 2) / (2 * ((std_dev_t * torch.sqrt(-1 * dt)) ** 2))
- torch.log(std_dev_t * torch.sqrt(-1 * dt))
- torch.log(torch.sqrt(2 * torch.as_tensor(math.pi)))
)
else:
log_prob = None
elif sde_type == "cps":
std_dev_t = sigma_prev * math.sin(noise_level * math.pi / 2)
pred_original_sample = sample - sigma * model_output
noise_estimate = sample + model_output * (1 - sigma)
prev_sample_mean = pred_original_sample * (1 - sigma_prev) + noise_estimate * torch.sqrt(
sigma_prev**2 - std_dev_t**2
)
if prev_sample is None:
variance_noise = randn_tensor(
model_output.shape,
generator=generator,
device=model_output.device,
dtype=model_output.dtype,
)
prev_sample = prev_sample_mean + std_dev_t * variance_noise
if return_logprobs:
log_prob = -((prev_sample.detach() - prev_sample_mean) ** 2)
else:
log_prob = None
# mean along all but batch dimension
log_prob = log_prob.mean(dim=tuple(range(1, log_prob.ndim))) if log_prob is not None else None
return prev_sample, log_prob, prev_sample_mean, std_dev_t
@@ -1,77 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Test worker extension that mirrors
``verl_omni.workers.rollout.vllm_rollout.utils.vLLMOmniColocateWorkerExtension``
Fidelity goals:
* ``__new__`` calls ``set_death_signal()`` (Linux ``PR_SET_PDEATHSIG``) so
spawned vLLM workers die with the parent.
* ``__new__`` calls ``VLLMOmniHijackForTest.hijack()`` which monkey-patches
``vllm_omni.diffusion.lora.manager.DiffusionLoRAManager._load_adapter``
to accept in-memory LoRA tensors (verbatim port of
``verl_omni.utils.vllm_omni.utils.VLLMOmniHijack.hijack``).
The ``verl.utils.vllm.VLLMHijack.hijack()`` call (which patches vLLM's
``LRUCacheWorkerLoRAManager._load_adapter`` for AR/text mode) is
intentionally skipped because (a) ``verl`` is not importable from this
repo and (b) the diffusion test path never reaches that manager.
"""
from __future__ import annotations
import ctypes
import logging
import os
import platform
import signal
from msgspec import field
from vllm_omni.diffusion.worker.diffusion_worker import CustomPipelineWorkerExtension
from vllm_omni.lora.request import LoRARequest as OmniLoRARequest
logger = logging.getLogger(__name__)
class OmniTensorLoRARequestForTest(OmniLoRARequest):
peft_config: dict = field(default=None)
lora_tensors: dict = field(default=None)
def set_death_signal() -> None:
"""Verbatim port of ``verl.workers.rollout.vllm_rollout.utils.set_death_signal``."""
if platform.system() != "Linux":
return
try:
libc = ctypes.CDLL("libc.so.6")
libc.prctl(1, signal.SIGKILL)
if os.getppid() == 1:
os.kill(os.getpid(), signal.SIGKILL)
except Exception: # noqa: BLE001
# libc.so.6 may not be present (musl, alpine) — best-effort only.
pass
# ---------------------------------------------------------------------
# The worker extension itself
# ---------------------------------------------------------------------
class vLLMOmniColocateWorkerExtensionForTest(CustomPipelineWorkerExtension):
"""Mirror of ``vLLMOmniColocateWorkerExtension`` (verl-omni).
The production ``__new__`` runs ``set_death_signal`` + ``VLLMOmniHijack.hijack``
on every vLLM worker process. Replicating both ensures this test
reproduces the same monkey-patched environment.
"""
def __new__(cls, **kwargs):
set_death_signal()
return super().__new__(cls)
@staticmethod
def test_extension_name() -> str:
"""Return a stable identifier for assertions in unit tests."""
return "vllm-omni-colocate-worker-extension-for-test"
@@ -1,37 +0,0 @@
# Stage config for Step-Audio2 CI tests
# Single Thinker stage (audio/text understanding → text). ASR/text-output
# tests don't need Token2Wav; a two-stage config would deadlock on stage-1 skip.
# The following config has been verified on 2x 24GB GPU (L4/RTX3090/RTX4090).
stage_args:
- stage_id: 0
runtime:
process: true
devices: "0"
max_batch_size: 1
engine_args:
model_stage: thinker
model_arch: StepAudio2ThinkerForConditionalGeneration
tensor_parallel_size: 1
worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
max_model_len: 4096
max_num_batched_tokens: 4096
max_num_seqs: 1
gpu_memory_utilization: 0.8
skip_mm_profiling: true
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: text # Output text + audio tokens
is_comprehension: true
final_output: true
final_output_type: text
default_sampling_params:
temperature: 0.7
top_p: 0.9
top_k: -1
max_tokens: 256
seed: 42
detokenize: true
repetition_penalty: 1.05
@@ -38,7 +38,7 @@ if not PROMPT_AUDIO.is_file() or not TARGET_AUDIO.is_file():
allow_module_level=True,
)
pytestmark = [pytest.mark.advanced_model, pytest.mark.diffusion, pytest.mark.tts]
pytestmark = [pytest.mark.slow, pytest.mark.diffusion, pytest.mark.tts]
_CASES = (
pytest.param(
@@ -6,7 +6,6 @@ E2E expansion tests for Step-Audio2 offline inference (nightly CI).
Full-model inference with audio input and audio output.
"""
from pathlib import Path
from typing import Any
import numpy as np
@@ -15,10 +14,11 @@ import pytest
from tests.helpers.mark import hardware_test
from tests.helpers.media import generate_synthetic_audio
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import get_deploy_config_path
from vllm_omni.outputs import OmniRequestOutput
MODEL = "stepfun-ai/Step-Audio-2-mini"
STAGE_CONFIG = str(Path(__file__).parent / "stage_configs" / "step_audio2_ci.yaml")
STAGE_CONFIG = get_deploy_config_path("step_audio2_thinker_ci.yaml")
TEST_PARAMS = [(MODEL, STAGE_CONFIG)]
pytestmark = [pytest.mark.slow, pytest.mark.tts]
@@ -1,34 +1,37 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""E2E online serving test for DreamZero OpenPI websocket serving."""
"""E2E online serving tests for DreamZero OpenPI websocket serving."""
from __future__ import annotations
import os
from pathlib import Path
import cv2
import numpy as np
import pytest
from tests.dreamzero import openpi_client_helper as openpi_client
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServerParams, get_open_port
from tests.helpers.runtime import (
DREAMZERO_ACTION_DIM,
DREAMZERO_ACTION_HORIZON,
DREAMZERO_CAMERA_FILES,
OmniServerParams,
OpenPIWebSocketResponse,
get_open_port,
)
from tests.helpers.stage_config import get_deploy_config_path
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
MODEL = "GEAR-Dreams/DreamZero-DROID"
test_params = [
OmniServerParams(
model=MODEL,
port=8091,
server_args=[
"--deploy-config",
"vllm_omni/deploy/dreamzero_tp1_cfg2.yaml",
],
stage_config_path=get_deploy_config_path("dreamzero_tp1_cfg2.yaml"),
env_dict={
"ATTENTION_BACKEND": "torch",
"DIFFUSION_ATTENTION_BACKEND": "TORCH_SDPA",
@@ -39,10 +42,10 @@ test_params = [
]
def _write_synthetic_video(path: Path, cv2_module, *, channel: int) -> None:
def _write_synthetic_video(path: Path, *, channel: int) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
height, width, num_frames = 180, 320, 24
writer = cv2_module.VideoWriter(str(path), cv2_module.VideoWriter_fourcc(*"mp4v"), 15.0, (width, height))
writer = cv2.VideoWriter(str(path), cv2.VideoWriter_fourcc(*"mp4v"), 15.0, (width, height))
if not writer.isOpened():
raise RuntimeError(f"Failed to open video writer for {path}")
try:
@@ -50,39 +53,47 @@ def _write_synthetic_video(path: Path, cv2_module, *, channel: int) -> None:
frame = np.zeros((height, width, 3), dtype=np.uint8)
frame[..., channel] = (frame_idx * 7) % 255
frame[..., (channel + 1) % 3] = 64
writer.write(cv2_module.cvtColor(frame, cv2_module.COLOR_RGB2BGR))
writer.write(cv2.cvtColor(frame, cv2.COLOR_RGB2BGR))
finally:
writer.release()
def _write_synthetic_dreamzero_videos(client_mod, video_dir: Path) -> None:
for channel, file_name in enumerate(client_mod.CAMERA_FILES.values()):
_write_synthetic_video(video_dir / file_name, client_mod.cv2, channel=channel)
def _write_synthetic_dreamzero_videos(video_dir: Path) -> None:
for channel, file_name in enumerate(DREAMZERO_CAMERA_FILES.values()):
_write_synthetic_video(video_dir / file_name, channel=channel)
def _validate_dreamzero_openpi_session(response: OpenPIWebSocketResponse) -> None:
metadata = response.server_metadata
assert tuple(metadata["image_resolution"]) == (180, 320)
assert metadata["n_external_cameras"] == 2
assert metadata["needs_wrist_camera"] is True
assert metadata["needs_stereo_camera"] is False
assert metadata["needs_session_id"] is True
assert metadata["action_space"] == "joint_position"
assert response.operation_responses[-1]["status"] == "reset successful"
action_tensors = response.action_tensors
assert action_tensors is not None
assert len(action_tensors) == 3
for index, action in enumerate(action_tensors):
assert action.shape == (DREAMZERO_ACTION_HORIZON, DREAMZERO_ACTION_DIM)
assert action.dtype == np.float32
assert np.isfinite(action).all()
@pytest.mark.slow
@pytest.mark.diffusion
@pytest.mark.distributed_cuda
@hardware_test(res={"cuda": "H100"}, num_cards=2)
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_dreamzero_openpi_online(omni_server, tmp_path: Path) -> None:
try:
openpi_client.require_dependencies()
except ModuleNotFoundError as exc:
pytest.skip(str(exc))
def test_dreamzero_openpi_online(omni_server, openai_client, tmp_path: Path) -> None:
video_dir = tmp_path / "dreamzero_videos"
_write_synthetic_dreamzero_videos(openpi_client, video_dir)
result = openpi_client.run_policy_session(
host=omni_server.host,
port=omni_server.port,
video_dir=video_dir,
session_id="dreamzero-online-e2e",
)
openpi_client.validate_session_result(result)
metadata = result["metadata"]
assert metadata["needs_session_id"] is True
assert metadata["needs_stereo_camera"] is False
assert tuple(metadata["image_resolution"]) == (180, 320)
_write_synthetic_dreamzero_videos(video_dir)
response = openai_client.send_robot_openpi_ws_request(
{
"run_dreamzero_policy_session": True,
"video_dir": video_dir,
"session_id": "dreamzero-online-e2e",
}
)[0]
_validate_dreamzero_openpi_session(response)
@@ -7,14 +7,19 @@ import os
import numpy as np
import pytest
from tests.gr00t import openpi_client_helper as openpi_client
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServerParams
from tests.helpers.runtime import (
OmniServerParams,
OpenPIWebSocketResponse,
build_openpi_droid_observation,
)
from tests.helpers.stage_config import get_deploy_config_path
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
MODEL = "nvidia/GR00T-N1.7-3B"
ACTION_KEYS = frozenset({"eef_9d", "gripper_position", "joint_position"})
EXPECTED_EMBODIMENT_TAG = "OXE_DROID_RELATIVE_EEF_RELATIVE_JOINT"
pytest.importorskip("websockets")
pytest.importorskip("openpi_client.msgpack_numpy")
@@ -34,7 +39,7 @@ test_params = [
]
# Reference values captured from the Isaac-GR00T ZMQ server (nvidia/GR00T-N1.7-3B,
# OXE_DROID_RELATIVE_EEF_RELATIVE_JOINT embodiment) using build_droid_observation()
# OXE_DROID_RELATIVE_EEF_RELATIVE_JOINT embodiment) using build_openpi_droid_observation()
# inputs: zero images (256×256), identity eef_9d, zero gripper/joint, "pick up the object".
# Outputs are bit-reproducible across resets and across runs (max_diff=0.0).
_REF_EEF_9D = np.array(
@@ -112,32 +117,65 @@ _REF_JOINT = np.array(
) # rows = step 0, step 4, step 39
@pytest.mark.full_model
def _validate_gr00t_openpi_session(response: OpenPIWebSocketResponse) -> None:
metadata = response.server_metadata
for key in ("action_horizon", "action_keys", "embodiment_tag", "needs_session_id"):
assert key in metadata, f"Missing GR00T metadata key: {key}"
assert metadata["embodiment_tag"] == EXPECTED_EMBODIMENT_TAG
assert metadata["needs_session_id"] is True
assert frozenset(metadata["action_keys"]) == ACTION_KEYS
assert response.operation_responses[-1]["status"] == "reset successful"
actions = response.actions
assert actions is not None
assert frozenset(actions) == ACTION_KEYS
action_horizon = int(metadata["action_horizon"])
expected_shapes = {
"eef_9d": (1, action_horizon, 9),
"gripper_position": (1, action_horizon, 1),
"joint_position": (1, action_horizon, 7),
}
for key, expected_shape in expected_shapes.items():
assert actions[key].shape == expected_shape
assert actions[key].dtype == np.float32
assert np.isfinite(actions[key]).all()
@pytest.mark.slow
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_gr00t_n1d7_openpi_online(omni_server) -> None:
result = openpi_client.run_policy_session(
host=omni_server.host,
port=omni_server.port,
session_id="gr00t-online-e2e",
)
openpi_client.validate_session_result(result)
def test_gr00t_n1d7_openpi_online(omni_server, openai_client) -> None:
response = openai_client.send_robot_openpi_ws_request(
{
"run_default_policy_session": True,
"session_id": "gr00t-online-e2e",
}
)[0]
_validate_gr00t_openpi_session(response)
@pytest.mark.full_model
@pytest.mark.slow
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_gr00t_n1d7_openpi_precision(omni_server) -> None:
def test_gr00t_n1d7_openpi_precision(omni_server, openai_client) -> None:
"""Assert actions match Isaac-GR00T reference (GR00T_NOISE_SEED=42, zero inputs)."""
client = openpi_client.OpenPIWebsocketClient(host=omni_server.host, port=omni_server.port)
try:
obs = openpi_client.build_droid_observation(session_id="gr00t-precision-e2e")
client.reset({})
actions = client.infer(obs)
finally:
client.close()
response = openai_client.send_robot_openpi_ws_request(
{
"operations": [
{"endpoint": "reset", "payload": {}},
{
"endpoint": "infer",
"payload": build_openpi_droid_observation(session_id="gr00t-precision-e2e"),
},
],
}
)[0]
actions = response.actions
assert actions is not None
steps = [0, 4, 39]
np.testing.assert_allclose(
@@ -25,7 +25,7 @@ try:
except ImportError:
_HAS_VOICE_CLONE = False
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
pytestmark = [pytest.mark.slow, pytest.mark.tts]
MODEL = "k2-fsa/OmniVoice"