[Refactor][CI] Reorganize test tree, and move slow e2e to weekly (#4951)

Signed-off-by: wangyu <410167048@qq.com>
This commit is contained in:
wangyu authored and GitHub committed 2026-07-29 23:53:03 +08:00
1 parent 925347332f
commit 7c17111ff5
158 files changed
+3479 -4130

No files matched your search

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.6 MiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 855 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.4 MiB

-225
View File
@@ -1,54 +1,15 @@
from __future__ import annotations
import os
import subprocess
from contextlib import contextmanager
from dataclasses import dataclass
from io import BytesIO
from pathlib import Path
import pytest
import requests
import torch
from PIL import Image
from tests.helpers.runtime import OmniServer, OmniServerParams
def pytest_addoption(parser):
group = parser.getgroup("accuracy-e2e")
group.addoption("--gebench-root", action="store", default=None, help="Local GEBench dataset root")
group.addoption("--gedit-root", action="store", default=None, help="Local GEdit-Bench dataset root")
group.addoption(
"--gebench-model", action="store", default="Qwen/Qwen-Image-2512", help="Generate model for GEBench smoke"
)
group.addoption(
"--gedit-model", action="store", default="Qwen/Qwen-Image-Edit", help="Generate model for GEdit-Bench smoke"
)
group.addoption(
"--accuracy-judge-model",
action="store",
default="QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ",
help="Judge model path",
)
group.addoption("--accuracy-gpu", action="store", default="0", help="Single GPU id used sequentially")
group.addoption("--gebench-port", action="store", type=int, default=8093, help="Generate port for GEBench")
group.addoption("--gedit-port", action="store", type=int, default=8093, help="Generate port for GEdit-Bench")
group.addoption(
"--gebench-samples-per-type",
action="store",
type=int,
default=10,
help="Balanced sample count per GEBench type",
)
group.addoption(
"--gedit-samples-per-group",
action="store",
type=int,
default=20,
help="Balanced sample count per GEdit task group",
)
group.addoption("--accuracy-workers", action="store", type=int, default=1, help="Worker count for accuracy benches")
group.addoption(
"--wan22-i2v-image-source",
action="store",
@@ -77,99 +38,6 @@ def pytest_addoption(parser):
)
def _hf_cache_root() -> Path:
return Path(os.environ.get("HF_HOME", "/root/.cache/huggingface"))
def _dataset_cache_dirs(dataset_id: str) -> list[Path]:
cache_root = _hf_cache_root() / "hub" / f"datasets--{dataset_id.replace('/', '--')}" / "snapshots"
if not cache_root.exists():
return []
return sorted(
(path for path in cache_root.iterdir() if path.is_dir()), key=lambda path: path.stat().st_mtime, reverse=True
)
def _ensure_dataset_snapshot(dataset_id: str) -> Path:
candidates = _dataset_cache_dirs(dataset_id)
if candidates:
return candidates[0]
subprocess.run(
["hf", "download", "--repo-type", "dataset", dataset_id],
check=True,
)
candidates = _dataset_cache_dirs(dataset_id)
if not candidates:
raise FileNotFoundError(
f"Dataset {dataset_id} was downloaded but no snapshot was found under {_hf_cache_root()}"
)
return candidates[0]
def _resolve_dataset_root(request: pytest.FixtureRequest, option_name: str, dataset_id: str) -> Path:
value = request.config.getoption(option_name)
if value:
path = Path(value)
if not path.exists():
pytest.skip(f"Dataset path does not exist: {path}")
return path
return _ensure_dataset_snapshot(dataset_id)
@dataclass
class AccuracyServerConfig:
generate_params: OmniServerParams
judge_params: OmniServerParams
run_level: str
model_prefix: str
@contextmanager
def generate_server(self):
params = self.generate_params
model = self.model_prefix + params.model
server_args = params.server_args or []
if params.use_omni and params.stage_init_timeout is not None:
server_args = ["--stage-init-timeout", str(params.stage_init_timeout), *server_args]
with OmniServer(
model,
server_args,
port=params.port,
env_dict=params.env_dict,
use_omni=params.use_omni,
) as server:
yield server
@contextmanager
def judge_server(self):
params = self.judge_params
model = self.model_prefix + params.model
server_args = params.server_args or []
with OmniServer(
model,
server_args,
port=params.port,
env_dict=params.env_dict,
use_omni=params.use_omni,
) as server:
yield server
@pytest.fixture(scope="session")
def gebench_dataset_root(request: pytest.FixtureRequest) -> Path:
return _resolve_dataset_root(request, "gebench_root", "stepfun-ai/GEBench")
@pytest.fixture(scope="session")
def gedit_dataset_root(request: pytest.FixtureRequest) -> Path:
return _resolve_dataset_root(request, "gedit_root", "stepfun-ai/GEdit-Bench")
@pytest.fixture(scope="session")
def accuracy_workers(request: pytest.FixtureRequest) -> int:
return int(request.config.getoption("accuracy_workers"))
@pytest.fixture(scope="session")
def wan22_i2v_image_source(request: pytest.FixtureRequest) -> str | None:
value = request.config.getoption("wan22_i2v_image_source")
@@ -192,16 +60,6 @@ def hunyuanvideo15_online_timeout_seconds(request: pytest.FixtureRequest) -> int
return int(request.config.getoption("hunyuanvideo15_online_timeout_seconds"))
@pytest.fixture(scope="session")
def gebench_samples_per_type(request: pytest.FixtureRequest) -> int:
return int(request.config.getoption("gebench_samples_per_type"))
@pytest.fixture(scope="session")
def gedit_samples_per_group(request: pytest.FixtureRequest) -> int:
return int(request.config.getoption("gedit_samples_per_group"))
@pytest.fixture(scope="session")
def accuracy_artifact_root() -> Path:
root = Path(__file__).resolve().parent / "artifacts"
@@ -209,12 +67,6 @@ def accuracy_artifact_root() -> Path:
return root
@pytest.fixture(scope="session")
def accuracy_assets_root() -> Path:
root = Path(__file__).resolve().parent / "assets"
return root
@pytest.fixture(scope="session")
def qwen_bear_image(accuracy_artifact_root: Path):
"""Download the Qwen bear image from the URL and save it to the accuracy artifact root."""
@@ -249,80 +101,3 @@ def rabbit_image(accuracy_artifact_root: Path):
image.save(image_path)
yield image
image.close()
def _build_accuracy_server_config(
*,
generate_model: str,
judge_model: str,
shared_gpu: str,
port: int,
run_level: str,
model_prefix: str,
) -> AccuracyServerConfig:
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 CUDA GPU for accuracy benchmark smoke tests.")
if not generate_model:
pytest.skip("No generate model configured for accuracy benchmark test.")
generate_server_args = ["--num-gpus", "1"]
judge_server_args = [
"--max-model-len",
"32768",
"--gpu-memory-utilization",
"0.8",
]
judge_env = {"CUDA_VISIBLE_DEVICES": shared_gpu}
return AccuracyServerConfig(
generate_params=OmniServerParams(
model=generate_model,
port=port,
server_args=generate_server_args,
env_dict={"CUDA_VISIBLE_DEVICES": shared_gpu},
use_omni=True,
stage_init_timeout=300,
),
judge_params=OmniServerParams(
model=judge_model,
port=port,
server_args=judge_server_args,
env_dict=judge_env,
use_omni=False,
),
run_level=run_level,
model_prefix=model_prefix,
)
@pytest.fixture
def gebench_accuracy_servers(
request: pytest.FixtureRequest,
run_level: str,
model_prefix: str,
) -> AccuracyServerConfig:
return _build_accuracy_server_config(
generate_model=request.config.getoption("gebench_model"),
judge_model=request.config.getoption("accuracy_judge_model"),
shared_gpu=str(request.config.getoption("accuracy_gpu")),
port=int(request.config.getoption("gebench_port")),
run_level=run_level,
model_prefix=model_prefix,
)
@pytest.fixture
def gedit_accuracy_servers(
request: pytest.FixtureRequest,
run_level: str,
model_prefix: str,
) -> AccuracyServerConfig:
return _build_accuracy_server_config(
generate_model=request.config.getoption("gedit_model"),
judge_model=request.config.getoption("accuracy_judge_model"),
shared_gpu=str(request.config.getoption("accuracy_gpu")),
port=int(request.config.getoption("gedit_port")),
run_level=run_level,
model_prefix=model_prefix,
)
+345 -2
View File
@@ -3,8 +3,12 @@ import os
import re
import shutil
import subprocess
import sys
import time
from abc import ABC, abstractmethod
from base64 import b64decode, b64encode
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from fractions import Fraction
from hashlib import sha1
from io import BytesIO
@@ -53,8 +57,6 @@ def env_to_apply_ftfy_mock_in_subproc(env: dict[str, str] | None = None) -> dict
def reset_artifact_dir(path: Path) -> Path:
import shutil
if path.exists():
shutil.rmtree(path)
path.mkdir(parents=True, exist_ok=True)
@@ -609,3 +611,344 @@ def download_images(urls: list[str]) -> list[Image.Image]:
images.append(image)
return images
_DREAMZERO_UPSTREAM_SERVER_SCRIPT = (
Path(__file__).resolve().parent / "upstream_mock" / "upstream_socket_server_no_compile.py"
)
_DREAMZERO_DEFAULT_MODEL = "GEAR-Dreams/DreamZero-DROID"
_DREAMZERO_DEFAULT_READY_TIMEOUT_S = int(os.environ.get("OPENPI_SERVICE_READY_TIMEOUT_S", "900"))
_DREAMZERO_DEFAULT_PROMPT = (
"Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan"
)
_DREAMZERO_DEFAULT_SESSION_ID = "openpi-e2e-parity-session"
_DREAMZERO_OPENPI_VLLM_PATH = "/v1/realtime/robot/openpi"
def require_dreamzero_parity_env() -> tuple[Path, Path]:
repo_env = os.environ.get("DREAMZERO_REPO")
if repo_env is None:
raise ValueError("DREAMZERO_REPO environment variable is required.")
repo = Path(repo_env).expanduser()
if not repo.is_dir():
raise FileNotFoundError(f"DREAMZERO_REPO does not exist: {repo}")
checkpoint_dir = repo / "checkpoints" / "dreamzero"
if not checkpoint_dir.is_dir():
raise FileNotFoundError(f"DreamZero checkpoint not found: {checkpoint_dir}")
if str(repo) not in sys.path:
sys.path.insert(0, str(repo))
return repo, checkpoint_dir
def load_dreamzero_upstream_modules():
require_dreamzero_parity_env()
import test_client_AR as dreamzero_upstream_client
from eval_utils.policy_client import WebsocketClientPolicy
return dreamzero_upstream_client, WebsocketClientPolicy
def dreamzero_cfg_parallel_size() -> int:
return int(os.environ.get("OPENPI_E2E_CFG_PARALLEL_SIZE", "2"))
def _dreamzero_parity_vllm_env() -> dict[str, str]:
from tests.helpers.env import pick_least_used_device_indices
from tests.helpers.runtime import get_open_port
parallel_size = dreamzero_cfg_parallel_size()
gpus = [str(index) for index in pick_least_used_device_indices(parallel_size)]
if parallel_size > len(gpus):
raise RuntimeError(
f"cfg_parallel_size={parallel_size} requires at least {parallel_size} GPUs, but only got {gpus}."
)
return {
"CUDA_VISIBLE_DEVICES": ",".join(gpus[:parallel_size]),
"ATTENTION_BACKEND": "torch",
"DIFFUSION_ATTENTION_BACKEND": "TORCH_SDPA",
"VLLM_DISABLE_COMPILE_CACHE": "1",
"MASTER_PORT": str(get_open_port()),
}
def dreamzero_parity_vllm_params(model: str | None = None):
from tests.helpers.runtime import OmniServerParams
from tests.helpers.stage_config import get_deploy_config_path
parallel_size = dreamzero_cfg_parallel_size()
server_args = [
"--served-model-name",
"dreamzero-droid",
"--enforce-eager",
]
if parallel_size > 1:
server_args.extend(["--cfg-parallel-size", str(parallel_size)])
return OmniServerParams(
model=model or os.environ.get("VLLM_DREAMZERO_MODEL", _DREAMZERO_DEFAULT_MODEL),
stage_config_path=get_deploy_config_path("dreamzero_tp1_cfg2.yaml"),
server_args=server_args,
env_dict=_dreamzero_parity_vllm_env(),
)
def _dreamzero_omni_server_from_params(params):
from tests.helpers.runtime import OmniServer
server_args = list(params.server_args or [])
if params.stage_config_path is not None:
server_args += ["--stage-configs-path", params.stage_config_path]
return OmniServer(
params.model,
server_args,
port=params.port,
env_dict=params.env_dict,
use_omni=params.use_omni,
)
@contextmanager
def dreamzero_parity_vllm_client(
params=None,
*,
ready_timeout_s: float = _DREAMZERO_DEFAULT_READY_TIMEOUT_S,
) -> Iterator["DreamZeroParityClient"]:
"""Start vLLM via ``OmniServer`` and return a ready OpenPI parity client."""
with _dreamzero_omni_server_from_params(params or dreamzero_parity_vllm_params()) as server:
client = _dreamzero_wait_for_client_ready(
lambda: _dreamzero_create_openpi_vllm_client(server.host, server.port),
timeout_s=ready_timeout_s,
proc=server.proc,
)
yield DreamZeroParityClient(client)
def _dreamzero_create_openpi_vllm_client(host: str, port: int) -> Any:
from openpi_client import msgpack_numpy
_, WebsocketClientPolicy = load_dreamzero_upstream_modules()
class _OpenPIWebsocketClientPolicy(WebsocketClientPolicy):
def __init__(
self,
host: str = "127.0.0.1",
port: int = 8000,
path: str = _DREAMZERO_OPENPI_VLLM_PATH,
) -> None:
self._uri = f"ws://{host}:{port}{path}"
self._packer = msgpack_numpy.Packer()
self._ws, self._server_metadata = self._wait_for_server()
return _OpenPIWebsocketClientPolicy(host=host, port=port)
def _dreamzero_stop_process(proc: subprocess.Popen[str]) -> None:
log_file = getattr(proc, "_codex_log_file", None)
if proc.poll() is None:
proc.terminate()
try:
proc.wait(timeout=30)
except subprocess.TimeoutExpired: # pragma: no cover - cleanup path
proc.kill()
proc.wait(timeout=10)
if log_file is not None:
log_file.close()
def _dreamzero_wait_for_client_ready(
client_factory: Callable[[], Any],
*,
timeout_s: float,
proc: subprocess.Popen[str] | None = None,
log_path: Path | None = None,
) -> Any:
deadline = time.time() + timeout_s
last_err: Exception | None = None
while time.time() < deadline:
if proc is not None and proc.poll() is not None:
details = ""
if log_path is not None and log_path.exists():
details = log_path.read_text(errors="replace")[-8000:]
raise RuntimeError(f"Service exited before becoming ready with code {proc.returncode}.\n{details}")
try:
return client_factory()
except Exception as exc: # pragma: no cover - retry path
last_err = exc
time.sleep(1)
raise TimeoutError(f"Timed out waiting for websocket service: {last_err}")
def _dreamzero_normalize_reset_response(response: Any) -> str:
from openpi_client import msgpack_numpy
if isinstance(response, str):
return response
decoded = msgpack_numpy.unpackb(response)
if isinstance(decoded, dict):
return str(decoded.get("status"))
return str(decoded)
def normalize_dreamzero_metadata(metadata: dict[str, Any]) -> dict[str, Any]:
normalized = dict(metadata)
if isinstance(normalized.get("image_resolution"), tuple):
normalized["image_resolution"] = list(normalized["image_resolution"])
return normalized
def assert_dreamzero_logs_clean(log_path: Path) -> None:
text = log_path.read_text(errors="replace")
if "SignalException: Process" in text and "got signal: 15" in text:
text = text.split("Traceback (most recent call last):", 1)[0]
assert "Traceback" not in text, text
assert "RuntimeError:" not in text, text
def assert_dreamzero_upstream_log_matches_vllm_baseline(log_path: Path) -> None:
text = log_path.read_text(errors="replace")
assert "DIT Compute Steps 8 steps" not in text, text
assert "DIT Compute Steps 16 steps" in text, text
class DreamZeroParityClient:
"""Connected OpenPI websocket client for DreamZero parity inference."""
def __init__(self, client: Any) -> None:
self._client = client
@property
def raw_client(self) -> Any:
return self._client
def build_obs_sequence(self) -> tuple[dict[str, Any], dict[str, Any]]:
dreamzero_upstream_client, _ = load_dreamzero_upstream_modules()
camera_frames = dreamzero_upstream_client.load_camera_frames()
chunks = dreamzero_upstream_client.build_frame_schedule(
min(v.shape[0] for v in camera_frames.values()),
1,
)
obs0 = dreamzero_upstream_client._make_obs_from_video(
camera_frames,
[0],
_DREAMZERO_DEFAULT_PROMPT,
_DREAMZERO_DEFAULT_SESSION_ID,
)
obs1 = dreamzero_upstream_client._make_obs_from_video(
camera_frames,
chunks[0],
_DREAMZERO_DEFAULT_PROMPT,
_DREAMZERO_DEFAULT_SESSION_ID,
)
return obs0, obs1
def collect_parity_outputs(self) -> tuple[dict[str, Any], list[np.ndarray]]:
metadata = self._client.get_server_metadata()
obs0, obs1 = self.build_obs_sequence()
outputs = [
self._client.infer(dict(obs0)),
self._client.infer(dict(obs1)),
]
assert _dreamzero_normalize_reset_response(self._client.reset({})) == "reset successful"
outputs.append(self._client.infer(dict(obs0)))
self._client._ws.close()
return metadata, outputs
class _DreamZeroParityServer(ABC):
host = "127.0.0.1"
def __init__(
self,
*,
log_path: Path,
port: int | None = None,
ready_timeout_s: float = _DREAMZERO_DEFAULT_READY_TIMEOUT_S,
) -> None:
from tests.helpers.runtime import get_open_port
self.log_path = log_path
self.port = get_open_port() if port is None else port
self.ready_timeout_s = ready_timeout_s
self.proc: subprocess.Popen[str] | None = None
self._log_file = None
@abstractmethod
def _build_argv(self) -> list[str]:
raise NotImplementedError
@abstractmethod
def _build_env(self) -> dict[str, str]:
raise NotImplementedError
@abstractmethod
def _create_client(self) -> Any:
raise NotImplementedError
def _start(self) -> None:
self.log_path.parent.mkdir(parents=True, exist_ok=True)
self._log_file = self.log_path.open("w")
self.proc = subprocess.Popen(
self._build_argv(),
stdout=self._log_file,
stderr=subprocess.STDOUT,
text=True,
cwd=str(Path.cwd()),
env=self._build_env(),
)
self.proc._codex_log_file = self._log_file # type: ignore[attr-defined]
def __enter__(self) -> DreamZeroParityClient:
self._start()
client = _dreamzero_wait_for_client_ready(
self._create_client,
timeout_s=self.ready_timeout_s,
proc=self.proc,
log_path=self.log_path,
)
return DreamZeroParityClient(client)
def __exit__(self, exc_type, exc_val, exc_tb) -> None:
if self.proc is not None:
_dreamzero_stop_process(self.proc)
self.proc = None
class DreamZeroUpstreamServer(_DreamZeroParityServer):
"""Official DreamZero upstream reference server launched via torchrun."""
def _build_argv(self) -> list[str]:
_, checkpoint_dir = require_dreamzero_parity_env()
return [
sys.executable,
"-m",
"torch.distributed.run",
"--standalone",
"--nproc_per_node",
str(dreamzero_cfg_parallel_size()),
str(_DREAMZERO_UPSTREAM_SERVER_SCRIPT),
"--port",
str(self.port),
"--model_path",
str(checkpoint_dir),
]
def _build_env(self) -> dict[str, str]:
from tests.helpers.env import pick_least_used_device_indices
dreamzero_repo, _ = require_dreamzero_parity_env()
env = os.environ.copy()
env.setdefault("PYTHONPATH", "")
env["PYTHONPATH"] = f"{Path.cwd()}:{dreamzero_repo}:{env['PYTHONPATH']}".rstrip(":")
env["CUDA_VISIBLE_DEVICES"] = ",".join(
str(index) for index in pick_least_used_device_indices(max(dreamzero_cfg_parallel_size(), 1))
)
env.setdefault("NO_ALBUMENTATIONS_UPDATE", "1")
env["ATTENTION_BACKEND"] = "torch"
env.setdefault("ENABLE_TENSORRT", "false")
env["ENABLE_DIT_CACHE"] = "false"
env["NUM_DIT_STEPS"] = "16"
env["DYNAMIC_CACHE_SCHEDULE"] = "false"
return env
def _create_client(self) -> Any:
_, WebsocketClientPolicy = load_dreamzero_upstream_modules()
return WebsocketClientPolicy(host=self.host, port=self.port)
+63
View File
@@ -0,0 +1,63 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DreamZero OpenPI upstream parity accuracy test."""
from __future__ import annotations
import os
from pathlib import Path
import numpy as np
import pytest
from tests.e2e.accuracy.helpers import (
DreamZeroUpstreamServer,
assert_dreamzero_logs_clean,
assert_dreamzero_upstream_log_matches_vllm_baseline,
dreamzero_parity_vllm_client,
load_dreamzero_upstream_modules,
normalize_dreamzero_metadata,
)
from tests.helpers.mark import hardware_test
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
EXPECTED_METADATA = {
"image_resolution": [180, 320],
"n_external_cameras": 2,
"needs_wrist_camera": True,
"needs_stereo_camera": False,
"needs_session_id": True,
"action_space": "joint_position",
}
@hardware_test(res={"cuda": "H100"}, num_cards=2)
def test_dreamzero_openpi_upstream_parity(tmp_path: Path) -> None:
"""Assert vLLM OpenPI actions match the upstream DreamZero reference server."""
load_dreamzero_upstream_modules()
upstream_log = tmp_path / "dreamzero_upstream.log"
with DreamZeroUpstreamServer(log_path=upstream_log) as upstream_client:
upstream_metadata, upstream_outputs = upstream_client.collect_parity_outputs()
assert_dreamzero_logs_clean(upstream_log)
assert_dreamzero_upstream_log_matches_vllm_baseline(upstream_log)
with dreamzero_parity_vllm_client() as vllm_client:
vllm_metadata, vllm_outputs = vllm_client.collect_parity_outputs()
assert normalize_dreamzero_metadata(upstream_metadata) == EXPECTED_METADATA
assert normalize_dreamzero_metadata(vllm_metadata) == EXPECTED_METADATA
for idx, (vllm_action, upstream_action) in enumerate(zip(vllm_outputs, upstream_outputs, strict=True)):
np.testing.assert_allclose(
vllm_action,
upstream_action,
rtol=1e-2,
atol=1e-3,
err_msg=f"OpenPI step {idx} output mismatch",
)
@@ -1,97 +0,0 @@
from __future__ import annotations
import json
from pathlib import Path
import pytest
from benchmarks.accuracy.text_to_image.gbench import main as gbench_main
from tests.e2e.accuracy.helpers import infer_model_label, reset_artifact_dir
from tests.helpers.mark import hardware_test
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
@pytest.mark.benchmark
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@pytest.mark.skip(reason="#3256")
def test_gebench_h100_smoke(
gebench_accuracy_servers,
accuracy_artifact_root: Path,
gebench_dataset_root: Path,
accuracy_workers: int,
) -> None:
model_label = infer_model_label(gebench_accuracy_servers.generate_params.model).lower()
output_root = reset_artifact_dir(accuracy_artifact_root / f"gebench_{model_label}")
with gebench_accuracy_servers.generate_server() as generate_server:
for data_type in ("type3", "type4"):
assert (
gbench_main(
[
"generate",
"--dataset-root",
str(gebench_dataset_root),
"--output-root",
str(output_root),
"--base-url",
f"http://{generate_server.host}:{generate_server.port}",
"--model",
generate_server.model,
"--data-type",
data_type,
"--width",
"768",
"--height",
"576",
"--output-compression",
"98",
"--num-inference-steps",
"8",
"--workers",
str(accuracy_workers),
]
)
== 0
)
with gebench_accuracy_servers.judge_server() as judge_server:
for data_type in ("type3", "type4"):
assert (
gbench_main(
[
"evaluate",
"--dataset-root",
str(gebench_dataset_root),
"--output-root",
str(output_root),
"--data-type",
data_type,
"--judge-base-url",
f"http://{judge_server.host}:{judge_server.port}",
"--judge-model",
judge_server.model,
"--judge-api-key",
"EMPTY",
"--workers",
str(accuracy_workers),
]
)
== 0
)
assert gbench_main(["summarize", "--output-root", str(output_root)]) == 0
summary = json.loads((output_root / "summary.json").read_text(encoding="utf-8"))
assert "generation" in summary
assert "evaluation" in summary
for data_type in ("type3", "type4"):
assert data_type in summary["generation"]["by_type"]
assert summary["generation"]["by_type"][data_type]["count"] > 0
assert data_type in summary["evaluation"]["by_type"]
assert summary["evaluation"]["by_type"][data_type]["count"] > 0
assert summary["evaluation"]["overall_mean"] >= 0.45
assert summary["evaluation"]["by_type"]["type3"]["overall_mean"] >= 0.45
assert summary["evaluation"]["by_type"]["type4"]["overall_mean"] >= 0.45
@@ -1,115 +0,0 @@
from __future__ import annotations
import json
from pathlib import Path
import pytest
from benchmarks.accuracy.image_to_image.gedit_bench import GROUPS
from benchmarks.accuracy.image_to_image.gedit_bench import main as gedit_main
from tests.e2e.accuracy.helpers import infer_model_label, reset_artifact_dir
from tests.helpers.mark import hardware_test
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
@pytest.mark.benchmark
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@pytest.mark.skip(reason="#3257")
def test_gedit_bench_h100_smoke(
gedit_accuracy_servers,
accuracy_artifact_root: Path,
gedit_dataset_root: Path,
gedit_samples_per_group: int,
accuracy_workers: int,
) -> None:
model_label = infer_model_label(gedit_accuracy_servers.generate_params.model).lower()
output_root = reset_artifact_dir(accuracy_artifact_root / f"gedit_results_{model_label}")
score_root = reset_artifact_dir(accuracy_artifact_root / f"gedit_scores_{model_label}")
model_name = model_label
with gedit_accuracy_servers.generate_server() as generate_server:
assert (
gedit_main(
[
"generate",
"--dataset-ref",
str(gedit_dataset_root),
"--output-root",
str(output_root),
"--base-url",
f"http://{generate_server.host}:{generate_server.port}",
"--model",
generate_server.model,
"--model-name",
model_name,
"--task-type",
"all",
"--instruction-language",
"all",
"--samples-per-group",
str(gedit_samples_per_group),
"--workers",
str(accuracy_workers),
]
)
== 0
)
with gedit_accuracy_servers.judge_server() as judge_server:
assert (
gedit_main(
[
"evaluate",
"--dataset-ref",
str(gedit_dataset_root),
"--output-root",
str(output_root),
"--model-name",
model_name,
"--save-dir",
str(score_root),
"--task-type",
"all",
"--instruction-language",
"all",
"--judge-base-url",
f"http://{judge_server.host}:{judge_server.port}",
"--judge-model",
judge_server.model,
"--judge-api-key",
"EMPTY",
"--samples-per-group",
str(gedit_samples_per_group),
"--workers",
str(accuracy_workers),
]
)
== 0
)
csv_path = score_root / f"{model_name}_all_all_vie_score.csv"
assert gedit_main(["summarize", "--csv-path", str(csv_path), "--language", "all"]) == 0
summary_path = score_root / f"{model_name}_all_all_summary.json"
summary = json.loads(summary_path.read_text(encoding="utf-8"))
assert set(summary["languages"]) == {"en", "cn"}
for language in ["en", "cn"]:
language_summary = summary["languages"][language]
assert language_summary["overall"]["count"] is not None
assert language_summary["intersection"]["count"] is not None
assert language_summary["overall"]["Q_SC"] is not None
assert language_summary["overall"]["Q_PQ"] is not None
assert language_summary["overall"]["Q_O"] is not None
for group in GROUPS:
group_summary = language_summary["by_group"][group]
assert set(group_summary) == {"count", "Q_SC", "Q_PQ", "Q_O"}
assert summary["languages"]["en"]["overall"]["Q_SC"] >= 6.95
assert summary["languages"]["en"]["overall"]["Q_PQ"] >= 5.8
assert summary["languages"]["en"]["overall"]["Q_O"] >= 6.15
assert summary["languages"]["cn"]["overall"]["Q_SC"] >= 6.9
assert summary["languages"]["cn"]["overall"]["Q_PQ"] >= 5.7
assert summary["languages"]["cn"]["overall"]["Q_O"] >= 6.1
+6 -4
View File
@@ -28,6 +28,7 @@ from tests.e2e.accuracy.helpers import (
model_output_dir,
)
from tests.helpers.mark import hardware_test
from tests.helpers.media import get_asset_path
from tests.helpers.runtime import OmniRunner, OmniServer
from vllm_omni.diffusion.models.hunyuan_image3.prompt_utils import (
build_prompt_tokens,
@@ -94,6 +95,7 @@ def _default_ar_dit_devices() -> tuple[str, str]:
AR_DEVICES, DIT_DEVICES = _default_ar_dit_devices()
MODEL_NAME = "tencent/HunyuanImage-3.0-Instruct"
HUNYUAN_IMAGE_REF_PATH = get_asset_path("hunyuan/hunyuan_image_ref.png")
NUM_INFERENCE_STEPS = 50
GUIDANCE_SCALE = 2.5
@@ -414,7 +416,7 @@ def _run_online(stage_configs_path: str, output_path: Path) -> tuple[Image.Image
torch.accelerator.device_count() < AR_TP_SIZE + DIT_TP_SIZE,
reason=f"Needs {AR_TP_SIZE + DIT_TP_SIZE}+ GPUs ({AR_TP_SIZE} AR + {DIT_TP_SIZE} DiT)",
)
def test_image_to_image_alignment_online(accuracy_artifact_root: Path, accuracy_assets_root: Path) -> None:
def test_image_to_image_alignment_online(accuracy_artifact_root: Path) -> None:
"""Online API test: same pipeline, same seed as offline → PSNR >= 10 dB."""
if importlib.util.find_spec("FlagEmbedding") is None:
raise ImportError("Missing dependency: FlagEmbedding\nInstall with: pip install FlagEmbedding")
@@ -431,7 +433,7 @@ def test_image_to_image_alignment_online(accuracy_artifact_root: Path, accuracy_
scorer = SemanticSimilarityScorer()
clip_scorer = CLIPScorer()
cot_results = scorer.text_similarity(online_cot, COT_REF)
image_ref = Image.open(str(accuracy_assets_root / "hunyuan_image_ref.png")).convert("RGB")
image_ref = Image.open(HUNYUAN_IMAGE_REF_PATH).convert("RGB")
image_clip_score = clip_scorer.image_image_score(online_image, image_ref)
ssim_value, psnr_value = compute_image_ssim_psnr(prediction=online_image, reference=image_ref, compare_mode="RGB")
@@ -529,7 +531,7 @@ def _run_dit_model(
torch.accelerator.device_count() < AR_TP_SIZE + DIT_TP_SIZE,
reason=f"Needs {AR_TP_SIZE + DIT_TP_SIZE}+ GPUs ({AR_TP_SIZE} AR + {DIT_TP_SIZE} DiT)",
)
def test_image_to_image_alignment(accuracy_artifact_root: Path, accuracy_assets_root: Path) -> None:
def test_image_to_image_alignment(accuracy_artifact_root: Path) -> None:
if importlib.util.find_spec("FlagEmbedding") is None:
raise ImportError("Missing dependency: FlagEmbedding\nInstall with: pip install FlagEmbedding")
from tabulate import tabulate # lazy import
@@ -545,7 +547,7 @@ def test_image_to_image_alignment(accuracy_artifact_root: Path, accuracy_assets_
scorer = SemanticSimilarityScorer()
clip_scorer = CLIPScorer()
cot_results = scorer.text_similarity(omni_cot, COT_REF)
image_ref = Image.open(str(accuracy_assets_root / "hunyuan_image_ref.png")).convert("RGB")
image_ref = Image.open(HUNYUAN_IMAGE_REF_PATH).convert("RGB")
image_clip_score = clip_scorer.image_image_score(omni_image, image_ref)
ssim_value, psnr_value = compute_image_ssim_psnr(prediction=omni_image, reference=image_ref, compare_mode="RGB")
@@ -15,6 +15,7 @@ from PIL import Image
from tests.e2e.accuracy.helpers import assert_images_pixel_close, assert_similarity, model_output_dir
from tests.helpers.mark import hardware_test
from tests.helpers.media import get_asset_path
from tests.helpers.runtime import OmniServer
pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]
@@ -33,8 +34,8 @@ P99_THRESHOLD = 3e-1
SSIM_THRESHOLD = 0.97
PSNR_THRESHOLD = 30.0
BASELINE_PATH = get_asset_path("hunyuan/hunyuan_baseline.png")
_REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent
BASELINE_PATH = _REPO_ROOT / "tests" / "assets" / "hunyuan" / "hunyuan_baseline.png"
_OFFLINE_SCRIPT = _REPO_ROOT / "examples" / "offline_inference" / "hunyuan_image3" / "end2end.py"
# DiT-only deploy config with trust_remote_code (based on hunyuan_image3_dit.yaml).
+3 -3
View File
@@ -14,6 +14,7 @@ from benchmarks.accuracy.common import decode_base64_image, pil_to_png_bytes
from tests.e2e.accuracy.helpers import assert_images_pixel_close, assert_similarity, model_output_dir
from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup
from tests.helpers.mark import hardware_test
from tests.helpers.media import get_asset_path
from tests.helpers.runtime import OmniServer
pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]
@@ -46,10 +47,9 @@ EDIT_2511_PROMPT = (
"(帽子紧贴头顶,未贴合部分可调整帽子摆放角度达到完全贴合), 头部与身体过渡必须自然,无拼接痕迹,"
"无额外元素出现在头部四周,纯白背景,不要出现绿色圆圈"
)
EDIT_2511_ASSET_DIR = Path(__file__).resolve().parent / "assets"
EDIT_2511_IMAGE_PATHS = [
EDIT_2511_ASSET_DIR / "qwen_image_edit_2511_test1.png",
EDIT_2511_ASSET_DIR / "qwen_image_edit_2511_test2.png",
get_asset_path("qwen_image_edit/qwen_image_edit_2511_test1.png"),
get_asset_path("qwen_image_edit/qwen_image_edit_2511_test2.png"),
]
PROMPT_SINGLE_IMAGE = "The input is a 2D cartoon bear mascot. Restyle it into a painterly oil artwork with warm colors while preserving the main structure."
@@ -0,0 +1,184 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Launch the upstream DreamZero websocket server with `torch.compile` disabled.
This wrapper is meant for formal parity tests against `vllm serve --omni`.
It monkeypatches `torch.compile` before importing DreamZero modules so that
all import-time decorators and `post_initialize()` compile calls become eager.
For the current DreamZero port baseline we also disable all upstream DiT cache
and step-skipping behavior so the reference server matches vLLM's current
eager/no-skip implementation:
- `ENABLE_DIT_CACHE=false`
- `NUM_DIT_STEPS=16`
- `DYNAMIC_CACHE_SCHEDULE=false`
In our CI/dev environment we also do not install Transformer Engine or
FlashAttention, so upstream's default `socket_test_optimized_AR.main()` path
(`ATTENTION_BACKEND=TE` and then fallback to `FA2`) cannot execute. To keep the
formal server-vs-server test runnable without pulling in those heavyweight
optional deps, this wrapper reproduces upstream `main()` but pins attention to
PyTorch SDPA (`ATTENTION_BACKEND=torch`) for this subprocess only.
Usage:
PYTHONPATH="${DREAMZERO_REPO}" \\
.venv/bin/python -m torch.distributed.run --standalone --nproc_per_node=2 \\
tests/e2e/accuracy/upstream_mock/upstream_socket_server_no_compile.py --port 18081 \\
--model_path "${DREAMZERO_REPO}/checkpoints/dreamzero"
"""
from __future__ import annotations
import os
import sys
from pathlib import Path
import torch
import torch.nn.functional as F
def _identity_compile(*args, **kwargs):
if args and callable(args[0]) and len(args) == 1 and not kwargs:
return args[0]
def deco(fn):
return fn
return deco
torch.compile = _identity_compile
os.environ.setdefault("NO_ALBUMENTATIONS_UPDATE", "1")
DREAMZERO_REPO_ENV = os.environ.get("DREAMZERO_REPO")
if not DREAMZERO_REPO_ENV:
raise RuntimeError("Set DREAMZERO_REPO to an upstream DreamZero checkout before launching this helper.")
DREAMZERO_REPO = Path(DREAMZERO_REPO_ENV).expanduser()
if str(DREAMZERO_REPO) not in sys.path:
sys.path.insert(0, str(DREAMZERO_REPO))
import socket_test_optimized_AR as upstream # noqa: E402
import tyro # noqa: E402
from groot.vla.model.dreamzero.modules import attention as upstream_attention # noqa: E402
from groot.vla.model.dreamzero.modules import wan2_1_submodule as upstream_submodule # noqa: E402
from groot.vla.model.dreamzero.modules import wan_video_dit as upstream_wan_video_dit # noqa: E402
def _torch_varlen_flash_attention(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
q_lens: torch.Tensor | None = None,
k_lens: torch.Tensor | None = None,
dropout_p: float = 0.0,
causal: bool = False,
dtype: torch.dtype = torch.bfloat16,
**_: object,
) -> torch.Tensor:
if q_lens is not None or k_lens is not None:
upstream_attention.warnings.warn(
"Padding mask is disabled in the test-only SDPA fallback.",
)
out_dtype = q.dtype
q = q.transpose(1, 2).to(dtype)
k = k.transpose(1, 2).to(dtype)
v = v.transpose(1, 2).to(dtype)
out = F.scaled_dot_product_attention(
q,
k,
v,
attn_mask=None,
is_causal=causal,
dropout_p=dropout_p,
)
return out.transpose(1, 2).contiguous().to(out_dtype)
upstream_attention.flash_attention = _torch_varlen_flash_attention
upstream_submodule.flash_attention = _torch_varlen_flash_attention
upstream_wan_video_dit.FLASH_ATTN_3_AVAILABLE = False
upstream_wan_video_dit.FLASH_ATTN_2_AVAILABLE = False
upstream_wan_video_dit.SAGE_ATTN_AVAILABLE = False
def main(args: upstream.Args) -> None:
os.environ["ENABLE_DIT_CACHE"] = "false"
os.environ["ATTENTION_BACKEND"] = "torch"
os.environ["NUM_DIT_STEPS"] = "16"
os.environ["DYNAMIC_CACHE_SCHEDULE"] = "false"
torch._dynamo.config.recompile_limit = 800
embodiment_tag = "oxe_droid"
model_path = args.model_path
device_mesh = upstream.init_mesh()
rank = upstream.dist.get_rank()
timeout_delta = upstream.datetime.timedelta(seconds=args.timeout_seconds)
signal_group = upstream.dist.new_group(backend="gloo", timeout=timeout_delta)
upstream.logger.info("Rank %s initialized signal_group (gloo)", rank)
policy = upstream.GrootSimPolicy(
embodiment_tag=upstream.EmbodimentTag(embodiment_tag),
model_path=model_path,
device="cuda" if torch.cuda.is_available() else "cpu",
device_mesh=device_mesh,
)
hostname = upstream.socket.gethostname()
local_ip = upstream.socket.gethostbyname(hostname)
if rank == 0:
upstream.logging.info("Creating server (host: %s, ip: %s)", hostname, local_ip)
parent_dir = os.path.dirname(model_path)
date_suffix = upstream.datetime.datetime.now().strftime("%Y%m%d")
checkpoint_name = os.path.basename(model_path)
output_dir = os.path.join(
parent_dir,
f"real_world_eval_gen_{date_suffix}_{args.index}",
checkpoint_name,
)
os.makedirs(output_dir, exist_ok=True)
upstream.logging.info("Videos will be saved to: %s", output_dir)
else:
output_dir = None
upstream.logging.info("Rank %s starting as worker for distributed inference...", rank)
wrapper_policy = upstream.ARDroidRoboarenaPolicy(
groot_policy=policy,
signal_group=signal_group,
output_dir=output_dir,
)
server_config = upstream.PolicyServerConfig(
image_resolution=(180, 320),
needs_wrist_camera=True,
n_external_cameras=2,
needs_stereo_camera=False,
needs_session_id=True,
action_space="joint_position",
)
if rank == 0:
upstream.logging.info("Using roboarena policy server interface")
upstream.logging.info("Server config: %s", server_config)
roboarena_server = upstream.RoboarenaServer(
policy=wrapper_policy,
server_config=server_config,
host="0.0.0.0",
port=args.port,
)
roboarena_server.serve_forever()
else:
worker = upstream.DistributedWorker(
policy=policy,
signal_group=signal_group,
)
upstream.asyncio.run(worker.run())
if __name__ == "__main__":
upstream.logging.basicConfig(level=upstream.logging.INFO, force=True)
main(tyro.cli(upstream.Args))