[Refactor][CI] Reorganize test tree, and move slow e2e to weekly (#4951)
Signed-off-by: wangyu <410167048@qq.com>
This commit is contained in:
1 parent
925347332f
commit
7c17111ff5
158 files changed
+3479
-4130
No files matched your search
Binary file not shown.
|
Before Width: | Height: | Size: 1.6 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 855 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.4 MiB |
@@ -1,54 +1,15 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from tests.helpers.runtime import OmniServer, OmniServerParams
|
||||
|
||||
|
||||
def pytest_addoption(parser):
|
||||
group = parser.getgroup("accuracy-e2e")
|
||||
group.addoption("--gebench-root", action="store", default=None, help="Local GEBench dataset root")
|
||||
group.addoption("--gedit-root", action="store", default=None, help="Local GEdit-Bench dataset root")
|
||||
group.addoption(
|
||||
"--gebench-model", action="store", default="Qwen/Qwen-Image-2512", help="Generate model for GEBench smoke"
|
||||
)
|
||||
group.addoption(
|
||||
"--gedit-model", action="store", default="Qwen/Qwen-Image-Edit", help="Generate model for GEdit-Bench smoke"
|
||||
)
|
||||
group.addoption(
|
||||
"--accuracy-judge-model",
|
||||
action="store",
|
||||
default="QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ",
|
||||
help="Judge model path",
|
||||
)
|
||||
group.addoption("--accuracy-gpu", action="store", default="0", help="Single GPU id used sequentially")
|
||||
group.addoption("--gebench-port", action="store", type=int, default=8093, help="Generate port for GEBench")
|
||||
group.addoption("--gedit-port", action="store", type=int, default=8093, help="Generate port for GEdit-Bench")
|
||||
group.addoption(
|
||||
"--gebench-samples-per-type",
|
||||
action="store",
|
||||
type=int,
|
||||
default=10,
|
||||
help="Balanced sample count per GEBench type",
|
||||
)
|
||||
group.addoption(
|
||||
"--gedit-samples-per-group",
|
||||
action="store",
|
||||
type=int,
|
||||
default=20,
|
||||
help="Balanced sample count per GEdit task group",
|
||||
)
|
||||
group.addoption("--accuracy-workers", action="store", type=int, default=1, help="Worker count for accuracy benches")
|
||||
group.addoption(
|
||||
"--wan22-i2v-image-source",
|
||||
action="store",
|
||||
@@ -77,99 +38,6 @@ def pytest_addoption(parser):
|
||||
)
|
||||
|
||||
|
||||
def _hf_cache_root() -> Path:
|
||||
return Path(os.environ.get("HF_HOME", "/root/.cache/huggingface"))
|
||||
|
||||
|
||||
def _dataset_cache_dirs(dataset_id: str) -> list[Path]:
|
||||
cache_root = _hf_cache_root() / "hub" / f"datasets--{dataset_id.replace('/', '--')}" / "snapshots"
|
||||
if not cache_root.exists():
|
||||
return []
|
||||
return sorted(
|
||||
(path for path in cache_root.iterdir() if path.is_dir()), key=lambda path: path.stat().st_mtime, reverse=True
|
||||
)
|
||||
|
||||
|
||||
def _ensure_dataset_snapshot(dataset_id: str) -> Path:
|
||||
candidates = _dataset_cache_dirs(dataset_id)
|
||||
if candidates:
|
||||
return candidates[0]
|
||||
|
||||
subprocess.run(
|
||||
["hf", "download", "--repo-type", "dataset", dataset_id],
|
||||
check=True,
|
||||
)
|
||||
candidates = _dataset_cache_dirs(dataset_id)
|
||||
if not candidates:
|
||||
raise FileNotFoundError(
|
||||
f"Dataset {dataset_id} was downloaded but no snapshot was found under {_hf_cache_root()}"
|
||||
)
|
||||
return candidates[0]
|
||||
|
||||
|
||||
def _resolve_dataset_root(request: pytest.FixtureRequest, option_name: str, dataset_id: str) -> Path:
|
||||
value = request.config.getoption(option_name)
|
||||
if value:
|
||||
path = Path(value)
|
||||
if not path.exists():
|
||||
pytest.skip(f"Dataset path does not exist: {path}")
|
||||
return path
|
||||
return _ensure_dataset_snapshot(dataset_id)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AccuracyServerConfig:
|
||||
generate_params: OmniServerParams
|
||||
judge_params: OmniServerParams
|
||||
run_level: str
|
||||
model_prefix: str
|
||||
|
||||
@contextmanager
|
||||
def generate_server(self):
|
||||
params = self.generate_params
|
||||
model = self.model_prefix + params.model
|
||||
server_args = params.server_args or []
|
||||
if params.use_omni and params.stage_init_timeout is not None:
|
||||
server_args = ["--stage-init-timeout", str(params.stage_init_timeout), *server_args]
|
||||
with OmniServer(
|
||||
model,
|
||||
server_args,
|
||||
port=params.port,
|
||||
env_dict=params.env_dict,
|
||||
use_omni=params.use_omni,
|
||||
) as server:
|
||||
yield server
|
||||
|
||||
@contextmanager
|
||||
def judge_server(self):
|
||||
params = self.judge_params
|
||||
model = self.model_prefix + params.model
|
||||
server_args = params.server_args or []
|
||||
with OmniServer(
|
||||
model,
|
||||
server_args,
|
||||
port=params.port,
|
||||
env_dict=params.env_dict,
|
||||
use_omni=params.use_omni,
|
||||
) as server:
|
||||
yield server
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def gebench_dataset_root(request: pytest.FixtureRequest) -> Path:
|
||||
return _resolve_dataset_root(request, "gebench_root", "stepfun-ai/GEBench")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def gedit_dataset_root(request: pytest.FixtureRequest) -> Path:
|
||||
return _resolve_dataset_root(request, "gedit_root", "stepfun-ai/GEdit-Bench")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def accuracy_workers(request: pytest.FixtureRequest) -> int:
|
||||
return int(request.config.getoption("accuracy_workers"))
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def wan22_i2v_image_source(request: pytest.FixtureRequest) -> str | None:
|
||||
value = request.config.getoption("wan22_i2v_image_source")
|
||||
@@ -192,16 +60,6 @@ def hunyuanvideo15_online_timeout_seconds(request: pytest.FixtureRequest) -> int
|
||||
return int(request.config.getoption("hunyuanvideo15_online_timeout_seconds"))
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def gebench_samples_per_type(request: pytest.FixtureRequest) -> int:
|
||||
return int(request.config.getoption("gebench_samples_per_type"))
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def gedit_samples_per_group(request: pytest.FixtureRequest) -> int:
|
||||
return int(request.config.getoption("gedit_samples_per_group"))
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def accuracy_artifact_root() -> Path:
|
||||
root = Path(__file__).resolve().parent / "artifacts"
|
||||
@@ -209,12 +67,6 @@ def accuracy_artifact_root() -> Path:
|
||||
return root
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def accuracy_assets_root() -> Path:
|
||||
root = Path(__file__).resolve().parent / "assets"
|
||||
return root
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qwen_bear_image(accuracy_artifact_root: Path):
|
||||
"""Download the Qwen bear image from the URL and save it to the accuracy artifact root."""
|
||||
@@ -249,80 +101,3 @@ def rabbit_image(accuracy_artifact_root: Path):
|
||||
image.save(image_path)
|
||||
yield image
|
||||
image.close()
|
||||
|
||||
|
||||
def _build_accuracy_server_config(
|
||||
*,
|
||||
generate_model: str,
|
||||
judge_model: str,
|
||||
shared_gpu: str,
|
||||
port: int,
|
||||
run_level: str,
|
||||
model_prefix: str,
|
||||
) -> AccuracyServerConfig:
|
||||
if torch.accelerator.device_count() < 1:
|
||||
pytest.skip("Need at least 1 CUDA GPU for accuracy benchmark smoke tests.")
|
||||
|
||||
if not generate_model:
|
||||
pytest.skip("No generate model configured for accuracy benchmark test.")
|
||||
generate_server_args = ["--num-gpus", "1"]
|
||||
judge_server_args = [
|
||||
"--max-model-len",
|
||||
"32768",
|
||||
"--gpu-memory-utilization",
|
||||
"0.8",
|
||||
]
|
||||
|
||||
judge_env = {"CUDA_VISIBLE_DEVICES": shared_gpu}
|
||||
|
||||
return AccuracyServerConfig(
|
||||
generate_params=OmniServerParams(
|
||||
model=generate_model,
|
||||
port=port,
|
||||
server_args=generate_server_args,
|
||||
env_dict={"CUDA_VISIBLE_DEVICES": shared_gpu},
|
||||
use_omni=True,
|
||||
stage_init_timeout=300,
|
||||
),
|
||||
judge_params=OmniServerParams(
|
||||
model=judge_model,
|
||||
port=port,
|
||||
server_args=judge_server_args,
|
||||
env_dict=judge_env,
|
||||
use_omni=False,
|
||||
),
|
||||
run_level=run_level,
|
||||
model_prefix=model_prefix,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gebench_accuracy_servers(
|
||||
request: pytest.FixtureRequest,
|
||||
run_level: str,
|
||||
model_prefix: str,
|
||||
) -> AccuracyServerConfig:
|
||||
return _build_accuracy_server_config(
|
||||
generate_model=request.config.getoption("gebench_model"),
|
||||
judge_model=request.config.getoption("accuracy_judge_model"),
|
||||
shared_gpu=str(request.config.getoption("accuracy_gpu")),
|
||||
port=int(request.config.getoption("gebench_port")),
|
||||
run_level=run_level,
|
||||
model_prefix=model_prefix,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gedit_accuracy_servers(
|
||||
request: pytest.FixtureRequest,
|
||||
run_level: str,
|
||||
model_prefix: str,
|
||||
) -> AccuracyServerConfig:
|
||||
return _build_accuracy_server_config(
|
||||
generate_model=request.config.getoption("gedit_model"),
|
||||
judge_model=request.config.getoption("accuracy_judge_model"),
|
||||
shared_gpu=str(request.config.getoption("accuracy_gpu")),
|
||||
port=int(request.config.getoption("gedit_port")),
|
||||
run_level=run_level,
|
||||
model_prefix=model_prefix,
|
||||
)
|
||||
@@ -3,8 +3,12 @@ import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from base64 import b64decode, b64encode
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import contextmanager
|
||||
from fractions import Fraction
|
||||
from hashlib import sha1
|
||||
from io import BytesIO
|
||||
@@ -53,8 +57,6 @@ def env_to_apply_ftfy_mock_in_subproc(env: dict[str, str] | None = None) -> dict
|
||||
|
||||
|
||||
def reset_artifact_dir(path: Path) -> Path:
|
||||
import shutil
|
||||
|
||||
if path.exists():
|
||||
shutil.rmtree(path)
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
@@ -609,3 +611,344 @@ def download_images(urls: list[str]) -> list[Image.Image]:
|
||||
images.append(image)
|
||||
|
||||
return images
|
||||
|
||||
|
||||
_DREAMZERO_UPSTREAM_SERVER_SCRIPT = (
|
||||
Path(__file__).resolve().parent / "upstream_mock" / "upstream_socket_server_no_compile.py"
|
||||
)
|
||||
_DREAMZERO_DEFAULT_MODEL = "GEAR-Dreams/DreamZero-DROID"
|
||||
_DREAMZERO_DEFAULT_READY_TIMEOUT_S = int(os.environ.get("OPENPI_SERVICE_READY_TIMEOUT_S", "900"))
|
||||
_DREAMZERO_DEFAULT_PROMPT = (
|
||||
"Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan"
|
||||
)
|
||||
_DREAMZERO_DEFAULT_SESSION_ID = "openpi-e2e-parity-session"
|
||||
_DREAMZERO_OPENPI_VLLM_PATH = "/v1/realtime/robot/openpi"
|
||||
|
||||
|
||||
def require_dreamzero_parity_env() -> tuple[Path, Path]:
|
||||
repo_env = os.environ.get("DREAMZERO_REPO")
|
||||
if repo_env is None:
|
||||
raise ValueError("DREAMZERO_REPO environment variable is required.")
|
||||
repo = Path(repo_env).expanduser()
|
||||
if not repo.is_dir():
|
||||
raise FileNotFoundError(f"DREAMZERO_REPO does not exist: {repo}")
|
||||
checkpoint_dir = repo / "checkpoints" / "dreamzero"
|
||||
if not checkpoint_dir.is_dir():
|
||||
raise FileNotFoundError(f"DreamZero checkpoint not found: {checkpoint_dir}")
|
||||
if str(repo) not in sys.path:
|
||||
sys.path.insert(0, str(repo))
|
||||
return repo, checkpoint_dir
|
||||
|
||||
|
||||
def load_dreamzero_upstream_modules():
|
||||
require_dreamzero_parity_env()
|
||||
import test_client_AR as dreamzero_upstream_client
|
||||
from eval_utils.policy_client import WebsocketClientPolicy
|
||||
|
||||
return dreamzero_upstream_client, WebsocketClientPolicy
|
||||
|
||||
|
||||
def dreamzero_cfg_parallel_size() -> int:
|
||||
return int(os.environ.get("OPENPI_E2E_CFG_PARALLEL_SIZE", "2"))
|
||||
|
||||
|
||||
def _dreamzero_parity_vllm_env() -> dict[str, str]:
|
||||
from tests.helpers.env import pick_least_used_device_indices
|
||||
from tests.helpers.runtime import get_open_port
|
||||
|
||||
parallel_size = dreamzero_cfg_parallel_size()
|
||||
gpus = [str(index) for index in pick_least_used_device_indices(parallel_size)]
|
||||
if parallel_size > len(gpus):
|
||||
raise RuntimeError(
|
||||
f"cfg_parallel_size={parallel_size} requires at least {parallel_size} GPUs, but only got {gpus}."
|
||||
)
|
||||
return {
|
||||
"CUDA_VISIBLE_DEVICES": ",".join(gpus[:parallel_size]),
|
||||
"ATTENTION_BACKEND": "torch",
|
||||
"DIFFUSION_ATTENTION_BACKEND": "TORCH_SDPA",
|
||||
"VLLM_DISABLE_COMPILE_CACHE": "1",
|
||||
"MASTER_PORT": str(get_open_port()),
|
||||
}
|
||||
|
||||
|
||||
def dreamzero_parity_vllm_params(model: str | None = None):
|
||||
from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
parallel_size = dreamzero_cfg_parallel_size()
|
||||
server_args = [
|
||||
"--served-model-name",
|
||||
"dreamzero-droid",
|
||||
"--enforce-eager",
|
||||
]
|
||||
if parallel_size > 1:
|
||||
server_args.extend(["--cfg-parallel-size", str(parallel_size)])
|
||||
return OmniServerParams(
|
||||
model=model or os.environ.get("VLLM_DREAMZERO_MODEL", _DREAMZERO_DEFAULT_MODEL),
|
||||
stage_config_path=get_deploy_config_path("dreamzero_tp1_cfg2.yaml"),
|
||||
server_args=server_args,
|
||||
env_dict=_dreamzero_parity_vllm_env(),
|
||||
)
|
||||
|
||||
|
||||
def _dreamzero_omni_server_from_params(params):
|
||||
from tests.helpers.runtime import OmniServer
|
||||
|
||||
server_args = list(params.server_args or [])
|
||||
if params.stage_config_path is not None:
|
||||
server_args += ["--stage-configs-path", params.stage_config_path]
|
||||
return OmniServer(
|
||||
params.model,
|
||||
server_args,
|
||||
port=params.port,
|
||||
env_dict=params.env_dict,
|
||||
use_omni=params.use_omni,
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def dreamzero_parity_vllm_client(
|
||||
params=None,
|
||||
*,
|
||||
ready_timeout_s: float = _DREAMZERO_DEFAULT_READY_TIMEOUT_S,
|
||||
) -> Iterator["DreamZeroParityClient"]:
|
||||
"""Start vLLM via ``OmniServer`` and return a ready OpenPI parity client."""
|
||||
with _dreamzero_omni_server_from_params(params or dreamzero_parity_vllm_params()) as server:
|
||||
client = _dreamzero_wait_for_client_ready(
|
||||
lambda: _dreamzero_create_openpi_vllm_client(server.host, server.port),
|
||||
timeout_s=ready_timeout_s,
|
||||
proc=server.proc,
|
||||
)
|
||||
yield DreamZeroParityClient(client)
|
||||
|
||||
|
||||
def _dreamzero_create_openpi_vllm_client(host: str, port: int) -> Any:
|
||||
from openpi_client import msgpack_numpy
|
||||
|
||||
_, WebsocketClientPolicy = load_dreamzero_upstream_modules()
|
||||
|
||||
class _OpenPIWebsocketClientPolicy(WebsocketClientPolicy):
|
||||
def __init__(
|
||||
self,
|
||||
host: str = "127.0.0.1",
|
||||
port: int = 8000,
|
||||
path: str = _DREAMZERO_OPENPI_VLLM_PATH,
|
||||
) -> None:
|
||||
self._uri = f"ws://{host}:{port}{path}"
|
||||
self._packer = msgpack_numpy.Packer()
|
||||
self._ws, self._server_metadata = self._wait_for_server()
|
||||
|
||||
return _OpenPIWebsocketClientPolicy(host=host, port=port)
|
||||
|
||||
|
||||
def _dreamzero_stop_process(proc: subprocess.Popen[str]) -> None:
|
||||
log_file = getattr(proc, "_codex_log_file", None)
|
||||
if proc.poll() is None:
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=30)
|
||||
except subprocess.TimeoutExpired: # pragma: no cover - cleanup path
|
||||
proc.kill()
|
||||
proc.wait(timeout=10)
|
||||
if log_file is not None:
|
||||
log_file.close()
|
||||
|
||||
|
||||
def _dreamzero_wait_for_client_ready(
|
||||
client_factory: Callable[[], Any],
|
||||
*,
|
||||
timeout_s: float,
|
||||
proc: subprocess.Popen[str] | None = None,
|
||||
log_path: Path | None = None,
|
||||
) -> Any:
|
||||
deadline = time.time() + timeout_s
|
||||
last_err: Exception | None = None
|
||||
while time.time() < deadline:
|
||||
if proc is not None and proc.poll() is not None:
|
||||
details = ""
|
||||
if log_path is not None and log_path.exists():
|
||||
details = log_path.read_text(errors="replace")[-8000:]
|
||||
raise RuntimeError(f"Service exited before becoming ready with code {proc.returncode}.\n{details}")
|
||||
try:
|
||||
return client_factory()
|
||||
except Exception as exc: # pragma: no cover - retry path
|
||||
last_err = exc
|
||||
time.sleep(1)
|
||||
raise TimeoutError(f"Timed out waiting for websocket service: {last_err}")
|
||||
|
||||
|
||||
def _dreamzero_normalize_reset_response(response: Any) -> str:
|
||||
from openpi_client import msgpack_numpy
|
||||
|
||||
if isinstance(response, str):
|
||||
return response
|
||||
decoded = msgpack_numpy.unpackb(response)
|
||||
if isinstance(decoded, dict):
|
||||
return str(decoded.get("status"))
|
||||
return str(decoded)
|
||||
|
||||
|
||||
def normalize_dreamzero_metadata(metadata: dict[str, Any]) -> dict[str, Any]:
|
||||
normalized = dict(metadata)
|
||||
if isinstance(normalized.get("image_resolution"), tuple):
|
||||
normalized["image_resolution"] = list(normalized["image_resolution"])
|
||||
return normalized
|
||||
|
||||
|
||||
def assert_dreamzero_logs_clean(log_path: Path) -> None:
|
||||
text = log_path.read_text(errors="replace")
|
||||
if "SignalException: Process" in text and "got signal: 15" in text:
|
||||
text = text.split("Traceback (most recent call last):", 1)[0]
|
||||
assert "Traceback" not in text, text
|
||||
assert "RuntimeError:" not in text, text
|
||||
|
||||
|
||||
def assert_dreamzero_upstream_log_matches_vllm_baseline(log_path: Path) -> None:
|
||||
text = log_path.read_text(errors="replace")
|
||||
assert "DIT Compute Steps 8 steps" not in text, text
|
||||
assert "DIT Compute Steps 16 steps" in text, text
|
||||
|
||||
|
||||
class DreamZeroParityClient:
|
||||
"""Connected OpenPI websocket client for DreamZero parity inference."""
|
||||
|
||||
def __init__(self, client: Any) -> None:
|
||||
self._client = client
|
||||
|
||||
@property
|
||||
def raw_client(self) -> Any:
|
||||
return self._client
|
||||
|
||||
def build_obs_sequence(self) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
dreamzero_upstream_client, _ = load_dreamzero_upstream_modules()
|
||||
camera_frames = dreamzero_upstream_client.load_camera_frames()
|
||||
chunks = dreamzero_upstream_client.build_frame_schedule(
|
||||
min(v.shape[0] for v in camera_frames.values()),
|
||||
1,
|
||||
)
|
||||
obs0 = dreamzero_upstream_client._make_obs_from_video(
|
||||
camera_frames,
|
||||
[0],
|
||||
_DREAMZERO_DEFAULT_PROMPT,
|
||||
_DREAMZERO_DEFAULT_SESSION_ID,
|
||||
)
|
||||
obs1 = dreamzero_upstream_client._make_obs_from_video(
|
||||
camera_frames,
|
||||
chunks[0],
|
||||
_DREAMZERO_DEFAULT_PROMPT,
|
||||
_DREAMZERO_DEFAULT_SESSION_ID,
|
||||
)
|
||||
return obs0, obs1
|
||||
|
||||
def collect_parity_outputs(self) -> tuple[dict[str, Any], list[np.ndarray]]:
|
||||
metadata = self._client.get_server_metadata()
|
||||
obs0, obs1 = self.build_obs_sequence()
|
||||
outputs = [
|
||||
self._client.infer(dict(obs0)),
|
||||
self._client.infer(dict(obs1)),
|
||||
]
|
||||
assert _dreamzero_normalize_reset_response(self._client.reset({})) == "reset successful"
|
||||
outputs.append(self._client.infer(dict(obs0)))
|
||||
self._client._ws.close()
|
||||
return metadata, outputs
|
||||
|
||||
|
||||
class _DreamZeroParityServer(ABC):
|
||||
host = "127.0.0.1"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
log_path: Path,
|
||||
port: int | None = None,
|
||||
ready_timeout_s: float = _DREAMZERO_DEFAULT_READY_TIMEOUT_S,
|
||||
) -> None:
|
||||
from tests.helpers.runtime import get_open_port
|
||||
|
||||
self.log_path = log_path
|
||||
self.port = get_open_port() if port is None else port
|
||||
self.ready_timeout_s = ready_timeout_s
|
||||
self.proc: subprocess.Popen[str] | None = None
|
||||
self._log_file = None
|
||||
|
||||
@abstractmethod
|
||||
def _build_argv(self) -> list[str]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def _build_env(self) -> dict[str, str]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def _create_client(self) -> Any:
|
||||
raise NotImplementedError
|
||||
|
||||
def _start(self) -> None:
|
||||
self.log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._log_file = self.log_path.open("w")
|
||||
self.proc = subprocess.Popen(
|
||||
self._build_argv(),
|
||||
stdout=self._log_file,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
cwd=str(Path.cwd()),
|
||||
env=self._build_env(),
|
||||
)
|
||||
self.proc._codex_log_file = self._log_file # type: ignore[attr-defined]
|
||||
|
||||
def __enter__(self) -> DreamZeroParityClient:
|
||||
self._start()
|
||||
client = _dreamzero_wait_for_client_ready(
|
||||
self._create_client,
|
||||
timeout_s=self.ready_timeout_s,
|
||||
proc=self.proc,
|
||||
log_path=self.log_path,
|
||||
)
|
||||
return DreamZeroParityClient(client)
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb) -> None:
|
||||
if self.proc is not None:
|
||||
_dreamzero_stop_process(self.proc)
|
||||
self.proc = None
|
||||
|
||||
|
||||
class DreamZeroUpstreamServer(_DreamZeroParityServer):
|
||||
"""Official DreamZero upstream reference server launched via torchrun."""
|
||||
|
||||
def _build_argv(self) -> list[str]:
|
||||
_, checkpoint_dir = require_dreamzero_parity_env()
|
||||
return [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"torch.distributed.run",
|
||||
"--standalone",
|
||||
"--nproc_per_node",
|
||||
str(dreamzero_cfg_parallel_size()),
|
||||
str(_DREAMZERO_UPSTREAM_SERVER_SCRIPT),
|
||||
"--port",
|
||||
str(self.port),
|
||||
"--model_path",
|
||||
str(checkpoint_dir),
|
||||
]
|
||||
|
||||
def _build_env(self) -> dict[str, str]:
|
||||
from tests.helpers.env import pick_least_used_device_indices
|
||||
|
||||
dreamzero_repo, _ = require_dreamzero_parity_env()
|
||||
env = os.environ.copy()
|
||||
env.setdefault("PYTHONPATH", "")
|
||||
env["PYTHONPATH"] = f"{Path.cwd()}:{dreamzero_repo}:{env['PYTHONPATH']}".rstrip(":")
|
||||
env["CUDA_VISIBLE_DEVICES"] = ",".join(
|
||||
str(index) for index in pick_least_used_device_indices(max(dreamzero_cfg_parallel_size(), 1))
|
||||
)
|
||||
env.setdefault("NO_ALBUMENTATIONS_UPDATE", "1")
|
||||
env["ATTENTION_BACKEND"] = "torch"
|
||||
env.setdefault("ENABLE_TENSORRT", "false")
|
||||
env["ENABLE_DIT_CACHE"] = "false"
|
||||
env["NUM_DIT_STEPS"] = "16"
|
||||
env["DYNAMIC_CACHE_SCHEDULE"] = "false"
|
||||
return env
|
||||
|
||||
def _create_client(self) -> Any:
|
||||
_, WebsocketClientPolicy = load_dreamzero_upstream_modules()
|
||||
return WebsocketClientPolicy(host=self.host, port=self.port)
|
||||
@@ -0,0 +1,63 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""DreamZero OpenPI upstream parity accuracy test."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from tests.e2e.accuracy.helpers import (
|
||||
DreamZeroUpstreamServer,
|
||||
assert_dreamzero_logs_clean,
|
||||
assert_dreamzero_upstream_log_matches_vllm_baseline,
|
||||
dreamzero_parity_vllm_client,
|
||||
load_dreamzero_upstream_modules,
|
||||
normalize_dreamzero_metadata,
|
||||
)
|
||||
from tests.helpers.mark import hardware_test
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
|
||||
|
||||
EXPECTED_METADATA = {
|
||||
"image_resolution": [180, 320],
|
||||
"n_external_cameras": 2,
|
||||
"needs_wrist_camera": True,
|
||||
"needs_stereo_camera": False,
|
||||
"needs_session_id": True,
|
||||
"action_space": "joint_position",
|
||||
}
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=2)
|
||||
def test_dreamzero_openpi_upstream_parity(tmp_path: Path) -> None:
|
||||
"""Assert vLLM OpenPI actions match the upstream DreamZero reference server."""
|
||||
load_dreamzero_upstream_modules()
|
||||
|
||||
upstream_log = tmp_path / "dreamzero_upstream.log"
|
||||
with DreamZeroUpstreamServer(log_path=upstream_log) as upstream_client:
|
||||
upstream_metadata, upstream_outputs = upstream_client.collect_parity_outputs()
|
||||
assert_dreamzero_logs_clean(upstream_log)
|
||||
assert_dreamzero_upstream_log_matches_vllm_baseline(upstream_log)
|
||||
|
||||
with dreamzero_parity_vllm_client() as vllm_client:
|
||||
vllm_metadata, vllm_outputs = vllm_client.collect_parity_outputs()
|
||||
|
||||
assert normalize_dreamzero_metadata(upstream_metadata) == EXPECTED_METADATA
|
||||
assert normalize_dreamzero_metadata(vllm_metadata) == EXPECTED_METADATA
|
||||
|
||||
for idx, (vllm_action, upstream_action) in enumerate(zip(vllm_outputs, upstream_outputs, strict=True)):
|
||||
np.testing.assert_allclose(
|
||||
vllm_action,
|
||||
upstream_action,
|
||||
rtol=1e-2,
|
||||
atol=1e-3,
|
||||
err_msg=f"OpenPI step {idx} output mismatch",
|
||||
)
|
||||
@@ -1,97 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from benchmarks.accuracy.text_to_image.gbench import main as gbench_main
|
||||
from tests.e2e.accuracy.helpers import infer_model_label, reset_artifact_dir
|
||||
from tests.helpers.mark import hardware_test
|
||||
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.skip(reason="#3256")
|
||||
def test_gebench_h100_smoke(
|
||||
gebench_accuracy_servers,
|
||||
accuracy_artifact_root: Path,
|
||||
gebench_dataset_root: Path,
|
||||
accuracy_workers: int,
|
||||
) -> None:
|
||||
model_label = infer_model_label(gebench_accuracy_servers.generate_params.model).lower()
|
||||
output_root = reset_artifact_dir(accuracy_artifact_root / f"gebench_{model_label}")
|
||||
|
||||
with gebench_accuracy_servers.generate_server() as generate_server:
|
||||
for data_type in ("type3", "type4"):
|
||||
assert (
|
||||
gbench_main(
|
||||
[
|
||||
"generate",
|
||||
"--dataset-root",
|
||||
str(gebench_dataset_root),
|
||||
"--output-root",
|
||||
str(output_root),
|
||||
"--base-url",
|
||||
f"http://{generate_server.host}:{generate_server.port}",
|
||||
"--model",
|
||||
generate_server.model,
|
||||
"--data-type",
|
||||
data_type,
|
||||
"--width",
|
||||
"768",
|
||||
"--height",
|
||||
"576",
|
||||
"--output-compression",
|
||||
"98",
|
||||
"--num-inference-steps",
|
||||
"8",
|
||||
"--workers",
|
||||
str(accuracy_workers),
|
||||
]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
|
||||
with gebench_accuracy_servers.judge_server() as judge_server:
|
||||
for data_type in ("type3", "type4"):
|
||||
assert (
|
||||
gbench_main(
|
||||
[
|
||||
"evaluate",
|
||||
"--dataset-root",
|
||||
str(gebench_dataset_root),
|
||||
"--output-root",
|
||||
str(output_root),
|
||||
"--data-type",
|
||||
data_type,
|
||||
"--judge-base-url",
|
||||
f"http://{judge_server.host}:{judge_server.port}",
|
||||
"--judge-model",
|
||||
judge_server.model,
|
||||
"--judge-api-key",
|
||||
"EMPTY",
|
||||
"--workers",
|
||||
str(accuracy_workers),
|
||||
]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
|
||||
assert gbench_main(["summarize", "--output-root", str(output_root)]) == 0
|
||||
|
||||
summary = json.loads((output_root / "summary.json").read_text(encoding="utf-8"))
|
||||
assert "generation" in summary
|
||||
assert "evaluation" in summary
|
||||
|
||||
for data_type in ("type3", "type4"):
|
||||
assert data_type in summary["generation"]["by_type"]
|
||||
assert summary["generation"]["by_type"][data_type]["count"] > 0
|
||||
assert data_type in summary["evaluation"]["by_type"]
|
||||
assert summary["evaluation"]["by_type"][data_type]["count"] > 0
|
||||
|
||||
assert summary["evaluation"]["overall_mean"] >= 0.45
|
||||
assert summary["evaluation"]["by_type"]["type3"]["overall_mean"] >= 0.45
|
||||
assert summary["evaluation"]["by_type"]["type4"]["overall_mean"] >= 0.45
|
||||
@@ -1,115 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from benchmarks.accuracy.image_to_image.gedit_bench import GROUPS
|
||||
from benchmarks.accuracy.image_to_image.gedit_bench import main as gedit_main
|
||||
from tests.e2e.accuracy.helpers import infer_model_label, reset_artifact_dir
|
||||
from tests.helpers.mark import hardware_test
|
||||
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.skip(reason="#3257")
|
||||
def test_gedit_bench_h100_smoke(
|
||||
gedit_accuracy_servers,
|
||||
accuracy_artifact_root: Path,
|
||||
gedit_dataset_root: Path,
|
||||
gedit_samples_per_group: int,
|
||||
accuracy_workers: int,
|
||||
) -> None:
|
||||
model_label = infer_model_label(gedit_accuracy_servers.generate_params.model).lower()
|
||||
output_root = reset_artifact_dir(accuracy_artifact_root / f"gedit_results_{model_label}")
|
||||
score_root = reset_artifact_dir(accuracy_artifact_root / f"gedit_scores_{model_label}")
|
||||
model_name = model_label
|
||||
|
||||
with gedit_accuracy_servers.generate_server() as generate_server:
|
||||
assert (
|
||||
gedit_main(
|
||||
[
|
||||
"generate",
|
||||
"--dataset-ref",
|
||||
str(gedit_dataset_root),
|
||||
"--output-root",
|
||||
str(output_root),
|
||||
"--base-url",
|
||||
f"http://{generate_server.host}:{generate_server.port}",
|
||||
"--model",
|
||||
generate_server.model,
|
||||
"--model-name",
|
||||
model_name,
|
||||
"--task-type",
|
||||
"all",
|
||||
"--instruction-language",
|
||||
"all",
|
||||
"--samples-per-group",
|
||||
str(gedit_samples_per_group),
|
||||
"--workers",
|
||||
str(accuracy_workers),
|
||||
]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
|
||||
with gedit_accuracy_servers.judge_server() as judge_server:
|
||||
assert (
|
||||
gedit_main(
|
||||
[
|
||||
"evaluate",
|
||||
"--dataset-ref",
|
||||
str(gedit_dataset_root),
|
||||
"--output-root",
|
||||
str(output_root),
|
||||
"--model-name",
|
||||
model_name,
|
||||
"--save-dir",
|
||||
str(score_root),
|
||||
"--task-type",
|
||||
"all",
|
||||
"--instruction-language",
|
||||
"all",
|
||||
"--judge-base-url",
|
||||
f"http://{judge_server.host}:{judge_server.port}",
|
||||
"--judge-model",
|
||||
judge_server.model,
|
||||
"--judge-api-key",
|
||||
"EMPTY",
|
||||
"--samples-per-group",
|
||||
str(gedit_samples_per_group),
|
||||
"--workers",
|
||||
str(accuracy_workers),
|
||||
]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
|
||||
csv_path = score_root / f"{model_name}_all_all_vie_score.csv"
|
||||
assert gedit_main(["summarize", "--csv-path", str(csv_path), "--language", "all"]) == 0
|
||||
|
||||
summary_path = score_root / f"{model_name}_all_all_summary.json"
|
||||
summary = json.loads(summary_path.read_text(encoding="utf-8"))
|
||||
assert set(summary["languages"]) == {"en", "cn"}
|
||||
|
||||
for language in ["en", "cn"]:
|
||||
language_summary = summary["languages"][language]
|
||||
assert language_summary["overall"]["count"] is not None
|
||||
assert language_summary["intersection"]["count"] is not None
|
||||
assert language_summary["overall"]["Q_SC"] is not None
|
||||
assert language_summary["overall"]["Q_PQ"] is not None
|
||||
assert language_summary["overall"]["Q_O"] is not None
|
||||
|
||||
for group in GROUPS:
|
||||
group_summary = language_summary["by_group"][group]
|
||||
assert set(group_summary) == {"count", "Q_SC", "Q_PQ", "Q_O"}
|
||||
|
||||
assert summary["languages"]["en"]["overall"]["Q_SC"] >= 6.95
|
||||
assert summary["languages"]["en"]["overall"]["Q_PQ"] >= 5.8
|
||||
assert summary["languages"]["en"]["overall"]["Q_O"] >= 6.15
|
||||
assert summary["languages"]["cn"]["overall"]["Q_SC"] >= 6.9
|
||||
assert summary["languages"]["cn"]["overall"]["Q_PQ"] >= 5.7
|
||||
assert summary["languages"]["cn"]["overall"]["Q_O"] >= 6.1
|
||||
@@ -28,6 +28,7 @@ from tests.e2e.accuracy.helpers import (
|
||||
model_output_dir,
|
||||
)
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.media import get_asset_path
|
||||
from tests.helpers.runtime import OmniRunner, OmniServer
|
||||
from vllm_omni.diffusion.models.hunyuan_image3.prompt_utils import (
|
||||
build_prompt_tokens,
|
||||
@@ -94,6 +95,7 @@ def _default_ar_dit_devices() -> tuple[str, str]:
|
||||
|
||||
AR_DEVICES, DIT_DEVICES = _default_ar_dit_devices()
|
||||
MODEL_NAME = "tencent/HunyuanImage-3.0-Instruct"
|
||||
HUNYUAN_IMAGE_REF_PATH = get_asset_path("hunyuan/hunyuan_image_ref.png")
|
||||
NUM_INFERENCE_STEPS = 50
|
||||
GUIDANCE_SCALE = 2.5
|
||||
|
||||
@@ -414,7 +416,7 @@ def _run_online(stage_configs_path: str, output_path: Path) -> tuple[Image.Image
|
||||
torch.accelerator.device_count() < AR_TP_SIZE + DIT_TP_SIZE,
|
||||
reason=f"Needs {AR_TP_SIZE + DIT_TP_SIZE}+ GPUs ({AR_TP_SIZE} AR + {DIT_TP_SIZE} DiT)",
|
||||
)
|
||||
def test_image_to_image_alignment_online(accuracy_artifact_root: Path, accuracy_assets_root: Path) -> None:
|
||||
def test_image_to_image_alignment_online(accuracy_artifact_root: Path) -> None:
|
||||
"""Online API test: same pipeline, same seed as offline → PSNR >= 10 dB."""
|
||||
if importlib.util.find_spec("FlagEmbedding") is None:
|
||||
raise ImportError("Missing dependency: FlagEmbedding\nInstall with: pip install FlagEmbedding")
|
||||
@@ -431,7 +433,7 @@ def test_image_to_image_alignment_online(accuracy_artifact_root: Path, accuracy_
|
||||
scorer = SemanticSimilarityScorer()
|
||||
clip_scorer = CLIPScorer()
|
||||
cot_results = scorer.text_similarity(online_cot, COT_REF)
|
||||
image_ref = Image.open(str(accuracy_assets_root / "hunyuan_image_ref.png")).convert("RGB")
|
||||
image_ref = Image.open(HUNYUAN_IMAGE_REF_PATH).convert("RGB")
|
||||
image_clip_score = clip_scorer.image_image_score(online_image, image_ref)
|
||||
ssim_value, psnr_value = compute_image_ssim_psnr(prediction=online_image, reference=image_ref, compare_mode="RGB")
|
||||
|
||||
@@ -529,7 +531,7 @@ def _run_dit_model(
|
||||
torch.accelerator.device_count() < AR_TP_SIZE + DIT_TP_SIZE,
|
||||
reason=f"Needs {AR_TP_SIZE + DIT_TP_SIZE}+ GPUs ({AR_TP_SIZE} AR + {DIT_TP_SIZE} DiT)",
|
||||
)
|
||||
def test_image_to_image_alignment(accuracy_artifact_root: Path, accuracy_assets_root: Path) -> None:
|
||||
def test_image_to_image_alignment(accuracy_artifact_root: Path) -> None:
|
||||
if importlib.util.find_spec("FlagEmbedding") is None:
|
||||
raise ImportError("Missing dependency: FlagEmbedding\nInstall with: pip install FlagEmbedding")
|
||||
from tabulate import tabulate # lazy import
|
||||
@@ -545,7 +547,7 @@ def test_image_to_image_alignment(accuracy_artifact_root: Path, accuracy_assets_
|
||||
scorer = SemanticSimilarityScorer()
|
||||
clip_scorer = CLIPScorer()
|
||||
cot_results = scorer.text_similarity(omni_cot, COT_REF)
|
||||
image_ref = Image.open(str(accuracy_assets_root / "hunyuan_image_ref.png")).convert("RGB")
|
||||
image_ref = Image.open(HUNYUAN_IMAGE_REF_PATH).convert("RGB")
|
||||
image_clip_score = clip_scorer.image_image_score(omni_image, image_ref)
|
||||
ssim_value, psnr_value = compute_image_ssim_psnr(prediction=omni_image, reference=image_ref, compare_mode="RGB")
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ from PIL import Image
|
||||
|
||||
from tests.e2e.accuracy.helpers import assert_images_pixel_close, assert_similarity, model_output_dir
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.media import get_asset_path
|
||||
from tests.helpers.runtime import OmniServer
|
||||
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]
|
||||
@@ -33,8 +34,8 @@ P99_THRESHOLD = 3e-1
|
||||
SSIM_THRESHOLD = 0.97
|
||||
PSNR_THRESHOLD = 30.0
|
||||
|
||||
BASELINE_PATH = get_asset_path("hunyuan/hunyuan_baseline.png")
|
||||
_REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent
|
||||
BASELINE_PATH = _REPO_ROOT / "tests" / "assets" / "hunyuan" / "hunyuan_baseline.png"
|
||||
_OFFLINE_SCRIPT = _REPO_ROOT / "examples" / "offline_inference" / "hunyuan_image3" / "end2end.py"
|
||||
|
||||
# DiT-only deploy config with trust_remote_code (based on hunyuan_image3_dit.yaml).
|
||||
|
||||
@@ -14,6 +14,7 @@ from benchmarks.accuracy.common import decode_base64_image, pil_to_png_bytes
|
||||
from tests.e2e.accuracy.helpers import assert_images_pixel_close, assert_similarity, model_output_dir
|
||||
from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.media import get_asset_path
|
||||
from tests.helpers.runtime import OmniServer
|
||||
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]
|
||||
@@ -46,10 +47,9 @@ EDIT_2511_PROMPT = (
|
||||
"(帽子紧贴头顶,未贴合部分可调整帽子摆放角度达到完全贴合), 头部与身体过渡必须自然,无拼接痕迹,"
|
||||
"无额外元素出现在头部四周,纯白背景,不要出现绿色圆圈"
|
||||
)
|
||||
EDIT_2511_ASSET_DIR = Path(__file__).resolve().parent / "assets"
|
||||
EDIT_2511_IMAGE_PATHS = [
|
||||
EDIT_2511_ASSET_DIR / "qwen_image_edit_2511_test1.png",
|
||||
EDIT_2511_ASSET_DIR / "qwen_image_edit_2511_test2.png",
|
||||
get_asset_path("qwen_image_edit/qwen_image_edit_2511_test1.png"),
|
||||
get_asset_path("qwen_image_edit/qwen_image_edit_2511_test2.png"),
|
||||
]
|
||||
|
||||
PROMPT_SINGLE_IMAGE = "The input is a 2D cartoon bear mascot. Restyle it into a painterly oil artwork with warm colors while preserving the main structure."
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Launch the upstream DreamZero websocket server with `torch.compile` disabled.
|
||||
|
||||
This wrapper is meant for formal parity tests against `vllm serve --omni`.
|
||||
It monkeypatches `torch.compile` before importing DreamZero modules so that
|
||||
all import-time decorators and `post_initialize()` compile calls become eager.
|
||||
|
||||
For the current DreamZero port baseline we also disable all upstream DiT cache
|
||||
and step-skipping behavior so the reference server matches vLLM's current
|
||||
eager/no-skip implementation:
|
||||
- `ENABLE_DIT_CACHE=false`
|
||||
- `NUM_DIT_STEPS=16`
|
||||
- `DYNAMIC_CACHE_SCHEDULE=false`
|
||||
|
||||
In our CI/dev environment we also do not install Transformer Engine or
|
||||
FlashAttention, so upstream's default `socket_test_optimized_AR.main()` path
|
||||
(`ATTENTION_BACKEND=TE` and then fallback to `FA2`) cannot execute. To keep the
|
||||
formal server-vs-server test runnable without pulling in those heavyweight
|
||||
optional deps, this wrapper reproduces upstream `main()` but pins attention to
|
||||
PyTorch SDPA (`ATTENTION_BACKEND=torch`) for this subprocess only.
|
||||
|
||||
Usage:
|
||||
PYTHONPATH="${DREAMZERO_REPO}" \\
|
||||
.venv/bin/python -m torch.distributed.run --standalone --nproc_per_node=2 \\
|
||||
tests/e2e/accuracy/upstream_mock/upstream_socket_server_no_compile.py --port 18081 \\
|
||||
--model_path "${DREAMZERO_REPO}/checkpoints/dreamzero"
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
|
||||
def _identity_compile(*args, **kwargs):
|
||||
if args and callable(args[0]) and len(args) == 1 and not kwargs:
|
||||
return args[0]
|
||||
|
||||
def deco(fn):
|
||||
return fn
|
||||
|
||||
return deco
|
||||
|
||||
|
||||
torch.compile = _identity_compile
|
||||
|
||||
os.environ.setdefault("NO_ALBUMENTATIONS_UPDATE", "1")
|
||||
|
||||
DREAMZERO_REPO_ENV = os.environ.get("DREAMZERO_REPO")
|
||||
if not DREAMZERO_REPO_ENV:
|
||||
raise RuntimeError("Set DREAMZERO_REPO to an upstream DreamZero checkout before launching this helper.")
|
||||
DREAMZERO_REPO = Path(DREAMZERO_REPO_ENV).expanduser()
|
||||
if str(DREAMZERO_REPO) not in sys.path:
|
||||
sys.path.insert(0, str(DREAMZERO_REPO))
|
||||
|
||||
import socket_test_optimized_AR as upstream # noqa: E402
|
||||
import tyro # noqa: E402
|
||||
from groot.vla.model.dreamzero.modules import attention as upstream_attention # noqa: E402
|
||||
from groot.vla.model.dreamzero.modules import wan2_1_submodule as upstream_submodule # noqa: E402
|
||||
from groot.vla.model.dreamzero.modules import wan_video_dit as upstream_wan_video_dit # noqa: E402
|
||||
|
||||
|
||||
def _torch_varlen_flash_attention(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
q_lens: torch.Tensor | None = None,
|
||||
k_lens: torch.Tensor | None = None,
|
||||
dropout_p: float = 0.0,
|
||||
causal: bool = False,
|
||||
dtype: torch.dtype = torch.bfloat16,
|
||||
**_: object,
|
||||
) -> torch.Tensor:
|
||||
if q_lens is not None or k_lens is not None:
|
||||
upstream_attention.warnings.warn(
|
||||
"Padding mask is disabled in the test-only SDPA fallback.",
|
||||
)
|
||||
out_dtype = q.dtype
|
||||
q = q.transpose(1, 2).to(dtype)
|
||||
k = k.transpose(1, 2).to(dtype)
|
||||
v = v.transpose(1, 2).to(dtype)
|
||||
out = F.scaled_dot_product_attention(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
attn_mask=None,
|
||||
is_causal=causal,
|
||||
dropout_p=dropout_p,
|
||||
)
|
||||
return out.transpose(1, 2).contiguous().to(out_dtype)
|
||||
|
||||
|
||||
upstream_attention.flash_attention = _torch_varlen_flash_attention
|
||||
upstream_submodule.flash_attention = _torch_varlen_flash_attention
|
||||
upstream_wan_video_dit.FLASH_ATTN_3_AVAILABLE = False
|
||||
upstream_wan_video_dit.FLASH_ATTN_2_AVAILABLE = False
|
||||
upstream_wan_video_dit.SAGE_ATTN_AVAILABLE = False
|
||||
|
||||
|
||||
def main(args: upstream.Args) -> None:
|
||||
os.environ["ENABLE_DIT_CACHE"] = "false"
|
||||
os.environ["ATTENTION_BACKEND"] = "torch"
|
||||
os.environ["NUM_DIT_STEPS"] = "16"
|
||||
os.environ["DYNAMIC_CACHE_SCHEDULE"] = "false"
|
||||
torch._dynamo.config.recompile_limit = 800
|
||||
|
||||
embodiment_tag = "oxe_droid"
|
||||
model_path = args.model_path
|
||||
|
||||
device_mesh = upstream.init_mesh()
|
||||
rank = upstream.dist.get_rank()
|
||||
|
||||
timeout_delta = upstream.datetime.timedelta(seconds=args.timeout_seconds)
|
||||
signal_group = upstream.dist.new_group(backend="gloo", timeout=timeout_delta)
|
||||
upstream.logger.info("Rank %s initialized signal_group (gloo)", rank)
|
||||
|
||||
policy = upstream.GrootSimPolicy(
|
||||
embodiment_tag=upstream.EmbodimentTag(embodiment_tag),
|
||||
model_path=model_path,
|
||||
device="cuda" if torch.cuda.is_available() else "cpu",
|
||||
device_mesh=device_mesh,
|
||||
)
|
||||
|
||||
hostname = upstream.socket.gethostname()
|
||||
local_ip = upstream.socket.gethostbyname(hostname)
|
||||
|
||||
if rank == 0:
|
||||
upstream.logging.info("Creating server (host: %s, ip: %s)", hostname, local_ip)
|
||||
parent_dir = os.path.dirname(model_path)
|
||||
date_suffix = upstream.datetime.datetime.now().strftime("%Y%m%d")
|
||||
checkpoint_name = os.path.basename(model_path)
|
||||
output_dir = os.path.join(
|
||||
parent_dir,
|
||||
f"real_world_eval_gen_{date_suffix}_{args.index}",
|
||||
checkpoint_name,
|
||||
)
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
upstream.logging.info("Videos will be saved to: %s", output_dir)
|
||||
else:
|
||||
output_dir = None
|
||||
upstream.logging.info("Rank %s starting as worker for distributed inference...", rank)
|
||||
|
||||
wrapper_policy = upstream.ARDroidRoboarenaPolicy(
|
||||
groot_policy=policy,
|
||||
signal_group=signal_group,
|
||||
output_dir=output_dir,
|
||||
)
|
||||
|
||||
server_config = upstream.PolicyServerConfig(
|
||||
image_resolution=(180, 320),
|
||||
needs_wrist_camera=True,
|
||||
n_external_cameras=2,
|
||||
needs_stereo_camera=False,
|
||||
needs_session_id=True,
|
||||
action_space="joint_position",
|
||||
)
|
||||
|
||||
if rank == 0:
|
||||
upstream.logging.info("Using roboarena policy server interface")
|
||||
upstream.logging.info("Server config: %s", server_config)
|
||||
roboarena_server = upstream.RoboarenaServer(
|
||||
policy=wrapper_policy,
|
||||
server_config=server_config,
|
||||
host="0.0.0.0",
|
||||
port=args.port,
|
||||
)
|
||||
roboarena_server.serve_forever()
|
||||
else:
|
||||
worker = upstream.DistributedWorker(
|
||||
policy=policy,
|
||||
signal_group=signal_group,
|
||||
)
|
||||
upstream.asyncio.run(worker.run())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
upstream.logging.basicConfig(level=upstream.logging.INFO, force=True)
|
||||
main(tyro.cli(upstream.Args))
|
||||
Reference in new issue
Block a user