[HunyuanImage][End2End Performance CI] Add hunyuan end2end test (#3849)
Signed-off-by: dengyunyang <584797741@qq.com>
This commit is contained in:
dengyunyang authored and GitHub committed
1 parent
ec94e83a25
commit
36eb8afba5
3 files changed
+340
-2
No files matched your search
@@ -47,6 +47,13 @@ Usage:
|
||||
--endpoint /v1/images/edits --dataset random --task ti2i --num-prompts 10 \
|
||||
--bot-task think
|
||||
|
||||
python benchmarks/diffusion/diffusion_benchmark_serving.py \
|
||||
--endpoint /v1/images/edits \
|
||||
--dataset custom \
|
||||
--dataset-path custom_requests.jsonl \
|
||||
--task ti2i \
|
||||
--bot-task think
|
||||
|
||||
i2i (chat-based models such as Qwen-Image-Edit):
|
||||
python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
|
||||
--endpoint /v1/chat/completions --dataset vbench --task i2i --num-prompts 10
|
||||
@@ -566,6 +573,135 @@ class TraceDataset(BaseDataset):
|
||||
return [self[i] for i in range(len(self))]
|
||||
|
||||
|
||||
class CustomDataset(BaseDataset):
|
||||
"""
|
||||
Custom dataset that loads requests from a JSONL file.
|
||||
|
||||
Each line in the JSONL file should be a JSON object with the following fields:
|
||||
- prompt (required): The text prompt for the request
|
||||
- width (optional): Image/video width
|
||||
- height (optional): Image height
|
||||
- num_inference_steps (optional): Number of diffusion steps
|
||||
- seed (optional): Random seed
|
||||
- image_paths (optional): List of input image paths for i2i/i2v/ti2i tasks
|
||||
- image_urls (optional): List of input image URLs (alternative to image_paths)
|
||||
|
||||
Example JSONL for ti2i:
|
||||
{"prompt": "Add sunset lighting", "width": 1024, "height": 1024, "image_urls": ["https://example.com/image.jpg"]}
|
||||
"""
|
||||
|
||||
def __init__(self, args, api_url: str, model: str):
|
||||
super().__init__(args, api_url, model)
|
||||
self.dataset_path = args.dataset_path
|
||||
if not self.dataset_path:
|
||||
raise ValueError("--dataset-path must be provided when using 'custom' dataset")
|
||||
self.load_data()
|
||||
|
||||
def load_data(self) -> None:
|
||||
"""Load data from JSONL file."""
|
||||
import pandas as pd
|
||||
|
||||
if not self.dataset_path.endswith(".jsonl"):
|
||||
raise ValueError("Custom dataset must be a JSONL file")
|
||||
|
||||
try:
|
||||
df = pd.read_json(path_or_buf=self.dataset_path, lines=True)
|
||||
except Exception as e:
|
||||
raise ValueError(f"Failed to load JSONL file {self.dataset_path}: {e}")
|
||||
|
||||
if "prompt" not in df.columns:
|
||||
raise ValueError("JSONL file must contain a 'prompt' column")
|
||||
|
||||
self.data = df.to_dict("records")
|
||||
print(f"Loaded {len(self.data)} requests from {self.dataset_path}")
|
||||
|
||||
def _resolve_image_paths(self, item: dict) -> list[str] | None:
|
||||
# Handle local file paths
|
||||
if "image_paths" in item and item["image_paths"]:
|
||||
paths = item["image_paths"]
|
||||
if isinstance(paths, str):
|
||||
paths = [paths]
|
||||
|
||||
# Verify all paths exist
|
||||
valid_paths = []
|
||||
for path in paths:
|
||||
if os.path.exists(path):
|
||||
valid_paths.append(path)
|
||||
else:
|
||||
raise ValueError(f"Image file not found: {path}")
|
||||
|
||||
return valid_paths if valid_paths else None
|
||||
|
||||
# Handle URLs - download to temp files
|
||||
if "image_urls" in item and item["image_urls"]:
|
||||
urls = item["image_urls"]
|
||||
if isinstance(urls, str):
|
||||
urls = [urls]
|
||||
|
||||
downloaded_paths = []
|
||||
for url in urls:
|
||||
try:
|
||||
response = requests.get(url, timeout=30)
|
||||
response.raise_for_status()
|
||||
|
||||
# Create temp file with appropriate extension
|
||||
suffix = os.path.splitext(url)[1] or ".png"
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
|
||||
tmp.write(response.content)
|
||||
downloaded_paths.append(tmp.name)
|
||||
except Exception as e:
|
||||
raise ValueError(f"Failed to download image from {url}: {e}")
|
||||
|
||||
return downloaded_paths if downloaded_paths else None
|
||||
|
||||
return None
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self.data)
|
||||
|
||||
def __getitem__(self, idx: int) -> RequestFuncInput:
|
||||
item = self.data[idx]
|
||||
|
||||
# Extract fields with fallback to args defaults
|
||||
prompt = item["prompt"]
|
||||
width = item.get("width", self.args.width)
|
||||
height = item.get("height", self.args.height)
|
||||
num_inference_steps = item.get("num_inference_steps", self.args.num_inference_steps)
|
||||
seed = item.get("seed", self.args.seed)
|
||||
reserved_keys = {"prompt", "width", "height", "num_inference_steps", "seed", "image_paths", "image_urls"}
|
||||
extra_body = {k: v for k, v in item.items() if k not in reserved_keys}
|
||||
|
||||
# Handle image paths/URLs
|
||||
image_paths = self._resolve_image_paths(item)
|
||||
|
||||
return RequestFuncInput(
|
||||
prompt=prompt,
|
||||
api_url=self.api_url,
|
||||
model=self.model,
|
||||
seed=seed,
|
||||
image_paths=image_paths,
|
||||
extra_body=extra_body,
|
||||
width=width,
|
||||
height=height,
|
||||
num_inference_steps=num_inference_steps,
|
||||
)
|
||||
|
||||
def get_requests(self) -> list[RequestFuncInput]:
|
||||
"""Get all requests, cycling through data if num_prompts > dataset size."""
|
||||
num_requests = getattr(self.args, "num_prompts", len(self.data))
|
||||
|
||||
if num_requests <= len(self.data):
|
||||
return [self[i] for i in range(num_requests)]
|
||||
|
||||
# Cycle through the dataset to reach num_prompts
|
||||
requests = []
|
||||
for i in range(num_requests):
|
||||
idx = i % len(self.data)
|
||||
requests.append(self[idx])
|
||||
|
||||
return requests
|
||||
|
||||
|
||||
class RandomDataset(BaseDataset):
|
||||
def __init__(self, args, api_url: str, model: str, enable_negative_prompt: bool = False):
|
||||
super().__init__(args, api_url, model)
|
||||
@@ -950,6 +1086,8 @@ async def benchmark(args):
|
||||
dataset = TraceDataset(args, api_url, args.model)
|
||||
elif args.dataset == "random":
|
||||
dataset = RandomDataset(args, api_url, args.model, args.enable_negative_prompt)
|
||||
elif args.dataset == "custom":
|
||||
dataset = CustomDataset(args, api_url, args.model)
|
||||
else:
|
||||
raise ValueError(f"Unknown dataset: {args.dataset}")
|
||||
|
||||
@@ -1097,7 +1235,7 @@ if __name__ == "__main__":
|
||||
"--dataset",
|
||||
type=str,
|
||||
default="vbench",
|
||||
choices=["vbench", "trace", "random"],
|
||||
choices=["vbench", "trace", "random", "custom"],
|
||||
help="Dataset to use.",
|
||||
)
|
||||
parser.add_argument(
|
||||
|
||||
@@ -41,6 +41,72 @@ pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ.setdefault("DIFFUSION_ATTENTION_BACKEND", "FLASH_ATTN")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Inline field processing
|
||||
# ---------------------------------------------------------------------------
|
||||
def _process_inline_fields(obj: Any, parent_key: str = "") -> None:
|
||||
"""Recursively process '*-inline' fields into temp files."""
|
||||
|
||||
if isinstance(obj, list):
|
||||
for item in obj:
|
||||
_process_inline_fields(item, parent_key)
|
||||
return
|
||||
|
||||
if not isinstance(obj, dict):
|
||||
return
|
||||
|
||||
import atexit
|
||||
|
||||
import yaml
|
||||
|
||||
for key in list(obj.keys()):
|
||||
value = obj[key]
|
||||
|
||||
if not key.endswith("-inline"):
|
||||
_process_inline_fields(value, key)
|
||||
continue
|
||||
|
||||
base_key = key[:-7]
|
||||
full_key = f"{parent_key}.{key}" if parent_key else key
|
||||
|
||||
try:
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError("must be a dict")
|
||||
|
||||
file_type = value.get("type")
|
||||
content = value.get("content")
|
||||
|
||||
if file_type not in {"yaml", "jsonl"}:
|
||||
raise ValueError(f"invalid type: {file_type}")
|
||||
|
||||
fd, path = tempfile.mkstemp(
|
||||
suffix=f".{file_type}",
|
||||
prefix=f"{base_key}_",
|
||||
)
|
||||
|
||||
atexit.register(Path(path).unlink, missing_ok=True)
|
||||
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
||||
if file_type == "jsonl":
|
||||
items = content if isinstance(content, list) else [content]
|
||||
f.writelines(json.dumps(x, ensure_ascii=False) + "\n" for x in items)
|
||||
else:
|
||||
yaml.dump(
|
||||
content,
|
||||
f,
|
||||
allow_unicode=True,
|
||||
sort_keys=False,
|
||||
indent=2,
|
||||
)
|
||||
|
||||
obj[base_key] = path
|
||||
del obj[key]
|
||||
|
||||
except Exception as e:
|
||||
print(f"Warning: failed processing '{full_key}': {e}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Paths
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -102,11 +168,14 @@ def _resolve_refs(configs: list[dict[str, Any]], config_dir: Path) -> list[dict[
|
||||
|
||||
|
||||
def load_configs(config_path: str) -> list[dict[str, Any]]:
|
||||
"""Load benchmark configs from JSON file and process inline fields."""
|
||||
try:
|
||||
abs_path = Path(config_path).resolve()
|
||||
with open(abs_path, encoding="utf-8") as f:
|
||||
configs = json.load(f)
|
||||
return _resolve_refs(configs, abs_path.parent)
|
||||
configs = _resolve_refs(configs, abs_path.parent)
|
||||
_process_inline_fields(configs)
|
||||
return configs
|
||||
except json.JSONDecodeError as e:
|
||||
raise ValueError(f"JSON parsing error: {str(e)}")
|
||||
except FileNotFoundError:
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_hunyuan_image3_ti2i_tp4_tp4_inline",
|
||||
"description": "AR (TP=4) + DiT (TP=4) - using inline dataset and deploy-config",
|
||||
"server_type": "vllm-omni",
|
||||
"benchmark_endpoint": "/v1/images/edits",
|
||||
"server_params": {
|
||||
"model": "tencent/HunyuanImage-3.0-Instruct",
|
||||
"serve_args": {
|
||||
"deploy-config-inline": {
|
||||
"type": "yaml",
|
||||
"content": {
|
||||
"pipeline": "hunyuan_image_3_moe",
|
||||
"async_chunk": false,
|
||||
"trust_remote_code": true,
|
||||
"connectors": {
|
||||
"rdma_connector": {
|
||||
"name": "MooncakeTransferEngineConnector",
|
||||
"extra": {
|
||||
"host": "auto",
|
||||
"zmq_port": 50051,
|
||||
"protocol": "rdma",
|
||||
"device_name": "",
|
||||
"memory_pool_size": 4294967296,
|
||||
"memory_pool_device": "cpu"
|
||||
}
|
||||
}
|
||||
},
|
||||
"stages": [
|
||||
{
|
||||
"stage_id": 0,
|
||||
"is_comprehension": true,
|
||||
"final_output": true,
|
||||
"final_output_type": "text",
|
||||
"max_num_seqs": 4,
|
||||
"gpu_memory_utilization": 0.95,
|
||||
"enforce_eager": false,
|
||||
"enable_prefix_caching": false,
|
||||
"max_num_batched_tokens": 32768,
|
||||
"devices": "0,1,2,3",
|
||||
"tensor_parallel_size": 4,
|
||||
"hf_overrides": {
|
||||
"rope_parameters": {
|
||||
"mrope_section": [0, 32, 32],
|
||||
"rope_type": "default"
|
||||
}
|
||||
},
|
||||
"omni_kv_config": {
|
||||
"need_send_cache": true
|
||||
},
|
||||
"output_connectors": {
|
||||
"to_stage_1": "rdma_connector"
|
||||
},
|
||||
"default_sampling_params": {
|
||||
"temperature": 0.0,
|
||||
"top_p": 1,
|
||||
"top_k": -1,
|
||||
"max_tokens": 8192,
|
||||
"detokenize": true,
|
||||
"skip_special_tokens": false,
|
||||
"stop_token_ids": [128025]
|
||||
}
|
||||
},
|
||||
{
|
||||
"stage_id": 1,
|
||||
"max_num_seqs": 1,
|
||||
"enforce_eager": true,
|
||||
"devices": "4,5,6,7",
|
||||
"distributed_executor_backend": "mp",
|
||||
"omni_kv_config": {
|
||||
"need_recv_cache": true
|
||||
},
|
||||
"parallel_config": {
|
||||
"tensor_parallel_size": 4,
|
||||
"enable_expert_parallel": true
|
||||
},
|
||||
"input_connectors": {
|
||||
"from_stage_0": "rdma_connector"
|
||||
},
|
||||
"default_sampling_params": {
|
||||
"num_inference_steps": 50,
|
||||
"guidance_scale": 0
|
||||
}
|
||||
}
|
||||
],
|
||||
"edges": [
|
||||
{
|
||||
"from": 0,
|
||||
"to": 1,
|
||||
"window_size": -1,
|
||||
"max_inflight": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "inline_dataset_5_requests",
|
||||
"dataset": "custom",
|
||||
"dataset-path-inline": {
|
||||
"type": "jsonl",
|
||||
"content": [
|
||||
{
|
||||
"prompt": "新年宠物海报，Q版圆润的可爱标题“新年快乐汪”，副标题“HAPPY NEW YEAR”。 鱼眼镜头，背景是房间门口，近景，上传的主体歪头笑，围着红色围巾，戴着红色毛线帽，高清，绒毛细节，面部特写。 宝丽莱相纸，超现实主义，写实主义，胶片摄影，打印颗粒感肌理。肌理，超写实，复古感。",
|
||||
"image_urls": ["https://raw.githubusercontent.com/Tencent-Hunyuan/HunyuanImage-3.0/main/assets/demo_instruct_imgs/input_0_0.png"],
|
||||
"num_inference_steps": 8,
|
||||
"seed": 42,
|
||||
"guidance_scale": 1.0
|
||||
}
|
||||
]
|
||||
},
|
||||
"task": "ti2i",
|
||||
"bot-task": "think_recaption",
|
||||
"max-concurrency": 4,
|
||||
"num-prompts": 16,
|
||||
"skip-performance-assertion": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.1,
|
||||
"latency_p99": 44.46,
|
||||
"latency_mean": 35.57,
|
||||
"latency_std_dev": 0.5,
|
||||
"peak_memory_mb_max": 70000,
|
||||
"peak_memory_mb_mean": 70000
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
Reference in new issue
Block a user