diff --git a/benchmarks/diffusion/diffusion_benchmark_serving.py b/benchmarks/diffusion/diffusion_benchmark_serving.py index 7e8ea97e1..2f385b538 100644 --- a/benchmarks/diffusion/diffusion_benchmark_serving.py +++ b/benchmarks/diffusion/diffusion_benchmark_serving.py @@ -47,6 +47,13 @@ Usage: --endpoint /v1/images/edits --dataset random --task ti2i --num-prompts 10 \ --bot-task think + python benchmarks/diffusion/diffusion_benchmark_serving.py \ + --endpoint /v1/images/edits \ + --dataset custom \ + --dataset-path custom_requests.jsonl \ + --task ti2i \ + --bot-task think + i2i (chat-based models such as Qwen-Image-Edit): python3 benchmarks/diffusion/diffusion_benchmark_serving.py \ --endpoint /v1/chat/completions --dataset vbench --task i2i --num-prompts 10 @@ -566,6 +573,135 @@ class TraceDataset(BaseDataset): return [self[i] for i in range(len(self))] +class CustomDataset(BaseDataset): + """ + Custom dataset that loads requests from a JSONL file. + + Each line in the JSONL file should be a JSON object with the following fields: + - prompt (required): The text prompt for the request + - width (optional): Image/video width + - height (optional): Image height + - num_inference_steps (optional): Number of diffusion steps + - seed (optional): Random seed + - image_paths (optional): List of input image paths for i2i/i2v/ti2i tasks + - image_urls (optional): List of input image URLs (alternative to image_paths) + + Example JSONL for ti2i: + {"prompt": "Add sunset lighting", "width": 1024, "height": 1024, "image_urls": ["https://example.com/image.jpg"]} + """ + + def __init__(self, args, api_url: str, model: str): + super().__init__(args, api_url, model) + self.dataset_path = args.dataset_path + if not self.dataset_path: + raise ValueError("--dataset-path must be provided when using 'custom' dataset") + self.load_data() + + def load_data(self) -> None: + """Load data from JSONL file.""" + import pandas as pd + + if not self.dataset_path.endswith(".jsonl"): + raise ValueError("Custom dataset must be a JSONL file") + + try: + df = pd.read_json(path_or_buf=self.dataset_path, lines=True) + except Exception as e: + raise ValueError(f"Failed to load JSONL file {self.dataset_path}: {e}") + + if "prompt" not in df.columns: + raise ValueError("JSONL file must contain a 'prompt' column") + + self.data = df.to_dict("records") + print(f"Loaded {len(self.data)} requests from {self.dataset_path}") + + def _resolve_image_paths(self, item: dict) -> list[str] | None: + # Handle local file paths + if "image_paths" in item and item["image_paths"]: + paths = item["image_paths"] + if isinstance(paths, str): + paths = [paths] + + # Verify all paths exist + valid_paths = [] + for path in paths: + if os.path.exists(path): + valid_paths.append(path) + else: + raise ValueError(f"Image file not found: {path}") + + return valid_paths if valid_paths else None + + # Handle URLs - download to temp files + if "image_urls" in item and item["image_urls"]: + urls = item["image_urls"] + if isinstance(urls, str): + urls = [urls] + + downloaded_paths = [] + for url in urls: + try: + response = requests.get(url, timeout=30) + response.raise_for_status() + + # Create temp file with appropriate extension + suffix = os.path.splitext(url)[1] or ".png" + with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp: + tmp.write(response.content) + downloaded_paths.append(tmp.name) + except Exception as e: + raise ValueError(f"Failed to download image from {url}: {e}") + + return downloaded_paths if downloaded_paths else None + + return None + + def __len__(self) -> int: + return len(self.data) + + def __getitem__(self, idx: int) -> RequestFuncInput: + item = self.data[idx] + + # Extract fields with fallback to args defaults + prompt = item["prompt"] + width = item.get("width", self.args.width) + height = item.get("height", self.args.height) + num_inference_steps = item.get("num_inference_steps", self.args.num_inference_steps) + seed = item.get("seed", self.args.seed) + reserved_keys = {"prompt", "width", "height", "num_inference_steps", "seed", "image_paths", "image_urls"} + extra_body = {k: v for k, v in item.items() if k not in reserved_keys} + + # Handle image paths/URLs + image_paths = self._resolve_image_paths(item) + + return RequestFuncInput( + prompt=prompt, + api_url=self.api_url, + model=self.model, + seed=seed, + image_paths=image_paths, + extra_body=extra_body, + width=width, + height=height, + num_inference_steps=num_inference_steps, + ) + + def get_requests(self) -> list[RequestFuncInput]: + """Get all requests, cycling through data if num_prompts > dataset size.""" + num_requests = getattr(self.args, "num_prompts", len(self.data)) + + if num_requests <= len(self.data): + return [self[i] for i in range(num_requests)] + + # Cycle through the dataset to reach num_prompts + requests = [] + for i in range(num_requests): + idx = i % len(self.data) + requests.append(self[idx]) + + return requests + + class RandomDataset(BaseDataset): def __init__(self, args, api_url: str, model: str, enable_negative_prompt: bool = False): super().__init__(args, api_url, model) @@ -950,6 +1086,8 @@ async def benchmark(args): dataset = TraceDataset(args, api_url, args.model) elif args.dataset == "random": dataset = RandomDataset(args, api_url, args.model, args.enable_negative_prompt) + elif args.dataset == "custom": + dataset = CustomDataset(args, api_url, args.model) else: raise ValueError(f"Unknown dataset: {args.dataset}") @@ -1097,7 +1235,7 @@ if __name__ == "__main__": "--dataset", type=str, default="vbench", - choices=["vbench", "trace", "random"], + choices=["vbench", "trace", "random", "custom"], help="Dataset to use.", ) parser.add_argument( diff --git a/tests/dfx/perf/scripts/run_diffusion_benchmark.py b/tests/dfx/perf/scripts/run_diffusion_benchmark.py index d4ae73fa6..92ce2740d 100644 --- a/tests/dfx/perf/scripts/run_diffusion_benchmark.py +++ b/tests/dfx/perf/scripts/run_diffusion_benchmark.py @@ -41,6 +41,72 @@ pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" os.environ.setdefault("DIFFUSION_ATTENTION_BACKEND", "FLASH_ATTN") + +# --------------------------------------------------------------------------- +# Inline field processing +# --------------------------------------------------------------------------- +def _process_inline_fields(obj: Any, parent_key: str = "") -> None: + """Recursively process '*-inline' fields into temp files.""" + + if isinstance(obj, list): + for item in obj: + _process_inline_fields(item, parent_key) + return + + if not isinstance(obj, dict): + return + + import atexit + + import yaml + + for key in list(obj.keys()): + value = obj[key] + + if not key.endswith("-inline"): + _process_inline_fields(value, key) + continue + + base_key = key[:-7] + full_key = f"{parent_key}.{key}" if parent_key else key + + try: + if not isinstance(value, dict): + raise ValueError("must be a dict") + + file_type = value.get("type") + content = value.get("content") + + if file_type not in {"yaml", "jsonl"}: + raise ValueError(f"invalid type: {file_type}") + + fd, path = tempfile.mkstemp( + suffix=f".{file_type}", + prefix=f"{base_key}_", + ) + + atexit.register(Path(path).unlink, missing_ok=True) + + with os.fdopen(fd, "w", encoding="utf-8") as f: + if file_type == "jsonl": + items = content if isinstance(content, list) else [content] + f.writelines(json.dumps(x, ensure_ascii=False) + "\n" for x in items) + else: + yaml.dump( + content, + f, + allow_unicode=True, + sort_keys=False, + indent=2, + ) + + obj[base_key] = path + del obj[key] + + except Exception as e: + print(f"Warning: failed processing '{full_key}': {e}") + + # --------------------------------------------------------------------------- # Paths # --------------------------------------------------------------------------- @@ -102,11 +168,14 @@ def _resolve_refs(configs: list[dict[str, Any]], config_dir: Path) -> list[dict[ def load_configs(config_path: str) -> list[dict[str, Any]]: + """Load benchmark configs from JSON file and process inline fields.""" try: abs_path = Path(config_path).resolve() with open(abs_path, encoding="utf-8") as f: configs = json.load(f) - return _resolve_refs(configs, abs_path.parent) + configs = _resolve_refs(configs, abs_path.parent) + _process_inline_fields(configs) + return configs except json.JSONDecodeError as e: raise ValueError(f"JSON parsing error: {str(e)}") except FileNotFoundError: diff --git a/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json b/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json new file mode 100644 index 000000000..661f6d263 --- /dev/null +++ b/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json @@ -0,0 +1,131 @@ +[ + { + "test_name": "test_hunyuan_image3_ti2i_tp4_tp4_inline", + "description": "AR (TP=4) + DiT (TP=4) - using inline dataset and deploy-config", + "server_type": "vllm-omni", + "benchmark_endpoint": "/v1/images/edits", + "server_params": { + "model": "tencent/HunyuanImage-3.0-Instruct", + "serve_args": { + "deploy-config-inline": { + "type": "yaml", + "content": { + "pipeline": "hunyuan_image_3_moe", + "async_chunk": false, + "trust_remote_code": true, + "connectors": { + "rdma_connector": { + "name": "MooncakeTransferEngineConnector", + "extra": { + "host": "auto", + "zmq_port": 50051, + "protocol": "rdma", + "device_name": "", + "memory_pool_size": 4294967296, + "memory_pool_device": "cpu" + } + } + }, + "stages": [ + { + "stage_id": 0, + "is_comprehension": true, + "final_output": true, + "final_output_type": "text", + "max_num_seqs": 4, + "gpu_memory_utilization": 0.95, + "enforce_eager": false, + "enable_prefix_caching": false, + "max_num_batched_tokens": 32768, + "devices": "0,1,2,3", + "tensor_parallel_size": 4, + "hf_overrides": { + "rope_parameters": { + "mrope_section": [0, 32, 32], + "rope_type": "default" + } + }, + "omni_kv_config": { + "need_send_cache": true + }, + "output_connectors": { + "to_stage_1": "rdma_connector" + }, + "default_sampling_params": { + "temperature": 0.0, + "top_p": 1, + "top_k": -1, + "max_tokens": 8192, + "detokenize": true, + "skip_special_tokens": false, + "stop_token_ids": [128025] + } + }, + { + "stage_id": 1, + "max_num_seqs": 1, + "enforce_eager": true, + "devices": "4,5,6,7", + "distributed_executor_backend": "mp", + "omni_kv_config": { + "need_recv_cache": true + }, + "parallel_config": { + "tensor_parallel_size": 4, + "enable_expert_parallel": true + }, + "input_connectors": { + "from_stage_0": "rdma_connector" + }, + "default_sampling_params": { + "num_inference_steps": 50, + "guidance_scale": 0 + } + } + ], + "edges": [ + { + "from": 0, + "to": 1, + "window_size": -1, + "max_inflight": 1 + } + ] + } + }, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "inline_dataset_5_requests", + "dataset": "custom", + "dataset-path-inline": { + "type": "jsonl", + "content": [ + { + "prompt": "新年宠物海报，Q版圆润的可爱标题“新年快乐汪”，副标题“HAPPY NEW YEAR”。 鱼眼镜头，背景是房间门口，近景，上传的主体歪头笑，围着红色围巾，戴着红色毛线帽，高清，绒毛细节，面部特写。 宝丽莱相纸，超现实主义，写实主义，胶片摄影，打印颗粒感肌理。肌理，超写实，复古感。", + "image_urls": ["https://raw.githubusercontent.com/Tencent-Hunyuan/HunyuanImage-3.0/main/assets/demo_instruct_imgs/input_0_0.png"], + "num_inference_steps": 8, + "seed": 42, + "guidance_scale": 1.0 + } + ] + }, + "task": "ti2i", + "bot-task": "think_recaption", + "max-concurrency": 4, + "num-prompts": 16, + "skip-performance-assertion": true, + "baseline": { + "throughput_qps": 0.1, + "latency_p99": 44.46, + "latency_mean": 35.57, + "latency_std_dev": 0.5, + "peak_memory_mb_max": 70000, + "peak_memory_mb_mean": 70000 + } + } + ] + } +]