[CI][Perf] Add JSON mark support and paired parametrize for L4 perf, so as to further support performance‑case selection for NPU‑CI. (#5084)
Signed-off-by: wangyu <410167048@qq.com> Signed-off-by: wangyu <53896905+yenuo26@users.noreply.github.com>
This commit is contained in:
wangyu authored and GitHub committed
1 parent
5c3900960b
commit
2a49b2b217
31 files changed
+3475
-2191
No files matched your search
@@ -1,6 +1,18 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_bagel_single_device_single_stage_t2i",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Single-stage BAGEL (TP=1, CFG=1), t2i Task",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
@@ -38,6 +50,18 @@
|
||||
},
|
||||
{
|
||||
"test_name": "test_bagel_single_device_single_stage_i2i",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Single-stage BAGEL (TP=1, CFG=1), i2i Task",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
@@ -76,6 +100,18 @@
|
||||
},
|
||||
{
|
||||
"test_name": "test_bagel_single_device_multi_stage_t2i",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Two-stage BAGEL (bagel.yaml), TP=1, CFG=1, t2i Task",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
@@ -113,6 +149,18 @@
|
||||
},
|
||||
{
|
||||
"test_name": "test_bagel_single_device_multi_stage_i2i",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Two-stage BAGEL (bagel.yaml), TP=1, CFG=1, i2i Task",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
|
||||
@@ -1,213 +1,261 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_cosmos3_t2i_official_demo_2gpu",
|
||||
"description": "Cosmos3-Nano text-to-image official demo workload on 2 GPUs",
|
||||
"server_type": "vllm-omni",
|
||||
"benchmark_endpoint": "/v1/images/generations",
|
||||
"server_params": {
|
||||
"model": "nvidia/Cosmos3-Nano",
|
||||
"serve_args": {
|
||||
"cfg-parallel-size": 2,
|
||||
"use-hsdp": true,
|
||||
"hsdp-shard-size": 2,
|
||||
"vae-patch-parallel-size": 2,
|
||||
"vae-use-tiling": true,
|
||||
"model-class-name": "Cosmos3OmniDiffusersPipeline",
|
||||
"no-guardrails": true,
|
||||
"stage-init-timeout": 900,
|
||||
"init-timeout": 1200,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "1024x1024_steps4",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"width": 1024,
|
||||
"height": 1024,
|
||||
"num-inference-steps": 4,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"seed": 42,
|
||||
"extra-body": {
|
||||
"guidance_scale": 7.0,
|
||||
"flow_shift": 3.0,
|
||||
"guardrails": false
|
||||
},
|
||||
"baseline": {
|
||||
"throughput_qps": 1.107,
|
||||
"latency_mean": 0.8929,
|
||||
"peak_memory_mb_mean": 80000.0
|
||||
}
|
||||
}
|
||||
]
|
||||
{
|
||||
"test_name": "test_cosmos3_t2i_official_demo_2gpu",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 2
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Cosmos3-Nano text-to-image official demo workload on 2 GPUs",
|
||||
"server_type": "vllm-omni",
|
||||
"benchmark_endpoint": "/v1/images/generations",
|
||||
"server_params": {
|
||||
"model": "nvidia/Cosmos3-Nano",
|
||||
"serve_args": {
|
||||
"cfg-parallel-size": 2,
|
||||
"use-hsdp": true,
|
||||
"hsdp-shard-size": 2,
|
||||
"vae-patch-parallel-size": 2,
|
||||
"vae-use-tiling": true,
|
||||
"model-class-name": "Cosmos3OmniDiffusersPipeline",
|
||||
"no-guardrails": true,
|
||||
"stage-init-timeout": 900,
|
||||
"init-timeout": 1200,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"test_name": "test_cosmos3_t2v_official_demo_2gpu",
|
||||
"description": "Cosmos3-Nano text-to-video official demo workload on 2 GPUs",
|
||||
"server_type": "vllm-omni",
|
||||
"benchmark_endpoint": "/v1/videos",
|
||||
"server_params": {
|
||||
"model": "nvidia/Cosmos3-Nano",
|
||||
"serve_args": {
|
||||
"cfg-parallel-size": 2,
|
||||
"use-hsdp": true,
|
||||
"hsdp-shard-size": 2,
|
||||
"vae-patch-parallel-size": 2,
|
||||
"vae-use-tiling": true,
|
||||
"model-class-name": "Cosmos3OmniDiffusersPipeline",
|
||||
"no-guardrails": true,
|
||||
"stage-init-timeout": 900,
|
||||
"init-timeout": 1200,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "1024x1024_steps4",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"width": 1024,
|
||||
"height": 1024,
|
||||
"num-inference-steps": 4,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"seed": 42,
|
||||
"extra-body": {
|
||||
"guidance_scale": 7.0,
|
||||
"flow_shift": 3.0,
|
||||
"guardrails": false
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "1280x720_frames189_steps4",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"seed": 42,
|
||||
"random-request-config": [
|
||||
{
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"prompt": "A robot arm is cleaning a plate in the kitchen.",
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 189,
|
||||
"fps": 24,
|
||||
"weight": 1
|
||||
}
|
||||
],
|
||||
"extra-body": {
|
||||
"guidance_scale": 6.0,
|
||||
"flow_shift": 10.0,
|
||||
"max_sequence_length": 4096,
|
||||
"guardrails": false,
|
||||
"use_resolution_template": false,
|
||||
"use_duration_template": false
|
||||
},
|
||||
"baseline": {
|
||||
"throughput_qps": 0.027,
|
||||
"latency_mean": 36.6824,
|
||||
"peak_memory_mb_mean": 30155.4
|
||||
}
|
||||
}
|
||||
]
|
||||
"baseline": {
|
||||
"throughput_qps": 1.107,
|
||||
"latency_mean": 0.8929,
|
||||
"peak_memory_mb_mean": 80000.0
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"test_name": "test_cosmos3_t2v_official_demo_2gpu",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 2
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Cosmos3-Nano text-to-video official demo workload on 2 GPUs",
|
||||
"server_type": "vllm-omni",
|
||||
"benchmark_endpoint": "/v1/videos",
|
||||
"server_params": {
|
||||
"model": "nvidia/Cosmos3-Nano",
|
||||
"serve_args": {
|
||||
"cfg-parallel-size": 2,
|
||||
"use-hsdp": true,
|
||||
"hsdp-shard-size": 2,
|
||||
"vae-patch-parallel-size": 2,
|
||||
"vae-use-tiling": true,
|
||||
"model-class-name": "Cosmos3OmniDiffusersPipeline",
|
||||
"no-guardrails": true,
|
||||
"stage-init-timeout": 900,
|
||||
"init-timeout": 1200,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"test_name": "test_cosmos3_i2v_official_demo_2gpu",
|
||||
"description": "Cosmos3-Nano image-to-video official demo workload on 2 GPUs",
|
||||
"server_type": "vllm-omni",
|
||||
"benchmark_endpoint": "/v1/videos",
|
||||
"server_params": {
|
||||
"model": "nvidia/Cosmos3-Nano",
|
||||
"serve_args": {
|
||||
"cfg-parallel-size": 2,
|
||||
"use-hsdp": true,
|
||||
"hsdp-shard-size": 2,
|
||||
"vae-patch-parallel-size": 2,
|
||||
"vae-use-tiling": true,
|
||||
"model-class-name": "Cosmos3OmniDiffusersPipeline",
|
||||
"no-guardrails": true,
|
||||
"stage-init-timeout": 900,
|
||||
"init-timeout": 1200,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "1280x720_frames189_steps4",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"seed": 42,
|
||||
"random-request-config": [
|
||||
{
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"prompt": "A robot arm is cleaning a plate in the kitchen.",
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 189,
|
||||
"fps": 24,
|
||||
"weight": 1
|
||||
}
|
||||
],
|
||||
"extra-body": {
|
||||
"guidance_scale": 6.0,
|
||||
"flow_shift": 10.0,
|
||||
"max_sequence_length": 4096,
|
||||
"guardrails": false,
|
||||
"use_resolution_template": false,
|
||||
"use_duration_template": false
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "1280x720_frames189_steps4",
|
||||
"dataset": "random",
|
||||
"task": "i2v",
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"num-input-images": 1,
|
||||
"seed": 42,
|
||||
"random-request-config": [
|
||||
{
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"prompt": "The scene comes to life with smooth, natural motion.",
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 189,
|
||||
"fps": 24,
|
||||
"weight": 1
|
||||
}
|
||||
],
|
||||
"extra-body": {
|
||||
"guidance_scale": 6.0,
|
||||
"flow_shift": 10.0,
|
||||
"max_sequence_length": 4096,
|
||||
"guardrails": false
|
||||
},
|
||||
"baseline": {
|
||||
"throughput_qps": 0.027,
|
||||
"latency_mean": 36.968,
|
||||
"peak_memory_mb_mean": 30443.6
|
||||
}
|
||||
}
|
||||
]
|
||||
"baseline": {
|
||||
"throughput_qps": 0.027,
|
||||
"latency_mean": 36.6824,
|
||||
"peak_memory_mb_mean": 30155.4
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"test_name": "test_cosmos3_i2v_official_demo_2gpu",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 2
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Cosmos3-Nano image-to-video official demo workload on 2 GPUs",
|
||||
"server_type": "vllm-omni",
|
||||
"benchmark_endpoint": "/v1/videos",
|
||||
"server_params": {
|
||||
"model": "nvidia/Cosmos3-Nano",
|
||||
"serve_args": {
|
||||
"cfg-parallel-size": 2,
|
||||
"use-hsdp": true,
|
||||
"hsdp-shard-size": 2,
|
||||
"vae-patch-parallel-size": 2,
|
||||
"vae-use-tiling": true,
|
||||
"model-class-name": "Cosmos3OmniDiffusersPipeline",
|
||||
"no-guardrails": true,
|
||||
"stage-init-timeout": 900,
|
||||
"init-timeout": 1200,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"test_name": "test_cosmos3_v2v_official_demo_2gpu",
|
||||
"description": "Cosmos3-Nano video-to-video official demo workload on 2 GPUs",
|
||||
"server_type": "vllm-omni",
|
||||
"benchmark_endpoint": "/v1/videos",
|
||||
"server_params": {
|
||||
"model": "nvidia/Cosmos3-Nano",
|
||||
"serve_args": {
|
||||
"cfg-parallel-size": 2,
|
||||
"use-hsdp": true,
|
||||
"hsdp-shard-size": 2,
|
||||
"vae-patch-parallel-size": 2,
|
||||
"vae-use-tiling": true,
|
||||
"model-class-name": "Cosmos3OmniDiffusersPipeline",
|
||||
"no-guardrails": true,
|
||||
"stage-init-timeout": 900,
|
||||
"init-timeout": 1200,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "1280x720_frames189_steps4",
|
||||
"dataset": "random",
|
||||
"task": "i2v",
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"num-input-images": 1,
|
||||
"seed": 42,
|
||||
"random-request-config": [
|
||||
{
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"prompt": "The scene comes to life with smooth, natural motion.",
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 189,
|
||||
"fps": 24,
|
||||
"weight": 1
|
||||
}
|
||||
],
|
||||
"extra-body": {
|
||||
"guidance_scale": 6.0,
|
||||
"flow_shift": 10.0,
|
||||
"max_sequence_length": 4096,
|
||||
"guardrails": false
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "1280x720_frames189_steps4",
|
||||
"dataset": "random",
|
||||
"task": "v2v",
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"seed": 42,
|
||||
"random-request-config": [
|
||||
{
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"prompt": "Continue the same scene with smooth natural motion.",
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 189,
|
||||
"fps": 24,
|
||||
"weight": 1
|
||||
}
|
||||
],
|
||||
"extra-body": {
|
||||
"guidance_scale": 6.0,
|
||||
"flow_shift": 10.0,
|
||||
"max_sequence_length": 4096,
|
||||
"guardrails": false,
|
||||
"condition_frame_indexes_vision": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"condition_video_keep": "first"
|
||||
},
|
||||
"baseline": {
|
||||
"throughput_qps": 0.027,
|
||||
"latency_mean": 37.2117,
|
||||
"peak_memory_mb_mean": 30135.6
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
"baseline": {
|
||||
"throughput_qps": 0.027,
|
||||
"latency_mean": 36.968,
|
||||
"peak_memory_mb_mean": 30443.6
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"test_name": "test_cosmos3_v2v_official_demo_2gpu",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 2
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Cosmos3-Nano video-to-video official demo workload on 2 GPUs",
|
||||
"server_type": "vllm-omni",
|
||||
"benchmark_endpoint": "/v1/videos",
|
||||
"server_params": {
|
||||
"model": "nvidia/Cosmos3-Nano",
|
||||
"serve_args": {
|
||||
"cfg-parallel-size": 2,
|
||||
"use-hsdp": true,
|
||||
"hsdp-shard-size": 2,
|
||||
"vae-patch-parallel-size": 2,
|
||||
"vae-use-tiling": true,
|
||||
"model-class-name": "Cosmos3OmniDiffusersPipeline",
|
||||
"no-guardrails": true,
|
||||
"stage-init-timeout": 900,
|
||||
"init-timeout": 1200,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "1280x720_frames189_steps4",
|
||||
"dataset": "random",
|
||||
"task": "v2v",
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"seed": 42,
|
||||
"random-request-config": [
|
||||
{
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"prompt": "Continue the same scene with smooth natural motion.",
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 189,
|
||||
"fps": 24,
|
||||
"weight": 1
|
||||
}
|
||||
],
|
||||
"extra-body": {
|
||||
"guidance_scale": 6.0,
|
||||
"flow_shift": 10.0,
|
||||
"max_sequence_length": 4096,
|
||||
"guardrails": false,
|
||||
"condition_frame_indexes_vision": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"condition_video_keep": "first"
|
||||
},
|
||||
"baseline": {
|
||||
"throughput_qps": 0.027,
|
||||
"latency_mean": 37.2117,
|
||||
"peak_memory_mb_mean": 30135.6
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1,6 +1,18 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_higgs_audio_v3",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"tts"
|
||||
],
|
||||
"server_params": {
|
||||
"model": "bosonai/higgs-audio-v3-tts-4b",
|
||||
"stage_overrides": {
|
||||
|
||||
@@ -1,140 +1,152 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_hunyuan_image3_ti2i_inline",
|
||||
"description": "AR (TP=2) + 2 * DiT (TP=2) - using inline dataset and deploy-config",
|
||||
"server_type": "vllm-omni",
|
||||
"benchmark_endpoint": "/v1/images/edits",
|
||||
"server_params": {
|
||||
"model": "tencent/HunyuanImage-3.0-Instruct",
|
||||
"serve_args": {
|
||||
"trust-remote-code": true,
|
||||
"deploy-config-inline": {
|
||||
"type": "yaml",
|
||||
"content": {
|
||||
"pipeline": "hunyuan_image_3_moe",
|
||||
"async_chunk": false,
|
||||
"trust_remote_code": true,
|
||||
"connectors": {
|
||||
"rdma_connector": {
|
||||
"name": "MooncakeTransferEngineConnector",
|
||||
"extra": {
|
||||
"host": "auto",
|
||||
"zmq_port": 50051,
|
||||
"protocol": "rdma",
|
||||
"device_name": "",
|
||||
"memory_pool_size": 17179869184,
|
||||
"memory_pool_device": "cpu"
|
||||
}
|
||||
}
|
||||
},
|
||||
"stages": [
|
||||
{
|
||||
"stage_id": 0,
|
||||
"is_comprehension": true,
|
||||
"final_output": true,
|
||||
"final_output_type": "text",
|
||||
"max_num_seqs": 16,
|
||||
"gpu_memory_utilization": 0.95,
|
||||
"enforce_eager": false,
|
||||
"enable_prefix_caching": false,
|
||||
"max_num_batched_tokens": 32768,
|
||||
"devices": "0,1",
|
||||
"tensor_parallel_size": 2,
|
||||
"hf_overrides": {
|
||||
"rope_parameters": {
|
||||
"mrope_section": [
|
||||
0,
|
||||
32,
|
||||
32
|
||||
],
|
||||
"rope_type": "default"
|
||||
}
|
||||
},
|
||||
"omni_kv_config": {
|
||||
"need_send_cache": true
|
||||
},
|
||||
"output_connectors": {
|
||||
"to_stage_1": "rdma_connector"
|
||||
},
|
||||
"default_sampling_params": {
|
||||
"temperature": 0.0,
|
||||
"top_p": 1,
|
||||
"top_k": -1,
|
||||
"max_tokens": 8192,
|
||||
"detokenize": true,
|
||||
"skip_special_tokens": false,
|
||||
"stop_token_ids": [
|
||||
128025
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"stage_id": 1,
|
||||
"max_num_seqs": 1,
|
||||
"enforce_eager": true,
|
||||
"devices": "2,3",
|
||||
"distributed_executor_backend": "mp",
|
||||
"omni_kv_config": {
|
||||
"need_recv_cache": true
|
||||
},
|
||||
"parallel_config": {
|
||||
"tensor_parallel_size": 2,
|
||||
"vae_patch_parallel_size": 2,
|
||||
"enable_expert_parallel": true
|
||||
},
|
||||
"input_connectors": {
|
||||
"from_stage_0": "rdma_connector"
|
||||
},
|
||||
"default_sampling_params": {
|
||||
"num_inference_steps": 50,
|
||||
"guidance_scale": 0
|
||||
}
|
||||
}
|
||||
],
|
||||
"edges": [
|
||||
{
|
||||
"from": 0,
|
||||
"to": 1,
|
||||
"window_size": -1,
|
||||
"max_inflight": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "inline_dataset_5_requests",
|
||||
"dataset": "custom",
|
||||
"dataset-path-inline": {
|
||||
"type": "jsonl",
|
||||
"content": [
|
||||
{
|
||||
"prompt": "新年宠物海报，Q版圆润的可爱标题“新年快乐汪”，副标题“HAPPY NEW YEAR”。 鱼眼镜头，背景是房间门口，近景，上传的主体歪头笑，围着红色围巾，戴着红色毛线帽，高清，绒毛细节，面部特写。 宝丽莱相纸，超现实主义，写实主义，胶片摄影，打印颗粒感肌理。肌理，超写实，复古感。",
|
||||
"image_urls": [
|
||||
"https://raw.githubusercontent.com/Tencent-Hunyuan/HunyuanImage-3.0/main/assets/demo_instruct_imgs/input_0_0.png"
|
||||
],
|
||||
"num_inference_steps": 8,
|
||||
"seed": 42,
|
||||
"guidance_scale": 1.0
|
||||
}
|
||||
]
|
||||
},
|
||||
"task": "ti2i",
|
||||
"bot-task": "think_recaption",
|
||||
"max-concurrency": 16,
|
||||
"num-prompts": 96,
|
||||
"skip-performance-assertion": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.1874,
|
||||
"latency_p99": 101.852,
|
||||
"latency_mean": 79.0385,
|
||||
"peak_memory_mb_max": 70000,
|
||||
"peak_memory_mb_mean": 70000
|
||||
{
|
||||
"test_name": "test_hunyuan_image3_ti2i_inline",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 4
|
||||
}
|
||||
},
|
||||
"diffusion",
|
||||
"local_model"
|
||||
],
|
||||
"description": "AR (TP=2) + 2 * DiT (TP=2) - using inline dataset and deploy-config",
|
||||
"server_type": "vllm-omni",
|
||||
"benchmark_endpoint": "/v1/images/edits",
|
||||
"server_params": {
|
||||
"model": "tencent/HunyuanImage-3.0-Instruct",
|
||||
"serve_args": {
|
||||
"trust-remote-code": true,
|
||||
"deploy-config-inline": {
|
||||
"type": "yaml",
|
||||
"content": {
|
||||
"pipeline": "hunyuan_image_3_moe",
|
||||
"async_chunk": false,
|
||||
"trust_remote_code": true,
|
||||
"connectors": {
|
||||
"rdma_connector": {
|
||||
"name": "MooncakeTransferEngineConnector",
|
||||
"extra": {
|
||||
"host": "auto",
|
||||
"zmq_port": 50051,
|
||||
"protocol": "rdma",
|
||||
"device_name": "",
|
||||
"memory_pool_size": 17179869184,
|
||||
"memory_pool_device": "cpu"
|
||||
}
|
||||
}
|
||||
},
|
||||
"stages": [
|
||||
{
|
||||
"stage_id": 0,
|
||||
"is_comprehension": true,
|
||||
"final_output": true,
|
||||
"final_output_type": "text",
|
||||
"max_num_seqs": 16,
|
||||
"gpu_memory_utilization": 0.95,
|
||||
"enforce_eager": false,
|
||||
"enable_prefix_caching": false,
|
||||
"max_num_batched_tokens": 32768,
|
||||
"devices": "0,1",
|
||||
"tensor_parallel_size": 2,
|
||||
"hf_overrides": {
|
||||
"rope_parameters": {
|
||||
"mrope_section": [
|
||||
0,
|
||||
32,
|
||||
32
|
||||
],
|
||||
"rope_type": "default"
|
||||
}
|
||||
},
|
||||
"omni_kv_config": {
|
||||
"need_send_cache": true
|
||||
},
|
||||
"output_connectors": {
|
||||
"to_stage_1": "rdma_connector"
|
||||
},
|
||||
"default_sampling_params": {
|
||||
"temperature": 0.0,
|
||||
"top_p": 1,
|
||||
"top_k": -1,
|
||||
"max_tokens": 8192,
|
||||
"detokenize": true,
|
||||
"skip_special_tokens": false,
|
||||
"stop_token_ids": [
|
||||
128025
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"stage_id": 1,
|
||||
"max_num_seqs": 1,
|
||||
"enforce_eager": true,
|
||||
"devices": "2,3",
|
||||
"distributed_executor_backend": "mp",
|
||||
"omni_kv_config": {
|
||||
"need_recv_cache": true
|
||||
},
|
||||
"parallel_config": {
|
||||
"tensor_parallel_size": 2,
|
||||
"vae_patch_parallel_size": 2,
|
||||
"enable_expert_parallel": true
|
||||
},
|
||||
"input_connectors": {
|
||||
"from_stage_0": "rdma_connector"
|
||||
},
|
||||
"default_sampling_params": {
|
||||
"num_inference_steps": 50,
|
||||
"guidance_scale": 0
|
||||
}
|
||||
}
|
||||
],
|
||||
"edges": [
|
||||
{
|
||||
"from": 0,
|
||||
"to": 1,
|
||||
"window_size": -1,
|
||||
"max_inflight": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "inline_dataset_5_requests",
|
||||
"dataset": "custom",
|
||||
"dataset-path-inline": {
|
||||
"type": "jsonl",
|
||||
"content": [
|
||||
{
|
||||
"prompt": "新年宠物海报，Q版圆润的可爱标题“新年快乐汪”，副标题“HAPPY NEW YEAR”。 鱼眼镜头，背景是房间门口，近景，上传的主体歪头笑，围着红色围巾，戴着红色毛线帽，高清，绒毛细节，面部特写。 宝丽莱相纸，超现实主义，写实主义，胶片摄影，打印颗粒感肌理。肌理，超写实，复古感。",
|
||||
"image_urls": [
|
||||
"https://raw.githubusercontent.com/Tencent-Hunyuan/HunyuanImage-3.0/main/assets/demo_instruct_imgs/input_0_0.png"
|
||||
],
|
||||
"num_inference_steps": 8,
|
||||
"seed": 42,
|
||||
"guidance_scale": 1.0
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"task": "ti2i",
|
||||
"bot-task": "think_recaption",
|
||||
"max-concurrency": 16,
|
||||
"num-prompts": 96,
|
||||
"skip-performance-assertion": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.1874,
|
||||
"latency_p99": 101.852,
|
||||
"latency_mean": 79.0385,
|
||||
"peak_memory_mb_max": 70000,
|
||||
"peak_memory_mb_mean": 70000
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1,35 +1,47 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_hunyuan_image_tp2_cfgp2",
|
||||
"description": "TP=2 CfgP=2 baseline",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "tencent/HunyuanImage-3.0-Instruct",
|
||||
"serve_args": {
|
||||
"deploy_config": "../vllm_omni/deploy/hunyuan_image3_dit.yaml",
|
||||
"tensor-parallel-size": 2,
|
||||
"cfg-parallel-size": 2,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "1024x1024_steps8",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"width": 1024,
|
||||
"height": 1024,
|
||||
"num-inference-steps": 8,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"skip-performance-assertion": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.21,
|
||||
"latency_p99": 4.7469,
|
||||
"peak_memory_mb_max": 101100,
|
||||
"peak_memory_mb_mean": 101100
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
{
|
||||
"test_name": "test_hunyuan_image_tp2_cfgp2",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 4
|
||||
}
|
||||
},
|
||||
"diffusion",
|
||||
"local_model"
|
||||
],
|
||||
"description": "TP=2 CfgP=2 baseline",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "tencent/HunyuanImage-3.0-Instruct",
|
||||
"serve_args": {
|
||||
"deploy_config": "../vllm_omni/deploy/hunyuan_image3_dit.yaml",
|
||||
"tensor-parallel-size": 2,
|
||||
"cfg-parallel-size": 2,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "1024x1024_steps8",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"width": 1024,
|
||||
"height": 1024,
|
||||
"num-inference-steps": 8,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"skip-performance-assertion": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.21,
|
||||
"latency_p99": 4.7469,
|
||||
"peak_memory_mb_max": 101100,
|
||||
"peak_memory_mb_mean": 101100
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1,35 +1,47 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_hunyuan_image_tp2_sp2",
|
||||
"description": "TP=2 SP=2 baseline",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "tencent/HunyuanImage-3.0-Instruct",
|
||||
"serve_args": {
|
||||
"deploy-config": "../vllm_omni/deploy/hunyuan_image3_dit.yaml",
|
||||
"tensor-parallel-size": 2,
|
||||
"ulysses-degree": 2,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "1024x1024_steps8",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"width": 1024,
|
||||
"height": 1024,
|
||||
"num-inference-steps": 8,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"skip-performance-assertion": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.20,
|
||||
"latency_p99": 5.1025,
|
||||
"peak_memory_mb_max": 97402,
|
||||
"peak_memory_mb_mean": 97402
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
{
|
||||
"test_name": "test_hunyuan_image_tp2_sp2",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 4
|
||||
}
|
||||
},
|
||||
"diffusion",
|
||||
"local_model"
|
||||
],
|
||||
"description": "TP=2 SP=2 baseline",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "tencent/HunyuanImage-3.0-Instruct",
|
||||
"serve_args": {
|
||||
"deploy-config": "../vllm_omni/deploy/hunyuan_image3_dit.yaml",
|
||||
"tensor-parallel-size": 2,
|
||||
"ulysses-degree": 2,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "1024x1024_steps8",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"width": 1024,
|
||||
"height": 1024,
|
||||
"num-inference-steps": 8,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"skip-performance-assertion": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.20,
|
||||
"latency_p99": 5.1025,
|
||||
"peak_memory_mb_max": 97402,
|
||||
"peak_memory_mb_mean": 97402
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1,35 +1,47 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_hunyuan_image_tp4",
|
||||
"description": "TP=4 baseline",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "tencent/HunyuanImage-3.0-Instruct",
|
||||
"serve_args": {
|
||||
"deploy-config": "../vllm_omni/deploy/hunyuan_image3_dit.yaml",
|
||||
"tensor-parallel-size": 4,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "1024x1024_steps8",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"width": 1024,
|
||||
"height": 1024,
|
||||
"num-inference-steps": 8,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"skip-performance-assertion": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.213,
|
||||
"latency_mean": 4.6953,
|
||||
"latency_p99": 4.8601,
|
||||
"peak_memory_mb_max": 56912.0,
|
||||
"peak_memory_mb_mean": 56912.0
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
{
|
||||
"test_name": "test_hunyuan_image_tp4",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 4
|
||||
}
|
||||
},
|
||||
"diffusion",
|
||||
"local_model"
|
||||
],
|
||||
"description": "TP=4 baseline",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "tencent/HunyuanImage-3.0-Instruct",
|
||||
"serve_args": {
|
||||
"deploy-config": "../vllm_omni/deploy/hunyuan_image3_dit.yaml",
|
||||
"tensor-parallel-size": 4,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "1024x1024_steps8",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"width": 1024,
|
||||
"height": 1024,
|
||||
"num-inference-steps": 8,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"skip-performance-assertion": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.213,
|
||||
"latency_mean": 4.6953,
|
||||
"latency_p99": 4.8601,
|
||||
"peak_memory_mb_max": 56912.0,
|
||||
"peak_memory_mb_mean": 56912.0
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1,217 +1,273 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_ltx2_baseline_eager",
|
||||
"description": "Single-device baseline with enforce-eager (no torch.compile)",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Lightricks/LTX-2",
|
||||
"serve_args": {
|
||||
"enforce-eager": true,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "256x256_145f_steps6",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 256,
|
||||
"height": 256,
|
||||
"num-frames": 145,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 6,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
},
|
||||
{
|
||||
"name": "480x768_41f_steps20",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 768,
|
||||
"height": 480,
|
||||
"num-frames": 41,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
}
|
||||
]
|
||||
{
|
||||
"test_name": "test_ltx2_baseline_eager",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Single-device baseline with enforce-eager (no torch.compile)",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Lightricks/LTX-2",
|
||||
"serve_args": {
|
||||
"enforce-eager": true,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
"test_name": "test_ltx2_torch_compile",
|
||||
"description": "Single-device with torch.compile (default, no enforce-eager)",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Lightricks/LTX-2",
|
||||
"serve_args": {
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "256x256_145f_steps6",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 256,
|
||||
"height": 256,
|
||||
"num-frames": 145,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 6,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
},
|
||||
{
|
||||
"name": "480x768_41f_steps20",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 768,
|
||||
"height": 480,
|
||||
"num-frames": 41,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
}
|
||||
]
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "256x256_145f_steps6",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 256,
|
||||
"height": 256,
|
||||
"num-frames": 145,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 6,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
},
|
||||
{
|
||||
"name": "480x768_41f_steps20",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 768,
|
||||
"height": 480,
|
||||
"num-frames": 41,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"test_name": "test_ltx2_torch_compile",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Single-device with torch.compile (default, no enforce-eager)",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Lightricks/LTX-2",
|
||||
"serve_args": {
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
"test_name": "test_ltx2_cfg2_eager",
|
||||
"description": "CFG-parallel=2 with enforce-eager",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Lightricks/LTX-2",
|
||||
"serve_args": {
|
||||
"cfg-parallel-size": 2,
|
||||
"enforce-eager": true,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "256x256_145f_steps6",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 256,
|
||||
"height": 256,
|
||||
"num-frames": 145,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 6,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
},
|
||||
{
|
||||
"name": "480x768_41f_steps20",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 768,
|
||||
"height": 480,
|
||||
"num-frames": 41,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
}
|
||||
]
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "256x256_145f_steps6",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 256,
|
||||
"height": 256,
|
||||
"num-frames": 145,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 6,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
},
|
||||
{
|
||||
"name": "480x768_41f_steps20",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 768,
|
||||
"height": 480,
|
||||
"num-frames": 41,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"test_name": "test_ltx2_cfg2_eager",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 2
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "CFG-parallel=2 with enforce-eager",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Lightricks/LTX-2",
|
||||
"serve_args": {
|
||||
"cfg-parallel-size": 2,
|
||||
"enforce-eager": true,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
"test_name": "test_ltx2_cfg2_compile",
|
||||
"description": "CFG-parallel=2 with torch.compile",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Lightricks/LTX-2",
|
||||
"serve_args": {
|
||||
"cfg-parallel-size": 2,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "256x256_145f_steps6",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 256,
|
||||
"height": 256,
|
||||
"num-frames": 145,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 6,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
},
|
||||
{
|
||||
"name": "480x768_41f_steps20",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 768,
|
||||
"height": 480,
|
||||
"num-frames": 41,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
}
|
||||
]
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "256x256_145f_steps6",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 256,
|
||||
"height": 256,
|
||||
"num-frames": 145,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 6,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
},
|
||||
{
|
||||
"name": "480x768_41f_steps20",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 768,
|
||||
"height": 480,
|
||||
"num-frames": 41,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"test_name": "test_ltx2_cfg2_compile",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 2
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "CFG-parallel=2 with torch.compile",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Lightricks/LTX-2",
|
||||
"serve_args": {
|
||||
"cfg-parallel-size": 2,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
"test_name": "test_ltx2_cache_dit_eager",
|
||||
"description": "CacheDiT with enforce-eager",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Lightricks/LTX-2",
|
||||
"serve_args": {
|
||||
"cache-backend": "cache_dit",
|
||||
"enforce-eager": true,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "256x256_145f_steps6",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 256,
|
||||
"height": 256,
|
||||
"num-frames": 145,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 6,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
},
|
||||
{
|
||||
"name": "480x768_41f_steps20",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 768,
|
||||
"height": 480,
|
||||
"num-frames": 41,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
}
|
||||
]
|
||||
}
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "256x256_145f_steps6",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 256,
|
||||
"height": 256,
|
||||
"num-frames": 145,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 6,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
},
|
||||
{
|
||||
"name": "480x768_41f_steps20",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 768,
|
||||
"height": 480,
|
||||
"num-frames": 41,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"test_name": "test_ltx2_cache_dit_eager",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "CacheDiT with enforce-eager",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Lightricks/LTX-2",
|
||||
"serve_args": {
|
||||
"cache-backend": "cache_dit",
|
||||
"enforce-eager": true,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "256x256_145f_steps6",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 256,
|
||||
"height": 256,
|
||||
"num-frames": 145,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 6,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
},
|
||||
{
|
||||
"name": "480x768_41f_steps20",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 768,
|
||||
"height": 480,
|
||||
"num-frames": 41,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1,331 +1,343 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_qwen3_omni_chunk",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"extra_cli_args": [
|
||||
"--async-chunk"
|
||||
]
|
||||
{
|
||||
"test_name": "test_qwen3_omni_chunk",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 2
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"omni"
|
||||
],
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"extra_cli_args": [
|
||||
"--async-chunk"
|
||||
]
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "random",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
4,
|
||||
16,
|
||||
32,
|
||||
64,
|
||||
128
|
||||
],
|
||||
"max_concurrency": [
|
||||
1,
|
||||
4,
|
||||
8,
|
||||
16,
|
||||
32
|
||||
],
|
||||
"random_input_len": 2500,
|
||||
"random_output_len": 900,
|
||||
"ignore_eos": true,
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
135.4379,
|
||||
187.0896,
|
||||
400,
|
||||
435.675,
|
||||
724.6836
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
17619.9851,
|
||||
34811.8335,
|
||||
43656.9501,
|
||||
56657.1548,
|
||||
85097.6016
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1279,
|
||||
0.2238,
|
||||
0.2735,
|
||||
0.3712,
|
||||
0.5634
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
10
|
||||
],
|
||||
"request_rate": [
|
||||
0.1
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"random_mm_base_items_per_request": 1,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"audio": 1
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "random",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
4,
|
||||
16,
|
||||
32,
|
||||
64,
|
||||
128
|
||||
],
|
||||
"max_concurrency": [
|
||||
1,
|
||||
4,
|
||||
8,
|
||||
16,
|
||||
32
|
||||
],
|
||||
"random_input_len": 2500,
|
||||
"random_output_len": 900,
|
||||
"ignore_eos": true,
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
135.4379,
|
||||
187.0896,
|
||||
400,
|
||||
435.675,
|
||||
724.6836
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
17619.9851,
|
||||
34811.8335,
|
||||
43656.9501,
|
||||
56657.1548,
|
||||
85097.6016
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1279,
|
||||
0.2238,
|
||||
0.2735,
|
||||
0.3712,
|
||||
0.5634
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
10
|
||||
],
|
||||
"request_rate": [
|
||||
0.1
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"random_mm_base_items_per_request": 1,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"audio": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(0, 60, 3)": 1.0
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
340.329
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
4305.477
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1305
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
40
|
||||
],
|
||||
"request_rate": [
|
||||
0.5
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"random_mm_base_items_per_request": 2,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1,
|
||||
"video": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(256, 256, 1)": 0.5,
|
||||
"(720, 1280, 2)": 0.5
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
220.0909
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
5475.9565
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1818
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
100
|
||||
],
|
||||
"request_rate": [
|
||||
1.0
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"random_mm_base_items_per_request": 3,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1,
|
||||
"video": 1,
|
||||
"audio": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(256, 256, 1)": 0.34,
|
||||
"(720, 1280, 2)": 0.33,
|
||||
"(0, 60, 3)": 0.33
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
450.0155
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
8516.8316
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.2717
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
4,
|
||||
16,
|
||||
32,
|
||||
64,
|
||||
128
|
||||
],
|
||||
"max_concurrency": [
|
||||
1,
|
||||
4,
|
||||
8,
|
||||
16,
|
||||
32
|
||||
],
|
||||
"random_input_len": 2500,
|
||||
"random_output_len": 900,
|
||||
"ignore_eos": true,
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text"
|
||||
]
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
78.4278,
|
||||
195.8037,
|
||||
333.7318,
|
||||
561.805,
|
||||
1038.043
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
4288.3029,
|
||||
7429.9429,
|
||||
9451.6184,
|
||||
12607.5292,
|
||||
18153.2547
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
10
|
||||
],
|
||||
"request_rate": [
|
||||
0.1
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text"
|
||||
]
|
||||
},
|
||||
"random_mm_base_items_per_request": 1,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"audio": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(0, 60, 3)": 1.0
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
275.3875
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
730.6995
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
40
|
||||
],
|
||||
"request_rate": [
|
||||
0.5
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text"
|
||||
]
|
||||
},
|
||||
"random_mm_base_items_per_request": 2,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1,
|
||||
"video": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(256, 256, 1)": 0.5,
|
||||
"(720, 1280, 2)": 0.5
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
166.2201
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
673.5789
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
100
|
||||
],
|
||||
"request_rate": [
|
||||
1.0
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text"
|
||||
]
|
||||
},
|
||||
"random_mm_base_items_per_request": 3,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1,
|
||||
"video": 1,
|
||||
"audio": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(256, 256, 1)": 0.34,
|
||||
"(720, 1280, 2)": 0.33,
|
||||
"(0, 60, 3)": 0.33
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
357.7781
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
953.3493
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
"random_mm_bucket_config": {
|
||||
"(0, 60, 3)": 1.0
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
340.329
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
4305.477
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1305
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
40
|
||||
],
|
||||
"request_rate": [
|
||||
0.5
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"random_mm_base_items_per_request": 2,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1,
|
||||
"video": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(256, 256, 1)": 0.5,
|
||||
"(720, 1280, 2)": 0.5
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
220.0909
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
5475.9565
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1818
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
100
|
||||
],
|
||||
"request_rate": [
|
||||
1.0
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"random_mm_base_items_per_request": 3,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1,
|
||||
"video": 1,
|
||||
"audio": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(256, 256, 1)": 0.34,
|
||||
"(720, 1280, 2)": 0.33,
|
||||
"(0, 60, 3)": 0.33
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
450.0155
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
8516.8316
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.2717
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
4,
|
||||
16,
|
||||
32,
|
||||
64,
|
||||
128
|
||||
],
|
||||
"max_concurrency": [
|
||||
1,
|
||||
4,
|
||||
8,
|
||||
16,
|
||||
32
|
||||
],
|
||||
"random_input_len": 2500,
|
||||
"random_output_len": 900,
|
||||
"ignore_eos": true,
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text"
|
||||
]
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
78.4278,
|
||||
195.8037,
|
||||
333.7318,
|
||||
561.805,
|
||||
1038.043
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
4288.3029,
|
||||
7429.9429,
|
||||
9451.6184,
|
||||
12607.5292,
|
||||
18153.2547
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
10
|
||||
],
|
||||
"request_rate": [
|
||||
0.1
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text"
|
||||
]
|
||||
},
|
||||
"random_mm_base_items_per_request": 1,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"audio": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(0, 60, 3)": 1.0
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
275.3875
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
730.6995
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
40
|
||||
],
|
||||
"request_rate": [
|
||||
0.5
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text"
|
||||
]
|
||||
},
|
||||
"random_mm_base_items_per_request": 2,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1,
|
||||
"video": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(256, 256, 1)": 0.5,
|
||||
"(720, 1280, 2)": 0.5
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
166.2201
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
673.5789
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
100
|
||||
],
|
||||
"request_rate": [
|
||||
1.0
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text"
|
||||
]
|
||||
},
|
||||
"random_mm_base_items_per_request": 3,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1,
|
||||
"video": 1,
|
||||
"audio": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(256, 256, 1)": 0.34,
|
||||
"(720, 1280, 2)": 0.33,
|
||||
"(0, 60, 3)": 0.33
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
357.7781
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
953.3493
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1,115 +1,127 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_qwen3_omni_3gpu_replica2",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"stage_overrides": {
|
||||
"1": {"num_replicas": 2, "devices": "1,2"},
|
||||
"2": {"num_replicas": 2, "devices": "1,2"}
|
||||
},
|
||||
"extra_cli_args": [
|
||||
"--async-chunk"
|
||||
]
|
||||
{
|
||||
"test_name": "test_qwen3_omni_3gpu_replica2",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 3
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"omni"
|
||||
],
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"stage_overrides": {
|
||||
"1": {"num_replicas": 2, "devices": "1,2"},
|
||||
"2": {"num_replicas": 2, "devices": "1,2"}
|
||||
},
|
||||
"extra_cli_args": [
|
||||
"--async-chunk"
|
||||
]
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "random",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [128, 128, 128, 128],
|
||||
"max_concurrency": [8, 16, 24, 32],
|
||||
"random_input_len": 2500,
|
||||
"random_output_len": 900,
|
||||
"ignore_eos": true,
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [203, 283, 470, 869],
|
||||
"mean_tpot_ms": [28, 36, 41, 45],
|
||||
"mean_audio_ttfp_ms": [861, 1200, 1528, 2100],
|
||||
"mean_audio_rtf": [0.24, 0.30, 0.35, 0.39]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [10],
|
||||
"request_rate": [0.1],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"random_mm_base_items_per_request": 1,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"audio": 1
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "random",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [128, 128, 128, 128],
|
||||
"max_concurrency": [8, 16, 24, 32],
|
||||
"random_input_len": 2500,
|
||||
"random_output_len": 900,
|
||||
"ignore_eos": true,
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [203, 283, 470, 869],
|
||||
"mean_tpot_ms": [28, 36, 41, 45],
|
||||
"mean_audio_ttfp_ms": [861, 1200, 1528, 2100],
|
||||
"mean_audio_rtf": [0.24, 0.30, 0.35, 0.39]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [10],
|
||||
"request_rate": [0.1],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"random_mm_base_items_per_request": 1,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"audio": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(0, 60, 3)": 1.0
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [377],
|
||||
"mean_audio_ttfp_ms": [684],
|
||||
"mean_audio_rtf": [0.14]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [40],
|
||||
"request_rate": [0.5],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"random_mm_base_items_per_request": 2,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1,
|
||||
"video": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(256, 256, 1)": 0.5,
|
||||
"(720, 1280, 2)": 0.5
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [228],
|
||||
"mean_audio_ttfp_ms": [686],
|
||||
"mean_audio_rtf": [0.16]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [100],
|
||||
"request_rate": [1.0],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"random_mm_base_items_per_request": 3,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1,
|
||||
"video": 1,
|
||||
"audio": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(256, 256, 1)": 0.34,
|
||||
"(720, 1280, 2)": 0.33,
|
||||
"(0, 60, 3)": 0.33
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [522],
|
||||
"mean_audio_ttfp_ms": [1119],
|
||||
"mean_audio_rtf": [0.21]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
"random_mm_bucket_config": {
|
||||
"(0, 60, 3)": 1.0
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [377],
|
||||
"mean_audio_ttfp_ms": [684],
|
||||
"mean_audio_rtf": [0.14]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [40],
|
||||
"request_rate": [0.5],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"random_mm_base_items_per_request": 2,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1,
|
||||
"video": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(256, 256, 1)": 0.5,
|
||||
"(720, 1280, 2)": 0.5
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [228],
|
||||
"mean_audio_ttfp_ms": [686],
|
||||
"mean_audio_rtf": [0.16]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [100],
|
||||
"request_rate": [1.0],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"random_mm_base_items_per_request": 3,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1,
|
||||
"video": 1,
|
||||
"audio": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(256, 256, 1)": 0.34,
|
||||
"(720, 1280, 2)": 0.33,
|
||||
"(0, 60, 3)": 0.33
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [522],
|
||||
"mean_audio_ttfp_ms": [1119],
|
||||
"mean_audio_rtf": [0.21]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1,170 +1,182 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_qwen3_omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"extra_cli_args": [
|
||||
"--no-async-chunk"
|
||||
]
|
||||
{
|
||||
"test_name": "test_qwen3_omni",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 2
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"omni"
|
||||
],
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"extra_cli_args": [
|
||||
"--no-async-chunk"
|
||||
]
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "random",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
4,
|
||||
16,
|
||||
32,
|
||||
64,
|
||||
128
|
||||
],
|
||||
"max_concurrency": [
|
||||
1,
|
||||
4,
|
||||
8,
|
||||
16,
|
||||
32
|
||||
],
|
||||
"random_input_len": 2500,
|
||||
"random_output_len": 900,
|
||||
"ignore_eos": true,
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
109.1218,
|
||||
247.4411,
|
||||
354.9384,
|
||||
447.7885,
|
||||
694.7487
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
22413.667,
|
||||
30808.9715,
|
||||
35677.844,
|
||||
46345.8978,
|
||||
68350.4853
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1695,
|
||||
0.2033,
|
||||
0.2467,
|
||||
0.3234,
|
||||
0.4438
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
10
|
||||
],
|
||||
"request_rate": [
|
||||
0.1
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"random_mm_base_items_per_request": 1,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"audio": 1
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "random",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
4,
|
||||
16,
|
||||
32,
|
||||
64,
|
||||
128
|
||||
],
|
||||
"max_concurrency": [
|
||||
1,
|
||||
4,
|
||||
8,
|
||||
16,
|
||||
32
|
||||
],
|
||||
"random_input_len": 2500,
|
||||
"random_output_len": 900,
|
||||
"ignore_eos": true,
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
109.1218,
|
||||
247.4411,
|
||||
354.9384,
|
||||
447.7885,
|
||||
694.7487
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
22413.667,
|
||||
30808.9715,
|
||||
35677.844,
|
||||
46345.8978,
|
||||
68350.4853
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1695,
|
||||
0.2033,
|
||||
0.2467,
|
||||
0.3234,
|
||||
0.4438
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
10
|
||||
],
|
||||
"request_rate": [
|
||||
0.1
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"random_mm_base_items_per_request": 1,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"audio": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(0, 60, 3)": 1.0
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
323.3573
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
4623.4173
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1406
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
40
|
||||
],
|
||||
"request_rate": [
|
||||
0.5
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"random_mm_base_items_per_request": 2,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1,
|
||||
"video": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(256, 256, 1)": 0.5,
|
||||
"(720, 1280, 2)": 0.5
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
215.4502
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
4924.149
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1637
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
100
|
||||
],
|
||||
"request_rate": [
|
||||
1.0
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"random_mm_base_items_per_request": 3,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1,
|
||||
"video": 1,
|
||||
"audio": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(256, 256, 1)": 0.34,
|
||||
"(720, 1280, 2)": 0.33,
|
||||
"(0, 60, 3)": 0.33
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
442.5815
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
6441.0986
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.2091
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
"random_mm_bucket_config": {
|
||||
"(0, 60, 3)": 1.0
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
323.3573
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
4623.4173
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1406
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
40
|
||||
],
|
||||
"request_rate": [
|
||||
0.5
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"random_mm_base_items_per_request": 2,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1,
|
||||
"video": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(256, 256, 1)": 0.5,
|
||||
"(720, 1280, 2)": 0.5
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
215.4502
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
4924.149
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1637
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
100
|
||||
],
|
||||
"request_rate": [
|
||||
1.0
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"random_mm_base_items_per_request": 3,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1,
|
||||
"video": 1,
|
||||
"audio": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(256, 256, 1)": 0.34,
|
||||
"(720, 1280, 2)": 0.33,
|
||||
"(0, 60, 3)": 0.33
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
442.5815
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
6441.0986
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.2091
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1,180 +1,192 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_qwen3_omni_vllm_text",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"use_omni": false,
|
||||
"extra_cli_args": [
|
||||
"--no-enable-prefix-caching",
|
||||
"--skip-mm-profiling"
|
||||
]
|
||||
{
|
||||
"test_name": "test_qwen3_omni_vllm_text",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 2
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"omni"
|
||||
],
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"use_omni": false,
|
||||
"extra_cli_args": [
|
||||
"--no-enable-prefix-caching",
|
||||
"--skip-mm-profiling"
|
||||
]
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "random",
|
||||
"backend": "vllm",
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"endpoint": "/v1/completions",
|
||||
"num_prompts": [
|
||||
4,
|
||||
16,
|
||||
32,
|
||||
64,
|
||||
128
|
||||
],
|
||||
"max_concurrency": [
|
||||
1,
|
||||
4,
|
||||
8,
|
||||
16,
|
||||
32
|
||||
],
|
||||
"random_input_len": 2500,
|
||||
"random_output_len": 900,
|
||||
"ignore_eos": true,
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text"
|
||||
]
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "random",
|
||||
"backend": "vllm",
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"endpoint": "/v1/completions",
|
||||
"num_prompts": [
|
||||
4,
|
||||
16,
|
||||
32,
|
||||
64,
|
||||
128
|
||||
],
|
||||
"max_concurrency": [
|
||||
1,
|
||||
4,
|
||||
8,
|
||||
16,
|
||||
32
|
||||
],
|
||||
"random_input_len": 2500,
|
||||
"random_output_len": 900,
|
||||
"ignore_eos": true,
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text"
|
||||
]
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
69.1276,
|
||||
255.1882,
|
||||
310.639,
|
||||
422.0337,
|
||||
532.0582
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
4382.7211,
|
||||
7510.3815,
|
||||
9440.4854,
|
||||
12553.0186,
|
||||
16932.6065
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "vllm",
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"endpoint": "/v1/completions",
|
||||
"num_prompts": [
|
||||
10
|
||||
],
|
||||
"request_rate": [
|
||||
0.1
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text"
|
||||
]
|
||||
},
|
||||
"random_mm_base_items_per_request": 1,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"audio": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(0, 60, 3)": 1.0
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
26.3381
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
485.2436
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "vllm",
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"endpoint": "/v1/completions",
|
||||
"num_prompts": [
|
||||
40
|
||||
],
|
||||
"request_rate": [
|
||||
0.5
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text"
|
||||
]
|
||||
},
|
||||
"random_mm_base_items_per_request": 2,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1,
|
||||
"video": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(256, 256, 1)": 0.5,
|
||||
"(720, 1280, 2)": 0.5
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
26.4517
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
517.2017
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "vllm",
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"endpoint": "/v1/completions",
|
||||
"num_prompts": [
|
||||
100
|
||||
],
|
||||
"request_rate": [
|
||||
1.0
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text"
|
||||
]
|
||||
},
|
||||
"random_mm_base_items_per_request": 3,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1,
|
||||
"video": 1,
|
||||
"audio": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(256, 256, 1)": 0.34,
|
||||
"(720, 1280, 2)": 0.33,
|
||||
"(0, 60, 3)": 0.33
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
27.6972
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
561.3684
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
69.1276,
|
||||
255.1882,
|
||||
310.639,
|
||||
422.0337,
|
||||
532.0582
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
4382.7211,
|
||||
7510.3815,
|
||||
9440.4854,
|
||||
12553.0186,
|
||||
16932.6065
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "vllm",
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"endpoint": "/v1/completions",
|
||||
"num_prompts": [
|
||||
10
|
||||
],
|
||||
"request_rate": [
|
||||
0.1
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text"
|
||||
]
|
||||
},
|
||||
"random_mm_base_items_per_request": 1,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"audio": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(0, 60, 3)": 1.0
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
26.3381
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
485.2436
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "vllm",
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"endpoint": "/v1/completions",
|
||||
"num_prompts": [
|
||||
40
|
||||
],
|
||||
"request_rate": [
|
||||
0.5
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text"
|
||||
]
|
||||
},
|
||||
"random_mm_base_items_per_request": 2,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1,
|
||||
"video": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(256, 256, 1)": 0.5,
|
||||
"(720, 1280, 2)": 0.5
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
26.4517
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
517.2017
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "vllm",
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"endpoint": "/v1/completions",
|
||||
"num_prompts": [
|
||||
100
|
||||
],
|
||||
"request_rate": [
|
||||
1.0
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text"
|
||||
]
|
||||
},
|
||||
"random_mm_base_items_per_request": 3,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1,
|
||||
"video": 1,
|
||||
"audio": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(256, 256, 1)": 0.34,
|
||||
"(720, 1280, 2)": 0.33,
|
||||
"(0, 60, 3)": 0.33
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [
|
||||
27.6972
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
561.3684
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1,149 +1,185 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_qwen_image_edit_2511_single_device",
|
||||
"description": "Single-device baseline (two input images)",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image-Edit-2511",
|
||||
"serve_args": {
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "512x512_steps20_i2i_2img",
|
||||
"dataset": "random",
|
||||
"task": "i2i",
|
||||
"width": 512,
|
||||
"height": 512,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"num-input-images": 2,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.0721,
|
||||
"latency_mean": 13.8398,
|
||||
"peak_memory_mb_max": 54731.4,
|
||||
"peak_memory_mb_mean": 54731.4
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "1536x1536_steps35_i2i_2img",
|
||||
"dataset": "random",
|
||||
"task": "i2i",
|
||||
"width": 1536,
|
||||
"height": 1536,
|
||||
"num-inference-steps": 35,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"num-input-images": 2,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.0186,
|
||||
"latency_mean": 53.6942,
|
||||
"peak_memory_mb_max": 64402.4,
|
||||
"peak_memory_mb_mean": 64402.4
|
||||
}
|
||||
}
|
||||
]
|
||||
{
|
||||
"test_name": "test_qwen_image_edit_2511_single_device",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Single-device baseline (two input images)",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image-Edit-2511",
|
||||
"serve_args": {
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"test_name": "test_qwen_image_edit_2511_ulysses2_cfg2_vae_patch4",
|
||||
"description": "Ulysses SP=2 + CFG=2 + VAE patch parallel=4",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image-Edit-2511",
|
||||
"serve_args": {
|
||||
"ulysses-degree": 2,
|
||||
"cfg-parallel-size": 2,
|
||||
"vae-patch-parallel-size": 4,
|
||||
"vae-use-tiling": true,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "1536x1536_steps35_i2i_2img",
|
||||
"dataset": "random",
|
||||
"task": "i2i",
|
||||
"width": 1536,
|
||||
"height": 1536,
|
||||
"num-inference-steps": 35,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"num-input-images": 2,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.0538,
|
||||
"latency_mean": 18.5591,
|
||||
"peak_memory_mb_max": 53485.0,
|
||||
"peak_memory_mb_mean": 53485.0
|
||||
}
|
||||
}
|
||||
]
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "512x512_steps20_i2i_2img",
|
||||
"dataset": "random",
|
||||
"task": "i2i",
|
||||
"width": 512,
|
||||
"height": 512,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"num-input-images": 2,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.0721,
|
||||
"latency_mean": 13.8398,
|
||||
"peak_memory_mb_max": 54731.4,
|
||||
"peak_memory_mb_mean": 54731.4
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "1536x1536_steps35_i2i_2img",
|
||||
"dataset": "random",
|
||||
"task": "i2i",
|
||||
"width": 1536,
|
||||
"height": 1536,
|
||||
"num-inference-steps": 35,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"num-input-images": 2,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.0186,
|
||||
"latency_mean": 53.6942,
|
||||
"peak_memory_mb_max": 64402.4,
|
||||
"peak_memory_mb_mean": 64402.4
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"test_name": "test_qwen_image_edit_2511_ulysses2_cfg2_vae_patch4",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 4
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Ulysses SP=2 + CFG=2 + VAE patch parallel=4",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image-Edit-2511",
|
||||
"serve_args": {
|
||||
"ulysses-degree": 2,
|
||||
"cfg-parallel-size": 2,
|
||||
"vae-patch-parallel-size": 4,
|
||||
"vae-use-tiling": true,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"test_name": "test_qwen_image_edit_2511_ulysses2_cfg2_cache_dit",
|
||||
"description": "Ulysses SP=2 + CFG=2 + CacheDiT",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image-Edit-2511",
|
||||
"serve_args": {
|
||||
"ulysses-degree": 2,
|
||||
"cfg-parallel-size": 2,
|
||||
"cache-backend": "cache_dit",
|
||||
"cache-config": {
|
||||
"Fn_compute_blocks": 1,
|
||||
"Bn_compute_blocks": 0,
|
||||
"max_warmup_steps": 4,
|
||||
"residual_diff_threshold": 0.24,
|
||||
"max_continuous_cached_steps": 3,
|
||||
"enable_taylorseer": false,
|
||||
"taylorseer_order": 1,
|
||||
"scm_steps_mask_policy": null,
|
||||
"scm_steps_policy": "dynamic"
|
||||
},
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "1536x1536_steps35_i2i_2img",
|
||||
"dataset": "random",
|
||||
"task": "i2i",
|
||||
"width": 1536,
|
||||
"height": 1536,
|
||||
"num-inference-steps": 35,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"num-input-images": 2,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.0538,
|
||||
"latency_mean": 18.5591,
|
||||
"peak_memory_mb_max": 53485.0,
|
||||
"peak_memory_mb_mean": 53485.0
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"test_name": "test_qwen_image_edit_2511_ulysses2_cfg2_cache_dit",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 4
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Ulysses SP=2 + CFG=2 + CacheDiT",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image-Edit-2511",
|
||||
"serve_args": {
|
||||
"ulysses-degree": 2,
|
||||
"cfg-parallel-size": 2,
|
||||
"cache-backend": "cache_dit",
|
||||
"cache-config": {
|
||||
"Fn_compute_blocks": 1,
|
||||
"Bn_compute_blocks": 0,
|
||||
"max_warmup_steps": 4,
|
||||
"residual_diff_threshold": 0.24,
|
||||
"max_continuous_cached_steps": 3,
|
||||
"enable_taylorseer": false,
|
||||
"taylorseer_order": 1,
|
||||
"scm_steps_mask_policy": null,
|
||||
"scm_steps_policy": "dynamic"
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "512x512_steps20_i2i_2img",
|
||||
"dataset": "random",
|
||||
"task": "i2i",
|
||||
"width": 512,
|
||||
"height": 512,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"num-input-images": 2,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.2776,
|
||||
"latency_mean": 3.5938,
|
||||
"peak_memory_mb_max": 55214.0,
|
||||
"peak_memory_mb_mean": 55214.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "1536x1536_steps35_i2i_2img",
|
||||
"dataset": "random",
|
||||
"task": "i2i",
|
||||
"width": 1536,
|
||||
"height": 1536,
|
||||
"num-inference-steps": 35,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"num-input-images": 2,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.1051,
|
||||
"latency_mean": 9.4918,
|
||||
"peak_memory_mb_max": 64265.6,
|
||||
"peak_memory_mb_mean": 64265.6
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "512x512_steps20_i2i_2img",
|
||||
"dataset": "random",
|
||||
"task": "i2i",
|
||||
"width": 512,
|
||||
"height": 512,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"num-input-images": 2,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.2776,
|
||||
"latency_mean": 3.5938,
|
||||
"peak_memory_mb_max": 55214.0,
|
||||
"peak_memory_mb_mean": 55214.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "1536x1536_steps35_i2i_2img",
|
||||
"dataset": "random",
|
||||
"task": "i2i",
|
||||
"width": 1536,
|
||||
"height": 1536,
|
||||
"num-inference-steps": 35,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"num-input-images": 2,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.1051,
|
||||
"latency_mean": 9.4918,
|
||||
"peak_memory_mb_max": 64265.6,
|
||||
"peak_memory_mb_mean": 64265.6
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1,144 +1,180 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_qwen_image_edit_single_device",
|
||||
"description": "Single-device baseline",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image-Edit",
|
||||
"serve_args": {
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "512x512_steps20_i2i",
|
||||
"dataset": "random",
|
||||
"task": "i2i",
|
||||
"width": 512,
|
||||
"height": 512,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.1129,
|
||||
"latency_mean": 8.8352,
|
||||
"peak_memory_mb_max": 55356.5,
|
||||
"peak_memory_mb_mean": 55356.5
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "1536x1536_steps35_i2i",
|
||||
"dataset": "random",
|
||||
"task": "i2i",
|
||||
"width": 1536,
|
||||
"height": 1536,
|
||||
"num-inference-steps": 35,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.0240,
|
||||
"latency_mean": 41.5686,
|
||||
"peak_memory_mb_max": 62762.7,
|
||||
"peak_memory_mb_mean": 62762.7
|
||||
}
|
||||
}
|
||||
]
|
||||
{
|
||||
"test_name": "test_qwen_image_edit_single_device",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Single-device baseline",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image-Edit",
|
||||
"serve_args": {
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"test_name": "test_qwen_image_edit_ulysses2_cfg2_vae_patch4",
|
||||
"description": "Ulysses SP=2 + CFG=2 + VAE patch parallel=4",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image-Edit",
|
||||
"serve_args": {
|
||||
"ulysses-degree": 2,
|
||||
"cfg-parallel-size": 2,
|
||||
"vae-patch-parallel-size": 4,
|
||||
"vae-use-tiling": true,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "1536x1536_steps35_i2i",
|
||||
"dataset": "random",
|
||||
"task": "i2i",
|
||||
"width": 1536,
|
||||
"height": 1536,
|
||||
"num-inference-steps": 35,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.0672,
|
||||
"latency_mean": 14.8517,
|
||||
"peak_memory_mb_max": 53931.5,
|
||||
"peak_memory_mb_mean": 53931.5
|
||||
}
|
||||
}
|
||||
]
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "512x512_steps20_i2i",
|
||||
"dataset": "random",
|
||||
"task": "i2i",
|
||||
"width": 512,
|
||||
"height": 512,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.1129,
|
||||
"latency_mean": 8.8352,
|
||||
"peak_memory_mb_max": 55356.5,
|
||||
"peak_memory_mb_mean": 55356.5
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "1536x1536_steps35_i2i",
|
||||
"dataset": "random",
|
||||
"task": "i2i",
|
||||
"width": 1536,
|
||||
"height": 1536,
|
||||
"num-inference-steps": 35,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.0240,
|
||||
"latency_mean": 41.5686,
|
||||
"peak_memory_mb_max": 62762.7,
|
||||
"peak_memory_mb_mean": 62762.7
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"test_name": "test_qwen_image_edit_ulysses2_cfg2_vae_patch4",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 4
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Ulysses SP=2 + CFG=2 + VAE patch parallel=4",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image-Edit",
|
||||
"serve_args": {
|
||||
"ulysses-degree": 2,
|
||||
"cfg-parallel-size": 2,
|
||||
"vae-patch-parallel-size": 4,
|
||||
"vae-use-tiling": true,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"test_name": "test_qwen_image_edit_ulysses2_cfg2_cache_dit",
|
||||
"description": "Ulysses SP=2 + CFG=2 + CacheDiT",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image-Edit",
|
||||
"serve_args": {
|
||||
"ulysses-degree": 2,
|
||||
"cfg-parallel-size": 2,
|
||||
"cache-backend": "cache_dit",
|
||||
"cache-config": {
|
||||
"Fn_compute_blocks": 1,
|
||||
"Bn_compute_blocks": 0,
|
||||
"max_warmup_steps": 4,
|
||||
"residual_diff_threshold": 0.24,
|
||||
"max_continuous_cached_steps": 3,
|
||||
"enable_taylorseer": false,
|
||||
"taylorseer_order": 1,
|
||||
"scm_steps_mask_policy": null,
|
||||
"scm_steps_policy": "dynamic"
|
||||
},
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "1536x1536_steps35_i2i",
|
||||
"dataset": "random",
|
||||
"task": "i2i",
|
||||
"width": 1536,
|
||||
"height": 1536,
|
||||
"num-inference-steps": 35,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.0672,
|
||||
"latency_mean": 14.8517,
|
||||
"peak_memory_mb_max": 53931.5,
|
||||
"peak_memory_mb_mean": 53931.5
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"test_name": "test_qwen_image_edit_ulysses2_cfg2_cache_dit",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 4
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Ulysses SP=2 + CFG=2 + CacheDiT",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image-Edit",
|
||||
"serve_args": {
|
||||
"ulysses-degree": 2,
|
||||
"cfg-parallel-size": 2,
|
||||
"cache-backend": "cache_dit",
|
||||
"cache-config": {
|
||||
"Fn_compute_blocks": 1,
|
||||
"Bn_compute_blocks": 0,
|
||||
"max_warmup_steps": 4,
|
||||
"residual_diff_threshold": 0.24,
|
||||
"max_continuous_cached_steps": 3,
|
||||
"enable_taylorseer": false,
|
||||
"taylorseer_order": 1,
|
||||
"scm_steps_mask_policy": null,
|
||||
"scm_steps_policy": "dynamic"
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "512x512_steps20_i2i",
|
||||
"dataset": "random",
|
||||
"task": "i2i",
|
||||
"width": 512,
|
||||
"height": 512,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.2879,
|
||||
"latency_mean": 3.4701,
|
||||
"peak_memory_mb_max": 55525.6,
|
||||
"peak_memory_mb_mean": 55525.6
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "1536x1536_steps35_i2i",
|
||||
"dataset": "random",
|
||||
"task": "i2i",
|
||||
"width": 1536,
|
||||
"height": 1536,
|
||||
"num-inference-steps": 35,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.1113,
|
||||
"latency_mean": 8.9656,
|
||||
"peak_memory_mb_max": 63758.3,
|
||||
"peak_memory_mb_mean": 63758.3
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "512x512_steps20_i2i",
|
||||
"dataset": "random",
|
||||
"task": "i2i",
|
||||
"width": 512,
|
||||
"height": 512,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.2879,
|
||||
"latency_mean": 3.4701,
|
||||
"peak_memory_mb_max": 55525.6,
|
||||
"peak_memory_mb_mean": 55525.6
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "1536x1536_steps35_i2i",
|
||||
"dataset": "random",
|
||||
"task": "i2i",
|
||||
"width": 1536,
|
||||
"height": 1536,
|
||||
"num-inference-steps": 35,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.1113,
|
||||
"latency_mean": 8.9656,
|
||||
"peak_memory_mb_max": 63758.3,
|
||||
"peak_memory_mb_mean": 63758.3
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1,49 +1,61 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_qwen_image_layered_single_device",
|
||||
"description": "Single-device baseline",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image-Layered",
|
||||
"serve_args": {
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "640x640_steps20_i2i",
|
||||
"dataset": "random",
|
||||
"task": "i2i",
|
||||
"width": 640,
|
||||
"height": 640,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.0703,
|
||||
"latency_mean": 14.208,
|
||||
"peak_memory_mb_max": 60543.5,
|
||||
"peak_memory_mb_mean": 60543.5
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "1024x1024_steps35_i2i",
|
||||
"dataset": "random",
|
||||
"task": "i2i",
|
||||
"width": 1024,
|
||||
"height": 1024,
|
||||
"num-inference-steps": 35,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.0417,
|
||||
"latency_mean": 23.914,
|
||||
"peak_memory_mb_max": 60543.5,
|
||||
"peak_memory_mb_mean": 60543.5
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
{
|
||||
"test_name": "test_qwen_image_layered_single_device",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Single-device baseline",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image-Layered",
|
||||
"serve_args": {
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "640x640_steps20_i2i",
|
||||
"dataset": "random",
|
||||
"task": "i2i",
|
||||
"width": 640,
|
||||
"height": 640,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.0703,
|
||||
"latency_mean": 14.208,
|
||||
"peak_memory_mb_max": 60543.5,
|
||||
"peak_memory_mb_mean": 60543.5
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "1024x1024_steps35_i2i",
|
||||
"dataset": "random",
|
||||
"task": "i2i",
|
||||
"width": 1024,
|
||||
"height": 1024,
|
||||
"num-inference-steps": 35,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.0417,
|
||||
"latency_mean": 23.914,
|
||||
"peak_memory_mb_max": 60543.5,
|
||||
"peak_memory_mb_mean": 60543.5
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1,219 +1,279 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_qwen_image_single_device",
|
||||
"description": "Single-device baseline (no parallelism)",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image",
|
||||
"serve_args": {
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "512x512_steps20",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"width": 512,
|
||||
"height": 512,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.4153,
|
||||
"latency_mean": 2.4065,
|
||||
"peak_memory_mb_mean": 52289.9
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "1536x1536_steps35",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"width": 1536,
|
||||
"height": 1536,
|
||||
"num-inference-steps": 35,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.0423,
|
||||
"latency_mean": 23.5991,
|
||||
"peak_memory_mb_mean": 61360.5
|
||||
}
|
||||
}
|
||||
]
|
||||
{
|
||||
"test_name": "test_qwen_image_single_device",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Single-device baseline (no parallelism)",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image",
|
||||
"serve_args": {
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"test_name": "test_qwen_image_single_device_step_execution",
|
||||
"description": "Single-device baseline (no parallelism) with step execution",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image",
|
||||
"serve_args": {
|
||||
"enable-diffusion-pipeline-profiler": true,
|
||||
"step-execution": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "512x512_steps20",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"width": 512,
|
||||
"height": 512,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.4175,
|
||||
"latency_mean": 2.3953,
|
||||
"peak_memory_mb_mean": 56000
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "1536x1536_steps35",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"width": 1536,
|
||||
"height": 1536,
|
||||
"num-inference-steps": 35,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.0423,
|
||||
"latency_mean": 23.5768,
|
||||
"peak_memory_mb_mean": 65000
|
||||
}
|
||||
}
|
||||
]
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "512x512_steps20",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"width": 512,
|
||||
"height": 512,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.4153,
|
||||
"latency_mean": 2.4065,
|
||||
"peak_memory_mb_mean": 52289.9
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "1536x1536_steps35",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"width": 1536,
|
||||
"height": 1536,
|
||||
"num-inference-steps": 35,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.0423,
|
||||
"latency_mean": 23.5991,
|
||||
"peak_memory_mb_mean": 61360.5
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"test_name": "test_qwen_image_single_device_step_execution",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Single-device baseline (no parallelism) with step execution",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image",
|
||||
"serve_args": {
|
||||
"enable-diffusion-pipeline-profiler": true,
|
||||
"step-execution": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"test_name": "test_qwen_image_ulysses2_cfg2_vae_patch4",
|
||||
"description": "Ulysses SP=2 + CFG-parallel=2 + VAE Patch Parallel=4",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image",
|
||||
"serve_args": {
|
||||
"ulysses-degree": 2,
|
||||
"cfg-parallel-size": 2,
|
||||
"vae-patch-parallel-size": 4,
|
||||
"vae-use-tiling": true,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "1536x1536_steps35",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"width": 1536,
|
||||
"height": 1536,
|
||||
"num-inference-steps": 35,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.1214,
|
||||
"latency_mean": 8.2211,
|
||||
"peak_memory_mb_mean": 51704.7
|
||||
}
|
||||
}
|
||||
]
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "512x512_steps20",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"width": 512,
|
||||
"height": 512,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.4175,
|
||||
"latency_mean": 2.3953,
|
||||
"peak_memory_mb_mean": 56000
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "1536x1536_steps35",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"width": 1536,
|
||||
"height": 1536,
|
||||
"num-inference-steps": 35,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.0423,
|
||||
"latency_mean": 23.5768,
|
||||
"peak_memory_mb_mean": 65000
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"test_name": "test_qwen_image_ulysses2_cfg2_vae_patch4",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 4
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Ulysses SP=2 + CFG-parallel=2 + VAE Patch Parallel=4",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image",
|
||||
"serve_args": {
|
||||
"ulysses-degree": 2,
|
||||
"cfg-parallel-size": 2,
|
||||
"vae-patch-parallel-size": 4,
|
||||
"vae-use-tiling": true,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"test_name": "test_qwen_image_ulysses2_cfg2_cache_dit",
|
||||
"description": "Ulysses SP=2 + CFG-parallel=2 + CacheDiT acceleration",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image",
|
||||
"serve_args": {
|
||||
"ulysses-degree": 2,
|
||||
"cfg-parallel-size": 2,
|
||||
"cache-backend": "cache_dit",
|
||||
"cache-config": {
|
||||
"Fn_compute_blocks": 1,
|
||||
"Bn_compute_blocks": 0,
|
||||
"max_warmup_steps": 4,
|
||||
"residual_diff_threshold": 0.24,
|
||||
"max_continuous_cached_steps": 3,
|
||||
"enable_taylorseer": false,
|
||||
"taylorseer_order": 1,
|
||||
"scm_steps_mask_policy": null,
|
||||
"scm_steps_policy": "dynamic"
|
||||
},
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "1536x1536_steps35",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"width": 1536,
|
||||
"height": 1536,
|
||||
"num-inference-steps": 35,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.1214,
|
||||
"latency_mean": 8.2211,
|
||||
"peak_memory_mb_mean": 51704.7
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"test_name": "test_qwen_image_ulysses2_cfg2_cache_dit",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 4
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Ulysses SP=2 + CFG-parallel=2 + CacheDiT acceleration",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image",
|
||||
"serve_args": {
|
||||
"ulysses-degree": 2,
|
||||
"cfg-parallel-size": 2,
|
||||
"cache-backend": "cache_dit",
|
||||
"cache-config": {
|
||||
"Fn_compute_blocks": 1,
|
||||
"Bn_compute_blocks": 0,
|
||||
"max_warmup_steps": 4,
|
||||
"residual_diff_threshold": 0.24,
|
||||
"max_continuous_cached_steps": 3,
|
||||
"enable_taylorseer": false,
|
||||
"taylorseer_order": 1,
|
||||
"scm_steps_mask_policy": null,
|
||||
"scm_steps_policy": "dynamic"
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "512x512_steps20",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"width": 512,
|
||||
"height": 512,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.559,
|
||||
"latency_mean": 1.794,
|
||||
"peak_memory_mb_mean": 52397.6775
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "1536x1536_steps35",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"width": 1536,
|
||||
"height": 1536,
|
||||
"num-inference-steps": 35,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.1918,
|
||||
"latency_mean": 5.2022,
|
||||
"peak_memory_mb_mean": 61468.8
|
||||
}
|
||||
}
|
||||
]
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"test_name": "test_qwen_image_single_device_step_execution_high_concurrency",
|
||||
"description": "Single-device Qwen-Image step-level execution benchmark at 512x512, 20 steps, concurrency sweep",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image",
|
||||
"serve_args": {
|
||||
"enable-diffusion-pipeline-profiler": true,
|
||||
"step-execution": true,
|
||||
"max-num-seqs": 8
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "512x512_steps20_high_concurrency",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"width": 512,
|
||||
"height": 512,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": [20, 40, 80, 160],
|
||||
"max-concurrency": [1, 2, 4, 8],
|
||||
"warmup-requests": 8,
|
||||
"warmup-concurrency": 8,
|
||||
"warmup-num-inference-steps": 20,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": [0.4125, 0.6714, 0.6674, 0.8159],
|
||||
"latency_mean": [2.4215, 2.992, 5.9293, 9.656],
|
||||
"peak_memory_mb_mean": [67000, 67000, 67000, 67000]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "512x512_steps20",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"width": 512,
|
||||
"height": 512,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.559,
|
||||
"latency_mean": 1.794,
|
||||
"peak_memory_mb_mean": 52397.6775
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "1536x1536_steps35",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"width": 1536,
|
||||
"height": 1536,
|
||||
"num-inference-steps": 35,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.1918,
|
||||
"latency_mean": 5.2022,
|
||||
"peak_memory_mb_mean": 61468.8
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"test_name": "test_qwen_image_single_device_step_execution_high_concurrency",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Single-device Qwen-Image step-level execution benchmark at 512x512, 20 steps, concurrency sweep",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image",
|
||||
"serve_args": {
|
||||
"enable-diffusion-pipeline-profiler": true,
|
||||
"step-execution": true,
|
||||
"max-num-seqs": 8
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "512x512_steps20_high_concurrency",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"width": 512,
|
||||
"height": 512,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": [20, 40, 80, 160],
|
||||
"max-concurrency": [1, 2, 4, 8],
|
||||
"warmup-requests": 8,
|
||||
"warmup-concurrency": 8,
|
||||
"warmup-num-inference-steps": 20,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": [0.4125, 0.6714, 0.6674, 0.8159],
|
||||
"latency_mean": [2.4215, 2.992, 5.9293, 9.656],
|
||||
"peak_memory_mb_mean": [67000, 67000, 67000, 67000]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -86,3 +86,202 @@ def test_enabled_false_entry_is_skipped():
|
||||
# Only the enabled=True entry should appear
|
||||
assert len(params) == 1
|
||||
assert params[0].get("task") == "voice_clone"
|
||||
|
||||
|
||||
def test_resolve_pytest_marks_hardware_dict_with_extra():
|
||||
from tests.dfx.conftest import resolve_pytest_marks
|
||||
|
||||
marks = resolve_pytest_marks(
|
||||
[
|
||||
{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}},
|
||||
"full_model",
|
||||
"diffusion",
|
||||
"local_model",
|
||||
]
|
||||
)
|
||||
names = {m.name for m in marks}
|
||||
assert "H100" in names
|
||||
assert "cuda" in names
|
||||
assert "gpu" in names
|
||||
assert "distributed_cuda" in names
|
||||
assert "full_model" in names
|
||||
assert "diffusion" in names
|
||||
assert "local_model" in names
|
||||
|
||||
|
||||
def test_resolve_pytest_marks_rejects_legacy_object_format():
|
||||
from tests.dfx.conftest import resolve_pytest_marks
|
||||
|
||||
with pytest.raises(ValueError, match="mark must be a list"):
|
||||
resolve_pytest_marks(
|
||||
{
|
||||
"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1},
|
||||
"marks": ["full_model"],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_extract_mark_resource_label():
|
||||
from tests.dfx.conftest import extract_mark_resource_label
|
||||
|
||||
assert extract_mark_resource_label(None) == "na"
|
||||
assert (
|
||||
extract_mark_resource_label(
|
||||
[
|
||||
{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}},
|
||||
"full_model",
|
||||
]
|
||||
)
|
||||
== "H100"
|
||||
)
|
||||
assert (
|
||||
extract_mark_resource_label(
|
||||
[
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {"cuda": "H100", "rocm": "MI325"},
|
||||
"num_cards": 2,
|
||||
}
|
||||
},
|
||||
"diffusion",
|
||||
]
|
||||
)
|
||||
== "H100-MI325"
|
||||
)
|
||||
|
||||
|
||||
def test_resource_label_for_filename():
|
||||
from tests.dfx.conftest import resource_label_for_filename
|
||||
|
||||
assert resource_label_for_filename("H100") == ""
|
||||
assert resource_label_for_filename("L4") == "L4"
|
||||
assert resource_label_for_filename("910B") == "910B"
|
||||
assert resource_label_for_filename("na") == "na"
|
||||
|
||||
|
||||
def test_hardware_json_value():
|
||||
from tests.dfx.conftest import hardware_json_value
|
||||
|
||||
assert hardware_json_value("H100") == "H100"
|
||||
assert hardware_json_value("na") == ""
|
||||
assert hardware_json_value(None) == ""
|
||||
|
||||
|
||||
def test_extract_configs_resource_label(monkeypatch):
|
||||
from tests.dfx.conftest import extract_configs_resource_label, get_runtime_resource_label
|
||||
|
||||
monkeypatch.setattr(
|
||||
"tests.dfx.conftest._read_runtime_device_name",
|
||||
lambda *, device_id=0: "NVIDIA H100 80GB HBM3",
|
||||
)
|
||||
get_runtime_resource_label(refresh=True)
|
||||
assert extract_configs_resource_label([]) == "H100"
|
||||
get_runtime_resource_label(refresh=True)
|
||||
monkeypatch.setattr(
|
||||
"tests.dfx.conftest._read_runtime_device_name",
|
||||
lambda *, device_id=0: "Ascend910B2",
|
||||
)
|
||||
assert get_runtime_resource_label(refresh=True) == "910B"
|
||||
|
||||
|
||||
def test_load_benchmark_configs_from_dir(tmp_path):
|
||||
from tests.dfx.conftest import load_benchmark_configs
|
||||
|
||||
(tmp_path / "a.json").write_text(
|
||||
json.dumps([{"test_name": "test_a", "server_params": {"model": "m/a"}, "benchmark_params": []}]),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(tmp_path / "b.json").write_text(
|
||||
json.dumps([{"test_name": "test_b", "server_params": {"model": "m/b"}, "benchmark_params": []}]),
|
||||
encoding="utf-8",
|
||||
)
|
||||
configs = load_benchmark_configs(config_dir=tmp_path)
|
||||
assert [c["test_name"] for c in configs] == ["test_a", "test_b"]
|
||||
|
||||
|
||||
def test_create_unique_server_pytest_params_applies_marks(tmp_path):
|
||||
from tests.dfx.conftest import create_unique_server_pytest_params
|
||||
|
||||
configs = [
|
||||
{
|
||||
"test_name": "test_with_mark",
|
||||
"mark": [
|
||||
{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}},
|
||||
"full_model",
|
||||
],
|
||||
"server_params": {"model": "some/model"},
|
||||
"benchmark_params": [{"name": "p0", "num_prompts": 1}],
|
||||
},
|
||||
{
|
||||
"test_name": "test_without_mark",
|
||||
"server_params": {"model": "other/model"},
|
||||
"benchmark_params": [{"name": "p0", "num_prompts": 1}],
|
||||
},
|
||||
]
|
||||
params = create_unique_server_pytest_params(configs, tmp_path)
|
||||
by_id = {p.id: p for p in params}
|
||||
assert len(by_id["test_with_mark"].values) == 1
|
||||
assert isinstance(by_id["test_with_mark"].values[0], tuple)
|
||||
assert any(m.name == "H100" for m in by_id["test_with_mark"].marks)
|
||||
assert not any(m.name == "H100" for m in by_id["test_without_mark"].marks)
|
||||
|
||||
|
||||
def test_is_diffusion_perf_config():
|
||||
from tests.dfx.conftest import is_diffusion_perf_config
|
||||
|
||||
assert not is_diffusion_perf_config(
|
||||
{"test_name": "omni_a", "mark": [{"hardware_marks": {"res": {"cuda": "H100"}}}, "omni"]}
|
||||
)
|
||||
assert is_diffusion_perf_config(
|
||||
{
|
||||
"test_name": "diff_a",
|
||||
"server_type": "vllm-omni",
|
||||
"mark": [{"hardware_marks": {"res": {"cuda": "H100"}}}, "diffusion"],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_benchmark_param_id_suffix_from_task_eval_phase():
|
||||
from tests.dfx.conftest import _unique_benchmark_param_id_suffixes
|
||||
|
||||
params = [
|
||||
{"task": "default_voice", "eval_phase": "latency"},
|
||||
{"task": "default_voice", "eval_phase": "throughput"},
|
||||
]
|
||||
assert _unique_benchmark_param_id_suffixes(params) == [
|
||||
"default_voice_latency",
|
||||
"default_voice_throughput",
|
||||
]
|
||||
|
||||
|
||||
def test_create_paired_omni_benchmark_pytest_params(tmp_path):
|
||||
from tests.dfx.conftest import create_paired_omni_benchmark_pytest_params
|
||||
|
||||
configs = [
|
||||
{
|
||||
"test_name": "test_omni",
|
||||
"mark": [
|
||||
{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}},
|
||||
"omni",
|
||||
],
|
||||
"server_params": {"model": "m/omni"},
|
||||
"benchmark_params": [{"name": "p0", "num_prompts": 1}, {"name": "p1", "num_prompts": 2}],
|
||||
},
|
||||
{
|
||||
"test_name": "test_tts",
|
||||
"mark": [
|
||||
{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}},
|
||||
"tts",
|
||||
],
|
||||
"server_params": {"model": "m/tts"},
|
||||
"benchmark_params": [{"name": "p0", "num_prompts": 1}],
|
||||
},
|
||||
]
|
||||
params = create_paired_omni_benchmark_pytest_params(configs, tmp_path)
|
||||
by_id = {p.id: p for p in params}
|
||||
assert set(by_id) == {"test_omni-p0", "test_omni-p1", "test_tts-p0"}
|
||||
omni_row, bench_row = by_id["test_tts-p0"].values
|
||||
assert omni_row[0] == "test_tts"
|
||||
assert bench_row == ("test_tts", 0)
|
||||
assert any(m.name == "tts" for m in by_id["test_tts-p0"].marks)
|
||||
assert not any(m.name == "tts" for m in by_id["test_omni-p0"].marks)
|
||||
@@ -1,6 +1,18 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_qwen3_tts_base",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 2
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"tts"
|
||||
],
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen3-TTS-12Hz-1.7B-Base",
|
||||
"stage_overrides": {
|
||||
@@ -98,6 +110,18 @@
|
||||
},
|
||||
{
|
||||
"test_name": "test_qwen3_tts_customvoice",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 2
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"tts"
|
||||
],
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
|
||||
"stage_overrides": {
|
||||
@@ -342,6 +366,18 @@
|
||||
},
|
||||
{
|
||||
"test_name": "test_voxtral_tts",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"tts"
|
||||
],
|
||||
"server_params": {
|
||||
"model": "mistralai/Voxtral-4B-TTS-2603"
|
||||
},
|
||||
|
||||
@@ -1,6 +1,18 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_voxcpm2",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"tts"
|
||||
],
|
||||
"server_params": {
|
||||
"model": "openbmb/VoxCPM2",
|
||||
"trust_remote_code": true,
|
||||
|
||||
@@ -1,107 +1,131 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_wan22_i2v_single_device",
|
||||
"description": "Single-device baseline",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers",
|
||||
"serve_args": {
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "832x480_frames81_steps4",
|
||||
"dataset": "random",
|
||||
"task": "i2v",
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"num-input-images": 1,
|
||||
"seed": 42,
|
||||
"enable-negative-prompt": true,
|
||||
"random-request-config": [
|
||||
{
|
||||
"width": 832,
|
||||
"height": 480,
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 81,
|
||||
"fps": 16,
|
||||
"weight": 1
|
||||
}
|
||||
],
|
||||
"baseline": {
|
||||
"throughput_qps": 0.0465,
|
||||
"latency_mean": 21.4354,
|
||||
"peak_memory_mb_mean": 72557.58
|
||||
}
|
||||
}
|
||||
]
|
||||
{
|
||||
"test_name": "test_wan22_i2v_single_device",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Single-device baseline",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers",
|
||||
"serve_args": {
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"test_name": "test_wan22_i2v_usp2_vae_patch2_hsdp_slicing",
|
||||
"description": "USP=2 + VAE patch parallel=2 + HSDP + VAE slicing",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers",
|
||||
"serve_args": {
|
||||
"usp": 2,
|
||||
"vae-patch-parallel-size": 2,
|
||||
"use-hsdp": true,
|
||||
"vae-use-slicing": true,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "832x480_frames81_steps4",
|
||||
"dataset": "random",
|
||||
"task": "i2v",
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"num-input-images": 1,
|
||||
"seed": 42,
|
||||
"enable-negative-prompt": true,
|
||||
"random-request-config": [
|
||||
{
|
||||
"width": 832,
|
||||
"height": 480,
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 81,
|
||||
"fps": 16,
|
||||
"weight": 1
|
||||
}
|
||||
],
|
||||
"baseline": {
|
||||
"throughput_qps": 0.0642,
|
||||
"latency_mean": 15.5589,
|
||||
"peak_memory_mb_mean": 45742.88
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "1280x720_frames121_steps4",
|
||||
"dataset": "random",
|
||||
"task": "i2v",
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"num-input-images": 1,
|
||||
"seed": 42,
|
||||
"enable-negative-prompt": true,
|
||||
"random-request-config": [
|
||||
{
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 121,
|
||||
"fps": 16,
|
||||
"weight": 1
|
||||
}
|
||||
],
|
||||
"baseline": {
|
||||
"throughput_qps": 0.0128,
|
||||
"latency_mean": 77.8565,
|
||||
"peak_memory_mb_mean": 53947.84
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "832x480_frames81_steps4",
|
||||
"dataset": "random",
|
||||
"task": "i2v",
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"num-input-images": 1,
|
||||
"seed": 42,
|
||||
"enable-negative-prompt": true,
|
||||
"random-request-config": [
|
||||
{
|
||||
"width": 832,
|
||||
"height": 480,
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 81,
|
||||
"fps": 16,
|
||||
"weight": 1
|
||||
}
|
||||
],
|
||||
"baseline": {
|
||||
"throughput_qps": 0.0465,
|
||||
"latency_mean": 21.4354,
|
||||
"peak_memory_mb_mean": 72557.58
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"test_name": "test_wan22_i2v_usp2_vae_patch2_hsdp_slicing",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 2
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "USP=2 + VAE patch parallel=2 + HSDP + VAE slicing",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers",
|
||||
"serve_args": {
|
||||
"usp": 2,
|
||||
"vae-patch-parallel-size": 2,
|
||||
"use-hsdp": true,
|
||||
"vae-use-slicing": true,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "832x480_frames81_steps4",
|
||||
"dataset": "random",
|
||||
"task": "i2v",
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"num-input-images": 1,
|
||||
"seed": 42,
|
||||
"enable-negative-prompt": true,
|
||||
"random-request-config": [
|
||||
{
|
||||
"width": 832,
|
||||
"height": 480,
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 81,
|
||||
"fps": 16,
|
||||
"weight": 1
|
||||
}
|
||||
],
|
||||
"baseline": {
|
||||
"throughput_qps": 0.0642,
|
||||
"latency_mean": 15.5589,
|
||||
"peak_memory_mb_mean": 45742.88
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "1280x720_frames121_steps4",
|
||||
"dataset": "random",
|
||||
"task": "i2v",
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"num-input-images": 1,
|
||||
"seed": 42,
|
||||
"enable-negative-prompt": true,
|
||||
"random-request-config": [
|
||||
{
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 121,
|
||||
"fps": 16,
|
||||
"weight": 1
|
||||
}
|
||||
],
|
||||
"baseline": {
|
||||
"throughput_qps": 0.0128,
|
||||
"latency_mean": 77.8565,
|
||||
"peak_memory_mb_mean": 53947.84
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
Reference in new issue
Block a user