Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Hongsheng Liu
|
4241081a67 | ||
|
hsliuustc0106andCursor
|
28773fbd7e | ||
|
hsliuustc0106andCursor
|
38c92ab0fe | ||
|
Y. Fisherandzzh
|
29179599fc | ||
|
hsliuustc0106andCursor
|
a9b6f71abf | ||
|
wangyuandHongsheng Liu
|
4e82192fd6 | ||
|
hsliuustc0106andCursor
|
e140c224b7 |
No files matched your search
@@ -39,11 +39,34 @@ Committers have write access and merge rights. They typically have deep expertis
|
||||
- [@linyueqian](https://github.com/linyueqian): TTS and Omni Support
|
||||
- [@lishunyang12](https://github.com/lishunyang12): Quantization and Configuration
|
||||
- [@princepride](https://github.com/princepride): Diffusion and Omni Support
|
||||
- [@SamitHuang](https://github.com/SamitHuang): RL and Diffusion
|
||||
- [@RuixiangMa](https://github.com/RuixiangMa): Diffusion models, parallel, cache, and docs
|
||||
- [@SamitHuang](https://github.com/SamitHuang): RL, Diffusion, and cache
|
||||
- [@tzhouam](https://github.com/tzhouam): Engine and New Model Support
|
||||
- [@wtomin](https://github.com/wtomin): Diffusion and Parallelism
|
||||
- [@wtomin](https://github.com/wtomin): Diffusion models, parallel, and docs
|
||||
- [@ZeldaHuang](https://github.com/ZeldaHuang): Omni Support
|
||||
- [@ZJY0516](https://github.com/ZJY0516): Diffusion and CustomOp
|
||||
- [@ZJY0516](https://github.com/ZJY0516): Diffusion attention backend, kernel fusion, and CustomOp
|
||||
- [@yuanheng-zhao](https://github.com/yuanheng-zhao): Diffusion cache, offload, and Omni Support
|
||||
|
||||
## Diffusion Workload Division
|
||||
|
||||
This section breaks down diffusion-related responsibilities. If you have PRs touching these areas, please ping the listed owners for review.
|
||||
|
||||
### Runtime-related
|
||||
|
||||
- Diffusion models: @RuixiangMa, @wtomin
|
||||
|
||||
### Optimization
|
||||
|
||||
- Parallel: @RuixiangMa, @wtomin
|
||||
- Attention backend: @ZJY0516
|
||||
- Cache: @yuanheng-zhao, @RuixiangMa, @SamitHuang
|
||||
- Offload: @yuanheng-zhao
|
||||
- Quantization: @lishunyang12, @david6666666
|
||||
- Kernel fusion / communication-computation: @ZJY0516
|
||||
|
||||
### Docs & Test
|
||||
|
||||
- Diffusion docs: @RuixiangMa, @wtomin
|
||||
|
||||
## Meetings
|
||||
|
||||
|
||||
+32
-5
@@ -279,6 +279,10 @@ def create_benchmark_indices(
|
||||
return indices
|
||||
|
||||
|
||||
# Omni perf/stability: ``vllm bench serve --omni`` result template (used if bench exits without writing JSON).
|
||||
OMNI_RESULT_TEMPLATE_PATH = Path(__file__).resolve().parent / "perf" / "scripts" / "result_omni_template.json"
|
||||
|
||||
|
||||
def _safe_filename_token(value: Any | None, *, default: str = "na") -> str:
|
||||
"""Make a single path segment safe for result filenames on common filesystems."""
|
||||
if value is None:
|
||||
@@ -289,7 +293,7 @@ def _safe_filename_token(value: Any | None, *, default: str = "na") -> str:
|
||||
return s if s else default
|
||||
|
||||
|
||||
def _resolve_baseline_value(
|
||||
def resolve_baseline_value(
|
||||
baseline_raw: Any,
|
||||
*,
|
||||
sweep_index: int | None,
|
||||
@@ -330,7 +334,7 @@ def _baseline_thresholds_for_step(
|
||||
) -> dict[str, Any]:
|
||||
"""Resolve baseline config to one threshold per metric for this iteration."""
|
||||
return {
|
||||
metric_name: _resolve_baseline_value(
|
||||
metric_name: resolve_baseline_value(
|
||||
baseline_raw,
|
||||
sweep_index=sweep_index,
|
||||
max_concurrency=max_concurrency,
|
||||
@@ -354,7 +358,12 @@ def run_benchmark(
|
||||
random_input_len: Any | None = None,
|
||||
random_output_len: Any | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Run one ``vllm bench serve --omni`` iteration and return parsed metrics."""
|
||||
"""Run one ``vllm bench serve --omni`` iteration and return parsed metrics.
|
||||
|
||||
After ``vllm bench`` writes the JSON, ``result["baseline"]`` holds the resolved per-metric thresholds
|
||||
(when ``baseline_config`` is provided). If the benchmark exits without writing a result file,
|
||||
``result_omni_template.json`` is used as a fallback.
|
||||
"""
|
||||
current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
ri = _safe_filename_token(random_input_len)
|
||||
ro = _safe_filename_token(random_output_len)
|
||||
@@ -403,8 +412,17 @@ def run_benchmark(
|
||||
result_dir = "./"
|
||||
|
||||
result_path = os.path.join(result_dir, result_filename)
|
||||
with open(result_path, encoding="utf-8") as f:
|
||||
result = json.load(f)
|
||||
if not os.path.exists(result_path):
|
||||
with open(OMNI_RESULT_TEMPLATE_PATH, encoding="utf-8") as f:
|
||||
template_result: dict[str, Any] = json.load(f)
|
||||
Path(result_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(result_path, "w", encoding="utf-8") as f:
|
||||
json.dump(template_result, f, ensure_ascii=False, indent=2)
|
||||
print(f"Benchmark result file not generated, fallback to template: {result_path}")
|
||||
result = template_result
|
||||
else:
|
||||
with open(result_path, encoding="utf-8") as f:
|
||||
result = json.load(f)
|
||||
|
||||
if baseline_config:
|
||||
result["baseline"] = _baseline_thresholds_for_step(
|
||||
@@ -432,3 +450,12 @@ def pytest_addoption(parser: pytest.Parser) -> None:
|
||||
default=None,
|
||||
help=("Path to benchmark config JSON. Example: --test-config-file tests/dfx/perf/tests/test_tts.json"),
|
||||
)
|
||||
parser.addoption(
|
||||
"--assert-baseline",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=(
|
||||
"When set, omni/diffusion perf runners compare metrics against the baseline block in the JSON config "
|
||||
"(default: off)."
|
||||
),
|
||||
)
|
||||
@@ -12,12 +12,17 @@ from tests.dfx.conftest import (
|
||||
create_unique_server_params,
|
||||
get_benchmark_params_for_server,
|
||||
load_configs,
|
||||
resolve_baseline_value,
|
||||
run_benchmark,
|
||||
)
|
||||
from tests.helpers.runtime import OmniServer
|
||||
|
||||
pytestmark = [pytest.mark.full_model]
|
||||
|
||||
# Compare metrics to each test JSON ``baseline`` block only when pytest is run with ``--assert-baseline``
|
||||
# (registered in ``tests/dfx/conftest.py``; default: off). ``run_benchmark`` and ``_resolve_baseline_value`` are
|
||||
# defined in the same module.
|
||||
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
@@ -46,7 +51,6 @@ if CONFIG_FILE_PATH is None:
|
||||
CONFIG_FILE_PATH = _DEFAULT_CONFIG_FILE
|
||||
|
||||
BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH)
|
||||
OMNI_RESULT_TEMPLATE_PATH = Path(__file__).parent / "result_omni_template.json"
|
||||
|
||||
|
||||
DEPLOY_CONFIGS_DIR = Path(__file__).parent.parent / "deploy"
|
||||
@@ -87,16 +91,6 @@ def omni_server(request):
|
||||
print("OmniServer stopped")
|
||||
|
||||
|
||||
def _safe_filename_token(value: Any | None, *, default: str = "na") -> str:
|
||||
"""Make a single path segment safe for result filenames on common filesystems."""
|
||||
if value is None:
|
||||
return default
|
||||
s = str(value).strip()
|
||||
for bad in ("/", "\\", ":", "*", "?", '"', "<", ">", "|"):
|
||||
s = s.replace(bad, "_")
|
||||
return s if s else default
|
||||
|
||||
|
||||
benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping)
|
||||
|
||||
|
||||
@@ -126,76 +120,23 @@ def benchmark_params(request, omni_server):
|
||||
}
|
||||
|
||||
|
||||
def _resolve_baseline_value(
|
||||
baseline_raw: Any,
|
||||
*,
|
||||
sweep_index: int | None,
|
||||
max_concurrency: Any = None,
|
||||
request_rate: Any = None,
|
||||
) -> Any:
|
||||
"""Pick the baseline threshold for this sweep step.
|
||||
|
||||
Supported shapes per metric:
|
||||
- **Scalar** — same threshold for every concurrency / QPS.
|
||||
- **List** — aligned with ``max_concurrency`` / ``request_rate`` sweep order; use ``sweep_index``.
|
||||
- **Dict** — keyed by concurrency or rate, e.g. ``{"1": 500, "4": 800}`` (keys are strings in JSON).
|
||||
|
||||
For dict lookup, ``max_concurrency`` is preferred when both are set (concurrency sweep).
|
||||
"""
|
||||
if baseline_raw is None:
|
||||
# If no baseline is set, the maximum value will be used.
|
||||
return 100000
|
||||
if isinstance(baseline_raw, dict):
|
||||
if max_concurrency is not None:
|
||||
for key in (max_concurrency, str(max_concurrency)):
|
||||
if key in baseline_raw:
|
||||
return baseline_raw[key]
|
||||
if request_rate is not None:
|
||||
for key in (request_rate, str(request_rate)):
|
||||
if key in baseline_raw:
|
||||
return baseline_raw[key]
|
||||
raise KeyError(
|
||||
f"baseline dict has no key for max_concurrency={max_concurrency!r} "
|
||||
f"or request_rate={request_rate!r}; keys={list(baseline_raw.keys())!r}"
|
||||
)
|
||||
if isinstance(baseline_raw, (list, tuple)):
|
||||
return baseline_raw[sweep_index]
|
||||
return baseline_raw
|
||||
|
||||
|
||||
def _baseline_thresholds_for_step(
|
||||
baseline_data: dict[str, Any],
|
||||
*,
|
||||
sweep_index: int | None = None,
|
||||
max_concurrency: Any = None,
|
||||
request_rate: Any = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Resolve ``test.json`` ``baseline`` block to one threshold per metric (same as ``assert_result``)."""
|
||||
return {
|
||||
metric_name: _resolve_baseline_value(
|
||||
baseline_raw,
|
||||
sweep_index=sweep_index,
|
||||
max_concurrency=max_concurrency,
|
||||
request_rate=request_rate,
|
||||
)
|
||||
for metric_name, baseline_raw in baseline_data.items()
|
||||
}
|
||||
|
||||
|
||||
def assert_result(
|
||||
result,
|
||||
params,
|
||||
num_prompt,
|
||||
*,
|
||||
assert_baseline: bool,
|
||||
sweep_index: int | None = None,
|
||||
max_concurrency: Any = None,
|
||||
request_rate: Any = None,
|
||||
max_concurrency: Any | None = None,
|
||||
request_rate: Any | None = None,
|
||||
) -> None:
|
||||
assert result["completed"] == num_prompt, "Request failures exist"
|
||||
if not assert_baseline:
|
||||
return
|
||||
baseline_data = params.get("baseline", {})
|
||||
for metric_name, baseline_raw in baseline_data.items():
|
||||
current_value = result[metric_name]
|
||||
baseline_value = _resolve_baseline_value(
|
||||
baseline_value = resolve_baseline_value(
|
||||
baseline_raw,
|
||||
sweep_index=sweep_index,
|
||||
max_concurrency=max_concurrency,
|
||||
@@ -214,7 +155,7 @@ def assert_result(
|
||||
@pytest.mark.benchmark
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
@pytest.mark.parametrize("benchmark_params", benchmark_indices, indirect=True)
|
||||
def test_performance_benchmark(omni_server, benchmark_params):
|
||||
def test_performance_benchmark(omni_server, benchmark_params, request):
|
||||
test_name = benchmark_params["test_name"]
|
||||
params = benchmark_params["params"]
|
||||
dataset_name = params.get("dataset_name", "")
|
||||
@@ -226,6 +167,8 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
print(f"Running benchmark for model: {model}")
|
||||
print(f"Benchmark parameters: {benchmark_params}")
|
||||
|
||||
assert_baseline = request.config.getoption("--assert-baseline", default=False)
|
||||
|
||||
def to_list(value, default=None):
|
||||
if value is None:
|
||||
return [] if default is None else [default]
|
||||
@@ -264,7 +207,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
elif not isinstance(value, bool):
|
||||
args.extend([arg_name, str(value)])
|
||||
|
||||
# QPS test (sweep_index aligns with qps_list / num_prompt_list for this loop)
|
||||
# QPS / request-rate sweep
|
||||
for i, (qps, num_prompt) in enumerate(zip(qps_list, num_prompt_list)):
|
||||
args = args + ["--request-rate", str(qps), "--num-prompts", str(num_prompt)]
|
||||
result = run_benchmark(
|
||||
@@ -283,12 +226,13 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
assert_result(
|
||||
result,
|
||||
params,
|
||||
num_prompt=num_prompt,
|
||||
num_prompt,
|
||||
assert_baseline=assert_baseline,
|
||||
sweep_index=i,
|
||||
request_rate=qps,
|
||||
)
|
||||
|
||||
# concurrency test (sweep_index aligns with max_concurrency_list for separate thresholds per concurrency)
|
||||
# concurrency test
|
||||
for i, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)):
|
||||
args = args + ["--max-concurrency", str(concurrency), "--num-prompts", str(num_prompt), "--request-rate", "inf"]
|
||||
result = run_benchmark(
|
||||
@@ -307,7 +251,8 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
assert_result(
|
||||
result,
|
||||
params,
|
||||
num_prompt=num_prompt,
|
||||
num_prompt,
|
||||
assert_baseline=assert_baseline,
|
||||
sweep_index=i,
|
||||
max_concurrency=concurrency,
|
||||
)
|
||||
@@ -11,6 +11,8 @@ This runner separates two concepts:
|
||||
A config JSON file is REQUIRED via --test-config-file:
|
||||
pytest run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
|
||||
|
||||
Optional: ``--assert-baseline`` compares metrics to the ``baseline`` block in each benchmark entry (default: off).
|
||||
|
||||
All benchmark results for a session are consolidated into a single JSON file under
|
||||
BENCHMARK_RESULT_DIR (override via the DIFFUSION_BENCHMARK_DIR environment variable).
|
||||
Each entry in the file contains the test metadata (test_name, endpoint, benchmark_params,
|
||||
@@ -652,12 +654,14 @@ def run_benchmark(
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def assert_result(result: dict[str, Any], params: dict[str, Any]) -> None:
|
||||
"""Assert that benchmark metrics satisfy the configured baselines."""
|
||||
def assert_result(result: dict[str, Any], params: dict[str, Any], *, assert_baseline: bool) -> None:
|
||||
"""Assert request completion; optionally assert metrics against ``baseline`` in *params*."""
|
||||
num_prompts = params.get("num-prompts", 10)
|
||||
completed = result.get("completed_requests", result.get("completed", 0))
|
||||
assert completed == num_prompts, f"Expected {num_prompts} completed requests, got {completed}"
|
||||
|
||||
if not assert_baseline:
|
||||
return
|
||||
if params.get("skip-performance-assertion", False):
|
||||
print("Skipping performance assertions.")
|
||||
return
|
||||
@@ -699,16 +703,14 @@ def _resolve_benchmark_endpoint(server_cfg: dict[str, Any], params: dict[str, An
|
||||
indirect=True,
|
||||
)
|
||||
@pytest.mark.parametrize("benchmark_params", benchmark_indices, indirect=True)
|
||||
def test_diffusion_performance_benchmark(diffusion_server, benchmark_params):
|
||||
"""Run the diffusion performance benchmark and assert against baselines.
|
||||
def test_diffusion_performance_benchmark(diffusion_server, benchmark_params, request):
|
||||
"""Run the diffusion performance benchmark and verify request completion.
|
||||
|
||||
One server is started per unique parallel configuration (module scope).
|
||||
For each server, all benchmark parameter sets defined in the config JSON
|
||||
are executed sequentially; results are asserted against the baselines.
|
||||
are executed sequentially; metrics are recorded to the aggregated result file.
|
||||
|
||||
Tracked metrics:
|
||||
- throughput_qps (higher is better)
|
||||
- latency_p50, latency_p99 (lower is better)
|
||||
Pass ``--assert-baseline`` to compare ``throughput_qps``, ``latency_mean``, etc. to the JSON ``baseline`` block.
|
||||
"""
|
||||
test_name = benchmark_params["test_name"]
|
||||
params = benchmark_params["params"]
|
||||
@@ -744,4 +746,5 @@ def test_diffusion_performance_benchmark(diffusion_server, benchmark_params):
|
||||
print(f"\n Aggregated results: {AGGREGATED_RESULT_FILE}")
|
||||
print("=" * 60)
|
||||
|
||||
assert_result(result, params)
|
||||
assert_baseline = request.config.getoption("--assert-baseline", default=False)
|
||||
assert_result(result, params, assert_baseline=assert_baseline)
|
||||
@@ -3,31 +3,67 @@
|
||||
"test_name": "test_qwen3_omni_chunk",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"extra_cli_args": ["--async-chunk"]
|
||||
"extra_cli_args": [
|
||||
"--async-chunk"
|
||||
]
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "random",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [4, 16, 32, 64, 128],
|
||||
"max_concurrency": [1, 4, 8, 16, 32],
|
||||
"num_prompts": [
|
||||
4,
|
||||
16,
|
||||
32,
|
||||
64,
|
||||
128
|
||||
],
|
||||
"max_concurrency": [
|
||||
1,
|
||||
4,
|
||||
8,
|
||||
16,
|
||||
32
|
||||
],
|
||||
"random_input_len": 2500,
|
||||
"random_output_len": 900,
|
||||
"ignore_eos": true,
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [1000, 3000, 5000, 7000, 9000],
|
||||
"mean_audio_ttfp_ms": [1000, 3000, 5000, 7000, 9000],
|
||||
"mean_audio_rtf": [0.2, 0.35, 0.6, 0.85, 0.9]
|
||||
"mean_ttft_ms": [
|
||||
400,
|
||||
300,
|
||||
400,
|
||||
600,
|
||||
900
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
26000,
|
||||
43000,
|
||||
52000,
|
||||
72000,
|
||||
109000
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.15,
|
||||
0.30,
|
||||
0.35,
|
||||
0.50,
|
||||
0.75
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [10],
|
||||
"request_rate": [0.1],
|
||||
"num_prompts": [
|
||||
10
|
||||
],
|
||||
"request_rate": [
|
||||
0.1
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
@@ -42,17 +78,27 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [2000],
|
||||
"mean_audio_ttfp_ms": [2000],
|
||||
"mean_audio_rtf": [0.25]
|
||||
"mean_ttft_ms": [
|
||||
600
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
6000
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.20
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [40],
|
||||
"request_rate": [0.5],
|
||||
"num_prompts": [
|
||||
40
|
||||
],
|
||||
"request_rate": [
|
||||
0.5
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
@@ -69,17 +115,27 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [6000],
|
||||
"mean_audio_ttfp_ms": [6000],
|
||||
"mean_audio_rtf": [0.7]
|
||||
"mean_ttft_ms": [
|
||||
300
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
7000
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.25
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [100],
|
||||
"request_rate": [1.0],
|
||||
"num_prompts": [
|
||||
100
|
||||
],
|
||||
"request_rate": [
|
||||
1.0
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
@@ -98,40 +154,79 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [12000],
|
||||
"mean_audio_ttfp_ms": [12000],
|
||||
"mean_audio_rtf": [1.0]
|
||||
"mean_ttft_ms": [
|
||||
700
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
11000
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.35
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [4, 16, 32, 64, 128],
|
||||
"max_concurrency": [1, 4, 8, 16, 32],
|
||||
"num_prompts": [
|
||||
4,
|
||||
16,
|
||||
32,
|
||||
64,
|
||||
128
|
||||
],
|
||||
"max_concurrency": [
|
||||
1,
|
||||
4,
|
||||
8,
|
||||
16,
|
||||
32
|
||||
],
|
||||
"random_input_len": 2500,
|
||||
"random_output_len": 900,
|
||||
"ignore_eos": true,
|
||||
"extra_body": {
|
||||
"modalities": ["text"]
|
||||
"modalities": [
|
||||
"text"
|
||||
]
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [1000, 3000, 5000, 7000, 9000]
|
||||
"mean_ttft_ms": [
|
||||
100,
|
||||
300,
|
||||
500,
|
||||
800,
|
||||
1500
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
5000,
|
||||
8000,
|
||||
10000,
|
||||
14000,
|
||||
20000
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [10],
|
||||
"request_rate": [0.1],
|
||||
"num_prompts": [
|
||||
10
|
||||
],
|
||||
"request_rate": [
|
||||
0.1
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"extra_body": {
|
||||
"modalities": ["text"]
|
||||
"modalities": [
|
||||
"text"
|
||||
]
|
||||
},
|
||||
"random_mm_base_items_per_request": 1,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
@@ -143,21 +238,32 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [2000]
|
||||
"mean_ttft_ms": [
|
||||
500
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
1000
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [40],
|
||||
"request_rate": [0.5],
|
||||
"num_prompts": [
|
||||
40
|
||||
],
|
||||
"request_rate": [
|
||||
0.5
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"extra_body": {
|
||||
"modalities": ["text"]
|
||||
"modalities": [
|
||||
"text"
|
||||
]
|
||||
},
|
||||
"random_mm_base_items_per_request": 2,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
@@ -171,21 +277,32 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [6000]
|
||||
"mean_ttft_ms": [
|
||||
200
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
800
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [100],
|
||||
"request_rate": [1.0],
|
||||
"num_prompts": [
|
||||
100
|
||||
],
|
||||
"request_rate": [
|
||||
1.0
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"extra_body": {
|
||||
"modalities": ["text"]
|
||||
"modalities": [
|
||||
"text"
|
||||
]
|
||||
},
|
||||
"random_mm_base_items_per_request": 3,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
@@ -201,7 +318,12 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [6000]
|
||||
"mean_ttft_ms": [
|
||||
600
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
1300
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -3,31 +3,67 @@
|
||||
"test_name": "test_qwen3_omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"extra_cli_args": ["--no-async-chunk"]
|
||||
"extra_cli_args": [
|
||||
"--no-async-chunk"
|
||||
]
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "random",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [4, 16, 32, 64, 128],
|
||||
"max_concurrency": [1, 4, 8, 16, 32],
|
||||
"num_prompts": [
|
||||
4,
|
||||
16,
|
||||
32,
|
||||
64,
|
||||
128
|
||||
],
|
||||
"max_concurrency": [
|
||||
1,
|
||||
4,
|
||||
8,
|
||||
16,
|
||||
32
|
||||
],
|
||||
"random_input_len": 2500,
|
||||
"random_output_len": 900,
|
||||
"ignore_eos": true,
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [1000, 3000, 5000, 7000, 9000],
|
||||
"mean_audio_ttfp_ms": [30000, 60000, 90000, 120000, 150000],
|
||||
"mean_audio_rtf": [0.35, 0.45, 0.55, 0.65, 0.75]
|
||||
"mean_ttft_ms": [
|
||||
400,
|
||||
300,
|
||||
400,
|
||||
600,
|
||||
900
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
32000,
|
||||
34000,
|
||||
45000,
|
||||
61000,
|
||||
90000
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.20,
|
||||
0.25,
|
||||
0.30,
|
||||
0.45,
|
||||
0.60
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [10],
|
||||
"request_rate": [0.1],
|
||||
"num_prompts": [
|
||||
10
|
||||
],
|
||||
"request_rate": [
|
||||
0.1
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
@@ -42,17 +78,27 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [2000],
|
||||
"mean_audio_ttfp_ms": [10000],
|
||||
"mean_audio_rtf": [0.25]
|
||||
"mean_ttft_ms": [
|
||||
500
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
6000
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.20
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [40],
|
||||
"request_rate": [0.5],
|
||||
"num_prompts": [
|
||||
40
|
||||
],
|
||||
"request_rate": [
|
||||
0.5
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
@@ -69,17 +115,27 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [6000],
|
||||
"mean_audio_ttfp_ms": [15000],
|
||||
"mean_audio_rtf": [0.45]
|
||||
"mean_ttft_ms": [
|
||||
300
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
7000
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.20
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [100],
|
||||
"request_rate": [1.0],
|
||||
"num_prompts": [
|
||||
100
|
||||
],
|
||||
"request_rate": [
|
||||
1.0
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
@@ -98,9 +154,15 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [12000],
|
||||
"mean_audio_ttfp_ms": [18000],
|
||||
"mean_audio_rtf": [0.9]
|
||||
"mean_ttft_ms": [
|
||||
700
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
9000
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.30
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -4,7 +4,9 @@
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"use_omni": false,
|
||||
"extra_cli_args": ["--no-enable-prefix-caching"]
|
||||
"extra_cli_args": [
|
||||
"--no-enable-prefix-caching"
|
||||
]
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
@@ -12,8 +14,20 @@
|
||||
"backend": "vllm",
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"endpoint": "/v1/completions",
|
||||
"num_prompts": [4, 16, 32, 64, 128],
|
||||
"max_concurrency": [1, 4, 8, 16, 32],
|
||||
"num_prompts": [
|
||||
4,
|
||||
16,
|
||||
32,
|
||||
64,
|
||||
128
|
||||
],
|
||||
"max_concurrency": [
|
||||
1,
|
||||
4,
|
||||
8,
|
||||
16,
|
||||
32
|
||||
],
|
||||
"random_input_len": 2500,
|
||||
"random_output_len": 900,
|
||||
"temperature": 0.4,
|
||||
@@ -23,11 +37,26 @@
|
||||
"repetition_penalty": 1.05,
|
||||
"ignore_eos": true,
|
||||
"extra_body": {
|
||||
"modalities": ["text"]
|
||||
"modalities": [
|
||||
"text"
|
||||
]
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [1000, 3000, 5000, 7000, 9000]
|
||||
"mean_ttft_ms": [
|
||||
300,
|
||||
300,
|
||||
1000,
|
||||
600,
|
||||
800
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
6000,
|
||||
8000,
|
||||
11000,
|
||||
13000,
|
||||
17000
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -35,8 +64,12 @@
|
||||
"backend": "vllm",
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"endpoint": "/v1/completions",
|
||||
"num_prompts": [10],
|
||||
"request_rate": [0.1],
|
||||
"num_prompts": [
|
||||
10
|
||||
],
|
||||
"request_rate": [
|
||||
0.1
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
@@ -47,7 +80,9 @@
|
||||
"repetition_penalty": 1.05,
|
||||
"ignore_eos": true,
|
||||
"extra_body": {
|
||||
"modalities": ["text"]
|
||||
"modalities": [
|
||||
"text"
|
||||
]
|
||||
},
|
||||
"random_mm_base_items_per_request": 1,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
@@ -59,7 +94,12 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [2000]
|
||||
"mean_ttft_ms": [
|
||||
30
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
540
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -67,8 +107,12 @@
|
||||
"backend": "vllm",
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"endpoint": "/v1/completions",
|
||||
"num_prompts": [40],
|
||||
"request_rate": [0.5],
|
||||
"num_prompts": [
|
||||
40
|
||||
],
|
||||
"request_rate": [
|
||||
0.5
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
@@ -79,7 +123,9 @@
|
||||
"repetition_penalty": 1.05,
|
||||
"ignore_eos": true,
|
||||
"extra_body": {
|
||||
"modalities": ["text"]
|
||||
"modalities": [
|
||||
"text"
|
||||
]
|
||||
},
|
||||
"random_mm_base_items_per_request": 2,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
@@ -93,7 +139,12 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [6000]
|
||||
"mean_ttft_ms": [
|
||||
30
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
600
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -101,8 +152,12 @@
|
||||
"backend": "vllm",
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"endpoint": "/v1/completions",
|
||||
"num_prompts": [100],
|
||||
"request_rate": [1.0],
|
||||
"num_prompts": [
|
||||
100
|
||||
],
|
||||
"request_rate": [
|
||||
1.0
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
@@ -113,7 +168,9 @@
|
||||
"repetition_penalty": 1.05,
|
||||
"ignore_eos": true,
|
||||
"extra_body": {
|
||||
"modalities": ["text"]
|
||||
"modalities": [
|
||||
"text"
|
||||
]
|
||||
},
|
||||
"random_mm_base_items_per_request": 3,
|
||||
"random_mm_num_mm_items_range_ratio": 0.5,
|
||||
@@ -129,7 +186,12 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": [6000]
|
||||
"mean_ttft_ms": [
|
||||
40
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
700
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -22,10 +22,10 @@
|
||||
"num-input-images": 2,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.05,
|
||||
"latency_mean": 18,
|
||||
"peak_memory_mb_max": 78500,
|
||||
"peak_memory_mb_mean": 78500
|
||||
"throughput_qps": 0.06,
|
||||
"latency_mean": 20,
|
||||
"peak_memory_mb_max": 57000,
|
||||
"peak_memory_mb_mean": 57000
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -41,9 +41,9 @@
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.01,
|
||||
"latency_mean": 70,
|
||||
"peak_memory_mb_max": 81000,
|
||||
"peak_memory_mb_mean": 81000
|
||||
"latency_mean": 60,
|
||||
"peak_memory_mb_max": 67000,
|
||||
"peak_memory_mb_mean": 67000
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -75,10 +75,10 @@
|
||||
"num-input-images": 2,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.1,
|
||||
"latency_mean": 12,
|
||||
"peak_memory_mb_max": 69000,
|
||||
"peak_memory_mb_mean": 69000
|
||||
"throughput_qps": 0.15,
|
||||
"latency_mean": 10,
|
||||
"peak_memory_mb_max": 55000,
|
||||
"peak_memory_mb_mean": 55000
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -93,10 +93,10 @@
|
||||
"num-input-images": 2,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.03,
|
||||
"latency_mean": 28,
|
||||
"peak_memory_mb_max": 69000,
|
||||
"peak_memory_mb_mean": 69000
|
||||
"throughput_qps": 0.05,
|
||||
"latency_mean": 20,
|
||||
"peak_memory_mb_max": 55000,
|
||||
"peak_memory_mb_mean": 55000
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -138,10 +138,10 @@
|
||||
"num-input-images": 2,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.10,
|
||||
"latency_mean": 12,
|
||||
"peak_memory_mb_max": 73000,
|
||||
"peak_memory_mb_mean": 73000
|
||||
"throughput_qps": 0.21,
|
||||
"latency_mean": 10,
|
||||
"peak_memory_mb_max": 57000,
|
||||
"peak_memory_mb_mean": 57000
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -156,10 +156,10 @@
|
||||
"num-input-images": 2,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.05,
|
||||
"throughput_qps": 0.08,
|
||||
"latency_mean": 20,
|
||||
"peak_memory_mb_max": 81000,
|
||||
"peak_memory_mb_mean": 81000
|
||||
"peak_memory_mb_max": 66000,
|
||||
"peak_memory_mb_mean": 66000
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -21,10 +21,10 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.05,
|
||||
"latency_mean": 15.0,
|
||||
"peak_memory_mb_max": 72500,
|
||||
"peak_memory_mb_mean": 72500
|
||||
"throughput_qps": 0.1,
|
||||
"latency_mean": 10,
|
||||
"peak_memory_mb_max": 57000,
|
||||
"peak_memory_mb_mean": 57000
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -38,10 +38,10 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.01,
|
||||
"latency_mean": 65.6,
|
||||
"peak_memory_mb_max": 80777,
|
||||
"peak_memory_mb_mean": 80777
|
||||
"throughput_qps": 0.02,
|
||||
"latency_mean": 50,
|
||||
"peak_memory_mb_max": 65000,
|
||||
"peak_memory_mb_mean": 65000
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -72,10 +72,10 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.10,
|
||||
"latency_mean": 7.2,
|
||||
"peak_memory_mb_max": 68100,
|
||||
"peak_memory_mb_mean": 68100
|
||||
"throughput_qps": 0.21,
|
||||
"latency_mean": 10,
|
||||
"peak_memory_mb_max": 56000,
|
||||
"peak_memory_mb_mean": 56000
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -89,10 +89,10 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.03,
|
||||
"latency_mean": 24.0,
|
||||
"peak_memory_mb_max": 68100,
|
||||
"peak_memory_mb_mean": 68100
|
||||
"throughput_qps": 0.06,
|
||||
"latency_mean": 20,
|
||||
"peak_memory_mb_max": 56000,
|
||||
"peak_memory_mb_mean": 56000
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -133,10 +133,10 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.1,
|
||||
"latency_mean": 6.5,
|
||||
"peak_memory_mb_max": 72600,
|
||||
"peak_memory_mb_mean": 72600
|
||||
"throughput_qps": 0.26,
|
||||
"latency_mean": 10,
|
||||
"peak_memory_mb_max": 58000,
|
||||
"peak_memory_mb_mean": 58000
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -150,10 +150,10 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.05,
|
||||
"latency_mean": 16.0,
|
||||
"peak_memory_mb_max": 81000,
|
||||
"peak_memory_mb_mean": 81000
|
||||
"throughput_qps": 0.1,
|
||||
"latency_mean": 10,
|
||||
"peak_memory_mb_max": 67000,
|
||||
"peak_memory_mb_mean": 67000
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -21,10 +21,10 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.02,
|
||||
"latency_mean": 40.0,
|
||||
"peak_memory_mb_max": 70000,
|
||||
"peak_memory_mb_mean": 70000
|
||||
"throughput_qps": 0.05,
|
||||
"latency_mean": 20,
|
||||
"peak_memory_mb_max": 63000,
|
||||
"peak_memory_mb_mean": 63000
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -38,10 +38,10 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.005,
|
||||
"latency_mean": 80.0,
|
||||
"peak_memory_mb_max": 70000,
|
||||
"peak_memory_mb_mean": 70000
|
||||
"throughput_qps": 0.03,
|
||||
"latency_mean": 30,
|
||||
"peak_memory_mb_max": 63000,
|
||||
"peak_memory_mb_mean": 63000
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -21,9 +21,9 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.30,
|
||||
"latency_mean": 3.50,
|
||||
"peak_memory_mb_mean": 67000
|
||||
"throughput_qps": 0.35,
|
||||
"latency_mean": 3,
|
||||
"peak_memory_mb_mean": 56000
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -37,9 +37,9 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.037,
|
||||
"latency_mean": 27.0,
|
||||
"peak_memory_mb_mean": 74000
|
||||
"throughput_qps": 0.03,
|
||||
"latency_mean": 26,
|
||||
"peak_memory_mb_mean": 65000
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -67,9 +67,9 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.30,
|
||||
"latency_mean": 3.50,
|
||||
"peak_memory_mb_mean": 67000
|
||||
"throughput_qps": 0.35,
|
||||
"latency_mean": 3,
|
||||
"peak_memory_mb_mean": 56000
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -83,9 +83,9 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.037,
|
||||
"latency_mean": 27.0,
|
||||
"peak_memory_mb_mean": 74000
|
||||
"throughput_qps": 0.03,
|
||||
"latency_mean": 26,
|
||||
"peak_memory_mb_mean": 65000
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -116,9 +116,9 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.1,
|
||||
"latency_mean": 9.1,
|
||||
"peak_memory_mb_mean": 61000
|
||||
"throughput_qps": 0.11,
|
||||
"latency_mean": 9,
|
||||
"peak_memory_mb_mean": 55000
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -159,9 +159,9 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.1,
|
||||
"latency_mean": 3.50,
|
||||
"peak_memory_mb_mean": 67000
|
||||
"throughput_qps": 0.46,
|
||||
"latency_mean": 3,
|
||||
"peak_memory_mb_mean": 56000
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -175,9 +175,9 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.1,
|
||||
"latency_mean": 6.15,
|
||||
"peak_memory_mb_mean": 74000
|
||||
"throughput_qps": 0.18,
|
||||
"latency_mean": 6,
|
||||
"peak_memory_mb_mean": 67000
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -4,7 +4,9 @@
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen3-TTS-12Hz-1.7B-Base",
|
||||
"stage_overrides": {
|
||||
"1": {"devices": "1"}
|
||||
"1": {
|
||||
"devices": "1"
|
||||
}
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
@@ -15,13 +17,21 @@
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"dataset_path": "linyueqian/seed-tts-eval-subset",
|
||||
"num_prompts": [20],
|
||||
"max_concurrency": [1],
|
||||
"num_prompts": [
|
||||
20
|
||||
],
|
||||
"max_concurrency": [
|
||||
1
|
||||
],
|
||||
"seed_tts_locale": "en",
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"median_audio_ttfp_ms": [350],
|
||||
"median_audio_rtf": [0.25]
|
||||
"median_audio_ttfp_ms": [
|
||||
3000
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.6
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -31,14 +41,34 @@
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"dataset_path": "linyueqian/seed-tts-eval-subset",
|
||||
"num_prompts": [80, 128, 128],
|
||||
"max_concurrency": [8, 16, 64],
|
||||
"num_prompts": [
|
||||
80,
|
||||
128,
|
||||
128
|
||||
],
|
||||
"max_concurrency": [
|
||||
8,
|
||||
16,
|
||||
64
|
||||
],
|
||||
"seed_tts_locale": "en",
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"median_audio_ttfp_ms": [3500, 30000, 7000],
|
||||
"median_audio_rtf": [0.75, 10.0, 1.5],
|
||||
"audio_throughput": [10.0, 1.0, 40.0]
|
||||
"median_audio_ttfp_ms": [
|
||||
1000,
|
||||
3000,
|
||||
14000
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.70,
|
||||
1.15,
|
||||
3.90
|
||||
],
|
||||
"audio_throughput": [
|
||||
12,
|
||||
14,
|
||||
14
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -49,13 +79,19 @@
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"dataset_path": "linyueqian/seed-tts-eval-subset",
|
||||
"num_prompts": [200],
|
||||
"max_concurrency": [4],
|
||||
"num_prompts": [
|
||||
200
|
||||
],
|
||||
"max_concurrency": [
|
||||
4
|
||||
],
|
||||
"seed_tts_locale": "en",
|
||||
"seed_tts_wer_eval": true,
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_audio_rtf": [0.45]
|
||||
"mean_audio_rtf": [
|
||||
0.45
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -65,7 +101,9 @@
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
|
||||
"stage_overrides": {
|
||||
"1": {"devices": "1"}
|
||||
"1": {
|
||||
"devices": "1"
|
||||
}
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
@@ -76,14 +114,26 @@
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
|
||||
"num_prompts": [20],
|
||||
"max_concurrency": [1],
|
||||
"num_prompts": [
|
||||
20
|
||||
],
|
||||
"max_concurrency": [
|
||||
1
|
||||
],
|
||||
"seed_tts_locale": "en",
|
||||
"extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"},
|
||||
"extra_body": {
|
||||
"voice": "Vivian",
|
||||
"language": "English",
|
||||
"task_type": "CustomVoice"
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"median_audio_ttfp_ms": [150],
|
||||
"median_audio_rtf": [0.15]
|
||||
"median_audio_ttfp_ms": [
|
||||
2000
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.40
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -93,15 +143,39 @@
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
|
||||
"num_prompts": [80, 128, 128],
|
||||
"max_concurrency": [8, 16, 64],
|
||||
"num_prompts": [
|
||||
80,
|
||||
128,
|
||||
128
|
||||
],
|
||||
"max_concurrency": [
|
||||
8,
|
||||
16,
|
||||
64
|
||||
],
|
||||
"seed_tts_locale": "en",
|
||||
"extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"},
|
||||
"extra_body": {
|
||||
"voice": "Vivian",
|
||||
"language": "English",
|
||||
"task_type": "CustomVoice"
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"median_audio_ttfp_ms": [1500, 30000, 3000],
|
||||
"median_audio_rtf": [0.30, 10.0, 0.60],
|
||||
"audio_throughput": [30.0, 1.0, 100.0]
|
||||
"median_audio_ttfp_ms": [
|
||||
300,
|
||||
1000,
|
||||
5000
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.25,
|
||||
0.35,
|
||||
1.10
|
||||
],
|
||||
"audio_throughput": [
|
||||
30,
|
||||
40,
|
||||
40
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -112,14 +186,24 @@
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
|
||||
"num_prompts": [200],
|
||||
"max_concurrency": [4],
|
||||
"num_prompts": [
|
||||
200
|
||||
],
|
||||
"max_concurrency": [
|
||||
4
|
||||
],
|
||||
"seed_tts_locale": "en",
|
||||
"extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"},
|
||||
"extra_body": {
|
||||
"voice": "Vivian",
|
||||
"language": "English",
|
||||
"task_type": "CustomVoice"
|
||||
},
|
||||
"seed_tts_wer_eval": true,
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_audio_rtf": [0.35]
|
||||
"mean_audio_rtf": [
|
||||
0.35
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -129,15 +213,29 @@
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
|
||||
"num_prompts": [100],
|
||||
"request_rate": [2.0],
|
||||
"num_prompts": [
|
||||
100
|
||||
],
|
||||
"request_rate": [
|
||||
2.0
|
||||
],
|
||||
"seed_tts_locale": "en",
|
||||
"extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"},
|
||||
"extra_body": {
|
||||
"voice": "Vivian",
|
||||
"language": "English",
|
||||
"task_type": "CustomVoice"
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"median_audio_ttfp_ms": [3000],
|
||||
"median_audio_rtf": [0.25],
|
||||
"audio_throughput": [4.0]
|
||||
"median_audio_ttfp_ms": [
|
||||
100
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.20
|
||||
],
|
||||
"audio_throughput": [
|
||||
10
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -147,14 +245,25 @@
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"dataset_path": "benchmarks/build_dataset/seed_tts_design",
|
||||
"num_prompts": [20],
|
||||
"max_concurrency": [1],
|
||||
"num_prompts": [
|
||||
20
|
||||
],
|
||||
"max_concurrency": [
|
||||
1
|
||||
],
|
||||
"seed_tts_locale": "en",
|
||||
"extra_body": {"task_type": "VoiceDesign", "language": "English"},
|
||||
"extra_body": {
|
||||
"task_type": "VoiceDesign",
|
||||
"language": "English"
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"median_audio_ttfp_ms": [150],
|
||||
"median_audio_rtf": [0.15]
|
||||
"median_audio_ttfp_ms": [
|
||||
100
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.15
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -164,15 +273,38 @@
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"dataset_path": "benchmarks/build_dataset/seed_tts_design",
|
||||
"num_prompts": [80, 128, 128],
|
||||
"max_concurrency": [8, 16, 64],
|
||||
"num_prompts": [
|
||||
80,
|
||||
128,
|
||||
128
|
||||
],
|
||||
"max_concurrency": [
|
||||
8,
|
||||
16,
|
||||
64
|
||||
],
|
||||
"seed_tts_locale": "en",
|
||||
"extra_body": {"task_type": "VoiceDesign", "language": "English"},
|
||||
"extra_body": {
|
||||
"task_type": "VoiceDesign",
|
||||
"language": "English"
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"median_audio_ttfp_ms": [1500, 30000, 3500],
|
||||
"median_audio_rtf": [0.35, 10.0, 0.70],
|
||||
"audio_throughput": [25.0, 1.0, 80.0]
|
||||
"median_audio_ttfp_ms": [
|
||||
100,
|
||||
1000,
|
||||
5000
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.25,
|
||||
0.40,
|
||||
1.20
|
||||
],
|
||||
"audio_throughput": [
|
||||
30,
|
||||
40,
|
||||
40
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -182,15 +314,28 @@
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"dataset_path": "benchmarks/build_dataset/seed_tts_design",
|
||||
"num_prompts": [100],
|
||||
"request_rate": [2.0],
|
||||
"num_prompts": [
|
||||
100
|
||||
],
|
||||
"request_rate": [
|
||||
2.0
|
||||
],
|
||||
"seed_tts_locale": "en",
|
||||
"extra_body": {"task_type": "VoiceDesign", "language": "English"},
|
||||
"extra_body": {
|
||||
"task_type": "VoiceDesign",
|
||||
"language": "English"
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"median_audio_ttfp_ms": [3500],
|
||||
"median_audio_rtf": [0.30],
|
||||
"audio_throughput": [4.0]
|
||||
"median_audio_ttfp_ms": [
|
||||
100
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.20
|
||||
],
|
||||
"audio_throughput": [
|
||||
9
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -30,9 +30,9 @@
|
||||
}
|
||||
],
|
||||
"baseline": {
|
||||
"throughput_qps": 0.034,
|
||||
"latency_mean": 26.0,
|
||||
"peak_memory_mb_mean": 80000
|
||||
"throughput_qps": 0.04,
|
||||
"latency_mean": 30,
|
||||
"peak_memory_mb_mean": 75000
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -72,9 +72,9 @@
|
||||
}
|
||||
],
|
||||
"baseline": {
|
||||
"throughput_qps": 0.042,
|
||||
"latency_mean": 21.6,
|
||||
"peak_memory_mb_mean": 55300
|
||||
"throughput_qps": 0.06,
|
||||
"latency_mean": 20,
|
||||
"peak_memory_mb_mean": 48000
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -97,9 +97,9 @@
|
||||
}
|
||||
],
|
||||
"baseline": {
|
||||
"throughput_qps": 0.0085,
|
||||
"latency_mean": 101.6,
|
||||
"peak_memory_mb_mean": 65200
|
||||
"throughput_qps": 0.01,
|
||||
"latency_mean": 90,
|
||||
"peak_memory_mb_mean": 56000
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_hunyuan import (
|
||||
DistributedAutoencoderKLHunyuan,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _DummyDistributedAutoencoderKLHunyuan(DistributedAutoencoderKLHunyuan):
|
||||
def __init__(self):
|
||||
torch.nn.Module.__init__(self)
|
||||
self.tile_latent_min_size = 8
|
||||
self.tile_sample_min_size = 8
|
||||
self.tile_overlap_factor = 0.25
|
||||
self.use_spatial_tiling = False
|
||||
|
||||
@property
|
||||
def dtype(self) -> torch.dtype:
|
||||
return torch.float32
|
||||
|
||||
def decoder(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return x + 10
|
||||
|
||||
def encoder(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return x + 20
|
||||
|
||||
def blend_v(self, _a: torch.Tensor, b: torch.Tensor, _blend_extent: int) -> torch.Tensor:
|
||||
return b
|
||||
|
||||
def blend_h(self, _a: torch.Tensor, b: torch.Tensor, _blend_extent: int) -> torch.Tensor:
|
||||
return b
|
||||
|
||||
|
||||
def test_hunyuan_vae_use_tiling_aliases_spatial_tiling():
|
||||
# Verify use_tiling property maps to use_spatial_tiling.
|
||||
vae = _DummyDistributedAutoencoderKLHunyuan()
|
||||
|
||||
assert not vae.use_tiling
|
||||
|
||||
vae.use_tiling = True
|
||||
|
||||
assert vae.use_spatial_tiling
|
||||
|
||||
|
||||
def test_hunyuan_vae_decode_tiles_round_trip():
|
||||
# Validate decode tile split/exec/merge returns expected reconstructed tensor.
|
||||
vae = _DummyDistributedAutoencoderKLHunyuan()
|
||||
z = torch.arange(144, dtype=torch.float32).reshape(1, 1, 1, 12, 12)
|
||||
|
||||
tile_tasks, grid_spec = vae.tile_split(z)
|
||||
decoded_tiles = {task.grid_coord: vae.tile_exec(task) for task in tile_tasks}
|
||||
output = vae.tile_merge(decoded_tiles, grid_spec)
|
||||
|
||||
assert grid_spec.split_dims == (3, 4)
|
||||
assert grid_spec.grid_shape == (2, 2)
|
||||
assert grid_spec.tile_spec == {"blend_extent": 2, "row_limit": 6}
|
||||
assert len(tile_tasks) == 4
|
||||
assert torch.equal(output, z + 10)
|
||||
|
||||
|
||||
def test_hunyuan_vae_encode_tiles_round_trip():
|
||||
# Validate encode tile split/exec/merge returns expected latent tensor.
|
||||
vae = _DummyDistributedAutoencoderKLHunyuan()
|
||||
x = torch.arange(144, dtype=torch.float32).reshape(1, 1, 1, 12, 12)
|
||||
|
||||
tile_tasks, grid_spec = vae.encode_tile_split(x)
|
||||
encoded_tiles = {task.grid_coord: vae.encode_tile_exec(task) for task in tile_tasks}
|
||||
output = vae.encode_tile_merge(encoded_tiles, grid_spec)
|
||||
|
||||
assert grid_spec.split_dims == (3, 4)
|
||||
assert grid_spec.grid_shape == (2, 2)
|
||||
assert grid_spec.tile_spec == {"blend_extent": 2, "row_limit": 6}
|
||||
assert len(tile_tasks) == 4
|
||||
assert torch.equal(output, x + 20)
|
||||
@@ -0,0 +1,145 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.distributed.autoencoders.distributed_vae_executor import (
|
||||
DistributedOperator,
|
||||
DistributedVaeMixin,
|
||||
GridSpec,
|
||||
TileTask,
|
||||
)
|
||||
from vllm_omni.diffusion.models.hunyuan_image3.autoencoder import AutoencoderKLConv3D
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class DistributedAutoencoderKLHunyuan(AutoencoderKLConv3D, DistributedVaeMixin):
|
||||
@classmethod
|
||||
def from_config(cls, config: Any, **kwargs: Any):
|
||||
model = super().from_config(config, **kwargs)
|
||||
model.init_distributed()
|
||||
return model
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(cls, *args: Any, **kwargs: Any):
|
||||
model = super().from_pretrained(*args, **kwargs)
|
||||
model.init_distributed()
|
||||
return model
|
||||
|
||||
@property
|
||||
def use_tiling(self) -> bool:
|
||||
return self.use_spatial_tiling
|
||||
|
||||
@use_tiling.setter
|
||||
def use_tiling(self, use_tiling: bool) -> None:
|
||||
self.use_spatial_tiling = use_tiling
|
||||
|
||||
def tile_split(self, z: torch.Tensor) -> tuple[list[TileTask], GridSpec]:
|
||||
_, _, _, height, width = z.shape
|
||||
overlap_size = int(self.tile_latent_min_size * (1 - self.tile_overlap_factor))
|
||||
blend_extent = int(self.tile_sample_min_size * self.tile_overlap_factor)
|
||||
row_limit = int(self.tile_sample_min_size - blend_extent)
|
||||
|
||||
tiletask_list = []
|
||||
for i in range(0, height, overlap_size):
|
||||
for j in range(0, width, overlap_size):
|
||||
tile = z[:, :, :, i : i + self.tile_latent_min_size, j : j + self.tile_latent_min_size]
|
||||
tiletask_list.append(
|
||||
TileTask(
|
||||
len(tiletask_list),
|
||||
(i // overlap_size, j // overlap_size),
|
||||
tile,
|
||||
workload=tile.shape[3] * tile.shape[4],
|
||||
)
|
||||
)
|
||||
|
||||
grid_spec = GridSpec(
|
||||
split_dims=(3, 4),
|
||||
grid_shape=(tiletask_list[-1].grid_coord[0] + 1, tiletask_list[-1].grid_coord[1] + 1),
|
||||
tile_spec={"blend_extent": blend_extent, "row_limit": row_limit},
|
||||
output_dtype=self.dtype,
|
||||
)
|
||||
return tiletask_list, grid_spec
|
||||
|
||||
def tile_exec(self, task: TileTask) -> torch.Tensor:
|
||||
return self.decoder(task.tensor)
|
||||
|
||||
def tile_merge(self, coord_tensor_map: dict[tuple[int, ...], torch.Tensor], grid_spec: GridSpec) -> torch.Tensor:
|
||||
grid_h, grid_w = grid_spec.grid_shape
|
||||
result_rows = []
|
||||
for i in range(grid_h):
|
||||
result_row = []
|
||||
for j in range(grid_w):
|
||||
tile = coord_tensor_map[(i, j)]
|
||||
if i > 0:
|
||||
tile = self.blend_v(coord_tensor_map[(i - 1, j)], tile, grid_spec.tile_spec["blend_extent"])
|
||||
if j > 0:
|
||||
tile = self.blend_h(coord_tensor_map[(i, j - 1)], tile, grid_spec.tile_spec["blend_extent"])
|
||||
result_row.append(tile[:, :, :, : grid_spec.tile_spec["row_limit"], : grid_spec.tile_spec["row_limit"]])
|
||||
result_rows.append(torch.cat(result_row, dim=-1))
|
||||
return torch.cat(result_rows, dim=-2)
|
||||
|
||||
def encode_tile_split(self, x: torch.Tensor) -> tuple[list[TileTask], GridSpec]:
|
||||
_, _, _, height, width = x.shape
|
||||
overlap_size = int(self.tile_sample_min_size * (1 - self.tile_overlap_factor))
|
||||
blend_extent = int(self.tile_latent_min_size * self.tile_overlap_factor)
|
||||
row_limit = int(self.tile_latent_min_size - blend_extent)
|
||||
|
||||
tiletask_list = []
|
||||
for i in range(0, height, overlap_size):
|
||||
for j in range(0, width, overlap_size):
|
||||
tile = x[:, :, :, i : i + self.tile_sample_min_size, j : j + self.tile_sample_min_size]
|
||||
tiletask_list.append(
|
||||
TileTask(
|
||||
len(tiletask_list),
|
||||
(i // overlap_size, j // overlap_size),
|
||||
tile,
|
||||
workload=tile.shape[3] * tile.shape[4],
|
||||
)
|
||||
)
|
||||
|
||||
grid_spec = GridSpec(
|
||||
split_dims=(3, 4),
|
||||
grid_shape=(tiletask_list[-1].grid_coord[0] + 1, tiletask_list[-1].grid_coord[1] + 1),
|
||||
tile_spec={"blend_extent": blend_extent, "row_limit": row_limit},
|
||||
output_dtype=self.dtype,
|
||||
)
|
||||
return tiletask_list, grid_spec
|
||||
|
||||
def encode_tile_exec(self, task: TileTask) -> torch.Tensor:
|
||||
return self.encoder(task.tensor)
|
||||
|
||||
def encode_tile_merge(
|
||||
self, coord_tensor_map: dict[tuple[int, ...], torch.Tensor], grid_spec: GridSpec
|
||||
) -> torch.Tensor:
|
||||
return self.tile_merge(coord_tensor_map, grid_spec)
|
||||
|
||||
def spatial_tiled_encode(self, x: torch.Tensor):
|
||||
if not self.is_distributed_enabled():
|
||||
return super().spatial_tiled_encode(x)
|
||||
|
||||
logger.debug("Encode running with distributed executor")
|
||||
return self.distributed_executor.execute(
|
||||
x,
|
||||
DistributedOperator(
|
||||
split=self.encode_tile_split,
|
||||
exec=self.encode_tile_exec,
|
||||
merge=self.encode_tile_merge,
|
||||
),
|
||||
broadcast_result=True,
|
||||
)
|
||||
|
||||
def spatial_tiled_decode(self, z: torch.Tensor):
|
||||
if not self.is_distributed_enabled():
|
||||
return super().spatial_tiled_decode(z)
|
||||
|
||||
logger.debug("Decode running with distributed executor")
|
||||
return self.distributed_executor.execute(
|
||||
z,
|
||||
DistributedOperator(split=self.tile_split, exec=self.tile_exec, merge=self.tile_merge),
|
||||
broadcast_result=True,
|
||||
)
|
||||
@@ -30,7 +30,6 @@ from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.inputs.data import OmniTextPrompt
|
||||
from vllm_omni.model_executor.models.hunyuan_image3.siglip2 import Siglip2VisionTransformer
|
||||
|
||||
from .autoencoder import AutoencoderKLConv3D
|
||||
from .hunyuan_image3_tokenizer import TokenizerWrapper
|
||||
from .hunyuan_image3_transformer import (
|
||||
CausalMMOutputWithPast,
|
||||
@@ -343,7 +342,13 @@ class HunyuanImage3Pipeline(
|
||||
quant_config = od_config.quantization_config
|
||||
self.model = HunyuanImage3Model(self.hf_config, quant_config=quant_config)
|
||||
self.transformer = self.model
|
||||
self.vae = AutoencoderKLConv3D.from_config(self.hf_config.vae)
|
||||
# Lazy import to break circular dependency:
|
||||
# autoencoder_kl_hunyuan -> hunyuan_image3/__init__ -> pipeline_hunyuan_image3 -> autoencoder_kl_hunyuan
|
||||
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_hunyuan import ( # noqa: PLC0415
|
||||
DistributedAutoencoderKLHunyuan,
|
||||
)
|
||||
|
||||
self.vae = DistributedAutoencoderKLHunyuan.from_config(self.hf_config.vae)
|
||||
self.vae.use_spatial_tiling = self.od_config.vae_use_tiling
|
||||
self._pipeline = None
|
||||
self._tkwrapper = TokenizerWrapper(od_config.model)
|
||||
|
||||
Reference in new issue
Block a user