[WIP] Optimize Helios accuracy-maintained path

Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>
This commit is contained in:
hsliuustc0106 committed 2026-05-21 16:34:09 +00:00
1 parent a3d4ed809d
commit 7bc59bd939
6 files changed
+728 -20

No files matched your search

+1 -1
View File
@@ -278,7 +278,7 @@ async def async_request_v1_videos(
return output
# invoke a poll request (GET /v1/videos/{video_id})
poll_interval = 2.0 # Unit(s)
poll_interval = 0.2 # Unit(s)
timeout_seconds = 600.0
deadline = time.perf_counter() + timeout_seconds
job_url = f"{input.api_url}/{job_id}"
@@ -68,6 +68,7 @@ import glob
import json
import logging
import os
import pathlib
import random
import tempfile
import time
@@ -598,6 +599,13 @@ class RandomDataset(BaseDataset):
if self.enable_negative_prompt:
extra_body["negative_prompt"] = f"Negative prompt {idx} for benchmarking diffusion models"
request_fields = {
"width",
"height",
"num_frames",
"num_inference_steps",
"fps",
}
params = {
"width": self.args.width,
"height": self.args.height,
@@ -607,9 +615,11 @@ class RandomDataset(BaseDataset):
}
if self._sampled_requests:
profile = self._sampled_requests[idx]
params.update(profile)
params.update({key: value for key, value in profile.items() if key in request_fields})
extra_body.update({key: value for key, value in profile.items() if key not in request_fields})
prompt = self.args.random_prompt or f"Random prompt {idx} for benchmarking diffusion models"
return RequestFuncInput(
prompt=f"Random prompt {idx} for benchmarking diffusion models",
prompt=prompt,
api_url=self.api_url,
model=self.model,
seed=self.args.seed,
@@ -869,6 +879,35 @@ def calculate_metrics(
return metrics
def _response_suffix(endpoint: str) -> str:
if endpoint == "/v1/videos":
return ".mp4"
return ".json"
def _save_success_responses(
outputs: list[RequestFuncOutput],
requests_list: list[RequestFuncInput],
output_dir: str,
endpoint: str,
) -> list[str]:
path = pathlib.Path(output_dir).expanduser().resolve()
path.mkdir(parents=True, exist_ok=True)
saved_paths: list[str] = []
suffix = _response_suffix(endpoint)
for index, (request, output) in enumerate(zip(requests_list, outputs, strict=False)):
if not output.success:
continue
item_path = path / f"{index:05d}_{request.request_id}{suffix}"
if isinstance(output.response_body, bytes):
item_path.write_bytes(output.response_body)
else:
item_path.write_text(json.dumps(output.response_body, indent=2, sort_keys=True), encoding="utf-8")
saved_paths.append(str(item_path))
return saved_paths
def wait_for_service(base_url: str, timeout: int = 120) -> None:
print(f"Waiting for service at {base_url}...")
start_time = time.time()
@@ -998,6 +1037,13 @@ async def benchmark(args):
metrics["model"] = args.model
metrics["dataset"] = args.dataset
metrics["task"] = args.task
if args.save_response_dir:
metrics["saved_responses"] = _save_success_responses(
outputs,
requests_list,
args.save_response_dir,
args.endpoint,
)
print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="="))
@@ -1154,6 +1200,12 @@ if __name__ == "__main__":
)
parser.add_argument("--fps", type=int, default=None, help="FPS (for video).")
parser.add_argument("--output-file", type=str, default=None, help="Output JSON file for metrics.")
parser.add_argument(
"--save-response-dir",
type=str,
default=None,
help="Directory to save successful raw responses. Video responses are saved as MP4 files.",
)
parser.add_argument(
"--slo",
action="store_true",
@@ -1189,6 +1241,12 @@ if __name__ == "__main__":
'{"width":768,"height":768,"num_inference_steps":20,"weight":0.85}]'
),
)
parser.add_argument(
"--random-prompt",
type=str,
default=None,
help="Prompt to use for every synthetic random request. Defaults to generated placeholder prompts.",
)
parser.add_argument(
"--num-input-images",
type=int,
@@ -0,0 +1,222 @@
# Helios-Distilled Serving Performance Dashboard
This document describes how to deploy and benchmark **BestWishYsh/Helios-Distilled** using vLLM-Omni. It follows the same serving benchmark structure as the Qwen-Image and Wan2.2 dashboards.
---
# 1. Overview
Helios-Distilled is a text-to-video diffusion model. Helios uses an autoregressive chunking pattern that generates 33 frames per chunk. For stable comparisons, choose `num_frames` as a multiple of 33, such as 99.
This document covers:
* Service launch configuration
* Benchmark scripts and usage
* Dataset and workload settings
* Performance and accuracy comparison methodology
* Reproducibility guidelines
---
# 2. Test Environment
| Component | Specification |
|------------|----------------|
| GPU | NVIDIA H100 for CI accuracy/perf gate; local bring-up also used L20X |
| Diffusion Attention Backend | FlashAttention |
| Model | `BestWishYsh/Helios-Distilled` |
---
# 3. Service Launch Configuration
## 3.1 Basic Serving Command
```bash
vllm serve BestWishYsh/Helios-Distilled --omni \
--port 8091
```
## 3.2 Key Parameters
| Parameter | Description |
| --- | --- |
| `--port` | Serving port |
| `--max-num-seqs` | Maximum number of active sequences |
| `--enable-diffusion-pipeline-profiler` | Optional stage-level profiling |
| `--profiler-config` | Optional torch profiler configuration |
Record these parameters when reporting performance results.
---
# 4. Benchmark Scripts
## 4.1 Online Serving Benchmark
Use the shared diffusion serving benchmark, as with Qwen-Image and Wan2.2:
```bash
python benchmarks/diffusion/diffusion_benchmark_serving.py \
--base-url http://localhost:8091 \
--endpoint /v1/videos \
--model BestWishYsh/Helios-Distilled \
--dataset random \
--task t2v \
--num-prompts 1 \
--max-concurrency 1 \
--disable-tqdm \
--random-request-config '[
{
"width": 640,
"height": 384,
"num_inference_steps": 50,
"num_frames": 99,
"fps": 16,
"guidance_scale": 1.0,
"is_enable_stage2": true,
"pyramid_num_stages": 3,
"pyramid_num_inference_steps_list": [1, 1, 1],
"is_amplify_first_chunk": false,
"weight": 1
}
]'
```
`--random-request-config` forwards standard fields such as `width`, `height`, `num_frames`, `fps`, and `num_inference_steps` as first-class request fields. Model-specific fields are forwarded in the request body, which is required for Helios stage-2 settings.
## 4.2 Warmed Serving Benchmark
For local bring-up, use the same shared serving benchmark with one warmup request and one measured request:
```bash
python benchmarks/diffusion/diffusion_benchmark_serving.py \
--base-url http://localhost:8091 \
--endpoint /v1/videos \
--model BestWishYsh/Helios-Distilled \
--dataset random \
--task t2v \
--num-prompts 1 \
--max-concurrency 1 \
--warmup-requests 1 \
--warmup-num-inference-steps 1 \
--random-prompt "A cat wearing sunglasses dances on a beach at sunset, cinematic lighting." \
--seed 42 \
--disable-tqdm \
--output-file metrics.json \
--save-response-dir responses \
--random-request-config '[
{
"width": 640,
"height": 384,
"num_inference_steps": 50,
"num_frames": 99,
"fps": 16,
"guidance_scale": 1.0,
"is_enable_stage2": true,
"pyramid_num_stages": 3,
"pyramid_num_inference_steps_list": [1, 1, 1],
"is_amplify_first_chunk": false,
"weight": 1
}
]'
```
## 4.3 PR-vs-Main Accuracy and Performance Comparison
The PR-vs-main accuracy/performance gate lives in the Helios accuracy pytest. It checks out `origin/main`, starts a server for each checkout, and drives both with `diffusion_benchmark_serving.py`:
```bash
pytest -s -v tests/e2e/accuracy/test_helios_distilled.py::test_helios_distilled_pr_matches_main_acc_perf \
-m 'full_model and benchmark' \
--run-level 'full_model'
```
---
# 5. Dataset & Workload Settings
## 5.1 Recommended Evaluation Configurations
### Dataset A (99 frames, 384x640)
* Dataset: `random`
* Task: t2v
* Concurrency: 1
* Frame count: 99 (`33 x 3`)
* FPS: 16
* Stage-2 steps: `[1, 1, 1]`
```json
[
{
"width": 640,
"height": 384,
"num_inference_steps": 50,
"num_frames": 99,
"fps": 16,
"guidance_scale": 1.0,
"is_enable_stage2": true,
"pyramid_num_stages": 3,
"pyramid_num_inference_steps_list": [1, 1, 1],
"is_amplify_first_chunk": false,
"weight": 1
}
]
```
---
# 6. Performance Metrics
| Metric | Description | Unit |
| --- | --- | --- |
| Mean Latency | Mean request latency | seconds |
| P99 Latency | P99 request latency | seconds |
| Generated FPS | `num_frames / measured_time_s` | frames/second |
| Candidate/reference latency ratio | PR latency divided by reference latency | ratio |
---
# 7. Accuracy Metrics
The PR-vs-main gate compares encoded video frames extracted from MP4 outputs:
| Metric | Threshold | Direction |
| --- | --- | --- |
| PSNR | `>= 35 dB` | Higher is better |
| MAE | `<= 3.0 / 255` | Lower is better |
| Candidate/reference latency ratio | `<= 1.03` | Lower is better |
Local L20X bring-up for this PR produced:
| Variant | Measured Latency (ms) | Generated FPS |
| --- | --- | --- |
| `origin/main` | 5317.95 | 18.62 |
| Candidate PR | 5283.27 | 18.74 |
Encoded-video similarity:
| Metric | Value |
| --- | --- |
| PSNR | 38.66 dB |
| MAE | 1.89 / 255 |
| Cosine similarity | 0.99978 |
---
# 8. Reproducibility Checklist
To ensure consistent and comparable benchmark results:
* Record GPU type
* Record vLLM and vLLM-Omni versions
* Record benchmark parameters: resolution, frame count, FPS, seed, concurrency, and stage-2 steps
* Use frame counts that are multiples of 33
* Keep prompt, seed, resolution, and sampling parameters identical when comparing PR vs main
* Ensure no background workload on GPUs during testing
* Use the same MP4 export and frame extraction path for both variants
---
This document serves as the Helios-Distilled serving performance reference under vLLM-Omni.