[WIP] Optimize Helios accuracy-maintained path
Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
a3d4ed809d
commit
7bc59bd939
6 files changed
+728
-20
No files matched your search
@@ -278,7 +278,7 @@ async def async_request_v1_videos(
|
||||
return output
|
||||
|
||||
# invoke a poll request (GET /v1/videos/{video_id})
|
||||
poll_interval = 2.0 # Unit(s)
|
||||
poll_interval = 0.2 # Unit(s)
|
||||
timeout_seconds = 600.0
|
||||
deadline = time.perf_counter() + timeout_seconds
|
||||
job_url = f"{input.api_url}/{job_id}"
|
||||
|
||||
@@ -68,6 +68,7 @@ import glob
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import random
|
||||
import tempfile
|
||||
import time
|
||||
@@ -598,6 +599,13 @@ class RandomDataset(BaseDataset):
|
||||
if self.enable_negative_prompt:
|
||||
extra_body["negative_prompt"] = f"Negative prompt {idx} for benchmarking diffusion models"
|
||||
|
||||
request_fields = {
|
||||
"width",
|
||||
"height",
|
||||
"num_frames",
|
||||
"num_inference_steps",
|
||||
"fps",
|
||||
}
|
||||
params = {
|
||||
"width": self.args.width,
|
||||
"height": self.args.height,
|
||||
@@ -607,9 +615,11 @@ class RandomDataset(BaseDataset):
|
||||
}
|
||||
if self._sampled_requests:
|
||||
profile = self._sampled_requests[idx]
|
||||
params.update(profile)
|
||||
params.update({key: value for key, value in profile.items() if key in request_fields})
|
||||
extra_body.update({key: value for key, value in profile.items() if key not in request_fields})
|
||||
prompt = self.args.random_prompt or f"Random prompt {idx} for benchmarking diffusion models"
|
||||
return RequestFuncInput(
|
||||
prompt=f"Random prompt {idx} for benchmarking diffusion models",
|
||||
prompt=prompt,
|
||||
api_url=self.api_url,
|
||||
model=self.model,
|
||||
seed=self.args.seed,
|
||||
@@ -869,6 +879,35 @@ def calculate_metrics(
|
||||
return metrics
|
||||
|
||||
|
||||
def _response_suffix(endpoint: str) -> str:
|
||||
if endpoint == "/v1/videos":
|
||||
return ".mp4"
|
||||
return ".json"
|
||||
|
||||
|
||||
def _save_success_responses(
|
||||
outputs: list[RequestFuncOutput],
|
||||
requests_list: list[RequestFuncInput],
|
||||
output_dir: str,
|
||||
endpoint: str,
|
||||
) -> list[str]:
|
||||
path = pathlib.Path(output_dir).expanduser().resolve()
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
saved_paths: list[str] = []
|
||||
suffix = _response_suffix(endpoint)
|
||||
|
||||
for index, (request, output) in enumerate(zip(requests_list, outputs, strict=False)):
|
||||
if not output.success:
|
||||
continue
|
||||
item_path = path / f"{index:05d}_{request.request_id}{suffix}"
|
||||
if isinstance(output.response_body, bytes):
|
||||
item_path.write_bytes(output.response_body)
|
||||
else:
|
||||
item_path.write_text(json.dumps(output.response_body, indent=2, sort_keys=True), encoding="utf-8")
|
||||
saved_paths.append(str(item_path))
|
||||
return saved_paths
|
||||
|
||||
|
||||
def wait_for_service(base_url: str, timeout: int = 120) -> None:
|
||||
print(f"Waiting for service at {base_url}...")
|
||||
start_time = time.time()
|
||||
@@ -998,6 +1037,13 @@ async def benchmark(args):
|
||||
metrics["model"] = args.model
|
||||
metrics["dataset"] = args.dataset
|
||||
metrics["task"] = args.task
|
||||
if args.save_response_dir:
|
||||
metrics["saved_responses"] = _save_success_responses(
|
||||
outputs,
|
||||
requests_list,
|
||||
args.save_response_dir,
|
||||
args.endpoint,
|
||||
)
|
||||
|
||||
print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="="))
|
||||
|
||||
@@ -1154,6 +1200,12 @@ if __name__ == "__main__":
|
||||
)
|
||||
parser.add_argument("--fps", type=int, default=None, help="FPS (for video).")
|
||||
parser.add_argument("--output-file", type=str, default=None, help="Output JSON file for metrics.")
|
||||
parser.add_argument(
|
||||
"--save-response-dir",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Directory to save successful raw responses. Video responses are saved as MP4 files.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--slo",
|
||||
action="store_true",
|
||||
@@ -1189,6 +1241,12 @@ if __name__ == "__main__":
|
||||
'{"width":768,"height":768,"num_inference_steps":20,"weight":0.85}]'
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--random-prompt",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Prompt to use for every synthetic random request. Defaults to generated placeholder prompts.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-input-images",
|
||||
type=int,
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
# Helios-Distilled Serving Performance Dashboard
|
||||
|
||||
This document describes how to deploy and benchmark **BestWishYsh/Helios-Distilled** using vLLM-Omni. It follows the same serving benchmark structure as the Qwen-Image and Wan2.2 dashboards.
|
||||
|
||||
---
|
||||
|
||||
# 1. Overview
|
||||
|
||||
Helios-Distilled is a text-to-video diffusion model. Helios uses an autoregressive chunking pattern that generates 33 frames per chunk. For stable comparisons, choose `num_frames` as a multiple of 33, such as 99.
|
||||
|
||||
This document covers:
|
||||
|
||||
* Service launch configuration
|
||||
* Benchmark scripts and usage
|
||||
* Dataset and workload settings
|
||||
* Performance and accuracy comparison methodology
|
||||
* Reproducibility guidelines
|
||||
|
||||
---
|
||||
|
||||
# 2. Test Environment
|
||||
|
||||
| Component | Specification |
|
||||
|------------|----------------|
|
||||
| GPU | NVIDIA H100 for CI accuracy/perf gate; local bring-up also used L20X |
|
||||
| Diffusion Attention Backend | FlashAttention |
|
||||
| Model | `BestWishYsh/Helios-Distilled` |
|
||||
|
||||
---
|
||||
|
||||
# 3. Service Launch Configuration
|
||||
|
||||
## 3.1 Basic Serving Command
|
||||
|
||||
```bash
|
||||
vllm serve BestWishYsh/Helios-Distilled --omni \
|
||||
--port 8091
|
||||
```
|
||||
|
||||
## 3.2 Key Parameters
|
||||
|
||||
| Parameter | Description |
|
||||
| --- | --- |
|
||||
| `--port` | Serving port |
|
||||
| `--max-num-seqs` | Maximum number of active sequences |
|
||||
| `--enable-diffusion-pipeline-profiler` | Optional stage-level profiling |
|
||||
| `--profiler-config` | Optional torch profiler configuration |
|
||||
|
||||
Record these parameters when reporting performance results.
|
||||
|
||||
---
|
||||
|
||||
# 4. Benchmark Scripts
|
||||
|
||||
## 4.1 Online Serving Benchmark
|
||||
|
||||
Use the shared diffusion serving benchmark, as with Qwen-Image and Wan2.2:
|
||||
|
||||
```bash
|
||||
python benchmarks/diffusion/diffusion_benchmark_serving.py \
|
||||
--base-url http://localhost:8091 \
|
||||
--endpoint /v1/videos \
|
||||
--model BestWishYsh/Helios-Distilled \
|
||||
--dataset random \
|
||||
--task t2v \
|
||||
--num-prompts 1 \
|
||||
--max-concurrency 1 \
|
||||
--disable-tqdm \
|
||||
--random-request-config '[
|
||||
{
|
||||
"width": 640,
|
||||
"height": 384,
|
||||
"num_inference_steps": 50,
|
||||
"num_frames": 99,
|
||||
"fps": 16,
|
||||
"guidance_scale": 1.0,
|
||||
"is_enable_stage2": true,
|
||||
"pyramid_num_stages": 3,
|
||||
"pyramid_num_inference_steps_list": [1, 1, 1],
|
||||
"is_amplify_first_chunk": false,
|
||||
"weight": 1
|
||||
}
|
||||
]'
|
||||
```
|
||||
|
||||
`--random-request-config` forwards standard fields such as `width`, `height`, `num_frames`, `fps`, and `num_inference_steps` as first-class request fields. Model-specific fields are forwarded in the request body, which is required for Helios stage-2 settings.
|
||||
|
||||
## 4.2 Warmed Serving Benchmark
|
||||
|
||||
For local bring-up, use the same shared serving benchmark with one warmup request and one measured request:
|
||||
|
||||
```bash
|
||||
python benchmarks/diffusion/diffusion_benchmark_serving.py \
|
||||
--base-url http://localhost:8091 \
|
||||
--endpoint /v1/videos \
|
||||
--model BestWishYsh/Helios-Distilled \
|
||||
--dataset random \
|
||||
--task t2v \
|
||||
--num-prompts 1 \
|
||||
--max-concurrency 1 \
|
||||
--warmup-requests 1 \
|
||||
--warmup-num-inference-steps 1 \
|
||||
--random-prompt "A cat wearing sunglasses dances on a beach at sunset, cinematic lighting." \
|
||||
--seed 42 \
|
||||
--disable-tqdm \
|
||||
--output-file metrics.json \
|
||||
--save-response-dir responses \
|
||||
--random-request-config '[
|
||||
{
|
||||
"width": 640,
|
||||
"height": 384,
|
||||
"num_inference_steps": 50,
|
||||
"num_frames": 99,
|
||||
"fps": 16,
|
||||
"guidance_scale": 1.0,
|
||||
"is_enable_stage2": true,
|
||||
"pyramid_num_stages": 3,
|
||||
"pyramid_num_inference_steps_list": [1, 1, 1],
|
||||
"is_amplify_first_chunk": false,
|
||||
"weight": 1
|
||||
}
|
||||
]'
|
||||
```
|
||||
|
||||
## 4.3 PR-vs-Main Accuracy and Performance Comparison
|
||||
|
||||
The PR-vs-main accuracy/performance gate lives in the Helios accuracy pytest. It checks out `origin/main`, starts a server for each checkout, and drives both with `diffusion_benchmark_serving.py`:
|
||||
|
||||
```bash
|
||||
pytest -s -v tests/e2e/accuracy/test_helios_distilled.py::test_helios_distilled_pr_matches_main_acc_perf \
|
||||
-m 'full_model and benchmark' \
|
||||
--run-level 'full_model'
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
# 5. Dataset & Workload Settings
|
||||
|
||||
## 5.1 Recommended Evaluation Configurations
|
||||
|
||||
### Dataset A (99 frames, 384x640)
|
||||
|
||||
* Dataset: `random`
|
||||
* Task: t2v
|
||||
* Concurrency: 1
|
||||
* Frame count: 99 (`33 x 3`)
|
||||
* FPS: 16
|
||||
* Stage-2 steps: `[1, 1, 1]`
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"width": 640,
|
||||
"height": 384,
|
||||
"num_inference_steps": 50,
|
||||
"num_frames": 99,
|
||||
"fps": 16,
|
||||
"guidance_scale": 1.0,
|
||||
"is_enable_stage2": true,
|
||||
"pyramid_num_stages": 3,
|
||||
"pyramid_num_inference_steps_list": [1, 1, 1],
|
||||
"is_amplify_first_chunk": false,
|
||||
"weight": 1
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
# 6. Performance Metrics
|
||||
|
||||
| Metric | Description | Unit |
|
||||
| --- | --- | --- |
|
||||
| Mean Latency | Mean request latency | seconds |
|
||||
| P99 Latency | P99 request latency | seconds |
|
||||
| Generated FPS | `num_frames / measured_time_s` | frames/second |
|
||||
| Candidate/reference latency ratio | PR latency divided by reference latency | ratio |
|
||||
|
||||
---
|
||||
|
||||
# 7. Accuracy Metrics
|
||||
|
||||
The PR-vs-main gate compares encoded video frames extracted from MP4 outputs:
|
||||
|
||||
| Metric | Threshold | Direction |
|
||||
| --- | --- | --- |
|
||||
| PSNR | `>= 35 dB` | Higher is better |
|
||||
| MAE | `<= 3.0 / 255` | Lower is better |
|
||||
| Candidate/reference latency ratio | `<= 1.03` | Lower is better |
|
||||
|
||||
Local L20X bring-up for this PR produced:
|
||||
|
||||
| Variant | Measured Latency (ms) | Generated FPS |
|
||||
| --- | --- | --- |
|
||||
| `origin/main` | 5317.95 | 18.62 |
|
||||
| Candidate PR | 5283.27 | 18.74 |
|
||||
|
||||
Encoded-video similarity:
|
||||
|
||||
| Metric | Value |
|
||||
| --- | --- |
|
||||
| PSNR | 38.66 dB |
|
||||
| MAE | 1.89 / 255 |
|
||||
| Cosine similarity | 0.99978 |
|
||||
|
||||
---
|
||||
|
||||
# 8. Reproducibility Checklist
|
||||
|
||||
To ensure consistent and comparable benchmark results:
|
||||
|
||||
* Record GPU type
|
||||
* Record vLLM and vLLM-Omni versions
|
||||
* Record benchmark parameters: resolution, frame count, FPS, seed, concurrency, and stage-2 steps
|
||||
* Use frame counts that are multiples of 33
|
||||
* Keep prompt, seed, resolution, and sampling parameters identical when comparing PR vs main
|
||||
* Ensure no background workload on GPUs during testing
|
||||
* Use the same MP4 export and frame extraction path for both variants
|
||||
|
||||
---
|
||||
|
||||
This document serves as the Helios-Distilled serving performance reference under vLLM-Omni.
|
||||
Reference in new issue
Block a user