[Perf] Optimize MP4 encoding latency in video generation (#2735)

Signed-off-by: samithuang <285365963@qq.com>
This commit is contained in:
Samit authored and GitHub committed 2026-04-14 14:20:38 +08:00
1 parent a5b38b5d0d
commit 644edac0b6
5 files changed
+158 -65

No files matched your search

+106 -38
View File
@@ -69,7 +69,7 @@ class BlockingVideoHandler:
if self.stage_configs is None:
self.stage_configs = stage_configs
async def generate_videos(self, request, reference_id, *, reference_image=None):
async def generate_video_bytes(self, request, reference_id, *, reference_image=None):
self.started.set()
try:
await asyncio.Future()
@@ -137,15 +137,81 @@ def _wait_until(predicate, timeout_s: float = 2.0, interval_s: float = 0.02):
raise AssertionError("Timed out waiting for condition")
def test_async_video_generation_bypasses_base64(test_client, mocker: MockerFixture):
"""Regression test: Ensure async video generation saves raw bytes directly
without bouncing through base64 encoding."""
# We mock _encode_video_bytes (the correct path)
mocker.patch(
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
return_value=b"raw-mp4-bytes",
)
# We assert that encode_video_base64 is never called
mock_base64 = mocker.patch(
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
side_effect=RuntimeError("Regression: async video path should not base64 encode"),
)
response = test_client.post(
"/v1/videos",
data={"prompt": "A base64 test."},
)
assert response.status_code == 200
video_id = response.json()["id"]
# Wait for completion. If it used base64, the RuntimeError would fail the task
_wait_for_status(test_client, video_id, VideoGenerationStatus.COMPLETED.value)
mock_base64.assert_not_called()
def test_async_video_generation_with_audio_bypasses_base64(test_client, mocker: MockerFixture):
"""Regression test: Ensure async video generation passes audio through
generate_video_bytes without bouncing through base64 encoding."""
mock_encode = mocker.patch(
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
return_value=b"raw-mp4-bytes",
)
mock_base64 = mocker.patch(
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
side_effect=RuntimeError("Regression: async video path should not base64 encode"),
)
engine = test_client.app.state.openai_serving_video._engine_client
async def _generate(prompt, request_id, sampling_params_list):
engine.captured_prompt = prompt
engine.captured_sampling_params_list = sampling_params_list
yield MockVideoResult([object()], audios=[object()], sample_rate=48000)
engine.generate = _generate
response = test_client.post(
"/v1/videos",
data={"prompt": "A base64 test with audio."},
)
assert response.status_code == 200
video_id = response.json()["id"]
_wait_for_status(test_client, video_id, VideoGenerationStatus.COMPLETED.value)
mock_base64.assert_not_called()
mock_encode.assert_called_once()
kwargs = mock_encode.call_args.kwargs
assert "audio" in kwargs
assert kwargs["audio"] is not None
assert kwargs["audio_sample_rate"] == 48000
def test_t2v_video_generation_form(test_client, mocker: MockerFixture):
fps_values = []
def _fake_encode(video, fps):
def _fake_encode(video, fps, audio=None, audio_sample_rate=None, **kwargs):
fps_values.append(fps)
return "Zg=="
return b"fake-video"
mocker.patch(
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
side_effect=_fake_encode,
)
response = test_client.post(
@@ -177,8 +243,8 @@ def test_i2v_video_generation_form(test_client, mocker: MockerFixture):
image_bytes = _make_test_image_bytes((48, 32))
mocker.patch(
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
return_value="Zg==",
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
return_value=b"fake-video",
)
response = test_client.post(
"/v1/videos",
@@ -203,8 +269,8 @@ def test_i2v_video_generation_resizes_input_to_requested_dimensions(test_client,
image_bytes = _make_test_image_bytes((48, 32))
mocker.patch(
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
return_value="Zg==",
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
return_value=b"fake-video",
)
response = test_client.post(
"/v1/videos",
@@ -229,8 +295,8 @@ def test_i2v_video_generation_resizes_input_to_requested_dimensions(test_client,
def test_i2v_video_generation_with_image_reference_form(test_client, mocker: MockerFixture):
mocker.patch(
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
return_value="Zg==",
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
return_value=b"fake-video",
)
response = test_client.post(
"/v1/videos",
@@ -254,12 +320,12 @@ def test_i2v_video_generation_with_image_reference_form(test_client, mocker: Moc
def test_seconds_defaults_fps_and_frames(test_client, mocker: MockerFixture):
fps_values = []
def _fake_encode(video, fps):
def _fake_encode(video, fps, audio=None, audio_sample_rate=None, **kwargs):
fps_values.append(fps)
return "Zg=="
return b"fake-video"
mocker.patch(
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
side_effect=_fake_encode,
)
response = test_client.post(
@@ -283,8 +349,8 @@ def test_seconds_defaults_fps_and_frames(test_client, mocker: MockerFixture):
def test_size_param_sets_width_height(test_client, mocker: MockerFixture):
mocker.patch(
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
return_value="Zg==",
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
return_value=b"fake-video",
)
response = test_client.post(
"/v1/videos",
@@ -305,8 +371,8 @@ def test_size_param_sets_width_height(test_client, mocker: MockerFixture):
def test_sampling_params_pass_through(test_client, mocker: MockerFixture):
mocker.patch(
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
return_value="Zg==",
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
return_value=b"fake-video",
)
response = test_client.post(
"/v1/videos",
@@ -337,10 +403,10 @@ def test_sampling_params_pass_through(test_client, mocker: MockerFixture):
def test_audio_sample_rate_comes_from_model_config(test_client, mocker: MockerFixture):
audio_sample_rates = []
def _fake_encode(video, fps, audio=None, audio_sample_rate=None):
del video, fps, audio
def _fake_encode(video, fps, audio=None, audio_sample_rate=None, video_codec_options=None):
del video, fps, audio, video_codec_options
audio_sample_rates.append(audio_sample_rate)
return "Zg=="
return b"fake-video"
engine = test_client.app.state.openai_serving_video._engine_client
engine.model_config = SimpleNamespace(
@@ -354,12 +420,14 @@ def test_audio_sample_rate_comes_from_model_config(test_client, mocker: MockerFi
async def _generate(prompt, request_id, sampling_params_list):
engine.captured_prompt = prompt
engine.captured_sampling_params_list = sampling_params_list
yield MockVideoResult([object()], audios=[object()])
import numpy as np
yield MockVideoResult([np.zeros((1, 64, 64, 3), dtype=np.uint8)], audios=[object()])
engine.generate = _generate
mocker.patch(
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
side_effect=_fake_encode,
)
response = test_client.post(
@@ -387,8 +455,8 @@ def test_video_job_persists_profiler_metadata(test_client, mocker: MockerFixture
engine.generate = _generate
mocker.patch(
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
return_value="Zg==",
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
return_value=b"fake-video",
)
response = test_client.post("/v1/videos", data={"prompt": "profile me"})
@@ -457,8 +525,8 @@ def test_invalid_seconds_returns_422(test_client):
def test_negative_prompt_and_seed_pass_through(test_client, mocker: MockerFixture):
mocker.patch(
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
return_value="Zg==",
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
return_value=b"fake-video",
)
response = test_client.post(
"/v1/videos",
@@ -531,8 +599,8 @@ def test_video_request_validation():
def test_list_videos_supports_order_after_and_limit(test_client, mocker: MockerFixture):
mocker.patch(
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
return_value="Zg==",
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
return_value=b"fake-video",
)
ids = []
for i in range(3):
@@ -600,8 +668,8 @@ def test_list_videos_supports_order_after_and_limit(test_client, mocker: MockerF
def test_delete_completed_job_removes_file_and_metadata(test_client, mocker: MockerFixture):
mocker.patch(
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
return_value="Zg==",
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
return_value=b"fake-video",
)
create_resp = test_client.post("/v1/videos", data={"prompt": "Delete this video"})
assert create_resp.status_code == 200
@@ -672,8 +740,8 @@ def test_video_response_file_extension_is_robust():
def test_extra_params_merged_into_extra_args(test_client, mocker: MockerFixture):
"""extra_params JSON object is merged into sampling_params.extra_args."""
mocker.patch(
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
return_value="Zg==",
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
return_value=b"fake-video",
)
extra_params = {
"is_enable_stage2": True,
@@ -703,8 +771,8 @@ def test_extra_params_merged_into_extra_args(test_client, mocker: MockerFixture)
def test_extra_params_none_by_default(test_client, mocker: MockerFixture):
"""When extra_params is omitted, extra_args stays empty."""
mocker.patch(
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
return_value="Zg==",
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
return_value=b"fake-video",
)
response = test_client.post(
"/v1/videos",
@@ -744,8 +812,8 @@ def test_extra_params_invalid_json(test_client):
def test_extra_params_merged_with_existing_extra_args(test_client, mocker: MockerFixture):
"""extra_params is merged on top of existing extra_args (e.g. flow_shift)."""
mocker.patch(
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
return_value="Zg==",
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
return_value=b"fake-video",
)
response = test_client.post(
"/v1/videos",
@@ -769,8 +837,8 @@ def test_extra_params_merged_with_existing_extra_args(test_client, mocker: Mocke
def test_sample_solver_forwarded_via_extra_params(test_client, mocker: MockerFixture):
"""sample_solver can be passed through existing extra_params for Wan2.2 online serving."""
mocker.patch(
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
return_value="Zg==",
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
return_value=b"fake-video",
)
response = test_client.post(
"/v1/videos",
+6 -1
View File
@@ -20,6 +20,7 @@ def mux_video_audio_bytes(
video_codec: str = "h264",
audio_codec: str = "aac",
crf: str = "18",
video_codec_options: dict[str, str] | None = None,
) -> bytes:
"""Mux video frames and optional audio waveform into MP4 bytes.
@@ -42,7 +43,11 @@ def mux_video_audio_bytes(
v_stream.width = video_frames.shape[2]
v_stream.height = video_frames.shape[1]
v_stream.pix_fmt = "yuv420p"
v_stream.options = {"crf": crf}
options = {"crf": str(crf)}
if video_codec_options:
options.update(video_codec_options)
v_stream.options = options
a_stream = None
if audio_waveform is not None:
+6 -21
View File
@@ -1955,18 +1955,6 @@ def video_response_from_request(model_name: str, req: VideoGenerationRequest) ->
return resp
async def decode_and_save_video_output(output: Any, file_name: str) -> str:
if not output.b64_json:
raise RuntimeError(f"Video output for {file_name} did not include b64_json content.")
try:
video_bytes = base64.b64decode(output.b64_json)
except Exception as decode_exc:
raise RuntimeError(f"Failed to decode generated video payload for {file_name}") from decode_exc
return await STORAGE_MANAGER.save(video_bytes, file_name)
def _cleanup_video(video_id: str, output_path: str | None):
try:
if output_path is not None:
@@ -1990,15 +1978,12 @@ async def _run_video_generation_job(
started_at = time.perf_counter()
output_path = None
try:
response = await handler.generate_videos(request, video_id, reference_image=reference_image)
if not response.data:
raise RuntimeError("Video generation completed but returned no outputs.")
if (video_count := len(response.data)) > 1:
logger.warning("Video request %s generated %s outputs but we only expected one.", video_id, video_count)
video_bytes, stage_durations, peak_memory_mb = await handler.generate_video_bytes(
request, video_id, reference_image=reference_image
)
file_name = f"{video_id}.{job.file_extension}"
output_path = await decode_and_save_video_output(response.data[0], file_name)
output_path = await STORAGE_MANAGER.save(video_bytes, file_name)
logger.info("Video request %s persisted %s output file.", video_id, output_path)
await VIDEO_STORE.update_fields(
@@ -2009,8 +1994,8 @@ async def _run_video_generation_job(
"file_name": file_name,
"completed_at": int(time.time()),
"inference_time_s": time.perf_counter() - started_at,
"stage_durations": response.stage_durations,
"peak_memory_mb": response.peak_memory_mb,
"stage_durations": stage_durations,
"peak_memory_mb": peak_memory_mb,
},
)
except Exception as exc:
+15 -1
View File
@@ -178,17 +178,24 @@ class OmniOpenAIServingVideo:
reference_image: ReferenceImage | None = None,
) -> VideoGenerationResponse:
artifacts = await self._run_and_extract(request, reference_id, reference_image=reference_image)
video_codec_options = {"preset": "ultrafast", "threads": "0"}
if request.extra_params is not None and isinstance(request.extra_params, dict):
if "video_codec_options" in request.extra_params:
video_codec_options = request.extra_params["video_codec_options"]
_t_encode_start = time.perf_counter()
video_data = [
VideoData(
b64_json=(
encode_video_base64(video, fps=artifacts.output_fps)
encode_video_base64(video, fps=artifacts.output_fps, video_codec_options=video_codec_options)
if artifacts.audios[idx] is None
else encode_video_base64(
video,
fps=artifacts.output_fps,
audio=artifacts.audios[idx],
audio_sample_rate=artifacts.audio_sample_rate,
video_codec_options=video_codec_options,
)
)
)
@@ -219,11 +226,18 @@ class OmniOpenAIServingVideo:
len(artifacts.videos),
)
audio = artifacts.audios[0]
video_codec_options = {"preset": "ultrafast", "threads": "0"}
if request.extra_params is not None and isinstance(request.extra_params, dict):
if "video_codec_options" in request.extra_params:
video_codec_options = request.extra_params["video_codec_options"]
_t_encode_start = time.perf_counter()
video_bytes = _encode_video_bytes(
artifacts.videos[0],
fps=artifacts.output_fps,
**({"audio": audio, "audio_sample_rate": artifacts.audio_sample_rate} if audio is not None else {}),
video_codec_options=video_codec_options,
)
_t_encode_ms = (time.perf_counter() - _t_encode_start) * 1000
logger.info("Video response encoding (MP4 bytes): %.2f ms", _t_encode_ms)
@@ -202,7 +202,13 @@ def _coerce_audio_to_numpy(audio: Any) -> np.ndarray:
return arr.astype(np.float32)
def _encode_video_bytes(video: Any, fps: int, audio: Any | None = None, audio_sample_rate: int | None = None) -> bytes:
def _encode_video_bytes(
video: Any,
fps: int,
audio: Any | None = None,
audio_sample_rate: int | None = None,
video_codec_options: dict[str, str] | None = None,
) -> bytes:
"""Encode a video payload into MP4 bytes, optionally muxing audio."""
from vllm_omni.diffusion.utils.media_utils import mux_video_audio_bytes
@@ -213,7 +219,13 @@ def _encode_video_bytes(video: Any, fps: int, audio: Any | None = None, audio_sa
frames_np = np.stack(frames, axis=0)
if frames_np.ndim == 4 and frames_np.shape[-1] == 4:
frames_np = frames_np[..., :3]
frames_u8 = (np.clip(frames_np, 0.0, 1.0) * 255).round().clip(0, 255).astype(np.uint8)
if frames_np.dtype == np.uint8:
frames_u8 = frames_np
else:
frames_np = np.clip(frames_np, 0.0, 1.0)
frames_np *= 255.0
frames_u8 = np.round(frames_np).astype(np.uint8)
audio_np = _coerce_audio_to_numpy(audio) if audio is not None else None
@@ -222,10 +234,19 @@ def _encode_video_bytes(video: Any, fps: int, audio: Any | None = None, audio_sa
audio_np,
fps=float(fps),
audio_sample_rate=audio_sample_rate or 24000,
video_codec_options=video_codec_options,
)
def encode_video_base64(video: Any, fps: int, audio: Any | None = None, audio_sample_rate: int | None = None) -> str:
def encode_video_base64(
video: Any,
fps: int,
audio: Any | None = None,
audio_sample_rate: int | None = None,
video_codec_options: dict[str, str] | None = None,
) -> str:
"""Encode a video (frames/array/tensor) to base64 MP4."""
video_bytes = _encode_video_bytes(video, fps=fps, audio=audio, audio_sample_rate=audio_sample_rate)
video_bytes = _encode_video_bytes(
video, fps=fps, audio=audio, audio_sample_rate=audio_sample_rate, video_codec_options=video_codec_options
)
return base64.b64encode(video_bytes).decode("utf-8")