[Perf] Optimize MP4 encoding latency in video generation (#2735)
Signed-off-by: samithuang <285365963@qq.com>
This commit is contained in:
1 parent
a5b38b5d0d
commit
644edac0b6
5 files changed
+158
-65
No files matched your search
@@ -69,7 +69,7 @@ class BlockingVideoHandler:
|
||||
if self.stage_configs is None:
|
||||
self.stage_configs = stage_configs
|
||||
|
||||
async def generate_videos(self, request, reference_id, *, reference_image=None):
|
||||
async def generate_video_bytes(self, request, reference_id, *, reference_image=None):
|
||||
self.started.set()
|
||||
try:
|
||||
await asyncio.Future()
|
||||
@@ -137,15 +137,81 @@ def _wait_until(predicate, timeout_s: float = 2.0, interval_s: float = 0.02):
|
||||
raise AssertionError("Timed out waiting for condition")
|
||||
|
||||
|
||||
def test_async_video_generation_bypasses_base64(test_client, mocker: MockerFixture):
|
||||
"""Regression test: Ensure async video generation saves raw bytes directly
|
||||
without bouncing through base64 encoding."""
|
||||
# We mock _encode_video_bytes (the correct path)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
||||
return_value=b"raw-mp4-bytes",
|
||||
)
|
||||
|
||||
# We assert that encode_video_base64 is never called
|
||||
mock_base64 = mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
|
||||
side_effect=RuntimeError("Regression: async video path should not base64 encode"),
|
||||
)
|
||||
|
||||
response = test_client.post(
|
||||
"/v1/videos",
|
||||
data={"prompt": "A base64 test."},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
video_id = response.json()["id"]
|
||||
|
||||
# Wait for completion. If it used base64, the RuntimeError would fail the task
|
||||
_wait_for_status(test_client, video_id, VideoGenerationStatus.COMPLETED.value)
|
||||
mock_base64.assert_not_called()
|
||||
|
||||
|
||||
def test_async_video_generation_with_audio_bypasses_base64(test_client, mocker: MockerFixture):
|
||||
"""Regression test: Ensure async video generation passes audio through
|
||||
generate_video_bytes without bouncing through base64 encoding."""
|
||||
mock_encode = mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
||||
return_value=b"raw-mp4-bytes",
|
||||
)
|
||||
|
||||
mock_base64 = mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
|
||||
side_effect=RuntimeError("Regression: async video path should not base64 encode"),
|
||||
)
|
||||
|
||||
engine = test_client.app.state.openai_serving_video._engine_client
|
||||
|
||||
async def _generate(prompt, request_id, sampling_params_list):
|
||||
engine.captured_prompt = prompt
|
||||
engine.captured_sampling_params_list = sampling_params_list
|
||||
yield MockVideoResult([object()], audios=[object()], sample_rate=48000)
|
||||
|
||||
engine.generate = _generate
|
||||
|
||||
response = test_client.post(
|
||||
"/v1/videos",
|
||||
data={"prompt": "A base64 test with audio."},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
video_id = response.json()["id"]
|
||||
|
||||
_wait_for_status(test_client, video_id, VideoGenerationStatus.COMPLETED.value)
|
||||
mock_base64.assert_not_called()
|
||||
|
||||
mock_encode.assert_called_once()
|
||||
kwargs = mock_encode.call_args.kwargs
|
||||
assert "audio" in kwargs
|
||||
assert kwargs["audio"] is not None
|
||||
assert kwargs["audio_sample_rate"] == 48000
|
||||
|
||||
|
||||
def test_t2v_video_generation_form(test_client, mocker: MockerFixture):
|
||||
fps_values = []
|
||||
|
||||
def _fake_encode(video, fps):
|
||||
def _fake_encode(video, fps, audio=None, audio_sample_rate=None, **kwargs):
|
||||
fps_values.append(fps)
|
||||
return "Zg=="
|
||||
return b"fake-video"
|
||||
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
|
||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
||||
side_effect=_fake_encode,
|
||||
)
|
||||
response = test_client.post(
|
||||
@@ -177,8 +243,8 @@ def test_i2v_video_generation_form(test_client, mocker: MockerFixture):
|
||||
image_bytes = _make_test_image_bytes((48, 32))
|
||||
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
|
||||
return_value="Zg==",
|
||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
||||
return_value=b"fake-video",
|
||||
)
|
||||
response = test_client.post(
|
||||
"/v1/videos",
|
||||
@@ -203,8 +269,8 @@ def test_i2v_video_generation_resizes_input_to_requested_dimensions(test_client,
|
||||
image_bytes = _make_test_image_bytes((48, 32))
|
||||
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
|
||||
return_value="Zg==",
|
||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
||||
return_value=b"fake-video",
|
||||
)
|
||||
response = test_client.post(
|
||||
"/v1/videos",
|
||||
@@ -229,8 +295,8 @@ def test_i2v_video_generation_resizes_input_to_requested_dimensions(test_client,
|
||||
|
||||
def test_i2v_video_generation_with_image_reference_form(test_client, mocker: MockerFixture):
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
|
||||
return_value="Zg==",
|
||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
||||
return_value=b"fake-video",
|
||||
)
|
||||
response = test_client.post(
|
||||
"/v1/videos",
|
||||
@@ -254,12 +320,12 @@ def test_i2v_video_generation_with_image_reference_form(test_client, mocker: Moc
|
||||
def test_seconds_defaults_fps_and_frames(test_client, mocker: MockerFixture):
|
||||
fps_values = []
|
||||
|
||||
def _fake_encode(video, fps):
|
||||
def _fake_encode(video, fps, audio=None, audio_sample_rate=None, **kwargs):
|
||||
fps_values.append(fps)
|
||||
return "Zg=="
|
||||
return b"fake-video"
|
||||
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
|
||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
||||
side_effect=_fake_encode,
|
||||
)
|
||||
response = test_client.post(
|
||||
@@ -283,8 +349,8 @@ def test_seconds_defaults_fps_and_frames(test_client, mocker: MockerFixture):
|
||||
|
||||
def test_size_param_sets_width_height(test_client, mocker: MockerFixture):
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
|
||||
return_value="Zg==",
|
||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
||||
return_value=b"fake-video",
|
||||
)
|
||||
response = test_client.post(
|
||||
"/v1/videos",
|
||||
@@ -305,8 +371,8 @@ def test_size_param_sets_width_height(test_client, mocker: MockerFixture):
|
||||
|
||||
def test_sampling_params_pass_through(test_client, mocker: MockerFixture):
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
|
||||
return_value="Zg==",
|
||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
||||
return_value=b"fake-video",
|
||||
)
|
||||
response = test_client.post(
|
||||
"/v1/videos",
|
||||
@@ -337,10 +403,10 @@ def test_sampling_params_pass_through(test_client, mocker: MockerFixture):
|
||||
def test_audio_sample_rate_comes_from_model_config(test_client, mocker: MockerFixture):
|
||||
audio_sample_rates = []
|
||||
|
||||
def _fake_encode(video, fps, audio=None, audio_sample_rate=None):
|
||||
del video, fps, audio
|
||||
def _fake_encode(video, fps, audio=None, audio_sample_rate=None, video_codec_options=None):
|
||||
del video, fps, audio, video_codec_options
|
||||
audio_sample_rates.append(audio_sample_rate)
|
||||
return "Zg=="
|
||||
return b"fake-video"
|
||||
|
||||
engine = test_client.app.state.openai_serving_video._engine_client
|
||||
engine.model_config = SimpleNamespace(
|
||||
@@ -354,12 +420,14 @@ def test_audio_sample_rate_comes_from_model_config(test_client, mocker: MockerFi
|
||||
async def _generate(prompt, request_id, sampling_params_list):
|
||||
engine.captured_prompt = prompt
|
||||
engine.captured_sampling_params_list = sampling_params_list
|
||||
yield MockVideoResult([object()], audios=[object()])
|
||||
import numpy as np
|
||||
|
||||
yield MockVideoResult([np.zeros((1, 64, 64, 3), dtype=np.uint8)], audios=[object()])
|
||||
|
||||
engine.generate = _generate
|
||||
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
|
||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
||||
side_effect=_fake_encode,
|
||||
)
|
||||
response = test_client.post(
|
||||
@@ -387,8 +455,8 @@ def test_video_job_persists_profiler_metadata(test_client, mocker: MockerFixture
|
||||
|
||||
engine.generate = _generate
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
|
||||
return_value="Zg==",
|
||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
||||
return_value=b"fake-video",
|
||||
)
|
||||
|
||||
response = test_client.post("/v1/videos", data={"prompt": "profile me"})
|
||||
@@ -457,8 +525,8 @@ def test_invalid_seconds_returns_422(test_client):
|
||||
|
||||
def test_negative_prompt_and_seed_pass_through(test_client, mocker: MockerFixture):
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
|
||||
return_value="Zg==",
|
||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
||||
return_value=b"fake-video",
|
||||
)
|
||||
response = test_client.post(
|
||||
"/v1/videos",
|
||||
@@ -531,8 +599,8 @@ def test_video_request_validation():
|
||||
|
||||
def test_list_videos_supports_order_after_and_limit(test_client, mocker: MockerFixture):
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
|
||||
return_value="Zg==",
|
||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
||||
return_value=b"fake-video",
|
||||
)
|
||||
ids = []
|
||||
for i in range(3):
|
||||
@@ -600,8 +668,8 @@ def test_list_videos_supports_order_after_and_limit(test_client, mocker: MockerF
|
||||
|
||||
def test_delete_completed_job_removes_file_and_metadata(test_client, mocker: MockerFixture):
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
|
||||
return_value="Zg==",
|
||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
||||
return_value=b"fake-video",
|
||||
)
|
||||
create_resp = test_client.post("/v1/videos", data={"prompt": "Delete this video"})
|
||||
assert create_resp.status_code == 200
|
||||
@@ -672,8 +740,8 @@ def test_video_response_file_extension_is_robust():
|
||||
def test_extra_params_merged_into_extra_args(test_client, mocker: MockerFixture):
|
||||
"""extra_params JSON object is merged into sampling_params.extra_args."""
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
|
||||
return_value="Zg==",
|
||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
||||
return_value=b"fake-video",
|
||||
)
|
||||
extra_params = {
|
||||
"is_enable_stage2": True,
|
||||
@@ -703,8 +771,8 @@ def test_extra_params_merged_into_extra_args(test_client, mocker: MockerFixture)
|
||||
def test_extra_params_none_by_default(test_client, mocker: MockerFixture):
|
||||
"""When extra_params is omitted, extra_args stays empty."""
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
|
||||
return_value="Zg==",
|
||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
||||
return_value=b"fake-video",
|
||||
)
|
||||
response = test_client.post(
|
||||
"/v1/videos",
|
||||
@@ -744,8 +812,8 @@ def test_extra_params_invalid_json(test_client):
|
||||
def test_extra_params_merged_with_existing_extra_args(test_client, mocker: MockerFixture):
|
||||
"""extra_params is merged on top of existing extra_args (e.g. flow_shift)."""
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
|
||||
return_value="Zg==",
|
||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
||||
return_value=b"fake-video",
|
||||
)
|
||||
response = test_client.post(
|
||||
"/v1/videos",
|
||||
@@ -769,8 +837,8 @@ def test_extra_params_merged_with_existing_extra_args(test_client, mocker: Mocke
|
||||
def test_sample_solver_forwarded_via_extra_params(test_client, mocker: MockerFixture):
|
||||
"""sample_solver can be passed through existing extra_params for Wan2.2 online serving."""
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
|
||||
return_value="Zg==",
|
||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
||||
return_value=b"fake-video",
|
||||
)
|
||||
response = test_client.post(
|
||||
"/v1/videos",
|
||||
|
||||
@@ -20,6 +20,7 @@ def mux_video_audio_bytes(
|
||||
video_codec: str = "h264",
|
||||
audio_codec: str = "aac",
|
||||
crf: str = "18",
|
||||
video_codec_options: dict[str, str] | None = None,
|
||||
) -> bytes:
|
||||
"""Mux video frames and optional audio waveform into MP4 bytes.
|
||||
|
||||
@@ -42,7 +43,11 @@ def mux_video_audio_bytes(
|
||||
v_stream.width = video_frames.shape[2]
|
||||
v_stream.height = video_frames.shape[1]
|
||||
v_stream.pix_fmt = "yuv420p"
|
||||
v_stream.options = {"crf": crf}
|
||||
|
||||
options = {"crf": str(crf)}
|
||||
if video_codec_options:
|
||||
options.update(video_codec_options)
|
||||
v_stream.options = options
|
||||
|
||||
a_stream = None
|
||||
if audio_waveform is not None:
|
||||
|
||||
@@ -1955,18 +1955,6 @@ def video_response_from_request(model_name: str, req: VideoGenerationRequest) ->
|
||||
return resp
|
||||
|
||||
|
||||
async def decode_and_save_video_output(output: Any, file_name: str) -> str:
|
||||
if not output.b64_json:
|
||||
raise RuntimeError(f"Video output for {file_name} did not include b64_json content.")
|
||||
|
||||
try:
|
||||
video_bytes = base64.b64decode(output.b64_json)
|
||||
except Exception as decode_exc:
|
||||
raise RuntimeError(f"Failed to decode generated video payload for {file_name}") from decode_exc
|
||||
|
||||
return await STORAGE_MANAGER.save(video_bytes, file_name)
|
||||
|
||||
|
||||
def _cleanup_video(video_id: str, output_path: str | None):
|
||||
try:
|
||||
if output_path is not None:
|
||||
@@ -1990,15 +1978,12 @@ async def _run_video_generation_job(
|
||||
started_at = time.perf_counter()
|
||||
output_path = None
|
||||
try:
|
||||
response = await handler.generate_videos(request, video_id, reference_image=reference_image)
|
||||
if not response.data:
|
||||
raise RuntimeError("Video generation completed but returned no outputs.")
|
||||
|
||||
if (video_count := len(response.data)) > 1:
|
||||
logger.warning("Video request %s generated %s outputs but we only expected one.", video_id, video_count)
|
||||
video_bytes, stage_durations, peak_memory_mb = await handler.generate_video_bytes(
|
||||
request, video_id, reference_image=reference_image
|
||||
)
|
||||
|
||||
file_name = f"{video_id}.{job.file_extension}"
|
||||
output_path = await decode_and_save_video_output(response.data[0], file_name)
|
||||
output_path = await STORAGE_MANAGER.save(video_bytes, file_name)
|
||||
logger.info("Video request %s persisted %s output file.", video_id, output_path)
|
||||
|
||||
await VIDEO_STORE.update_fields(
|
||||
@@ -2009,8 +1994,8 @@ async def _run_video_generation_job(
|
||||
"file_name": file_name,
|
||||
"completed_at": int(time.time()),
|
||||
"inference_time_s": time.perf_counter() - started_at,
|
||||
"stage_durations": response.stage_durations,
|
||||
"peak_memory_mb": response.peak_memory_mb,
|
||||
"stage_durations": stage_durations,
|
||||
"peak_memory_mb": peak_memory_mb,
|
||||
},
|
||||
)
|
||||
except Exception as exc:
|
||||
|
||||
@@ -178,17 +178,24 @@ class OmniOpenAIServingVideo:
|
||||
reference_image: ReferenceImage | None = None,
|
||||
) -> VideoGenerationResponse:
|
||||
artifacts = await self._run_and_extract(request, reference_id, reference_image=reference_image)
|
||||
|
||||
video_codec_options = {"preset": "ultrafast", "threads": "0"}
|
||||
if request.extra_params is not None and isinstance(request.extra_params, dict):
|
||||
if "video_codec_options" in request.extra_params:
|
||||
video_codec_options = request.extra_params["video_codec_options"]
|
||||
|
||||
_t_encode_start = time.perf_counter()
|
||||
video_data = [
|
||||
VideoData(
|
||||
b64_json=(
|
||||
encode_video_base64(video, fps=artifacts.output_fps)
|
||||
encode_video_base64(video, fps=artifacts.output_fps, video_codec_options=video_codec_options)
|
||||
if artifacts.audios[idx] is None
|
||||
else encode_video_base64(
|
||||
video,
|
||||
fps=artifacts.output_fps,
|
||||
audio=artifacts.audios[idx],
|
||||
audio_sample_rate=artifacts.audio_sample_rate,
|
||||
video_codec_options=video_codec_options,
|
||||
)
|
||||
)
|
||||
)
|
||||
@@ -219,11 +226,18 @@ class OmniOpenAIServingVideo:
|
||||
len(artifacts.videos),
|
||||
)
|
||||
audio = artifacts.audios[0]
|
||||
|
||||
video_codec_options = {"preset": "ultrafast", "threads": "0"}
|
||||
if request.extra_params is not None and isinstance(request.extra_params, dict):
|
||||
if "video_codec_options" in request.extra_params:
|
||||
video_codec_options = request.extra_params["video_codec_options"]
|
||||
|
||||
_t_encode_start = time.perf_counter()
|
||||
video_bytes = _encode_video_bytes(
|
||||
artifacts.videos[0],
|
||||
fps=artifacts.output_fps,
|
||||
**({"audio": audio, "audio_sample_rate": artifacts.audio_sample_rate} if audio is not None else {}),
|
||||
video_codec_options=video_codec_options,
|
||||
)
|
||||
_t_encode_ms = (time.perf_counter() - _t_encode_start) * 1000
|
||||
logger.info("Video response encoding (MP4 bytes): %.2f ms", _t_encode_ms)
|
||||
|
||||
@@ -202,7 +202,13 @@ def _coerce_audio_to_numpy(audio: Any) -> np.ndarray:
|
||||
return arr.astype(np.float32)
|
||||
|
||||
|
||||
def _encode_video_bytes(video: Any, fps: int, audio: Any | None = None, audio_sample_rate: int | None = None) -> bytes:
|
||||
def _encode_video_bytes(
|
||||
video: Any,
|
||||
fps: int,
|
||||
audio: Any | None = None,
|
||||
audio_sample_rate: int | None = None,
|
||||
video_codec_options: dict[str, str] | None = None,
|
||||
) -> bytes:
|
||||
"""Encode a video payload into MP4 bytes, optionally muxing audio."""
|
||||
from vllm_omni.diffusion.utils.media_utils import mux_video_audio_bytes
|
||||
|
||||
@@ -213,7 +219,13 @@ def _encode_video_bytes(video: Any, fps: int, audio: Any | None = None, audio_sa
|
||||
frames_np = np.stack(frames, axis=0)
|
||||
if frames_np.ndim == 4 and frames_np.shape[-1] == 4:
|
||||
frames_np = frames_np[..., :3]
|
||||
frames_u8 = (np.clip(frames_np, 0.0, 1.0) * 255).round().clip(0, 255).astype(np.uint8)
|
||||
|
||||
if frames_np.dtype == np.uint8:
|
||||
frames_u8 = frames_np
|
||||
else:
|
||||
frames_np = np.clip(frames_np, 0.0, 1.0)
|
||||
frames_np *= 255.0
|
||||
frames_u8 = np.round(frames_np).astype(np.uint8)
|
||||
|
||||
audio_np = _coerce_audio_to_numpy(audio) if audio is not None else None
|
||||
|
||||
@@ -222,10 +234,19 @@ def _encode_video_bytes(video: Any, fps: int, audio: Any | None = None, audio_sa
|
||||
audio_np,
|
||||
fps=float(fps),
|
||||
audio_sample_rate=audio_sample_rate or 24000,
|
||||
video_codec_options=video_codec_options,
|
||||
)
|
||||
|
||||
|
||||
def encode_video_base64(video: Any, fps: int, audio: Any | None = None, audio_sample_rate: int | None = None) -> str:
|
||||
def encode_video_base64(
|
||||
video: Any,
|
||||
fps: int,
|
||||
audio: Any | None = None,
|
||||
audio_sample_rate: int | None = None,
|
||||
video_codec_options: dict[str, str] | None = None,
|
||||
) -> str:
|
||||
"""Encode a video (frames/array/tensor) to base64 MP4."""
|
||||
video_bytes = _encode_video_bytes(video, fps=fps, audio=audio, audio_sample_rate=audio_sample_rate)
|
||||
video_bytes = _encode_video_bytes(
|
||||
video, fps=fps, audio=audio, audio_sample_rate=audio_sample_rate, video_codec_options=video_codec_options
|
||||
)
|
||||
return base64.b64encode(video_bytes).decode("utf-8")
|
||||
Reference in new issue
Block a user