From 644edac0b6e29b153380a2a3796c328918c2d614 Mon Sep 17 00:00:00 2001 From: Samit <285365963@qq.com> Date: Tue, 14 Apr 2026 14:20:38 +0800 Subject: [PATCH] [Perf] Optimize MP4 encoding latency in video generation (#2735) Signed-off-by: samithuang <285365963@qq.com> --- .../openai_api/test_video_server.py | 144 +++++++++++++----- vllm_omni/diffusion/utils/media_utils.py | 7 +- vllm_omni/entrypoints/openai/api_server.py | 27 +--- vllm_omni/entrypoints/openai/serving_video.py | 16 +- .../entrypoints/openai/video_api_utils.py | 29 +++- 5 files changed, 158 insertions(+), 65 deletions(-) diff --git a/tests/entrypoints/openai_api/test_video_server.py b/tests/entrypoints/openai_api/test_video_server.py index fd7d4df60..82c34f87e 100644 --- a/tests/entrypoints/openai_api/test_video_server.py +++ b/tests/entrypoints/openai_api/test_video_server.py @@ -69,7 +69,7 @@ class BlockingVideoHandler: if self.stage_configs is None: self.stage_configs = stage_configs - async def generate_videos(self, request, reference_id, *, reference_image=None): + async def generate_video_bytes(self, request, reference_id, *, reference_image=None): self.started.set() try: await asyncio.Future() @@ -137,15 +137,81 @@ def _wait_until(predicate, timeout_s: float = 2.0, interval_s: float = 0.02): raise AssertionError("Timed out waiting for condition") +def test_async_video_generation_bypasses_base64(test_client, mocker: MockerFixture): + """Regression test: Ensure async video generation saves raw bytes directly + without bouncing through base64 encoding.""" + # We mock _encode_video_bytes (the correct path) + mocker.patch( + "vllm_omni.entrypoints.openai.serving_video._encode_video_bytes", + return_value=b"raw-mp4-bytes", + ) + + # We assert that encode_video_base64 is never called + mock_base64 = mocker.patch( + "vllm_omni.entrypoints.openai.serving_video.encode_video_base64", + side_effect=RuntimeError("Regression: async video path should not base64 encode"), + ) + + response = test_client.post( + "/v1/videos", + data={"prompt": "A base64 test."}, + ) + assert response.status_code == 200 + video_id = response.json()["id"] + + # Wait for completion. If it used base64, the RuntimeError would fail the task + _wait_for_status(test_client, video_id, VideoGenerationStatus.COMPLETED.value) + mock_base64.assert_not_called() + + +def test_async_video_generation_with_audio_bypasses_base64(test_client, mocker: MockerFixture): + """Regression test: Ensure async video generation passes audio through + generate_video_bytes without bouncing through base64 encoding.""" + mock_encode = mocker.patch( + "vllm_omni.entrypoints.openai.serving_video._encode_video_bytes", + return_value=b"raw-mp4-bytes", + ) + + mock_base64 = mocker.patch( + "vllm_omni.entrypoints.openai.serving_video.encode_video_base64", + side_effect=RuntimeError("Regression: async video path should not base64 encode"), + ) + + engine = test_client.app.state.openai_serving_video._engine_client + + async def _generate(prompt, request_id, sampling_params_list): + engine.captured_prompt = prompt + engine.captured_sampling_params_list = sampling_params_list + yield MockVideoResult([object()], audios=[object()], sample_rate=48000) + + engine.generate = _generate + + response = test_client.post( + "/v1/videos", + data={"prompt": "A base64 test with audio."}, + ) + assert response.status_code == 200 + video_id = response.json()["id"] + + _wait_for_status(test_client, video_id, VideoGenerationStatus.COMPLETED.value) + mock_base64.assert_not_called() + + mock_encode.assert_called_once() + kwargs = mock_encode.call_args.kwargs + assert "audio" in kwargs + assert kwargs["audio"] is not None + assert kwargs["audio_sample_rate"] == 48000 + + def test_t2v_video_generation_form(test_client, mocker: MockerFixture): fps_values = [] - def _fake_encode(video, fps): + def _fake_encode(video, fps, audio=None, audio_sample_rate=None, **kwargs): fps_values.append(fps) - return "Zg==" + return b"fake-video" mocker.patch( - "vllm_omni.entrypoints.openai.serving_video.encode_video_base64", + "vllm_omni.entrypoints.openai.serving_video._encode_video_bytes", side_effect=_fake_encode, ) response = test_client.post( @@ -177,8 +243,8 @@ def test_i2v_video_generation_form(test_client, mocker: MockerFixture): image_bytes = _make_test_image_bytes((48, 32)) mocker.patch( - "vllm_omni.entrypoints.openai.serving_video.encode_video_base64", - return_value="Zg==", + "vllm_omni.entrypoints.openai.serving_video._encode_video_bytes", + return_value=b"fake-video", ) response = test_client.post( "/v1/videos", @@ -203,8 +269,8 @@ def test_i2v_video_generation_resizes_input_to_requested_dimensions(test_client, image_bytes = _make_test_image_bytes((48, 32)) mocker.patch( - "vllm_omni.entrypoints.openai.serving_video.encode_video_base64", - return_value="Zg==", + "vllm_omni.entrypoints.openai.serving_video._encode_video_bytes", + return_value=b"fake-video", ) response = test_client.post( "/v1/videos", @@ -229,8 +295,8 @@ def test_i2v_video_generation_resizes_input_to_requested_dimensions(test_client, def test_i2v_video_generation_with_image_reference_form(test_client, mocker: MockerFixture): mocker.patch( - "vllm_omni.entrypoints.openai.serving_video.encode_video_base64", - return_value="Zg==", + "vllm_omni.entrypoints.openai.serving_video._encode_video_bytes", + return_value=b"fake-video", ) response = test_client.post( "/v1/videos", @@ -254,12 +320,12 @@ def test_i2v_video_generation_with_image_reference_form(test_client, mocker: Moc def test_seconds_defaults_fps_and_frames(test_client, mocker: MockerFixture): fps_values = [] - def _fake_encode(video, fps): + def _fake_encode(video, fps, audio=None, audio_sample_rate=None, **kwargs): fps_values.append(fps) - return "Zg==" + return b"fake-video" mocker.patch( - "vllm_omni.entrypoints.openai.serving_video.encode_video_base64", + "vllm_omni.entrypoints.openai.serving_video._encode_video_bytes", side_effect=_fake_encode, ) response = test_client.post( @@ -283,8 +349,8 @@ def test_seconds_defaults_fps_and_frames(test_client, mocker: MockerFixture): def test_size_param_sets_width_height(test_client, mocker: MockerFixture): mocker.patch( - "vllm_omni.entrypoints.openai.serving_video.encode_video_base64", - return_value="Zg==", + "vllm_omni.entrypoints.openai.serving_video._encode_video_bytes", + return_value=b"fake-video", ) response = test_client.post( "/v1/videos", @@ -305,8 +371,8 @@ def test_size_param_sets_width_height(test_client, mocker: MockerFixture): def test_sampling_params_pass_through(test_client, mocker: MockerFixture): mocker.patch( - "vllm_omni.entrypoints.openai.serving_video.encode_video_base64", - return_value="Zg==", + "vllm_omni.entrypoints.openai.serving_video._encode_video_bytes", + return_value=b"fake-video", ) response = test_client.post( "/v1/videos", @@ -337,10 +403,10 @@ def test_sampling_params_pass_through(test_client, mocker: MockerFixture): def test_audio_sample_rate_comes_from_model_config(test_client, mocker: MockerFixture): audio_sample_rates = [] - def _fake_encode(video, fps, audio=None, audio_sample_rate=None): - del video, fps, audio + def _fake_encode(video, fps, audio=None, audio_sample_rate=None, video_codec_options=None): + del video, fps, audio, video_codec_options audio_sample_rates.append(audio_sample_rate) - return "Zg==" + return b"fake-video" engine = test_client.app.state.openai_serving_video._engine_client engine.model_config = SimpleNamespace( @@ -354,12 +420,14 @@ def test_audio_sample_rate_comes_from_model_config(test_client, mocker: MockerFi async def _generate(prompt, request_id, sampling_params_list): engine.captured_prompt = prompt engine.captured_sampling_params_list = sampling_params_list - yield MockVideoResult([object()], audios=[object()]) + import numpy as np + + yield MockVideoResult([np.zeros((1, 64, 64, 3), dtype=np.uint8)], audios=[object()]) engine.generate = _generate mocker.patch( - "vllm_omni.entrypoints.openai.serving_video.encode_video_base64", + "vllm_omni.entrypoints.openai.serving_video._encode_video_bytes", side_effect=_fake_encode, ) response = test_client.post( @@ -387,8 +455,8 @@ def test_video_job_persists_profiler_metadata(test_client, mocker: MockerFixture engine.generate = _generate mocker.patch( - "vllm_omni.entrypoints.openai.serving_video.encode_video_base64", - return_value="Zg==", + "vllm_omni.entrypoints.openai.serving_video._encode_video_bytes", + return_value=b"fake-video", ) response = test_client.post("/v1/videos", data={"prompt": "profile me"}) @@ -457,8 +525,8 @@ def test_invalid_seconds_returns_422(test_client): def test_negative_prompt_and_seed_pass_through(test_client, mocker: MockerFixture): mocker.patch( - "vllm_omni.entrypoints.openai.serving_video.encode_video_base64", - return_value="Zg==", + "vllm_omni.entrypoints.openai.serving_video._encode_video_bytes", + return_value=b"fake-video", ) response = test_client.post( "/v1/videos", @@ -531,8 +599,8 @@ def test_video_request_validation(): def test_list_videos_supports_order_after_and_limit(test_client, mocker: MockerFixture): mocker.patch( - "vllm_omni.entrypoints.openai.serving_video.encode_video_base64", - return_value="Zg==", + "vllm_omni.entrypoints.openai.serving_video._encode_video_bytes", + return_value=b"fake-video", ) ids = [] for i in range(3): @@ -600,8 +668,8 @@ def test_list_videos_supports_order_after_and_limit(test_client, mocker: MockerF def test_delete_completed_job_removes_file_and_metadata(test_client, mocker: MockerFixture): mocker.patch( - "vllm_omni.entrypoints.openai.serving_video.encode_video_base64", - return_value="Zg==", + "vllm_omni.entrypoints.openai.serving_video._encode_video_bytes", + return_value=b"fake-video", ) create_resp = test_client.post("/v1/videos", data={"prompt": "Delete this video"}) assert create_resp.status_code == 200 @@ -672,8 +740,8 @@ def test_video_response_file_extension_is_robust(): def test_extra_params_merged_into_extra_args(test_client, mocker: MockerFixture): """extra_params JSON object is merged into sampling_params.extra_args.""" mocker.patch( - "vllm_omni.entrypoints.openai.serving_video.encode_video_base64", - return_value="Zg==", + "vllm_omni.entrypoints.openai.serving_video._encode_video_bytes", + return_value=b"fake-video", ) extra_params = { "is_enable_stage2": True, @@ -703,8 +771,8 @@ def test_extra_params_merged_into_extra_args(test_client, mocker: MockerFixture) def test_extra_params_none_by_default(test_client, mocker: MockerFixture): """When extra_params is omitted, extra_args stays empty.""" mocker.patch( - "vllm_omni.entrypoints.openai.serving_video.encode_video_base64", - return_value="Zg==", + "vllm_omni.entrypoints.openai.serving_video._encode_video_bytes", + return_value=b"fake-video", ) response = test_client.post( "/v1/videos", @@ -744,8 +812,8 @@ def test_extra_params_invalid_json(test_client): def test_extra_params_merged_with_existing_extra_args(test_client, mocker: MockerFixture): """extra_params is merged on top of existing extra_args (e.g. flow_shift).""" mocker.patch( - "vllm_omni.entrypoints.openai.serving_video.encode_video_base64", - return_value="Zg==", + "vllm_omni.entrypoints.openai.serving_video._encode_video_bytes", + return_value=b"fake-video", ) response = test_client.post( "/v1/videos", @@ -769,8 +837,8 @@ def test_extra_params_merged_with_existing_extra_args(test_client, mocker: Mocke def test_sample_solver_forwarded_via_extra_params(test_client, mocker: MockerFixture): """sample_solver can be passed through existing extra_params for Wan2.2 online serving.""" mocker.patch( - "vllm_omni.entrypoints.openai.serving_video.encode_video_base64", - return_value="Zg==", + "vllm_omni.entrypoints.openai.serving_video._encode_video_bytes", + return_value=b"fake-video", ) response = test_client.post( "/v1/videos", diff --git a/vllm_omni/diffusion/utils/media_utils.py b/vllm_omni/diffusion/utils/media_utils.py index f96a28fbd..a09cd4595 100644 --- a/vllm_omni/diffusion/utils/media_utils.py +++ b/vllm_omni/diffusion/utils/media_utils.py @@ -20,6 +20,7 @@ def mux_video_audio_bytes( video_codec: str = "h264", audio_codec: str = "aac", crf: str = "18", + video_codec_options: dict[str, str] | None = None, ) -> bytes: """Mux video frames and optional audio waveform into MP4 bytes. @@ -42,7 +43,11 @@ def mux_video_audio_bytes( v_stream.width = video_frames.shape[2] v_stream.height = video_frames.shape[1] v_stream.pix_fmt = "yuv420p" - v_stream.options = {"crf": crf} + + options = {"crf": str(crf)} + if video_codec_options: + options.update(video_codec_options) + v_stream.options = options a_stream = None if audio_waveform is not None: diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py index defaa9822..6a65f4433 100644 --- a/vllm_omni/entrypoints/openai/api_server.py +++ b/vllm_omni/entrypoints/openai/api_server.py @@ -1955,18 +1955,6 @@ def video_response_from_request(model_name: str, req: VideoGenerationRequest) -> return resp -async def decode_and_save_video_output(output: Any, file_name: str) -> str: - if not output.b64_json: - raise RuntimeError(f"Video output for {file_name} did not include b64_json content.") - - try: - video_bytes = base64.b64decode(output.b64_json) - except Exception as decode_exc: - raise RuntimeError(f"Failed to decode generated video payload for {file_name}") from decode_exc - - return await STORAGE_MANAGER.save(video_bytes, file_name) - - def _cleanup_video(video_id: str, output_path: str | None): try: if output_path is not None: @@ -1990,15 +1978,12 @@ async def _run_video_generation_job( started_at = time.perf_counter() output_path = None try: - response = await handler.generate_videos(request, video_id, reference_image=reference_image) - if not response.data: - raise RuntimeError("Video generation completed but returned no outputs.") - - if (video_count := len(response.data)) > 1: - logger.warning("Video request %s generated %s outputs but we only expected one.", video_id, video_count) + video_bytes, stage_durations, peak_memory_mb = await handler.generate_video_bytes( + request, video_id, reference_image=reference_image + ) file_name = f"{video_id}.{job.file_extension}" - output_path = await decode_and_save_video_output(response.data[0], file_name) + output_path = await STORAGE_MANAGER.save(video_bytes, file_name) logger.info("Video request %s persisted %s output file.", video_id, output_path) await VIDEO_STORE.update_fields( @@ -2009,8 +1994,8 @@ async def _run_video_generation_job( "file_name": file_name, "completed_at": int(time.time()), "inference_time_s": time.perf_counter() - started_at, - "stage_durations": response.stage_durations, - "peak_memory_mb": response.peak_memory_mb, + "stage_durations": stage_durations, + "peak_memory_mb": peak_memory_mb, }, ) except Exception as exc: diff --git a/vllm_omni/entrypoints/openai/serving_video.py b/vllm_omni/entrypoints/openai/serving_video.py index 3e05a1eed..0001fa65f 100644 --- a/vllm_omni/entrypoints/openai/serving_video.py +++ b/vllm_omni/entrypoints/openai/serving_video.py @@ -178,17 +178,24 @@ class OmniOpenAIServingVideo: reference_image: ReferenceImage | None = None, ) -> VideoGenerationResponse: artifacts = await self._run_and_extract(request, reference_id, reference_image=reference_image) + + video_codec_options = {"preset": "ultrafast", "threads": "0"} + if request.extra_params is not None and isinstance(request.extra_params, dict): + if "video_codec_options" in request.extra_params: + video_codec_options = request.extra_params["video_codec_options"] + _t_encode_start = time.perf_counter() video_data = [ VideoData( b64_json=( - encode_video_base64(video, fps=artifacts.output_fps) + encode_video_base64(video, fps=artifacts.output_fps, video_codec_options=video_codec_options) if artifacts.audios[idx] is None else encode_video_base64( video, fps=artifacts.output_fps, audio=artifacts.audios[idx], audio_sample_rate=artifacts.audio_sample_rate, + video_codec_options=video_codec_options, ) ) ) @@ -219,11 +226,18 @@ class OmniOpenAIServingVideo: len(artifacts.videos), ) audio = artifacts.audios[0] + + video_codec_options = {"preset": "ultrafast", "threads": "0"} + if request.extra_params is not None and isinstance(request.extra_params, dict): + if "video_codec_options" in request.extra_params: + video_codec_options = request.extra_params["video_codec_options"] + _t_encode_start = time.perf_counter() video_bytes = _encode_video_bytes( artifacts.videos[0], fps=artifacts.output_fps, **({"audio": audio, "audio_sample_rate": artifacts.audio_sample_rate} if audio is not None else {}), + video_codec_options=video_codec_options, ) _t_encode_ms = (time.perf_counter() - _t_encode_start) * 1000 logger.info("Video response encoding (MP4 bytes): %.2f ms", _t_encode_ms) diff --git a/vllm_omni/entrypoints/openai/video_api_utils.py b/vllm_omni/entrypoints/openai/video_api_utils.py index 69178fb3d..193546979 100644 --- a/vllm_omni/entrypoints/openai/video_api_utils.py +++ b/vllm_omni/entrypoints/openai/video_api_utils.py @@ -202,7 +202,13 @@ def _coerce_audio_to_numpy(audio: Any) -> np.ndarray: return arr.astype(np.float32) -def _encode_video_bytes(video: Any, fps: int, audio: Any | None = None, audio_sample_rate: int | None = None) -> bytes: +def _encode_video_bytes( + video: Any, + fps: int, + audio: Any | None = None, + audio_sample_rate: int | None = None, + video_codec_options: dict[str, str] | None = None, +) -> bytes: """Encode a video payload into MP4 bytes, optionally muxing audio.""" from vllm_omni.diffusion.utils.media_utils import mux_video_audio_bytes @@ -213,7 +219,13 @@ def _encode_video_bytes(video: Any, fps: int, audio: Any | None = None, audio_sa frames_np = np.stack(frames, axis=0) if frames_np.ndim == 4 and frames_np.shape[-1] == 4: frames_np = frames_np[..., :3] - frames_u8 = (np.clip(frames_np, 0.0, 1.0) * 255).round().clip(0, 255).astype(np.uint8) + + if frames_np.dtype == np.uint8: + frames_u8 = frames_np + else: + frames_np = np.clip(frames_np, 0.0, 1.0) + frames_np *= 255.0 + frames_u8 = np.round(frames_np).astype(np.uint8) audio_np = _coerce_audio_to_numpy(audio) if audio is not None else None @@ -222,10 +234,19 @@ def _encode_video_bytes(video: Any, fps: int, audio: Any | None = None, audio_sa audio_np, fps=float(fps), audio_sample_rate=audio_sample_rate or 24000, + video_codec_options=video_codec_options, ) -def encode_video_base64(video: Any, fps: int, audio: Any | None = None, audio_sample_rate: int | None = None) -> str: +def encode_video_base64( + video: Any, + fps: int, + audio: Any | None = None, + audio_sample_rate: int | None = None, + video_codec_options: dict[str, str] | None = None, +) -> str: """Encode a video (frames/array/tensor) to base64 MP4.""" - video_bytes = _encode_video_bytes(video, fps=fps, audio=audio, audio_sample_rate=audio_sample_rate) + video_bytes = _encode_video_bytes( + video, fps=fps, audio=audio, audio_sample_rate=audio_sample_rate, video_codec_options=video_codec_options + ) return base64.b64encode(video_bytes).decode("utf-8")