diff --git a/recipes/MiniMaxAI/MiniMax-H3-NPU.md b/recipes/MiniMaxAI/MiniMax-H3-NPU.md index ffe8eee09..b4563c1f7 100755 --- a/recipes/MiniMaxAI/MiniMax-H3-NPU.md +++ b/recipes/MiniMaxAI/MiniMax-H3-NPU.md @@ -118,6 +118,21 @@ On Atlas 800I A3 (64 GB HBM per device) the combined service does not fit at above) or HSDP — see [§ Memory and attention optimizations](#memory-and-attention-optimizations-a3). +### CPU MP4 response encoding (Atlas A2) + +Non-streaming MP4 responses use one public automatic encoder. It checks the +runtime frame shape, common dtype, and RGB channel-plane contiguity for every +request; compatible inputs use direct planar PyAV frames, while unsupported +inputs fall back to the legacy muxer before opening the PyAV container. No CLI +flag, model declaration, or user configuration is required. Streaming fMP4 +output remains on its existing incremental path. + +Current performance and correctness validation is limited to one Atlas A2 host +with 8x Ascend 910B4-1 NPUs, the `FL2VA`/`t2va` partition, one request at a +time, 1344x768 at 24 fps, and 5, 8.7, and 15 second requests. This optimization +only changes CPU MP4 response encoding; it does not change DiT execution or +stage 0. + ### Optional optimizations Two independent optimizations may be enabled on top of the configuration diff --git a/recipes/MiniMaxAI/MiniMax-H3.md b/recipes/MiniMaxAI/MiniMax-H3.md index e8c66ca37..fb2edfc3d 100644 --- a/recipes/MiniMaxAI/MiniMax-H3.md +++ b/recipes/MiniMaxAI/MiniMax-H3.md @@ -65,6 +65,31 @@ On AMD ROCm, install without the `[fa4]` extra (FA4 is CUDA-only) and use `ffmpeg` and `ffprobe` must be available on `PATH`. They are used for reference-video preparation and MP4 output. +## CPU MP4 response encoding + +For CUDA and ROCm deployments, non-streaming MP4 responses are encoded on the +host CPU through PyAV/libx264 after generation. The response encoder selects the +path automatically at runtime: inputs with a supported frame shape, common dtype, +and RGB per-channel contiguous layout use direct planar frames; other inputs +fall back to the legacy path before the PyAV container is opened. No CLI flag, +model declaration, or user configuration is required. Streaming fMP4 output is +unchanged. + +A community benchmark on 2x Xeon 8480C reported the following comparison +between the legacy and direct planar paths +([full result](https://github.com/vllm-project/vllm-omni/pull/6288#issuecomment-5337546499)): + +| Metric | Legacy | Direct planar | Change | +|---|---:|---:|---:| +| Median wall time | 1.805 s | 1.394 s | -22.8% | +| Median process CPU time | 3.613 s | 3.207 s | -11.2% | +| Peak RSS | 3182 MiB | 2794 MiB | -387 MiB (-12.2%) | + +Across the 1.0-8.7 s sweep, wall-time improvement was approximately 21.8-22.6%; +outputs were byte-identical and full decode passed. This is evidence for host +CPU response encoding. Actual gains depend on the CPU and runtime, and should +not be interpreted as GPU, DiT, or stage 0 speedups. + ## Start a server Pass the repository ID directly. The pipeline uses `FL2VA` for model discovery diff --git a/tests/benchmarks/benchmark_video_response_encoding.py b/tests/benchmarks/benchmark_video_response_encoding.py new file mode 100644 index 000000000..575d13664 --- /dev/null +++ b/tests/benchmarks/benchmark_video_response_encoding.py @@ -0,0 +1,211 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Benchmark legacy and automatic MP4 response encoding on synthetic input.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import statistics +import time +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +import numpy as np + +from vllm_omni.entrypoints.openai.video_api_utils import ( + _encode_video_bytes, + _encode_video_bytes_legacy, +) + + +@dataclass(frozen=True) +class BenchmarkVariant: + label: str + encoder: Callable[..., bytes] + + +def _positive_int(value: str) -> int: + parsed = int(value) + if parsed < 1: + raise argparse.ArgumentTypeError("value must be at least 1") + return parsed + + +def _build_inputs( + *, + frames: int, + height: int, + width: int, + fps: int, + audio_sample_rate: int, + seed: int, +) -> tuple[np.ndarray, np.ndarray]: + rng = np.random.default_rng(seed) + planar = rng.random((3, frames, height, width), dtype=np.float32) + video = planar.transpose(1, 2, 3, 0) + if not all(video[..., channel].flags.c_contiguous for channel in range(3)): + raise RuntimeError("synthetic video does not expose contiguous channel planes") + + sample_count = max(1, round(frames / fps * audio_sample_rate)) + timeline = np.arange(sample_count, dtype=np.float32) / audio_sample_rate + left = np.sin(2 * np.pi * 220 * timeline).astype(np.float32, copy=False) + right = np.sin(2 * np.pi * 330 * timeline).astype(np.float32, copy=False) + return video, np.stack((left, right)) + + +def _measure( + variant: BenchmarkVariant, + *, + video: np.ndarray, + audio: np.ndarray, + fps: int, + audio_sample_rate: int, +) -> tuple[bytes, dict[str, object]]: + cpu_start = time.process_time_ns() + wall_start = time.perf_counter_ns() + output = variant.encoder( + video, + fps=fps, + audio=audio, + audio_sample_rate=audio_sample_rate, + video_codec_options={"preset": "ultrafast", "threads": "0"}, + ) + wall_ms = (time.perf_counter_ns() - wall_start) / 1_000_000 + process_cpu_ms = (time.process_time_ns() - cpu_start) / 1_000_000 + return output, { + "label": variant.label, + "wall_ms": wall_ms, + "process_cpu_ms": process_cpu_ms, + "output_bytes": len(output), + "output_sha256": hashlib.sha256(output).hexdigest(), + } + + +def _summarize(records: list[dict[str, object]], label: str) -> dict[str, object]: + selected = [record for record in records if record["label"] == label] + wall_values = [float(record["wall_ms"]) for record in selected] + cpu_values = [float(record["process_cpu_ms"]) for record in selected] + return { + "runs": len(selected), + "wall_ms": { + "median": statistics.median(wall_values), + "min": min(wall_values), + "max": max(wall_values), + }, + "process_cpu_ms": { + "median": statistics.median(cpu_values), + "min": min(cpu_values), + "max": max(cpu_values), + }, + } + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--frames", type=_positive_int, default=12) + parser.add_argument("--height", type=_positive_int, default=96) + parser.add_argument("--width", type=_positive_int, default=160) + parser.add_argument("--fps", type=_positive_int, default=24) + parser.add_argument("--audio-sample-rate", type=_positive_int, default=32000) + parser.add_argument("--warmup", type=int, default=1) + parser.add_argument("--rounds", type=_positive_int, default=3) + parser.add_argument("--seed", type=int, default=20260817) + parser.add_argument("--output-json", type=Path) + args = parser.parse_args() + if args.warmup < 0: + parser.error("--warmup must be non-negative") + return args + + +def main() -> None: + args = _parse_args() + video, audio = _build_inputs( + frames=args.frames, + height=args.height, + width=args.width, + fps=args.fps, + audio_sample_rate=args.audio_sample_rate, + seed=args.seed, + ) + baseline = BenchmarkVariant("legacy", _encode_video_bytes_legacy) + candidate = BenchmarkVariant("automatic", _encode_video_bytes) + + for _ in range(args.warmup): + for variant in (baseline, candidate): + _measure( + variant, + video=video, + audio=audio, + fps=args.fps, + audio_sample_rate=args.audio_sample_rate, + ) + + records: list[dict[str, object]] = [] + for round_index in range(args.rounds): + order = (baseline, candidate) if round_index % 2 == 0 else (candidate, baseline) + for variant in order: + _, record = _measure( + variant, + video=video, + audio=audio, + fps=args.fps, + audio_sample_rate=args.audio_sample_rate, + ) + record["round"] = round_index + 1 + records.append(record) + + output_hashes = {str(record["output_sha256"]) for record in records} + if len(output_hashes) != 1: + raise RuntimeError(f"benchmark variants produced different outputs: {sorted(output_hashes)}") + + paired_savings_ms = [] + paired_savings_percent = [] + for round_index in range(args.rounds): + pair = [record for record in records if record["round"] == round_index + 1] + baseline_ms = float(next(record["wall_ms"] for record in pair if record["label"] == baseline.label)) + candidate_ms = float(next(record["wall_ms"] for record in pair if record["label"] == candidate.label)) + paired_savings_ms.append(baseline_ms - candidate_ms) + paired_savings_percent.append((baseline_ms - candidate_ms) / baseline_ms * 100) + + result = { + "config": { + "frames": args.frames, + "height": args.height, + "width": args.width, + "fps": args.fps, + "audio_sample_rate": args.audio_sample_rate, + "warmup": args.warmup, + "rounds": args.rounds, + "seed": args.seed, + "video_shape": list(video.shape), + "video_strides": list(video.strides), + "codec_options": {"preset": "ultrafast", "threads": "0"}, + }, + "records": records, + "summary": { + baseline.label: _summarize(records, baseline.label), + candidate.label: _summarize(records, candidate.label), + "paired_wall_saving_ms": { + "median": statistics.median(paired_savings_ms), + "min": min(paired_savings_ms), + "max": max(paired_savings_ms), + }, + "paired_wall_saving_percent": { + "median": statistics.median(paired_savings_percent), + "min": min(paired_savings_percent), + "max": max(paired_savings_percent), + }, + "output_sha256": output_hashes.pop(), + }, + } + output_json = json.dumps(result, indent=2, sort_keys=True) + if args.output_json is not None: + args.output_json.write_text(output_json + "\n", encoding="utf-8") + print(output_json) + + +if __name__ == "__main__": + main() diff --git a/tests/entrypoints/openai_api/test_video_api_utils.py b/tests/entrypoints/openai_api/test_video_api_utils.py index bb47356c6..a7334c177 100644 --- a/tests/entrypoints/openai_api/test_video_api_utils.py +++ b/tests/entrypoints/openai_api/test_video_api_utils.py @@ -5,6 +5,7 @@ import base64 from io import BytesIO +import av import httpx import numpy as np import pytest @@ -13,6 +14,7 @@ from PIL import Image from vllm import envs from vllm_omni.diffusion.postprocess import rife_interpolator +from vllm_omni.diffusion.utils import media_utils from vllm_omni.entrypoints.openai import video_api_utils from vllm_omni.entrypoints.openai.errors import InvalidInputReferenceError @@ -219,6 +221,462 @@ def test_channel_first_video_tensor_is_converted_to_channel_last(): np.testing.assert_array_equal(frames, video.permute(1, 2, 3, 0).numpy()) +def test_channel_first_video_tensor_uses_direct_planar_mux(monkeypatch): + calls = [] + + def fake_planar_mux(frames, width, height, audio_waveform, **kwargs): + calls.append((list(frames), width, height, audio_waveform, kwargs)) + return b"planar-video" + + def fail_compat_mux(*args, **kwargs): + raise AssertionError("CFHW-backed FHWC frames should use the planar path") + + monkeypatch.setattr(media_utils, "mux_av_video_audio_bytes", fake_planar_mux) + monkeypatch.setattr(media_utils, "mux_video_audio_bytes", fail_compat_mux) + + video = torch.linspace(-1, 1, 3 * 2 * 4 * 6, dtype=torch.float32).reshape(3, 2, 4, 6) + encoded = video_api_utils._encode_video_bytes( + video, + fps=12, + ) + + assert encoded == b"planar-video" + assert len(calls) == 1 + frames, width, height, audio, options = calls[0] + assert (len(frames), width, height) == (2, 6, 4) + assert all(frame.format.name == "gbrp" for frame in frames) + assert audio is None + assert options["fps"] == 12.0 + + +@pytest.mark.parametrize( + ("video", "expected_path", "expected_reason", "expected_mux"), + [ + ( + np.transpose(np.zeros((3, 2, 4, 4), dtype=np.float32), (1, 2, 3, 0)), + "direct_planar", + None, + "direct", + ), + ( + np.zeros((2, 4, 6, 3), dtype=np.float32), + "legacy_fallback", + "non_contiguous_rgb_planes", + "legacy", + ), + ], + ids=["automatic-direct", "interleaved-legacy-fallback"], +) +def test_audio_sample_rate_defaults_and_path_log(monkeypatch, video, expected_path, expected_reason, expected_mux): + mux_calls = [] + path_logs = [] + default_rate_logs = [] + monkeypatch.setattr( + video_api_utils.logger, + "info", + lambda message, *args, **kwargs: path_logs.append(message % args), + ) + monkeypatch.setattr( + video_api_utils.logger, + "info_once", + lambda message, *args, **kwargs: default_rate_logs.append(message % args), + ) + + def fake_planar_mux(frames, width, height, audio_waveform, **kwargs): + mux_calls.append({"kind": "direct", "audio_sample_rate": kwargs["audio_sample_rate"]}) + return b"planar-video" + + def fake_compat_mux(frames, audio, **kwargs): + mux_calls.append({"kind": "legacy", "audio_sample_rate": kwargs["audio_sample_rate"]}) + return b"legacy-video" + + monkeypatch.setattr(media_utils, "mux_av_video_audio_bytes", fake_planar_mux) + monkeypatch.setattr(media_utils, "mux_video_audio_bytes", fake_compat_mux) + + encoded = video_api_utils._encode_video_bytes( + video, + fps=12, + audio=np.zeros((1, 8), dtype=np.float32), + ) + + assert encoded == (b"planar-video" if expected_mux == "direct" else b"legacy-video") + assert mux_calls == [{"kind": expected_mux, "audio_sample_rate": 24000}] + assert len(default_rate_logs) == 1 + assert "default sample rate of 24000 Hz" in default_rate_logs[0] + assert len(path_logs) == 1 + assert f"selected_path={expected_path}" in path_logs[0] + if expected_reason is not None: + assert f"reason={expected_reason}" in path_logs[0] + assert "frames=2" in path_logs[0] + expected_shape = "(4, 4, 3)" if expected_mux == "direct" else "(4, 6, 3)" + assert f"frame_shape={expected_shape}" in path_logs[0] + assert "dtype=float32" in path_logs[0] + assert "fps=12" in path_logs[0] + assert "audio_present=True" in path_logs[0] + assert "effective_audio_sample_rate=24000" in path_logs[0] + + +def test_explicit_audio_sample_rate_bypasses_default_log(monkeypatch): + log_messages = [] + monkeypatch.setattr( + video_api_utils.logger, + "info_once", + lambda message, *args, **kwargs: log_messages.append(message % args), + ) + + audio = [[0.0, 1.0], [1.0, 0.0]] + assert video_api_utils._resolve_audio_sample_rate(audio, 32000) == 32000 + assert log_messages == [] + + +def test_video_only_does_not_log_default_audio_sample_rate(monkeypatch): + log_messages = [] + monkeypatch.setattr( + video_api_utils.logger, + "info_once", + lambda message, *args, **kwargs: log_messages.append(message % args), + ) + monkeypatch.setattr( + media_utils, + "mux_video_audio_bytes", + lambda frames, audio, **kwargs: b"legacy-video", + ) + + video_api_utils._encode_video_bytes(np.zeros((1, 2, 2, 3), dtype=np.uint8), fps=12) + + assert log_messages == [] + + +def test_interleaved_video_uses_legacy_fallback_automatically(monkeypatch): + calls = [] + path_logs = [] + coerce_calls = [] + + real_coerce_video_to_frames = video_api_utils._coerce_video_to_frames + + def count_coerce_video_to_frames(video): + coerce_calls.append(video) + return real_coerce_video_to_frames(video) + + def fail_planar_mux(*args, **kwargs): + raise AssertionError("interleaved FHWC frames must use the legacy path") + + def fake_compat_mux(frames, audio, **kwargs): + calls.append((frames, audio, kwargs)) + return b"legacy-video" + + monkeypatch.setattr( + video_api_utils.logger, "info", lambda message, *args, **kwargs: path_logs.append(message % args) + ) + monkeypatch.setattr(video_api_utils, "_coerce_video_to_frames", count_coerce_video_to_frames) + monkeypatch.setattr(media_utils, "mux_av_video_audio_bytes", fail_planar_mux) + monkeypatch.setattr(media_utils, "mux_video_audio_bytes", fake_compat_mux) + + video = np.zeros((2, 4, 6, 3), dtype=np.float32) + encoded = video_api_utils._encode_video_bytes(video, fps=12) + + assert encoded == b"legacy-video" + assert calls[0][0].shape == (2, 4, 6, 3) + assert calls[0][0].dtype == np.uint8 + assert len(path_logs) == 1 + assert "selected_path=legacy_fallback" in path_logs[0] + assert "reason=non_contiguous_rgb_planes" in path_logs[0] + assert "audio_present=False" in path_logs[0] + assert "effective_audio_sample_rate=None" in path_logs[0] + assert len(coerce_calls) == 1 + + +@pytest.mark.parametrize( + "video", + [ + np.array( + [[[[0, 1, 127], [128, 254, 255]], [[255, 127, 1], [2, 3, 4]]]], + dtype=np.uint8, + ), + np.array( + [[[[-1.0, -0.5, 0.0], [0.5, 1.0, 2.0]], [[1.5, -2.0, 0.25], [0.75, 0.125, 0.875]]]], + dtype=np.float32, + ), + np.array( + [[[[-512, -1, 0], [128, 255, 512]], [[1024, -256, 64], [127, 300, -300]]]], + dtype=np.int16, + ), + ], + ids=["uint8", "float-bounded-and-out-of-range", "signed-int-out-of-range"], +) +def test_prepared_automatic_fallback_preserves_legacy_quantization(monkeypatch, video): + mux_inputs = [] + + def fail_planar_mux(*args, **kwargs): + raise AssertionError("interleaved frames must use the legacy mux") + + def fake_compat_mux(frames, audio, **kwargs): + mux_inputs.append(frames.copy()) + return b"legacy-video" + + monkeypatch.setattr(media_utils, "mux_av_video_audio_bytes", fail_planar_mux) + monkeypatch.setattr(media_utils, "mux_video_audio_bytes", fake_compat_mux) + + prepared_frames, frame_shape, _ = video_api_utils._prepare_video_frames(video) + reference = np.rint(np.clip(np.stack([frame[..., :3] for frame in prepared_frames]), 0.0, 1.0) * 255.0).astype( + np.uint8 + ) + + assert video_api_utils._encode_video_bytes(video, fps=12) == b"legacy-video" + assert len(mux_inputs) == 1 + assert mux_inputs[0].dtype == np.uint8 + assert mux_inputs[0].shape == (len(prepared_frames), *frame_shape[:-1], 3) + np.testing.assert_array_equal(mux_inputs[0], reference) + + +@pytest.mark.parametrize( + ("video", "reason"), + [ + (np.zeros((2, 4, 6), dtype=np.float32), "unsupported_shape"), + (np.empty((2, 0, 6, 3), dtype=np.float32), "unsupported_shape"), + (np.empty((2, 4, 0, 3), dtype=np.float32), "unsupported_shape"), + (np.zeros((2, 2, 2, 3), dtype=np.complex64), "unsupported_dtype"), + (np.zeros((2, 2, 2, 3), dtype=object), "unsupported_dtype"), + (np.zeros((2, 2, 2, 3), dtype="U1"), "unsupported_dtype"), + (np.zeros((2, 2, 2, 3), dtype="datetime64[D]"), "unsupported_dtype"), + ], +) +def test_unsupported_direct_capability_uses_legacy_with_reason(monkeypatch, video, reason): + events = [] + + def fake_legacy(*args, **kwargs): + events.append("legacy") + return b"legacy-video" + + monkeypatch.setattr(video_api_utils, "_encode_prepared_video_bytes_legacy", fake_legacy) + monkeypatch.setattr( + video_api_utils.logger, + "info", + lambda message, *args, **kwargs: events.append(message % args), + ) + + assert video_api_utils._encode_video_bytes(video, fps=12) == b"legacy-video" + assert events[-1] == "legacy" + assert f"selected_path=legacy_fallback reason={reason}" in events[0] + assert "frames=" in events[0] + assert "fps=12" in events[0] + assert "audio_present=False" in events[0] + assert "effective_audio_sample_rate=None" in events[0] + + +def test_non_contiguous_fallback_is_logged_before_legacy_mux(monkeypatch): + events = [] + video = np.zeros((2, 4, 6, 3), dtype=np.float32) + + def fake_legacy(*args, **kwargs): + events.append("legacy") + return b"legacy-video" + + monkeypatch.setattr(video_api_utils, "_encode_prepared_video_bytes_legacy", fake_legacy) + monkeypatch.setattr( + video_api_utils.logger, + "info", + lambda message, *args, **kwargs: events.append(message % args), + ) + + assert video_api_utils._encode_video_bytes(video, fps=12) == b"legacy-video" + assert events[0].startswith("Video response encoding route selected: selected_path=legacy_fallback") + assert events[1] == "legacy" + + +def test_direct_mux_error_is_propagated_without_legacy_retry(monkeypatch): + legacy_calls = [] + + def fail_direct(*args, **kwargs): + raise RuntimeError("direct mux failed") + + def fail_legacy(*args, **kwargs): + legacy_calls.append((args, kwargs)) + raise AssertionError("direct failures must not retry through legacy") + + monkeypatch.setattr(media_utils, "mux_av_video_audio_bytes", fail_direct) + monkeypatch.setattr(video_api_utils, "_encode_prepared_video_bytes_legacy", fail_legacy) + planar = np.transpose(np.zeros((3, 2, 4, 4), dtype=np.float32), (1, 2, 3, 0)) + + with pytest.raises(RuntimeError, match="direct mux failed"): + video_api_utils._encode_video_bytes(planar, fps=12) + + assert legacy_calls == [] + + +@pytest.mark.parametrize("dtype", [np.float16, np.float32]) +@pytest.mark.parametrize("channels", [3, 4]) +def test_planar_frames_decode_with_bounded_rgb_quantization(dtype, channels): + channel_values = np.array( + [ + [[-0.1, 0.0, 0.1, 0.5, 0.9, 1.0, 1.1]], + [[1.0, 0.9, 0.5, 0.1, 0.0, -0.1, 1.1]], + [[0.25, 0.75, 0.501, 0.499, 0.125, 0.875, 0.5]], + [[0.2, 0.4, 0.6, 0.8, 1.0, 0.0, 0.5]], + ], + dtype=dtype, + )[:channels] + fhwc = np.transpose(channel_values[:, None, :, :], (1, 2, 3, 0)) + frames = list(video_api_utils._iter_planar_video_frames(list(fhwc), np.dtype(dtype))) + + decoded = frames[0].to_ndarray(format="rgb24") + expected = np.rint(np.clip(fhwc[0, ..., :3], 0.0, 1.0) * 255.0).astype(np.uint8) + np.testing.assert_array_equal(decoded, expected) + + +def test_planar_frame_writes_gbr_planes_and_clears_padding(): + width = 17 + planar = np.stack( + [ + np.full((2, width), 11, dtype=np.uint8), + np.full((2, width), 22, dtype=np.uint8), + np.full((2, width), 33, dtype=np.uint8), + ] + ) + fhwc = np.transpose(planar[:, None, :, :], (1, 2, 3, 0)) + + frame = next(video_api_utils._iter_planar_video_frames(list(fhwc), np.dtype(np.uint8))) + + for plane, expected in zip(frame.planes, (22, 33, 11)): + plane_data = np.frombuffer(plane, dtype=np.uint8).reshape(plane.height, plane.line_size) + np.testing.assert_array_equal(plane_data[:2, :width], expected) + np.testing.assert_array_equal(plane_data[:2, width:], 0) + np.testing.assert_array_equal(plane_data[2:], 0) + + +def test_planar_bool_frames_match_bounded_compatible_output(): + planar = np.array( + [ + [[[False, True], [True, False]]], + [[[True, False], [True, False]]], + [[[False, False], [True, True]]], + ], + dtype=np.bool_, + ) + direct_video = np.transpose(planar, (1, 2, 3, 0)) + compatible_video = np.ascontiguousarray(direct_video) + + prepared_frames, _, common_dtype = video_api_utils._prepare_video_frames(direct_video) + direct_frames = list(video_api_utils._iter_planar_video_frames(prepared_frames, common_dtype)) + decoded_direct = np.stack([frame.to_ndarray(format="rgb24") for frame in direct_frames]) + bounded_compatible = video_api_utils._coerce_video_to_uint8_frames(compatible_video) + + np.testing.assert_array_equal(decoded_direct, bounded_compatible) + + +@pytest.mark.parametrize( + ("plane_height", "line_size"), + [(1, 2), (2, 1)], +) +def test_planar_frame_rejects_undersized_plane(monkeypatch, plane_height, line_size): + class FakePlane: + pass + + plane = FakePlane() + plane.height = plane_height + plane.line_size = line_size + + class FakeFrame: + planes = [plane] + + monkeypatch.setattr(av, "VideoFrame", lambda *args, **kwargs: FakeFrame()) + planar = np.zeros((3, 1, 2, 2), dtype=np.uint8) + fhwc = np.transpose(planar, (1, 2, 3, 0)) + + with pytest.raises(ValueError, match="smaller than"): + next(video_api_utils._iter_planar_video_frames(list(fhwc), np.dtype(np.uint8))) + + +def test_unequal_frame_shapes_fail_before_output_allocation(monkeypatch): + def fail_empty(*args, **kwargs): + raise AssertionError("output allocation must happen after shape validation") + + monkeypatch.setattr(video_api_utils.np, "empty", fail_empty) + + with pytest.raises(ValueError, match="same shape"): + video_api_utils._encode_video_bytes( + [np.zeros((2, 3, 3), dtype=np.float32), np.zeros((3, 3, 3), dtype=np.float32)], + fps=12, + ) + + +@pytest.mark.parametrize("with_audio", [False, True]) +def test_direct_and_compatible_paths_produce_equivalent_mp4(with_audio): + rng = np.random.default_rng(7) + planar = rng.random((3, 3, 32, 32), dtype=np.float32) + direct_video = np.transpose(planar, (1, 2, 3, 0)) + compatible_video = np.ascontiguousarray(direct_video) + audio = np.zeros((2, 2400), dtype=np.float32) if with_audio else None + + direct_bytes = video_api_utils._encode_video_bytes( + direct_video, + fps=12, + audio=audio, + audio_sample_rate=24000, + video_codec_options={"preset": "ultrafast", "threads": "1"}, + ) + compatible_bytes = video_api_utils._encode_video_bytes( + compatible_video, + fps=12, + audio=audio, + audio_sample_rate=24000, + video_codec_options={"preset": "ultrafast", "threads": "1"}, + ) + + def decode(mp4_bytes): + with av.open(BytesIO(mp4_bytes), mode="r", format="mp4") as container: + stream_types = [stream.type for stream in container.streams] + video_stream = container.streams.video[0] + frames = np.stack([frame.to_ndarray(format="rgb24") for frame in container.decode(video_stream)]) + return stream_types, frames + + direct_streams, direct_frames = decode(direct_bytes) + compatible_streams, compatible_frames = decode(compatible_bytes) + assert direct_bytes == compatible_bytes + assert direct_streams == compatible_streams + assert ("audio" in direct_streams) is with_audio + np.testing.assert_array_equal(direct_frames, compatible_frames) + + +def test_mux_closes_container_and_preserves_generator_error(monkeypatch): + class GeneratorError(RuntimeError): + pass + + class FakeStream: + options = None + + def encode(self, frame=None): + return [] + + class FakeContainer: + def __init__(self): + self.closed = False + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.closed = True + return False + + def add_stream(self, codec, rate): + return FakeStream() + + def mux(self, packet): + pass + + container = FakeContainer() + monkeypatch.setattr(media_utils.av, "open", lambda *args, **kwargs: container) + + def failing_frames(): + yield object() + raise GeneratorError("frame generation failed") + + with pytest.raises(GeneratorError, match="frame generation failed"): + media_utils.mux_av_video_audio_bytes(failing_frames(), width=2, height=2) + assert container.closed + + def test_fragmented_mp4_video_encoder_reuses_single_muxer(monkeypatch): muxers = [] diff --git a/tests/entrypoints/openai_api/test_video_server.py b/tests/entrypoints/openai_api/test_video_server.py index a83edfcb9..1d2bfe204 100644 --- a/tests/entrypoints/openai_api/test_video_server.py +++ b/tests/entrypoints/openai_api/test_video_server.py @@ -89,6 +89,56 @@ class FakeAsyncOmni: yield MockVideoResult(videos) +def test_raw_and_base64_encoders_receive_no_policy_config(mocker: MockerFixture): + engine = FakeAsyncOmni() + handler = OmniOpenAIServingVideo.for_diffusion( + engine, + model_name="test-model", + ) + raw_encoder = mocker.patch( + "vllm_omni.entrypoints.openai.serving_video._encode_video_bytes", + return_value=b"encoded-video", + ) + base64_encoder = mocker.patch( + "vllm_omni.entrypoints.openai.serving_video.encode_video_base64", + return_value="encoded-video", + ) + + async def _generate_both_response_types(): + request = VideoGenerationRequest(prompt="test prompt") + await handler.generate_video_bytes(request, "raw-request") + await handler.generate_videos(request, "base64-request") + + asyncio.run(_generate_both_response_types()) + + assert "encoding_config" not in raw_encoder.call_args.kwargs + assert "encoding_config" not in base64_encoder.call_args.kwargs + + +def test_resolve_diffusion_od_config_falls_back_to_attribute(): + od_config = SimpleNamespace(model_class_name="WanPipeline") + handler = OmniOpenAIServingVideo.for_diffusion( + SimpleNamespace(od_config=od_config), + model_name="test-model", + ) + + assert handler._resolve_diffusion_od_config() is od_config + + +def test_resolve_diffusion_od_config_prefers_getter_over_attribute(): + attribute_config = SimpleNamespace(model_class_name="WanPipeline") + getter_config = SimpleNamespace(model_class_name="MiniMaxH3Pipeline") + handler = OmniOpenAIServingVideo.for_diffusion( + SimpleNamespace( + od_config=attribute_config, + get_diffusion_od_config=lambda: getter_config, + ), + model_name="test-model", + ) + + assert handler._resolve_diffusion_od_config() is getter_config + + class BlockingVideoHandler: def __init__(self): self.model_name = "Wan-AI/Wan2.2-T2V-A14B-Diffusers" @@ -1157,7 +1207,13 @@ def test_worker_fps_multiplier_is_applied_to_async_encoding(test_client, mocker: def test_audio_sample_rate_comes_from_model_config(test_client, mocker: MockerFixture): audio_sample_rates = [] - def _fake_encode(video, fps, audio=None, audio_sample_rate=None, video_codec_options=None): + def _fake_encode( + video, + fps, + audio=None, + audio_sample_rate=None, + video_codec_options=None, + ): del video, fps, audio, video_codec_options audio_sample_rates.append(audio_sample_rate) return b"fake-video" diff --git a/vllm_omni/diffusion/utils/media_utils.py b/vllm_omni/diffusion/utils/media_utils.py index 1ace493db..5c210e996 100644 --- a/vllm_omni/diffusion/utils/media_utils.py +++ b/vllm_omni/diffusion/utils/media_utils.py @@ -5,6 +5,7 @@ from __future__ import annotations import io +from collections.abc import Iterable from fractions import Fraction from typing import Any, cast @@ -200,3 +201,70 @@ def mux_video_audio_bytes( container.close() return buf.getvalue() + + +def mux_av_video_audio_bytes( + video_frames: Iterable[av.VideoFrame], + width: int, + height: int, + audio_waveform: np.ndarray | None = None, + *, + fps: float = 25.0, + audio_sample_rate: int = 44100, + video_codec: str = "h264", + audio_codec: str = "aac", + crf: str = "18", + video_codec_options: dict[str, str] | None = None, +) -> bytes: + """Mux preconstructed video frames and optional audio into MP4 bytes.""" + buf = io.BytesIO() + with cast(Any, av.open(buf, mode="w", format="mp4")) as container: + v_stream = cast( + av.VideoStream, + container.add_stream(video_codec, rate=Fraction(fps).limit_denominator(10000)), + ) + v_stream.width = width + v_stream.height = height + v_stream.pix_fmt = "yuv420p" + + options: dict[str, object] = {"crf": str(crf)} + if video_codec_options: + options.update(video_codec_options) + v_stream.options = options + + a_stream: av.AudioStream | None = None + samples: np.ndarray | None = None + layout: str | None = None + if audio_waveform is not None: + samples = audio_waveform.astype(np.float32) + if samples.ndim == 1: + samples = samples.reshape(1, -1) + elif samples.ndim == 2 and samples.shape[0] > samples.shape[1]: + samples = np.ascontiguousarray(samples.T) + num_channels = samples.shape[0] + layout = "stereo" if num_channels >= 2 else "mono" + a_stream = cast(av.AudioStream, container.add_stream(audio_codec, rate=audio_sample_rate)) + a_stream.layout = layout + + for frame in video_frames: + for packet in v_stream.encode(frame): + container.mux(packet) + for packet in v_stream.encode(): + container.mux(packet) + + if a_stream is not None and audio_waveform is not None: + if samples is None or layout is None: + raise ValueError("Audio samples were not prepared for muxing.") + audio_frame = av.AudioFrame.from_ndarray(samples, format="fltp", layout=layout) + audio_frame.sample_rate = audio_sample_rate + # AAC has a one-frame encoder delay. Mark the input waveform as + # starting at t=0 so the MP4 muxer writes the corresponding negative + # priming timestamp instead of exposing the delay as leading silence. + audio_frame.pts = 0 + audio_frame.time_base = Fraction(1, audio_sample_rate) + for packet in a_stream.encode(audio_frame): + container.mux(packet) + for packet in a_stream.encode(): + container.mux(packet) + + return buf.getvalue() diff --git a/vllm_omni/entrypoints/cli/serve.py b/vllm_omni/entrypoints/cli/serve.py index cf80a2afa..c104f1e72 100644 --- a/vllm_omni/entrypoints/cli/serve.py +++ b/vllm_omni/entrypoints/cli/serve.py @@ -823,7 +823,6 @@ class OmniServeCommand(CLISubcommand): default=False, help="Enable chunked streaming output for diffusion (mainly video generation) models that support it.", ) - # TTS-specific parameters omni_config_group.add_argument( "--tts-max-instructions-length", diff --git a/vllm_omni/entrypoints/openai/serving_video.py b/vllm_omni/entrypoints/openai/serving_video.py index 2628c14f8..cdb5114e2 100644 --- a/vllm_omni/entrypoints/openai/serving_video.py +++ b/vllm_omni/entrypoints/openai/serving_video.py @@ -9,7 +9,8 @@ from collections.abc import Mapping from dataclasses import dataclass from functools import cached_property from http import HTTPStatus -from typing import Any, cast +from types import SimpleNamespace +from typing import TYPE_CHECKING, Any, cast from fastapi import HTTPException from PIL import Image @@ -29,7 +30,10 @@ from vllm_omni.entrypoints.openai.stage_params import ( get_default_sampling_params_list, ) from vllm_omni.entrypoints.openai.utils import get_stage_type, parse_lora_request -from vllm_omni.entrypoints.openai.video_api_utils import _encode_video_bytes, encode_video_base64 +from vllm_omni.entrypoints.openai.video_api_utils import ( + _encode_video_bytes, + encode_video_base64, +) from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniTextPrompt from vllm_omni.model_extras import should_preserve_reference_image_size from vllm_omni.outputs.output_metadata import ( @@ -40,6 +44,9 @@ from vllm_omni.outputs.output_metadata import ( logger = init_logger(__name__) +if TYPE_CHECKING: + from vllm_omni.diffusion.data import OmniDiffusionConfig + @dataclass class ReferenceImage: @@ -90,6 +97,12 @@ class OmniOpenAIServingVideo: self._model_name = model_name self._stage_configs = stage_configs + def _resolve_diffusion_od_config(self) -> OmniDiffusionConfig | SimpleNamespace | None: + get_od_config = getattr(self._engine_client, "get_diffusion_od_config", None) + if callable(get_od_config): + return get_od_config() + return getattr(self._engine_client, "od_config", None) + @property def model_name(self) -> str | None: return self._model_name @@ -105,8 +118,7 @@ class OmniOpenAIServingVideo: @cached_property def preserves_reference_image_size(self) -> bool: """Return whether the active pipeline owns reference-image resizing.""" - get_od_config = getattr(self._engine_client, "get_diffusion_od_config", None) - od_config = get_od_config() if callable(get_od_config) else getattr(self._engine_client, "od_config", None) + od_config = self._resolve_diffusion_od_config() model_class_name = None if od_config is None else getattr(od_config, "model_class_name", None) model = getattr(od_config, "model", None) if od_config is not None else None model = model or self.model_name @@ -120,8 +132,7 @@ class OmniOpenAIServingVideo: @property def supports_mixed_reference_inputs(self) -> bool: """Return whether the configured diffusion model accepts mixed refs.""" - get_od_config = getattr(self._engine_client, "get_diffusion_od_config", None) - od_config = get_od_config() if callable(get_od_config) else getattr(self._engine_client, "od_config", None) + od_config = self._resolve_diffusion_od_config() if od_config is None: return False @@ -332,7 +343,11 @@ class OmniOpenAIServingVideo: video_data = [ VideoData( b64_json=( - encode_video_base64(video, fps=artifacts.output_fps, video_codec_options=video_codec_options) + encode_video_base64( + video, + fps=artifacts.output_fps, + video_codec_options=video_codec_options, + ) if artifacts.audios[idx] is None else encode_video_base64( video, diff --git a/vllm_omni/entrypoints/openai/video_api_utils.py b/vllm_omni/entrypoints/openai/video_api_utils.py index d954bd821..53d087af4 100644 --- a/vllm_omni/entrypoints/openai/video_api_utils.py +++ b/vllm_omni/entrypoints/openai/video_api_utils.py @@ -10,14 +10,16 @@ import base64 import binascii import os import tempfile +from collections.abc import Iterator from io import BytesIO -from typing import Any, Literal +from typing import TYPE_CHECKING, Any, Literal, cast import httpx import numpy as np import torch from PIL import Image, UnidentifiedImageError from vllm import envs +from vllm.logger import init_logger from vllm.multimodal.video import ( VIDEO_LOADER_REGISTRY, VideoBackend, @@ -35,6 +37,20 @@ from vllm_omni.entrypoints.openai.protocol.videos import ( VideoReference, ) +if TYPE_CHECKING: + import av + + +logger = init_logger(__name__) + + +DEFAULT_AUDIO_SAMPLE_RATE = 24_000 + + +VideoInput = torch.Tensor | np.ndarray | list[torch.Tensor | np.ndarray | Image.Image] +AudioSample = int | float +AudioInput = torch.Tensor | np.ndarray | list[AudioSample] | list[list[AudioSample]] + class VideoFrames(list[Image.Image]): """Decoded video frames plus source metadata.""" @@ -398,7 +414,7 @@ def _normalize_single_video_array(video_array: np.ndarray) -> np.ndarray: video_array = np.transpose(video_array, (0, 2, 3, 1)) if np.issubdtype(video_array.dtype, np.floating): - if video_array.min() < 0.0 or video_array.max() > 1.0: + if video_array.size and (video_array.min() < 0.0 or video_array.max() > 1.0): video_array = np.clip(video_array, -1.0, 1.0) * 0.5 + 0.5 elif np.issubdtype(video_array.dtype, np.integer): video_array = video_array.astype(np.float32) / 255.0 @@ -432,7 +448,7 @@ def _normalize_frames(frames: list[Any]) -> list[np.ndarray]: frame_array = np.transpose(frame_array, (1, 2, 0)) if np.issubdtype(frame_array.dtype, np.floating): - if frame_array.min() < 0.0 or frame_array.max() > 1.0: + if frame_array.size and (frame_array.min() < 0.0 or frame_array.max() > 1.0): frame_array = np.clip(frame_array, -1.0, 1.0) * 0.5 + 0.5 elif np.issubdtype(frame_array.dtype, np.integer): frame_array = frame_array.astype(np.float32) / 255.0 @@ -486,17 +502,29 @@ def _coerce_audio_to_numpy(audio: Any) -> np.ndarray: return arr.astype(np.float32) -def _coerce_video_to_uint8_frames(video: Any) -> np.ndarray: - """Convert a video payload into contiguous uint8 frames shaped (F, H, W, 3).""" +def _prepare_video_frames(video: Any) -> tuple[list[np.ndarray], tuple[int, ...], np.dtype]: + """Normalize and validate frames for the common video encoding dispatcher.""" frames = _coerce_video_to_frames(video) if not frames: raise ValueError("No frames found to encode.") frame_shape = frames[0].shape + if any(frame.shape != frame_shape for frame in frames[1:]): + raise ValueError("All video frames must have the same shape.") + + common_dtype = np.result_type(*(frame.dtype for frame in frames)) + return frames, frame_shape, common_dtype + + +def _coerce_prepared_video_to_uint8_frames( + frames: list[np.ndarray], + frame_shape: tuple[int, ...], + common_dtype: np.dtype, +) -> np.ndarray: + """Convert prepared frames into contiguous uint8 frames for the legacy muxer.""" has_alpha = len(frame_shape) == 3 and frame_shape[-1] == 4 output_shape = (*frame_shape[:-1], 3) if has_alpha else frame_shape frames_u8 = np.empty((len(frames), *output_shape), dtype=np.uint8) - common_dtype = np.result_type(*(frame.dtype for frame in frames)) # Convert one frame at a time instead of stacking the normalized float # payload first. Long videos can otherwise require another full-size @@ -520,6 +548,145 @@ def _coerce_video_to_uint8_frames(video: Any) -> np.ndarray: return frames_u8 +def _coerce_video_to_uint8_frames(video: Any) -> np.ndarray: + """Convert a video payload into contiguous uint8 frames shaped (F, H, W, 3).""" + frames, frame_shape, common_dtype = _prepare_video_frames(video) + return _coerce_prepared_video_to_uint8_frames(frames, frame_shape, common_dtype) + + +def _direct_planar_fallback_reason( + frames: list[np.ndarray], + frame_shape: tuple[int, ...], + common_dtype: np.dtype, +) -> str | None: + """Return a stable reason when direct planar muxing cannot consume frames.""" + if len(frame_shape) != 3 or frame_shape[0] <= 0 or frame_shape[1] <= 0 or frame_shape[2] not in (3, 4): + return "unsupported_shape" + + if not ( + common_dtype == np.dtype(np.uint8) + or np.issubdtype(common_dtype, np.bool_) + or np.issubdtype(common_dtype, np.floating) + ): + return "unsupported_dtype" + + if not all(frame[..., channel].flags.c_contiguous for frame in frames for channel in range(3)): + return "non_contiguous_rgb_planes" + + return None + + +def _log_video_encoding_path( + *, + selected_path: str, + frames: list[np.ndarray], + frame_shape: tuple[int, ...], + common_dtype: np.dtype, + fps: int, + audio: AudioInput | None, + audio_sample_rate: int | None, + reason: str | None = None, +) -> None: + reason_field = "" if reason is None else f" reason={reason}" + logger.info( + "Video response encoding route selected: selected_path=%s%s frames=%d frame_shape=%s dtype=%s fps=%s " + "audio_present=%s effective_audio_sample_rate=%s", + selected_path, + reason_field, + len(frames), + frame_shape, + np.dtype(common_dtype).name, + fps, + audio is not None, + audio_sample_rate, + ) + + +def _resolve_audio_sample_rate(audio: AudioInput | None, audio_sample_rate: int | None) -> int: + if audio is not None and audio_sample_rate is None: + logger.info_once( + "Audio sample rate was not provided; using default sample rate of %s Hz.", + DEFAULT_AUDIO_SAMPLE_RATE, + ) + return audio_sample_rate or DEFAULT_AUDIO_SAMPLE_RATE + + +def _iter_planar_video_frames( + frames: list[np.ndarray], + common_dtype: np.dtype, +) -> Iterator[av.VideoFrame]: + """Yield planar PyAV frames while retaining only one channel scratch buffer.""" + import av + + height, width = frames[0].shape[:2] + scratch_dtype = np.float64 if np.issubdtype(common_dtype, np.bool_) else common_dtype + scratch = None if common_dtype == np.uint8 else np.empty((height, width), dtype=scratch_dtype) + + for frame in frames: + av_frame = av.VideoFrame(width, height, format="gbrp") + for plane, channel in zip(av_frame.planes, (1, 2, 0)): + if plane.height < height or plane.line_size < width: + raise ValueError("PyAV video plane is smaller than the requested frame dimensions.") + plane_view = np.frombuffer( + plane, + dtype=np.uint8, + count=plane.height * plane.line_size, + ).reshape(plane.height, plane.line_size) + plane_view.fill(0) + if frame.dtype == np.uint8: + plane_view[:height, :width] = frame[..., channel] + else: + scratch_buffer = cast(np.ndarray, scratch) + np.copyto(scratch_buffer, frame[..., channel], casting="unsafe") + np.clip(scratch_buffer, 0.0, 1.0, out=scratch_buffer) + scratch_buffer *= 255.0 + np.rint(scratch_buffer, out=scratch_buffer) + plane_view[:height, :width] = scratch_buffer + yield av_frame + + +def _encode_prepared_video_bytes_legacy( + frames: list[np.ndarray], + frame_shape: tuple[int, ...], + common_dtype: np.dtype, + fps: int, + audio: Any | None = None, + audio_sample_rate: int | None = None, + video_codec_options: dict[str, str] | None = None, +) -> bytes: + """Encode validated frames through the compatibility path used before planar encoding.""" + from vllm_omni.diffusion.utils.media_utils import mux_video_audio_bytes + + audio_np = _coerce_audio_to_numpy(audio) if audio is not None else None + return mux_video_audio_bytes( + _coerce_prepared_video_to_uint8_frames(frames, frame_shape, common_dtype), + audio_np, + fps=float(fps), + audio_sample_rate=audio_sample_rate or DEFAULT_AUDIO_SAMPLE_RATE, + video_codec_options=video_codec_options, + ) + + +def _encode_video_bytes_legacy( + video: Any, + fps: int, + audio: Any | None = None, + audio_sample_rate: int | None = None, + video_codec_options: dict[str, str] | None = None, +) -> bytes: + """Encode through the compatibility path used before planar encoding.""" + frames, frame_shape, common_dtype = _prepare_video_frames(video) + return _encode_prepared_video_bytes_legacy( + frames, + frame_shape, + common_dtype, + fps, + audio=audio, + audio_sample_rate=_resolve_audio_sample_rate(audio, audio_sample_rate), + video_codec_options=video_codec_options, + ) + + def _encode_video_bytes( video: Any, fps: int, @@ -527,16 +694,52 @@ def _encode_video_bytes( audio_sample_rate: int | None = None, video_codec_options: dict[str, str] | None = None, ) -> bytes: - """Encode a video payload into MP4 bytes, optionally muxing audio.""" - from vllm_omni.diffusion.utils.media_utils import mux_video_audio_bytes + """Encode a video payload through the automatic capability dispatcher.""" + from vllm_omni.diffusion.utils.media_utils import mux_av_video_audio_bytes + # Prepare once so validation is shared by both paths and malformed common + # input is reported before any muxer is opened. + frames, frame_shape, common_dtype = _prepare_video_frames(video) + effective_audio_sample_rate = _resolve_audio_sample_rate(audio, audio_sample_rate) if audio is not None else None + fallback_reason = _direct_planar_fallback_reason(frames, frame_shape, common_dtype) + if fallback_reason is not None: + _log_video_encoding_path( + selected_path="legacy_fallback", + reason=fallback_reason, + frames=frames, + frame_shape=frame_shape, + common_dtype=common_dtype, + fps=fps, + audio=audio, + audio_sample_rate=effective_audio_sample_rate, + ) + return _encode_prepared_video_bytes_legacy( + frames, + frame_shape, + common_dtype, + fps, + audio=audio, + audio_sample_rate=effective_audio_sample_rate, + video_codec_options=video_codec_options, + ) + + _log_video_encoding_path( + selected_path="direct_planar", + frames=frames, + frame_shape=frame_shape, + common_dtype=common_dtype, + fps=fps, + audio=audio, + audio_sample_rate=effective_audio_sample_rate, + ) audio_np = _coerce_audio_to_numpy(audio) if audio is not None else None - - return mux_video_audio_bytes( - _coerce_video_to_uint8_frames(video), - audio_np, + return mux_av_video_audio_bytes( + _iter_planar_video_frames(frames, common_dtype), + width=frame_shape[1], + height=frame_shape[0], + audio_waveform=audio_np, fps=float(fps), - audio_sample_rate=audio_sample_rate or 24000, + audio_sample_rate=effective_audio_sample_rate, video_codec_options=video_codec_options, ) @@ -599,6 +802,10 @@ def encode_video_base64( ) -> str: """Encode a video (frames/array/tensor) to base64 MP4.""" video_bytes = _encode_video_bytes( - video, fps=fps, audio=audio, audio_sample_rate=audio_sample_rate, video_codec_options=video_codec_options + video, + fps=fps, + audio=audio, + audio_sample_rate=audio_sample_rate, + video_codec_options=video_codec_options, ) return base64.b64encode(video_bytes).decode("utf-8")