622 lines
23 KiB
Python
622 lines
23 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
"""MiniMax H3 Ref2VA reference-video preparation."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import math
|
|
import os
|
|
import subprocess
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import numpy as np
|
|
import torch
|
|
|
|
from vllm_omni.errors import OmniClientError
|
|
|
|
MINIMAX_H3_FPS = 24.0
|
|
MINIMAX_H3_QWEN_VIDEO_SAMPLE_FPS = 2.0
|
|
MINIMAX_H3_QWEN_TEMPORAL_PATCH = 2
|
|
MINIMAX_H3_BASE_SHORT_EDGE = 768
|
|
MINIMAX_H3_MAX_PIXELS = 768 * 1344
|
|
MINIMAX_H3_CANVAS_MULTIPLE = 32
|
|
MINIMAX_H3_MIN_REFERENCE_DIMENSION = 256
|
|
MINIMAX_H3_MAX_REFERENCE_DIMENSION = 5760
|
|
MINIMAX_H3_MIN_REFERENCE_FPS = 23.976
|
|
MINIMAX_H3_MAX_REFERENCE_FPS = 60.0
|
|
MINIMAX_H3_MIN_REFERENCE_DURATION = 2.0
|
|
MINIMAX_H3_MAX_REFERENCE_DURATION = 15.0
|
|
MINIMAX_H3_MAX_REFERENCE_TOTAL_DURATION = 15.0
|
|
# ffprobe reports container durations at a finer precision than the official
|
|
# example's rounded values. Trim only this small probe-rounding excess.
|
|
MINIMAX_H3_REFERENCE_DURATION_EPSILON = 1e-2
|
|
MINIMAX_H3_MAX_VIDEO_BYTES = 50 * 1024 * 1024
|
|
MINIMAX_H3_MAX_AUDIO_BYTES = 15 * 1024 * 1024
|
|
MINIMAX_H3_VIDEO_FORMATS = frozenset({"mp4", "mov"})
|
|
MINIMAX_H3_VIDEO_CODECS = frozenset({"h264", "h265", "hevc"})
|
|
MINIMAX_H3_AUDIO_FORMATS = frozenset({"wav", "mp3"})
|
|
MINIMAX_H3_AUDIO_CODECS = frozenset({"aac", "mp3"})
|
|
|
|
|
|
def _nearest_multiple(value: float, multiple: int) -> int:
|
|
return max(multiple, int(round(float(value) / multiple)) * multiple)
|
|
|
|
|
|
def _probe_video(path: str) -> dict[str, Any]:
|
|
show_entries = (
|
|
"stream=codec_type,codec_name,width,height,r_frame_rate,nb_read_frames,nb_frames,duration,"
|
|
"sample_aspect_ratio:format=format_name,size,duration"
|
|
)
|
|
command = ["ffprobe", "-v", "error", "-show_entries", show_entries, "-of", "json", path]
|
|
result = subprocess.run(command, check=True, capture_output=True, text=True)
|
|
probe = json.loads(result.stdout)
|
|
streams = probe.get("streams") or []
|
|
video_streams = [stream for stream in streams if stream.get("codec_type") == "video"]
|
|
if not video_streams:
|
|
raise OmniClientError(f"media has no video stream: {path}")
|
|
stream = video_streams[0]
|
|
raw_count = stream.get("nb_read_frames") or stream.get("nb_frames")
|
|
if raw_count in (None, "", "N/A"):
|
|
result = subprocess.run(
|
|
["ffprobe", "-v", "error", "-count_frames", *command[3:]],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
probe = json.loads(result.stdout)
|
|
streams = probe.get("streams") or []
|
|
video_streams = [stream for stream in streams if stream.get("codec_type") == "video"]
|
|
if not video_streams:
|
|
raise OmniClientError(f"media has no video stream: {path}")
|
|
stream = video_streams[0]
|
|
raw_count = stream.get("nb_read_frames") or stream.get("nb_frames")
|
|
try:
|
|
numerator, denominator = str(stream["r_frame_rate"]).split("/", 1)
|
|
fps = float(numerator) / float(denominator)
|
|
except (KeyError, TypeError, ValueError, ZeroDivisionError) as exc:
|
|
raise OmniClientError(f"cannot determine video FPS: {path}") from exc
|
|
if not math.isfinite(fps) or fps <= 0:
|
|
raise OmniClientError(f"cannot determine video FPS: {path}")
|
|
if raw_count in (None, "", "N/A"):
|
|
raise OmniClientError(f"cannot determine video frame count: {path}")
|
|
raw_duration = stream.get("duration")
|
|
if raw_duration in (None, "", "N/A"):
|
|
raw_duration = probe.get("format", {}).get("duration")
|
|
duration = float(raw_duration) if raw_duration not in (None, "", "N/A") else int(raw_count) / fps
|
|
format_info = probe.get("format") or {}
|
|
format_names = tuple(
|
|
name.strip().lower() for name in str(format_info.get("format_name", "")).split(",") if name.strip()
|
|
)
|
|
raw_size = format_info.get("size")
|
|
file_size = int(raw_size) if raw_size not in (None, "", "N/A") else os.path.getsize(path)
|
|
audio_codecs = tuple(
|
|
str(stream.get("codec_name", "")).lower()
|
|
for stream in streams
|
|
if stream.get("codec_type") == "audio" and stream.get("codec_name")
|
|
)
|
|
return {
|
|
"width": int(stream["width"]),
|
|
"height": int(stream["height"]),
|
|
"fps": fps,
|
|
"frame_count": int(raw_count),
|
|
"duration": duration,
|
|
"format_names": format_names,
|
|
"video_codec": str(stream.get("codec_name", "")).lower(),
|
|
"audio_codecs": audio_codecs,
|
|
"file_size": file_size,
|
|
}
|
|
|
|
|
|
def _validate_reference_video_metadata(meta: dict[str, Any], *, index: int, source: str) -> None:
|
|
width = int(meta["width"])
|
|
height = int(meta["height"])
|
|
if (
|
|
min(width, height) < MINIMAX_H3_MIN_REFERENCE_DIMENSION
|
|
or max(width, height) > MINIMAX_H3_MAX_REFERENCE_DIMENSION
|
|
):
|
|
raise OmniClientError(f"reference video {index} dimensions must be in [256, 5760] pixels, got {width}x{height}")
|
|
ratio = width / height
|
|
if not 0.4 <= ratio <= 2.5:
|
|
raise OmniClientError(f"reference video {index} aspect ratio must be in [0.4, 2.5], got {width}x{height}")
|
|
|
|
fps = float(meta["fps"])
|
|
if not MINIMAX_H3_MIN_REFERENCE_FPS <= fps <= MINIMAX_H3_MAX_REFERENCE_FPS:
|
|
raise OmniClientError(f"reference video {index} FPS must be in [23.976, 60], got {fps:.3f}")
|
|
duration = float(meta["duration"])
|
|
if (
|
|
not math.isfinite(duration)
|
|
or not MINIMAX_H3_MIN_REFERENCE_DURATION <= duration <= MINIMAX_H3_MAX_REFERENCE_DURATION
|
|
):
|
|
raise OmniClientError(f"reference video {index} duration must be in [2, 15] seconds, got {duration:.3f}")
|
|
|
|
raw_file_size = meta.get("file_size")
|
|
file_size = int(raw_file_size) if raw_file_size is not None else os.path.getsize(source)
|
|
if file_size > MINIMAX_H3_MAX_VIDEO_BYTES:
|
|
raise OmniClientError(f"reference video {index} exceeds the 50 MiB size limit")
|
|
format_names = set(meta.get("format_names", ()))
|
|
if not format_names.intersection(MINIMAX_H3_VIDEO_FORMATS):
|
|
formats = ", ".join(sorted(format_names)) or "unknown"
|
|
raise OmniClientError(f"reference video {index} must use an MP4 or MOV container, got {formats}")
|
|
video_codec = str(meta.get("video_codec", "")).lower()
|
|
if video_codec not in MINIMAX_H3_VIDEO_CODECS:
|
|
raise OmniClientError(f"reference video {index} must use H.264 or H.265 video, got {video_codec or 'unknown'}")
|
|
audio_codecs = tuple(str(codec).lower() for codec in meta.get("audio_codecs", ()))
|
|
invalid_audio = [codec for codec in audio_codecs if codec not in MINIMAX_H3_AUDIO_CODECS]
|
|
if invalid_audio:
|
|
raise OmniClientError(
|
|
f"reference video {index} must use AAC or MP3 audio when audio is present, got {invalid_audio[0]}"
|
|
)
|
|
|
|
|
|
def _audio_items(values: Any) -> list[Any]:
|
|
if isinstance(values, (str, os.PathLike)):
|
|
return [values]
|
|
if isinstance(values, tuple) and len(values) == 2 and isinstance(values[1], (int, np.integer)):
|
|
return [values]
|
|
if isinstance(values, (list, tuple)):
|
|
return list(values)
|
|
return [values]
|
|
|
|
|
|
def _probe_audio(path: str) -> dict[str, Any]:
|
|
result = subprocess.run(
|
|
[
|
|
"ffprobe",
|
|
"-v",
|
|
"error",
|
|
"-show_entries",
|
|
"stream=codec_type,codec_name,duration:format=format_name,size,duration",
|
|
"-of",
|
|
"json",
|
|
path,
|
|
],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
probe = json.loads(result.stdout)
|
|
streams = [stream for stream in probe.get("streams", []) if stream.get("codec_type") == "audio"]
|
|
if not streams:
|
|
raise OmniClientError(f"media has no audio stream: {path}")
|
|
stream = streams[0]
|
|
raw_duration = stream.get("duration")
|
|
if raw_duration in (None, "", "N/A"):
|
|
raw_duration = (probe.get("format") or {}).get("duration")
|
|
if raw_duration in (None, "", "N/A"):
|
|
raise OmniClientError(f"cannot determine audio duration: {path}")
|
|
format_info = probe.get("format") or {}
|
|
format_names = tuple(
|
|
name.strip().lower() for name in str(format_info.get("format_name", "")).split(",") if name.strip()
|
|
)
|
|
raw_size = format_info.get("size")
|
|
file_size = int(raw_size) if raw_size not in (None, "", "N/A") else os.path.getsize(path)
|
|
return {
|
|
"duration": float(raw_duration),
|
|
"format_names": format_names,
|
|
"codec": str(stream.get("codec_name", "")).lower(),
|
|
"file_size": file_size,
|
|
}
|
|
|
|
|
|
def validate_reference_audio_files(values: Any) -> None:
|
|
"""Validate path-backed standalone audio references against the H3 spec."""
|
|
total_duration = 0.0
|
|
for index, value in enumerate(_audio_items(values)):
|
|
if not isinstance(value, (str, os.PathLike)):
|
|
continue
|
|
source = str(value)
|
|
meta = _probe_audio(source)
|
|
file_size = int(meta["file_size"])
|
|
if file_size > MINIMAX_H3_MAX_AUDIO_BYTES:
|
|
raise OmniClientError(f"reference audio {index} exceeds the 15 MiB size limit")
|
|
if not set(meta["format_names"]).intersection(MINIMAX_H3_AUDIO_FORMATS):
|
|
formats = ", ".join(meta["format_names"]) or "unknown"
|
|
raise OmniClientError(f"reference audio {index} must be WAV or MP3, got {formats}")
|
|
codec = str(meta["codec"]).lower()
|
|
if codec not in MINIMAX_H3_AUDIO_CODECS and not codec.startswith("pcm_"):
|
|
raise OmniClientError(f"reference audio {index} must use MP3 or PCM audio, got {codec or 'unknown'}")
|
|
duration = float(meta["duration"])
|
|
if (
|
|
not math.isfinite(duration)
|
|
or not MINIMAX_H3_MIN_REFERENCE_DURATION <= duration <= MINIMAX_H3_MAX_REFERENCE_DURATION
|
|
):
|
|
raise OmniClientError(f"reference audio {index} duration must be in [2, 15] seconds, got {duration:.3f}")
|
|
total_duration += duration
|
|
if total_duration > MINIMAX_H3_MAX_REFERENCE_TOTAL_DURATION:
|
|
raise OmniClientError("reference audio must be at most 15 seconds in total")
|
|
|
|
|
|
def validate_reference_audio_waveforms(values: list[tuple[torch.Tensor, int]]) -> None:
|
|
"""Validate in-memory standalone audio references using sample counts."""
|
|
total_duration = 0.0
|
|
for index, (waveform, sample_rate) in enumerate(values):
|
|
if int(sample_rate) <= 0:
|
|
raise OmniClientError(f"reference audio {index} has an invalid sample rate")
|
|
samples = int(waveform.shape[-1])
|
|
duration = samples / int(sample_rate)
|
|
if not MINIMAX_H3_MIN_REFERENCE_DURATION <= duration <= MINIMAX_H3_MAX_REFERENCE_DURATION:
|
|
raise OmniClientError(f"reference audio {index} duration must be in [2, 15] seconds, got {duration:.3f}")
|
|
total_duration += duration
|
|
if total_duration > MINIMAX_H3_MAX_REFERENCE_TOTAL_DURATION:
|
|
raise OmniClientError("reference audio must be at most 15 seconds in total")
|
|
|
|
|
|
def _reference_video_shape(width: int, height: int) -> tuple[int, int]:
|
|
if (
|
|
min(width, height) < MINIMAX_H3_MIN_REFERENCE_DIMENSION
|
|
or max(width, height) > MINIMAX_H3_MAX_REFERENCE_DIMENSION
|
|
):
|
|
raise OmniClientError(f"reference video dimensions must be in [256, 5760] pixels, got {width}x{height}")
|
|
ratio = float(width) / float(height)
|
|
if not 0.4 <= ratio <= 2.5:
|
|
raise OmniClientError(f"reference video aspect ratio must be in [0.4, 2.5], got {width}x{height}")
|
|
if ratio >= 1.0:
|
|
target_width = MINIMAX_H3_BASE_SHORT_EDGE * ratio
|
|
target_height = float(MINIMAX_H3_BASE_SHORT_EDGE)
|
|
else:
|
|
target_width = float(MINIMAX_H3_BASE_SHORT_EDGE)
|
|
target_height = MINIMAX_H3_BASE_SHORT_EDGE / ratio
|
|
area = target_width * target_height
|
|
if area > MINIMAX_H3_MAX_PIXELS:
|
|
scale = math.sqrt(MINIMAX_H3_MAX_PIXELS / area)
|
|
target_width *= scale
|
|
target_height *= scale
|
|
return (
|
|
_nearest_multiple(target_width, MINIMAX_H3_CANVAS_MULTIPLE),
|
|
_nearest_multiple(target_height, MINIMAX_H3_CANVAS_MULTIPLE),
|
|
)
|
|
|
|
|
|
def _transcode_reference_video(
|
|
source: str,
|
|
*,
|
|
target_width: int,
|
|
target_height: int,
|
|
target_frame_count: int,
|
|
workdir: str,
|
|
start_time_seconds: float = 0.0,
|
|
duration_seconds: float | None = None,
|
|
) -> str:
|
|
output = str(Path(workdir) / "prepared.mp4")
|
|
duration_args = ["-t", f"{float(duration_seconds):.6f}"] if duration_seconds is not None else []
|
|
subprocess.run(
|
|
[
|
|
"ffmpeg",
|
|
"-y",
|
|
"-loglevel",
|
|
"error",
|
|
"-ss",
|
|
f"{float(start_time_seconds):.6f}",
|
|
"-i",
|
|
source,
|
|
"-map",
|
|
"0:v:0",
|
|
"-an",
|
|
"-vf",
|
|
(f"fps={MINIMAX_H3_FPS:g},scale={target_width}:{target_height}:flags=lanczos,setsar=1"),
|
|
*duration_args,
|
|
"-frames:v",
|
|
str(int(target_frame_count)),
|
|
"-metadata:s:v:0",
|
|
"rotate=0",
|
|
# Keep the transformed RGB pixels lossless. The same prepared
|
|
# stream is decoded by Qwen3VL and the video VAE; a yuv420p
|
|
# intermediate would make the two consumers see different
|
|
# pixels from the model-card reference recipe.
|
|
"-c:v",
|
|
"libx264rgb",
|
|
"-crf",
|
|
"0",
|
|
"-preset",
|
|
"veryfast",
|
|
"-pix_fmt",
|
|
"rgb24",
|
|
output,
|
|
],
|
|
check=True,
|
|
)
|
|
return output
|
|
|
|
|
|
def prepare_reference_videos(
|
|
values: Any,
|
|
*,
|
|
target_frame_count: int,
|
|
workdir: str,
|
|
start_time_seconds: Any = None,
|
|
) -> list[dict[str, Any]]:
|
|
if isinstance(values, (str, os.PathLike)):
|
|
values = [values]
|
|
if not isinstance(values, (list, tuple)) or not values:
|
|
raise OmniClientError("MiniMax H3 Ref2VA video input must be a path or a non-empty list of paths")
|
|
if isinstance(start_time_seconds, (list, tuple)):
|
|
start_times = list(start_time_seconds)
|
|
else:
|
|
start_times = [start_time_seconds] * len(values)
|
|
if len(start_times) != len(values):
|
|
raise OmniClientError("start_time_seconds must contain one value per reference video")
|
|
|
|
prepared: list[dict[str, Any]] = []
|
|
total_duration = 0.0
|
|
for index, value in enumerate(values):
|
|
item_start = start_times[index]
|
|
if isinstance(value, dict):
|
|
item_start = value.get("start_time_seconds", item_start)
|
|
value = value.get("path", value.get("video_path", value.get("video_url")))
|
|
if not isinstance(value, (str, os.PathLike)):
|
|
raise OmniClientError(
|
|
f"MiniMax H3 Ref2VA video references require file paths, got item {index}: {type(value)!r}"
|
|
)
|
|
source = str(value)
|
|
meta = _probe_video(source)
|
|
_validate_reference_video_metadata(meta, index=index, source=source)
|
|
start = 0.0 if item_start is None else float(item_start)
|
|
if not math.isfinite(start) or start < 0 or start >= float(meta["duration"]):
|
|
raise OmniClientError(f"reference video {index} start_time_seconds is outside the source duration")
|
|
duration = float(meta["duration"]) - start
|
|
if duration < MINIMAX_H3_MIN_REFERENCE_DURATION:
|
|
raise OmniClientError(
|
|
f"reference video {index} must provide at least 2 seconds after start_time_seconds, got {duration:.3f}"
|
|
)
|
|
remaining_duration = MINIMAX_H3_MAX_REFERENCE_TOTAL_DURATION - total_duration
|
|
if duration > remaining_duration:
|
|
if duration - remaining_duration > MINIMAX_H3_REFERENCE_DURATION_EPSILON:
|
|
raise OmniClientError("reference videos must be at most 15 seconds in total")
|
|
duration = remaining_duration
|
|
if duration < MINIMAX_H3_MIN_REFERENCE_DURATION:
|
|
raise OmniClientError("reference videos must be at most 15 seconds in total")
|
|
total_duration += duration
|
|
width, height = _reference_video_shape(meta["width"], meta["height"])
|
|
item_workdir = Path(workdir) / f"video_{index}"
|
|
item_workdir.mkdir(parents=True)
|
|
prepared_path = _transcode_reference_video(
|
|
source,
|
|
target_width=width,
|
|
target_height=height,
|
|
target_frame_count=target_frame_count,
|
|
workdir=str(item_workdir),
|
|
start_time_seconds=start,
|
|
duration_seconds=duration,
|
|
)
|
|
prepared.append(
|
|
{
|
|
"original_path": source,
|
|
"prepared_path": prepared_path,
|
|
"input_has_audio": bool(meta.get("audio_codecs")),
|
|
"width": width,
|
|
"height": height,
|
|
"start_time_seconds": start,
|
|
"duration_seconds": duration,
|
|
# Reference video frames and their soundtrack are conditioned
|
|
# only for the generated temporal extent. Keep the source
|
|
# segment duration for metadata/validation, and expose the
|
|
# bounded duration separately for audio extraction.
|
|
"audio_duration_seconds": min(
|
|
duration,
|
|
float(target_frame_count) / MINIMAX_H3_FPS,
|
|
),
|
|
}
|
|
)
|
|
return prepared
|
|
|
|
|
|
def load_video_frames(path: str) -> np.ndarray:
|
|
try:
|
|
import decord
|
|
except ImportError:
|
|
meta = _probe_video(path)
|
|
return _decode_video_frames_ffmpeg(
|
|
path,
|
|
frame_count=int(meta["frame_count"]),
|
|
width=int(meta["width"]),
|
|
height=int(meta["height"]),
|
|
)
|
|
|
|
reader = decord.VideoReader(path)
|
|
if len(reader) <= 0:
|
|
raise OmniClientError(f"video has no frames: {path}")
|
|
frames = reader.get_batch(list(range(len(reader))))
|
|
return frames.asnumpy() if hasattr(frames, "asnumpy") else np.asarray(frames)
|
|
|
|
|
|
def _decode_video_frames_ffmpeg(
|
|
path: str,
|
|
*,
|
|
frame_count: int,
|
|
width: int,
|
|
height: int,
|
|
indices: list[int] | None = None,
|
|
) -> np.ndarray:
|
|
frame_count = int(frame_count)
|
|
width = int(width)
|
|
height = int(height)
|
|
if frame_count <= 0:
|
|
raise OmniClientError(f"video has no frames: {path}")
|
|
if width <= 0 or height <= 0:
|
|
raise OmniClientError(f"video has invalid dimensions {width}x{height}: {path}")
|
|
if indices is not None and (not indices or any(index < 0 or index >= frame_count for index in indices)):
|
|
raise OmniClientError(f"invalid frame indices for {path}: {indices}")
|
|
|
|
output_frame_count = frame_count if indices is None else len(indices)
|
|
command = [
|
|
"ffmpeg",
|
|
"-loglevel",
|
|
"error",
|
|
"-threads",
|
|
"0",
|
|
"-i",
|
|
path,
|
|
"-map",
|
|
"0:v:0",
|
|
"-an",
|
|
]
|
|
if indices is not None:
|
|
select = "+".join(f"eq(n\\,{index})" for index in indices)
|
|
command.extend(["-vf", f"select={select}"])
|
|
command.extend(
|
|
[
|
|
"-vsync",
|
|
"0",
|
|
"-frames:v",
|
|
str(output_frame_count),
|
|
"-f",
|
|
"rawvideo",
|
|
"-pix_fmt",
|
|
"rgb24",
|
|
"pipe:1",
|
|
]
|
|
)
|
|
result = subprocess.run(
|
|
command,
|
|
check=True,
|
|
capture_output=True,
|
|
)
|
|
frame_size = width * height * 3
|
|
expected_size = output_frame_count * frame_size
|
|
if len(result.stdout) != expected_size:
|
|
raise OmniClientError(f"decoded {len(result.stdout)} bytes from {path}, expected {expected_size}")
|
|
return np.frombuffer(result.stdout, dtype=np.uint8).reshape(
|
|
output_frame_count,
|
|
height,
|
|
width,
|
|
3,
|
|
)
|
|
|
|
|
|
def sample_reference_video_frames(prepared_path: str) -> dict[str, Any]:
|
|
meta = _probe_video(prepared_path)
|
|
ratio = MINIMAX_H3_FPS / MINIMAX_H3_QWEN_VIDEO_SAMPLE_FPS
|
|
indices: list[int] = []
|
|
cursor = 0.0
|
|
while True:
|
|
frame_index = int(round(cursor))
|
|
if frame_index >= meta["frame_count"]:
|
|
break
|
|
if not indices or frame_index > indices[-1]:
|
|
indices.append(frame_index)
|
|
cursor += ratio
|
|
if not indices:
|
|
raise OmniClientError(f"no frames sampled from {prepared_path}")
|
|
|
|
# The preparation step emits a lossless RGB stream. Decode only the sampled
|
|
# frames in one ffmpeg process so Qwen3VL sees the exact prepared pixels
|
|
# without decoding the entire high-bitrate stream into host memory.
|
|
decoded_frames = _decode_video_frames_ffmpeg(
|
|
prepared_path,
|
|
frame_count=int(meta["frame_count"]),
|
|
indices=indices,
|
|
width=meta["width"],
|
|
height=meta["height"],
|
|
)
|
|
frames = [np.asarray(frame) for frame in decoded_frames]
|
|
|
|
timestamps = [index / MINIMAX_H3_QWEN_VIDEO_SAMPLE_FPS for index in range(len(indices))]
|
|
timestamps += [timestamps[-1]] * ((-len(timestamps)) % MINIMAX_H3_QWEN_TEMPORAL_PATCH)
|
|
block_timestamps = [
|
|
(timestamps[index] + timestamps[index + MINIMAX_H3_QWEN_TEMPORAL_PATCH - 1]) / 2
|
|
for index in range(
|
|
0,
|
|
len(timestamps),
|
|
MINIMAX_H3_QWEN_TEMPORAL_PATCH,
|
|
)
|
|
]
|
|
return {
|
|
"frames": frames,
|
|
"block_timestamps": block_timestamps,
|
|
}
|
|
|
|
|
|
def _soundfile_to_waveform(path: str) -> tuple[torch.Tensor, int]:
|
|
import soundfile as sf
|
|
|
|
data, sample_rate = sf.read(path, dtype="float32", always_2d=True)
|
|
# soundfile is (samples, channels); torchaudio.load is (channels, samples).
|
|
waveform = torch.from_numpy(data).t().contiguous()
|
|
return waveform, int(sample_rate)
|
|
|
|
|
|
def load_audio_file(path: str) -> tuple[torch.Tensor, int]:
|
|
"""Load an audio file as ``(waveform[C, T] float32, sample_rate)``.
|
|
|
|
torchaudio 2.6+ routes ``load`` through TorchCodec, whose wheels do not load
|
|
on aarch64 with a CPU-only torch (they link CUDA libraries that are absent).
|
|
Fall back to soundfile; if the container is not libsndfile-readable (e.g.
|
|
mp3/m4a/mp4), demux to wav with ffmpeg first so any ffmpeg-supported input
|
|
works at its native sample rate.
|
|
"""
|
|
try:
|
|
import torchaudio
|
|
|
|
return torchaudio.load(path)
|
|
except (ImportError, RuntimeError, OSError):
|
|
pass
|
|
try:
|
|
return _soundfile_to_waveform(path)
|
|
except Exception:
|
|
import tempfile
|
|
|
|
with tempfile.TemporaryDirectory(prefix="minimax_h3_audio_") as tmpdir:
|
|
wav = str(Path(tmpdir) / "audio.wav")
|
|
subprocess.run(
|
|
[
|
|
"ffmpeg",
|
|
"-y",
|
|
"-loglevel",
|
|
"error",
|
|
"-i",
|
|
path,
|
|
"-vn",
|
|
"-f",
|
|
"wav",
|
|
wav,
|
|
],
|
|
check=True,
|
|
)
|
|
return _soundfile_to_waveform(wav)
|
|
|
|
|
|
def load_video_audio(
|
|
path: str,
|
|
*,
|
|
start_time_seconds: float = 0.0,
|
|
duration_seconds: float | None = None,
|
|
) -> tuple[torch.Tensor, int]:
|
|
import tempfile
|
|
|
|
with tempfile.TemporaryDirectory(prefix="minimax_h3_video_audio_") as tmpdir:
|
|
output = str(Path(tmpdir) / "audio.wav")
|
|
command = [
|
|
"ffmpeg",
|
|
"-y",
|
|
"-loglevel",
|
|
"error",
|
|
"-ss",
|
|
f"{float(start_time_seconds):.6f}",
|
|
"-i",
|
|
path,
|
|
"-vn",
|
|
"-ac",
|
|
"2",
|
|
"-ar",
|
|
"44100",
|
|
"-f",
|
|
"wav",
|
|
]
|
|
if duration_seconds is not None:
|
|
command.extend(["-t", f"{float(duration_seconds):.6f}"])
|
|
command.append(output)
|
|
subprocess.run(command, check=True)
|
|
return load_audio_file(output)
|
|
|
|
|
|
__all__ = [
|
|
"load_audio_file",
|
|
"load_video_audio",
|
|
"load_video_frames",
|
|
"prepare_reference_videos",
|
|
"sample_reference_video_frames",
|
|
"validate_reference_audio_files",
|
|
"validate_reference_audio_waveforms",
|
|
]
|