[Model] Add soundfile fallback for MiniMax H3 audio loading (#5699)

Signed-off-by: brandneway <gyuan4892@gmail.com>
This commit is contained in:
ooooooye authored and GitHub committed 2026-08-03 16:24:50 +08:00
1 parent e0c2b9d73f
commit a4ea67a21b
3 files changed
+215 -6

No files matched your search

@@ -65,6 +65,7 @@ from .presentation import (
minimax_h3_text_only_ids,
)
from .reference_video import (
load_audio_file,
load_video_audio,
load_video_frames,
prepare_reference_videos,
@@ -146,14 +147,12 @@ def _load_image(value: Any) -> Image.Image:
def _load_audio(value: Any) -> tuple[torch.Tensor, int]:
import torchaudio
if isinstance(value, list):
if len(value) != 1:
raise ValueError("MiniMax H3 currently supports exactly one audio")
value = value[0]
if isinstance(value, (str, os.PathLike)):
return torchaudio.load(str(value))
return load_audio_file(str(value))
if isinstance(value, tuple) and len(value) == 2:
waveform, sample_rate = value
waveform = torch.as_tensor(waveform).float()
@@ -260,11 +260,58 @@ def sample_reference_video_frames(
}
def _soundfile_to_waveform(path: str) -> tuple[torch.Tensor, int]:
import soundfile as sf
data, sample_rate = sf.read(path, dtype="float32", always_2d=True)
# soundfile is (samples, channels); torchaudio.load is (channels, samples).
waveform = torch.from_numpy(data).t().contiguous()
return waveform, int(sample_rate)
def load_audio_file(path: str) -> tuple[torch.Tensor, int]:
"""Load an audio file as ``(waveform[C, T] float32, sample_rate)``.
torchaudio 2.6+ routes ``load`` through TorchCodec, whose wheels do not load
on aarch64 with a CPU-only torch (they link CUDA libraries that are absent).
Fall back to soundfile; if the container is not libsndfile-readable (e.g.
mp3/m4a/mp4), demux to wav with ffmpeg first so any ffmpeg-supported input
works at its native sample rate.
"""
try:
import torchaudio
return torchaudio.load(path)
except (ImportError, RuntimeError, OSError):
pass
try:
return _soundfile_to_waveform(path)
except Exception:
import tempfile
with tempfile.TemporaryDirectory(prefix="minimax_h3_audio_") as tmpdir:
wav = str(Path(tmpdir) / "audio.wav")
subprocess.run(
[
"ffmpeg",
"-y",
"-loglevel",
"error",
"-i",
path,
"-vn",
"-f",
"wav",
wav,
],
check=True,
)
return _soundfile_to_waveform(wav)
def load_video_audio(path: str) -> tuple[torch.Tensor, int]:
import tempfile
import torchaudio
with tempfile.TemporaryDirectory(prefix="minimax_h3_video_audio_") as tmpdir:
output = str(Path(tmpdir) / "audio.wav")
subprocess.run(
@@ -286,10 +333,11 @@ def load_video_audio(path: str) -> tuple[torch.Tensor, int]:
],
check=True,
)
return torchaudio.load(output)
return load_audio_file(output)
__all__ = [
"load_audio_file",
"load_video_audio",
"load_video_frames",
"prepare_reference_videos",