[Model] Add soundfile fallback for MiniMax H3 audio loading (#5699)
Signed-off-by: brandneway <gyuan4892@gmail.com>
This commit is contained in:
1 parent
e0c2b9d73f
commit
a4ea67a21b
3 files changed
+215
-6
No files matched your search
@@ -65,6 +65,7 @@ from .presentation import (
|
||||
minimax_h3_text_only_ids,
|
||||
)
|
||||
from .reference_video import (
|
||||
load_audio_file,
|
||||
load_video_audio,
|
||||
load_video_frames,
|
||||
prepare_reference_videos,
|
||||
@@ -146,14 +147,12 @@ def _load_image(value: Any) -> Image.Image:
|
||||
|
||||
|
||||
def _load_audio(value: Any) -> tuple[torch.Tensor, int]:
|
||||
import torchaudio
|
||||
|
||||
if isinstance(value, list):
|
||||
if len(value) != 1:
|
||||
raise ValueError("MiniMax H3 currently supports exactly one audio")
|
||||
value = value[0]
|
||||
if isinstance(value, (str, os.PathLike)):
|
||||
return torchaudio.load(str(value))
|
||||
return load_audio_file(str(value))
|
||||
if isinstance(value, tuple) and len(value) == 2:
|
||||
waveform, sample_rate = value
|
||||
waveform = torch.as_tensor(waveform).float()
|
||||
|
||||
@@ -260,11 +260,58 @@ def sample_reference_video_frames(
|
||||
}
|
||||
|
||||
|
||||
def _soundfile_to_waveform(path: str) -> tuple[torch.Tensor, int]:
|
||||
import soundfile as sf
|
||||
|
||||
data, sample_rate = sf.read(path, dtype="float32", always_2d=True)
|
||||
# soundfile is (samples, channels); torchaudio.load is (channels, samples).
|
||||
waveform = torch.from_numpy(data).t().contiguous()
|
||||
return waveform, int(sample_rate)
|
||||
|
||||
|
||||
def load_audio_file(path: str) -> tuple[torch.Tensor, int]:
|
||||
"""Load an audio file as ``(waveform[C, T] float32, sample_rate)``.
|
||||
|
||||
torchaudio 2.6+ routes ``load`` through TorchCodec, whose wheels do not load
|
||||
on aarch64 with a CPU-only torch (they link CUDA libraries that are absent).
|
||||
Fall back to soundfile; if the container is not libsndfile-readable (e.g.
|
||||
mp3/m4a/mp4), demux to wav with ffmpeg first so any ffmpeg-supported input
|
||||
works at its native sample rate.
|
||||
"""
|
||||
try:
|
||||
import torchaudio
|
||||
|
||||
return torchaudio.load(path)
|
||||
except (ImportError, RuntimeError, OSError):
|
||||
pass
|
||||
try:
|
||||
return _soundfile_to_waveform(path)
|
||||
except Exception:
|
||||
import tempfile
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="minimax_h3_audio_") as tmpdir:
|
||||
wav = str(Path(tmpdir) / "audio.wav")
|
||||
subprocess.run(
|
||||
[
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-i",
|
||||
path,
|
||||
"-vn",
|
||||
"-f",
|
||||
"wav",
|
||||
wav,
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
return _soundfile_to_waveform(wav)
|
||||
|
||||
|
||||
def load_video_audio(path: str) -> tuple[torch.Tensor, int]:
|
||||
import tempfile
|
||||
|
||||
import torchaudio
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="minimax_h3_video_audio_") as tmpdir:
|
||||
output = str(Path(tmpdir) / "audio.wav")
|
||||
subprocess.run(
|
||||
@@ -286,10 +333,11 @@ def load_video_audio(path: str) -> tuple[torch.Tensor, int]:
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
return torchaudio.load(output)
|
||||
return load_audio_file(output)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"load_audio_file",
|
||||
"load_video_audio",
|
||||
"load_video_frames",
|
||||
"prepare_reference_videos",
|
||||
|
||||
Reference in new issue
Block a user