[Model] Add YuE2-3B text-to-music (model + /v1/audio/speech adapter) (#7886)
Signed-off-by: Youyuan Li <105433145+Darcy-Lee@users.noreply.github.com> Signed-off-by: princepride <wangzhipeng628@gmail.com> Co-authored-by: princepride <wangzhipeng628@gmail.com>
This commit is contained in:
1 parent
a038b38179
commit
423f34326e
27 files changed
+3294
-6
No files matched your search
@@ -0,0 +1,6 @@
|
||||
X:1
|
||||
T:Twinkle Twinkle Little Star (SheetSage2 transcription of the Wikimedia piano rendition, single voice)
|
||||
M:4/4
|
||||
L:1/16
|
||||
K:C
|
||||
C4C4G4G4|A4A4G8|F4F4E4E4|D4D4C8|G4G4F4F4|E4E4D8|G4G4F4F4|E4E4D8|C4C4G4G4|A4A4G8|F4F4E4E4|D4D4C8|
|
||||
@@ -167,7 +167,7 @@ def test_inventory_matches_reviewed_snapshot_counts():
|
||||
EnvironmentVariableCategory.PUBLIC_OMNI: 32,
|
||||
EnvironmentVariableCategory.INHERITED_VLLM: 20,
|
||||
EnvironmentVariableCategory.PLATFORM_EXTERNAL: 29,
|
||||
EnvironmentVariableCategory.MODEL_SPECIFIC: 74,
|
||||
EnvironmentVariableCategory.MODEL_SPECIFIC: 75,
|
||||
EnvironmentVariableCategory.BENCHMARK_TRANSITIONAL: 21,
|
||||
EnvironmentVariableCategory.INTERNAL: 6,
|
||||
}
|
||||
@@ -178,7 +178,7 @@ def test_inventory_matches_reviewed_snapshot_counts():
|
||||
if item.category is EnvironmentVariableCategory.MODEL_SPECIFIC
|
||||
)
|
||||
assert {disposition: disposition_counts[disposition] for disposition in ModelEnvironmentVariableDisposition} == {
|
||||
ModelEnvironmentVariableDisposition.PROMOTE: 45,
|
||||
ModelEnvironmentVariableDisposition.PROMOTE: 46,
|
||||
ModelEnvironmentVariableDisposition.REQUEST_SCOPE: 5,
|
||||
ModelEnvironmentVariableDisposition.EXTERNAL: 0,
|
||||
ModelEnvironmentVariableDisposition.INTERNALIZE: 17,
|
||||
|
||||
@@ -1048,3 +1048,83 @@ def test_minimax_music3_speech_invalid_field_values(
|
||||
online_client.send_audio_speech_http_request(
|
||||
{"json": body, "timeout": 120, "err_code": 400, "err_message": err_message}
|
||||
)
|
||||
|
||||
|
||||
# ─── POST /v1/audio/speech · YuE2-3B ───
|
||||
|
||||
# Text-to-music like MiniMax Music 3, with one twist: the ABC score span is
|
||||
# request-controlled via extra_params.cot/abc and the two must agree, and
|
||||
# sampling is pinned by the checkpoint preset so sampling knobs are rejected.
|
||||
_YUE2_SPEECH = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model="m-a-p/YuE2-3B",
|
||||
stage_config_path=get_deploy_config_path("yue2.yaml"),
|
||||
server_args=_SPEECH_SERVER_ARGS,
|
||||
),
|
||||
id="yue2",
|
||||
marks=hardware_marks(res={"cuda": "H100"}),
|
||||
),
|
||||
]
|
||||
|
||||
_YUE2_LYRICS = "一闪一闪亮晶晶\n满天都是小星星"
|
||||
_YUE2_CAPTION = "Chinese heavy metal, distorted electric guitars, 140 BPM, key of C."
|
||||
_YUE2_ABC = "X:1\nT:Twinkle\nM:4/4\nL:1/4\nK:C\nC C G G|A A G2|"
|
||||
# 25 frames per second; the semantic preset bounds the frame budget.
|
||||
_YUE2_MAX_FRAMES = 9000
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"overrides, err_message",
|
||||
[
|
||||
pytest.param({"instructions": None}, ("instructions",), id="caption_missing"),
|
||||
pytest.param({"instructions": " "}, ("instructions",), id="caption_blank"),
|
||||
pytest.param({"input": " "}, ("input",), id="lyrics_blank"),
|
||||
pytest.param({"voice": "alloy"}, ("voice",), id="voice_rejected"),
|
||||
pytest.param({"speed": 1.5}, ("speed",), id="speed_rejected"),
|
||||
pytest.param(
|
||||
{"max_new_tokens": _YUE2_MAX_FRAMES + 1},
|
||||
("max_new_tokens", str(_YUE2_MAX_FRAMES)),
|
||||
id="max_new_tokens_over_cap",
|
||||
),
|
||||
pytest.param({"max_new_tokens": 199}, ("max_new_tokens",), id="max_new_tokens_below_min"),
|
||||
pytest.param(
|
||||
{"extra_params": {"cot": "melody"}},
|
||||
("abc",),
|
||||
id="cot_melody_without_abc",
|
||||
),
|
||||
pytest.param(
|
||||
{"extra_params": {"abc": _YUE2_ABC}},
|
||||
("cot",),
|
||||
id="abc_without_cot",
|
||||
),
|
||||
pytest.param(
|
||||
{"extra_params": {"temperature": 0.3}},
|
||||
("temperature",),
|
||||
id="temperature_rejected",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("omni_server", _YUE2_SPEECH, indirect=True)
|
||||
def test_yue2_speech_invalid_field_values(
|
||||
omni_server: OmniServer,
|
||||
online_client: OnlineOmniClient,
|
||||
overrides: dict[str, object],
|
||||
err_message: str | tuple[str, ...],
|
||||
) -> None:
|
||||
body: dict[str, Any] = {
|
||||
"model": omni_server.model,
|
||||
"input": _YUE2_LYRICS,
|
||||
"instructions": _YUE2_CAPTION,
|
||||
"seed": 1,
|
||||
"max_new_tokens": 250,
|
||||
"response_format": "wav",
|
||||
}
|
||||
for key, value in overrides.items():
|
||||
if value is None:
|
||||
body.pop(key, None)
|
||||
else:
|
||||
body[key] = value
|
||||
online_client.send_audio_speech_http_request(
|
||||
{"json": body, "timeout": 120, "err_code": 400, "err_message": err_message}
|
||||
)
|
||||
@@ -0,0 +1,82 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""End-to-end regressions for YuE2 offline WAV export."""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import soundfile as sf
|
||||
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.runtime import get_model_prefix
|
||||
from vllm_omni.transformers_utils.repo_utils import hf_api
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.full_model,
|
||||
pytest.mark.tts,
|
||||
*hardware_marks(res={"cuda": ["L4", "H100", "B200"]}, num_cards=1),
|
||||
]
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def vae_path() -> Path:
|
||||
path = Path(os.environ.get("YUE2_VAE", get_model_prefix() + "m-a-p/YuE2-Vae"))
|
||||
if not path.is_dir():
|
||||
path = Path(hf_api().snapshot_download(str(path), allow_patterns=["*.json", "*.safetensors"]))
|
||||
return path
|
||||
|
||||
|
||||
@pytest.mark.parametrize("max_frames,truncated", [(200, True), (9000, False)], ids=["budget", "natural-eos"])
|
||||
def test_offline_example_saves_audio(vae_path: Path, tmp_path: Path, max_frames: int, truncated: bool) -> None:
|
||||
model = Path(os.environ.get("YUE2_MODEL_DIR", get_model_prefix() + "m-a-p/YuE2-3B"))
|
||||
if not model.is_dir():
|
||||
model = Path(
|
||||
hf_api().snapshot_download(str(model), allow_patterns=["*.json", "*.safetensors", "qwen.tiktoken"])
|
||||
)
|
||||
output = tmp_path / "song.wav"
|
||||
env = os.environ.copy()
|
||||
env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
env["PYTHONPATH"] = str(REPO_ROOT) + os.pathsep + env.get("PYTHONPATH", "")
|
||||
completed = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "examples/offline_inference/yue2/end2end.py"),
|
||||
"--model",
|
||||
str(model),
|
||||
"--vae",
|
||||
str(vae_path),
|
||||
"--style",
|
||||
"gentle acoustic folk, clear English vocals, 90 BPM",
|
||||
"--lyrics",
|
||||
"[Verse]\nMorning light across the bay\nCarry all my dreams away\n[Outro]",
|
||||
"--cot",
|
||||
"off",
|
||||
"--seed",
|
||||
"831001",
|
||||
"--max-frames",
|
||||
str(max_frames),
|
||||
"--output",
|
||||
str(output),
|
||||
],
|
||||
cwd=REPO_ROOT,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=900,
|
||||
)
|
||||
assert completed.returncode == 0, completed.stdout + completed.stderr
|
||||
assert f"truncated={truncated}" in completed.stdout
|
||||
audio, sample_rate = sf.read(output, always_2d=True)
|
||||
assert sample_rate == 48000 and audio.shape[1] == 2
|
||||
assert np.isfinite(audio).all() and np.abs(audio).max() > 0
|
||||
duration = len(audio) / sample_rate
|
||||
if truncated:
|
||||
assert duration == pytest.approx(max_frames / 25, abs=0.1)
|
||||
else:
|
||||
assert 8 <= duration < max_frames / 25
|
||||
@@ -0,0 +1,237 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""E2E online tests for YuE2-3B text-to-music on ``/v1/audio/speech``.
|
||||
|
||||
Scenario: heavy-metal "Twinkle Twinkle Little Star" — Chinese lyrics plus a
|
||||
metal style caption plus an ABC score of the melody (``extra_params.cot =
|
||||
"melody"``). The golden score is a SheetSage2 transcription of the Wikimedia
|
||||
piano rendition, reduced to a single voice (``tests/assets/yue2``).
|
||||
|
||||
Two programmatic content checks, both calibrated against a same-seed control
|
||||
request with ``cot="off"`` (no ABC conditioning):
|
||||
|
||||
- Style: CLAP zero-shot probability of "heavy metal" must clear 0.9.
|
||||
- Melody: the generated song is transcribed back with SheetSage2 and its
|
||||
pitch-class sequence is LCS-matched against the golden score (max over
|
||||
major/minor variants and 12 transpositions, tolerating pedal-tone
|
||||
insertions). SheetSage2's transcription of distorted metal vocals carries
|
||||
roughly ±0.05 noise, so the bar sits at 0.80; the ABC-conditioned request
|
||||
must also beat the control.
|
||||
|
||||
The content checks run in a dedicated verification interpreter because
|
||||
SheetSage2 wants a dependency set (mir_eval, pretty_midi, numpy<2) that must
|
||||
not leak into the serving environment. They are enabled by three env vars:
|
||||
|
||||
- ``YUE2_E2E_VERIFY_PYTHON``: python with sheetsage2 + CLAP deps installed
|
||||
- ``YUE2_E2E_CLAP_DIR``: local laion/clap-htsat-unfused snapshot (~0.6 GB)
|
||||
- ``YUE2_E2E_SHEETSAGE2_DIR``: local m-a-p/SheetSage2 snapshot (~0.25 GB,
|
||||
plus its MERT-v2 backbone ~2.5 GB from the HF cache)
|
||||
|
||||
Without them the test still asserts the structural floor (non-empty payload
|
||||
above the byte floor) and passes on that alone, following the MiniMax Music 3
|
||||
precedent.
|
||||
|
||||
Weekly rather than per-PR: each request is a 16 s song and the verification
|
||||
checkpoints add ~5.5 GB of downloads. Validated on a single RTX 4090
|
||||
(24 GB); the hardware mark targets an H100 because the CI fleet has no
|
||||
consumer cards.
|
||||
"""
|
||||
|
||||
import concurrent.futures
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import wave
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.media import get_asset_path
|
||||
from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
MODEL = "m-a-p/YuE2-3B"
|
||||
DEFAULT_AUDIO_SPEECH_TIMEOUT_S = 900.0
|
||||
|
||||
# 400 frames = 16 s of song. The byte floor sits far below the observed
|
||||
# payload (~3.0 MB of 48 kHz stereo 16-bit) to leave room for an early EOS.
|
||||
_FRAMES = 400
|
||||
# Concurrent pair stays short: it exists to exercise row re-indexing, not
|
||||
# to double the wall time.
|
||||
_CONCURRENT_FRAMES = 250
|
||||
_MIN_BYTES = 1_000_000
|
||||
|
||||
LYRICS = "一闪一闪亮晶晶\n满天都是小星星\n挂在天空放光明\n好像千万小眼睛"
|
||||
CAPTION = (
|
||||
"Chinese heavy metal, distorted electric guitars, aggressive double-kick drums, powerful vocals, 140 BPM, key of C"
|
||||
)
|
||||
SEED = 831001
|
||||
|
||||
_VERIFY_PYTHON = os.environ.get("YUE2_E2E_VERIFY_PYTHON")
|
||||
_CLAP_DIR = os.environ.get("YUE2_E2E_CLAP_DIR")
|
||||
_SHEETSAGE2_DIR = os.environ.get("YUE2_E2E_SHEETSAGE2_DIR")
|
||||
_CONTENT_CHECKS_ENABLED = all([_VERIFY_PYTHON, _CLAP_DIR, _SHEETSAGE2_DIR])
|
||||
_VERIFY_HELPER = os.path.join(os.path.dirname(__file__), "yue2_metal_verify.py")
|
||||
|
||||
|
||||
tts_server_params = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
stage_config_path=get_deploy_config_path("yue2.yaml"),
|
||||
server_args=["--disable-log-stats"],
|
||||
),
|
||||
id="yue2",
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def _http_speech(omni_server, *, seed: int, frames: int) -> bytes:
|
||||
"""Raw-HTTP speech request returning (and structurally checking) the WAV.
|
||||
|
||||
The OpenAI SDK client swallows response headers, so the truncation flag is
|
||||
cross-checked here: ``X-Audio-Truncated`` must agree with the audio — the
|
||||
full frame budget (a 25 frames/s song of exactly frames/25 seconds) means
|
||||
truncated, a shorter song means a natural end token.
|
||||
"""
|
||||
response = httpx.post(
|
||||
f"http://{omni_server.host}:{omni_server.port}/v1/audio/speech",
|
||||
json={
|
||||
"model": omni_server.model,
|
||||
"input": LYRICS,
|
||||
"instructions": CAPTION,
|
||||
"seed": seed,
|
||||
"max_new_tokens": frames,
|
||||
"stream": False,
|
||||
"response_format": "wav",
|
||||
"extra_params": {"cot": "off"},
|
||||
},
|
||||
timeout=DEFAULT_AUDIO_SPEECH_TIMEOUT_S,
|
||||
)
|
||||
assert response.status_code == 200, response.text[:500]
|
||||
truncated = response.headers.get("X-Audio-Truncated")
|
||||
assert truncated in ("true", "false"), f"X-Audio-Truncated missing/invalid: {truncated!r}"
|
||||
audio = response.content
|
||||
with wave.open(io.BytesIO(audio)) as wav:
|
||||
assert wav.getnchannels() == 2
|
||||
assert wav.getframerate() == 48000
|
||||
duration = wav.getnframes() / wav.getframerate()
|
||||
hit_budget = duration >= frames / 25 - 0.5
|
||||
assert truncated == ("true" if hit_budget else "false"), (
|
||||
f"X-Audio-Truncated={truncated} but duration={duration:.2f}s vs {frames / 25:.1f}s budget"
|
||||
)
|
||||
return audio
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards={"cuda": 1})
|
||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
||||
def test_yue2_metal_twinkle_001(omni_server, openai_client, tmp_path) -> None:
|
||||
"""Heavy-metal Twinkle: ABC-conditioned request vs same-seed control.
|
||||
|
||||
Deploy Setting: yue2.yaml
|
||||
Input Modal: text (lyrics) + text (style caption) + ABC score (extra_params)
|
||||
Output Modal: audio, 48 kHz stereo
|
||||
Input Setting: stream=False, seed pinned, 400 frames
|
||||
"""
|
||||
golden_abc = get_asset_path("yue2/twinkle_golden.abc").read_text()
|
||||
|
||||
def _request(extra_params: dict) -> bytes:
|
||||
responses = openai_client.send_audio_speech_request(
|
||||
{
|
||||
"model": omni_server.model,
|
||||
"input": LYRICS,
|
||||
"instructions": CAPTION,
|
||||
"seed": SEED,
|
||||
"max_new_tokens": _FRAMES,
|
||||
"stream": False,
|
||||
"response_format": "wav",
|
||||
"timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S,
|
||||
"min_audio_bytes": _MIN_BYTES,
|
||||
# Sung metal vocals are not whisper-transcribable; skip the
|
||||
# full_model transcript check (our content checks do the work).
|
||||
"transcript_expected_text": "",
|
||||
"extra_params": extra_params,
|
||||
}
|
||||
)
|
||||
audio = responses[0].audio_bytes
|
||||
assert audio, "empty audio payload"
|
||||
return audio
|
||||
|
||||
metal_path = tmp_path / "metal.wav"
|
||||
control_path = tmp_path / "control.wav"
|
||||
metal_bytes = _request({"cot": "melody", "abc": golden_abc})
|
||||
control_bytes = _request({"cot": "off"})
|
||||
metal_path.write_bytes(metal_bytes)
|
||||
control_path.write_bytes(control_bytes)
|
||||
|
||||
# CI runs without the verification assets, so the structural checks below
|
||||
# are the only guard against the two real bugs this PR fixed during review:
|
||||
# mono/half-speed output (WAV header + duration bounds) and the int32
|
||||
# crash under concurrent row re-indexing (two simultaneous requests).
|
||||
# Duration bounds: the end token is masked for the first 200 steps
|
||||
# (min_tokens), and the 400-frame budget caps the song, so a healthy
|
||||
# request lands in [8, 16.5] s; the half-speed bug produced ~32 s.
|
||||
for label, audio in (("metal", metal_bytes), ("control", control_bytes)):
|
||||
with wave.open(io.BytesIO(audio)) as wav:
|
||||
assert wav.getnchannels() == 2, f"{label}: {wav.getnchannels()} channels, want stereo"
|
||||
assert wav.getframerate() == 48000, f"{label}: {wav.getframerate()} Hz, want 48 kHz"
|
||||
duration = wav.getnframes() / wav.getframerate()
|
||||
assert _FRAMES / 50 <= duration <= _FRAMES / 25 + 0.5, (
|
||||
f"{label}: {duration:.2f}s outside [{_FRAMES / 50}, {_FRAMES / 25 + 0.5}]s"
|
||||
)
|
||||
|
||||
# A pinned-seed request must flag truncation consistently with the audio
|
||||
# (see _http_speech); the OpenAI SDK response does not expose headers.
|
||||
_http_speech(omni_server, seed=SEED, frames=_FRAMES)
|
||||
|
||||
# Two concurrent requests with different seeds: both must succeed and
|
||||
# produce different songs (the model-owned sampler re-indexes rows across
|
||||
# concurrent requests; a dtype mismatch here crashed the engine before).
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
|
||||
futures = [pool.submit(_http_speech, omni_server, seed=SEED + i, frames=_CONCURRENT_FRAMES) for i in range(2)]
|
||||
pair = [f.result() for f in futures]
|
||||
assert pair[0] != pair[1], "different seeds produced identical audio"
|
||||
|
||||
# Without the verification assets the structural assertions above
|
||||
# (non-empty payload, byte floor) are the whole test — pass on them.
|
||||
if not _CONTENT_CHECKS_ENABLED:
|
||||
return
|
||||
assert _VERIFY_PYTHON is not None
|
||||
assert _CLAP_DIR is not None
|
||||
assert _SHEETSAGE2_DIR is not None
|
||||
|
||||
proc = subprocess.run(
|
||||
[
|
||||
_VERIFY_PYTHON,
|
||||
_VERIFY_HELPER,
|
||||
"--clap-dir",
|
||||
_CLAP_DIR,
|
||||
"--sheetsage2-dir",
|
||||
_SHEETSAGE2_DIR,
|
||||
"--work-dir",
|
||||
str(tmp_path),
|
||||
str(metal_path),
|
||||
str(control_path),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=1800,
|
||||
)
|
||||
assert proc.returncode == 0, f"verification helper failed:\n{proc.stderr}"
|
||||
report = json.loads(proc.stdout.strip().splitlines()[-1])
|
||||
|
||||
metal_prob = report["metal_probability"]
|
||||
assert metal_prob >= 0.9, f"style check: P(heavy metal)={metal_prob:.3f} < 0.9"
|
||||
|
||||
melody_score = report["melody_score"]
|
||||
control_score = report["control_score"]
|
||||
assert melody_score >= 0.80, f"melody check: score={melody_score:.3f} < 0.80"
|
||||
assert melody_score > control_score, (
|
||||
f"calibration: ABC-conditioned {melody_score:.3f} not above control {control_score:.3f}"
|
||||
)
|
||||
@@ -0,0 +1,185 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Content-gate helper for the YuE2 heavy-metal Twinkle e2e test.
|
||||
|
||||
Run as ``__main__`` by a dedicated verification interpreter (see
|
||||
``YUE2_E2E_VERIFY_PYTHON`` in test_yue2.py), NOT by the serving environment:
|
||||
SheetSage2 wants its own dependency set (mir_eval, pretty_midi, numpy<2),
|
||||
which must not leak into the serving venv.
|
||||
|
||||
Usage:
|
||||
python yue2_metal_verify.py --clap-dir DIR --sheetsage2-dir DIR \
|
||||
metal.wav control.wav
|
||||
|
||||
Prints one JSON object on stdout:
|
||||
{"metal_probability": float, "melody_score": float, "control_score": float}
|
||||
|
||||
Gates (applied by the caller):
|
||||
- metal_probability >= 0.9 (CLAP zero-shot heavy-metal style)
|
||||
- melody_score >= 0.80 and melody_score > control_score
|
||||
(SheetSage2 re-transcription, pitch-class LCS vs the golden score, max over
|
||||
major/minor variants and 12 transpositions, pedal-tone tolerant)
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import wave
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
_NOTE_BASE = {"C": 0, "D": 2, "E": 4, "F": 5, "G": 7, "A": 9, "B": 11}
|
||||
_GOLDEN_MAJOR = "C4C4G4G4|A4A4G8|F4F4E4E4|D4D4C8|G4G4F4F4|E4E4D8|G4G4F4F4|E4E4D8|C4C4G4G4|A4A4G8|F4F4E4E4|D4D4C8|"
|
||||
# Metal arrangements commonly minor-ize the melody; accept both modes.
|
||||
_GOLDEN_MINOR = (
|
||||
"C4C4G4G4|_A4_A4G8|F4F4_E4_E4|D4D4C8|G4G4F4F4|_E4_E4D8|G4G4F4F4|_E4_E4D8|C4C4G4G4|_A4_A4G8|F4F4_E4_E4|D4D4C8|"
|
||||
)
|
||||
|
||||
_STYLE_CANDIDATES = [
|
||||
"heavy metal music with distorted guitars and aggressive drums",
|
||||
"a soft children's lullaby",
|
||||
"classical piano music",
|
||||
"pop music",
|
||||
]
|
||||
|
||||
|
||||
def _abc_note_to_midi(token):
|
||||
m = re.match(r"([\^_=]*)([A-Ga-g])([,']*)(\d*)", token)
|
||||
if not m:
|
||||
return None
|
||||
acc, letter, oct_marks, _ = m.groups()
|
||||
pc = _NOTE_BASE[letter.upper()] + acc.count("^") - acc.count("_")
|
||||
octave = 5 if letter.isupper() else 6
|
||||
octave += oct_marks.count("'") - oct_marks.count(",")
|
||||
return 12 * octave + pc
|
||||
|
||||
|
||||
def _collapse(seq):
|
||||
out: list[Any] = []
|
||||
for x in seq:
|
||||
if not out or x != out[-1]:
|
||||
out.append(x)
|
||||
return out
|
||||
|
||||
|
||||
def _golden_events(body):
|
||||
return _collapse([_abc_note_to_midi(t) for t in re.findall(r"[\^_=]*[A-G]\d*", body)])
|
||||
|
||||
|
||||
def _melody_events(score_abc):
|
||||
"""Note events of the voice carrying the most notes in a SheetSage2 score."""
|
||||
voices: dict[str | None, list[int]] = {}
|
||||
current = None
|
||||
for line in score_abc.splitlines():
|
||||
line = line.strip()
|
||||
if line.startswith("V:"):
|
||||
parts = line.split()
|
||||
current = parts[1] if len(parts) > 1 else None
|
||||
continue
|
||||
if current is None or not line or line.startswith(("%", "X:", "T:", "M:", "L:", "Q:", "K:")):
|
||||
continue
|
||||
for tok in re.findall(r"[\^_=]*[A-Ga-g][,']*\d*", line.replace("|", " ")):
|
||||
midi = _abc_note_to_midi(tok)
|
||||
if midi is not None:
|
||||
voices.setdefault(current, []).append(midi)
|
||||
return _collapse(max(voices.values(), key=len, default=[]))
|
||||
|
||||
|
||||
def _lcs(a, b):
|
||||
dp = [0] * (len(b) + 1)
|
||||
for x in a:
|
||||
prev = 0
|
||||
for j, y in enumerate(b, 1):
|
||||
tmp = dp[j]
|
||||
dp[j] = prev + 1 if x == y else max(dp[j], dp[j - 1])
|
||||
prev = tmp
|
||||
return dp[-1]
|
||||
|
||||
|
||||
def melody_score(score_abc):
|
||||
gen = [m % 12 for m in _melody_events(score_abc)]
|
||||
if len(gen) < 4:
|
||||
return 0.0
|
||||
best = 0
|
||||
for body in (_GOLDEN_MAJOR, _GOLDEN_MINOR):
|
||||
gold = [m % 12 for m in _golden_events(body)]
|
||||
best = max(best, max(_lcs(gen, [(p + r) % 12 for p in gold]) for r in range(12)))
|
||||
return best / len(_golden_events(_GOLDEN_MAJOR))
|
||||
|
||||
|
||||
def _wav_to_mono(path):
|
||||
with wave.open(path) as wf:
|
||||
sr = wf.getframerate()
|
||||
channels = wf.getnchannels()
|
||||
frames = wf.readframes(wf.getnframes())
|
||||
data = np.frombuffer(frames, dtype=np.int16).astype(np.float32) / 32768.0
|
||||
if channels > 1:
|
||||
data = data.reshape(-1, channels).mean(axis=1)
|
||||
return data, sr
|
||||
|
||||
|
||||
def metal_probability(wav_path, clap_dir):
|
||||
import torch
|
||||
from transformers import ClapModel, ClapProcessor
|
||||
|
||||
model = ClapModel.from_pretrained(clap_dir)
|
||||
proc = ClapProcessor.from_pretrained(clap_dir)
|
||||
audio, sr = _wav_to_mono(wav_path)
|
||||
if sr != 48000:
|
||||
n = int(len(audio) * 48000 / sr)
|
||||
audio = np.interp(np.linspace(0, len(audio), n), np.arange(len(audio)), audio).astype(np.float32)
|
||||
audio = audio[: 48000 * 10]
|
||||
inputs = proc(
|
||||
text=_STYLE_CANDIDATES,
|
||||
audios=[audio] * len(_STYLE_CANDIDATES),
|
||||
sampling_rate=48000,
|
||||
return_tensors="pt",
|
||||
padding=True,
|
||||
)
|
||||
with torch.no_grad():
|
||||
probs = model(**inputs).logits_per_audio[0].softmax(dim=-1)
|
||||
return float(probs[0])
|
||||
|
||||
|
||||
def transcribe_melody_abc(wav_path, sheetsage2_dir, output_dir):
|
||||
import torch
|
||||
from transformers import AutoModel
|
||||
|
||||
audio, sr = _wav_to_mono(wav_path)
|
||||
model = AutoModel.from_pretrained(sheetsage2_dir, trust_remote_code=True)
|
||||
result = model.transcribe(
|
||||
torch.from_numpy(audio[None, :]),
|
||||
sampling_rate=sr,
|
||||
output_dir=str(output_dir),
|
||||
melody_only=True,
|
||||
dtype="fp32",
|
||||
)
|
||||
abc = result.get("abc")
|
||||
if not abc:
|
||||
raise RuntimeError(f"SheetSage2 produced no melody ABC: {result.get('abc_error')}")
|
||||
return abc
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--clap-dir", required=True)
|
||||
parser.add_argument("--sheetsage2-dir", required=True)
|
||||
parser.add_argument("metal_wav")
|
||||
parser.add_argument("control_wav")
|
||||
parser.add_argument("--work-dir", default=None)
|
||||
args = parser.parse_args()
|
||||
|
||||
work = args.work_dir or "."
|
||||
report = {"metal_probability": metal_probability(args.metal_wav, args.clap_dir)}
|
||||
report["melody_score"] = melody_score(
|
||||
transcribe_melody_abc(args.metal_wav, args.sheetsage2_dir, f"{work}/metal_score")
|
||||
)
|
||||
report["control_score"] = melody_score(
|
||||
transcribe_melody_abc(args.control_wav, args.sheetsage2_dir, f"{work}/control_score")
|
||||
)
|
||||
print(json.dumps(report))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1105,6 +1105,38 @@ class TestTTSMethods:
|
||||
assert isinstance(audio_obj, CreateAudio)
|
||||
assert audio_obj.speed == 1.0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_audio_synthesis_error_flag_raises_tts_generation_error(
|
||||
self,
|
||||
speech_server,
|
||||
mocker: MockerFixture,
|
||||
):
|
||||
"""A model-flagged synthesis failure surfaces as a non-retryable
|
||||
TTSGenerationError, so create_speech answers 500 instead of shipping a
|
||||
zero-length WAV or crashing on an unpacked Response."""
|
||||
|
||||
async def mock_generate():
|
||||
yield create_mock_audio_output_for_test()
|
||||
|
||||
mocker.patch.object(
|
||||
speech_server,
|
||||
"_prepare_speech_generation",
|
||||
new=mocker.AsyncMock(return_value=("speech-synth-err", mock_generate(), {})),
|
||||
)
|
||||
adapter = mocker.MagicMock()
|
||||
adapter.collect_response_metadata = lambda _audio_output, collect: collect.__setitem__(
|
||||
"audio_synthesis_error", True
|
||||
)
|
||||
mocker.patch.object(speech_server, "_get_tts_adapter", return_value=adapter)
|
||||
|
||||
with pytest.raises(TTSGenerationError, match="failed to synthesize audio") as exc_info:
|
||||
await speech_server._generate_audio_bytes(
|
||||
OpenAICreateSpeechRequest(input="Hello"),
|
||||
collect={},
|
||||
)
|
||||
|
||||
assert exc_info.value.retryable is False
|
||||
|
||||
def test_is_tts_detection_with_tts_stage(self, mocker: MockerFixture):
|
||||
"""Test TTS model detection when TTS stage exists."""
|
||||
mock_engine_client = mocker.MagicMock()
|
||||
|
||||
@@ -183,6 +183,7 @@ _PIPELINE_STAGES = [
|
||||
"token2wav",
|
||||
"tts",
|
||||
"encoder",
|
||||
"yue2",
|
||||
]
|
||||
|
||||
_STAGES = [*_PIPELINE_STAGES, None, "vae", "not_a_real_stage"]
|
||||
|
||||
@@ -15,6 +15,7 @@ import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.worker import gpu_input_batch
|
||||
from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
|
||||
|
||||
@@ -103,6 +104,8 @@ def test_post_sample_talker_mtp_uses_current_sample_and_hidden() -> None:
|
||||
)
|
||||
|
||||
assert received["req_ids"] == ["ready"]
|
||||
assert isinstance(received["input_ids"], torch.Tensor)
|
||||
assert isinstance(received["hidden_states"], torch.Tensor)
|
||||
assert received["input_ids"].tolist() == [101]
|
||||
assert received["hidden_states"].tolist() == [[1.0, 2.0]]
|
||||
assert received["req_infos"] == [{"duplex": {"data_plane": True}}]
|
||||
@@ -139,6 +142,8 @@ def test_post_sample_talker_mtp_uses_gpu_token_with_async_scheduling() -> None:
|
||||
)
|
||||
|
||||
assert received["req_ids"] == ["ready"]
|
||||
assert isinstance(received["input_ids"], torch.Tensor)
|
||||
assert isinstance(received["hidden_states"], torch.Tensor)
|
||||
assert received["input_ids"].tolist() == [101]
|
||||
assert received["hidden_states"].tolist() == [[1.0, 2.0]]
|
||||
assert received["req_infos"] == [{"duplex": {"data_plane": True}}]
|
||||
@@ -275,7 +280,12 @@ def test_speech_extra_params_reach_model_sampler_as_sampling_metadata(monkeypatc
|
||||
)
|
||||
input_batch.sampling_metadata = input_batch._make_sampling_metadata()
|
||||
|
||||
received = []
|
||||
received: list[SamplingMetadata] = []
|
||||
|
||||
def sample(logits: torch.Tensor, metadata: SamplingMetadata) -> str:
|
||||
received.append(metadata)
|
||||
return "model-sampler"
|
||||
|
||||
runner = object.__new__(GPUARModelRunner)
|
||||
runner._omni_cache_policy = ModelCachePolicy(needs_full_hidden_states=True)
|
||||
runner._pooler_payload_include_hidden_flag = True
|
||||
@@ -283,7 +293,7 @@ def test_speech_extra_params_reach_model_sampler_as_sampling_metadata(monkeypatc
|
||||
runner.model = SimpleNamespace(
|
||||
prefer_model_sampler=True,
|
||||
skips_model_sampler_output_token_history=True,
|
||||
sample=lambda logits, metadata: received.append(metadata) or "model-sampler",
|
||||
sample=sample,
|
||||
)
|
||||
runner.sampler = SimpleNamespace()
|
||||
logits = torch.zeros((1, 4))
|
||||
@@ -1670,7 +1680,6 @@ class TestMergeModelKvTransferMetadata:
|
||||
# ...while the engine-shared originals are untouched.
|
||||
assert original["r1"]["custom_metadata"] is original_meta
|
||||
assert original_meta == {"a": 1}
|
||||
assert "talker_codes" not in original["r1"]["custom_metadata"]
|
||||
|
||||
def test_merge_without_model_meta_passes_entry_through(self):
|
||||
class _Model:
|
||||
@@ -1713,7 +1722,7 @@ class TestDownstreamPayloadMemoization:
|
||||
return runner
|
||||
|
||||
def test_missing_marker_defaults_true_without_memoizing(self):
|
||||
stages = {"r1": None}
|
||||
stages: dict[str, int | None] = {"r1": None}
|
||||
runner = self._runner(stages)
|
||||
|
||||
assert runner._request_needs_downstream_stage_payload("r1") is True
|
||||
@@ -1861,6 +1870,7 @@ class TestPreferModelSamplerNoneFallback:
|
||||
"minicpmo_4_5",
|
||||
"minimax_music3",
|
||||
"nemotron_voicechat",
|
||||
"yue2", # Always supplies SamplerOutput, including empty/prefill steps.
|
||||
}
|
||||
assert declarers == expected, (
|
||||
"The set of models declaring `prefer_model_sampler` changed:\n"
|
||||
|
||||
Reference in new issue
Block a user