[Model] Add YuE2-3B text-to-music (model + /v1/audio/speech adapter) (#7886)

Signed-off-by: Youyuan Li <105433145+Darcy-Lee@users.noreply.github.com>
Signed-off-by: princepride <wangzhipeng628@gmail.com>
Co-authored-by: princepride <wangzhipeng628@gmail.com>
This commit is contained in:
Youyuan Liandprincepride authored and GitHub committed 2026-10-01 18:04:31 +08:00
1 parent a038b38179
commit 423f34326e
27 files changed
+3294 -6

No files matched your search

+6
View File
@@ -0,0 +1,6 @@
X:1
T:Twinkle Twinkle Little Star (SheetSage2 transcription of the Wikimedia piano rendition, single voice)
M:4/4
L:1/16
K:C
C4C4G4G4|A4A4G8|F4F4E4E4|D4D4C8|G4G4F4F4|E4E4D8|G4G4F4F4|E4E4D8|C4C4G4G4|A4A4G8|F4F4E4E4|D4D4C8|
+2 -2
View File
@@ -167,7 +167,7 @@ def test_inventory_matches_reviewed_snapshot_counts():
EnvironmentVariableCategory.PUBLIC_OMNI: 32,
EnvironmentVariableCategory.INHERITED_VLLM: 20,
EnvironmentVariableCategory.PLATFORM_EXTERNAL: 29,
EnvironmentVariableCategory.MODEL_SPECIFIC: 74,
EnvironmentVariableCategory.MODEL_SPECIFIC: 75,
EnvironmentVariableCategory.BENCHMARK_TRANSITIONAL: 21,
EnvironmentVariableCategory.INTERNAL: 6,
}
@@ -178,7 +178,7 @@ def test_inventory_matches_reviewed_snapshot_counts():
if item.category is EnvironmentVariableCategory.MODEL_SPECIFIC
)
assert {disposition: disposition_counts[disposition] for disposition in ModelEnvironmentVariableDisposition} == {
ModelEnvironmentVariableDisposition.PROMOTE: 45,
ModelEnvironmentVariableDisposition.PROMOTE: 46,
ModelEnvironmentVariableDisposition.REQUEST_SCOPE: 5,
ModelEnvironmentVariableDisposition.EXTERNAL: 0,
ModelEnvironmentVariableDisposition.INTERNALIZE: 17,
@@ -1048,3 +1048,83 @@ def test_minimax_music3_speech_invalid_field_values(
online_client.send_audio_speech_http_request(
{"json": body, "timeout": 120, "err_code": 400, "err_message": err_message}
)
# ─── POST /v1/audio/speech · YuE2-3B ───
# Text-to-music like MiniMax Music 3, with one twist: the ABC score span is
# request-controlled via extra_params.cot/abc and the two must agree, and
# sampling is pinned by the checkpoint preset so sampling knobs are rejected.
_YUE2_SPEECH = [
pytest.param(
OmniServerParams(
model="m-a-p/YuE2-3B",
stage_config_path=get_deploy_config_path("yue2.yaml"),
server_args=_SPEECH_SERVER_ARGS,
),
id="yue2",
marks=hardware_marks(res={"cuda": "H100"}),
),
]
_YUE2_LYRICS = "一闪一闪亮晶晶\n满天都是小星星"
_YUE2_CAPTION = "Chinese heavy metal, distorted electric guitars, 140 BPM, key of C."
_YUE2_ABC = "X:1\nT:Twinkle\nM:4/4\nL:1/4\nK:C\nC C G G|A A G2|"
# 25 frames per second; the semantic preset bounds the frame budget.
_YUE2_MAX_FRAMES = 9000
@pytest.mark.parametrize(
"overrides, err_message",
[
pytest.param({"instructions": None}, ("instructions",), id="caption_missing"),
pytest.param({"instructions": " "}, ("instructions",), id="caption_blank"),
pytest.param({"input": " "}, ("input",), id="lyrics_blank"),
pytest.param({"voice": "alloy"}, ("voice",), id="voice_rejected"),
pytest.param({"speed": 1.5}, ("speed",), id="speed_rejected"),
pytest.param(
{"max_new_tokens": _YUE2_MAX_FRAMES + 1},
("max_new_tokens", str(_YUE2_MAX_FRAMES)),
id="max_new_tokens_over_cap",
),
pytest.param({"max_new_tokens": 199}, ("max_new_tokens",), id="max_new_tokens_below_min"),
pytest.param(
{"extra_params": {"cot": "melody"}},
("abc",),
id="cot_melody_without_abc",
),
pytest.param(
{"extra_params": {"abc": _YUE2_ABC}},
("cot",),
id="abc_without_cot",
),
pytest.param(
{"extra_params": {"temperature": 0.3}},
("temperature",),
id="temperature_rejected",
),
],
)
@pytest.mark.parametrize("omni_server", _YUE2_SPEECH, indirect=True)
def test_yue2_speech_invalid_field_values(
omni_server: OmniServer,
online_client: OnlineOmniClient,
overrides: dict[str, object],
err_message: str | tuple[str, ...],
) -> None:
body: dict[str, Any] = {
"model": omni_server.model,
"input": _YUE2_LYRICS,
"instructions": _YUE2_CAPTION,
"seed": 1,
"max_new_tokens": 250,
"response_format": "wav",
}
for key, value in overrides.items():
if value is None:
body.pop(key, None)
else:
body[key] = value
online_client.send_audio_speech_http_request(
{"json": body, "timeout": 120, "err_code": 400, "err_message": err_message}
)
+82
View File
@@ -0,0 +1,82 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""End-to-end regressions for YuE2 offline WAV export."""
import os
import subprocess
import sys
from pathlib import Path
import numpy as np
import pytest
import soundfile as sf
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import get_model_prefix
from vllm_omni.transformers_utils.repo_utils import hf_api
pytestmark = [
pytest.mark.full_model,
pytest.mark.tts,
*hardware_marks(res={"cuda": ["L4", "H100", "B200"]}, num_cards=1),
]
REPO_ROOT = Path(__file__).resolve().parents[3]
@pytest.fixture(scope="module")
def vae_path() -> Path:
path = Path(os.environ.get("YUE2_VAE", get_model_prefix() + "m-a-p/YuE2-Vae"))
if not path.is_dir():
path = Path(hf_api().snapshot_download(str(path), allow_patterns=["*.json", "*.safetensors"]))
return path
@pytest.mark.parametrize("max_frames,truncated", [(200, True), (9000, False)], ids=["budget", "natural-eos"])
def test_offline_example_saves_audio(vae_path: Path, tmp_path: Path, max_frames: int, truncated: bool) -> None:
model = Path(os.environ.get("YUE2_MODEL_DIR", get_model_prefix() + "m-a-p/YuE2-3B"))
if not model.is_dir():
model = Path(
hf_api().snapshot_download(str(model), allow_patterns=["*.json", "*.safetensors", "qwen.tiktoken"])
)
output = tmp_path / "song.wav"
env = os.environ.copy()
env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
env["PYTHONPATH"] = str(REPO_ROOT) + os.pathsep + env.get("PYTHONPATH", "")
completed = subprocess.run(
[
sys.executable,
str(REPO_ROOT / "examples/offline_inference/yue2/end2end.py"),
"--model",
str(model),
"--vae",
str(vae_path),
"--style",
"gentle acoustic folk, clear English vocals, 90 BPM",
"--lyrics",
"[Verse]\nMorning light across the bay\nCarry all my dreams away\n[Outro]",
"--cot",
"off",
"--seed",
"831001",
"--max-frames",
str(max_frames),
"--output",
str(output),
],
cwd=REPO_ROOT,
env=env,
capture_output=True,
text=True,
timeout=900,
)
assert completed.returncode == 0, completed.stdout + completed.stderr
assert f"truncated={truncated}" in completed.stdout
audio, sample_rate = sf.read(output, always_2d=True)
assert sample_rate == 48000 and audio.shape[1] == 2
assert np.isfinite(audio).all() and np.abs(audio).max() > 0
duration = len(audio) / sample_rate
if truncated:
assert duration == pytest.approx(max_frames / 25, abs=0.1)
else:
assert 8 <= duration < max_frames / 25
+237
View File
@@ -0,0 +1,237 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""E2E online tests for YuE2-3B text-to-music on ``/v1/audio/speech``.
Scenario: heavy-metal "Twinkle Twinkle Little Star" — Chinese lyrics plus a
metal style caption plus an ABC score of the melody (``extra_params.cot =
"melody"``). The golden score is a SheetSage2 transcription of the Wikimedia
piano rendition, reduced to a single voice (``tests/assets/yue2``).
Two programmatic content checks, both calibrated against a same-seed control
request with ``cot="off"`` (no ABC conditioning):
- Style: CLAP zero-shot probability of "heavy metal" must clear 0.9.
- Melody: the generated song is transcribed back with SheetSage2 and its
pitch-class sequence is LCS-matched against the golden score (max over
major/minor variants and 12 transpositions, tolerating pedal-tone
insertions). SheetSage2's transcription of distorted metal vocals carries
roughly ±0.05 noise, so the bar sits at 0.80; the ABC-conditioned request
must also beat the control.
The content checks run in a dedicated verification interpreter because
SheetSage2 wants a dependency set (mir_eval, pretty_midi, numpy<2) that must
not leak into the serving environment. They are enabled by three env vars:
- ``YUE2_E2E_VERIFY_PYTHON``: python with sheetsage2 + CLAP deps installed
- ``YUE2_E2E_CLAP_DIR``: local laion/clap-htsat-unfused snapshot (~0.6 GB)
- ``YUE2_E2E_SHEETSAGE2_DIR``: local m-a-p/SheetSage2 snapshot (~0.25 GB,
plus its MERT-v2 backbone ~2.5 GB from the HF cache)
Without them the test still asserts the structural floor (non-empty payload
above the byte floor) and passes on that alone, following the MiniMax Music 3
precedent.
Weekly rather than per-PR: each request is a 16 s song and the verification
checkpoints add ~5.5 GB of downloads. Validated on a single RTX 4090
(24 GB); the hardware mark targets an H100 because the CI fleet has no
consumer cards.
"""
import concurrent.futures
import io
import json
import os
import subprocess
import wave
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
import httpx
import pytest
from tests.helpers.mark import hardware_test
from tests.helpers.media import get_asset_path
from tests.helpers.runtime import OmniServerParams
from tests.helpers.stage_config import get_deploy_config_path
MODEL = "m-a-p/YuE2-3B"
DEFAULT_AUDIO_SPEECH_TIMEOUT_S = 900.0
# 400 frames = 16 s of song. The byte floor sits far below the observed
# payload (~3.0 MB of 48 kHz stereo 16-bit) to leave room for an early EOS.
_FRAMES = 400
# Concurrent pair stays short: it exists to exercise row re-indexing, not
# to double the wall time.
_CONCURRENT_FRAMES = 250
_MIN_BYTES = 1_000_000
LYRICS = "一闪一闪亮晶晶\n满天都是小星星\n挂在天空放光明\n好像千万小眼睛"
CAPTION = (
"Chinese heavy metal, distorted electric guitars, aggressive double-kick drums, powerful vocals, 140 BPM, key of C"
)
SEED = 831001
_VERIFY_PYTHON = os.environ.get("YUE2_E2E_VERIFY_PYTHON")
_CLAP_DIR = os.environ.get("YUE2_E2E_CLAP_DIR")
_SHEETSAGE2_DIR = os.environ.get("YUE2_E2E_SHEETSAGE2_DIR")
_CONTENT_CHECKS_ENABLED = all([_VERIFY_PYTHON, _CLAP_DIR, _SHEETSAGE2_DIR])
_VERIFY_HELPER = os.path.join(os.path.dirname(__file__), "yue2_metal_verify.py")
tts_server_params = [
pytest.param(
OmniServerParams(
model=MODEL,
stage_config_path=get_deploy_config_path("yue2.yaml"),
server_args=["--disable-log-stats"],
),
id="yue2",
)
]
def _http_speech(omni_server, *, seed: int, frames: int) -> bytes:
"""Raw-HTTP speech request returning (and structurally checking) the WAV.
The OpenAI SDK client swallows response headers, so the truncation flag is
cross-checked here: ``X-Audio-Truncated`` must agree with the audio — the
full frame budget (a 25 frames/s song of exactly frames/25 seconds) means
truncated, a shorter song means a natural end token.
"""
response = httpx.post(
f"http://{omni_server.host}:{omni_server.port}/v1/audio/speech",
json={
"model": omni_server.model,
"input": LYRICS,
"instructions": CAPTION,
"seed": seed,
"max_new_tokens": frames,
"stream": False,
"response_format": "wav",
"extra_params": {"cot": "off"},
},
timeout=DEFAULT_AUDIO_SPEECH_TIMEOUT_S,
)
assert response.status_code == 200, response.text[:500]
truncated = response.headers.get("X-Audio-Truncated")
assert truncated in ("true", "false"), f"X-Audio-Truncated missing/invalid: {truncated!r}"
audio = response.content
with wave.open(io.BytesIO(audio)) as wav:
assert wav.getnchannels() == 2
assert wav.getframerate() == 48000
duration = wav.getnframes() / wav.getframerate()
hit_budget = duration >= frames / 25 - 0.5
assert truncated == ("true" if hit_budget else "false"), (
f"X-Audio-Truncated={truncated} but duration={duration:.2f}s vs {frames / 25:.1f}s budget"
)
return audio
@pytest.mark.slow
@pytest.mark.tts
@hardware_test(res={"cuda": "H100"}, num_cards={"cuda": 1})
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
def test_yue2_metal_twinkle_001(omni_server, openai_client, tmp_path) -> None:
"""Heavy-metal Twinkle: ABC-conditioned request vs same-seed control.
Deploy Setting: yue2.yaml
Input Modal: text (lyrics) + text (style caption) + ABC score (extra_params)
Output Modal: audio, 48 kHz stereo
Input Setting: stream=False, seed pinned, 400 frames
"""
golden_abc = get_asset_path("yue2/twinkle_golden.abc").read_text()
def _request(extra_params: dict) -> bytes:
responses = openai_client.send_audio_speech_request(
{
"model": omni_server.model,
"input": LYRICS,
"instructions": CAPTION,
"seed": SEED,
"max_new_tokens": _FRAMES,
"stream": False,
"response_format": "wav",
"timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S,
"min_audio_bytes": _MIN_BYTES,
# Sung metal vocals are not whisper-transcribable; skip the
# full_model transcript check (our content checks do the work).
"transcript_expected_text": "",
"extra_params": extra_params,
}
)
audio = responses[0].audio_bytes
assert audio, "empty audio payload"
return audio
metal_path = tmp_path / "metal.wav"
control_path = tmp_path / "control.wav"
metal_bytes = _request({"cot": "melody", "abc": golden_abc})
control_bytes = _request({"cot": "off"})
metal_path.write_bytes(metal_bytes)
control_path.write_bytes(control_bytes)
# CI runs without the verification assets, so the structural checks below
# are the only guard against the two real bugs this PR fixed during review:
# mono/half-speed output (WAV header + duration bounds) and the int32
# crash under concurrent row re-indexing (two simultaneous requests).
# Duration bounds: the end token is masked for the first 200 steps
# (min_tokens), and the 400-frame budget caps the song, so a healthy
# request lands in [8, 16.5] s; the half-speed bug produced ~32 s.
for label, audio in (("metal", metal_bytes), ("control", control_bytes)):
with wave.open(io.BytesIO(audio)) as wav:
assert wav.getnchannels() == 2, f"{label}: {wav.getnchannels()} channels, want stereo"
assert wav.getframerate() == 48000, f"{label}: {wav.getframerate()} Hz, want 48 kHz"
duration = wav.getnframes() / wav.getframerate()
assert _FRAMES / 50 <= duration <= _FRAMES / 25 + 0.5, (
f"{label}: {duration:.2f}s outside [{_FRAMES / 50}, {_FRAMES / 25 + 0.5}]s"
)
# A pinned-seed request must flag truncation consistently with the audio
# (see _http_speech); the OpenAI SDK response does not expose headers.
_http_speech(omni_server, seed=SEED, frames=_FRAMES)
# Two concurrent requests with different seeds: both must succeed and
# produce different songs (the model-owned sampler re-indexes rows across
# concurrent requests; a dtype mismatch here crashed the engine before).
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
futures = [pool.submit(_http_speech, omni_server, seed=SEED + i, frames=_CONCURRENT_FRAMES) for i in range(2)]
pair = [f.result() for f in futures]
assert pair[0] != pair[1], "different seeds produced identical audio"
# Without the verification assets the structural assertions above
# (non-empty payload, byte floor) are the whole test — pass on them.
if not _CONTENT_CHECKS_ENABLED:
return
assert _VERIFY_PYTHON is not None
assert _CLAP_DIR is not None
assert _SHEETSAGE2_DIR is not None
proc = subprocess.run(
[
_VERIFY_PYTHON,
_VERIFY_HELPER,
"--clap-dir",
_CLAP_DIR,
"--sheetsage2-dir",
_SHEETSAGE2_DIR,
"--work-dir",
str(tmp_path),
str(metal_path),
str(control_path),
],
capture_output=True,
text=True,
timeout=1800,
)
assert proc.returncode == 0, f"verification helper failed:\n{proc.stderr}"
report = json.loads(proc.stdout.strip().splitlines()[-1])
metal_prob = report["metal_probability"]
assert metal_prob >= 0.9, f"style check: P(heavy metal)={metal_prob:.3f} < 0.9"
melody_score = report["melody_score"]
control_score = report["control_score"]
assert melody_score >= 0.80, f"melody check: score={melody_score:.3f} < 0.80"
assert melody_score > control_score, (
f"calibration: ABC-conditioned {melody_score:.3f} not above control {control_score:.3f}"
)
@@ -0,0 +1,185 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Content-gate helper for the YuE2 heavy-metal Twinkle e2e test.
Run as ``__main__`` by a dedicated verification interpreter (see
``YUE2_E2E_VERIFY_PYTHON`` in test_yue2.py), NOT by the serving environment:
SheetSage2 wants its own dependency set (mir_eval, pretty_midi, numpy<2),
which must not leak into the serving venv.
Usage:
python yue2_metal_verify.py --clap-dir DIR --sheetsage2-dir DIR \
metal.wav control.wav
Prints one JSON object on stdout:
{"metal_probability": float, "melody_score": float, "control_score": float}
Gates (applied by the caller):
- metal_probability >= 0.9 (CLAP zero-shot heavy-metal style)
- melody_score >= 0.80 and melody_score > control_score
(SheetSage2 re-transcription, pitch-class LCS vs the golden score, max over
major/minor variants and 12 transpositions, pedal-tone tolerant)
"""
import argparse
import json
import re
import wave
from typing import Any
import numpy as np
_NOTE_BASE = {"C": 0, "D": 2, "E": 4, "F": 5, "G": 7, "A": 9, "B": 11}
_GOLDEN_MAJOR = "C4C4G4G4|A4A4G8|F4F4E4E4|D4D4C8|G4G4F4F4|E4E4D8|G4G4F4F4|E4E4D8|C4C4G4G4|A4A4G8|F4F4E4E4|D4D4C8|"
# Metal arrangements commonly minor-ize the melody; accept both modes.
_GOLDEN_MINOR = (
"C4C4G4G4|_A4_A4G8|F4F4_E4_E4|D4D4C8|G4G4F4F4|_E4_E4D8|G4G4F4F4|_E4_E4D8|C4C4G4G4|_A4_A4G8|F4F4_E4_E4|D4D4C8|"
)
_STYLE_CANDIDATES = [
"heavy metal music with distorted guitars and aggressive drums",
"a soft children's lullaby",
"classical piano music",
"pop music",
]
def _abc_note_to_midi(token):
m = re.match(r"([\^_=]*)([A-Ga-g])([,']*)(\d*)", token)
if not m:
return None
acc, letter, oct_marks, _ = m.groups()
pc = _NOTE_BASE[letter.upper()] + acc.count("^") - acc.count("_")
octave = 5 if letter.isupper() else 6
octave += oct_marks.count("'") - oct_marks.count(",")
return 12 * octave + pc
def _collapse(seq):
out: list[Any] = []
for x in seq:
if not out or x != out[-1]:
out.append(x)
return out
def _golden_events(body):
return _collapse([_abc_note_to_midi(t) for t in re.findall(r"[\^_=]*[A-G]\d*", body)])
def _melody_events(score_abc):
"""Note events of the voice carrying the most notes in a SheetSage2 score."""
voices: dict[str | None, list[int]] = {}
current = None
for line in score_abc.splitlines():
line = line.strip()
if line.startswith("V:"):
parts = line.split()
current = parts[1] if len(parts) > 1 else None
continue
if current is None or not line or line.startswith(("%", "X:", "T:", "M:", "L:", "Q:", "K:")):
continue
for tok in re.findall(r"[\^_=]*[A-Ga-g][,']*\d*", line.replace("|", " ")):
midi = _abc_note_to_midi(tok)
if midi is not None:
voices.setdefault(current, []).append(midi)
return _collapse(max(voices.values(), key=len, default=[]))
def _lcs(a, b):
dp = [0] * (len(b) + 1)
for x in a:
prev = 0
for j, y in enumerate(b, 1):
tmp = dp[j]
dp[j] = prev + 1 if x == y else max(dp[j], dp[j - 1])
prev = tmp
return dp[-1]
def melody_score(score_abc):
gen = [m % 12 for m in _melody_events(score_abc)]
if len(gen) < 4:
return 0.0
best = 0
for body in (_GOLDEN_MAJOR, _GOLDEN_MINOR):
gold = [m % 12 for m in _golden_events(body)]
best = max(best, max(_lcs(gen, [(p + r) % 12 for p in gold]) for r in range(12)))
return best / len(_golden_events(_GOLDEN_MAJOR))
def _wav_to_mono(path):
with wave.open(path) as wf:
sr = wf.getframerate()
channels = wf.getnchannels()
frames = wf.readframes(wf.getnframes())
data = np.frombuffer(frames, dtype=np.int16).astype(np.float32) / 32768.0
if channels > 1:
data = data.reshape(-1, channels).mean(axis=1)
return data, sr
def metal_probability(wav_path, clap_dir):
import torch
from transformers import ClapModel, ClapProcessor
model = ClapModel.from_pretrained(clap_dir)
proc = ClapProcessor.from_pretrained(clap_dir)
audio, sr = _wav_to_mono(wav_path)
if sr != 48000:
n = int(len(audio) * 48000 / sr)
audio = np.interp(np.linspace(0, len(audio), n), np.arange(len(audio)), audio).astype(np.float32)
audio = audio[: 48000 * 10]
inputs = proc(
text=_STYLE_CANDIDATES,
audios=[audio] * len(_STYLE_CANDIDATES),
sampling_rate=48000,
return_tensors="pt",
padding=True,
)
with torch.no_grad():
probs = model(**inputs).logits_per_audio[0].softmax(dim=-1)
return float(probs[0])
def transcribe_melody_abc(wav_path, sheetsage2_dir, output_dir):
import torch
from transformers import AutoModel
audio, sr = _wav_to_mono(wav_path)
model = AutoModel.from_pretrained(sheetsage2_dir, trust_remote_code=True)
result = model.transcribe(
torch.from_numpy(audio[None, :]),
sampling_rate=sr,
output_dir=str(output_dir),
melody_only=True,
dtype="fp32",
)
abc = result.get("abc")
if not abc:
raise RuntimeError(f"SheetSage2 produced no melody ABC: {result.get('abc_error')}")
return abc
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--clap-dir", required=True)
parser.add_argument("--sheetsage2-dir", required=True)
parser.add_argument("metal_wav")
parser.add_argument("control_wav")
parser.add_argument("--work-dir", default=None)
args = parser.parse_args()
work = args.work_dir or "."
report = {"metal_probability": metal_probability(args.metal_wav, args.clap_dir)}
report["melody_score"] = melody_score(
transcribe_melody_abc(args.metal_wav, args.sheetsage2_dir, f"{work}/metal_score")
)
report["control_score"] = melody_score(
transcribe_melody_abc(args.control_wav, args.sheetsage2_dir, f"{work}/control_score")
)
print(json.dumps(report))
if __name__ == "__main__":
main()
@@ -1105,6 +1105,38 @@ class TestTTSMethods:
assert isinstance(audio_obj, CreateAudio)
assert audio_obj.speed == 1.0
@pytest.mark.asyncio
async def test_audio_synthesis_error_flag_raises_tts_generation_error(
self,
speech_server,
mocker: MockerFixture,
):
"""A model-flagged synthesis failure surfaces as a non-retryable
TTSGenerationError, so create_speech answers 500 instead of shipping a
zero-length WAV or crashing on an unpacked Response."""
async def mock_generate():
yield create_mock_audio_output_for_test()
mocker.patch.object(
speech_server,
"_prepare_speech_generation",
new=mocker.AsyncMock(return_value=("speech-synth-err", mock_generate(), {})),
)
adapter = mocker.MagicMock()
adapter.collect_response_metadata = lambda _audio_output, collect: collect.__setitem__(
"audio_synthesis_error", True
)
mocker.patch.object(speech_server, "_get_tts_adapter", return_value=adapter)
with pytest.raises(TTSGenerationError, match="failed to synthesize audio") as exc_info:
await speech_server._generate_audio_bytes(
OpenAICreateSpeechRequest(input="Hello"),
collect={},
)
assert exc_info.value.retryable is False
def test_is_tts_detection_with_tts_stage(self, mocker: MockerFixture):
"""Test TTS model detection when TTS stage exists."""
mock_engine_client = mocker.MagicMock()
@@ -183,6 +183,7 @@ _PIPELINE_STAGES = [
"token2wav",
"tts",
"encoder",
"yue2",
]
_STAGES = [*_PIPELINE_STAGES, None, "vae", "not_a_real_stage"]
+14 -4
View File
@@ -15,6 +15,7 @@ import numpy as np
import pytest
import torch
from vllm.sampling_params import SamplingParams
from vllm.v1.sample.metadata import SamplingMetadata
from vllm.v1.worker import gpu_input_batch
from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
@@ -103,6 +104,8 @@ def test_post_sample_talker_mtp_uses_current_sample_and_hidden() -> None:
)
assert received["req_ids"] == ["ready"]
assert isinstance(received["input_ids"], torch.Tensor)
assert isinstance(received["hidden_states"], torch.Tensor)
assert received["input_ids"].tolist() == [101]
assert received["hidden_states"].tolist() == [[1.0, 2.0]]
assert received["req_infos"] == [{"duplex": {"data_plane": True}}]
@@ -139,6 +142,8 @@ def test_post_sample_talker_mtp_uses_gpu_token_with_async_scheduling() -> None:
)
assert received["req_ids"] == ["ready"]
assert isinstance(received["input_ids"], torch.Tensor)
assert isinstance(received["hidden_states"], torch.Tensor)
assert received["input_ids"].tolist() == [101]
assert received["hidden_states"].tolist() == [[1.0, 2.0]]
assert received["req_infos"] == [{"duplex": {"data_plane": True}}]
@@ -275,7 +280,12 @@ def test_speech_extra_params_reach_model_sampler_as_sampling_metadata(monkeypatc
)
input_batch.sampling_metadata = input_batch._make_sampling_metadata()
received = []
received: list[SamplingMetadata] = []
def sample(logits: torch.Tensor, metadata: SamplingMetadata) -> str:
received.append(metadata)
return "model-sampler"
runner = object.__new__(GPUARModelRunner)
runner._omni_cache_policy = ModelCachePolicy(needs_full_hidden_states=True)
runner._pooler_payload_include_hidden_flag = True
@@ -283,7 +293,7 @@ def test_speech_extra_params_reach_model_sampler_as_sampling_metadata(monkeypatc
runner.model = SimpleNamespace(
prefer_model_sampler=True,
skips_model_sampler_output_token_history=True,
sample=lambda logits, metadata: received.append(metadata) or "model-sampler",
sample=sample,
)
runner.sampler = SimpleNamespace()
logits = torch.zeros((1, 4))
@@ -1670,7 +1680,6 @@ class TestMergeModelKvTransferMetadata:
# ...while the engine-shared originals are untouched.
assert original["r1"]["custom_metadata"] is original_meta
assert original_meta == {"a": 1}
assert "talker_codes" not in original["r1"]["custom_metadata"]
def test_merge_without_model_meta_passes_entry_through(self):
class _Model:
@@ -1713,7 +1722,7 @@ class TestDownstreamPayloadMemoization:
return runner
def test_missing_marker_defaults_true_without_memoizing(self):
stages = {"r1": None}
stages: dict[str, int | None] = {"r1": None}
runner = self._runner(stages)
assert runner._request_needs_downstream_stage_payload("r1") is True
@@ -1861,6 +1870,7 @@ class TestPreferModelSamplerNoneFallback:
"minicpmo_4_5",
"minimax_music3",
"nemotron_voicechat",
"yue2", # Always supplies SamplerOutput, including empty/prefill steps.
}
assert declarers == expected, (
"The set of models declaring `prefer_model_sampler` changed:\n"