From 5ca23106fd6a0f500deea818bc2b7089306a210f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B1=AA=E5=BF=97=E9=B9=8F?= Date: Mon, 5 Oct 2026 11:09:37 +0800 Subject: [PATCH] [Model] Add SheetSage2 preprocessing and YuE2 score handoff (#8371) Signed-off-by: princepride --- docs/models/supported_models.md | 2 +- recipes/README.md | 2 + recipes/m-a-p/SheetSage2-H200.md | 259 ++++++++++++++++++++++ recipes/m-a-p/YuE2-3B.md | 5 + tests/tools/test_sheetsage2_transcribe.py | 193 ++++++++++++++++ tools/requirements/sheetsage2.txt | 13 ++ tools/sheetsage2_transcribe.py | 178 +++++++++++++++ 7 files changed, 651 insertions(+), 1 deletion(-) create mode 100644 recipes/m-a-p/SheetSage2-H200.md create mode 100644 tests/tools/test_sheetsage2_transcribe.py create mode 100644 tools/requirements/sheetsage2.txt create mode 100644 tools/sheetsage2_transcribe.py diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 336b41d8b..5d2178cc2 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -83,7 +83,7 @@ th { | `OmniGen2Pipeline` | OmniGen2 | `OmniGen2/OmniGen2` | ✅︎ | ✅︎ | | ✅︎ | — | | `StableAudioPipeline` | Stable-Audio-Open | `stabilityai/stable-audio-open-1.0` | ✅︎ | ✅︎ | | ✅︎ | — | | `MiniMaxMusic3ForConditionalGeneration` | MiniMax Music 3 (text-to-music) | `MiniMaxAI/MiniMax-Music3` | ✅︎ | | | | — | -| `Yue2ForCausalLM` | YuE2-3B (text-to-music with ABC plan) | `m-a-p/YuE2-3B` | ✅︎ | | | | — | +| `Yue2ForCausalLM` | YuE2-3B (text-to-music with ABC plan) | `m-a-p/YuE2-3B` | ✅︎ | | | | [Recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/m-a-p/YuE2-3B.md) / [Audio-to-score](https://github.com/vllm-project/vllm-omni/blob/main/recipes/m-a-p/SheetSage2-H200.md) | | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | ✅︎ | | | ✅︎ | [Published](https://recipes.vllm.ai/Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice) | | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — | | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — | diff --git a/recipes/README.md b/recipes/README.md index c6d02f0c5..9a3837367 100644 --- a/recipes/README.md +++ b/recipes/README.md @@ -66,6 +66,8 @@ recipes/ | [`krea/Krea-2.md`](./krea/Krea-2.md) | Text-to-image (Turbo + Raw), offline + online, with LoRA | 1x H100 80GB / 1x Intel Arc Pro B70 32GB | | [`LTX/LTX-2.md`](./LTX/LTX-2.md) | LTX-2/LTX-2.3 text-to-video and image-to-video with synchronized audio | H200 141GB / 96GB-class GPU / 1x Intel Arc Pro B70 32GB | | [`LTX/LTX-2.5.md`](./LTX/LTX-2.5.md) | LTX-2.5-Diffusers: Full/SFT one-stage and distilled two-stage T2V/I2V with synchronized audio | NVIDIA B300; cuDNN-qualified / 1x Intel Arc Pro B70 32GB | +| [`m-a-p/SheetSage2-H200.md`](./m-a-p/SheetSage2-H200.md) | Official audio-to-score preprocessing and YuE2 request export | 1x H200 141GB | +| [`m-a-p/YuE2-3B.md`](./m-a-p/YuE2-3B.md) | Offline and online text-to-music with optional ABC score | 1x RTX 4090 24GB | | [`MammothModa2/MammothModa2.md`](./MammothModa2/MammothModa2.md) | Preview and Dev text-to-image (AR → DiT); Dev text/image understanding | Preview: 1x L40S 48GB / 1x ≥40GB GPU; Dev: 1x NVIDIA GPU with sufficient cache headroom | | [`NVIDIA/SANA-Video-2B.md`](./NVIDIA/SANA-Video-2B.md) | Native and Diffusers-adapter 480p/720p T2V+I2V | 1x RTX 5090 32GB | | [`meituan-longcat/LongCat-Video-Avatar-1.5.md`](./meituan-longcat/LongCat-Video-Avatar-1.5.md) | Audio-driven avatar video generation (AT2V / AI2V, single- and multi-speaker, AVC continuation) | 1x H100 80GB | diff --git a/recipes/m-a-p/SheetSage2-H200.md b/recipes/m-a-p/SheetSage2-H200.md new file mode 100644 index 000000000..9b278ca68 --- /dev/null +++ b/recipes/m-a-p/SheetSage2-H200.md @@ -0,0 +1,259 @@ +# SheetSage2 audio-to-score preprocessing — H200 + +## Summary + +- Vendor: m-a-p +- Model: `m-a-p/SheetSage2` with the `m-a-p/MERT-v2-FullSong` parent +- Task: recording → ABC score, MIDI and timed music annotations +- Mode: standalone official Transformers inference; optional YuE2 request export +- Hardware: one NVIDIA H200 (141 GB) +- Maintainer: [@princepride](https://github.com/princepride) + +## When to use this recipe + +Use a recording as the musical score for a YuE2 cover, or inspect its melodies, +chords and structure. The tool calls SheetSage2's official `transcribe()` API. +It does not register SheetSage2 as a native vLLM model or start an Omni server. + +The transcription tool works independently of YuE2. For score-conditioned +generation, use an Omni checkout containing the merged +[PR #7886](https://github.com/vllm-project/vllm-omni/pull/7886) +(`423f34326ed420e5acf0b1fb862a1b5ffb7e0fa7` or later). See the +[YuE2 recipe](YuE2-3B.md) for the generation model's runtime contract. + +## Supported model contract + +| Item | Contract | +| --- | --- | +| Input | One local recording decodable by FFmpeg; upstream converts it to mono, 24 kHz | +| Duration | Entire recording by default; `--max-seconds` limits preprocessing to a prefix | +| Full score | Default: melody voices and chords; exported request uses `cot=full` | +| Melody score | `--melody-only`: vocal and instrumental melodies, without score/playback chords; request uses `cot=melody` | +| Outputs | `score.abc`, `transcription.mid`, upstream annotations and `manifest.json` | +| YuE2 handoff | `--lyrics-file` and `--style` together also write `yue2_request.json` | +| Output directory | Must be new or empty; failed exports produce no YuE2 request | +| Deployment profile | One process, one H200, FP32 weight loading and BF16 inference autocast | + +`--melody-only` does not isolate or clone a singer. The upstream model still +decodes its task annotations and retains those raw annotation files. YuE2 +uses the exported score to condition newly generated audio; a transcription +does not guarantee an exact reproduction of the recording. + +## References + +- [Official SheetSage2 model card](https://huggingface.co/m-a-p/SheetSage2) +- [Pinned SheetSage2 source](https://huggingface.co/m-a-p/SheetSage2/tree/cafc0df1021e14f49e928c4b345f5959d414ef64) +- [Official YuE2 model card and cover documentation](https://huggingface.co/m-a-p/YuE2-3B) +- Tool: [`tools/sheetsage2_transcribe.py`](../../tools/sheetsage2_transcribe.py) +- Offline generation: [`examples/offline_inference/yue2/end2end.py`](../../examples/offline_inference/yue2/end2end.py) +- [Supported generation models](../../docs/models/supported_models.md) + +## Hardware + +- Accelerator: one NVIDIA H200, 141 GB; no multi-device interconnect required. +- Qualification: local M4A transcription, melody-only prefix, full-score + whole-song export and one offline YuE2 cover. Run transcription and generation + sequentially in separate processes. Other devices, CPU inference and + performance scaling are not qualified by this recipe. + +## Software environment + +- Ubuntu 22.04, Python 3.10, NVIDIA driver 590.48.01. +- PyTorch/torchaudio 2.8.0 (CUDA 12.8 wheels), Transformers 4.45.2. +- FFmpeg 4.4.2 was used for the local M4A check; upstream recommends FFmpeg 6.1. +- vLLM: not required in the preprocessing environment. +- vLLM-Omni: tool developed against main `a038b38179e9c788d3af6a8e73f94652afb247e4`. +- YuE2 qualification: vLLM 0.30.0, PyTorch 2.13.0+cu130, Transformers 5.14.1; + #7886 at `5cc8acc942e76fa6f12f8b2f2558c2663abc2aa5` with the FP32 VAE-loading + and scalar `truncated` fixes subsequently included in the merged integration. + +## Command + +### Set up transcription + +Run from this repository's root. Create a **separate environment**: the +official transcriber's Transformers version differs from Omni's serving +dependencies. FFmpeg must be on `PATH`. + +```bash +uv venv --python 3.10 .venv-sheetsage2 +uv pip install --python .venv-sheetsage2/bin/python \ + -r tools/requirements/sheetsage2.txt + +.venv-sheetsage2/bin/python tools/sheetsage2_transcribe.py reference.wav \ + --trust-remote-code --device cuda:0 --melody-only \ + --output-dir outputs/reference-score +``` + +The default Hub checkpoint and its custom Python code are pinned to +`cafc0df1021e14f49e928c4b345f5959d414ef64`. Its configuration pins the MERT-v2 +parent to `d8ba1c745e733b3908ce6ad16ebeb17ac7600a42`. Review that code before +passing `--trust-remote-code`. `--revision` can override the default pin; +custom Hub IDs use their default revision unless one is supplied. + +Local snapshots must preserve this pinned source layout: `modeling_sheetsage2.py` +and its transitive relative-import dependencies are sibling `.py` files in the +snapshot root. The local module-cache workaround targets that flat layout; +reorganized snapshots with nested Python packages are not supported by it. + +For offline use, download **both** model snapshots, including their Python, +JSON and safetensors files, before disconnecting: + +```bash +.venv-sheetsage2/bin/hf download m-a-p/SheetSage2 \ + --revision cafc0df1021e14f49e928c4b345f5959d414ef64 \ + --local-dir models/SheetSage2 +.venv-sheetsage2/bin/hf download m-a-p/MERT-v2-FullSong \ + --revision d8ba1c745e733b3908ce6ad16ebeb17ac7600a42 \ + --local-dir models/MERT-v2-FullSong +``` + +### Transcribe a full recording for a cover + +Supply target lyrics in `lyrics.txt`, including YuE2 section tags such as +`[Verse]` and `[Chorus]`. Match the lyric phrasing to the reference melody. +This example retains the melody and chords of the whole recording: + +```bash +.venv-sheetsage2/bin/python tools/sheetsage2_transcribe.py reference.wav \ + --model models/SheetSage2 --base-model-path models/MERT-v2-FullSong \ + --local-files-only --trust-remote-code --device cuda:0 \ + --lyrics-file lyrics.txt --style 'Chinese folk, guzheng, gentle vocals' \ + --seed 831001 --output-dir outputs/cover-score +``` + +Add `--melody-only` for a new arrangement without the original chords; +this also changes the exported request from `cot=full` to `cot=melody`. +For a short transcription smoke test, add `--max-seconds 45` and use lyrics +appropriate to that excerpt. Use a fresh output directory for each run. + +### Generate offline + +Switch to your **Omni environment** for generation. Download `m-a-p/YuE2-3B` +and `m-a-p/YuE2-Vae` to `models/YuE2-3B` and `models/YuE2-Vae`, respectively. +The offline entrypoint needs a local YuE2 directory containing `qwen.tiktoken`. +From the repository root, pass the exported ABC to the existing entrypoint: + +```bash +CUDA_VISIBLE_DEVICES=0 python examples/offline_inference/yue2/end2end.py \ + --model models/YuE2-3B --vae models/YuE2-Vae \ + --lyrics "$(cat lyrics.txt)" \ + --style 'Chinese folk, guzheng, gentle vocals' \ + --abc-file outputs/cover-score/score.abc --cot full \ + --seed 831001 --max-frames 9000 --gpu-memory-utilization 0.25 \ + --output cover.wav +``` + +Use `--cot melody` if the score was transcribed with `--melody-only`. +Changing `--cot` alone does not remove chords from an existing score. +The 9,000-frame budget permits up to 360 seconds at 25 frames/s; it does +not force that duration. The entrypoint's default 200-frame budget caps +audio at 8 seconds. Check that the final report says `truncated=False`. +The memory fraction above was used on one H200; it is not a measured +minimum memory requirement. + +### Submit over HTTP (configuration-only) + +The exported JSON can also be submitted to YuE2's speech endpoint. These +commands have not been qualified with an HTTP synthesis run in this recipe. +Start YuE2 in its **Omni environment**: + +```bash +vllm serve m-a-p/YuE2-3B --omni --port 8091 +``` + +Submit the exported file: + +```bash +curl --fail-with-body http://localhost:8091/v1/audio/speech \ + -H 'Content-Type: application/json' \ + --data-binary @outputs/cover-score/yue2_request.json \ + --output cover.wav +``` + +The JSON contains the score itself, not a server-local file path. It sets +`stream=false`, requests WAV, and preserves an optional generation seed. +Use `--yue2-model` to match a custom served model name. The tool does not +submit the request or set a generation token budget; the server's own +duration and context limits apply. + +## Verification + +After a successful transcription, set `output` below to its output directory +(`outputs/reference-score` for the transcription-only example). The YuE2 +request is checked when exported with `--lyrics-file` and `--style`: + +```bash +.venv-sheetsage2/bin/python - <<'PY' +import json +from pathlib import Path +import mido + +output = Path("outputs/cover-score") +abc = (output / "score.abc").read_text(encoding="utf-8") +manifest = json.loads((output / "manifest.json").read_text(encoding="utf-8")) +assert abc.strip() +request_path = output / "yue2_request.json" +if request_path.exists(): + request = json.loads(request_path.read_text(encoding="utf-8")) + assert request["extra_params"]["abc"] == abc + assert request["extra_params"]["cot"] == ("melody" if manifest["melody_only"] else "full") + print("YuE2 request verified") +midi = mido.MidiFile(output / "transcription.mid") +assert any(msg.type == "note_on" and msg.velocity for track in midi.tracks for msg in track) +print("ABC, MIDI and manifest verified") +PY +``` + +Run the CPU CLI contract tests in the repository's normal test +environment (with pytest and pytest-mock installed): + +```bash +python -m pytest tests/tools/test_sheetsage2_transcribe.py -m 'core_model and cpu' -q +``` + +These tests cover request construction, loading options, validation and +failure handling with a mocked backend. They do not measure transcription +accuracy or generated cover similarity. + +Local H200 checks with the pinned backend: + +| Input / mode | Result | +| --- | --- | +| 45 s M4A prefix, melody-only | 579 ABC characters; parseable MIDI with 118 note-on events | +| 242.219 s M4A, full score | 3,033 ABC characters; parseable MIDI with 1,087 note-on events | +| Default Hub ID, 10 s prefix | Successful ABC and MIDI export | +| Local snapshots, fresh Transformers module cache | Successful offline full-score export | +| Both generated JSON requests, #7886 adapter at `5cc8acc942e76fa6f12f8b2f2558c2663abc2aa5` | Validation and prompt construction passed (919 / 2,821 prompt tokens) | +| Full score + new Mandarin lyrics, offline YuE2, `cot=full`, seed 831001, 9,000-frame budget | 241.319 s, 48 kHz stereo WAV; 6,034 generated tokens; `truncated=False` | + +The offline cover used a 108 BPM Chinese traditional pop style and one H200 +with `--gpu-memory-utilization 0.25`. The saved waveform contained finite +samples, peak amplitude 0.9501, RMS 0.1232 and no full-scale samples. This is +one functional run, not an accuracy or performance benchmark. It does not +establish perceptual equivalence to the recording. HTTP synthesis and +recordings longer than 300 seconds were not exercised. The local recording, +lyrics, generated scores and cover audio are not included in the repository. + +## Notes + +- Weights load in FP32 because upstream merges MERT adapters before inference + autocast. `--dtype fp32` disables BF16 autocast. +- For local snapshots the tool prepares transitive Python dependencies in + Transformers' module cache. This avoids the pinned Transformers version's + direct-import-only copy behavior without editing upstream files. +- SheetSage2 manages long recordings with its own overlapping windows. This + tool leaves windowing and notation generation to the official implementation. +- Failed runs may retain upstream diagnostic annotations. Choose a new output + directory when retrying; no incomplete request is published for synthesis. +- Model weights have upstream license terms (CC BY-NC 4.0); this repository + does not bundle weights or recordings. + +## Supported features + +| Feature | Status | +| --- | --- | +| Audio-to-score preprocessing | Official backend; local file input | +| YuE2 score-conditioned generation | Offline ABC handoff qualified on H200; HTTP request export validated, HTTP synthesis configuration-only | +| Native Omni inference, continuous batching, TP/PP/SP | Not implemented for SheetSage2 | +| Streaming transcription or cover audio | Not supported by this tool | diff --git a/recipes/m-a-p/YuE2-3B.md b/recipes/m-a-p/YuE2-3B.md index 197156e66..64ede18eb 100644 --- a/recipes/m-a-p/YuE2-3B.md +++ b/recipes/m-a-p/YuE2-3B.md @@ -19,6 +19,11 @@ renders it with the lyrics. `cot=off` skips the score and generates music directly. Weights are **CC BY-NC 4.0** (non-commercial); this recipe loads user-downloaded weights and bundles none. +To start from a recording, use the [SheetSage2 H200 recipe](SheetSage2-H200.md) +to transcribe it into ABC in a separate environment, then pass the score to +YuE2 with `--abc-file`. It includes full-song frame budgets and offline cover +validation details. + ## References - Model card: [m-a-p/YuE2-3B](https://huggingface.co/m-a-p/YuE2-3B) diff --git a/tests/tools/test_sheetsage2_transcribe.py b/tests/tools/test_sheetsage2_transcribe.py new file mode 100644 index 000000000..c38b2f41f --- /dev/null +++ b/tests/tools/test_sheetsage2_transcribe.py @@ -0,0 +1,193 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Contracts for the standalone audio-to-score/YuE2 handoff tool.""" + +import json +import sys +from pathlib import Path + +import pytest + +from tools import sheetsage2_transcribe as cli + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +@pytest.fixture +def arguments(tmp_path): + audio = tmp_path / "song.wav" + audio.write_bytes(b"audio is decoded by the upstream transcriber") + return cli.create_parser().parse_args([str(audio), "--output-dir", str(tmp_path / "score"), "--trust-remote-code"]) + + +@pytest.fixture +def backend(mocker): + model = mocker.Mock() + model.config = mocker.Mock( + spec_set=["model_type", "base_model_revision"], model_type="sheetsage2", base_model_revision="parent-sha" + ) + model.eval.return_value = model + model.to.return_value = model + model.transcribe.return_value = {"abc": "X:1\nM:4/4\nK:C\nC4|\n", "midi": b"MThd\x00\x00\x00\x06"} + loader = mocker.Mock(return_value=model) + torch = mocker.Mock( + float32=object(), + __version__="test", + get_num_threads=lambda: 8, + set_num_threads=mocker.Mock(), + accelerator=mocker.Mock(is_available=lambda: False), + ) + mocker.patch.dict( + sys.modules, + {"torch": torch, "transformers": mocker.Mock(AutoModel=mocker.Mock(from_pretrained=loader))}, + ) + return mocker.Mock(model=model, loader=loader, torch=torch) + + +@pytest.mark.parametrize("melody_only,expected_cot", [(True, "melody"), (False, "full")]) +def test_score_handoff_preserves_unicode_and_selects_matching_cot(arguments, backend, melody_only, expected_cot): + arguments.lyrics_file = arguments.audio.with_suffix(".txt") + arguments.lyrics_file.write_text("[Verse]\n江边的清晨", encoding="utf-8") + arguments.style = " Chinese folk, bamboo flute " + arguments.yue2_model = "music-server" + arguments.melody_only = melody_only + arguments.seed = 123 + + output = cli.transcribe(arguments) + + request = json.loads((output / "yue2_request.json").read_text()) + assert request == { + "model": "music-server", + "input": "[Verse]\n江边的清晨", + "instructions": "Chinese folk, bamboo flute", + "response_format": "wav", + "stream": False, + "seed": 123, + "extra_params": {"cot": expected_cot, "abc": backend.model.transcribe.return_value["abc"]}, + } + assert backend.model.transcribe.call_args.kwargs["melody_only"] == melody_only + assert "max_new_tokens" not in request # Leave natural song length to the server. + + +def test_default_model_pins_code_and_loads_adapters_in_fp32(arguments, backend): + output = cli.transcribe(arguments) + options = backend.loader.call_args.kwargs + assert options["revision"] == options["code_revision"] == cli.DEFAULT_REVISION + assert options["torch_dtype"] is backend.torch.float32 + backend.model.to.assert_called_once_with("cpu") + assert not (output / "yue2_request.json").exists() + + +def test_local_offline_checkpoint_forwards_parent_without_default_hub_revision(arguments, backend, tmp_path, mocker): + arguments.model = str(tmp_path / "SheetSage2") + snapshot = Path(arguments.model) + snapshot.mkdir() + source = snapshot / "modeling_sheetsage2.py" + direct = snapshot / "exports_sheetsage2.py" + transitive = snapshot / "chord_spelling_sheetsage2.py" + source.write_text("from .exports_sheetsage2 import export\n") + direct.write_text("from .chord_spelling_sheetsage2 import chord\n") + transitive.write_text("chord = 'C'\n") + dynamic_modules = mocker.Mock() + dynamic_modules.get_relative_import_files.return_value = [str(direct), str(transitive)] + mocker.patch.dict(sys.modules, {"transformers.dynamic_module_utils": dynamic_modules}) + calls = mocker.Mock() + calls.attach_mock(dynamic_modules.get_cached_module_file, "prime") + calls.attach_mock(backend.loader, "load") + arguments.base_model_path = tmp_path / "MERT" + arguments.base_model_path.mkdir() + arguments.local_files_only = True + cli.transcribe(arguments) + dynamic_modules.get_relative_import_files.assert_called_once_with(str(source)) + dynamic_modules.get_cached_module_file.assert_has_calls( + [ + mocker.call(arguments.model, direct.name, local_files_only=True), + mocker.call(arguments.model, transitive.name, local_files_only=True), + ] + ) + assert [call[0] for call in calls.mock_calls] == ["prime", "prime", "load"] + backend.model.transcribe.assert_called_once() + options = backend.loader.call_args.kwargs + assert options["revision"] is None + assert options["local_files_only"] is True + assert options["base_model_path"] == str(arguments.base_model_path) + + +@pytest.mark.parametrize("limit", [0, -1, float("inf"), float("nan")]) +def test_invalid_duration_fails_before_model_loading(arguments, backend, limit): + arguments.max_seconds = limit + with pytest.raises(ValueError, match="finite and positive"): + cli.transcribe(arguments) + backend.loader.assert_not_called() + assert not arguments.output_dir.exists() + + +def test_nonempty_output_is_not_reused(arguments, backend): + arguments.output_dir.mkdir() + old = arguments.output_dir / "yue2_request.json" + old.write_text("old request") + with pytest.raises(ValueError, match="new or empty"): + cli.transcribe(arguments) + assert old.read_text() == "old request" + backend.loader.assert_not_called() + + +def test_explicit_code_trust_is_required(arguments, backend): + arguments.trust_remote_code = False + with pytest.raises(ValueError, match="--trust-remote-code"): + cli.transcribe(arguments) + backend.loader.assert_not_called() + + +@pytest.mark.parametrize( + "result,message", + [ + ({"abc": "", "midi": b"MThd"}, "ABC export failed"), + ({"abc": "X:1", "abc_error": "invalid beat grid", "midi": b"MThd"}, "invalid beat grid"), + ({"abc": "X:1", "midi": b"invalid"}, "MIDI export failed"), + ], +) +def test_failed_export_never_writes_a_yue2_request(arguments, backend, result, message): + arguments.lyrics_file = arguments.audio.with_suffix(".txt") + arguments.lyrics_file.write_text("lyrics") + arguments.style = "folk" + backend.model.transcribe.return_value = result + with pytest.raises(RuntimeError, match=message): + cli.transcribe(arguments) + assert not (arguments.output_dir / "yue2_request.json").exists() + assert not (arguments.output_dir / "manifest.json").exists() + + +def test_style_requires_lyrics_before_loading(arguments, backend): + arguments.style = "folk" + with pytest.raises(ValueError, match="supplied together"): + cli.transcribe(arguments) + backend.loader.assert_not_called() + + +def test_seed_requires_request_context_before_loading(arguments, backend): + arguments.seed = 123 + with pytest.raises(ValueError, match="supply --lyrics-file and --style"): + cli.transcribe(arguments) + backend.loader.assert_not_called() + assert not arguments.output_dir.exists() + + +@pytest.mark.parametrize("seed", [-1, 2**63]) +def test_out_of_range_seed_fails_before_loading(arguments, backend, seed): + arguments.lyrics_file = arguments.audio.with_suffix(".txt") + arguments.lyrics_file.write_text("lyrics", encoding="utf-8") + arguments.style = "folk" + arguments.seed = seed + with pytest.raises(ValueError, match="between 0 and"): + cli.transcribe(arguments) + backend.loader.assert_not_called() + assert not arguments.output_dir.exists() + + +def test_cli_reports_export_failure_as_nonzero(arguments, backend, capsys): + backend.model.transcribe.return_value = {"abc": "", "midi": b"MThd"} + with pytest.raises(SystemExit) as exc: + cli.main([str(arguments.audio), "--output-dir", str(arguments.output_dir), "--trust-remote-code"]) + assert exc.value.code == 1 + assert "ABC export failed" in capsys.readouterr().err diff --git a/tools/requirements/sheetsage2.txt b/tools/requirements/sheetsage2.txt new file mode 100644 index 000000000..c60096e12 --- /dev/null +++ b/tools/requirements/sheetsage2.txt @@ -0,0 +1,13 @@ +# Isolated preprocessing environment; do not install into the Omni server venv. +# Matches m-a-p/SheetSage2 at cafc0df1021e14f49e928c4b345f5959d414ef64. +torch==2.8.0 +torchaudio==2.8.0 +transformers==4.45.2 +huggingface-hub==0.36.0 +safetensors==0.5.3 +numpy==1.24.3 +scipy==1.13.1 +mir_eval==0.8.2 +pretty_midi==0.2.10 +mido==1.3.3 +setuptools==78.1.1 diff --git a/tools/sheetsage2_transcribe.py b/tools/sheetsage2_transcribe.py new file mode 100644 index 000000000..775beb975 --- /dev/null +++ b/tools/sheetsage2_transcribe.py @@ -0,0 +1,178 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Convert music to a score and optionally prepare a YuE2 speech request. + +Run this preprocessing tool in the separate environment documented in +recipes/m-a-p/SheetSage2-H200.md. It uses the upstream transcriber, not an Omni +engine, and does not import vLLM or change the serving environment. +""" + +from __future__ import annotations + +import argparse +import json +import math +from pathlib import Path + +DEFAULT_MODEL = "m-a-p/SheetSage2" +DEFAULT_REVISION = "cafc0df1021e14f49e928c4b345f5959d414ef64" + + +def create_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("audio", type=Path, help="Local music recording (decoded by FFmpeg)") + parser.add_argument("--output-dir", required=True, type=Path, help="New or empty output directory") + parser.add_argument("--model", default=DEFAULT_MODEL, help="SheetSage2 Hub ID or local snapshot") + parser.add_argument("--revision", help="Model/code revision; the default Hub model uses a pinned commit") + parser.add_argument("--base-model-path", type=Path, help="Local MERT-v2 parent for offline loading") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument( + "--trust-remote-code", action="store_true", help="Allow the upstream model's custom Python code" + ) + parser.add_argument("--device", default="auto", help="auto, cpu, cuda:0, etc.") + parser.add_argument( + "--dtype", choices=("bf16", "fp32"), default="bf16", help="Inference autocast; weights load in FP32" + ) + parser.add_argument("--melody-only", action="store_true", help="Retain vocal/instrumental melodies and omit chords") + parser.add_argument( + "--max-seconds", type=float, help="Transcribe only this many seconds; otherwise process the song" + ) + parser.add_argument("--lyrics-file", type=Path, help="With --style, write yue2_request.json for /v1/audio/speech") + parser.add_argument("--style", help="Target musical arrangement for the optional YuE2 request") + parser.add_argument("--yue2-model", default="m-a-p/YuE2-3B", help="Served model name in the optional request") + parser.add_argument("--seed", type=int, help="Optional YuE2 generation seed") + return parser + + +def validate_args(args: argparse.Namespace) -> str | None: + if not args.audio.is_file(): + raise ValueError(f"Audio file does not exist: {args.audio}") + if not args.trust_remote_code: + raise ValueError( + "SheetSage2 uses custom model code; pass --trust-remote-code after reviewing the model repository" + ) + if args.max_seconds is not None and (not math.isfinite(args.max_seconds) or args.max_seconds <= 0): + raise ValueError("--max-seconds must be finite and positive") + if args.output_dir.exists() and (not args.output_dir.is_dir() or any(args.output_dir.iterdir())): + raise ValueError("--output-dir must be a new or empty directory") + if args.base_model_path is not None and not args.base_model_path.is_dir(): + raise ValueError(f"MERT-v2 directory does not exist: {args.base_model_path}") + if (args.lyrics_file is None) != (args.style is None): + raise ValueError("--lyrics-file and --style must be supplied together") + if args.lyrics_file is None: + if args.seed is not None: + raise ValueError("--seed applies to the YuE2 request; supply --lyrics-file and --style") + return None + lyrics = args.lyrics_file.read_text(encoding="utf-8").strip() + if not lyrics or not args.style.strip(): + raise ValueError("Lyrics and --style must not be blank") + if not args.yue2_model.strip(): + raise ValueError("--yue2-model must not be blank") + if args.seed is not None and not 0 <= args.seed < 2**63: + raise ValueError("--seed must be between 0 and 2**63 - 1") + return lyrics + + +def build_yue2_request(abc: str, lyrics: str, args: argparse.Namespace) -> dict[str, object]: + request: dict[str, object] = { + "model": args.yue2_model, + "input": lyrics, + "instructions": args.style.strip(), + "response_format": "wav", + "stream": False, + "extra_params": {"cot": "melody" if args.melody_only else "full", "abc": abc}, + } + if args.seed is not None: + request["seed"] = args.seed + return request + + +def transcribe(args: argparse.Namespace) -> Path: + # Validate before loading dependencies/weights or touching an existing output. + lyrics = validate_args(args) + import torch + from transformers import AutoModel + + if Path(args.model).is_dir(): + # Transformers 4.45 only copies direct imports for local snapshots. + # Prime transitive dependencies too (e.g. chord_spelling_sheetsage2), + # using its cache API without modifying the upstream source files. + # The pinned snapshot keeps the entry point and all relative imports + # as sibling .py files; local snapshots must preserve that flat layout. + from transformers.dynamic_module_utils import get_cached_module_file, get_relative_import_files + + source = Path(args.model) / "modeling_sheetsage2.py" + for dependency in get_relative_import_files(str(source)): + get_cached_module_file(args.model, Path(dependency).name, local_files_only=True) + + torch.set_num_threads(min(4, torch.get_num_threads())) + device = args.device + if device == "auto": + device = torch.accelerator.current_accelerator().type if torch.accelerator.is_available() else "cpu" + revision = args.revision + if revision is None and args.model == DEFAULT_MODEL: + revision = DEFAULT_REVISION + load_options = { + "trust_remote_code": args.trust_remote_code, + "revision": revision, + "code_revision": revision, + "local_files_only": args.local_files_only, + # Upstream merges the MERT adapters before any reduced-precision cast. + "torch_dtype": torch.float32, + } + if args.base_model_path is not None: + load_options["base_model_path"] = str(args.base_model_path) + model = AutoModel.from_pretrained(args.model, **load_options).eval().to(device) + if model.config.model_type != "sheetsage2": + raise ValueError("--model must identify a SheetSage2 checkpoint") + args.output_dir.mkdir(parents=True, exist_ok=True) + result = model.transcribe( + str(args.audio), + output_dir=str(args.output_dir), + melody_only=args.melody_only, + dtype=args.dtype, + max_seconds=args.max_seconds, + ) + abc = result.get("abc") + if result.get("abc_error") or not isinstance(abc, str) or not abc.strip(): + raise RuntimeError( + f"ABC export failed: {result.get('abc_error') or 'empty score'}; no YuE2 request was written" + ) + midi = result.get("midi") + if not isinstance(midi, bytes) or not midi.startswith(b"MThd"): + raise RuntimeError("MIDI export failed; no YuE2 request was written") + if lyrics is not None: + request = build_yue2_request(abc, lyrics, args) + (args.output_dir / "yue2_request.json").write_text( + json.dumps(request, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + manifest = { + "model": args.model, + "revision": revision, + "base_model_revision": getattr(model.config, "base_model_revision", None), + "audio": str(args.audio.resolve()), + "melody_only": args.melody_only, + "max_seconds": args.max_seconds, + "device": str(device), + "inference_dtype": args.dtype, + "torch_version": torch.__version__, + "backend": "upstream SheetSage2 transcribe", + } + (args.output_dir / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + return args.output_dir + + +def main(argv: list[str] | None = None) -> None: + parser = create_parser() + args = parser.parse_args(argv) + try: + output_dir = transcribe(args) + except (ValueError, RuntimeError, OSError, ImportError) as exc: + parser.exit(1, f"SheetSage2: {exc}\nSee recipes/m-a-p/SheetSage2-H200.md for the isolated environment setup.\n") + print(f"Saved score.abc, transcription.mid and annotations to {output_dir.resolve()}") + if args.lyrics_file is not None: + print(f"YuE2 request: {output_dir.resolve() / 'yue2_request.json'}") + + +if __name__ == "__main__": + main()