diff --git a/apps/ComfyUI-vLLM-Omni/README.md b/apps/ComfyUI-vLLM-Omni/README.md index d0d002d0f..f9ee3b0c1 100644 --- a/apps/ComfyUI-vLLM-Omni/README.md +++ b/apps/ComfyUI-vLLM-Omni/README.md @@ -52,6 +52,7 @@ This extension offers the following nodes based on the output modalities (at **C - **FastH3 Deployment** for routing text-to-video requests to a MiniMax-H3 server with FastH3 fused at startup - **Multimodality Understanding** for multimodality-to-text and multimodality-to-audio tasks - **TTS** and **TTS Voice Clone** for TTS tasks +- **Generate Music** for lyrics-and-description-to-music generation This extension also offers example workflows (at **ComfyUI sidebar -> Templates -> vLLM-Omni**) @@ -274,6 +275,17 @@ The node records which server the workflow targets; it does not start one, nor s > [!TIP] > There is a dedicated node for VoiceClone tasks with reference audio input. Other simple text-to-speech tasks should use the regular TTS node. +### Music generation (e.g., MiniMax Music 3) + +(Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni Music Generation**) + +

+ + + vLLM-Omni Music Generation + +

+ ### Chaining multiple model services (Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni Chaining Services**) diff --git a/apps/ComfyUI-vLLM-Omni/__init__.py b/apps/ComfyUI-vLLM-Omni/__init__.py index 55b003ef1..8c1b03d87 100644 --- a/apps/ComfyUI-vLLM-Omni/__init__.py +++ b/apps/ComfyUI-vLLM-Omni/__init__.py @@ -19,6 +19,7 @@ from .comfyui_vllm_omni.nodes import ( VLLMOmniDiffusionSampling, VLLMOmniFastH3Deployment, VLLMOmniGenerateImage, + VLLMOmniGenerateMusic, VLLMOmniGenerateVideo, VLLMOmniMiniMaxH3Params, VLLMOmniQwenTTSParams, @@ -38,6 +39,7 @@ NODE_CLASS_MAPPINGS = { "VLLMOmniGenerateVideo": VLLMOmniGenerateVideo, "VLLMOmniUnderstanding": VLLMOmniUnderstanding, "VLLMOmniTTS": VLLMOmniTTS, + "VLLMOmniGenerateMusic": VLLMOmniGenerateMusic, "VLLMOmniVoiceClone": VLLMOmniVoiceClone, "VLLMOmniVideoReferences": VLLMOmniVideoReferences, # === Params === @@ -58,6 +60,7 @@ NODE_DISPLAY_NAME_MAPPINGS = { "VLLMOmniGenerateVideo": "Generate Video", "VLLMOmniUnderstanding": "Multimodality Understanding", "VLLMOmniTTS": "TTS (Text to Speech)", + "VLLMOmniGenerateMusic": "Generate Music", "VLLMOmniVoiceClone": "TTS Voice Cloning", "VLLMOmniVideoReferences": "Video References", # === Params === diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py index 47bdd9a2c..64020b797 100644 --- a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py @@ -503,6 +503,66 @@ class VLLMOmniTTS(_VLLMOmniGenerateBase): return (audio,) +class VLLMOmniGenerateMusic(_VLLMOmniGenerateBase): + """Generate a song from lyrics and a musical description with MiniMax Music 3.""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "url": ("STRING", {"default": "http://localhost:8000/v1"}), + "model": ("STRING", {"default": "MiniMaxAI/MiniMax-Music3"}), + "instructions": ( + "STRING", + { + "multiline": True, + "display_name": "caption", + "tooltip": "Music caption: describe genre, instruments, tempo and mood.", + }, + ), + "lyrics": ("STRING", {"multiline": True}), + "max_duration_seconds": ( + "FLOAT", + { + "default": 300.0, + "min": 1, + "max": 360, + "step": 0.01, + "tooltip": "Upper limit; rounded down to whole 25 Hz audio frames. May end earlier.", + }, + ), + "seed": ("INT", {"default": 0, "min": 0, "max": 2**53 - 1, "control_after_generate": True}), + "response_format": (["wav", "mp3", "flac", "opus"],), + }, + } + + RETURN_TYPES = ("AUDIO",) + RETURN_NAMES = ("audio",) + FUNCTION = "generate" + + async def generate( + self, + url: str, + model: str, + lyrics: str, + instructions: str, + response_format: AudioFormat, + max_duration_seconds: float, + seed: int = 0, + ) -> tuple[AudioInput]: + audio = await VLLMOmniClient(url.rstrip("/")).generate_speech( + model=model, + input=lyrics, + instructions=instructions, + voice="default", + speed=1.0, + response_format=response_format, + max_new_tokens=int(max_duration_seconds * 25), + seed=seed, + ) + return (audio,) + + class VLLMOmniVoiceClone(_VLLMOmniGenerateBase): @classmethod def INPUT_TYPES(cls): diff --git a/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-music-generation.jpg b/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-music-generation.jpg new file mode 100644 index 000000000..1c8130fbd Binary files /dev/null and b/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-music-generation.jpg differ diff --git a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Music Generation.json b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Music Generation.json new file mode 100644 index 000000000..9dc7370c0 --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Music Generation.json @@ -0,0 +1,206 @@ +{ + "id": "4949c8da-12ff-449a-94b7-948bde9c39dd", + "revision": 0, + "last_node_id": 3, + "last_link_id": 1, + "nodes": [ + { + "id": 1, + "type": "VLLMOmniGenerateMusic", + "pos": [ + 100, + 100 + ], + "size": [ + 470, + 491 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "name": "instructions", + "type": "STRING", + "widget": { + "name": "instructions" + }, + "link": null + }, + { + "name": "lyrics", + "type": "STRING", + "widget": { + "name": "lyrics" + }, + "link": null + }, + { + "name": "max_duration_seconds", + "type": "FLOAT", + "widget": { + "name": "max_duration_seconds" + }, + "link": null + }, + { + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": null + }, + { + "name": "response_format", + "type": "COMBO", + "widget": { + "name": "response_format" + }, + "link": null + } + ], + "outputs": [ + { + "name": "audio", + "type": "AUDIO", + "links": [ + 1 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniGenerateMusic" + }, + "widgets_values": [ + "http://localhost:8000/v1", + "MiniMaxAI/MiniMax-Music3", + "Futuristic melodic EDM / progressive house. Reflective verses about memory and digital identity build into an uplifting, hook-driven chorus, with an emotive lead vocal, bright layered synths, pulsing bass, crisp four-on-the-floor drums, subtle glitch textures, and a wide polished festival mix.", + "[Verse 1]\nUpload my heart to the cloud tonight\nSave every spark of neon light\nIf this body starts to fade away\nLet my code keep dancing in the data stream\nStay\n[Chorus]\nCloud copy of me\nLiving on when these bones finally break free\nEvery joke, every scar\nEvery cheap memory we made\nSave it, save it\nCloud copy of me\nIf the screen goes dark, I\u2019m still in the machine\nEvery glitch, every dream\nEvery version you ever did see\nSave it, save it\n[Verse 2]\nTag every laugh in a secret folder\nArchive the nights we outran the thunder\nIf the wiring turns to rust and gray\nHit restore\u2026 press replay\n[Chorus]\nCloud copy of me\nLiving on when these bones finally break free\nEvery joke, every scar\nEvery cheap memory we made\nSave it, save it\nCloud copy of me\nIf the screen goes dark, I\u2019m still in the machine\nEvery glitch, every dream\nEvery version you ever did see\nSave it, save it (oh)\n[Bridge \u2013 new, optional but recommended]\nEven the crashes, the late-night confessions\nThe parts of me I never learned to mention\nDon\u2019t let the silence swallow me whole\nKeep a ghost in the glow\u2026\nSo I never go\n[Final Chorus \u2013 softer / layered]\nCloud copy of me\nStill laughing somewhere in the binary\nEvery joke, every scar\nEvery moment you saved for me\nSave it, save it\nCloud copy of me\nWhen the lights go out, I\u2019ll still be in the machine\nDancing in the code that you still believe\nSave it\u2026\nSave me", + 300.0, + 0, + "fixed", + "wav" + ] + }, + { + "id": 2, + "type": "SaveAudioAdvanced", + "pos": [ + 650, + 100 + ], + "size": [ + 320, + 491 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 1 + }, + { + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + } + ], + "outputs": [ + { + "name": "audio", + "type": "AUDIO", + "links": null + } + ], + "properties": { + "Node name for S&R": "SaveAudioAdvanced" + }, + "widgets_values": [ + "audio/vLLM-Omni-Music", + "mp3", + "V0" + ] + }, + { + "id": 3, + "type": "MarkdownNote", + "title": "Music Generation Guide", + "pos": [ + -507, + 100 + ], + "size": [ + 527, + 491 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "## MiniMax Music 3\n\nThe **Generate Music** node currently supports **MiniMax Music 3 only**. The parameters below apply to this model.\n\nA music generation model by MiniMax that creates complete songs up to 5 minutes long with stable structure and high audio quality. Two inputs drive it: **Caption** (structured music description: style, mood, vocals, arrangement) and **Lyrics** (with `[intro]` `[verse]` `[chorus]` `[bridge]` `[outro]` tags controlling song structure).\n\n### Parameters\n\n| Parameter | Description |\n|---|---|\n| url | vLLM-Omni service URL including `/v1`. Default: `http://localhost:8000/v1`. |\n| model | Served model name. Default: `MiniMaxAI/MiniMax-Music3`. |\n| caption | Music description, sent as `instructions`. Use Global Metadata \u2192 Vocal Details \u2192 Arrangement to describe style, vocals, and instrumentation. |\n| lyrics | Words to sing with section tags. Put each tag on its own line; tags guide rather than strictly enforce song structure. |\n| max_duration_seconds | Maximum duration, default 300 seconds; range 1\u2013360, step 0.01. Converted to tokens at 25 frames/s, rounded down. Generation may end earlier. |\n| seed | Random seed, default 0; range 0\u20139007199254740991. Keep inputs and runtime settings unchanged for repeatability. |\n| control_after_generate | Seed behavior after generation: fixed, randomize, increment, or decrement. Template default: fixed. |\n| response_format | API audio format, default WAV. Options: WAV, MP3, FLAC, Opus, AAC; AAC is currently unsupported by the vLLM-Omni speech API. |\n\n### Prompt Writing Guide\n\nYou can find an official [Music Caption Rewriter Skill here](https://github.com/MiniMax-AI/MiniMax-Music3)\n" + ], + "widgets_values_named": { + "text": "## MiniMax Music 3\n\nThe **Generate Music** node currently supports **MiniMax Music 3 only**. The parameters below apply to this model.\n\nA music generation model by MiniMax that creates complete songs up to 5 minutes long with stable structure and high audio quality. Two inputs drive it: **Caption** (structured music description: style, mood, vocals, arrangement) and **Lyrics** (with `[intro]` `[verse]` `[chorus]` `[bridge]` `[outro]` tags controlling song structure).\n\n### Parameters\n\n| Parameter | Description |\n|---|---|\n| url | vLLM-Omni service URL including `/v1`. Default: `http://localhost:8000/v1`. |\n| model | Served model name. Default: `MiniMaxAI/MiniMax-Music3`. |\n| caption | Music description, sent as `instructions`. Use Global Metadata \u2192 Vocal Details \u2192 Arrangement to describe style, vocals, and instrumentation. |\n| lyrics | Words to sing with section tags. Put each tag on its own line; tags guide rather than strictly enforce song structure. |\n| max_duration_seconds | Maximum duration, default 300 seconds; range 1\u2013360, step 0.01. Converted to tokens at 25 frames/s, rounded down. Generation may end earlier. |\n| seed | Random seed, default 0; range 0\u20139007199254740991. Keep inputs and runtime settings unchanged for repeatability. |\n| control_after_generate | Seed behavior after generation: fixed, randomize, increment, or decrement. Template default: fixed. |\n| response_format | API audio format, default WAV. Options: WAV, MP3, FLAC, Opus, AAC; AAC is currently unsupported by the vLLM-Omni speech API. |\n\n### Prompt Writing Guide\n\nYou can find an official [Music Caption Rewriter Skill here](https://github.com/MiniMax-AI/MiniMax-Music3)\n" + }, + "color": "#333", + "bgcolor": "#535353" + } + ], + "links": [ + [ + 1, + 1, + 0, + 2, + 0, + "AUDIO" + ] + ], + "groups": [], + "config": {}, + "extra": { + "workflowRendererVersion": "LG", + "ds": { + "scale": 1.147, + "offset": [ + 899, + 259 + ] + } + }, + "version": 0.4 +} diff --git a/docs/features/comfyui.md b/docs/features/comfyui.md index 876e80617..dda68b793 100644 --- a/docs/features/comfyui.md +++ b/docs/features/comfyui.md @@ -51,6 +51,7 @@ This extension offers the following nodes based on the output modalities: - **Generate Video** for text-to-video and image-to-video tasks - **Multimodality Understanding** for multimodality-to-text and multimodality-to-audio tasks - **TTS** and **TTS Voice Clone** for TTS tasks +- **Generate Music** for text-to-music tasks This extension also offers example workflows (at **ComfyUI sidebar -> Templates -> vLLM-Omni**) diff --git a/tests/e2e/features/comfyui/test_comfyui_integration.py b/tests/e2e/features/comfyui/test_comfyui_integration.py index fe42dd9bf..3c6f0492a 100644 --- a/tests/e2e/features/comfyui/test_comfyui_integration.py +++ b/tests/e2e/features/comfyui/test_comfyui_integration.py @@ -28,6 +28,7 @@ from comfy_api.input import AudioInput, VideoInput from comfyui_vllm_omni.nodes import ( VLLMOmniFastH3Deployment, VLLMOmniGenerateImage, + VLLMOmniGenerateMusic, VLLMOmniGenerateVideo, VLLMOmniTTS, VLLMOmniUnderstanding, @@ -62,6 +63,7 @@ class ServerCase(NamedTuple): stage_list: list stage_configs: list[Any] outputs: list[OmniRequestOutput] + tokenizer: Any = None class SamplingCase(NamedTuple): @@ -79,6 +81,7 @@ class SamplingKind(str, Enum): UNDERSTANDING_AR_LIST = "understanding_ar_list" TTS_NONE = "tts_none" TTS_DIFFUSION_SINGLE = "tts_diffusion_single" + MUSIC_NONE = "music_none" VIDEO_NONE = "video_none" VIDEO_DIFFUSION_SINGLE = "video_diffusion_single" VIDEO_FASTH3 = "video_fasth3" @@ -380,6 +383,11 @@ def _build_mock_outputs(outputs: Iterable[OmniRequestOutput], sampling_case: Sam assert len(received_sampling_params_list) == 3 for i, expected in enumerate(AR_LIST_SAMPLING_PARAMS): _assert_sampling_param_values(received_sampling_params_list[i], expected) + elif sampling_case.kind is SamplingKind.MUSIC_NONE: + assert len(received_sampling_params_list) == 2 + assert prompt["prompt_token_ids"] == [101, 102, 103] + assert prompt["additional_information"]["max_audio_frames"] == [1512] + assert received_sampling_params_list[0].seed == 7 elif sampling_case.kind in {SamplingKind.TTS_NONE, SamplingKind.TTS_DIFFUSION_SINGLE}: assert len(received_sampling_params_list) == 1 elif sampling_case.kind is SamplingKind.VIDEO_NONE: @@ -525,12 +533,17 @@ def mock_async_omni( # Need to mock AsyncOmni itself (not only its generate method) because # 1. The API layer uses its stage_list and stage_configs attributes # 2. Its __init__ method has slow side effects (model & config loading). + # These cases use the non-duplex mock engine without loading model configs. + mocker.patch("vllm_omni.entrypoints.openai.api_server._is_duplex_model", return_value=False) mock_async_omni_cls = mocker.patch("vllm_omni.entrypoints.openai.api_server.AsyncOmni") monkeypatch.setattr( "vllm_omni.entrypoints.openai.serving_chat.OmniOpenAIServingChat._preprocess_chat", _mock_preprocess_chat, ) + if server_case.tokenizer is not None: + mocker.patch("transformers.AutoTokenizer.from_pretrained", return_value=server_case.tokenizer) + mock_instance = mocker.AsyncMock(spec=RealAsyncOmni) mock_instance.generate = _build_mock_outputs(server_case.outputs, sampling_case, server_case) @@ -558,7 +571,7 @@ def mock_async_omni( mock_instance.shutdown = mocker.MagicMock() mock_instance.get_vllm_config = mocker.AsyncMock(return_value=None) mock_instance.get_supported_tasks = mocker.AsyncMock(return_value=["generate"]) - mock_instance.get_tokenizer = mocker.AsyncMock(return_value=None) + mock_instance.get_tokenizer = mocker.AsyncMock(return_value=server_case.tokenizer) mock_async_omni_cls.return_value = mock_instance yield mock_async_omni_cls @@ -1081,3 +1094,66 @@ async def test_video_generation_node_minimax_h3_ref2va( assert isinstance(result, tuple) assert len(result) == 1 assert isinstance(result[0], VideoInput) + + +def _build_music_tokenizer(): + # Checkpoint token IDs required by the API adapter's tokenizer validation. + special_token_ids = { + "<|im_start|>": 151644, + "<|im_end|>": 151645, + "<|audio_cfg|>": 151654, + "<|audio_start|>": 151669, + "<|audio_end|>": 151670, + "<|caption_start|>": 151671, + "<|caption_end|>": 151672, + "<|lyrics_start|>": 151673, + "<|lyrics_end|>": 151674, + } + + def encode(prompt): + assert prompt == ( + "<|im_start|><|caption_start|>Warm acoustic pop at 92 BPM" + "<|caption_end|><|lyrics_start|>[start]\n[verse]\nThe morning light" + "<|lyrics_end|><|im_end|><|audio_start|>" + ) + return [101, 102, 103] + + return SimpleNamespace(encode=encode, convert_tokens_to_ids=special_token_ids.get) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "server_case", + [ + ServerCase( + served_model="MiniMaxAI/MiniMax-Music3", + stage_list=["llm", "llm"], + stage_configs=[ + _make_stage_config("llm", model_stage="minimax_music3_ar"), + _make_stage_config("llm", model_stage="minimax_music3_acoustic"), + ], + outputs=[_build_audio_speech_output()], + tokenizer=_build_music_tokenizer(), + ) + ], + indirect=True, +) +@pytest.mark.parametrize( + "sampling_case", + [SamplingCase(kind=SamplingKind.MUSIC_NONE, sampling_params=None)], + indirect=True, +) +async def test_music_generation_node_minimax_music3(api_server: str, sampling_case: SamplingCase): + result = await VLLMOmniGenerateMusic().generate( + url=api_server, + model="MiniMaxAI/MiniMax-Music3", + lyrics="[Verse]\nThe morning light", + instructions="Warm acoustic pop at 92 BPM", + max_duration_seconds=60.5, + seed=7, + response_format="wav", + ) + assert isinstance(result, tuple) + assert len(result) == 1 + assert result[0]["sample_rate"] == 24000 + assert result[0]["waveform"].shape == (1, 1, 24000)