[CI] Remove AudioX and MammothModa2 support (#6353)
Signed-off-by: wangyu <410167048@qq.com>
This commit is contained in:
1 parent
39c16d75a9
commit
1e2a6446f7
59 files changed
+42
-7029
No files matched your search
@@ -17,7 +17,7 @@ Shared plumbing (corpus/WAV IO, tokenizer loading, the CFG cond/uncond
|
||||
pair contract) lives in `common.py`; each task script keeps only its own
|
||||
prompt recipe and flow. Note `text_to_audio.py` here drives the Audex
|
||||
two-stage AR pipeline — for diffusion text-to-audio models (Stable Audio
|
||||
Open, AudioX) use the generic
|
||||
Open) use the generic
|
||||
`examples/offline_inference/text_to_audio/text_to_audio.py` instead.
|
||||
|
||||
## text_to_speech.py — text → speech
|
||||
|
||||
@@ -7,7 +7,7 @@ thinker generates interleaved 4-codebook <audiocodec_N> RVQ tokens under an
|
||||
RVQ phase mask, and the external XCodec1 checkpoint decodes them to a 16 kHz
|
||||
mono WAV.
|
||||
|
||||
(For DIFFUSION text-to-audio models — Stable Audio Open, AudioX — use
|
||||
(For DIFFUSION text-to-audio models — Stable Audio Open — use
|
||||
``examples/offline_inference/text_to_audio/text_to_audio.py`` instead; Audex
|
||||
TTA is a two-stage autoregressive pipeline with a different interface.)
|
||||
|
||||
|
||||
@@ -1,117 +0,0 @@
|
||||
"""
|
||||
Offline inference example: MammothModa2 image summarization (single AR stage).
|
||||
|
||||
Example:
|
||||
uv run python examples/offline_inference/mammothmodal2_preview/run_mammothmoda2_image_summarize.py \
|
||||
--model path/to/MammothModa2-Preview \
|
||||
--deploy-config vllm_omni/deploy/mammoth_moda2_ar.yaml \
|
||||
--image /path/to/input.jpg \
|
||||
--question "Please summarize the content of this image."
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
|
||||
from PIL import Image
|
||||
from vllm import SamplingParams
|
||||
from vllm.multimodal.image import convert_image_mode
|
||||
|
||||
from vllm_omni import Omni
|
||||
|
||||
DEFAULT_SYSTEM = "You are a helpful assistant."
|
||||
DEFAULT_QUESTION = "Please summarize the content of this image."
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="MammothModa2 image summarization (offline, AR only).")
|
||||
parser.add_argument("--model", type=str, required=True, help="Path to model directory or model id.")
|
||||
parser.add_argument("--deploy-config", type=str, required=True, help="Path to deploy config yaml.")
|
||||
parser.add_argument("--image", type=str, required=True, help="Path to input image.")
|
||||
parser.add_argument("--question", type=str, default=DEFAULT_QUESTION, help="Question/instruction for the model.")
|
||||
parser.add_argument("--system", type=str, default=DEFAULT_SYSTEM, help="System prompt.")
|
||||
parser.add_argument(
|
||||
"--max-tokens",
|
||||
type=int,
|
||||
default=512,
|
||||
help="Max new tokens to generate.",
|
||||
)
|
||||
parser.add_argument("--temperature", type=float, default=0.2)
|
||||
parser.add_argument("--top-p", type=float, default=0.9)
|
||||
parser.add_argument("--seed", type=int, default=42)
|
||||
parser.add_argument("--trust-remote-code", action="store_true")
|
||||
parser.add_argument(
|
||||
"--enable-diffusion-pipeline-profiler",
|
||||
action="store_true",
|
||||
help="Enable diffusion pipeline profiler to display stage durations.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def build_prompt(system: str, question: str) -> str:
|
||||
return (
|
||||
f"<|im_start|>system\n{system}<|im_end|>\n"
|
||||
"<|im_start|>user\n"
|
||||
"<|vision_start|><|image_pad|><|vision_end|>"
|
||||
f"{question}<|im_end|>\n"
|
||||
"<|im_start|>assistant\n"
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
|
||||
if not os.path.exists(args.image):
|
||||
raise FileNotFoundError(f"Image file not found: {args.image}")
|
||||
|
||||
pil_image = Image.open(args.image)
|
||||
image_data = convert_image_mode(pil_image, "RGB")
|
||||
prompt = build_prompt(args.system, args.question)
|
||||
|
||||
omni = Omni(
|
||||
model=args.model,
|
||||
deploy_config=args.deploy_config,
|
||||
trust_remote_code=args.trust_remote_code,
|
||||
enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler,
|
||||
)
|
||||
try:
|
||||
sp = SamplingParams(
|
||||
temperature=float(args.temperature),
|
||||
top_p=float(args.top_p),
|
||||
top_k=-1,
|
||||
max_tokens=int(args.max_tokens),
|
||||
seed=int(args.seed),
|
||||
detokenize=True,
|
||||
)
|
||||
# NOTE: omni.generate() returns a Generator[OmniRequestOutput, None, None].
|
||||
# Consume it inside the try block so the worker isn't closed early.
|
||||
outputs = list(
|
||||
omni.generate(
|
||||
[
|
||||
{
|
||||
"prompt": prompt,
|
||||
"multi_modal_data": {"image": image_data},
|
||||
"additional_information": {"omni_task": ["chat"]},
|
||||
}
|
||||
],
|
||||
[sp],
|
||||
)
|
||||
)
|
||||
finally:
|
||||
omni.close()
|
||||
|
||||
lines: list[str] = []
|
||||
for stage_outputs in outputs:
|
||||
ro = stage_outputs
|
||||
text = ro.outputs[0].text if getattr(ro, "outputs", None) else str(ro)
|
||||
lines.append(f"request_id: {getattr(ro, 'request_id', 'unknown')}\n")
|
||||
lines.append("answer:\n")
|
||||
lines.append(text.strip() + "\n")
|
||||
lines.append("\n")
|
||||
|
||||
print("\n".join(lines))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,11 +1,10 @@
|
||||
# Text-To-Audio
|
||||
|
||||
A unified script for text/video-to-audio generation. Supported models:
|
||||
A unified script for text-to-audio generation. Supported models:
|
||||
|
||||
| Model | Tasks | Notes |
|
||||
|-------|-------|-------|
|
||||
| `stabilityai/stable-audio-open-1.0` | text-to-audio | gated; uses `--audio-length` |
|
||||
| `zhangj1an/AudioX` | `t2a` / `t2m` / `v2a` / `v2m` / `tv2a` / `tv2m` | pass `--task`; video tasks need `--video` |
|
||||
|
||||
The `stabilityai/stable-audio-open-1.0` pipeline generates audio from text prompts.
|
||||
|
||||
@@ -50,42 +49,12 @@ python text_to_audio.py \
|
||||
--output stable_audio_output.wav
|
||||
```
|
||||
|
||||
### AudioX
|
||||
|
||||
AudioX supports six tasks. Sampler and reference knobs (declared in
|
||||
`vllm_omni/model_extras/audiox.py`) are passed via the generic `--extra-body`
|
||||
JSON flag, routed into sampling `extra_args`.
|
||||
|
||||
Text tasks (`t2a` / `t2m`):
|
||||
|
||||
```bash
|
||||
python text_to_audio.py \
|
||||
--model zhangj1an/AudioX --task t2a \
|
||||
--prompt "Fireworks burst twice, followed by a clock ticking." \
|
||||
--num-inference-steps 250 --guidance-scale 6.0 --audio-length 10.0 --seed 42 \
|
||||
--extra-body '{"sigma_min": 0.03, "sigma_max": 1000.0}' \
|
||||
--output t2a.wav
|
||||
```
|
||||
|
||||
Video-conditioned tasks (`v2a` / `v2m` / `tv2a` / `tv2m`) require `--video`:
|
||||
|
||||
```bash
|
||||
python text_to_audio.py \
|
||||
--model zhangj1an/AudioX --task tv2a \
|
||||
--prompt "drum beating sound and human talking" \
|
||||
--video https://zeyuet.github.io/AudioX/static/samples/V2M/1XeBotOFqHA.mp4 \
|
||||
--num-inference-steps 250 --guidance-scale 6.0 --audio-length 10.0 \
|
||||
--output tv2a.wav
|
||||
```
|
||||
|
||||
Key arguments:
|
||||
|
||||
- `--prompt`: text description (string).
|
||||
- `--task`: [AudioX] one of `t2a`/`t2m`/`v2a`/`v2m`/`tv2a`/`tv2m`.
|
||||
- `--video`: [AudioX `v2*`/`tv2*`] video file/URL for conditioning (→ `video_path`).
|
||||
- `--audio-start`: audio start offset in seconds (→ `audio_start_in_s` for Stable Audio, `seconds_start` for AudioX).
|
||||
- `--audio-length`: audio duration in seconds (audio length for Stable Audio, `seconds_total` for AudioX).
|
||||
- `--extra-body`: JSON dict of model-specific knobs (declared in `vllm_omni/model_extras/`), merged into sampling `extra_args`. For AudioX, sampler/reference knobs go here, e.g. `'{"sigma_min": 0.03, "sigma_max": 1000.0, "cfg_rescale": 0.0, "audio_path": "ref.wav"}'`.
|
||||
- `--audio-start`: audio start offset in seconds (→ `audio_start_in_s`).
|
||||
- `--audio-length`: audio duration in seconds (audio length for Stable Audio).
|
||||
- `--extra-body`: JSON dict of model-specific knobs, merged into sampling `extra_args`.
|
||||
- `--negative-prompt`: negative prompt for classifier-free guidance.
|
||||
- `--seed`: integer seed for deterministic generation.
|
||||
- `--guidance-scale`: classifier-free guidance scale.
|
||||
|
||||
@@ -2,15 +2,13 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Example script for text/video-to-audio generation with diffusion audio models.
|
||||
Example script for text-to-audio generation with diffusion audio models.
|
||||
|
||||
This script supports:
|
||||
* Stable Audio Open (`stabilityai/stable-audio-open-1.0`) — text-to-audio.
|
||||
* AudioX (`zhangj1an/AudioX`) — six tasks: t2a / t2m / v2a / v2m / tv2a / tv2m.
|
||||
|
||||
Model-specific generation knobs are declared in `vllm_omni/model_extras` and
|
||||
routed into `sampling_params.extra_args` via `apply_declared_extra_args`, so
|
||||
the same script serves both models without per-model branching in the engine.
|
||||
Model-specific generation knobs can be passed via `--extra-body` and are
|
||||
merged into `sampling_params.extra_args`.
|
||||
|
||||
Usage:
|
||||
# Stable Audio Open
|
||||
@@ -18,15 +16,6 @@ Usage:
|
||||
python text_to_audio.py --prompt "A piano playing a gentle melody" --audio-length 10.0
|
||||
python text_to_audio.py --prompt "Thunder and rain sounds" --negative-prompt "Low quality"
|
||||
python text_to_audio.py --prompt "A soft synth pad" --cache-backend tea_cache
|
||||
|
||||
# AudioX (text-to-audio / text-to-music)
|
||||
python text_to_audio.py --model zhangj1an/AudioX --task t2a \
|
||||
--prompt "Fireworks burst twice, then a clock ticking." \
|
||||
--num-inference-steps 250 --guidance-scale 6.0 --audio-length 10.0
|
||||
# AudioX video-conditioned (v2*/tv2*) — pass a video file/URL
|
||||
python text_to_audio.py --model zhangj1an/AudioX --task tv2a \
|
||||
--prompt "drum beating sound and human talking" --video clip.mp4 \
|
||||
--extra-body '{"sigma_min": 0.03, "sigma_max": 1000.0}'
|
||||
"""
|
||||
|
||||
import argparse
|
||||
@@ -38,16 +27,11 @@ from typing import Any
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.model_extras import get_extra_body_params, get_model_class_name
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
AUDIOX_TASKS = ("t2a", "t2m", "v2a", "v2m", "tv2a", "tv2m")
|
||||
AUDIOX_VIDEO_TASKS = frozenset({"v2a", "v2m", "tv2a", "tv2m"})
|
||||
|
||||
|
||||
def parse_extra_body(value: str) -> dict[str, Any]:
|
||||
"""Parse a JSON object string for --extra-body."""
|
||||
@@ -65,7 +49,7 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
default="stabilityai/stable-audio-open-1.0",
|
||||
help="Audio diffusion model name or local path. Supported: stabilityai/stable-audio-open-1.0, zhangj1an/AudioX.",
|
||||
help="Audio diffusion model name or local path. Supported: stabilityai/stable-audio-open-1.0.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prompt",
|
||||
@@ -96,19 +80,13 @@ def parse_args() -> argparse.Namespace:
|
||||
"--audio-start",
|
||||
type=float,
|
||||
default=0.0,
|
||||
help=(
|
||||
"Audio start offset in seconds. Applies to both models "
|
||||
"(audio_start_in_s for Stable Audio, seconds_start for AudioX)."
|
||||
),
|
||||
help="Audio start offset in seconds (maps to audio_start_in_s).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--audio-length",
|
||||
type=float,
|
||||
default=10.0,
|
||||
help=(
|
||||
"Audio duration in seconds. Maps to audio length for Stable Audio "
|
||||
"(max ~47s for stable-audio-open-1.0) and seconds_total for AudioX."
|
||||
),
|
||||
help=("Audio duration in seconds. Maps to audio length for Stable Audio (max ~47s for stable-audio-open-1.0)."),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-inference-steps",
|
||||
@@ -122,41 +100,20 @@ def parse_args() -> argparse.Namespace:
|
||||
default=1,
|
||||
help="Number of audio waveforms to generate for the given prompt.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--task",
|
||||
type=str,
|
||||
default=None,
|
||||
choices=list(AUDIOX_TASKS),
|
||||
help=(
|
||||
"[AudioX only] Generation task: t2a/t2m/v2a/v2m/tv2a/tv2m. "
|
||||
"v2*/tv2* require --video. Ignored for Stable Audio Open."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--video",
|
||||
type=str,
|
||||
default=None,
|
||||
help="[AudioX v2*/tv2*] Video file path or URL used for conditioning.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--extra-body",
|
||||
type=parse_extra_body,
|
||||
default=None,
|
||||
help=(
|
||||
"Model-specific generation params as a JSON object, e.g. "
|
||||
'\'{"sigma_min": 0.03, "sigma_max": 1000.0}\'. Each key is filtered '
|
||||
"against the model's declared extra_body_params (see vllm_omni/model_extras), "
|
||||
"so unknown keys are silently dropped. Values here override the equivalent flags."
|
||||
"Model-specific generation params as a JSON object, merged into "
|
||||
"sampling extra_args. Values here override the equivalent flags."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=str,
|
||||
default=None,
|
||||
help=(
|
||||
"Path to save the generated audio (WAV format). Defaults to "
|
||||
"audiox_output.wav for AudioX and stable_audio_output.wav otherwise."
|
||||
),
|
||||
help=("Path to save the generated audio (WAV format). Defaults to stable_audio_output.wav."),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--sample-rate",
|
||||
@@ -275,8 +232,6 @@ def save_audio(audio_data: np.ndarray, output_path: str, sample_rate: int = 4410
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
model_name = str(args.model).lower() if args.model else ""
|
||||
is_audiox = "audiox" in model_name
|
||||
generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
|
||||
cache_config = None
|
||||
if args.cache_backend == "tea_cache":
|
||||
@@ -285,11 +240,9 @@ def main():
|
||||
}
|
||||
|
||||
print(f"\n{'=' * 60}")
|
||||
print("Text/Video-to-Audio Generation")
|
||||
print("Text-to-Audio Generation")
|
||||
print(f"{'=' * 60}")
|
||||
print(f" Model: {args.model}")
|
||||
if args.task is not None:
|
||||
print(f" Task: {args.task}")
|
||||
print(f" Prompt: {args.prompt}")
|
||||
print(f" Negative prompt: {args.negative_prompt}")
|
||||
print(f" Duration: {args.audio_length}s")
|
||||
@@ -305,8 +258,6 @@ def main():
|
||||
print(f" Seed: {args.seed}")
|
||||
print(f"{'=' * 60}\n")
|
||||
|
||||
# Initialize Omni. AudioX requires an explicit pipeline class; Stable Audio
|
||||
# is resolved from its config.
|
||||
omni_kwargs: dict[str, Any] = dict(
|
||||
model=args.model,
|
||||
use_hsdp=args.use_hsdp,
|
||||
@@ -318,11 +269,7 @@ def main():
|
||||
enable_cpu_offload=args.enable_cpu_offload,
|
||||
enable_layerwise_offload=args.enable_layerwise_offload,
|
||||
)
|
||||
if is_audiox:
|
||||
omni_kwargs["model_class_name"] = "AudioXPipeline"
|
||||
omni = Omni(**omni_kwargs)
|
||||
model_class_name = get_model_class_name(omni)
|
||||
declared_extra_body_params = get_extra_body_params(model_class_name)
|
||||
|
||||
diffusion_params = OmniDiffusionSamplingParams(
|
||||
generator=generator,
|
||||
@@ -332,38 +279,19 @@ def main():
|
||||
seed=args.seed,
|
||||
)
|
||||
|
||||
# Negative prompt is opt-in for both models (off by default). Including a
|
||||
# Negative prompt is opt-in (off by default). Including a
|
||||
# non-empty negative prompt enables the separate CFG conditioning branch.
|
||||
prompt: dict[str, Any] = {"prompt": args.prompt}
|
||||
if args.negative_prompt:
|
||||
prompt["negative_prompt"] = args.negative_prompt
|
||||
|
||||
if model_class_name == "AudioXPipeline":
|
||||
if args.task is None:
|
||||
raise ValueError("AudioX requires --task (one of t2a/t2m/v2a/v2m/tv2a/tv2m).")
|
||||
if args.task in AUDIOX_VIDEO_TASKS and not args.video:
|
||||
raise ValueError(f"AudioX task {args.task!r} requires --video.")
|
||||
# AudioX core knobs are surfaced as shared flags; sampler/reference knobs
|
||||
# (sigma_min/sigma_max/cfg_rescale/audio_path) arrive via --extra-body and
|
||||
# are filtered against the declared extra_body contract.
|
||||
user_extra: dict[str, Any] = {
|
||||
"audiox_task": args.task,
|
||||
"seconds_start": args.audio_start,
|
||||
"seconds_total": args.audio_length,
|
||||
"video_path": args.video,
|
||||
}
|
||||
if args.extra_body:
|
||||
user_extra.update(args.extra_body)
|
||||
apply_declared_extra_args(diffusion_params, declared_extra_body_params, user_extra)
|
||||
else:
|
||||
# Stable Audio Open path (backward-compatible).
|
||||
audio_end_in_s = args.audio_start + args.audio_length
|
||||
diffusion_params.extra_args = {
|
||||
"audio_start_in_s": args.audio_start,
|
||||
"audio_end_in_s": audio_end_in_s,
|
||||
}
|
||||
if args.extra_body:
|
||||
diffusion_params.extra_args.update(args.extra_body)
|
||||
audio_end_in_s = args.audio_start + args.audio_length
|
||||
diffusion_params.extra_args = {
|
||||
"audio_start_in_s": args.audio_start,
|
||||
"audio_end_in_s": audio_end_in_s,
|
||||
}
|
||||
if args.extra_body:
|
||||
diffusion_params.extra_args.update(args.extra_body)
|
||||
|
||||
generation_start = time.perf_counter()
|
||||
|
||||
@@ -374,7 +302,7 @@ def main():
|
||||
|
||||
print(f"Total generation time: {generation_time:.2f} seconds")
|
||||
|
||||
output = args.output or ("audiox_output.wav" if model_class_name == "AudioXPipeline" else "stable_audio_output.wav")
|
||||
output = args.output or "stable_audio_output.wav"
|
||||
output_path = Path(output)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
suffix = output_path.suffix or ".wav"
|
||||
|
||||
@@ -599,17 +599,6 @@ def main():
|
||||
if args.seed is not None and hasattr(params, "seed"):
|
||||
params.seed = args.seed
|
||||
|
||||
# MammothModa2's AR stage emits one visual token per grid cell,
|
||||
# one EOL token per row, and one final look-ahead token whose hidden
|
||||
# state is unavailable. Size the first stage from the prompt metadata
|
||||
# instead of SamplingParams' default of 16 tokens.
|
||||
prompt_info = prompt_dict.get("additional_information", {})
|
||||
if idx == 0 and prompt_info.get("omni_task") == ["t2i"]:
|
||||
ar_width = int(prompt_info.get("ar_width", [0])[0])
|
||||
ar_height = int(prompt_info.get("ar_height", [0])[0])
|
||||
if ar_width > 0 and ar_height > 0:
|
||||
params.max_tokens = ar_height * (ar_width + 1) + 1
|
||||
|
||||
if not diffusion_replaced and len(sampling_params_list) == 1:
|
||||
sampling_params_list = [diffusion_params]
|
||||
|
||||
|
||||
@@ -10,12 +10,10 @@ Generate text from text or image inputs with vLLM-Omni's shared offline entrypoi
|
||||
| --- | --- | --- | --- |
|
||||
| `ByteDance-Seed/BAGEL-7B-MoT` | Yes | Yes | `vllm_omni/deploy/bagel.yaml` |
|
||||
| `tencent/HunyuanImage-3.0-Instruct` | Yes | Yes | `vllm_omni/deploy/hunyuan_image3_ar.yaml` |
|
||||
| `bytedance-research/MammothModa2-Preview` | Yes | Yes | `vllm_omni/deploy/mammoth_moda2_ar.yaml` |
|
||||
| `bytedance-research/MammothModa2-Dev` | Yes | Yes | `vllm_omni/deploy/mammoth_moda2_ar.yaml` |
|
||||
|
||||
The script recognizes these model families from `config.json`, applies the
|
||||
model-specific prompt format, and selects an AR-only deploy for HunyuanImage-3
|
||||
and MammothModa2. BAGEL uses its registered default deploy.
|
||||
model-specific prompt format, and selects an AR-only deploy for HunyuanImage-3.
|
||||
BAGEL uses its registered default deploy.
|
||||
|
||||
## Text-To-Text
|
||||
|
||||
@@ -35,22 +33,6 @@ python examples/offline_inference/x_to_text/x_to_text.py \
|
||||
--prompt "Explain multimodal inference in three concise sentences."
|
||||
```
|
||||
|
||||
### MammothModa2
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/x_to_text/x_to_text.py \
|
||||
--model bytedance-research/MammothModa2-Preview \
|
||||
--prompt "Explain multimodal inference in three concise sentences."
|
||||
```
|
||||
|
||||
The same command supports the Dev checkpoint:
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/x_to_text/x_to_text.py \
|
||||
--model bytedance-research/MammothModa2-Dev \
|
||||
--prompt "Explain multimodal inference in three concise sentences."
|
||||
```
|
||||
|
||||
## Image-To-Text
|
||||
|
||||
Pass one image with `--image`. The default question can be replaced with any
|
||||
@@ -72,16 +54,6 @@ python examples/offline_inference/x_to_text/x_to_text.py \
|
||||
--prompt "Describe this image in detail."
|
||||
```
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/x_to_text/x_to_text.py \
|
||||
--model bytedance-research/MammothModa2-Preview \
|
||||
--image image.png \
|
||||
--prompt "Describe this image in detail."
|
||||
```
|
||||
|
||||
Use `bytedance-research/MammothModa2-Dev` in the same command to run I2T with
|
||||
the Dev checkpoint.
|
||||
|
||||
## Key Arguments
|
||||
|
||||
| Argument | Default | Description |
|
||||
@@ -102,4 +74,4 @@ the Dev checkpoint.
|
||||
|
||||
The committed HunyuanImage-3 AR-only default uses four GPUs with TP=4. To use a
|
||||
different topology, pass a custom AR-only YAML through `--deploy-config`.
|
||||
BAGEL and MammothModa2 use their repository deploy defaults unless overridden.
|
||||
BAGEL uses its repository deploy default unless overridden.
|
||||
@@ -15,7 +15,6 @@ from vllm_omni.model_extras import build_x_to_text_prompt, get_x_to_text_model_f
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
_HUNYUAN_AR_DEPLOY_CONFIG = _REPO_ROOT / "vllm_omni" / "deploy" / "hunyuan_image3_ar.yaml"
|
||||
_MAMMOTH_AR_DEPLOY_CONFIG = _REPO_ROOT / "vllm_omni" / "deploy" / "mammoth_moda2_ar.yaml"
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
@@ -62,7 +61,7 @@ def main() -> None:
|
||||
omni_kwargs: dict[str, Any] = {
|
||||
"model": args.model,
|
||||
"mode": "image-to-text" if image is not None else "text-to-text",
|
||||
"trust_remote_code": args.trust_remote_code or family in {"hunyuan_image3", "mammoth_moda2"},
|
||||
"trust_remote_code": args.trust_remote_code or family == "hunyuan_image3",
|
||||
"log_stats": args.log_stats,
|
||||
}
|
||||
if args.enforce_eager is not None:
|
||||
@@ -71,8 +70,6 @@ def main() -> None:
|
||||
omni_kwargs["deploy_config"] = args.deploy_config
|
||||
elif family == "hunyuan_image3":
|
||||
omni_kwargs["deploy_config"] = str(_HUNYUAN_AR_DEPLOY_CONFIG)
|
||||
elif family == "mammoth_moda2":
|
||||
omni_kwargs["deploy_config"] = str(_MAMMOTH_AR_DEPLOY_CONFIG)
|
||||
|
||||
omni = Omni(**omni_kwargs)
|
||||
try:
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
# AudioX online serving
|
||||
|
||||
Launches the `AudioXPipeline` behind vLLM-Omni's OpenAI-compatible chat endpoint and provides a
|
||||
minimal Python client that covers all six tasks (`t2a`, `t2m`, `v2a`, `v2m`, `tv2a`, `tv2m`).
|
||||
|
||||
## Start the server
|
||||
|
||||
```bash
|
||||
cd examples/online_serving/audiox
|
||||
bash run_server.sh # defaults: MODEL=zhangj1an/AudioX, PORT=8099
|
||||
```
|
||||
|
||||
Environment overrides: `MODEL`, `PORT`, `DIFFUSION_ATTENTION_BACKEND`.
|
||||
|
||||
## Call from Python
|
||||
|
||||
```bash
|
||||
# text-to-audio
|
||||
python openai_chat_client.py --task t2a \
|
||||
--prompt "Fireworks burst twice, followed by a period of silence before a clock begins ticking." \
|
||||
--output t2a.wav
|
||||
|
||||
# text-to-music
|
||||
python openai_chat_client.py --task t2m \
|
||||
--prompt "Uplifting ukulele tune for a travel vlog" \
|
||||
--output t2m.wav
|
||||
|
||||
# video-to-audio (no text)
|
||||
python openai_chat_client.py --task v2a --video path/to/clip.mp4 --output v2a.wav
|
||||
|
||||
# text+video-to-audio
|
||||
python openai_chat_client.py --task tv2a \
|
||||
--prompt "drum beating sound and human talking" \
|
||||
--video path/to/clip.mp4 \
|
||||
--output tv2a.wav
|
||||
```
|
||||
|
||||
The client sends:
|
||||
|
||||
- `num_inference_steps`, `guidance_scale`, `seed`, `audiox_task`, `seconds_start`,
|
||||
`seconds_total`, `sigma_min`, `sigma_max` as flattened keys under `extra_body`. The server
|
||||
routes the declared knobs (see `vllm_omni/model_extras/audiox.py`) into the pipeline's
|
||||
`sampling_params.extra_args`.
|
||||
- For `v2*` / `tv2*` tasks, the video as a `video_url` content item (data URI for local files)
|
||||
|
||||
## curl
|
||||
|
||||
```bash
|
||||
curl -sS -X POST http://localhost:8099/v1/chat/completions \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"model": "zhangj1an/AudioX",
|
||||
"messages": [{"role": "user", "content": [{"type": "text", "text": "Uplifting ukulele"}]}],
|
||||
"extra_body": {
|
||||
"num_inference_steps": 250,
|
||||
"guidance_scale": 7.0,
|
||||
"seed": 42,
|
||||
"audiox_task": "t2m",
|
||||
"seconds_total": 10.0,
|
||||
"sigma_min": 0.3,
|
||||
"sigma_max": 500.0
|
||||
}
|
||||
}' > t2m.json
|
||||
```
|
||||
@@ -1,119 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""AudioX OpenAI-compatible chat client.
|
||||
|
||||
AudioX supports 6 tasks (t2a, t2m, v2a, v2m, tv2a, tv2m). Text-only tasks send the prompt as the
|
||||
chat message; video-conditioned tasks additionally attach the video via a ``video_url`` content
|
||||
item (data URI for local files). Task + generation knobs (steps, cfg, sigma range, seconds, seed)
|
||||
are sent as flattened keys under ``extra_body``; the server routes the declared knobs into
|
||||
``sampling_params.extra_args`` (see ``vllm_omni/model_extras/audiox.py``).
|
||||
|
||||
Usage:
|
||||
python openai_chat_client.py --task t2a --prompt "Fireworks burst twice..." --output t2a.wav
|
||||
python openai_chat_client.py --task tv2a --prompt "drum beating" --video clip.mp4 -o tv2a.wav
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import io
|
||||
import mimetypes
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
import soundfile
|
||||
import torch
|
||||
|
||||
VIDEO_TASKS = frozenset({"v2a", "v2m", "tv2a", "tv2m"})
|
||||
TEXT_TASKS = frozenset({"t2a", "t2m", "tv2a", "tv2m"})
|
||||
|
||||
|
||||
def _to_data_url(path: str) -> str:
|
||||
mime, _ = mimetypes.guess_type(path)
|
||||
mime = mime or "video/mp4"
|
||||
with open(path, "rb") as f:
|
||||
data = base64.b64encode(f.read()).decode("ascii")
|
||||
return f"data:{mime};base64,{data}"
|
||||
|
||||
|
||||
def _save_wav(audio: torch.Tensor, path: Path, sample_rate: int) -> None:
|
||||
audio = audio.to(torch.float32)
|
||||
audio = audio / audio.abs().max().clamp(min=1e-8)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
# soundfile expects channels-last (T, C); project convention is (C, T).
|
||||
soundfile.write(str(path), audio.clamp(-1.0, 1.0).cpu().T.numpy(), sample_rate, subtype="PCM_16")
|
||||
|
||||
|
||||
def _decode_audio_from_response(body: dict) -> tuple[torch.Tensor, int]:
|
||||
for choice in body.get("choices", []):
|
||||
audio_obj = choice.get("message", {}).get("audio")
|
||||
if not (isinstance(audio_obj, dict) and audio_obj.get("data")):
|
||||
continue
|
||||
data, sr = soundfile.read(io.BytesIO(base64.b64decode(audio_obj["data"])), dtype="float32", always_2d=True)
|
||||
return torch.from_numpy(data).transpose(0, 1), sr
|
||||
brief = {k: v for k, v in body.items() if k != "choices"}
|
||||
raise RuntimeError(f"no audio in response message.audio: {brief}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser(description="AudioX OpenAI chat client")
|
||||
p.add_argument("--task", required=True, choices=["t2a", "t2m", "v2a", "v2m", "tv2a", "tv2m"])
|
||||
p.add_argument("--prompt", "-p", default="", help="Text prompt (required for t2*/tv2*).")
|
||||
p.add_argument("--video", help="Video path or URL (required for v2*/tv2*).")
|
||||
p.add_argument("--output", "-o", default="audiox_out.wav")
|
||||
p.add_argument("--server", "-s", default="http://localhost:8099")
|
||||
p.add_argument("--model", default="zhangj1an/AudioX")
|
||||
p.add_argument("--steps", type=int, default=250)
|
||||
p.add_argument("--guidance-scale", type=float, default=7.0)
|
||||
p.add_argument("--seed", type=int, default=42)
|
||||
p.add_argument("--seconds-total", type=float, default=10.0)
|
||||
p.add_argument("--seconds-start", type=float, default=0.0)
|
||||
p.add_argument("--sigma-min", type=float, default=0.03)
|
||||
p.add_argument("--sigma-max", type=float, default=1000.0)
|
||||
args = p.parse_args()
|
||||
|
||||
if args.task in VIDEO_TASKS and not args.video:
|
||||
print(f"ERROR: task {args.task!r} requires --video", file=sys.stderr)
|
||||
return 2
|
||||
if args.task in TEXT_TASKS and not args.prompt.strip() and args.task not in {"v2a", "v2m"}:
|
||||
print(f"ERROR: task {args.task!r} requires --prompt", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
content: list[dict] = [{"type": "text", "text": args.prompt}]
|
||||
if args.task in VIDEO_TASKS:
|
||||
vurl = args.video if args.video.startswith(("http://", "https://")) else _to_data_url(args.video)
|
||||
content.append({"type": "video_url", "video_url": {"url": vurl}})
|
||||
|
||||
payload = {
|
||||
"model": args.model,
|
||||
"messages": [{"role": "user", "content": content}],
|
||||
"extra_body": {
|
||||
"num_inference_steps": args.steps,
|
||||
"guidance_scale": args.guidance_scale,
|
||||
"seed": args.seed,
|
||||
"audiox_task": args.task,
|
||||
"seconds_start": args.seconds_start,
|
||||
"seconds_total": args.seconds_total,
|
||||
"sigma_min": args.sigma_min,
|
||||
"sigma_max": args.sigma_max,
|
||||
},
|
||||
}
|
||||
|
||||
print(f"POST {args.server}/v1/chat/completions task={args.task} steps={args.steps}")
|
||||
r = requests.post(
|
||||
f"{args.server}/v1/chat/completions",
|
||||
headers={"Content-Type": "application/json"},
|
||||
json=payload,
|
||||
timeout=600,
|
||||
)
|
||||
r.raise_for_status()
|
||||
audio, sr = _decode_audio_from_response(r.json())
|
||||
_save_wav(audio, Path(args.output), sr)
|
||||
dur = audio.shape[-1] / sr
|
||||
print(f"saved {args.output} sr={sr}Hz duration={dur:.2f}s channels={audio.shape[0]}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,16 +0,0 @@
|
||||
#!/bin/bash
|
||||
# AudioX online serving startup script.
|
||||
|
||||
MODEL="${MODEL:-zhangj1an/AudioX}"
|
||||
PORT="${PORT:-8099}"
|
||||
DIFFUSION_ATTENTION_BACKEND="${DIFFUSION_ATTENTION_BACKEND:-FLASH_ATTN}"
|
||||
|
||||
echo "Starting AudioX server..."
|
||||
echo "Model: $MODEL"
|
||||
echo "Port: $PORT"
|
||||
echo "Diffusion attention backend: $DIFFUSION_ATTENTION_BACKEND"
|
||||
|
||||
DIFFUSION_ATTENTION_BACKEND="$DIFFUSION_ATTENTION_BACKEND" \
|
||||
vllm serve "$MODEL" --omni \
|
||||
--model-class-name AudioXPipeline \
|
||||
--port "$PORT"
|
||||
Reference in new issue
Block a user