[CI] Remove AudioX and MammothModa2 support (#6353)

Signed-off-by: wangyu <410167048@qq.com>
This commit is contained in:
wangyu authored and GitHub committed 2026-08-23 04:22:05 +00:00
1 parent 39c16d75a9
commit 1e2a6446f7
59 files changed
+42 -7029

No files matched your search

+1 -1
View File
@@ -17,7 +17,7 @@ Shared plumbing (corpus/WAV IO, tokenizer loading, the CFG cond/uncond
pair contract) lives in `common.py`; each task script keeps only its own
prompt recipe and flow. Note `text_to_audio.py` here drives the Audex
two-stage AR pipeline — for diffusion text-to-audio models (Stable Audio
Open, AudioX) use the generic
Open) use the generic
`examples/offline_inference/text_to_audio/text_to_audio.py` instead.
## text_to_speech.py — text → speech
@@ -7,7 +7,7 @@ thinker generates interleaved 4-codebook <audiocodec_N> RVQ tokens under an
RVQ phase mask, and the external XCodec1 checkpoint decodes them to a 16 kHz
mono WAV.
(For DIFFUSION text-to-audio models — Stable Audio Open, AudioX — use
(For DIFFUSION text-to-audio models — Stable Audio Open — use
``examples/offline_inference/text_to_audio/text_to_audio.py`` instead; Audex
TTA is a two-stage autoregressive pipeline with a different interface.)
@@ -1,117 +0,0 @@
"""
Offline inference example: MammothModa2 image summarization (single AR stage).
Example:
uv run python examples/offline_inference/mammothmodal2_preview/run_mammothmoda2_image_summarize.py \
--model path/to/MammothModa2-Preview \
--deploy-config vllm_omni/deploy/mammoth_moda2_ar.yaml \
--image /path/to/input.jpg \
--question "Please summarize the content of this image."
"""
from __future__ import annotations
import argparse
import os
from PIL import Image
from vllm import SamplingParams
from vllm.multimodal.image import convert_image_mode
from vllm_omni import Omni
DEFAULT_SYSTEM = "You are a helpful assistant."
DEFAULT_QUESTION = "Please summarize the content of this image."
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="MammothModa2 image summarization (offline, AR only).")
parser.add_argument("--model", type=str, required=True, help="Path to model directory or model id.")
parser.add_argument("--deploy-config", type=str, required=True, help="Path to deploy config yaml.")
parser.add_argument("--image", type=str, required=True, help="Path to input image.")
parser.add_argument("--question", type=str, default=DEFAULT_QUESTION, help="Question/instruction for the model.")
parser.add_argument("--system", type=str, default=DEFAULT_SYSTEM, help="System prompt.")
parser.add_argument(
"--max-tokens",
type=int,
default=512,
help="Max new tokens to generate.",
)
parser.add_argument("--temperature", type=float, default=0.2)
parser.add_argument("--top-p", type=float, default=0.9)
parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--trust-remote-code", action="store_true")
parser.add_argument(
"--enable-diffusion-pipeline-profiler",
action="store_true",
help="Enable diffusion pipeline profiler to display stage durations.",
)
return parser.parse_args()
def build_prompt(system: str, question: str) -> str:
return (
f"<|im_start|>system\n{system}<|im_end|>\n"
"<|im_start|>user\n"
"<|vision_start|><|image_pad|><|vision_end|>"
f"{question}<|im_end|>\n"
"<|im_start|>assistant\n"
)
def main() -> None:
args = parse_args()
if not os.path.exists(args.image):
raise FileNotFoundError(f"Image file not found: {args.image}")
pil_image = Image.open(args.image)
image_data = convert_image_mode(pil_image, "RGB")
prompt = build_prompt(args.system, args.question)
omni = Omni(
model=args.model,
deploy_config=args.deploy_config,
trust_remote_code=args.trust_remote_code,
enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler,
)
try:
sp = SamplingParams(
temperature=float(args.temperature),
top_p=float(args.top_p),
top_k=-1,
max_tokens=int(args.max_tokens),
seed=int(args.seed),
detokenize=True,
)
# NOTE: omni.generate() returns a Generator[OmniRequestOutput, None, None].
# Consume it inside the try block so the worker isn't closed early.
outputs = list(
omni.generate(
[
{
"prompt": prompt,
"multi_modal_data": {"image": image_data},
"additional_information": {"omni_task": ["chat"]},
}
],
[sp],
)
)
finally:
omni.close()
lines: list[str] = []
for stage_outputs in outputs:
ro = stage_outputs
text = ro.outputs[0].text if getattr(ro, "outputs", None) else str(ro)
lines.append(f"request_id: {getattr(ro, 'request_id', 'unknown')}\n")
lines.append("answer:\n")
lines.append(text.strip() + "\n")
lines.append("\n")
print("\n".join(lines))
if __name__ == "__main__":
main()
@@ -1,11 +1,10 @@
# Text-To-Audio
A unified script for text/video-to-audio generation. Supported models:
A unified script for text-to-audio generation. Supported models:
| Model | Tasks | Notes |
|-------|-------|-------|
| `stabilityai/stable-audio-open-1.0` | text-to-audio | gated; uses `--audio-length` |
| `zhangj1an/AudioX` | `t2a` / `t2m` / `v2a` / `v2m` / `tv2a` / `tv2m` | pass `--task`; video tasks need `--video` |
The `stabilityai/stable-audio-open-1.0` pipeline generates audio from text prompts.
@@ -50,42 +49,12 @@ python text_to_audio.py \
--output stable_audio_output.wav
```
### AudioX
AudioX supports six tasks. Sampler and reference knobs (declared in
`vllm_omni/model_extras/audiox.py`) are passed via the generic `--extra-body`
JSON flag, routed into sampling `extra_args`.
Text tasks (`t2a` / `t2m`):
```bash
python text_to_audio.py \
--model zhangj1an/AudioX --task t2a \
--prompt "Fireworks burst twice, followed by a clock ticking." \
--num-inference-steps 250 --guidance-scale 6.0 --audio-length 10.0 --seed 42 \
--extra-body '{"sigma_min": 0.03, "sigma_max": 1000.0}' \
--output t2a.wav
```
Video-conditioned tasks (`v2a` / `v2m` / `tv2a` / `tv2m`) require `--video`:
```bash
python text_to_audio.py \
--model zhangj1an/AudioX --task tv2a \
--prompt "drum beating sound and human talking" \
--video https://zeyuet.github.io/AudioX/static/samples/V2M/1XeBotOFqHA.mp4 \
--num-inference-steps 250 --guidance-scale 6.0 --audio-length 10.0 \
--output tv2a.wav
```
Key arguments:
- `--prompt`: text description (string).
- `--task`: [AudioX] one of `t2a`/`t2m`/`v2a`/`v2m`/`tv2a`/`tv2m`.
- `--video`: [AudioX `v2*`/`tv2*`] video file/URL for conditioning (→ `video_path`).
- `--audio-start`: audio start offset in seconds (→ `audio_start_in_s` for Stable Audio, `seconds_start` for AudioX).
- `--audio-length`: audio duration in seconds (audio length for Stable Audio, `seconds_total` for AudioX).
- `--extra-body`: JSON dict of model-specific knobs (declared in `vllm_omni/model_extras/`), merged into sampling `extra_args`. For AudioX, sampler/reference knobs go here, e.g. `'{"sigma_min": 0.03, "sigma_max": 1000.0, "cfg_rescale": 0.0, "audio_path": "ref.wav"}'`.
- `--audio-start`: audio start offset in seconds (→ `audio_start_in_s`).
- `--audio-length`: audio duration in seconds (audio length for Stable Audio).
- `--extra-body`: JSON dict of model-specific knobs, merged into sampling `extra_args`.
- `--negative-prompt`: negative prompt for classifier-free guidance.
- `--seed`: integer seed for deterministic generation.
- `--guidance-scale`: classifier-free guidance scale.
@@ -2,15 +2,13 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Example script for text/video-to-audio generation with diffusion audio models.
Example script for text-to-audio generation with diffusion audio models.
This script supports:
* Stable Audio Open (`stabilityai/stable-audio-open-1.0`) — text-to-audio.
* AudioX (`zhangj1an/AudioX`) — six tasks: t2a / t2m / v2a / v2m / tv2a / tv2m.
Model-specific generation knobs are declared in `vllm_omni/model_extras` and
routed into `sampling_params.extra_args` via `apply_declared_extra_args`, so
the same script serves both models without per-model branching in the engine.
Model-specific generation knobs can be passed via `--extra-body` and are
merged into `sampling_params.extra_args`.
Usage:
# Stable Audio Open
@@ -18,15 +16,6 @@ Usage:
python text_to_audio.py --prompt "A piano playing a gentle melody" --audio-length 10.0
python text_to_audio.py --prompt "Thunder and rain sounds" --negative-prompt "Low quality"
python text_to_audio.py --prompt "A soft synth pad" --cache-backend tea_cache
# AudioX (text-to-audio / text-to-music)
python text_to_audio.py --model zhangj1an/AudioX --task t2a \
--prompt "Fireworks burst twice, then a clock ticking." \
--num-inference-steps 250 --guidance-scale 6.0 --audio-length 10.0
# AudioX video-conditioned (v2*/tv2*) — pass a video file/URL
python text_to_audio.py --model zhangj1an/AudioX --task tv2a \
--prompt "drum beating sound and human talking" --video clip.mp4 \
--extra-body '{"sigma_min": 0.03, "sigma_max": 1000.0}'
"""
import argparse
@@ -38,16 +27,11 @@ from typing import Any
import numpy as np
import torch
from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args
from vllm_omni.entrypoints.omni import Omni
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.model_extras import get_extra_body_params, get_model_class_name
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
AUDIOX_TASKS = ("t2a", "t2m", "v2a", "v2m", "tv2a", "tv2m")
AUDIOX_VIDEO_TASKS = frozenset({"v2a", "v2m", "tv2a", "tv2m"})
def parse_extra_body(value: str) -> dict[str, Any]:
"""Parse a JSON object string for --extra-body."""
@@ -65,7 +49,7 @@ def parse_args() -> argparse.Namespace:
parser.add_argument(
"--model",
default="stabilityai/stable-audio-open-1.0",
help="Audio diffusion model name or local path. Supported: stabilityai/stable-audio-open-1.0, zhangj1an/AudioX.",
help="Audio diffusion model name or local path. Supported: stabilityai/stable-audio-open-1.0.",
)
parser.add_argument(
"--prompt",
@@ -96,19 +80,13 @@ def parse_args() -> argparse.Namespace:
"--audio-start",
type=float,
default=0.0,
help=(
"Audio start offset in seconds. Applies to both models "
"(audio_start_in_s for Stable Audio, seconds_start for AudioX)."
),
help="Audio start offset in seconds (maps to audio_start_in_s).",
)
parser.add_argument(
"--audio-length",
type=float,
default=10.0,
help=(
"Audio duration in seconds. Maps to audio length for Stable Audio "
"(max ~47s for stable-audio-open-1.0) and seconds_total for AudioX."
),
help=("Audio duration in seconds. Maps to audio length for Stable Audio (max ~47s for stable-audio-open-1.0)."),
)
parser.add_argument(
"--num-inference-steps",
@@ -122,41 +100,20 @@ def parse_args() -> argparse.Namespace:
default=1,
help="Number of audio waveforms to generate for the given prompt.",
)
parser.add_argument(
"--task",
type=str,
default=None,
choices=list(AUDIOX_TASKS),
help=(
"[AudioX only] Generation task: t2a/t2m/v2a/v2m/tv2a/tv2m. "
"v2*/tv2* require --video. Ignored for Stable Audio Open."
),
)
parser.add_argument(
"--video",
type=str,
default=None,
help="[AudioX v2*/tv2*] Video file path or URL used for conditioning.",
)
parser.add_argument(
"--extra-body",
type=parse_extra_body,
default=None,
help=(
"Model-specific generation params as a JSON object, e.g. "
'\'{"sigma_min": 0.03, "sigma_max": 1000.0}\'. Each key is filtered '
"against the model's declared extra_body_params (see vllm_omni/model_extras), "
"so unknown keys are silently dropped. Values here override the equivalent flags."
"Model-specific generation params as a JSON object, merged into "
"sampling extra_args. Values here override the equivalent flags."
),
)
parser.add_argument(
"--output",
type=str,
default=None,
help=(
"Path to save the generated audio (WAV format). Defaults to "
"audiox_output.wav for AudioX and stable_audio_output.wav otherwise."
),
help=("Path to save the generated audio (WAV format). Defaults to stable_audio_output.wav."),
)
parser.add_argument(
"--sample-rate",
@@ -275,8 +232,6 @@ def save_audio(audio_data: np.ndarray, output_path: str, sample_rate: int = 4410
def main():
args = parse_args()
model_name = str(args.model).lower() if args.model else ""
is_audiox = "audiox" in model_name
generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
cache_config = None
if args.cache_backend == "tea_cache":
@@ -285,11 +240,9 @@ def main():
}
print(f"\n{'=' * 60}")
print("Text/Video-to-Audio Generation")
print("Text-to-Audio Generation")
print(f"{'=' * 60}")
print(f" Model: {args.model}")
if args.task is not None:
print(f" Task: {args.task}")
print(f" Prompt: {args.prompt}")
print(f" Negative prompt: {args.negative_prompt}")
print(f" Duration: {args.audio_length}s")
@@ -305,8 +258,6 @@ def main():
print(f" Seed: {args.seed}")
print(f"{'=' * 60}\n")
# Initialize Omni. AudioX requires an explicit pipeline class; Stable Audio
# is resolved from its config.
omni_kwargs: dict[str, Any] = dict(
model=args.model,
use_hsdp=args.use_hsdp,
@@ -318,11 +269,7 @@ def main():
enable_cpu_offload=args.enable_cpu_offload,
enable_layerwise_offload=args.enable_layerwise_offload,
)
if is_audiox:
omni_kwargs["model_class_name"] = "AudioXPipeline"
omni = Omni(**omni_kwargs)
model_class_name = get_model_class_name(omni)
declared_extra_body_params = get_extra_body_params(model_class_name)
diffusion_params = OmniDiffusionSamplingParams(
generator=generator,
@@ -332,38 +279,19 @@ def main():
seed=args.seed,
)
# Negative prompt is opt-in for both models (off by default). Including a
# Negative prompt is opt-in (off by default). Including a
# non-empty negative prompt enables the separate CFG conditioning branch.
prompt: dict[str, Any] = {"prompt": args.prompt}
if args.negative_prompt:
prompt["negative_prompt"] = args.negative_prompt
if model_class_name == "AudioXPipeline":
if args.task is None:
raise ValueError("AudioX requires --task (one of t2a/t2m/v2a/v2m/tv2a/tv2m).")
if args.task in AUDIOX_VIDEO_TASKS and not args.video:
raise ValueError(f"AudioX task {args.task!r} requires --video.")
# AudioX core knobs are surfaced as shared flags; sampler/reference knobs
# (sigma_min/sigma_max/cfg_rescale/audio_path) arrive via --extra-body and
# are filtered against the declared extra_body contract.
user_extra: dict[str, Any] = {
"audiox_task": args.task,
"seconds_start": args.audio_start,
"seconds_total": args.audio_length,
"video_path": args.video,
}
if args.extra_body:
user_extra.update(args.extra_body)
apply_declared_extra_args(diffusion_params, declared_extra_body_params, user_extra)
else:
# Stable Audio Open path (backward-compatible).
audio_end_in_s = args.audio_start + args.audio_length
diffusion_params.extra_args = {
"audio_start_in_s": args.audio_start,
"audio_end_in_s": audio_end_in_s,
}
if args.extra_body:
diffusion_params.extra_args.update(args.extra_body)
audio_end_in_s = args.audio_start + args.audio_length
diffusion_params.extra_args = {
"audio_start_in_s": args.audio_start,
"audio_end_in_s": audio_end_in_s,
}
if args.extra_body:
diffusion_params.extra_args.update(args.extra_body)
generation_start = time.perf_counter()
@@ -374,7 +302,7 @@ def main():
print(f"Total generation time: {generation_time:.2f} seconds")
output = args.output or ("audiox_output.wav" if model_class_name == "AudioXPipeline" else "stable_audio_output.wav")
output = args.output or "stable_audio_output.wav"
output_path = Path(output)
output_path.parent.mkdir(parents=True, exist_ok=True)
suffix = output_path.suffix or ".wav"
@@ -599,17 +599,6 @@ def main():
if args.seed is not None and hasattr(params, "seed"):
params.seed = args.seed
# MammothModa2's AR stage emits one visual token per grid cell,
# one EOL token per row, and one final look-ahead token whose hidden
# state is unavailable. Size the first stage from the prompt metadata
# instead of SamplingParams' default of 16 tokens.
prompt_info = prompt_dict.get("additional_information", {})
if idx == 0 and prompt_info.get("omni_task") == ["t2i"]:
ar_width = int(prompt_info.get("ar_width", [0])[0])
ar_height = int(prompt_info.get("ar_height", [0])[0])
if ar_width > 0 and ar_height > 0:
params.max_tokens = ar_height * (ar_width + 1) + 1
if not diffusion_replaced and len(sampling_params_list) == 1:
sampling_params_list = [diffusion_params]
+3 -31
View File
@@ -10,12 +10,10 @@ Generate text from text or image inputs with vLLM-Omni's shared offline entrypoi
| --- | --- | --- | --- |
| `ByteDance-Seed/BAGEL-7B-MoT` | Yes | Yes | `vllm_omni/deploy/bagel.yaml` |
| `tencent/HunyuanImage-3.0-Instruct` | Yes | Yes | `vllm_omni/deploy/hunyuan_image3_ar.yaml` |
| `bytedance-research/MammothModa2-Preview` | Yes | Yes | `vllm_omni/deploy/mammoth_moda2_ar.yaml` |
| `bytedance-research/MammothModa2-Dev` | Yes | Yes | `vllm_omni/deploy/mammoth_moda2_ar.yaml` |
The script recognizes these model families from `config.json`, applies the
model-specific prompt format, and selects an AR-only deploy for HunyuanImage-3
and MammothModa2. BAGEL uses its registered default deploy.
model-specific prompt format, and selects an AR-only deploy for HunyuanImage-3.
BAGEL uses its registered default deploy.
## Text-To-Text
@@ -35,22 +33,6 @@ python examples/offline_inference/x_to_text/x_to_text.py \
--prompt "Explain multimodal inference in three concise sentences."
```
### MammothModa2
```bash
python examples/offline_inference/x_to_text/x_to_text.py \
--model bytedance-research/MammothModa2-Preview \
--prompt "Explain multimodal inference in three concise sentences."
```
The same command supports the Dev checkpoint:
```bash
python examples/offline_inference/x_to_text/x_to_text.py \
--model bytedance-research/MammothModa2-Dev \
--prompt "Explain multimodal inference in three concise sentences."
```
## Image-To-Text
Pass one image with `--image`. The default question can be replaced with any
@@ -72,16 +54,6 @@ python examples/offline_inference/x_to_text/x_to_text.py \
--prompt "Describe this image in detail."
```
```bash
python examples/offline_inference/x_to_text/x_to_text.py \
--model bytedance-research/MammothModa2-Preview \
--image image.png \
--prompt "Describe this image in detail."
```
Use `bytedance-research/MammothModa2-Dev` in the same command to run I2T with
the Dev checkpoint.
## Key Arguments
| Argument | Default | Description |
@@ -102,4 +74,4 @@ the Dev checkpoint.
The committed HunyuanImage-3 AR-only default uses four GPUs with TP=4. To use a
different topology, pass a custom AR-only YAML through `--deploy-config`.
BAGEL and MammothModa2 use their repository deploy defaults unless overridden.
BAGEL uses its repository deploy default unless overridden.
@@ -15,7 +15,6 @@ from vllm_omni.model_extras import build_x_to_text_prompt, get_x_to_text_model_f
_REPO_ROOT = Path(__file__).resolve().parents[3]
_HUNYUAN_AR_DEPLOY_CONFIG = _REPO_ROOT / "vllm_omni" / "deploy" / "hunyuan_image3_ar.yaml"
_MAMMOTH_AR_DEPLOY_CONFIG = _REPO_ROOT / "vllm_omni" / "deploy" / "mammoth_moda2_ar.yaml"
def parse_args() -> argparse.Namespace:
@@ -62,7 +61,7 @@ def main() -> None:
omni_kwargs: dict[str, Any] = {
"model": args.model,
"mode": "image-to-text" if image is not None else "text-to-text",
"trust_remote_code": args.trust_remote_code or family in {"hunyuan_image3", "mammoth_moda2"},
"trust_remote_code": args.trust_remote_code or family == "hunyuan_image3",
"log_stats": args.log_stats,
}
if args.enforce_eager is not None:
@@ -71,8 +70,6 @@ def main() -> None:
omni_kwargs["deploy_config"] = args.deploy_config
elif family == "hunyuan_image3":
omni_kwargs["deploy_config"] = str(_HUNYUAN_AR_DEPLOY_CONFIG)
elif family == "mammoth_moda2":
omni_kwargs["deploy_config"] = str(_MAMMOTH_AR_DEPLOY_CONFIG)
omni = Omni(**omni_kwargs)
try:
-64
View File
@@ -1,64 +0,0 @@
# AudioX online serving
Launches the `AudioXPipeline` behind vLLM-Omni's OpenAI-compatible chat endpoint and provides a
minimal Python client that covers all six tasks (`t2a`, `t2m`, `v2a`, `v2m`, `tv2a`, `tv2m`).
## Start the server
```bash
cd examples/online_serving/audiox
bash run_server.sh # defaults: MODEL=zhangj1an/AudioX, PORT=8099
```
Environment overrides: `MODEL`, `PORT`, `DIFFUSION_ATTENTION_BACKEND`.
## Call from Python
```bash
# text-to-audio
python openai_chat_client.py --task t2a \
--prompt "Fireworks burst twice, followed by a period of silence before a clock begins ticking." \
--output t2a.wav
# text-to-music
python openai_chat_client.py --task t2m \
--prompt "Uplifting ukulele tune for a travel vlog" \
--output t2m.wav
# video-to-audio (no text)
python openai_chat_client.py --task v2a --video path/to/clip.mp4 --output v2a.wav
# text+video-to-audio
python openai_chat_client.py --task tv2a \
--prompt "drum beating sound and human talking" \
--video path/to/clip.mp4 \
--output tv2a.wav
```
The client sends:
- `num_inference_steps`, `guidance_scale`, `seed`, `audiox_task`, `seconds_start`,
`seconds_total`, `sigma_min`, `sigma_max` as flattened keys under `extra_body`. The server
routes the declared knobs (see `vllm_omni/model_extras/audiox.py`) into the pipeline's
`sampling_params.extra_args`.
- For `v2*` / `tv2*` tasks, the video as a `video_url` content item (data URI for local files)
## curl
```bash
curl -sS -X POST http://localhost:8099/v1/chat/completions \
-H 'Content-Type: application/json' \
-d '{
"model": "zhangj1an/AudioX",
"messages": [{"role": "user", "content": [{"type": "text", "text": "Uplifting ukulele"}]}],
"extra_body": {
"num_inference_steps": 250,
"guidance_scale": 7.0,
"seed": 42,
"audiox_task": "t2m",
"seconds_total": 10.0,
"sigma_min": 0.3,
"sigma_max": 500.0
}
}' > t2m.json
```
@@ -1,119 +0,0 @@
#!/usr/bin/env python3
"""AudioX OpenAI-compatible chat client.
AudioX supports 6 tasks (t2a, t2m, v2a, v2m, tv2a, tv2m). Text-only tasks send the prompt as the
chat message; video-conditioned tasks additionally attach the video via a ``video_url`` content
item (data URI for local files). Task + generation knobs (steps, cfg, sigma range, seconds, seed)
are sent as flattened keys under ``extra_body``; the server routes the declared knobs into
``sampling_params.extra_args`` (see ``vllm_omni/model_extras/audiox.py``).
Usage:
python openai_chat_client.py --task t2a --prompt "Fireworks burst twice..." --output t2a.wav
python openai_chat_client.py --task tv2a --prompt "drum beating" --video clip.mp4 -o tv2a.wav
"""
from __future__ import annotations
import argparse
import base64
import io
import mimetypes
import sys
from pathlib import Path
import requests
import soundfile
import torch
VIDEO_TASKS = frozenset({"v2a", "v2m", "tv2a", "tv2m"})
TEXT_TASKS = frozenset({"t2a", "t2m", "tv2a", "tv2m"})
def _to_data_url(path: str) -> str:
mime, _ = mimetypes.guess_type(path)
mime = mime or "video/mp4"
with open(path, "rb") as f:
data = base64.b64encode(f.read()).decode("ascii")
return f"data:{mime};base64,{data}"
def _save_wav(audio: torch.Tensor, path: Path, sample_rate: int) -> None:
audio = audio.to(torch.float32)
audio = audio / audio.abs().max().clamp(min=1e-8)
path.parent.mkdir(parents=True, exist_ok=True)
# soundfile expects channels-last (T, C); project convention is (C, T).
soundfile.write(str(path), audio.clamp(-1.0, 1.0).cpu().T.numpy(), sample_rate, subtype="PCM_16")
def _decode_audio_from_response(body: dict) -> tuple[torch.Tensor, int]:
for choice in body.get("choices", []):
audio_obj = choice.get("message", {}).get("audio")
if not (isinstance(audio_obj, dict) and audio_obj.get("data")):
continue
data, sr = soundfile.read(io.BytesIO(base64.b64decode(audio_obj["data"])), dtype="float32", always_2d=True)
return torch.from_numpy(data).transpose(0, 1), sr
brief = {k: v for k, v in body.items() if k != "choices"}
raise RuntimeError(f"no audio in response message.audio: {brief}")
def main() -> int:
p = argparse.ArgumentParser(description="AudioX OpenAI chat client")
p.add_argument("--task", required=True, choices=["t2a", "t2m", "v2a", "v2m", "tv2a", "tv2m"])
p.add_argument("--prompt", "-p", default="", help="Text prompt (required for t2*/tv2*).")
p.add_argument("--video", help="Video path or URL (required for v2*/tv2*).")
p.add_argument("--output", "-o", default="audiox_out.wav")
p.add_argument("--server", "-s", default="http://localhost:8099")
p.add_argument("--model", default="zhangj1an/AudioX")
p.add_argument("--steps", type=int, default=250)
p.add_argument("--guidance-scale", type=float, default=7.0)
p.add_argument("--seed", type=int, default=42)
p.add_argument("--seconds-total", type=float, default=10.0)
p.add_argument("--seconds-start", type=float, default=0.0)
p.add_argument("--sigma-min", type=float, default=0.03)
p.add_argument("--sigma-max", type=float, default=1000.0)
args = p.parse_args()
if args.task in VIDEO_TASKS and not args.video:
print(f"ERROR: task {args.task!r} requires --video", file=sys.stderr)
return 2
if args.task in TEXT_TASKS and not args.prompt.strip() and args.task not in {"v2a", "v2m"}:
print(f"ERROR: task {args.task!r} requires --prompt", file=sys.stderr)
return 2
content: list[dict] = [{"type": "text", "text": args.prompt}]
if args.task in VIDEO_TASKS:
vurl = args.video if args.video.startswith(("http://", "https://")) else _to_data_url(args.video)
content.append({"type": "video_url", "video_url": {"url": vurl}})
payload = {
"model": args.model,
"messages": [{"role": "user", "content": content}],
"extra_body": {
"num_inference_steps": args.steps,
"guidance_scale": args.guidance_scale,
"seed": args.seed,
"audiox_task": args.task,
"seconds_start": args.seconds_start,
"seconds_total": args.seconds_total,
"sigma_min": args.sigma_min,
"sigma_max": args.sigma_max,
},
}
print(f"POST {args.server}/v1/chat/completions task={args.task} steps={args.steps}")
r = requests.post(
f"{args.server}/v1/chat/completions",
headers={"Content-Type": "application/json"},
json=payload,
timeout=600,
)
r.raise_for_status()
audio, sr = _decode_audio_from_response(r.json())
_save_wav(audio, Path(args.output), sr)
dur = audio.shape[-1] / sr
print(f"saved {args.output} sr={sr}Hz duration={dur:.2f}s channels={audio.shape[0]}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -1,16 +0,0 @@
#!/bin/bash
# AudioX online serving startup script.
MODEL="${MODEL:-zhangj1an/AudioX}"
PORT="${PORT:-8099}"
DIFFUSION_ATTENTION_BACKEND="${DIFFUSION_ATTENTION_BACKEND:-FLASH_ATTN}"
echo "Starting AudioX server..."
echo "Model: $MODEL"
echo "Port: $PORT"
echo "Diffusion attention backend: $DIFFUSION_ATTENTION_BACKEND"
DIFFUSION_ATTENTION_BACKEND="$DIFFUSION_ATTENTION_BACKEND" \
vllm serve "$MODEL" --omni \
--model-class-name AudioXPipeline \
--port "$PORT"