[New Model] Add support for tencent/Covo-Audio-Chat (#2293)

Signed-off-by: Dnoob <dxpouo@gmail.com>
Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>

examples/offline_inference/covo_audio/README.md

@@ -0,0 +1,76 @@
+# Covo-Audio-Chat (Offline Inference)
+
+## Setup
+
+Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.
+
+> **Note**
+> Covo-Audio code2wav requires `torchdiffeq`. Install it with: `pip install torchdiffeq`
+
+## Run examples
+
+Get into the example folder:
+```bash
+cd examples/offline_inference/covo_audio
+```
+
+### Audio input chat
+
+Using the default audio asset:
+```bash
+python end2end.py
+```
+
+Using a custom audio file:
+```bash
+python end2end.py --audio-path /path/to/audio.wav
+```
+
+Using a local model:
+```bash
+python end2end.py -m /path/to/Covo-Audio-Chat --output-dir ./my_output
+```
+
+### Command-line Arguments
+
+| Argument | Short | Default | Description |
+|----------|-------|---------|-------------|
+| `--model-name` | `-m` | `tencent/Covo-Audio-Chat` | Model path or HuggingFace model ID |
+| `--text` | `-t` | `请回答这段音频里的问题。` | Text prompt / question for the audio |
+| `--audio-path` | `-a` | default audio asset | Path to local audio file |
+| `--sampling-rate` | | `16000` | Sampling rate for audio loading (Hz) |
+| `--output-dir` | | `./output_audio` | Output directory for generated files |
+| `--num-prompts` | | `1` | Number of prompts to generate |
+| `--stage-configs-path` | | (auto) | Path to stage configs YAML file |
+| `--log-stats` | | `false` | Enable detailed statistics logging |
+| `--stage-init-timeout` | | `300` | Stage initialization timeout (seconds) |
+| `--batch-timeout` | | `5` | Batching timeout (seconds) |
+| `--init-timeout` | | `300` | Overall initialization timeout (seconds) |
+| `--shm-threshold-bytes` | | `65536` | Shared memory threshold (bytes) |
+
+## Pipeline
+
+Covo-Audio-Chat uses a 2-stage pipeline:
+
+- **Stage 0 (fused_thinker_talker):** The 7B LLM generates interleaved text and audio tokens in a single autoregressive pass.
+- **Stage 1 (code2wav):** A BigVGAN-based vocoder converts the extracted audio codes into a 24kHz WAV waveform.
+
+## Output
+
+The script generates two files per request in the output directory:
+
+- `{request_id}.txt` -- prompt and generated text
+- `{request_id}.wav` -- generated audio (24kHz WAV)
+
+## FAQ
+
+If you encounter `ModuleNotFoundError: No module named 'librosa'`, install it with:
+```bash
+pip install librosa
+```
+
+## Environment
+
+- GPU: 1x A100 (80 GiB)
+- Stage 0 (7B LLM): ~16 GiB VRAM
+- Stage 1 (BigVGAN vocoder): ~2 GiB VRAM

examples/offline_inference/covo_audio/end2end.py

@@ -0,0 +1,222 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+This example shows how to use vLLM-Omni for running offline inference
+with the correct prompt format on Covo-Audio-Chat.
+
+Usage:
+    python end2end.py --audio-path /path/to/audio.wav
+"""
+
+import os
+
+import soundfile as sf
+from vllm.assets.audio import AudioAsset
+from vllm.multimodal.media.audio import load_audio
+from vllm.sampling_params import SamplingParams
+from vllm.utils.argparse_utils import FlexibleArgumentParser
+
+from vllm_omni.entrypoints.omni import Omni
+from vllm_omni.model_executor.models.covo_audio.prompt_utils import (
+    COVO_AUDIO_INPUT_PREFIX,
+    build_covo_audio_chat_prompt,
+)
+
+SEED = 42
+
+
+def get_audio_query(
+    question: str | None = None,
+    audio_path: str | None = None,
+    sampling_rate: int = 16000,
+) -> dict:
+    if question is None:
+        question = "请回答这段音频里的问题。"
+    user_content = COVO_AUDIO_INPUT_PREFIX + question
+    prompt = build_covo_audio_chat_prompt(user_content)
+
+    if audio_path is None:
+        audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate
+    else:
+        import numpy as np
+
+        audio_signal, sr = load_audio(audio_path, sr=sampling_rate)
+        audio_data = (audio_signal.astype(np.float32), sr)
+
+    return {
+        "prompt": prompt,
+        "multi_modal_data": {"audio": audio_data},
+        "modalities": ["audio"],
+    }
+
+
+def main(args):
+    query_result = get_audio_query(
+        question=args.text,
+        audio_path=args.audio_path,
+        sampling_rate=args.sampling_rate,
+    )
+
+    omni = Omni(
+        model=args.model_name,
+        stage_configs_path=args.stage_configs_path,
+        log_stats=args.log_stats,
+        stage_init_timeout=args.stage_init_timeout,
+        batch_timeout=args.batch_timeout,
+        init_timeout=args.init_timeout,
+        shm_threshold_bytes=args.shm_threshold_bytes,
+    )
+
+    # Stage 0: fused_thinker_talker
+    # stop_token_ids=[151645] (<|im_end|>) and ignore_eos=True are required
+    # so the model generates interleaved text+audio tokens before stopping.
+    thinker_sampling_params = SamplingParams(
+        temperature=0.0,
+        top_p=1.0,
+        top_k=-1,
+        max_tokens=2048,
+        seed=SEED,
+        detokenize=True,
+        repetition_penalty=1.05,
+        stop_token_ids=[151645],
+        ignore_eos=True,
+    )
+    # Stage 1: code2wav (audio codes, not real token IDs — skip detokenize)
+    code2wav_sampling_params = SamplingParams(
+        temperature=0.0,
+        top_p=1.0,
+        top_k=-1,
+        max_tokens=2048,
+        seed=SEED,
+        detokenize=False,
+        repetition_penalty=1.1,
+    )
+
+    sampling_params_list = [
+        thinker_sampling_params,
+        code2wav_sampling_params,
+    ]
+
+    prompts = [query_result for _ in range(args.num_prompts)]
+
+    omni_outputs = omni.generate(prompts, sampling_params_list)
+
+    output_dir = args.output_dir
+    os.makedirs(output_dir, exist_ok=True)
+
+    for stage_outputs in omni_outputs:
+        output = stage_outputs.request_output
+        if stage_outputs.final_output_type == "text":
+            request_id = output.request_id
+            text_output = output.outputs[0].text
+            prompt_text = output.prompt
+            out_txt = os.path.join(output_dir, f"{request_id}.txt")
+            lines = [
+                "Prompt:\n",
+                str(prompt_text) + "\n",
+                "vllm_text_output:\n",
+                str(text_output).strip() + "\n",
+            ]
+            try:
+                with open(out_txt, "w", encoding="utf-8") as f:
+                    f.writelines(lines)
+            except Exception as e:
+                print(f"[Warn] Failed writing text file {out_txt}: {e}")
+            print(f"Request ID: {request_id}, Text saved to {out_txt}")
+        elif stage_outputs.final_output_type == "audio":
+            request_id = output.request_id
+            audio_tensor = output.outputs[0].multimodal_output.get("audio")
+            if audio_tensor is None:
+                continue
+            output_wav = os.path.join(output_dir, f"{request_id}.wav")
+            audio_numpy = audio_tensor.float().detach().cpu().numpy()
+            if audio_numpy.ndim > 1:
+                audio_numpy = audio_numpy.flatten()
+            sf.write(output_wav, audio_numpy, samplerate=24000, format="WAV")
+            print(f"Request ID: {request_id}, Audio saved to {output_wav}")
+
+    omni.close()
+
+
+def parse_args():
+    parser = FlexibleArgumentParser(description="Offline inference demo for Covo-Audio-Chat")
+    parser.add_argument(
+        "--model-name",
+        "-m",
+        type=str,
+        default="tencent/Covo-Audio-Chat",
+        help="Model path or HuggingFace model ID.",
+    )
+    parser.add_argument(
+        "--text",
+        "-t",
+        type=str,
+        default=None,
+        help="Text prompt / question for the audio.",
+    )
+    parser.add_argument(
+        "--audio-path",
+        "-a",
+        type=str,
+        default=None,
+        help="Path to local audio file. Uses default asset if not provided.",
+    )
+    parser.add_argument(
+        "--sampling-rate",
+        type=int,
+        default=16000,
+        help="Sampling rate for audio loading (default: 16000).",
+    )
+    parser.add_argument(
+        "--stage-configs-path",
+        type=str,
+        default=None,
+        help="Path to stage configs YAML file.",
+    )
+    parser.add_argument(
+        "--log-stats",
+        action="store_true",
+        default=False,
+        help="Enable writing detailed statistics.",
+    )
+    parser.add_argument(
+        "--stage-init-timeout",
+        type=int,
+        default=300,
+        help="Timeout for initializing a single stage in seconds.",
+    )
+    parser.add_argument(
+        "--batch-timeout",
+        type=int,
+        default=5,
+        help="Timeout for batching in seconds.",
+    )
+    parser.add_argument(
+        "--init-timeout",
+        type=int,
+        default=300,
+        help="Timeout for initializing stages in seconds.",
+    )
+    parser.add_argument(
+        "--shm-threshold-bytes",
+        type=int,
+        default=65536,
+        help="Threshold for using shared memory in bytes.",
+    )
+    parser.add_argument(
+        "--output-dir",
+        default="./output_audio",
+        help="Output directory for generated files.",
+    )
+    parser.add_argument(
+        "--num-prompts",
+        type=int,
+        default=1,
+        help="Number of prompts to generate.",
+    )
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    main(args)

examples/online_serving/covo_audio/README.md

@@ -0,0 +1,93 @@
+# Covo-Audio-Chat
+
+## Installation
+
+Please refer to [README.md](../../../README.md)
+
+> **Note**
+> Covo-Audio code2wav requires `torchdiffeq`. Install it with: `pip install torchdiffeq`
+
+## Run examples (Covo-Audio-Chat)
+
+### Launch the Server
+
+```bash
+CUDA_VISIBLE_DEVICES=0 vllm serve tencent/Covo-Audio-Chat --omni \
+    --trust-remote-code --port 18091
+```
+
+> The default deploy config is located at `vllm_omni/deploy/covo_audio.yaml` and is loaded automatically by the model registry — no `--deploy-config` flag needed for default use.
+
+### Send Request
+
+Get into the example folder:
+```bash
+cd examples/online_serving/covo_audio
+```
+
+#### Audio input chat
+
+Using the default audio asset:
+```bash
+python openai_chat_completion_client.py
+```
+
+Using a custom audio file:
+```bash
+python openai_chat_completion_client.py --audio-path /path/to/audio.wav
+```
+
+#### Streaming mode
+
+```bash
+python openai_chat_completion_client.py --audio-path /path/to/audio.wav --stream
+```
+
+### Command-line Arguments
+
+The Python client supports the following arguments:
+
+- `--prompt` (or `-p`): Text prompt (default: `"请回答这段音频里的问题。"`)
+- `--audio-path`: Path to audio file. Supports local file paths or HTTP/HTTPS URLs. Common audio formats: WAV, MP3, OGG, FLAC, M4A. Uses the default audio asset if omitted
+- `--output-audio-path` (or `-o`): Output path for generated audio (default: `./audio_0.wav`)
+- `--model`: Model name for the API. Auto-detected from server if omitted
+- `--stream`: Enable streaming mode
+
+## Pipeline
+
+Covo-Audio-Chat uses a 2-stage pipeline:
+
+- **Stage 0 (fused_thinker_talker):** The 7B LLM generates interleaved text and audio tokens in a single autoregressive pass.
+- **Stage 1 (code2wav):** A BigVGAN-based vocoder converts the extracted audio codes into a 24kHz WAV waveform.
+
+## Environment
+
+- GPU: 1x H100 (80 GiB)
+- Stage 0 (7B LLM): ~16 GiB VRAM
+- Stage 1 (BigVGAN vocoder): ~2 GiB VRAM
+
+## Important: System Prompt
+
+Covo-Audio-Chat requires a specific system prompt to enable interleaved text+audio generation. Without it, Stage 0 produces text-only tokens, and the output audio will be **silent**.
+
+The system prompt must include the instruction:
+
+```
+请用文本和音频进行对话，交替生成5个文本token和15个音频token，音频部分使用发音人：default_female
+```
+
+The full system prompt is defined in `vllm_omni/model_executor/models/covo_audio/prompt_utils.py` (`COVO_AUDIO_SYSTEM_PROMPT`).
+
+## FAQ
+
+If you encounter `ModuleNotFoundError: No module named 'librosa'`, install it with:
+```bash
+pip install librosa
+```
+
+### Audio output is silent (0.1s, all zeros)
+
+This means Stage 0 did not generate any audio tokens. Most likely causes:
+
+1. **Missing system prompt** -- see the "Important: System Prompt" section above. The model needs the interleaved generation instruction in the system message.
+2. **Missing `ignore_eos: true` / `stop_token_ids: [151645]`** in Stage 0 sampling params -- without these, the model stops at `<|endoftext|>` before generating audio tokens.

examples/online_serving/covo_audio/openai_chat_completion_client.py

@@ -0,0 +1,216 @@
+"""
+Client example for Covo-Audio-Chat with vllm-omni.
+
+Usage
+-----
+# Start the server first:
+#   CUDA_VISIBLE_DEVICES=0 vllm serve tencent/Covo-Audio-Chat --omni --trust-remote-code
+
+# Audio input chat (uses default audio asset if --audio-path not provided):
+python openai_chat_completion_client.py
+python openai_chat_completion_client.py --audio-path /path/to/audio.wav
+
+# Streaming mode:
+python openai_chat_completion_client.py --stream
+"""
+
+import base64
+import os
+import sys
+import time
+
+from openai import OpenAI
+from vllm.assets.audio import AudioAsset
+from vllm.utils.argparse_utils import FlexibleArgumentParser
+
+from vllm_omni.model_executor.models.covo_audio.prompt_utils import (
+    COVO_AUDIO_SYSTEM_PROMPT,
+)
+
+openai_api_key = "EMPTY"
+openai_api_base = "http://localhost:18091/v1"
+
+client = OpenAI(api_key=openai_api_key, base_url=openai_api_base)
+
+SEED = 42
+
+
+def encode_base64_content_from_file(file_path: str) -> str:
+    """Encode a local file to base64 format."""
+    with open(file_path, "rb") as f:
+        return base64.b64encode(f.read()).decode("utf-8")
+
+
+def get_audio_url(audio_path: str | None) -> str:
+    """Convert an audio path to a data URL for the API."""
+    if not audio_path:
+        return AudioAsset("mary_had_lamb").url
+
+    if audio_path.startswith(("data:", "http://", "https://")):
+        return audio_path
+
+    if not os.path.exists(audio_path):
+        raise FileNotFoundError(f"Audio file not found: {audio_path}")
+
+    ext = audio_path.rsplit(".", 1)[-1].lower() if "." in audio_path else "wav"
+    mime_map = {
+        "wav": "audio/wav",
+        "mp3": "audio/mpeg",
+        "mpeg": "audio/mpeg",
+        "ogg": "audio/ogg",
+        "flac": "audio/flac",
+        "m4a": "audio/mp4",
+    }
+    mime = mime_map.get(ext, "audio/wav")
+    b64 = encode_base64_content_from_file(audio_path)
+    return f"data:{mime};base64,{b64}"
+
+
+def build_audio_messages(audio_path: str | None, prompt: str) -> list[dict]:
+    """Build messages for an audio chat request."""
+    audio_url = get_audio_url(audio_path)
+    return [
+        {"role": "system", "content": COVO_AUDIO_SYSTEM_PROMPT},
+        {
+            "role": "user",
+            "content": [
+                {"type": "audio_url", "audio_url": {"url": audio_url}},
+                {"type": "text", "text": prompt},
+            ],
+        },
+    ]
+
+
+def get_model_name(override: str | None = None) -> str:
+    """Return the model name to use in API calls."""
+    if override:
+        return override
+    # Auto-detect from the server
+    models = client.models.list()
+    return models.data[0].id
+
+
+def run(args) -> None:
+    model_name = get_model_name(args.model)
+    thinker_params = {
+        "temperature": 0.0,
+        "top_p": 1.0,
+        "top_k": -1,
+        "max_tokens": 2048,
+        "seed": SEED,
+        "detokenize": True,
+        "repetition_penalty": 1.05,
+        "stop_token_ids": [151645],
+        "ignore_eos": True,
+    }
+    code2wav_params = {
+        "temperature": 0.0,
+        "top_p": 1.0,
+        "top_k": -1,
+        "max_tokens": 2048,
+        "seed": SEED,
+        "detokenize": False,
+        "repetition_penalty": 1.05,
+    }
+
+    extra_body = {
+        "sampling_params_list": [thinker_params, code2wav_params],
+    }
+
+    messages = build_audio_messages(args.audio_path, args.prompt)
+
+    start = time.perf_counter()
+
+    if not args.stream:
+        completion = client.chat.completions.create(
+            messages=messages,
+            model=model_name,
+            extra_body=extra_body,
+            stream=False,
+        )
+        elapsed = time.perf_counter() - start
+
+        for choice in completion.choices:
+            if choice.message.audio:
+                audio_bytes = base64.b64decode(choice.message.audio.data)
+                out_path = args.output_audio_path
+                os.makedirs(os.path.dirname(out_path) or ".", exist_ok=True)
+                with open(out_path, "wb") as f:
+                    f.write(audio_bytes)
+                print(f"Audio saved to {out_path} ({len(audio_bytes)} bytes)")
+            elif choice.message.content:
+                print("Chat completion output from text:", choice.message.content)
+
+        print(f"Elapsed: {elapsed:.2f}s", file=sys.stderr)
+
+    else:
+        stream = client.chat.completions.create(
+            messages=messages,
+            model=model_name,
+            extra_body=extra_body,
+            stream=True,
+        )
+        audio_idx = 0
+        for chunk in stream:
+            modality = getattr(chunk, "modality", None)
+            elapsed = time.perf_counter() - start
+            for choice in chunk.choices:
+                content = getattr(choice.delta, "content", None) if hasattr(choice, "delta") else None
+                if not content:
+                    continue
+                if modality == "audio":
+                    audio_bytes = base64.b64decode(content)
+                    out_dir = os.path.dirname(args.output_audio_path) or "."
+                    out_path = os.path.join(out_dir, f"audio_{audio_idx}.wav")
+                    os.makedirs(out_dir, exist_ok=True)
+                    with open(out_path, "wb") as f:
+                        f.write(audio_bytes)
+                    print(f"[{elapsed:.2f}s] Audio chunk saved to {out_path}", file=sys.stderr)
+                    audio_idx += 1
+                elif modality == "text":
+                    print(content, end="", flush=True)
+
+        print()  # trailing newline
+        elapsed = time.perf_counter() - start
+        print(f"Elapsed: {elapsed:.2f}s", file=sys.stderr)
+
+
+def parse_args():
+    parser = FlexibleArgumentParser(description="Covo-Audio-Chat client example")
+    parser.add_argument(
+        "--prompt",
+        "-p",
+        type=str,
+        default="请回答这段音频里的问题。",
+        help="Text prompt.",
+    )
+    parser.add_argument(
+        "--audio-path",
+        type=str,
+        default=None,
+        help="Path to audio file. Uses default audio asset if omitted.",
+    )
+    parser.add_argument(
+        "--output-audio-path",
+        "-o",
+        type=str,
+        default="./audio_0.wav",
+        help="Output path for generated audio.",
+    )
+    parser.add_argument(
+        "--model",
+        type=str,
+        default=None,
+        help="Model name for the API. Auto-detected from server if omitted.",
+    )
+    parser.add_argument(
+        "--stream",
+        action="store_true",
+        help="Enable streaming mode.",
+    )
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    run(args)