[Feature] support to change the speaker of qwen3-omni (#1963)
Signed-off-by: Rein Yang <ruiruyang2@gmail.com>
This commit is contained in:
1 parent
d16be833e1
commit
3cdf28828e
18 files changed
+486
-533
No files matched your search
+98
-30
@@ -16,6 +16,14 @@ class QueryResult(NamedTuple):
|
||||
limit_mm_per_prompt: dict[str, int]
|
||||
|
||||
|
||||
def make_audio_output_filename(request_id: str | None, index: int) -> str:
|
||||
"""Build a stable output filename using request ID when available."""
|
||||
if not request_id:
|
||||
request_id = f"unknown_{index}"
|
||||
safe_request_id = "".join(ch if (ch.isalnum() or ch in ("-", "_")) else "_" for ch in request_id)
|
||||
return f"audio_{safe_request_id}_{index}.wav"
|
||||
|
||||
|
||||
def encode_base64_content_from_url(content_url: str) -> str:
|
||||
"""Encode a content retrieved from a remote url to base64 format."""
|
||||
|
||||
@@ -165,6 +173,34 @@ def get_system_prompt():
|
||||
}
|
||||
|
||||
|
||||
def _parse_csv_arg(value: str | None) -> list[str]:
|
||||
if not value:
|
||||
return []
|
||||
return [item.strip() for item in value.split(",") if item.strip()]
|
||||
|
||||
|
||||
def _build_prompt_for_query_type(
|
||||
query_type: str,
|
||||
custom_prompt: str | None,
|
||||
video_path: str | None,
|
||||
image_path: str | None,
|
||||
audio_path: str | None,
|
||||
):
|
||||
query_func = query_map[query_type]
|
||||
if query_type == "use_video":
|
||||
return query_func(video_path=video_path, custom_prompt=custom_prompt)
|
||||
if query_type == "use_image":
|
||||
return query_func(image_path=image_path, custom_prompt=custom_prompt)
|
||||
if query_type == "use_audio":
|
||||
return query_func(audio_path=audio_path, custom_prompt=custom_prompt)
|
||||
if query_type == "text":
|
||||
return query_func(custom_prompt=custom_prompt)
|
||||
if query_type == "use_audio_in_video":
|
||||
return query_func(video_path=video_path, custom_prompt=custom_prompt)
|
||||
# use_mixed_modalities / use_multi_audios
|
||||
return query_func(custom_prompt=custom_prompt)
|
||||
|
||||
|
||||
def get_text_query(custom_prompt: str | None = None):
|
||||
question = (
|
||||
custom_prompt or "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words."
|
||||
@@ -379,31 +415,6 @@ def run_multimodal_generation(args, client: OpenAI) -> None:
|
||||
audio_path = getattr(args, "audio_path", None)
|
||||
custom_prompt = getattr(args, "prompt", None)
|
||||
|
||||
# Get the query function and call it with appropriate parameters
|
||||
query_func = query_map[args.query_type]
|
||||
if args.query_type == "use_video":
|
||||
prompt = query_func(video_path=video_path, custom_prompt=custom_prompt)
|
||||
elif args.query_type == "use_image":
|
||||
prompt = query_func(image_path=image_path, custom_prompt=custom_prompt)
|
||||
elif args.query_type == "use_audio":
|
||||
prompt = query_func(audio_path=audio_path, custom_prompt=custom_prompt)
|
||||
elif args.query_type == "text":
|
||||
prompt = query_func(custom_prompt=custom_prompt)
|
||||
elif args.query_type == "use_audio_in_video":
|
||||
prompt = query_func(
|
||||
video_path=video_path,
|
||||
custom_prompt=custom_prompt,
|
||||
)
|
||||
else:
|
||||
prompt = query_func()
|
||||
|
||||
extra_body = {
|
||||
"sampling_params_list": sampling_params_list # Optional, it has a default setting in stage_configs of the corresponding model.
|
||||
}
|
||||
|
||||
if args.query_type == "use_audio_in_video":
|
||||
extra_body["mm_processor_kwargs"] = {"use_audio_in_video": True}
|
||||
|
||||
if args.modalities is not None:
|
||||
output_modalities = args.modalities.split(",")
|
||||
else:
|
||||
@@ -411,6 +422,37 @@ def run_multimodal_generation(args, client: OpenAI) -> None:
|
||||
|
||||
# Test multiple concurrent completions
|
||||
num_concurrent_requests = args.num_concurrent_requests
|
||||
prompt_list = _parse_csv_arg(getattr(args, "prompts", None))
|
||||
speaker_list = _parse_csv_arg(getattr(args, "speakers", None))
|
||||
|
||||
request_payloads = []
|
||||
for idx in range(num_concurrent_requests):
|
||||
per_req_prompt = (
|
||||
prompt_list[idx]
|
||||
if idx < len(prompt_list)
|
||||
else (custom_prompt if idx == 0 or not prompt_list else prompt_list[-1])
|
||||
)
|
||||
per_req_speaker = (
|
||||
speaker_list[idx]
|
||||
if idx < len(speaker_list)
|
||||
else (args.speaker if idx == 0 or not speaker_list else speaker_list[-1])
|
||||
)
|
||||
prompt = _build_prompt_for_query_type(
|
||||
query_type=args.query_type,
|
||||
custom_prompt=per_req_prompt,
|
||||
video_path=video_path,
|
||||
image_path=image_path,
|
||||
audio_path=audio_path,
|
||||
)
|
||||
extra_body = {
|
||||
# Optional, it has default settings in stage configs.
|
||||
"sampling_params_list": sampling_params_list
|
||||
}
|
||||
if args.query_type == "use_audio_in_video":
|
||||
extra_body["mm_processor_kwargs"] = {"use_audio_in_video": True}
|
||||
if per_req_speaker and per_req_speaker.strip():
|
||||
extra_body["speaker"] = per_req_speaker.strip()
|
||||
request_payloads.append({"prompt": prompt, "extra_body": extra_body})
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=num_concurrent_requests) as executor:
|
||||
# Submit multiple completion requests concurrently
|
||||
@@ -419,14 +461,14 @@ def run_multimodal_generation(args, client: OpenAI) -> None:
|
||||
client.chat.completions.create,
|
||||
messages=[
|
||||
get_system_prompt(),
|
||||
prompt,
|
||||
payload["prompt"],
|
||||
],
|
||||
model=model_name,
|
||||
modalities=output_modalities,
|
||||
extra_body=extra_body,
|
||||
extra_body=payload["extra_body"],
|
||||
stream=args.stream,
|
||||
)
|
||||
for _ in range(num_concurrent_requests)
|
||||
for payload in request_payloads
|
||||
]
|
||||
|
||||
# Wait for all requests to complete and collect results
|
||||
@@ -437,10 +479,11 @@ def run_multimodal_generation(args, client: OpenAI) -> None:
|
||||
if not args.stream:
|
||||
# Verify all completions succeeded
|
||||
for chat_completion in chat_completions:
|
||||
request_id = getattr(chat_completion, "id", None)
|
||||
for choice in chat_completion.choices:
|
||||
if choice.message.audio:
|
||||
audio_data = base64.b64decode(choice.message.audio.data)
|
||||
audio_file_path = f"audio_{count}.wav"
|
||||
audio_file_path = make_audio_output_filename(request_id=request_id, index=count)
|
||||
with open(audio_file_path, "wb") as f:
|
||||
f.write(audio_data)
|
||||
print(f"Audio saved to {audio_file_path}")
|
||||
@@ -459,7 +502,8 @@ def run_multimodal_generation(args, client: OpenAI) -> None:
|
||||
|
||||
if getattr(chunk, "modality", None) == "audio" and content:
|
||||
audio_data = base64.b64decode(content)
|
||||
audio_file_path = f"audio_{count}.wav"
|
||||
request_id = getattr(chunk, "id", None)
|
||||
audio_file_path = make_audio_output_filename(request_id=request_id, index=count)
|
||||
with open(audio_file_path, "wb") as f:
|
||||
f.write(audio_data)
|
||||
print(f"\nAudio saved to {audio_file_path}")
|
||||
@@ -546,6 +590,30 @@ def parse_args():
|
||||
default="localhost",
|
||||
help="Host/IP of the vLLM Omni API server.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--speaker",
|
||||
type=str,
|
||||
default=None,
|
||||
help="TTS speaker/voice for audio output (e.g. Ethan, Vivian). Passed via extra_body to the talker stage.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--speakers",
|
||||
type=str,
|
||||
default=None,
|
||||
help=(
|
||||
"Comma-separated speakers for concurrent requests, e.g. "
|
||||
"'Ethan,Vivian,Ryan'. Overrides --speaker per request."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prompts",
|
||||
type=str,
|
||||
default=None,
|
||||
help=(
|
||||
"Comma-separated prompts for concurrent requests. "
|
||||
"If fewer than --num-concurrent-requests, the last prompt is reused."
|
||||
),
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
-459
@@ -1,459 +0,0 @@
|
||||
import base64
|
||||
import os
|
||||
|
||||
import requests
|
||||
from openai import OpenAI
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
SEED = 42
|
||||
|
||||
|
||||
def encode_base64_content_from_url(content_url: str) -> str:
|
||||
"""Encode a content retrieved from a remote url to base64 format."""
|
||||
|
||||
with requests.get(content_url) as response:
|
||||
response.raise_for_status()
|
||||
result = base64.b64encode(response.content).decode("utf-8")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def encode_base64_content_from_file(file_path: str) -> str:
|
||||
"""Encode a local file to base64 format."""
|
||||
with open(file_path, "rb") as f:
|
||||
content = f.read()
|
||||
result = base64.b64encode(content).decode("utf-8")
|
||||
return result
|
||||
|
||||
|
||||
def get_video_url_from_path(video_path: str | None) -> str:
|
||||
"""Convert a video path (local file or URL) to a video URL format for the API.
|
||||
|
||||
If video_path is None or empty, returns the default URL.
|
||||
If video_path is a local file path, encodes it to base64 data URL.
|
||||
If video_path is a URL, returns it as-is.
|
||||
"""
|
||||
if not video_path:
|
||||
# Default video URL
|
||||
return "https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4"
|
||||
|
||||
# Check if it's a URL (starts with http:// or https://)
|
||||
if video_path.startswith(("http://", "https://")):
|
||||
return video_path
|
||||
|
||||
# Otherwise, treat it as a local file path
|
||||
if not os.path.exists(video_path):
|
||||
raise FileNotFoundError(f"Video file not found: {video_path}")
|
||||
|
||||
# Detect video MIME type from file extension
|
||||
video_path_lower = video_path.lower()
|
||||
if video_path_lower.endswith(".mp4"):
|
||||
mime_type = "video/mp4"
|
||||
elif video_path_lower.endswith(".webm"):
|
||||
mime_type = "video/webm"
|
||||
elif video_path_lower.endswith(".mov"):
|
||||
mime_type = "video/quicktime"
|
||||
elif video_path_lower.endswith(".avi"):
|
||||
mime_type = "video/x-msvideo"
|
||||
elif video_path_lower.endswith(".mkv"):
|
||||
mime_type = "video/x-matroska"
|
||||
else:
|
||||
# Default to mp4 if extension is unknown
|
||||
mime_type = "video/mp4"
|
||||
|
||||
video_base64 = encode_base64_content_from_file(video_path)
|
||||
return f"data:{mime_type};base64,{video_base64}"
|
||||
|
||||
|
||||
def get_image_url_from_path(image_path: str | None) -> str:
|
||||
"""Convert an image path (local file or URL) to an image URL format for the API.
|
||||
|
||||
If image_path is None or empty, returns the default URL.
|
||||
If image_path is a local file path, encodes it to base64 data URL.
|
||||
If image_path is a URL, returns it as-is.
|
||||
"""
|
||||
if not image_path:
|
||||
# Default image URL
|
||||
return "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg"
|
||||
|
||||
# Check if it's a URL (starts with http:// or https://)
|
||||
if image_path.startswith(("http://", "https://")):
|
||||
return image_path
|
||||
|
||||
# Otherwise, treat it as a local file path
|
||||
if not os.path.exists(image_path):
|
||||
raise FileNotFoundError(f"Image file not found: {image_path}")
|
||||
|
||||
# Detect image MIME type from file extension
|
||||
image_path_lower = image_path.lower()
|
||||
if image_path_lower.endswith((".jpg", ".jpeg")):
|
||||
mime_type = "image/jpeg"
|
||||
elif image_path_lower.endswith(".png"):
|
||||
mime_type = "image/png"
|
||||
elif image_path_lower.endswith(".gif"):
|
||||
mime_type = "image/gif"
|
||||
elif image_path_lower.endswith(".webp"):
|
||||
mime_type = "image/webp"
|
||||
else:
|
||||
# Default to jpeg if extension is unknown
|
||||
mime_type = "image/jpeg"
|
||||
|
||||
image_base64 = encode_base64_content_from_file(image_path)
|
||||
return f"data:{mime_type};base64,{image_base64}"
|
||||
|
||||
|
||||
def get_audio_url_from_path(audio_path: str | None) -> str:
|
||||
"""Convert an audio path (local file or URL) to an audio URL format for the API.
|
||||
|
||||
If audio_path is None or empty, returns the default URL.
|
||||
If audio_path is a local file path, encodes it to base64 data URL.
|
||||
If audio_path is a URL, returns it as-is.
|
||||
"""
|
||||
if not audio_path:
|
||||
# Default audio URL
|
||||
return AudioAsset("mary_had_lamb").url
|
||||
|
||||
# Check if it's a URL (starts with http:// or https://)
|
||||
if audio_path.startswith(("http://", "https://")):
|
||||
return audio_path
|
||||
|
||||
# Otherwise, treat it as a local file path
|
||||
if not os.path.exists(audio_path):
|
||||
raise FileNotFoundError(f"Audio file not found: {audio_path}")
|
||||
|
||||
# Detect audio MIME type from file extension
|
||||
audio_path_lower = audio_path.lower()
|
||||
if audio_path_lower.endswith((".mp3", ".mpeg")):
|
||||
mime_type = "audio/mpeg"
|
||||
elif audio_path_lower.endswith(".wav"):
|
||||
mime_type = "audio/wav"
|
||||
elif audio_path_lower.endswith(".ogg"):
|
||||
mime_type = "audio/ogg"
|
||||
elif audio_path_lower.endswith(".flac"):
|
||||
mime_type = "audio/flac"
|
||||
elif audio_path_lower.endswith(".m4a"):
|
||||
mime_type = "audio/mp4"
|
||||
else:
|
||||
# Default to wav if extension is unknown
|
||||
mime_type = "audio/wav"
|
||||
|
||||
audio_base64 = encode_base64_content_from_file(audio_path)
|
||||
return f"data:{mime_type};base64,{audio_base64}"
|
||||
|
||||
|
||||
def get_system_prompt():
|
||||
return {
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": (
|
||||
"You are Qwen, a virtual human developed by the Qwen Team, "
|
||||
"Alibaba Group, capable of perceiving auditory and visual inputs, "
|
||||
"as well as generating text and speech."
|
||||
),
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def get_text_query(custom_prompt: str | None = None):
|
||||
question = (
|
||||
custom_prompt or "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words."
|
||||
)
|
||||
prompt = {
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": f"{question}",
|
||||
}
|
||||
],
|
||||
}
|
||||
return prompt
|
||||
|
||||
|
||||
def get_mixed_modalities_query(
|
||||
video_path: str | None = None,
|
||||
image_path: str | None = None,
|
||||
audio_path: str | None = None,
|
||||
custom_prompt: str | None = None,
|
||||
):
|
||||
question = (
|
||||
custom_prompt or "What is recited in the audio? What is the content of this image? Why is this video funny?"
|
||||
)
|
||||
video_url = get_video_url_from_path(video_path)
|
||||
image_url = get_image_url_from_path(image_path)
|
||||
audio_url = get_audio_url_from_path(audio_path)
|
||||
prompt = {
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "audio_url",
|
||||
"audio_url": {"url": audio_url},
|
||||
},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_url},
|
||||
},
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": {"url": video_url},
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": f"{question}",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
return prompt
|
||||
|
||||
|
||||
def get_use_audio_in_video_query(video_path: str | None = None, custom_prompt: str | None = None):
|
||||
question = custom_prompt or "Describe the content of the video, then convert what the baby say into text."
|
||||
video_url = get_video_url_from_path(video_path)
|
||||
|
||||
prompt = {
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": {
|
||||
"url": video_url,
|
||||
"num_frames": 16,
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": f"{question}",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
return prompt
|
||||
|
||||
|
||||
def get_multi_audios_query(audio_path: str | None = None, custom_prompt: str | None = None):
|
||||
question = custom_prompt or "Are these two audio clips the same?"
|
||||
audio_url = get_audio_url_from_path(audio_path)
|
||||
prompt = {
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "audio_url",
|
||||
"audio_url": {"url": audio_url},
|
||||
},
|
||||
{
|
||||
"type": "audio_url",
|
||||
"audio_url": {"url": AudioAsset("winning_call").url},
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": f"{question}",
|
||||
},
|
||||
],
|
||||
}
|
||||
return prompt
|
||||
|
||||
|
||||
query_map = {
|
||||
"mixed_modalities": get_mixed_modalities_query,
|
||||
"use_audio_in_video": get_use_audio_in_video_query,
|
||||
"multi_audios": get_multi_audios_query,
|
||||
"text": get_text_query,
|
||||
}
|
||||
|
||||
|
||||
def run_multimodal_generation(args, client: OpenAI) -> None:
|
||||
model_name = "Qwen/Qwen2.5-Omni-7B"
|
||||
thinker_sampling_params = {
|
||||
"temperature": 0.0, # Deterministic - no randomness
|
||||
"top_p": 1.0, # Disable nucleus sampling
|
||||
"top_k": -1, # Disable top-k sampling
|
||||
"max_tokens": 2048,
|
||||
"seed": SEED, # Fixed seed for sampling
|
||||
"detokenize": True,
|
||||
"repetition_penalty": 1.1,
|
||||
}
|
||||
talker_sampling_params = {
|
||||
"temperature": 0.9,
|
||||
"top_p": 0.8,
|
||||
"top_k": 40,
|
||||
"max_tokens": 2048,
|
||||
"seed": SEED, # Fixed seed for sampling
|
||||
"detokenize": True,
|
||||
"repetition_penalty": 1.05,
|
||||
"stop_token_ids": [8294],
|
||||
}
|
||||
code2wav_sampling_params = {
|
||||
"temperature": 0.0, # Deterministic - no randomness
|
||||
"top_p": 1.0, # Disable nucleus sampling
|
||||
"top_k": -1, # Disable top-k sampling
|
||||
"max_tokens": 2048,
|
||||
"seed": SEED, # Fixed seed for sampling
|
||||
"detokenize": True,
|
||||
"repetition_penalty": 1.1,
|
||||
}
|
||||
|
||||
sampling_params_list = [
|
||||
thinker_sampling_params,
|
||||
talker_sampling_params,
|
||||
code2wav_sampling_params,
|
||||
]
|
||||
|
||||
# Get paths and custom prompt from args
|
||||
video_path = getattr(args, "video_path", None)
|
||||
image_path = getattr(args, "image_path", None)
|
||||
audio_path = getattr(args, "audio_path", None)
|
||||
custom_prompt = getattr(args, "prompt", None)
|
||||
|
||||
# Get the query function and call it with appropriate parameters
|
||||
query_func = query_map[args.query_type]
|
||||
if args.query_type == "mixed_modalities":
|
||||
prompt = query_func(
|
||||
video_path=video_path, image_path=image_path, audio_path=audio_path, custom_prompt=custom_prompt
|
||||
)
|
||||
elif args.query_type == "use_audio_in_video":
|
||||
prompt = query_func(video_path=video_path, custom_prompt=custom_prompt)
|
||||
elif args.query_type == "multi_audios":
|
||||
prompt = query_func(audio_path=audio_path, custom_prompt=custom_prompt)
|
||||
elif args.query_type == "text":
|
||||
prompt = query_func(custom_prompt=custom_prompt)
|
||||
else:
|
||||
prompt = query_func()
|
||||
|
||||
extra_body = {
|
||||
"sampling_params_list": sampling_params_list # Optional, it has a default setting in stage_configs of the corresponding model.
|
||||
}
|
||||
|
||||
if args.query_type == "use_audio_in_video":
|
||||
extra_body["mm_processor_kwargs"] = {"use_audio_in_video": True}
|
||||
|
||||
if args.modalities is not None:
|
||||
output_modalities = args.modalities.split(",")
|
||||
else:
|
||||
output_modalities = None
|
||||
|
||||
chat_completion = client.chat.completions.create(
|
||||
messages=[
|
||||
get_system_prompt(),
|
||||
prompt,
|
||||
],
|
||||
model=model_name,
|
||||
modalities=output_modalities,
|
||||
extra_body=extra_body,
|
||||
stream=args.stream,
|
||||
)
|
||||
|
||||
count = 0
|
||||
if not args.stream:
|
||||
for choice in chat_completion.choices:
|
||||
if choice.message.audio:
|
||||
audio_data = base64.b64decode(choice.message.audio.data)
|
||||
audio_file_path = f"audio_{count}.wav"
|
||||
with open(audio_file_path, "wb") as f:
|
||||
f.write(audio_data)
|
||||
print(f"Audio saved to {audio_file_path}")
|
||||
count += 1
|
||||
elif choice.message.content:
|
||||
print("Chat completion output from text:", choice.message.content)
|
||||
else:
|
||||
printed_content = False
|
||||
for chunk in chat_completion:
|
||||
for choice in chunk.choices:
|
||||
if hasattr(choice, "delta"):
|
||||
content = getattr(choice.delta, "content", None)
|
||||
else:
|
||||
content = None
|
||||
|
||||
if getattr(chunk, "modality", None) == "audio" and content:
|
||||
audio_data = base64.b64decode(content)
|
||||
audio_file_path = f"audio_{count}.wav"
|
||||
with open(audio_file_path, "wb") as f:
|
||||
f.write(audio_data)
|
||||
print(f"\nAudio saved to {audio_file_path}")
|
||||
count += 1
|
||||
|
||||
elif getattr(chunk, "modality", None) == "text":
|
||||
if not printed_content:
|
||||
printed_content = True
|
||||
print("\ncontent:", end="", flush=True)
|
||||
print(content, end="", flush=True)
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = FlexibleArgumentParser(description="Demo on using vLLM for offline inference with audio language models")
|
||||
parser.add_argument(
|
||||
"--query-type",
|
||||
"-q",
|
||||
type=str,
|
||||
default="mixed_modalities",
|
||||
choices=query_map.keys(),
|
||||
help="Query type.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--video-path",
|
||||
"-v",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to local video file or URL. If not provided and query-type uses video, uses default video URL.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--image-path",
|
||||
"-i",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to local image file or URL. If not provided and query-type uses image, uses default image URL.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--audio-path",
|
||||
"-a",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to local audio file or URL. If not provided and query-type uses audio, uses default audio URL.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prompt",
|
||||
"-p",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Custom text prompt/question to use instead of the default prompt for the selected query type.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--modalities",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Output modalities to use for the prompts.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stream",
|
||||
action="store_true",
|
||||
help="Stream the response.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--port",
|
||||
type=int,
|
||||
default=8091,
|
||||
help="Port of the vLLM Omni API server.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--host",
|
||||
type=str,
|
||||
default="localhost",
|
||||
help="Host/IP of the vLLM Omni API server.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = parse_args()
|
||||
host = args.host
|
||||
port = args.port
|
||||
openai_api_base = f"http://{host}:{port}/v1"
|
||||
client = OpenAI(
|
||||
api_key="EMPTY",
|
||||
base_url=openai_api_base,
|
||||
)
|
||||
run_multimodal_generation(args, client)
|
||||
@@ -44,6 +44,7 @@ The Python client supports the following command-line arguments:
|
||||
- `--image-path` (or `-i`): Path to local image file or URL. If not provided and query-type is `use_image`, uses default image URL. Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs and common image formats: JPEG, PNG, GIF, WebP. Example: `--image-path /path/to/image.jpg` or `--image-path https://example.com/image.png`
|
||||
- `--audio-path` (or `-a`): Path to local audio file or URL. If not provided and query-type is `use_audio`, uses default audio URL. Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs and common audio formats: MP3, WAV, OGG, FLAC, M4A. Example: `--audio-path /path/to/audio.wav` or `--audio-path https://example.com/audio.mp3`
|
||||
- `--prompt` (or `-p`): Custom text prompt/question. If not provided, uses default prompt for the selected query type. Example: `--prompt "What are the main activities shown in this video?"`
|
||||
- `--speaker`: TTS speaker/voice for audio output when requesting audio (e.g. `ethan`, `chelsie`, `aiden`). Omit to use the model default. Example: `--speaker "chelsie"`
|
||||
|
||||
|
||||
For example, to use a local video file with custom prompt:
|
||||
@@ -150,6 +151,57 @@ print(response.choices[0].message.content) # Text response
|
||||
print(response.choices[1].message.audio) # Audio response
|
||||
```
|
||||
|
||||
## Speaker selection
|
||||
|
||||
When requesting audio output, you can choose the TTS speaker (voice) used for synthesis. If not specified, the model uses its default speaker.
|
||||
|
||||
### Using curl
|
||||
|
||||
Pass a `speaker` field in the request body:
|
||||
|
||||
```bash
|
||||
curl http://localhost:8091/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"messages": [{"role": "user", "content": "Say hello in one sentence."}],
|
||||
"modalities": ["audio"],
|
||||
"speaker": "chelsie"
|
||||
}'
|
||||
```
|
||||
|
||||
### Using Python client
|
||||
|
||||
Use the `--speaker` argument when generating audio:
|
||||
|
||||
```bash
|
||||
python openai_chat_completion_client_for_multimodal_generation.py \
|
||||
--query-type use_image \
|
||||
--modalities audio \
|
||||
--speaker "chelsie"
|
||||
```
|
||||
|
||||
### Using OpenAI Python SDK
|
||||
|
||||
Pass `speaker` in `extra_body`:
|
||||
|
||||
```python
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(base_url="http://localhost:8091/v1", api_key="EMPTY")
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
messages=[{"role": "user", "content": "Say hello in one sentence."}],
|
||||
modalities=["audio"],
|
||||
extra_body={"speaker": "chelsie"}
|
||||
)
|
||||
# Audio uses the specified speaker
|
||||
print(response.choices[1].message.audio)
|
||||
```
|
||||
|
||||
Supported speaker names depend on the model (e.g. `Ethan`, `Chelsie`, `Aiden`). Omit `speaker` to use the default.
|
||||
|
||||
## Streaming Output
|
||||
If you want to enable streaming output, please set the argument as below. The final output will be obtained just after generated by corresponding stage. Now we only support text streaming output. Other modalities can output normally.
|
||||
```bash
|
||||
|
||||
@@ -65,7 +65,7 @@ def run_tts_generation(args) -> None:
|
||||
payload = {
|
||||
"model": args.model,
|
||||
"input": args.text,
|
||||
"voice": args.voice,
|
||||
"speaker": args.speaker,
|
||||
"response_format": args.response_format,
|
||||
}
|
||||
|
||||
@@ -95,7 +95,7 @@ def run_tts_generation(args) -> None:
|
||||
print(f"Model: {args.model}")
|
||||
print(f"Task type: {args.task_type or 'CustomVoice'}")
|
||||
print(f"Text: {args.text}")
|
||||
print(f"Voice: {args.voice}")
|
||||
print(f"Speaker: {args.speaker}")
|
||||
print("Generating audio...")
|
||||
|
||||
# Make the API call
|
||||
@@ -178,10 +178,10 @@ def parse_args():
|
||||
|
||||
# Voice/speaker
|
||||
parser.add_argument(
|
||||
"--voice",
|
||||
"--speaker",
|
||||
type=str,
|
||||
default="vivian",
|
||||
help="Speaker/voice name (default: vivian). Options: vivian, ryan, aiden, etc.",
|
||||
help="Speaker name (default: vivian). Options: vivian, ryan, aiden, etc.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--language",
|
||||
|
||||
@@ -163,7 +163,7 @@ def main():
|
||||
|
||||
# Session config options
|
||||
parser.add_argument("--model", default=None, help="Model name")
|
||||
parser.add_argument("--voice", default="Vivian", help="Speaker voice")
|
||||
parser.add_argument("--speaker", default="Vivian", help="Speaker name")
|
||||
parser.add_argument(
|
||||
"--task-type",
|
||||
default="CustomVoice",
|
||||
@@ -215,7 +215,7 @@ def main():
|
||||
config = {}
|
||||
for key in [
|
||||
"model",
|
||||
"voice",
|
||||
"speaker",
|
||||
"task_type",
|
||||
"language",
|
||||
"instructions",
|
||||
|
||||
@@ -100,7 +100,7 @@ def build_payload(
|
||||
|
||||
if task_type == "CustomVoice":
|
||||
if voice:
|
||||
payload["voice"] = voice
|
||||
payload["speaker"] = voice
|
||||
if instructions and instructions.strip():
|
||||
payload["instructions"] = instructions.strip()
|
||||
|
||||
|
||||
Reference in new issue
Block a user