[Debug] Debug qwen3 mix modality output empty string (#431)
Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
1 parent
099a36dfdc
commit
59ae174bba
5 files changed
+127
-10
No files matched your search
@@ -143,11 +143,61 @@ def get_audio_query(question: str = None, audio_path: str | None = None, samplin
|
||||
)
|
||||
|
||||
|
||||
def get_mixed_modalities_query() -> QueryResult:
|
||||
question = "What is recited in the audio? What is the content of this image? Why is this video funny?"
|
||||
prompt = (
|
||||
f"<|im_start|>system\n{default_system}<|im_end|>\n"
|
||||
"<|im_start|>user\n<|audio_start|><|audio_pad|><|audio_end|>"
|
||||
"<|vision_start|><|image_pad|><|vision_end|>"
|
||||
"<|vision_start|><|video_pad|><|vision_end|>"
|
||||
f"{question}<|im_end|>\n"
|
||||
f"<|im_start|>assistant\n"
|
||||
)
|
||||
return QueryResult(
|
||||
inputs={
|
||||
"prompt": prompt,
|
||||
"multi_modal_data": {
|
||||
"audio": AudioAsset("mary_had_lamb").audio_and_sample_rate,
|
||||
"image": convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB"),
|
||||
"video": VideoAsset(name="baby_reading", num_frames=16).np_ndarrays,
|
||||
},
|
||||
},
|
||||
limit_mm_per_prompt={"audio": 1, "image": 1, "video": 1},
|
||||
)
|
||||
|
||||
|
||||
def get_multi_audios_query() -> QueryResult:
|
||||
question = "Are these two audio clips the same?"
|
||||
prompt = (
|
||||
f"<|im_start|>system\n{default_system}<|im_end|>\n"
|
||||
"<|im_start|>user\n<|audio_start|><|audio_pad|><|audio_end|>"
|
||||
"<|audio_start|><|audio_pad|><|audio_end|>"
|
||||
f"{question}<|im_end|>\n"
|
||||
f"<|im_start|>assistant\n"
|
||||
)
|
||||
return QueryResult(
|
||||
inputs={
|
||||
"prompt": prompt,
|
||||
"multi_modal_data": {
|
||||
"audio": [
|
||||
AudioAsset("winning_call").audio_and_sample_rate,
|
||||
AudioAsset("mary_had_lamb").audio_and_sample_rate,
|
||||
],
|
||||
},
|
||||
},
|
||||
limit_mm_per_prompt={
|
||||
"audio": 2,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
query_map = {
|
||||
"text": get_text_query,
|
||||
"use_audio": get_audio_query,
|
||||
"use_image": get_image_query,
|
||||
"use_video": get_video_query,
|
||||
"use_mixed_modalities": get_mixed_modalities_query,
|
||||
"use_multi_audios": get_multi_audios_query,
|
||||
}
|
||||
|
||||
|
||||
@@ -276,7 +326,7 @@ def parse_args():
|
||||
"--query-type",
|
||||
"-q",
|
||||
type=str,
|
||||
default="use_video",
|
||||
default="use_mixed_modalities",
|
||||
choices=query_map.keys(),
|
||||
help="Query type.",
|
||||
)
|
||||
|
||||
+54
-1
@@ -254,11 +254,64 @@ def get_audio_query(audio_path: str | None = None, custom_prompt: str | None = N
|
||||
return prompt
|
||||
|
||||
|
||||
def get_mixed_modalities_query(
|
||||
video_path: str | None = None,
|
||||
image_path: str | None = None,
|
||||
audio_path: str | None = None,
|
||||
custom_prompt: str | None = None,
|
||||
):
|
||||
"""
|
||||
Online-friendly multimodal user message:
|
||||
- Uses URLs (or base64 data URLs) for audio / image / video.
|
||||
- Returns the OpenAI-style message dict directly (not the offline QueryResult).
|
||||
"""
|
||||
question = (
|
||||
custom_prompt or "What is recited in the audio? What is the content of this image? Why is this video funny?"
|
||||
)
|
||||
|
||||
audio_url = get_audio_url_from_path(audio_path)
|
||||
image_url = get_image_url_from_path(image_path)
|
||||
video_url = get_video_url_from_path(video_path)
|
||||
|
||||
return {
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "audio_url", "audio_url": {"url": audio_url}},
|
||||
{"type": "image_url", "image_url": {"url": image_url}},
|
||||
{"type": "video_url", "video_url": {"url": video_url}},
|
||||
{"type": "text", "text": question},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def get_multi_audios_query(custom_prompt: str | None = None):
|
||||
"""
|
||||
Online-friendly two-audio comparison request.
|
||||
- Encodes both audio clips as URLs (or data URLs).
|
||||
- Returns the OpenAI-style message dict.
|
||||
"""
|
||||
question = custom_prompt or "Are these two audio clips the same?"
|
||||
# Use default demo clips; you can point to your own via --audio-path if needed.
|
||||
audio_url_1 = get_audio_url_from_path(AudioAsset("winning_call").url)
|
||||
audio_url_2 = get_audio_url_from_path(AudioAsset("mary_had_lamb").url)
|
||||
|
||||
return {
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "audio_url", "audio_url": {"url": audio_url_1}},
|
||||
{"type": "audio_url", "audio_url": {"url": audio_url_2}},
|
||||
{"type": "text", "text": question},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
query_map = {
|
||||
"text": get_text_query,
|
||||
"use_audio": get_audio_query,
|
||||
"use_image": get_image_query,
|
||||
"use_video": get_video_query,
|
||||
"use_mixed_modalities": get_mixed_modalities_query,
|
||||
"use_multi_audios": get_multi_audios_query,
|
||||
}
|
||||
|
||||
|
||||
@@ -363,7 +416,7 @@ def parse_args():
|
||||
"--query-type",
|
||||
"-q",
|
||||
type=str,
|
||||
default="use_video",
|
||||
default="use_mixed_modalities",
|
||||
choices=query_map.keys(),
|
||||
help="Query type.",
|
||||
)
|
||||
|
||||
Reference in new issue
Block a user