[Test] L2 & L3 Test Case Stratification Design for Omni Model (#1272)
Signed-off-by: wangyu31577 <wangyu31577@hundsun.com> Signed-off-by: yenuo26 <410167048@qq.com> Signed-off-by: wangyu <53896905+yenuo26@users.noreply.github.com> Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: wangyu31577 <wangyu31577@hundsun.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
20 files changed
+1193
-1183
No files matched your search
@@ -1,103 +0,0 @@
|
||||
# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
|
||||
# Stage 0: Thinker (multimodal understanding + text generation)
|
||||
# Stage 1: Talker (text embeddings → 16-layer RVQ codec codes)
|
||||
# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
|
||||
|
||||
# The following config has been verified on 2x H100-80G GPUs.
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
stage_type: llm # Use llm stage type to launch OmniLLM
|
||||
runtime:
|
||||
devices: "0"
|
||||
max_batch_size: 5
|
||||
engine_args:
|
||||
model_stage: thinker
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.9
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output hidden states for talker
|
||||
distributed_executor_backend: "mp"
|
||||
enable_prefix_caching: false
|
||||
max_num_batched_tokens: 32768
|
||||
hf_config_name: thinker_config
|
||||
tensor_parallel_size: 1
|
||||
load_format: dummy
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
is_comprehension: true
|
||||
default_sampling_params:
|
||||
temperature: 0.4
|
||||
top_p: 0.9
|
||||
top_k: 1
|
||||
max_tokens: 100
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.05
|
||||
|
||||
- stage_id: 1
|
||||
stage_type: llm # Use llm stage type to launch OmniLLM
|
||||
runtime:
|
||||
devices: "1"
|
||||
max_batch_size: 5
|
||||
engine_args:
|
||||
model_stage: talker
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.6
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output codec codes for code2wav
|
||||
# tensor_parallel_size: 2
|
||||
enable_prefix_caching: false
|
||||
distributed_executor_backend: "mp"
|
||||
hf_config_name: talker_config
|
||||
load_format: dummy
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
|
||||
# final_output: true
|
||||
# final_output_type: text
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_k: 50
|
||||
max_tokens: 100
|
||||
seed: 42
|
||||
detokenize: False
|
||||
repetition_penalty: 1.05
|
||||
stop_token_ids: [2150]
|
||||
|
||||
- stage_id: 2
|
||||
stage_type: llm # Use llm stage type to launch OmniLLM
|
||||
runtime:
|
||||
devices: "1"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: code2wav
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
async_scheduling: false
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio # Final output: audio waveform
|
||||
gpu_memory_utilization: 0.1
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 1000000
|
||||
hf_config_name: thinker_config
|
||||
load_format: dummy
|
||||
engine_input_source: [1]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 200
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
@@ -1,31 +0,0 @@
|
||||
# The following config has been verified on 2x H100-80G GPUs.
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
runtime:
|
||||
devices: "0,1"
|
||||
max_batch_size: 5
|
||||
engine_args:
|
||||
model_stage: thinker
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.6
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output hidden states for talker
|
||||
distributed_executor_backend: "mp"
|
||||
enable_prefix_caching: false
|
||||
hf_config_name: thinker_config
|
||||
tensor_parallel_size: 2
|
||||
load_format: dummy
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
is_comprehension: true
|
||||
default_sampling_params:
|
||||
temperature: 0.4
|
||||
top_p: 0.9
|
||||
top_k: 1
|
||||
max_tokens: 100
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.05
|
||||
@@ -1,95 +0,0 @@
|
||||
# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
|
||||
# Stage 0: Thinker (multimodal understanding + text generation)
|
||||
# Stage 1: Talker (text embeddings → 16-layer RVQ codec codes)
|
||||
# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
|
||||
# The following config has been verified on 2x H100-80G GPUs.
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
runtime:
|
||||
devices: "0"
|
||||
max_batch_size: 5
|
||||
engine_args:
|
||||
model_stage: thinker
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.9
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output hidden states for talker
|
||||
distributed_executor_backend: "mp"
|
||||
enable_prefix_caching: false
|
||||
hf_config_name: thinker_config
|
||||
tensor_parallel_size: 1
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
is_comprehension: true
|
||||
default_sampling_params:
|
||||
temperature: 0.4
|
||||
top_p: 0.9
|
||||
top_k: 1
|
||||
max_tokens: 100
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.05
|
||||
|
||||
- stage_id: 1
|
||||
runtime:
|
||||
devices: "1"
|
||||
max_batch_size: 5
|
||||
engine_args:
|
||||
model_stage: talker
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.6
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output codec codes for code2wav
|
||||
# tensor_parallel_size: 2
|
||||
enable_prefix_caching: false
|
||||
distributed_executor_backend: "mp"
|
||||
hf_config_name: talker_config
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
|
||||
# final_output: true
|
||||
# final_output_type: text
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_k: 50
|
||||
max_tokens: 1000
|
||||
seed: 42
|
||||
detokenize: False
|
||||
repetition_penalty: 1.05
|
||||
stop_token_ids: [2150]
|
||||
|
||||
- stage_id: 2
|
||||
runtime:
|
||||
devices: "1"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: code2wav
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio # Final output: audio waveform
|
||||
gpu_memory_utilization: 0.1
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 1000000
|
||||
hf_config_name: thinker_config
|
||||
async_scheduling: false
|
||||
engine_input_source: [1]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 2000
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
@@ -1,117 +0,0 @@
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
from contextlib import ExitStack
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from vllm import SamplingParams
|
||||
from vllm.inputs import PromptType
|
||||
|
||||
from tests.utils import hardware_test
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
SEED = 42
|
||||
|
||||
stage_config = str(Path(__file__).parent / "stage_configs" / "qwen3_omni_thinker_ci.yaml")
|
||||
model = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
|
||||
|
||||
|
||||
async def generate(
|
||||
engine: AsyncOmni,
|
||||
request_id: str,
|
||||
prompt: PromptType,
|
||||
max_tokens: int,
|
||||
) -> tuple[int, str]:
|
||||
# Ensure generate doesn't complete too fast for cancellation test.
|
||||
await asyncio.sleep(0.2)
|
||||
thinker_sampling_params = SamplingParams(
|
||||
temperature=0.4, # Deterministic
|
||||
top_p=0.9,
|
||||
top_k=1,
|
||||
max_tokens=max_tokens,
|
||||
repetition_penalty=1.05,
|
||||
stop_token_ids=[151645], # Qwen EOS token <|im_end|>
|
||||
seed=SEED,
|
||||
)
|
||||
|
||||
sampling_params_list = [
|
||||
thinker_sampling_params,
|
||||
]
|
||||
count = 0
|
||||
async for omni_output in engine.generate(
|
||||
prompt=prompt,
|
||||
request_id=request_id,
|
||||
sampling_params_list=sampling_params_list,
|
||||
output_modalities=["text"],
|
||||
):
|
||||
stage_id = omni_output.stage_id
|
||||
out = omni_output.request_output
|
||||
if stage_id == 0:
|
||||
num_tokens = sum(len(output.token_ids) for output in out.outputs)
|
||||
count = num_tokens
|
||||
|
||||
await asyncio.sleep(0.0)
|
||||
|
||||
return count, request_id
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@pytest.mark.asyncio
|
||||
async def test_abort():
|
||||
with ExitStack() as after:
|
||||
# Avoid SHM IPC in tests to prevent /dev/shm exhaustion and SIGBUS.
|
||||
engine = AsyncOmni(
|
||||
model=model,
|
||||
stage_configs_path=stage_config,
|
||||
shm_threshold_bytes=sys.maxsize,
|
||||
)
|
||||
after.callback(engine.shutdown)
|
||||
|
||||
# Keep token counts modest to reduce flakiness on slow test hardware.
|
||||
NUM_REQUESTS = 3
|
||||
NUM_EXPECTED_TOKENS = 64
|
||||
NUM_EXPECTED_TOKENS_LONG = 256
|
||||
REQUEST_IDS_TO_ABORT = [1]
|
||||
|
||||
prompt = "Hello my name is Robert and "
|
||||
|
||||
request_ids = [f"request-{i}" for i in range(NUM_REQUESTS)]
|
||||
|
||||
# Create concurrent requests.
|
||||
tasks: list[asyncio.Task] = []
|
||||
for idx, request_id in enumerate(request_ids):
|
||||
max_tokens = NUM_EXPECTED_TOKENS_LONG if (idx in REQUEST_IDS_TO_ABORT) else NUM_EXPECTED_TOKENS
|
||||
tasks.append(asyncio.create_task(generate(engine, request_id, prompt, max_tokens)))
|
||||
|
||||
# API server cancels requests when they disconnect.
|
||||
# Explicitly abort in the engine to avoid orphaned requests hanging.
|
||||
for idx in REQUEST_IDS_TO_ABORT:
|
||||
tasks[idx].cancel()
|
||||
await engine.abort(request_ids[idx])
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
# Confirm the other requests are okay.
|
||||
for idx, task in enumerate(tasks):
|
||||
# Confirm that it was actually canceled.
|
||||
if idx in REQUEST_IDS_TO_ABORT:
|
||||
with pytest.raises((asyncio.CancelledError, GeneratorExit)):
|
||||
await asyncio.wait_for(task, timeout=60)
|
||||
else:
|
||||
# Otherwise, make sure the request was not impacted.
|
||||
num_generated_tokens, request_id = await asyncio.wait_for(task, timeout=180)
|
||||
expected_tokens = NUM_EXPECTED_TOKENS
|
||||
assert num_generated_tokens == expected_tokens, (
|
||||
f"{request_id} generated {num_generated_tokens} but expected {expected_tokens}"
|
||||
)
|
||||
|
||||
# Confirm we can do another generation.
|
||||
request_id = f"request-{REQUEST_IDS_TO_ABORT[0]}"
|
||||
task = asyncio.create_task(generate(engine, request_id, prompt, NUM_EXPECTED_TOKENS))
|
||||
num_generated_tokens, request_id = await task
|
||||
assert num_generated_tokens == NUM_EXPECTED_TOKENS
|
||||
await asyncio.sleep(5)
|
||||
@@ -1,5 +1,3 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
E2E Online tests for Qwen3-Omni model with video input and audio output.
|
||||
"""
|
||||
@@ -9,23 +7,15 @@ import os
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
import concurrent.futures
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
|
||||
from tests.conftest import (
|
||||
OmniServer,
|
||||
convert_audio_to_text,
|
||||
cosine_similarity_text,
|
||||
dummy_messages_from_mix_data,
|
||||
generate_synthetic_audio,
|
||||
generate_synthetic_image,
|
||||
generate_synthetic_video,
|
||||
merge_base64_and_convert_to_text,
|
||||
modify_stage_config,
|
||||
)
|
||||
from tests.utils import hardware_test
|
||||
@@ -34,13 +24,9 @@ from vllm_omni.platforms import current_omni_platform
|
||||
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
|
||||
|
||||
|
||||
def get_default_config():
|
||||
return str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml")
|
||||
|
||||
|
||||
def get_chunk_config():
|
||||
path = modify_stage_config(
|
||||
get_default_config(),
|
||||
str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml"),
|
||||
updates={
|
||||
"async_chunk": True,
|
||||
"stage_args": {
|
||||
@@ -57,49 +43,17 @@ def get_chunk_config():
|
||||
return path
|
||||
|
||||
|
||||
CHUNK_CONFIG_PATH = get_chunk_config()
|
||||
# CI stage config for 2xH100-80G GPUs or AMD GPU MI325
|
||||
if current_omni_platform.is_rocm():
|
||||
# ROCm stage config optimized for MI325 GPU
|
||||
stage_configs = [str(Path(__file__).parent / "stage_configs" / "rocm" / "qwen3_omni_ci.yaml")]
|
||||
stage_configs = [str(Path(__file__).parent.parent / "stage_configs" / "rocm" / "qwen3_omni_ci.yaml")]
|
||||
else:
|
||||
stage_configs = [get_default_config(), CHUNK_CONFIG_PATH]
|
||||
stage_configs = [get_chunk_config()]
|
||||
|
||||
# Create parameter combinations for model and stage config
|
||||
test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
|
||||
|
||||
|
||||
_omni_server_lock = threading.Lock()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def omni_server(request):
|
||||
"""Start vLLM-Omni server as a subprocess with actual model weights.
|
||||
Uses session scope so the server starts only once for the entire test session.
|
||||
Multi-stage initialization can take 10-20+ minutes.
|
||||
"""
|
||||
with _omni_server_lock:
|
||||
model, stage_config_path = request.param
|
||||
|
||||
print(f"Starting OmniServer with model: {model}")
|
||||
|
||||
with OmniServer(model, ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "120"]) as server:
|
||||
print("OmniServer started successfully")
|
||||
yield server
|
||||
print("OmniServer stopping...")
|
||||
|
||||
print("OmniServer stopped")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(omni_server):
|
||||
"""OpenAI client for the running vLLM-Omni server."""
|
||||
return openai.OpenAI(
|
||||
base_url=f"http://{omni_server.host}:{omni_server.port}/v1",
|
||||
api_key="EMPTY",
|
||||
)
|
||||
|
||||
|
||||
def get_system_prompt():
|
||||
return {
|
||||
"role": "system",
|
||||
@@ -116,23 +70,6 @@ def get_system_prompt():
|
||||
}
|
||||
|
||||
|
||||
def dummy_messages_from_video_data(
|
||||
video_data_url: str,
|
||||
content_text: str = "Describe the video briefly.",
|
||||
):
|
||||
"""Create messages with video data URL for OpenAI API."""
|
||||
return [
|
||||
get_system_prompt(),
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "video_url", "video_url": {"url": video_data_url}},
|
||||
{"type": "text", "text": content_text},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def get_prompt(prompt_type="text_only"):
|
||||
prompts = {
|
||||
"text_only": "What is the capital of China? Answer in 20 words.",
|
||||
@@ -146,11 +83,12 @@ def get_max_batch_size(size_type="few"):
|
||||
return batch_sizes.get(size_type, 5)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server, request) -> None:
|
||||
def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test multi-modal input processing and text/audio output generation via OpenAI API.
|
||||
Deploy Setting: default yaml
|
||||
@@ -160,8 +98,6 @@ def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server, request) -> N
|
||||
Datasets: single request
|
||||
"""
|
||||
|
||||
# Test single completion
|
||||
e2e_list = list()
|
||||
video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(224, 224, 300)['base64']}"
|
||||
image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(224, 224)['base64']}"
|
||||
audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(5, 1)['base64']}"
|
||||
@@ -173,57 +109,25 @@ def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server, request) -> N
|
||||
content_text=get_prompt("mix"),
|
||||
)
|
||||
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"stream": True,
|
||||
"key_words": {
|
||||
"audio": ["water", "chirping"],
|
||||
"image": ["square", "quadrate"],
|
||||
},
|
||||
}
|
||||
|
||||
# Test single completion
|
||||
start_time = time.perf_counter()
|
||||
chat_completion = client.chat.completions.create(model=omni_server.model, messages=messages, stream=True)
|
||||
|
||||
text_content = ""
|
||||
audio_data = []
|
||||
for chunk in chat_completion:
|
||||
for choice in chunk.choices:
|
||||
if hasattr(choice, "delta"):
|
||||
content = getattr(choice.delta, "content", None)
|
||||
else:
|
||||
content = None
|
||||
|
||||
modality = getattr(chunk, "modality", None)
|
||||
|
||||
if modality == "audio" and content:
|
||||
audio_data.append(content)
|
||||
elif modality == "text" and content:
|
||||
# Text chunk - accumulate text content
|
||||
text_content += content if content else ""
|
||||
|
||||
# Verify E2E
|
||||
current_e2e = time.perf_counter() - start_time
|
||||
print(f"the request e2e is: {current_e2e}")
|
||||
# TODO: Verify the E2E latency after confirmation baseline.
|
||||
e2e_list.append(current_e2e)
|
||||
|
||||
print(f"the avg e2e is: {sum(e2e_list) / len(e2e_list)}")
|
||||
# Verify all completions succeeded
|
||||
assert audio_data is not None, "No audio output is generated"
|
||||
|
||||
# Verify text output success
|
||||
assert text_content is not None and len(text_content) >= 2, "No text output is generated"
|
||||
assert any(
|
||||
keyword in text_content.lower() for keyword in ["square", "quadrate", "sphere", "globe", "circle", "round"]
|
||||
), "The output does not contain any of the keywords."
|
||||
|
||||
# Verify text output same as audio output
|
||||
audio_content = merge_base64_and_convert_to_text(audio_data)
|
||||
print(f"text content is: {text_content}")
|
||||
print(f"audio content is: {audio_content}")
|
||||
similarity = cosine_similarity_text(audio_content.lower(), text_content.lower())
|
||||
print(f"similarity is: {similarity}")
|
||||
assert similarity > 0.9, "The audio content is not same as the text"
|
||||
openai_client.send_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_text_to_text_audio_001(client: openai.OpenAI, omni_server) -> None:
|
||||
def test_text_to_text_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test text input processing and text/audio output generation via OpenAI API.
|
||||
Deploy Setting: default yaml
|
||||
@@ -231,50 +135,14 @@ def test_text_to_text_audio_001(client: openai.OpenAI, omni_server) -> None:
|
||||
Output Modal: text + audio
|
||||
Datasets: few requests
|
||||
"""
|
||||
|
||||
num_concurrent_requests = get_max_batch_size()
|
||||
messages = dummy_messages_from_mix_data(system_prompt=get_system_prompt(), content_text=get_prompt())
|
||||
|
||||
e2e_list = list()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=num_concurrent_requests) as executor:
|
||||
# Submit multiple completion requests concurrently
|
||||
futures = [
|
||||
executor.submit(client.chat.completions.create, model=omni_server.model, messages=messages)
|
||||
for _ in range(num_concurrent_requests)
|
||||
]
|
||||
start_time = time.perf_counter()
|
||||
# Wait for all requests to complete and collect results
|
||||
chat_completions = list()
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
chat_completions.append(future.result())
|
||||
# Verify E2E
|
||||
current_e2e = time.perf_counter() - start_time
|
||||
print(f"the request e2e is: {current_e2e}")
|
||||
# TODO: Verify the E2E latency after confirmation baseline.
|
||||
e2e_list.append(current_e2e)
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
"modalities": ["text"],
|
||||
"key_words": {"text": ["beijing"]},
|
||||
}
|
||||
|
||||
print(f"the avg e2e is: {sum(e2e_list) / len(e2e_list)}")
|
||||
# Verify all completions succeeded
|
||||
assert len(chat_completions) == num_concurrent_requests, "Not all requests succeeded."
|
||||
for chat_completion in chat_completions:
|
||||
# Verify audio output success
|
||||
audio_data = None
|
||||
text_content = None
|
||||
for choice in chat_completion.choices:
|
||||
if choice.message.audio is not None:
|
||||
audio_message = choice.message
|
||||
audio_data = audio_message.audio.data
|
||||
assert audio_message.audio.expires_at > time.time(), "The generated audio has expired."
|
||||
|
||||
if choice.message.content is not None:
|
||||
# Verify text output success
|
||||
text_content = choice.message.content
|
||||
assert "beijing" in text_content.lower(), "The output do not contain keywords."
|
||||
|
||||
# Verify text output same as audio output
|
||||
audio_content = convert_audio_to_text(audio_data)
|
||||
print(f"text content is: {text_content}")
|
||||
print(f"audio content is: {audio_content}")
|
||||
similarity = cosine_similarity_text(audio_content.lower(), text_content.lower())
|
||||
print(f"similarity is: {similarity}")
|
||||
assert similarity > 0.9, "The audio content is not same as the text"
|
||||
openai_client.send_request(request_config, request_num=get_max_batch_size())
|
||||
Reference in new issue
Block a user