[CI] Update test markers and configurations to use 'full_model' for L4 nightly tests (#2641)

Signed-off-by: wangyu <410167048@qq.com>
This commit is contained in:
wangyu authored and GitHub committed 2026-04-22 15:14:20 +08:00
1 parent 554233208f
commit e18cb89d5a
50 files changed
+173 -322

No files matched your search

+27 -11
View File
@@ -11,7 +11,9 @@ import numpy as np
import soundfile as sf
from PIL import Image
from tests.helpers.media import cosine_similarity_text
from tests.helpers.media import (
cosine_similarity_text,
)
_GENDER_PIPELINE = None
_GENDER_PIPELINE_LOCK = threading.Lock()
@@ -56,7 +58,7 @@ def assert_image_diffusion_response(
f"Expected {num_outputs_per_prompt} images, got {len(response.images)}"
)
if run_level == "advanced_model":
if run_level in {"advanced_model", "full_model"}:
width = extra_body.get("width")
height = extra_body.get("height")
@@ -407,7 +409,8 @@ def assert_omni_response(response: Any, request_config: dict[str, Any], run_leve
modalities = request_config.get("modalities", ["text", "audio"])
if run_level == "advanced_model":
if run_level in {"advanced_model", "full_model"}:
# Verify output success
if "audio" in modalities:
assert response.audio_content is not None, "No audio output is generated"
print(f"audio content is: {response.audio_content}")
@@ -417,12 +420,11 @@ def assert_omni_response(response: Any, request_config: dict[str, Any], run_leve
response.audio_bytes,
speaker,
)
if "text" in modalities:
assert response.text_content is not None, "No text output is generated"
print(f"text content is: {response.text_content}")
# Verify image description
# Verify keywords in output
word_types = ["text", "image", "audio", "video"]
keywords_dict = request_config.get("key_words", {})
for word_type in word_types:
@@ -441,11 +443,25 @@ def assert_omni_response(response: Any, request_config: dict[str, Any], run_leve
)
# Verify similarity (Whisper transcript vs streamed/detokenized text)
if "text" in modalities and "audio" in modalities:
assert response.similarity is not None and response.similarity > 0.9, (
"The audio content is not same as the text"
)
print(f"similarity is: {response.similarity}")
if "audio" in modalities:
audio_ref_text = request_config.get("audio_ref_text")
if "text" in modalities:
transcript = (response.audio_content or "").strip()
text_output = (response.text_content or "").strip()
similarity = cosine_similarity_text(
transcript.lower(),
text_output.lower(),
)
assert similarity > 0.9, "The audio content is not same as the text"
print(f"similarity is: {similarity}")
if audio_ref_text:
audio_similarity = cosine_similarity_text(
response.audio_content.lower(),
str(audio_ref_text).lower(),
)
assert audio_similarity > 0.9, (
f"The audio content does not match reference text: similarity={audio_similarity:.3f}"
)
def assert_audio_speech_response(response: Any, request_config: dict[str, Any], run_level: str) -> None:
@@ -464,7 +480,7 @@ def assert_audio_speech_response(response: Any, request_config: dict[str, Any],
elif req_fmt == "wav" and response.audio_format:
assert req_fmt in response.audio_format
if run_level == "advanced_model" and req_fmt != "pcm":
if run_level in {"advanced_model", "full_model"} and req_fmt != "pcm":
expected_text = request_config.get("input")
if expected_text:
transcript = (response.audio_content or "").strip()
+2 -2
View File
@@ -6,8 +6,8 @@ def pytest_addoption(parser):
"--run-level",
action="store",
default="core_model",
choices=["core_model", "advanced_model"],
help="Test level to run: L2, L3",
choices=["core_model", "advanced_model", "full_model"],
help="Test level to run: L2, L3, L4",
)
+1 -1
View File
@@ -36,7 +36,7 @@ def omni_server(request: pytest.FixtureRequest, run_level: str, model_prefix: st
model = model_prefix + params.model
port = params.port
stage_config_path = params.stage_config_path
if run_level == "advanced_model" and stage_config_path is not None:
if run_level in {"advanced_model", "full_model"} and stage_config_path is not None:
with open(stage_config_path, encoding="utf-8") as f:
cfg = yaml.safe_load(f) or {}
# Strip ``load_format: dummy`` (CI overlay default) so advanced_model
-10
View File
@@ -35,7 +35,6 @@ from tests.helpers.env import run_forced_gpu_cleanup_round
from tests.helpers.media import (
_merge_base64_audio_to_segment,
convert_audio_bytes_to_text,
cosine_similarity_text,
decode_b64_image,
)
from vllm_omni.config.stage_config import resolve_deploy_yaml
@@ -500,7 +499,6 @@ class OmniResponse:
audio_content: str | None = None
audio_format: str | None = None
audio_bytes: bytes | None = None
similarity: float | None = None
e2e_latency: float | None = None
success: bool = False
error_message: str | None = None
@@ -542,19 +540,15 @@ class OpenAIClientHandler:
text_content += content
result.e2e_latency = time.perf_counter() - start_time
audio_content = None
similarity = None
if audio_data:
merged_seg = _merge_base64_audio_to_segment(audio_data)
wav_buf = BytesIO()
merged_seg.export(wav_buf, format="wav")
result.audio_bytes = wav_buf.getvalue()
audio_content = convert_audio_bytes_to_text(result.audio_bytes)
if audio_content and text_content:
similarity = cosine_similarity_text(audio_content.lower(), text_content.lower())
result.text_content = text_content
result.audio_data = audio_data
result.audio_content = audio_content
result.similarity = similarity
result.success = True
except Exception as e:
result.error_message = f"Stream processing error: {str(e)}"
@@ -578,15 +572,11 @@ class OpenAIClientHandler:
result.cached_tokens = details.cached_tokens
result.e2e_latency = time.perf_counter() - start_time
audio_content = None
similarity = None
if audio_data:
result.audio_bytes = base64.b64decode(audio_data)
audio_content = convert_audio_bytes_to_text(result.audio_bytes)
if audio_content and text_content:
similarity = cosine_similarity_text(audio_content.lower(), text_content.lower())
result.text_content = text_content
result.audio_content = audio_content
result.similarity = similarity
result.success = True
except Exception as e:
result.error_message = f"Non-stream processing error: {str(e)}"