[CI] Update test markers and configurations to use 'full_model' for L4 nightly tests (#2641)
Signed-off-by: wangyu <410167048@qq.com>
This commit is contained in:
1 parent
554233208f
commit
e18cb89d5a
50 files changed
+173
-322
No files matched your search
+27
-11
@@ -11,7 +11,9 @@ import numpy as np
|
||||
import soundfile as sf
|
||||
from PIL import Image
|
||||
|
||||
from tests.helpers.media import cosine_similarity_text
|
||||
from tests.helpers.media import (
|
||||
cosine_similarity_text,
|
||||
)
|
||||
|
||||
_GENDER_PIPELINE = None
|
||||
_GENDER_PIPELINE_LOCK = threading.Lock()
|
||||
@@ -56,7 +58,7 @@ def assert_image_diffusion_response(
|
||||
f"Expected {num_outputs_per_prompt} images, got {len(response.images)}"
|
||||
)
|
||||
|
||||
if run_level == "advanced_model":
|
||||
if run_level in {"advanced_model", "full_model"}:
|
||||
width = extra_body.get("width")
|
||||
height = extra_body.get("height")
|
||||
|
||||
@@ -407,7 +409,8 @@ def assert_omni_response(response: Any, request_config: dict[str, Any], run_leve
|
||||
|
||||
modalities = request_config.get("modalities", ["text", "audio"])
|
||||
|
||||
if run_level == "advanced_model":
|
||||
if run_level in {"advanced_model", "full_model"}:
|
||||
# Verify output success
|
||||
if "audio" in modalities:
|
||||
assert response.audio_content is not None, "No audio output is generated"
|
||||
print(f"audio content is: {response.audio_content}")
|
||||
@@ -417,12 +420,11 @@ def assert_omni_response(response: Any, request_config: dict[str, Any], run_leve
|
||||
response.audio_bytes,
|
||||
speaker,
|
||||
)
|
||||
|
||||
if "text" in modalities:
|
||||
assert response.text_content is not None, "No text output is generated"
|
||||
print(f"text content is: {response.text_content}")
|
||||
|
||||
# Verify image description
|
||||
# Verify keywords in output
|
||||
word_types = ["text", "image", "audio", "video"]
|
||||
keywords_dict = request_config.get("key_words", {})
|
||||
for word_type in word_types:
|
||||
@@ -441,11 +443,25 @@ def assert_omni_response(response: Any, request_config: dict[str, Any], run_leve
|
||||
)
|
||||
|
||||
# Verify similarity (Whisper transcript vs streamed/detokenized text)
|
||||
if "text" in modalities and "audio" in modalities:
|
||||
assert response.similarity is not None and response.similarity > 0.9, (
|
||||
"The audio content is not same as the text"
|
||||
)
|
||||
print(f"similarity is: {response.similarity}")
|
||||
if "audio" in modalities:
|
||||
audio_ref_text = request_config.get("audio_ref_text")
|
||||
if "text" in modalities:
|
||||
transcript = (response.audio_content or "").strip()
|
||||
text_output = (response.text_content or "").strip()
|
||||
similarity = cosine_similarity_text(
|
||||
transcript.lower(),
|
||||
text_output.lower(),
|
||||
)
|
||||
assert similarity > 0.9, "The audio content is not same as the text"
|
||||
print(f"similarity is: {similarity}")
|
||||
if audio_ref_text:
|
||||
audio_similarity = cosine_similarity_text(
|
||||
response.audio_content.lower(),
|
||||
str(audio_ref_text).lower(),
|
||||
)
|
||||
assert audio_similarity > 0.9, (
|
||||
f"The audio content does not match reference text: similarity={audio_similarity:.3f}"
|
||||
)
|
||||
|
||||
|
||||
def assert_audio_speech_response(response: Any, request_config: dict[str, Any], run_level: str) -> None:
|
||||
@@ -464,7 +480,7 @@ def assert_audio_speech_response(response: Any, request_config: dict[str, Any],
|
||||
elif req_fmt == "wav" and response.audio_format:
|
||||
assert req_fmt in response.audio_format
|
||||
|
||||
if run_level == "advanced_model" and req_fmt != "pcm":
|
||||
if run_level in {"advanced_model", "full_model"} and req_fmt != "pcm":
|
||||
expected_text = request_config.get("input")
|
||||
if expected_text:
|
||||
transcript = (response.audio_content or "").strip()
|
||||
|
||||
@@ -6,8 +6,8 @@ def pytest_addoption(parser):
|
||||
"--run-level",
|
||||
action="store",
|
||||
default="core_model",
|
||||
choices=["core_model", "advanced_model"],
|
||||
help="Test level to run: L2, L3",
|
||||
choices=["core_model", "advanced_model", "full_model"],
|
||||
help="Test level to run: L2, L3, L4",
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ def omni_server(request: pytest.FixtureRequest, run_level: str, model_prefix: st
|
||||
model = model_prefix + params.model
|
||||
port = params.port
|
||||
stage_config_path = params.stage_config_path
|
||||
if run_level == "advanced_model" and stage_config_path is not None:
|
||||
if run_level in {"advanced_model", "full_model"} and stage_config_path is not None:
|
||||
with open(stage_config_path, encoding="utf-8") as f:
|
||||
cfg = yaml.safe_load(f) or {}
|
||||
# Strip ``load_format: dummy`` (CI overlay default) so advanced_model
|
||||
|
||||
@@ -35,7 +35,6 @@ from tests.helpers.env import run_forced_gpu_cleanup_round
|
||||
from tests.helpers.media import (
|
||||
_merge_base64_audio_to_segment,
|
||||
convert_audio_bytes_to_text,
|
||||
cosine_similarity_text,
|
||||
decode_b64_image,
|
||||
)
|
||||
from vllm_omni.config.stage_config import resolve_deploy_yaml
|
||||
@@ -500,7 +499,6 @@ class OmniResponse:
|
||||
audio_content: str | None = None
|
||||
audio_format: str | None = None
|
||||
audio_bytes: bytes | None = None
|
||||
similarity: float | None = None
|
||||
e2e_latency: float | None = None
|
||||
success: bool = False
|
||||
error_message: str | None = None
|
||||
@@ -542,19 +540,15 @@ class OpenAIClientHandler:
|
||||
text_content += content
|
||||
result.e2e_latency = time.perf_counter() - start_time
|
||||
audio_content = None
|
||||
similarity = None
|
||||
if audio_data:
|
||||
merged_seg = _merge_base64_audio_to_segment(audio_data)
|
||||
wav_buf = BytesIO()
|
||||
merged_seg.export(wav_buf, format="wav")
|
||||
result.audio_bytes = wav_buf.getvalue()
|
||||
audio_content = convert_audio_bytes_to_text(result.audio_bytes)
|
||||
if audio_content and text_content:
|
||||
similarity = cosine_similarity_text(audio_content.lower(), text_content.lower())
|
||||
result.text_content = text_content
|
||||
result.audio_data = audio_data
|
||||
result.audio_content = audio_content
|
||||
result.similarity = similarity
|
||||
result.success = True
|
||||
except Exception as e:
|
||||
result.error_message = f"Stream processing error: {str(e)}"
|
||||
@@ -578,15 +572,11 @@ class OpenAIClientHandler:
|
||||
result.cached_tokens = details.cached_tokens
|
||||
result.e2e_latency = time.perf_counter() - start_time
|
||||
audio_content = None
|
||||
similarity = None
|
||||
if audio_data:
|
||||
result.audio_bytes = base64.b64decode(audio_data)
|
||||
audio_content = convert_audio_bytes_to_text(result.audio_bytes)
|
||||
if audio_content and text_content:
|
||||
similarity = cosine_similarity_text(audio_content.lower(), text_content.lower())
|
||||
result.text_content = text_content
|
||||
result.audio_content = audio_content
|
||||
result.similarity = similarity
|
||||
result.success = True
|
||||
except Exception as e:
|
||||
result.error_message = f"Non-stream processing error: {str(e)}"
|
||||
|
||||
Reference in new issue
Block a user