[Test] Add BuildKite test-full script for full CI. (#867)

Signed-off-by: wangyu31577 <wangyu31577@hundsun.com>
Co-authored-by: wangyu31577 <wangyu31577@hundsun.com>
This commit is contained in:
wangyuandwangyu31577 authored and GitHub committed 2026-02-06 01:19:46 +08:00
1 parent ffbc1305e6
commit 9f00a37437
2 files changed
+93 -72

No files matched your search

+45
View File
@@ -0,0 +1,45 @@
steps:
- label: ":docker: Build image"
key: image-build
commands:
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
- "docker build --file docker/Dockerfile.ci -t vllm-omni-ci ."
- "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
- "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
agents:
queue: "cpu_queue_premerge"
- label: "Omni Model Test with H100"
timeout_in_minutes: 180
depends_on: image-build
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -s -v tests/e2e/online_serving/test_qwen3_omni_expansion.py
agents:
queue: "mithril-h100-pool"
plugins:
- kubernetes:
podSpec:
containers:
- image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
resources:
limits:
nvidia.com/gpu: 2
volumeMounts:
- name: devshm
mountPath: /dev/shm
- name: hf-cache
mountPath: /root/.cache/huggingface
env:
- name: HF_HOME
value: /root/.cache/huggingface
nodeSelector:
node.kubernetes.io/instance-type: gpu-h100-sxm
volumes:
- name: devshm
emptyDir:
medium: Memory
- name: hf-cache
hostPath:
path: /mnt/hf-cache
type: DirectoryOrCreate
@@ -8,6 +8,7 @@ import concurrent.futures
import os import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
import threading
import time import time
from pathlib import Path from pathlib import Path
@@ -21,7 +22,6 @@ from tests.conftest import (
dummy_messages_from_mix_data, dummy_messages_from_mix_data,
generate_synthetic_audio, generate_synthetic_audio,
generate_synthetic_image, generate_synthetic_image,
modify_stage_config,
) )
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"] models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
@@ -33,6 +33,29 @@ stage_configs = [str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omn
test_params = [(model, stage_config) for model in models for stage_config in stage_configs] test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
_omni_server_lock = threading.Lock()
@pytest.fixture(scope="module")
def omni_server(request):
"""Start vLLM-Omni server as a subprocess with actual model weights.
Uses session scope so the server starts only once for the entire test session.
Multi-stage initialization can take 10-20+ minutes.
"""
with _omni_server_lock:
model, stage_config_path = request.param
print(f"Starting OmniServer with model: {model}")
with OmniServer(model, ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "120"]) as server:
print("OmniServer started successfully")
yield server
print("OmniServer stopping...")
print("OmniServer stopped")
@pytest.fixture
def client(omni_server): def client(omni_server):
"""OpenAI client for the running vLLM-Omni server.""" """OpenAI client for the running vLLM-Omni server."""
return openai.OpenAI( return openai.OpenAI(
@@ -86,18 +109,15 @@ def get_deploy_config(deploy_type="TP1"):
return result.get(deploy_type, result["TP1"]) return result.get(deploy_type, result["TP1"])
@pytest.mark.parametrize("test_config", test_params) @pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_text_to_text_001(test_config: tuple[str, str]) -> None: def test_text_to_text_001(client: openai.OpenAI, omni_server) -> None:
"""Test processing text, generating text output via OpenAI API.""" """Test processing text, generating text output via OpenAI API."""
model, stage_config_path = test_config
with OmniServer(model, ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "90"]) as server:
messages = dummy_messages_from_mix_data(system_prompt=get_system_prompt(), content_text=get_prompt()) messages = dummy_messages_from_mix_data(system_prompt=get_system_prompt(), content_text=get_prompt())
# Test single completion # Test single completion
api_client = client(server)
start_time = time.perf_counter() start_time = time.perf_counter()
chat_completion = api_client.chat.completions.create( chat_completion = client.chat.completions.create(
model=server.model, messages=messages, max_tokens=20, modalities=["text"] model=omni_server.model, messages=messages, max_tokens=20, modalities=["text"]
) )
# Verify E2E # Verify E2E
print(f"the request e2e is: {time.perf_counter() - start_time}") print(f"the request e2e is: {time.perf_counter() - start_time}")
@@ -113,66 +133,41 @@ def test_text_to_text_001(test_config: tuple[str, str]) -> None:
assert "beijing" in text_choice.message.content.lower(), "The output do not contain keywords." assert "beijing" in text_choice.message.content.lower(), "The output do not contain keywords."
@pytest.mark.parametrize("test_config", test_params) @pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_audio_to_text_001(test_config: tuple[str, str]) -> None: def test_audio_to_text_001(client: openai.OpenAI, omni_server) -> None:
"""Test processing text, generating text output via OpenAI API.""" """Test processing text, generating text output via OpenAI API."""
model, stage_config_path = test_config
deploy_config = get_deploy_config()
deploy_config[0]["default_sampling_params.ignore_eos"] = True
stage_config_path = modify_stage_config(stage_config_path, deploy_config)
with OmniServer(model, ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "90"]) as server:
audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(1, 1)['base64']}" audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(1, 1)['base64']}"
messages = dummy_messages_from_mix_data(audio_data_url=audio_data_url) messages = dummy_messages_from_mix_data(audio_data_url=audio_data_url)
# Test single completion # Test single completion
api_client = client(server)
start_time = time.perf_counter() start_time = time.perf_counter()
chat_completion = api_client.chat.completions.create( chat_completion = client.chat.completions.create(model=omni_server.model, messages=messages, modalities=["text"])
model=server.model, messages=messages, max_tokens=200, modalities=["text"]
)
# Verify only output text # Verify only output text
assert len(chat_completion.choices) == 1, "The generated content includes more than just text." assert len(chat_completion.choices) == 1, "The generated content includes more than just text."
# Verify text output success # Verify text output success
text_choice = chat_completion.choices[0] text_choice = chat_completion.choices[0]
assert text_choice.message.content is not None, "No text output is generated" assert text_choice.message.content is not None, "No text output is generated"
assert chat_completion.usage.completion_tokens == 200, ( assert "water" in text_choice.message.content.lower(), "The output do not contain keywords."
"The output length differs from the requested max_tokens."
)
# Verify E2E # Verify E2E
print(f"the request e2e is: {time.perf_counter() - start_time}") print(f"the request e2e is: {time.perf_counter() - start_time}")
# TODO: Verify the E2E latency after confirmation baseline. # TODO: Verify the E2E latency after confirmation baseline.
@pytest.mark.parametrize("test_config", test_params) @pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_audio_to_text_audio_001(test_config: tuple[str, str]) -> None: def test_audio_to_text_audio_001(client: openai.OpenAI, omni_server) -> None:
"""Test processing text, generating audio output via OpenAI API.""" """Test processing text, generating audio output via OpenAI API."""
model, stage_config_path = test_config audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(1, 5)['base64']}"
num_concurrent_requests = get_max_batch_size()
stage_config_path = modify_stage_config(
stage_config_path,
{
"stage_args": {
0: {"runtime.max_batch_size": num_concurrent_requests},
1: {"runtime.max_batch_size": num_concurrent_requests},
}
},
)
with OmniServer(model, ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "90"]) as server:
audio_data_url = []
for _ in range(5):
audio_data_url.append(f"data:audio/wav;base64,{generate_synthetic_audio(1, 5)['base64']}")
messages = dummy_messages_from_mix_data(audio_data_url=audio_data_url) messages = dummy_messages_from_mix_data(audio_data_url=audio_data_url)
num_concurrent_requests = get_max_batch_size()
# Test single completion # Test single completion
api_client = client(server)
e2e_list = list() e2e_list = list()
with concurrent.futures.ThreadPoolExecutor(max_workers=num_concurrent_requests) as executor: with concurrent.futures.ThreadPoolExecutor(max_workers=num_concurrent_requests) as executor:
# Submit multiple completion requests concurrently # Submit multiple completion requests concurrently
futures = [ futures = [
executor.submit(api_client.chat.completions.create, model=server.model, messages=messages) executor.submit(client.chat.completions.create, model=omni_server.model, messages=messages)
for _ in range(num_concurrent_requests) for _ in range(num_concurrent_requests)
] ]
start_time = time.perf_counter() start_time = time.perf_counter()
@@ -210,21 +205,15 @@ def test_audio_to_text_audio_001(test_config: tuple[str, str]) -> None:
assert similarity > 0.9, "The audio content is not same as the text" assert similarity > 0.9, "The audio content is not same as the text"
@pytest.mark.parametrize("test_config", test_params) @pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_image_to_text_001(test_config: tuple[str, str]) -> None: def test_image_to_text_001(client: openai.OpenAI, omni_server) -> None:
"""Test processing text, generating text output via OpenAI API.""" """Test processing text, generating text output via OpenAI API."""
model, stage_config_path = test_config
deploy_config = get_deploy_config()
stage_config_path = modify_stage_config(stage_config_path, deploy_config)
with OmniServer(model, ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "90"]) as server:
image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(224, 224)['base64']}" image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(224, 224)['base64']}"
messages = dummy_messages_from_mix_data(image_data_url=image_data_url) messages = dummy_messages_from_mix_data(image_data_url=image_data_url)
# Test single completion # Test single completion
api_client = client(server)
start_time = time.perf_counter() start_time = time.perf_counter()
chat_completion = api_client.chat.completions.create( chat_completion = client.chat.completions.create(
model=server.model, messages=messages, max_tokens=100, modalities=["text"] model=omni_server.model, messages=messages, max_tokens=100, modalities=["text"]
) )
# Verify E2E # Verify E2E
print(f"the request e2e is: {time.perf_counter() - start_time}") print(f"the request e2e is: {time.perf_counter() - start_time}")
@@ -241,37 +230,24 @@ def test_image_to_text_001(test_config: tuple[str, str]) -> None:
assert "square" in text_content.lower(), "The output do not contain keywords." assert "square" in text_content.lower(), "The output do not contain keywords."
@pytest.mark.parametrize("test_config", test_params) @pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_image_to_text_audio_001(test_config: tuple[str, str]) -> None: def test_image_to_text_audio_001(client: openai.OpenAI, omni_server) -> None:
"""Test processing text, generating audio output via OpenAI API.""" """Test processing text, generating audio output via OpenAI API."""
model, stage_config_path = test_config num_concurrent_requests = get_max_batch_size()
num_concurrent_requests = 5
stage_config_path = modify_stage_config( image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(1280, 720)['base64']}"
stage_config_path,
{
"stage_args": {
0: {"runtime.max_batch_size": num_concurrent_requests},
1: {"runtime.max_batch_size": num_concurrent_requests},
}
},
)
with OmniServer(model, ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "90"]) as server:
image_data_url = []
for _ in range(4):
image_data_url.append(f"data:image/jpeg;base64,{generate_synthetic_image(1280, 720)['base64']}")
messages = dummy_messages_from_mix_data(image_data_url=image_data_url) messages = dummy_messages_from_mix_data(image_data_url=image_data_url)
# Test single completion # Test single completion
api_client = client(server)
e2e_list = list() e2e_list = list()
with concurrent.futures.ThreadPoolExecutor(max_workers=num_concurrent_requests) as executor: with concurrent.futures.ThreadPoolExecutor(max_workers=num_concurrent_requests) as executor:
# Submit multiple completion requests concurrently # Submit multiple completion requests concurrently
futures = [ futures = [
executor.submit( executor.submit(
api_client.chat.completions.create, client.chat.completions.create,
model=server.model, model=omni_server.model,
messages=messages, messages=messages,
) )
for _ in range(num_concurrent_requests) for _ in range(num_concurrent_requests)