Compare commits

..
Author SHA1 Message Date
tzhouam d2e6b4b5c7 ci: run focused vLLM 0.30.0 weekly baseline
Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
2026-10-04 02:17:47 +08:00
4 changed files with 84 additions and 109 deletions

No files matched your search

+45 -34
View File
@@ -1,35 +1,46 @@
# Focused weekly comparison against unchanged Omni main and official vLLM 0.30.0.
steps: steps:
- label: ":docker: Build image" - label: ':docker: Build image'
key: image-build key: image-build
commands: commands:
- "bash .buildkite/cuda/scripts/build-ci-image.sh" - bash .buildkite/cuda/scripts/build-ci-image.sh
agents: agents:
queue: "cpu_queue_premerge" queue: cpu_queue_premerge
- label: Upload selected weekly baseline
# L2 Test depends_on: image-build
- label: "Upload Ready Pipeline" key: upload-weekly-pipeline
depends_on: image-build commands:
key: upload-ready-pipeline - |
commands: python3 - <<'PYBASELINE'
- python3 .buildkite/common/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-ready.yml import tempfile
agents: from pathlib import Path
queue: "cpu_queue_premerge" import yaml
labels = set(['TTS · Audex · CosyVoice · Qwen3-TTS · L4', 'TTS · H100', 'Diffusion · Bagel · H100 · Single-GPU', 'Diffusion · SenseNova-U1 · H100 · Single-GPU', 'Diffusion · DreamZero · H100 · 2-GPU', 'Diffusion · H100 · 4-GPU', 'Diffusion · MAGI-2 Preview Serving Smoke · H100 · 4-GPU'])
# L3 Test source = yaml.safe_load(Path('.buildkite/cuda/test-weekly.yml').read_text())
- label: "Upload Merge Pipeline" selected = set()
depends_on: image-build guard = 'python3 -c \'import vllm, torch, flashinfer, vllm._C_stable_libtorch; assert vllm.__version__ == \'"\'"\'0.30.0\'"\'"\'; assert torch.cuda.is_available(); print(\'"\'"\'Verified baseline runtime:\'"\'"\', vllm.__version__, torch.__version__)\''
key: upload-merge-pipeline def select(steps):
commands: kept = []
- python3 .buildkite/common/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-merge.yml for step in steps:
agents: if not isinstance(step, dict):
queue: "cpu_queue_premerge" continue
if 'steps' in step:
# L4 Test children = select(step['steps'])
- label: "Upload Nightly Pipeline" if children:
depends_on: image-build kept.append(dict(step, steps=children))
key: upload-nightly-pipeline elif step.get('label') in labels:
if: build.env("NIGHTLY") == "1" item = dict(step)
commands: commands = item['commands']
- python3 .buildkite/common/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-nightly.yml item['commands'] = [guard] + ([commands] if isinstance(commands, str) else commands)
agents: kept.append(item)
queue: "cpu_queue_premerge" selected.add(item['label'])
return kept
source['steps'] = select(source['steps'])
assert selected == labels, (selected, labels)
path = Path(tempfile.gettempdir()) / 'vllm-omni-selected-weekly-baseline.yml'
path.write_text(yaml.safe_dump(source, sort_keys=False, allow_unicode=True))
print('Selected baseline suites:', sorted(selected), flush=True)
PYBASELINE
python3 .buildkite/common/scripts/upload_pipeline.py --upload --all /tmp/vllm-omni-selected-weekly-baseline.yml
agents:
queue: cpu_queue_premerge
+2 -1
View File
@@ -36,7 +36,8 @@ RUN apt-get update && \
# for CI # for CI
COPY pyproject.toml setup.py ./ COPY pyproject.toml setup.py ./
COPY requirements/ requirements/ COPY requirements/ requirements/
ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 # Keep source-built dependency versions independent of the Omni CI version.
ENV VLLM_OMNI_VERSION_OVERRIDE=0.0.0
# Editable install: deps + project metadata are cached in this layer, while # Editable install: deps + project metadata are cached in this layer, while
# the package itself is linked to APP_DIR (populated later by COPY --link). # the package itself is linked to APP_DIR (populated later by COPY --link).
# Do NOT use a non-editable install here — that copies an empty stub into # Do NOT use a non-editable install here — that copies an empty stub into
@@ -216,40 +216,44 @@ def test_video_chat_stream_invalid_requests(
# WS /v1/realtime # WS /v1/realtime
# ───────────────────────────────────────────────────────────────────────────── # ─────────────────────────────────────────────────────────────────────────────
# Resolved in ``test_realtime_invalid_requests`` (skip ``session.created`` handshake).
_REALTIME_WS_MODEL_MISMATCH = object()
_REALTIME_WS_INVALID_AUDIO_APPEND = object()
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
@pytest.mark.parametrize( @pytest.mark.parametrize(
"send_frames_spec, ws_error_code, err_message", "send_frames_spec, ws_error_code, err_message",
[ [
pytest.param("{not-json", "invalid_event", "Invalid or unrecognized client event", id="invalid_json"), pytest.param("{not-json", "invalid_json", "Invalid JSON", id="invalid_json"),
pytest.param( pytest.param(
json.dumps({"type": "session.update"}), json.dumps({"type": "session.update"}),
"invalid_event", "invalid_event",
"Invalid or unrecognized client event", ("model", "Missing required field"),
id="session_update_missing_session", id="session_update_missing_model",
), ),
pytest.param( pytest.param(
json.dumps({"type": "session.update", "session": []}), _REALTIME_WS_MODEL_MISMATCH,
"invalid_event", "model_not_found",
"Invalid or unrecognized client event", ("model", "does not exist"),
id="session_update_invalid_session", id="session_update_model_mismatch",
), ),
pytest.param( pytest.param(
json.dumps({"type": "input_audio_buffer.commit"}), json.dumps({"type": "input_audio_buffer.commit", "final": False}),
"invalid_request_error", "model_not_validated",
"Input audio buffer is empty", ("session.update", "validate the model"),
id="commit_without_audio", id="commit_without_model_validation",
), ),
pytest.param( pytest.param(
json.dumps({"type": "unknown.event"}), json.dumps({"type": "unknown.event"}),
"invalid_event", "unknown_event",
"Invalid or unrecognized client event", "Unknown event type",
id="unknown_event_type", id="unknown_event_type",
), ),
pytest.param( pytest.param(
json.dumps({"type": "input_audio_buffer.append", "audio": "not-valid-base64!!!"}), _REALTIME_WS_INVALID_AUDIO_APPEND,
"invalid_request_error", "invalid_audio",
"Invalid base64 audio data", "Invalid audio data",
id="invalid_audio_append", id="invalid_audio_append",
), ),
], ],
@@ -262,12 +266,27 @@ def test_realtime_invalid_requests(
ws_error_code: str, ws_error_code: str,
err_message: str | tuple[str, ...], err_message: str | tuple[str, ...],
) -> None: ) -> None:
send_frames: Any
if send_frames_spec is _REALTIME_WS_MODEL_MISMATCH:
send_frames = json.dumps(
{
"type": "session.update",
"model": "this-model-is-not-served-on-this-instance",
}
)
elif send_frames_spec is _REALTIME_WS_INVALID_AUDIO_APPEND:
send_frames = [
json.dumps({"type": "session.update", "model": omni_server.model}),
json.dumps({"type": "input_audio_buffer.append", "audio": "not-valid-base64!!!"}),
]
else:
send_frames = send_frames_spec
online_client.send_realtime_ws_request( online_client.send_realtime_ws_request(
{ {
"send_frames": send_frames_spec, "send_frames": send_frames,
"timeout": 120, "timeout": 120,
"ws_max_size": None, "ws_max_size": None,
"ws_skip_types": ["session.created", "conversation.created"], "ws_skip_types": ["session.created"],
"ws_json_type": "error", "ws_json_type": "error",
"ws_error_code": ws_error_code, "ws_error_code": ws_error_code,
"err_message": err_message, "err_message": err_message,
@@ -4,10 +4,7 @@
from __future__ import annotations from __future__ import annotations
import asyncio
import base64 import base64
import json
from types import SimpleNamespace
import numpy as np import numpy as np
import pytest import pytest
@@ -87,56 +84,3 @@ class TestAsyncOmniStreamingParamsValidation:
p = SamplingParams(n=1, stop=["\n"], output_kind=RequestOutputKind.DELTA) p = SamplingParams(n=1, stop=["\n"], output_kind=RequestOutputKind.DELTA)
with pytest.raises(ValueError, match="Input streaming"): with pytest.raises(ValueError, match="Input streaming"):
AsyncOmni._validate_streaming_input_sampling_params(p) AsyncOmni._validate_streaming_input_sampling_params(p)
@pytest.mark.parametrize(
"frame,code,message",
[
("{not-json", "invalid_event", "Invalid or unrecognized client event"),
(json.dumps({"type": "session.update"}), "invalid_event", "Invalid or unrecognized client event"),
(
json.dumps({"type": "session.update", "session": []}),
"invalid_event",
"Invalid or unrecognized client event",
),
(json.dumps({"type": "input_audio_buffer.commit"}), "invalid_request_error", "Input audio buffer is empty"),
(json.dumps({"type": "unknown.event"}), "invalid_event", "Invalid or unrecognized client event"),
(
json.dumps({"type": "input_audio_buffer.append", "audio": "not-valid-base64!!!"}),
"invalid_request_error",
"Invalid base64 audio data",
),
],
)
def test_full_duplex_invalid_request_contract(frame, code, message):
from starlette.websockets import WebSocketDisconnect
from vllm_omni.entrypoints.openai.realtime.connection import OpenAIFullDuplexConnection
class Socket:
def __init__(self):
self.sent = []
self.frames = iter([frame])
async def accept(self):
pass
async def receive_text(self):
value = next(self.frames, None)
if value is None:
raise WebSocketDisconnect()
return value
async def send_text(self, payload):
self.sent.append(json.loads(payload))
async def run():
socket = Socket()
connection = OpenAIFullDuplexConnection(socket, SimpleNamespace(), "test-model", None)
await connection.handle_connection()
assert [event["type"] for event in socket.sent] == ["session.created", "conversation.created", "error"]
error = socket.sent[-1]["error"]
assert error["code"] == code
assert error["message"] == message
asyncio.run(run())