Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d2e6b4b5c7 |
No files matched your search
@@ -1,35 +1,46 @@
|
|||||||
|
# Focused weekly comparison against unchanged Omni main and official vLLM 0.30.0.
|
||||||
steps:
|
steps:
|
||||||
- label: ":docker: Build image"
|
- label: ':docker: Build image'
|
||||||
key: image-build
|
key: image-build
|
||||||
commands:
|
commands:
|
||||||
- "bash .buildkite/cuda/scripts/build-ci-image.sh"
|
- bash .buildkite/cuda/scripts/build-ci-image.sh
|
||||||
agents:
|
agents:
|
||||||
queue: "cpu_queue_premerge"
|
queue: cpu_queue_premerge
|
||||||
|
- label: Upload selected weekly baseline
|
||||||
# L2 Test
|
depends_on: image-build
|
||||||
- label: "Upload Ready Pipeline"
|
key: upload-weekly-pipeline
|
||||||
depends_on: image-build
|
commands:
|
||||||
key: upload-ready-pipeline
|
- |
|
||||||
commands:
|
python3 - <<'PYBASELINE'
|
||||||
- python3 .buildkite/common/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-ready.yml
|
import tempfile
|
||||||
agents:
|
from pathlib import Path
|
||||||
queue: "cpu_queue_premerge"
|
import yaml
|
||||||
|
labels = set(['TTS · Audex · CosyVoice · Qwen3-TTS · L4', 'TTS · H100', 'Diffusion · Bagel · H100 · Single-GPU', 'Diffusion · SenseNova-U1 · H100 · Single-GPU', 'Diffusion · DreamZero · H100 · 2-GPU', 'Diffusion · H100 · 4-GPU', 'Diffusion · MAGI-2 Preview Serving Smoke · H100 · 4-GPU'])
|
||||||
# L3 Test
|
source = yaml.safe_load(Path('.buildkite/cuda/test-weekly.yml').read_text())
|
||||||
- label: "Upload Merge Pipeline"
|
selected = set()
|
||||||
depends_on: image-build
|
guard = 'python3 -c \'import vllm, torch, flashinfer, vllm._C_stable_libtorch; assert vllm.__version__ == \'"\'"\'0.30.0\'"\'"\'; assert torch.cuda.is_available(); print(\'"\'"\'Verified baseline runtime:\'"\'"\', vllm.__version__, torch.__version__)\''
|
||||||
key: upload-merge-pipeline
|
def select(steps):
|
||||||
commands:
|
kept = []
|
||||||
- python3 .buildkite/common/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-merge.yml
|
for step in steps:
|
||||||
agents:
|
if not isinstance(step, dict):
|
||||||
queue: "cpu_queue_premerge"
|
continue
|
||||||
|
if 'steps' in step:
|
||||||
# L4 Test
|
children = select(step['steps'])
|
||||||
- label: "Upload Nightly Pipeline"
|
if children:
|
||||||
depends_on: image-build
|
kept.append(dict(step, steps=children))
|
||||||
key: upload-nightly-pipeline
|
elif step.get('label') in labels:
|
||||||
if: build.env("NIGHTLY") == "1"
|
item = dict(step)
|
||||||
commands:
|
commands = item['commands']
|
||||||
- python3 .buildkite/common/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-nightly.yml
|
item['commands'] = [guard] + ([commands] if isinstance(commands, str) else commands)
|
||||||
agents:
|
kept.append(item)
|
||||||
queue: "cpu_queue_premerge"
|
selected.add(item['label'])
|
||||||
|
return kept
|
||||||
|
source['steps'] = select(source['steps'])
|
||||||
|
assert selected == labels, (selected, labels)
|
||||||
|
path = Path(tempfile.gettempdir()) / 'vllm-omni-selected-weekly-baseline.yml'
|
||||||
|
path.write_text(yaml.safe_dump(source, sort_keys=False, allow_unicode=True))
|
||||||
|
print('Selected baseline suites:', sorted(selected), flush=True)
|
||||||
|
PYBASELINE
|
||||||
|
python3 .buildkite/common/scripts/upload_pipeline.py --upload --all /tmp/vllm-omni-selected-weekly-baseline.yml
|
||||||
|
agents:
|
||||||
|
queue: cpu_queue_premerge
|
||||||
@@ -36,7 +36,8 @@ RUN apt-get update && \
|
|||||||
# for CI
|
# for CI
|
||||||
COPY pyproject.toml setup.py ./
|
COPY pyproject.toml setup.py ./
|
||||||
COPY requirements/ requirements/
|
COPY requirements/ requirements/
|
||||||
ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0
|
# Keep source-built dependency versions independent of the Omni CI version.
|
||||||
|
ENV VLLM_OMNI_VERSION_OVERRIDE=0.0.0
|
||||||
# Editable install: deps + project metadata are cached in this layer, while
|
# Editable install: deps + project metadata are cached in this layer, while
|
||||||
# the package itself is linked to APP_DIR (populated later by COPY --link).
|
# the package itself is linked to APP_DIR (populated later by COPY --link).
|
||||||
# Do NOT use a non-editable install here — that copies an empty stub into
|
# Do NOT use a non-editable install here — that copies an empty stub into
|
||||||
|
|||||||
@@ -216,40 +216,44 @@ def test_video_chat_stream_invalid_requests(
|
|||||||
# WS /v1/realtime
|
# WS /v1/realtime
|
||||||
# ─────────────────────────────────────────────────────────────────────────────
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
# Resolved in ``test_realtime_invalid_requests`` (skip ``session.created`` handshake).
|
||||||
|
_REALTIME_WS_MODEL_MISMATCH = object()
|
||||||
|
_REALTIME_WS_INVALID_AUDIO_APPEND = object()
|
||||||
|
|
||||||
|
|
||||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"send_frames_spec, ws_error_code, err_message",
|
"send_frames_spec, ws_error_code, err_message",
|
||||||
[
|
[
|
||||||
pytest.param("{not-json", "invalid_event", "Invalid or unrecognized client event", id="invalid_json"),
|
pytest.param("{not-json", "invalid_json", "Invalid JSON", id="invalid_json"),
|
||||||
pytest.param(
|
pytest.param(
|
||||||
json.dumps({"type": "session.update"}),
|
json.dumps({"type": "session.update"}),
|
||||||
"invalid_event",
|
"invalid_event",
|
||||||
"Invalid or unrecognized client event",
|
("model", "Missing required field"),
|
||||||
id="session_update_missing_session",
|
id="session_update_missing_model",
|
||||||
),
|
),
|
||||||
pytest.param(
|
pytest.param(
|
||||||
json.dumps({"type": "session.update", "session": []}),
|
_REALTIME_WS_MODEL_MISMATCH,
|
||||||
"invalid_event",
|
"model_not_found",
|
||||||
"Invalid or unrecognized client event",
|
("model", "does not exist"),
|
||||||
id="session_update_invalid_session",
|
id="session_update_model_mismatch",
|
||||||
),
|
),
|
||||||
pytest.param(
|
pytest.param(
|
||||||
json.dumps({"type": "input_audio_buffer.commit"}),
|
json.dumps({"type": "input_audio_buffer.commit", "final": False}),
|
||||||
"invalid_request_error",
|
"model_not_validated",
|
||||||
"Input audio buffer is empty",
|
("session.update", "validate the model"),
|
||||||
id="commit_without_audio",
|
id="commit_without_model_validation",
|
||||||
),
|
),
|
||||||
pytest.param(
|
pytest.param(
|
||||||
json.dumps({"type": "unknown.event"}),
|
json.dumps({"type": "unknown.event"}),
|
||||||
"invalid_event",
|
"unknown_event",
|
||||||
"Invalid or unrecognized client event",
|
"Unknown event type",
|
||||||
id="unknown_event_type",
|
id="unknown_event_type",
|
||||||
),
|
),
|
||||||
pytest.param(
|
pytest.param(
|
||||||
json.dumps({"type": "input_audio_buffer.append", "audio": "not-valid-base64!!!"}),
|
_REALTIME_WS_INVALID_AUDIO_APPEND,
|
||||||
"invalid_request_error",
|
"invalid_audio",
|
||||||
"Invalid base64 audio data",
|
"Invalid audio data",
|
||||||
id="invalid_audio_append",
|
id="invalid_audio_append",
|
||||||
),
|
),
|
||||||
],
|
],
|
||||||
@@ -262,12 +266,27 @@ def test_realtime_invalid_requests(
|
|||||||
ws_error_code: str,
|
ws_error_code: str,
|
||||||
err_message: str | tuple[str, ...],
|
err_message: str | tuple[str, ...],
|
||||||
) -> None:
|
) -> None:
|
||||||
|
send_frames: Any
|
||||||
|
if send_frames_spec is _REALTIME_WS_MODEL_MISMATCH:
|
||||||
|
send_frames = json.dumps(
|
||||||
|
{
|
||||||
|
"type": "session.update",
|
||||||
|
"model": "this-model-is-not-served-on-this-instance",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
elif send_frames_spec is _REALTIME_WS_INVALID_AUDIO_APPEND:
|
||||||
|
send_frames = [
|
||||||
|
json.dumps({"type": "session.update", "model": omni_server.model}),
|
||||||
|
json.dumps({"type": "input_audio_buffer.append", "audio": "not-valid-base64!!!"}),
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
send_frames = send_frames_spec
|
||||||
online_client.send_realtime_ws_request(
|
online_client.send_realtime_ws_request(
|
||||||
{
|
{
|
||||||
"send_frames": send_frames_spec,
|
"send_frames": send_frames,
|
||||||
"timeout": 120,
|
"timeout": 120,
|
||||||
"ws_max_size": None,
|
"ws_max_size": None,
|
||||||
"ws_skip_types": ["session.created", "conversation.created"],
|
"ws_skip_types": ["session.created"],
|
||||||
"ws_json_type": "error",
|
"ws_json_type": "error",
|
||||||
"ws_error_code": ws_error_code,
|
"ws_error_code": ws_error_code,
|
||||||
"err_message": err_message,
|
"err_message": err_message,
|
||||||
|
|||||||
@@ -4,10 +4,7 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import base64
|
import base64
|
||||||
import json
|
|
||||||
from types import SimpleNamespace
|
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import pytest
|
import pytest
|
||||||
@@ -87,56 +84,3 @@ class TestAsyncOmniStreamingParamsValidation:
|
|||||||
p = SamplingParams(n=1, stop=["\n"], output_kind=RequestOutputKind.DELTA)
|
p = SamplingParams(n=1, stop=["\n"], output_kind=RequestOutputKind.DELTA)
|
||||||
with pytest.raises(ValueError, match="Input streaming"):
|
with pytest.raises(ValueError, match="Input streaming"):
|
||||||
AsyncOmni._validate_streaming_input_sampling_params(p)
|
AsyncOmni._validate_streaming_input_sampling_params(p)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"frame,code,message",
|
|
||||||
[
|
|
||||||
("{not-json", "invalid_event", "Invalid or unrecognized client event"),
|
|
||||||
(json.dumps({"type": "session.update"}), "invalid_event", "Invalid or unrecognized client event"),
|
|
||||||
(
|
|
||||||
json.dumps({"type": "session.update", "session": []}),
|
|
||||||
"invalid_event",
|
|
||||||
"Invalid or unrecognized client event",
|
|
||||||
),
|
|
||||||
(json.dumps({"type": "input_audio_buffer.commit"}), "invalid_request_error", "Input audio buffer is empty"),
|
|
||||||
(json.dumps({"type": "unknown.event"}), "invalid_event", "Invalid or unrecognized client event"),
|
|
||||||
(
|
|
||||||
json.dumps({"type": "input_audio_buffer.append", "audio": "not-valid-base64!!!"}),
|
|
||||||
"invalid_request_error",
|
|
||||||
"Invalid base64 audio data",
|
|
||||||
),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_full_duplex_invalid_request_contract(frame, code, message):
|
|
||||||
from starlette.websockets import WebSocketDisconnect
|
|
||||||
|
|
||||||
from vllm_omni.entrypoints.openai.realtime.connection import OpenAIFullDuplexConnection
|
|
||||||
|
|
||||||
class Socket:
|
|
||||||
def __init__(self):
|
|
||||||
self.sent = []
|
|
||||||
self.frames = iter([frame])
|
|
||||||
|
|
||||||
async def accept(self):
|
|
||||||
pass
|
|
||||||
|
|
||||||
async def receive_text(self):
|
|
||||||
value = next(self.frames, None)
|
|
||||||
if value is None:
|
|
||||||
raise WebSocketDisconnect()
|
|
||||||
return value
|
|
||||||
|
|
||||||
async def send_text(self, payload):
|
|
||||||
self.sent.append(json.loads(payload))
|
|
||||||
|
|
||||||
async def run():
|
|
||||||
socket = Socket()
|
|
||||||
connection = OpenAIFullDuplexConnection(socket, SimpleNamespace(), "test-model", None)
|
|
||||||
await connection.handle_connection()
|
|
||||||
assert [event["type"] for event in socket.sent] == ["session.created", "conversation.created", "error"]
|
|
||||||
error = socket.sent[-1]["error"]
|
|
||||||
assert error["code"] == code
|
|
||||||
assert error["message"] == message
|
|
||||||
|
|
||||||
asyncio.run(run())
|
|
||||||
Reference in new issue
Block a user