diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 25b14ce67..a2d0cdfe0 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -178,24 +178,60 @@ steps: volumes: - "/fsx/hf_cache:/fsx/hf_cache" + + - label: "Benchmark Test" + timeout_in_minutes: 15 + depends_on: image-build + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v tests/benchmarks/test_serve_cli.py + agents: + queue: "mithril-h100-pool" + plugins: + - kubernetes: + podSpec: + containers: + - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 2 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate + - label: "Omni Model Test" timeout_in_minutes: 15 depends_on: image-build commands: - - export VLLM_LOGGING_LEVEL=DEBUG - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py + - export VLLM_LOGGING_LEVEL=DEBUG + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py agents: queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - environment: - - "HF_HOME=/fsx/hf_cache" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + - docker#v5.2.0: + image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + always-pull: true + propagate-environment: true + environment: + - "HF_HOME=/fsx/hf_cache" + volumes: + - "/fsx/hf_cache:/fsx/hf_cache" - label: "Omni Model Test with H100" timeout_in_minutes: 30 diff --git a/docs/cli/bench/serve.md b/docs/cli/bench/serve.md index bcb1df74c..cc47bfc3c 100644 --- a/docs/cli/bench/serve.md +++ b/docs/cli/bench/serve.md @@ -128,7 +128,7 @@ Specify to save benchmark results to a json file Second item: videos with resolution 720x1280 and 16 frames Third item: audios with 1s duration and 5 channels w.p. 0.1 OBS.: If the probabilities do not sum to 1, they are normalized. - Only the random-mm mode supports this parameter. + Only the random-mm mode supports this parameter ## Usage Examples diff --git a/tests/benchmarks/test_serve_cli.py b/tests/benchmarks/test_serve_cli.py new file mode 100644 index 000000000..2c624b8e7 --- /dev/null +++ b/tests/benchmarks/test_serve_cli.py @@ -0,0 +1,60 @@ +import subprocess +from pathlib import Path + +import pytest + +from tests.conftest import OmniServer + +models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"] +stage_configs = [str(Path(__file__).parent.parent / "e2e" / "stage_configs" / "qwen3_omni_ci.yaml")] + +# Create parameter combinations for model and stage config +test_params = [(model, stage_config) for model in models for stage_config in stage_configs] + + +@pytest.fixture(scope="module") +def omni_server(request): + """Start vLLM-Omni server as a subprocess with actual model weights. + Uses session scope so the server starts only once for the entire test session. + Multi-stage initialization can take 10-20+ minutes. + """ + model, stage_config_path = request.param + + print(f"Starting OmniServer with model: {model}") + print("This may take 10-20+ minutes for initialization...") + + with OmniServer(model, ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "120"]) as server: + print("OmniServer started successfully") + yield server + print("OmniServer stopped") + + +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_bench_serve_chat(omni_server): + command = [ + "vllm", + "bench", + "serve", + "--omni", + "--model", + omni_server.model, + "--port", + str(omni_server.port), + "--dataset-name", + "random", + "--random-input-len", + "32", + "--random-output-len", + "4", + "--num-prompts", + "5", + "--endpoint", + "/v1/chat/completions", + "--backend", + "openai-chat-omni", + ] + result = subprocess.run(command, capture_output=True, text=True) + print(result.stdout) + print(result.stderr) + + assert result.returncode == 0, f"Benchmark failed: {result.stderr}" diff --git a/vllm_omni/benchmarks/patch/patch.py b/vllm_omni/benchmarks/patch/patch.py index 6591af014..cf2e5c195 100644 --- a/vllm_omni/benchmarks/patch/patch.py +++ b/vllm_omni/benchmarks/patch/patch.py @@ -108,7 +108,7 @@ async def async_request_openai_chat_omni_completions( output.prompt_len = request_func_input.prompt_len generated_text = "" - generated_audio = "" + generated_audio = None ttft = 0.0 st = time.perf_counter() output.start_time = st @@ -160,7 +160,7 @@ async def async_request_openai_chat_omni_completions( if generated_audio is None: generated_audio = seg else: - generated_audio = seg + generated_audio + generated_audio = generated_audio + seg elif usage := data.get("usage"): output.output_tokens = usage.get("completion_tokens")