[Feature] Support Omni serving abort request (#486)

Signed-off-by: ZeldaHuang <hzm414167@alibaba-inc.com>
Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
Ziming HuangandHongsheng Liu authored and GitHub committed 2025-12-31 16:14:03 +08:00
1 parent 6f8deb4a5a
commit ad0224f2e6
10 files changed
+354 -187

No files matched your search

@@ -0,0 +1,31 @@
# The following config has been verified on 2x H100-80G GPUs.
stage_args:
- stage_id: 0
runtime:
devices: "0,1"
max_batch_size: 5
engine_args:
model_stage: thinker
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.6
enforce_eager: true
trust_remote_code: true
engine_output_type: latent # Output hidden states for talker
distributed_executor_backend: "mp"
enable_prefix_caching: false
hf_config_name: thinker_config
tensor_parallel_size: 2
load_format: dummy
final_output: true
final_output_type: text
is_comprehension: true
default_sampling_params:
temperature: 0.4
top_p: 0.9
top_k: 1
max_tokens: 100
seed: 42
detokenize: True
repetition_penalty: 1.05
+100
View File
@@ -0,0 +1,100 @@
import asyncio
from contextlib import ExitStack
from pathlib import Path
import pytest
from vllm.inputs import PromptType
from vllm_omni.entrypoints.async_omni import AsyncOmni
SEED = 42
stage_config = str(Path(__file__).parent / "stage_configs" / "qwen3_omni_thinker_ci.yaml")
model = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
async def generate(
engine: AsyncOmni,
request_id: str,
prompt: PromptType,
max_tokens: int,
) -> tuple[int, str]:
# Ensure generate doesn't complete too fast for cancellation test.
await asyncio.sleep(0.2)
thinker_sampling_params = {
"temperature": 0.4, # Deterministic
"top_p": 0.9,
"top_k": 1,
"max_tokens": max_tokens,
"repetition_penalty": 1.05,
"stop_token_ids": [151645], # Qwen EOS token <|im_end|>
"seed": SEED,
}
sampling_params_list = [
thinker_sampling_params,
]
count = 0
async for omni_output in engine.generate(
prompt=prompt,
request_id=request_id,
sampling_params_list=sampling_params_list,
output_modalities=["text"],
):
stage_id = omni_output.stage_id
out = omni_output.request_output
if stage_id == 0:
num_tokens = sum(len(output.token_ids) for output in out.outputs)
count = num_tokens
await asyncio.sleep(0.0)
return count, request_id
@pytest.mark.asyncio
async def test_abort():
with ExitStack() as after:
engine = AsyncOmni(model=model, stage_configs_path=stage_config)
after.callback(engine.shutdown)
NUM_REQUESTS = 5
NUM_EXPECTED_TOKENS = 100
NUM_EXPECTED_TOKENS_LONG = 1000
REQUEST_IDS_TO_ABORT = [1, 2, 3]
prompt = "Hello my name is Robert and "
request_ids = [f"request-{i}" for i in range(NUM_REQUESTS)]
# Create concurrent requests.
tasks: list[asyncio.Task] = []
for idx, request_id in enumerate(request_ids):
max_tokens = NUM_EXPECTED_TOKENS_LONG if (idx in REQUEST_IDS_TO_ABORT) else NUM_EXPECTED_TOKENS
tasks.append(asyncio.create_task(generate(engine, request_id, prompt, max_tokens)))
# API server cancels requests when they disconnect.
for idx in REQUEST_IDS_TO_ABORT:
tasks[idx].cancel()
await asyncio.sleep(0.1)
# Confirm the other requests are okay.
for idx, task in enumerate(tasks):
# Confirm that it was actually canceled.
if idx in REQUEST_IDS_TO_ABORT:
with pytest.raises((asyncio.CancelledError, GeneratorExit)):
await task
else:
# Otherwise, make sure the request was not impacted.
num_generated_tokens, request_id = await task
expected_tokens = NUM_EXPECTED_TOKENS
assert num_generated_tokens == expected_tokens, (
f"{request_id} generated {num_generated_tokens} but expected {expected_tokens}"
)
# Confirm we can do another generation.
request_id = f"request-{REQUEST_IDS_TO_ABORT[0]}"
task = asyncio.create_task(generate(engine, request_id, prompt, NUM_EXPECTED_TOKENS))
num_generated_tokens, request_id = await task
assert num_generated_tokens == NUM_EXPECTED_TOKENS
await asyncio.sleep(5)
+4 -2
View File
@@ -6,6 +6,8 @@ from unittest.mock import MagicMock
import pytest
from vllm_omni.entrypoints.stage_utils import SHUTDOWN_TASK
# Suppress noisy DeprecationWarnings from optional Swig bindings imported by vLLM dependencies.
warnings.filterwarnings(
"ignore",
@@ -127,7 +129,7 @@ class _FakeStage:
"""Mock stop_stage_worker: clean up queue references."""
if self._in_q is not None:
try:
self._in_q.put_nowait(None)
self._in_q.put_nowait(SHUTDOWN_TASK)
except Exception:
pass
@@ -977,7 +979,7 @@ def test_close_sends_shutdown_signal(monkeypatch, fake_stage_config):
# Use get_nowait to avoid blocking (close() uses put_nowait, so should be safe)
try:
shutdown_signal = omni.stage_list[0]._in_q.get_nowait()
assert shutdown_signal is None
assert shutdown_signal == SHUTDOWN_TASK
except Empty:
# If queue was already empty or only had stage_ready, that's also acceptable
# The important thing is that close() was called without error