[Feature] Support Omni serving abort request (#486)
Signed-off-by: ZeldaHuang <hzm414167@alibaba-inc.com> Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
6f8deb4a5a
commit
ad0224f2e6
10 files changed
+354
-187
No files matched your search
@@ -0,0 +1,31 @@
|
||||
# The following config has been verified on 2x H100-80G GPUs.
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
runtime:
|
||||
devices: "0,1"
|
||||
max_batch_size: 5
|
||||
engine_args:
|
||||
model_stage: thinker
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.6
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output hidden states for talker
|
||||
distributed_executor_backend: "mp"
|
||||
enable_prefix_caching: false
|
||||
hf_config_name: thinker_config
|
||||
tensor_parallel_size: 2
|
||||
load_format: dummy
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
is_comprehension: true
|
||||
default_sampling_params:
|
||||
temperature: 0.4
|
||||
top_p: 0.9
|
||||
top_k: 1
|
||||
max_tokens: 100
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.05
|
||||
@@ -0,0 +1,100 @@
|
||||
import asyncio
|
||||
from contextlib import ExitStack
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from vllm.inputs import PromptType
|
||||
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
|
||||
SEED = 42
|
||||
|
||||
stage_config = str(Path(__file__).parent / "stage_configs" / "qwen3_omni_thinker_ci.yaml")
|
||||
model = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
|
||||
|
||||
|
||||
async def generate(
|
||||
engine: AsyncOmni,
|
||||
request_id: str,
|
||||
prompt: PromptType,
|
||||
max_tokens: int,
|
||||
) -> tuple[int, str]:
|
||||
# Ensure generate doesn't complete too fast for cancellation test.
|
||||
await asyncio.sleep(0.2)
|
||||
thinker_sampling_params = {
|
||||
"temperature": 0.4, # Deterministic
|
||||
"top_p": 0.9,
|
||||
"top_k": 1,
|
||||
"max_tokens": max_tokens,
|
||||
"repetition_penalty": 1.05,
|
||||
"stop_token_ids": [151645], # Qwen EOS token <|im_end|>
|
||||
"seed": SEED,
|
||||
}
|
||||
|
||||
sampling_params_list = [
|
||||
thinker_sampling_params,
|
||||
]
|
||||
count = 0
|
||||
async for omni_output in engine.generate(
|
||||
prompt=prompt,
|
||||
request_id=request_id,
|
||||
sampling_params_list=sampling_params_list,
|
||||
output_modalities=["text"],
|
||||
):
|
||||
stage_id = omni_output.stage_id
|
||||
out = omni_output.request_output
|
||||
if stage_id == 0:
|
||||
num_tokens = sum(len(output.token_ids) for output in out.outputs)
|
||||
count = num_tokens
|
||||
|
||||
await asyncio.sleep(0.0)
|
||||
|
||||
return count, request_id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_abort():
|
||||
with ExitStack() as after:
|
||||
engine = AsyncOmni(model=model, stage_configs_path=stage_config)
|
||||
after.callback(engine.shutdown)
|
||||
|
||||
NUM_REQUESTS = 5
|
||||
NUM_EXPECTED_TOKENS = 100
|
||||
NUM_EXPECTED_TOKENS_LONG = 1000
|
||||
REQUEST_IDS_TO_ABORT = [1, 2, 3]
|
||||
|
||||
prompt = "Hello my name is Robert and "
|
||||
|
||||
request_ids = [f"request-{i}" for i in range(NUM_REQUESTS)]
|
||||
|
||||
# Create concurrent requests.
|
||||
tasks: list[asyncio.Task] = []
|
||||
for idx, request_id in enumerate(request_ids):
|
||||
max_tokens = NUM_EXPECTED_TOKENS_LONG if (idx in REQUEST_IDS_TO_ABORT) else NUM_EXPECTED_TOKENS
|
||||
tasks.append(asyncio.create_task(generate(engine, request_id, prompt, max_tokens)))
|
||||
|
||||
# API server cancels requests when they disconnect.
|
||||
for idx in REQUEST_IDS_TO_ABORT:
|
||||
tasks[idx].cancel()
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
# Confirm the other requests are okay.
|
||||
for idx, task in enumerate(tasks):
|
||||
# Confirm that it was actually canceled.
|
||||
if idx in REQUEST_IDS_TO_ABORT:
|
||||
with pytest.raises((asyncio.CancelledError, GeneratorExit)):
|
||||
await task
|
||||
else:
|
||||
# Otherwise, make sure the request was not impacted.
|
||||
num_generated_tokens, request_id = await task
|
||||
expected_tokens = NUM_EXPECTED_TOKENS
|
||||
assert num_generated_tokens == expected_tokens, (
|
||||
f"{request_id} generated {num_generated_tokens} but expected {expected_tokens}"
|
||||
)
|
||||
|
||||
# Confirm we can do another generation.
|
||||
request_id = f"request-{REQUEST_IDS_TO_ABORT[0]}"
|
||||
task = asyncio.create_task(generate(engine, request_id, prompt, NUM_EXPECTED_TOKENS))
|
||||
num_generated_tokens, request_id = await task
|
||||
assert num_generated_tokens == NUM_EXPECTED_TOKENS
|
||||
await asyncio.sleep(5)
|
||||
@@ -6,6 +6,8 @@ from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.entrypoints.stage_utils import SHUTDOWN_TASK
|
||||
|
||||
# Suppress noisy DeprecationWarnings from optional Swig bindings imported by vLLM dependencies.
|
||||
warnings.filterwarnings(
|
||||
"ignore",
|
||||
@@ -127,7 +129,7 @@ class _FakeStage:
|
||||
"""Mock stop_stage_worker: clean up queue references."""
|
||||
if self._in_q is not None:
|
||||
try:
|
||||
self._in_q.put_nowait(None)
|
||||
self._in_q.put_nowait(SHUTDOWN_TASK)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -977,7 +979,7 @@ def test_close_sends_shutdown_signal(monkeypatch, fake_stage_config):
|
||||
# Use get_nowait to avoid blocking (close() uses put_nowait, so should be safe)
|
||||
try:
|
||||
shutdown_signal = omni.stage_list[0]._in_q.get_nowait()
|
||||
assert shutdown_signal is None
|
||||
assert shutdown_signal == SHUTDOWN_TASK
|
||||
except Empty:
|
||||
# If queue was already empty or only had stage_ready, that's also acceptable
|
||||
# The important thing is that close() was called without error
|
||||
|
||||
Reference in new issue
Block a user