[Core] Supports stage abstraction in the diffusion model (#391)

Signed-off-by: Chenguang ZHENG <645327136@qq.com>
Signed-off-by: yinpeiqi <yinpeiqi809@gmail.com>
Co-authored-by: yinpeiqi <yinpeiqi809@gmail.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
authored and GitHub committed 2025-12-27 01:59:33 +13:00
1 parent 1f875f06d9
commit 595e7c0032
41 files changed
+2473 -1242

No files matched your search

@@ -132,8 +132,9 @@ def test_client(mock_async_diffusion):
app.include_router(router)
# Set up app state with diffusion engine
app.state.diffusion_engine = mock_async_diffusion
app.state.diffusion_model_name = "Qwen/Qwen-Image"
app.state.engine_client = mock_async_diffusion
app.state.stage_configs = [{"stage_type": "diffusion"}]
app.state.served_model_names = "Qwen/Qwen-Image"
return TestClient(app)
+172 -106
View File
@@ -83,6 +83,8 @@ class _FakeStage:
self.stage_id = getattr(config, "stage_id", 0)
self.engine_args = config.engine_args
self.model_stage = getattr(config.engine_args, "model_stage", None)
# set default sampling params
self.default_sampling_params = {"temperature": 1.0}
# Allow configuring final_output and final_output_type
self.final_output = config.final_output if hasattr(config, "final_output") else False
self.final_output_type = getattr(config, "final_output_type", None)
@@ -171,18 +173,6 @@ class _FakeEngine:
yield from self._outputs
class _FakeStageLLM:
"""Replace OmniStageLLM to avoid constructing real engine."""
def __init__(self, **kwargs):
# Allow injecting custom fake outputs, default returns single placeholder output
fake_outputs = kwargs.get("_fake_outputs", [[{"text": "ok"}]])
self._fake_engine = _FakeEngine(fake_outputs)
def generate(self, prompts, sampling_params, use_tqdm=False):
yield from self._fake_engine.generate(prompts, sampling_params)
@pytest.fixture
def fake_stage_config():
return {
@@ -298,22 +288,22 @@ def _setup_ipc_mocks(monkeypatch):
def _fake_set(obj):
return str(obj).encode()
monkeypatch.setattr("vllm_omni.entrypoints.omni_llm._encode", _fake_encode, raising=False)
monkeypatch.setattr("vllm_omni.entrypoints.omni_llm._load", _fake_load, raising=False)
monkeypatch.setattr("vllm_omni.entrypoints.omni_llm._set", _fake_set, raising=False)
monkeypatch.setattr("vllm_omni.entrypoints.omni._encode", _fake_encode, raising=False)
monkeypatch.setattr("vllm_omni.entrypoints.omni._load", _fake_load, raising=False)
monkeypatch.setattr("vllm_omni.entrypoints.omni._set", _fake_set, raising=False)
def _setup_log_mocks(monkeypatch):
"""Helper function to set up logging and stats mocks."""
# Mock init_stats_paths to return None files (no stats logging)
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_llm.init_stats_paths",
"vllm_omni.entrypoints.omni.init_stats_paths",
lambda enable_stats, log_file: (None, None),
raising=False,
)
# Mock configure_orchestrator_logger to do nothing
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_llm.configure_orchestrator_logger",
"vllm_omni.entrypoints.omni.configure_orchestrator_logger",
lambda logger, log_file: None,
raising=False,
)
@@ -340,7 +330,7 @@ def _setup_log_mocks(monkeypatch):
return "Fake summary"
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_llm.OrchestratorMetrics",
"vllm_omni.entrypoints.omni.OrchestratorMetrics",
_FakeOrchestratorMetrics,
raising=False,
)
@@ -482,7 +472,8 @@ def test_initialize_stage_configs_called_when_none(monkeypatch, fake_stage_confi
for module_name in [
"vllm_omni.entrypoints.utils",
"vllm_omni.entrypoints.omni_llm",
"vllm_omni.entrypoints.omni",
"vllm_omni.entrypoints.log_utils",
"vllm_omni.entrypoints.omni_stage",
]:
if module_name in sys.modules:
@@ -496,7 +487,7 @@ def test_initialize_stage_configs_called_when_none(monkeypatch, fake_stage_confi
# Mock remove_old_logs to avoid bug (called before stage_list is created)
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_llm.remove_old_logs",
"vllm_omni.entrypoints.log_utils.remove_old_logs",
lambda log_file, num_stages: None,
raising=False,
)
@@ -516,28 +507,28 @@ def test_initialize_stage_configs_called_when_none(monkeypatch, fake_stage_confi
)
# Import the module after mocks are set
import vllm_omni.entrypoints.omni_llm as omni_llm_module
import vllm_omni.entrypoints.omni as omni_module
# Patch the imported function and class in the module
monkeypatch.setattr(omni_llm_module, "load_stage_configs_from_model", _fake_loader)
monkeypatch.setattr(omni_llm_module, "OmniStage", lambda cfg: _FakeStage(cfg))
monkeypatch.setattr(omni_module, "load_stage_configs_from_model", _fake_loader)
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg: _FakeStage(cfg))
from vllm_omni.entrypoints.omni_llm import OmniLLM
from vllm_omni.entrypoints.omni import Omni
llm = OmniLLM(model="any", init_timeout=1)
omni = Omni(model="any", init_timeout=1)
# Verify: auto-loaded stage_configs and stage_list have consistent count
assert isinstance(llm.stage_configs, list)
assert len(llm.stage_configs) == 2
assert len(llm.stage_list) == 2
assert isinstance(omni.stage_configs, list)
assert len(omni.stage_configs) == 2
assert len(omni.stage_list) == 2
# Verify: each Stage is _FakeStage instance
for st in llm.stage_list:
for st in omni.stage_list:
assert isinstance(st, _FakeStage)
# Verify: queues are attached
for st in llm.stage_list:
for st in omni.stage_list:
assert st._in_q is not None
assert st._out_q is not None
# Verify: all stages are ready
assert len(llm._stages_ready) == 2
assert len(omni._stages_ready) == 2
def test_generate_raises_on_length_mismatch(monkeypatch, fake_stage_config):
@@ -550,7 +541,8 @@ def test_generate_raises_on_length_mismatch(monkeypatch, fake_stage_config):
for module_name in [
"vllm_omni.entrypoints.utils",
"vllm_omni.entrypoints.omni_llm",
"vllm_omni.entrypoints.omni",
"vllm_omni.entrypoints.log_utils",
"vllm_omni.entrypoints.omni_stage",
]:
if module_name in sys.modules:
@@ -562,7 +554,7 @@ def test_generate_raises_on_length_mismatch(monkeypatch, fake_stage_config):
_setup_log_mocks(monkeypatch)
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_llm.remove_old_logs",
"vllm_omni.entrypoints.log_utils.remove_old_logs",
lambda log_file, num_stages: None,
raising=False,
)
@@ -573,7 +565,7 @@ def test_generate_raises_on_length_mismatch(monkeypatch, fake_stage_config):
raising=False,
)
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_llm.load_stage_configs_from_model",
"vllm_omni.entrypoints.utils.load_stage_configs_from_model",
_fake_loader,
raising=False,
)
@@ -583,16 +575,16 @@ def test_generate_raises_on_length_mismatch(monkeypatch, fake_stage_config):
raising=False,
)
import vllm_omni.entrypoints.omni_llm as omni_llm_module
import vllm_omni.entrypoints.omni as omni_module
monkeypatch.setattr(omni_llm_module, "load_stage_configs_from_model", _fake_loader)
monkeypatch.setattr(omni_llm_module, "OmniStage", lambda cfg: _FakeStage(cfg))
monkeypatch.setattr(omni_module, "load_stage_configs_from_model", _fake_loader)
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg: _FakeStage(cfg))
from vllm_omni.entrypoints.omni_llm import OmniLLM
from vllm_omni.entrypoints.omni import Omni
llm = OmniLLM(model="any", init_timeout=1)
omni = Omni(model="any", init_timeout=1)
with pytest.raises(ValueError):
llm.generate(prompts=["hi"], sampling_params_list=[])
omni.generate(prompts=["hi"], sampling_params_list=[])
def test_generate_pipeline_and_final_outputs(monkeypatch, fake_stage_config):
@@ -608,7 +600,8 @@ def test_generate_pipeline_and_final_outputs(monkeypatch, fake_stage_config):
for module_name in [
"vllm_omni.entrypoints.utils",
"vllm_omni.entrypoints.omni_llm",
"vllm_omni.entrypoints.omni",
"vllm_omni.entrypoints.log_utils",
"vllm_omni.entrypoints.omni_stage",
]:
if module_name in sys.modules:
@@ -620,7 +613,7 @@ def test_generate_pipeline_and_final_outputs(monkeypatch, fake_stage_config):
_setup_log_mocks(monkeypatch)
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_llm.remove_old_logs",
"vllm_omni.entrypoints.log_utils.remove_old_logs",
lambda log_file, num_stages: None,
raising=False,
)
@@ -631,7 +624,7 @@ def test_generate_pipeline_and_final_outputs(monkeypatch, fake_stage_config):
raising=False,
)
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_llm.load_stage_configs_from_model",
"vllm_omni.entrypoints.utils.load_stage_configs_from_model",
_fake_loader,
raising=False,
)
@@ -641,19 +634,19 @@ def test_generate_pipeline_and_final_outputs(monkeypatch, fake_stage_config):
raising=False,
)
import vllm_omni.entrypoints.omni_llm as omni_llm_module
import vllm_omni.entrypoints.omni as omni_module
monkeypatch.setattr(omni_llm_module, "load_stage_configs_from_model", _fake_loader)
monkeypatch.setattr(omni_llm_module, "OmniStage", lambda cfg: _FakeStage(cfg))
monkeypatch.setattr(omni_module, "load_stage_configs_from_model", _fake_loader)
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg: _FakeStage(cfg))
# Mock uuid.uuid4() to return a predictable value for request ID generation
test_uuid = uuid.UUID("00000000-0000-0000-0000-000000000000")
monkeypatch.setattr(uuid, "uuid4", lambda: test_uuid)
monkeypatch.setattr(omni_llm_module, "uuid", uuid)
monkeypatch.setattr(omni_module, "uuid", uuid)
from vllm_omni.entrypoints.omni_llm import OmniLLM
from vllm_omni.entrypoints.omni import Omni
llm = OmniLLM(model="any", init_timeout=1)
omni = Omni(model="any", init_timeout=1)
# Generate the expected request ID format: "0_<uuid>"
expected_request_id = f"0_{test_uuid}"
@@ -662,7 +655,7 @@ def test_generate_pipeline_and_final_outputs(monkeypatch, fake_stage_config):
# Note: We put results before calling generate, which simulates worker processes
# that have already completed. The polling loop will collect them in stage order.
# Stage 0 output (will be collected first)
llm.stage_list[0]._out_q.put_nowait(
omni.stage_list[0]._out_q.put_nowait(
{
"request_id": expected_request_id,
"engine_outputs": [{"stage": 0, "text": "s0"}],
@@ -674,7 +667,7 @@ def test_generate_pipeline_and_final_outputs(monkeypatch, fake_stage_config):
# but for testing we pre-populate it. The polling loop processes stages
# in order, so stage 0 result will be collected first, then forwarded,
# then stage 1 result will be collected.
llm.stage_list[1]._out_q.put_nowait(
omni.stage_list[1]._out_q.put_nowait(
{
"request_id": expected_request_id,
"engine_outputs": [{"stage": 1, "text": "s1"}],
@@ -685,17 +678,17 @@ def test_generate_pipeline_and_final_outputs(monkeypatch, fake_stage_config):
# Use dicts instead of object() for serializable sampling params
sampling_params_list = [{"temperature": 0.7}, {"temperature": 0.8}]
prompts = ["hi"]
outputs = llm.generate(prompts=prompts, sampling_params_list=sampling_params_list)
outputs = omni.generate(prompts=prompts, sampling_params_list=sampling_params_list)
# Both stages have final_output=True, so should aggregate two OmniRequestOutput
assert len(outputs) == 2
# Verify stage outputs are set
assert llm.stage_list[0].engine_outputs == [{"stage": 0, "text": "s0"}]
assert llm.stage_list[1].engine_outputs == [{"stage": 1, "text": "s1"}]
assert omni.stage_list[0].engine_outputs == [{"stage": 0, "text": "s0"}]
assert omni.stage_list[1].engine_outputs == [{"stage": 1, "text": "s1"}]
# Verify stage 0 input queue received the task
assert not llm.stage_list[0]._in_q.empty()
assert not omni.stage_list[0]._in_q.empty()
# Verify stage 1 received forwarded task (process_engine_inputs was called)
assert llm.stage_list[1].process_engine_inputs([], []) is not None
assert omni.stage_list[1].process_engine_inputs([], []) is not None
def test_generate_no_final_output_returns_empty(monkeypatch, fake_stage_config):
@@ -712,7 +705,8 @@ def test_generate_no_final_output_returns_empty(monkeypatch, fake_stage_config):
for module_name in [
"vllm_omni.entrypoints.utils",
"vllm_omni.entrypoints.omni_llm",
"vllm_omni.entrypoints.omni",
"vllm_omni.entrypoints.log_utils",
"vllm_omni.entrypoints.omni_stage",
]:
if module_name in sys.modules:
@@ -724,7 +718,7 @@ def test_generate_no_final_output_returns_empty(monkeypatch, fake_stage_config):
_setup_log_mocks(monkeypatch)
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_llm.remove_old_logs",
"vllm_omni.entrypoints.log_utils.remove_old_logs",
lambda log_file, num_stages: None,
raising=False,
)
@@ -735,7 +729,83 @@ def test_generate_no_final_output_returns_empty(monkeypatch, fake_stage_config):
raising=False,
)
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_llm.load_stage_configs_from_model",
"vllm_omni.entrypoints.omni_stage.OmniStage",
lambda cfg: _FakeStage(cfg),
raising=False,
)
import vllm_omni.entrypoints.omni as omni_module
monkeypatch.setattr(omni_module, "load_stage_configs_from_model", _fake_loader)
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg: _FakeStage(cfg))
# Mock uuid.uuid4() to return a predictable value for request ID generation
test_uuid = uuid.UUID("00000000-0000-0000-0000-000000000000")
monkeypatch.setattr(uuid, "uuid4", lambda: test_uuid)
monkeypatch.setattr(omni_module, "uuid", uuid)
from vllm_omni.entrypoints.omni import Omni
omni = Omni(model="any", init_timeout=1)
# Generate the expected request ID format: "0_<uuid>"
expected_request_id = f"0_{test_uuid}"
# Simulate worker behavior: put results into output queues
omni.stage_list[0]._out_q.put_nowait(
{
"request_id": expected_request_id,
"engine_outputs": [{"stage": 0}],
"metrics": {"num_tokens_out": 1, "stage_gen_time_ms": 10.0},
}
)
omni.stage_list[1]._out_q.put_nowait(
{
"request_id": expected_request_id,
"engine_outputs": [{"stage": 1}],
"metrics": {"num_tokens_out": 1, "stage_gen_time_ms": 10.0},
}
)
# Use dicts instead of object() for serializable sampling params
outputs = omni.generate(prompts=["p"], sampling_params_list=[{"temperature": 0.7}, {"temperature": 0.8}])
assert outputs == []
def test_generate_sampling_params_none_use_default(monkeypatch, fake_stage_config):
"""Test that generate uses default sampling params when sampling_params_list is None."""
stage_cfg0 = dict(fake_stage_config)
stage_cfg1 = dict(fake_stage_config)
stage_cfg0["final_output"] = False
stage_cfg1["final_output"] = False
def _fake_loader(model: str):
return [_FakeStageConfig(stage_cfg0), _FakeStageConfig(stage_cfg1)]
import sys
for module_name in [
"vllm_omni.entrypoints.utils",
"vllm_omni.entrypoints.omni",
"vllm_omni.entrypoints.log_utils",
"vllm_omni.entrypoints.omni_stage",
]:
if module_name in sys.modules:
del sys.modules[module_name]
_setup_engine_mocks(monkeypatch)
_setup_multiprocessing_mocks(monkeypatch)
_setup_ipc_mocks(monkeypatch)
_setup_log_mocks(monkeypatch)
monkeypatch.setattr(
"vllm_omni.entrypoints.log_utils.remove_old_logs",
lambda log_file, num_stages: None,
raising=False,
)
monkeypatch.setattr(
"vllm_omni.entrypoints.utils.load_stage_configs_from_model",
_fake_loader,
raising=False,
)
@@ -745,42 +815,40 @@ def test_generate_no_final_output_returns_empty(monkeypatch, fake_stage_config):
raising=False,
)
import vllm_omni.entrypoints.omni_llm as omni_llm_module
import vllm_omni.entrypoints.omni as omni_module
monkeypatch.setattr(omni_llm_module, "load_stage_configs_from_model", _fake_loader)
monkeypatch.setattr(omni_llm_module, "OmniStage", lambda cfg: _FakeStage(cfg))
monkeypatch.setattr(omni_module, "load_stage_configs_from_model", _fake_loader)
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg: _FakeStage(cfg))
# Mock uuid.uuid4() to return a predictable value for request ID generation
test_uuid = uuid.UUID("00000000-0000-0000-0000-000000000000")
monkeypatch.setattr(uuid, "uuid4", lambda: test_uuid)
monkeypatch.setattr(omni_llm_module, "uuid", uuid)
monkeypatch.setattr(omni_module, "uuid", uuid)
from vllm_omni.entrypoints.omni_llm import OmniLLM
from vllm_omni.entrypoints.omni import Omni
llm = OmniLLM(model="any", init_timeout=1)
omni = Omni(model="any", init_timeout=1)
# Generate the expected request ID format: "0_<uuid>"
expected_request_id = f"0_{test_uuid}"
# Simulate worker behavior: put results into output queues
llm.stage_list[0]._out_q.put_nowait(
omni.stage_list[0]._out_q.put_nowait(
{
"request_id": expected_request_id,
"engine_outputs": [{"stage": 0}],
"metrics": {"num_tokens_out": 1, "stage_gen_time_ms": 10.0},
}
)
llm.stage_list[1]._out_q.put_nowait(
omni.stage_list[1]._out_q.put_nowait(
{
"request_id": expected_request_id,
"engine_outputs": [{"stage": 1}],
"metrics": {"num_tokens_out": 1, "stage_gen_time_ms": 10.0},
}
)
# Use dicts instead of object() for serializable sampling params
outputs = llm.generate(prompts=["p"], sampling_params_list=[{"temperature": 0.7}, {"temperature": 0.8}])
assert outputs == []
# Use the default sampling params
omni.generate(prompts=["p"], sampling_params_list=None)
def test_wait_for_stages_ready_timeout(monkeypatch, fake_stage_config):
@@ -793,7 +861,8 @@ def test_wait_for_stages_ready_timeout(monkeypatch, fake_stage_config):
for module_name in [
"vllm_omni.entrypoints.utils",
"vllm_omni.entrypoints.omni_llm",
"vllm_omni.entrypoints.omni",
"vllm_omni.entrypoints.log_utils",
"vllm_omni.entrypoints.omni_stage",
]:
if module_name in sys.modules:
@@ -805,7 +874,7 @@ def test_wait_for_stages_ready_timeout(monkeypatch, fake_stage_config):
_setup_log_mocks(monkeypatch)
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_llm.remove_old_logs",
"vllm_omni.entrypoints.log_utils.remove_old_logs",
lambda log_file, num_stages: None,
raising=False,
)
@@ -832,17 +901,17 @@ def test_wait_for_stages_ready_timeout(monkeypatch, fake_stage_config):
raising=False,
)
import vllm_omni.entrypoints.omni_llm as omni_llm_module
import vllm_omni.entrypoints.omni as omni_module
monkeypatch.setattr(omni_llm_module, "load_stage_configs_from_model", _fake_loader)
monkeypatch.setattr(omni_llm_module, "OmniStage", lambda cfg: _FakeStageNoReady(cfg))
monkeypatch.setattr(omni_module, "load_stage_configs_from_model", _fake_loader)
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg: _FakeStageNoReady(cfg))
from vllm_omni.entrypoints.omni_llm import OmniLLM
from vllm_omni.entrypoints.omni import Omni
# Use very short timeout
llm = OmniLLM(model="any", init_timeout=0.01)
omni = Omni(model="any", init_timeout=0.01)
# Verify that no stages are ready
assert len(llm._stages_ready) == 0
assert len(omni._stages_ready) == 0
def test_generate_handles_error_messages(monkeypatch, fake_stage_config):
@@ -855,7 +924,8 @@ def test_generate_handles_error_messages(monkeypatch, fake_stage_config):
for module_name in [
"vllm_omni.entrypoints.utils",
"vllm_omni.entrypoints.omni_llm",
"vllm_omni.entrypoints.omni",
"vllm_omni.entrypoints.log_utils",
"vllm_omni.entrypoints.omni_stage",
]:
if module_name in sys.modules:
@@ -867,7 +937,7 @@ def test_generate_handles_error_messages(monkeypatch, fake_stage_config):
_setup_log_mocks(monkeypatch)
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_llm.remove_old_logs",
"vllm_omni.entrypoints.log_utils.remove_old_logs",
lambda log_file, num_stages: None,
raising=False,
)
@@ -877,36 +947,31 @@ def test_generate_handles_error_messages(monkeypatch, fake_stage_config):
_fake_loader,
raising=False,
)
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_llm.load_stage_configs_from_model",
_fake_loader,
raising=False,
)
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_stage.OmniStage",
lambda cfg: _FakeStage(cfg),
raising=False,
)
import vllm_omni.entrypoints.omni_llm as omni_llm_module
import vllm_omni.entrypoints.omni as omni_module
monkeypatch.setattr(omni_llm_module, "load_stage_configs_from_model", _fake_loader)
monkeypatch.setattr(omni_llm_module, "OmniStage", lambda cfg: _FakeStage(cfg))
monkeypatch.setattr(omni_module, "load_stage_configs_from_model", _fake_loader)
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg: _FakeStage(cfg))
# Mock uuid.uuid4() to return a predictable value for request ID generation
test_uuid = uuid.UUID("00000000-0000-0000-0000-000000000000")
monkeypatch.setattr(uuid, "uuid4", lambda: test_uuid)
monkeypatch.setattr(omni_llm_module, "uuid", uuid)
monkeypatch.setattr(omni_module, "uuid", uuid)
from vllm_omni.entrypoints.omni_llm import OmniLLM
from vllm_omni.entrypoints.omni import Omni
llm = OmniLLM(model="any", init_timeout=1)
omni = Omni(model="any", init_timeout=1)
# Generate the expected request ID format: "0_<uuid>"
expected_request_id = f"0_{test_uuid}"
# Put error message in output queue
llm.stage_list[0]._out_q.put_nowait(
omni.stage_list[0]._out_q.put_nowait(
{
"request_id": expected_request_id,
"error": "test error",
@@ -914,7 +979,7 @@ def test_generate_handles_error_messages(monkeypatch, fake_stage_config):
)
# Also put a valid result after error to allow the loop to complete
# (error handling continues the loop, so we need a valid result to finish)
llm.stage_list[0]._out_q.put_nowait(
omni.stage_list[0]._out_q.put_nowait(
{
"request_id": expected_request_id,
"engine_outputs": [{"stage": 0, "text": "result"}],
@@ -925,7 +990,7 @@ def test_generate_handles_error_messages(monkeypatch, fake_stage_config):
# Generate should handle error gracefully (log but continue)
# Use dict instead of object() for serializable sampling params
sampling_params_list = [{"temperature": 0.7}]
outputs = llm.generate(prompts=["hi"], sampling_params_list=sampling_params_list)
outputs = omni.generate(prompts=["hi"], sampling_params_list=sampling_params_list)
# Should return final output (error was logged but didn't stop processing)
assert isinstance(outputs, list)
# Since final_output=True, should have one output
@@ -942,7 +1007,8 @@ def test_close_sends_shutdown_signal(monkeypatch, fake_stage_config):
for module_name in [
"vllm_omni.entrypoints.utils",
"vllm_omni.entrypoints.omni_llm",
"vllm_omni.entrypoints.omni",
"vllm_omni.entrypoints.log_utils",
"vllm_omni.entrypoints.omni_stage",
]:
if module_name in sys.modules:
@@ -954,7 +1020,7 @@ def test_close_sends_shutdown_signal(monkeypatch, fake_stage_config):
_setup_log_mocks(monkeypatch)
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_llm.remove_old_logs",
"vllm_omni.entrypoints.log_utils.remove_old_logs",
lambda log_file, num_stages: None,
raising=False,
)
@@ -965,7 +1031,7 @@ def test_close_sends_shutdown_signal(monkeypatch, fake_stage_config):
raising=False,
)
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_llm.load_stage_configs_from_model",
"vllm_omni.entrypoints.utils.load_stage_configs_from_model",
_fake_loader,
raising=False,
)
@@ -975,22 +1041,22 @@ def test_close_sends_shutdown_signal(monkeypatch, fake_stage_config):
raising=False,
)
import vllm_omni.entrypoints.omni_llm as omni_llm_module
import vllm_omni.entrypoints.omni as omni_module
monkeypatch.setattr(omni_llm_module, "load_stage_configs_from_model", _fake_loader)
monkeypatch.setattr(omni_llm_module, "OmniStage", lambda cfg: _FakeStage(cfg))
monkeypatch.setattr(omni_module, "load_stage_configs_from_model", _fake_loader)
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg: _FakeStage(cfg))
from vllm_omni.entrypoints.omni_llm import OmniLLM
from vllm_omni.entrypoints.omni import Omni
llm = OmniLLM(model="any", init_timeout=1)
omni = Omni(model="any", init_timeout=1)
# Call close
llm.close()
omni.close()
# Verify shutdown signal (None) was sent to input queue
# Use get_nowait to avoid blocking (close() uses put_nowait, so should be safe)
try:
shutdown_signal = llm.stage_list[0]._in_q.get_nowait()
shutdown_signal = omni.stage_list[0]._in_q.get_nowait()
assert shutdown_signal is None
except Empty:
# If queue was already empty or only had stage_ready, that's also acceptable
@@ -998,4 +1064,4 @@ def test_close_sends_shutdown_signal(monkeypatch, fake_stage_config):
pass
# Verify stop_stage_worker was called (process should be set)
assert llm.stage_list[0]._proc is not None
assert omni.stage_list[0]._proc is not None