[Core] Supports stage abstraction in the diffusion model (#391)
Signed-off-by: Chenguang ZHENG <645327136@qq.com> Signed-off-by: yinpeiqi <yinpeiqi809@gmail.com> Co-authored-by: yinpeiqi <yinpeiqi809@gmail.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
41 files changed
+2473
-1242
No files matched your search
@@ -132,8 +132,9 @@ def test_client(mock_async_diffusion):
|
||||
app.include_router(router)
|
||||
|
||||
# Set up app state with diffusion engine
|
||||
app.state.diffusion_engine = mock_async_diffusion
|
||||
app.state.diffusion_model_name = "Qwen/Qwen-Image"
|
||||
app.state.engine_client = mock_async_diffusion
|
||||
app.state.stage_configs = [{"stage_type": "diffusion"}]
|
||||
app.state.served_model_names = "Qwen/Qwen-Image"
|
||||
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
+172
-106
@@ -83,6 +83,8 @@ class _FakeStage:
|
||||
self.stage_id = getattr(config, "stage_id", 0)
|
||||
self.engine_args = config.engine_args
|
||||
self.model_stage = getattr(config.engine_args, "model_stage", None)
|
||||
# set default sampling params
|
||||
self.default_sampling_params = {"temperature": 1.0}
|
||||
# Allow configuring final_output and final_output_type
|
||||
self.final_output = config.final_output if hasattr(config, "final_output") else False
|
||||
self.final_output_type = getattr(config, "final_output_type", None)
|
||||
@@ -171,18 +173,6 @@ class _FakeEngine:
|
||||
yield from self._outputs
|
||||
|
||||
|
||||
class _FakeStageLLM:
|
||||
"""Replace OmniStageLLM to avoid constructing real engine."""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
# Allow injecting custom fake outputs, default returns single placeholder output
|
||||
fake_outputs = kwargs.get("_fake_outputs", [[{"text": "ok"}]])
|
||||
self._fake_engine = _FakeEngine(fake_outputs)
|
||||
|
||||
def generate(self, prompts, sampling_params, use_tqdm=False):
|
||||
yield from self._fake_engine.generate(prompts, sampling_params)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_stage_config():
|
||||
return {
|
||||
@@ -298,22 +288,22 @@ def _setup_ipc_mocks(monkeypatch):
|
||||
def _fake_set(obj):
|
||||
return str(obj).encode()
|
||||
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.omni_llm._encode", _fake_encode, raising=False)
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.omni_llm._load", _fake_load, raising=False)
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.omni_llm._set", _fake_set, raising=False)
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.omni._encode", _fake_encode, raising=False)
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.omni._load", _fake_load, raising=False)
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.omni._set", _fake_set, raising=False)
|
||||
|
||||
|
||||
def _setup_log_mocks(monkeypatch):
|
||||
"""Helper function to set up logging and stats mocks."""
|
||||
# Mock init_stats_paths to return None files (no stats logging)
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.entrypoints.omni_llm.init_stats_paths",
|
||||
"vllm_omni.entrypoints.omni.init_stats_paths",
|
||||
lambda enable_stats, log_file: (None, None),
|
||||
raising=False,
|
||||
)
|
||||
# Mock configure_orchestrator_logger to do nothing
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.entrypoints.omni_llm.configure_orchestrator_logger",
|
||||
"vllm_omni.entrypoints.omni.configure_orchestrator_logger",
|
||||
lambda logger, log_file: None,
|
||||
raising=False,
|
||||
)
|
||||
@@ -340,7 +330,7 @@ def _setup_log_mocks(monkeypatch):
|
||||
return "Fake summary"
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.entrypoints.omni_llm.OrchestratorMetrics",
|
||||
"vllm_omni.entrypoints.omni.OrchestratorMetrics",
|
||||
_FakeOrchestratorMetrics,
|
||||
raising=False,
|
||||
)
|
||||
@@ -482,7 +472,8 @@ def test_initialize_stage_configs_called_when_none(monkeypatch, fake_stage_confi
|
||||
|
||||
for module_name in [
|
||||
"vllm_omni.entrypoints.utils",
|
||||
"vllm_omni.entrypoints.omni_llm",
|
||||
"vllm_omni.entrypoints.omni",
|
||||
"vllm_omni.entrypoints.log_utils",
|
||||
"vllm_omni.entrypoints.omni_stage",
|
||||
]:
|
||||
if module_name in sys.modules:
|
||||
@@ -496,7 +487,7 @@ def test_initialize_stage_configs_called_when_none(monkeypatch, fake_stage_confi
|
||||
|
||||
# Mock remove_old_logs to avoid bug (called before stage_list is created)
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.entrypoints.omni_llm.remove_old_logs",
|
||||
"vllm_omni.entrypoints.log_utils.remove_old_logs",
|
||||
lambda log_file, num_stages: None,
|
||||
raising=False,
|
||||
)
|
||||
@@ -516,28 +507,28 @@ def test_initialize_stage_configs_called_when_none(monkeypatch, fake_stage_confi
|
||||
)
|
||||
|
||||
# Import the module after mocks are set
|
||||
import vllm_omni.entrypoints.omni_llm as omni_llm_module
|
||||
import vllm_omni.entrypoints.omni as omni_module
|
||||
|
||||
# Patch the imported function and class in the module
|
||||
monkeypatch.setattr(omni_llm_module, "load_stage_configs_from_model", _fake_loader)
|
||||
monkeypatch.setattr(omni_llm_module, "OmniStage", lambda cfg: _FakeStage(cfg))
|
||||
monkeypatch.setattr(omni_module, "load_stage_configs_from_model", _fake_loader)
|
||||
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg: _FakeStage(cfg))
|
||||
|
||||
from vllm_omni.entrypoints.omni_llm import OmniLLM
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
|
||||
llm = OmniLLM(model="any", init_timeout=1)
|
||||
omni = Omni(model="any", init_timeout=1)
|
||||
# Verify: auto-loaded stage_configs and stage_list have consistent count
|
||||
assert isinstance(llm.stage_configs, list)
|
||||
assert len(llm.stage_configs) == 2
|
||||
assert len(llm.stage_list) == 2
|
||||
assert isinstance(omni.stage_configs, list)
|
||||
assert len(omni.stage_configs) == 2
|
||||
assert len(omni.stage_list) == 2
|
||||
# Verify: each Stage is _FakeStage instance
|
||||
for st in llm.stage_list:
|
||||
for st in omni.stage_list:
|
||||
assert isinstance(st, _FakeStage)
|
||||
# Verify: queues are attached
|
||||
for st in llm.stage_list:
|
||||
for st in omni.stage_list:
|
||||
assert st._in_q is not None
|
||||
assert st._out_q is not None
|
||||
# Verify: all stages are ready
|
||||
assert len(llm._stages_ready) == 2
|
||||
assert len(omni._stages_ready) == 2
|
||||
|
||||
|
||||
def test_generate_raises_on_length_mismatch(monkeypatch, fake_stage_config):
|
||||
@@ -550,7 +541,8 @@ def test_generate_raises_on_length_mismatch(monkeypatch, fake_stage_config):
|
||||
|
||||
for module_name in [
|
||||
"vllm_omni.entrypoints.utils",
|
||||
"vllm_omni.entrypoints.omni_llm",
|
||||
"vllm_omni.entrypoints.omni",
|
||||
"vllm_omni.entrypoints.log_utils",
|
||||
"vllm_omni.entrypoints.omni_stage",
|
||||
]:
|
||||
if module_name in sys.modules:
|
||||
@@ -562,7 +554,7 @@ def test_generate_raises_on_length_mismatch(monkeypatch, fake_stage_config):
|
||||
_setup_log_mocks(monkeypatch)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.entrypoints.omni_llm.remove_old_logs",
|
||||
"vllm_omni.entrypoints.log_utils.remove_old_logs",
|
||||
lambda log_file, num_stages: None,
|
||||
raising=False,
|
||||
)
|
||||
@@ -573,7 +565,7 @@ def test_generate_raises_on_length_mismatch(monkeypatch, fake_stage_config):
|
||||
raising=False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.entrypoints.omni_llm.load_stage_configs_from_model",
|
||||
"vllm_omni.entrypoints.utils.load_stage_configs_from_model",
|
||||
_fake_loader,
|
||||
raising=False,
|
||||
)
|
||||
@@ -583,16 +575,16 @@ def test_generate_raises_on_length_mismatch(monkeypatch, fake_stage_config):
|
||||
raising=False,
|
||||
)
|
||||
|
||||
import vllm_omni.entrypoints.omni_llm as omni_llm_module
|
||||
import vllm_omni.entrypoints.omni as omni_module
|
||||
|
||||
monkeypatch.setattr(omni_llm_module, "load_stage_configs_from_model", _fake_loader)
|
||||
monkeypatch.setattr(omni_llm_module, "OmniStage", lambda cfg: _FakeStage(cfg))
|
||||
monkeypatch.setattr(omni_module, "load_stage_configs_from_model", _fake_loader)
|
||||
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg: _FakeStage(cfg))
|
||||
|
||||
from vllm_omni.entrypoints.omni_llm import OmniLLM
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
|
||||
llm = OmniLLM(model="any", init_timeout=1)
|
||||
omni = Omni(model="any", init_timeout=1)
|
||||
with pytest.raises(ValueError):
|
||||
llm.generate(prompts=["hi"], sampling_params_list=[])
|
||||
omni.generate(prompts=["hi"], sampling_params_list=[])
|
||||
|
||||
|
||||
def test_generate_pipeline_and_final_outputs(monkeypatch, fake_stage_config):
|
||||
@@ -608,7 +600,8 @@ def test_generate_pipeline_and_final_outputs(monkeypatch, fake_stage_config):
|
||||
|
||||
for module_name in [
|
||||
"vllm_omni.entrypoints.utils",
|
||||
"vllm_omni.entrypoints.omni_llm",
|
||||
"vllm_omni.entrypoints.omni",
|
||||
"vllm_omni.entrypoints.log_utils",
|
||||
"vllm_omni.entrypoints.omni_stage",
|
||||
]:
|
||||
if module_name in sys.modules:
|
||||
@@ -620,7 +613,7 @@ def test_generate_pipeline_and_final_outputs(monkeypatch, fake_stage_config):
|
||||
_setup_log_mocks(monkeypatch)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.entrypoints.omni_llm.remove_old_logs",
|
||||
"vllm_omni.entrypoints.log_utils.remove_old_logs",
|
||||
lambda log_file, num_stages: None,
|
||||
raising=False,
|
||||
)
|
||||
@@ -631,7 +624,7 @@ def test_generate_pipeline_and_final_outputs(monkeypatch, fake_stage_config):
|
||||
raising=False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.entrypoints.omni_llm.load_stage_configs_from_model",
|
||||
"vllm_omni.entrypoints.utils.load_stage_configs_from_model",
|
||||
_fake_loader,
|
||||
raising=False,
|
||||
)
|
||||
@@ -641,19 +634,19 @@ def test_generate_pipeline_and_final_outputs(monkeypatch, fake_stage_config):
|
||||
raising=False,
|
||||
)
|
||||
|
||||
import vllm_omni.entrypoints.omni_llm as omni_llm_module
|
||||
import vllm_omni.entrypoints.omni as omni_module
|
||||
|
||||
monkeypatch.setattr(omni_llm_module, "load_stage_configs_from_model", _fake_loader)
|
||||
monkeypatch.setattr(omni_llm_module, "OmniStage", lambda cfg: _FakeStage(cfg))
|
||||
monkeypatch.setattr(omni_module, "load_stage_configs_from_model", _fake_loader)
|
||||
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg: _FakeStage(cfg))
|
||||
|
||||
# Mock uuid.uuid4() to return a predictable value for request ID generation
|
||||
test_uuid = uuid.UUID("00000000-0000-0000-0000-000000000000")
|
||||
monkeypatch.setattr(uuid, "uuid4", lambda: test_uuid)
|
||||
monkeypatch.setattr(omni_llm_module, "uuid", uuid)
|
||||
monkeypatch.setattr(omni_module, "uuid", uuid)
|
||||
|
||||
from vllm_omni.entrypoints.omni_llm import OmniLLM
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
|
||||
llm = OmniLLM(model="any", init_timeout=1)
|
||||
omni = Omni(model="any", init_timeout=1)
|
||||
|
||||
# Generate the expected request ID format: "0_<uuid>"
|
||||
expected_request_id = f"0_{test_uuid}"
|
||||
@@ -662,7 +655,7 @@ def test_generate_pipeline_and_final_outputs(monkeypatch, fake_stage_config):
|
||||
# Note: We put results before calling generate, which simulates worker processes
|
||||
# that have already completed. The polling loop will collect them in stage order.
|
||||
# Stage 0 output (will be collected first)
|
||||
llm.stage_list[0]._out_q.put_nowait(
|
||||
omni.stage_list[0]._out_q.put_nowait(
|
||||
{
|
||||
"request_id": expected_request_id,
|
||||
"engine_outputs": [{"stage": 0, "text": "s0"}],
|
||||
@@ -674,7 +667,7 @@ def test_generate_pipeline_and_final_outputs(monkeypatch, fake_stage_config):
|
||||
# but for testing we pre-populate it. The polling loop processes stages
|
||||
# in order, so stage 0 result will be collected first, then forwarded,
|
||||
# then stage 1 result will be collected.
|
||||
llm.stage_list[1]._out_q.put_nowait(
|
||||
omni.stage_list[1]._out_q.put_nowait(
|
||||
{
|
||||
"request_id": expected_request_id,
|
||||
"engine_outputs": [{"stage": 1, "text": "s1"}],
|
||||
@@ -685,17 +678,17 @@ def test_generate_pipeline_and_final_outputs(monkeypatch, fake_stage_config):
|
||||
# Use dicts instead of object() for serializable sampling params
|
||||
sampling_params_list = [{"temperature": 0.7}, {"temperature": 0.8}]
|
||||
prompts = ["hi"]
|
||||
outputs = llm.generate(prompts=prompts, sampling_params_list=sampling_params_list)
|
||||
outputs = omni.generate(prompts=prompts, sampling_params_list=sampling_params_list)
|
||||
|
||||
# Both stages have final_output=True, so should aggregate two OmniRequestOutput
|
||||
assert len(outputs) == 2
|
||||
# Verify stage outputs are set
|
||||
assert llm.stage_list[0].engine_outputs == [{"stage": 0, "text": "s0"}]
|
||||
assert llm.stage_list[1].engine_outputs == [{"stage": 1, "text": "s1"}]
|
||||
assert omni.stage_list[0].engine_outputs == [{"stage": 0, "text": "s0"}]
|
||||
assert omni.stage_list[1].engine_outputs == [{"stage": 1, "text": "s1"}]
|
||||
# Verify stage 0 input queue received the task
|
||||
assert not llm.stage_list[0]._in_q.empty()
|
||||
assert not omni.stage_list[0]._in_q.empty()
|
||||
# Verify stage 1 received forwarded task (process_engine_inputs was called)
|
||||
assert llm.stage_list[1].process_engine_inputs([], []) is not None
|
||||
assert omni.stage_list[1].process_engine_inputs([], []) is not None
|
||||
|
||||
|
||||
def test_generate_no_final_output_returns_empty(monkeypatch, fake_stage_config):
|
||||
@@ -712,7 +705,8 @@ def test_generate_no_final_output_returns_empty(monkeypatch, fake_stage_config):
|
||||
|
||||
for module_name in [
|
||||
"vllm_omni.entrypoints.utils",
|
||||
"vllm_omni.entrypoints.omni_llm",
|
||||
"vllm_omni.entrypoints.omni",
|
||||
"vllm_omni.entrypoints.log_utils",
|
||||
"vllm_omni.entrypoints.omni_stage",
|
||||
]:
|
||||
if module_name in sys.modules:
|
||||
@@ -724,7 +718,7 @@ def test_generate_no_final_output_returns_empty(monkeypatch, fake_stage_config):
|
||||
_setup_log_mocks(monkeypatch)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.entrypoints.omni_llm.remove_old_logs",
|
||||
"vllm_omni.entrypoints.log_utils.remove_old_logs",
|
||||
lambda log_file, num_stages: None,
|
||||
raising=False,
|
||||
)
|
||||
@@ -735,7 +729,83 @@ def test_generate_no_final_output_returns_empty(monkeypatch, fake_stage_config):
|
||||
raising=False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.entrypoints.omni_llm.load_stage_configs_from_model",
|
||||
"vllm_omni.entrypoints.omni_stage.OmniStage",
|
||||
lambda cfg: _FakeStage(cfg),
|
||||
raising=False,
|
||||
)
|
||||
|
||||
import vllm_omni.entrypoints.omni as omni_module
|
||||
|
||||
monkeypatch.setattr(omni_module, "load_stage_configs_from_model", _fake_loader)
|
||||
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg: _FakeStage(cfg))
|
||||
|
||||
# Mock uuid.uuid4() to return a predictable value for request ID generation
|
||||
test_uuid = uuid.UUID("00000000-0000-0000-0000-000000000000")
|
||||
monkeypatch.setattr(uuid, "uuid4", lambda: test_uuid)
|
||||
monkeypatch.setattr(omni_module, "uuid", uuid)
|
||||
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
|
||||
omni = Omni(model="any", init_timeout=1)
|
||||
|
||||
# Generate the expected request ID format: "0_<uuid>"
|
||||
expected_request_id = f"0_{test_uuid}"
|
||||
|
||||
# Simulate worker behavior: put results into output queues
|
||||
omni.stage_list[0]._out_q.put_nowait(
|
||||
{
|
||||
"request_id": expected_request_id,
|
||||
"engine_outputs": [{"stage": 0}],
|
||||
"metrics": {"num_tokens_out": 1, "stage_gen_time_ms": 10.0},
|
||||
}
|
||||
)
|
||||
omni.stage_list[1]._out_q.put_nowait(
|
||||
{
|
||||
"request_id": expected_request_id,
|
||||
"engine_outputs": [{"stage": 1}],
|
||||
"metrics": {"num_tokens_out": 1, "stage_gen_time_ms": 10.0},
|
||||
}
|
||||
)
|
||||
|
||||
# Use dicts instead of object() for serializable sampling params
|
||||
outputs = omni.generate(prompts=["p"], sampling_params_list=[{"temperature": 0.7}, {"temperature": 0.8}])
|
||||
assert outputs == []
|
||||
|
||||
|
||||
def test_generate_sampling_params_none_use_default(monkeypatch, fake_stage_config):
|
||||
"""Test that generate uses default sampling params when sampling_params_list is None."""
|
||||
stage_cfg0 = dict(fake_stage_config)
|
||||
stage_cfg1 = dict(fake_stage_config)
|
||||
stage_cfg0["final_output"] = False
|
||||
stage_cfg1["final_output"] = False
|
||||
|
||||
def _fake_loader(model: str):
|
||||
return [_FakeStageConfig(stage_cfg0), _FakeStageConfig(stage_cfg1)]
|
||||
|
||||
import sys
|
||||
|
||||
for module_name in [
|
||||
"vllm_omni.entrypoints.utils",
|
||||
"vllm_omni.entrypoints.omni",
|
||||
"vllm_omni.entrypoints.log_utils",
|
||||
"vllm_omni.entrypoints.omni_stage",
|
||||
]:
|
||||
if module_name in sys.modules:
|
||||
del sys.modules[module_name]
|
||||
|
||||
_setup_engine_mocks(monkeypatch)
|
||||
_setup_multiprocessing_mocks(monkeypatch)
|
||||
_setup_ipc_mocks(monkeypatch)
|
||||
_setup_log_mocks(monkeypatch)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.entrypoints.log_utils.remove_old_logs",
|
||||
lambda log_file, num_stages: None,
|
||||
raising=False,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.entrypoints.utils.load_stage_configs_from_model",
|
||||
_fake_loader,
|
||||
raising=False,
|
||||
)
|
||||
@@ -745,42 +815,40 @@ def test_generate_no_final_output_returns_empty(monkeypatch, fake_stage_config):
|
||||
raising=False,
|
||||
)
|
||||
|
||||
import vllm_omni.entrypoints.omni_llm as omni_llm_module
|
||||
import vllm_omni.entrypoints.omni as omni_module
|
||||
|
||||
monkeypatch.setattr(omni_llm_module, "load_stage_configs_from_model", _fake_loader)
|
||||
monkeypatch.setattr(omni_llm_module, "OmniStage", lambda cfg: _FakeStage(cfg))
|
||||
monkeypatch.setattr(omni_module, "load_stage_configs_from_model", _fake_loader)
|
||||
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg: _FakeStage(cfg))
|
||||
|
||||
# Mock uuid.uuid4() to return a predictable value for request ID generation
|
||||
test_uuid = uuid.UUID("00000000-0000-0000-0000-000000000000")
|
||||
monkeypatch.setattr(uuid, "uuid4", lambda: test_uuid)
|
||||
monkeypatch.setattr(omni_llm_module, "uuid", uuid)
|
||||
monkeypatch.setattr(omni_module, "uuid", uuid)
|
||||
|
||||
from vllm_omni.entrypoints.omni_llm import OmniLLM
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
|
||||
llm = OmniLLM(model="any", init_timeout=1)
|
||||
omni = Omni(model="any", init_timeout=1)
|
||||
|
||||
# Generate the expected request ID format: "0_<uuid>"
|
||||
expected_request_id = f"0_{test_uuid}"
|
||||
|
||||
# Simulate worker behavior: put results into output queues
|
||||
llm.stage_list[0]._out_q.put_nowait(
|
||||
omni.stage_list[0]._out_q.put_nowait(
|
||||
{
|
||||
"request_id": expected_request_id,
|
||||
"engine_outputs": [{"stage": 0}],
|
||||
"metrics": {"num_tokens_out": 1, "stage_gen_time_ms": 10.0},
|
||||
}
|
||||
)
|
||||
llm.stage_list[1]._out_q.put_nowait(
|
||||
omni.stage_list[1]._out_q.put_nowait(
|
||||
{
|
||||
"request_id": expected_request_id,
|
||||
"engine_outputs": [{"stage": 1}],
|
||||
"metrics": {"num_tokens_out": 1, "stage_gen_time_ms": 10.0},
|
||||
}
|
||||
)
|
||||
|
||||
# Use dicts instead of object() for serializable sampling params
|
||||
outputs = llm.generate(prompts=["p"], sampling_params_list=[{"temperature": 0.7}, {"temperature": 0.8}])
|
||||
assert outputs == []
|
||||
# Use the default sampling params
|
||||
omni.generate(prompts=["p"], sampling_params_list=None)
|
||||
|
||||
|
||||
def test_wait_for_stages_ready_timeout(monkeypatch, fake_stage_config):
|
||||
@@ -793,7 +861,8 @@ def test_wait_for_stages_ready_timeout(monkeypatch, fake_stage_config):
|
||||
|
||||
for module_name in [
|
||||
"vllm_omni.entrypoints.utils",
|
||||
"vllm_omni.entrypoints.omni_llm",
|
||||
"vllm_omni.entrypoints.omni",
|
||||
"vllm_omni.entrypoints.log_utils",
|
||||
"vllm_omni.entrypoints.omni_stage",
|
||||
]:
|
||||
if module_name in sys.modules:
|
||||
@@ -805,7 +874,7 @@ def test_wait_for_stages_ready_timeout(monkeypatch, fake_stage_config):
|
||||
_setup_log_mocks(monkeypatch)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.entrypoints.omni_llm.remove_old_logs",
|
||||
"vllm_omni.entrypoints.log_utils.remove_old_logs",
|
||||
lambda log_file, num_stages: None,
|
||||
raising=False,
|
||||
)
|
||||
@@ -832,17 +901,17 @@ def test_wait_for_stages_ready_timeout(monkeypatch, fake_stage_config):
|
||||
raising=False,
|
||||
)
|
||||
|
||||
import vllm_omni.entrypoints.omni_llm as omni_llm_module
|
||||
import vllm_omni.entrypoints.omni as omni_module
|
||||
|
||||
monkeypatch.setattr(omni_llm_module, "load_stage_configs_from_model", _fake_loader)
|
||||
monkeypatch.setattr(omni_llm_module, "OmniStage", lambda cfg: _FakeStageNoReady(cfg))
|
||||
monkeypatch.setattr(omni_module, "load_stage_configs_from_model", _fake_loader)
|
||||
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg: _FakeStageNoReady(cfg))
|
||||
|
||||
from vllm_omni.entrypoints.omni_llm import OmniLLM
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
|
||||
# Use very short timeout
|
||||
llm = OmniLLM(model="any", init_timeout=0.01)
|
||||
omni = Omni(model="any", init_timeout=0.01)
|
||||
# Verify that no stages are ready
|
||||
assert len(llm._stages_ready) == 0
|
||||
assert len(omni._stages_ready) == 0
|
||||
|
||||
|
||||
def test_generate_handles_error_messages(monkeypatch, fake_stage_config):
|
||||
@@ -855,7 +924,8 @@ def test_generate_handles_error_messages(monkeypatch, fake_stage_config):
|
||||
|
||||
for module_name in [
|
||||
"vllm_omni.entrypoints.utils",
|
||||
"vllm_omni.entrypoints.omni_llm",
|
||||
"vllm_omni.entrypoints.omni",
|
||||
"vllm_omni.entrypoints.log_utils",
|
||||
"vllm_omni.entrypoints.omni_stage",
|
||||
]:
|
||||
if module_name in sys.modules:
|
||||
@@ -867,7 +937,7 @@ def test_generate_handles_error_messages(monkeypatch, fake_stage_config):
|
||||
_setup_log_mocks(monkeypatch)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.entrypoints.omni_llm.remove_old_logs",
|
||||
"vllm_omni.entrypoints.log_utils.remove_old_logs",
|
||||
lambda log_file, num_stages: None,
|
||||
raising=False,
|
||||
)
|
||||
@@ -877,36 +947,31 @@ def test_generate_handles_error_messages(monkeypatch, fake_stage_config):
|
||||
_fake_loader,
|
||||
raising=False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.entrypoints.omni_llm.load_stage_configs_from_model",
|
||||
_fake_loader,
|
||||
raising=False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.entrypoints.omni_stage.OmniStage",
|
||||
lambda cfg: _FakeStage(cfg),
|
||||
raising=False,
|
||||
)
|
||||
|
||||
import vllm_omni.entrypoints.omni_llm as omni_llm_module
|
||||
import vllm_omni.entrypoints.omni as omni_module
|
||||
|
||||
monkeypatch.setattr(omni_llm_module, "load_stage_configs_from_model", _fake_loader)
|
||||
monkeypatch.setattr(omni_llm_module, "OmniStage", lambda cfg: _FakeStage(cfg))
|
||||
monkeypatch.setattr(omni_module, "load_stage_configs_from_model", _fake_loader)
|
||||
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg: _FakeStage(cfg))
|
||||
|
||||
# Mock uuid.uuid4() to return a predictable value for request ID generation
|
||||
test_uuid = uuid.UUID("00000000-0000-0000-0000-000000000000")
|
||||
monkeypatch.setattr(uuid, "uuid4", lambda: test_uuid)
|
||||
monkeypatch.setattr(omni_llm_module, "uuid", uuid)
|
||||
monkeypatch.setattr(omni_module, "uuid", uuid)
|
||||
|
||||
from vllm_omni.entrypoints.omni_llm import OmniLLM
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
|
||||
llm = OmniLLM(model="any", init_timeout=1)
|
||||
omni = Omni(model="any", init_timeout=1)
|
||||
|
||||
# Generate the expected request ID format: "0_<uuid>"
|
||||
expected_request_id = f"0_{test_uuid}"
|
||||
|
||||
# Put error message in output queue
|
||||
llm.stage_list[0]._out_q.put_nowait(
|
||||
omni.stage_list[0]._out_q.put_nowait(
|
||||
{
|
||||
"request_id": expected_request_id,
|
||||
"error": "test error",
|
||||
@@ -914,7 +979,7 @@ def test_generate_handles_error_messages(monkeypatch, fake_stage_config):
|
||||
)
|
||||
# Also put a valid result after error to allow the loop to complete
|
||||
# (error handling continues the loop, so we need a valid result to finish)
|
||||
llm.stage_list[0]._out_q.put_nowait(
|
||||
omni.stage_list[0]._out_q.put_nowait(
|
||||
{
|
||||
"request_id": expected_request_id,
|
||||
"engine_outputs": [{"stage": 0, "text": "result"}],
|
||||
@@ -925,7 +990,7 @@ def test_generate_handles_error_messages(monkeypatch, fake_stage_config):
|
||||
# Generate should handle error gracefully (log but continue)
|
||||
# Use dict instead of object() for serializable sampling params
|
||||
sampling_params_list = [{"temperature": 0.7}]
|
||||
outputs = llm.generate(prompts=["hi"], sampling_params_list=sampling_params_list)
|
||||
outputs = omni.generate(prompts=["hi"], sampling_params_list=sampling_params_list)
|
||||
# Should return final output (error was logged but didn't stop processing)
|
||||
assert isinstance(outputs, list)
|
||||
# Since final_output=True, should have one output
|
||||
@@ -942,7 +1007,8 @@ def test_close_sends_shutdown_signal(monkeypatch, fake_stage_config):
|
||||
|
||||
for module_name in [
|
||||
"vllm_omni.entrypoints.utils",
|
||||
"vllm_omni.entrypoints.omni_llm",
|
||||
"vllm_omni.entrypoints.omni",
|
||||
"vllm_omni.entrypoints.log_utils",
|
||||
"vllm_omni.entrypoints.omni_stage",
|
||||
]:
|
||||
if module_name in sys.modules:
|
||||
@@ -954,7 +1020,7 @@ def test_close_sends_shutdown_signal(monkeypatch, fake_stage_config):
|
||||
_setup_log_mocks(monkeypatch)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.entrypoints.omni_llm.remove_old_logs",
|
||||
"vllm_omni.entrypoints.log_utils.remove_old_logs",
|
||||
lambda log_file, num_stages: None,
|
||||
raising=False,
|
||||
)
|
||||
@@ -965,7 +1031,7 @@ def test_close_sends_shutdown_signal(monkeypatch, fake_stage_config):
|
||||
raising=False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.entrypoints.omni_llm.load_stage_configs_from_model",
|
||||
"vllm_omni.entrypoints.utils.load_stage_configs_from_model",
|
||||
_fake_loader,
|
||||
raising=False,
|
||||
)
|
||||
@@ -975,22 +1041,22 @@ def test_close_sends_shutdown_signal(monkeypatch, fake_stage_config):
|
||||
raising=False,
|
||||
)
|
||||
|
||||
import vllm_omni.entrypoints.omni_llm as omni_llm_module
|
||||
import vllm_omni.entrypoints.omni as omni_module
|
||||
|
||||
monkeypatch.setattr(omni_llm_module, "load_stage_configs_from_model", _fake_loader)
|
||||
monkeypatch.setattr(omni_llm_module, "OmniStage", lambda cfg: _FakeStage(cfg))
|
||||
monkeypatch.setattr(omni_module, "load_stage_configs_from_model", _fake_loader)
|
||||
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg: _FakeStage(cfg))
|
||||
|
||||
from vllm_omni.entrypoints.omni_llm import OmniLLM
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
|
||||
llm = OmniLLM(model="any", init_timeout=1)
|
||||
omni = Omni(model="any", init_timeout=1)
|
||||
|
||||
# Call close
|
||||
llm.close()
|
||||
omni.close()
|
||||
|
||||
# Verify shutdown signal (None) was sent to input queue
|
||||
# Use get_nowait to avoid blocking (close() uses put_nowait, so should be safe)
|
||||
try:
|
||||
shutdown_signal = llm.stage_list[0]._in_q.get_nowait()
|
||||
shutdown_signal = omni.stage_list[0]._in_q.get_nowait()
|
||||
assert shutdown_signal is None
|
||||
except Empty:
|
||||
# If queue was already empty or only had stage_ready, that's also acceptable
|
||||
@@ -998,4 +1064,4 @@ def test_close_sends_shutdown_signal(monkeypatch, fake_stage_config):
|
||||
pass
|
||||
|
||||
# Verify stop_stage_worker was called (process should be set)
|
||||
assert llm.stage_list[0]._proc is not None
|
||||
assert omni.stage_list[0]._proc is not None
|
||||
Reference in new issue
Block a user