[Bugfix] Gate async-chunk segment resume by session mode (#6680)
Signed-off-by: natureofnature <wzliu@connect.hku.hk>
This commit is contained in:
1 parent
c6df39e9f7
commit
54c1fc64b7
9 files changed
+67
-9
No files matched your search
@@ -96,6 +96,7 @@ def test_duplex_session_capacity_propagates_to_every_structured_stage() -> None:
|
||||
omni_config = _from_pipeline_key("personaplex")
|
||||
|
||||
assert [stage.model_config.duplex_max_sessions for stage in omni_config.stage_configs] == [2, 2]
|
||||
assert [stage.model_config.session_mode for stage in omni_config.stage_configs] == ["duplex", "duplex"]
|
||||
|
||||
|
||||
def test_non_duplex_deploy_keeps_model_session_capacity_at_one(tmp_path: Path) -> None:
|
||||
@@ -105,6 +106,7 @@ def test_non_duplex_deploy_keeps_model_session_capacity_at_one(tmp_path: Path) -
|
||||
omni_config = _from_pipeline_key("personaplex", deploy_config_path=str(deploy_path))
|
||||
|
||||
assert [stage.model_config.duplex_max_sessions for stage in omni_config.stage_configs] == [1, 1]
|
||||
assert [stage.model_config.session_mode for stage in omni_config.stage_configs] == ["turn", "turn"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_type", sorted(OMNI_PIPELINES))
|
||||
@@ -132,6 +134,7 @@ def test_vllm_omni_config_from_pipeline_config_matches_merge_pipeline_deploy(mod
|
||||
|
||||
engine_args = legacy_stage.yaml_engine_args
|
||||
assert omni_stage.model_config.duplex_max_sessions == engine_args.get("duplex_max_sessions", 1)
|
||||
assert omni_stage.model_config.session_mode == engine_args.get("session_mode", "turn")
|
||||
assert omni_stage.model_config.enforce_eager == engine_args.get("enforce_eager", False)
|
||||
assert omni_stage.load_config.load_format == engine_args.get("load_format", "auto")
|
||||
assert omni_stage.load_config.tokenizer_mode == engine_args.get("tokenizer_mode", "auto")
|
||||
@@ -579,6 +582,7 @@ def test_sub_config_fields_match_structured_scopes():
|
||||
"interleave_mm_strings",
|
||||
"media_io_kwargs",
|
||||
"active_stream_window",
|
||||
"session_mode",
|
||||
"duplex_max_sessions",
|
||||
"enable_sleep_mode",
|
||||
"default_sampling_params",
|
||||
|
||||
@@ -25,10 +25,10 @@ from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _make_scheduler(*, stage_id: int = 0) -> OmniARScheduler:
|
||||
def _make_scheduler(*, stage_id: int = 0, session_mode: str = "turn") -> OmniARScheduler:
|
||||
sched = OmniARScheduler.__new__(OmniARScheduler)
|
||||
sched._new_prompt_len_snapshot = {}
|
||||
sched.vllm_config = SimpleNamespace(model_config=SimpleNamespace(stage_id=stage_id))
|
||||
sched.vllm_config = SimpleNamespace(model_config=SimpleNamespace(stage_id=stage_id, session_mode=session_mode))
|
||||
sched.num_waiting_for_streaming_input = 0
|
||||
sched.log_stats = False
|
||||
sched.chunk_transfer_adapter = None
|
||||
@@ -446,11 +446,15 @@ def test_chunk_segment_cleanup_keeps_requeued_resumable_receiver() -> None:
|
||||
return result
|
||||
|
||||
sched = OmniARScheduler.__new__(OmniARScheduler)
|
||||
sched.vllm_config = SimpleNamespace(model_config=SimpleNamespace(session_mode="duplex"))
|
||||
sched.running = []
|
||||
sched.waiting = queue()
|
||||
sched.skipped_waiting = queue(session)
|
||||
sched.num_waiting_for_streaming_input = 1
|
||||
sched.chunk_transfer_adapter = SimpleNamespace(segment_finished_requests={session.request_id})
|
||||
sched.chunk_transfer_adapter = SimpleNamespace(
|
||||
receives_chunks=True,
|
||||
segment_finished_requests={session.request_id},
|
||||
)
|
||||
|
||||
sched._resume_downstream_chunk_receiver(session)
|
||||
sched._remove_stopped_requests_from_queues(set(), {session})
|
||||
@@ -460,3 +464,37 @@ def test_chunk_segment_cleanup_keeps_requeued_resumable_receiver() -> None:
|
||||
assert session not in sched.skipped_waiting.requests
|
||||
assert sched.num_waiting_for_streaming_input == 0
|
||||
assert session.request_id not in sched.chunk_transfer_adapter.segment_finished_requests
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("receives_chunks", "session_mode"),
|
||||
[
|
||||
(False, "duplex"),
|
||||
(True, "turn"),
|
||||
],
|
||||
)
|
||||
def test_chunk_segment_cleanup_keeps_explicit_update_stage_parked(
|
||||
receives_chunks: bool,
|
||||
session_mode: str,
|
||||
) -> None:
|
||||
"""Only duplex connector-driven receivers resume without an update."""
|
||||
session = _make_request()
|
||||
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
|
||||
sched = OmniARScheduler.__new__(OmniARScheduler)
|
||||
sched.vllm_config = SimpleNamespace(model_config=SimpleNamespace(session_mode=session_mode))
|
||||
sched.num_waiting_for_streaming_input = 1
|
||||
sched.chunk_transfer_adapter = SimpleNamespace(
|
||||
receives_chunks=receives_chunks,
|
||||
segment_finished_requests={session.request_id},
|
||||
)
|
||||
sched.skipped_waiting = MagicMock()
|
||||
sched._enqueue_waiting_request = MagicMock()
|
||||
|
||||
sched._resume_downstream_chunk_receiver(session)
|
||||
|
||||
assert session.status == RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
assert sched.num_waiting_for_streaming_input == 1
|
||||
assert session.request_id not in sched.chunk_transfer_adapter.segment_finished_requests
|
||||
sched.skipped_waiting.remove_requests.assert_not_called()
|
||||
sched._enqueue_waiting_request.assert_not_called()
|
||||
@@ -34,6 +34,11 @@ def test_sync_config_is_omni():
|
||||
assert isinstance(cfg, OmniModelConfig)
|
||||
|
||||
|
||||
def test_session_mode_reaches_omni_model_config():
|
||||
assert OmniEngineArgs().create_model_config().session_mode == "turn"
|
||||
assert OmniEngineArgs(session_mode="duplex").create_model_config().session_mode == "duplex"
|
||||
|
||||
|
||||
def test_default_stage_id_is_concrete_int():
|
||||
"""Ensure `stage_id` stays safe for downstream arithmetic/indexing."""
|
||||
engine_args = OmniEngineArgs()
|
||||
|
||||
@@ -93,6 +93,7 @@ class OmniModelConfig(ModelConfig):
|
||||
hf_text_config: The sub text_config of the model's hf_config (default: None)
|
||||
stage_id: Identifier for the stage in a multi-stage pipeline (default: 0)
|
||||
async_chunk: If set to True, perform async chunk
|
||||
session_mode: Request lifecycle mode, either turn-based or duplex
|
||||
model_stage: Stage type identifier, e.g., "thinker" or "talker"
|
||||
(default: "thinker")
|
||||
model_arch: Model architecture name
|
||||
@@ -121,6 +122,7 @@ class OmniModelConfig(ModelConfig):
|
||||
|
||||
stage_id: int = 0
|
||||
async_chunk: bool = False
|
||||
session_mode: str = "turn"
|
||||
retains_state_across_chunks: bool = False
|
||||
# Stage-1 active stream slots; 0 keeps legacy chunk-level round-robin.
|
||||
active_stream_window: int = 0
|
||||
|
||||
@@ -384,6 +384,7 @@ class OmniStageModelConfig:
|
||||
interleave_mm_strings: bool | None = None
|
||||
media_io_kwargs: dict[str, Any] | None = None
|
||||
active_stream_window: int = Field(default=0, ge=0)
|
||||
session_mode: str = "turn"
|
||||
duplex_max_sessions: int = Field(default=1, ge=1)
|
||||
enable_sleep_mode: bool = False
|
||||
default_sampling_params: dict[str, Any] | None = None
|
||||
@@ -1632,6 +1633,7 @@ def _build_model_config(
|
||||
kwargs["tokenizer_subdir"] = topology.tokenizer_subdir
|
||||
return cast(Any, OmniStageModelConfig)(
|
||||
default_sampling_params=default_sampling_params,
|
||||
session_mode=deploy.session_mode,
|
||||
duplex_max_sessions=duplex_max_sessions,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@@ -851,6 +851,7 @@ def _build_engine_args(
|
||||
# Materialize the resolved pipeline-wide async_chunk value into every
|
||||
# stage so explicit False overrides do not get lost downstream.
|
||||
engine_args["async_chunk"] = bool(deploy.async_chunk)
|
||||
engine_args["session_mode"] = deploy.session_mode
|
||||
if deploy.session_mode == "duplex":
|
||||
# The engine admission limit is also the authoritative capacity for
|
||||
# model-owned streaming state. Propagate it to every stage instead of
|
||||
|
||||
@@ -589,10 +589,9 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler):
|
||||
# for streaming input request only
|
||||
if self.chunk_transfer_adapter:
|
||||
if self.vllm_config.model_config.stage_id != 0:
|
||||
# Downstream async-chunk stages receive real payloads from the
|
||||
# connector, not an external StreamingUpdate. Move the
|
||||
# resumable request back to the ordinary waiting queue so
|
||||
# connector polling starts again immediately.
|
||||
# Only a connector-fed receiver in native duplex can
|
||||
# poll the next segment without an external update.
|
||||
# Sender-only and turn-mode stages remain parked.
|
||||
self._resume_downstream_chunk_receiver(request)
|
||||
outstanding_async_tokens = request.num_output_placeholders
|
||||
# Always record the discard signal (0 when nothing is in
|
||||
|
||||
@@ -588,10 +588,14 @@ class OmniSchedulerMixin:
|
||||
self.skipped_waiting.remove_requests(removable)
|
||||
|
||||
def _resume_downstream_chunk_receiver(self, request: Request) -> None:
|
||||
"""Resume connector polling without waiting for an external update."""
|
||||
"""Resume duplex connector polling without an external update."""
|
||||
adapter = self.chunk_transfer_adapter
|
||||
adapter.segment_finished_requests.discard(request.request_id)
|
||||
if request.status != RequestStatus.WAITING_FOR_STREAMING_REQ:
|
||||
if (
|
||||
not adapter.receives_chunks
|
||||
or getattr(self.vllm_config.model_config, "session_mode", "turn") != "duplex"
|
||||
or request.status != RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
):
|
||||
return
|
||||
self.num_waiting_for_streaming_input -= 1
|
||||
request.status = RequestStatus.WAITING
|
||||
|
||||
@@ -163,6 +163,7 @@ class OmniEngineArgs(EngineArgs):
|
||||
If None, default processing is used.
|
||||
stage_connector_spec: Extra configuration for stage connector
|
||||
async_chunk: If set to True, perform async chunk
|
||||
session_mode: Request lifecycle mode, either turn-based or duplex
|
||||
worker_type: Model Type, e.g., "ar" or "generation"
|
||||
task_type: Model-defined startup task type. Consumers validate the
|
||||
supported values and decide whether it selects request behavior,
|
||||
@@ -192,6 +193,7 @@ class OmniEngineArgs(EngineArgs):
|
||||
subtalker_sampling_params: dict[str, Any] | None = None
|
||||
silence_ban_frames: int = 0
|
||||
async_chunk: bool = False
|
||||
session_mode: str = "turn"
|
||||
retains_state_across_chunks: bool = False
|
||||
# WS-A: Stage-1 active stream slots. 0 = legacy preempt-everything.
|
||||
# Must be declared here so engine_args dict propagation does not silently
|
||||
@@ -415,6 +417,7 @@ class OmniEngineArgs(EngineArgs):
|
||||
# All kwargs below are Omni specific
|
||||
stage_id=self.stage_id,
|
||||
async_chunk=self.async_chunk,
|
||||
session_mode=self.session_mode,
|
||||
retains_state_across_chunks=self.retains_state_across_chunks,
|
||||
active_stream_window=self.active_stream_window,
|
||||
duplex_max_sessions=self.duplex_max_sessions,
|
||||
|
||||
Reference in new issue
Block a user