[Frontend] Add a shared realtime web UI for MiniCPM-o and Qwen3-Omni (#7222) (#7585)

Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>

tests/engine/duplex/test_engine_session.py

@@ -567,10 +567,10 @@ def test_overlap_policy_defaults_and_invalid_values_to_listen_only():
     assert DuplexSessionConfig.from_event({"session": {"overlap_policy": "bogus"}}).overlap_policy == "listen_only"
 
 
-def test_capabilities_as_dict_reports_model_native_duplex_constants():
+def test_capabilities_as_dict_reports_declared_implementation():
     caps = DuplexCapabilities().as_dict()
 
-    assert caps["implementation_level"] == "model_native_duplex"
+    assert caps["implementation_level"] == "turn_based_duplex"
     assert caps["input_modes"] == ["append_audio_chunk"]
     assert caps["supports_kv_lease"] is False
     assert caps["supports_core_kv_lease"] is False

tests/engine/duplex/test_session_manager.py

@@ -196,7 +196,7 @@ class FakePlugin(DuplexModelPlugin):
 
     def capabilities(self, *, max_sessions: int) -> DuplexCapabilities:
         del max_sessions
-        return DuplexCapabilities(supports_input_append=True)
+        return DuplexCapabilities(supports_input_append=True, supports_core_resumable_request=True)
 
     def validate_client_extra_body(self, extra_body: object) -> None:
         pass
@@ -408,7 +408,9 @@ async def test_open_answers_with_capabilities_and_emits_session_created() -> Non
         assert result.control_id == "open-sid-open"
         assert result.session_id == "sid-open"
         assert result.lease_generation == 0
-        assert result.capabilities == DuplexCapabilities(supports_input_append=True)
+        assert result.capabilities == DuplexCapabilities(
+            supports_input_append=True, supports_core_resumable_request=True
+        )
         assert result.public_session is not None
         assert result.public_session["id"] == "sid-open"
         assert result.public_session["voice"] == "test"

tests/engine/duplex/test_vad_backend.py

@@ -107,8 +107,8 @@ def test_endpoint_decisions_match_the_serving_side_policy(
 # --------------------------------------------------------------------------- #
 
 
-def test_a_loud_frame_does_not_restart_the_silence_timer() -> None:
-    """Silero v6.2 hysteresis: once silence is running, a loud frame only delays it.
+def test_hysteresis_band_does_not_restart_the_silence_timer() -> None:
+    """Scores between the negative and activation thresholds preserve the timer.
 
     The frame at 0.45 sits above the negative threshold (0.35) but below the
     activation threshold. It must not cancel the pending endpoint, or a speaker
@@ -164,9 +164,14 @@ def _pcm16_b64(samples: np.ndarray) -> str:
 
 def test_pcm16_input_is_accepted_alongside_float32() -> None:
     config = SileroVADConfig(threshold=0.5)
-    tone = np.zeros(FRAME * 2, dtype=np.float32)
+    tone: np.ndarray = np.zeros(FRAME * 2, dtype=np.float32)
     seen: list[int] = []
-    vad = SileroStreamingVAD(config, frame_scorer=lambda frame: seen.append(frame.size) or 0.0)
+
+    def _score_frame(frame: np.ndarray) -> float:
+        seen.append(frame.size)
+        return 0.0
+
+    vad = SileroStreamingVAD(config, frame_scorer=_score_frame)
     vad.process_base64(_pcm16_b64(tone), fmt="pcm16", sample_rate_hz=SAMPLE_RATE_HZ)
     assert seen == [FRAME, FRAME]
 
@@ -272,7 +277,7 @@ def test_the_onnx_backend_holds_the_upstream_silero_v62_contract(monkeypatch, tm
         intra_op_num_threads = 0
 
     class FakeInferenceSession:
-        def __init__(self, path: str, *, providers: list[str], sess_options: object) -> None:
+        def __init__(self, path: str, *, providers: list[str], sess_options: FakeSessionOptions) -> None:
             self.providers = providers
             self.sess_options = sess_options
             self.calls: list[dict[str, np.ndarray]] = []
@@ -313,7 +318,7 @@ def test_the_onnx_backend_holds_the_upstream_silero_v62_contract(monkeypatch, tm
     assert session.calls[0]["sr"].item() == SAMPLE_RATE_HZ
 
     state = backend.new_state()
-    first_frame = np.arange(backend.frame_samples, dtype=np.float32)
+    first_frame: np.ndarray = np.arange(backend.frame_samples, dtype=np.float32)
     probability, state = backend.infer(first_frame, state)
     first_call = session.calls[1]
 
@@ -377,3 +382,17 @@ def test_the_documented_server_vad_fields_are_all_accepted() -> None:
         )
         is None
     )
+
+
+@pytest.mark.parametrize("speech_probability", [0.5, 0.9])
+def test_resumed_speech_cancels_pending_silence(speech_probability):
+    config = SileroVADConfig(threshold=0.5, prefix_padding_ms=0, silence_duration_ms=96, min_speech_duration_ms=32)
+    # 32 ms pause, resumed speech, then a full 96 ms pause: one utterance.
+    scores = [0.9, 0.1, speech_probability, speech_probability, 0.1, 0.1, 0.1]
+    assert _drive(config, scores) == [("start", 0), ("stop", 224)]
+
+
+def test_short_pauses_do_not_split_continuous_speech_into_repeated_turns():
+    config = SileroVADConfig(threshold=0.5, prefix_padding_ms=0, silence_duration_ms=500, min_speech_duration_ms=32)
+    scores = [0.9] + [0.1, 0.9, 0.9, 0.9] * 20
+    assert _drive(config, scores) == [("start", 0)]

tests/engine/test_duplex_orchestrator.py

@@ -515,3 +515,48 @@ async def test_a_dead_replica_closes_the_sessions_it_was_serving() -> None:
     messages = [output_q.get_nowait() for _ in range(output_q.qsize())]
     types = [getattr(getattr(m, "event", None), "type", type(m).__name__) for m in messages]
     assert types[-1] in {"session.expired", "session.closed"}, f"the client needs a terminal event, got {types}"
+
+
+@pytest.mark.asyncio
+async def test_turn_plugin_processes_multimodal_prompt_before_stage_submission(monkeypatch):
+    from vllm_omni.engine.orchestrator import build_engine_core_request_from_tokens
+    from vllm_omni.model_executor.models.qwen3_omni.duplex.plugin import Qwen3OmniDuplexPlugin
+
+    orchestrator, clients, rpc_q, _ = _build()
+    plugin = Qwen3OmniDuplexPlugin(_encode_audio)
+    plugin.processor = SimpleNamespace(apply_chat_template=lambda messages, **kwargs: "audio prompt")
+    orchestrator.plugin = orchestrator.session_manager.plugin = plugin
+    seen = []
+
+    def process_inputs(**kwargs):
+        seen.append(kwargs)
+        return build_engine_core_request_from_tokens(
+            request_id=kwargs["request_id"],
+            prompt={"prompt_token_ids": [1, 2]},
+            params=kwargs["params"],
+            model_config=orchestrator.stage_pools[0].stage_vllm_config.model_config,
+            resumable=kwargs["resumable"],
+        )
+
+    monkeypatch.setattr(
+        orchestrator, "_get_stage_input_processor", lambda stage_id: SimpleNamespace(process_inputs=process_inputs)
+    )
+    await orchestrator._dispatch_message(
+        OpenDuplexSessionMessage(
+            control_id="open-qwen",
+            session_id=SESSION_ID,
+            session_config=DuplexSessionConfig(model="qwen", modalities=["text", "audio"]),
+        )
+    )
+    assert (await rpc_q.get()).ok
+    try:
+        await _submit(orchestrator, _append_audio())
+        assert not clients[0].add_request_calls
+        await _submit(orchestrator, commands.Commit(final=True, create_response=True))
+        assert len(seen) == 1
+        assert seen[0]["prompt"]["multi_modal_data"]["audio"][0][0].shape == (16000,)
+        submitted = clients[0].add_request_calls[0][0]
+        assert submitted.resumable is False
+        assert not orchestrator.request_states[submitted.request_id].streaming.enabled
+    finally:
+        await _close(orchestrator, rpc_q)