[Perf][MiniCPM-o] Stage 0 streaming audio/vision encoder CUDA graphs and candidate sampling (#8430)

Signed-off-by: BeatSeat <wendavid552@gmail.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>

vllm_omni/model_executor/duplex_sampling.py

@@ -22,6 +22,20 @@ class DuplexSamplingRow:
     seq: int | None
     payload: dict[str, object] | None
     max_tokens: int | None
+    # Host copies of the row's sampling parameters (None when the runner has none).
+    temperature: float | None = None
+    top_k: int | None = None
+    top_p: float | None = None
+
+
+def _host_value(values: Any, row_idx: int, cast: type) -> Any:
+    """``values[row_idx]`` of a host array as ``cast``, or None when there is none."""
+    if values is None:
+        return None
+    try:
+        return cast(values[row_idx])
+    except (IndexError, TypeError, ValueError):
+        return None
 
 
 class DuplexSamplingHelper:
@@ -53,15 +67,19 @@ class DuplexSamplingHelper:
         else:
             self.active_request_ids.discard(req_id)
 
-    def update_states(self, runner: object, scheduler_output: object) -> None:
+    def update_states(self, runner: Any, scheduler_output: Any) -> None:
         self.active_request_ids.difference_update(str(req_id) for req_id in scheduler_output.finished_req_ids)
         for request in scheduler_output.scheduled_new_reqs:
             self.refresh_active_request(runner, str(request.req_id))
 
-    def rows(self, runner: object) -> tuple[DuplexSamplingRow, ...]:
+    def rows(self, runner: Any) -> tuple[DuplexSamplingRow, ...]:
         rows: list[DuplexSamplingRow] = []
-        req_ids = [str(req_id) for req_id in getattr(runner.input_batch, "req_ids", [])]
+        input_batch = runner.input_batch
+        req_ids = [str(req_id) for req_id in getattr(input_batch, "req_ids", [])]
         requests = getattr(runner, "requests", {})
+        temperature_cpu = getattr(input_batch, "temperature_cpu", None)
+        top_k_cpu = getattr(input_batch, "top_k_cpu", None)
+        top_p_cpu = getattr(input_batch, "top_p_cpu", None)
         for row_idx, req_id in enumerate(req_ids):
             if req_id not in self.active_request_ids:
                 continue
@@ -73,7 +91,8 @@ class DuplexSamplingHelper:
             if not isinstance(session_id, str) or not session_id:
                 session_id = None
             try:
-                seq = int(duplex.get("seq"))
+                seq_raw = duplex.get("seq")
+                seq = int(seq_raw) if seq_raw is not None else None
             except (TypeError, ValueError):
                 seq = None
             payload = duplex.get("payload")
@@ -93,6 +112,9 @@ class DuplexSamplingHelper:
                     seq=seq,
                     payload=payload,
                     max_tokens=max_tokens if max_tokens > 0 else None,
+                    temperature=_host_value(temperature_cpu, row_idx, float),
+                    top_k=_host_value(top_k_cpu, row_idx, int),
+                    top_p=_host_value(top_p_cpu, row_idx, float),
                 )
             )
         return tuple(rows)
@@ -142,8 +164,9 @@ class DuplexSamplingRunnerMixin:
             return
         helper = getattr(self, "_duplex_sampling_helper", None)
         rows = helper.rows(self) if helper is not None and helper.active_request_ids else ()
-        if rows or (helper is not None and helper.hook_active):
-            prepare_duplex_sampling(logits, prepared_sampling_metadata, rows)
+        # Publish the row set even when empty, so the model skips its per-row
+        # prompt-token scan (a host sync per row) in turn-based serving.
+        prepare_duplex_sampling(logits, prepared_sampling_metadata, rows)
         if helper is not None:
             helper.hook_active = bool(rows)