[Bugfix][MiniCPM-o] Stop Talker mid-utterance truncation and load cod… (#6346)
Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
47b1893676
commit
9c61d5b67b
27 files changed
+1118
-678
No files matched your search
@@ -4,7 +4,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -103,4 +108,55 @@ class DuplexSamplingHelper:
|
||||
self.hook_active = False
|
||||
|
||||
|
||||
__all__ = ["DuplexSamplingHelper", "DuplexSamplingRow"]
|
||||
class DuplexSamplingRunnerMixin:
|
||||
"""Runner-side wiring for the experimental duplex sampling hook.
|
||||
|
||||
GPU and NPU AR runners are siblings, not parent and child, so neither
|
||||
inherits the other's ``_sample``. Keep the hook here and have each runner
|
||||
opt in, instead of copying the GPU ``_sample`` path and dropping the
|
||||
``prepare_duplex_sampling`` call.
|
||||
"""
|
||||
|
||||
def _init_duplex_sampling_state(self) -> None:
|
||||
"""Reset hook resolution. Call from ``__init__``, before ``load_model``."""
|
||||
self._duplex_sampling_hook = None
|
||||
self._duplex_sampling_hook_resolved = False
|
||||
|
||||
def _resolve_duplex_sampling_hook(self, *, force: bool = False):
|
||||
"""Bind ``model.prepare_duplex_sampling``. Call with ``force`` after load."""
|
||||
if not force and getattr(self, "_duplex_sampling_hook_resolved", False):
|
||||
return self._duplex_sampling_hook
|
||||
candidate = getattr(getattr(self, "model", None), "prepare_duplex_sampling", None)
|
||||
self._duplex_sampling_hook = candidate if callable(candidate) else None
|
||||
self._duplex_sampling_hook_resolved = True
|
||||
if self._duplex_sampling_hook is not None and not hasattr(self, "_duplex_sampling_helper"):
|
||||
self._duplex_sampling_helper = DuplexSamplingHelper()
|
||||
return self._duplex_sampling_hook
|
||||
|
||||
def _update_duplex_sampling_states(self, scheduler_output: SchedulerOutput) -> None:
|
||||
"""Refresh which batch rows are duplex rows. Call from ``_update_states``."""
|
||||
if self._resolve_duplex_sampling_hook() is None:
|
||||
return
|
||||
helper = getattr(self, "_duplex_sampling_helper", None)
|
||||
if helper is not None:
|
||||
helper.update_states(self, scheduler_output)
|
||||
|
||||
def _apply_duplex_sampling(self, logits: torch.Tensor, prepared_sampling_metadata: SamplingMetadata) -> None:
|
||||
"""Hand the model its duplex rows. Call from ``_sample``, before the sampler."""
|
||||
prepare_duplex_sampling = self._resolve_duplex_sampling_hook()
|
||||
if prepare_duplex_sampling is None:
|
||||
return
|
||||
helper = getattr(self, "_duplex_sampling_helper", None)
|
||||
rows = helper.rows(self) if helper is not None and helper.active_request_ids else ()
|
||||
if rows or (helper is not None and helper.hook_active):
|
||||
prepare_duplex_sampling(logits, prepared_sampling_metadata, rows)
|
||||
if helper is not None:
|
||||
helper.hook_active = bool(rows)
|
||||
|
||||
def _clear_duplex_sampling(self) -> None:
|
||||
helper = getattr(self, "_duplex_sampling_helper", None)
|
||||
if helper is not None:
|
||||
helper.clear()
|
||||
|
||||
|
||||
__all__ = ["DuplexSamplingHelper", "DuplexSamplingRow", "DuplexSamplingRunnerMixin"]
|
||||
Reference in new issue
Block a user