[Model] Aura support - Non async chunk path (#4257)
Signed-off-by: Rein Yang <ruiruyang2@gmail.com> Signed-off-by: R2-Y <ruiruyang2@gmail.com>
This commit is contained in:
1 parent
1292c47804
commit
afbcc51672
30 files changed
+2958
-26
No files matched your search
@@ -1,4 +1,5 @@
|
||||
import contextlib
|
||||
import inspect
|
||||
from collections.abc import Callable
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
@@ -48,6 +49,30 @@ else:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _filter_mrope_kwargs_for_model(model: object, kwargs: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Return only M-RoPE kwargs accepted by the model implementation."""
|
||||
method = getattr(model, "get_mrope_input_positions")
|
||||
try:
|
||||
signature = inspect.signature(method)
|
||||
except (TypeError, ValueError):
|
||||
return kwargs
|
||||
|
||||
if any(param.kind == inspect.Parameter.VAR_KEYWORD for param in signature.parameters.values()):
|
||||
return kwargs
|
||||
|
||||
accepted = {
|
||||
name
|
||||
for name, param in signature.parameters.items()
|
||||
if name not in {"self", "input_tokens"}
|
||||
and param.kind
|
||||
in {
|
||||
inspect.Parameter.POSITIONAL_OR_KEYWORD,
|
||||
inspect.Parameter.KEYWORD_ONLY,
|
||||
}
|
||||
}
|
||||
return {key: value for key, value in kwargs.items() if key in accepted}
|
||||
|
||||
|
||||
class OmniGPUModelRunner(GPUModelRunner):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
@@ -338,7 +363,7 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
kwargs["target_w"] = target_w
|
||||
req_state.mrope_positions, req_state.mrope_position_delta = self.model.get_mrope_input_positions(
|
||||
req_state.prompt_token_ids,
|
||||
**kwargs,
|
||||
**_filter_mrope_kwargs_for_model(self.model, kwargs),
|
||||
)
|
||||
else:
|
||||
req_state.mrope_positions, req_state.mrope_position_delta = MRotaryEmbedding.get_input_positions_tensor(
|
||||
|
||||
Reference in new issue
Block a user