ab1acb27ff
Signed-off-by: CHEN <116010019@link.cuhk.edu.cn> Co-authored-by: Bhanu068 <voutharoja.bhanu06@gmail.com> Co-authored-by: Gao Han <gaohan19@huawei.com>
42 lines
2.1 KiB
Python
42 lines
2.1 KiB
Python
import sys
|
|
|
|
from aenum import extend_enum
|
|
from vllm.inputs.data import TokensPrompt as _OriginalTokensPrompt
|
|
from vllm.model_executor.layers.rotary_embedding import (
|
|
MRotaryEmbedding as _OriginalMRotaryEmbedding,
|
|
)
|
|
from vllm.v1.engine import EngineCoreOutput as _OriginalEngineCoreOutput
|
|
from vllm.v1.engine import EngineCoreOutputs as _OriginalEngineCoreOutputs
|
|
from vllm.v1.engine import EngineCoreRequest as _OriginalEngineCoreRequest
|
|
from vllm.v1.request import Request as _OriginalRequest
|
|
from vllm.v1.request import RequestStatus
|
|
|
|
import vllm_omni.logger # noqa: F401
|
|
from vllm_omni.engine import OmniEngineCoreOutput, OmniEngineCoreOutputs, OmniEngineCoreRequest
|
|
from vllm_omni.inputs.data import OmniTokensPrompt
|
|
from vllm_omni.model_executor.layers.rotary_embedding import OmniMRotaryEmbedding
|
|
from vllm_omni.request import OmniRequest
|
|
|
|
# Extend RequestStatus enum with omni-specific statuses
|
|
if not hasattr(RequestStatus, "WAITING_FOR_CHUNK"):
|
|
# The value - 1 is intentionally chosen to ensure it is treated
|
|
# as a non-finished state and remains compatible with existing comparisons.
|
|
extend_enum(RequestStatus, "WAITING_FOR_CHUNK", -1)
|
|
|
|
for module_name, module in sys.modules.items():
|
|
# only do patch on module of vllm, pass others
|
|
if "vllm" not in module_name:
|
|
continue
|
|
if hasattr(module, "EngineCoreOutput") and module.EngineCoreOutput == _OriginalEngineCoreOutput:
|
|
module.EngineCoreOutput = OmniEngineCoreOutput
|
|
if hasattr(module, "EngineCoreOutputs") and module.EngineCoreOutputs == _OriginalEngineCoreOutputs:
|
|
module.EngineCoreOutputs = OmniEngineCoreOutputs
|
|
if hasattr(module, "TokensPrompt") and module.TokensPrompt == _OriginalTokensPrompt:
|
|
module.TokensPrompt = OmniTokensPrompt
|
|
if hasattr(module, "MRotaryEmbedding") and module.MRotaryEmbedding == _OriginalMRotaryEmbedding:
|
|
module.MRotaryEmbedding = OmniMRotaryEmbedding
|
|
if hasattr(module, "Request") and module.Request == _OriginalRequest:
|
|
module.Request = OmniRequest
|
|
if hasattr(module, "EngineCoreRequest") and module.EngineCoreRequest == _OriginalEngineCoreRequest:
|
|
module.EngineCoreRequest = OmniEngineCoreRequest
|