[Kernel][Boogu-Image] Fuse Q/K RMSNorm + interleaved RoPE via fused_qk_norm_rope (#6982)
Signed-off-by: Qihan Kang <rollykanggg@gmail.com>
This commit is contained in:
1 parent
7e520f9cc4
commit
f7d9deb45a
8 files changed
+703
-53
No files matched your search
@@ -15,6 +15,7 @@ if TYPE_CHECKING:
|
||||
MASTER_PORT: int | None = None
|
||||
CUDA_HOME: str | None = None
|
||||
LOCAL_RANK: int = 0
|
||||
VLLM_OMNI_FUSED_QK_NORM_ROPE_MIN_TOKENS: str | None = None
|
||||
|
||||
environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# ================== Runtime Env Vars ==================
|
||||
@@ -28,6 +29,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# local rank of the process in the distributed setting, used to determine
|
||||
# the GPU device id
|
||||
"LOCAL_RANK": lambda: int(os.environ.get("LOCAL_RANK", "0")),
|
||||
# Minimum rotary-table span (B*S positions) at which consumers of the shared
|
||||
# fused_qk_norm_rope op take the fused path instead of their eager chain;
|
||||
# "0" = always fuse; unset = each consumer's own measured default. Raw
|
||||
# string or None; validated by fused_qk_norm_rope_min_tokens().
|
||||
"VLLM_OMNI_FUSED_QK_NORM_ROPE_MIN_TOKENS": lambda: os.environ.get("VLLM_OMNI_FUSED_QK_NORM_ROPE_MIN_TOKENS", None),
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user