Add normalized online FP8 cache for DLO
Signed-off-by: hsliuustc0106 <hsliuustc0106@users.noreply.github.com>
This commit is contained in:
1 parent
14bd8f8772
commit
1ae56cfcfb
14 files changed
+714
-11
No files matched your search
@@ -41,6 +41,8 @@ class _MiniMaxH3DirectMmapAdapter:
|
||||
target: torch.Tensor,
|
||||
) -> DirectMmapTensorPolicy:
|
||||
del target
|
||||
if getattr(self.pipeline, "_dlo_fp8_cache_normalized", False):
|
||||
return DirectMmapTensorPolicy(allow_custom_loader=True)
|
||||
transform = None
|
||||
suffix = ".qkv_proj.weight"
|
||||
if runtime_name.endswith(suffix):
|
||||
|
||||
Reference in new issue
Block a user