Rebase multi-API serving on latest main
Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
1 parent
78934753ff
commit
dc3c7dbe16
25 files changed
+1832
-202
No files matched your search
@@ -690,7 +690,7 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
self._init_mrope_positions(req_state)
|
||||
|
||||
# Only relevant for models using XD-RoPE (e.g, HunYuan-VL)
|
||||
if self.uses_xdrope_dim > 0:
|
||||
if getattr(self, "uses_xdrope_dim", 0) > 0:
|
||||
self._init_xdrope_positions(req_state)
|
||||
|
||||
reqs_to_add.append(self.requests[req_id])
|
||||
@@ -1164,7 +1164,7 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
|
||||
if self.uses_mrope:
|
||||
positions = self.mrope_positions.gpu[:, :num_tokens_padded]
|
||||
elif self.uses_xdrope_dim > 0:
|
||||
elif getattr(self, "uses_xdrope_dim", 0) > 0:
|
||||
positions = self.xdrope_positions.gpu[:, :num_tokens_padded]
|
||||
else:
|
||||
positions = self.positions[:num_tokens_padded]
|
||||
@@ -1701,7 +1701,7 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
|
||||
if self.uses_mrope:
|
||||
positions = self.mrope_positions.gpu[:, :num_input_tokens]
|
||||
elif self.uses_xdrope_dim > 0:
|
||||
elif getattr(self, "uses_xdrope_dim", 0) > 0:
|
||||
positions = self.xdrope_positions.gpu[:, :num_input_tokens]
|
||||
else:
|
||||
positions = self.positions[:num_input_tokens]
|
||||
|
||||
Reference in new issue
Block a user