[BugFix][NPU] Honor prefer_model_sampler in NPU AR runner (#3517)
Signed-off-by: gcanlin <canlinguosdu@gmail.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
6c1bb27b69
commit
c890b3a99c
3 files changed
+82
-47
No files matched your search
@@ -109,53 +109,6 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
|
||||
with maybe_disable_pin_memory_for_ray(self, total_bytes):
|
||||
return super()._make_buffer(*size, dtype=dtype, numpy=numpy)
|
||||
|
||||
def _build_model_sampler_output_token_ids(self) -> list[list[int]]:
|
||||
"""Build decoded-token history for custom model samplers.
|
||||
|
||||
vLLM only populates sampling_metadata.output_token_ids when penalties or
|
||||
logits processors require it. CosyVoice3's custom RAS sampler also
|
||||
depends on this history, so we reconstruct it directly from the input
|
||||
batch for prefer_model_sampler models.
|
||||
"""
|
||||
req_output_token_ids = getattr(self.input_batch, "req_output_token_ids", [])
|
||||
req_ids = list(getattr(self.input_batch, "req_ids", []))
|
||||
output_token_ids = [list(req_output_token_ids[idx] or []) for idx in range(len(req_ids))]
|
||||
|
||||
sampled_token_ids_cpu = getattr(self.input_batch, "sampled_token_ids_cpu", None)
|
||||
async_copy_ready_event = getattr(self.input_batch, "async_copy_ready_event", None)
|
||||
prev_req_id_to_index = getattr(self.input_batch, "prev_req_id_to_index", None)
|
||||
if sampled_token_ids_cpu is None or not output_token_ids or prev_req_id_to_index is None:
|
||||
return output_token_ids
|
||||
|
||||
sampled_token_ids: list[list[int]] | None = None
|
||||
for index, req_id in enumerate(req_ids):
|
||||
prev_index = prev_req_id_to_index.get(req_id)
|
||||
if prev_index is None:
|
||||
continue
|
||||
req_history = output_token_ids[index]
|
||||
if not req_history or req_history[-1] != -1:
|
||||
continue
|
||||
if sampled_token_ids is None:
|
||||
assert async_copy_ready_event is not None
|
||||
async_copy_ready_event.synchronize()
|
||||
sampled_token_ids = sampled_token_ids_cpu.tolist()
|
||||
new_ids = list(sampled_token_ids[prev_index])
|
||||
if not new_ids:
|
||||
continue
|
||||
num_sampled_ids = len(new_ids) if new_ids[-1] != -1 else new_ids.index(-1)
|
||||
first_placeholder = req_history.index(-1)
|
||||
num_placeholders = len(req_history) - first_placeholder
|
||||
num_to_replace = min(num_sampled_ids, num_placeholders)
|
||||
req_history[first_placeholder : first_placeholder + num_to_replace] = new_ids[:num_to_replace]
|
||||
|
||||
return output_token_ids
|
||||
|
||||
def _sampling_metadata_for_model_sampler(self, sampling_metadata):
|
||||
output_token_ids = self._build_model_sampler_output_token_ids()
|
||||
if output_token_ids == sampling_metadata.output_token_ids:
|
||||
return sampling_metadata
|
||||
return replace(sampling_metadata, output_token_ids=output_token_ids)
|
||||
|
||||
def _request_final_stage_id(self, req_id: str) -> int | None:
|
||||
info = self.model_intermediate_buffer.get(req_id)
|
||||
if not isinstance(info, dict):
|
||||
|
||||
Reference in new issue
Block a user