[BugFix][NPU] Honor prefer_model_sampler in NPU AR runner (#3517)

Signed-off-by: gcanlin <canlinguosdu@gmail.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
Canlin GuoandHongsheng Liu authored and GitHub committed 2026-05-25 09:08:18 +08:00
1 parent 6c1bb27b69
commit c890b3a99c
3 files changed
+82 -47

No files matched your search

-47
View File
@@ -109,53 +109,6 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
with maybe_disable_pin_memory_for_ray(self, total_bytes):
return super()._make_buffer(*size, dtype=dtype, numpy=numpy)
def _build_model_sampler_output_token_ids(self) -> list[list[int]]:
"""Build decoded-token history for custom model samplers.
vLLM only populates sampling_metadata.output_token_ids when penalties or
logits processors require it. CosyVoice3's custom RAS sampler also
depends on this history, so we reconstruct it directly from the input
batch for prefer_model_sampler models.
"""
req_output_token_ids = getattr(self.input_batch, "req_output_token_ids", [])
req_ids = list(getattr(self.input_batch, "req_ids", []))
output_token_ids = [list(req_output_token_ids[idx] or []) for idx in range(len(req_ids))]
sampled_token_ids_cpu = getattr(self.input_batch, "sampled_token_ids_cpu", None)
async_copy_ready_event = getattr(self.input_batch, "async_copy_ready_event", None)
prev_req_id_to_index = getattr(self.input_batch, "prev_req_id_to_index", None)
if sampled_token_ids_cpu is None or not output_token_ids or prev_req_id_to_index is None:
return output_token_ids
sampled_token_ids: list[list[int]] | None = None
for index, req_id in enumerate(req_ids):
prev_index = prev_req_id_to_index.get(req_id)
if prev_index is None:
continue
req_history = output_token_ids[index]
if not req_history or req_history[-1] != -1:
continue
if sampled_token_ids is None:
assert async_copy_ready_event is not None
async_copy_ready_event.synchronize()
sampled_token_ids = sampled_token_ids_cpu.tolist()
new_ids = list(sampled_token_ids[prev_index])
if not new_ids:
continue
num_sampled_ids = len(new_ids) if new_ids[-1] != -1 else new_ids.index(-1)
first_placeholder = req_history.index(-1)
num_placeholders = len(req_history) - first_placeholder
num_to_replace = min(num_sampled_ids, num_placeholders)
req_history[first_placeholder : first_placeholder + num_to_replace] = new_ids[:num_to_replace]
return output_token_ids
def _sampling_metadata_for_model_sampler(self, sampling_metadata):
output_token_ids = self._build_model_sampler_output_token_ids()
if output_token_ids == sampling_metadata.output_token_ids:
return sampling_metadata
return replace(sampling_metadata, output_token_ids=output_token_ids)
def _request_final_stage_id(self, req_id: str) -> int | None:
info = self.model_intermediate_buffer.get(req_id)
if not isinstance(info, dict):