[Bugfix] fix kernel error for qwen3-omni (#1602)
Signed-off-by: Rein Yang <ruiruyang2@gmail.com>
This commit is contained in:
1 parent
1ca198eaff
commit
f082f6eedb
2 files changed
+9
-1
No files matched your search
@@ -72,7 +72,7 @@ stage_args:
|
||||
stage_type: llm # Use llm stage type to launch OmniLLM
|
||||
runtime:
|
||||
devices: "1"
|
||||
max_batch_size: 64
|
||||
max_batch_size: 32
|
||||
engine_args:
|
||||
model_stage: code2wav
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
|
||||
@@ -416,6 +416,14 @@ class GPUARModelRunner(OmniGPUModelRunner):
|
||||
if grammar_output is not None:
|
||||
apply_grammar_bitmask(scheduler_output, grammar_output, self.input_batch, logits)
|
||||
|
||||
# Correct padding values of prompt_token_ids to match the logits vocabulary size
|
||||
if logits is not None and not self.input_batch.sampling_metadata.no_penalties:
|
||||
smd = self.input_batch.sampling_metadata
|
||||
if smd.prompt_token_ids is not None:
|
||||
logits_vocab = logits.shape[-1]
|
||||
if self.input_batch.vocab_size > logits_vocab:
|
||||
smd.prompt_token_ids = smd.prompt_token_ids.clamp(max=logits_vocab)
|
||||
|
||||
with record_function_or_nullcontext("gpu_model_runner: sample"):
|
||||
sampler_output = self._sample(logits, spec_decode_metadata)
|
||||
|
||||
|
||||
Reference in new issue
Block a user