[Bugfix] Fix Qwen3-TTS code predictor crash due to missing vLLM config context (#1619)

Signed-off-by: Zhanqiu <zh338@cornell.edu>
This commit is contained in:
zhanqiuhu authored and GitHub committed 2026-03-03 11:02:13 +08:00
1 parent a09b6499f8
commit 80c89e00cb
1 file changed
+11 -10
@@ -370,16 +370,17 @@ class Qwen3TTSTalkerCodePredictorForConditionalGenerationVLLM(nn.Module):
def _maybe_init_kv_cache(self, device: torch.device) -> None:
if self._kv_cache is not None:
return
max_seq_len = int(getattr(self.config, "num_code_groups", 16) or 16)
# Upper bound on batch size: vLLM scheduler max_num_seqs (fallback 8).
max_batch = int(getattr(self._vllm_config.scheduler_config, "max_num_seqs", 8) or 8)
max_batch = max(1, max_batch)
self._kv_cache = _LocalPredictorKVCache(
vllm_config=self._vllm_config,
max_seq_len=max_seq_len,
max_batch_size=max_batch,
device=device,
)
with set_current_vllm_config(self._vllm_config):
max_seq_len = int(getattr(self.config, "num_code_groups", 16) or 16)
# Upper bound on batch size: vLLM scheduler max_num_seqs (fallback 8).
max_batch = int(getattr(self._vllm_config.scheduler_config, "max_num_seqs", 8) or 8)
max_batch = max(1, max_batch)
self._kv_cache = _LocalPredictorKVCache(
vllm_config=self._vllm_config,
max_seq_len=max_seq_len,
max_batch_size=max_batch,
device=device,
)
@torch.inference_mode()
def reset_cache(self) -> None: