From 57227dc7f32e61110b120ab82931cfc51a65c65e Mon Sep 17 00:00:00 2001 From: Juan Pablo Zuluaga <46724788+JuanPZuluaga@users.noreply.github.com> Date: Tue, 9 Jun 2026 19:36:57 +0200 Subject: [PATCH] [Perf][Bugfix] qwen3-tts hot path: prefix-cache OOM guards + talker/orchestrator micro-opts (#3689) Signed-off-by: JuanPZuluaga Co-authored-by: JuanPZuluaga Co-authored-by: Gao Han --- tests/worker/test_omni_gpu_model_runner.py | 1 + vllm_omni/core/prefix_cache.py | 28 ++++++++++++++++++- vllm_omni/engine/orchestrator.py | 2 +- .../qwen3_tts/cuda_graph_decoder_wrapper.py | 9 ++++-- .../models/qwen3_tts/qwen3_tts_code2wav.py | 3 +- .../models/qwen3_tts/qwen3_tts_talker.py | 5 ++-- .../modeling_qwen3_tts_tokenizer_v2.py | 13 ++++++++- vllm_omni/worker/gpu_model_runner.py | 6 ++-- 8 files changed, 54 insertions(+), 13 deletions(-) diff --git a/tests/worker/test_omni_gpu_model_runner.py b/tests/worker/test_omni_gpu_model_runner.py index ae6ecc57c..6f4994b2c 100644 --- a/tests/worker/test_omni_gpu_model_runner.py +++ b/tests/worker/test_omni_gpu_model_runner.py @@ -22,6 +22,7 @@ class DummyInputBatch: def __init__(self, req_ids): self.req_ids = req_ids + self.req_id_to_index = {r: i for i, r in enumerate(req_ids)} class DummyReqState: diff --git a/vllm_omni/core/prefix_cache.py b/vllm_omni/core/prefix_cache.py index 25acfe8ed..5e127fd44 100644 --- a/vllm_omni/core/prefix_cache.py +++ b/vllm_omni/core/prefix_cache.py @@ -10,6 +10,9 @@ from vllm_omni.utils.mm_outputs import build_mm_cpu, to_payload_element logger = init_logger(__name__) +# Skip oversized per-key caches; otherwise hidden_states.layer_N can OOM on big models. +_MAX_MM_CACHE_BYTES_PER_KEY = 512 * 1024 * 1024 + class OmniTensorPrefixCache: """Prefix cache for hidden states (model outputs) and model specific @@ -46,6 +49,7 @@ class OmniTensorPrefixCache: # actually see mm output tensors dependent on num tokens. self.mm_outputs_cache = {} self.mm_cache_keys = set() + self._mm_oversized_keys: set[str] = set() self._new_req_cache_hit_ids: set[str] = set() self._deferred_mm_outputs: dict[str, dict[str, list[tuple[int, torch.Tensor]]]] = {} @@ -75,6 +79,17 @@ class OmniTensorPrefixCache: and key not in self.mm_cache_keys ): feat_dim = val.shape[-1] + # Bound per-key cache to avoid OOM on wide per-token features. + key_bytes = self.num_blocks * self.block_size * feat_dim * val.element_size() + if key_bytes > _MAX_MM_CACHE_BYTES_PER_KEY: + logger.warning_once( + "Skipping mm prefix cache key '%s': %.1f MiB > %.1f MiB cap.", + key, + key_bytes / 2**20, + _MAX_MM_CACHE_BYTES_PER_KEY / 2**20, + ) + self._mm_oversized_keys.add(key) + continue self.mm_outputs_cache[key] = self._get_cache_tensor( dtype=val.dtype, hidden_size=feat_dim, @@ -417,7 +432,18 @@ class OmniTensorPrefixCache: # tensors similarly to the non prefix cached path, and then populate # the subdicts mapping request IDs -> payload objects passthrough_keys = set(mm_outputs.keys()) - self.mm_cache_keys - passthrough_mm_data = {k: v for k, v in mm_outputs.items() if k in passthrough_keys} + total_scheduled_tokens = sum(int(num_scheduled_tokens[r]) for r in input_batch.req_ids) + passthrough_mm_data: dict[str, object] = {} + for k in passthrough_keys: + v = mm_outputs[k] + if ( + k in self._mm_oversized_keys + and isinstance(v, torch.Tensor) + and v.dim() >= 2 + and v.shape[0] >= total_scheduled_tokens + ): + v = v[:total_scheduled_tokens] + passthrough_mm_data[k] = v mm_cpu = build_mm_cpu(multimodal_outputs=passthrough_mm_data) for mm_key, mm_val in mm_cpu.items(): diff --git a/vllm_omni/engine/orchestrator.py b/vllm_omni/engine/orchestrator.py index ea20c650b..ae4d2bbcd 100644 --- a/vllm_omni/engine/orchestrator.py +++ b/vllm_omni/engine/orchestrator.py @@ -689,7 +689,7 @@ class Orchestrator: await self._handle_kv_ready_raw_outputs(stage_id, raw_outputs) for eco in raw_outputs.outputs: req_state = self.request_states.get(getattr(eco, "request_id", None)) - if req_state is None: + if req_state is None or not req_state.streaming.enabled: continue req_state.streaming.segment_finished = bool(getattr(eco, "is_segment_finished", False)) req_state.streaming.new_prompt_len_snapshot = getattr( diff --git a/vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py b/vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py index 1c25e0842..604946a35 100644 --- a/vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py +++ b/vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py @@ -7,6 +7,7 @@ This module provides CUDA Graph acceleration for the speech tokenizer decoder, reducing kernel launch overhead during inference. """ +import bisect import os import time from collections import Counter @@ -225,9 +226,11 @@ class CUDAGraphDecoderWrapper: return sorted(sizes) def _get_padded_size(self, actual_size: int) -> int | None: - for size in self._bucket_sizes: - if actual_size <= size: - return size + # bisect_left over the pre-sorted _bucket_sizes is O(log n) vs the linear scan; + # this matters because _decode invokes it on every per-chunk replay. + idx = bisect.bisect_left(self._bucket_sizes, actual_size) + if idx < len(self._bucket_sizes): + return self._bucket_sizes[idx] return None def _get_capture_shapes(self) -> list[tuple[int, int]]: diff --git a/vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code2wav.py b/vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code2wav.py index 7991d85a1..6a5866c7e 100644 --- a/vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code2wav.py +++ b/vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code2wav.py @@ -545,7 +545,8 @@ class Qwen3TTSCode2Wav(nn.Module): continue wav = wav[start : min(end, wav.shape[0])] if wav.shape[0] > 0: - audios[idx] = wav.to(dtype=torch.float32).reshape(-1) + # Decoder already runs in fp32, so the .to(float32) is a redundant dispatch. + audios[idx] = (wav if wav.dtype == torch.float32 else wav.to(torch.float32)).reshape(-1) for req_id, finished in zip(ref_context_request_ids, finished_flags, strict=False): if req_id is not None and finished: diff --git a/vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py b/vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py index d97f21233..541efb580 100644 --- a/vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py +++ b/vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py @@ -368,6 +368,7 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module): ) # Constant logit mask: allow only codec ids [1, codebook_vocab_size) plus codec EOS. + # Register the *disallowed* form so compute_logits doesn't recompute ~mask per step. vocab = int(self.talker_config.vocab_size) codec_mask = torch.zeros((vocab,), dtype=torch.bool) lo, hi = 1, min(self._codebook_vocab_size, vocab) @@ -375,7 +376,7 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module): codec_mask[lo:hi] = True if 0 <= self._codec_eos_token_id < vocab: codec_mask[self._codec_eos_token_id] = True - self.register_buffer("_codec_allowed_mask", codec_mask, persistent=False) + self.register_buffer("_codec_disallowed_mask", ~codec_mask, persistent=False) # Keys that should stay on GPU in model_intermediate_buffer to avoid # CPU-to-GPU round-trips on every decode step. @@ -513,7 +514,7 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module): return None # Mask out invalid codec ids using the pre-built constant buffer. - logits = logits.masked_fill(~self._codec_allowed_mask, float("-inf")) + logits = logits.masked_fill(self._codec_disallowed_mask, float("-inf")) if self._eos_logit_bias != 0.0: eos_id = self._codec_eos_token_id diff --git a/vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py b/vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py index 99b26ea1e..6eb78eeae 100644 --- a/vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py +++ b/vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py @@ -198,10 +198,21 @@ class Qwen3TTSTokenizerV2CausalConvNet(nn.Module): self.padding = self.kernel_size - self.stride def _get_extra_padding_for_conv1d(self, hidden_state: torch.Tensor) -> int: + # Cache the result per input length — under cudagraph capture the set of lengths + # is fixed, so the LUT converges quickly and replaces the math.ceil call per replay. length = hidden_state.shape[-1] + cache = self.__dict__.get("_pad_lut") + if cache is None: + cache = {} + self.__dict__["_pad_lut"] = cache + cached = cache.get(length) + if cached is not None: + return cached n_frames = (length - self.kernel_size + self.padding) / self.stride + 1 ideal_length = (math.ceil(n_frames) - 1) * self.stride + (self.kernel_size - self.padding) - return ideal_length - length + extra_padding = ideal_length - length + cache[length] = extra_padding + return extra_padding def forward(self, hidden_state): extra_padding = self._get_extra_padding_for_conv1d(hidden_state) diff --git a/vllm_omni/worker/gpu_model_runner.py b/vllm_omni/worker/gpu_model_runner.py index 70f1f58dc..abef62f31 100644 --- a/vllm_omni/worker/gpu_model_runner.py +++ b/vllm_omni/worker/gpu_model_runner.py @@ -1790,10 +1790,8 @@ class OmniGPUModelRunner(GPUModelRunner): if not isinstance(out_key, tuple) or len(out_key) != 2: raise TypeError(f"talker_mtp_output_key must be a 2-tuple, got {type(out_key).__name__}: {out_key!r}") if start_offsets is None: - start_offsets = [] - for req_id in decode_req_ids: - req_index = self.input_batch.req_ids.index(req_id) - start_offsets.append(int(self.query_start_loc.cpu[req_index])) + id_to_index = self.input_batch.req_id_to_index + start_offsets = [int(self.query_start_loc.cpu[id_to_index[req_id]]) for req_id in decode_req_ids] for idx, (req_id, start_offset) in enumerate(zip(decode_req_ids, start_offsets, strict=True)): inputs_embeds[start_offset : start_offset + 1] = req_embeds[idx : idx + 1] update_dict = {out_key[0]: {out_key[1]: code_predictor_codes[idx : idx + 1]}}