[Perf][Bugfix] qwen3-tts hot path: prefix-cache OOM guards + talker/orchestrator micro-opts (#3689)

Signed-off-by: JuanPZuluaga <juanz9312@gmal.com>
Co-authored-by: JuanPZuluaga <juanz9312@gmal.com>
Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
authored and GitHub committed 2026-06-09 10:36:57 -07:00
1 parent ca75623ff3
commit 57227dc7f3
8 files changed
+54 -13

No files matched your search

@@ -22,6 +22,7 @@ class DummyInputBatch:
def __init__(self, req_ids):
self.req_ids = req_ids
self.req_id_to_index = {r: i for i, r in enumerate(req_ids)}
class DummyReqState:
+27 -1
View File
@@ -10,6 +10,9 @@ from vllm_omni.utils.mm_outputs import build_mm_cpu, to_payload_element
logger = init_logger(__name__)
# Skip oversized per-key caches; otherwise hidden_states.layer_N can OOM on big models.
_MAX_MM_CACHE_BYTES_PER_KEY = 512 * 1024 * 1024
class OmniTensorPrefixCache:
"""Prefix cache for hidden states (model outputs) and model specific
@@ -46,6 +49,7 @@ class OmniTensorPrefixCache:
# actually see mm output tensors dependent on num tokens.
self.mm_outputs_cache = {}
self.mm_cache_keys = set()
self._mm_oversized_keys: set[str] = set()
self._new_req_cache_hit_ids: set[str] = set()
self._deferred_mm_outputs: dict[str, dict[str, list[tuple[int, torch.Tensor]]]] = {}
@@ -75,6 +79,17 @@ class OmniTensorPrefixCache:
and key not in self.mm_cache_keys
):
feat_dim = val.shape[-1]
# Bound per-key cache to avoid OOM on wide per-token features.
key_bytes = self.num_blocks * self.block_size * feat_dim * val.element_size()
if key_bytes > _MAX_MM_CACHE_BYTES_PER_KEY:
logger.warning_once(
"Skipping mm prefix cache key '%s': %.1f MiB > %.1f MiB cap.",
key,
key_bytes / 2**20,
_MAX_MM_CACHE_BYTES_PER_KEY / 2**20,
)
self._mm_oversized_keys.add(key)
continue
self.mm_outputs_cache[key] = self._get_cache_tensor(
dtype=val.dtype,
hidden_size=feat_dim,
@@ -417,7 +432,18 @@ class OmniTensorPrefixCache:
# tensors similarly to the non prefix cached path, and then populate
# the subdicts mapping request IDs -> payload objects
passthrough_keys = set(mm_outputs.keys()) - self.mm_cache_keys
passthrough_mm_data = {k: v for k, v in mm_outputs.items() if k in passthrough_keys}
total_scheduled_tokens = sum(int(num_scheduled_tokens[r]) for r in input_batch.req_ids)
passthrough_mm_data: dict[str, object] = {}
for k in passthrough_keys:
v = mm_outputs[k]
if (
k in self._mm_oversized_keys
and isinstance(v, torch.Tensor)
and v.dim() >= 2
and v.shape[0] >= total_scheduled_tokens
):
v = v[:total_scheduled_tokens]
passthrough_mm_data[k] = v
mm_cpu = build_mm_cpu(multimodal_outputs=passthrough_mm_data)
for mm_key, mm_val in mm_cpu.items():
+1 -1
View File
@@ -689,7 +689,7 @@ class Orchestrator:
await self._handle_kv_ready_raw_outputs(stage_id, raw_outputs)
for eco in raw_outputs.outputs:
req_state = self.request_states.get(getattr(eco, "request_id", None))
if req_state is None:
if req_state is None or not req_state.streaming.enabled:
continue
req_state.streaming.segment_finished = bool(getattr(eco, "is_segment_finished", False))
req_state.streaming.new_prompt_len_snapshot = getattr(
@@ -7,6 +7,7 @@ This module provides CUDA Graph acceleration for the speech tokenizer decoder,
reducing kernel launch overhead during inference.
"""
import bisect
import os
import time
from collections import Counter
@@ -225,9 +226,11 @@ class CUDAGraphDecoderWrapper:
return sorted(sizes)
def _get_padded_size(self, actual_size: int) -> int | None:
for size in self._bucket_sizes:
if actual_size <= size:
return size
# bisect_left over the pre-sorted _bucket_sizes is O(log n) vs the linear scan;
# this matters because _decode invokes it on every per-chunk replay.
idx = bisect.bisect_left(self._bucket_sizes, actual_size)
if idx < len(self._bucket_sizes):
return self._bucket_sizes[idx]
return None
def _get_capture_shapes(self) -> list[tuple[int, int]]:
@@ -545,7 +545,8 @@ class Qwen3TTSCode2Wav(nn.Module):
continue
wav = wav[start : min(end, wav.shape[0])]
if wav.shape[0] > 0:
audios[idx] = wav.to(dtype=torch.float32).reshape(-1)
# Decoder already runs in fp32, so the .to(float32) is a redundant dispatch.
audios[idx] = (wav if wav.dtype == torch.float32 else wav.to(torch.float32)).reshape(-1)
for req_id, finished in zip(ref_context_request_ids, finished_flags, strict=False):
if req_id is not None and finished:
@@ -368,6 +368,7 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
)
# Constant logit mask: allow only codec ids [1, codebook_vocab_size) plus codec EOS.
# Register the *disallowed* form so compute_logits doesn't recompute ~mask per step.
vocab = int(self.talker_config.vocab_size)
codec_mask = torch.zeros((vocab,), dtype=torch.bool)
lo, hi = 1, min(self._codebook_vocab_size, vocab)
@@ -375,7 +376,7 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
codec_mask[lo:hi] = True
if 0 <= self._codec_eos_token_id < vocab:
codec_mask[self._codec_eos_token_id] = True
self.register_buffer("_codec_allowed_mask", codec_mask, persistent=False)
self.register_buffer("_codec_disallowed_mask", ~codec_mask, persistent=False)
# Keys that should stay on GPU in model_intermediate_buffer to avoid
# CPU-to-GPU round-trips on every decode step.
@@ -513,7 +514,7 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
return None
# Mask out invalid codec ids using the pre-built constant buffer.
logits = logits.masked_fill(~self._codec_allowed_mask, float("-inf"))
logits = logits.masked_fill(self._codec_disallowed_mask, float("-inf"))
if self._eos_logit_bias != 0.0:
eos_id = self._codec_eos_token_id
@@ -198,10 +198,21 @@ class Qwen3TTSTokenizerV2CausalConvNet(nn.Module):
self.padding = self.kernel_size - self.stride
def _get_extra_padding_for_conv1d(self, hidden_state: torch.Tensor) -> int:
# Cache the result per input length — under cudagraph capture the set of lengths
# is fixed, so the LUT converges quickly and replaces the math.ceil call per replay.
length = hidden_state.shape[-1]
cache = self.__dict__.get("_pad_lut")
if cache is None:
cache = {}
self.__dict__["_pad_lut"] = cache
cached = cache.get(length)
if cached is not None:
return cached
n_frames = (length - self.kernel_size + self.padding) / self.stride + 1
ideal_length = (math.ceil(n_frames) - 1) * self.stride + (self.kernel_size - self.padding)
return ideal_length - length
extra_padding = ideal_length - length
cache[length] = extra_padding
return extra_padding
def forward(self, hidden_state):
extra_padding = self._get_extra_padding_for_conv1d(hidden_state)
+2 -4
View File
@@ -1790,10 +1790,8 @@ class OmniGPUModelRunner(GPUModelRunner):
if not isinstance(out_key, tuple) or len(out_key) != 2:
raise TypeError(f"talker_mtp_output_key must be a 2-tuple, got {type(out_key).__name__}: {out_key!r}")
if start_offsets is None:
start_offsets = []
for req_id in decode_req_ids:
req_index = self.input_batch.req_ids.index(req_id)
start_offsets.append(int(self.query_start_loc.cpu[req_index]))
id_to_index = self.input_batch.req_id_to_index
start_offsets = [int(self.query_start_loc.cpu[id_to_index[req_id]]) for req_id in decode_req_ids]
for idx, (req_id, start_offset) in enumerate(zip(decode_req_ids, start_offsets, strict=True)):
inputs_embeds[start_offset : start_offset + 1] = req_embeds[idx : idx + 1]
update_dict = {out_key[0]: {out_key[1]: code_predictor_codes[idx : idx + 1]}}