[Perf][Bugfix] qwen3-tts hot path: prefix-cache OOM guards + talker/orchestrator micro-opts (#3689)
Signed-off-by: JuanPZuluaga <juanz9312@gmal.com> Co-authored-by: JuanPZuluaga <juanz9312@gmal.com> Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
8 files changed
+54
-13
No files matched your search
@@ -22,6 +22,7 @@ class DummyInputBatch:
|
||||
|
||||
def __init__(self, req_ids):
|
||||
self.req_ids = req_ids
|
||||
self.req_id_to_index = {r: i for i, r in enumerate(req_ids)}
|
||||
|
||||
|
||||
class DummyReqState:
|
||||
|
||||
@@ -10,6 +10,9 @@ from vllm_omni.utils.mm_outputs import build_mm_cpu, to_payload_element
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Skip oversized per-key caches; otherwise hidden_states.layer_N can OOM on big models.
|
||||
_MAX_MM_CACHE_BYTES_PER_KEY = 512 * 1024 * 1024
|
||||
|
||||
|
||||
class OmniTensorPrefixCache:
|
||||
"""Prefix cache for hidden states (model outputs) and model specific
|
||||
@@ -46,6 +49,7 @@ class OmniTensorPrefixCache:
|
||||
# actually see mm output tensors dependent on num tokens.
|
||||
self.mm_outputs_cache = {}
|
||||
self.mm_cache_keys = set()
|
||||
self._mm_oversized_keys: set[str] = set()
|
||||
self._new_req_cache_hit_ids: set[str] = set()
|
||||
self._deferred_mm_outputs: dict[str, dict[str, list[tuple[int, torch.Tensor]]]] = {}
|
||||
|
||||
@@ -75,6 +79,17 @@ class OmniTensorPrefixCache:
|
||||
and key not in self.mm_cache_keys
|
||||
):
|
||||
feat_dim = val.shape[-1]
|
||||
# Bound per-key cache to avoid OOM on wide per-token features.
|
||||
key_bytes = self.num_blocks * self.block_size * feat_dim * val.element_size()
|
||||
if key_bytes > _MAX_MM_CACHE_BYTES_PER_KEY:
|
||||
logger.warning_once(
|
||||
"Skipping mm prefix cache key '%s': %.1f MiB > %.1f MiB cap.",
|
||||
key,
|
||||
key_bytes / 2**20,
|
||||
_MAX_MM_CACHE_BYTES_PER_KEY / 2**20,
|
||||
)
|
||||
self._mm_oversized_keys.add(key)
|
||||
continue
|
||||
self.mm_outputs_cache[key] = self._get_cache_tensor(
|
||||
dtype=val.dtype,
|
||||
hidden_size=feat_dim,
|
||||
@@ -417,7 +432,18 @@ class OmniTensorPrefixCache:
|
||||
# tensors similarly to the non prefix cached path, and then populate
|
||||
# the subdicts mapping request IDs -> payload objects
|
||||
passthrough_keys = set(mm_outputs.keys()) - self.mm_cache_keys
|
||||
passthrough_mm_data = {k: v for k, v in mm_outputs.items() if k in passthrough_keys}
|
||||
total_scheduled_tokens = sum(int(num_scheduled_tokens[r]) for r in input_batch.req_ids)
|
||||
passthrough_mm_data: dict[str, object] = {}
|
||||
for k in passthrough_keys:
|
||||
v = mm_outputs[k]
|
||||
if (
|
||||
k in self._mm_oversized_keys
|
||||
and isinstance(v, torch.Tensor)
|
||||
and v.dim() >= 2
|
||||
and v.shape[0] >= total_scheduled_tokens
|
||||
):
|
||||
v = v[:total_scheduled_tokens]
|
||||
passthrough_mm_data[k] = v
|
||||
mm_cpu = build_mm_cpu(multimodal_outputs=passthrough_mm_data)
|
||||
|
||||
for mm_key, mm_val in mm_cpu.items():
|
||||
|
||||
@@ -689,7 +689,7 @@ class Orchestrator:
|
||||
await self._handle_kv_ready_raw_outputs(stage_id, raw_outputs)
|
||||
for eco in raw_outputs.outputs:
|
||||
req_state = self.request_states.get(getattr(eco, "request_id", None))
|
||||
if req_state is None:
|
||||
if req_state is None or not req_state.streaming.enabled:
|
||||
continue
|
||||
req_state.streaming.segment_finished = bool(getattr(eco, "is_segment_finished", False))
|
||||
req_state.streaming.new_prompt_len_snapshot = getattr(
|
||||
|
||||
@@ -7,6 +7,7 @@ This module provides CUDA Graph acceleration for the speech tokenizer decoder,
|
||||
reducing kernel launch overhead during inference.
|
||||
"""
|
||||
|
||||
import bisect
|
||||
import os
|
||||
import time
|
||||
from collections import Counter
|
||||
@@ -225,9 +226,11 @@ class CUDAGraphDecoderWrapper:
|
||||
return sorted(sizes)
|
||||
|
||||
def _get_padded_size(self, actual_size: int) -> int | None:
|
||||
for size in self._bucket_sizes:
|
||||
if actual_size <= size:
|
||||
return size
|
||||
# bisect_left over the pre-sorted _bucket_sizes is O(log n) vs the linear scan;
|
||||
# this matters because _decode invokes it on every per-chunk replay.
|
||||
idx = bisect.bisect_left(self._bucket_sizes, actual_size)
|
||||
if idx < len(self._bucket_sizes):
|
||||
return self._bucket_sizes[idx]
|
||||
return None
|
||||
|
||||
def _get_capture_shapes(self) -> list[tuple[int, int]]:
|
||||
|
||||
@@ -545,7 +545,8 @@ class Qwen3TTSCode2Wav(nn.Module):
|
||||
continue
|
||||
wav = wav[start : min(end, wav.shape[0])]
|
||||
if wav.shape[0] > 0:
|
||||
audios[idx] = wav.to(dtype=torch.float32).reshape(-1)
|
||||
# Decoder already runs in fp32, so the .to(float32) is a redundant dispatch.
|
||||
audios[idx] = (wav if wav.dtype == torch.float32 else wav.to(torch.float32)).reshape(-1)
|
||||
|
||||
for req_id, finished in zip(ref_context_request_ids, finished_flags, strict=False):
|
||||
if req_id is not None and finished:
|
||||
|
||||
@@ -368,6 +368,7 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
)
|
||||
|
||||
# Constant logit mask: allow only codec ids [1, codebook_vocab_size) plus codec EOS.
|
||||
# Register the *disallowed* form so compute_logits doesn't recompute ~mask per step.
|
||||
vocab = int(self.talker_config.vocab_size)
|
||||
codec_mask = torch.zeros((vocab,), dtype=torch.bool)
|
||||
lo, hi = 1, min(self._codebook_vocab_size, vocab)
|
||||
@@ -375,7 +376,7 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
codec_mask[lo:hi] = True
|
||||
if 0 <= self._codec_eos_token_id < vocab:
|
||||
codec_mask[self._codec_eos_token_id] = True
|
||||
self.register_buffer("_codec_allowed_mask", codec_mask, persistent=False)
|
||||
self.register_buffer("_codec_disallowed_mask", ~codec_mask, persistent=False)
|
||||
|
||||
# Keys that should stay on GPU in model_intermediate_buffer to avoid
|
||||
# CPU-to-GPU round-trips on every decode step.
|
||||
@@ -513,7 +514,7 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
return None
|
||||
|
||||
# Mask out invalid codec ids using the pre-built constant buffer.
|
||||
logits = logits.masked_fill(~self._codec_allowed_mask, float("-inf"))
|
||||
logits = logits.masked_fill(self._codec_disallowed_mask, float("-inf"))
|
||||
|
||||
if self._eos_logit_bias != 0.0:
|
||||
eos_id = self._codec_eos_token_id
|
||||
|
||||
+12
-1
@@ -198,10 +198,21 @@ class Qwen3TTSTokenizerV2CausalConvNet(nn.Module):
|
||||
self.padding = self.kernel_size - self.stride
|
||||
|
||||
def _get_extra_padding_for_conv1d(self, hidden_state: torch.Tensor) -> int:
|
||||
# Cache the result per input length — under cudagraph capture the set of lengths
|
||||
# is fixed, so the LUT converges quickly and replaces the math.ceil call per replay.
|
||||
length = hidden_state.shape[-1]
|
||||
cache = self.__dict__.get("_pad_lut")
|
||||
if cache is None:
|
||||
cache = {}
|
||||
self.__dict__["_pad_lut"] = cache
|
||||
cached = cache.get(length)
|
||||
if cached is not None:
|
||||
return cached
|
||||
n_frames = (length - self.kernel_size + self.padding) / self.stride + 1
|
||||
ideal_length = (math.ceil(n_frames) - 1) * self.stride + (self.kernel_size - self.padding)
|
||||
return ideal_length - length
|
||||
extra_padding = ideal_length - length
|
||||
cache[length] = extra_padding
|
||||
return extra_padding
|
||||
|
||||
def forward(self, hidden_state):
|
||||
extra_padding = self._get_extra_padding_for_conv1d(hidden_state)
|
||||
|
||||
@@ -1790,10 +1790,8 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
if not isinstance(out_key, tuple) or len(out_key) != 2:
|
||||
raise TypeError(f"talker_mtp_output_key must be a 2-tuple, got {type(out_key).__name__}: {out_key!r}")
|
||||
if start_offsets is None:
|
||||
start_offsets = []
|
||||
for req_id in decode_req_ids:
|
||||
req_index = self.input_batch.req_ids.index(req_id)
|
||||
start_offsets.append(int(self.query_start_loc.cpu[req_index]))
|
||||
id_to_index = self.input_batch.req_id_to_index
|
||||
start_offsets = [int(self.query_start_loc.cpu[id_to_index[req_id]]) for req_id in decode_req_ids]
|
||||
for idx, (req_id, start_offset) in enumerate(zip(decode_req_ids, start_offsets, strict=True)):
|
||||
inputs_embeds[start_offset : start_offset + 1] = req_embeds[idx : idx + 1]
|
||||
update_dict = {out_key[0]: {out_key[1]: code_predictor_codes[idx : idx + 1]}}
|
||||
|
||||
Reference in new issue
Block a user