[Perf][Qwen3-TTS] Sample code predictor outputs via Gumbel-max (#4261)
Signed-off-by: 林鑫 <77673148+l-wave@users.noreply.github.com> Co-authored-by: yurain <yurain26@users.noreply.github.com>
This commit is contained in:
1 parent
b3a4819694
commit
020792e6be
2 files changed
+136
-7
No files matched your search
@@ -28,6 +28,9 @@ from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_GeneratorLike = torch.Generator | Sequence[torch.Generator | None] | None
|
||||
_UNIFORM_EPS = 1e-20
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# HF-numerics-compatible layers for code predictor
|
||||
@@ -605,6 +608,30 @@ class CodePredictorWrapper(nn.Module):
|
||||
values.add(parsed)
|
||||
return values
|
||||
|
||||
@staticmethod
|
||||
def _normalize_generators(
|
||||
generator: _GeneratorLike, batch_size: int
|
||||
) -> torch.Generator | list[torch.Generator | None] | None:
|
||||
if generator is None or isinstance(generator, torch.Generator):
|
||||
return generator
|
||||
|
||||
row_generators = list(generator)
|
||||
if len(row_generators) != batch_size:
|
||||
raise ValueError(f"Expected {batch_size} per-row generators, but got {len(row_generators)}.")
|
||||
return row_generators
|
||||
|
||||
@classmethod
|
||||
def _sample_codes_gumbel(cls, logits: torch.Tensor, generator: _GeneratorLike = None) -> torch.Tensor:
|
||||
"""Sample ``logits`` via Gumbel-max with optional per-row generators."""
|
||||
row_generators = cls._normalize_generators(generator, int(logits.shape[0]))
|
||||
u = torch.empty_like(logits, dtype=torch.float32)
|
||||
if isinstance(row_generators, list):
|
||||
for row, row_generator in enumerate(row_generators):
|
||||
u[row : row + 1].uniform_(_UNIFORM_EPS, 1.0 - _UNIFORM_EPS, generator=row_generator)
|
||||
else:
|
||||
u.uniform_(_UNIFORM_EPS, 1.0 - _UNIFORM_EPS, generator=row_generators)
|
||||
return (logits.float() - torch.log(-torch.log(u))).argmax(dim=-1, keepdim=True)
|
||||
|
||||
def _prefix_seq_lens(self, max_seq: int) -> list[int]:
|
||||
all_seq_lens = list(range(2, max_seq))
|
||||
if not self._prefix_graph_seq_lens:
|
||||
@@ -780,6 +807,7 @@ class CodePredictorWrapper(nn.Module):
|
||||
bsz = int(layer0_code.shape[0])
|
||||
if generators is not None and len(generators) != bsz:
|
||||
raise ValueError(f"generators must have one entry per row: got {len(generators)} for batch {bsz}")
|
||||
sample_generator: _GeneratorLike = generators if generators is not None else generator
|
||||
num_groups = self._num_groups
|
||||
device = layer0_code.device
|
||||
|
||||
@@ -856,9 +884,20 @@ class CodePredictorWrapper(nn.Module):
|
||||
|
||||
logits = lm_heads[step - 1](hidden_out[:bsz, step, :])
|
||||
|
||||
# Sample next code
|
||||
# Sample next code via Gumbel-max.
|
||||
#
|
||||
# ``argmax_i(logits_i + Gumbel_i)`` with
|
||||
# ``Gumbel_i = -log(-log(u_i)), u_i ~ Uniform(0, 1)`` is
|
||||
# distributionally identical to sampling from ``softmax(logits)``.
|
||||
# In this file the motivations are practical rather than graph
|
||||
# related: it is measurably cheaper than ``softmax + multinomial``
|
||||
# on the B x 2048 shapes used here, it stays well-defined for
|
||||
# degenerate masked rows with a surviving finite entry (and is more
|
||||
# defensive than ``multinomial`` around fully-masked/NaN inputs),
|
||||
# and the helper below can honor either one batch generator or one
|
||||
# generator per seeded row.
|
||||
if stored_mode:
|
||||
# "stored" mode: top-k -> top-p -> softmax -> multinomial
|
||||
# "stored" mode: top-k -> top-p -> Gumbel-max
|
||||
if s_top_k > 0:
|
||||
topk_vals, _ = logits.topk(s_top_k, dim=-1)
|
||||
logits = logits.masked_fill(logits < topk_vals[:, -1:], float("-inf"))
|
||||
@@ -869,17 +908,15 @@ class CodePredictorWrapper(nn.Module):
|
||||
remove_mask = (cumulative_probs - sorted_probs) >= s_top_p
|
||||
sorted_logits[remove_mask] = float("-inf")
|
||||
logits = sorted_logits.scatter(1, sorted_idx, sorted_logits)
|
||||
probs = F.softmax(logits, dim=-1, dtype=torch.float32)
|
||||
code = self._multinomial(probs, generator, generators)
|
||||
code = self._sample_codes_gumbel(logits, generator=sample_generator)
|
||||
else:
|
||||
# "per_call" mode: temperature-scaled + top-k
|
||||
# "per_call" mode: temperature-scaled + top-k -> Gumbel-max
|
||||
if use_sampling:
|
||||
scaled = logits * inv_temperature
|
||||
if top_k > 0:
|
||||
topk_vals, _ = scaled.topk(top_k, dim=-1)
|
||||
scaled = scaled.masked_fill(scaled < topk_vals[:, -1:], float("-inf"))
|
||||
probs = F.softmax(scaled, dim=-1, dtype=torch.float32)
|
||||
code = self._multinomial(probs, generator, generators)
|
||||
code = self._sample_codes_gumbel(scaled, generator=sample_generator)
|
||||
else:
|
||||
code = logits.argmax(dim=-1, keepdim=True)
|
||||
|
||||
|
||||
Reference in new issue
Block a user