[Bugfix] Add seed support to TTS API for deterministic Fish Speech voice generation (#2624)
Signed-off-by: Yiyang Liu <yiyangliu@microsoft.com> Signed-off-by: Yiyang Liu <37043548+ianliuy@users.noreply.github.com> Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
7 files changed
+94
-9
No files matched your search
@@ -400,6 +400,7 @@ class FishSpeechFastAR(nn.Module):
|
||||
temperature: float = 0.8,
|
||||
top_k: int = 30,
|
||||
top_p: float = 0.9,
|
||||
seed: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Predict residual codebook codes 0..num_codebooks-1 autoregressively.
|
||||
|
||||
@@ -443,6 +444,12 @@ class FishSpeechFastAR(nn.Module):
|
||||
use_sampling = do_sample and temperature > 0
|
||||
inv_temperature = 1.0 / max(temperature, 1e-6) if use_sampling else 0.0
|
||||
|
||||
# Create a seeded generator for deterministic residual codebook sampling.
|
||||
generator = None
|
||||
if seed is not None and use_sampling:
|
||||
generator = torch.Generator(device=device)
|
||||
generator.manual_seed(seed)
|
||||
|
||||
# Residual codebook size (1024) vs semantic codebook size (4096).
|
||||
# The fast_output head has codebook_size (4096) outputs, but residual
|
||||
# codebooks only have 1024 entries. Truncate logits for steps > 0.
|
||||
@@ -474,7 +481,7 @@ class FishSpeechFastAR(nn.Module):
|
||||
sorted_logits[sorted_indices_to_remove] = float("-inf")
|
||||
scaled = sorted_logits.scatter(1, sorted_indices, sorted_logits)
|
||||
probs = F.softmax(scaled, dim=-1)
|
||||
next_ids = torch.multinomial(probs, num_samples=1)
|
||||
next_ids = torch.multinomial(probs, num_samples=1, generator=generator)
|
||||
else:
|
||||
next_ids = logits.argmax(dim=-1, keepdim=True)
|
||||
|
||||
|
||||
@@ -652,6 +652,7 @@ class FishSpeechSlowARForConditionalGeneration(nn.Module):
|
||||
input_embeds: torch.Tensor,
|
||||
last_talker_hidden: torch.Tensor,
|
||||
text_step: torch.Tensor,
|
||||
seed: int | None = None,
|
||||
**kwargs: Any,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""GPU fast-path: run Fast AR to predict residual codebook codes.
|
||||
@@ -680,6 +681,7 @@ class FishSpeechSlowARForConditionalGeneration(nn.Module):
|
||||
temperature=0.8,
|
||||
top_k=30,
|
||||
top_p=0.9,
|
||||
seed=seed,
|
||||
) # [B, num_codebooks]
|
||||
|
||||
# Add codebook embeddings to the input embedding (from preprocess).
|
||||
|
||||
@@ -1656,6 +1656,7 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
temperature: float | None = None,
|
||||
top_k: int | None = None,
|
||||
top_p: float | None = None,
|
||||
**kwargs: Any,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""GPU fast-path used by OmniGPUModelRunner to predict residual codebooks (1..Q-1).
|
||||
Returns (inputs_embeds, audio_codes) for the current step."""
|
||||
|
||||
Reference in new issue
Block a user