[Bugfix] Add seed support to TTS API for deterministic Fish Speech voice generation (#2624)

Signed-off-by: Yiyang Liu <yiyangliu@microsoft.com>
Signed-off-by: Yiyang Liu <37043548+ianliuy@users.noreply.github.com>
Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>

vllm_omni/entrypoints/openai/protocol/audio.py

@@ -69,6 +69,13 @@ class OpenAICreateSpeechRequest(BaseModel):
         default=None,
         description="Maximum tokens to generate",
     )
+    seed: int | None = Field(
+        default=None,
+        ge=0,
+        le=2**63 - 1,
+        description="Random seed for reproducible generation. When set, ensures "
+        "deterministic output for the same input text and seed value.",
+    )
     initial_codec_chunk_frames: int | None = Field(
         default=None,
         ge=0,

vllm_omni/entrypoints/openai/serving_speech.py

@@ -1797,6 +1797,22 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
             sampling_params_list = copy.deepcopy(sampling_params_list)
             sampling_params_list[0].max_tokens = request.max_new_tokens
 
+        # Propagate per-request seed to sampling params so both Slow AR
+        # and Fast AR produce deterministic output for the same seed.
+        if request.seed is not None and sampling_params_list:
+            if not self._is_fish_speech:
+                logger.warning(
+                    "seed=%d requested but deterministic Fast AR seeding is "
+                    "only implemented for Fish Speech; other TTS models will "
+                    "use the seed for the main AR sampler only.",
+                    request.seed,
+                )
+            if sampling_params_list is self.engine_client.default_sampling_params_list:
+                import copy
+
+                sampling_params_list = copy.deepcopy(sampling_params_list)
+            sampling_params_list[0].seed = request.seed
+
         generator = self.engine_client.generate(
             prompt=prompt,
             request_id=request_id,

vllm_omni/model_executor/models/fish_speech/fish_speech_fast_ar.py

@@ -400,6 +400,7 @@ class FishSpeechFastAR(nn.Module):
         temperature: float = 0.8,
         top_k: int = 30,
         top_p: float = 0.9,
+        seed: int | None = None,
     ) -> torch.Tensor:
         """Predict residual codebook codes 0..num_codebooks-1 autoregressively.
 
@@ -443,6 +444,12 @@ class FishSpeechFastAR(nn.Module):
         use_sampling = do_sample and temperature > 0
         inv_temperature = 1.0 / max(temperature, 1e-6) if use_sampling else 0.0
 
+        # Create a seeded generator for deterministic residual codebook sampling.
+        generator = None
+        if seed is not None and use_sampling:
+            generator = torch.Generator(device=device)
+            generator.manual_seed(seed)
+
         # Residual codebook size (1024) vs semantic codebook size (4096).
         # The fast_output head has codebook_size (4096) outputs, but residual
         # codebooks only have 1024 entries.  Truncate logits for steps > 0.
@@ -474,7 +481,7 @@ class FishSpeechFastAR(nn.Module):
                     sorted_logits[sorted_indices_to_remove] = float("-inf")
                     scaled = sorted_logits.scatter(1, sorted_indices, sorted_logits)
                 probs = F.softmax(scaled, dim=-1)
-                next_ids = torch.multinomial(probs, num_samples=1)
+                next_ids = torch.multinomial(probs, num_samples=1, generator=generator)
             else:
                 next_ids = logits.argmax(dim=-1, keepdim=True)
 

vllm_omni/model_executor/models/fish_speech/fish_speech_slow_ar.py

@@ -652,6 +652,7 @@ class FishSpeechSlowARForConditionalGeneration(nn.Module):
         input_embeds: torch.Tensor,
         last_talker_hidden: torch.Tensor,
         text_step: torch.Tensor,
+        seed: int | None = None,
         **kwargs: Any,
     ) -> tuple[torch.Tensor, torch.Tensor]:
         """GPU fast-path: run Fast AR to predict residual codebook codes.
@@ -680,6 +681,7 @@ class FishSpeechSlowARForConditionalGeneration(nn.Module):
             temperature=0.8,
             top_k=30,
             top_p=0.9,
+            seed=seed,
         )  # [B, num_codebooks]
 
         # Add codebook embeddings to the input embedding (from preprocess).

vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py

@@ -1656,6 +1656,7 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
         temperature: float | None = None,
         top_k: int | None = None,
         top_p: float | None = None,
+        **kwargs: Any,
     ) -> tuple[torch.Tensor, torch.Tensor]:
         """GPU fast-path used by OmniGPUModelRunner to predict residual codebooks (1..Q-1).
         Returns (inputs_embeds, audio_codes) for the current step."""

vllm_omni/platforms/npu/worker/npu_model_runner.py

@@ -420,6 +420,35 @@ class OmniNPUModelRunner(OmniGPUModelRunner, NPUModelRunner):
         subtalker_params = getattr(self.vllm_config.model_config, "subtalker_sampling_params", None)
         if not isinstance(subtalker_params, dict):
             subtalker_params = {}
+        # Extract seed from the first request's sampling params for Fast AR
+        # determinism.  NOTE: when batch_size > 1, all requests share the first
+        # request's seed.  Per-request Fast AR seeding requires row-by-row
+        # torch.multinomial calls and is left as a follow-up optimisation.
+        _seed = None
+        if decode_req_ids:
+            _first_sp = getattr(self.requests[decode_req_ids[0]], "sampling_params", None)
+            if _first_sp is not None and getattr(_first_sp, "seed", None) is not None:
+                _seed = _first_sp.seed
+            # Warn when batched requests have different seeds.
+            if len(decode_req_ids) > 1 and _seed is not None:
+                _other_seeds = {
+                    getattr(getattr(self.requests[rid], "sampling_params", None), "seed", None)
+                    for rid in decode_req_ids[1:]
+                }
+                if _other_seeds != {_seed}:
+                    logger.warning(
+                        "Fast AR seed: batch has mixed seeds; using first request's seed=%d for all %d requests.",
+                        _seed,
+                        len(decode_req_ids),
+                    )
+        talker_kwargs = {
+            "do_sample": subtalker_params.get("do_sample"),
+            "temperature": subtalker_params.get("temperature"),
+            "top_k": subtalker_params.get("top_k"),
+            "top_p": subtalker_params.get("top_p"),
+        }
+        if _seed is not None:
+            talker_kwargs["seed"] = _seed
         with set_ascend_forward_context(
             None, self.vllm_config, aclgraph_runtime_mode=_cudagraph_mode, batch_descriptor=batch_desc
         ):
@@ -428,10 +457,7 @@ class OmniNPUModelRunner(OmniGPUModelRunner, NPUModelRunner):
                 req_embeds,
                 last_talker_hidden,
                 text_step,
-                do_sample=subtalker_params.get("do_sample"),
-                temperature=subtalker_params.get("temperature"),
-                top_k=subtalker_params.get("top_k"),
-                top_p=subtalker_params.get("top_p"),
+                **talker_kwargs,
             )
         # update the inputs_embeds and code_predictor_codes
         code_predictor_codes_cpu = code_predictor_codes.detach().to("cpu").contiguous()

vllm_omni/worker/gpu_model_runner.py

@@ -1372,6 +1372,35 @@ class OmniGPUModelRunner(GPUModelRunner):
         subtalker_params = getattr(self.vllm_config.model_config, "subtalker_sampling_params", None)
         if not isinstance(subtalker_params, dict):
             subtalker_params = {}
+        # Extract seed from the first request's sampling params for Fast AR
+        # determinism.  NOTE: when batch_size > 1, all requests share the first
+        # request's seed.  Per-request Fast AR seeding requires row-by-row
+        # torch.multinomial calls and is left as a follow-up optimisation.
+        _seed = None
+        if decode_req_ids:
+            _first_sp = getattr(self.requests[decode_req_ids[0]], "sampling_params", None)
+            if _first_sp is not None and getattr(_first_sp, "seed", None) is not None:
+                _seed = _first_sp.seed
+            # Warn when batched requests have different seeds.
+            if len(decode_req_ids) > 1 and _seed is not None:
+                _other_seeds = {
+                    getattr(getattr(self.requests[rid], "sampling_params", None), "seed", None)
+                    for rid in decode_req_ids[1:]
+                }
+                if _other_seeds != {_seed}:
+                    logger.warning(
+                        "Fast AR seed: batch has mixed seeds; using first request's seed=%d for all %d requests.",
+                        _seed,
+                        len(decode_req_ids),
+                    )
+        talker_kwargs = {
+            "do_sample": subtalker_params.get("do_sample"),
+            "temperature": subtalker_params.get("temperature"),
+            "top_k": subtalker_params.get("top_k"),
+            "top_p": subtalker_params.get("top_p"),
+        }
+        if _seed is not None:
+            talker_kwargs["seed"] = _seed
         with set_forward_context(
             None, self.vllm_config, cudagraph_runtime_mode=_cudagraph_mode, batch_descriptor=batch_desc
         ):
@@ -1380,10 +1409,7 @@ class OmniGPUModelRunner(GPUModelRunner):
                 req_embeds,
                 last_talker_hidden,
                 text_step,
-                do_sample=subtalker_params.get("do_sample"),
-                temperature=subtalker_params.get("temperature"),
-                top_k=subtalker_params.get("top_k"),
-                top_p=subtalker_params.get("top_p"),
+                **talker_kwargs,
             )
         # update the inputs_embeds and code_predictor_codes
         out_key = getattr(self.model, "talker_mtp_output_key", ("codes", "audio"))