From 62c86c9d626e931ee048175aede2ea7b26f4f1d5 Mon Sep 17 00:00:00 2001 From: "Yiyang \"Ian\" Liu" Date: Sun, 26 Apr 2026 12:15:30 -0700 Subject: [PATCH] [Bugfix] Add seed support to TTS API for deterministic Fish Speech voice generation (#2624) Signed-off-by: Yiyang Liu Signed-off-by: Yiyang Liu <37043548+ianliuy@users.noreply.github.com> Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> --- .../entrypoints/openai/protocol/audio.py | 7 ++++ .../entrypoints/openai/serving_speech.py | 16 +++++++++ .../models/fish_speech/fish_speech_fast_ar.py | 9 ++++- .../models/fish_speech/fish_speech_slow_ar.py | 2 ++ .../models/qwen3_tts/qwen3_tts_talker.py | 1 + .../platforms/npu/worker/npu_model_runner.py | 34 ++++++++++++++++--- vllm_omni/worker/gpu_model_runner.py | 34 ++++++++++++++++--- 7 files changed, 94 insertions(+), 9 deletions(-) diff --git a/vllm_omni/entrypoints/openai/protocol/audio.py b/vllm_omni/entrypoints/openai/protocol/audio.py index 59b5777a8..5144e0744 100644 --- a/vllm_omni/entrypoints/openai/protocol/audio.py +++ b/vllm_omni/entrypoints/openai/protocol/audio.py @@ -69,6 +69,13 @@ class OpenAICreateSpeechRequest(BaseModel): default=None, description="Maximum tokens to generate", ) + seed: int | None = Field( + default=None, + ge=0, + le=2**63 - 1, + description="Random seed for reproducible generation. When set, ensures " + "deterministic output for the same input text and seed value.", + ) initial_codec_chunk_frames: int | None = Field( default=None, ge=0, diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index eb8a12c66..08cf44d81 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -1797,6 +1797,22 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): sampling_params_list = copy.deepcopy(sampling_params_list) sampling_params_list[0].max_tokens = request.max_new_tokens + # Propagate per-request seed to sampling params so both Slow AR + # and Fast AR produce deterministic output for the same seed. + if request.seed is not None and sampling_params_list: + if not self._is_fish_speech: + logger.warning( + "seed=%d requested but deterministic Fast AR seeding is " + "only implemented for Fish Speech; other TTS models will " + "use the seed for the main AR sampler only.", + request.seed, + ) + if sampling_params_list is self.engine_client.default_sampling_params_list: + import copy + + sampling_params_list = copy.deepcopy(sampling_params_list) + sampling_params_list[0].seed = request.seed + generator = self.engine_client.generate( prompt=prompt, request_id=request_id, diff --git a/vllm_omni/model_executor/models/fish_speech/fish_speech_fast_ar.py b/vllm_omni/model_executor/models/fish_speech/fish_speech_fast_ar.py index 22a2744ff..cc38c200a 100644 --- a/vllm_omni/model_executor/models/fish_speech/fish_speech_fast_ar.py +++ b/vllm_omni/model_executor/models/fish_speech/fish_speech_fast_ar.py @@ -400,6 +400,7 @@ class FishSpeechFastAR(nn.Module): temperature: float = 0.8, top_k: int = 30, top_p: float = 0.9, + seed: int | None = None, ) -> torch.Tensor: """Predict residual codebook codes 0..num_codebooks-1 autoregressively. @@ -443,6 +444,12 @@ class FishSpeechFastAR(nn.Module): use_sampling = do_sample and temperature > 0 inv_temperature = 1.0 / max(temperature, 1e-6) if use_sampling else 0.0 + # Create a seeded generator for deterministic residual codebook sampling. + generator = None + if seed is not None and use_sampling: + generator = torch.Generator(device=device) + generator.manual_seed(seed) + # Residual codebook size (1024) vs semantic codebook size (4096). # The fast_output head has codebook_size (4096) outputs, but residual # codebooks only have 1024 entries. Truncate logits for steps > 0. @@ -474,7 +481,7 @@ class FishSpeechFastAR(nn.Module): sorted_logits[sorted_indices_to_remove] = float("-inf") scaled = sorted_logits.scatter(1, sorted_indices, sorted_logits) probs = F.softmax(scaled, dim=-1) - next_ids = torch.multinomial(probs, num_samples=1) + next_ids = torch.multinomial(probs, num_samples=1, generator=generator) else: next_ids = logits.argmax(dim=-1, keepdim=True) diff --git a/vllm_omni/model_executor/models/fish_speech/fish_speech_slow_ar.py b/vllm_omni/model_executor/models/fish_speech/fish_speech_slow_ar.py index e46451180..582dbf344 100644 --- a/vllm_omni/model_executor/models/fish_speech/fish_speech_slow_ar.py +++ b/vllm_omni/model_executor/models/fish_speech/fish_speech_slow_ar.py @@ -652,6 +652,7 @@ class FishSpeechSlowARForConditionalGeneration(nn.Module): input_embeds: torch.Tensor, last_talker_hidden: torch.Tensor, text_step: torch.Tensor, + seed: int | None = None, **kwargs: Any, ) -> tuple[torch.Tensor, torch.Tensor]: """GPU fast-path: run Fast AR to predict residual codebook codes. @@ -680,6 +681,7 @@ class FishSpeechSlowARForConditionalGeneration(nn.Module): temperature=0.8, top_k=30, top_p=0.9, + seed=seed, ) # [B, num_codebooks] # Add codebook embeddings to the input embedding (from preprocess). diff --git a/vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py b/vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py index 53e2a5480..3324d9196 100644 --- a/vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py +++ b/vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py @@ -1656,6 +1656,7 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module): temperature: float | None = None, top_k: int | None = None, top_p: float | None = None, + **kwargs: Any, ) -> tuple[torch.Tensor, torch.Tensor]: """GPU fast-path used by OmniGPUModelRunner to predict residual codebooks (1..Q-1). Returns (inputs_embeds, audio_codes) for the current step.""" diff --git a/vllm_omni/platforms/npu/worker/npu_model_runner.py b/vllm_omni/platforms/npu/worker/npu_model_runner.py index 77b6e9ef0..a7000604f 100644 --- a/vllm_omni/platforms/npu/worker/npu_model_runner.py +++ b/vllm_omni/platforms/npu/worker/npu_model_runner.py @@ -420,6 +420,35 @@ class OmniNPUModelRunner(OmniGPUModelRunner, NPUModelRunner): subtalker_params = getattr(self.vllm_config.model_config, "subtalker_sampling_params", None) if not isinstance(subtalker_params, dict): subtalker_params = {} + # Extract seed from the first request's sampling params for Fast AR + # determinism. NOTE: when batch_size > 1, all requests share the first + # request's seed. Per-request Fast AR seeding requires row-by-row + # torch.multinomial calls and is left as a follow-up optimisation. + _seed = None + if decode_req_ids: + _first_sp = getattr(self.requests[decode_req_ids[0]], "sampling_params", None) + if _first_sp is not None and getattr(_first_sp, "seed", None) is not None: + _seed = _first_sp.seed + # Warn when batched requests have different seeds. + if len(decode_req_ids) > 1 and _seed is not None: + _other_seeds = { + getattr(getattr(self.requests[rid], "sampling_params", None), "seed", None) + for rid in decode_req_ids[1:] + } + if _other_seeds != {_seed}: + logger.warning( + "Fast AR seed: batch has mixed seeds; using first request's seed=%d for all %d requests.", + _seed, + len(decode_req_ids), + ) + talker_kwargs = { + "do_sample": subtalker_params.get("do_sample"), + "temperature": subtalker_params.get("temperature"), + "top_k": subtalker_params.get("top_k"), + "top_p": subtalker_params.get("top_p"), + } + if _seed is not None: + talker_kwargs["seed"] = _seed with set_ascend_forward_context( None, self.vllm_config, aclgraph_runtime_mode=_cudagraph_mode, batch_descriptor=batch_desc ): @@ -428,10 +457,7 @@ class OmniNPUModelRunner(OmniGPUModelRunner, NPUModelRunner): req_embeds, last_talker_hidden, text_step, - do_sample=subtalker_params.get("do_sample"), - temperature=subtalker_params.get("temperature"), - top_k=subtalker_params.get("top_k"), - top_p=subtalker_params.get("top_p"), + **talker_kwargs, ) # update the inputs_embeds and code_predictor_codes code_predictor_codes_cpu = code_predictor_codes.detach().to("cpu").contiguous() diff --git a/vllm_omni/worker/gpu_model_runner.py b/vllm_omni/worker/gpu_model_runner.py index d914f1b39..efad5f3d4 100644 --- a/vllm_omni/worker/gpu_model_runner.py +++ b/vllm_omni/worker/gpu_model_runner.py @@ -1372,6 +1372,35 @@ class OmniGPUModelRunner(GPUModelRunner): subtalker_params = getattr(self.vllm_config.model_config, "subtalker_sampling_params", None) if not isinstance(subtalker_params, dict): subtalker_params = {} + # Extract seed from the first request's sampling params for Fast AR + # determinism. NOTE: when batch_size > 1, all requests share the first + # request's seed. Per-request Fast AR seeding requires row-by-row + # torch.multinomial calls and is left as a follow-up optimisation. + _seed = None + if decode_req_ids: + _first_sp = getattr(self.requests[decode_req_ids[0]], "sampling_params", None) + if _first_sp is not None and getattr(_first_sp, "seed", None) is not None: + _seed = _first_sp.seed + # Warn when batched requests have different seeds. + if len(decode_req_ids) > 1 and _seed is not None: + _other_seeds = { + getattr(getattr(self.requests[rid], "sampling_params", None), "seed", None) + for rid in decode_req_ids[1:] + } + if _other_seeds != {_seed}: + logger.warning( + "Fast AR seed: batch has mixed seeds; using first request's seed=%d for all %d requests.", + _seed, + len(decode_req_ids), + ) + talker_kwargs = { + "do_sample": subtalker_params.get("do_sample"), + "temperature": subtalker_params.get("temperature"), + "top_k": subtalker_params.get("top_k"), + "top_p": subtalker_params.get("top_p"), + } + if _seed is not None: + talker_kwargs["seed"] = _seed with set_forward_context( None, self.vllm_config, cudagraph_runtime_mode=_cudagraph_mode, batch_descriptor=batch_desc ): @@ -1380,10 +1409,7 @@ class OmniGPUModelRunner(GPUModelRunner): req_embeds, last_talker_hidden, text_step, - do_sample=subtalker_params.get("do_sample"), - temperature=subtalker_params.get("temperature"), - top_k=subtalker_params.get("top_k"), - top_p=subtalker_params.get("top_p"), + **talker_kwargs, ) # update the inputs_embeds and code_predictor_codes out_key = getattr(self.model, "talker_mtp_output_key", ("codes", "audio"))