[Model] Add Ming-omni-tts dense 0.5B pipeline (#2906)

Signed-off-by: akshatvishu <akshatnayak197@gmail.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
akshatvishuandHongsheng Liu authored and GitHub committed 2026-06-10 20:11:04 +08:00
1 parent 39ae6e12f8
commit 2fa4c7a4a9
55 files changed
+6263 -549

No files matched your search

+61 -5
View File
@@ -7,6 +7,43 @@ from pydantic import AliasChoices, BaseModel, Field, field_validator, model_vali
_MAX_EMBEDDING_DIM = 8192
def _normalize_ref_audio_value(value):
if value is None:
return None
if isinstance(value, str):
return value
if isinstance(value, (list, tuple)):
items = []
for item in value:
if not isinstance(item, str):
raise TypeError("'ref_audio' list entries must be strings")
items.append(item)
if not items:
raise ValueError("'ref_audio' list cannot be empty")
return items
raise TypeError("'ref_audio' must be a string or list of strings")
def _normalize_speaker_embedding_value(value):
if value is None:
return None
if not isinstance(value, (list, tuple)):
raise TypeError("'speaker_embedding' must be a list of numbers or list of embedding vectors")
if not value:
return []
first = value[0]
if isinstance(first, (list, tuple)):
embeddings = []
for item in value:
if not isinstance(item, (list, tuple)):
raise TypeError("'speaker_embedding' must not mix flat and nested values")
embeddings.append([float(x) for x in item])
return embeddings
return [float(x) for x in value]
class OpenAICreateSpeechRequest(BaseModel):
input: str
model: str | None = None
@@ -46,7 +83,7 @@ class OpenAICreateSpeechRequest(BaseModel):
default=None,
description="Language code (e.g., 'Chinese', 'English', 'Auto')",
)
ref_audio: str | None = Field(
ref_audio: str | list[str] | None = Field(
default=None,
description="Reference audio for voice cloning (Base task). URL, base64, or file URI.",
)
@@ -73,7 +110,7 @@ class OpenAICreateSpeechRequest(BaseModel):
default=None,
description="Use speaker embedding only without in-context learning (Base task)",
)
speaker_embedding: list[float] | None = Field(
speaker_embedding: list[float] | list[list[float]] | None = Field(
default=None,
max_length=_MAX_EMBEDDING_DIM,
description="Pre-computed speaker embedding vector (1024-dim for 0.6B, "
@@ -108,10 +145,29 @@ class OpenAICreateSpeechRequest(BaseModel):
raise ValueError("'sse' is not a supported stream_format yet. Please use 'audio'.")
return v
@field_validator("ref_audio", mode="before")
@classmethod
def normalize_ref_audio(cls, v):
return _normalize_ref_audio_value(v)
@field_validator("speaker_embedding")
@classmethod
def validate_speaker_embedding(cls, v: list[float] | None) -> list[float] | None:
if v is not None and not all(math.isfinite(x) for x in v):
def validate_speaker_embedding(
cls, v: list[float] | list[list[float]] | None
) -> list[float] | list[list[float]] | None:
v = _normalize_speaker_embedding_value(v)
if v is None:
return None
if not v:
return []
if isinstance(v[0], list):
for item in v:
if not item:
raise ValueError("'speaker_embedding' nested vectors must be non-empty")
if not all(math.isfinite(x) for x in item):
raise ValueError("'speaker_embedding' values must be finite (no NaN or Inf)")
return v
if not all(math.isfinite(x) for x in v):
raise ValueError("'speaker_embedding' values must be finite (no NaN or Inf)")
return v
@@ -210,7 +266,7 @@ class OpenAICreateSpeechRequest(BaseModel):
@model_validator(mode="after")
def validate_embedding_constraints(self) -> "OpenAICreateSpeechRequest":
if self.speaker_embedding is not None:
if self.ref_audio is not None:
if self.ref_audio is not None and not isinstance(self.ref_audio, list):
raise ValueError("'speaker_embedding' and 'ref_audio' are mutually exclusive")
return self
+320 -22
View File
@@ -66,6 +66,7 @@ from vllm_omni.utils.speaker_cache import (
logger = init_logger(__name__)
# TTS Configuration
_MING_TTS_MODEL_ARCHS = {"MingTTSForConditionalGeneration"}
_VOXTRAL_TTS_MODEL_STAGES = {"audio_generation"}
_QWEN3_TTS_MODEL_STAGES = {"qwen3_tts"}
_FISH_TTS_MODEL_STAGES = {"fish_speech_slow_ar"}
@@ -126,6 +127,7 @@ _QWEN3_TTS_REF_AUDIO_CACHE_KEY = "_qwen3_tts_ref_audio_cache_key"
_TTS_MAX_INSTRUCTIONS_LENGTH = 500
_TTS_MAX_NEW_TOKENS_MIN = 1
_TTS_MAX_NEW_TOKENS_MAX = 4096
_MING_DEFAULT_PROMPT = MING_DEFAULT_PROMPT
def _create_wav_header(sample_rate: int, num_channels: int = 1, bits_per_sample: int = 16) -> bytes:
@@ -528,6 +530,32 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
def _load_codec_frame_rate(self) -> float | None:
"""Load codec frame rate from speech tokenizer config for prompt length estimation."""
if self._tts_model_type == "ming_tts":
try:
from vllm_omni.model_executor.models.ming_tts.config_ming_tts import MingTTSConfig
hf_config = self.engine_client.model_config.hf_config
ming_cfg = MingTTSConfig.from_hf_config(hf_config)
patch_size = int(ming_cfg.patch_size)
audio_frame_hop = int(ming_cfg.audio_frame_hop)
sample_rate = int(ming_cfg.sample_rate)
if patch_size <= 0 or audio_frame_hop <= 0 or sample_rate <= 0:
raise ValueError(
"Ming config has invalid tokenizer timing values: "
f"patch_size={patch_size}, audio_frame_hop={audio_frame_hop}, sample_rate={sample_rate}"
)
rate = float(sample_rate) / float(audio_frame_hop * patch_size)
logger.info(
"Derived Ming codec frame rate: %.1f Hz (sample_rate=%s, audio_frame_hop=%s, patch_size=%s)",
rate,
sample_rate,
audio_frame_hop,
patch_size,
)
return rate
except Exception as e:
logger.warning(f"Failed to derive Ming codec frame rate from hf_config: {e}")
try:
model_path = self.engine_client.model_config.model
st_config_path = os.path.join(model_path, "speech_tokenizer", "config.json")
@@ -572,7 +600,15 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
def _find_tts_stage(self):
"""Find and return the TTS stage config, or None if not found."""
for stage in self.engine_client.stage_configs:
if stage.engine_args.model_stage in _TTS_MODEL_STAGES:
engine_args = stage.engine_args
model_stage = engine_args.model_stage
model_arch = getattr(engine_args, "model_arch", None)
worker_type = getattr(engine_args, "worker_type", None)
if model_stage in _TTS_MODEL_STAGES:
return stage
# Ming dense identifies its AR entry stage by architecture because
# it does not use a dedicated TTS model_stage value.
if model_arch in _MING_TTS_MODEL_ARCHS and worker_type == "ar":
return stage
return None
@@ -602,6 +638,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
return "voxcpm2"
if model_stage in _MING_TTS_MODEL_STAGES:
return "ming_flash_omni_tts"
if model_arch in _MING_TTS_MODEL_ARCHS:
return "ming_tts"
if model_stage in _MOSS_TTS_MODEL_STAGES:
return "moss_tts_nano"
if model_stage in _MOSS_TTS_FULL_MODEL_STAGES:
@@ -682,6 +720,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
try:
if self._tts_model_type == "glm_tts":
return set()
if self._tts_model_type == "ming_tts":
return set()
if self._tts_model_type == "voxcpm2":
return {"default"}
if self._tts_model_type == "voxtral_tts":
@@ -1214,9 +1254,13 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
raise ValueError("'speaker_embedding' values must be finite (no NaN or Inf)")
emb_dim = len(embedding)
dim_err = self._validate_qwen_tts_speaker_embedding_dim(emb_dim)
if dim_err is not None:
raise ValueError(dim_err)
if self._tts_model_type == "ming_tts":
if emb_dim != 192:
logger.warning("speaker_embedding has %d dimensions; Ming dense expects 192", emb_dim)
else:
dim_err = self._validate_qwen_tts_speaker_embedding_dim(emb_dim)
if dim_err is not None:
raise ValueError(dim_err)
async with self._upload_lock:
voice_name_lower = name.lower()
@@ -1303,7 +1347,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
def _is_tts_model(self) -> bool:
"""Check if the current model is a supported TTS model."""
return any(stage.engine_args.model_stage in _TTS_MODEL_STAGES for stage in self.engine_client.stage_configs)
return self._find_tts_stage() is not None
def _validate_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None:
"""Validate TTS request parameters. Returns error message or None."""
@@ -1316,6 +1360,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
if self._tts_model_type == "voxcpm2":
return self._validate_voxcpm2_request(request)
if self._tts_model_type == "ming_flash_omni_tts":
return self._validate_ming_flash_omni_tts_request(request)
if self._tts_model_type == "ming_tts":
return self._validate_ming_tts_request(request)
if self._tts_model_type in ("moss_tts_nano", "moss_tts"):
return self._validate_moss_tts_request(request)
@@ -1362,7 +1408,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
ids = self._voxcpm2_tokenizer.encode(text, add_special_tokens=True)
return split_multichar_chinese(ids, self._voxcpm2_split_map)
def _validate_ming_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None:
def _validate_ming_flash_omni_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None:
"""Validate Ming-flash-omni standalone-talker request parameters."""
if not request.input or not request.input.strip():
return "Input text cannot be empty"
@@ -1399,6 +1445,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
def _validate_ref_audio_format(self, ref_audio: str) -> str | None:
"""Validate ref_audio is a supported URI format. Returns error or None."""
if not isinstance(ref_audio, str):
return "ref_audio must be a URL (http/https), base64 data URL (data:...), or file URI (file://...)"
if not (
ref_audio.startswith(("http://", "https://"))
or ref_audio.startswith("data:")
@@ -2066,6 +2114,99 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
return None
def _validate_ming_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None:
"""Validate Ming TTS request parameters. Returns error message or None."""
if not request.input or not request.input.strip():
return "Input text cannot be empty"
if isinstance(request.ref_audio, list):
return self._validate_ming_tts_podcast_request(request)
return self._validate_ming_tts_single_speaker_request(request)
def _validate_ming_tts_single_speaker_request(self, request: OpenAICreateSpeechRequest) -> str | None:
if request.ref_audio is not None:
fmt_err = self._validate_ref_audio_format(request.ref_audio)
if fmt_err:
return fmt_err
if request.speaker_embedding is not None:
if not request.speaker_embedding:
return "'speaker_embedding' must be a non-empty list of floats"
emb_len = len(request.speaker_embedding)
if emb_len != 192:
logger.warning(
"speaker_embedding has %d dimensions; Ming dense expects 192. "
"Wrong dimensions will likely fail or degrade output.",
emb_len,
)
voice_lower = request.voice.lower() if isinstance(request.voice, str) else None
uploaded_voice = bool(voice_lower and voice_lower in self.uploaded_speakers)
clone_source_present = request.ref_audio is not None or request.speaker_embedding is not None or uploaded_voice
if request.task_type == "Base" and not clone_source_present:
return "Base task requires 'ref_audio', 'speaker_embedding', or an uploaded voice sample"
if request.ref_audio is not None and request.ref_text is not None and not request.ref_text.strip():
return "'ref_text' must be non-empty when provided with 'ref_audio'"
# Ming offline ref-audio cases use prompt_waveform without prompt_text;
# keep the transcript requirement for other TTS models.
if request.ref_audio is not None and request.speaker_embedding is None and not self._is_ming_tts_model():
uploaded_ref_text = self.uploaded_speakers[voice_lower].get("ref_text") if uploaded_voice else None
if not (request.ref_text and request.ref_text.strip()) and not uploaded_ref_text:
return "Reference-audio cloning requires non-empty 'ref_text'"
if request.ref_text is not None and request.ref_audio is None and not uploaded_voice:
return "'ref_text' requires 'ref_audio' or an uploaded voice sample"
if request.instructions and len(request.instructions) > self._max_instructions_length:
return f"Instructions too long (max {self._max_instructions_length} characters)"
if request.max_new_tokens is not None:
if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN:
return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}"
if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX:
return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}"
return None
def _validate_ming_tts_podcast_request(self, request: OpenAICreateSpeechRequest) -> str | None:
if len(request.ref_audio) < 2:
return "Podcast-style Ming requests require at least two 'ref_audio' clips"
for ref_audio in request.ref_audio:
fmt_err = self._validate_ref_audio_format(ref_audio)
if fmt_err:
return fmt_err
if not request.ref_text or not request.ref_text.strip():
return "Podcast-style Ming requests require non-empty 'ref_text'"
if request.speaker_embedding is not None:
embeddings = request.speaker_embedding
embedding_count = len(embeddings) if embeddings and isinstance(embeddings[0], list) else 1
if embedding_count != len(request.ref_audio):
return (
"Podcast-style Ming requests require one speaker embedding per ref_audio clip; "
f"got {embedding_count} embeddings for {len(request.ref_audio)} clips"
)
if embeddings and isinstance(embeddings[0], list):
for item in embeddings:
if len(item) != 192:
return "Podcast-style Ming speaker embeddings must each have 192 dimensions"
if request.instructions and len(request.instructions) > self._max_instructions_length:
return f"Instructions too long (max {self._max_instructions_length} characters)"
if request.max_new_tokens is not None:
if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN:
return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}"
if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX:
return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}"
return None
async def _resolve_ref_audio(self, ref_audio_str: str) -> tuple[list[float], int]:
"""Resolve ref_audio to (wav_samples, sample_rate).
@@ -2192,6 +2333,142 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
def _discard_ref_audio_artifact_warmup(self, request_id: str) -> None:
self._request_ref_audio_artifact_keys.pop(request_id, None)
async def _resolve_ref_audio_many(self, ref_audio_list: list[str]) -> list[tuple[list[float], int]]:
resolved = []
for ref_audio in ref_audio_list:
resolved.append(await self._resolve_ref_audio(ref_audio))
return resolved
# ---- Ming TTS helpers ----
def _is_ming_tts_model(self) -> bool:
return self._tts_model_type == "ming_tts"
def _coerce_ming_prompt_waveform(self, wav_samples, sample_rate):
from torchaudio.functional import resample as resample_audio
from vllm_omni.model_executor.models.ming_tts.config_ming_tts import SAMPLE_RATE
waveform = torch.as_tensor(wav_samples, dtype=torch.float32).reshape(1, -1)
if int(sample_rate) != SAMPLE_RATE:
waveform = resample_audio(waveform, int(sample_rate), SAMPLE_RATE)
return waveform
def _build_ming_prompt_waveform(
self,
ref_audio_data: tuple[list[float], int] | list[tuple[list[float], int]] | None,
):
if isinstance(ref_audio_data, list):
return torch.cat(
[self._coerce_ming_prompt_waveform(item[0], item[1]) for item in ref_audio_data],
dim=-1,
)
if ref_audio_data is not None:
return self._coerce_ming_prompt_waveform(ref_audio_data[0], ref_audio_data[1])
return None
def _extract_ming_speaker_embeddings_from_ref_audio(
self,
ref_audio_data_list: list[tuple[list[float], int]],
) -> list[list[float]]:
from vllm_omni.model_executor.models.ming_tts.speaker_extractor import MingSpeakerEmbeddingExtractor
extractor = MingSpeakerEmbeddingExtractor(self.engine_client.model_config.model, target_sr=16000)
embeddings = []
for wav_samples, sr in ref_audio_data_list:
waveform = torch.as_tensor(wav_samples, dtype=torch.float32).reshape(1, -1)
embedding = extractor.extract_from_waveform(waveform, int(sr))
flat = embedding.detach().reshape(-1).to(torch.float32).cpu()
if int(flat.numel()) != 192:
raise ValueError(f"Ming speaker extractor returned {int(flat.numel())} dims; expected 192")
embeddings.append(flat.tolist())
return embeddings
def _parse_ming_instruction_fields(
self,
request,
*,
include_language=False,
include_voice=False,
plain_text_passthrough=False,
):
instruction_text = request.instructions.strip() if isinstance(request.instructions, str) else None
instruction_dict: dict[str, Any] = {}
if include_language and request.language not in (None, "", "Auto"):
instruction_dict["方言"] = request.language
voice_lower = request.voice.lower() if isinstance(request.voice, str) else None
if include_voice and request.voice and not (voice_lower and voice_lower in self.uploaded_speakers):
instruction_dict["IP"] = request.voice
if instruction_text:
try:
parsed = json.loads(instruction_text)
except json.JSONDecodeError:
parsed = None
if isinstance(parsed, dict):
instruction_dict.update(parsed)
elif instruction_dict or not plain_text_passthrough:
instruction_dict["风格"] = instruction_text
else:
return instruction_text
return instruction_dict or None
def _parse_ming_instruction(self, request: OpenAICreateSpeechRequest) -> Any:
"""Build a Ming instruction payload from OpenAI speech fields."""
return self._parse_ming_instruction_fields(
request,
include_language=True,
include_voice=True,
plain_text_passthrough=True,
)
def _build_ming_dense_prompt(
self,
request: OpenAICreateSpeechRequest,
*,
ref_audio_data: tuple[list[float], int] | list[tuple[list[float], int]] | None = None,
) -> dict[str, Any]:
"""Build a Ming dense prompt directly from the OpenAI speech request."""
from transformers import AutoTokenizer
from vllm_omni.model_executor.models.ming_tts.config_ming_tts import KEY_MAX_DECODE_STEPS
from vllm_omni.model_executor.models.ming_tts.prompt_assembly import build_ming_dense_prompt
if self._tts_tokenizer is None:
model_name = self.engine_client.model_config.model
self._tts_tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=False)
ref_text = request.ref_text
prompt_waveform = self._build_ming_prompt_waveform(ref_audio_data) if ref_text is not None else None
speaker_embedding = request.speaker_embedding
use_zero_spk_emb = prompt_waveform is None and speaker_embedding is None
runtime_controls = {}
if request.max_new_tokens is not None:
runtime_controls[KEY_MAX_DECODE_STEPS] = request.max_new_tokens
prompt_dict = build_ming_dense_prompt(
self._tts_tokenizer,
# bgm / music-prompt mode not supported online;
# requires prompt_mode API extension (deferred).
prompt=_MING_DEFAULT_PROMPT,
text=request.input,
runtime_controls=runtime_controls or None,
instruction=self._parse_ming_instruction(request),
prompt_text=ref_text,
prompt_waveform=prompt_waveform,
speaker_embedding=speaker_embedding,
use_zero_spk_emb=use_zero_spk_emb,
)
prompt = tokens_input(prompt_token_ids=prompt_dict["prompt_token_ids"])
prompt["prompt"] = prompt_dict["prompt"]
prompt["text"] = prompt_dict["text"]
prompt["additional_information"] = prompt_dict["additional_information"]
return prompt
async def _generate_audio_chunks(
self,
generator,
@@ -2796,25 +3073,12 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
# ---- Ming-flash-omni standalone-talker (TTS) helpers ----
def _build_ming_prompt(self, request: OpenAICreateSpeechRequest) -> dict[str, Any]:
def _build_ming_flash_omni_prompt(self, request: OpenAICreateSpeechRequest) -> dict[str, Any]:
# request.instructions accepts two forms:
# 1. Plain text: mapped to the caption's 风格 (style) field
# 2. JSON object: parsed and splatted into the caption. Unlocks
# Unknown keys are dropped by `ming_create_instruction`.
caption_fields: dict[str, Any] = {}
if request.instructions:
stripped = request.instructions.strip()
if stripped.startswith("{"):
try:
parsed = json.loads(stripped)
except json.JSONDecodeError:
parsed = None
if isinstance(parsed, dict):
caption_fields.update(parsed)
else:
caption_fields["风格"] = request.instructions
else:
caption_fields["风格"] = request.instructions
caption_fields = self._parse_ming_instruction_fields(request) or {}
has_spk_emb = request.speaker_embedding is not None
@@ -2982,8 +3246,29 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
prompt = await self._build_glm_tts_prompt(request, has_inline_ref_audio=has_inline_ref_audio)
tts_params = {}
elif self._tts_model_type == "ming_flash_omni_tts":
prompt = self._build_ming_prompt(request)
prompt = self._build_ming_flash_omni_prompt(request)
tts_params = {}
elif self._tts_model_type == "ming_tts":
ref_audio_source = request.ref_audio
voice_lower = request.voice.lower() if isinstance(request.voice, str) else None
if ref_audio_source is None and voice_lower in self.uploaded_speakers:
ref_audio_source = self._get_uploaded_audio_data(request.voice)
if request.ref_text is None:
request.ref_text = self.uploaded_speakers[voice_lower].get("ref_text")
ref_audio_data = None
if isinstance(ref_audio_source, list):
ref_audio_data = await self._resolve_ref_audio_many(ref_audio_source)
if request.speaker_embedding is None:
request.speaker_embedding = self._extract_ming_speaker_embeddings_from_ref_audio(ref_audio_data)
elif ref_audio_source is not None and isinstance(ref_audio_source, str):
wav_list, sr = await self._resolve_ref_audio(ref_audio_source)
ref_audio_data = (wav_list, sr)
if request.speaker_embedding is None:
request.speaker_embedding = self._extract_ming_speaker_embeddings_from_ref_audio(
[ref_audio_data]
)[0]
prompt = self._build_ming_dense_prompt(request, ref_audio_data=ref_audio_data)
tts_params = prompt.get("additional_information", {})
elif self._tts_model_type in ("moss_tts_nano", "moss_tts"):
tts_params = await self._build_moss_tts_params(request)
if request.voice:
@@ -3069,6 +3354,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
model_type = "voxcpm2"
elif self._tts_model_type == "ming_flash_omni_tts":
model_type = "ming_flash_omni_tts"
elif self._tts_model_type == "ming_tts":
model_type = "ming_tts"
elif self._tts_model_type == "moss_tts_nano":
model_type = "moss_tts_nano"
elif self._tts_model_type == "moss_tts":
@@ -3177,6 +3464,17 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
getattr(sampling_params_list[0], "min_tokens", 0),
request.max_new_tokens,
)
elif self._tts_model_type == "ming_tts" and sampling_params_list:
import copy
from vllm_omni.model_executor.models.ming_tts.config_ming_tts import TEXT_EOS_TOKEN_ID
sampling_params_list = copy.deepcopy(sampling_params_list)
sampling_params_list[0].stop_token_ids = [int(TEXT_EOS_TOKEN_ID)]
if request.max_new_tokens is not None:
# Ming emits TEXT_EOS after the latent decode budget is exhausted, so
# Stage-0 needs one extra token beyond ming_max_decode_steps.
sampling_params_list[0].max_tokens = int(request.max_new_tokens) + 1
if request.seed is not None and sampling_params_list:
if sampling_params_list is self.engine_client.default_sampling_params_list: