import base64 import io import os import random import re from collections.abc import Callable import numpy as np import torch import torchaudio from process_speechdata import InputSegment, StreamingInputSegment from torchaudio.transforms import MelSpectrogram from vllm.multimodal.media.audio import load_audio speech_zeroemb_idx = 151667 empty_token = "<|empty|>" mimo_audio_tokenizer = None device = "cpu" asr_zh_templates = [ "请将这段语音转换为文字", "帮我识别这个音频文件中的内容", "把这段录音转成文本", "请转录这段语音", "将音频内容转换成文字格式", "识别并转写这段语音", "把语音内容写成文字", "转录这个音频片段", "将这段对话转换为文本", "麻烦帮我把这段录音整理成详细的文字记录", ] asr_en_templates = [ "Please transcribe this audio file", "Convert this speech recording to text", "Transcribe the following voice message", "Turn this audio into readable text", "Please convert the recording to written format", "Transcribe what you hear in this audio", "Convert this spoken content to text", "Please write down what is said in this recording", "Transcribe this voice recording", "Could you please help me transcribe this important recording?", "Would you mind converting this voice message into a readable text format?", "I'd really appreciate it if you could turn this audio file into a written document", ] tts_zh_templates = [ "请将这段文字转换为语音", "帮我把这个文本读出来", "将这些文字生成音频", "请朗读这段内容", "把这段话转换成语音文件", "生成这段文字的语音版本", "请用语音播报这些内容", "将文本转换为可听的音频", "帮我朗读这段文字", "把这些内容念出来", ] tts_en_templates = [ "Please convert this text to speech", "Turn this writing into audio", "Generate speech from this text", "Read this content out loud", "Convert these words to voice", "Create an audio version of this text", "Please vocalize this content", "Turn this text into audible format", "Help me convert this writing to speech", "Make this text into spoken audio", ] def detect_language(text): if re.search(r"[\u4e00-\u9fff]", text): return "zh" else: return "en" # ============================================ # Common Helper Functions - InputSegment Creation # ============================================ def create_segment(text: str = "", audio=None) -> InputSegment: """Create a standard InputSegment with default zeroemb parameters""" return InputSegment( text=text, audio=audio, speech_zeroemb_idx=speech_zeroemb_idx, text_zeroemb_idx=empty_token, ) def create_streaming_segment( text: str, audio, tokenizer, group_size: int, audio_channels: int ) -> StreamingInputSegment: """Create a StreamingInputSegment""" return StreamingInputSegment( text=text, audio=audio, tokenizer=tokenizer, group_size=group_size, audio_channels=audio_channels, speech_zeroemb_idx=speech_zeroemb_idx, text_zeroemb_idx=empty_token, ) # ============================================ # Common Helper Functions - Common Token Segment Creation # ============================================ def create_user_start() -> InputSegment: """Create user role start token""" return create_segment(text="<|im_start|>user\n") def create_user_end() -> InputSegment: """Create role end token""" return create_segment(text="<|im_end|>\n") def create_assistant_start() -> InputSegment: """Create assistant role start token""" return create_segment(text="<|im_start|>assistant\n") def create_system_start() -> InputSegment: """Create system role start token""" return create_segment(text="<|im_start|>system\n") def create_thinking_segment(thinking: bool = False) -> InputSegment: """Create thinking token, closed or open based on thinking parameter""" if thinking: return create_segment(text="\n") else: return create_segment(text="\n\n\n") def create_sostm_segment() -> InputSegment: """Create streaming output start token""" return create_segment(text="<|sostm|>") def create_assistant_start_with_sostm() -> InputSegment: """Create assistant start token with sostm""" return create_segment(text="<|im_start|>assistant\n<|sostm|>") def create_assistant_start_with_think() -> InputSegment: """Create assistant start token with think opening""" return create_segment(text="<|im_start|>assistant\n\n") # ============================================ # Common Helper Functions - Composite Segment Creation # ============================================ def create_user_turn_with_audio(audio_tokenized, extra_text: str = None) -> list[InputSegment]: """Create a user turn containing audio""" segments = [ create_user_start(), create_segment(audio=audio_tokenized), ] if extra_text: segments.append(create_segment(text=extra_text)) segments.append(create_user_end()) return segments def create_user_turn_with_text(text: str) -> list[InputSegment]: """Create a text-only user turn""" return [ create_user_start(), create_segment(text=text), create_user_end(), ] def create_system_turn_with_voice_prompt(prompt_text: str, audio_token) -> list[InputSegment]: """Create a system turn with voice prompt""" return [ create_system_start(), create_segment(text=prompt_text), create_segment(text="", audio=audio_token), create_user_end(), ] def create_system_turn_text_only(system_text: str) -> list[InputSegment]: """Create a text-only system turn""" return [ create_system_start(), create_segment(text=system_text), create_user_end(), ] # ============================================ # Common Helper Functions - Multi-turn Dialogue Processing # ============================================ def process_multiturn_messages( message_list: list[dict], user_processor: Callable[[dict], list[InputSegment]], assistant_processor: Callable[[dict], list[InputSegment]], ) -> list[InputSegment]: """ Generic multi-turn dialogue message processing function Args: message_list: List of messages, each containing 'role' and 'content' user_processor: Function to process user messages assistant_processor: Function to process assistant messages Returns: Processed list of InputSegments """ lm_prompt = [] for message in message_list: role = message["role"] if role == "user": lm_prompt.extend(user_processor(message)) elif role == "assistant": lm_prompt.extend(assistant_processor(message)) else: raise ValueError(f"Invalid role: {role}") return lm_prompt def create_text_user_message(message: dict) -> list[InputSegment]: """Process a text-only user message""" return [ create_user_start(), create_segment(text=message["content"]), create_user_end(), ] def create_text_assistant_message(message: dict) -> list[InputSegment]: """Process a text-only assistant message""" return [ create_assistant_start(), create_segment(text=message["content"]), create_user_end(), ] def create_audio_user_message(message: dict) -> list[InputSegment]: """Process an audio user message""" return [ create_user_start(), create_segment(audio=preprocess_input(message["content"])), create_user_end(), ] def append_assistant_ending( lm_prompt: list[InputSegment], thinking: bool = False, use_sostm: bool = False ) -> list[InputSegment]: """ Append assistant ending to the prompt Args: lm_prompt: Existing prompt list thinking: Whether to use open thinking token use_sostm: Whether to use sostm token (for speech output) """ if use_sostm: lm_prompt.append(create_assistant_start_with_sostm()) else: lm_prompt.append(create_assistant_start()) lm_prompt.append(create_thinking_segment(thinking)) return lm_prompt def get_asr_sft_prompt( input: None | str = None, ): """Build prompt for ASR (Automatic Speech Recognition) task""" audio_tokenized = preprocess_input(input) template = random.choice(asr_zh_templates + asr_en_templates) lm_prompt = create_user_turn_with_audio(audio_tokenized, extra_text=template) lm_prompt = append_assistant_ending(lm_prompt, thinking=False) return lm_prompt def resample_audio_if_needed(wav_tensor: torch.Tensor, original_sr: int): target_sr = 24000 if original_sr != target_sr: wav_tensor = torchaudio.functional.resample(wav_tensor, original_sr, target_sr) return wav_tensor def wav2mel(wav, device="cpu"): mel_transform = MelSpectrogram( sample_rate=mimo_audio_tokenizer.config.sampling_rate, n_fft=mimo_audio_tokenizer.config.nfft, hop_length=mimo_audio_tokenizer.config.hop_length, win_length=mimo_audio_tokenizer.config.window_size, f_min=mimo_audio_tokenizer.config.fmin, f_max=mimo_audio_tokenizer.config.fmax, n_mels=mimo_audio_tokenizer.config.n_mels, power=1.0, center=True, ).to(device) spec = mel_transform(wav[None, :]) return torch.log(torch.clip(spec, min=1e-7)).squeeze() def group_by_length(features: torch.Tensor, lengths: torch.Tensor, max_length: int): if features.size(0) != lengths.sum().item(): raise ValueError(f"Feature size mismatch: {features.size(0)} vs {lengths.sum().item()}") split_points = [] current_sum = 0 for i, seq_len in enumerate(lengths): if current_sum + seq_len > max_length and current_sum > 0: split_points.append(i) current_sum = seq_len.item() else: current_sum += seq_len.item() # Convert split points to group sizes group_sizes = [] prev = 0 for point in split_points: group_sizes.append(point - prev) prev = point if prev < len(lengths): group_sizes.append(len(lengths) - prev) len_groups = torch.split(lengths, group_sizes) feature_sizes = [group.sum().item() for group in len_groups] feature_groups = torch.split(features, feature_sizes) return feature_groups, len_groups def encode_batch(input_features: torch.Tensor, input_lens: torch.Tensor, max_length: int = 256000): feature_groups, len_groups = group_by_length(input_features, input_lens, max_length) encoded_parts = [] for features, lengths in zip(feature_groups, len_groups): with torch.no_grad(): codes, _ = mimo_audio_tokenizer.encoder.encode( input_features=features.to(device), input_lens=lengths.to(device), return_codes_only=True ) encoded_parts.append(codes) return torch.cat(encoded_parts, dim=-1) def preprocess_input(input: None | str | torch.Tensor = None, device="cpu", audio_channels=4, group_size=8): if isinstance(input, torch.Tensor) or (isinstance(input, str) and os.path.isfile(input)): return "<|sosp|><|empty|><|eosp|>" else: text = input if ( text.isupper() or text.islower() ): # If the text only contains upper-case or lower-case letters, capitalize it. text = text.capitalize() return text def _build_tts_system_prompt(has_voice_prompt: bool, voice_audio_token=None) -> list[InputSegment]: """Build system prompt for TTS task""" if has_voice_prompt and voice_audio_token is not None: return [ create_system_start(), create_segment( # text="You need to generate a speech with the same timbre as the speech prompt, based on the specified style instructions and text content. Your timbre should be: " text="你需要根据指定的风格指令和文本内容来生成和语音prompt具有相同音色的语音。你的音色应该是:" ), create_segment(text="", audio=voice_audio_token), create_user_end(), ] else: return create_system_turn_text_only("你需要根据指定的风格指令和文本内容来生成语音。") # return create_system_turn_text_only( # "You need to generate speech based on the specified style instructions and text content." # ) def _build_tts_system_prompt_no_instruct(has_voice_prompt: bool, voice_audio_token=None) -> list[InputSegment]: """Build system prompt for TTS task""" if has_voice_prompt and voice_audio_token is not None: return [ create_system_start(), create_segment( # text="You need to generate a speech with the same timbre as the speech prompt, based on the specified style instructions and text content. Your timbre should be:" text="你需要根据指定的风格指令和文本内容来生成和语音prompt具有相同音色的语音。你的音色应该是:" ), create_segment(text="", audio=voice_audio_token), create_user_end(), ] else: return [] def get_tts_sft_prompt( input: None | str = None, instruct=None, read_text_only=True, prompt_speech=None, ): """ Build prompt for TTS (Text-to-Speech) task Args: input: Input text instruct: Style instruction (e.g., "speak happily in a child's voice") read_text_only: Whether to read only plain text (False means text contains template) prompt_speech: Reference audio (for voice cloning) """ assistant_prompt_audio_token = preprocess_input(prompt_speech) if prompt_speech is not None else None if not read_text_only: # Not just reading text, text contains template (template:text format) text = preprocess_input(input) lm_prompt = _build_tts_system_prompt( has_voice_prompt=assistant_prompt_audio_token is not None, voice_audio_token=assistant_prompt_audio_token, ) lm_prompt.append(create_segment(text=f"<|im_start|>user\n{text}<|im_end|>\n")) lm_prompt.append(create_assistant_start_with_think()) else: # Plain text (no instruction inside) language = detect_language(input) template = random.choice(tts_zh_templates if language == "zh" else tts_en_templates) text = preprocess_input(input) if instruct is None: # No instruct instruction lm_prompt = _build_tts_system_prompt_no_instruct( has_voice_prompt=assistant_prompt_audio_token is not None, voice_audio_token=assistant_prompt_audio_token, ) lm_prompt.extend( [ create_segment(text=f"<|im_start|>user\n{template}: {text}<|im_end|>\n"), create_assistant_start_with_sostm(), ] ) else: # Has instruct instruction lm_prompt = _build_tts_system_prompt( has_voice_prompt=assistant_prompt_audio_token is not None, voice_audio_token=assistant_prompt_audio_token, ) lm_prompt.append(create_segment(text=f"<|im_start|>user\n{template}: {text}({instruct})<|im_end|>\n")) lm_prompt.append(create_assistant_start_with_think()) return lm_prompt def get_audio_understanding_sft_prompt( input_speech, input_text, thinking=False, use_sostm=False, ): """Build prompt for audio understanding task""" audio_tokenized = preprocess_input(input_speech) lm_prompt = create_user_turn_with_audio(audio_tokenized, extra_text=input_text) lm_prompt = append_assistant_ending(lm_prompt, thinking=thinking, use_sostm=use_sostm) return lm_prompt def _build_voice_prompt_system(prompt_speech) -> list[InputSegment]: """Build system prompt with voice prompt""" return create_system_turn_with_voice_prompt( prompt_text="Your voice should be:", audio_token=preprocess_input(prompt_speech) ) def get_spoken_dialogue_sft_prompt( input_speech, system_prompt=None, prompt_speech=None, add_history=False, ): """ Build prompt for spoken dialogue task Args: input_speech: Input speech system_prompt: System prompt text prompt_speech: Reference audio (for voice cloning) add_history: Whether to add history (Note: history variable is undefined in original code) """ audio_tokenized = preprocess_input(input_speech) lm_prompt = [] # Note: history variable is undefined in original code, this branch may never execute # To use history feature, history should be passed as a parameter if add_history: # Simplified form of adding history lm_prompt = create_user_turn_with_audio(audio_tokenized) lm_prompt.append(create_assistant_start_with_sostm()) else: # Add voice prompt (if available) if prompt_speech: lm_prompt.extend(_build_voice_prompt_system(prompt_speech)) # Add user turn lm_prompt.append(create_user_start()) if system_prompt: lm_prompt.append(create_segment(text=system_prompt)) lm_prompt.append(create_segment(audio=audio_tokenized)) lm_prompt.append(create_user_end()) lm_prompt.append(create_assistant_start_with_sostm()) return lm_prompt def get_spoken_dialogue_sft_multiturn_prompt( message_list, system_prompt=None, prompt_speech=None, tokenizer=None, group_size=8, audio_channels=4, ): """ Build prompt for multi-turn spoken dialogue task Args: message_list: List of messages containing role and content system_prompt: System prompt text prompt_speech: Reference audio (for voice cloning) tokenizer: Tokenizer group_size: Group size audio_channels: Number of audio channels """ lm_prompt = [] # Add voice prompt (if available) if prompt_speech: lm_prompt.extend( create_system_turn_with_voice_prompt( prompt_text="Your voice should be:", audio_token=preprocess_input(prompt_speech) ) ) # Define message processors def user_processor(msg): segments = [create_user_start()] if system_prompt: segments.append(create_segment(text=system_prompt)) segments.append(create_segment(audio=preprocess_input(msg["content"]))) segments.append(create_user_end()) return segments def assistant_processor(msg): return [ create_assistant_start(), create_streaming_segment( text=msg["content"]["text"], audio=preprocess_input(msg["content"]["audio"]), tokenizer=tokenizer, group_size=group_size, audio_channels=audio_channels, ), create_user_end(), ] # Process message list lm_prompt.extend(process_multiturn_messages(message_list, user_processor, assistant_processor)) lm_prompt.append(create_assistant_start_with_sostm()) return lm_prompt def get_s2t_dialogue_sft_prompt( input_speech, thinking=False, ): """Build prompt for speech-to-text dialogue task""" audio_tokenized = preprocess_input(input_speech) lm_prompt = create_user_turn_with_audio(audio_tokenized) lm_prompt = append_assistant_ending(lm_prompt, thinking=thinking) return lm_prompt def get_s2t_dialogue_sft_multiturn_prompt(message_list, thinking=False): """Build prompt for multi-turn speech-to-text dialogue task""" lm_prompt = process_multiturn_messages( message_list, user_processor=create_audio_user_message, assistant_processor=create_text_assistant_message ) lm_prompt = append_assistant_ending(lm_prompt, thinking=thinking) return lm_prompt def get_text_dialogue_sft_prompt( input_text, thinking=False, ): """Build prompt for text-only dialogue task""" lm_prompt = create_user_turn_with_text(input_text) lm_prompt = append_assistant_ending(lm_prompt, thinking=thinking) return lm_prompt def get_text_dialogue_sft_multiturn_prompt( message_list, thinking=False, ): """Build prompt for multi-turn text-only dialogue task""" lm_prompt = process_multiturn_messages( message_list, user_processor=create_text_user_message, assistant_processor=create_text_assistant_message ) lm_prompt = append_assistant_ending(lm_prompt, thinking=thinking) return lm_prompt def get_in_context_learning_s2s_prompt( instruction, prompt_examples, audio, tokenizer=None, group_size=8, audio_channels=4, ): """ Build prompt for In-Context Learning speech-to-speech task Args: instruction: Instruction text prompt_examples: List of examples, each containing input_audio, output_transcription, output_audio audio: Input audio to be processed tokenizer: Tokenizer group_size: Group size audio_channels: Number of audio channels """ prompt = [create_segment(text=f"[Int]:{instruction}\n")] # Add examples for example in prompt_examples: prompt.extend( [ create_segment(audio=preprocess_input(example["input_audio"])), create_segment(text="\n"), create_streaming_segment( text=example["output_transcription"], audio=preprocess_input(example["output_audio"]), tokenizer=tokenizer, group_size=group_size, audio_channels=audio_channels, ), create_segment(text=" \n\n"), ] ) # Add input audio to be processed prompt.extend( [ create_segment(audio=preprocess_input(audio)), create_segment(text="\n"), create_sostm_segment(), ] ) return prompt def get_audio_data(audio_url): if audio_url.startswith("data:"): header, b64_data = audio_url.split(",", 1) audio_bytes = base64.b64decode(b64_data.strip()) audio_file = io.BytesIO(audio_bytes) else: # File path audio_file = audio_url audio_signal, sr = load_audio(audio_file, sr=24000) audio_data = (audio_signal.astype(np.float32), sr) return audio_data def to_prompt(input_segs): out_put = [] for input_seg in input_segs: if isinstance(input_seg, StreamingInputSegment) and input_seg.text: out_put.append("<|sostm|>") if input_seg.audio is not None and isinstance(input_seg.audio, str): out_put.append(input_seg.text) out_put.append("<|eot|>") out_put.append("<|empty|>") else: out_put.append(input_seg.text) out_put.append("<|eot|>") out_put.append("<|eostm|>") else: out_put.append(input_seg.text) if input_seg.audio is not None: out_put.append(input_seg.audio) prompt = "".join(out_put) return prompt