[Edge][WP3] Add CPU OmniPlatform (AR + generation stages on vLLM's CPU backend)
* vllm_omni/platforms/cpu/platform.py: CPUOmniPlatform(OmniPlatform, CpuPlatform) selected when VLLM_TARGET_DEVICE=cpu or a +cpu vLLM wheel is installed (builtin plugin "cpu"; explicit CPU target short-circuits accelerator probes like vLLM does). NUMA-based memory queries, one logical device, no graph wrapper (sentinel type for isinstance checks), inductor opt-in via VLLM_OMNI_CPU_INDUCTOR=1, diffusion stages unsupported. * vllm_omni/platforms/cpu/worker: CPUARWorker / CPUGenerationWorker compose vLLM's CPUWorker with the Omni mixin (XPU pattern); CPUARModelRunner / CPUGenerationModelRunner compose the Omni runners with CPUModelRunner (no graphs, no async omni output; sleep/wake return error ACKs). * patch.py skips the CuMem patch on CPU-only torch; Qwen3-TTS speaker encoder and prompt-embeds dtype follow the model dtype (float32/float16 on CPU). * vllm_omni/deploy/qwen3_tts.yaml: `platforms: cpu:` overlay (eager, no async scheduling, absolute kv_cache_memory_bytes, disjoint OMP thread ranges). * requirements/cpu.txt (+psutil, intel-openmp), docs/getting_started/installation/cpu.md. Measured (this x86 Xeon 8480C host, vllm 0.28.0+cpu, bf16, 12+4 threads): Qwen3-TTS 0.6B CustomVoice end-to-end on CPU, 12 prompts: TTFA p50 292 ms, RTF ~1.7 (25-frame chunk ~4.6 s), init 185 s. Result JSON + init timeline committed under benchmarks/tts/edge_results/. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
55192de984
commit
35e43b9983
18 files changed
+1740
-7
No files matched your search
@@ -363,7 +363,12 @@ class Qwen3TTSPromptEmbedsBuilder:
|
||||
self._tts_pad_embed_buffer = tts_pad_embed
|
||||
self._encode_ref_audio_batch_fn = encode_ref_audio_batch
|
||||
self._speaker_cache = speaker_cache
|
||||
self._embedding_dtype = torch.bfloat16
|
||||
# Follow the talker's pad-embedding dtype (model dtype) so CPU float32/float16 deployments work.
|
||||
self._embedding_dtype = (
|
||||
tts_pad_embed.dtype
|
||||
if isinstance(tts_pad_embed, torch.Tensor) and tts_pad_embed.is_floating_point()
|
||||
else torch.bfloat16
|
||||
)
|
||||
|
||||
self._text_tokenizer: Any | None = None
|
||||
|
||||
|
||||
@@ -352,6 +352,13 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
self.use_async_omni_output = True
|
||||
self.eager_omni_postprocess_before_async_output = True
|
||||
self.omni_pooler_payload_include_hidden = False
|
||||
# Terminal talker (no downstream stage, e.g. qwen3_tts_talker_only):
|
||||
# publish the per-step codec frames to the client so an on-device
|
||||
# decoder can consume them (codec-token streaming).
|
||||
if getattr(vllm_config.model_config, "custom_process_next_stage_input_func", None) is None and not getattr(
|
||||
vllm_config.model_config, "async_chunk_process_next_stage_input_func", None
|
||||
):
|
||||
self.omni_client_multimodal_output_keys = ("codes.audio",)
|
||||
self.model = Qwen3Model(vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model"))
|
||||
|
||||
if get_pp_group().is_last_rank:
|
||||
@@ -465,7 +472,9 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
torch.zeros(1, int(self.talker_config.hidden_size), dtype=model_dtype),
|
||||
persistent=False,
|
||||
)
|
||||
self._embedding_dtype = torch.bfloat16
|
||||
# Follows the model dtype so CPU deployments can run float32/float16;
|
||||
# on CUDA this is bfloat16 as before.
|
||||
self._embedding_dtype = model_dtype if isinstance(model_dtype, torch.dtype) else torch.bfloat16
|
||||
|
||||
tokenizer_config = Qwen3TTSTokenizerV2Config.from_pretrained(
|
||||
self.model_path,
|
||||
@@ -475,7 +484,7 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
tokenizer_config.encoder_config,
|
||||
)
|
||||
self.encoder.eval()
|
||||
self.encoder.to(dtype=torch.bfloat16)
|
||||
self.encoder.to(dtype=self._embedding_dtype)
|
||||
self._encoder_valid_num_quantizers = int(tokenizer_config.encoder_valid_num_quantizers)
|
||||
self._encoder_downsample_rate = int(tokenizer_config.encode_downsample_rate)
|
||||
|
||||
@@ -1048,7 +1057,7 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
sampling_rate=target_sr,
|
||||
return_tensors="pt",
|
||||
)
|
||||
inputs = inputs.to(device).to(torch.bfloat16)
|
||||
inputs = inputs.to(device).to(self._embedding_dtype)
|
||||
|
||||
input_values = inputs["input_values"].squeeze(1)
|
||||
padding_mask = inputs["padding_mask"].squeeze(1)
|
||||
@@ -1125,7 +1134,7 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
loaded.add(name)
|
||||
|
||||
device = self.vllm_config.device_config.device
|
||||
self.encoder.to(device=device, dtype=torch.bfloat16)
|
||||
self.encoder.to(device=device, dtype=self._embedding_dtype)
|
||||
|
||||
self._init_runtime_buffers()
|
||||
self._init_silence_mask()
|
||||
|
||||
Reference in new issue
Block a user