anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[Edge][WP3] Add CPU OmniPlatform (AR + generation stages on vLLM's CPU backend)

Browse Source 
* vllm_omni/platforms/cpu/platform.py: CPUOmniPlatform(OmniPlatform, CpuPlatform)
  selected when VLLM_TARGET_DEVICE=cpu or a +cpu vLLM wheel is installed
  (builtin plugin "cpu"; explicit CPU target short-circuits accelerator probes
  like vLLM does). NUMA-based memory queries, one logical device, no graph
  wrapper (sentinel type for isinstance checks), inductor opt-in via
  VLLM_OMNI_CPU_INDUCTOR=1, diffusion stages unsupported.
* vllm_omni/platforms/cpu/worker: CPUARWorker / CPUGenerationWorker compose
  vLLM's CPUWorker with the Omni mixin (XPU pattern); CPUARModelRunner /
  CPUGenerationModelRunner compose the Omni runners with CPUModelRunner
  (no graphs, no async omni output; sleep/wake return error ACKs).
* patch.py skips the CuMem patch on CPU-only torch; Qwen3-TTS speaker encoder
  and prompt-embeds dtype follow the model dtype (float32/float16 on CPU).
* vllm_omni/deploy/qwen3_tts.yaml: `platforms: cpu:` overlay (eager, no async
  scheduling, absolute kv_cache_memory_bytes, disjoint OMP thread ranges).
* requirements/cpu.txt (+psutil, intel-openmp), docs/getting_started/installation/cpu.md.

Measured (this x86 Xeon 8480C host, vllm 0.28.0+cpu, bf16, 12+4 threads):
Qwen3-TTS 0.6B CustomVoice end-to-end on CPU, 12 prompts: TTFA p50 292 ms,
RTF ~1.7 (25-frame chunk ~4.6 s), init 185 s. Result JSON + init timeline
committed under benchmarks/tts/edge_results/.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
tzhouamandClaude Fable 5.1 committed 2026-09-09 11:02:14 +08:00
1 parent 55192de984
commit 35e43b9983
18 files changed
+1740 -7
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

benchmarks/tts/edge_results/stream_latency/Qwen3-TTS-12Hz-0.6B-CustomVoice/default/20260909-105334_cpu_smoke.json
+867
View File
@@ -0,0 +1,867 @@
{
"schema": 1,
"meta": {
"model": "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice",
"deploy_config": null,
"query_type": "CustomVoice",
"txt_prompts": "benchmarks/tts/prompts_12.txt",
"git_sha": "f804bbd6",
"gpu_name": "NVIDIA L20X",
"cuda_visible_devices": null,
"timestamp": "20260909-105334",
"argv": [
"--query-type",
"CustomVoice",
"--txt-prompts",
"benchmarks/tts/prompts_12.txt",
"--init-timeout",
"1500",
"--stage-init-timeout",
"1200"
],
"bench_args": {
"model": "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice",
"warmup": 1,
"repeat": 1,
"init_only": false,
"tag": "cpu_smoke",
"output_dir": "benchmarks/tts/edge_results/stream_latency",
"save_wav": false,
"gpu_poll_interval_s": 0.2,
"parallel_stage_init": false,
"deploy_profile": null,
"bench_help": false
},
"warmup": 1,
"repeat": 1,
"init_only": false
},
"init": {
"init_s": 185.0914630489424,
"init_timeline": {
"schema": 1,
"t0": 1788922042.76263,
"t_end": 1788922223.6046276,
"total_s": 180.8419976234436,
"phases": [
{
"name": "stages_init",
"start": 1788922042.7633224,
"end": 1788922223.6009316,
"stage_id": null,
"replica_id": 0,
"pid": 2543391,
"role": "engine",
"extra": {},
"dur_s": 180.83760929107666
},
{
"name": "g3_device_lock",
"start": 1788922106.0638428,
"end": 1788922106.0639107,
"stage_id": 0,
"replica_id": 0,
"pid": 2543391,
"role": "engine",
"extra": {},
"dur_s": 6.794929504394531e-05
},
{
"name": "g2_launch_lock_wait",
"start": 1788922106.0639355,
"end": 1788922106.063938,
"stage_id": 0,
"replica_id": 0,
"pid": 2543391,
"role": "engine",
"extra": {
"acquired": true
},
"dur_s": 2.384185791015625e-06
},
{
"name": "g2_spawn_and_ready",
"start": 1788922106.0640821,
"end": 1788922164.965892,
"stage_id": 0,
"replica_id": 0,
"pid": 2543391,
"role": "engine",
"extra": {},
"dur_s": 58.90180993080139
},
{
"name": "proc_entry",
"start": 1788922130.877344,
"end": 1788922130.877344,
"stage_id": 0,
"replica_id": 0,
"pid": 2564074,
"role": "worker",
"extra": {},
"dur_s": 0.0
},
{
"name": "engine_core_init",
"start": 1788922130.8864088,
"end": 1788922164.9667127,
"stage_id": 0,
"replica_id": 0,
"pid": 2564074,
"role": "worker",
"extra": {},
"dur_s": 34.08030390739441
},
{
"name": "init_device",
"start": 1788922159.3280494,
"end": 1788922159.5362585,
"stage_id": 0,
"replica_id": 0,
"pid": 2567908,
"role": "worker",
"extra": {},
"dur_s": 0.20820903778076172
},
{
"name": "load_model",
"start": 1788922159.5429327,
"end": 1788922164.054677,
"stage_id": 0,
"replica_id": 0,
"pid": 2567908,
"role": "worker",
"extra": {},
"dur_s": 4.511744260787964
},
{
"name": "load_weights",
"start": 1788922159.5429373,
"end": 1788922164.0436099,
"stage_id": 0,
"replica_id": 0,
"pid": 2567908,
"role": "worker",
"extra": {},
"dur_s": 4.5006725788116455
},
{
"name": "talker_mtp_init",
"start": 1788922164.0478003,
"end": 1788922164.0480099,
"stage_id": 0,
"replica_id": 0,
"pid": 2567908,
"role": "worker",
"extra": {},
"dur_s": 0.00020956993103027344
},
{
"name": "attn_prewarm",
"start": 1788922164.0528018,
"end": 1788922164.0528057,
"stage_id": 0,
"replica_id": 0,
"pid": 2567908,
"role": "worker",
"extra": {},
"dur_s": 3.814697265625e-06
},
{
"name": "g3_device_lock",
"start": 1788922165.1493986,
"end": 1788922165.1495192,
"stage_id": 1,
"replica_id": 0,
"pid": 2543391,
"role": "engine",
"extra": {},
"dur_s": 0.00012063980102539062
},
{
"name": "g2_launch_lock_wait",
"start": 1788922165.149549,
"end": 1788922165.1495519,
"stage_id": 1,
"replica_id": 0,
"pid": 2543391,
"role": "engine",
"extra": {
"acquired": true
},
"dur_s": 2.86102294921875e-06
},
{
"name": "g2_spawn_and_ready",
"start": 1788922165.149846,
"end": 1788922220.5601192,
"stage_id": 1,
"replica_id": 0,
"pid": 2543391,
"role": "engine",
"extra": {},
"dur_s": 55.41027307510376
},
{
"name": "proc_entry",
"start": 1788922191.4990642,
"end": 1788922191.4990642,
"stage_id": 1,
"replica_id": 0,
"pid": 2572020,
"role": "worker",
"extra": {},
"dur_s": 0.0
},
{
"name": "engine_core_init",
"start": 1788922191.50477,
"end": 1788922220.5615401,
"stage_id": 1,
"replica_id": 0,
"pid": 2572020,
"role": "worker",
"extra": {},
"dur_s": 29.056770086288452
},
{
"name": "init_device",
"start": 1788922218.1614058,
"end": 1788922218.3837707,
"stage_id": 1,
"replica_id": 0,
"pid": 2576162,
"role": "worker",
"extra": {},
"dur_s": 0.2223649024963379
},
{
"name": "load_model",
"start": 1788922218.3888378,
"end": 1788922219.7271593,
"stage_id": 1,
"replica_id": 0,
"pid": 2576162,
"role": "worker",
"extra": {},
"dur_s": 1.3383214473724365
},
{
"name": "load_weights",
"start": 1788922218.3888428,
"end": 1788922219.721446,
"stage_id": 1,
"replica_id": 0,
"pid": 2576162,
"role": "worker",
"extra": {},
"dur_s": 1.3326032161712646
},
{
"name": "talker_mtp_init",
"start": 1788922219.7234585,
"end": 1788922219.7234626,
"stage_id": 1,
"replica_id": 0,
"pid": 2576162,
"role": "worker",
"extra": {},
"dur_s": 4.0531158447265625e-06
},
{
"name": "attn_prewarm",
"start": 1788922219.7253046,
"end": 1788922219.725309,
"stage_id": 1,
"replica_id": 0,
"pid": 2576162,
"role": "worker",
"extra": {},
"dur_s": 4.291534423828125e-06
}
]
}
},
"resources": {
"peak_rss_bytes": 14409445376,
"peak_gpu_mem_mib": 95959,
"gpu_mem_samples": 379,
"gpu_mem_mode": "whole-gpu"
},
"requests": [
{
"request_id": "w0",
"sr": 24000,
"chunks": [
{
"idx": 0,
"t_ms": 9093.21565227583,
"gap_ms": 9093.21565227583,
"samples": 1920
},
{
"idx": 1,
"t_ms": 12637.973092030734,
"gap_ms": 3544.7574397549033,
"samples": 48000
},
{
"idx": 2,
"t_ms": 15888.486401177943,
"gap_ms": 3250.513309147209,
"samples": 48000
}
],
"t_final_ms": 16496.613026130944,
"ttfa_ms": 9093.21565227583,
"playback_start_ms": 13808.486401177943,
"stall_at_ttfa_ms": 4715.270748902112,
"n_chunks": 3,
"streamed_audio_s": 4.08,
"tail_audio_s": 0.16,
"total_wall_s": 16.496613026130944,
"rtf_streamed": 3.8942368630338096,
"rtf_total": 3.8907106193705054,
"gap_ms": [
3544.7574397549033,
3250.513309147209
],
"text": "Hello, welcome to the voice synthesis benchmark test.",
"warmup": true
},
{
"request_id": "r0_0",
"sr": 24000,
"chunks": [
{
"idx": 0,
"t_ms": 291.6870419867337,
"gap_ms": 291.6870419867337,
"samples": 1920
},
{
"idx": 1,
"t_ms": 3767.898346297443,
"gap_ms": 3476.2113043107092,
"samples": 48000
},
{
"idx": 2,
"t_ms": 7128.282887395471,
"gap_ms": 3360.3845410980284,
"samples": 48000
}
],
"t_final_ms": 8370.939870364964,
"ttfa_ms": 291.6870419867337,
"playback_start_ms": 5048.282887395471,
"stall_at_ttfa_ms": 4756.595845408738,
"n_chunks": 3,
"streamed_audio_s": 4.08,
"tail_audio_s": 0.8,
"total_wall_s": 8.370939870364964,
"rtf_streamed": 1.7471281586753606,
"rtf_total": 1.7153565308124927,
"gap_ms": [
3476.2113043107092,
3360.3845410980284
],
"text": "Hello, welcome to the voice synthesis benchmark test.",
"warmup": false
},
{
"request_id": "r0_1",
"sr": 24000,
"chunks": [
{
"idx": 0,
"t_ms": 289.57725409418344,
"gap_ms": 289.57725409418344,
"samples": 1920
},
{
"idx": 1,
"t_ms": 3722.4736879579723,
"gap_ms": 3432.896433863789,
"samples": 48000
},
{
"idx": 2,
"t_ms": 6917.577911168337,
"gap_ms": 3195.1042232103646,
"samples": 48000
}
],
"t_final_ms": 7475.041428115219,
"ttfa_ms": 289.57725409418344,
"playback_start_ms": 4837.577911168337,
"stall_at_ttfa_ms": 4548.000657074153,
"n_chunks": 3,
"streamed_audio_s": 4.08,
"tail_audio_s": 0.08,
"total_wall_s": 7.475041428115219,
"rtf_streamed": 1.6954847821491021,
"rtf_total": 1.796884958681543,
"gap_ms": [
3432.896433863789,
3195.1042232103646
],
"text": "She said she would be here by noon, but nobody showed up.",
"warmup": false
},
{
"request_id": "r0_2",
"sr": 24000,
"chunks": [
{
"idx": 0,
"t_ms": 273.937676101923,
"gap_ms": 273.937676101923,
"samples": 1920
},
{
"idx": 1,
"t_ms": 3683.945945929736,
"gap_ms": 3410.008269827813,
"samples": 48000
},
{
"idx": 2,
"t_ms": 7284.630519337952,
"gap_ms": 3600.6845734082162,
"samples": 48000
}
],
"t_final_ms": 7858.37772814557,
"ttfa_ms": 273.937676101923,
"playback_start_ms": 5204.630519337952,
"stall_at_ttfa_ms": 4930.692843236029,
"n_chunks": 3,
"streamed_audio_s": 4.08,
"tail_audio_s": 0.32,
"total_wall_s": 7.85837772814557,
"rtf_streamed": 1.7854486567004784,
"rtf_total": 1.785994938214902,
"gap_ms": [
3410.008269827813,
3600.6845734082162
],
"text": "The quick brown fox jumps over the lazy dog near the riverbank.",
"warmup": false
},
{
"request_id": "r0_3",
"sr": 24000,
"chunks": [
{
"idx": 0,
"t_ms": 291.9175820425153,
"gap_ms": 291.9175820425153,
"samples": 1920
},
{
"idx": 1,
"t_ms": 3912.9916448146105,
"gap_ms": 3621.074062772095,
"samples": 48000
},
{
"idx": 2,
"t_ms": 7129.715716000646,
"gap_ms": 3216.724071186036,
"samples": 48000
}
],
"t_final_ms": 9064.493948128074,
"ttfa_ms": 291.9175820425153,
"playback_start_ms": 5049.715716000646,
"stall_at_ttfa_ms": 4757.798133958131,
"n_chunks": 3,
"streamed_audio_s": 4.08,
"tail_audio_s": 1.04,
"total_wall_s": 9.064493948128074,
"rtf_streamed": 1.7474793421570212,
"rtf_total": 1.7704089742437645,
"gap_ms": [
3621.074062772095,
3216.724071186036
],
"text": "I can't believe how beautiful the sunset looks from up here on the mountain.",
"warmup": false
},
{
"request_id": "r0_4",
"sr": 24000,
"chunks": [
{
"idx": 0,
"t_ms": 285.0063256919384,
"gap_ms": 285.0063256919384,
"samples": 1920
},
{
"idx": 1,
"t_ms": 3749.287095852196,
"gap_ms": 3464.280770160258,
"samples": 48000
},
{
"idx": 2,
"t_ms": 7012.798972893506,
"gap_ms": 3263.51187704131,
"samples": 48000
}
],
"t_final_ms": 8748.178598005325,
"ttfa_ms": 285.0063256919384,
"playback_start_ms": 4932.798972893506,
"stall_at_ttfa_ms": 4647.792647201568,
"n_chunks": 3,
"streamed_audio_s": 4.08,
"tail_audio_s": 1.12,
"total_wall_s": 8.748178598005325,
"rtf_streamed": 1.718823277669977,
"rtf_total": 1.682342038077947,
"gap_ms": [
3464.280770160258,
3263.51187704131
],
"text": "Please remember to bring your identification documents to the appointment tomorrow morning.",
"warmup": false
},
{
"request_id": "r0_5",
"sr": 24000,
"chunks": [
{
"idx": 0,
"t_ms": 275.7884138263762,
"gap_ms": 275.7884138263762,
"samples": 1920
},
{
"idx": 1,
"t_ms": 3673.660270869732,
"gap_ms": 3397.8718570433557,
"samples": 48000
},
{
"idx": 2,
"t_ms": 12653.649115003645,
"gap_ms": 8979.988844133914,
"samples": 48000
}
],
"t_final_ms": 75011.55206793919,
"ttfa_ms": 275.7884138263762,
"playback_start_ms": 10573.649115003645,
"stall_at_ttfa_ms": 10297.86070117727,
"n_chunks": 3,
"streamed_audio_s": 4.08,
"tail_audio_s": 1.68,
"total_wall_s": 75.01155206793919,
"rtf_streamed": 3.1013845870106973,
"rtf_total": 13.022838900683888,
"gap_ms": [
3397.8718570433557,
8979.988844133914
],
"text": "Have you ever wondered what it would be like to travel through time and visit ancient civilizations?",
"warmup": false
},
{
"request_id": "r0_6",
"sr": 24000,
"chunks": [
{
"idx": 0,
"t_ms": 290.9621219150722,
"gap_ms": 290.9621219150722,
"samples": 1920
},
{
"idx": 1,
"t_ms": 3827.7830346487463,
"gap_ms": 3536.820912733674,
"samples": 48000
},
{
"idx": 2,
"t_ms": 6955.3797277621925,
"gap_ms": 3127.5966931134462,
"samples": 48000
}
],
"t_final_ms": 9968.388166744262,
"ttfa_ms": 290.9621219150722,
"playback_start_ms": 4875.3797277621925,
"stall_at_ttfa_ms": 4584.41760584712,
"n_chunks": 3,
"streamed_audio_s": 4.08,
"tail_audio_s": 2.0,
"total_wall_s": 9.968388166744262,
"rtf_streamed": 1.704749933275047,
"rtf_total": 1.639537527425043,
"gap_ms": [
3536.820912733674,
3127.5966931134462
],
"text": "The restaurant on the corner serves the best pasta I have ever tasted in my entire life.",
"warmup": false
},
{
"request_id": "r0_7",
"sr": 24000,
"chunks": [
{
"idx": 0,
"t_ms": 301.0973362252116,
"gap_ms": 301.0973362252116,
"samples": 1920
},
{
"idx": 1,
"t_ms": 3678.3662801608443,
"gap_ms": 3377.2689439356327,
"samples": 48000
},
{
"idx": 2,
"t_ms": 6857.406801078469,
"gap_ms": 3179.0405209176242,
"samples": 48000
}
],
"t_final_ms": 9085.033813957125,
"ttfa_ms": 301.0973362252116,
"playback_start_ms": 4777.406801078469,
"stall_at_ttfa_ms": 4476.309464853257,
"n_chunks": 3,
"streamed_audio_s": 4.08,
"tail_audio_s": 1.36,
"total_wall_s": 9.085033813957125,
"rtf_streamed": 1.6807369610486442,
"rtf_total": 1.6700429805068244,
"gap_ms": [
3377.2689439356327,
3179.0405209176242
],
"text": "After the meeting, we should discuss the quarterly results and plan for the next phase.",
"warmup": false
},
{
"request_id": "r0_8",
"sr": 24000,
"chunks": [
{
"idx": 0,
"t_ms": 294.4919569417834,
"gap_ms": 294.4919569417834,
"samples": 1920
},
{
"idx": 1,
"t_ms": 3713.8455058448017,
"gap_ms": 3419.3535489030182,
"samples": 48000
},
{
"idx": 2,
"t_ms": 6958.719541784376,
"gap_ms": 3244.8740359395742,
"samples": 48000
},
{
"idx": 3,
"t_ms": 9996.23750988394,
"gap_ms": 3037.5179680995643,
"samples": 48000
}
],
"t_final_ms": 10559.09016681835,
"ttfa_ms": 294.4919569417834,
"playback_start_ms": 5916.23750988394,
"stall_at_ttfa_ms": 5621.745552942157,
"n_chunks": 4,
"streamed_audio_s": 6.08,
"tail_audio_s": 0.16,
"total_wall_s": 10.55909016681835,
"rtf_streamed": 1.644118011494069,
"rtf_total": 1.692161885708069,
"gap_ms": [
3419.3535489030182,
3244.8740359395742,
3037.5179680995643
],
"text": "Learning a new language takes patience, practice, and a genuine curiosity about other cultures.",
"warmup": false
},
{
"request_id": "r0_9",
"sr": 24000,
"chunks": [
{
"idx": 0,
"t_ms": 316.74008816480637,
"gap_ms": 316.74008816480637,
"samples": 1920
},
{
"idx": 1,
"t_ms": 3735.6491410173476,
"gap_ms": 3418.909052852541,
"samples": 48000
},
{
"idx": 2,
"t_ms": 7355.19207501784,
"gap_ms": 3619.542934000492,
"samples": 48000
}
],
"t_final_ms": 10337.802770081908,
"ttfa_ms": 316.74008816480637,
"playback_start_ms": 5275.19207501784,
"stall_at_ttfa_ms": 4958.451986853033,
"n_chunks": 3,
"streamed_audio_s": 4.08,
"tail_audio_s": 2.0,
"total_wall_s": 10.337802770081908,
"rtf_streamed": 1.8027431556416273,
"rtf_total": 1.7002965082371557,
"gap_ms": [
3418.909052852541,
3619.542934000492
],
"text": "The train leaves at half past seven, so we need to arrive at the station before then.",
"warmup": false
},
{
"request_id": "r0_10",
"sr": 24000,
"chunks": [
{
"idx": 0,
"t_ms": 310.7302547432482,
"gap_ms": 310.7302547432482,
"samples": 1920
},
{
"idx": 1,
"t_ms": 3758.4262755699456,
"gap_ms": 3447.6960208266973,
"samples": 48000
},
{
"idx": 2,
"t_ms": 6996.824599802494,
"gap_ms": 3238.3983242325485,
"samples": 48000
}
],
"t_final_ms": 8471.956291701645,
"ttfa_ms": 310.7302547432482,
"playback_start_ms": 4916.824599802494,
"stall_at_ttfa_ms": 4606.094345059246,
"n_chunks": 3,
"streamed_audio_s": 4.08,
"tail_audio_s": 0.96,
"total_wall_s": 8.471956291701645,
"rtf_streamed": 1.71490799014767,
"rtf_total": 1.6809437086709613,
"gap_ms": [
3447.6960208266973,
3238.3983242325485
],
"text": "Could you please turn down the music a little bit, I'm trying to concentrate on my work.",
"warmup": false
},
{
"request_id": "r0_11",
"sr": 24000,
"chunks": [
{
"idx": 0,
"t_ms": 296.68450681492686,
"gap_ms": 296.68450681492686,
"samples": 1920
},
{
"idx": 1,
"t_ms": 3681.493826210499,
"gap_ms": 3384.809319395572,
"samples": 48000
},
{
"idx": 2,
"t_ms": 7026.184834074229,
"gap_ms": 3344.69100786373,
"samples": 48000
}
],
"t_final_ms": 9525.669381022453,
"ttfa_ms": 296.68450681492686,
"playback_start_ms": 4946.184834074229,
"stall_at_ttfa_ms": 4649.500327259302,
"n_chunks": 3,
"streamed_audio_s": 4.08,
"tail_audio_s": 1.6,
"total_wall_s": 9.525669381022453,
"rtf_streamed": 1.7221041259985854,
"rtf_total": 1.6770544684898687,
"gap_ms": [
3384.809319395572,
3344.69100786373
],
"text": "It was a dark and stormy night when the old lighthouse keeper heard a knock at the door.",
"warmup": false
}
],
"summary": {
"n_requests": 12,
"ttfa_ms": {
"n": 12,
"p50": 291.6870419867337,
"p90": 310.7302547432482,
"max": 316.74008816480637,
"mean": 293.2183798790599
},
"playback_start_ms": {
"n": 12,
"p50": 4946.184834074229,
"p90": 5916.23750988394,
"max": 10573.649115003645,
"mean": 5529.490055784893
},
"stall_at_ttfa_ms": {
"n": 12,
"p50": 4649.500327259302,
"p90": 5621.745552942157,
"max": 10297.86070117727,
"mean": 5236.271675905834
},
"rtf_streamed": {
"n": 12,
"p50": 1.718823277669977,
"p90": 1.8027431556416273,
"max": 3.1013845870106973,
"mean": 1.83875908183069
},
"rtf_total": {
"n": 12,
"p50": 1.692161885708069,
"p90": 1.796884958681543,
"max": 13.022838900683888,
"mean": 2.6528219516460383
},
"gap_ms": {
"n": 25,
"p50": 3397.8718570433557,
"p90": 3619.542934000492,
"max": 8979.988844133914,
"mean": 3591.8104044348
},
"total_audio_s": 50.96,
"total_wall_s": 174.4765242310241
}
}
benchmarks/tts/edge_results/stream_latency/timelines/0.6B_cpu_smoke.json
+243
View File
@@ -0,0 +1,243 @@
{
"schema": 1,
"t0": 1788922042.76263,
"t_end": 1788922223.6046276,
"total_s": 180.8419976234436,
"phases": [
{
"name": "stages_init",
"start": 1788922042.7633224,
"end": 1788922223.6009316,
"stage_id": null,
"replica_id": 0,
"pid": 2543391,
"role": "engine",
"extra": {},
"dur_s": 180.83760929107666
},
{
"name": "g3_device_lock",
"start": 1788922106.0638428,
"end": 1788922106.0639107,
"stage_id": 0,
"replica_id": 0,
"pid": 2543391,
"role": "engine",
"extra": {},
"dur_s": 6.794929504394531e-05
},
{
"name": "g2_launch_lock_wait",
"start": 1788922106.0639355,
"end": 1788922106.063938,
"stage_id": 0,
"replica_id": 0,
"pid": 2543391,
"role": "engine",
"extra": {
"acquired": true
},
"dur_s": 2.384185791015625e-06
},
{
"name": "g2_spawn_and_ready",
"start": 1788922106.0640821,
"end": 1788922164.965892,
"stage_id": 0,
"replica_id": 0,
"pid": 2543391,
"role": "engine",
"extra": {},
"dur_s": 58.90180993080139
},
{
"name": "proc_entry",
"start": 1788922130.877344,
"end": 1788922130.877344,
"stage_id": 0,
"replica_id": 0,
"pid": 2564074,
"role": "worker",
"extra": {},
"dur_s": 0.0
},
{
"name": "engine_core_init",
"start": 1788922130.8864088,
"end": 1788922164.9667127,
"stage_id": 0,
"replica_id": 0,
"pid": 2564074,
"role": "worker",
"extra": {},
"dur_s": 34.08030390739441
},
{
"name": "init_device",
"start": 1788922159.3280494,
"end": 1788922159.5362585,
"stage_id": 0,
"replica_id": 0,
"pid": 2567908,
"role": "worker",
"extra": {},
"dur_s": 0.20820903778076172
},
{
"name": "load_model",
"start": 1788922159.5429327,
"end": 1788922164.054677,
"stage_id": 0,
"replica_id": 0,
"pid": 2567908,
"role": "worker",
"extra": {},
"dur_s": 4.511744260787964
},
{
"name": "load_weights",
"start": 1788922159.5429373,
"end": 1788922164.0436099,
"stage_id": 0,
"replica_id": 0,
"pid": 2567908,
"role": "worker",
"extra": {},
"dur_s": 4.5006725788116455
},
{
"name": "talker_mtp_init",
"start": 1788922164.0478003,
"end": 1788922164.0480099,
"stage_id": 0,
"replica_id": 0,
"pid": 2567908,
"role": "worker",
"extra": {},
"dur_s": 0.00020956993103027344
},
{
"name": "attn_prewarm",
"start": 1788922164.0528018,
"end": 1788922164.0528057,
"stage_id": 0,
"replica_id": 0,
"pid": 2567908,
"role": "worker",
"extra": {},
"dur_s": 3.814697265625e-06
},
{
"name": "g3_device_lock",
"start": 1788922165.1493986,
"end": 1788922165.1495192,
"stage_id": 1,
"replica_id": 0,
"pid": 2543391,
"role": "engine",
"extra": {},
"dur_s": 0.00012063980102539062
},
{
"name": "g2_launch_lock_wait",
"start": 1788922165.149549,
"end": 1788922165.1495519,
"stage_id": 1,
"replica_id": 0,
"pid": 2543391,
"role": "engine",
"extra": {
"acquired": true
},
"dur_s": 2.86102294921875e-06
},
{
"name": "g2_spawn_and_ready",
"start": 1788922165.149846,
"end": 1788922220.5601192,
"stage_id": 1,
"replica_id": 0,
"pid": 2543391,
"role": "engine",
"extra": {},
"dur_s": 55.41027307510376
},
{
"name": "proc_entry",
"start": 1788922191.4990642,
"end": 1788922191.4990642,
"stage_id": 1,
"replica_id": 0,
"pid": 2572020,
"role": "worker",
"extra": {},
"dur_s": 0.0
},
{
"name": "engine_core_init",
"start": 1788922191.50477,
"end": 1788922220.5615401,
"stage_id": 1,
"replica_id": 0,
"pid": 2572020,
"role": "worker",
"extra": {},
"dur_s": 29.056770086288452
},
{
"name": "init_device",
"start": 1788922218.1614058,
"end": 1788922218.3837707,
"stage_id": 1,
"replica_id": 0,
"pid": 2576162,
"role": "worker",
"extra": {},
"dur_s": 0.2223649024963379
},
{
"name": "load_model",
"start": 1788922218.3888378,
"end": 1788922219.7271593,
"stage_id": 1,
"replica_id": 0,
"pid": 2576162,
"role": "worker",
"extra": {},
"dur_s": 1.3383214473724365
},
{
"name": "load_weights",
"start": 1788922218.3888428,
"end": 1788922219.721446,
"stage_id": 1,
"replica_id": 0,
"pid": 2576162,
"role": "worker",
"extra": {},
"dur_s": 1.3326032161712646
},
{
"name": "talker_mtp_init",
"start": 1788922219.7234585,
"end": 1788922219.7234626,
"stage_id": 1,
"replica_id": 0,
"pid": 2576162,
"role": "worker",
"extra": {},
"dur_s": 4.0531158447265625e-06
},
{
"name": "attn_prewarm",
"start": 1788922219.7253046,
"end": 1788922219.725309,
"stage_id": 1,
"replica_id": 0,
"pid": 2576162,
"role": "worker",
"extra": {},
"dur_s": 4.291534423828125e-06
}
]
}
docs/getting_started/installation/cpu.md
+56
View File
@@ -0,0 +1,56 @@
# CPU
vLLM-Omni can run **autoregressive and generation stages** (for example the
Qwen3-TTS talker → code2wav pipeline) on vLLM's CPU backend. Diffusion stages
are not supported on CPU.
## Requirements
- x86-64 with AVX-512 (bf16 needs `avx512_bf16` or AMX) or aarch64 with NEON
(`fp16`/`bf16` extensions recommended). See the vLLM CPU installation page
for the supported ISA matrix.
- A vLLM **CPU** build (`+cpu` wheel). The CUDA wheel does not contain the
CPU kernels.
- Python 3.10–3.13.
## Install
```bash
uv venv --python 3.12 && source .venv/bin/activate
# vLLM CPU wheel (x86-64 example; see vLLM docs for aarch64 and per-commit wheels)
uv pip install --torch-backend cpu \
https://github.com/vllm-project/vllm/releases/download/v0.28.0/vllm-0.28.0+cpu-cp38-abi3-manylinux_2_34_x86_64.whl
# vLLM-Omni without re-resolving torch, then its CPU requirements
VLLM_TARGET_DEVICE=cpu uv pip install --torch-backend cpu --no-deps -e .
VLLM_TARGET_DEVICE=cpu uv pip install --torch-backend cpu -r requirements/cpu.txt
```
## Run
```bash
export VLLM_TARGET_DEVICE=cpu # selects the CPU OmniPlatform
export LD_PRELOAD=$(python -c "import glob,sys; print(glob.glob(sys.prefix+'/lib/libiomp5.so')[0])")
# one utterance, Qwen3-TTS 0.6B, CPU overlay from vllm_omni/deploy/qwen3_tts.yaml (platforms: cpu:)
python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-type CustomVoice
# or let the hardware probe size everything (threads, KV budget, dtype, chunking):
python benchmarks/tts/stream_latency_bench.py --model Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice \
--query-type CustomVoice --deploy-profile auto
```
What the CPU platform changes (`vllm_omni/platforms/cpu/platform.py`):
| Item | CPU behaviour |
|---|---|
| Workers | `CPUARWorker` / `CPUGenerationWorker` compose vLLM's `CPUWorker` with the Omni runners |
| Scheduling | `async_scheduling=False` (forced by vLLM's `CpuPlatform`), no CUDA graphs |
| Memory | `gpu_memory_utilization` is a NUMA-node memory fraction; prefer `kv_cache_memory_bytes` |
| Threads | per-stage `VLLM_CPU_OMP_THREADS_BIND` ranges (`platforms: cpu:` overlay or `deploy_profile="auto"`) |
| Compile | code-predictor `torch.compile` off by default; `VLLM_OMNI_CPU_INDUCTOR=1` enables it |
| dtype | bf16 with `avx512_bf16`/AMX or ARM `bf16`, otherwise `float32` (`deploy_profile="auto"`) |
| Sleep mode | not supported (returns an error ACK) |
## Tests
```bash
VLLM_TARGET_DEVICE=cpu pytest tests/platforms/test_cpu_platform.py tests/worker/test_cpu_ar_model_runner.py -m "core_model and cpu"
```
requirements/cpu.txt
+4
View File
@@ -1,2 +1,6 @@
-r common.txt
onnxruntime>=1.23.2
# CPU OmniPlatform runtime (vllm_omni/platforms/cpu): NUMA/RSS accounting and
# the OpenMP runtime vLLM's CpuPlatform expects in LD_PRELOAD.
psutil
intel-openmp; platform_machine == "x86_64"
tests/platforms/test_cpu_platform.py
+91
View File
@@ -0,0 +1,91 @@
"""CPU OmniPlatform: registration, MRO, config forcing (CPU, no accelerator needed)."""
from __future__ import annotations
import pytest
import torch
from vllm.platforms.cpu import CpuPlatform
from vllm.platforms.interface import Platform
import vllm_omni.platforms as omni_platforms
from vllm_omni.platforms.cpu.platform import CPUOmniPlatform
from vllm_omni.platforms.interface import OmniPlatform, OmniPlatformEnum
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_registered_and_selected_when_cpu_target(monkeypatch):
assert "cpu" in omni_platforms.builtin_omni_platform_plugins
monkeypatch.setattr(omni_platforms, "_cpu_target_requested", lambda: True)
assert omni_platforms.cpu_omni_platform_plugin() == omni_platforms._CPU_OMNI_PLATFORM
assert omni_platforms.resolve_current_omni_platform_cls_qualname() == omni_platforms._CPU_OMNI_PLATFORM
monkeypatch.setattr(omni_platforms, "_cpu_target_requested", lambda: False)
assert omni_platforms.cpu_omni_platform_plugin() is None
def test_cpu_target_requested_reads_env(monkeypatch):
import vllm.envs as envs
monkeypatch.setattr(envs, "VLLM_TARGET_DEVICE", "cpu", raising=False)
assert omni_platforms._cpu_target_requested() is True
def test_mro_and_attributes():
mro = CPUOmniPlatform.__mro__
assert mro.index(OmniPlatform) < mro.index(CpuPlatform) < mro.index(Platform)
p = CPUOmniPlatform()
assert p.is_cpu() and not p.is_cuda() and not p.is_npu()
assert p._omni_enum is OmniPlatformEnum.CPU
assert CPUOmniPlatform.device_name == "cpu" and CPUOmniPlatform.device_type == "cpu"
assert CPUOmniPlatform.dist_backend == "gloo"
assert CPUOmniPlatform.get_device_count() == 1
assert CPUOmniPlatform.get_torch_device() == torch.device("cpu")
assert CPUOmniPlatform.get_device_version() is None
assert CPUOmniPlatform.synchronize() is None
assert CPUOmniPlatform.get_free_memory() > 0
avail, total = CPUOmniPlatform.get_device_memory()
assert 0 < avail <= total
assert CPUOmniPlatform.get_device_total_memory() > 0
assert CPUOmniPlatform.supports_talker_mtp_graph_capture() is False
assert CPUOmniPlatform.supports_torch_inductor() is False
assert CPUOmniPlatform.get_default_stage_config_path() == "vllm_omni/deploy"
assert "cpu_ar_worker.CPUARWorker" in CPUOmniPlatform.get_omni_ar_worker_cls()
assert "CPUGenerationWorker" in CPUOmniPlatform.get_omni_generation_worker_cls()
wrapper_cls = CPUOmniPlatform.get_graph_wrapper_cls()
assert isinstance(wrapper_cls, type) and not isinstance(object(), wrapper_cls)
with pytest.raises(NotImplementedError):
wrapper_cls()
with pytest.raises(NotImplementedError):
CPUOmniPlatform.get_diffusion_worker_cls()
def test_inductor_opt_in(monkeypatch):
monkeypatch.setenv("VLLM_OMNI_CPU_INDUCTOR", "1")
assert CPUOmniPlatform.supports_torch_inductor() is True
def test_check_and_update_config_forces_cpu_rules():
from vllm.config import VllmConfig
from vllm.config.device import DeviceConfig
cfg = VllmConfig(device_config=DeviceConfig(device="cpu"))
cfg.scheduler_config.async_scheduling = True
CPUOmniPlatform.check_and_update_config(cfg)
assert cfg.scheduler_config.async_scheduling is False
assert cfg.compilation_config.cudagraph_capture_sizes == []
# worker_cls is resolved by the Omni platform (resolve_worker_cls), not here.
assert cfg.cache_config.block_size == 128
def test_deploy_overlay_for_cpu_platform():
from vllm_omni.config.stage_config import _DEPLOY_DIR, _apply_platform_overrides, load_deploy_config
deploy = load_deploy_config(_DEPLOY_DIR / "qwen3_tts.yaml")
deploy = _apply_platform_overrides(deploy, platform="cpu")
s0, s1 = deploy.stages
assert s0.devices == "cpu" and s1.devices == "cpu"
assert s0.enforce_eager is True and s1.enforce_eager is True
assert s0.async_scheduling is False
assert s0.engine_extras["dtype"] == "bfloat16"
assert s0.engine_extras["kv_cache_memory_bytes"] == 4 * 2**30
assert s0.env["VLLM_CPU_OMP_THREADS_BIND"] != s1.env["VLLM_CPU_OMP_THREADS_BIND"]
tests/worker/test_cpu_ar_model_runner.py
+74
View File
@@ -0,0 +1,74 @@
"""CPU Omni runners/workers: MRO and override resolution (import-only, CPU)."""
from __future__ import annotations
import pytest
from vllm.v1.worker.cpu_model_runner import CPUModelRunner
from vllm.v1.worker.cpu_worker import CPUWorker
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
from vllm_omni.platforms.cpu.worker.cpu_ar_model_runner import (
CPUARModelRunner,
CPUGenerationModelRunner,
_CPUOmniRunnerMixin,
)
from vllm_omni.platforms.cpu.worker.cpu_ar_worker import CPUARWorker, CPUGenerationWorker
from vllm_omni.worker.gpu_ar_model_runner import GPUARModelRunner
from vllm_omni.worker.gpu_generation_model_runner import GPUGenerationModelRunner
from vllm_omni.worker.mixins import OmniWorkerMixin
from vllm_omni.worker.omni_connector_model_runner_mixin import OmniConnectorModelRunnerMixin
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_ar_runner_mro():
mro = CPUARModelRunner.__mro__
assert (
mro.index(_CPUOmniRunnerMixin)
< mro.index(GPUARModelRunner)
< mro.index(CPUModelRunner)
< mro.index(GPUModelRunner)
)
# CPU overrides win over the GPU runner implementations
assert CPUARModelRunner._init_device_properties is _CPUOmniRunnerMixin._init_device_properties
assert CPUARModelRunner._sync_device is _CPUOmniRunnerMixin._sync_device
assert CPUARModelRunner.capture_model is _CPUOmniRunnerMixin.capture_model
assert CPUARModelRunner._capture_talker_mtp_graphs is _CPUOmniRunnerMixin._capture_talker_mtp_graphs
# vLLM CPU shims (triton/tensor post-processing, warm-up) are reachable
assert CPUARModelRunner.warming_up_model is CPUModelRunner.warming_up_model
assert CPUARModelRunner._postprocess_triton is CPUModelRunner._postprocess_triton
# omni connector mixin is present for the worker-class validation
assert issubclass(CPUARModelRunner, OmniConnectorModelRunnerMixin)
def test_generation_runner_mro():
mro = CPUGenerationModelRunner.__mro__
assert mro.index(GPUGenerationModelRunner) < mro.index(CPUModelRunner)
assert CPUGenerationModelRunner.capture_model is _CPUOmniRunnerMixin.capture_model
def test_workers_compose_cpu_worker_and_omni_mixin():
for worker_cls, runner_cls in ((CPUARWorker, CPUARModelRunner), (CPUGenerationWorker, CPUGenerationModelRunner)):
assert issubclass(worker_cls, CPUWorker) and issubclass(worker_cls, OmniWorkerMixin)
assert worker_cls.model_runner_cls is runner_cls
assert hasattr(worker_cls, "handle_sleep_task") and hasattr(worker_cls, "handle_wake_task")
def test_worker_cls_validation_accepts_cpu_workers():
from vllm_omni.worker.omni_connector_validation import validate_worker_omni_connector
validate_worker_omni_connector("vllm_omni.platforms.cpu.worker.cpu_ar_worker.CPUARWorker", required=True)
validate_worker_omni_connector(
"vllm_omni.platforms.cpu.worker.cpu_generation_worker.CPUGenerationWorker", required=True
)
def test_sleep_and_wake_return_error_acks():
class _Task:
task_id = "t1"
level = 1
ack = CPUARWorker.handle_sleep_task(object.__new__(CPUARWorker), _Task())
assert ack.status == "ERROR" and ack.task_id == "t1"
ack = CPUARWorker.handle_wake_task(object.__new__(CPUARWorker), _Task())
assert ack.status == "ERROR"
vllm_omni/deploy/qwen3_tts.yaml
+38
View File
@@ -136,3 +136,41 @@ platforms:
- stage_id: 1
compilation_config:
cudagraph_mode: PIECEWISE
# CPU OmniPlatform (VLLM_TARGET_DEVICE=cpu or a +cpu vLLM wheel). vLLM's
# CpuPlatform already forces async_scheduling=False, no CUDA graphs and
# CPU_ATTN; the values below size the two stages for a CPU box. Nested
# layout: engine args under ``engine_args``, placement under ``runtime``.
# gpu_memory_utilization is read by the CPU worker as a NUMA-node memory
# fraction, so an absolute kv_cache_memory_bytes is set instead.
# Thread ranges are per-stage (disjoint) so the two workers do not pin the
# same cores; deploy_profile="auto" derives them from the probed topology.
cpu:
stages:
- stage_id: 0
engine_args:
enforce_eager: true
async_scheduling: false
max_num_seqs: 2
max_num_batched_tokens: 2048
max_model_len: 2048
gpu_memory_utilization: 0.05
dtype: bfloat16
kv_cache_memory_bytes: 4294967296
runtime:
devices: "cpu"
env:
VLLM_CPU_OMP_THREADS_BIND: "0-11"
- stage_id: 1
engine_args:
enforce_eager: true
async_scheduling: false
max_num_seqs: 2
max_num_batched_tokens: 8192
max_model_len: 8192
gpu_memory_utilization: 0.05
dtype: bfloat16
kv_cache_memory_bytes: 2147483648
runtime:
devices: "cpu"
env:
VLLM_CPU_OMP_THREADS_BIND: "12-15"
vllm_omni/model_executor/models/qwen3_tts/prompt_embeds_builder.py
+6 -1
View File
@@ -363,7 +363,12 @@ class Qwen3TTSPromptEmbedsBuilder:
self._tts_pad_embed_buffer = tts_pad_embed
self._encode_ref_audio_batch_fn = encode_ref_audio_batch
self._speaker_cache = speaker_cache
self._embedding_dtype = torch.bfloat16
# Follow the talker's pad-embedding dtype (model dtype) so CPU float32/float16 deployments work.
self._embedding_dtype = (
tts_pad_embed.dtype
if isinstance(tts_pad_embed, torch.Tensor) and tts_pad_embed.is_floating_point()
else torch.bfloat16
)
self._text_tokenizer: Any | None = None
vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py
+13 -4
View File
@@ -352,6 +352,13 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
self.use_async_omni_output = True
self.eager_omni_postprocess_before_async_output = True
self.omni_pooler_payload_include_hidden = False
# Terminal talker (no downstream stage, e.g. qwen3_tts_talker_only):
# publish the per-step codec frames to the client so an on-device
# decoder can consume them (codec-token streaming).
if getattr(vllm_config.model_config, "custom_process_next_stage_input_func", None) is None and not getattr(
vllm_config.model_config, "async_chunk_process_next_stage_input_func", None
):
self.omni_client_multimodal_output_keys = ("codes.audio",)
self.model = Qwen3Model(vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model"))
if get_pp_group().is_last_rank:
@@ -465,7 +472,9 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
torch.zeros(1, int(self.talker_config.hidden_size), dtype=model_dtype),
persistent=False,
)
self._embedding_dtype = torch.bfloat16
# Follows the model dtype so CPU deployments can run float32/float16;
# on CUDA this is bfloat16 as before.
self._embedding_dtype = model_dtype if isinstance(model_dtype, torch.dtype) else torch.bfloat16
tokenizer_config = Qwen3TTSTokenizerV2Config.from_pretrained(
self.model_path,
@@ -475,7 +484,7 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
tokenizer_config.encoder_config,
)
self.encoder.eval()
self.encoder.to(dtype=torch.bfloat16)
self.encoder.to(dtype=self._embedding_dtype)
self._encoder_valid_num_quantizers = int(tokenizer_config.encoder_valid_num_quantizers)
self._encoder_downsample_rate = int(tokenizer_config.encode_downsample_rate)
@@ -1048,7 +1057,7 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
sampling_rate=target_sr,
return_tensors="pt",
)
inputs = inputs.to(device).to(torch.bfloat16)
inputs = inputs.to(device).to(self._embedding_dtype)
input_values = inputs["input_values"].squeeze(1)
padding_mask = inputs["padding_mask"].squeeze(1)
@@ -1125,7 +1134,7 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
loaded.add(name)
device = self.vllm_config.device_config.device
self.encoder.to(device=device, dtype=torch.bfloat16)
self.encoder.to(device=device, dtype=self._embedding_dtype)
self._init_runtime_buffers()
self._init_silence_mask()
vllm_omni/patch.py
+10 -2
View File
@@ -328,8 +328,8 @@ def _patch_chat_template_registry():
)
if "qwen3_omni_moe" not in _MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK:
_MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK["qwen3_omni_moe"] = (
lambda _: CHAT_TEMPLATES_DIR / "template_chatml.jinja"
_MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK["qwen3_omni_moe"] = lambda _: (
CHAT_TEMPLATES_DIR / "template_chatml.jinja"
)
except ImportError:
pass
@@ -529,6 +529,14 @@ _patch_inductor_factorable_divisibility()
# (not a slot or C extension). If CuMemAllocator is rewritten in C/Cython,
# this monkey-patch will silently become a no-op.
def _patch_cumem_free_callback_cuda() -> None:
try:
import torch
if torch.version.cuda is None and getattr(torch.version, "hip", None) is None:
_PATCH_LOGGER.debug("[cumem-cuda] CPU-only torch; skipping patch")
return
except Exception:
pass
try:
from vllm.device_allocator.cumem import CuMemAllocator
except ImportError:
vllm_omni/platforms/__init__.py
+38
View File
@@ -120,17 +120,55 @@ def musa_omni_platform_plugin() -> str | None:
return "vllm_omni.platforms.musa.platform.MUSAOmniPlatform" if is_musa else None
_CPU_OMNI_PLATFORM = "vllm_omni.platforms.cpu.platform.CPUOmniPlatform"
def _cpu_target_requested() -> bool:
"""True when vLLM itself would select its CPU platform (env or ``+cpu`` wheel)."""
try:
import vllm.envs as envs
if getattr(envs, "VLLM_TARGET_DEVICE", None) == "cpu":
return True
except Exception: # pragma: no cover - envs import never fails in practice
pass
try:
from vllm.platforms import vllm_version_matches_substr
return bool(vllm_version_matches_substr("cpu"))
except Exception:
return False
def cpu_omni_platform_plugin() -> str | None:
"""Check if the CPU OmniPlatform should be activated.
Mirrors vLLM's own ``cpu_platform_plugin``: active when
``VLLM_TARGET_DEVICE=cpu`` or a ``+cpu`` vLLM wheel is installed.
"""
logger.debug("Checking if CPU OmniPlatform is available.")
return _CPU_OMNI_PLATFORM if _cpu_target_requested() else None
builtin_omni_platform_plugins = {
"cuda": cuda_omni_platform_plugin,
"rocm": rocm_omni_platform_plugin,
"npu": npu_omni_platform_plugin,
"xpu": xpu_omni_platform_plugin,
"musa": musa_omni_platform_plugin,
"cpu": cpu_omni_platform_plugin,
}
def resolve_current_omni_platform_cls_qualname() -> str:
"""Resolve the current OmniPlatform class qualified name."""
# Like vLLM (vllm/platforms/__init__.py), an explicit CPU target wins before
# any accelerator probe so a +cpu wheel on a GPU host does not activate two
# builtin platforms.
if _cpu_target_requested():
logger.debug("CPU target requested; activating CPU OmniPlatform.")
return _CPU_OMNI_PLATFORM
platform_plugins = load_omni_plugins_by_group(OMNI_PLATFORM_PLUGINS_GROUP)
activated_plugins = []
vllm_omni/platforms/cpu/__init__.py
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
vllm_omni/platforms/cpu/platform.py
+160
View File
@@ -0,0 +1,160 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""CPU OmniPlatform: AR / generation stages on vLLM's CPU backend.
Composes vLLM's ``CpuPlatform`` (device attributes, ``check_and_update_config``
forcing: no async scheduling, no CUDA graphs, ``CPU_ATTN``, thread binding,
LD_PRELOAD) with the Omni interface, exactly as ``CudaOmniPlatform`` composes
``CudaPlatformBase``. Diffusion stages are not supported on CPU.
Activate with ``VLLM_TARGET_DEVICE=cpu`` (or a ``+cpu`` vLLM wheel). Deploy
YAMLs can pin CPU values under ``platforms: cpu:``.
"""
from __future__ import annotations
import os
from typing import Any
import torch
from vllm.logger import init_logger
from vllm.platforms.cpu import CpuPlatform
from vllm_omni.platforms.interface import OmniPlatform, OmniPlatformEnum
logger = init_logger(__name__)
def _numa_memory(device: torch.device | None = None) -> tuple[int, int]:
"""(available, total) bytes of the NUMA node this process is allowed on, else host memory."""
try:
from vllm.utils.cpu_resource_utils import get_allowed_cpu_list, get_memory_node_info
node = get_allowed_cpu_list()[0].numa_node
info = get_memory_node_info(node)
return int(info.available_memory), int(info.total_memory)
except Exception:
try:
import psutil
vm = psutil.virtual_memory()
return int(vm.available), int(vm.total)
except Exception:
return 0, 0
class _NoGraphWrapper:
"""Sentinel graph-wrapper type for the CPU platform (never instantiated)."""
def __init__(self, *args: Any, **kwargs: Any) -> None: # pragma: no cover - guard
raise NotImplementedError("CUDA graphs are not available on the CPU OmniPlatform")
class CPUOmniPlatform(OmniPlatform, CpuPlatform):
"""CPU implementation of OmniPlatform (AR + generation stages only)."""
_omni_enum = OmniPlatformEnum.CPU
# ----------------------------------------------------------- workers
@classmethod
def get_omni_ar_worker_cls(cls) -> str:
return "vllm_omni.platforms.cpu.worker.cpu_ar_worker.CPUARWorker"
@classmethod
def get_omni_generation_worker_cls(cls) -> str:
return "vllm_omni.platforms.cpu.worker.cpu_generation_worker.CPUGenerationWorker"
@classmethod
def get_default_stage_config_path(cls) -> str:
return "vllm_omni/deploy"
# ---------------------------------------------------------- diffusion
@classmethod
def get_diffusion_worker_cls(cls) -> str:
raise NotImplementedError("Diffusion stages are not supported on the CPU OmniPlatform")
@classmethod
def get_diffusion_model_runner_cls(cls) -> str:
raise NotImplementedError("Diffusion stages are not supported on the CPU OmniPlatform")
@classmethod
def get_diffusion_attn_backend_cls(cls, *args: Any, **kwargs: Any) -> str:
raise NotImplementedError("Diffusion attention backends are not supported on the CPU OmniPlatform")
@classmethod
def supports_diffusion_dense_flash_attention(cls) -> bool:
return False
@classmethod
def has_flash_attn_package(cls) -> bool:
return False
# ------------------------------------------------------------ compile
@classmethod
def supports_torch_inductor(cls) -> bool:
# The code predictor's torch.compile costs a long C++ codegen on first
# use; keep it off unless explicitly requested (measured variant).
return os.environ.get("VLLM_OMNI_CPU_INDUCTOR", "0") == "1"
@classmethod
def supports_talker_mtp_graph_capture(cls) -> bool:
return False
@classmethod
def get_graph_wrapper_cls(cls) -> type:
# Runners use this in ``isinstance`` checks on every decode step
# (``_talker_mtp_forward``); on CPU nothing is ever wrapped, so return a
# sentinel class that no object is an instance of.
return _NoGraphWrapper
# ------------------------------------------------------------- device
@classmethod
def get_torch_device(cls, local_rank: int | None = None) -> torch.device:
return torch.device("cpu")
@classmethod
def get_device_count(cls) -> int:
# One logical device: stage device locks / admission iterate range(count).
return 1
@classmethod
def get_device_version(cls) -> str | None:
return None
@classmethod
def synchronize(cls) -> None:
return None
@classmethod
def record_device_event(cls):
return None
@classmethod
def get_free_memory(cls, device: torch.device | None = None) -> int:
return _numa_memory(device)[0]
@classmethod
def get_device_memory(cls, device: torch.device | None = None) -> tuple[int, int]:
return _numa_memory(device)
@classmethod
def get_device_total_memory(cls, device_id: int = 0) -> int:
try:
return int(CpuPlatform.get_device_total_memory(device_id))
except Exception:
return _numa_memory()[1]
@classmethod
def supports_cpu_offload(cls) -> bool:
return False
@classmethod
def is_fully_connected(cls, physical_device_ids: list[int] | None = None) -> bool:
return True
@classmethod
def get_default_ir_op_priority(cls, *args: Any, **kwargs: Any):
parent = getattr(CpuPlatform, "get_default_ir_op_priority", None)
if parent is not None:
return parent(*args, **kwargs)
return super().get_default_ir_op_priority(*args, **kwargs) # type: ignore[misc]
vllm_omni/platforms/cpu/worker/__init__.py
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
vllm_omni/platforms/cpu/worker/cpu_ar_model_runner.py
+62
View File
@@ -0,0 +1,62 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""CPU model runners for Omni AR and generation stages.
MRO: ``CPUARModelRunner -> GPUARModelRunner -> OmniGPUModelRunner -> (omni
mixins) -> CPUModelRunner -> GPUModelRunner``. ``CPUModelRunner.__init__``
runs vLLM's CPU shims (torch.cuda placeholders, ``_postprocess_tensors``,
``_postprocess_triton``) around ``GPUModelRunner.__init__``; the Omni layers
then add their buffers on the CPU device. Nothing CUDA-graph related can be
reached because ``CpuPlatform.check_and_update_config`` forces
``cudagraph_capture_sizes=[]`` and ``async_scheduling=False``.
"""
from __future__ import annotations
from vllm.logger import init_logger
from vllm.v1.worker.cpu_model_runner import CPUModelRunner
from vllm_omni.worker.gpu_ar_model_runner import GPUARModelRunner
from vllm_omni.worker.gpu_generation_model_runner import GPUGenerationModelRunner
logger = init_logger(__name__)
class _CPUOmniRunnerMixin:
"""Overrides shared by the CPU AR and generation runners."""
def _assert_cpu_invariants(self) -> None:
if getattr(self, "use_async_scheduling", False):
raise RuntimeError(
"CPU Omni runners require async_scheduling=False (vLLM's CpuPlatform forces it); "
"the async omni-output path uses CUDA streams."
)
def _init_device_properties(self) -> None:
self.num_sms = None
def _sync_device(self) -> None:
return None
def capture_model(self) -> int:
# No graphs on CPU; keep the call cheap and explicit.
return 0
def _capture_talker_mtp_graphs(self) -> None:
return None
class CPUARModelRunner(_CPUOmniRunnerMixin, GPUARModelRunner, CPUModelRunner):
"""AR (talker / thinker) runner on CPU."""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._assert_cpu_invariants()
class CPUGenerationModelRunner(_CPUOmniRunnerMixin, GPUGenerationModelRunner, CPUModelRunner):
"""Generation (code2wav / decoder) runner on CPU."""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._assert_cpu_invariants()
vllm_omni/platforms/cpu/worker/cpu_ar_worker.py
+63
View File
@@ -0,0 +1,63 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""CPU workers for Omni AR and generation stages (XPU-worker pattern).
``CPUWorker`` owns the CPU specifics (NUMA binding, ``init_cpu_memory_env``,
gloo, NUMA-based ``determine_available_memory``, no cudagraph warm-up); the
Omni mixin only loads omni plugins in the worker process. Sleep/wake are not
supported on CPU and return an error ACK instead of raising in the RPC loop.
"""
from __future__ import annotations
import vllm.v1.worker.cpu.shm # noqa: F401 # patches torch.Event/torch.cuda.Stream; must be first
from vllm.logger import init_logger
from vllm.v1.worker.cpu_worker import CPUWorker
from vllm_omni.diffusion.data import OmniACK
from vllm_omni.engine.init_timeline import timed_phase
from vllm_omni.platforms.cpu.worker.cpu_ar_model_runner import CPUARModelRunner, CPUGenerationModelRunner
from vllm_omni.worker.mixins import OmniWorkerMixin
logger = init_logger(__name__)
class _CPUOmniWorkerMixin:
model_runner_cls: type
@timed_phase("init_device")
def init_device(self) -> None:
super().init_device() # type: ignore[misc]
if getattr(self, "use_v2_model_runner", False):
logger.warning("OMNI CPU worker forces v1 model runner for omni hooks.")
self.use_v2_model_runner = False
# CPUWorker.init_device built a plain CPUModelRunner; replace it with the omni runner.
self.model_runner = self.model_runner_cls(self.vllm_config, self.device)
@staticmethod
def _task_id(task) -> str:
if isinstance(task, dict):
return str(task.get("task_id", ""))
return str(getattr(task, "task_id", ""))
def handle_sleep_task(self, task) -> OmniACK:
return OmniACK(
task_id=self._task_id(task), status="ERROR", error_msg="sleep mode is not supported on the CPU OmniPlatform"
)
def handle_wake_task(self, task) -> OmniACK:
return OmniACK(
task_id=self._task_id(task), status="ERROR", error_msg="wake-up is not supported on the CPU OmniPlatform"
)
class CPUARWorker(_CPUOmniWorkerMixin, OmniWorkerMixin, CPUWorker):
"""CPU AR worker for thinker / talker stages."""
model_runner_cls = CPUARModelRunner
class CPUGenerationWorker(_CPUOmniWorkerMixin, OmniWorkerMixin, CPUWorker):
"""CPU generation worker for decoder / code2wav stages."""
model_runner_cls = CPUGenerationModelRunner
vllm_omni/platforms/cpu/worker/cpu_generation_worker.py
+7
View File
@@ -0,0 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Import-path shim: the CPU generation worker lives next to the AR worker."""
from vllm_omni.platforms.cpu.worker.cpu_ar_worker import CPUGenerationWorker
__all__ = ["CPUGenerationWorker"]
vllm_omni/platforms/interface.py
+4
View File
@@ -24,6 +24,7 @@ class OmniPlatformEnum(Enum):
NPU = "npu"
XPU = "xpu"
MUSA = "musa"
CPU = "cpu"
OOT = "oot"
UNSPECIFIED = "unspecified"
@@ -54,6 +55,9 @@ class OmniPlatform(Platform):
def is_musa(self) -> bool:
return self._omni_enum == OmniPlatformEnum.MUSA
def is_cpu(self) -> bool:
return self._omni_enum == OmniPlatformEnum.CPU
def is_out_of_tree(self) -> bool:
return self._omni_enum == OmniPlatformEnum.OOT