diff --git a/benchmarks/tts/edge_results/stream_latency/Qwen3-TTS-12Hz-0.6B-CustomVoice/default/20260909-105334_cpu_smoke.json b/benchmarks/tts/edge_results/stream_latency/Qwen3-TTS-12Hz-0.6B-CustomVoice/default/20260909-105334_cpu_smoke.json new file mode 100644 index 000000000..a2f756e01 --- /dev/null +++ b/benchmarks/tts/edge_results/stream_latency/Qwen3-TTS-12Hz-0.6B-CustomVoice/default/20260909-105334_cpu_smoke.json @@ -0,0 +1,867 @@ +{ + "schema": 1, + "meta": { + "model": "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice", + "deploy_config": null, + "query_type": "CustomVoice", + "txt_prompts": "benchmarks/tts/prompts_12.txt", + "git_sha": "f804bbd6", + "gpu_name": "NVIDIA L20X", + "cuda_visible_devices": null, + "timestamp": "20260909-105334", + "argv": [ + "--query-type", + "CustomVoice", + "--txt-prompts", + "benchmarks/tts/prompts_12.txt", + "--init-timeout", + "1500", + "--stage-init-timeout", + "1200" + ], + "bench_args": { + "model": "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice", + "warmup": 1, + "repeat": 1, + "init_only": false, + "tag": "cpu_smoke", + "output_dir": "benchmarks/tts/edge_results/stream_latency", + "save_wav": false, + "gpu_poll_interval_s": 0.2, + "parallel_stage_init": false, + "deploy_profile": null, + "bench_help": false + }, + "warmup": 1, + "repeat": 1, + "init_only": false + }, + "init": { + "init_s": 185.0914630489424, + "init_timeline": { + "schema": 1, + "t0": 1788922042.76263, + "t_end": 1788922223.6046276, + "total_s": 180.8419976234436, + "phases": [ + { + "name": "stages_init", + "start": 1788922042.7633224, + "end": 1788922223.6009316, + "stage_id": null, + "replica_id": 0, + "pid": 2543391, + "role": "engine", + "extra": {}, + "dur_s": 180.83760929107666 + }, + { + "name": "g3_device_lock", + "start": 1788922106.0638428, + "end": 1788922106.0639107, + "stage_id": 0, + "replica_id": 0, + "pid": 2543391, + "role": "engine", + "extra": {}, + "dur_s": 6.794929504394531e-05 + }, + { + "name": "g2_launch_lock_wait", + "start": 1788922106.0639355, + "end": 1788922106.063938, + "stage_id": 0, + "replica_id": 0, + "pid": 2543391, + "role": "engine", + "extra": { + "acquired": true + }, + "dur_s": 2.384185791015625e-06 + }, + { + "name": "g2_spawn_and_ready", + "start": 1788922106.0640821, + "end": 1788922164.965892, + "stage_id": 0, + "replica_id": 0, + "pid": 2543391, + "role": "engine", + "extra": {}, + "dur_s": 58.90180993080139 + }, + { + "name": "proc_entry", + "start": 1788922130.877344, + "end": 1788922130.877344, + "stage_id": 0, + "replica_id": 0, + "pid": 2564074, + "role": "worker", + "extra": {}, + "dur_s": 0.0 + }, + { + "name": "engine_core_init", + "start": 1788922130.8864088, + "end": 1788922164.9667127, + "stage_id": 0, + "replica_id": 0, + "pid": 2564074, + "role": "worker", + "extra": {}, + "dur_s": 34.08030390739441 + }, + { + "name": "init_device", + "start": 1788922159.3280494, + "end": 1788922159.5362585, + "stage_id": 0, + "replica_id": 0, + "pid": 2567908, + "role": "worker", + "extra": {}, + "dur_s": 0.20820903778076172 + }, + { + "name": "load_model", + "start": 1788922159.5429327, + "end": 1788922164.054677, + "stage_id": 0, + "replica_id": 0, + "pid": 2567908, + "role": "worker", + "extra": {}, + "dur_s": 4.511744260787964 + }, + { + "name": "load_weights", + "start": 1788922159.5429373, + "end": 1788922164.0436099, + "stage_id": 0, + "replica_id": 0, + "pid": 2567908, + "role": "worker", + "extra": {}, + "dur_s": 4.5006725788116455 + }, + { + "name": "talker_mtp_init", + "start": 1788922164.0478003, + "end": 1788922164.0480099, + "stage_id": 0, + "replica_id": 0, + "pid": 2567908, + "role": "worker", + "extra": {}, + "dur_s": 0.00020956993103027344 + }, + { + "name": "attn_prewarm", + "start": 1788922164.0528018, + "end": 1788922164.0528057, + "stage_id": 0, + "replica_id": 0, + "pid": 2567908, + "role": "worker", + "extra": {}, + "dur_s": 3.814697265625e-06 + }, + { + "name": "g3_device_lock", + "start": 1788922165.1493986, + "end": 1788922165.1495192, + "stage_id": 1, + "replica_id": 0, + "pid": 2543391, + "role": "engine", + "extra": {}, + "dur_s": 0.00012063980102539062 + }, + { + "name": "g2_launch_lock_wait", + "start": 1788922165.149549, + "end": 1788922165.1495519, + "stage_id": 1, + "replica_id": 0, + "pid": 2543391, + "role": "engine", + "extra": { + "acquired": true + }, + "dur_s": 2.86102294921875e-06 + }, + { + "name": "g2_spawn_and_ready", + "start": 1788922165.149846, + "end": 1788922220.5601192, + "stage_id": 1, + "replica_id": 0, + "pid": 2543391, + "role": "engine", + "extra": {}, + "dur_s": 55.41027307510376 + }, + { + "name": "proc_entry", + "start": 1788922191.4990642, + "end": 1788922191.4990642, + "stage_id": 1, + "replica_id": 0, + "pid": 2572020, + "role": "worker", + "extra": {}, + "dur_s": 0.0 + }, + { + "name": "engine_core_init", + "start": 1788922191.50477, + "end": 1788922220.5615401, + "stage_id": 1, + "replica_id": 0, + "pid": 2572020, + "role": "worker", + "extra": {}, + "dur_s": 29.056770086288452 + }, + { + "name": "init_device", + "start": 1788922218.1614058, + "end": 1788922218.3837707, + "stage_id": 1, + "replica_id": 0, + "pid": 2576162, + "role": "worker", + "extra": {}, + "dur_s": 0.2223649024963379 + }, + { + "name": "load_model", + "start": 1788922218.3888378, + "end": 1788922219.7271593, + "stage_id": 1, + "replica_id": 0, + "pid": 2576162, + "role": "worker", + "extra": {}, + "dur_s": 1.3383214473724365 + }, + { + "name": "load_weights", + "start": 1788922218.3888428, + "end": 1788922219.721446, + "stage_id": 1, + "replica_id": 0, + "pid": 2576162, + "role": "worker", + "extra": {}, + "dur_s": 1.3326032161712646 + }, + { + "name": "talker_mtp_init", + "start": 1788922219.7234585, + "end": 1788922219.7234626, + "stage_id": 1, + "replica_id": 0, + "pid": 2576162, + "role": "worker", + "extra": {}, + "dur_s": 4.0531158447265625e-06 + }, + { + "name": "attn_prewarm", + "start": 1788922219.7253046, + "end": 1788922219.725309, + "stage_id": 1, + "replica_id": 0, + "pid": 2576162, + "role": "worker", + "extra": {}, + "dur_s": 4.291534423828125e-06 + } + ] + } + }, + "resources": { + "peak_rss_bytes": 14409445376, + "peak_gpu_mem_mib": 95959, + "gpu_mem_samples": 379, + "gpu_mem_mode": "whole-gpu" + }, + "requests": [ + { + "request_id": "w0", + "sr": 24000, + "chunks": [ + { + "idx": 0, + "t_ms": 9093.21565227583, + "gap_ms": 9093.21565227583, + "samples": 1920 + }, + { + "idx": 1, + "t_ms": 12637.973092030734, + "gap_ms": 3544.7574397549033, + "samples": 48000 + }, + { + "idx": 2, + "t_ms": 15888.486401177943, + "gap_ms": 3250.513309147209, + "samples": 48000 + } + ], + "t_final_ms": 16496.613026130944, + "ttfa_ms": 9093.21565227583, + "playback_start_ms": 13808.486401177943, + "stall_at_ttfa_ms": 4715.270748902112, + "n_chunks": 3, + "streamed_audio_s": 4.08, + "tail_audio_s": 0.16, + "total_wall_s": 16.496613026130944, + "rtf_streamed": 3.8942368630338096, + "rtf_total": 3.8907106193705054, + "gap_ms": [ + 3544.7574397549033, + 3250.513309147209 + ], + "text": "Hello, welcome to the voice synthesis benchmark test.", + "warmup": true + }, + { + "request_id": "r0_0", + "sr": 24000, + "chunks": [ + { + "idx": 0, + "t_ms": 291.6870419867337, + "gap_ms": 291.6870419867337, + "samples": 1920 + }, + { + "idx": 1, + "t_ms": 3767.898346297443, + "gap_ms": 3476.2113043107092, + "samples": 48000 + }, + { + "idx": 2, + "t_ms": 7128.282887395471, + "gap_ms": 3360.3845410980284, + "samples": 48000 + } + ], + "t_final_ms": 8370.939870364964, + "ttfa_ms": 291.6870419867337, + "playback_start_ms": 5048.282887395471, + "stall_at_ttfa_ms": 4756.595845408738, + "n_chunks": 3, + "streamed_audio_s": 4.08, + "tail_audio_s": 0.8, + "total_wall_s": 8.370939870364964, + "rtf_streamed": 1.7471281586753606, + "rtf_total": 1.7153565308124927, + "gap_ms": [ + 3476.2113043107092, + 3360.3845410980284 + ], + "text": "Hello, welcome to the voice synthesis benchmark test.", + "warmup": false + }, + { + "request_id": "r0_1", + "sr": 24000, + "chunks": [ + { + "idx": 0, + "t_ms": 289.57725409418344, + "gap_ms": 289.57725409418344, + "samples": 1920 + }, + { + "idx": 1, + "t_ms": 3722.4736879579723, + "gap_ms": 3432.896433863789, + "samples": 48000 + }, + { + "idx": 2, + "t_ms": 6917.577911168337, + "gap_ms": 3195.1042232103646, + "samples": 48000 + } + ], + "t_final_ms": 7475.041428115219, + "ttfa_ms": 289.57725409418344, + "playback_start_ms": 4837.577911168337, + "stall_at_ttfa_ms": 4548.000657074153, + "n_chunks": 3, + "streamed_audio_s": 4.08, + "tail_audio_s": 0.08, + "total_wall_s": 7.475041428115219, + "rtf_streamed": 1.6954847821491021, + "rtf_total": 1.796884958681543, + "gap_ms": [ + 3432.896433863789, + 3195.1042232103646 + ], + "text": "She said she would be here by noon, but nobody showed up.", + "warmup": false + }, + { + "request_id": "r0_2", + "sr": 24000, + "chunks": [ + { + "idx": 0, + "t_ms": 273.937676101923, + "gap_ms": 273.937676101923, + "samples": 1920 + }, + { + "idx": 1, + "t_ms": 3683.945945929736, + "gap_ms": 3410.008269827813, + "samples": 48000 + }, + { + "idx": 2, + "t_ms": 7284.630519337952, + "gap_ms": 3600.6845734082162, + "samples": 48000 + } + ], + "t_final_ms": 7858.37772814557, + "ttfa_ms": 273.937676101923, + "playback_start_ms": 5204.630519337952, + "stall_at_ttfa_ms": 4930.692843236029, + "n_chunks": 3, + "streamed_audio_s": 4.08, + "tail_audio_s": 0.32, + "total_wall_s": 7.85837772814557, + "rtf_streamed": 1.7854486567004784, + "rtf_total": 1.785994938214902, + "gap_ms": [ + 3410.008269827813, + 3600.6845734082162 + ], + "text": "The quick brown fox jumps over the lazy dog near the riverbank.", + "warmup": false + }, + { + "request_id": "r0_3", + "sr": 24000, + "chunks": [ + { + "idx": 0, + "t_ms": 291.9175820425153, + "gap_ms": 291.9175820425153, + "samples": 1920 + }, + { + "idx": 1, + "t_ms": 3912.9916448146105, + "gap_ms": 3621.074062772095, + "samples": 48000 + }, + { + "idx": 2, + "t_ms": 7129.715716000646, + "gap_ms": 3216.724071186036, + "samples": 48000 + } + ], + "t_final_ms": 9064.493948128074, + "ttfa_ms": 291.9175820425153, + "playback_start_ms": 5049.715716000646, + "stall_at_ttfa_ms": 4757.798133958131, + "n_chunks": 3, + "streamed_audio_s": 4.08, + "tail_audio_s": 1.04, + "total_wall_s": 9.064493948128074, + "rtf_streamed": 1.7474793421570212, + "rtf_total": 1.7704089742437645, + "gap_ms": [ + 3621.074062772095, + 3216.724071186036 + ], + "text": "I can't believe how beautiful the sunset looks from up here on the mountain.", + "warmup": false + }, + { + "request_id": "r0_4", + "sr": 24000, + "chunks": [ + { + "idx": 0, + "t_ms": 285.0063256919384, + "gap_ms": 285.0063256919384, + "samples": 1920 + }, + { + "idx": 1, + "t_ms": 3749.287095852196, + "gap_ms": 3464.280770160258, + "samples": 48000 + }, + { + "idx": 2, + "t_ms": 7012.798972893506, + "gap_ms": 3263.51187704131, + "samples": 48000 + } + ], + "t_final_ms": 8748.178598005325, + "ttfa_ms": 285.0063256919384, + "playback_start_ms": 4932.798972893506, + "stall_at_ttfa_ms": 4647.792647201568, + "n_chunks": 3, + "streamed_audio_s": 4.08, + "tail_audio_s": 1.12, + "total_wall_s": 8.748178598005325, + "rtf_streamed": 1.718823277669977, + "rtf_total": 1.682342038077947, + "gap_ms": [ + 3464.280770160258, + 3263.51187704131 + ], + "text": "Please remember to bring your identification documents to the appointment tomorrow morning.", + "warmup": false + }, + { + "request_id": "r0_5", + "sr": 24000, + "chunks": [ + { + "idx": 0, + "t_ms": 275.7884138263762, + "gap_ms": 275.7884138263762, + "samples": 1920 + }, + { + "idx": 1, + "t_ms": 3673.660270869732, + "gap_ms": 3397.8718570433557, + "samples": 48000 + }, + { + "idx": 2, + "t_ms": 12653.649115003645, + "gap_ms": 8979.988844133914, + "samples": 48000 + } + ], + "t_final_ms": 75011.55206793919, + "ttfa_ms": 275.7884138263762, + "playback_start_ms": 10573.649115003645, + "stall_at_ttfa_ms": 10297.86070117727, + "n_chunks": 3, + "streamed_audio_s": 4.08, + "tail_audio_s": 1.68, + "total_wall_s": 75.01155206793919, + "rtf_streamed": 3.1013845870106973, + "rtf_total": 13.022838900683888, + "gap_ms": [ + 3397.8718570433557, + 8979.988844133914 + ], + "text": "Have you ever wondered what it would be like to travel through time and visit ancient civilizations?", + "warmup": false + }, + { + "request_id": "r0_6", + "sr": 24000, + "chunks": [ + { + "idx": 0, + "t_ms": 290.9621219150722, + "gap_ms": 290.9621219150722, + "samples": 1920 + }, + { + "idx": 1, + "t_ms": 3827.7830346487463, + "gap_ms": 3536.820912733674, + "samples": 48000 + }, + { + "idx": 2, + "t_ms": 6955.3797277621925, + "gap_ms": 3127.5966931134462, + "samples": 48000 + } + ], + "t_final_ms": 9968.388166744262, + "ttfa_ms": 290.9621219150722, + "playback_start_ms": 4875.3797277621925, + "stall_at_ttfa_ms": 4584.41760584712, + "n_chunks": 3, + "streamed_audio_s": 4.08, + "tail_audio_s": 2.0, + "total_wall_s": 9.968388166744262, + "rtf_streamed": 1.704749933275047, + "rtf_total": 1.639537527425043, + "gap_ms": [ + 3536.820912733674, + 3127.5966931134462 + ], + "text": "The restaurant on the corner serves the best pasta I have ever tasted in my entire life.", + "warmup": false + }, + { + "request_id": "r0_7", + "sr": 24000, + "chunks": [ + { + "idx": 0, + "t_ms": 301.0973362252116, + "gap_ms": 301.0973362252116, + "samples": 1920 + }, + { + "idx": 1, + "t_ms": 3678.3662801608443, + "gap_ms": 3377.2689439356327, + "samples": 48000 + }, + { + "idx": 2, + "t_ms": 6857.406801078469, + "gap_ms": 3179.0405209176242, + "samples": 48000 + } + ], + "t_final_ms": 9085.033813957125, + "ttfa_ms": 301.0973362252116, + "playback_start_ms": 4777.406801078469, + "stall_at_ttfa_ms": 4476.309464853257, + "n_chunks": 3, + "streamed_audio_s": 4.08, + "tail_audio_s": 1.36, + "total_wall_s": 9.085033813957125, + "rtf_streamed": 1.6807369610486442, + "rtf_total": 1.6700429805068244, + "gap_ms": [ + 3377.2689439356327, + 3179.0405209176242 + ], + "text": "After the meeting, we should discuss the quarterly results and plan for the next phase.", + "warmup": false + }, + { + "request_id": "r0_8", + "sr": 24000, + "chunks": [ + { + "idx": 0, + "t_ms": 294.4919569417834, + "gap_ms": 294.4919569417834, + "samples": 1920 + }, + { + "idx": 1, + "t_ms": 3713.8455058448017, + "gap_ms": 3419.3535489030182, + "samples": 48000 + }, + { + "idx": 2, + "t_ms": 6958.719541784376, + "gap_ms": 3244.8740359395742, + "samples": 48000 + }, + { + "idx": 3, + "t_ms": 9996.23750988394, + "gap_ms": 3037.5179680995643, + "samples": 48000 + } + ], + "t_final_ms": 10559.09016681835, + "ttfa_ms": 294.4919569417834, + "playback_start_ms": 5916.23750988394, + "stall_at_ttfa_ms": 5621.745552942157, + "n_chunks": 4, + "streamed_audio_s": 6.08, + "tail_audio_s": 0.16, + "total_wall_s": 10.55909016681835, + "rtf_streamed": 1.644118011494069, + "rtf_total": 1.692161885708069, + "gap_ms": [ + 3419.3535489030182, + 3244.8740359395742, + 3037.5179680995643 + ], + "text": "Learning a new language takes patience, practice, and a genuine curiosity about other cultures.", + "warmup": false + }, + { + "request_id": "r0_9", + "sr": 24000, + "chunks": [ + { + "idx": 0, + "t_ms": 316.74008816480637, + "gap_ms": 316.74008816480637, + "samples": 1920 + }, + { + "idx": 1, + "t_ms": 3735.6491410173476, + "gap_ms": 3418.909052852541, + "samples": 48000 + }, + { + "idx": 2, + "t_ms": 7355.19207501784, + "gap_ms": 3619.542934000492, + "samples": 48000 + } + ], + "t_final_ms": 10337.802770081908, + "ttfa_ms": 316.74008816480637, + "playback_start_ms": 5275.19207501784, + "stall_at_ttfa_ms": 4958.451986853033, + "n_chunks": 3, + "streamed_audio_s": 4.08, + "tail_audio_s": 2.0, + "total_wall_s": 10.337802770081908, + "rtf_streamed": 1.8027431556416273, + "rtf_total": 1.7002965082371557, + "gap_ms": [ + 3418.909052852541, + 3619.542934000492 + ], + "text": "The train leaves at half past seven, so we need to arrive at the station before then.", + "warmup": false + }, + { + "request_id": "r0_10", + "sr": 24000, + "chunks": [ + { + "idx": 0, + "t_ms": 310.7302547432482, + "gap_ms": 310.7302547432482, + "samples": 1920 + }, + { + "idx": 1, + "t_ms": 3758.4262755699456, + "gap_ms": 3447.6960208266973, + "samples": 48000 + }, + { + "idx": 2, + "t_ms": 6996.824599802494, + "gap_ms": 3238.3983242325485, + "samples": 48000 + } + ], + "t_final_ms": 8471.956291701645, + "ttfa_ms": 310.7302547432482, + "playback_start_ms": 4916.824599802494, + "stall_at_ttfa_ms": 4606.094345059246, + "n_chunks": 3, + "streamed_audio_s": 4.08, + "tail_audio_s": 0.96, + "total_wall_s": 8.471956291701645, + "rtf_streamed": 1.71490799014767, + "rtf_total": 1.6809437086709613, + "gap_ms": [ + 3447.6960208266973, + 3238.3983242325485 + ], + "text": "Could you please turn down the music a little bit, I'm trying to concentrate on my work.", + "warmup": false + }, + { + "request_id": "r0_11", + "sr": 24000, + "chunks": [ + { + "idx": 0, + "t_ms": 296.68450681492686, + "gap_ms": 296.68450681492686, + "samples": 1920 + }, + { + "idx": 1, + "t_ms": 3681.493826210499, + "gap_ms": 3384.809319395572, + "samples": 48000 + }, + { + "idx": 2, + "t_ms": 7026.184834074229, + "gap_ms": 3344.69100786373, + "samples": 48000 + } + ], + "t_final_ms": 9525.669381022453, + "ttfa_ms": 296.68450681492686, + "playback_start_ms": 4946.184834074229, + "stall_at_ttfa_ms": 4649.500327259302, + "n_chunks": 3, + "streamed_audio_s": 4.08, + "tail_audio_s": 1.6, + "total_wall_s": 9.525669381022453, + "rtf_streamed": 1.7221041259985854, + "rtf_total": 1.6770544684898687, + "gap_ms": [ + 3384.809319395572, + 3344.69100786373 + ], + "text": "It was a dark and stormy night when the old lighthouse keeper heard a knock at the door.", + "warmup": false + } + ], + "summary": { + "n_requests": 12, + "ttfa_ms": { + "n": 12, + "p50": 291.6870419867337, + "p90": 310.7302547432482, + "max": 316.74008816480637, + "mean": 293.2183798790599 + }, + "playback_start_ms": { + "n": 12, + "p50": 4946.184834074229, + "p90": 5916.23750988394, + "max": 10573.649115003645, + "mean": 5529.490055784893 + }, + "stall_at_ttfa_ms": { + "n": 12, + "p50": 4649.500327259302, + "p90": 5621.745552942157, + "max": 10297.86070117727, + "mean": 5236.271675905834 + }, + "rtf_streamed": { + "n": 12, + "p50": 1.718823277669977, + "p90": 1.8027431556416273, + "max": 3.1013845870106973, + "mean": 1.83875908183069 + }, + "rtf_total": { + "n": 12, + "p50": 1.692161885708069, + "p90": 1.796884958681543, + "max": 13.022838900683888, + "mean": 2.6528219516460383 + }, + "gap_ms": { + "n": 25, + "p50": 3397.8718570433557, + "p90": 3619.542934000492, + "max": 8979.988844133914, + "mean": 3591.8104044348 + }, + "total_audio_s": 50.96, + "total_wall_s": 174.4765242310241 + } +} \ No newline at end of file diff --git a/benchmarks/tts/edge_results/stream_latency/timelines/0.6B_cpu_smoke.json b/benchmarks/tts/edge_results/stream_latency/timelines/0.6B_cpu_smoke.json new file mode 100644 index 000000000..b696ba1d0 --- /dev/null +++ b/benchmarks/tts/edge_results/stream_latency/timelines/0.6B_cpu_smoke.json @@ -0,0 +1,243 @@ +{ + "schema": 1, + "t0": 1788922042.76263, + "t_end": 1788922223.6046276, + "total_s": 180.8419976234436, + "phases": [ + { + "name": "stages_init", + "start": 1788922042.7633224, + "end": 1788922223.6009316, + "stage_id": null, + "replica_id": 0, + "pid": 2543391, + "role": "engine", + "extra": {}, + "dur_s": 180.83760929107666 + }, + { + "name": "g3_device_lock", + "start": 1788922106.0638428, + "end": 1788922106.0639107, + "stage_id": 0, + "replica_id": 0, + "pid": 2543391, + "role": "engine", + "extra": {}, + "dur_s": 6.794929504394531e-05 + }, + { + "name": "g2_launch_lock_wait", + "start": 1788922106.0639355, + "end": 1788922106.063938, + "stage_id": 0, + "replica_id": 0, + "pid": 2543391, + "role": "engine", + "extra": { + "acquired": true + }, + "dur_s": 2.384185791015625e-06 + }, + { + "name": "g2_spawn_and_ready", + "start": 1788922106.0640821, + "end": 1788922164.965892, + "stage_id": 0, + "replica_id": 0, + "pid": 2543391, + "role": "engine", + "extra": {}, + "dur_s": 58.90180993080139 + }, + { + "name": "proc_entry", + "start": 1788922130.877344, + "end": 1788922130.877344, + "stage_id": 0, + "replica_id": 0, + "pid": 2564074, + "role": "worker", + "extra": {}, + "dur_s": 0.0 + }, + { + "name": "engine_core_init", + "start": 1788922130.8864088, + "end": 1788922164.9667127, + "stage_id": 0, + "replica_id": 0, + "pid": 2564074, + "role": "worker", + "extra": {}, + "dur_s": 34.08030390739441 + }, + { + "name": "init_device", + "start": 1788922159.3280494, + "end": 1788922159.5362585, + "stage_id": 0, + "replica_id": 0, + "pid": 2567908, + "role": "worker", + "extra": {}, + "dur_s": 0.20820903778076172 + }, + { + "name": "load_model", + "start": 1788922159.5429327, + "end": 1788922164.054677, + "stage_id": 0, + "replica_id": 0, + "pid": 2567908, + "role": "worker", + "extra": {}, + "dur_s": 4.511744260787964 + }, + { + "name": "load_weights", + "start": 1788922159.5429373, + "end": 1788922164.0436099, + "stage_id": 0, + "replica_id": 0, + "pid": 2567908, + "role": "worker", + "extra": {}, + "dur_s": 4.5006725788116455 + }, + { + "name": "talker_mtp_init", + "start": 1788922164.0478003, + "end": 1788922164.0480099, + "stage_id": 0, + "replica_id": 0, + "pid": 2567908, + "role": "worker", + "extra": {}, + "dur_s": 0.00020956993103027344 + }, + { + "name": "attn_prewarm", + "start": 1788922164.0528018, + "end": 1788922164.0528057, + "stage_id": 0, + "replica_id": 0, + "pid": 2567908, + "role": "worker", + "extra": {}, + "dur_s": 3.814697265625e-06 + }, + { + "name": "g3_device_lock", + "start": 1788922165.1493986, + "end": 1788922165.1495192, + "stage_id": 1, + "replica_id": 0, + "pid": 2543391, + "role": "engine", + "extra": {}, + "dur_s": 0.00012063980102539062 + }, + { + "name": "g2_launch_lock_wait", + "start": 1788922165.149549, + "end": 1788922165.1495519, + "stage_id": 1, + "replica_id": 0, + "pid": 2543391, + "role": "engine", + "extra": { + "acquired": true + }, + "dur_s": 2.86102294921875e-06 + }, + { + "name": "g2_spawn_and_ready", + "start": 1788922165.149846, + "end": 1788922220.5601192, + "stage_id": 1, + "replica_id": 0, + "pid": 2543391, + "role": "engine", + "extra": {}, + "dur_s": 55.41027307510376 + }, + { + "name": "proc_entry", + "start": 1788922191.4990642, + "end": 1788922191.4990642, + "stage_id": 1, + "replica_id": 0, + "pid": 2572020, + "role": "worker", + "extra": {}, + "dur_s": 0.0 + }, + { + "name": "engine_core_init", + "start": 1788922191.50477, + "end": 1788922220.5615401, + "stage_id": 1, + "replica_id": 0, + "pid": 2572020, + "role": "worker", + "extra": {}, + "dur_s": 29.056770086288452 + }, + { + "name": "init_device", + "start": 1788922218.1614058, + "end": 1788922218.3837707, + "stage_id": 1, + "replica_id": 0, + "pid": 2576162, + "role": "worker", + "extra": {}, + "dur_s": 0.2223649024963379 + }, + { + "name": "load_model", + "start": 1788922218.3888378, + "end": 1788922219.7271593, + "stage_id": 1, + "replica_id": 0, + "pid": 2576162, + "role": "worker", + "extra": {}, + "dur_s": 1.3383214473724365 + }, + { + "name": "load_weights", + "start": 1788922218.3888428, + "end": 1788922219.721446, + "stage_id": 1, + "replica_id": 0, + "pid": 2576162, + "role": "worker", + "extra": {}, + "dur_s": 1.3326032161712646 + }, + { + "name": "talker_mtp_init", + "start": 1788922219.7234585, + "end": 1788922219.7234626, + "stage_id": 1, + "replica_id": 0, + "pid": 2576162, + "role": "worker", + "extra": {}, + "dur_s": 4.0531158447265625e-06 + }, + { + "name": "attn_prewarm", + "start": 1788922219.7253046, + "end": 1788922219.725309, + "stage_id": 1, + "replica_id": 0, + "pid": 2576162, + "role": "worker", + "extra": {}, + "dur_s": 4.291534423828125e-06 + } + ] +} \ No newline at end of file diff --git a/docs/getting_started/installation/cpu.md b/docs/getting_started/installation/cpu.md new file mode 100644 index 000000000..c995b0656 --- /dev/null +++ b/docs/getting_started/installation/cpu.md @@ -0,0 +1,56 @@ +# CPU + +vLLM-Omni can run **autoregressive and generation stages** (for example the +Qwen3-TTS talker → code2wav pipeline) on vLLM's CPU backend. Diffusion stages +are not supported on CPU. + +## Requirements + +- x86-64 with AVX-512 (bf16 needs `avx512_bf16` or AMX) or aarch64 with NEON + (`fp16`/`bf16` extensions recommended). See the vLLM CPU installation page + for the supported ISA matrix. +- A vLLM **CPU** build (`+cpu` wheel). The CUDA wheel does not contain the + CPU kernels. +- Python 3.10–3.13. + +## Install + +```bash +uv venv --python 3.12 && source .venv/bin/activate +# vLLM CPU wheel (x86-64 example; see vLLM docs for aarch64 and per-commit wheels) +uv pip install --torch-backend cpu \ + https://github.com/vllm-project/vllm/releases/download/v0.28.0/vllm-0.28.0+cpu-cp38-abi3-manylinux_2_34_x86_64.whl +# vLLM-Omni without re-resolving torch, then its CPU requirements +VLLM_TARGET_DEVICE=cpu uv pip install --torch-backend cpu --no-deps -e . +VLLM_TARGET_DEVICE=cpu uv pip install --torch-backend cpu -r requirements/cpu.txt +``` + +## Run + +```bash +export VLLM_TARGET_DEVICE=cpu # selects the CPU OmniPlatform +export LD_PRELOAD=$(python -c "import glob,sys; print(glob.glob(sys.prefix+'/lib/libiomp5.so')[0])") +# one utterance, Qwen3-TTS 0.6B, CPU overlay from vllm_omni/deploy/qwen3_tts.yaml (platforms: cpu:) +python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-type CustomVoice +# or let the hardware probe size everything (threads, KV budget, dtype, chunking): +python benchmarks/tts/stream_latency_bench.py --model Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice \ + --query-type CustomVoice --deploy-profile auto +``` + +What the CPU platform changes (`vllm_omni/platforms/cpu/platform.py`): + +| Item | CPU behaviour | +|---|---| +| Workers | `CPUARWorker` / `CPUGenerationWorker` compose vLLM's `CPUWorker` with the Omni runners | +| Scheduling | `async_scheduling=False` (forced by vLLM's `CpuPlatform`), no CUDA graphs | +| Memory | `gpu_memory_utilization` is a NUMA-node memory fraction; prefer `kv_cache_memory_bytes` | +| Threads | per-stage `VLLM_CPU_OMP_THREADS_BIND` ranges (`platforms: cpu:` overlay or `deploy_profile="auto"`) | +| Compile | code-predictor `torch.compile` off by default; `VLLM_OMNI_CPU_INDUCTOR=1` enables it | +| dtype | bf16 with `avx512_bf16`/AMX or ARM `bf16`, otherwise `float32` (`deploy_profile="auto"`) | +| Sleep mode | not supported (returns an error ACK) | + +## Tests + +```bash +VLLM_TARGET_DEVICE=cpu pytest tests/platforms/test_cpu_platform.py tests/worker/test_cpu_ar_model_runner.py -m "core_model and cpu" +``` diff --git a/requirements/cpu.txt b/requirements/cpu.txt index c7732b1e1..d90b83f30 100644 --- a/requirements/cpu.txt +++ b/requirements/cpu.txt @@ -1,2 +1,6 @@ -r common.txt onnxruntime>=1.23.2 +# CPU OmniPlatform runtime (vllm_omni/platforms/cpu): NUMA/RSS accounting and +# the OpenMP runtime vLLM's CpuPlatform expects in LD_PRELOAD. +psutil +intel-openmp; platform_machine == "x86_64" diff --git a/tests/platforms/test_cpu_platform.py b/tests/platforms/test_cpu_platform.py new file mode 100644 index 000000000..e744d24a9 --- /dev/null +++ b/tests/platforms/test_cpu_platform.py @@ -0,0 +1,91 @@ +"""CPU OmniPlatform: registration, MRO, config forcing (CPU, no accelerator needed).""" + +from __future__ import annotations + +import pytest +import torch +from vllm.platforms.cpu import CpuPlatform +from vllm.platforms.interface import Platform + +import vllm_omni.platforms as omni_platforms +from vllm_omni.platforms.cpu.platform import CPUOmniPlatform +from vllm_omni.platforms.interface import OmniPlatform, OmniPlatformEnum + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_registered_and_selected_when_cpu_target(monkeypatch): + assert "cpu" in omni_platforms.builtin_omni_platform_plugins + monkeypatch.setattr(omni_platforms, "_cpu_target_requested", lambda: True) + assert omni_platforms.cpu_omni_platform_plugin() == omni_platforms._CPU_OMNI_PLATFORM + assert omni_platforms.resolve_current_omni_platform_cls_qualname() == omni_platforms._CPU_OMNI_PLATFORM + monkeypatch.setattr(omni_platforms, "_cpu_target_requested", lambda: False) + assert omni_platforms.cpu_omni_platform_plugin() is None + + +def test_cpu_target_requested_reads_env(monkeypatch): + import vllm.envs as envs + + monkeypatch.setattr(envs, "VLLM_TARGET_DEVICE", "cpu", raising=False) + assert omni_platforms._cpu_target_requested() is True + + +def test_mro_and_attributes(): + mro = CPUOmniPlatform.__mro__ + assert mro.index(OmniPlatform) < mro.index(CpuPlatform) < mro.index(Platform) + p = CPUOmniPlatform() + assert p.is_cpu() and not p.is_cuda() and not p.is_npu() + assert p._omni_enum is OmniPlatformEnum.CPU + assert CPUOmniPlatform.device_name == "cpu" and CPUOmniPlatform.device_type == "cpu" + assert CPUOmniPlatform.dist_backend == "gloo" + assert CPUOmniPlatform.get_device_count() == 1 + assert CPUOmniPlatform.get_torch_device() == torch.device("cpu") + assert CPUOmniPlatform.get_device_version() is None + assert CPUOmniPlatform.synchronize() is None + assert CPUOmniPlatform.get_free_memory() > 0 + avail, total = CPUOmniPlatform.get_device_memory() + assert 0 < avail <= total + assert CPUOmniPlatform.get_device_total_memory() > 0 + assert CPUOmniPlatform.supports_talker_mtp_graph_capture() is False + assert CPUOmniPlatform.supports_torch_inductor() is False + assert CPUOmniPlatform.get_default_stage_config_path() == "vllm_omni/deploy" + assert "cpu_ar_worker.CPUARWorker" in CPUOmniPlatform.get_omni_ar_worker_cls() + assert "CPUGenerationWorker" in CPUOmniPlatform.get_omni_generation_worker_cls() + wrapper_cls = CPUOmniPlatform.get_graph_wrapper_cls() + assert isinstance(wrapper_cls, type) and not isinstance(object(), wrapper_cls) + with pytest.raises(NotImplementedError): + wrapper_cls() + with pytest.raises(NotImplementedError): + CPUOmniPlatform.get_diffusion_worker_cls() + + +def test_inductor_opt_in(monkeypatch): + monkeypatch.setenv("VLLM_OMNI_CPU_INDUCTOR", "1") + assert CPUOmniPlatform.supports_torch_inductor() is True + + +def test_check_and_update_config_forces_cpu_rules(): + from vllm.config import VllmConfig + from vllm.config.device import DeviceConfig + + cfg = VllmConfig(device_config=DeviceConfig(device="cpu")) + cfg.scheduler_config.async_scheduling = True + CPUOmniPlatform.check_and_update_config(cfg) + assert cfg.scheduler_config.async_scheduling is False + assert cfg.compilation_config.cudagraph_capture_sizes == [] + # worker_cls is resolved by the Omni platform (resolve_worker_cls), not here. + assert cfg.cache_config.block_size == 128 + + +def test_deploy_overlay_for_cpu_platform(): + from vllm_omni.config.stage_config import _DEPLOY_DIR, _apply_platform_overrides, load_deploy_config + + deploy = load_deploy_config(_DEPLOY_DIR / "qwen3_tts.yaml") + deploy = _apply_platform_overrides(deploy, platform="cpu") + s0, s1 = deploy.stages + assert s0.devices == "cpu" and s1.devices == "cpu" + assert s0.enforce_eager is True and s1.enforce_eager is True + assert s0.async_scheduling is False + assert s0.engine_extras["dtype"] == "bfloat16" + assert s0.engine_extras["kv_cache_memory_bytes"] == 4 * 2**30 + assert s0.env["VLLM_CPU_OMP_THREADS_BIND"] != s1.env["VLLM_CPU_OMP_THREADS_BIND"] diff --git a/tests/worker/test_cpu_ar_model_runner.py b/tests/worker/test_cpu_ar_model_runner.py new file mode 100644 index 000000000..b598f03b7 --- /dev/null +++ b/tests/worker/test_cpu_ar_model_runner.py @@ -0,0 +1,74 @@ +"""CPU Omni runners/workers: MRO and override resolution (import-only, CPU).""" + +from __future__ import annotations + +import pytest +from vllm.v1.worker.cpu_model_runner import CPUModelRunner +from vllm.v1.worker.cpu_worker import CPUWorker +from vllm.v1.worker.gpu_model_runner import GPUModelRunner + +from vllm_omni.platforms.cpu.worker.cpu_ar_model_runner import ( + CPUARModelRunner, + CPUGenerationModelRunner, + _CPUOmniRunnerMixin, +) +from vllm_omni.platforms.cpu.worker.cpu_ar_worker import CPUARWorker, CPUGenerationWorker +from vllm_omni.worker.gpu_ar_model_runner import GPUARModelRunner +from vllm_omni.worker.gpu_generation_model_runner import GPUGenerationModelRunner +from vllm_omni.worker.mixins import OmniWorkerMixin +from vllm_omni.worker.omni_connector_model_runner_mixin import OmniConnectorModelRunnerMixin + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_ar_runner_mro(): + mro = CPUARModelRunner.__mro__ + assert ( + mro.index(_CPUOmniRunnerMixin) + < mro.index(GPUARModelRunner) + < mro.index(CPUModelRunner) + < mro.index(GPUModelRunner) + ) + # CPU overrides win over the GPU runner implementations + assert CPUARModelRunner._init_device_properties is _CPUOmniRunnerMixin._init_device_properties + assert CPUARModelRunner._sync_device is _CPUOmniRunnerMixin._sync_device + assert CPUARModelRunner.capture_model is _CPUOmniRunnerMixin.capture_model + assert CPUARModelRunner._capture_talker_mtp_graphs is _CPUOmniRunnerMixin._capture_talker_mtp_graphs + # vLLM CPU shims (triton/tensor post-processing, warm-up) are reachable + assert CPUARModelRunner.warming_up_model is CPUModelRunner.warming_up_model + assert CPUARModelRunner._postprocess_triton is CPUModelRunner._postprocess_triton + # omni connector mixin is present for the worker-class validation + assert issubclass(CPUARModelRunner, OmniConnectorModelRunnerMixin) + + +def test_generation_runner_mro(): + mro = CPUGenerationModelRunner.__mro__ + assert mro.index(GPUGenerationModelRunner) < mro.index(CPUModelRunner) + assert CPUGenerationModelRunner.capture_model is _CPUOmniRunnerMixin.capture_model + + +def test_workers_compose_cpu_worker_and_omni_mixin(): + for worker_cls, runner_cls in ((CPUARWorker, CPUARModelRunner), (CPUGenerationWorker, CPUGenerationModelRunner)): + assert issubclass(worker_cls, CPUWorker) and issubclass(worker_cls, OmniWorkerMixin) + assert worker_cls.model_runner_cls is runner_cls + assert hasattr(worker_cls, "handle_sleep_task") and hasattr(worker_cls, "handle_wake_task") + + +def test_worker_cls_validation_accepts_cpu_workers(): + from vllm_omni.worker.omni_connector_validation import validate_worker_omni_connector + + validate_worker_omni_connector("vllm_omni.platforms.cpu.worker.cpu_ar_worker.CPUARWorker", required=True) + validate_worker_omni_connector( + "vllm_omni.platforms.cpu.worker.cpu_generation_worker.CPUGenerationWorker", required=True + ) + + +def test_sleep_and_wake_return_error_acks(): + class _Task: + task_id = "t1" + level = 1 + + ack = CPUARWorker.handle_sleep_task(object.__new__(CPUARWorker), _Task()) + assert ack.status == "ERROR" and ack.task_id == "t1" + ack = CPUARWorker.handle_wake_task(object.__new__(CPUARWorker), _Task()) + assert ack.status == "ERROR" diff --git a/vllm_omni/deploy/qwen3_tts.yaml b/vllm_omni/deploy/qwen3_tts.yaml index 747b68852..d7cc489e2 100644 --- a/vllm_omni/deploy/qwen3_tts.yaml +++ b/vllm_omni/deploy/qwen3_tts.yaml @@ -136,3 +136,41 @@ platforms: - stage_id: 1 compilation_config: cudagraph_mode: PIECEWISE + # CPU OmniPlatform (VLLM_TARGET_DEVICE=cpu or a +cpu vLLM wheel). vLLM's + # CpuPlatform already forces async_scheduling=False, no CUDA graphs and + # CPU_ATTN; the values below size the two stages for a CPU box. Nested + # layout: engine args under ``engine_args``, placement under ``runtime``. + # gpu_memory_utilization is read by the CPU worker as a NUMA-node memory + # fraction, so an absolute kv_cache_memory_bytes is set instead. + # Thread ranges are per-stage (disjoint) so the two workers do not pin the + # same cores; deploy_profile="auto" derives them from the probed topology. + cpu: + stages: + - stage_id: 0 + engine_args: + enforce_eager: true + async_scheduling: false + max_num_seqs: 2 + max_num_batched_tokens: 2048 + max_model_len: 2048 + gpu_memory_utilization: 0.05 + dtype: bfloat16 + kv_cache_memory_bytes: 4294967296 + runtime: + devices: "cpu" + env: + VLLM_CPU_OMP_THREADS_BIND: "0-11" + - stage_id: 1 + engine_args: + enforce_eager: true + async_scheduling: false + max_num_seqs: 2 + max_num_batched_tokens: 8192 + max_model_len: 8192 + gpu_memory_utilization: 0.05 + dtype: bfloat16 + kv_cache_memory_bytes: 2147483648 + runtime: + devices: "cpu" + env: + VLLM_CPU_OMP_THREADS_BIND: "12-15" diff --git a/vllm_omni/model_executor/models/qwen3_tts/prompt_embeds_builder.py b/vllm_omni/model_executor/models/qwen3_tts/prompt_embeds_builder.py index ddb55cd51..1cf97da4f 100644 --- a/vllm_omni/model_executor/models/qwen3_tts/prompt_embeds_builder.py +++ b/vllm_omni/model_executor/models/qwen3_tts/prompt_embeds_builder.py @@ -363,7 +363,12 @@ class Qwen3TTSPromptEmbedsBuilder: self._tts_pad_embed_buffer = tts_pad_embed self._encode_ref_audio_batch_fn = encode_ref_audio_batch self._speaker_cache = speaker_cache - self._embedding_dtype = torch.bfloat16 + # Follow the talker's pad-embedding dtype (model dtype) so CPU float32/float16 deployments work. + self._embedding_dtype = ( + tts_pad_embed.dtype + if isinstance(tts_pad_embed, torch.Tensor) and tts_pad_embed.is_floating_point() + else torch.bfloat16 + ) self._text_tokenizer: Any | None = None diff --git a/vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py b/vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py index e65334ff5..67be40eac 100644 --- a/vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py +++ b/vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py @@ -352,6 +352,13 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module): self.use_async_omni_output = True self.eager_omni_postprocess_before_async_output = True self.omni_pooler_payload_include_hidden = False + # Terminal talker (no downstream stage, e.g. qwen3_tts_talker_only): + # publish the per-step codec frames to the client so an on-device + # decoder can consume them (codec-token streaming). + if getattr(vllm_config.model_config, "custom_process_next_stage_input_func", None) is None and not getattr( + vllm_config.model_config, "async_chunk_process_next_stage_input_func", None + ): + self.omni_client_multimodal_output_keys = ("codes.audio",) self.model = Qwen3Model(vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")) if get_pp_group().is_last_rank: @@ -465,7 +472,9 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module): torch.zeros(1, int(self.talker_config.hidden_size), dtype=model_dtype), persistent=False, ) - self._embedding_dtype = torch.bfloat16 + # Follows the model dtype so CPU deployments can run float32/float16; + # on CUDA this is bfloat16 as before. + self._embedding_dtype = model_dtype if isinstance(model_dtype, torch.dtype) else torch.bfloat16 tokenizer_config = Qwen3TTSTokenizerV2Config.from_pretrained( self.model_path, @@ -475,7 +484,7 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module): tokenizer_config.encoder_config, ) self.encoder.eval() - self.encoder.to(dtype=torch.bfloat16) + self.encoder.to(dtype=self._embedding_dtype) self._encoder_valid_num_quantizers = int(tokenizer_config.encoder_valid_num_quantizers) self._encoder_downsample_rate = int(tokenizer_config.encode_downsample_rate) @@ -1048,7 +1057,7 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module): sampling_rate=target_sr, return_tensors="pt", ) - inputs = inputs.to(device).to(torch.bfloat16) + inputs = inputs.to(device).to(self._embedding_dtype) input_values = inputs["input_values"].squeeze(1) padding_mask = inputs["padding_mask"].squeeze(1) @@ -1125,7 +1134,7 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module): loaded.add(name) device = self.vllm_config.device_config.device - self.encoder.to(device=device, dtype=torch.bfloat16) + self.encoder.to(device=device, dtype=self._embedding_dtype) self._init_runtime_buffers() self._init_silence_mask() diff --git a/vllm_omni/patch.py b/vllm_omni/patch.py index 770673120..37086986a 100644 --- a/vllm_omni/patch.py +++ b/vllm_omni/patch.py @@ -328,8 +328,8 @@ def _patch_chat_template_registry(): ) if "qwen3_omni_moe" not in _MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK: - _MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK["qwen3_omni_moe"] = ( - lambda _: CHAT_TEMPLATES_DIR / "template_chatml.jinja" + _MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK["qwen3_omni_moe"] = lambda _: ( + CHAT_TEMPLATES_DIR / "template_chatml.jinja" ) except ImportError: pass @@ -529,6 +529,14 @@ _patch_inductor_factorable_divisibility() # (not a slot or C extension). If CuMemAllocator is rewritten in C/Cython, # this monkey-patch will silently become a no-op. def _patch_cumem_free_callback_cuda() -> None: + try: + import torch + + if torch.version.cuda is None and getattr(torch.version, "hip", None) is None: + _PATCH_LOGGER.debug("[cumem-cuda] CPU-only torch; skipping patch") + return + except Exception: + pass try: from vllm.device_allocator.cumem import CuMemAllocator except ImportError: diff --git a/vllm_omni/platforms/__init__.py b/vllm_omni/platforms/__init__.py index 942eb7c24..225e18e29 100644 --- a/vllm_omni/platforms/__init__.py +++ b/vllm_omni/platforms/__init__.py @@ -120,17 +120,55 @@ def musa_omni_platform_plugin() -> str | None: return "vllm_omni.platforms.musa.platform.MUSAOmniPlatform" if is_musa else None +_CPU_OMNI_PLATFORM = "vllm_omni.platforms.cpu.platform.CPUOmniPlatform" + + +def _cpu_target_requested() -> bool: + """True when vLLM itself would select its CPU platform (env or ``+cpu`` wheel).""" + try: + import vllm.envs as envs + + if getattr(envs, "VLLM_TARGET_DEVICE", None) == "cpu": + return True + except Exception: # pragma: no cover - envs import never fails in practice + pass + try: + from vllm.platforms import vllm_version_matches_substr + + return bool(vllm_version_matches_substr("cpu")) + except Exception: + return False + + +def cpu_omni_platform_plugin() -> str | None: + """Check if the CPU OmniPlatform should be activated. + + Mirrors vLLM's own ``cpu_platform_plugin``: active when + ``VLLM_TARGET_DEVICE=cpu`` or a ``+cpu`` vLLM wheel is installed. + """ + logger.debug("Checking if CPU OmniPlatform is available.") + return _CPU_OMNI_PLATFORM if _cpu_target_requested() else None + + builtin_omni_platform_plugins = { "cuda": cuda_omni_platform_plugin, "rocm": rocm_omni_platform_plugin, "npu": npu_omni_platform_plugin, "xpu": xpu_omni_platform_plugin, "musa": musa_omni_platform_plugin, + "cpu": cpu_omni_platform_plugin, } def resolve_current_omni_platform_cls_qualname() -> str: """Resolve the current OmniPlatform class qualified name.""" + # Like vLLM (vllm/platforms/__init__.py), an explicit CPU target wins before + # any accelerator probe so a +cpu wheel on a GPU host does not activate two + # builtin platforms. + if _cpu_target_requested(): + logger.debug("CPU target requested; activating CPU OmniPlatform.") + return _CPU_OMNI_PLATFORM + platform_plugins = load_omni_plugins_by_group(OMNI_PLATFORM_PLUGINS_GROUP) activated_plugins = [] diff --git a/vllm_omni/platforms/cpu/__init__.py b/vllm_omni/platforms/cpu/__init__.py new file mode 100644 index 000000000..4244ce4b4 --- /dev/null +++ b/vllm_omni/platforms/cpu/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project diff --git a/vllm_omni/platforms/cpu/platform.py b/vllm_omni/platforms/cpu/platform.py new file mode 100644 index 000000000..a8f5bad14 --- /dev/null +++ b/vllm_omni/platforms/cpu/platform.py @@ -0,0 +1,160 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""CPU OmniPlatform: AR / generation stages on vLLM's CPU backend. + +Composes vLLM's ``CpuPlatform`` (device attributes, ``check_and_update_config`` +forcing: no async scheduling, no CUDA graphs, ``CPU_ATTN``, thread binding, +LD_PRELOAD) with the Omni interface, exactly as ``CudaOmniPlatform`` composes +``CudaPlatformBase``. Diffusion stages are not supported on CPU. + +Activate with ``VLLM_TARGET_DEVICE=cpu`` (or a ``+cpu`` vLLM wheel). Deploy +YAMLs can pin CPU values under ``platforms: cpu:``. +""" + +from __future__ import annotations + +import os +from typing import Any + +import torch +from vllm.logger import init_logger +from vllm.platforms.cpu import CpuPlatform + +from vllm_omni.platforms.interface import OmniPlatform, OmniPlatformEnum + +logger = init_logger(__name__) + + +def _numa_memory(device: torch.device | None = None) -> tuple[int, int]: + """(available, total) bytes of the NUMA node this process is allowed on, else host memory.""" + try: + from vllm.utils.cpu_resource_utils import get_allowed_cpu_list, get_memory_node_info + + node = get_allowed_cpu_list()[0].numa_node + info = get_memory_node_info(node) + return int(info.available_memory), int(info.total_memory) + except Exception: + try: + import psutil + + vm = psutil.virtual_memory() + return int(vm.available), int(vm.total) + except Exception: + return 0, 0 + + +class _NoGraphWrapper: + """Sentinel graph-wrapper type for the CPU platform (never instantiated).""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: # pragma: no cover - guard + raise NotImplementedError("CUDA graphs are not available on the CPU OmniPlatform") + + +class CPUOmniPlatform(OmniPlatform, CpuPlatform): + """CPU implementation of OmniPlatform (AR + generation stages only).""" + + _omni_enum = OmniPlatformEnum.CPU + + # ----------------------------------------------------------- workers + @classmethod + def get_omni_ar_worker_cls(cls) -> str: + return "vllm_omni.platforms.cpu.worker.cpu_ar_worker.CPUARWorker" + + @classmethod + def get_omni_generation_worker_cls(cls) -> str: + return "vllm_omni.platforms.cpu.worker.cpu_generation_worker.CPUGenerationWorker" + + @classmethod + def get_default_stage_config_path(cls) -> str: + return "vllm_omni/deploy" + + # ---------------------------------------------------------- diffusion + @classmethod + def get_diffusion_worker_cls(cls) -> str: + raise NotImplementedError("Diffusion stages are not supported on the CPU OmniPlatform") + + @classmethod + def get_diffusion_model_runner_cls(cls) -> str: + raise NotImplementedError("Diffusion stages are not supported on the CPU OmniPlatform") + + @classmethod + def get_diffusion_attn_backend_cls(cls, *args: Any, **kwargs: Any) -> str: + raise NotImplementedError("Diffusion attention backends are not supported on the CPU OmniPlatform") + + @classmethod + def supports_diffusion_dense_flash_attention(cls) -> bool: + return False + + @classmethod + def has_flash_attn_package(cls) -> bool: + return False + + # ------------------------------------------------------------ compile + @classmethod + def supports_torch_inductor(cls) -> bool: + # The code predictor's torch.compile costs a long C++ codegen on first + # use; keep it off unless explicitly requested (measured variant). + return os.environ.get("VLLM_OMNI_CPU_INDUCTOR", "0") == "1" + + @classmethod + def supports_talker_mtp_graph_capture(cls) -> bool: + return False + + @classmethod + def get_graph_wrapper_cls(cls) -> type: + # Runners use this in ``isinstance`` checks on every decode step + # (``_talker_mtp_forward``); on CPU nothing is ever wrapped, so return a + # sentinel class that no object is an instance of. + return _NoGraphWrapper + + # ------------------------------------------------------------- device + @classmethod + def get_torch_device(cls, local_rank: int | None = None) -> torch.device: + return torch.device("cpu") + + @classmethod + def get_device_count(cls) -> int: + # One logical device: stage device locks / admission iterate range(count). + return 1 + + @classmethod + def get_device_version(cls) -> str | None: + return None + + @classmethod + def synchronize(cls) -> None: + return None + + @classmethod + def record_device_event(cls): + return None + + @classmethod + def get_free_memory(cls, device: torch.device | None = None) -> int: + return _numa_memory(device)[0] + + @classmethod + def get_device_memory(cls, device: torch.device | None = None) -> tuple[int, int]: + return _numa_memory(device) + + @classmethod + def get_device_total_memory(cls, device_id: int = 0) -> int: + try: + return int(CpuPlatform.get_device_total_memory(device_id)) + except Exception: + return _numa_memory()[1] + + @classmethod + def supports_cpu_offload(cls) -> bool: + return False + + @classmethod + def is_fully_connected(cls, physical_device_ids: list[int] | None = None) -> bool: + return True + + @classmethod + def get_default_ir_op_priority(cls, *args: Any, **kwargs: Any): + parent = getattr(CpuPlatform, "get_default_ir_op_priority", None) + if parent is not None: + return parent(*args, **kwargs) + return super().get_default_ir_op_priority(*args, **kwargs) # type: ignore[misc] diff --git a/vllm_omni/platforms/cpu/worker/__init__.py b/vllm_omni/platforms/cpu/worker/__init__.py new file mode 100644 index 000000000..4244ce4b4 --- /dev/null +++ b/vllm_omni/platforms/cpu/worker/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project diff --git a/vllm_omni/platforms/cpu/worker/cpu_ar_model_runner.py b/vllm_omni/platforms/cpu/worker/cpu_ar_model_runner.py new file mode 100644 index 000000000..11daa0ff5 --- /dev/null +++ b/vllm_omni/platforms/cpu/worker/cpu_ar_model_runner.py @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""CPU model runners for Omni AR and generation stages. + +MRO: ``CPUARModelRunner -> GPUARModelRunner -> OmniGPUModelRunner -> (omni +mixins) -> CPUModelRunner -> GPUModelRunner``. ``CPUModelRunner.__init__`` +runs vLLM's CPU shims (torch.cuda placeholders, ``_postprocess_tensors``, +``_postprocess_triton``) around ``GPUModelRunner.__init__``; the Omni layers +then add their buffers on the CPU device. Nothing CUDA-graph related can be +reached because ``CpuPlatform.check_and_update_config`` forces +``cudagraph_capture_sizes=[]`` and ``async_scheduling=False``. +""" + +from __future__ import annotations + +from vllm.logger import init_logger +from vllm.v1.worker.cpu_model_runner import CPUModelRunner + +from vllm_omni.worker.gpu_ar_model_runner import GPUARModelRunner +from vllm_omni.worker.gpu_generation_model_runner import GPUGenerationModelRunner + +logger = init_logger(__name__) + + +class _CPUOmniRunnerMixin: + """Overrides shared by the CPU AR and generation runners.""" + + def _assert_cpu_invariants(self) -> None: + if getattr(self, "use_async_scheduling", False): + raise RuntimeError( + "CPU Omni runners require async_scheduling=False (vLLM's CpuPlatform forces it); " + "the async omni-output path uses CUDA streams." + ) + + def _init_device_properties(self) -> None: + self.num_sms = None + + def _sync_device(self) -> None: + return None + + def capture_model(self) -> int: + # No graphs on CPU; keep the call cheap and explicit. + return 0 + + def _capture_talker_mtp_graphs(self) -> None: + return None + + +class CPUARModelRunner(_CPUOmniRunnerMixin, GPUARModelRunner, CPUModelRunner): + """AR (talker / thinker) runner on CPU.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._assert_cpu_invariants() + + +class CPUGenerationModelRunner(_CPUOmniRunnerMixin, GPUGenerationModelRunner, CPUModelRunner): + """Generation (code2wav / decoder) runner on CPU.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._assert_cpu_invariants() diff --git a/vllm_omni/platforms/cpu/worker/cpu_ar_worker.py b/vllm_omni/platforms/cpu/worker/cpu_ar_worker.py new file mode 100644 index 000000000..976147574 --- /dev/null +++ b/vllm_omni/platforms/cpu/worker/cpu_ar_worker.py @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""CPU workers for Omni AR and generation stages (XPU-worker pattern). + +``CPUWorker`` owns the CPU specifics (NUMA binding, ``init_cpu_memory_env``, +gloo, NUMA-based ``determine_available_memory``, no cudagraph warm-up); the +Omni mixin only loads omni plugins in the worker process. Sleep/wake are not +supported on CPU and return an error ACK instead of raising in the RPC loop. +""" + +from __future__ import annotations + +import vllm.v1.worker.cpu.shm # noqa: F401 # patches torch.Event/torch.cuda.Stream; must be first +from vllm.logger import init_logger +from vllm.v1.worker.cpu_worker import CPUWorker + +from vllm_omni.diffusion.data import OmniACK +from vllm_omni.engine.init_timeline import timed_phase +from vllm_omni.platforms.cpu.worker.cpu_ar_model_runner import CPUARModelRunner, CPUGenerationModelRunner +from vllm_omni.worker.mixins import OmniWorkerMixin + +logger = init_logger(__name__) + + +class _CPUOmniWorkerMixin: + model_runner_cls: type + + @timed_phase("init_device") + def init_device(self) -> None: + super().init_device() # type: ignore[misc] + if getattr(self, "use_v2_model_runner", False): + logger.warning("OMNI CPU worker forces v1 model runner for omni hooks.") + self.use_v2_model_runner = False + # CPUWorker.init_device built a plain CPUModelRunner; replace it with the omni runner. + self.model_runner = self.model_runner_cls(self.vllm_config, self.device) + + @staticmethod + def _task_id(task) -> str: + if isinstance(task, dict): + return str(task.get("task_id", "")) + return str(getattr(task, "task_id", "")) + + def handle_sleep_task(self, task) -> OmniACK: + return OmniACK( + task_id=self._task_id(task), status="ERROR", error_msg="sleep mode is not supported on the CPU OmniPlatform" + ) + + def handle_wake_task(self, task) -> OmniACK: + return OmniACK( + task_id=self._task_id(task), status="ERROR", error_msg="wake-up is not supported on the CPU OmniPlatform" + ) + + +class CPUARWorker(_CPUOmniWorkerMixin, OmniWorkerMixin, CPUWorker): + """CPU AR worker for thinker / talker stages.""" + + model_runner_cls = CPUARModelRunner + + +class CPUGenerationWorker(_CPUOmniWorkerMixin, OmniWorkerMixin, CPUWorker): + """CPU generation worker for decoder / code2wav stages.""" + + model_runner_cls = CPUGenerationModelRunner diff --git a/vllm_omni/platforms/cpu/worker/cpu_generation_worker.py b/vllm_omni/platforms/cpu/worker/cpu_generation_worker.py new file mode 100644 index 000000000..93a2e0a44 --- /dev/null +++ b/vllm_omni/platforms/cpu/worker/cpu_generation_worker.py @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Import-path shim: the CPU generation worker lives next to the AR worker.""" + +from vllm_omni.platforms.cpu.worker.cpu_ar_worker import CPUGenerationWorker + +__all__ = ["CPUGenerationWorker"] diff --git a/vllm_omni/platforms/interface.py b/vllm_omni/platforms/interface.py index 7ab4dd084..b18f73a99 100644 --- a/vllm_omni/platforms/interface.py +++ b/vllm_omni/platforms/interface.py @@ -24,6 +24,7 @@ class OmniPlatformEnum(Enum): NPU = "npu" XPU = "xpu" MUSA = "musa" + CPU = "cpu" OOT = "oot" UNSPECIFIED = "unspecified" @@ -54,6 +55,9 @@ class OmniPlatform(Platform): def is_musa(self) -> bool: return self._omni_enum == OmniPlatformEnum.MUSA + def is_cpu(self) -> bool: + return self._omni_enum == OmniPlatformEnum.CPU + def is_out_of_tree(self) -> bool: return self._omni_enum == OmniPlatformEnum.OOT