From bb8735effca1cf1403092ccfbdec783738e72cd4 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 23 Sep 2026 21:18:21 +0800 Subject: [PATCH] Add native Windows CPU Qwen TTS Omni stage and profile evidence --- .../experiments/probe_omni_qwen_tts_cpu.py | 216 + .../probe_omni_qwen_tts_cpu_entrypoint.py | 107 + .../results/e2e_expansion_20260923/README.md | 5 +- .../evidence/qwen_tts_native_cpu/README.md | 4 +- .../evidence/qwen_tts_omni_cpu/README.md | 43 + .../omni_qwen_tts_cpu_asr_20260923.json | 16 + .../omni_qwen_tts_cpu_asr_20260923.log | 17 + ...qwen_tts_cpu_profile20_20260923.driver.log | 11995 ++++++++++++++++ .../omni_qwen_tts_cpu_profile20_20260923.json | 11991 +++++++++++++++ .../omni_qwen_tts_cpu_profile20_20260923.log | 88 + ...ni_qwen_tts_cpu_public_20260923.driver.log | 52 + .../omni_qwen_tts_cpu_public_20260923.json | 26 + .../omni_qwen_tts_cpu_public_20260923.log | 21 + ...ni_qwen_tts_cpu_reject_20260923.driver.log | 99 + .../omni_qwen_tts_cpu_reject_20260923.json | 25 + .../evidence/summary/README.md | 6 +- docs/design/omni_edge_backends.md | 10 +- docs/design/omni_edge_e2e_roadmap.md | 11 +- vllm_omni/config/pipeline_registry.py | 2 + vllm_omni/engine/backends/__init__.py | 4 + vllm_omni/engine/backends/crisp_tts.py | 7 +- vllm_omni/engine/backends/qwen_tts_cpu.py | 205 + .../engine/backends/qwen_tts_cpu_worker.py | 138 + .../models/qwen3_tts/pipeline.py | 16 + 24 files changed, 25090 insertions(+), 14 deletions(-) create mode 100644 benchmarks/edge_harness/experiments/probe_omni_qwen_tts_cpu.py create mode 100644 benchmarks/edge_harness/experiments/probe_omni_qwen_tts_cpu_entrypoint.py create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_omni_cpu/README.md create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_omni_cpu/omni_qwen_tts_cpu_asr_20260923.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_omni_cpu/omni_qwen_tts_cpu_asr_20260923.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_omni_cpu/omni_qwen_tts_cpu_profile20_20260923.driver.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_omni_cpu/omni_qwen_tts_cpu_profile20_20260923.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_omni_cpu/omni_qwen_tts_cpu_profile20_20260923.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_omni_cpu/omni_qwen_tts_cpu_public_20260923.driver.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_omni_cpu/omni_qwen_tts_cpu_public_20260923.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_omni_cpu/omni_qwen_tts_cpu_public_20260923.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_omni_cpu/omni_qwen_tts_cpu_reject_20260923.driver.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_omni_cpu/omni_qwen_tts_cpu_reject_20260923.json create mode 100644 vllm_omni/engine/backends/qwen_tts_cpu.py create mode 100644 vllm_omni/engine/backends/qwen_tts_cpu_worker.py diff --git a/benchmarks/edge_harness/experiments/probe_omni_qwen_tts_cpu.py b/benchmarks/edge_harness/experiments/probe_omni_qwen_tts_cpu.py new file mode 100644 index 000000000..0a69187d9 --- /dev/null +++ b/benchmarks/edge_harness/experiments/probe_omni_qwen_tts_cpu.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +"""Profile the isolated official Qwen3-TTS CPU wrapper through Omni.""" + +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import json +import math +import time +import wave +from pathlib import Path +from types import SimpleNamespace + +import numpy as np + + +def _rank(values: list[float], fraction: float) -> float: + return sorted(values)[math.ceil(len(values) * fraction) - 1] + + +def _save_wav(path: Path, audio) -> None: + pcm = np.rint(audio.numpy() * 32768).astype(" None: + parser = argparse.ArgumentParser(description=__doc__) + for name in ( + "python_bin", "python_sha256", "model_dir", "overlay_dir", "talker_sha256", + "tokenizer_sha256", "server_log", "output_report", "output_wav", + ): + parser.add_argument("--" + name.replace("_", "-"), required=True) + parser.add_argument("--warmups", type=int, default=1) + parser.add_argument("--repeats", type=int, default=20) + parser.add_argument("--reserve-gib", type=int, default=10) + parser.add_argument("--abort-check", action="store_true") + args = parser.parse_args() + if args.warmups < 0 or args.repeats <= 0 or not 0 < args.reserve_gib <= 16: + parser.error("invalid profile or budget") + + from vllm_omni.config.stage_config import DeployConfig, StageDeployConfig, merge_pipeline_deploy + from vllm_omni.engine.stage_runtime import StageRuntime + from vllm_omni.model_executor.models.qwen3_tts.pipeline import QWEN3_TTS_CPU_WHOLE_PIPELINE + + backend = { + "name": "external.qwen_tts.cpu.v1", + "python_bin": args.python_bin, + "python_sha256": args.python_sha256, + "model_dir": args.model_dir, + "overlay_dir": args.overlay_dir, + "talker_sha256": args.talker_sha256, + "tokenizer_sha256": args.tokenizer_sha256, + "log_file": args.server_log, + "memory_overhead_bytes": 6 << 30, + "expected_torch": "2.13.0+cu130", + "max_text_bytes": 4096, + "max_wav_bytes": 4 << 20, + "max_audio_s": 10, + "request_timeout_s": 180, + "start_timeout_s": 180, + } + deploy = DeployConfig( + async_chunk=False, + stages=[StageDeployConfig( + stage_id=0, + backend=backend, + resource_budget={ + "capacities": {"host_ram": 16 << 30}, + "demands": {"host_ram": args.reserve_gib << 30}, + }, + )], + ) + pipeline = QWEN3_TTS_CPU_WHOLE_PIPELINE + configs = [stage.to_omegaconf() for stage in merge_pipeline_deploy(pipeline, deploy)] + runtime = StageRuntime(configs, "local-qwen-tts-cpu", "", stage_init_timeout=180, async_chunk=False) + report = { + "scope": "complete resident native Windows CPU Qwen3-TTS text-to-WAV through Omni", + "warmup_count": args.warmups, + "measured_count": args.repeats, + "memory_budget": deploy.stages[0].resource_budget, + } + sample_task = None + stop_sampling = asyncio.Event() + try: + started = time.perf_counter() + runtime.initialize() + report["startup_s"] = time.perf_counter() - started + pool = runtime.stage_pools[0] + report["execution_plan"] = pool.stage_client.execution_plan + state = SimpleNamespace(sampling_params_list=[None]) + import psutil + + memory_samples = [] + + async def sample_memory() -> None: + while not stop_sampling.is_set(): + try: + root = psutil.Process(pool.stage_client._proc.pid) + members = [root, *root.children(recursive=True)] + memories = [process.memory_full_info() for process in members] + memory_samples.append({ + "unix": time.time(), + "rss_bytes": sum(int(memory.rss) for memory in memories), + "private_bytes": sum( + int(getattr(memory, "private", memory.rss)) for memory in memories + ), + "host_available_bytes": psutil.virtual_memory().available, + "pids": [process.pid for process in members], + }) + except psutil.Error: + break + await asyncio.sleep(0.25) + + sample_task = asyncio.create_task(sample_memory()) + + async def request_one(text: str, *, save: bool = False) -> dict: + request_id = f"cpu-tts-{time.monotonic_ns()}" + started = time.perf_counter() + await pool.submit_initial(request_id, state, {"text": text, "voice": "Ryan", "seed": 42}) + deadline = time.monotonic() + 180 + while True: + output = pool.poll_graph_output(0) + if output is not None: + break + if time.monotonic() >= deadline: + raise TimeoutError("Omni Qwen CPU TTS stage request timed out") + await asyncio.sleep(0.005) + wall_s = time.perf_counter() - started + try: + if output.error: + raise RuntimeError(output.error) + payload = output.multimodal_output + audio, sample_rate = payload["audio"], payload["sr"] + metadata = output.custom_output["audio_metadata"] + assert sample_rate == 24000 and audio.ndim == 1 + assert len(audio) == metadata["pcm_frames"] and len(audio) > 0 + assert float(audio.square().mean().sqrt()) > 1e-5 + if save: + _save_wav(Path(args.output_wav), audio) + pcm = np.rint(audio.numpy() * 32768).astype(" None: + parser = argparse.ArgumentParser(description=__doc__) + for name in ( + "python_bin", "python_sha256", "model_dir", "overlay_dir", "talker_sha256", + "tokenizer_sha256", "server_log", "output_report", + ): + parser.add_argument("--" + name.replace("_", "-"), required=True) + args = parser.parse_args() + + from vllm_omni.entrypoints.async_omni import AsyncOmni + from vllm_omni.model_executor.models.qwen3_tts.pipeline import QWEN3_TTS_CPU_WHOLE_PIPELINE + + backend = { + "name": "external.qwen_tts.cpu.v1", + "python_bin": args.python_bin, + "python_sha256": args.python_sha256, + "model_dir": args.model_dir, + "overlay_dir": args.overlay_dir, + "talker_sha256": args.talker_sha256, + "tokenizer_sha256": args.tokenizer_sha256, + "log_file": args.server_log, + "memory_overhead_bytes": 6 << 30, + "expected_torch": "2.13.0+cu130", + "max_text_bytes": 4096, + "max_wav_bytes": 4 << 20, + "max_audio_s": 10, + "request_timeout_s": 180, + "start_timeout_s": 180, + } + report = {"entrypoint": "AsyncOmni.generate", "pipeline": QWEN3_TTS_CPU_WHOLE_PIPELINE.model_type} + engine = None + try: + with tempfile.TemporaryDirectory(prefix="omni-qwen-cpu-tts-") as temporary: + deployment = Path(temporary) / "deploy.yaml" + deployment.write_text(yaml.safe_dump({ + "pipeline": QWEN3_TTS_CPU_WHOLE_PIPELINE.model_type, + "async_chunk": False, + "stages": [{ + "stage_id": 0, + "backend": backend, + "resource_budget": { + "capacities": {"host_ram": 16 << 30}, + "demands": {"host_ram": 10 << 30}, + }, + }], + }), encoding="utf-8") + started = time.perf_counter() + engine = AsyncOmni( + model=str(Path(args.model_dir).resolve(strict=True)), + deploy_config=str(deployment), stage_init_timeout=180, init_timeout=240, + ) + report["startup_s"] = time.perf_counter() - started + outputs = [] + started = time.perf_counter() + async for output in engine.generate( + {"text": "Hello from the local computer.", "voice": "Ryan", "seed": 42}, + request_id="public-cpu-tts-1", + ): + if output.error: + raise RuntimeError(output.error) + audio = output.multimodal_output["audio"] + pcm = np.rint(audio.numpy() * 32768).astype(" + asyncio.run(main()) + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\asyncio\runners.py", line 195, in run + return runner.run(main) + ^^^^^^^^^^^^^^^^ + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\asyncio\runners.py", line 118, in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\asyncio\base_events.py", line 691, in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\benchmarks\edge_harness\experiments\probe_omni_qwen_tts_cpu.py", line 93, in main + runtime.initialize() + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 278, in initialize + raise exc + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 252, in initialize + initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 589, in _initialize_stage_replicas + raise primary_exc + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 550, in _init_group + client = self._initialize_replica( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 817, in _initialize_replica + return create_graph_client( + ^^^^^^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\backends\__init__.py", line 23, in create_graph_client + return QwenTTSCPUStageClient(metadata, backend, ledger, reservation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\backends\qwen_tts_cpu.py", line 79, in __init__ + raise ResourceUnavailable("CPU TTS weights plus state/workspace/headroom exceed reservation") +vllm_omni.engine.resource_ledger.ResourceUnavailable: CPU TTS weights plus state/workspace/headroom exceed reservation diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_omni_cpu/omni_qwen_tts_cpu_reject_20260923.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_omni_cpu/omni_qwen_tts_cpu_reject_20260923.json new file mode 100644 index 000000000..bd71fbe15 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_omni_cpu/omni_qwen_tts_cpu_reject_20260923.json @@ -0,0 +1,25 @@ +{ + "scope": "complete resident native Windows CPU Qwen3-TTS text-to-WAV through Omni", + "warmup_count": 0, + "measured_count": 1, + "memory_budget": { + "capacities": { + "host_ram": 17179869184 + }, + "demands": { + "host_ram": 4294967296 + } + }, + "status": "failed", + "error": "ResourceUnavailable: CPU TTS weights plus state/workspace/headroom exceed reservation", + "ledger_after_shutdown": { + "capacities": { + "host_ram": 17179869184 + }, + "reserved": { + "host_ram": 0 + }, + "owners": [], + "quarantined": [] + } +} diff --git a/benchmarks/edge_harness/results/e2e_profiling_20260922/evidence/summary/README.md b/benchmarks/edge_harness/results/e2e_profiling_20260922/evidence/summary/README.md index 81c13067c..adc4d3ba4 100644 --- a/benchmarks/edge_harness/results/e2e_profiling_20260922/evidence/summary/README.md +++ b/benchmarks/edge_harness/results/e2e_profiling_20260922/evidence/summary/README.md @@ -1,6 +1,6 @@ # E2E check and profiling -All 60 pairings have a disposition. The timing tables below remain the 2026-09-22 measurements. The configuration matrix and affected per-cell reasons incorporate the [2026-09-23 scoped recovery runs](../../../e2e_recovery_20260923/README.md) and [WSL CPU/native Windows expansion](../../../e2e_expansion_20260923/README.md). The expansion adds separate 20-request serial MiniCPM-o text-to-speech profiles on WSL CPU and native Windows GPU, a Spark output-head component profile on the HX370 AMD NPU, complete Spark text-request profiles through standalone llama.cpp and a bounded Omni whole-session stage on native Windows CPU and Radeon 890M, a standalone Qwen3-TTS text-to-WAV profile on native Windows CPU, and a hybrid CPU+Radeon 890M Qwen3-TTS profile now repeated through a bounded Omni whole-session stage. Profiling completion does not imply full release qualification. +All 60 pairings have a disposition. The timing tables below remain the 2026-09-22 measurements. The configuration matrix and affected per-cell reasons incorporate the [2026-09-23 scoped recovery runs](../../../e2e_recovery_20260923/README.md) and [WSL CPU/native Windows expansion](../../../e2e_expansion_20260923/README.md). The expansion adds separate 20-request serial MiniCPM-o text-to-speech profiles on WSL CPU and native Windows GPU, a Spark output-head component profile on the HX370 AMD NPU, complete Spark text-request profiles through standalone llama.cpp and a bounded Omni whole-session stage on native Windows CPU and Radeon 890M, native Windows CPU Qwen3-TTS text-to-WAV profiles through both standalone PyTorch and a bounded Omni whole-session stage, and a hybrid CPU+Radeon 890M Qwen3-TTS profile likewise repeated through a bounded Omni whole-session stage. Profiling completion does not imply full release qualification. ## Configuration × model @@ -9,7 +9,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E� i | Device | Spark-X2.5 | Qwen3-TTS 0.6B CustomVoice | Qwen3.8-27B | MiniCPM-o 4.5 | InternVLA-A1 | |---|---|---|---|---|---| | PC HX370: CPU / WSL | profile completed | scoped E2E: short stream | NOT E2E | scoped E2E: text/image/audio → speech | synthetic policy pass | -| PC HX370: CPU / native Windows | scoped E2E: Omni Spark text | scoped E2E: standalone text → WAV | NOT E2E | NOT E2E | synthetic policy pass | +| PC HX370: CPU / native Windows | scoped E2E: Omni Spark text | scoped E2E: Omni CPU text → complete WAV | NOT E2E | NOT E2E | synthetic policy pass | | PC HX370 + RTX 5090 Laptop / WSL | profile completed | profile completed | scoped E2E: text + image | scoped E2E: text/image/audio → speech | synthetic policy pass | | PC HX370 + RTX 5090 Laptop / Windows | profile completed | profile completed | NOT E2E | scoped E2E: text/image/audio → speech | synthetic policy pass | | PC HX370 + Radeon 890M / Windows worker | scoped E2E: Omni Spark text | scoped E2E: Omni Radeon talker/codec + CPU predictor | NOT E2E | NOT E2E | NOT E2E | @@ -31,7 +31,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E� i | PC HX370: CPU / WSL | MiniCPM-o 4.5 | SCOPED_E2E | The verified checkpoint completed real three-stage text, one synthetic image and one synthetic audio input to text+speech on the vLLM 0.28 CPU backend in separate bounded plans. The 448×448 red square was described correctly with a nonzero 5.2 s WAV in one 37.70 s request; the 2 s/440 Hz tone was described as “A loud and high pitched beep� with a nonzero 2.16 s WAV in one 17.30 s request. CPU wording/token IDs differ from the WSL GPU runs, with no task-quality threshold established. A separate 1-warmup/20-measured serial **text** run had request wall p50 21.86 s/p95 22.97 s. WSL swap reached 5.75 GB during that text profile; manual snapshots in the multimodal runs showed heavy swap use but do not prove peak memory. Both multimodal runs force-killed stage 0 at shutdown and reported shared-memory cleanup warnings. | Validate real image/audio suites, speech intelligibility, video input, concurrency, interruption, admission/loading peak, clean shutdown and sustained thermal behavior before broader CPU qualification; [CPU evidence](../../../e2e_expansion_20260923/README.md). | | PC HX370: CPU / WSL | InternVLA-A1 | SYNTHETIC_POLICY_PASS | The Place_Markpen checkpoint strict-loaded in CPU-only PyTorch and produced finite `[1, 50, 32]` actions from zero observations/noise in one 3.25 s forward. Compared with the same CUDA checkpoint/input, relative L2 error was 1.30% and cosine similarity 0.999917; no task-quality tolerance is established. | Validate real observations against reference actions, units/timestamps, repeated latency and memory before robot-task qualification; [CPU evidence](../../../e2e_expansion_20260923/README.md). | | PC HX370: CPU / native Windows | Spark-X2.5 | SCOPED_E2E_OMNI_COMPLETE_REQUEST | The pinned Spark-X2.5-1.7B Q4_K_M GGUF ran in native Windows llama.cpp CPU (`-dev none -ngl 0`) as a whole-session Omni StageRuntime/StagePool backend. `Paris` and the correct `1511` inventory answer passed; one warmup plus 20 serial complete requests were 20/20 correct, nearest-rank wall p50/p95 6.93/7.05 s. The public AsyncOmni short request passed. The stage verifies artifact hashes and CPU placement (0/29 layers offloaded), reserves 4 GiB host RAM, emits a terminal text event and releases reservation after in-flight cancellation. The separate prior standalone profile was 6.65/6.86 s. No vLLM CPU attention repair, incremental stream, restart-after-cancel, loading peak or sustained power/thermal gate is claimed. | Validate broader text/token quality, streaming and cancellation recovery, memory peak and sustained load; [Omni raw evidence](../../../e2e_expansion_20260923/evidence/spark_omni_llamacpp/README.md), [standalone evidence](../../../e2e_expansion_20260923/evidence/spark_radeon890m_llamacpp/README.md). | -| PC HX370: CPU / native Windows | Qwen3-TTS 0.6B CustomVoice | SCOPED_E2E_STANDALONE | The pinned 0.6B CustomVoice checkpoint ran wholly on CPU through Qwen's standalone PyTorch wrapper (BF16/SDPA) and produced nonzero 24 kHz WAVs for two English sentences. For one repeated short prompt, 1 warmup + 20 serial measured complete generation requests had nearest-rank p50/p95 13.94/14.68 s for 4.56 s of audio. Independent Whisper tiny.en ASR gave word error rates 0.20 and 0.00 on the two WAVs; this is only an intelligibility proxy. The native vLLM CPU extension still lacks the required attention operators. No Omni binding, playable streaming, listening/quality threshold, loading peak or sustained power/thermal run was verified. | Bind the whole CPU TTS backend to Omni's stage and streaming/state contracts; validate playback, quality, cancellation, admission and sustained use; [standalone TTS evidence](../../../e2e_expansion_20260923/evidence/qwen_tts_native_cpu/README.md). | +| PC HX370: CPU / native Windows | Qwen3-TTS 0.6B CustomVoice | SCOPED_E2E_OMNI_COMPLETE_REQUEST | The pinned 0.6B CustomVoice BF16/SDPA checkpoint ran wholly on CPU through an isolated Qwen wrapper worker owned by one bounded Omni StageRuntime stage. Two named 24 kHz WAVs matched the earlier standalone PCM SHA256 values exactly. One warmup + 20 serial complete Omni requests for 4.56 s of audio had 20/20 identical PCM and nearest-rank wall p50/p95 15.32/15.60 s (RTF 3.36/3.42); independent Whisper tiny.en ASR gave WER 0.20 on the first Omni WAV, an intelligibility proxy only. A public AsyncOmni audio request passed. The stage verified pinned weights/interpreter and CPU placement, reserved 10 GiB host RAM, rejected an insufficient 4 GiB demand before worker launch, and after in-flight cancellation emitted no stale output and returned the ledger to zero. Sampled process-tree private memory peaked at 5.39 GB, not a loading peak. Native vLLM CPU attention operators remain absent; playable streaming, post-cancel restart, broader speech quality, loading peak and sustained power/thermal behavior remain unverified. | Validate playable streaming and restart/recovery, broader voice quality, peak memory, concurrency and sustained behavior; [Omni raw evidence](../../../e2e_expansion_20260923/evidence/qwen_tts_omni_cpu/README.md), [standalone baseline](../../../e2e_expansion_20260923/evidence/qwen_tts_native_cpu/README.md). | | PC HX370: CPU / native Windows | Qwen3.8-27B | NOT_E2E_PROFILEABLE | The installed native Windows vLLM extension lacks cpu_attn_reshape_and_cache and cpu_attn_get_scheduler_metadata. The separate ORT CPU graph worker is not a whole-model text/TTS adapter; a qualified CPU build/binding is missing. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | | PC HX370: CPU / native Windows | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | The exact checkpoint is verified in the local Linux model store, but the installed native Windows vLLM extension lacks required CPU attention operators; no native full-model run or memory admission exists. | Establish a native Windows CPU backend and model access, then run the full requested path; [weight evidence](../../../e2e_recovery_20260923/evidence/weights/minicpmo.json). | | PC HX370: CPU / native Windows | InternVLA-A1 | SYNTHETIC_POLICY_PASS | The Place_Markpen checkpoint strict-loaded from a UNC path and the direct CPU policy produced finite `[1, 50, 32]` actions on synthetic zero observations/noise. Policy/output CPU placement was checked; one forward took 3.36 s. Relative L2 difference from WSL CPU was 1.08% (no task tolerance). This direct policy route does not require native vLLM CPU attention operators. | Validate real observations/reference actions, units/timestamps, memory and repeated latency before task qualification; [native evidence](../../../e2e_expansion_20260923/README.md). | diff --git a/docs/design/omni_edge_backends.md b/docs/design/omni_edge_backends.md index aa611aef3..41468c95c 100644 --- a/docs/design/omni_edge_backends.md +++ b/docs/design/omni_edge_backends.md @@ -5,7 +5,9 @@ per request, with a fixed input bucket and copied host tensors. A second backend `external.llamacpp.text.v1`, owns one pinned GGUF whole-model server process and returns complete text requests. `external.crisp.tts.v1` owns a pinned CustomVoice GGUF talker, codec and local hybrid CPU+Radeon process, returning -complete PCM audio. All use the normal `Omni` / `AsyncOmni` APIs, +complete PCM audio. `external.qwen_tts.cpu.v1` owns the original BF16 CustomVoice +checkpoint in an isolated native Windows CPU process and also returns complete +PCM audio. All use the normal `Omni` / `AsyncOmni` APIs, `PipelineConfig`, `StageRuntime`, `StagePool` and orchestrator. Native vLLM stages retain their model execution, KV cache, sampling and batching. No accelerator is mandatory. The llama.cpp backend currently uses the complete-request graph @@ -13,7 +15,9 @@ control path for admission and acknowledgement; it is not an ONNX graph and does not yet support incremental token events or restart after active cancellation. The CrispASR route likewise lacks playable incremental PCM and post-cancel restart; its current placement gate is specific to the measured HX370 Radeon -890M shared-RAM configuration. +890M shared-RAM configuration. The Qwen CPU route likewise returns a terminal +WAV only; its worker pins an isolated Transformers 4.57.3 overlay and verifies +CPU placement and weight hashes before serving. ## Boundaries @@ -64,6 +68,8 @@ This section preserves the 2026-09-22 audit. For the latest per-cell disposition The [Qwen3-TTS CPU+Radeon whole-session path](../../benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_omni_hybrid/README.md) also completed two named text-to-audio inputs and 20 serial Omni requests with exact PCM parity to its standalone record. It verifies pinned executable and weights, logged hybrid placement, one-slot shared-RAM admission, terminal audio and cancellation drain. Playable streaming, post-cancel restart, loading/iGPU peaks, sustained use and broader speech quality remain open; RTF still exceeds 1. No joint CPU+iGPU+NPU claim follows from this result. +The [native Windows CPU Qwen3-TTS Omni path](../../benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_omni_cpu/README.md) completed two named text-to-WAV inputs and 20 serial Omni requests with exact PCM parity to the standalone PyTorch record. It verifies pinned weights, CPU-only placement, a 10 GiB host reservation, a 4 GiB pre-load refusal, terminal audio and cancellation drain. Request-wall p50/p95 were 15.32/15.60 s for 4.56 s of audio. Playable streaming, post-cancel restart, loading peak, broad voice quality and sustained use remain open. This CPU route does not qualify AMD or mobile execution. + **All 60 named pairings were checked on 2026-09-22** through actual runs, raw-record review or artifact/runtime preflight. This is not 60 successful executions. Each cell links to its evidence, finding and next required step. diff --git a/docs/design/omni_edge_e2e_roadmap.md b/docs/design/omni_edge_e2e_roadmap.md index fd44ca377..5c9713da1 100644 --- a/docs/design/omni_edge_e2e_roadmap.md +++ b/docs/design/omni_edge_e2e_roadmap.md @@ -53,8 +53,15 @@ prompt took p50/p95 13.94/14.68 s to generate 4.56 s of audio. Whisper tiny.en ASR gave word error rates 0.20 and 0.00 for the two WAVs, an intelligibility proxy rather than a quality pass. The [raw record](../../benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/README.md) documents the isolated Transformers 4.57.3 dependency and short process-memory -trace. Omni CPU stage binding, playable streaming, loading-peak admission and -sustained power/thermal tests remain M2 work. +trace. The subsequent [Omni CPU whole-session stage](../../benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_omni_cpu/README.md) +matched both standalone PCM hashes exactly, returned a public `AsyncOmni` +audio event, and completed one warmup plus 20 serial measured requests at +p50/p95 15.32/15.60 s for 4.56 s of audio. It reserves 10 GiB host RAM, +rejects an insufficient 4 GiB demand before worker launch, and drains an +in-flight cancellation without stale output or remaining reservation. This +closes the scoped CPU stage-binding task. Playable streaming, post-cancel +restart, loading-peak admission, broad speech quality and sustained +power/thermal tests remain M2 work. A separate native Windows CPU+Radeon 890M Qwen3-TTS route now completes two named text-to-WAV requests in standalone CrispASR using a Q8_0 talker, F16 diff --git a/vllm_omni/config/pipeline_registry.py b/vllm_omni/config/pipeline_registry.py index b27cc8f50..808fb6ad4 100644 --- a/vllm_omni/config/pipeline_registry.py +++ b/vllm_omni/config/pipeline_registry.py @@ -113,6 +113,7 @@ from vllm_omni.model_executor.models.qwen3_omni.pipeline import ( ) from vllm_omni.model_executor.models.qwen3_tts.pipeline import ( QWEN3_TTS_CRISP_HYBRID_PIPELINE, + QWEN3_TTS_CPU_WHOLE_PIPELINE, QWEN3_TTS_PIPELINE, QWEN3_TTS_TALKER_ONLY_PIPELINE, ) @@ -149,6 +150,7 @@ OMNI_PIPELINES: dict[str, PipelineConfig | PipelineResolverFunc] = { "qwen3_tts": QWEN3_TTS_PIPELINE, "qwen3_tts_talker_only": QWEN3_TTS_TALKER_ONLY_PIPELINE, "qwen3_tts_crisp_hybrid": QWEN3_TTS_CRISP_HYBRID_PIPELINE, + "qwen3_tts_cpu_whole": QWEN3_TTS_CPU_WHOLE_PIPELINE, # Single-stage text: an AR decoder with no companion stage. It lives in # the same registry as the multi-stage pipelines so that one code path # resolves every model (see the M0 local text mode). diff --git a/vllm_omni/engine/backends/__init__.py b/vllm_omni/engine/backends/__init__.py index 34cadbf96..a9d344c59 100644 --- a/vllm_omni/engine/backends/__init__.py +++ b/vllm_omni/engine/backends/__init__.py @@ -17,4 +17,8 @@ def create_graph_client(metadata, config, ledger, reservation): from .crisp_tts import CrispTTSStageClient return CrispTTSStageClient(metadata, backend, ledger, reservation) + if name == "external.qwen_tts.cpu.v1": + from .qwen_tts_cpu import QwenTTSCPUStageClient + + return QwenTTSCPUStageClient(metadata, backend, ledger, reservation) raise ValueError(f"unsupported complete-request stage backend: {name!r}") diff --git a/vllm_omni/engine/backends/crisp_tts.py b/vllm_omni/engine/backends/crisp_tts.py index e0eeab3e1..be76562f6 100644 --- a/vllm_omni/engine/backends/crisp_tts.py +++ b/vllm_omni/engine/backends/crisp_tts.py @@ -100,6 +100,7 @@ class CrispTTSStageClient(StageClientBase): self._request_timeout_s = float(config.get("request_timeout_s", 180)) self._voice = str(config.get("voice", "ryan")) self._seed = int(config.get("seed", 42)) + self._metric_name = "crisp_tts_wall_s" self._port = int(config.get("port") or _local_port()) try: if ( @@ -217,7 +218,7 @@ class CrispTTSStageClient(StageClientBase): async def add_request_async(self, request_id: str, prompt: Any, params: Any = None) -> None: self.check_health() if self._active is not None: - raise ResourceUnavailable("CrispASR stage has an unacknowledged request; capacity is one") + raise ResourceUnavailable("audio stage has an unacknowledged request; capacity is one") if not isinstance(prompt, dict) or not isinstance(prompt.get("text"), str) or not prompt["text"]: raise ValueError("TTS prompt must contain nonempty text") text = prompt["text"] @@ -261,7 +262,7 @@ class CrispTTSStageClient(StageClientBase): "pcm_sha256": hashlib.sha256(pcm).hexdigest(), }, }, - metrics={"crisp_tts_wall_s": wall_s}, + metrics={self._metric_name: wall_s}, ) except asyncio.CancelledError: raise @@ -336,7 +337,7 @@ class CrispTTSStageClient(StageClientBase): raise EngineDeadError() async def collective_rpc_async(self, method, timeout=None, args=(), kwargs=None): - raise NotImplementedError(f"CrispASR backend does not implement collective RPC {method}") + raise NotImplementedError(f"complete audio backend does not implement collective RPC {method}") def shutdown(self) -> None: self._closed = True diff --git a/vllm_omni/engine/backends/qwen_tts_cpu.py b/vllm_omni/engine/backends/qwen_tts_cpu.py new file mode 100644 index 000000000..3bd2b6450 --- /dev/null +++ b/vllm_omni/engine/backends/qwen_tts_cpu.py @@ -0,0 +1,205 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Native Windows CPU Qwen3-TTS whole-session stage with isolated dependencies. + +The worker alone imports Qwen's PyTorch wrapper and Transformers 4.57.3. +This adapter reuses the bounded complete-WAV request, fencing and ownership +methods from the CrispASR audio client; its model and launch path are separate. +""" + +from __future__ import annotations + +import asyncio +import json +import os +import subprocess +import time +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +from vllm_omni.engine.resource_ledger import ResourceUnavailable + +from .crisp_tts import CrispTTSStageClient, _local_port, _sha256 + + +class QwenTTSCPUStageClient(CrispTTSStageClient): + def __init__(self, metadata, config: dict, ledger, reservation) -> None: + for name, value in vars(metadata).items(): + setattr(self, name, value) + self.stage_type = "graph" # Omni complete-request control path. + self._ledger, self._reservation = ledger, reservation + self._generation = uuid.uuid4().hex + self._proc: subprocess.Popen | None = None + self._log_stream = None + self._closed = False + self._active = None + self._task = None + self._output = None + self._epoch = 0 + self._memory_pool = str(config.get("memory_pool", "host_ram")) + self._max_text_bytes = int(config.get("max_text_bytes", 4096)) + self._max_wav_bytes = int(config.get("max_wav_bytes", 4 << 20)) + self._max_audio_s = float(config.get("max_audio_s", 10)) + self._request_timeout_s = float(config.get("request_timeout_s", 180)) + self._voice = "Ryan" + self._seed = 42 + self._metric_name = "qwen_cpu_tts_wall_s" + self._port = int(config.get("port") or _local_port()) + try: + if ( + self._max_text_bytes <= 0 or self._max_wav_bytes <= 0 or self._max_audio_s <= 0 + or self._request_timeout_s <= 0 or not 0 < self._port < 65536 + ): + raise ValueError("invalid Qwen CPU TTS input, output, timeout or port bound") + if self._memory_pool not in reservation.demands: + raise ResourceUnavailable("CPU TTS memory pool absent from stage reservation") + demand = reservation.demands[self._memory_pool] + if self._max_wav_bytes > demand: + raise ResourceUnavailable("CPU TTS output bound exceeds stage reservation") + self._python = Path(config["python_bin"]).resolve(strict=True) + self._model_dir = Path(config["model_dir"]).resolve(strict=True) + self._overlay = Path(config["overlay_dir"]).resolve(strict=True) + self._talker = self._model_dir / "model.safetensors" + self._tokenizer = self._model_dir / "speech_tokenizer" / "model.safetensors" + self._worker = Path(__file__).with_name("qwen_tts_cpu_worker.py").resolve(strict=True) + expected = { + "python_sha256": (self._python, str(config["python_sha256"]).lower()), + "talker_sha256": (self._talker, str(config["talker_sha256"]).lower()), + "tokenizer_sha256": (self._tokenizer, str(config["tokenizer_sha256"]).lower()), + } + if not (self._overlay / "transformers" / "__init__.py").is_file(): + raise FileNotFoundError("pinned Transformers overlay is absent") + for label, (path, digest) in expected.items(): + if _sha256(path) != digest: + raise ValueError(f"CPU TTS {label} differs from the declared artifact hash") + overhead = int(config["memory_overhead_bytes"]) + weight_bytes = self._talker.stat().st_size + self._tokenizer.stat().st_size + if overhead <= 0 or weight_bytes + overhead > demand: + raise ResourceUnavailable("CPU TTS weights plus state/workspace/headroom exceed reservation") + self._log_path = Path(config["log_file"]).resolve() + self._log_path.parent.mkdir(parents=True, exist_ok=True) + self._log_stream = self._log_path.open("wb") + env = os.environ.copy() + env.update({ + "PYTHONPATH": str(self._overlay), + "QWEN_TTS_OVERLAY_DIR": str(self._overlay), + "CUDA_VISIBLE_DEVICES": "", + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", + "OMP_NUM_THREADS": "8", + "PYTHONUTF8": "1", + }) + command = [ + str(self._python), "-u", str(self._worker), + "--model-dir", str(self._model_dir), "--port", str(self._port), + "--threads", "8", + ] + flags = subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0 + self._proc = subprocess.Popen( + command, stdout=self._log_stream, stderr=subprocess.STDOUT, + env=env, creationflags=flags, + ) + base = f"http://127.0.0.1:{self._port}" + deadline = time.monotonic() + float(config.get("start_timeout_s", 180)) + while True: + if self._proc.poll() is not None: + raise RuntimeError(f"CPU TTS worker exited during load; see {self._log_path}") + try: + with urllib.request.urlopen(base + "/health", timeout=1) as response: + if response.status == 200: + break + except (urllib.error.URLError, TimeoutError, OSError): + pass + if time.monotonic() >= deadline: + raise TimeoutError(f"CPU TTS worker did not become ready; see {self._log_path}") + time.sleep(0.2) + with urllib.request.urlopen(base + "/props", timeout=5) as response: + props = json.load(response) + if ( + Path(props.get("model_dir", "")).resolve() != self._model_dir + or props.get("placement") != "cpu" + or props.get("transformers") != "4.57.3" + or props.get("qwen_tts") != "0.1.1" + or props.get("torch") != str(config["expected_torch"]) + or props.get("dtype") != "bfloat16" + or props.get("sample_rate") != 24000 + or props.get("speaker") != "Ryan" + or props.get("max_new_tokens") != 64 + ): + raise RuntimeError("CPU TTS worker environment, model or placement differs from plan") + import psutil + + root = psutil.Process(self._proc.pid) + processes = [root, *root.children(recursive=True)] + self._loaded_process_tree = [] + for process in processes: + memory = process.memory_full_info() + self._loaded_process_tree.append({ + "pid": process.pid, + "rss_bytes": int(memory.rss), + "private_bytes": int(getattr(memory, "private", memory.rss)), + }) + self._loaded_rss_bytes = sum(row["rss_bytes"] for row in self._loaded_process_tree) + self._loaded_private_bytes = sum(row["private_bytes"] for row in self._loaded_process_tree) + if max(self._loaded_rss_bytes, self._loaded_private_bytes) > demand: + raise ResourceUnavailable("loaded CPU TTS process tree exceeds stage reservation") + self._base_url = base + self.execution_plan = { + "backend": "external.qwen_tts.cpu.v1", + "stage_id": self.stage_id, + "worker_generation": self._generation, + "worker_pid": self._proc.pid, + "worker_script_sha256": _sha256(self._worker), + "artifact_sha256": {label: digest for label, (_, digest) in expected.items()}, + "model_dir": str(self._model_dir), + "placement": "CPU BF16 SDPA, 8 threads; isolated Transformers 4.57.3", + "worker_props": props, + "loaded_rss_bytes": self._loaded_rss_bytes, + "loaded_private_bytes": self._loaded_private_bytes, + "loaded_process_tree": self._loaded_process_tree, + "reserved_bytes": dict(reservation.demands), + "memory_overhead_bytes": overhead, + "max_text_bytes": self._max_text_bytes, + "max_wav_bytes": self._max_wav_bytes, + "max_audio_s": self._max_audio_s, + "voice": self._voice, + "seed": self._seed, + "request_capacity": 1, + "stateful_session": "Qwen PyTorch worker-owned; complete WAV per request", + "evidence": "B", + } + except BaseException: + self.shutdown() + raise + + def _terminate(self) -> bool: + import psutil + + children = [] + try: + if self._proc is not None: + children = psutil.Process(self._proc.pid).children(recursive=True) + except psutil.NoSuchProcess: + pass + except psutil.Error: + return False + drained = True + for child in reversed(children): + try: + child.terminate() + except psutil.NoSuchProcess: + pass + except psutil.Error: + drained = False + _, alive = psutil.wait_procs(children, timeout=5) + for child in alive: + try: + child.kill() + except psutil.NoSuchProcess: + pass + except psutil.Error: + drained = False + if psutil.wait_procs(alive, timeout=5)[1]: + drained = False + return super()._terminate() and drained diff --git a/vllm_omni/engine/backends/qwen_tts_cpu_worker.py b/vllm_omni/engine/backends/qwen_tts_cpu_worker.py new file mode 100644 index 000000000..6f629290b --- /dev/null +++ b/vllm_omni/engine/backends/qwen_tts_cpu_worker.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Isolated local HTTP worker for the official Qwen3-TTS CPU wrapper. + +This file is launched as a script by Omni's parent process. Its interpreter +loads a pinned Transformers 4.57 overlay; the Omni interpreter keeps its own +Transformers version. The worker owns all model and vocoder state. +""" + +from __future__ import annotations + +import argparse +import importlib.metadata +import io +import json +import os +import time +import traceback +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model-dir", type=Path, required=True) + parser.add_argument("--port", type=int, required=True) + parser.add_argument("--threads", type=int, default=8) + args = parser.parse_args() + if not 0 < args.port < 65536 or not 0 < args.threads <= 24: + parser.error("invalid port or CPU thread bound") + if os.environ.get("CUDA_VISIBLE_DEVICES") != "": + raise RuntimeError("CPU TTS worker requires CUDA_VISIBLE_DEVICES empty") + if os.environ.get("HF_HUB_OFFLINE") != "1" or os.environ.get("TRANSFORMERS_OFFLINE") != "1": + raise RuntimeError("CPU TTS worker requires offline model loading") + + import numpy as np + import soundfile as sf + import torch + import transformers + from qwen_tts import Qwen3TTSModel + + overlay = os.path.normcase(os.path.realpath(os.environ["QWEN_TTS_OVERLAY_DIR"])) + loaded_transformers = os.path.normcase(os.path.realpath(transformers.__file__)) + if transformers.__version__ != "4.57.3" or not loaded_transformers.startswith(overlay + os.sep): + raise RuntimeError("Qwen TTS worker loaded the wrong Transformers dependency") + if importlib.metadata.version("qwen-tts") != "0.1.1": + raise RuntimeError("Qwen TTS worker loaded the wrong model wrapper") + torch.set_num_threads(args.threads) + started = time.perf_counter() + model = Qwen3TTSModel.from_pretrained( + str(args.model_dir.resolve(strict=True)), + device_map="cpu", dtype=torch.bfloat16, attn_implementation="sdpa", + local_files_only=True, + ) + if {parameter.device.type for parameter in model.model.parameters()} != {"cpu"}: + raise RuntimeError("Qwen TTS worker loaded a non-CPU parameter") + if "ryan" not in model.get_supported_speakers(): + raise RuntimeError("pinned CustomVoice checkpoint does not provide Ryan") + props = { + "model_dir": str(args.model_dir.resolve()), + "placement": "cpu", + "torch": torch.__version__, + "transformers": transformers.__version__, + "qwen_tts": "0.1.1", + "dtype": "bfloat16", + "attention": "sdpa", + "threads": args.threads, + "sample_rate": 24000, + "speaker": "Ryan", + "max_new_tokens": 64, + "load_s": time.perf_counter() - started, + } + + class Handler(BaseHTTPRequestHandler): + def _json(self, status: int, payload: dict) -> None: + data = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_GET(self) -> None: + if self.path == "/health": + self._json(200, {"status": "ok"}) + elif self.path == "/props": + self._json(200, props) + else: + self._json(404, {"error": "unknown endpoint"}) + + def do_POST(self) -> None: + if self.path != "/v1/audio/speech": + self._json(404, {"error": "unknown endpoint"}) + return + size = int(self.headers.get("Content-Length", "0")) + if not 0 < size <= 8192: + self._json(413, {"error": "request exceeds worker bound"}) + return + try: + payload = json.loads(self.rfile.read(size)) + text = payload["input"] + if not isinstance(text, str) or not 0 < len(text.encode()) <= 4096: + raise ValueError("invalid bounded TTS input") + if payload.get("response_format") != "wav" or payload.get("seed") != 42: + raise ValueError("worker requires WAV and pinned seed 42") + torch.manual_seed(42) + started = time.perf_counter() + wavs, sample_rate = model.generate_custom_voice( + text=text, language="English", speaker="Ryan", + max_new_tokens=64, do_sample=True, non_streaming_mode=True, + ) + if len(wavs) != 1 or sample_rate != 24000: + raise RuntimeError("model did not return one 24 kHz waveform") + audio = np.asarray(wavs[0], dtype=np.float32).reshape(-1) + if not audio.size or not np.isfinite(audio).all(): + raise RuntimeError("model returned empty or non-finite audio") + with io.BytesIO() as buffer: + sf.write(buffer, audio, sample_rate, subtype="PCM_16", format="WAV") + data = buffer.getvalue() + if len(data) > (4 << 20): + raise RuntimeError("model returned audio above worker bound") + self.send_response(200) + self.send_header("Content-Type", "audio/wav") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + print(f"tts request duration_s={len(audio) / sample_rate:.3f} wall_s={time.perf_counter() - started:.3f}", flush=True) + except Exception as exc: + traceback.print_exc() + self._json(500, {"error": f"{type(exc).__name__}: {exc}"}) + + with HTTPServer(("127.0.0.1", args.port), Handler) as server: + print(f"qwen-tts-cpu-worker ready port={args.port} props={json.dumps(props)}", flush=True) + server.serve_forever(poll_interval=0.1) + + +if __name__ == "__main__": + main() diff --git a/vllm_omni/model_executor/models/qwen3_tts/pipeline.py b/vllm_omni/model_executor/models/qwen3_tts/pipeline.py index 6dce12d50..ee61de75a 100644 --- a/vllm_omni/model_executor/models/qwen3_tts/pipeline.py +++ b/vllm_omni/model_executor/models/qwen3_tts/pipeline.py @@ -100,3 +100,19 @@ QWEN3_TTS_CRISP_HYBRID_PIPELINE = PipelineConfig( ), ), ) + + +# Native Windows CPU reference path with the official PyTorch wrapper in an +# isolated worker, preserving Omni's installed Transformers dependency. +QWEN3_TTS_CPU_WHOLE_PIPELINE = PipelineConfig( + model_type="qwen3_tts_cpu_whole", + stages=( + StagePipelineConfig( + stage_id=0, + model_stage="audio", + execution_type=StageExecutionType.GRAPH, + final_output=True, + final_output_type="audio", + ), + ), +)