Add native Windows CPU Qwen TTS Omni stage and profile evidence
This commit is contained in:
1 parent
8569995591
commit
bb8735effc
24 files changed
+25090
-14
No files matched your search
@@ -17,4 +17,8 @@ def create_graph_client(metadata, config, ledger, reservation):
|
||||
from .crisp_tts import CrispTTSStageClient
|
||||
|
||||
return CrispTTSStageClient(metadata, backend, ledger, reservation)
|
||||
if name == "external.qwen_tts.cpu.v1":
|
||||
from .qwen_tts_cpu import QwenTTSCPUStageClient
|
||||
|
||||
return QwenTTSCPUStageClient(metadata, backend, ledger, reservation)
|
||||
raise ValueError(f"unsupported complete-request stage backend: {name!r}")
|
||||
@@ -100,6 +100,7 @@ class CrispTTSStageClient(StageClientBase):
|
||||
self._request_timeout_s = float(config.get("request_timeout_s", 180))
|
||||
self._voice = str(config.get("voice", "ryan"))
|
||||
self._seed = int(config.get("seed", 42))
|
||||
self._metric_name = "crisp_tts_wall_s"
|
||||
self._port = int(config.get("port") or _local_port())
|
||||
try:
|
||||
if (
|
||||
@@ -217,7 +218,7 @@ class CrispTTSStageClient(StageClientBase):
|
||||
async def add_request_async(self, request_id: str, prompt: Any, params: Any = None) -> None:
|
||||
self.check_health()
|
||||
if self._active is not None:
|
||||
raise ResourceUnavailable("CrispASR stage has an unacknowledged request; capacity is one")
|
||||
raise ResourceUnavailable("audio stage has an unacknowledged request; capacity is one")
|
||||
if not isinstance(prompt, dict) or not isinstance(prompt.get("text"), str) or not prompt["text"]:
|
||||
raise ValueError("TTS prompt must contain nonempty text")
|
||||
text = prompt["text"]
|
||||
@@ -261,7 +262,7 @@ class CrispTTSStageClient(StageClientBase):
|
||||
"pcm_sha256": hashlib.sha256(pcm).hexdigest(),
|
||||
},
|
||||
},
|
||||
metrics={"crisp_tts_wall_s": wall_s},
|
||||
metrics={self._metric_name: wall_s},
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
@@ -336,7 +337,7 @@ class CrispTTSStageClient(StageClientBase):
|
||||
raise EngineDeadError()
|
||||
|
||||
async def collective_rpc_async(self, method, timeout=None, args=(), kwargs=None):
|
||||
raise NotImplementedError(f"CrispASR backend does not implement collective RPC {method}")
|
||||
raise NotImplementedError(f"complete audio backend does not implement collective RPC {method}")
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._closed = True
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Native Windows CPU Qwen3-TTS whole-session stage with isolated dependencies.
|
||||
|
||||
The worker alone imports Qwen's PyTorch wrapper and Transformers 4.57.3.
|
||||
This adapter reuses the bounded complete-WAV request, fencing and ownership
|
||||
methods from the CrispASR audio client; its model and launch path are separate.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from vllm_omni.engine.resource_ledger import ResourceUnavailable
|
||||
|
||||
from .crisp_tts import CrispTTSStageClient, _local_port, _sha256
|
||||
|
||||
|
||||
class QwenTTSCPUStageClient(CrispTTSStageClient):
|
||||
def __init__(self, metadata, config: dict, ledger, reservation) -> None:
|
||||
for name, value in vars(metadata).items():
|
||||
setattr(self, name, value)
|
||||
self.stage_type = "graph" # Omni complete-request control path.
|
||||
self._ledger, self._reservation = ledger, reservation
|
||||
self._generation = uuid.uuid4().hex
|
||||
self._proc: subprocess.Popen | None = None
|
||||
self._log_stream = None
|
||||
self._closed = False
|
||||
self._active = None
|
||||
self._task = None
|
||||
self._output = None
|
||||
self._epoch = 0
|
||||
self._memory_pool = str(config.get("memory_pool", "host_ram"))
|
||||
self._max_text_bytes = int(config.get("max_text_bytes", 4096))
|
||||
self._max_wav_bytes = int(config.get("max_wav_bytes", 4 << 20))
|
||||
self._max_audio_s = float(config.get("max_audio_s", 10))
|
||||
self._request_timeout_s = float(config.get("request_timeout_s", 180))
|
||||
self._voice = "Ryan"
|
||||
self._seed = 42
|
||||
self._metric_name = "qwen_cpu_tts_wall_s"
|
||||
self._port = int(config.get("port") or _local_port())
|
||||
try:
|
||||
if (
|
||||
self._max_text_bytes <= 0 or self._max_wav_bytes <= 0 or self._max_audio_s <= 0
|
||||
or self._request_timeout_s <= 0 or not 0 < self._port < 65536
|
||||
):
|
||||
raise ValueError("invalid Qwen CPU TTS input, output, timeout or port bound")
|
||||
if self._memory_pool not in reservation.demands:
|
||||
raise ResourceUnavailable("CPU TTS memory pool absent from stage reservation")
|
||||
demand = reservation.demands[self._memory_pool]
|
||||
if self._max_wav_bytes > demand:
|
||||
raise ResourceUnavailable("CPU TTS output bound exceeds stage reservation")
|
||||
self._python = Path(config["python_bin"]).resolve(strict=True)
|
||||
self._model_dir = Path(config["model_dir"]).resolve(strict=True)
|
||||
self._overlay = Path(config["overlay_dir"]).resolve(strict=True)
|
||||
self._talker = self._model_dir / "model.safetensors"
|
||||
self._tokenizer = self._model_dir / "speech_tokenizer" / "model.safetensors"
|
||||
self._worker = Path(__file__).with_name("qwen_tts_cpu_worker.py").resolve(strict=True)
|
||||
expected = {
|
||||
"python_sha256": (self._python, str(config["python_sha256"]).lower()),
|
||||
"talker_sha256": (self._talker, str(config["talker_sha256"]).lower()),
|
||||
"tokenizer_sha256": (self._tokenizer, str(config["tokenizer_sha256"]).lower()),
|
||||
}
|
||||
if not (self._overlay / "transformers" / "__init__.py").is_file():
|
||||
raise FileNotFoundError("pinned Transformers overlay is absent")
|
||||
for label, (path, digest) in expected.items():
|
||||
if _sha256(path) != digest:
|
||||
raise ValueError(f"CPU TTS {label} differs from the declared artifact hash")
|
||||
overhead = int(config["memory_overhead_bytes"])
|
||||
weight_bytes = self._talker.stat().st_size + self._tokenizer.stat().st_size
|
||||
if overhead <= 0 or weight_bytes + overhead > demand:
|
||||
raise ResourceUnavailable("CPU TTS weights plus state/workspace/headroom exceed reservation")
|
||||
self._log_path = Path(config["log_file"]).resolve()
|
||||
self._log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._log_stream = self._log_path.open("wb")
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PYTHONPATH": str(self._overlay),
|
||||
"QWEN_TTS_OVERLAY_DIR": str(self._overlay),
|
||||
"CUDA_VISIBLE_DEVICES": "",
|
||||
"HF_HUB_OFFLINE": "1",
|
||||
"TRANSFORMERS_OFFLINE": "1",
|
||||
"OMP_NUM_THREADS": "8",
|
||||
"PYTHONUTF8": "1",
|
||||
})
|
||||
command = [
|
||||
str(self._python), "-u", str(self._worker),
|
||||
"--model-dir", str(self._model_dir), "--port", str(self._port),
|
||||
"--threads", "8",
|
||||
]
|
||||
flags = subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0
|
||||
self._proc = subprocess.Popen(
|
||||
command, stdout=self._log_stream, stderr=subprocess.STDOUT,
|
||||
env=env, creationflags=flags,
|
||||
)
|
||||
base = f"http://127.0.0.1:{self._port}"
|
||||
deadline = time.monotonic() + float(config.get("start_timeout_s", 180))
|
||||
while True:
|
||||
if self._proc.poll() is not None:
|
||||
raise RuntimeError(f"CPU TTS worker exited during load; see {self._log_path}")
|
||||
try:
|
||||
with urllib.request.urlopen(base + "/health", timeout=1) as response:
|
||||
if response.status == 200:
|
||||
break
|
||||
except (urllib.error.URLError, TimeoutError, OSError):
|
||||
pass
|
||||
if time.monotonic() >= deadline:
|
||||
raise TimeoutError(f"CPU TTS worker did not become ready; see {self._log_path}")
|
||||
time.sleep(0.2)
|
||||
with urllib.request.urlopen(base + "/props", timeout=5) as response:
|
||||
props = json.load(response)
|
||||
if (
|
||||
Path(props.get("model_dir", "")).resolve() != self._model_dir
|
||||
or props.get("placement") != "cpu"
|
||||
or props.get("transformers") != "4.57.3"
|
||||
or props.get("qwen_tts") != "0.1.1"
|
||||
or props.get("torch") != str(config["expected_torch"])
|
||||
or props.get("dtype") != "bfloat16"
|
||||
or props.get("sample_rate") != 24000
|
||||
or props.get("speaker") != "Ryan"
|
||||
or props.get("max_new_tokens") != 64
|
||||
):
|
||||
raise RuntimeError("CPU TTS worker environment, model or placement differs from plan")
|
||||
import psutil
|
||||
|
||||
root = psutil.Process(self._proc.pid)
|
||||
processes = [root, *root.children(recursive=True)]
|
||||
self._loaded_process_tree = []
|
||||
for process in processes:
|
||||
memory = process.memory_full_info()
|
||||
self._loaded_process_tree.append({
|
||||
"pid": process.pid,
|
||||
"rss_bytes": int(memory.rss),
|
||||
"private_bytes": int(getattr(memory, "private", memory.rss)),
|
||||
})
|
||||
self._loaded_rss_bytes = sum(row["rss_bytes"] for row in self._loaded_process_tree)
|
||||
self._loaded_private_bytes = sum(row["private_bytes"] for row in self._loaded_process_tree)
|
||||
if max(self._loaded_rss_bytes, self._loaded_private_bytes) > demand:
|
||||
raise ResourceUnavailable("loaded CPU TTS process tree exceeds stage reservation")
|
||||
self._base_url = base
|
||||
self.execution_plan = {
|
||||
"backend": "external.qwen_tts.cpu.v1",
|
||||
"stage_id": self.stage_id,
|
||||
"worker_generation": self._generation,
|
||||
"worker_pid": self._proc.pid,
|
||||
"worker_script_sha256": _sha256(self._worker),
|
||||
"artifact_sha256": {label: digest for label, (_, digest) in expected.items()},
|
||||
"model_dir": str(self._model_dir),
|
||||
"placement": "CPU BF16 SDPA, 8 threads; isolated Transformers 4.57.3",
|
||||
"worker_props": props,
|
||||
"loaded_rss_bytes": self._loaded_rss_bytes,
|
||||
"loaded_private_bytes": self._loaded_private_bytes,
|
||||
"loaded_process_tree": self._loaded_process_tree,
|
||||
"reserved_bytes": dict(reservation.demands),
|
||||
"memory_overhead_bytes": overhead,
|
||||
"max_text_bytes": self._max_text_bytes,
|
||||
"max_wav_bytes": self._max_wav_bytes,
|
||||
"max_audio_s": self._max_audio_s,
|
||||
"voice": self._voice,
|
||||
"seed": self._seed,
|
||||
"request_capacity": 1,
|
||||
"stateful_session": "Qwen PyTorch worker-owned; complete WAV per request",
|
||||
"evidence": "B",
|
||||
}
|
||||
except BaseException:
|
||||
self.shutdown()
|
||||
raise
|
||||
|
||||
def _terminate(self) -> bool:
|
||||
import psutil
|
||||
|
||||
children = []
|
||||
try:
|
||||
if self._proc is not None:
|
||||
children = psutil.Process(self._proc.pid).children(recursive=True)
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
except psutil.Error:
|
||||
return False
|
||||
drained = True
|
||||
for child in reversed(children):
|
||||
try:
|
||||
child.terminate()
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
except psutil.Error:
|
||||
drained = False
|
||||
_, alive = psutil.wait_procs(children, timeout=5)
|
||||
for child in alive:
|
||||
try:
|
||||
child.kill()
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
except psutil.Error:
|
||||
drained = False
|
||||
if psutil.wait_procs(alive, timeout=5)[1]:
|
||||
drained = False
|
||||
return super()._terminate() and drained
|
||||
@@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Isolated local HTTP worker for the official Qwen3-TTS CPU wrapper.
|
||||
|
||||
This file is launched as a script by Omni's parent process. Its interpreter
|
||||
loads a pinned Transformers 4.57 overlay; the Omni interpreter keeps its own
|
||||
Transformers version. The worker owns all model and vocoder state.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import importlib.metadata
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import traceback
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--model-dir", type=Path, required=True)
|
||||
parser.add_argument("--port", type=int, required=True)
|
||||
parser.add_argument("--threads", type=int, default=8)
|
||||
args = parser.parse_args()
|
||||
if not 0 < args.port < 65536 or not 0 < args.threads <= 24:
|
||||
parser.error("invalid port or CPU thread bound")
|
||||
if os.environ.get("CUDA_VISIBLE_DEVICES") != "":
|
||||
raise RuntimeError("CPU TTS worker requires CUDA_VISIBLE_DEVICES empty")
|
||||
if os.environ.get("HF_HUB_OFFLINE") != "1" or os.environ.get("TRANSFORMERS_OFFLINE") != "1":
|
||||
raise RuntimeError("CPU TTS worker requires offline model loading")
|
||||
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
import torch
|
||||
import transformers
|
||||
from qwen_tts import Qwen3TTSModel
|
||||
|
||||
overlay = os.path.normcase(os.path.realpath(os.environ["QWEN_TTS_OVERLAY_DIR"]))
|
||||
loaded_transformers = os.path.normcase(os.path.realpath(transformers.__file__))
|
||||
if transformers.__version__ != "4.57.3" or not loaded_transformers.startswith(overlay + os.sep):
|
||||
raise RuntimeError("Qwen TTS worker loaded the wrong Transformers dependency")
|
||||
if importlib.metadata.version("qwen-tts") != "0.1.1":
|
||||
raise RuntimeError("Qwen TTS worker loaded the wrong model wrapper")
|
||||
torch.set_num_threads(args.threads)
|
||||
started = time.perf_counter()
|
||||
model = Qwen3TTSModel.from_pretrained(
|
||||
str(args.model_dir.resolve(strict=True)),
|
||||
device_map="cpu", dtype=torch.bfloat16, attn_implementation="sdpa",
|
||||
local_files_only=True,
|
||||
)
|
||||
if {parameter.device.type for parameter in model.model.parameters()} != {"cpu"}:
|
||||
raise RuntimeError("Qwen TTS worker loaded a non-CPU parameter")
|
||||
if "ryan" not in model.get_supported_speakers():
|
||||
raise RuntimeError("pinned CustomVoice checkpoint does not provide Ryan")
|
||||
props = {
|
||||
"model_dir": str(args.model_dir.resolve()),
|
||||
"placement": "cpu",
|
||||
"torch": torch.__version__,
|
||||
"transformers": transformers.__version__,
|
||||
"qwen_tts": "0.1.1",
|
||||
"dtype": "bfloat16",
|
||||
"attention": "sdpa",
|
||||
"threads": args.threads,
|
||||
"sample_rate": 24000,
|
||||
"speaker": "Ryan",
|
||||
"max_new_tokens": 64,
|
||||
"load_s": time.perf_counter() - started,
|
||||
}
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def _json(self, status: int, payload: dict) -> None:
|
||||
data = json.dumps(payload).encode()
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(data)))
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
|
||||
def do_GET(self) -> None:
|
||||
if self.path == "/health":
|
||||
self._json(200, {"status": "ok"})
|
||||
elif self.path == "/props":
|
||||
self._json(200, props)
|
||||
else:
|
||||
self._json(404, {"error": "unknown endpoint"})
|
||||
|
||||
def do_POST(self) -> None:
|
||||
if self.path != "/v1/audio/speech":
|
||||
self._json(404, {"error": "unknown endpoint"})
|
||||
return
|
||||
size = int(self.headers.get("Content-Length", "0"))
|
||||
if not 0 < size <= 8192:
|
||||
self._json(413, {"error": "request exceeds worker bound"})
|
||||
return
|
||||
try:
|
||||
payload = json.loads(self.rfile.read(size))
|
||||
text = payload["input"]
|
||||
if not isinstance(text, str) or not 0 < len(text.encode()) <= 4096:
|
||||
raise ValueError("invalid bounded TTS input")
|
||||
if payload.get("response_format") != "wav" or payload.get("seed") != 42:
|
||||
raise ValueError("worker requires WAV and pinned seed 42")
|
||||
torch.manual_seed(42)
|
||||
started = time.perf_counter()
|
||||
wavs, sample_rate = model.generate_custom_voice(
|
||||
text=text, language="English", speaker="Ryan",
|
||||
max_new_tokens=64, do_sample=True, non_streaming_mode=True,
|
||||
)
|
||||
if len(wavs) != 1 or sample_rate != 24000:
|
||||
raise RuntimeError("model did not return one 24 kHz waveform")
|
||||
audio = np.asarray(wavs[0], dtype=np.float32).reshape(-1)
|
||||
if not audio.size or not np.isfinite(audio).all():
|
||||
raise RuntimeError("model returned empty or non-finite audio")
|
||||
with io.BytesIO() as buffer:
|
||||
sf.write(buffer, audio, sample_rate, subtype="PCM_16", format="WAV")
|
||||
data = buffer.getvalue()
|
||||
if len(data) > (4 << 20):
|
||||
raise RuntimeError("model returned audio above worker bound")
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "audio/wav")
|
||||
self.send_header("Content-Length", str(len(data)))
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
print(f"tts request duration_s={len(audio) / sample_rate:.3f} wall_s={time.perf_counter() - started:.3f}", flush=True)
|
||||
except Exception as exc:
|
||||
traceback.print_exc()
|
||||
self._json(500, {"error": f"{type(exc).__name__}: {exc}"})
|
||||
|
||||
with HTTPServer(("127.0.0.1", args.port), Handler) as server:
|
||||
print(f"qwen-tts-cpu-worker ready port={args.port} props={json.dumps(props)}", flush=True)
|
||||
server.serve_forever(poll_interval=0.1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user