Add native Windows CPU Qwen TTS Omni stage and profile evidence
This commit is contained in:
tzhouam committed
1 parent
8569995591
commit
bb8735effc
24 files changed
+25090
-14
No files matched your search
@@ -0,0 +1,216 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Profile the isolated official Qwen3-TTS CPU wrapper through Omni."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import time
|
||||
import wave
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
def _rank(values: list[float], fraction: float) -> float:
|
||||
return sorted(values)[math.ceil(len(values) * fraction) - 1]
|
||||
|
||||
|
||||
def _save_wav(path: Path, audio) -> None:
|
||||
pcm = np.rint(audio.numpy() * 32768).astype("<i2").tobytes()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with wave.open(str(path), "wb") as wav:
|
||||
wav.setnchannels(1)
|
||||
wav.setsampwidth(2)
|
||||
wav.setframerate(24000)
|
||||
wav.writeframes(pcm)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
for name in (
|
||||
"python_bin", "python_sha256", "model_dir", "overlay_dir", "talker_sha256",
|
||||
"tokenizer_sha256", "server_log", "output_report", "output_wav",
|
||||
):
|
||||
parser.add_argument("--" + name.replace("_", "-"), required=True)
|
||||
parser.add_argument("--warmups", type=int, default=1)
|
||||
parser.add_argument("--repeats", type=int, default=20)
|
||||
parser.add_argument("--reserve-gib", type=int, default=10)
|
||||
parser.add_argument("--abort-check", action="store_true")
|
||||
args = parser.parse_args()
|
||||
if args.warmups < 0 or args.repeats <= 0 or not 0 < args.reserve_gib <= 16:
|
||||
parser.error("invalid profile or budget")
|
||||
|
||||
from vllm_omni.config.stage_config import DeployConfig, StageDeployConfig, merge_pipeline_deploy
|
||||
from vllm_omni.engine.stage_runtime import StageRuntime
|
||||
from vllm_omni.model_executor.models.qwen3_tts.pipeline import QWEN3_TTS_CPU_WHOLE_PIPELINE
|
||||
|
||||
backend = {
|
||||
"name": "external.qwen_tts.cpu.v1",
|
||||
"python_bin": args.python_bin,
|
||||
"python_sha256": args.python_sha256,
|
||||
"model_dir": args.model_dir,
|
||||
"overlay_dir": args.overlay_dir,
|
||||
"talker_sha256": args.talker_sha256,
|
||||
"tokenizer_sha256": args.tokenizer_sha256,
|
||||
"log_file": args.server_log,
|
||||
"memory_overhead_bytes": 6 << 30,
|
||||
"expected_torch": "2.13.0+cu130",
|
||||
"max_text_bytes": 4096,
|
||||
"max_wav_bytes": 4 << 20,
|
||||
"max_audio_s": 10,
|
||||
"request_timeout_s": 180,
|
||||
"start_timeout_s": 180,
|
||||
}
|
||||
deploy = DeployConfig(
|
||||
async_chunk=False,
|
||||
stages=[StageDeployConfig(
|
||||
stage_id=0,
|
||||
backend=backend,
|
||||
resource_budget={
|
||||
"capacities": {"host_ram": 16 << 30},
|
||||
"demands": {"host_ram": args.reserve_gib << 30},
|
||||
},
|
||||
)],
|
||||
)
|
||||
pipeline = QWEN3_TTS_CPU_WHOLE_PIPELINE
|
||||
configs = [stage.to_omegaconf() for stage in merge_pipeline_deploy(pipeline, deploy)]
|
||||
runtime = StageRuntime(configs, "local-qwen-tts-cpu", "", stage_init_timeout=180, async_chunk=False)
|
||||
report = {
|
||||
"scope": "complete resident native Windows CPU Qwen3-TTS text-to-WAV through Omni",
|
||||
"warmup_count": args.warmups,
|
||||
"measured_count": args.repeats,
|
||||
"memory_budget": deploy.stages[0].resource_budget,
|
||||
}
|
||||
sample_task = None
|
||||
stop_sampling = asyncio.Event()
|
||||
try:
|
||||
started = time.perf_counter()
|
||||
runtime.initialize()
|
||||
report["startup_s"] = time.perf_counter() - started
|
||||
pool = runtime.stage_pools[0]
|
||||
report["execution_plan"] = pool.stage_client.execution_plan
|
||||
state = SimpleNamespace(sampling_params_list=[None])
|
||||
import psutil
|
||||
|
||||
memory_samples = []
|
||||
|
||||
async def sample_memory() -> None:
|
||||
while not stop_sampling.is_set():
|
||||
try:
|
||||
root = psutil.Process(pool.stage_client._proc.pid)
|
||||
members = [root, *root.children(recursive=True)]
|
||||
memories = [process.memory_full_info() for process in members]
|
||||
memory_samples.append({
|
||||
"unix": time.time(),
|
||||
"rss_bytes": sum(int(memory.rss) for memory in memories),
|
||||
"private_bytes": sum(
|
||||
int(getattr(memory, "private", memory.rss)) for memory in memories
|
||||
),
|
||||
"host_available_bytes": psutil.virtual_memory().available,
|
||||
"pids": [process.pid for process in members],
|
||||
})
|
||||
except psutil.Error:
|
||||
break
|
||||
await asyncio.sleep(0.25)
|
||||
|
||||
sample_task = asyncio.create_task(sample_memory())
|
||||
|
||||
async def request_one(text: str, *, save: bool = False) -> dict:
|
||||
request_id = f"cpu-tts-{time.monotonic_ns()}"
|
||||
started = time.perf_counter()
|
||||
await pool.submit_initial(request_id, state, {"text": text, "voice": "Ryan", "seed": 42})
|
||||
deadline = time.monotonic() + 180
|
||||
while True:
|
||||
output = pool.poll_graph_output(0)
|
||||
if output is not None:
|
||||
break
|
||||
if time.monotonic() >= deadline:
|
||||
raise TimeoutError("Omni Qwen CPU TTS stage request timed out")
|
||||
await asyncio.sleep(0.005)
|
||||
wall_s = time.perf_counter() - started
|
||||
try:
|
||||
if output.error:
|
||||
raise RuntimeError(output.error)
|
||||
payload = output.multimodal_output
|
||||
audio, sample_rate = payload["audio"], payload["sr"]
|
||||
metadata = output.custom_output["audio_metadata"]
|
||||
assert sample_rate == 24000 and audio.ndim == 1
|
||||
assert len(audio) == metadata["pcm_frames"] and len(audio) > 0
|
||||
assert float(audio.square().mean().sqrt()) > 1e-5
|
||||
if save:
|
||||
_save_wav(Path(args.output_wav), audio)
|
||||
pcm = np.rint(audio.numpy() * 32768).astype("<i2").tobytes()
|
||||
assert hashlib.sha256(pcm).hexdigest() == metadata["pcm_sha256"]
|
||||
return {
|
||||
"text": text,
|
||||
"wall_s": wall_s,
|
||||
"frames": len(audio),
|
||||
"duration_s": metadata["duration_s"],
|
||||
"pcm_sha256": metadata["pcm_sha256"],
|
||||
"stage_event": output.custom_output["stage_event"],
|
||||
}
|
||||
finally:
|
||||
output.release_stage_buffers()
|
||||
await asyncio.sleep(0)
|
||||
|
||||
first = "Hello from the local computer."
|
||||
second = "The blue car is parked beside the library."
|
||||
report["checks"] = [await request_one(first), await request_one(second)]
|
||||
expected = (
|
||||
"c8a49b5c73efd642a1eb04be973d6aefdc8ed9cfd18b34ff67690b8464906bed",
|
||||
"4edc7a62f749b1fde5ea39bedbc15a35f291ab1db8d24d7b7eb3df7270885e16",
|
||||
)
|
||||
report["standalone_pcm_parity"] = [
|
||||
row["pcm_sha256"] == digest for row, digest in zip(report["checks"], expected)
|
||||
]
|
||||
if report["standalone_pcm_parity"] != [True, True]:
|
||||
raise RuntimeError("CPU Omni PCM differs from the same checkpoint's standalone output")
|
||||
report["warmups"] = [await request_one(first) for _ in range(args.warmups)]
|
||||
report["measured"] = [await request_one(first, save=i == 0) for i in range(args.repeats)]
|
||||
walls = [row["wall_s"] for row in report["measured"]]
|
||||
report["nearest_rank_p50_wall_s"] = _rank(walls, 0.5)
|
||||
report["nearest_rank_p95_wall_s"] = _rank(walls, 0.95)
|
||||
report["all_same_pcm_sha256"] = len({row["pcm_sha256"] for row in report["measured"]}) == 1
|
||||
report["memory_samples"] = memory_samples
|
||||
report["sampled_max_rss_bytes"] = max((row["rss_bytes"] for row in memory_samples), default=None)
|
||||
report["sampled_max_private_bytes"] = max(
|
||||
(row["private_bytes"] for row in memory_samples), default=None
|
||||
)
|
||||
if args.abort_check:
|
||||
request_id = f"cpu-tts-abort-{time.monotonic_ns()}"
|
||||
await pool.submit_initial(request_id, state, {"text": second, "voice": "Ryan", "seed": 42})
|
||||
await asyncio.sleep(0.01)
|
||||
await pool.abort_requests([request_id])
|
||||
await asyncio.sleep(0.1)
|
||||
report["abort_check"] = {
|
||||
"stale_output": pool.poll_graph_output(0) is not None,
|
||||
"worker_exited": pool.stage_client._proc.poll() is not None,
|
||||
"ledger_after_abort": runtime.resource_ledger.snapshot(),
|
||||
}
|
||||
if report["abort_check"]["stale_output"] or not report["abort_check"]["worker_exited"]:
|
||||
raise RuntimeError("CPU TTS cancellation left stale output or live worker")
|
||||
report["status"] = "passed"
|
||||
except BaseException as exc:
|
||||
report["status"] = "failed"
|
||||
report["error"] = f"{type(exc).__name__}: {exc}"
|
||||
raise
|
||||
finally:
|
||||
stop_sampling.set()
|
||||
if sample_task is not None:
|
||||
await sample_task
|
||||
runtime.shutdown()
|
||||
if runtime.resource_ledger is not None:
|
||||
report["ledger_after_shutdown"] = runtime.resource_ledger.snapshot()
|
||||
destination = Path(args.output_report)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(report, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify public AsyncOmni WAV delivery from the isolated CPU Qwen3-TTS worker."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import yaml
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
for name in (
|
||||
"python_bin", "python_sha256", "model_dir", "overlay_dir", "talker_sha256",
|
||||
"tokenizer_sha256", "server_log", "output_report",
|
||||
):
|
||||
parser.add_argument("--" + name.replace("_", "-"), required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.model_executor.models.qwen3_tts.pipeline import QWEN3_TTS_CPU_WHOLE_PIPELINE
|
||||
|
||||
backend = {
|
||||
"name": "external.qwen_tts.cpu.v1",
|
||||
"python_bin": args.python_bin,
|
||||
"python_sha256": args.python_sha256,
|
||||
"model_dir": args.model_dir,
|
||||
"overlay_dir": args.overlay_dir,
|
||||
"talker_sha256": args.talker_sha256,
|
||||
"tokenizer_sha256": args.tokenizer_sha256,
|
||||
"log_file": args.server_log,
|
||||
"memory_overhead_bytes": 6 << 30,
|
||||
"expected_torch": "2.13.0+cu130",
|
||||
"max_text_bytes": 4096,
|
||||
"max_wav_bytes": 4 << 20,
|
||||
"max_audio_s": 10,
|
||||
"request_timeout_s": 180,
|
||||
"start_timeout_s": 180,
|
||||
}
|
||||
report = {"entrypoint": "AsyncOmni.generate", "pipeline": QWEN3_TTS_CPU_WHOLE_PIPELINE.model_type}
|
||||
engine = None
|
||||
try:
|
||||
with tempfile.TemporaryDirectory(prefix="omni-qwen-cpu-tts-") as temporary:
|
||||
deployment = Path(temporary) / "deploy.yaml"
|
||||
deployment.write_text(yaml.safe_dump({
|
||||
"pipeline": QWEN3_TTS_CPU_WHOLE_PIPELINE.model_type,
|
||||
"async_chunk": False,
|
||||
"stages": [{
|
||||
"stage_id": 0,
|
||||
"backend": backend,
|
||||
"resource_budget": {
|
||||
"capacities": {"host_ram": 16 << 30},
|
||||
"demands": {"host_ram": 10 << 30},
|
||||
},
|
||||
}],
|
||||
}), encoding="utf-8")
|
||||
started = time.perf_counter()
|
||||
engine = AsyncOmni(
|
||||
model=str(Path(args.model_dir).resolve(strict=True)),
|
||||
deploy_config=str(deployment), stage_init_timeout=180, init_timeout=240,
|
||||
)
|
||||
report["startup_s"] = time.perf_counter() - started
|
||||
outputs = []
|
||||
started = time.perf_counter()
|
||||
async for output in engine.generate(
|
||||
{"text": "Hello from the local computer.", "voice": "Ryan", "seed": 42},
|
||||
request_id="public-cpu-tts-1",
|
||||
):
|
||||
if output.error:
|
||||
raise RuntimeError(output.error)
|
||||
audio = output.multimodal_output["audio"]
|
||||
pcm = np.rint(audio.numpy() * 32768).astype("<i2").tobytes()
|
||||
outputs.append({
|
||||
"request_id": output.request_id,
|
||||
"sample_rate": output.multimodal_output["sr"],
|
||||
"frames": len(audio),
|
||||
"pcm_sha256": hashlib.sha256(pcm).hexdigest(),
|
||||
"stage_event": output.custom_output.get("stage_event"),
|
||||
})
|
||||
report["request_wall_s"] = time.perf_counter() - started
|
||||
report["outputs"] = outputs
|
||||
assert len(outputs) == 1 and outputs[0]["request_id"] == "public-cpu-tts-1"
|
||||
assert outputs[0]["sample_rate"] == 24000 and outputs[0]["frames"] == 109440
|
||||
assert outputs[0]["pcm_sha256"] == "c8a49b5c73efd642a1eb04be973d6aefdc8ed9cfd18b34ff67690b8464906bed"
|
||||
report["status"] = "passed"
|
||||
except BaseException as exc:
|
||||
report["status"] = "failed"
|
||||
report["error"] = f"{type(exc).__name__}: {exc}"
|
||||
raise
|
||||
finally:
|
||||
if engine is not None:
|
||||
engine.shutdown()
|
||||
destination = Path(args.output_report)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(report, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -1,12 +1,13 @@
|
||||
# 2026-09-23 WSL CPU and native Windows expansion
|
||||
|
||||
This extends the [60-cell device × model matrix](../e2e_profiling_20260922/evidence/summary/README.md) with previously unrun WSL CPU and native Windows paths. The September 22 timing tables remain historical. MiniCPM-o now has separate 20-request serial text-to-speech timing runs on WSL CPU and native Windows GPU, plus one synthetic image and one synthetic audio input on each platform; each InternVLA timing remains one observation. Spark has a 20-request Omni output-head component profile on the HX370 AMD NPU and separate 20-request **complete text** profiles through standalone llama.cpp on native Windows CPU and Radeon 890M. The latter now also run as bounded whole-session Omni stages, with public AsyncOmni requests checked separately. Qwen3-TTS has a 20-request complete standalone text-to-WAV profile on native Windows CPU; the CPU+Radeon 890M hybrid route now has separate standalone and bounded Omni whole-session 20-request profiles. None is a sustained or full-modality profile; the native CPU TTS path still needs Omni integration. The [checkpoint manifests](../e2e_recovery_20260923/README.md) pin the earlier real weights; the [Spark NPU component record](evidence/spark_amd_npu_output_head/README.md), [standalone CPU/Radeon whole-model record](evidence/spark_radeon890m_llamacpp/README.md), [Omni CPU/Radeon whole-session record](evidence/spark_omni_llamacpp/README.md), [native CPU Qwen TTS record](evidence/qwen_tts_native_cpu/README.md), [standalone joint CPU+Radeon TTS record](evidence/qwen_tts_radeon890m_joint/README.md), and [Omni joint TTS record](evidence/qwen_tts_omni_hybrid/README.md) pin their respective artifacts. No model, precision or context was silently substituted.
|
||||
This extends the [60-cell device × model matrix](../e2e_profiling_20260922/evidence/summary/README.md) with previously unrun WSL CPU and native Windows paths. The September 22 timing tables remain historical. MiniCPM-o now has separate 20-request serial text-to-speech timing runs on WSL CPU and native Windows GPU, plus one synthetic image and one synthetic audio input on each platform; each InternVLA timing remains one observation. Spark has a 20-request Omni output-head component profile on the HX370 AMD NPU and separate 20-request **complete text** profiles through standalone llama.cpp on native Windows CPU and Radeon 890M. The latter now also run as bounded whole-session Omni stages, with public AsyncOmni requests checked separately. Qwen3-TTS has separate 20-request standalone and bounded Omni whole-session text-to-WAV profiles on native Windows CPU; the CPU+Radeon 890M hybrid route likewise has standalone and Omni profiles. None is a sustained or full-modality profile. The [checkpoint manifests](../e2e_recovery_20260923/README.md) pin the earlier real weights; the [Spark NPU component record](evidence/spark_amd_npu_output_head/README.md), [standalone CPU/Radeon whole-model record](evidence/spark_radeon890m_llamacpp/README.md), [Omni CPU/Radeon whole-session record](evidence/spark_omni_llamacpp/README.md), [standalone CPU Qwen TTS record](evidence/qwen_tts_native_cpu/README.md), [Omni CPU Qwen TTS record](evidence/qwen_tts_omni_cpu/README.md), [standalone joint CPU+Radeon TTS record](evidence/qwen_tts_radeon890m_joint/README.md), and [Omni joint TTS record](evidence/qwen_tts_omni_hybrid/README.md) pin their respective artifacts. No model, precision or context was silently substituted.
|
||||
|
||||
| Device / model | Verified depth | Observed output and time | Remaining gate |
|
||||
|---|---|---|---|
|
||||
| Native Windows HX370 CPU+Radeon 890M / Qwen3-TTS 0.6B CustomVoice Q8_0 GGUF plus F16 codec | Scoped complete standalone CrispASR text-to-WAV path with Vulkan talker/codec and CPU FP32 code predictor (P for two named sentences; outside Omni; **not** all-Radeon or NPU) | Two 24 kHz WAVs of 2.56 s and 3.20 s; independently pinned Whisper tiny.en ASR WER 0 on both. One warmup + 20 serial measured resident-server requests for the first prompt had client wall p50/p95 3.310/3.345 s, all PCM hashes identical; RTF 1.29/1.31. The default Vulkan setting fell back to CPU, while all-Vulkan talker+predictor hit a 360-frame runaway cap. | Explicit Omni hybrid stage binding, quality and playback gates, loading-peak admission, cancellation, broader prompts, concurrency and sustained power/thermal behavior; [joint raw evidence](evidence/qwen_tts_radeon890m_joint/README.md), [failed DirectML attempt](evidence/qwen_tts_radeon890m_directml/README.md). |
|
||||
| Native Windows HX370 CPU+Radeon 890M / same Qwen3-TTS 0.6B CustomVoice GGUFs | Scoped complete-request Omni CrispASR hybrid stage (P for two named sentences; **not** all-Radeon or NPU) | Both 24 kHz PCM outputs exactly match the earlier standalone hashes. One warmup + 20 serial Omni requests for 2.56 s of audio had 20/20 identical PCM, nearest-rank wall p50/p95 3.298/3.336 s (RTF 1.29/1.30). Public AsyncOmni audio passed; cancellation drained the worker and ledger. | Playable streaming, post-cancel restart, broad speech quality, loading/iGPU peak, concurrency and sustained real-time/thermal gates; [Omni raw evidence](evidence/qwen_tts_omni_hybrid/README.md). |
|
||||
| Native Windows HX370 CPU / Qwen3-TTS 0.6B CustomVoice, revision `85e237c12c027371202489a0ec509ded67b5e4b5` | Scoped complete standalone PyTorch text-to-WAV path (P for two named sentences; outside Omni) | Nonzero 24 kHz WAVs of 4.56 s and 3.44 s. For the first prompt, 1 warmup + 20 serial measured complete generations p50/p95 13.94/14.68 s; Whisper tiny.en ASR WER 0.20 on the first and 0.00 on the second sentence. | Omni CPU stage binding and dependency isolation, playable streaming, speech quality, memory loading peak and sustained behavior; [raw evidence](evidence/qwen_tts_native_cpu/README.md). |
|
||||
| Native Windows HX370 CPU / Qwen3-TTS 0.6B CustomVoice, revision `85e237c12c027371202489a0ec509ded67b5e4b5` | Scoped complete standalone PyTorch text-to-WAV path (P for two named sentences; outside Omni) | Nonzero 24 kHz WAVs of 4.56 s and 3.44 s. For the first prompt, 1 warmup + 20 serial measured complete generations p50/p95 13.94/14.68 s; Whisper tiny.en ASR WER 0.20 on the first and 0.00 on the second sentence. | See the subsequent Omni CPU binding below; playable streaming, speech quality, memory loading peak and sustained behavior remain open; [standalone raw evidence](evidence/qwen_tts_native_cpu/README.md). |
|
||||
| Native Windows HX370 CPU / same Qwen3-TTS 0.6B CustomVoice checkpoint | Scoped complete-request Omni whole-session PyTorch text-to-WAV path (P for two named sentences) | Both 24 kHz PCM hashes exactly match the standalone outputs. One warmup + 20 serial measured Omni requests for 4.56 s of audio had 20/20 identical PCM and nearest-rank request-wall p50/p95 15.32/15.60 s (RTF 3.36/3.42). Public AsyncOmni audio, 10 GiB admission, 4 GiB refusal, cancellation drain and ledger release passed. Whisper tiny.en ASR WER was 0.20 for the first Omni WAV. | Playable streaming, post-cancel restart, broader speech quality, loading peak, concurrency and sustained use; [Omni raw evidence](evidence/qwen_tts_omni_cpu/README.md). |
|
||||
| Native Windows HX370 CPU / Spark-X2.5-1.7B Q4_K_M GGUF, revision `1f7fa33b1245c14730da39e125714ad3a327901b` | Scoped complete standalone llama.cpp text path (P for two named inputs; outside Omni) | Correct `Paris` and 120-item inventory answer `1511`; 1 warmup + 20 uncached serial full inventory requests p50/p95 6.65/6.86 s, 20/20 correct. Stream-close recovery passed. | Omni whole-model stage binding, broader quality, explicit loading-peak admission and sustained behavior; [raw evidence](evidence/spark_radeon890m_llamacpp/README.md). |
|
||||
| Native Windows HX370 Radeon 890M / Spark-X2.5-1.7B same GGUF | Scoped complete standalone llama.cpp Vulkan1 text path (P for two named inputs; outside Omni) | All 29 logged layer assignments on Vulkan1, with a separate Vulkan_Host buffer; correct `Paris` and `1511`; 1 warmup + 20 uncached serial inventory requests p50/p95 2.18/2.20 s, 20/20 correct. Stream-close recovery passed. | Omni binding, state and memory contract, broad quality, GPU-memory attribution and sustained power/thermal behavior; [raw evidence](evidence/spark_radeon890m_llamacpp/README.md). |
|
||||
| Native Windows HX370 CPU / Spark-X2.5-1.7B same GGUF | Scoped complete-request Omni whole-session llama.cpp stage (P for two named inputs) | Correct `Paris` and `1511`; 1 warmup + 20 serial inventory requests p50/p95 6.93/7.05 s, 20/20 correct. CPU 0/29 GPU layers, 4 GiB explicit host reservation, cancellation drain and zero remaining ledger reservation. | Incremental output, restart after active cancellation, broad quality, memory peak and sustained use; [Omni raw evidence](evidence/spark_omni_llamacpp/README.md). |
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
# Native Windows CPU Qwen3-TTS standalone text-to-WAV evidence
|
||||
|
||||
This is a **scoped complete text-to-WAV request outside Omni**, not an Omni streaming or full speech-quality qualification. The CPU-only model placement was asserted on the Ryzen AI 9 HX 370 laptop running Windows 11 build 26200. The installed native vLLM extension still lacks its CPU attention operators, so this experiment used the official Qwen PyTorch wrapper as a candidate whole-model stage backend.
|
||||
This is a **scoped complete text-to-WAV request outside Omni**, not an Omni streaming or full speech-quality qualification. A subsequent [Omni whole-session CPU stage](../qwen_tts_omni_cpu/README.md) reproduces these PCM outputs through the public API; the timings below remain the standalone baseline. The CPU-only model placement was asserted on the Ryzen AI 9 HX 370 laptop running Windows 11 build 26200. The installed native vLLM extension still lacks its CPU attention operators, so this experiment used the official Qwen PyTorch wrapper as a candidate whole-model stage backend.
|
||||
|
||||
The local checkpoint is [Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice](https://huggingface.co/Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice/tree/85e237c12c027371202489a0ec509ded67b5e4b5) at revision `85e237c12c027371202489a0ec509ded67b5e4b5`. The talker `model.safetensors` is 1,811,626,576 bytes, SHA256 `bc3c7e785eb961179c25450d1acff03f839e0002f2f3a5aeb67b5735c0fa2adb`; `speech_tokenizer/model.safetensors` is 682,293,092 bytes, SHA256 `836b7b357f5ea43e889936a3709af68dfe3751881acefe4ecf0dbd30ba571258`. Both match the remote revision's LFS hashes. Weights stay in the local model cache, not Git. The [official `qwen-tts` wrapper](https://github.com/QwenLM/Qwen3-TTS) was version 0.1.1. The process used Python 3.12.10, PyTorch 2.13.0+cu130 with CUDA masked, BF16, SDPA, eight CPU threads, `Ryan`/English, seed 42, `max_new_tokens=64`, sampling enabled and `non_streaming_mode=True`. All model parameters were checked on CPU.
|
||||
|
||||
@@ -12,7 +12,7 @@ The local checkpoint is [Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice](https://huggingfa
|
||||
|
||||
The memory trace covers only 120 samples over 32.07 s at roughly 0.25 s intervals during the serial profile. Maximum observed Python process working set was 3,202,650,112 bytes and private bytes 5,336,956,928. It is **not** the loading peak, a full-run maximum, or a shared-RAM/iGPU admission measurement. No device power mode or thermal history was captured. The ASR proxy used locally pinned [Whisper tiny.en](https://huggingface.co/openai/whisper-tiny.en/tree/87c7102498dcde7456f24cfd30239ca606ed9063) revision `87c7102498dcde7456f24cfd30239ca606ed9063`, weights SHA256 `db59695928ded6043adaef491a53ef4e12da9611184d77c53baa691a60b958ad`. Neither two transcriptions nor nonzero audio establishes perceptual quality, speaker fidelity, or general intelligibility.
|
||||
|
||||
The native Omni environment has Transformers 5.14.1. Importing `qwen-tts` 0.1.1 with that version [failed](qwen_tts_native_transformers5_import_failure_20260923.log) at `check_model_inputs()`. The successful process overlaid the wrapper's pinned Transformers 4.57.3 and compatible dependencies through an isolated `PYTHONPATH` directory, leaving the Omni environment unchanged. The missing SoX executable emitted a warning; it did not prevent these WAV outputs. This dependency split must be resolved when implementing an Omni stage binding.
|
||||
The native Omni environment has Transformers 5.14.1. Importing `qwen-tts` 0.1.1 with that version [failed](qwen_tts_native_transformers5_import_failure_20260923.log) at `check_model_inputs()`. The successful process overlaid the wrapper's pinned Transformers 4.57.3 and compatible dependencies through an isolated `PYTHONPATH` directory, leaving the Omni environment unchanged. The missing SoX executable emitted a warning; it did not prevent these WAV outputs. The [later Omni binding](../qwen_tts_omni_cpu/README.md) isolates this dependency in a worker process.
|
||||
|
||||
Reproduce in PowerShell after downloading the exact checkpoint and installing `qwen-tts==0.1.1` and its pinned dependencies into isolated local environments:
|
||||
|
||||
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
# Native Windows CPU Qwen3-TTS through Omni
|
||||
|
||||
This is a **scoped complete-request text-to-WAV pass**, not a playable streaming or general speech-quality qualification. On the Ryzen AI 9 HX 370 laptop with Windows 11 build 26200, `AsyncOmni` and `StageRuntime` ran the pinned Qwen3-TTS 0.6B CustomVoice checkpoint through one `external.qwen_tts.cpu.v1` whole-session stage. The stage owns a bounded local worker, request epoch and terminal audio event; it reserves 10 GiB of host RAM before loading and releases the reservation on cancellation or shutdown. The model and speech tokenizer remain in the worker, with CUDA masked. No NPU, Radeon or RTX execution is claimed.
|
||||
|
||||
The checkpoint is [Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice](https://huggingface.co/Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice/tree/85e237c12c027371202489a0ec509ded67b5e4b5) revision `85e237c12c027371202489a0ec509ded67b5e4b5`. The talker weight SHA256 is `bc3c7e785eb961179c25450d1acff03f839e0002f2f3a5aeb67b5735c0fa2adb` (1,811,626,576 bytes); the speech tokenizer weight SHA256 is `836b7b357f5ea43e889936a3709af68dfe3751881acefe4ecf0dbd30ba571258` (682,293,092 bytes). The worker verifies both, interpreter SHA256 `0b471133e110cfb53a061cad528ce8e517d7b9ac41a0a396c39ad795a487fc14`, and the loaded Python package versions before accepting a request. It uses Python 3.12.10, PyTorch 2.13.0+cu130 on CPU, `qwen-tts` 0.1.1, an isolated Transformers 4.57.3 overlay, BF16/SDPA, eight CPU threads, `Ryan`/English, seed 42, `max_new_tokens=64`, and non-streaming generation. The Omni interpreter retains Transformers 5.14.1. The [standalone baseline](../qwen_tts_native_cpu/README.md) used the same checkpoint and generation settings.
|
||||
|
||||
| Check | Result | Raw evidence |
|
||||
|---|---|---|
|
||||
| Two named sentences | “Hello from the local computer.” produced 109,440 frames (4.56 s); “The blue car is parked beside the library.” produced 82,560 frames (3.44 s), both at 24 kHz mono PCM16. Their PCM SHA256 values exactly match the standalone outputs: `c8a49b5c73efd642a1eb04be973d6aefdc8ed9cfd18b34ff67690b8464906bed` and `4edc7a62f749b1fde5ea39bedbc15a35f291ab1db8d24d7b7eb3df7270885e16`. | [profile report](omni_qwen_tts_cpu_profile20_20260923.json), [worker log](omni_qwen_tts_cpu_profile20_20260923.log) |
|
||||
| Resident serial profile | One warmup excluded; 20/20 measured complete requests returned the identical first-sentence PCM. Nearest-rank request-wall p50/p95 were **15.32/15.60 s** for 4.56 s of audio (RTF 3.36/3.42). These times include the Omni stage handoff and exclude startup; they are not first-audio latency. | [all requests and 1,042 memory samples](omni_qwen_tts_cpu_profile20_20260923.json), [driver log](omni_qwen_tts_cpu_profile20_20260923.driver.log), [WAV](omni_qwen_tts_cpu_profile20_20260923.wav) |
|
||||
| Public API | `AsyncOmni.generate` returned one terminal 24 kHz audio event with 109,440 frames and the same PCM SHA256 in 15.87 s after 13.83 s initialization. | [public API report](omni_qwen_tts_cpu_public_20260923.json), [driver log](omni_qwen_tts_cpu_public_20260923.driver.log), [worker log](omni_qwen_tts_cpu_public_20260923.log) |
|
||||
| Admission and cancellation | A deliberate 4 GiB demand was refused as `ResourceUnavailable` before worker launch, with zero remaining reservation. An in-flight request was cancelled: no stale output, worker exited, and the 10 GiB reservation returned to zero. | [rejection report](omni_qwen_tts_cpu_reject_20260923.json), [rejection log](omni_qwen_tts_cpu_reject_20260923.driver.log), [profile report](omni_qwen_tts_cpu_profile20_20260923.json) |
|
||||
| Independent transcription proxy | Pinned Whisper tiny.en transcribed the Omni first-sentence WAV as “Hello from the local compute”, WER 0.20, the same as the standalone WAV. This does not establish perceptual quality or speaker fidelity. | [ASR report](omni_qwen_tts_cpu_asr_20260923.json), [ASR log](omni_qwen_tts_cpu_asr_20260923.log) |
|
||||
|
||||
The loaded launcher-plus-child process tree reported 1,856,651,264 bytes RSS and 4,673,892,352 bytes private memory. The sampled maximum during the profile was 3,255,246,848 bytes RSS and 5,392,134,144 bytes private memory, below the declared 10 GiB reservation. Process-tree accounting matters here: the virtualenv launcher by itself has a small working set, while its descendant holds the model. These 0.25 s samples do **not** establish loading peak, unique physical RAM use, pagefile peak, concurrency admission or a 30-minute thermal/power result. The worker returns only a complete WAV, so playable incremental audio, post-cancel restart, a broad voice-quality suite and sustained real-time performance remain open.
|
||||
|
||||
Reproduce from the repository root in PowerShell with the exact checkpoint and isolated overlay already installed locally:
|
||||
|
||||
```powershell
|
||||
$root = '\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge'
|
||||
$env:PYTHONUTF8 = '1'
|
||||
$env:PYTHONPATH = $root
|
||||
$env:VLLM_ENABLE_V1_MULTIPROCESSING = '0'
|
||||
$python = 'C:\Users\zhout\w2\omni029venv\Scripts\python.exe'
|
||||
$common = @(
|
||||
'--python-bin', $python,
|
||||
'--python-sha256', '0b471133e110cfb53a061cad528ce8e517d7b9ac41a0a396c39ad795a487fc14',
|
||||
'--model-dir', 'C:\Users\zhout\w2\models\Qwen3-TTS-0.6B-85e237c',
|
||||
'--overlay-dir', 'C:\Users\zhout\w2\qwentts_pins',
|
||||
'--talker-sha256', 'bc3c7e785eb961179c25450d1acff03f839e0002f2f3a5aeb67b5735c0fa2adb',
|
||||
'--tokenizer-sha256', '836b7b357f5ea43e889936a3709af68dfe3751881acefe4ecf0dbd30ba571258'
|
||||
)
|
||||
& $python "$root\benchmarks\edge_harness\experiments\probe_omni_qwen_tts_cpu.py" @common `
|
||||
--server-log 'C:\Users\zhout\w2\omni_qwen_tts_cpu_profile20_20260923.log' `
|
||||
--output-report 'C:\Users\zhout\w2\omni_qwen_tts_cpu_profile20_20260923.json' `
|
||||
--output-wav 'C:\Users\zhout\w2\omni_qwen_tts_cpu_profile20_20260923.wav' `
|
||||
--warmups 1 --repeats 20 --abort-check
|
||||
& $python "$root\benchmarks\edge_harness\experiments\probe_omni_qwen_tts_cpu_entrypoint.py" @common `
|
||||
--server-log 'C:\Users\zhout\w2\omni_qwen_tts_cpu_public_20260923.log' `
|
||||
--output-report 'C:\Users\zhout\w2\omni_qwen_tts_cpu_public_20260923.json'
|
||||
```
|
||||
|
||||
The matrix cell advances only the named native Windows CPU × Qwen3-TTS text-to-WAV route. It does not advance the NPU, mobile, joint-accelerator or other model rows.
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"scope": "local Whisper tiny.en ASR proxy for generated speech; no perceptual quality claim",
|
||||
"asr_model": "openai/whisper-tiny.en",
|
||||
"asr_revision": "87c7102498dcde7456f24cfd30239ca606ed9063",
|
||||
"asr_weights_sha256": "db59695928ded6043adaef491a53ef4e12da9611184d77c53baa691a60b958ad",
|
||||
"transformers": "4.57.3",
|
||||
"torch": "2.4.1+cpu",
|
||||
"wav_sha256": "8a41cd36a87981b0ee5b8f77e8a411a4e89b76211fd65b88ad63789f1dd813f4",
|
||||
"input_sample_rate": 24000,
|
||||
"input_duration_s": 4.56,
|
||||
"reference_text": "Hello from the local computer.",
|
||||
"transcript": "Hello from the local compute",
|
||||
"word_error_rate": 0.2,
|
||||
"asr_load_s": 0.8955639000050724,
|
||||
"asr_inference_s": 0.8573601000243798
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
Using custom `forced_decoder_ids` from the (generation) config. This is deprecated in favor of the `task` and `language` flags/config options.
|
||||
{
|
||||
"scope": "local Whisper tiny.en ASR proxy for generated speech; no perceptual quality claim",
|
||||
"asr_model": "openai/whisper-tiny.en",
|
||||
"asr_revision": "87c7102498dcde7456f24cfd30239ca606ed9063",
|
||||
"asr_weights_sha256": "db59695928ded6043adaef491a53ef4e12da9611184d77c53baa691a60b958ad",
|
||||
"transformers": "4.57.3",
|
||||
"torch": "2.4.1+cpu",
|
||||
"wav_sha256": "8a41cd36a87981b0ee5b8f77e8a411a4e89b76211fd65b88ad63789f1dd813f4",
|
||||
"input_sample_rate": 24000,
|
||||
"input_duration_s": 4.56,
|
||||
"reference_text": "Hello from the local computer.",
|
||||
"transcript": "Hello from the local compute",
|
||||
"word_error_rate": 0.2,
|
||||
"asr_load_s": 0.8955639000050724,
|
||||
"asr_inference_s": 0.8573601000243798
|
||||
}
|
||||
+11995
File diff suppressed because it is too large.
Load diff
+11991
File diff suppressed because it is too large.
Load diff
+88
@@ -0,0 +1,88 @@
|
||||
|
||||
********
|
||||
Warning: flash-attn is not installed. Will only run the manual PyTorch version. Please install flash-attn for faster inference.
|
||||
********
|
||||
|
||||
'sox' 不是内部或外部命令，也不是可运行的程序
|
||||
或批处理文件。
|
||||
SoX could not be found!
|
||||
|
||||
If you do not have SoX, proceed here:
|
||||
- - - http://sox.sourceforge.net/ - - -
|
||||
|
||||
If you do (or think that you should) have SoX, double-check your
|
||||
path variables.
|
||||
|
||||
qwen-tts-cpu-worker ready port=54108 props={"model_dir": "C:\\Users\\zhout\\w2\\models\\Qwen3-TTS-0.6B-85e237c", "placement": "cpu", "torch": "2.13.0+cu130", "transformers": "4.57.3", "qwen_tts": "0.1.1", "dtype": "bfloat16", "attention": "sdpa", "threads": 8, "sample_rate": 24000, "speaker": "Ryan", "max_new_tokens": 64, "load_s": 1.746043000021018}
|
||||
127.0.0.1 - - [23/Sep/2026 21:04:39] "GET /health HTTP/1.1" 200 -
|
||||
127.0.0.1 - - [23/Sep/2026 21:04:39] "GET /props HTTP/1.1" 200 -
|
||||
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
|
||||
127.0.0.1 - - [23/Sep/2026 21:04:55] "POST /v1/audio/speech HTTP/1.1" 200 -
|
||||
tts request duration_s=4.560 wall_s=15.639
|
||||
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
|
||||
127.0.0.1 - - [23/Sep/2026 21:05:07] "POST /v1/audio/speech HTTP/1.1" 200 -
|
||||
tts request duration_s=3.440 wall_s=11.662
|
||||
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
|
||||
127.0.0.1 - - [23/Sep/2026 21:05:22] "POST /v1/audio/speech HTTP/1.1" 200 -
|
||||
tts request duration_s=4.560 wall_s=15.285
|
||||
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
|
||||
127.0.0.1 - - [23/Sep/2026 21:05:37] "POST /v1/audio/speech HTTP/1.1" 200 -
|
||||
tts request duration_s=4.560 wall_s=15.313
|
||||
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
|
||||
127.0.0.1 - - [23/Sep/2026 21:05:53] "POST /v1/audio/speech HTTP/1.1" 200 -
|
||||
tts request duration_s=4.560 wall_s=15.593
|
||||
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
|
||||
127.0.0.1 - - [23/Sep/2026 21:06:08] "POST /v1/audio/speech HTTP/1.1" 200 -
|
||||
tts request duration_s=4.560 wall_s=15.352
|
||||
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
|
||||
127.0.0.1 - - [23/Sep/2026 21:06:24] "POST /v1/audio/speech HTTP/1.1" 200 -
|
||||
tts request duration_s=4.560 wall_s=15.238
|
||||
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
|
||||
127.0.0.1 - - [23/Sep/2026 21:06:39] "POST /v1/audio/speech HTTP/1.1" 200 -
|
||||
tts request duration_s=4.560 wall_s=15.426
|
||||
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
|
||||
127.0.0.1 - - [23/Sep/2026 21:06:55] "POST /v1/audio/speech HTTP/1.1" 200 -
|
||||
tts request duration_s=4.560 wall_s=15.306
|
||||
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
|
||||
127.0.0.1 - - [23/Sep/2026 21:07:10] "POST /v1/audio/speech HTTP/1.1" 200 -
|
||||
tts request duration_s=4.560 wall_s=15.247
|
||||
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
|
||||
127.0.0.1 - - [23/Sep/2026 21:07:25] "POST /v1/audio/speech HTTP/1.1" 200 -
|
||||
tts request duration_s=4.560 wall_s=15.270
|
||||
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
|
||||
127.0.0.1 - - [23/Sep/2026 21:07:40] "POST /v1/audio/speech HTTP/1.1" 200 -
|
||||
tts request duration_s=4.560 wall_s=15.244
|
||||
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
|
||||
127.0.0.1 - - [23/Sep/2026 21:07:56] "POST /v1/audio/speech HTTP/1.1" 200 -
|
||||
tts request duration_s=4.560 wall_s=15.354
|
||||
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
|
||||
127.0.0.1 - - [23/Sep/2026 21:08:11] "POST /v1/audio/speech HTTP/1.1" 200 -
|
||||
tts request duration_s=4.560 wall_s=15.311
|
||||
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
|
||||
127.0.0.1 - - [23/Sep/2026 21:08:26] "POST /v1/audio/speech HTTP/1.1" 200 -
|
||||
tts request duration_s=4.560 wall_s=15.284
|
||||
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
|
||||
127.0.0.1 - - [23/Sep/2026 21:08:42] "POST /v1/audio/speech HTTP/1.1" 200 -
|
||||
tts request duration_s=4.560 wall_s=15.613
|
||||
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
|
||||
127.0.0.1 - - [23/Sep/2026 21:08:57] "POST /v1/audio/speech HTTP/1.1" 200 -
|
||||
tts request duration_s=4.560 wall_s=15.178
|
||||
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
|
||||
127.0.0.1 - - [23/Sep/2026 21:09:13] "POST /v1/audio/speech HTTP/1.1" 200 -
|
||||
tts request duration_s=4.560 wall_s=15.266
|
||||
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
|
||||
127.0.0.1 - - [23/Sep/2026 21:09:28] "POST /v1/audio/speech HTTP/1.1" 200 -
|
||||
tts request duration_s=4.560 wall_s=15.255
|
||||
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
|
||||
127.0.0.1 - - [23/Sep/2026 21:09:43] "POST /v1/audio/speech HTTP/1.1" 200 -
|
||||
tts request duration_s=4.560 wall_s=15.290
|
||||
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
|
||||
127.0.0.1 - - [23/Sep/2026 21:09:58] "POST /v1/audio/speech HTTP/1.1" 200 -
|
||||
tts request duration_s=4.560 wall_s=15.245
|
||||
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
|
||||
127.0.0.1 - - [23/Sep/2026 21:10:14] "POST /v1/audio/speech HTTP/1.1" 200 -
|
||||
tts request duration_s=4.560 wall_s=15.403
|
||||
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
|
||||
127.0.0.1 - - [23/Sep/2026 21:10:29] "POST /v1/audio/speech HTTP/1.1" 200 -
|
||||
tts request duration_s=4.560 wall_s=15.364
|
||||
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
INFO 09-23 21:11:22 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 21:11:22 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 21:11:26 [omni_base.py:216] [AsyncOmni] Initializing with model C:\Users\zhout\w2\models\Qwen3-TTS-0.6B-85e237c
|
||||
INFO 09-23 21:11:26 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model C:\Users\zhout\w2\models\Qwen3-TTS-0.6B-85e237c
|
||||
ERROR 09-23 21:11:26 [repo_utils.py:119] Error parsing config for C:\Users\zhout\w2\models\Qwen3-TTS-0.6B-85e237c: The checkpoint you are trying to load has model type `qwen3_tts` but Transformers does not recognize this architecture. This could be because of an issue with the checkpoint, or because your version of Transformers is out of date.
|
||||
|
||||
ERROR 09-23 21:11:26 [repo_utils.py:119]
|
||||
|
||||
ERROR 09-23 21:11:26 [repo_utils.py:119] You can update Transformers with the command `pip install --upgrade transformers`. If this does not work, and the checkpoint is very new, then there may not be a release version that supports this model yet. In this case, you can get the most up-to-date code by installing Transformers from source with the command `pip install git+https://github.com/huggingface/transformers.git`, retrying 1 of 2
|
||||
ERROR 09-23 21:11:28 [repo_utils.py:117] Error parsing config for C:\Users\zhout\w2\models\Qwen3-TTS-0.6B-85e237c: The checkpoint you are trying to load has model type `qwen3_tts` but Transformers does not recognize this architecture. This could be because of an issue with the checkpoint, or because your version of Transformers is out of date.
|
||||
|
||||
ERROR 09-23 21:11:28 [repo_utils.py:117]
|
||||
|
||||
ERROR 09-23 21:11:28 [repo_utils.py:117] You can update Transformers with the command `pip install --upgrade transformers`. If this does not work, and the checkpoint is very new, then there may not be a release version that supports this model yet. In this case, you can get the most up-to-date code by installing Transformers from source with the command `pip install git+https://github.com/huggingface/transformers.git`
|
||||
INFO 09-23 21:11:28 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 1 stages
|
||||
INFO 09-23 21:11:28 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
INFO 09-23 21:11:28 [initialization.py:356] Loaded OmniTransferConfig with 0 connector configurations
|
||||
INFO 09-23 21:11:39 [orchestrator.py:604] [Orchestrator] Starting event loop
|
||||
INFO 09-23 21:11:39 [async_omni_engine.py:323] [AsyncOmniEngine] Orchestrator ready with 1 stages
|
||||
INFO 09-23 21:11:39 [omni_base.py:235] [AsyncOmni] AsyncOmniEngine initialized in 13.83 seconds
|
||||
INFO 09-23 21:11:39 [omni_base.py:255] [AsyncOmni] Initialized with 1 stages for model C:\Users\zhout\w2\models\Qwen3-TTS-0.6B-85e237c
|
||||
INFO 09-23 21:11:55 [omni_base.py:781] [AsyncOmni] Shutting down
|
||||
INFO 09-23 21:11:55 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator
|
||||
INFO 09-23 21:11:55 [orchestrator.py:705] [Orchestrator] Received shutdown signal
|
||||
INFO 09-23 21:11:55 [orchestrator.py:3092] [Orchestrator] Shutting down all 1 client(s)
|
||||
INFO 09-23 21:11:56 [stage_pool.py:1354] [StagePool] Stage 0 replica 0 shut down
|
||||
{
|
||||
"entrypoint": "AsyncOmni.generate",
|
||||
"pipeline": "qwen3_tts_cpu_whole",
|
||||
"startup_s": 13.828751699998975,
|
||||
"request_wall_s": 15.871287400019355,
|
||||
"outputs": [
|
||||
{
|
||||
"request_id": "public-cpu-tts-1",
|
||||
"sample_rate": 24000,
|
||||
"frames": 109440,
|
||||
"pcm_sha256": "c8a49b5c73efd642a1eb04be973d6aefdc8ed9cfd18b34ff67690b8464906bed",
|
||||
"stage_event": {
|
||||
"request_id": "public-cpu-tts-1-b41a79f3",
|
||||
"stage_id": 0,
|
||||
"epoch": 1,
|
||||
"seq": 1,
|
||||
"kind": "audio",
|
||||
"worker_generation": "0e4f76e367a048c5a1b5a5bd231b459d",
|
||||
"buffers": [],
|
||||
"terminal": true,
|
||||
"error": null
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": "passed"
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"entrypoint": "AsyncOmni.generate",
|
||||
"pipeline": "qwen3_tts_cpu_whole",
|
||||
"startup_s": 13.828751699998975,
|
||||
"request_wall_s": 15.871287400019355,
|
||||
"outputs": [
|
||||
{
|
||||
"request_id": "public-cpu-tts-1",
|
||||
"sample_rate": 24000,
|
||||
"frames": 109440,
|
||||
"pcm_sha256": "c8a49b5c73efd642a1eb04be973d6aefdc8ed9cfd18b34ff67690b8464906bed",
|
||||
"stage_event": {
|
||||
"request_id": "public-cpu-tts-1-b41a79f3",
|
||||
"stage_id": 0,
|
||||
"epoch": 1,
|
||||
"seq": 1,
|
||||
"kind": "audio",
|
||||
"worker_generation": "0e4f76e367a048c5a1b5a5bd231b459d",
|
||||
"buffers": [],
|
||||
"terminal": true,
|
||||
"error": null
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": "passed"
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
|
||||
********
|
||||
Warning: flash-attn is not installed. Will only run the manual PyTorch version. Please install flash-attn for faster inference.
|
||||
********
|
||||
|
||||
'sox' 不是内部或外部命令，也不是可运行的程序
|
||||
或批处理文件。
|
||||
SoX could not be found!
|
||||
|
||||
If you do not have SoX, proceed here:
|
||||
- - - http://sox.sourceforge.net/ - - -
|
||||
|
||||
If you do (or think that you should) have SoX, double-check your
|
||||
path variables.
|
||||
|
||||
qwen-tts-cpu-worker ready port=64072 props={"model_dir": "C:\\Users\\zhout\\w2\\models\\Qwen3-TTS-0.6B-85e237c", "placement": "cpu", "torch": "2.13.0+cu130", "transformers": "4.57.3", "qwen_tts": "0.1.1", "dtype": "bfloat16", "attention": "sdpa", "threads": 8, "sample_rate": 24000, "speaker": "Ryan", "max_new_tokens": 64, "load_s": 1.6077488999580964}
|
||||
127.0.0.1 - - [23/Sep/2026 21:11:39] "GET /health HTTP/1.1" 200 -
|
||||
127.0.0.1 - - [23/Sep/2026 21:11:39] "GET /props HTTP/1.1" 200 -
|
||||
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
|
||||
127.0.0.1 - - [23/Sep/2026 21:11:55] "POST /v1/audio/speech HTTP/1.1" 200 -
|
||||
tts request duration_s=4.560 wall_s=15.830
|
||||
+99
@@ -0,0 +1,99 @@
|
||||
INFO 09-23 21:12:59 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 21:12:59 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 21:13:02 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
WARNING 09-23 21:13:02 [stage_init_utils.py:1887] [stage_init] Failed to load transfer config: [Errno 13] Permission denied: 'C:\\Users\\zhout\\w2'
|
||||
ERROR 09-23 21:13:04 [stage_runtime.py:265] [StageRuntime] Stage initialization failed; shutting down 0 initialized client(s)
|
||||
|
||||
ERROR 09-23 21:13:04 [stage_runtime.py:265] Traceback (most recent call last):
|
||||
|
||||
ERROR 09-23 21:13:04 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 252, in initialize
|
||||
|
||||
ERROR 09-23 21:13:04 [stage_runtime.py:265] initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
|
||||
ERROR 09-23 21:13:04 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
ERROR 09-23 21:13:04 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
|
||||
ERROR 09-23 21:13:04 [stage_runtime.py:265] raise primary_exc
|
||||
|
||||
ERROR 09-23 21:13:04 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 550, in _init_group
|
||||
|
||||
ERROR 09-23 21:13:04 [stage_runtime.py:265] client = self._initialize_replica(
|
||||
|
||||
ERROR 09-23 21:13:04 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
ERROR 09-23 21:13:04 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 817, in _initialize_replica
|
||||
|
||||
ERROR 09-23 21:13:04 [stage_runtime.py:265] return create_graph_client(
|
||||
|
||||
ERROR 09-23 21:13:04 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
ERROR 09-23 21:13:04 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\backends\__init__.py", line 23, in create_graph_client
|
||||
|
||||
ERROR 09-23 21:13:04 [stage_runtime.py:265] return QwenTTSCPUStageClient(metadata, backend, ledger, reservation)
|
||||
|
||||
ERROR 09-23 21:13:04 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
ERROR 09-23 21:13:04 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\backends\qwen_tts_cpu.py", line 79, in __init__
|
||||
|
||||
ERROR 09-23 21:13:04 [stage_runtime.py:265] raise ResourceUnavailable("CPU TTS weights plus state/workspace/headroom exceed reservation")
|
||||
|
||||
ERROR 09-23 21:13:04 [stage_runtime.py:265] vllm_omni.engine.resource_ledger.ResourceUnavailable: CPU TTS weights plus state/workspace/headroom exceed reservation
|
||||
{
|
||||
"scope": "complete resident native Windows CPU Qwen3-TTS text-to-WAV through Omni",
|
||||
"warmup_count": 0,
|
||||
"measured_count": 1,
|
||||
"memory_budget": {
|
||||
"capacities": {
|
||||
"host_ram": 17179869184
|
||||
},
|
||||
"demands": {
|
||||
"host_ram": 4294967296
|
||||
}
|
||||
},
|
||||
"status": "failed",
|
||||
"error": "ResourceUnavailable: CPU TTS weights plus state/workspace/headroom exceed reservation",
|
||||
"ledger_after_shutdown": {
|
||||
"capacities": {
|
||||
"host_ram": 17179869184
|
||||
},
|
||||
"reserved": {
|
||||
"host_ram": 0
|
||||
},
|
||||
"owners": [],
|
||||
"quarantined": []
|
||||
}
|
||||
}
|
||||
Traceback (most recent call last):
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\benchmarks\edge_harness\experiments\probe_omni_qwen_tts_cpu.py", line 216, in <module>
|
||||
asyncio.run(main())
|
||||
File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\asyncio\runners.py", line 195, in run
|
||||
return runner.run(main)
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\asyncio\runners.py", line 118, in run
|
||||
return self._loop.run_until_complete(task)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\asyncio\base_events.py", line 691, in run_until_complete
|
||||
return future.result()
|
||||
^^^^^^^^^^^^^^^
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\benchmarks\edge_harness\experiments\probe_omni_qwen_tts_cpu.py", line 93, in main
|
||||
runtime.initialize()
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 278, in initialize
|
||||
raise exc
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 252, in initialize
|
||||
initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
raise primary_exc
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 550, in _init_group
|
||||
client = self._initialize_replica(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 817, in _initialize_replica
|
||||
return create_graph_client(
|
||||
^^^^^^^^^^^^^^^^^^^^
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\backends\__init__.py", line 23, in create_graph_client
|
||||
return QwenTTSCPUStageClient(metadata, backend, ledger, reservation)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\backends\qwen_tts_cpu.py", line 79, in __init__
|
||||
raise ResourceUnavailable("CPU TTS weights plus state/workspace/headroom exceed reservation")
|
||||
vllm_omni.engine.resource_ledger.ResourceUnavailable: CPU TTS weights plus state/workspace/headroom exceed reservation
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"scope": "complete resident native Windows CPU Qwen3-TTS text-to-WAV through Omni",
|
||||
"warmup_count": 0,
|
||||
"measured_count": 1,
|
||||
"memory_budget": {
|
||||
"capacities": {
|
||||
"host_ram": 17179869184
|
||||
},
|
||||
"demands": {
|
||||
"host_ram": 4294967296
|
||||
}
|
||||
},
|
||||
"status": "failed",
|
||||
"error": "ResourceUnavailable: CPU TTS weights plus state/workspace/headroom exceed reservation",
|
||||
"ledger_after_shutdown": {
|
||||
"capacities": {
|
||||
"host_ram": 17179869184
|
||||
},
|
||||
"reserved": {
|
||||
"host_ram": 0
|
||||
},
|
||||
"owners": [],
|
||||
"quarantined": []
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
# E2E check and profiling
|
||||
|
||||
All 60 pairings have a disposition. The timing tables below remain the 2026-09-22 measurements. The configuration matrix and affected per-cell reasons incorporate the [2026-09-23 scoped recovery runs](../../../e2e_recovery_20260923/README.md) and [WSL CPU/native Windows expansion](../../../e2e_expansion_20260923/README.md). The expansion adds separate 20-request serial MiniCPM-o text-to-speech profiles on WSL CPU and native Windows GPU, a Spark output-head component profile on the HX370 AMD NPU, complete Spark text-request profiles through standalone llama.cpp and a bounded Omni whole-session stage on native Windows CPU and Radeon 890M, a standalone Qwen3-TTS text-to-WAV profile on native Windows CPU, and a hybrid CPU+Radeon 890M Qwen3-TTS profile now repeated through a bounded Omni whole-session stage. Profiling completion does not imply full release qualification.
|
||||
All 60 pairings have a disposition. The timing tables below remain the 2026-09-22 measurements. The configuration matrix and affected per-cell reasons incorporate the [2026-09-23 scoped recovery runs](../../../e2e_recovery_20260923/README.md) and [WSL CPU/native Windows expansion](../../../e2e_expansion_20260923/README.md). The expansion adds separate 20-request serial MiniCPM-o text-to-speech profiles on WSL CPU and native Windows GPU, a Spark output-head component profile on the HX370 AMD NPU, complete Spark text-request profiles through standalone llama.cpp and a bounded Omni whole-session stage on native Windows CPU and Radeon 890M, native Windows CPU Qwen3-TTS text-to-WAV profiles through both standalone PyTorch and a bounded Omni whole-session stage, and a hybrid CPU+Radeon 890M Qwen3-TTS profile likewise repeated through a bounded Omni whole-session stage. Profiling completion does not imply full release qualification.
|
||||
|
||||
## Configuration × model
|
||||
|
||||
@@ -9,7 +9,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
|
||||
| Device | Spark-X2.5 | Qwen3-TTS 0.6B CustomVoice | Qwen3.8-27B | MiniCPM-o 4.5 | InternVLA-A1 |
|
||||
|---|---|---|---|---|---|
|
||||
| PC HX370: CPU / WSL | profile completed | scoped E2E: short stream | NOT E2E | scoped E2E: text/image/audio → speech | synthetic policy pass |
|
||||
| PC HX370: CPU / native Windows | scoped E2E: Omni Spark text | scoped E2E: standalone text → WAV | NOT E2E | NOT E2E | synthetic policy pass |
|
||||
| PC HX370: CPU / native Windows | scoped E2E: Omni Spark text | scoped E2E: Omni CPU text → complete WAV | NOT E2E | NOT E2E | synthetic policy pass |
|
||||
| PC HX370 + RTX 5090 Laptop / WSL | profile completed | profile completed | scoped E2E: text + image | scoped E2E: text/image/audio → speech | synthetic policy pass |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | profile completed | profile completed | NOT E2E | scoped E2E: text/image/audio → speech | synthetic policy pass |
|
||||
| PC HX370 + Radeon 890M / Windows worker | scoped E2E: Omni Spark text | scoped E2E: Omni Radeon talker/codec + CPU predictor | NOT E2E | NOT E2E | NOT E2E |
|
||||
@@ -31,7 +31,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
|
||||
| PC HX370: CPU / WSL | MiniCPM-o 4.5 | SCOPED_E2E | The verified checkpoint completed real three-stage text, one synthetic image and one synthetic audio input to text+speech on the vLLM 0.28 CPU backend in separate bounded plans. The 448×448 red square was described correctly with a nonzero 5.2 s WAV in one 37.70 s request; the 2 s/440 Hz tone was described as “A loud and high pitched beep” with a nonzero 2.16 s WAV in one 17.30 s request. CPU wording/token IDs differ from the WSL GPU runs, with no task-quality threshold established. A separate 1-warmup/20-measured serial **text** run had request wall p50 21.86 s/p95 22.97 s. WSL swap reached 5.75 GB during that text profile; manual snapshots in the multimodal runs showed heavy swap use but do not prove peak memory. Both multimodal runs force-killed stage 0 at shutdown and reported shared-memory cleanup warnings. | Validate real image/audio suites, speech intelligibility, video input, concurrency, interruption, admission/loading peak, clean shutdown and sustained thermal behavior before broader CPU qualification; [CPU evidence](../../../e2e_expansion_20260923/README.md). |
|
||||
| PC HX370: CPU / WSL | InternVLA-A1 | SYNTHETIC_POLICY_PASS | The Place_Markpen checkpoint strict-loaded in CPU-only PyTorch and produced finite `[1, 50, 32]` actions from zero observations/noise in one 3.25 s forward. Compared with the same CUDA checkpoint/input, relative L2 error was 1.30% and cosine similarity 0.999917; no task-quality tolerance is established. | Validate real observations against reference actions, units/timestamps, repeated latency and memory before robot-task qualification; [CPU evidence](../../../e2e_expansion_20260923/README.md). |
|
||||
| PC HX370: CPU / native Windows | Spark-X2.5 | SCOPED_E2E_OMNI_COMPLETE_REQUEST | The pinned Spark-X2.5-1.7B Q4_K_M GGUF ran in native Windows llama.cpp CPU (`-dev none -ngl 0`) as a whole-session Omni StageRuntime/StagePool backend. `Paris` and the correct `1511` inventory answer passed; one warmup plus 20 serial complete requests were 20/20 correct, nearest-rank wall p50/p95 6.93/7.05 s. The public AsyncOmni short request passed. The stage verifies artifact hashes and CPU placement (0/29 layers offloaded), reserves 4 GiB host RAM, emits a terminal text event and releases reservation after in-flight cancellation. The separate prior standalone profile was 6.65/6.86 s. No vLLM CPU attention repair, incremental stream, restart-after-cancel, loading peak or sustained power/thermal gate is claimed. | Validate broader text/token quality, streaming and cancellation recovery, memory peak and sustained load; [Omni raw evidence](../../../e2e_expansion_20260923/evidence/spark_omni_llamacpp/README.md), [standalone evidence](../../../e2e_expansion_20260923/evidence/spark_radeon890m_llamacpp/README.md). |
|
||||
| PC HX370: CPU / native Windows | Qwen3-TTS 0.6B CustomVoice | SCOPED_E2E_STANDALONE | The pinned 0.6B CustomVoice checkpoint ran wholly on CPU through Qwen's standalone PyTorch wrapper (BF16/SDPA) and produced nonzero 24 kHz WAVs for two English sentences. For one repeated short prompt, 1 warmup + 20 serial measured complete generation requests had nearest-rank p50/p95 13.94/14.68 s for 4.56 s of audio. Independent Whisper tiny.en ASR gave word error rates 0.20 and 0.00 on the two WAVs; this is only an intelligibility proxy. The native vLLM CPU extension still lacks the required attention operators. No Omni binding, playable streaming, listening/quality threshold, loading peak or sustained power/thermal run was verified. | Bind the whole CPU TTS backend to Omni's stage and streaming/state contracts; validate playback, quality, cancellation, admission and sustained use; [standalone TTS evidence](../../../e2e_expansion_20260923/evidence/qwen_tts_native_cpu/README.md). |
|
||||
| PC HX370: CPU / native Windows | Qwen3-TTS 0.6B CustomVoice | SCOPED_E2E_OMNI_COMPLETE_REQUEST | The pinned 0.6B CustomVoice BF16/SDPA checkpoint ran wholly on CPU through an isolated Qwen wrapper worker owned by one bounded Omni StageRuntime stage. Two named 24 kHz WAVs matched the earlier standalone PCM SHA256 values exactly. One warmup + 20 serial complete Omni requests for 4.56 s of audio had 20/20 identical PCM and nearest-rank wall p50/p95 15.32/15.60 s (RTF 3.36/3.42); independent Whisper tiny.en ASR gave WER 0.20 on the first Omni WAV, an intelligibility proxy only. A public AsyncOmni audio request passed. The stage verified pinned weights/interpreter and CPU placement, reserved 10 GiB host RAM, rejected an insufficient 4 GiB demand before worker launch, and after in-flight cancellation emitted no stale output and returned the ledger to zero. Sampled process-tree private memory peaked at 5.39 GB, not a loading peak. Native vLLM CPU attention operators remain absent; playable streaming, post-cancel restart, broader speech quality, loading peak and sustained power/thermal behavior remain unverified. | Validate playable streaming and restart/recovery, broader voice quality, peak memory, concurrency and sustained behavior; [Omni raw evidence](../../../e2e_expansion_20260923/evidence/qwen_tts_omni_cpu/README.md), [standalone baseline](../../../e2e_expansion_20260923/evidence/qwen_tts_native_cpu/README.md). |
|
||||
| PC HX370: CPU / native Windows | Qwen3.8-27B | NOT_E2E_PROFILEABLE | The installed native Windows vLLM extension lacks cpu_attn_reshape_and_cache and cpu_attn_get_scheduler_metadata. The separate ORT CPU graph worker is not a whole-model text/TTS adapter; a qualified CPU build/binding is missing. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| PC HX370: CPU / native Windows | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | The exact checkpoint is verified in the local Linux model store, but the installed native Windows vLLM extension lacks required CPU attention operators; no native full-model run or memory admission exists. | Establish a native Windows CPU backend and model access, then run the full requested path; [weight evidence](../../../e2e_recovery_20260923/evidence/weights/minicpmo.json). |
|
||||
| PC HX370: CPU / native Windows | InternVLA-A1 | SYNTHETIC_POLICY_PASS | The Place_Markpen checkpoint strict-loaded from a UNC path and the direct CPU policy produced finite `[1, 50, 32]` actions on synthetic zero observations/noise. Policy/output CPU placement was checked; one forward took 3.36 s. Relative L2 difference from WSL CPU was 1.08% (no task tolerance). This direct policy route does not require native vLLM CPU attention operators. | Validate real observations/reference actions, units/timestamps, memory and repeated latency before task qualification; [native evidence](../../../e2e_expansion_20260923/README.md). |
|
||||
|
||||
@@ -5,7 +5,9 @@ per request, with a fixed input bucket and copied host tensors. A second backend
|
||||
`external.llamacpp.text.v1`, owns one pinned GGUF whole-model server process and
|
||||
returns complete text requests. `external.crisp.tts.v1` owns a pinned
|
||||
CustomVoice GGUF talker, codec and local hybrid CPU+Radeon process, returning
|
||||
complete PCM audio. All use the normal `Omni` / `AsyncOmni` APIs,
|
||||
complete PCM audio. `external.qwen_tts.cpu.v1` owns the original BF16 CustomVoice
|
||||
checkpoint in an isolated native Windows CPU process and also returns complete
|
||||
PCM audio. All use the normal `Omni` / `AsyncOmni` APIs,
|
||||
`PipelineConfig`, `StageRuntime`, `StagePool` and orchestrator. Native vLLM stages
|
||||
retain their model execution, KV cache, sampling and batching. No accelerator is
|
||||
mandatory. The llama.cpp backend currently uses the complete-request graph
|
||||
@@ -13,7 +15,9 @@ control path for admission and acknowledgement; it is not an ONNX graph and does
|
||||
not yet support incremental token events or restart after active cancellation.
|
||||
The CrispASR route likewise lacks playable incremental PCM and post-cancel
|
||||
restart; its current placement gate is specific to the measured HX370 Radeon
|
||||
890M shared-RAM configuration.
|
||||
890M shared-RAM configuration. The Qwen CPU route likewise returns a terminal
|
||||
WAV only; its worker pins an isolated Transformers 4.57.3 overlay and verifies
|
||||
CPU placement and weight hashes before serving.
|
||||
|
||||
## Boundaries
|
||||
|
||||
@@ -64,6 +68,8 @@ This section preserves the 2026-09-22 audit. For the latest per-cell disposition
|
||||
|
||||
The [Qwen3-TTS CPU+Radeon whole-session path](../../benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_omni_hybrid/README.md) also completed two named text-to-audio inputs and 20 serial Omni requests with exact PCM parity to its standalone record. It verifies pinned executable and weights, logged hybrid placement, one-slot shared-RAM admission, terminal audio and cancellation drain. Playable streaming, post-cancel restart, loading/iGPU peaks, sustained use and broader speech quality remain open; RTF still exceeds 1. No joint CPU+iGPU+NPU claim follows from this result.
|
||||
|
||||
The [native Windows CPU Qwen3-TTS Omni path](../../benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_omni_cpu/README.md) completed two named text-to-WAV inputs and 20 serial Omni requests with exact PCM parity to the standalone PyTorch record. It verifies pinned weights, CPU-only placement, a 10 GiB host reservation, a 4 GiB pre-load refusal, terminal audio and cancellation drain. Request-wall p50/p95 were 15.32/15.60 s for 4.56 s of audio. Playable streaming, post-cancel restart, loading peak, broad voice quality and sustained use remain open. This CPU route does not qualify AMD or mobile execution.
|
||||
|
||||
**All 60 named pairings were checked on 2026-09-22** through actual runs,
|
||||
raw-record review or artifact/runtime preflight. This is not 60 successful
|
||||
executions. Each cell links to its evidence, finding and next required step.
|
||||
|
||||
@@ -53,8 +53,15 @@ prompt took p50/p95 13.94/14.68 s to generate 4.56 s of audio. Whisper tiny.en
|
||||
ASR gave word error rates 0.20 and 0.00 for the two WAVs, an intelligibility
|
||||
proxy rather than a quality pass. The [raw record](../../benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/README.md)
|
||||
documents the isolated Transformers 4.57.3 dependency and short process-memory
|
||||
trace. Omni CPU stage binding, playable streaming, loading-peak admission and
|
||||
sustained power/thermal tests remain M2 work.
|
||||
trace. The subsequent [Omni CPU whole-session stage](../../benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_omni_cpu/README.md)
|
||||
matched both standalone PCM hashes exactly, returned a public `AsyncOmni`
|
||||
audio event, and completed one warmup plus 20 serial measured requests at
|
||||
p50/p95 15.32/15.60 s for 4.56 s of audio. It reserves 10 GiB host RAM,
|
||||
rejects an insufficient 4 GiB demand before worker launch, and drains an
|
||||
in-flight cancellation without stale output or remaining reservation. This
|
||||
closes the scoped CPU stage-binding task. Playable streaming, post-cancel
|
||||
restart, loading-peak admission, broad speech quality and sustained
|
||||
power/thermal tests remain M2 work.
|
||||
|
||||
A separate native Windows CPU+Radeon 890M Qwen3-TTS route now completes two
|
||||
named text-to-WAV requests in standalone CrispASR using a Q8_0 talker, F16
|
||||
|
||||
@@ -113,6 +113,7 @@ from vllm_omni.model_executor.models.qwen3_omni.pipeline import (
|
||||
)
|
||||
from vllm_omni.model_executor.models.qwen3_tts.pipeline import (
|
||||
QWEN3_TTS_CRISP_HYBRID_PIPELINE,
|
||||
QWEN3_TTS_CPU_WHOLE_PIPELINE,
|
||||
QWEN3_TTS_PIPELINE,
|
||||
QWEN3_TTS_TALKER_ONLY_PIPELINE,
|
||||
)
|
||||
@@ -149,6 +150,7 @@ OMNI_PIPELINES: dict[str, PipelineConfig | PipelineResolverFunc] = {
|
||||
"qwen3_tts": QWEN3_TTS_PIPELINE,
|
||||
"qwen3_tts_talker_only": QWEN3_TTS_TALKER_ONLY_PIPELINE,
|
||||
"qwen3_tts_crisp_hybrid": QWEN3_TTS_CRISP_HYBRID_PIPELINE,
|
||||
"qwen3_tts_cpu_whole": QWEN3_TTS_CPU_WHOLE_PIPELINE,
|
||||
# Single-stage text: an AR decoder with no companion stage. It lives in
|
||||
# the same registry as the multi-stage pipelines so that one code path
|
||||
# resolves every model (see the M0 local text mode).
|
||||
|
||||
@@ -17,4 +17,8 @@ def create_graph_client(metadata, config, ledger, reservation):
|
||||
from .crisp_tts import CrispTTSStageClient
|
||||
|
||||
return CrispTTSStageClient(metadata, backend, ledger, reservation)
|
||||
if name == "external.qwen_tts.cpu.v1":
|
||||
from .qwen_tts_cpu import QwenTTSCPUStageClient
|
||||
|
||||
return QwenTTSCPUStageClient(metadata, backend, ledger, reservation)
|
||||
raise ValueError(f"unsupported complete-request stage backend: {name!r}")
|
||||
@@ -100,6 +100,7 @@ class CrispTTSStageClient(StageClientBase):
|
||||
self._request_timeout_s = float(config.get("request_timeout_s", 180))
|
||||
self._voice = str(config.get("voice", "ryan"))
|
||||
self._seed = int(config.get("seed", 42))
|
||||
self._metric_name = "crisp_tts_wall_s"
|
||||
self._port = int(config.get("port") or _local_port())
|
||||
try:
|
||||
if (
|
||||
@@ -217,7 +218,7 @@ class CrispTTSStageClient(StageClientBase):
|
||||
async def add_request_async(self, request_id: str, prompt: Any, params: Any = None) -> None:
|
||||
self.check_health()
|
||||
if self._active is not None:
|
||||
raise ResourceUnavailable("CrispASR stage has an unacknowledged request; capacity is one")
|
||||
raise ResourceUnavailable("audio stage has an unacknowledged request; capacity is one")
|
||||
if not isinstance(prompt, dict) or not isinstance(prompt.get("text"), str) or not prompt["text"]:
|
||||
raise ValueError("TTS prompt must contain nonempty text")
|
||||
text = prompt["text"]
|
||||
@@ -261,7 +262,7 @@ class CrispTTSStageClient(StageClientBase):
|
||||
"pcm_sha256": hashlib.sha256(pcm).hexdigest(),
|
||||
},
|
||||
},
|
||||
metrics={"crisp_tts_wall_s": wall_s},
|
||||
metrics={self._metric_name: wall_s},
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
@@ -336,7 +337,7 @@ class CrispTTSStageClient(StageClientBase):
|
||||
raise EngineDeadError()
|
||||
|
||||
async def collective_rpc_async(self, method, timeout=None, args=(), kwargs=None):
|
||||
raise NotImplementedError(f"CrispASR backend does not implement collective RPC {method}")
|
||||
raise NotImplementedError(f"complete audio backend does not implement collective RPC {method}")
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._closed = True
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Native Windows CPU Qwen3-TTS whole-session stage with isolated dependencies.
|
||||
|
||||
The worker alone imports Qwen's PyTorch wrapper and Transformers 4.57.3.
|
||||
This adapter reuses the bounded complete-WAV request, fencing and ownership
|
||||
methods from the CrispASR audio client; its model and launch path are separate.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from vllm_omni.engine.resource_ledger import ResourceUnavailable
|
||||
|
||||
from .crisp_tts import CrispTTSStageClient, _local_port, _sha256
|
||||
|
||||
|
||||
class QwenTTSCPUStageClient(CrispTTSStageClient):
|
||||
def __init__(self, metadata, config: dict, ledger, reservation) -> None:
|
||||
for name, value in vars(metadata).items():
|
||||
setattr(self, name, value)
|
||||
self.stage_type = "graph" # Omni complete-request control path.
|
||||
self._ledger, self._reservation = ledger, reservation
|
||||
self._generation = uuid.uuid4().hex
|
||||
self._proc: subprocess.Popen | None = None
|
||||
self._log_stream = None
|
||||
self._closed = False
|
||||
self._active = None
|
||||
self._task = None
|
||||
self._output = None
|
||||
self._epoch = 0
|
||||
self._memory_pool = str(config.get("memory_pool", "host_ram"))
|
||||
self._max_text_bytes = int(config.get("max_text_bytes", 4096))
|
||||
self._max_wav_bytes = int(config.get("max_wav_bytes", 4 << 20))
|
||||
self._max_audio_s = float(config.get("max_audio_s", 10))
|
||||
self._request_timeout_s = float(config.get("request_timeout_s", 180))
|
||||
self._voice = "Ryan"
|
||||
self._seed = 42
|
||||
self._metric_name = "qwen_cpu_tts_wall_s"
|
||||
self._port = int(config.get("port") or _local_port())
|
||||
try:
|
||||
if (
|
||||
self._max_text_bytes <= 0 or self._max_wav_bytes <= 0 or self._max_audio_s <= 0
|
||||
or self._request_timeout_s <= 0 or not 0 < self._port < 65536
|
||||
):
|
||||
raise ValueError("invalid Qwen CPU TTS input, output, timeout or port bound")
|
||||
if self._memory_pool not in reservation.demands:
|
||||
raise ResourceUnavailable("CPU TTS memory pool absent from stage reservation")
|
||||
demand = reservation.demands[self._memory_pool]
|
||||
if self._max_wav_bytes > demand:
|
||||
raise ResourceUnavailable("CPU TTS output bound exceeds stage reservation")
|
||||
self._python = Path(config["python_bin"]).resolve(strict=True)
|
||||
self._model_dir = Path(config["model_dir"]).resolve(strict=True)
|
||||
self._overlay = Path(config["overlay_dir"]).resolve(strict=True)
|
||||
self._talker = self._model_dir / "model.safetensors"
|
||||
self._tokenizer = self._model_dir / "speech_tokenizer" / "model.safetensors"
|
||||
self._worker = Path(__file__).with_name("qwen_tts_cpu_worker.py").resolve(strict=True)
|
||||
expected = {
|
||||
"python_sha256": (self._python, str(config["python_sha256"]).lower()),
|
||||
"talker_sha256": (self._talker, str(config["talker_sha256"]).lower()),
|
||||
"tokenizer_sha256": (self._tokenizer, str(config["tokenizer_sha256"]).lower()),
|
||||
}
|
||||
if not (self._overlay / "transformers" / "__init__.py").is_file():
|
||||
raise FileNotFoundError("pinned Transformers overlay is absent")
|
||||
for label, (path, digest) in expected.items():
|
||||
if _sha256(path) != digest:
|
||||
raise ValueError(f"CPU TTS {label} differs from the declared artifact hash")
|
||||
overhead = int(config["memory_overhead_bytes"])
|
||||
weight_bytes = self._talker.stat().st_size + self._tokenizer.stat().st_size
|
||||
if overhead <= 0 or weight_bytes + overhead > demand:
|
||||
raise ResourceUnavailable("CPU TTS weights plus state/workspace/headroom exceed reservation")
|
||||
self._log_path = Path(config["log_file"]).resolve()
|
||||
self._log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._log_stream = self._log_path.open("wb")
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PYTHONPATH": str(self._overlay),
|
||||
"QWEN_TTS_OVERLAY_DIR": str(self._overlay),
|
||||
"CUDA_VISIBLE_DEVICES": "",
|
||||
"HF_HUB_OFFLINE": "1",
|
||||
"TRANSFORMERS_OFFLINE": "1",
|
||||
"OMP_NUM_THREADS": "8",
|
||||
"PYTHONUTF8": "1",
|
||||
})
|
||||
command = [
|
||||
str(self._python), "-u", str(self._worker),
|
||||
"--model-dir", str(self._model_dir), "--port", str(self._port),
|
||||
"--threads", "8",
|
||||
]
|
||||
flags = subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0
|
||||
self._proc = subprocess.Popen(
|
||||
command, stdout=self._log_stream, stderr=subprocess.STDOUT,
|
||||
env=env, creationflags=flags,
|
||||
)
|
||||
base = f"http://127.0.0.1:{self._port}"
|
||||
deadline = time.monotonic() + float(config.get("start_timeout_s", 180))
|
||||
while True:
|
||||
if self._proc.poll() is not None:
|
||||
raise RuntimeError(f"CPU TTS worker exited during load; see {self._log_path}")
|
||||
try:
|
||||
with urllib.request.urlopen(base + "/health", timeout=1) as response:
|
||||
if response.status == 200:
|
||||
break
|
||||
except (urllib.error.URLError, TimeoutError, OSError):
|
||||
pass
|
||||
if time.monotonic() >= deadline:
|
||||
raise TimeoutError(f"CPU TTS worker did not become ready; see {self._log_path}")
|
||||
time.sleep(0.2)
|
||||
with urllib.request.urlopen(base + "/props", timeout=5) as response:
|
||||
props = json.load(response)
|
||||
if (
|
||||
Path(props.get("model_dir", "")).resolve() != self._model_dir
|
||||
or props.get("placement") != "cpu"
|
||||
or props.get("transformers") != "4.57.3"
|
||||
or props.get("qwen_tts") != "0.1.1"
|
||||
or props.get("torch") != str(config["expected_torch"])
|
||||
or props.get("dtype") != "bfloat16"
|
||||
or props.get("sample_rate") != 24000
|
||||
or props.get("speaker") != "Ryan"
|
||||
or props.get("max_new_tokens") != 64
|
||||
):
|
||||
raise RuntimeError("CPU TTS worker environment, model or placement differs from plan")
|
||||
import psutil
|
||||
|
||||
root = psutil.Process(self._proc.pid)
|
||||
processes = [root, *root.children(recursive=True)]
|
||||
self._loaded_process_tree = []
|
||||
for process in processes:
|
||||
memory = process.memory_full_info()
|
||||
self._loaded_process_tree.append({
|
||||
"pid": process.pid,
|
||||
"rss_bytes": int(memory.rss),
|
||||
"private_bytes": int(getattr(memory, "private", memory.rss)),
|
||||
})
|
||||
self._loaded_rss_bytes = sum(row["rss_bytes"] for row in self._loaded_process_tree)
|
||||
self._loaded_private_bytes = sum(row["private_bytes"] for row in self._loaded_process_tree)
|
||||
if max(self._loaded_rss_bytes, self._loaded_private_bytes) > demand:
|
||||
raise ResourceUnavailable("loaded CPU TTS process tree exceeds stage reservation")
|
||||
self._base_url = base
|
||||
self.execution_plan = {
|
||||
"backend": "external.qwen_tts.cpu.v1",
|
||||
"stage_id": self.stage_id,
|
||||
"worker_generation": self._generation,
|
||||
"worker_pid": self._proc.pid,
|
||||
"worker_script_sha256": _sha256(self._worker),
|
||||
"artifact_sha256": {label: digest for label, (_, digest) in expected.items()},
|
||||
"model_dir": str(self._model_dir),
|
||||
"placement": "CPU BF16 SDPA, 8 threads; isolated Transformers 4.57.3",
|
||||
"worker_props": props,
|
||||
"loaded_rss_bytes": self._loaded_rss_bytes,
|
||||
"loaded_private_bytes": self._loaded_private_bytes,
|
||||
"loaded_process_tree": self._loaded_process_tree,
|
||||
"reserved_bytes": dict(reservation.demands),
|
||||
"memory_overhead_bytes": overhead,
|
||||
"max_text_bytes": self._max_text_bytes,
|
||||
"max_wav_bytes": self._max_wav_bytes,
|
||||
"max_audio_s": self._max_audio_s,
|
||||
"voice": self._voice,
|
||||
"seed": self._seed,
|
||||
"request_capacity": 1,
|
||||
"stateful_session": "Qwen PyTorch worker-owned; complete WAV per request",
|
||||
"evidence": "B",
|
||||
}
|
||||
except BaseException:
|
||||
self.shutdown()
|
||||
raise
|
||||
|
||||
def _terminate(self) -> bool:
|
||||
import psutil
|
||||
|
||||
children = []
|
||||
try:
|
||||
if self._proc is not None:
|
||||
children = psutil.Process(self._proc.pid).children(recursive=True)
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
except psutil.Error:
|
||||
return False
|
||||
drained = True
|
||||
for child in reversed(children):
|
||||
try:
|
||||
child.terminate()
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
except psutil.Error:
|
||||
drained = False
|
||||
_, alive = psutil.wait_procs(children, timeout=5)
|
||||
for child in alive:
|
||||
try:
|
||||
child.kill()
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
except psutil.Error:
|
||||
drained = False
|
||||
if psutil.wait_procs(alive, timeout=5)[1]:
|
||||
drained = False
|
||||
return super()._terminate() and drained
|
||||
@@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Isolated local HTTP worker for the official Qwen3-TTS CPU wrapper.
|
||||
|
||||
This file is launched as a script by Omni's parent process. Its interpreter
|
||||
loads a pinned Transformers 4.57 overlay; the Omni interpreter keeps its own
|
||||
Transformers version. The worker owns all model and vocoder state.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import importlib.metadata
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import traceback
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--model-dir", type=Path, required=True)
|
||||
parser.add_argument("--port", type=int, required=True)
|
||||
parser.add_argument("--threads", type=int, default=8)
|
||||
args = parser.parse_args()
|
||||
if not 0 < args.port < 65536 or not 0 < args.threads <= 24:
|
||||
parser.error("invalid port or CPU thread bound")
|
||||
if os.environ.get("CUDA_VISIBLE_DEVICES") != "":
|
||||
raise RuntimeError("CPU TTS worker requires CUDA_VISIBLE_DEVICES empty")
|
||||
if os.environ.get("HF_HUB_OFFLINE") != "1" or os.environ.get("TRANSFORMERS_OFFLINE") != "1":
|
||||
raise RuntimeError("CPU TTS worker requires offline model loading")
|
||||
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
import torch
|
||||
import transformers
|
||||
from qwen_tts import Qwen3TTSModel
|
||||
|
||||
overlay = os.path.normcase(os.path.realpath(os.environ["QWEN_TTS_OVERLAY_DIR"]))
|
||||
loaded_transformers = os.path.normcase(os.path.realpath(transformers.__file__))
|
||||
if transformers.__version__ != "4.57.3" or not loaded_transformers.startswith(overlay + os.sep):
|
||||
raise RuntimeError("Qwen TTS worker loaded the wrong Transformers dependency")
|
||||
if importlib.metadata.version("qwen-tts") != "0.1.1":
|
||||
raise RuntimeError("Qwen TTS worker loaded the wrong model wrapper")
|
||||
torch.set_num_threads(args.threads)
|
||||
started = time.perf_counter()
|
||||
model = Qwen3TTSModel.from_pretrained(
|
||||
str(args.model_dir.resolve(strict=True)),
|
||||
device_map="cpu", dtype=torch.bfloat16, attn_implementation="sdpa",
|
||||
local_files_only=True,
|
||||
)
|
||||
if {parameter.device.type for parameter in model.model.parameters()} != {"cpu"}:
|
||||
raise RuntimeError("Qwen TTS worker loaded a non-CPU parameter")
|
||||
if "ryan" not in model.get_supported_speakers():
|
||||
raise RuntimeError("pinned CustomVoice checkpoint does not provide Ryan")
|
||||
props = {
|
||||
"model_dir": str(args.model_dir.resolve()),
|
||||
"placement": "cpu",
|
||||
"torch": torch.__version__,
|
||||
"transformers": transformers.__version__,
|
||||
"qwen_tts": "0.1.1",
|
||||
"dtype": "bfloat16",
|
||||
"attention": "sdpa",
|
||||
"threads": args.threads,
|
||||
"sample_rate": 24000,
|
||||
"speaker": "Ryan",
|
||||
"max_new_tokens": 64,
|
||||
"load_s": time.perf_counter() - started,
|
||||
}
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def _json(self, status: int, payload: dict) -> None:
|
||||
data = json.dumps(payload).encode()
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(data)))
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
|
||||
def do_GET(self) -> None:
|
||||
if self.path == "/health":
|
||||
self._json(200, {"status": "ok"})
|
||||
elif self.path == "/props":
|
||||
self._json(200, props)
|
||||
else:
|
||||
self._json(404, {"error": "unknown endpoint"})
|
||||
|
||||
def do_POST(self) -> None:
|
||||
if self.path != "/v1/audio/speech":
|
||||
self._json(404, {"error": "unknown endpoint"})
|
||||
return
|
||||
size = int(self.headers.get("Content-Length", "0"))
|
||||
if not 0 < size <= 8192:
|
||||
self._json(413, {"error": "request exceeds worker bound"})
|
||||
return
|
||||
try:
|
||||
payload = json.loads(self.rfile.read(size))
|
||||
text = payload["input"]
|
||||
if not isinstance(text, str) or not 0 < len(text.encode()) <= 4096:
|
||||
raise ValueError("invalid bounded TTS input")
|
||||
if payload.get("response_format") != "wav" or payload.get("seed") != 42:
|
||||
raise ValueError("worker requires WAV and pinned seed 42")
|
||||
torch.manual_seed(42)
|
||||
started = time.perf_counter()
|
||||
wavs, sample_rate = model.generate_custom_voice(
|
||||
text=text, language="English", speaker="Ryan",
|
||||
max_new_tokens=64, do_sample=True, non_streaming_mode=True,
|
||||
)
|
||||
if len(wavs) != 1 or sample_rate != 24000:
|
||||
raise RuntimeError("model did not return one 24 kHz waveform")
|
||||
audio = np.asarray(wavs[0], dtype=np.float32).reshape(-1)
|
||||
if not audio.size or not np.isfinite(audio).all():
|
||||
raise RuntimeError("model returned empty or non-finite audio")
|
||||
with io.BytesIO() as buffer:
|
||||
sf.write(buffer, audio, sample_rate, subtype="PCM_16", format="WAV")
|
||||
data = buffer.getvalue()
|
||||
if len(data) > (4 << 20):
|
||||
raise RuntimeError("model returned audio above worker bound")
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "audio/wav")
|
||||
self.send_header("Content-Length", str(len(data)))
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
print(f"tts request duration_s={len(audio) / sample_rate:.3f} wall_s={time.perf_counter() - started:.3f}", flush=True)
|
||||
except Exception as exc:
|
||||
traceback.print_exc()
|
||||
self._json(500, {"error": f"{type(exc).__name__}: {exc}"})
|
||||
|
||||
with HTTPServer(("127.0.0.1", args.port), Handler) as server:
|
||||
print(f"qwen-tts-cpu-worker ready port={args.port} props={json.dumps(props)}", flush=True)
|
||||
server.serve_forever(poll_interval=0.1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -100,3 +100,19 @@ QWEN3_TTS_CRISP_HYBRID_PIPELINE = PipelineConfig(
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# Native Windows CPU reference path with the official PyTorch wrapper in an
|
||||
# isolated worker, preserving Omni's installed Transformers dependency.
|
||||
QWEN3_TTS_CPU_WHOLE_PIPELINE = PipelineConfig(
|
||||
model_type="qwen3_tts_cpu_whole",
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="audio",
|
||||
execution_type=StageExecutionType.GRAPH,
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
),
|
||||
),
|
||||
)
|
||||
Reference in new issue
Block a user