[Edge][WP4] Windowed Code2Wav decoder export (torch.export / ExecuTorch) + client backend; results summarizer
vllm_omni/edge/decoder_export.py exports a stateless windowed decoder (context 72 frames + chunk, one program per ramp chunk size) via torch.export (.pt2, verified here on CPU for chunks 2 and 25 with the real 0.6B decoder: exact match vs eager, 460 MB fp32 per program) or ExecuTorch XNNPACK (.pte; written against the executorch 1.x API, not executed: package not installed, install is permission gated; QNN partitioner export-only/unverified). The WP5 client gains --decoder-backend executorch --pte-manifest for phone/NPU targets. benchmarks/tts/edge_results/summarize.py renders all committed JSON as tables. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
37c0ea9a6b
commit
580a201631
5 files changed
+549
-1
No files matched your search
@@ -0,0 +1,216 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Export the Qwen3-TTS Code2Wav decoder for on-device (NPU / phone) runtimes (WP4).
|
||||
|
||||
vLLM-Omni cannot run on phone NPUs; the ``npu_phone`` hardware class is served
|
||||
by the codec-token stream (WP5) plus an exported decoder artifact. This module
|
||||
exports a *windowed, stateless* variant of ``Qwen3TTSTokenizerV2Decoder``:
|
||||
|
||||
* input ``codes``: ``int64[1, num_quantizers, context_frames + chunk_frames]``
|
||||
(the last ``context_frames`` frames already decoded plus the new chunk);
|
||||
* output ``wav``: ``float32[1, chunk_frames * hop]`` (only the new chunk's audio,
|
||||
``hop = total_upsample`` samples per frame, 1920 at 24 kHz / 12.5 Hz).
|
||||
|
||||
Fixed shapes (one program per chunk size of the ramp schedule) keep the export
|
||||
static, which is what ExecuTorch / QNN / CoreML partitioners need; the
|
||||
72-frame window is explicit input state instead of the decoder's Python
|
||||
caches, so the artifact carries no mutable state. Backends:
|
||||
|
||||
* ``torch_export`` - ``torch.export`` + ``.pt2`` (always available);
|
||||
* ``executorch`` - XNNPACK-lowered ``.pte`` (needs ``pip install executorch``;
|
||||
the QNN partitioner is export-only and listed as unverified).
|
||||
|
||||
Status: the ExecuTorch path is written against the executorch 1.x API but was
|
||||
not executed in this checkout (package not installed; install is permission
|
||||
gated). ``tests/edge/test_decoder_export.py`` skips the lowering without it and
|
||||
still checks the windowed wrapper against the eager decoder.
|
||||
|
||||
CLI:
|
||||
python -m vllm_omni.edge.decoder_export --model Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice \
|
||||
--ramp 2 4 8 16 25 --context 72 --backend torch_export --out-dir ./code2wav_export
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
DEFAULT_RAMP = (2, 4, 8, 16, 25)
|
||||
DEFAULT_CONTEXT = 72
|
||||
|
||||
|
||||
class WindowedCode2Wav(nn.Module):
|
||||
"""Stateless windowed decode: full window in, trailing ``chunk_frames`` of audio out."""
|
||||
|
||||
def __init__(self, decoder: nn.Module, chunk_frames: int, context_frames: int = DEFAULT_CONTEXT):
|
||||
super().__init__()
|
||||
self.decoder = decoder
|
||||
self.chunk_frames = int(chunk_frames)
|
||||
self.context_frames = int(context_frames)
|
||||
self.hop = int(getattr(decoder, "total_upsample", 1920))
|
||||
self.num_quantizers = int(getattr(getattr(decoder, "config", None), "num_quantizers", 16))
|
||||
|
||||
@property
|
||||
def window_frames(self) -> int:
|
||||
return self.context_frames + self.chunk_frames
|
||||
|
||||
def example_input(self) -> torch.Tensor:
|
||||
return torch.zeros(1, self.num_quantizers, self.window_frames, dtype=torch.long)
|
||||
|
||||
def forward(self, codes: torch.Tensor) -> torch.Tensor:
|
||||
wav = self.decoder._forward_exact(codes) # [1, 1, samples] or [1, samples]
|
||||
wav = wav.reshape(wav.shape[0], -1)
|
||||
return wav[:, -self.chunk_frames * self.hop :]
|
||||
|
||||
|
||||
def resolve_model_dir(model: str) -> Path:
|
||||
p = Path(model)
|
||||
if p.is_dir():
|
||||
return p
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
return Path(snapshot_download(model))
|
||||
|
||||
|
||||
def load_code2wav_decoder(model: str, dtype: torch.dtype = torch.float32) -> nn.Module:
|
||||
"""Load ``Qwen3TTSTokenizerV2Decoder`` from ``<model>/speech_tokenizer`` (eager, CPU)."""
|
||||
from safetensors.torch import load_file
|
||||
|
||||
from vllm_omni.model_executor.models.qwen3_tts.tokenizer_12hz.configuration_qwen3_tts_tokenizer_v2 import (
|
||||
Qwen3TTSTokenizerV2Config,
|
||||
)
|
||||
from vllm_omni.model_executor.models.qwen3_tts.tokenizer_12hz.modeling_qwen3_tts_tokenizer_v2 import (
|
||||
Qwen3TTSTokenizerV2Decoder,
|
||||
)
|
||||
|
||||
model_dir = resolve_model_dir(model)
|
||||
cfg = Qwen3TTSTokenizerV2Config.from_pretrained(str(model_dir), subfolder="speech_tokenizer")
|
||||
decoder = Qwen3TTSTokenizerV2Decoder._from_config(cfg.decoder_config)
|
||||
state = load_file(str(model_dir / "speech_tokenizer" / "model.safetensors"))
|
||||
dec_state = {k[len("decoder.") :]: v for k, v in state.items() if k.startswith("decoder.")}
|
||||
_, unexpected = decoder.load_state_dict(dec_state, strict=False)
|
||||
if unexpected:
|
||||
raise RuntimeError(f"unexpected decoder keys: {unexpected[:5]}")
|
||||
decoder.eval().to(dtype=dtype)
|
||||
if hasattr(decoder, "precompute_snake_caches"):
|
||||
decoder.precompute_snake_caches()
|
||||
return decoder
|
||||
|
||||
|
||||
def export_program(module: WindowedCode2Wav) -> Any:
|
||||
"""``torch.export`` the windowed module with its fixed example input."""
|
||||
example = module.example_input()
|
||||
with torch.inference_mode():
|
||||
return torch.export.export(module.eval(), (example,))
|
||||
|
||||
|
||||
def lower_executorch(exported_program: Any, partitioner: str = "xnnpack") -> bytes:
|
||||
"""Lower an exported program to an ExecuTorch ``.pte`` buffer (requires executorch)."""
|
||||
from executorch.exir import to_edge_transform_and_lower # type: ignore[import-not-found]
|
||||
|
||||
partitioners = []
|
||||
if partitioner == "xnnpack":
|
||||
from executorch.backends.xnnpack.partition.xnnpack_partitioner import ( # type: ignore[import-not-found]
|
||||
XnnpackPartitioner,
|
||||
)
|
||||
|
||||
partitioners = [XnnpackPartitioner()]
|
||||
elif partitioner == "qnn": # unverified: needs the Qualcomm SDK build of executorch
|
||||
from executorch.backends.qualcomm.partition.qnn_partitioner import ( # type: ignore[import-not-found]
|
||||
QnnPartitioner,
|
||||
)
|
||||
|
||||
partitioners = [QnnPartitioner()]
|
||||
elif partitioner not in ("none", ""):
|
||||
raise ValueError(f"unknown partitioner {partitioner!r}")
|
||||
edge = to_edge_transform_and_lower(exported_program, partitioner=partitioners or None)
|
||||
return edge.to_executorch().buffer
|
||||
|
||||
|
||||
def export_windowed_decoder(
|
||||
model: str,
|
||||
out_dir: str | Path,
|
||||
*,
|
||||
ramp: tuple[int, ...] = DEFAULT_RAMP,
|
||||
context_frames: int = DEFAULT_CONTEXT,
|
||||
backend: str = "torch_export",
|
||||
partitioner: str = "xnnpack",
|
||||
check: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""Export one program per chunk size; returns a manifest (also written as ``manifest.json``)."""
|
||||
out = Path(out_dir)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
decoder = load_code2wav_decoder(model)
|
||||
manifest: dict[str, Any] = {
|
||||
"model": model,
|
||||
"backend": backend,
|
||||
"partitioner": partitioner if backend == "executorch" else None,
|
||||
"context_frames": context_frames,
|
||||
"num_quantizers": int(decoder.config.num_quantizers),
|
||||
"hop": int(decoder.total_upsample),
|
||||
"sample_rate": int(getattr(decoder.config, "output_sample_rate", 24000) or 24000),
|
||||
"programs": {},
|
||||
}
|
||||
for chunk in sorted(set(int(c) for c in ramp)):
|
||||
module = WindowedCode2Wav(decoder, chunk, context_frames)
|
||||
t0 = time.perf_counter()
|
||||
ep = export_program(module)
|
||||
entry: dict[str, Any] = {"chunk_frames": chunk, "window_frames": module.window_frames}
|
||||
if check:
|
||||
codes = torch.randint(0, int(decoder.config.codebook_size), module.example_input().shape, dtype=torch.long)
|
||||
with torch.inference_mode():
|
||||
ref = module(codes)
|
||||
got = ep.module()(codes)
|
||||
entry["max_abs_diff_vs_eager"] = float((ref - got).abs().max())
|
||||
entry["out_samples"] = int(got.shape[-1])
|
||||
if backend == "torch_export":
|
||||
path = out / f"code2wav_c{chunk}_ctx{context_frames}.pt2"
|
||||
torch.export.save(ep, str(path))
|
||||
elif backend == "executorch":
|
||||
path = out / f"code2wav_c{chunk}_ctx{context_frames}.pte"
|
||||
path.write_bytes(lower_executorch(ep, partitioner))
|
||||
else:
|
||||
raise ValueError(f"unknown backend {backend!r}")
|
||||
entry["path"] = str(path)
|
||||
entry["export_s"] = time.perf_counter() - t0
|
||||
entry["bytes"] = path.stat().st_size
|
||||
manifest["programs"][str(chunk)] = entry
|
||||
print(
|
||||
f"[decoder_export] chunk={chunk} -> {path} ({entry['bytes'] / 1e6:.1f} MB, {entry['export_s']:.1f} s)",
|
||||
flush=True,
|
||||
)
|
||||
(out / "manifest.json").write_text(json.dumps(manifest, indent=1))
|
||||
return manifest
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--model", required=True)
|
||||
p.add_argument("--out-dir", required=True)
|
||||
p.add_argument("--ramp", type=int, nargs="*", default=list(DEFAULT_RAMP))
|
||||
p.add_argument("--context", type=int, default=DEFAULT_CONTEXT)
|
||||
p.add_argument("--backend", choices=["torch_export", "executorch"], default="torch_export")
|
||||
p.add_argument("--partitioner", choices=["xnnpack", "qnn", "none"], default="xnnpack")
|
||||
p.add_argument("--no-check", action="store_true")
|
||||
args = p.parse_args(argv)
|
||||
export_windowed_decoder(
|
||||
args.model,
|
||||
args.out_dir,
|
||||
ramp=tuple(args.ramp),
|
||||
context_frames=args.context,
|
||||
backend=args.backend,
|
||||
partitioner=args.partitioner,
|
||||
check=not args.no_check,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user