Youyuan Liandprincepride
423f34326e
Signed-off-by: Youyuan Li <105433145+Darcy-Lee@users.noreply.github.com> Signed-off-by: princepride <wangzhipeng628@gmail.com> Co-authored-by: princepride <wangzhipeng628@gmail.com>
56 lines
2.5 KiB
YAML
56 lines
2.5 KiB
YAML
# YuE2-3B deploy: single-stage native-AR text-to-music (~3.6B bf16 + VAE).
|
|
# Target 1x 24 GB card; verified on a single RTX 4090.
|
|
#
|
|
# The architectures pin is required: the checkpoint's config.json names
|
|
# "YuE2ForCausalLM" with an auto_map that only makes sense under
|
|
# trust_remote_code, so the deploy pins vLLM to this package's model class.
|
|
# model_type stays "yue2" so the registered Yue2Config is used; no
|
|
# trust_remote_code is needed (the registered config/model win over the
|
|
# checkpoint's auto_map).
|
|
pipeline: yue2
|
|
async_chunk: false
|
|
dtype: bfloat16
|
|
|
|
stages:
|
|
- stage_id: 0
|
|
max_num_seqs: 4
|
|
# Measured on a 24 GB RTX 4090 with 4 concurrent requests pinned to the
|
|
# 9000-frame cap (worst case): the engine reserves ~15.8 GiB at 0.70 and
|
|
# peak usage during the terminal NAR/VAE finishing pass reaches 20.9 GiB,
|
|
# leaving ~3 GiB of headroom. 0.85 reserves ~19.4 GiB and the same
|
|
# workload OOMs inside the finishing pass.
|
|
gpu_memory_utilization: 0.70
|
|
# Phase 1 keeps the backbone eager: the model-owned sampler runs after the
|
|
# forward, and the terminal NAR/VAE pass runs inside the finishing step.
|
|
# CUDA-graph capture of the standard Qwen3 backbone is a perf follow-up.
|
|
enforce_eager: true
|
|
# Prefix caching stays off: the omni prefix-cache save snapshots the
|
|
# step's mm payload before the model-owned sampler runs, but YuE2
|
|
# appends the decoded waveform inside sample() (the terminal NAR/VAE
|
|
# pass), so the cached payload would ship empty audio. Re-enable once
|
|
# the runner saves post-sample.
|
|
enable_prefix_caching: false
|
|
max_num_batched_tokens: 24576
|
|
max_model_len: 24576
|
|
devices: "0"
|
|
# The checkpoint ships no HF tokenizer files (only qwen.tiktoken, which
|
|
# the offline driver reads itself); every prompt arrives as token ids,
|
|
# so the engine builds no tokenizer. Placeholder tokenizer dir follows
|
|
# the indextts2 precedent.
|
|
skip_tokenizer_init: true
|
|
tokenizer: gpt2
|
|
hf_overrides:
|
|
architectures: ["Yue2ForCausalLM"]
|
|
default_sampling_params:
|
|
# Placeholder values: the model's own sampler overrides everything from
|
|
# extra_args. Their presence turns on has_sampling_extra_args, which is
|
|
# what threads the per-request yue2_* keys into the model's forward.
|
|
extra_args:
|
|
yue2_phase: "semantic"
|
|
yue2_seed: 831001
|
|
yue2_max_audio_frames: 200
|
|
detokenize: false
|
|
# Engine-side backstop; the model stops on its own end token first.
|
|
max_tokens: 9000
|
|
temperature: 1.0
|