# CosyVoice3 deploy: 2-stage talker → flow-matching code2wav.
#
# Default mode is async-chunk streaming through SharedMemoryConnector. Pass
# ``--no-async-chunk`` to the serve CLI for the legacy sync path (stage 1
# builds flow input from the full speech-token sequence via ``text2flow``).
# The shared-memory connector definition and input/output_connectors
# references stay in the yaml unconditionally; the runtime only activates
# them when ``async_chunk: true``.
#
# enforce_eager:
#   * Stage 0 (talker) — enforce_eager: false (CUDA graphs enabled).
#   * Stage 1 (code2wav) — enforce_eager: true (CUDA graphs don't work with
#     dynamic conv shapes).
#
# Streaming continuity is GPU-class sensitive — TUNE PER GPU.
#   The defaults below (max_num_seqs: 8, codec_chunk_frames: 15) are tuned for
#   H100, where streaming continuity at concurrency 4 stays ~100%. They push
#   more audio per scheduler step through the SharedMemoryConnector than a
#   slower GPU can keep up with: on H20-3e the same config regressed continuity
#   to ~78% at c=4 (vs ~86% with smaller batch/chunk), because code2wav can't
#   produce mel chunks fast enough to feed the streaming output → buffer
#   underruns → audible gaps.
#   On a slower GPU class, lower max_num_seqs (e.g. 4) and/or codec_chunk_frames
#   first. Raising connector_get_sleep_s also relaxes the consumer's polling
#   cadence, but trades against TTFP — keep it as a last resort.
pipeline: cosyvoice3
async_chunk: true
connectors:
  connector_of_shared_memory:
    name: SharedMemoryConnector
    extra:
      codec_streaming: true
      connector_get_sleep_s: 0.01
      connector_get_max_wait_first_chunk: 3000
      connector_get_max_wait: 300
      codec_chunk_frames: 15
      codec_left_context_frames: 25
      codec_vocab_size: 6561

stages:
  - stage_id: 0
    max_num_batched_tokens: 32768
    max_num_seqs: 8
    gpu_memory_utilization: 0.4
    enforce_eager: false
    trust_remote_code: true
    enable_prefix_caching: false
    devices: "0"
    output_connectors:
      to_stage_1: connector_of_shared_memory
    default_sampling_params:
      max_tokens: 2048
      top_k: 25
      top_p: 0.8
      # near-identity repetition penalty forces vLLM to track
      # output_token_ids for RAS (stop-token logit logsumexp).
      repetition_penalty: 1.0001
    disable_hybrid_kv_cache_manager: true
    mm_processor_cache_gb: 0
    skip_mm_profiling: true

  - stage_id: 1
    max_num_batched_tokens: 32768
    max_num_seqs: 8
    gpu_memory_utilization: 0.2
    enforce_eager: true
    trust_remote_code: true
    enable_prefix_caching: false
    max_model_len: 32768
    devices: "0"
    input_connectors:
      from_stage_0: connector_of_shared_memory
    default_sampling_params:
      max_tokens: 2048
    disable_hybrid_kv_cache_manager: true
    skip_mm_profiling: true