170 lines
3.5 KiB
YAML
170 lines
3.5 KiB
YAML
# Higgs v3 Model Runner V2 streaming profile, validated on one H200.
|
|
# Select an available physical GPU with CUDA_VISIBLE_DEVICES before launch.
|
|
# First audio: 8 frames (320 ms); following chunks: 25 frames (1 s).
|
|
# Codec graph windows are exact: 8 first frames, 25 + 8 left context.
|
|
# AR and codec capacities are 64; tested client concurrencies are 32 and 64.
|
|
# Keep model weights and sampling parameters identical for comparisons.
|
|
# Server flags used in measurements: --disable-log-stats --disable-uvicorn-access-log.
|
|
# Auxiliary graphs are captured after KV sizing and are outside the utilization
|
|
# budget. Leave at least 10 GiB free for codec graph pools on this H200 profile;
|
|
# inspect both allocated and reserved capture logs before raising stage budgets.
|
|
# Sampler graphs cover tensor-valued temperature/top_k/top_p. Disabling top-k
|
|
# or top-p for an entire batch (None metadata) uses eager sampling.
|
|
async_chunk: true
|
|
connectors:
|
|
connector_of_shared_memory:
|
|
name: SharedMemoryConnector
|
|
extra:
|
|
codec_streaming: true
|
|
codec_chunk_frames: 25
|
|
codec_left_context_frames: 8
|
|
codec_right_holdback_frames: 0
|
|
initial_codec_chunk_frames: 8
|
|
connector_get_sleep_s: 0.01
|
|
connector_get_max_wait_first_chunk: 5000
|
|
connector_get_max_wait: 500
|
|
stages:
|
|
- stage_id: 0
|
|
max_num_seqs: 64
|
|
gpu_memory_utilization: 0.6
|
|
enforce_eager: false
|
|
attention_backend: FLASH_ATTN
|
|
trust_remote_code: true
|
|
enable_prefix_caching: false
|
|
async_scheduling: true
|
|
compilation_config:
|
|
cudagraph_capture_sizes:
|
|
- 1
|
|
- 2
|
|
- 4
|
|
- 8
|
|
- 16
|
|
- 24
|
|
- 32
|
|
- 40
|
|
- 48
|
|
- 56
|
|
- 64
|
|
- 96
|
|
- 128
|
|
- 192
|
|
- 256
|
|
- 384
|
|
- 512
|
|
cudagraph_mode: FULL
|
|
cudagraph_num_of_warmups: 1
|
|
max_cudagraph_capture_size: 512
|
|
cudagraph_copy_inputs: true
|
|
max_num_batched_tokens: 4096
|
|
max_model_len: 8192
|
|
devices: '0'
|
|
output_connectors:
|
|
to_stage_1: connector_of_shared_memory
|
|
default_sampling_params:
|
|
temperature: 1.0
|
|
top_p: 0.95
|
|
top_k: 50
|
|
max_tokens: 2048
|
|
seed: 42
|
|
repetition_penalty: 1.0
|
|
hf_overrides:
|
|
audio_async_prompt_mode: true
|
|
audio_async_payload: true
|
|
audio_mrv2_static_inputs: true
|
|
audio_mrv2_slot_parameters: true
|
|
audio_mrv2_direct_payload: true
|
|
audio_mrv2_sample_graph_buckets: true
|
|
audio_mrv2_async_state_indices: true
|
|
audio_mrv2_fused_state: true
|
|
- stage_id: 1
|
|
max_num_seqs: 64
|
|
gpu_memory_utilization: 0.25
|
|
enforce_eager: true
|
|
trust_remote_code: true
|
|
enable_prefix_caching: false
|
|
async_scheduling: false
|
|
max_num_batched_tokens: 65536
|
|
max_model_len: 65536
|
|
devices: '0'
|
|
input_connectors:
|
|
from_stage_0: connector_of_shared_memory
|
|
default_sampling_params:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
top_k: -1
|
|
max_tokens: 65536
|
|
seed: 42
|
|
repetition_penalty: 1.0
|
|
hf_overrides:
|
|
codec_graph_single_max_frames: 33
|
|
codec_graph_batch_sizes:
|
|
- 1
|
|
- 2
|
|
- 3
|
|
- 4
|
|
- 5
|
|
- 6
|
|
- 7
|
|
- 8
|
|
- 9
|
|
- 10
|
|
- 11
|
|
- 12
|
|
- 13
|
|
- 14
|
|
- 15
|
|
- 16
|
|
- 17
|
|
- 18
|
|
- 19
|
|
- 20
|
|
- 21
|
|
- 22
|
|
- 23
|
|
- 24
|
|
- 25
|
|
- 26
|
|
- 27
|
|
- 28
|
|
- 29
|
|
- 30
|
|
- 31
|
|
- 32
|
|
- 33
|
|
- 34
|
|
- 35
|
|
- 36
|
|
- 37
|
|
- 38
|
|
- 39
|
|
- 40
|
|
- 41
|
|
- 42
|
|
- 43
|
|
- 44
|
|
- 45
|
|
- 46
|
|
- 47
|
|
- 48
|
|
- 49
|
|
- 50
|
|
- 51
|
|
- 52
|
|
- 53
|
|
- 54
|
|
- 55
|
|
- 56
|
|
- 57
|
|
- 58
|
|
- 59
|
|
- 60
|
|
- 61
|
|
- 62
|
|
- 63
|
|
- 64
|
|
codec_graph_frame_sizes:
|
|
- 8
|
|
- 33
|
|
cuda_mps: true
|
|
model_runner: v2
|