c32a26a845
Signed-off-by: Joshna Medisetty <joshna.medisetty@intel.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Chendi.Xue <chendi.xue@intel.com>
25 lines
604 B
YAML
25 lines
604 B
YAML
# MammothModa2 AR-only (image/video understanding and summarization).
|
|
|
|
async_chunk: false
|
|
pipeline: mammoth_moda2_ar
|
|
|
|
stages:
|
|
- stage_id: 0
|
|
devices: "0"
|
|
max_num_seqs: 16
|
|
max_model_len: 8192
|
|
gpu_memory_utilization: 0.5
|
|
enforce_eager: true
|
|
trust_remote_code: true
|
|
enable_prefix_caching: false
|
|
|
|
platforms:
|
|
xpu:
|
|
stages:
|
|
- stage_id: 0
|
|
# BF16 weights are ~31 GiB and do not fit one 32 GiB card.
|
|
# 0.9 leaves KV room once the weights are split across two cards.
|
|
devices: "0,1"
|
|
tensor_parallel_size: 2
|
|
gpu_memory_utilization: 0.9
|