c32a26a845
Signed-off-by: Joshna Medisetty <joshna.medisetty@intel.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Chendi.Xue <chendi.xue@intel.com>
51 lines
1.4 KiB
YAML
51 lines
1.4 KiB
YAML
# Ming-flash-omni-2.0 thinker-only deploy: multimodal understanding -> text.
|
|
#
|
|
# Single-stage configuration that drops the talker; useful when only text
|
|
# output is required (chat / understanding / reasoning)
|
|
#
|
|
# Stage 0: Thinker — TP=4 across GPUs 0-3 with full memory budget since no
|
|
# talker shares the last device.
|
|
pipeline: ming_flash_omni_thinker_only
|
|
async_chunk: false
|
|
|
|
trust_remote_code: true
|
|
distributed_executor_backend: mp
|
|
enable_prefix_caching: false
|
|
|
|
stages:
|
|
- stage_id: 0
|
|
max_num_seqs: 1
|
|
gpu_memory_utilization: 0.9
|
|
enforce_eager: false
|
|
tensor_parallel_size: 4
|
|
max_num_batched_tokens: 32768
|
|
devices: "0,1,2,3"
|
|
# disable fused allreduce+rms for flashinfer version mismatch for now
|
|
compilation_config:
|
|
pass_config:
|
|
fuse_allreduce_rms: false
|
|
default_sampling_params:
|
|
temperature: 0.4
|
|
top_p: 0.9
|
|
max_tokens: 2048
|
|
repetition_penalty: 1.05
|
|
detokenize: true
|
|
ignore_eos: false
|
|
|
|
platforms:
|
|
xpu:
|
|
stages:
|
|
- stage_id: 0
|
|
# BF16 stays ~26 GiB per card even at TP=8. FP8 halves that.
|
|
devices: "0,1,2,3,4,5,6,7"
|
|
tensor_parallel_size: 8
|
|
quantization: fp8
|
|
max_num_batched_tokens: 4096
|
|
max_model_len: 8192
|
|
enforce_eager: true
|
|
skip_mm_profiling: true
|
|
limit_mm_per_prompt:
|
|
image: 1
|
|
video: 0
|
|
audio: 1
|