Adjust GPU memory utilization and device settings in qwen2_5_omni_ci.yaml for improved performance
Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
1 parent
11b6c16007
commit
72cb4f661b
1 file changed
+3
-3
@@ -47,7 +47,7 @@ stage_args:
|
||||
max_model_len: 16384
|
||||
max_num_batched_tokens: 16384
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.9
|
||||
gpu_memory_utilization: 0.4
|
||||
skip_mm_profiling: true
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
@@ -67,14 +67,14 @@ stage_args:
|
||||
- stage_id: 2
|
||||
runtime:
|
||||
process: true
|
||||
devices: "2" # Example: use a different GPU than the previous stage; use "0" if single GPU
|
||||
devices: "1" # Example: use a different GPU than the previous stage; use "0" if single GPU
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: code2wav
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
gpu_memory_utilization: 0.9 #increase the gpu memory utilization to enable the test on H800
|
||||
gpu_memory_utilization: 0.5 #increase the gpu memory utilization to enable the test on H800
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
|
||||
Reference in new issue
Block a user