[Bugfix] Remove mandatory flash-attn dependency and optimzie docs (#113)
Signed-off-by: Gaohan123 <gaohan19@huawei.com>
This commit is contained in:
1 parent
228f4e5070
commit
88c00fb271
29 files changed
+857
-176
No files matched your search
@@ -23,7 +23,7 @@ Then run the command below.
|
||||
```bash
|
||||
bash run_multiple_prompts.sh
|
||||
```
|
||||
### Single Prompts
|
||||
### Single Prompt
|
||||
Get into the example folder
|
||||
```bash
|
||||
cd examples/offline_inference/qwen2_5_omni
|
||||
|
||||
@@ -16,7 +16,7 @@ from vllm.multimodal.image import convert_image_mode
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils import FlexibleArgumentParser
|
||||
|
||||
from vllm_omni.entrypoints.omni_llm import OmniLLM
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
|
||||
SEED = 42
|
||||
|
||||
@@ -140,7 +140,7 @@ def main(args):
|
||||
model_name = "Qwen/Qwen2.5-Omni-7B"
|
||||
query_result = query_map[args.query_type]()
|
||||
|
||||
omni_llm = OmniLLM(
|
||||
omni_llm = Omni(
|
||||
model=model_name,
|
||||
log_stats=args.enable_stats,
|
||||
log_file=("omni_llm_pipeline.log" if args.enable_stats else None),
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
python end2end.py --output-wav output_audio \
|
||||
--query-type use_audio_in_video
|
||||
--query-type text
|
||||
@@ -1,4 +1,4 @@
|
||||
# Offline Example of vLLM-Omni for Qwen2.5-omni
|
||||
# Offline Example of vLLM-Omni for Qwen3-omni
|
||||
|
||||
## 🛠️ Installation
|
||||
|
||||
@@ -15,7 +15,7 @@ Then run the command below.
|
||||
```bash
|
||||
bash run_multiple_prompts.sh
|
||||
```
|
||||
### Single Prompts
|
||||
### Single Prompt
|
||||
Get into the example folder
|
||||
```bash
|
||||
cd examples/offline_inference/qwen3_omni
|
||||
@@ -24,6 +24,10 @@ Then run the command below.
|
||||
```bash
|
||||
bash run_single_prompt.sh
|
||||
```
|
||||
If you have not enough memory, you can set thinker with tensor parallel. Just run the command below.
|
||||
```bash
|
||||
bash run_single_prompt_tp.sh
|
||||
```
|
||||
|
||||
### FAQ
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ from vllm.assets.video import VideoAsset
|
||||
from vllm.multimodal.image import convert_image_mode
|
||||
from vllm.utils import FlexibleArgumentParser
|
||||
|
||||
from vllm_omni.entrypoints.omni_llm import OmniLLM
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
|
||||
SEED = 42
|
||||
|
||||
@@ -127,8 +127,9 @@ def main(args):
|
||||
model_name = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
|
||||
query_result = query_map[args.query_type]()
|
||||
|
||||
omni_llm = OmniLLM(
|
||||
omni_llm = Omni(
|
||||
model=model_name,
|
||||
stage_configs_path=args.stage_configs_path,
|
||||
)
|
||||
|
||||
thinker_sampling_params = SamplingParams(
|
||||
@@ -276,6 +277,12 @@ def parse_args():
|
||||
default=None,
|
||||
help="Path to a .txt file with one prompt per line (preferred).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stage-configs-path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to a stage configs file.",
|
||||
)
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
@@ -283,7 +290,3 @@ def parse_args():
|
||||
if __name__ == "__main__":
|
||||
args = parse_args()
|
||||
main(args)
|
||||
|
||||
# use examples:
|
||||
# python end2end.py --model Qwen/Qwen3-Omni-30B-A3B-Instruct --query-type text
|
||||
# python end2end.py --model /custom_path/Qwen3-Omni-30B-A3B-Instruct --query-type use_video_only
|
||||
@@ -0,0 +1,94 @@
|
||||
# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
|
||||
# Stage 0: Thinker (multimodal understanding + text generation)
|
||||
# Stage 1: Talker (text embeddings → 8-layer RVQ codec codes)
|
||||
# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
|
||||
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
runtime:
|
||||
devices: "0,1"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: thinker
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
|
||||
scheduler_cls: vllm_omni.core.sched.scheduler.OmniScheduler
|
||||
gpu_memory_utilization: 0.8
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output hidden states for talker
|
||||
distributed_executor_backend: "mp"
|
||||
enable_prefix_caching: false
|
||||
hf_config_name: thinker_config
|
||||
tensor_parallel_size: 2
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
is_comprehension: true
|
||||
default_sampling_params:
|
||||
temperature: 0.4
|
||||
top_p: 0.9
|
||||
top_k: 1
|
||||
max_tokens: 2048
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.05
|
||||
|
||||
- stage_id: 1
|
||||
runtime:
|
||||
devices: "1"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: talker
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
|
||||
scheduler_cls: vllm_omni.core.sched.scheduler.OmniScheduler
|
||||
gpu_memory_utilization: 0.5
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output codec codes for code2wav
|
||||
# tensor_parallel_size: 2
|
||||
enable_prefix_caching: false
|
||||
distributed_executor_backend: "mp"
|
||||
hf_config_name: talker_config
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
|
||||
# final_output: true
|
||||
# final_output_type: text
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_k: 50
|
||||
max_tokens: 4096
|
||||
seed: 42
|
||||
detokenize: False
|
||||
repetition_penalty: 1.05
|
||||
stop_token_ids: [2150]
|
||||
|
||||
- stage_id: 2
|
||||
runtime:
|
||||
devices: "1"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: code2wav
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker
|
||||
scheduler_cls: vllm_omni.core.sched.generation_scheduler.GenerationScheduler
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio # Final output: audio waveform
|
||||
gpu_memory_utilization: 0.1
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 1000000
|
||||
hf_config_name: thinker_config
|
||||
engine_input_source: [1]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 65536
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
@@ -0,0 +1,6 @@
|
||||
python end2end.py --output-wav output_audio \
|
||||
--query-type use_audio \
|
||||
--init-sleep-seconds 90 \
|
||||
--stage-configs-path qwen3_omni_moe_tp.yaml
|
||||
|
||||
# init-sleep-seconds works to avoid two vLLM stages initialized at the same time within a card.
|
||||
@@ -1,4 +1,4 @@
|
||||
# Online serving Example of vLLM-omni for Qwen2.5-omni
|
||||
# Online serving Example of vLLM-Omni for Qwen3-omni
|
||||
|
||||
## 🛠️ Installation
|
||||
|
||||
|
||||
Reference in new issue
Block a user