[Bugfix] Remove mandatory flash-attn dependency and optimzie docs (#113)

Signed-off-by: Gaohan123 <gaohan19@huawei.com>
This commit is contained in:
Gao Han authored and GitHub committed 2025-11-30 18:20:44 +08:00
1 parent 228f4e5070
commit 88c00fb271
29 files changed
+857 -176

No files matched your search

@@ -23,7 +23,7 @@ Then run the command below.
```bash
bash run_multiple_prompts.sh
```
### Single Prompts
### Single Prompt
Get into the example folder
```bash
cd examples/offline_inference/qwen2_5_omni
@@ -16,7 +16,7 @@ from vllm.multimodal.image import convert_image_mode
from vllm.sampling_params import SamplingParams
from vllm.utils import FlexibleArgumentParser
from vllm_omni.entrypoints.omni_llm import OmniLLM
from vllm_omni.entrypoints.omni import Omni
SEED = 42
@@ -140,7 +140,7 @@ def main(args):
model_name = "Qwen/Qwen2.5-Omni-7B"
query_result = query_map[args.query_type]()
omni_llm = OmniLLM(
omni_llm = Omni(
model=model_name,
log_stats=args.enable_stats,
log_file=("omni_llm_pipeline.log" if args.enable_stats else None),
@@ -1,2 +1,2 @@
python end2end.py --output-wav output_audio \
--query-type use_audio_in_video
--query-type text
@@ -1,4 +1,4 @@
# Offline Example of vLLM-Omni for Qwen2.5-omni
# Offline Example of vLLM-Omni for Qwen3-omni
## 🛠️ Installation
@@ -15,7 +15,7 @@ Then run the command below.
```bash
bash run_multiple_prompts.sh
```
### Single Prompts
### Single Prompt
Get into the example folder
```bash
cd examples/offline_inference/qwen3_omni
@@ -24,6 +24,10 @@ Then run the command below.
```bash
bash run_single_prompt.sh
```
If you have not enough memory, you can set thinker with tensor parallel. Just run the command below.
```bash
bash run_single_prompt_tp.sh
```
### FAQ
@@ -16,7 +16,7 @@ from vllm.assets.video import VideoAsset
from vllm.multimodal.image import convert_image_mode
from vllm.utils import FlexibleArgumentParser
from vllm_omni.entrypoints.omni_llm import OmniLLM
from vllm_omni.entrypoints.omni import Omni
SEED = 42
@@ -127,8 +127,9 @@ def main(args):
model_name = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
query_result = query_map[args.query_type]()
omni_llm = OmniLLM(
omni_llm = Omni(
model=model_name,
stage_configs_path=args.stage_configs_path,
)
thinker_sampling_params = SamplingParams(
@@ -276,6 +277,12 @@ def parse_args():
default=None,
help="Path to a .txt file with one prompt per line (preferred).",
)
parser.add_argument(
"--stage-configs-path",
type=str,
default=None,
help="Path to a stage configs file.",
)
return parser.parse_args()
@@ -283,7 +290,3 @@ def parse_args():
if __name__ == "__main__":
args = parse_args()
main(args)
# use examples:
# python end2end.py --model Qwen/Qwen3-Omni-30B-A3B-Instruct --query-type text
# python end2end.py --model /custom_path/Qwen3-Omni-30B-A3B-Instruct --query-type use_video_only
@@ -0,0 +1,94 @@
# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
# Stage 0: Thinker (multimodal understanding + text generation)
# Stage 1: Talker (text embeddings → 8-layer RVQ codec codes)
# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
stage_args:
- stage_id: 0
runtime:
devices: "0,1"
max_batch_size: 1
engine_args:
model_stage: thinker
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
scheduler_cls: vllm_omni.core.sched.scheduler.OmniScheduler
gpu_memory_utilization: 0.8
enforce_eager: true
trust_remote_code: true
engine_output_type: latent # Output hidden states for talker
distributed_executor_backend: "mp"
enable_prefix_caching: false
hf_config_name: thinker_config
tensor_parallel_size: 2
final_output: true
final_output_type: text
is_comprehension: true
default_sampling_params:
temperature: 0.4
top_p: 0.9
top_k: 1
max_tokens: 2048
seed: 42
detokenize: True
repetition_penalty: 1.05
- stage_id: 1
runtime:
devices: "1"
max_batch_size: 1
engine_args:
model_stage: talker
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
scheduler_cls: vllm_omni.core.sched.scheduler.OmniScheduler
gpu_memory_utilization: 0.5
enforce_eager: true
trust_remote_code: true
engine_output_type: latent # Output codec codes for code2wav
# tensor_parallel_size: 2
enable_prefix_caching: false
distributed_executor_backend: "mp"
hf_config_name: talker_config
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
# final_output: true
# final_output_type: text
default_sampling_params:
temperature: 0.9
top_k: 50
max_tokens: 4096
seed: 42
detokenize: False
repetition_penalty: 1.05
stop_token_ids: [2150]
- stage_id: 2
runtime:
devices: "1"
max_batch_size: 1
engine_args:
model_stage: code2wav
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker
scheduler_cls: vllm_omni.core.sched.generation_scheduler.GenerationScheduler
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: audio # Final output: audio waveform
gpu_memory_utilization: 0.1
distributed_executor_backend: "mp"
max_num_batched_tokens: 1000000
hf_config_name: thinker_config
engine_input_source: [1]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
final_output: true
final_output_type: audio
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 65536
seed: 42
detokenize: True
repetition_penalty: 1.1
@@ -0,0 +1,6 @@
python end2end.py --output-wav output_audio \
--query-type use_audio \
--init-sleep-seconds 90 \
--stage-configs-path qwen3_omni_moe_tp.yaml
# init-sleep-seconds works to avoid two vLLM stages initialized at the same time within a card.
+1 -1
View File
@@ -1,4 +1,4 @@
# Online serving Example of vLLM-omni for Qwen2.5-omni
# Online serving Example of vLLM-Omni for Qwen3-omni
## 🛠️ Installation