From 36ada0478aebfdb9b0599af6e3b279911544b052 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Mon, 10 Nov 2025 15:07:01 +0800 Subject: [PATCH 1/6] add back the sleep time for the stage init to avoid single card conflict Signed-off-by: tzhouam --- vllm_omni/entrypoints/omni_llm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vllm_omni/entrypoints/omni_llm.py b/vllm_omni/entrypoints/omni_llm.py index 458e000e5..63dab9fde 100644 --- a/vllm_omni/entrypoints/omni_llm.py +++ b/vllm_omni/entrypoints/omni_llm.py @@ -178,6 +178,7 @@ class OmniLLM(BaseOmniLLM): batch_timeout=self.batch_timeout, ) logger.debug("[Orchestrator] Stage-%s process started", stage_id) + time.sleep(self._init_sleep_seconds) def close(self) -> None: for q in self._stage_in_queues: From 0ef1fefd4522cb4767f77b8ccde0a9fc1121872e Mon Sep 17 00:00:00 2001 From: tzhouam Date: Mon, 10 Nov 2025 15:44:44 +0800 Subject: [PATCH 2/6] add prompt file Signed-off-by: tzhouam --- .../offline_inference/qwen_2_5_omni/top100.pt | Bin 0 -> 9319 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 examples/offline_inference/qwen_2_5_omni/top100.pt diff --git a/examples/offline_inference/qwen_2_5_omni/top100.pt b/examples/offline_inference/qwen_2_5_omni/top100.pt new file mode 100644 index 0000000000000000000000000000000000000000..017c60ac806dc45952b6bf652fd60ad362ece700 GIT binary patch literal 9319 zcmd6tU6UNudB+zpg4i*TIKd`1<{&#Pah7JqN?P$M5FoP_fZeeU$nqZ6E6rGvdV5UV#n)_BK z>i)DwtDW_kshl*L`4$c5mVf=&SkrnxTMx!&z7JiI1+=o2nNMA1w5V$rMKv*#nNCln-*A3-ZB$!%XoC$Rhd~)roK6Nxhj3c#=a1NTF}*wg~)WyZCD;el{FYx8kVUCCx)Clk{+U z37u|ui&Z9oB`;kF7}2)AAFys}+a7GNvqQhC18E+Enkmjon!mWWjYl7WTm{+20Bj7f0KYlGaHvSFP%R8ba#dz$^$bKO%~wuC=wGI;H?(9}~RWk6ZzhF`%Kb~j;W+b;f`G>@jQBYWSK?yv~rh73}f z*#l=5zBhS48*P|T---0}i!C|%Nb%)!CVhc< zaA%n-AS`Twd82v69ikhE7Nhfax1i8?8|-g94rqy+S<-y%6lDHQ`U-l!Lr}I(6bS7P zTpk>pUv$gdgs=3?j6J%TF=eV z+%90AF7y=?7#aAcOluVzwv3XsO)-NYISbtFGg~g|0Q>#jsqgU3^fG6-Qna{y>=YDJ zSF{{v9LN%QHi9RObb28VZ^v1QDpz*YQ%gy{l5I)VRq8s$c_Y+fkQZk9d>JhS9I^(lM+Y4 zWug}G#&Cp1!~R1{pDqsc}|T_2U<}A$;x6jx^)aY{Dx*Z@kEAgx zmG*R+a?%e&48g1|>duTx@QJS!J6K*AJAVAz1lzi%FIqTTW`H_T)2sez;axK-NOWh@ zJsC2xYwIW?dK?K+>+E=MPQvzzmDUj?T%P>+!| zHYlE|h(3?mWHCa1)cU!!%2dcmx1h>0)~N(xYjI$x5u7yNjl8ojlBAY^A;wXO&d(uT z)y}pYqhyF=QP`6Fsw@t+qT7}7zEU97_2pXQ&G1OlWBEGPs4Kh^6gV%Yd&(XTN8N?yk?#8z>r?QkT*y6mRZ0NWehzA720>47{m0lH?W>xgrUwg3 z3f;WGMY%0XZ`tXF)|VPR=H?SVZ_8mI<#H~VT*ieWpGlWbbQ;5J!bMIzis z#WWO5$^Qo>o{wRw99t`mS3`AJ79;`XmO$ED;)rsh&?TK%o$?awNwXPaYM(lN9V)uh zgvtU#3q2&PLp$m5G;B{=!yee;CJZ0pA^;+gC1l{FIk$R|FQlL1@;zKGzv3y<-jb7M z^prG~bzeeqv2@iLBj$5Jo~g1Cy~et}4}D~B0X{%gl&vH);JJ0$8=Ox+-~=Dpve$+N z3!=k@DaMNA5lK)@w53S0qoOPnsypjLt%bESd2t}}%Jj9dkyXhXNps94ncx8i=5JQ%@zwMOSi0VZ#O$}0K6N=A*imt$3S?mp z3d&_1Mi= zyBqe46hHT+aNH@S+5Aw60s0@Pgq1J?O@mFE%cmy!^_b<~q?Pu8IOoKqsUg|ZjK6I5 z$W#Hy#6cew8{-HiV>4nHm^5$1sEU=iHI)+a+?JZS=|0S=Lu;Zi^b!mm30vAq7%_DH zc$_p>(hYp^uF3;s_AOTC!w}shf5w$FL}&qFj*qDV-b}wI>Y}+DFGN@vnIEfhG3ncw zRuy@%8a=dSK`r`Ly32xVuBylglo}j{6OL>?lvP@<^vI~`X&TzYrAQc3 zH&(RlP*6`^c{@&thA~P|m8^jlDNSTBnU1(;nD558W9hQ*j1&=vVKfYp9ZJohDh+&+ z=IUt({r>G}4Zyb;7bIfNUinz<`ijq!2>b|StdBlmP$d?%AN41(q{A!CseB?3LhV(B zF&ZiW{yXX0Sa_dX@X`pxMzjdVZ9l7>5Kl|bAmIoHIoFxwL`tDa>F7_Ym<*FC~(Qt(`9#R zP9l(0J7Yk90J_bs=UQ58hD4Mtt*s+<$^3B`bbY1L-iyt_*-w103TxRmmokbhaZvgB z<+IYdV}KPQK(oJOFU-YwZQ*E|-yg;b1MSi$j@*PJ4)M-nLX66W7^`8A*#^X_W}QKc zo0BkU`QQX>yt@)9AEf8-@_TfQM1BQvlWK#b|LZ5}38-dMKw~xI8{)Pg(bsBpQk&JIA zkoj`t9K*qC+RXf!7M~>f(MqL#oW6mJ-kds~AW;di-$18xHzMmqvK}k=pk+om;^7=4 z$AJw^9aO9pxGY_foKI4RK5q=8e_D~6%reB~+yek?)PmF_9Py;f1K#CCTkS^X)y$DM zC<4F-&(JWaiUTInYC-!-iB1VMX+He|_xfRMYxm-4V`m*_Pbh?8A&WgH!vR}2iM@$V zGfXw*{jtAAaIgUOqg^95e-zc;>*0P3;zLGZFaS6&YEOclk?sJN#`x~mP)G9khiYkA z?C5$4#2J>@38}ZpKq5Pk7J@S0So0~Yg^7Or1@85e^a`$jKSFS+Zh(?9f|P)C_$`hL zi(Fv~UCem~0S0Mtc#t7$Ov%*hD?eM6pr5AippoKK4GTab2B<;FmZZ;&HbZ2jM<21o zB1kdboWY(ht7M|dAgzE!svTm)#(wvDe1J5>P2^?h8G;Sb;W^;2tWdhdSjqy5L%&aEvU?I#2?wES5NR$pDc^4!TM z{Kud7pZs=CyZ-Bc{o60^!-IN%{;p4~UAn!idPkDXXVj;YN>Oz6%K1NU zH!q#Pcv|JxO^Dob09ldw2 z-6w@_|Id@Vf%~X_IQBa6|4>dYz8im!A6xi);^0ps{iKY$A@syC86TKCGX4m+C&k>2 zg2#@D`Q05cx6yZU_1%7a3f_csb~UboH;e_8jCK0Wx|;rkH(560WiuuAt2_x?9;1?{B( literal 0 HcmV?d00001 From 519c140d2018ec7d3177fb813285da8fda226d71 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Mon, 10 Nov 2025 15:47:28 +0800 Subject: [PATCH 3/6] Update run script to include initialization sleep time and support for loading prompts from a .pt file. Signed-off-by: tzhouam --- examples/offline_inference/qwen_2_5_omni/run.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/offline_inference/qwen_2_5_omni/run.sh b/examples/offline_inference/qwen_2_5_omni/run.sh index 7f02a672c..bee28e2c4 100644 --- a/examples/offline_inference/qwen_2_5_omni/run.sh +++ b/examples/offline_inference/qwen_2_5_omni/run.sh @@ -1,8 +1,9 @@ export PYTHONPATH=/path/to/vllm-omni:$PYTHONPATH python end2end.py --model Qwen/Qwen2.5-Omni-7B \ - --prompts "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words." \ --voice-type "m02" \ --dit-ckpt none \ --bigvgan-ckpt none \ --output-wav output_audio \ - --prompt_type text + --prompt_type text \ + --init-sleep-seconds 0 \ + --pt-prompts top100.pt From c18fc40f104c665b020e76163aa6cc52e2610581 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Mon, 10 Nov 2025 15:47:28 +0800 Subject: [PATCH 4/6] Update run script to include initialization sleep time and support for loading prompts from a .pt file. Signed-off-by: tzhouam --- .../offline_inference/qwen_2_5_omni/run.sh | 5 ++-- .../offline_inference/qwen_2_5_omni/utils.py | 27 ++++++++++++++----- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/examples/offline_inference/qwen_2_5_omni/run.sh b/examples/offline_inference/qwen_2_5_omni/run.sh index 7f02a672c..bee28e2c4 100644 --- a/examples/offline_inference/qwen_2_5_omni/run.sh +++ b/examples/offline_inference/qwen_2_5_omni/run.sh @@ -1,8 +1,9 @@ export PYTHONPATH=/path/to/vllm-omni:$PYTHONPATH python end2end.py --model Qwen/Qwen2.5-Omni-7B \ - --prompts "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words." \ --voice-type "m02" \ --dit-ckpt none \ --bigvgan-ckpt none \ --output-wav output_audio \ - --prompt_type text + --prompt_type text \ + --init-sleep-seconds 0 \ + --pt-prompts top100.pt diff --git a/examples/offline_inference/qwen_2_5_omni/utils.py b/examples/offline_inference/qwen_2_5_omni/utils.py index 730d2a077..42894c4a7 100644 --- a/examples/offline_inference/qwen_2_5_omni/utils.py +++ b/examples/offline_inference/qwen_2_5_omni/utils.py @@ -13,6 +13,10 @@ from processing_omni import fetch_image, fetch_video from vllm.inputs import TextPrompt from vllm_omni.inputs.data import OmniTokensPrompt +# Simple caches to avoid repeated heavy HF loads per prompt +_PROCESSOR_CACHE: Dict[str, "AutoProcessor"] = {} +_CONFIG_CACHE: Dict[str, "AutoConfig"] = {} + def get_system_prompt(): @@ -113,16 +117,25 @@ def make_inputs_qwen2_omni( from transformers import AutoConfig, AutoProcessor - processor = AutoProcessor.from_pretrained(args.model) + # Cached processor/config to prevent per-prompt reloading and repeated warnings + if args.model not in _PROCESSOR_CACHE: + _PROCESSOR_CACHE[args.model] = AutoProcessor.from_pretrained(args.model) + processor = _PROCESSOR_CACHE[args.model] - try: - config = AutoConfig.from_pretrained(args.model) - if "Qwen2_5OmniModel" in config.architectures: - args.legacy_omni_video = False + config = _CONFIG_CACHE.get(args.model) + if config is None: + try: + config = AutoConfig.from_pretrained(args.model) + except Exception: + config = None + _CONFIG_CACHE[args.model] = config # cache even if None to avoid retry storms + + # Decide legacy flag only once based on config (default True if unknown) + if getattr(args, "legacy_omni_video", None) is None: + if config is not None and hasattr(config, "architectures"): + args.legacy_omni_video = not ("Qwen2_5OmniModel" in config.architectures) else: args.legacy_omni_video = True - except Exception: - args.legacy_omni_video = True audios, images, videos = [], [], [] for message in messages: From 5f3f53a39d2b3e98850664ed32e0bf980b64f20e Mon Sep 17 00:00:00 2001 From: tzhouam Date: Mon, 10 Nov 2025 16:36:55 +0800 Subject: [PATCH 5/6] Revert "Update run script to include initialization sleep time and support for loading prompts from a .pt file." This reverts commit c18fc40f104c665b020e76163aa6cc52e2610581. --- .../offline_inference/qwen_2_5_omni/run.sh | 5 ++-- .../offline_inference/qwen_2_5_omni/utils.py | 27 +++++-------------- 2 files changed, 9 insertions(+), 23 deletions(-) diff --git a/examples/offline_inference/qwen_2_5_omni/run.sh b/examples/offline_inference/qwen_2_5_omni/run.sh index bee28e2c4..7f02a672c 100644 --- a/examples/offline_inference/qwen_2_5_omni/run.sh +++ b/examples/offline_inference/qwen_2_5_omni/run.sh @@ -1,9 +1,8 @@ export PYTHONPATH=/path/to/vllm-omni:$PYTHONPATH python end2end.py --model Qwen/Qwen2.5-Omni-7B \ + --prompts "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words." \ --voice-type "m02" \ --dit-ckpt none \ --bigvgan-ckpt none \ --output-wav output_audio \ - --prompt_type text \ - --init-sleep-seconds 0 \ - --pt-prompts top100.pt + --prompt_type text diff --git a/examples/offline_inference/qwen_2_5_omni/utils.py b/examples/offline_inference/qwen_2_5_omni/utils.py index 42894c4a7..730d2a077 100644 --- a/examples/offline_inference/qwen_2_5_omni/utils.py +++ b/examples/offline_inference/qwen_2_5_omni/utils.py @@ -13,10 +13,6 @@ from processing_omni import fetch_image, fetch_video from vllm.inputs import TextPrompt from vllm_omni.inputs.data import OmniTokensPrompt -# Simple caches to avoid repeated heavy HF loads per prompt -_PROCESSOR_CACHE: Dict[str, "AutoProcessor"] = {} -_CONFIG_CACHE: Dict[str, "AutoConfig"] = {} - def get_system_prompt(): @@ -117,25 +113,16 @@ def make_inputs_qwen2_omni( from transformers import AutoConfig, AutoProcessor - # Cached processor/config to prevent per-prompt reloading and repeated warnings - if args.model not in _PROCESSOR_CACHE: - _PROCESSOR_CACHE[args.model] = AutoProcessor.from_pretrained(args.model) - processor = _PROCESSOR_CACHE[args.model] + processor = AutoProcessor.from_pretrained(args.model) - config = _CONFIG_CACHE.get(args.model) - if config is None: - try: - config = AutoConfig.from_pretrained(args.model) - except Exception: - config = None - _CONFIG_CACHE[args.model] = config # cache even if None to avoid retry storms - - # Decide legacy flag only once based on config (default True if unknown) - if getattr(args, "legacy_omni_video", None) is None: - if config is not None and hasattr(config, "architectures"): - args.legacy_omni_video = not ("Qwen2_5OmniModel" in config.architectures) + try: + config = AutoConfig.from_pretrained(args.model) + if "Qwen2_5OmniModel" in config.architectures: + args.legacy_omni_video = False else: args.legacy_omni_video = True + except Exception: + args.legacy_omni_video = True audios, images, videos = [], [], [] for message in messages: From da385277a210d3c0b4b588404f4bdf27a53279c4 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Mon, 10 Nov 2025 16:37:37 +0800 Subject: [PATCH 6/6] Revert "Revert "Update run script to include initialization sleep time and support for loading prompts from a .pt file."" This reverts commit 5f3f53a39d2b3e98850664ed32e0bf980b64f20e. --- .../offline_inference/qwen_2_5_omni/run.sh | 5 ++-- .../offline_inference/qwen_2_5_omni/utils.py | 27 ++++++++++++++----- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/examples/offline_inference/qwen_2_5_omni/run.sh b/examples/offline_inference/qwen_2_5_omni/run.sh index 7f02a672c..bee28e2c4 100644 --- a/examples/offline_inference/qwen_2_5_omni/run.sh +++ b/examples/offline_inference/qwen_2_5_omni/run.sh @@ -1,8 +1,9 @@ export PYTHONPATH=/path/to/vllm-omni:$PYTHONPATH python end2end.py --model Qwen/Qwen2.5-Omni-7B \ - --prompts "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words." \ --voice-type "m02" \ --dit-ckpt none \ --bigvgan-ckpt none \ --output-wav output_audio \ - --prompt_type text + --prompt_type text \ + --init-sleep-seconds 0 \ + --pt-prompts top100.pt diff --git a/examples/offline_inference/qwen_2_5_omni/utils.py b/examples/offline_inference/qwen_2_5_omni/utils.py index 730d2a077..42894c4a7 100644 --- a/examples/offline_inference/qwen_2_5_omni/utils.py +++ b/examples/offline_inference/qwen_2_5_omni/utils.py @@ -13,6 +13,10 @@ from processing_omni import fetch_image, fetch_video from vllm.inputs import TextPrompt from vllm_omni.inputs.data import OmniTokensPrompt +# Simple caches to avoid repeated heavy HF loads per prompt +_PROCESSOR_CACHE: Dict[str, "AutoProcessor"] = {} +_CONFIG_CACHE: Dict[str, "AutoConfig"] = {} + def get_system_prompt(): @@ -113,16 +117,25 @@ def make_inputs_qwen2_omni( from transformers import AutoConfig, AutoProcessor - processor = AutoProcessor.from_pretrained(args.model) + # Cached processor/config to prevent per-prompt reloading and repeated warnings + if args.model not in _PROCESSOR_CACHE: + _PROCESSOR_CACHE[args.model] = AutoProcessor.from_pretrained(args.model) + processor = _PROCESSOR_CACHE[args.model] - try: - config = AutoConfig.from_pretrained(args.model) - if "Qwen2_5OmniModel" in config.architectures: - args.legacy_omni_video = False + config = _CONFIG_CACHE.get(args.model) + if config is None: + try: + config = AutoConfig.from_pretrained(args.model) + except Exception: + config = None + _CONFIG_CACHE[args.model] = config # cache even if None to avoid retry storms + + # Decide legacy flag only once based on config (default True if unknown) + if getattr(args, "legacy_omni_video", None) is None: + if config is not None and hasattr(config, "architectures"): + args.legacy_omni_video = not ("Qwen2_5OmniModel" in config.architectures) else: args.legacy_omni_video = True - except Exception: - args.legacy_omni_video = True audios, images, videos = [], [], [] for message in messages: