[Feat] Rename AsyncOmniLLM -> AsyncOmni (#103)
Signed-off-by: WANG Cong <115451386+congw729@users.noreply.github.com>
This commit is contained in:
1 parent
0c245dd1f9
commit
3427fb4774
16 files changed
+98
-95
No files matched your search
@@ -45,7 +45,7 @@ This Web UI demo allows users to interact with the model through a web browser.
|
||||
|
||||
### Running Gradio Demo
|
||||
|
||||
Once vllm and vllm-omni are installed, you can launch the web service built on AsyncOmniLLM by
|
||||
Once vllm and vllm-omni are installed, you can launch the web service built on AsyncOmni by
|
||||
|
||||
```bash
|
||||
python gradio_demo.py --model Qwen/Qwen2.5-Omni-7B --port 7861
|
||||
|
||||
@@ -15,7 +15,7 @@ from PIL import Image
|
||||
from vllm.assets.video import video_get_metadata, video_to_ndarrays
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
from vllm_omni.entrypoints.async_omni_llm import AsyncOmniLLM
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
|
||||
# Import utils from offline inference example
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../offline_inference/qwen2_5_omni"))
|
||||
@@ -236,8 +236,8 @@ def process_video_file(
|
||||
return frames, metadata, audio_tuple
|
||||
|
||||
|
||||
async def run_inference_async_omni_llm(
|
||||
omni_llm: AsyncOmniLLM,
|
||||
async def run_inference_async_omni(
|
||||
omni: AsyncOmni,
|
||||
sampling_params: list[SamplingParams],
|
||||
prompt_args_template: SimpleNamespace,
|
||||
user_prompt: str,
|
||||
@@ -246,7 +246,7 @@ async def run_inference_async_omni_llm(
|
||||
video_file: Optional[str] = None,
|
||||
use_audio_in_video: bool = False,
|
||||
):
|
||||
"""Run inference using AsyncOmniLLM directly with multimodal support."""
|
||||
"""Run inference using AsyncOmni directly with multimodal support."""
|
||||
if not user_prompt.strip() and not audio_file and not image_file and not video_file:
|
||||
return "Please provide at least a text prompt or multimodal input.", None
|
||||
|
||||
@@ -328,7 +328,7 @@ async def run_inference_async_omni_llm(
|
||||
text_outputs: list[str] = []
|
||||
audio_output = None
|
||||
|
||||
async for stage_outputs in omni_llm.generate(
|
||||
async for stage_outputs in omni.generate(
|
||||
prompt=omni_prompt,
|
||||
request_id=request_id,
|
||||
sampling_params_list=sampling_params,
|
||||
@@ -362,12 +362,12 @@ async def run_inference_async_omni_llm(
|
||||
|
||||
|
||||
def build_interface(
|
||||
omni_llm: AsyncOmniLLM,
|
||||
omni: AsyncOmni,
|
||||
sampling_params: list[SamplingParams],
|
||||
prompt_args_template: SimpleNamespace,
|
||||
model: str,
|
||||
):
|
||||
"""Build Gradio interface for AsyncOmniLLM mode."""
|
||||
"""Build Gradio interface for AsyncOmni mode."""
|
||||
|
||||
async def run_inference(
|
||||
user_prompt: str,
|
||||
@@ -376,8 +376,8 @@ def build_interface(
|
||||
video_file: Optional[str],
|
||||
use_audio_in_video: bool,
|
||||
):
|
||||
return await run_inference_async_omni_llm(
|
||||
omni_llm,
|
||||
return await run_inference_async_omni(
|
||||
omni,
|
||||
sampling_params,
|
||||
prompt_args_template,
|
||||
user_prompt,
|
||||
@@ -475,7 +475,7 @@ def build_interface(
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
omni_llm = None
|
||||
omni = None
|
||||
|
||||
model_name = "/".join(args.model.split("/")[-2:])
|
||||
assert model_name in SUPPORTED_MODELS, (
|
||||
@@ -485,9 +485,9 @@ def main():
|
||||
# Register signal handlers for graceful shutdown
|
||||
def signal_handler(sig, frame):
|
||||
print("\nReceived interrupt signal, shutting down...")
|
||||
if omni_llm is not None:
|
||||
if omni is not None:
|
||||
try:
|
||||
omni_llm.shutdown()
|
||||
omni.shutdown()
|
||||
except Exception as e:
|
||||
print(f"Error during shutdown: {e}")
|
||||
sys.exit(0)
|
||||
@@ -495,21 +495,21 @@ def main():
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
|
||||
print(f"Initializing AsyncOmniLLM with model: {args.model}")
|
||||
print(f"Initializing AsyncOmni with model: {args.model}")
|
||||
if args.stage_configs_path:
|
||||
print(f"Using custom stage configs: {args.stage_configs_path}")
|
||||
|
||||
sampling_params = build_sampling_params(SEED, model_name)
|
||||
omni_llm = AsyncOmniLLM(
|
||||
omni = AsyncOmni(
|
||||
model=args.model,
|
||||
stage_configs_path=args.stage_configs_path,
|
||||
init_timeout=ASYNC_INIT_TIMEOUT,
|
||||
)
|
||||
print("✓ AsyncOmniLLM initialized successfully")
|
||||
print("✓ AsyncOmni initialized successfully")
|
||||
prompt_args_template = create_prompt_args(args)
|
||||
|
||||
demo = build_interface(
|
||||
omni_llm,
|
||||
omni,
|
||||
sampling_params,
|
||||
prompt_args_template,
|
||||
args.model,
|
||||
@@ -524,9 +524,9 @@ def main():
|
||||
print("\nShutting down...")
|
||||
finally:
|
||||
# Cleanup
|
||||
if omni_llm is not None:
|
||||
if omni is not None:
|
||||
try:
|
||||
omni_llm.shutdown()
|
||||
omni.shutdown()
|
||||
except Exception as e:
|
||||
print(f"Error during cleanup: {e}")
|
||||
|
||||
|
||||
@@ -45,7 +45,7 @@ This Web UI demo allows users to interact with the model through a web browser.
|
||||
|
||||
### Running Gradio Demo
|
||||
|
||||
Once vllm and vllm-omni are installed, you can launch the web service built on AsyncOmniLLM by
|
||||
Once vllm and vllm-omni are installed, you can launch the web service built on AsyncOmni by
|
||||
|
||||
```bash
|
||||
python gradio_demo.py --model Qwen/Qwen3-Omni-30B-A3B-Instruct --port 7861
|
||||
|
||||
@@ -15,7 +15,7 @@ from PIL import Image
|
||||
from vllm.assets.video import video_get_metadata, video_to_ndarrays
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
from vllm_omni.entrypoints.async_omni_llm import AsyncOmniLLM
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
|
||||
# Import utils from offline inference example
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../offline_inference/qwen3_omni"))
|
||||
@@ -239,8 +239,8 @@ def process_video_file(
|
||||
return frames, metadata, audio_tuple
|
||||
|
||||
|
||||
async def run_inference_async_omni_llm(
|
||||
omni_llm: AsyncOmniLLM,
|
||||
async def run_inference_async_omni(
|
||||
omni: AsyncOmni,
|
||||
sampling_params: list[SamplingParams],
|
||||
prompt_args_template: SimpleNamespace,
|
||||
user_prompt: str,
|
||||
@@ -249,7 +249,7 @@ async def run_inference_async_omni_llm(
|
||||
video_file: Optional[str] = None,
|
||||
use_audio_in_video: bool = False,
|
||||
):
|
||||
"""Run inference using AsyncOmniLLM directly with multimodal support."""
|
||||
"""Run inference using AsyncOmni directly with multimodal support."""
|
||||
if not user_prompt.strip() and not audio_file and not image_file and not video_file:
|
||||
return "Please provide at least a text prompt or multimodal input.", None
|
||||
|
||||
@@ -331,7 +331,7 @@ async def run_inference_async_omni_llm(
|
||||
text_outputs: list[str] = []
|
||||
audio_output = None
|
||||
|
||||
async for stage_outputs in omni_llm.generate(
|
||||
async for stage_outputs in omni.generate(
|
||||
prompt=omni_prompt,
|
||||
request_id=request_id,
|
||||
sampling_params_list=sampling_params,
|
||||
@@ -366,12 +366,12 @@ async def run_inference_async_omni_llm(
|
||||
|
||||
|
||||
def build_interface(
|
||||
omni_llm: AsyncOmniLLM,
|
||||
omni: AsyncOmni,
|
||||
sampling_params: list[SamplingParams],
|
||||
prompt_args_template: SimpleNamespace,
|
||||
model: str,
|
||||
):
|
||||
"""Build Gradio interface for AsyncOmniLLM mode."""
|
||||
"""Build Gradio interface for AsyncOmni mode."""
|
||||
|
||||
async def run_inference(
|
||||
user_prompt: str,
|
||||
@@ -380,8 +380,8 @@ def build_interface(
|
||||
video_file: Optional[str],
|
||||
use_audio_in_video: bool,
|
||||
):
|
||||
return await run_inference_async_omni_llm(
|
||||
omni_llm,
|
||||
return await run_inference_async_omni(
|
||||
omni,
|
||||
sampling_params,
|
||||
prompt_args_template,
|
||||
user_prompt,
|
||||
@@ -479,7 +479,7 @@ def build_interface(
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
omni_llm = None
|
||||
omni = None
|
||||
|
||||
model_name = "/".join(args.model.split("/")[-2:])
|
||||
assert model_name in SUPPORTED_MODELS, (
|
||||
@@ -489,9 +489,9 @@ def main():
|
||||
# Register signal handlers for graceful shutdown
|
||||
def signal_handler(sig, frame):
|
||||
print("\nReceived interrupt signal, shutting down...")
|
||||
if omni_llm is not None:
|
||||
if omni is not None:
|
||||
try:
|
||||
omni_llm.shutdown()
|
||||
omni.shutdown()
|
||||
except Exception as e:
|
||||
print(f"Error during shutdown: {e}")
|
||||
sys.exit(0)
|
||||
@@ -499,21 +499,21 @@ def main():
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
|
||||
print(f"Initializing AsyncOmniLLM with model: {args.model}")
|
||||
print(f"Initializing AsyncOmni with model: {args.model}")
|
||||
if args.stage_configs_path:
|
||||
print(f"Using custom stage configs: {args.stage_configs_path}")
|
||||
|
||||
sampling_params = build_sampling_params(SEED, model_name)
|
||||
omni_llm = AsyncOmniLLM(
|
||||
omni = AsyncOmni(
|
||||
model=args.model,
|
||||
stage_configs_path=args.stage_configs_path,
|
||||
init_timeout=ASYNC_INIT_TIMEOUT,
|
||||
)
|
||||
print("✓ AsyncOmniLLM initialized successfully")
|
||||
print("✓ AsyncOmni initialized successfully")
|
||||
prompt_args_template = create_prompt_args(args)
|
||||
|
||||
demo = build_interface(
|
||||
omni_llm,
|
||||
omni,
|
||||
sampling_params,
|
||||
prompt_args_template,
|
||||
args.model,
|
||||
@@ -528,9 +528,9 @@ def main():
|
||||
print("\nShutting down...")
|
||||
finally:
|
||||
# Cleanup
|
||||
if omni_llm is not None:
|
||||
if omni is not None:
|
||||
try:
|
||||
omni_llm.shutdown()
|
||||
omni.shutdown()
|
||||
except Exception as e:
|
||||
print(f"Error during cleanup: {e}")
|
||||
|
||||
|
||||
Reference in new issue
Block a user