[Feat] Rename AsyncOmniLLM -> AsyncOmni (#103)
Signed-off-by: WANG Cong <115451386+congw729@users.noreply.github.com>
This commit is contained in:
1 parent
0c245dd1f9
commit
3427fb4774
16 files changed
+98
-95
No files matched your search
+2
-6
@@ -4,8 +4,8 @@
|
||||
|
||||
Main entry points for vLLM-Omni inference and serving.
|
||||
|
||||
- [vllm_omni.entrypoints.async_omni_llm.AsyncOmniLLM][]
|
||||
- [vllm_omni.entrypoints.async_omni_llm.AsyncOmniStageLLM][]
|
||||
- [vllm_omni.entrypoints.async_omni.AsyncOmni][]
|
||||
- [vllm_omni.entrypoints.async_omni.AsyncOmniStageLLM][]
|
||||
- [vllm_omni.entrypoints.chat_utils.OmniAsyncMultiModalContentParser][]
|
||||
- [vllm_omni.entrypoints.chat_utils.OmniAsyncMultiModalItemTracker][]
|
||||
- [vllm_omni.entrypoints.chat_utils.parse_chat_messages_futures][]
|
||||
@@ -102,10 +102,6 @@ Configuration classes.
|
||||
|
||||
Worker classes and model runners for distributed inference.
|
||||
|
||||
- [vllm_omni.diffusion.worker.gpu_worker.GPUWorker][]
|
||||
- [vllm_omni.diffusion.worker.gpu_worker.WorkerProc][]
|
||||
- [vllm_omni.diffusion.worker.npu.npu_worker.NPUWorker][]
|
||||
- [vllm_omni.diffusion.worker.npu.npu_worker.NPUWorkerProc][]
|
||||
- [vllm_omni.worker.gpu_ar_model_runner.GPUARModelRunner][]
|
||||
- [vllm_omni.worker.gpu_ar_worker.GPUARWorker][]
|
||||
- [vllm_omni.worker.gpu_diffusion_model_runner.GPUDiffusionModelRunner][]
|
||||
|
||||
@@ -44,7 +44,7 @@ stage_args:
|
||||
trust_remote_code: true # Needed by huggingface config parsing
|
||||
engine_output_type: latent # It claims that the stage will input latent hiddenstates besides token ids
|
||||
enable_prefix_caching: false # For request with hiddenstates output, the prefix caching is not supported now
|
||||
is_comprehension: true # If the stage is a text or multimodal comprehension module. If it is, the AsyncOmniLLM will use its tokenizer as default
|
||||
is_comprehension: true # If the stage is a text or multimodal comprehension module. If it is, the AsyncOmni will use its tokenizer as default
|
||||
final_output: true # If the stage has output as part of final outputs. If it is false, which means that the stage only works as a intermediate role.
|
||||
final_output_type: text # What is the final output type. It can be text and audio now.
|
||||
default_sampling_params: # sampling parameters for the stage. Their meaning aligns with vLLM.
|
||||
@@ -206,7 +206,7 @@ Default: `false`
|
||||
|
||||
### `is_comprehension`
|
||||
|
||||
Whether this stage is a text or multimodal comprehension module. When set to `true`, the stage acts as a comprehension module that processes input text or multimodal content. If this is the first comprehension stage, `AsyncOmniLLM` will use its tokenizer as the default tokenizer for the entire pipeline.
|
||||
Whether this stage is a text or multimodal comprehension module. When set to `true`, the stage acts as a comprehension module that processes input text or multimodal content. If this is the first comprehension stage, `AsyncOmni` will use its tokenizer as the default tokenizer for the entire pipeline.
|
||||
|
||||
Default: `true`
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ vLLM-Omni is a multi-modality extension for vLLM that supports non-autoregressiv
|
||||
|
||||
## Key Data Flow
|
||||
|
||||
API Server --> OmniLLM/AsyncOmniLLM (New, including multi engines) --> LLMEngine/AsyncLLM --> Engine Core
|
||||
API Server --> OmniLLM/AsyncOmni (New, including multi engines) --> LLMEngine/AsyncLLM --> Engine Core
|
||||
--> Scheduler (New one for DiT) --> Executor (New one for diffusers) --> Worker (New one for DiT)
|
||||
--> ModelRunner (New one for AR hiddenstate, New one for DiT) --> RequestState --> OutputProcessoer (New one for final multimodal output)
|
||||
|
||||
@@ -40,7 +40,7 @@ graph TD
|
||||
B --> C{Detect --omni flag}
|
||||
C -->|Yes| D[Parse OmniConfig]
|
||||
C -->|No| E[Forward to vLLM CLI]
|
||||
D --> F[Initialize AsyncOmniLLM]
|
||||
D --> F[Initialize AsyncOmni]
|
||||
F --> G[Start omni Server]
|
||||
G --> H[Multi-stage Processing]
|
||||
E --> I[Standard vLLM Pipeline]
|
||||
@@ -209,7 +209,7 @@ Similar to OmniLLM in offline inference, add some asynchronous processing, refer
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
|
||||
class AsyncOmniLLM(AsyncLLM):
|
||||
class AsyncOmni(AsyncLLM):
|
||||
"""Extended AsyncLLM supporting multiple engines and stage-based processing"""
|
||||
|
||||
def __init__(self, stage_configs: List[StageConfig]):
|
||||
|
||||
@@ -188,6 +188,11 @@ def scan_package(package_name: str = "vllm_omni") -> dict[str, list[str]]:
|
||||
# Add classes (filter out internal ones)
|
||||
for class_name in classes:
|
||||
class_short_name = class_name.split(".")[-1]
|
||||
|
||||
# TODO: Implement worker API reference
|
||||
if "diffusion.worker.gpu_worker" in class_name:
|
||||
continue
|
||||
|
||||
# Skip if it matches internal patterns (unless it's a main model class)
|
||||
if any(pattern in class_short_name for pattern in internal_patterns):
|
||||
# But include main model classes
|
||||
|
||||
@@ -48,7 +48,7 @@ This Web UI demo allows users to interact with the model through a web browser.
|
||||
|
||||
### Running Gradio Demo
|
||||
|
||||
Once vllm and vllm-omni are installed, you can launch the web service built on AsyncOmniLLM by
|
||||
Once vllm and vllm-omni are installed, you can launch the web service built on AsyncOmni by
|
||||
|
||||
```bash
|
||||
python gradio_demo.py --model Qwen/Qwen2.5-Omni-7B --port 7861
|
||||
|
||||
@@ -48,7 +48,7 @@ This Web UI demo allows users to interact with the model through a web browser.
|
||||
|
||||
### Running Gradio Demo
|
||||
|
||||
Once vllm and vllm-omni are installed, you can launch the web service built on AsyncOmniLLM by
|
||||
Once vllm and vllm-omni are installed, you can launch the web service built on AsyncOmni by
|
||||
|
||||
```bash
|
||||
python gradio_demo.py --model Qwen/Qwen3-Omni-30B-A3B-Instruct --port 7861
|
||||
@@ -72,7 +72,7 @@ python gradio_demo.py \
|
||||
```
|
||||
|
||||
- `--model`: Model name
|
||||
- `--use-api-server`: If set, connect to an existing vLLM HTTP API server instead of running AsyncOmniLLM locally.
|
||||
- `--use-api-server`: If set, connect to an existing vLLM HTTP API server instead of running AsyncOmni locally.
|
||||
- `--api-base`: Base URL for vllm serve (only used when `use-api-server` is set, default: http://localhost:8091/v1)
|
||||
- `--ip`: Host/IP for Gradio server (default: 127.0.0.1)
|
||||
- `--port`: Port for Gradio server (default: 7861)
|
||||
|
||||
Reference in new issue
Block a user