update ar_dit_test
This commit is contained in:
1 parent
ff9c7d4536
commit
81f04d3cf9
32 files changed
+1192
-1908
No files matched your search
+11
-44
@@ -1,49 +1,16 @@
|
||||
# vLLM-omni Examples
|
||||
|
||||
This directory contains examples demonstrating how to use vLLM-omni for multi-modality model inference and serving.
|
||||
A curated set of runnable samples demonstrating vLLM-omni usage.
|
||||
|
||||
## Directory Structure
|
||||
## Directory Overview
|
||||
|
||||
- `basic/` - Simple usage examples and API client
|
||||
- `advanced/` - Advanced features and configurations
|
||||
- `multimodal/` - Multi-modality specific examples
|
||||
- **basic/** – Quick-start scripts for text-only flows.
|
||||
- `text_generation.py` – Minimal CPU-friendly text generation with vLLM.
|
||||
- `api_client.py` – Calls the FastAPI server using the JSON API.
|
||||
- `docker_run.py` – Template for launching the server in Docker.
|
||||
- **omni/** – Multi-stage AR → DiT pipelines powered by diffusers.
|
||||
- `ar_dit_diffusers.py` – Runs Qwen3 (AR) feeding Stable Diffusion 2.1 (DiT).
|
||||
- `configs/ar_dit_local.yaml` – Sample plugin config referencing local weights.
|
||||
- **advanced/** – Placeholder for upcoming complex flows (Ray, benchmarking).
|
||||
|
||||
## Quick Start
|
||||
|
||||
1. **Basic Usage:**
|
||||
```bash
|
||||
cd examples/basic
|
||||
python simple_usage.py
|
||||
```
|
||||
|
||||
2. **API Server:**
|
||||
```bash
|
||||
# Start server
|
||||
vllm serve Qwen/Qwen3-0.6B --omni --port 8000
|
||||
|
||||
# Test with client
|
||||
python api_client.py
|
||||
```
|
||||
|
||||
## Examples Overview
|
||||
|
||||
### Basic Examples (`basic/`)
|
||||
- `simple_usage.py` - Direct library usage (sync/async)
|
||||
- `api_client.py` - HTTP API client examples
|
||||
- `README.md` - Basic examples documentation
|
||||
|
||||
### Advanced Examples (`advanced/`)
|
||||
- Multi-stage processing
|
||||
- Custom configurations
|
||||
- Performance optimization
|
||||
|
||||
### Multi-modality Examples (`multimodal/`)
|
||||
- Text-to-image generation
|
||||
- Image-to-text processing
|
||||
- Multi-modal pipelines
|
||||
|
||||
## Requirements
|
||||
|
||||
- Python 3.8+
|
||||
- vLLM-omni installed
|
||||
- For API examples: HTTP server running
|
||||
See the subdirectory READMEs for usage instructions.
|
||||
@@ -1,76 +1,10 @@
|
||||
# Basic vLLM-omni Examples
|
||||
# Basic Examples
|
||||
|
||||
This directory contains simple examples demonstrating how to use vLLM-omni for text generation.
|
||||
Quick-start scripts for experimenting with vLLM-omni.
|
||||
|
||||
## Examples
|
||||
## Scripts
|
||||
|
||||
### 1. `simple_usage.py` - Direct Library Usage
|
||||
|
||||
Demonstrates how to use vLLM-omni directly in your Python code:
|
||||
|
||||
```bash
|
||||
# Run the example
|
||||
python simple_usage.py
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- Synchronous text generation
|
||||
- Asynchronous text generation
|
||||
- Multi-prompt processing
|
||||
- Shows both text output and token information
|
||||
|
||||
### 2. `api_client.py` - API Server Usage
|
||||
|
||||
Demonstrates how to interact with the vLLM-omni API server:
|
||||
|
||||
```bash
|
||||
# First, start the server
|
||||
vllm serve Qwen/Qwen3-0.6B --omni --port 8000
|
||||
|
||||
# Then run the client (in another terminal)
|
||||
python api_client.py
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- Health check
|
||||
- Text generation via HTTP API
|
||||
- Multiple prompt types
|
||||
- Full response structure inspection
|
||||
|
||||
## Quick Start
|
||||
|
||||
1. **Install vLLM-omni:**
|
||||
```bash
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
2. **Run direct usage example:**
|
||||
```bash
|
||||
cd examples/basic
|
||||
python simple_usage.py
|
||||
```
|
||||
|
||||
3. **Run API server example:**
|
||||
```bash
|
||||
# Terminal 1: Start server
|
||||
vllm serve Qwen/Qwen3-0.6B --omni --port 8000
|
||||
|
||||
# Terminal 2: Run client
|
||||
cd examples/basic
|
||||
python api_client.py
|
||||
```
|
||||
|
||||
## Expected Output
|
||||
|
||||
The examples will show:
|
||||
- ✅ Generated text responses
|
||||
- ✅ Token counts and completion status
|
||||
- ✅ Stage processing information
|
||||
- ✅ Error handling
|
||||
|
||||
## Requirements
|
||||
|
||||
- Python 3.8+
|
||||
- vLLM-omni installed
|
||||
- For API examples: HTTP server running
|
||||
- `text_generation.py` – Minimal standalone usage showing how to create an LLM and generate text locally.
|
||||
- `api_client.py` – Demonstrates calling the FastAPI `/generate` endpoint from Python.
|
||||
|
||||
Run any script with `python examples/basic/<script>.py`. Ensure the `vllm_omni` conda environment is active and dependencies are installed.
|
||||
@@ -1,178 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Docker example for vLLM-omni.
|
||||
|
||||
This example shows how to test the vLLM-omni API server running in Docker.
|
||||
"""
|
||||
|
||||
import requests
|
||||
import json
|
||||
import time
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def check_docker_running():
|
||||
"""Check if Docker is running."""
|
||||
try:
|
||||
result = subprocess.run(['docker', 'info'],
|
||||
capture_output=True, text=True, timeout=5)
|
||||
return result.returncode == 0
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||
return False
|
||||
|
||||
|
||||
def start_docker_server():
|
||||
"""Start the vLLM-omni server in Docker."""
|
||||
print("🐳 Starting vLLM-omni server in Docker...")
|
||||
|
||||
# Check if container is already running
|
||||
result = subprocess.run(['docker', 'ps', '--filter', 'name=vllm-omni-cpu', '--format', '{{.Names}}'],
|
||||
capture_output=True, text=True)
|
||||
|
||||
if 'vllm-omni-cpu' in result.stdout:
|
||||
print("✅ Server already running")
|
||||
return True
|
||||
|
||||
# Start the server
|
||||
cmd = [
|
||||
'docker', 'run', '-d', '--name', 'vllm-omni-cpu',
|
||||
'-p', '8000:8000',
|
||||
'vllm-omni-cpu:latest',
|
||||
'vllm', 'serve', 'Qwen/Qwen3-0.6B', '--omni', '--port', '8000'
|
||||
]
|
||||
|
||||
try:
|
||||
subprocess.run(cmd, check=True)
|
||||
print("✅ Server started successfully")
|
||||
|
||||
# Wait for server to be ready
|
||||
print("⏳ Waiting for server to be ready...")
|
||||
for i in range(30): # Wait up to 30 seconds
|
||||
if test_server_health():
|
||||
print("✅ Server is ready!")
|
||||
return True
|
||||
time.sleep(1)
|
||||
|
||||
print("❌ Server failed to start within 30 seconds")
|
||||
return False
|
||||
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"❌ Failed to start server: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def test_server_health():
|
||||
"""Test if the server is healthy."""
|
||||
try:
|
||||
response = requests.get("http://localhost:8000/health", timeout=2)
|
||||
return response.status_code == 200
|
||||
except requests.exceptions.RequestException:
|
||||
return False
|
||||
|
||||
|
||||
def test_generation():
|
||||
"""Test text generation via API."""
|
||||
print("\n🧪 Testing text generation...")
|
||||
|
||||
test_cases = [
|
||||
{
|
||||
"prompt": "Hello, how are you?",
|
||||
"max_tokens": 50,
|
||||
"temperature": 0.7
|
||||
},
|
||||
{
|
||||
"prompt": "Tell me a joke",
|
||||
"max_tokens": 100,
|
||||
"temperature": 0.9
|
||||
},
|
||||
{
|
||||
"prompt": "What is artificial intelligence?",
|
||||
"max_tokens": 150,
|
||||
"temperature": 0.5
|
||||
}
|
||||
]
|
||||
|
||||
for i, test_case in enumerate(test_cases, 1):
|
||||
print(f"\nTest {i}: {test_case['prompt']}")
|
||||
|
||||
try:
|
||||
response = requests.post(
|
||||
"http://localhost:8000/generate",
|
||||
json={
|
||||
"prompts": [test_case["prompt"]],
|
||||
"max_tokens": test_case["max_tokens"],
|
||||
"temperature": test_case["temperature"]
|
||||
},
|
||||
timeout=30
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
text = result["outputs"][0]["text"]
|
||||
print(f"✅ Response: {text}")
|
||||
else:
|
||||
print(f"❌ Error: {response.status_code} - {response.text}")
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f"❌ Request failed: {e}")
|
||||
|
||||
|
||||
def cleanup():
|
||||
"""Clean up Docker container."""
|
||||
print("\n🧹 Cleaning up...")
|
||||
try:
|
||||
subprocess.run(['docker', 'stop', 'vllm-omni-cpu'],
|
||||
capture_output=True, text=True)
|
||||
subprocess.run(['docker', 'rm', 'vllm-omni-cpu'],
|
||||
capture_output=True, text=True)
|
||||
print("✅ Cleanup completed")
|
||||
except Exception as e:
|
||||
print(f"⚠️ Cleanup warning: {e}")
|
||||
|
||||
|
||||
def main():
|
||||
"""Main example function."""
|
||||
print("🐳 vLLM-omni Docker Example")
|
||||
print("=" * 40)
|
||||
|
||||
# Check Docker
|
||||
if not check_docker_running():
|
||||
print("❌ Docker is not running. Please start Docker Desktop.")
|
||||
sys.exit(1)
|
||||
|
||||
print("✅ Docker is running")
|
||||
|
||||
# Check if image exists
|
||||
result = subprocess.run(['docker', 'images', 'vllm-omni-cpu:latest'],
|
||||
capture_output=True, text=True)
|
||||
if 'vllm-omni-cpu' not in result.stdout:
|
||||
print("❌ Docker image not found. Please build it first:")
|
||||
print(" docker build -f docker/Dockerfile.cpu -t vllm-omni-cpu .")
|
||||
sys.exit(1)
|
||||
|
||||
print("✅ Docker image found")
|
||||
|
||||
try:
|
||||
# Start server
|
||||
if not start_docker_server():
|
||||
sys.exit(1)
|
||||
|
||||
# Test generation
|
||||
test_generation()
|
||||
|
||||
print("\n🎉 Docker example completed successfully!")
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n⏹️ Example interrupted by user")
|
||||
finally:
|
||||
# Ask if user wants to keep server running
|
||||
keep_running = input("\n🤔 Keep the server running? (y/N): ").lower().strip()
|
||||
if keep_running != 'y':
|
||||
cleanup()
|
||||
else:
|
||||
print("✅ Server kept running. Access it at http://localhost:8000")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File renamed without changes.
@@ -0,0 +1,31 @@
|
||||
# Omni Examples
|
||||
|
||||
Examples showcasing multi-stage AR + DiT pipelines powered by vLLM-omni.
|
||||
|
||||
## AR ➜ DiT with Diffusers Backend
|
||||
|
||||
```bash
|
||||
# Optional: customise models
|
||||
export AR_MODEL=Qwen/Qwen3-0.6B
|
||||
export DIT_MODEL=./models/stable-diffusion-2-1/
|
||||
|
||||
# Run with defaults (20 steps @ 512x512)
|
||||
python examples/omni/ar_dit_diffusers.py
|
||||
|
||||
# Faster run
|
||||
python examples/omni/ar_dit_diffusers.py --steps 14 --height 384 --width 384 --guidance 4.5
|
||||
```
|
||||
|
||||
Arguments:
|
||||
- `--ar-model`: Override AR stage model (default `Qwen/Qwen3-0.6B`).
|
||||
- `--dit-model`: Override DiT stage model (default env `DIT_MODEL`, else diffusers repo).
|
||||
- `--steps`: Number of diffusion steps (default 20).
|
||||
- `--guidance`: Guidance scale (default 5.0).
|
||||
- `--height` / `--width`: Output resolution (default 512).
|
||||
- `--seed`: Optional RNG seed.
|
||||
- `--prompt`, `--temperature`, `--max-tokens`: Control AR stage generation.
|
||||
- `--output`: Destination path for the generated image file.
|
||||
|
||||
Generated images are saved next to the script as `omni_dit_output.png` by default.
|
||||
|
||||
See the script for additional tweaks (scheduler, prompt chaining, etc.).
|
||||
@@ -0,0 +1,175 @@
|
||||
"""Run the AR → DiT (diffusers) pipeline using YAML configuration defaults."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Dict, List
|
||||
|
||||
import yaml
|
||||
|
||||
from vllm_omni import (
|
||||
OmniLLM,
|
||||
DiTConfig,
|
||||
create_ar_stage_config,
|
||||
create_dit_stage_config,
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Run a two-stage AR → DiT pipeline with config-derived defaults."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--config",
|
||||
type=Path,
|
||||
default=Path(__file__).resolve().parent / "configs" / "ar_dit_local.yaml",
|
||||
help="Path to a YAML configuration describing the pipeline stages.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prompt",
|
||||
default="A scenic watercolor painting of a lighthouse at sunset",
|
||||
help="Prompt passed to the AR stage.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--negative-prompt",
|
||||
default=None,
|
||||
help="Optional negative prompt forwarded to the diffusion stage.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--seed",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Optional seed for deterministic diffusion sampling.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=Path,
|
||||
default=Path("./omni_dit_output.png"),
|
||||
help="Destination path for the generated image.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _normalize_modalities(values: List[str] | None, default: List[str]) -> List[str]:
|
||||
if not values:
|
||||
return default
|
||||
return list(values)
|
||||
|
||||
|
||||
def _load_stage_configs_from_yaml(config_path: Path):
|
||||
with config_path.open("r", encoding="utf-8") as f:
|
||||
config = yaml.safe_load(f) or {}
|
||||
|
||||
stages = config.get("stages", [])
|
||||
if not stages:
|
||||
raise ValueError(f"No stages defined in config: {config_path}")
|
||||
|
||||
stage_configs = []
|
||||
for stage in stages:
|
||||
stage_id = stage["stage_id"]
|
||||
engine_type = stage["engine_type"]
|
||||
default_stage_args = stage.get("default_stage_args")
|
||||
|
||||
if engine_type == "AR":
|
||||
stage_configs.append(
|
||||
create_ar_stage_config(
|
||||
stage_id=stage_id,
|
||||
model_path=stage["model_path"],
|
||||
input_modalities=_normalize_modalities(
|
||||
stage.get("input_modalities"), ["text"]
|
||||
),
|
||||
output_modalities=_normalize_modalities(
|
||||
stage.get("output_modalities"), ["text"]
|
||||
),
|
||||
default_stage_args=default_stage_args,
|
||||
)
|
||||
)
|
||||
elif engine_type == "DiT":
|
||||
dit_cfg_dict: Dict = dict(stage.get("dit_config", {}))
|
||||
if "num_inference_steps" not in dit_cfg_dict:
|
||||
raise ValueError(
|
||||
"DiT stage requires 'num_inference_steps' in dit_config"
|
||||
)
|
||||
dit_cfg = DiTConfig(**dit_cfg_dict)
|
||||
|
||||
stage_configs.append(
|
||||
create_dit_stage_config(
|
||||
stage_id=stage_id,
|
||||
model_path=stage["model_path"],
|
||||
input_modalities=_normalize_modalities(
|
||||
stage.get("input_modalities"), ["text"]
|
||||
),
|
||||
output_modalities=_normalize_modalities(
|
||||
stage.get("output_modalities"), ["image"]
|
||||
),
|
||||
dit_config=dit_cfg,
|
||||
default_stage_args=default_stage_args,
|
||||
)
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported engine_type '{engine_type}' in config")
|
||||
|
||||
return stage_configs
|
||||
|
||||
|
||||
def _apply_env_overrides(stage_configs):
|
||||
ar_override = os.environ.get("AR_MODEL")
|
||||
dit_override = os.environ.get("DIT_MODEL")
|
||||
|
||||
for stage_config in stage_configs:
|
||||
if ar_override and stage_config.engine_type == "AR":
|
||||
stage_config.model_path = ar_override
|
||||
if dit_override and stage_config.engine_type == "DiT":
|
||||
stage_config.model_path = dit_override
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
|
||||
stage_configs = _load_stage_configs_from_yaml(args.config)
|
||||
_apply_env_overrides(stage_configs)
|
||||
|
||||
omni = OmniLLM(stage_configs)
|
||||
|
||||
stage_overrides: Dict[int, Dict[str, object]] = {}
|
||||
if args.seed is not None or args.negative_prompt:
|
||||
for stage_config in stage_configs:
|
||||
if stage_config.engine_type != "DiT":
|
||||
continue
|
||||
override = stage_overrides.setdefault(stage_config.stage_id, {})
|
||||
if args.seed is not None:
|
||||
override["seed"] = args.seed
|
||||
if args.negative_prompt:
|
||||
override["negative_prompt"] = args.negative_prompt
|
||||
|
||||
outputs = omni.generate(
|
||||
prompt=args.prompt,
|
||||
stage_overrides=stage_overrides if stage_overrides else None,
|
||||
)
|
||||
|
||||
image = None
|
||||
for request_output in outputs:
|
||||
for completion in getattr(request_output, "outputs", []) or []:
|
||||
candidate = getattr(completion, "image", None)
|
||||
if candidate is not None:
|
||||
image = candidate
|
||||
break
|
||||
if image is not None:
|
||||
break
|
||||
|
||||
if image is None:
|
||||
print("No image found in outputs.")
|
||||
return
|
||||
|
||||
out_path = args.output.resolve()
|
||||
try:
|
||||
image.save(out_path)
|
||||
print(f"Saved image to {out_path}")
|
||||
except Exception:
|
||||
print("Generated output is not a PIL.Image; skipping save.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,28 @@
|
||||
# Example config for running OmniLLM with local AR + DiT models.
|
||||
stages:
|
||||
- stage_id: 0
|
||||
engine_type: AR
|
||||
model_path: Qwen/Qwen3-0.6B
|
||||
input_modalities: [text]
|
||||
output_modalities: [text]
|
||||
default_stage_args:
|
||||
max_tokens: 16
|
||||
temperature: 0.7
|
||||
- stage_id: 1
|
||||
engine_type: DiT
|
||||
model_path: stabilityai/stable-diffusion-2-1
|
||||
input_modalities: [text]
|
||||
output_modalities: [image]
|
||||
default_stage_args:
|
||||
height: 512
|
||||
width: 512
|
||||
dit_config:
|
||||
num_inference_steps: 20
|
||||
guidance_scale: 5.0
|
||||
use_diffusers: true
|
||||
diffusers_pipeline: auto
|
||||
height: 512
|
||||
width: 512
|
||||
device_config:
|
||||
device: mps
|
||||
dtype: float16
|
||||
Reference in new issue
Block a user