[Hardware] Support platforms and plugin system (#774)
Signed-off-by: gcanlin <canlinguosdu@gmail.com>
This commit is contained in:
1 parent
b11d436e19
commit
c9392dff01
103 files changed
+1770
-1218
No files matched your search
@@ -96,8 +96,12 @@ Key arguments:
|
||||
- `--num_inference_steps`: diffusion sampling steps (more steps = higher quality, slower).
|
||||
- `--height/--width`: output resolution (defaults 1024x1024).
|
||||
- `--output`: path to save the generated PNG.
|
||||
- `--vae_use_slicing`: enable VAE slicing for memory optimization.
|
||||
- `--vae_use_tiling`: enable VAE tiling for memory optimization.
|
||||
- `--enable-cpu-offload`: enable CPU offloading for diffusion models.
|
||||
|
||||
> ℹ️ If you encounter OOM errors, try using `--vae_use_slicing` and `--vae_use_tiling` to reduce memory usage.
|
||||
|
||||
> ℹ️ Qwen-Image currently publishes best-effort presets at `1328x1328`, `1664x928`, `928x1664`, `1472x1140`, `1140x1472`, `1584x1056`, and `1056x1584`. Adjust `--height/--width` accordingly for the most reliable outcomes.
|
||||
|
||||
## Web UI Demo
|
||||
|
||||
@@ -7,7 +7,7 @@ import torch
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.utils.platform_utils import detect_device_type, is_npu
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
ASPECT_RATIOS: dict[str, tuple[int, int]] = {
|
||||
"1:1": (1328, 1328),
|
||||
@@ -62,8 +62,8 @@ def parse_args() -> argparse.Namespace:
|
||||
@lru_cache(maxsize=1)
|
||||
def get_omni(model_name: str) -> Omni:
|
||||
# Enable VAE memory optimizations on NPU
|
||||
vae_use_slicing = is_npu()
|
||||
vae_use_tiling = is_npu()
|
||||
vae_use_slicing = current_omni_platform.is_npu()
|
||||
vae_use_tiling = current_omni_platform.is_npu()
|
||||
return Omni(
|
||||
model=model_name,
|
||||
vae_use_slicing=vae_use_slicing,
|
||||
@@ -72,7 +72,6 @@ def get_omni(model_name: str) -> Omni:
|
||||
|
||||
|
||||
def build_demo(args: argparse.Namespace) -> gr.Blocks:
|
||||
device = detect_device_type()
|
||||
omni = get_omni(args.model)
|
||||
|
||||
def run_inference(
|
||||
@@ -99,7 +98,7 @@ def build_demo(args: argparse.Namespace) -> gr.Blocks:
|
||||
raise gr.Error("Inference steps must be a positive integer.")
|
||||
if num_images not in {1, 2, 3, 4}:
|
||||
raise gr.Error("Number of images must be 1, 2, 3, or 4.")
|
||||
generator = torch.Generator(device=device).manual_seed(seed)
|
||||
generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(seed)
|
||||
outputs = omni.generate(
|
||||
prompt.strip(),
|
||||
OmniDiffusionSamplingParams(
|
||||
|
||||
@@ -12,7 +12,7 @@ from vllm_omni.diffusion.data import DiffusionParallelConfig, logger
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.utils.platform_utils import detect_device_type, is_npu
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
@@ -113,17 +113,22 @@ def parse_args() -> argparse.Namespace:
|
||||
default=1,
|
||||
help="Number of GPUs used for tensor parallelism (TP) inside the DiT.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--vae_use_slicing",
|
||||
action="store_true",
|
||||
help="Enable VAE slicing for memory optimization.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--vae_use_tiling",
|
||||
action="store_true",
|
||||
help="Enable VAE tiling for memory optimization.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
device = detect_device_type()
|
||||
generator = torch.Generator(device=device).manual_seed(args.seed)
|
||||
|
||||
# Enable VAE memory optimizations on NPU
|
||||
vae_use_slicing = is_npu()
|
||||
vae_use_tiling = is_npu()
|
||||
generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
|
||||
|
||||
# Configure cache based on backend type
|
||||
cache_config = None
|
||||
@@ -167,8 +172,8 @@ def main():
|
||||
|
||||
omni = Omni(
|
||||
model=args.model,
|
||||
vae_use_slicing=vae_use_slicing,
|
||||
vae_use_tiling=vae_use_tiling,
|
||||
vae_use_slicing=args.vae_use_slicing,
|
||||
vae_use_tiling=args.vae_use_tiling,
|
||||
cache_backend=args.cache_backend,
|
||||
cache_config=cache_config,
|
||||
enable_cache_dit_summary=args.enable_cache_dit_summary,
|
||||
|
||||
Reference in new issue
Block a user