Merge remote-tracking branch 'origin/main' into qwen-image-2.1

# Conflicts:
#	vllm_omni/diffusion/executor/multiproc_executor.py
#	vllm_omni/diffusion/offloader/sequential_backend.py
This commit is contained in:
Isotr0py committed 2026-09-21 07:32:04 +00:00
commit 9174554e40
1332 files changed
+160051 -60504

No files matched your search

+159 -44
View File
@@ -6,6 +6,7 @@ import json
import math
import os
import random
import warnings
from collections.abc import Callable, Mapping
from dataclasses import dataclass, field, fields
from enum import Enum
@@ -29,6 +30,7 @@ from vllm_omni.diffusion.diffusion_kv.config import (
parse_diffusion_kv_cache_mode,
)
from vllm_omni.diffusion.lora.manager import LoRABackend
from vllm_omni.diffusion.media import DiffusionMediaOutput
from vllm_omni.diffusion.model_metadata import get_diffusion_model_metadata
from vllm_omni.diffusion.utils.network_utils import is_port_available
from vllm_omni.errors import client_error_metadata
@@ -43,55 +45,63 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
def normalize_omni_diffusion_kwargs(raw_kwargs: Mapping[str, Any]) -> dict[str, Any]:
"""Normalize legacy diffusion kwargs before config construction."""
def _move_diffusion_alias(
normalized: dict[str, Any],
legacy_name: str,
canonical_name: str,
) -> None:
legacy_value = normalized.pop(legacy_name, None)
if legacy_value is None:
return
if normalized.get(canonical_name) is not None:
raise ValueError(f"Diffusion config fields {legacy_name!r} and {canonical_name!r} cannot both be provided.")
warnings.warn(
f"Diffusion config field {legacy_name!r} is deprecated; use {canonical_name!r}.",
FutureWarning,
stacklevel=3,
)
normalized[canonical_name] = legacy_value
def normalize_omni_diffusion_kwargs(
raw_kwargs: Mapping[str, Any],
*,
apply_defaults: bool = True,
) -> dict[str, Any]:
"""Normalize diffusion kwargs, deferring defaults until sources are merged."""
config_kwargs = dict(raw_kwargs)
dtype = config_kwargs.get("dtype")
if dtype is None:
config_kwargs["dtype"] = "auto"
if apply_defaults:
config_kwargs["dtype"] = "auto"
elif isinstance(dtype, torch.dtype):
config_kwargs["dtype"] = str(dtype).removeprefix("torch.")
elif not isinstance(dtype, str):
raise TypeError(f"Provided dtype must be a string or torch.dtype, got {type(dtype).__name__}")
# Backwards-compatibility: older callers may use a diffusion-specific
# "static_lora_scale" kwarg. Normalize it to the canonical "lora_scale".
if "static_lora_scale" in config_kwargs:
if "lora_scale" not in config_kwargs:
config_kwargs["lora_scale"] = config_kwargs["static_lora_scale"]
config_kwargs.pop("static_lora_scale", None)
diffusion_quantization = config_kwargs.pop("diffusion_quantization_config", None)
if config_kwargs.get("quantization_config") is None and diffusion_quantization is not None:
config_kwargs["quantization_config"] = diffusion_quantization
# Backwards-compatibility: map "quantization" to "quantization_config"
# so callers using the old field name still work.
if "quantization" in config_kwargs and config_kwargs.get("quantization_config", None) is None:
config_kwargs["quantization_config"] = config_kwargs.pop("quantization")
else:
config_kwargs.pop("quantization", None)
# Renamed from kv_cache_* to avoid clashing with vLLM's --kv-cache-dtype.
if config_kwargs.get("diffusion_kv_cache_dtype") is None and "kv_cache_dtype" in config_kwargs:
config_kwargs["diffusion_kv_cache_dtype"] = config_kwargs.pop("kv_cache_dtype")
else:
config_kwargs.pop("kv_cache_dtype", None)
if config_kwargs.get("diffusion_kv_cache_skip_steps") is None and "kv_cache_skip_steps" in config_kwargs:
config_kwargs["diffusion_kv_cache_skip_steps"] = config_kwargs.pop("kv_cache_skip_steps")
else:
config_kwargs.pop("kv_cache_skip_steps", None)
if config_kwargs.get("diffusion_kv_cache_skip_layers") is None and "kv_cache_skip_layers" in config_kwargs:
config_kwargs["diffusion_kv_cache_skip_layers"] = config_kwargs.pop("kv_cache_skip_layers")
else:
config_kwargs.pop("kv_cache_skip_layers", None)
for legacy, canonical in (
("static_lora_scale", "lora_scale"),
("quantization", "quantization_config"),
("diffusion_quantization_config", "quantization_config"),
("max_batch_size", "max_num_seqs"),
("kv_cache_skip_steps", "diffusion_kv_cache_skip_steps"),
("kv_cache_skip_layers", "diffusion_kv_cache_skip_layers"),
):
_move_diffusion_alias(config_kwargs, legacy, canonical)
_move_diffusion_alias(config_kwargs, "kv_cache_dtype", "diffusion_kv_cache_dtype")
# Handle "diffusion_attention_backend" shorthand: merge into
# diffusion_attention_config before field filtering.
diffusion_attn_backend = config_kwargs.pop("diffusion_attention_backend", None)
fastvideo_vsa_topk = config_kwargs.pop("fastvideo_vsa_topk", None)
if diffusion_attn_backend is not None or fastvideo_vsa_topk is not None:
if diffusion_attn_backend is not None:
warnings.warn(
"Diffusion config field 'diffusion_attention_backend' is deprecated; use 'diffusion_attention_config'.",
FutureWarning,
stacklevel=2,
)
existing = config_kwargs.get("diffusion_attention_config")
config_kwargs["diffusion_attention_config"] = parse_attention_config(
existing,
@@ -99,12 +109,23 @@ def normalize_omni_diffusion_kwargs(raw_kwargs: Mapping[str, Any]) -> dict[str,
fastvideo_vsa_topk=fastvideo_vsa_topk,
)
auxiliary_text_encoder = config_kwargs.pop("auxiliary_text_encoder", None)
if auxiliary_text_encoder is not None:
extras = dict(config_kwargs.get("extras") or {})
if extras.get("auxiliary_text_encoder") is not None:
raise ValueError(
"Diffusion engine field 'auxiliary_text_encoder' cannot be provided both at the top level and in "
"'extras'."
)
extras["auxiliary_text_encoder"] = auxiliary_text_encoder
config_kwargs["extras"] = extras
# Check environment variable as fallback for cache_backend.
# Support both old DIFFUSION_CACHE_ADAPTER and new DIFFUSION_CACHE_BACKEND.
if "cache_backend" not in config_kwargs:
if "cache_backend" not in config_kwargs and apply_defaults:
cache_backend = os.environ.get("DIFFUSION_CACHE_BACKEND") or os.environ.get("DIFFUSION_CACHE_ADAPTER")
config_kwargs["cache_backend"] = cache_backend.lower() if cache_backend else "none"
elif config_kwargs["cache_backend"] is None:
elif "cache_backend" in config_kwargs and config_kwargs["cache_backend"] is None and apply_defaults:
# Callers (e.g. example CLIs with `default=None`) pass an explicit
# None for "no cache"; canonicalize it so every consumer sees the
# declared `str` value instead of relying on per-model None handling.
@@ -161,6 +182,20 @@ def validate_dlo_host_registration_options(
return value
def validate_omni_diffusion_kwargs(
kwargs: Mapping[str, Any],
allowed_fields: set[str] | frozenset[str],
*,
stage_id: int | str | None = None,
) -> None:
"""Reject every field without an owner."""
unknown = set(kwargs) - allowed_fields
if unknown:
names = ", ".join(repr(name) for name in sorted(unknown))
suffix = "" if stage_id is None else f" for stage {stage_id}"
raise ValueError(f"Unknown diffusion config field(s){suffix}: {names}")
def parse_kv_cache_skip_selector(
selector: str | list[int] | tuple[int, ...] | set[int] | None,
) -> set[int] | None:
@@ -540,6 +575,8 @@ class DiffusionCacheConfig:
scm_steps_mask_policy, scm_steps_policy
- MagCache: mag_threshold, mag_max_skip_steps, mag_retention_ratio,
mag_ratios, mag_calibrate
- SeaCache: sea_threshold, sea_residual_order,
sea_max_consecutive_cached, sea_power_exp
- step_cache: step_cache_dit_enabled, velocity_sim_thresholds,
velocity_skip_countdowns, step_cache_dit_min_history
@@ -559,6 +596,12 @@ class DiffusionCacheConfig:
rel_l1_thresh: float | None = None
coefficients: list[float] | None = None # Uses model-specific defaults if None
# SeaCache parameters [sea_cache only]
sea_threshold: float = 0.25
sea_residual_order: int = 1
sea_max_consecutive_cached: int = 2
sea_power_exp: float = 3.0
# MagCache parameters [mag_cache only]
# Default: 0.24 threshold for accumulated magnitude error
mag_threshold: float = 0.24
@@ -758,6 +801,15 @@ def uses_diffusers_adapter(od_config: object) -> bool:
)
@dataclass
class VideoOutputTransportConfig:
enable_device_postprocess: bool = False
def __post_init__(self) -> None:
if not isinstance(self.enable_device_postprocess, bool):
raise TypeError("enable_device_postprocess must be a bool")
@dataclass
class OmniDiffusionConfig:
# Model and path configuration (for convenience)
@@ -793,6 +845,7 @@ class OmniDiffusionConfig:
# Cache backend configuration (NEW)
cache_backend: str = "none" # "tea_cache", "deep_cache", etc.
cache_config: DiffusionCacheConfig | dict[str, Any] = field(default_factory=dict)
video_output_transport: VideoOutputTransportConfig = field(default_factory=VideoOutputTransportConfig)
enable_cache_dit_summary: bool = False
# Prompt-embedding cache. When enabled, ``DiffusionModelRunner`` wraps the
@@ -909,6 +962,7 @@ class OmniDiffusionConfig:
# Parallel weight loading (for faster diffusion model startup)
enable_multithread_weight_load: bool = True
enable_broadcast_weight_load: bool = False
num_weight_load_threads: int = 4
# Enable sleep mode
@@ -964,6 +1018,7 @@ class OmniDiffusionConfig:
"transformer": True,
"vae": True,
"text_encoder": True,
"vae_encoder": True,
}
)
override_transformer_cls_name: str | None = None
@@ -1000,16 +1055,15 @@ class OmniDiffusionConfig:
# str is resolved to {"method": <str>} internally.
# Per-component: {"transformer": {"method": "fp8"}, "vae": None}
quantization_config: str | QuantizationConfig | dict[str, Any] | None = None
# Internal provenance, retained across config projection and worker transport.
quantization_config_is_auto_detected: bool = False
# Explicit runtime override for ModelOpt FP8 diffusion checkpoints. This
# does not enable FP8 by itself; it only selects CUTLASS once the checkpoint
# has already resolved to vLLM's ModelOpt FP8 linear method.
force_cutlass_fp8: bool = False
# Diffusion attention KV cache dtype (not vLLM's --kv-cache-dtype for AR models).
# None = native dtype (no quantization).
# "fp8" = dynamic FP8 (float8_e4m3fn) quantization per forward pass.
# On Hopper+FA3: native FP8 attention (memory + compute savings).
# On other backends: no benefit, backends skip quantization.
# Runtime diffusion attention method (not vLLM's --kv-cache-dtype for AR models).
# None/"auto" keeps native dtype; other values are validated by the selected backend.
diffusion_kv_cache_dtype: str | None = None
# Optional skip selectors for KV-cache quantization. Format: "0-9,20,25-30".
# Listed steps/layers skip quantization; others keep quantized execution.
@@ -1171,6 +1225,8 @@ class OmniDiffusionConfig:
if self.diffusion_kv_mode is not DiffusionKVCacheMode.PAGED_SCHEDULER:
raise ValueError("native kv_transfer_config requires diffusion_kv_mode='paged_scheduler'")
self.kv_transfer_config = parse_kv_transfer_config(self.kv_transfer_config)
if self.enable_sleep_mode:
raise ValueError("Native KV transfer does not support sleep mode: registered pages must remain mapped")
self.master_port = self._resolve_master_port()
self.request_batch_max_wait_ms = float(self.request_batch_max_wait_ms or 0.0)
@@ -1258,6 +1314,13 @@ class OmniDiffusionConfig:
# If it's neither dict nor DiffusionCacheConfig, convert to empty config
self.cache_config = DiffusionCacheConfig()
if self.video_output_transport is None:
self.video_output_transport = VideoOutputTransportConfig()
elif isinstance(self.video_output_transport, Mapping):
self.video_output_transport = VideoOutputTransportConfig(**dict(self.video_output_transport))
elif not isinstance(self.video_output_transport, VideoOutputTransportConfig):
raise TypeError("video_output_transport must be a VideoOutputTransportConfig or mapping")
# Auto-detect quantization from TransformerConfig if not explicitly set.
# This covers the case where tf_model_config is passed at construction
# time. For late (post-construction) assignment, callers should use
@@ -1318,6 +1381,10 @@ class OmniDiffusionConfig:
self.model,
)
@property
def is_single_file(self) -> bool:
return isinstance(self.model, str) and os.path.isfile(self.model)
def _propagate_quantization_from_tf_config(self, tf_config: "TransformerConfig") -> None:
if tf_config.quant_config is None:
return
@@ -1330,6 +1397,8 @@ class OmniDiffusionConfig:
or (is_checkpoint_nvfp4 and self._is_generic_nvfp4_quant_config(self.quantization_config))
)
if should_use_checkpoint_config:
if self.quantization_config is None:
self.quantization_config_is_auto_detected = True
self.quantization_config = tf_config.quant_config
logger.info(
"Auto-detected quantization '%s' from model config",
@@ -1420,12 +1489,22 @@ class OmniDiffusionConfig:
"""
from vllm.transformers_utils.config import get_hf_file_to_dict
from vllm_omni.diffusion.registry import resolve_native_single_file
from vllm_omni.diffusion.utils.hf_utils import (
get_diffusion_model_index,
resolve_native_diffusion_model_class,
)
assert self.model is not None
native_single_file_model = resolve_native_single_file(self.model_class_name)
if self.is_single_file and native_single_file_model is not None:
self.diffusion_load_format = "default"
self.model_class_name = native_single_file_model
self.diffusers_pipeline_cls = None
self.set_tf_model_config(TransformerConfig())
return
try:
config_dict = get_diffusion_model_index(
self.model,
@@ -1583,6 +1662,11 @@ class OmniDiffusionConfig:
self.model_class_name = "Pi0Pipeline"
self.set_tf_model_config(TransformerConfig())
self.update_multimodal_support()
elif cfg.get("type") == "pi05":
if self.model_class_name is None:
self.model_class_name = "Pi05Pipeline"
self.set_tf_model_config(TransformerConfig())
self.update_multimodal_support()
elif architectures and len(architectures) == 1:
architecture = architectures[0]
from vllm_omni.diffusion.registry import DiffusionModelRegistry
@@ -1597,8 +1681,9 @@ class OmniDiffusionConfig:
@classmethod
def normalize_init_kwargs(cls, raw_kwargs: Mapping[str, Any]) -> dict[str, Any]:
valid_fields = frozenset(f.name for f in fields(cls))
config_kwargs = normalize_omni_diffusion_kwargs(raw_kwargs)
valid_fields = {f.name for f in fields(cls)}
validate_omni_diffusion_kwargs(config_kwargs, valid_fields)
# Remaining ``None`` values mean "unset" at the CLI/deploy boundary.
# Drop them so non-optional dataclass defaults are not overwritten.
# Fields where ``None`` has normalization semantics (for example dtype
@@ -1610,6 +1695,18 @@ class OmniDiffusionConfig:
return cls(**cls.normalize_init_kwargs(kwargs))
DIFFUSION_REQUEST_LIFECYCLE_KEY = "_diffusion_request_lifecycle"
DIFFUSION_REQUEST_STARTED = "started"
def is_diffusion_request_started_output(output: Any) -> bool:
custom_output = getattr(output, "custom_output", None)
return (
isinstance(custom_output, dict)
and custom_output.get(DIFFUSION_REQUEST_LIFECYCLE_KEY) == DIFFUSION_REQUEST_STARTED
)
@dataclass
class DiffusionOutput:
"""
@@ -1618,8 +1715,8 @@ class DiffusionOutput:
output: torch.Tensor | tuple[Any, ...] | dict[str, Any] | None = None
# Legacy compatibility fields. New pipeline-specific payloads should be
# carried by output["payload"] instead.
# Legacy compatibility fields. Decoded video uses ``media``; other new
# payloads should be carried by output["payload"] instead.
trajectory_timesteps: torch.Tensor | dict[str, Any] | None = None
trajectory_latents: torch.Tensor | dict[str, Any] | None = None
trajectory_log_probs: torch.Tensor | dict[str, Any] | None = None
@@ -1658,7 +1755,21 @@ class DiffusionOutput:
# mode) and the receiving side must not initialise a stray CUDA context.
to_cpu: bool = False
# Internal control-plane event emitted on first scheduler admission.
request_started: bool = False
# Typed video-media contract. Declared last so the pre-existing positional
# constructor order (output, trajectory_timesteps, ...) that out-of-tree
# pipelines rely on is preserved. Mutually exclusive with ``output``.
media: DiffusionMediaOutput | None = None
def __post_init__(self) -> None:
if self.media is not None and not isinstance(self.media, DiffusionMediaOutput):
raise TypeError(f"media must be DiffusionMediaOutput, got {type(self.media).__name__}")
if self.media is not None and self.output is not None:
raise ValueError("DiffusionOutput cannot contain both media and legacy output")
if self.media is not None and self.post_process_func is not None:
raise ValueError("Typed diffusion media cannot carry a model-specific post_process_func")
if not self.to_cpu:
return
@@ -1674,6 +1785,10 @@ class DiffusionOutput:
return value
self.output = _maybe_to_cpu(self.output)
if self.media is not None:
if not self.media.prepared_for_transport:
raise ValueError("Diffusion media must be prepared before to_cpu=True")
self.media = self.media.to_cpu()
self.trajectory_timesteps = _maybe_to_cpu(self.trajectory_timesteps)
self.trajectory_latents = _maybe_to_cpu(self.trajectory_latents)
self.trajectory_log_probs = _maybe_to_cpu(self.trajectory_log_probs)