Merge branch 'main' into feat/diffusion-prefetch-protection
This commit is contained in:
53 files changed
+1341
-1273
No files matched your search
@@ -30,15 +30,6 @@ def get_pipeline_path(model_dir: str, filename: str) -> Path:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _warn_deprecated_kwargs(kwargs: dict[str, Any]) -> None:
|
||||
if "cli_explicit_keys" in kwargs:
|
||||
warnings.warn(
|
||||
"cli_explicit_keys= is deprecated and ignored. Remove the kwarg.",
|
||||
DeprecationWarning,
|
||||
stacklevel=3,
|
||||
)
|
||||
|
||||
|
||||
_STAGE_OVERRIDE_PATTERN = re.compile(r"^stage_(\d+)_(.+)$")
|
||||
|
||||
|
||||
@@ -837,15 +828,6 @@ _PIPELINE_WIDE_ENGINE_FIELDS: tuple[str, ...] = (
|
||||
)
|
||||
|
||||
|
||||
def deploy_override_field_names() -> frozenset[str]:
|
||||
"""Return deploy-schema fields whose CLI defaults must not override YAML."""
|
||||
return (
|
||||
frozenset(_STAGE_DEPLOY_FIELDS)
|
||||
| frozenset(_PIPELINE_WIDE_ENGINE_FIELDS)
|
||||
| frozenset({"async_chunk", "devices"})
|
||||
)
|
||||
|
||||
|
||||
def _build_engine_args(
|
||||
ps: StagePipelineConfig,
|
||||
ds: StageDeployConfig | None,
|
||||
@@ -1169,14 +1151,11 @@ class StageConfigFactory:
|
||||
model: str,
|
||||
cli_overrides: dict[str, Any] | None = None,
|
||||
deploy_config_path: str | None = None,
|
||||
**deprecated_kwargs: Any,
|
||||
) -> list[StageConfig] | None:
|
||||
"""Load pipeline + deploy config, merge with CLI overrides.
|
||||
|
||||
Checks _PIPELINE_REGISTRY first (new path), falls back to legacy YAML.
|
||||
"""
|
||||
_warn_deprecated_kwargs(deprecated_kwargs)
|
||||
|
||||
if cli_overrides is None:
|
||||
cli_overrides = {}
|
||||
|
||||
@@ -1256,15 +1235,12 @@ class StageConfigFactory:
|
||||
model_type: str,
|
||||
cli_overrides: dict[str, Any],
|
||||
deploy_config_path: str | None = None,
|
||||
**deprecated_kwargs: Any,
|
||||
) -> list[StageConfig]:
|
||||
"""Create StageConfigs from pipeline registry + deploy YAML.
|
||||
|
||||
Precedence: caller-typed (non-None) value > deploy YAML >
|
||||
StageDeployConfig dataclass default.
|
||||
"""
|
||||
_warn_deprecated_kwargs(deprecated_kwargs)
|
||||
|
||||
# Resolve deploy config path
|
||||
if deploy_config_path is None:
|
||||
deploy_path = _DEPLOY_DIR / f"{model_type}.yaml"
|
||||
|
||||
@@ -14,6 +14,16 @@ from dataclasses import dataclass
|
||||
import torch
|
||||
from einops import rearrange
|
||||
from torch import Tensor, nn
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.distributed.autoencoders.distributed_vae_executor import (
|
||||
DistributedOperator,
|
||||
DistributedVaeMixin,
|
||||
GridSpec,
|
||||
TileTask,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -322,3 +332,229 @@ class AutoEncoder(nn.Module):
|
||||
|
||||
def forward(self, x: Tensor) -> Tensor:
|
||||
return self.decode(self.encode(x))
|
||||
|
||||
|
||||
class DistributedAutoEncoder(AutoEncoder, DistributedVaeMixin):
|
||||
def __init__(self, params: AutoEncoderParams):
|
||||
super().__init__(params)
|
||||
self.init_distributed()
|
||||
|
||||
self.use_tiling = False
|
||||
self.tile_sample_min_height = 512
|
||||
self.tile_sample_min_width = 512
|
||||
self.tile_sample_stride_height = 448
|
||||
self.tile_sample_stride_width = 448
|
||||
self.spatial_compression_ratio = params.downsample
|
||||
|
||||
def blend_v(self, above: Tensor, current: Tensor, blend_extent: int) -> Tensor:
|
||||
blend_extent = min(above.shape[-2], current.shape[-2], blend_extent)
|
||||
if blend_extent <= 0:
|
||||
return current
|
||||
|
||||
for y in range(blend_extent):
|
||||
alpha = y / blend_extent
|
||||
current[:, :, y, :] = above[:, :, -blend_extent + y, :] * (1 - alpha) + current[:, :, y, :] * alpha
|
||||
return current
|
||||
|
||||
def blend_h(self, left: Tensor, current: Tensor, blend_extent: int) -> Tensor:
|
||||
blend_extent = min(left.shape[-1], current.shape[-1], blend_extent)
|
||||
if blend_extent <= 0:
|
||||
return current
|
||||
|
||||
for x in range(blend_extent):
|
||||
alpha = x / blend_extent
|
||||
current[:, :, :, x] = left[:, :, :, -blend_extent + x] * (1 - alpha) + current[:, :, :, x] * alpha
|
||||
return current
|
||||
|
||||
def decode_tile_split(self, z: Tensor) -> tuple[list[TileTask], GridSpec]:
|
||||
_, _, latent_height, latent_width = z.shape
|
||||
|
||||
sample_height = latent_height * self.spatial_compression_ratio
|
||||
sample_width = latent_width * self.spatial_compression_ratio
|
||||
|
||||
tile_latent_min_height = self.tile_sample_min_height // self.spatial_compression_ratio
|
||||
tile_latent_min_width = self.tile_sample_min_width // self.spatial_compression_ratio
|
||||
tile_latent_stride_height = self.tile_sample_stride_height // self.spatial_compression_ratio
|
||||
tile_latent_stride_width = self.tile_sample_stride_width // self.spatial_compression_ratio
|
||||
|
||||
blend_height = self.tile_sample_min_height - self.tile_sample_stride_height
|
||||
blend_width = self.tile_sample_min_width - self.tile_sample_stride_width
|
||||
|
||||
tiletask_list = []
|
||||
for i in range(0, latent_height, tile_latent_stride_height):
|
||||
for j in range(0, latent_width, tile_latent_stride_width):
|
||||
tile = z[
|
||||
:,
|
||||
:,
|
||||
i : i + tile_latent_min_height,
|
||||
j : j + tile_latent_min_width,
|
||||
]
|
||||
tiletask_list.append(
|
||||
TileTask(
|
||||
tile_id=len(tiletask_list),
|
||||
grid_coord=(i // tile_latent_stride_height, j // tile_latent_stride_width),
|
||||
tensor=tile,
|
||||
workload=tile.shape[-2] * tile.shape[-1],
|
||||
)
|
||||
)
|
||||
grid_spec = GridSpec(
|
||||
split_dims=(2, 3),
|
||||
grid_shape=(tiletask_list[-1].grid_coord[0] + 1, tiletask_list[-1].grid_coord[1] + 1),
|
||||
tile_spec={
|
||||
"sample_height": sample_height,
|
||||
"sample_width": sample_width,
|
||||
"blend_height": blend_height,
|
||||
"blend_width": blend_width,
|
||||
},
|
||||
output_dtype=z.dtype,
|
||||
)
|
||||
return tiletask_list, grid_spec
|
||||
|
||||
def decode_tile_exec(self, task: TileTask) -> Tensor:
|
||||
tile = task.tensor
|
||||
tile = tile / self.scale_factor + self.shift_factor
|
||||
return self.decoder(tile)
|
||||
|
||||
def decode_tile_merge(self, coord_tensor_map: dict[tuple[int, ...], Tensor], grid_spec: GridSpec) -> Tensor:
|
||||
grid_h, grid_w = grid_spec.grid_shape
|
||||
result_rows = []
|
||||
|
||||
for i in range(grid_h):
|
||||
result_row = []
|
||||
for j in range(grid_w):
|
||||
tile = coord_tensor_map[(i, j)]
|
||||
if i > 0:
|
||||
tile = self.blend_v(coord_tensor_map[(i - 1, j)], tile, grid_spec.tile_spec["blend_height"])
|
||||
if j > 0:
|
||||
tile = self.blend_h(coord_tensor_map[(i, j - 1)], tile, grid_spec.tile_spec["blend_width"])
|
||||
|
||||
result_row.append(
|
||||
tile[
|
||||
:,
|
||||
:,
|
||||
: self.tile_sample_stride_height,
|
||||
: self.tile_sample_stride_width,
|
||||
]
|
||||
)
|
||||
result_rows.append(torch.cat(result_row, dim=-1))
|
||||
|
||||
dec = torch.cat(result_rows, dim=-2)[
|
||||
:,
|
||||
:,
|
||||
: grid_spec.tile_spec["sample_height"],
|
||||
: grid_spec.tile_spec["sample_width"],
|
||||
]
|
||||
return dec
|
||||
|
||||
def decode(self, z: Tensor) -> Tensor:
|
||||
if not self.is_distributed_enabled():
|
||||
return super().decode(z)
|
||||
|
||||
logger.debug("Bagel VAE decode running with distributed executor")
|
||||
return self.distributed_executor.execute(
|
||||
z,
|
||||
DistributedOperator(
|
||||
split=self.decode_tile_split,
|
||||
exec=self.decode_tile_exec,
|
||||
merge=self.decode_tile_merge,
|
||||
),
|
||||
broadcast_result=True,
|
||||
)
|
||||
|
||||
def encode_tile_split(self, x: Tensor) -> tuple[list[TileTask], GridSpec]:
|
||||
_, _, height, width = x.shape
|
||||
|
||||
latent_height = height // self.spatial_compression_ratio
|
||||
latent_width = width // self.spatial_compression_ratio
|
||||
|
||||
tile_latent_min_height = self.tile_sample_min_height // self.spatial_compression_ratio
|
||||
tile_latent_min_width = self.tile_sample_min_width // self.spatial_compression_ratio
|
||||
|
||||
tile_latent_stride_height = self.tile_sample_stride_height // self.spatial_compression_ratio
|
||||
tile_latent_stride_width = self.tile_sample_stride_width // self.spatial_compression_ratio
|
||||
|
||||
blend_height = tile_latent_min_height - tile_latent_stride_height
|
||||
blend_width = tile_latent_min_width - tile_latent_stride_width
|
||||
|
||||
tiletask_list = []
|
||||
for i in range(0, height, self.tile_sample_stride_height):
|
||||
for j in range(0, width, self.tile_sample_stride_width):
|
||||
tile = x[
|
||||
:,
|
||||
:,
|
||||
i : i + self.tile_sample_min_height,
|
||||
j : j + self.tile_sample_min_width,
|
||||
]
|
||||
tiletask_list.append(
|
||||
TileTask(
|
||||
tile_id=len(tiletask_list),
|
||||
grid_coord=(i // self.tile_sample_stride_height, j // self.tile_sample_stride_width),
|
||||
tensor=tile,
|
||||
workload=tile.shape[-2] * tile.shape[-1],
|
||||
)
|
||||
)
|
||||
grid_spec = GridSpec(
|
||||
split_dims=(2, 3),
|
||||
grid_shape=(tiletask_list[-1].grid_coord[0] + 1, tiletask_list[-1].grid_coord[1] + 1),
|
||||
tile_spec={
|
||||
"latent_height": latent_height,
|
||||
"latent_width": latent_width,
|
||||
"blend_height": blend_height,
|
||||
"blend_width": blend_width,
|
||||
"tile_latent_stride_height": tile_latent_stride_height,
|
||||
"tile_latent_stride_width": tile_latent_stride_width,
|
||||
},
|
||||
output_dtype=next(self.parameters()).dtype,
|
||||
)
|
||||
return tiletask_list, grid_spec
|
||||
|
||||
def encode_tile_exec(self, task: TileTask) -> Tensor:
|
||||
tile = task.tensor
|
||||
z = self.reg(self.encoder(tile))
|
||||
z = self.scale_factor * (z - self.shift_factor)
|
||||
return z
|
||||
|
||||
def encode_tile_merge(self, coord_tensor_map: dict[tuple[int, ...], Tensor], grid_spec: GridSpec) -> Tensor:
|
||||
grid_h, grid_w = grid_spec.grid_shape
|
||||
result_rows = []
|
||||
|
||||
for i in range(grid_h):
|
||||
result_row = []
|
||||
for j in range(grid_w):
|
||||
tile = coord_tensor_map[(i, j)]
|
||||
if i > 0:
|
||||
tile = self.blend_v(coord_tensor_map[(i - 1, j)], tile, grid_spec.tile_spec["blend_height"])
|
||||
if j > 0:
|
||||
tile = self.blend_h(coord_tensor_map[(i, j - 1)], tile, grid_spec.tile_spec["blend_width"])
|
||||
|
||||
result_row.append(
|
||||
tile[
|
||||
:,
|
||||
:,
|
||||
: grid_spec.tile_spec["tile_latent_stride_height"],
|
||||
: grid_spec.tile_spec["tile_latent_stride_width"],
|
||||
]
|
||||
)
|
||||
result_rows.append(torch.cat(result_row, dim=-1))
|
||||
|
||||
enc = torch.cat(result_rows, dim=-2)[
|
||||
:,
|
||||
:,
|
||||
: grid_spec.tile_spec["latent_height"],
|
||||
: grid_spec.tile_spec["latent_width"],
|
||||
]
|
||||
return enc
|
||||
|
||||
def encode(self, x: Tensor) -> Tensor:
|
||||
if not self.is_distributed_enabled():
|
||||
return super().encode(x)
|
||||
logger.debug("Bagel VAE encode running with distributed executor")
|
||||
return self.distributed_executor.execute(
|
||||
x,
|
||||
DistributedOperator(
|
||||
split=self.encode_tile_split,
|
||||
exec=self.encode_tile_exec,
|
||||
merge=self.encode_tile_merge,
|
||||
),
|
||||
broadcast_result=True,
|
||||
)
|
||||
@@ -32,7 +32,7 @@ from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPi
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific
|
||||
|
||||
from .autoencoder import AutoEncoder, AutoEncoderParams
|
||||
from .autoencoder import AutoEncoder, AutoEncoderParams, DistributedAutoEncoder
|
||||
from .bagel_transformer import Bagel, NaiveCache, Qwen2MoTConfig, Qwen2MoTForCausalLM
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -253,7 +253,7 @@ class BagelPipeline(nn.Module, SupportsComponentDiscovery, DiffusionPipelineProf
|
||||
)
|
||||
self.transformer = self.language_model.model
|
||||
ae_params: AutoEncoderParams = default_ae_params()
|
||||
self.vae = AutoEncoder(ae_params)
|
||||
self.vae = DistributedAutoEncoder(ae_params)
|
||||
|
||||
self.bagel = Bagel(
|
||||
language_model=self.language_model,
|
||||
|
||||
@@ -43,7 +43,6 @@ from vllm_omni.diffusion.distributed.sp_plan import (
|
||||
)
|
||||
from vllm_omni.diffusion.forward_context import get_forward_context
|
||||
from vllm_omni.diffusion.layers.adalayernorm import AdaLayerNorm
|
||||
from vllm_omni.diffusion.layers.norm import RMSNorm
|
||||
from vllm_omni.diffusion.layers.rope import RotaryEmbedding
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -538,8 +537,8 @@ class QwenImageCrossAttention(nn.Module):
|
||||
self.query_num_heads = self.to_qkv.num_heads
|
||||
self.kv_num_heads = self.to_qkv.num_kv_heads
|
||||
|
||||
self.norm_q = RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()
|
||||
self.norm_k = RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()
|
||||
self.norm_q = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()
|
||||
self.norm_k = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()
|
||||
|
||||
self.inner_dim = out_dim if out_dim is not None else head_dim * self.total_num_heads
|
||||
|
||||
@@ -576,8 +575,8 @@ class QwenImageCrossAttention(nn.Module):
|
||||
prefix=_join_prefix(prefix, "to_out"),
|
||||
)
|
||||
|
||||
self.norm_added_q = RMSNorm(head_dim, eps=eps)
|
||||
self.norm_added_k = RMSNorm(head_dim, eps=eps)
|
||||
self.norm_added_q = nn.RMSNorm(head_dim, eps=eps)
|
||||
self.norm_added_k = nn.RMSNorm(head_dim, eps=eps)
|
||||
|
||||
self.attn = Attention(
|
||||
num_heads=self.query_num_heads,
|
||||
@@ -992,7 +991,7 @@ class QwenImageTransformer2DModel(CachedTransformer):
|
||||
quant_config=quant_config,
|
||||
)
|
||||
|
||||
self.txt_norm = RMSNorm(joint_attention_dim, eps=1e-6)
|
||||
self.txt_norm = nn.RMSNorm(joint_attention_dim, eps=1e-6)
|
||||
|
||||
# Entry projections (image/text) are kept full precision —
|
||||
# small sensitive layers at the network boundary (see #2728).
|
||||
|
||||
@@ -402,12 +402,12 @@ class StableDiffusion3Pipeline(nn.Module, CFGParallelMixin, DiffusionPipelinePro
|
||||
truncation=True,
|
||||
add_special_tokens=True,
|
||||
return_tensors="pt",
|
||||
).to(self.device)
|
||||
)
|
||||
text_input_ids = text_inputs.input_ids
|
||||
untruncated_ids = self.tokenizer_3(prompt, padding="longest", return_tensors="pt").input_ids
|
||||
|
||||
if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids):
|
||||
removed_text = self.tokenizer_3.batch_decode(untruncated_ids[:, self.tokenizer_max_length - 1 : -1])
|
||||
removed_text = self.tokenizer_3.batch_decode(untruncated_ids[:, max_sequence_length - 1 : -1])
|
||||
logger.warning(
|
||||
"The following part of your input was truncated because `max_sequence_length` is set to "
|
||||
f" {max_sequence_length} tokens: {removed_text}"
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import argparse
|
||||
import dataclasses
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
@@ -191,30 +190,6 @@ class OmniEngineArgs(EngineArgs):
|
||||
load_omni_general_plugins()
|
||||
super().__post_init__()
|
||||
|
||||
@classmethod
|
||||
def from_cli_args(cls, args: argparse.Namespace) -> "OmniEngineArgs":
|
||||
attrs = [attr.name for attr in dataclasses.fields(cls)]
|
||||
engine_args = cls(**{attr: getattr(args, attr) for attr in attrs if hasattr(args, attr)})
|
||||
engine_args._explicit_fields = frozenset(
|
||||
attr for attr in attrs if hasattr(args, attr) and getattr(args, attr) is not None
|
||||
)
|
||||
return engine_args
|
||||
|
||||
@classmethod
|
||||
def create(cls, **explicit: Any) -> "OmniEngineArgs":
|
||||
"""Tracks caller-set fields for ``Omni(..., engine_args=ea)``."""
|
||||
ea = cls(**explicit)
|
||||
ea._explicit_fields = frozenset(explicit.keys())
|
||||
return ea
|
||||
|
||||
def explicit_kwargs(self) -> dict[str, Any]:
|
||||
explicit = getattr(self, "_explicit_fields", None)
|
||||
if explicit is None:
|
||||
return {
|
||||
f.name: getattr(self, f.name) for f in dataclasses.fields(self) if getattr(self, f.name) is not None
|
||||
}
|
||||
return {k: getattr(self, k) for k in explicit}
|
||||
|
||||
def _ensure_omni_models_registered(self):
|
||||
if hasattr(self, "_omni_models_registered"):
|
||||
return True
|
||||
@@ -420,8 +395,7 @@ class OrchestratorArgs:
|
||||
forwarding).
|
||||
|
||||
Fields that BOTH orchestrator and engine genuinely need (e.g. ``model``,
|
||||
``log_stats``) should be listed in ``SHARED_FIELDS`` below; ``split_kwargs``
|
||||
will copy them to both buckets.
|
||||
``log_stats``) should be listed in ``SHARED_FIELDS`` below.
|
||||
"""
|
||||
|
||||
# === Lifecycle ===
|
||||
@@ -530,163 +504,3 @@ def internal_blacklist_keys() -> frozenset[str]:
|
||||
dataclass — this function updates automatically.
|
||||
"""
|
||||
return orchestrator_field_names() - SHARED_FIELDS
|
||||
|
||||
|
||||
def split_kwargs(
|
||||
kwargs: dict[str, Any],
|
||||
*,
|
||||
engine_cls: type | None = None,
|
||||
user_typed: set[str] | None = None,
|
||||
strict: bool = False,
|
||||
) -> tuple[OrchestratorArgs, dict[str, Any]]:
|
||||
"""Partition CLI kwargs into (orchestrator, engine) buckets.
|
||||
|
||||
Args:
|
||||
kwargs: Raw dict, typically ``vars(args)``.
|
||||
engine_cls: Engine dataclass used to whitelist-filter the engine
|
||||
bucket. Defaults to ``OmniEngineArgs``. Pass a custom class
|
||||
for testing.
|
||||
user_typed: Keys the user actually typed on the command line. Used
|
||||
to warn when a user-typed flag is unclassifiable.
|
||||
strict: If True, raise ``ValueError`` on ambiguous (double-classified
|
||||
but not in ``SHARED_FIELDS``) fields. Default False to keep the
|
||||
rollout non-breaking; flip to True in tests and CI.
|
||||
|
||||
Returns:
|
||||
``(orchestrator_args, engine_kwargs)``. ``engine_kwargs`` has already
|
||||
been whitelist-filtered against ``engine_cls`` — safe to pass directly
|
||||
to ``engine_cls(**engine_kwargs)``.
|
||||
"""
|
||||
if engine_cls is None:
|
||||
engine_cls = OmniEngineArgs
|
||||
|
||||
orch_fields = orchestrator_field_names()
|
||||
engine_fields = {f.name for f in fields(engine_cls)}
|
||||
|
||||
orch_kwargs: dict[str, Any] = {}
|
||||
engine_candidate: dict[str, Any] = {}
|
||||
shared_values: dict[str, Any] = {}
|
||||
unclassified: dict[str, Any] = {}
|
||||
|
||||
for key, value in kwargs.items():
|
||||
in_orch = key in orch_fields
|
||||
in_engine = key in engine_fields
|
||||
is_shared = key in SHARED_FIELDS
|
||||
|
||||
if is_shared:
|
||||
shared_values[key] = value
|
||||
elif in_orch and in_engine:
|
||||
# Declared in both but not marked shared → ambiguous.
|
||||
msg = (
|
||||
f"Field {key!r} is defined on both OrchestratorArgs and "
|
||||
f"{engine_cls.__name__} but is not in SHARED_FIELDS. "
|
||||
f"This causes double-routing. Either remove the duplicate or "
|
||||
f"add {key!r} to SHARED_FIELDS if the sharing is intentional."
|
||||
)
|
||||
if strict:
|
||||
raise ValueError(msg)
|
||||
logger.error(msg)
|
||||
# Default: treat as orchestrator-only to preserve existing behavior.
|
||||
orch_kwargs[key] = value
|
||||
elif in_orch:
|
||||
orch_kwargs[key] = value
|
||||
elif in_engine:
|
||||
engine_candidate[key] = value
|
||||
else:
|
||||
unclassified[key] = value
|
||||
|
||||
# Warn on user-typed but unclassifiable flags so we don't silently drop
|
||||
# something the user cared about (fixes the class of bug that spawned #873).
|
||||
if unclassified and user_typed:
|
||||
user_typed_unknown = sorted(k for k in unclassified if k in user_typed)
|
||||
if user_typed_unknown:
|
||||
logger.warning(
|
||||
"CLI flags not consumed by vllm-omni and dropped before "
|
||||
"per-stage engine construction: %s. If these are vllm "
|
||||
"frontend/uvicorn flags (host, port, ssl_*, api_key, …) this "
|
||||
"is expected; otherwise check your spelling.",
|
||||
user_typed_unknown,
|
||||
)
|
||||
|
||||
# Engine bucket: shared + engine-only. We do NOT pass through unclassified
|
||||
# fields — that's exactly the server/uvicorn noise we want to shed.
|
||||
engine_kwargs = {**shared_values, **engine_candidate}
|
||||
|
||||
# Construct the orchestrator dataclass. Shared fields that OrchestratorArgs
|
||||
# also declares get copied into its constructor.
|
||||
orch_init: dict[str, Any] = dict(orch_kwargs)
|
||||
for key, value in shared_values.items():
|
||||
if key in orch_fields:
|
||||
orch_init[key] = value
|
||||
orch_args = OrchestratorArgs(**orch_init)
|
||||
|
||||
return orch_args, engine_kwargs
|
||||
|
||||
|
||||
def derive_server_dests_from_vllm_parser() -> frozenset[str]:
|
||||
"""Derive the set of argparse dests that belong to vllm's frontend/server.
|
||||
|
||||
Returns every dest registered by ``make_arg_parser`` that is NOT a field
|
||||
of ``OmniEngineArgs`` and NOT a field of ``OrchestratorArgs``. Useful for
|
||||
CI tests to assert all CLI flags are classifiable without maintaining
|
||||
a hardcoded server list.
|
||||
|
||||
Returns empty frozenset if vllm's parser cannot be built (e.g. in a
|
||||
minimal test environment).
|
||||
"""
|
||||
try:
|
||||
from vllm.entrypoints.openai.cli_args import make_arg_parser
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
except ImportError:
|
||||
logger.debug("Cannot import vllm parser — server-dest derivation skipped")
|
||||
return frozenset()
|
||||
|
||||
try:
|
||||
parser = make_arg_parser(FlexibleArgumentParser())
|
||||
all_dests = {a.dest for a in parser._actions if a.dest and a.dest != "help"}
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to build vllm parser: %s", exc)
|
||||
return frozenset()
|
||||
|
||||
engine_fields = {f.name for f in fields(OmniEngineArgs)}
|
||||
orch_fields = orchestrator_field_names()
|
||||
|
||||
return frozenset(all_dests - engine_fields - orch_fields - SHARED_FIELDS)
|
||||
|
||||
|
||||
def orchestrator_args_from_argparse(args: Any) -> OrchestratorArgs:
|
||||
"""Build an ``OrchestratorArgs`` from an ``argparse.Namespace``.
|
||||
|
||||
Only copies attributes that exist on the namespace — missing fields fall
|
||||
back to the dataclass default. Useful when the full parser is already
|
||||
built and ``vars(args)`` would include noise.
|
||||
"""
|
||||
kwargs: dict[str, Any] = {}
|
||||
for f in fields(OrchestratorArgs):
|
||||
if hasattr(args, f.name):
|
||||
value = getattr(args, f.name)
|
||||
if value is not None or f.default is None:
|
||||
kwargs[f.name] = value
|
||||
return OrchestratorArgs(**kwargs)
|
||||
|
||||
|
||||
def nullify_stage_engine_defaults(parser: argparse.ArgumentParser) -> None:
|
||||
"""Reset stage-level engine flag defaults to ``None``; preserve real
|
||||
default in help text. Only deploy-YAML override fields are touched.
|
||||
Idempotent."""
|
||||
from vllm_omni.config.stage_config import deploy_override_field_names
|
||||
|
||||
override_dests = deploy_override_field_names()
|
||||
|
||||
for action in parser._actions:
|
||||
if action.dest in ("help", "version") or not action.option_strings:
|
||||
continue
|
||||
if action.dest not in override_dests:
|
||||
continue
|
||||
if action.default is None or action.default is argparse.SUPPRESS:
|
||||
continue
|
||||
if action.help and "(default:" not in action.help and "%(default)" not in action.help:
|
||||
action.help = f"{action.help} (default: {action.default})"
|
||||
action.default = None
|
||||
|
||||
parser._omni_nullified = True # type: ignore[attr-defined]
|
||||
@@ -21,7 +21,7 @@ import weakref
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from contextlib import ExitStack
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import TYPE_CHECKING, Any, Literal, cast
|
||||
from typing import Any, Literal, cast
|
||||
|
||||
import janus
|
||||
import torch
|
||||
@@ -113,9 +113,6 @@ from vllm_omni.inputs.data import OmniSamplingParams
|
||||
from vllm_omni.metrics.prometheus import OmniRequestCounter
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.engine.arg_utils import OmniEngineArgs
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_STARTUP_POLL_INTERVAL_S = 1.0
|
||||
@@ -269,16 +266,17 @@ class AsyncOmniEngine:
|
||||
def __init__(
|
||||
self,
|
||||
model: str,
|
||||
engine_args: OmniEngineArgs | None = None,
|
||||
stage_init_timeout: int = 300,
|
||||
init_timeout: int = 600,
|
||||
diffusion_batch_size: int = 1,
|
||||
single_stage_mode: bool = False,
|
||||
transfer_emitter: Any = None,
|
||||
log_stats: bool = False,
|
||||
tokenizer: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
self.model = model
|
||||
self.tokenizer = tokenizer
|
||||
self.diffusion_batch_size = diffusion_batch_size
|
||||
startup_timeout = int(init_timeout)
|
||||
# Forwarded into Orchestrator so its _forward_to_next_stage path can
|
||||
@@ -294,22 +292,6 @@ class AsyncOmniEngine:
|
||||
|
||||
logger.info(f"[AsyncOmniEngine] Initializing with model {model}")
|
||||
|
||||
# Merge tracked engine_args fields into kwargs; explicit kwargs take priority.
|
||||
if engine_args is not None:
|
||||
if not hasattr(engine_args, "_explicit_fields"):
|
||||
raise TypeError(
|
||||
"engine_args=OmniEngineArgs(...) is ambiguous under "
|
||||
"sentinel-default precedence. Use "
|
||||
"OmniEngineArgs.create(**explicit) or pass explicit kwargs "
|
||||
"directly."
|
||||
)
|
||||
ea_dict = engine_args.explicit_kwargs()
|
||||
# Remove model since it is passed as a positional arg already.
|
||||
ea_dict.pop("model", None)
|
||||
kwargs = {**ea_dict, **kwargs}
|
||||
|
||||
self.tokenizer: str | None = kwargs.get("tokenizer")
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Single-stage mode detection #
|
||||
# ------------------------------------------------------------------ #
|
||||
@@ -2072,7 +2054,6 @@ class AsyncOmniEngine:
|
||||
stage_configs_path = kwargs.get("stage_configs_path", None)
|
||||
deploy_config_path = kwargs.pop("deploy_config", None)
|
||||
stage_overrides_json = kwargs.pop("stage_overrides", None)
|
||||
kwargs.pop("_cli_explicit_keys", None)
|
||||
explicit_stage_configs = kwargs.pop("stage_configs", None)
|
||||
if explicit_stage_configs is not None:
|
||||
logger.warning(
|
||||
|
||||
@@ -482,6 +482,15 @@ def prepare_engine_environment() -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _maybe_set_qwen3_omni_moe_env(engine_args_dict: dict[str, Any]) -> None:
|
||||
if (
|
||||
engine_args_dict.get("model_arch") == "Qwen3OmniMoeForConditionalGeneration"
|
||||
and "VLLM_USE_FLASHINFER_MOE_FP16" not in os.environ
|
||||
):
|
||||
os.environ["VLLM_USE_FLASHINFER_MOE_FP16"] = "0"
|
||||
logger.info("[stage_init] Set VLLM_USE_FLASHINFER_MOE_FP16=0 for Qwen3-Omni stage")
|
||||
|
||||
|
||||
def split_devices_for_replicas(
|
||||
devices_str: str | None,
|
||||
num_replicas: int,
|
||||
@@ -762,6 +771,10 @@ def build_engine_args_dict(
|
||||
default_sp = _to_dict(getattr(stage_config, "default_sampling_params", {}))
|
||||
engine_args_dict["has_sampling_extra_args"] = bool(default_sp.get("extra_args"))
|
||||
|
||||
# TODO: Remove this after the performance regression is fixed
|
||||
# Set VLLM_USE_FLASHINFER_MOE_FP16=0 for Qwen3-Omni to avoid performance regression
|
||||
_maybe_set_qwen3_omni_moe_env(engine_args_dict)
|
||||
|
||||
return engine_args_dict
|
||||
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG
|
||||
from vllm_omni.entrypoints.cli.benchmark.base import OmniBenchmarkSubcommandBase
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
|
||||
|
||||
|
||||
class OmniBenchmarkSubcommand(CLISubcommand):
|
||||
@@ -25,7 +25,7 @@ class OmniBenchmarkSubcommand(CLISubcommand):
|
||||
def validate(self, args: argparse.Namespace) -> None:
|
||||
pass
|
||||
|
||||
def subparser_init(self, subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
|
||||
def subparser_init(self, subparsers: argparse._SubParsersAction) -> TrackingArgumentParser:
|
||||
bench_parser = subparsers.add_parser(
|
||||
self.name, description=self.help, usage=f"vllm {self.name} <bench_type> [options]"
|
||||
)
|
||||
|
||||
@@ -24,10 +24,10 @@ def main():
|
||||
os.environ["VLLM_LOGGING_COLOR"] = "1"
|
||||
|
||||
from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG, cli_env_setup
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
import vllm_omni.entrypoints.cli.benchmark.main
|
||||
import vllm_omni.entrypoints.cli.serve
|
||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
|
||||
|
||||
CMD_MODULES = [
|
||||
vllm_omni.entrypoints.cli.serve,
|
||||
@@ -40,7 +40,7 @@ def main():
|
||||
|
||||
_ensure_vllm_platform()
|
||||
|
||||
parser = FlexibleArgumentParser(
|
||||
parser = TrackingArgumentParser(
|
||||
description="vLLM OMNI CLI",
|
||||
epilog=VLLM_SUBCMD_PARSER_EPILOG.format(subcmd="[subcommand]"),
|
||||
)
|
||||
|
||||
@@ -19,11 +19,10 @@ from vllm.entrypoints.cli.types import CLISubcommand
|
||||
from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
|
||||
from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
from vllm_omni.entrypoints.cli.logo import log_logo
|
||||
from vllm_omni.entrypoints.openai.api_server import omni_run_server
|
||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser, TrackingNamespace
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -84,10 +83,10 @@ class OmniServeCommand(CLISubcommand):
|
||||
name = "serve"
|
||||
# Parser stashed at subparser_init so ``cmd`` can resolve each user-typed
|
||||
# flag to its real ``dest`` via the parser's action table.
|
||||
_parser: FlexibleArgumentParser | None = None
|
||||
_parser: TrackingArgumentParser
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
def cmd(args: TrackingNamespace) -> None:
|
||||
if not os.environ.get("VLLM_DISABLE_LOG_LOGO"):
|
||||
os.environ["VLLM_DISABLE_LOG_LOGO"] = "1"
|
||||
log_logo()
|
||||
@@ -176,7 +175,7 @@ class OmniServeCommand(CLISubcommand):
|
||||
return
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
def subparser_init(self, subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
|
||||
def subparser_init(self, subparsers: argparse._SubParsersAction) -> TrackingArgumentParser:
|
||||
serve_parser = subparsers.add_parser(
|
||||
self.name,
|
||||
description=DESCRIPTION,
|
||||
@@ -682,22 +681,10 @@ class OmniServeCommand(CLISubcommand):
|
||||
# sandboxed globals dict via ``DummySelf``) doesn't fail on a NameError.
|
||||
type(self)._parser = serve_parser
|
||||
|
||||
nullify_stage_engine_defaults(serve_parser)
|
||||
return serve_parser
|
||||
|
||||
|
||||
def _create_default_diffusion_stage_cfg(args: argparse.Namespace) -> list[dict[str, Any]]:
|
||||
"""Create default diffusion stage configuration.
|
||||
|
||||
Uses AsyncOmniEngine's implementation which doesn't have OmegaConf
|
||||
compatibility issues.
|
||||
"""
|
||||
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
|
||||
|
||||
return AsyncOmniEngine._create_default_diffusion_stage_cfg(vars(args))
|
||||
|
||||
|
||||
def run_headless(args: argparse.Namespace) -> None:
|
||||
def run_headless(args: TrackingNamespace) -> None:
|
||||
"""Run a single stage in headless mode.
|
||||
|
||||
Honors ``--omni-dp-size-local``: launches that many replicas locally for
|
||||
@@ -738,43 +725,39 @@ def run_headless(args: argparse.Namespace) -> None:
|
||||
stage_id: int | None = args.stage_id
|
||||
omni_master_address: str | None = args.omni_master_address
|
||||
omni_master_port: int | None = args.omni_master_port
|
||||
worker_backend: str | None = args.worker_backend
|
||||
stage_configs_path: str | None = args.stage_configs_path
|
||||
omni_replica_address: str | None = getattr(args, "omni_replica_address", None)
|
||||
omni_dp_size_local: int = max(1, int(getattr(args, "omni_dp_size_local", 1) or 1))
|
||||
|
||||
if not model:
|
||||
raise ValueError("Failed to pass model from kwargs")
|
||||
if stage_id is None:
|
||||
raise ValueError("--stage-id is required in headless mode")
|
||||
if omni_master_address is None or omni_master_port is None:
|
||||
raise ValueError("--omni-master-address and --omni-master-port are required in headless mode")
|
||||
if worker_backend != "multi_process":
|
||||
raise ValueError("headless mode requires worker_backend=multi_process")
|
||||
|
||||
# Filter down to a dict of things explicitly requested by the user
|
||||
args_dict = args.get_explicit_kwargs_dict()
|
||||
|
||||
# ``--replica-id`` is deprecated and ignored — replica ids are
|
||||
# auto-assigned by ``OmniMasterServer`` so headless processes carry
|
||||
# no knowledge of their per-replica id at launch time. Warn (don't
|
||||
# error) when the operator still supplies it so existing launchers
|
||||
# keep working with a single log line.
|
||||
explicit_cli_keys: set[str] = getattr(args, "_cli_explicit_keys", set()) or set()
|
||||
if "replica_id" in explicit_cli_keys:
|
||||
if "replica_id" in args_dict:
|
||||
logger.warning(
|
||||
"[Headless] --replica-id is deprecated and ignored "
|
||||
"(supplied value: %s). Replica ids are auto-assigned by the "
|
||||
"master server.",
|
||||
args.replica_id,
|
||||
)
|
||||
if omni_master_address is None or omni_master_port is None:
|
||||
raise ValueError("--omni-master-address and --omni-master-port are required in headless mode")
|
||||
api_server_count = args.api_server_count or 0
|
||||
if api_server_count > 1:
|
||||
raise ValueError("api_server_count can't be set in headless mode")
|
||||
if args.worker_backend != "multi_process":
|
||||
raise ValueError("headless mode requires worker_backend=multi_process")
|
||||
|
||||
args_dict = vars(args).copy()
|
||||
args_dict.pop("_cli_explicit_keys", None)
|
||||
# Forward ``--deploy-config`` so the headless reads the same YAML the
|
||||
# head was launched with — otherwise ``load_and_resolve_stage_configs``
|
||||
# falls back to ``vllm_omni/deploy/<model>.yaml`` and the headless's
|
||||
# view of ``stage.runtime.devices`` diverges from the head's, breaking
|
||||
# the per-replica device split.
|
||||
config_path, stage_configs = load_and_resolve_stage_configs(
|
||||
model,
|
||||
args_dict.get("stage_configs_path"),
|
||||
stage_configs_path,
|
||||
args_dict,
|
||||
deploy_config_path=args_dict.get("deploy_config"),
|
||||
)
|
||||
|
||||
@@ -1,13 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import warnings
|
||||
import weakref
|
||||
from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING, Any, Literal
|
||||
from typing import Any, Literal
|
||||
|
||||
import huggingface_hub
|
||||
from vllm.logger import init_logger
|
||||
@@ -29,9 +26,7 @@ from vllm_omni.metrics.stats import OrchestratorAggregator as OrchestratorMetric
|
||||
from vllm_omni.metrics.transfer import OmniTransferMetrics
|
||||
from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.engine.arg_utils import OmniEngineArgs
|
||||
from vllm_omni.utils.tracking_parser import TrackingNamespace
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -105,48 +100,44 @@ class OmniBase(PDDisaggregationMixin):
|
||||
@classmethod
|
||||
def from_cli_args(
|
||||
cls,
|
||||
args: argparse.Namespace,
|
||||
*,
|
||||
parser: argparse.ArgumentParser | None = None,
|
||||
**overrides: Any,
|
||||
args: TrackingNamespace,
|
||||
model: str | None = None,
|
||||
) -> OmniBase:
|
||||
"""Deprecated argparse builder.
|
||||
|
||||
Build from argparse. If ``parser`` is passed and not yet nullified,
|
||||
un-typed engine fields are reset to ``None``. New callers should
|
||||
nullify deploy-overriding parser defaults with
|
||||
``nullify_stage_engine_defaults(parser)`` and construct Omni/AsyncOmni
|
||||
directly.
|
||||
"""Build from a TrackingNamespace parsed by TrackingArgumentParser.
|
||||
Only args that are explicitly passed to parse_args are forwarded.
|
||||
"""
|
||||
warnings.warn(
|
||||
"`from_cli_args()` is deprecated. Nullify deploy-overriding parser defaults "
|
||||
"with `nullify_stage_engine_defaults(parser)` and construct Omni/AsyncOmni "
|
||||
"directly from `vars(args)`.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
kwargs: dict[str, Any] = {k: v for k, v in vars(args).items() if not k.startswith("_")}
|
||||
if not isinstance(args, TrackingNamespace):
|
||||
raise TypeError(
|
||||
f"expected args to be of type TrackingNamespace, got {type(args)}. "
|
||||
"Hint: did you parse your args with TrackingArgumentParser?"
|
||||
)
|
||||
|
||||
if parser is not None and not getattr(parser, "_omni_nullified", False):
|
||||
from vllm_omni.config.stage_config import deploy_override_field_names
|
||||
from vllm_omni.entrypoints.utils import detect_explicit_cli_keys
|
||||
explicit_kwargs = args.get_explicit_kwargs_dict()
|
||||
args_model = explicit_kwargs.pop("model", None) or args.model
|
||||
if model is not None and args_model is not None and model != args_model:
|
||||
raise ValueError(
|
||||
f"explicit model kwarg and args.model were both provided, but do not match [{model} != {args_model}]"
|
||||
)
|
||||
|
||||
explicit = detect_explicit_cli_keys(sys.argv[1:], parser) or set()
|
||||
override_dests = deploy_override_field_names()
|
||||
for key in list(kwargs):
|
||||
if key in override_dests and key not in explicit:
|
||||
kwargs[key] = None
|
||||
if model is None and args_model is None:
|
||||
raise ValueError(
|
||||
"model must be explicitly passed as a parsed arg in the TrackingNamespace or directly provided."
|
||||
)
|
||||
|
||||
kwargs.update(overrides)
|
||||
return cls(**kwargs)
|
||||
resolved_model = model or args_model
|
||||
return cls(model=resolved_model, **explicit_kwargs)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: str,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
engine_args: OmniEngineArgs | None = kwargs.pop("engine_args", None)
|
||||
|
||||
if "engine_args" in kwargs:
|
||||
logger.warning(
|
||||
"engine_args were passed as a kwarg to an Omni instance; this is not supported. "
|
||||
"You should instead, pass the keyword arguments used to initialize the engine args "
|
||||
"directly to this object's initializer."
|
||||
)
|
||||
stage_init_timeout = kwargs.pop("stage_init_timeout", 300)
|
||||
init_timeout = kwargs.pop("init_timeout", 600)
|
||||
log_stats = kwargs.pop("log_stats", False)
|
||||
@@ -180,7 +171,6 @@ class OmniBase(PDDisaggregationMixin):
|
||||
st = time.time()
|
||||
self.engine = AsyncOmniEngine(
|
||||
model=model,
|
||||
engine_args=engine_args,
|
||||
init_timeout=init_timeout,
|
||||
stage_init_timeout=stage_init_timeout,
|
||||
diffusion_batch_size=diffusion_batch_size,
|
||||
|
||||
@@ -136,6 +136,7 @@ from vllm_omni.entrypoints.openai.utils import get_stage_type, parse_lora_reques
|
||||
from vllm_omni.entrypoints.openai.video_api_utils import decode_input_reference
|
||||
from vllm_omni.entrypoints.openpi.serving import ServingRealtimeRobotOpenPI
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniTextPrompt
|
||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser, TrackingNamespace
|
||||
|
||||
logger = init_logger(__name__)
|
||||
router = APIRouter()
|
||||
@@ -510,7 +511,7 @@ async def omni_run_server_worker(listen_address, sock, args, client_config=None,
|
||||
|
||||
@asynccontextmanager
|
||||
async def build_async_omni(
|
||||
args: Namespace,
|
||||
args: TrackingNamespace,
|
||||
*,
|
||||
disable_frontend_multiprocessing: bool | None = None,
|
||||
client_config: dict[str, Any] | None = None,
|
||||
@@ -550,7 +551,7 @@ async def build_async_omni(
|
||||
|
||||
@asynccontextmanager
|
||||
async def build_async_omni_from_stage_config(
|
||||
args: Namespace,
|
||||
args: TrackingNamespace,
|
||||
*,
|
||||
disable_frontend_multiprocessing: bool = False,
|
||||
) -> AsyncIterator[EngineClient]:
|
||||
@@ -604,9 +605,9 @@ async def build_async_omni_from_stage_config(
|
||||
logger.debug("Pre-loading transformers_modules failed: %s", e)
|
||||
|
||||
try:
|
||||
kwargs = vars(args).copy()
|
||||
kwargs.pop("model", None)
|
||||
async_omni = AsyncOmni(model=args.model, **kwargs)
|
||||
kwargs = args.get_explicit_kwargs_dict()
|
||||
model = kwargs.pop("model", None) or args.model
|
||||
async_omni = AsyncOmni(model=model, **kwargs)
|
||||
|
||||
# # Don't keep the dummy data in memory
|
||||
# await async_llm.reset_mm_cache()
|
||||
@@ -3040,25 +3041,23 @@ async def omni_wakeup(request: OmniWakeupRequest, raw_request: Request):
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
import asyncio
|
||||
|
||||
from vllm.entrypoints.openai.cli_args import make_arg_parser
|
||||
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
|
||||
parser = argparse.ArgumentParser(description="vLLM-Omni OpenAI-Compatible REST API server")
|
||||
parser = TrackingArgumentParser(description="vLLM-Omni OpenAI-Compatible REST API server")
|
||||
parser = make_arg_parser(parser)
|
||||
registered_flags = set()
|
||||
for action in parser._actions:
|
||||
registered_flags.update(action.option_strings)
|
||||
|
||||
# FIXME - this is broken on `main`. `make_arg_parser` does not handle omni engine args properly.
|
||||
if "--omni" not in registered_flags:
|
||||
parser.add_argument("--omni", action="store_true", default=False, help="Enable vLLM-Omni mode.")
|
||||
if "--enable-sleep-mode" not in registered_flags:
|
||||
parser.add_argument(
|
||||
"--enable-sleep-mode", action="store_true", default=False, help="Enable GPU memory pool for sleep mode."
|
||||
)
|
||||
nullify_stage_engine_defaults(parser)
|
||||
args = parser.parse_args()
|
||||
if not hasattr(args, "model_tag"):
|
||||
setattr(args, "model_tag", args.model)
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import argparse
|
||||
import os
|
||||
import types
|
||||
from collections import Counter
|
||||
@@ -24,81 +23,11 @@ PROJECT_ROOT = Path(__file__).parent.parent.parent
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _warn_deprecated_explicit_keys(kwargs: dict[str, Any]) -> None:
|
||||
if "cli_explicit_keys" in kwargs:
|
||||
import warnings
|
||||
|
||||
warnings.warn(
|
||||
"cli_explicit_keys= is deprecated and ignored. Remove the kwarg.",
|
||||
DeprecationWarning,
|
||||
stacklevel=3,
|
||||
)
|
||||
|
||||
|
||||
_DIFFUSERS_CLASS_TO_CONFIG: dict[str, str] = {
|
||||
"GlmImagePipeline": "glm_image",
|
||||
}
|
||||
|
||||
|
||||
def detect_explicit_cli_keys(
|
||||
argv: list[str],
|
||||
parser: argparse.ArgumentParser | None = None,
|
||||
) -> set[str]:
|
||||
"""Walk ``argv`` and return the set of ``dest`` attribute names the user
|
||||
explicitly provided (e.g. ``--max-num-seqs 64`` → ``max_num_seqs``).
|
||||
|
||||
Used to distinguish user-typed CLI args from argparse default values so
|
||||
deploy YAMLs are not silently overridden by parser defaults. Shared
|
||||
across online (``vllm serve``) and offline (scripts, examples, tests,
|
||||
CI) entry points — offline callers that parse CLI args via argparse
|
||||
should invoke this on ``sys.argv[1:]`` and pass the result through to
|
||||
``AsyncOmni`` / ``Omni`` via the ``_cli_explicit_keys`` kwarg.
|
||||
|
||||
When ``parser`` is provided, each token is looked up in the parser's
|
||||
action table to find its real ``dest``. This correctly handles flags
|
||||
with ``dest=`` overrides, alias flags (e.g. ``--usp`` /
|
||||
``--ulysses-degree`` both mapping to ``ulysses_degree``), and
|
||||
``--disable-foo`` / ``store_false`` patterns that map to a differently
|
||||
named dest. Callers with access to an ``argparse.ArgumentParser`` should
|
||||
always pass it.
|
||||
|
||||
When ``parser`` is ``None``, a name-based heuristic is used as a
|
||||
fallback (hyphens → underscores, plus a ``no_`` prefix strip for
|
||||
``argparse.BooleanOptionalAction``). This is correct for simple flags
|
||||
but silently misidentifies ``--disable-X``-style flags and explicit
|
||||
``dest=`` overrides, so prefer the parser-aware form.
|
||||
"""
|
||||
if parser is not None:
|
||||
dest_map: dict[str, str] = {}
|
||||
for action in parser._actions:
|
||||
for opt in action.option_strings:
|
||||
dest_map[opt] = action.dest
|
||||
explicit: set[str] = set()
|
||||
for tok in argv:
|
||||
if not tok.startswith("--"):
|
||||
continue
|
||||
flag = tok.split("=", 1)[0]
|
||||
dest = dest_map.get(flag)
|
||||
if dest is not None:
|
||||
explicit.add(dest)
|
||||
return explicit
|
||||
|
||||
# Fallback: name-based heuristic (legacy path for callers without a parser).
|
||||
explicit = set()
|
||||
for tok in argv:
|
||||
if not tok.startswith("--"):
|
||||
continue
|
||||
name = tok[2:].split("=", 1)[0]
|
||||
if not name:
|
||||
continue
|
||||
attr = name.replace("-", "_")
|
||||
explicit.add(attr)
|
||||
# BooleanOptionalAction: --no-foo records as dest `foo`, not `no_foo`.
|
||||
if attr.startswith("no_"):
|
||||
explicit.add(attr[3:])
|
||||
return explicit
|
||||
|
||||
|
||||
def inject_omni_kv_config(stage: Any, omni_conn_cfg: dict[str, Any], omni_from: str, omni_to: str) -> None:
|
||||
"""Inject connector configuration into stage engine arguments."""
|
||||
# Prepare omni_kv_config dict
|
||||
@@ -383,7 +312,6 @@ def load_stage_configs_from_model(
|
||||
base_engine_args: dict | None = None,
|
||||
deploy_config_path: str | None = None,
|
||||
stage_overrides: dict[str, dict[str, Any]] | None = None,
|
||||
**deprecated_kwargs: Any,
|
||||
) -> list:
|
||||
"""Load stage configurations from model's default config file.
|
||||
|
||||
@@ -402,8 +330,6 @@ def load_stage_configs_from_model(
|
||||
Returns:
|
||||
List of stage configuration dictionaries
|
||||
"""
|
||||
_warn_deprecated_explicit_keys(deprecated_kwargs)
|
||||
|
||||
if base_engine_args is None:
|
||||
base_engine_args = {}
|
||||
|
||||
@@ -577,7 +503,6 @@ def load_and_resolve_stage_configs(
|
||||
default_stage_cfg_factory: Any = None,
|
||||
deploy_config_path: str | None = None,
|
||||
stage_overrides: dict[str, dict[str, Any]] | None = None,
|
||||
**deprecated_kwargs: Any,
|
||||
) -> tuple[str, list]:
|
||||
"""Load stage configurations from model or YAML file with fallback to defaults.
|
||||
|
||||
@@ -621,8 +546,6 @@ def load_and_resolve_stage_configs(
|
||||
stage_configs_path,
|
||||
)
|
||||
|
||||
_warn_deprecated_explicit_keys(deprecated_kwargs)
|
||||
|
||||
if deploy_config_path is not None:
|
||||
config_path = deploy_config_path
|
||||
stage_configs = load_stage_configs_from_model(
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import argparse
|
||||
from typing import Any
|
||||
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
UNSET = object()
|
||||
|
||||
|
||||
class TrackingNamespace(argparse.Namespace):
|
||||
"""Proxy that wraps an argparse namespace with explicit keys, which
|
||||
can be filtered down to a dict containing only explicitly passed values.
|
||||
"""
|
||||
|
||||
def __init__(self, unfiltered_ns: argparse.Namespace, explicit_keys: frozenset[str]) -> None:
|
||||
# We never have nested tracking namespaces, but explicitly guard
|
||||
# against them to prevent bad behavior with nested __dict__ overrides.
|
||||
if isinstance(unfiltered_ns, TrackingNamespace):
|
||||
raise ValueError("Tracking namespaces cannot be nested")
|
||||
|
||||
self.unfiltered_ns = unfiltered_ns
|
||||
self.explicit_keys = explicit_keys
|
||||
|
||||
def __setattr__(self, name: str, value: Any) -> None:
|
||||
if name in ("unfiltered_ns", "explicit_keys"):
|
||||
object.__setattr__(self, name, value)
|
||||
else:
|
||||
setattr(self.unfiltered_ns, name, value)
|
||||
|
||||
def get_explicit_kwargs_dict(self):
|
||||
"""Return a dict containing only the explicitly passed key-value pairs."""
|
||||
return {k: v for k, v in vars(self.unfiltered_ns).items() if k in self.explicit_keys}
|
||||
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
return getattr(self.unfiltered_ns, name)
|
||||
|
||||
@property
|
||||
def __dict__(self):
|
||||
# NOTE: We do this so that vars() etc forward directly into the encapsulated namespace,
|
||||
# which makes this class a drop-in replacement for the original namespace, while also
|
||||
# ensuring that updates to the encapsulated namespace are correctly reflected.
|
||||
return self.unfiltered_ns.__dict__
|
||||
|
||||
|
||||
class TrackingGroup:
|
||||
"""Proxy that wraps an argument group and its corresponding shadow group."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
real_group: argparse._ArgumentGroup,
|
||||
shadow_group: argparse._ArgumentGroup,
|
||||
):
|
||||
self._real = real_group
|
||||
self._shadow = shadow_group
|
||||
|
||||
def add_argument(self, *args: Any, **kwargs: Any) -> argparse.Action:
|
||||
"""Add an argument to the real group and to the shadow group."""
|
||||
action = self._real.add_argument(*args, **kwargs)
|
||||
default_kwargs = {**kwargs, "default": UNSET}
|
||||
self._shadow.add_argument(*args, **default_kwargs)
|
||||
return action
|
||||
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
# Any attribute access is forwarded to the real argument group.
|
||||
return getattr(self._real, name)
|
||||
|
||||
|
||||
class TrackingSubparsers:
|
||||
"""Proxy that wraps a subparser and its corresponding shadow subparser."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
real_sub: argparse._SubParsersAction,
|
||||
shadow_sub: argparse._SubParsersAction,
|
||||
):
|
||||
self._real = real_sub
|
||||
self._shadow = shadow_sub
|
||||
|
||||
def add_parser(self, name, *args, **kwargs):
|
||||
"""Add a parser to the encapsulated real parser and its shadow."""
|
||||
real_parser = self._real.add_parser(name, *args, **kwargs)
|
||||
# real_parser is a TrackingArgumentParser with its own _shadow.
|
||||
# Reuse that shadow as the parent shadow's child — so when
|
||||
# real_parser.add_argument() mirrors to real_parser._shadow,
|
||||
# the parent's shadow sees it too.
|
||||
self._shadow._name_parser_map[name] = real_parser._shadow
|
||||
return real_parser
|
||||
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
# Any attribute access is forwarded to the real subparser.
|
||||
return getattr(self._real, name)
|
||||
|
||||
|
||||
class TrackingArgumentParser(FlexibleArgumentParser):
|
||||
"""Drop-in replacement for FlexibleArgumentParser, which tracks keys that
|
||||
were explicitly passed as args on the parser namespace.
|
||||
|
||||
Unfortunately, Argparse does not provide an easy way of doing this without
|
||||
depending on a lot of internal attributes, so we implement it by instead
|
||||
using a 'shadow' parser, which is essentially a clone of the parser, where
|
||||
defaults are overridden to `None`. By comparing the parser against its
|
||||
shadow, we can tell which values were passed in a non-destructive manner.
|
||||
"""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
# NOTE: We have to define the shadow parser before calling init,
|
||||
# with add_help=False, since otherwise init will call add_argument
|
||||
# and delegate to the override on this class and cause problems.
|
||||
shadow_kwargs = {**kwargs, "add_help": False}
|
||||
self._shadow = FlexibleArgumentParser(*args, **shadow_kwargs)
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def add_argument(self, *args: Any, **kwargs: Any) -> argparse.Action:
|
||||
"""Add an arg to the parser & the shadow, where the latter has UNSET for the default."""
|
||||
action = super().add_argument(*args, **kwargs)
|
||||
shadow_kwargs = {**kwargs, "default": UNSET}
|
||||
self._shadow.add_argument(*args, **shadow_kwargs)
|
||||
return action
|
||||
|
||||
def add_argument_group(self, *args, **kwargs) -> TrackingGroup:
|
||||
real_group = super().add_argument_group(*args, **kwargs)
|
||||
shadow_group = self._shadow.add_argument_group(*args, **kwargs)
|
||||
return TrackingGroup(real_group, shadow_group)
|
||||
|
||||
def add_mutually_exclusive_group(self, *args, **kwargs) -> TrackingGroup:
|
||||
real_group = super().add_mutually_exclusive_group(*args, **kwargs)
|
||||
shadow_group: argparse._MutuallyExclusiveGroup = self._shadow.add_mutually_exclusive_group(*args, **kwargs)
|
||||
return TrackingGroup(real_group, shadow_group)
|
||||
|
||||
def add_subparsers(self, *args, **kwargs) -> TrackingSubparsers:
|
||||
real_sub = super().add_subparsers(*args, **kwargs)
|
||||
shadow_sub = self._shadow.add_subparsers(*args, **kwargs)
|
||||
return TrackingSubparsers(real_sub, shadow_sub)
|
||||
|
||||
def build_tracking_namespace(self, real_ns: argparse.Namespace, shadow_ns: argparse.Namespace) -> TrackingNamespace:
|
||||
"""Build a tracking namespace for the real / shadow namespaces."""
|
||||
explicit_keys = frozenset(k for k, v in vars(shadow_ns).items() if v is not UNSET)
|
||||
return TrackingNamespace(real_ns, explicit_keys)
|
||||
|
||||
def parse_args(
|
||||
self,
|
||||
args: list[str] | None = None,
|
||||
namespace: argparse.Namespace | None = None,
|
||||
) -> TrackingNamespace:
|
||||
"""Parse the args on the real/shadow parser."""
|
||||
# Only the real parser should use the namespace if one is,
|
||||
# given since shadow parser will set its own defaults to None.
|
||||
real_ns = super().parse_args(args, namespace)
|
||||
shadow_ns = self._shadow.parse_args(args)
|
||||
if real_ns is None or shadow_ns is None:
|
||||
raise ValueError("Parse args created empty namespaces")
|
||||
|
||||
# If this is called through parse_known_args on self, we will already
|
||||
# get a TrackingNamespace back, which will already have set the explicit
|
||||
# keys through build_tracking_namespace, so no need to do it again.
|
||||
if isinstance(real_ns, TrackingNamespace):
|
||||
return real_ns
|
||||
|
||||
return self.build_tracking_namespace(real_ns, shadow_ns)
|
||||
|
||||
def parse_known_args(
|
||||
self,
|
||||
args: list[str] | None = None,
|
||||
namespace: argparse.Namespace | None = None,
|
||||
) -> tuple[TrackingNamespace, list[str]]:
|
||||
"""Parse the known args on the real/shadow parser."""
|
||||
real_ns, remaining = super().parse_known_args(args, namespace)
|
||||
shadow_ns, _ = self._shadow.parse_known_args(args)
|
||||
tracked_ns = self.build_tracking_namespace(real_ns, shadow_ns)
|
||||
|
||||
return tracked_ns, remaining
|
||||
Reference in new issue
Block a user