[Model] Support request-level batching for MammothModa2 DiT on shared runtime (#7476)
Signed-off-by: BeatSeat <wendavid552@gmail.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
1 parent
9e7e3a802c
commit
b308e196c9
6 files changed
+992
-175
No files matched your search
@@ -631,8 +631,33 @@ class Transformer2DModel(ModelMixin, ConfigMixin):
|
||||
dtype=torch.bool,
|
||||
device=image_hidden_states.device,
|
||||
)
|
||||
text_hidden_states = torch.cat([text_hidden_states, image_hidden_states], dim=1)
|
||||
text_attention_mask = torch.cat([text_attention_mask, image_attention_mask], dim=1)
|
||||
# When batch requests have unequal text lengths, right-padded text_hidden_states
|
||||
# followed by concatenated image_hidden_states places padding in the middle of
|
||||
# the row: [valid text, pad, valid image]. Since downstream rope_embedder and
|
||||
# joint_hidden_states prefix-pack using encoder_seq_lengths (sum of valid mask),
|
||||
# middle padding causes image queries to be truncated.
|
||||
# Compact each row so valid tokens are placed contiguously at the start.
|
||||
if batch_size > 1 and not text_attention_mask.all():
|
||||
text_lens = text_attention_mask.sum(dim=-1).tolist()
|
||||
img_lens = image_attention_mask.sum(dim=-1).tolist()
|
||||
max_valid_len = max(t + img for t, img in zip(text_lens, img_lens))
|
||||
compact_hidden_states = text_hidden_states.new_zeros(
|
||||
batch_size, max_valid_len, text_hidden_states.shape[-1]
|
||||
)
|
||||
compact_attention_mask = torch.zeros((batch_size, max_valid_len), dtype=torch.bool, device=device)
|
||||
for i in range(batch_size):
|
||||
t_len = text_lens[i]
|
||||
i_len = img_lens[i]
|
||||
if t_len > 0:
|
||||
compact_hidden_states[i, :t_len] = text_hidden_states[i, :t_len]
|
||||
if i_len > 0:
|
||||
compact_hidden_states[i, t_len : t_len + i_len] = image_hidden_states[i, :i_len]
|
||||
compact_attention_mask[i, : t_len + i_len] = True
|
||||
text_hidden_states = compact_hidden_states
|
||||
text_attention_mask = compact_attention_mask
|
||||
else:
|
||||
text_hidden_states = torch.cat([text_hidden_states, image_hidden_states], dim=1)
|
||||
text_attention_mask = torch.cat([text_attention_mask, image_attention_mask], dim=1)
|
||||
|
||||
img_tokens = rearrange(hidden_states, "b c (h p1) (w p2) -> b (h w) (p1 p2 c)", p1=p, p2=p)
|
||||
img_tokens = self.x_embedder(img_tokens)
|
||||
|
||||
@@ -24,7 +24,9 @@ from vllm_omni.diffusion.data import DiffusionCacheConfig, DiffusionOutput, Omni
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.transformers_utils.configs.mammoth_moda2 import Mammothmoda2Config
|
||||
|
||||
from .mammothmoda2_dit_model import SimpleQFormerImageRefiner, Transformer2DModel
|
||||
@@ -75,8 +77,235 @@ def _root_weight_source(
|
||||
)
|
||||
|
||||
|
||||
def _pad_cond_sequence(
|
||||
embeds: list[torch.Tensor],
|
||||
masks: list[torch.Tensor],
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Right-pad per-request ``[1, T_i, H]`` embeds into ``[B, T_max, H]``.
|
||||
|
||||
The transformer derives per-row ``encoder_seq_lengths`` from the boolean
|
||||
mask, so padding must sit on the right and masked positions are excluded
|
||||
from the packed joint sequence.
|
||||
"""
|
||||
if not embeds:
|
||||
raise ValueError("Cannot pad an empty conditioning sequence list")
|
||||
if len(embeds) != len(masks):
|
||||
raise ValueError(f"Conditioning embeds/mask count mismatch: {len(embeds)} vs {len(masks)}")
|
||||
|
||||
# Fast-path for single-item batch: bypass padding tensor allocation
|
||||
if len(embeds) == 1:
|
||||
emb0, m0 = embeds[0], masks[0]
|
||||
if emb0.ndim != 3 or emb0.shape[0] != 1:
|
||||
raise ValueError(f"Conditioning embeds[0] must be [1, T, H], got {tuple(emb0.shape)}")
|
||||
if m0.ndim != 2 or m0.shape != emb0.shape[:2]:
|
||||
raise ValueError(f"Conditioning embeds/mask length mismatch: {tuple(emb0.shape)} vs {tuple(m0.shape)}")
|
||||
return emb0, m0
|
||||
|
||||
hidden = embeds[0].shape[-1]
|
||||
lengths = [int(e.shape[1]) for e in embeds]
|
||||
max_len = max(lengths)
|
||||
|
||||
# Fast-path for homogeneous lengths: concatenate directly in C++
|
||||
if all(length == max_len for length in lengths):
|
||||
for i, (emb, m) in enumerate(zip(embeds, masks)):
|
||||
if emb.ndim != 3 or emb.shape[0] != 1 or emb.shape[-1] != hidden:
|
||||
raise ValueError(f"Conditioning embeds[{i}] must be [1, T, {hidden}], got {tuple(emb.shape)}")
|
||||
if m.ndim != 2 or m.shape[0] != 1 or m.shape[1] != max_len:
|
||||
raise ValueError(f"Conditioning embeds/mask length mismatch: {tuple(emb.shape)} vs {tuple(m.shape)}")
|
||||
return torch.cat(embeds, dim=0), torch.cat(masks, dim=0)
|
||||
|
||||
out = embeds[0].new_zeros((len(embeds), max_len, hidden))
|
||||
mask = torch.zeros((len(embeds), max_len), dtype=torch.bool, device=out.device)
|
||||
for i, (emb, m) in enumerate(zip(embeds, masks)):
|
||||
if emb.ndim != 3 or emb.shape[0] != 1 or emb.shape[-1] != hidden:
|
||||
raise ValueError(f"Conditioning embeds[{i}] must be [1, T, {hidden}], got {tuple(emb.shape)}")
|
||||
if m.ndim != 2 or m.shape[0] != 1 or m.shape[1] != emb.shape[1]:
|
||||
raise ValueError(f"Conditioning embeds/mask length mismatch: {tuple(emb.shape)} vs {tuple(m.shape)}")
|
||||
out[i, : emb.shape[1]] = emb[0]
|
||||
mask[i, : m.shape[1]] = m[0].bool()
|
||||
return out, mask
|
||||
|
||||
|
||||
def _resolve_request_sampling_and_dims(
|
||||
prompt: dict,
|
||||
sampling: OmniDiffusionSamplingParams | None,
|
||||
request_id: str,
|
||||
) -> tuple[int, int, float, tuple[float, float], int]:
|
||||
"""Shared extraction and validation of geometry and sampling knobs."""
|
||||
dimensions = []
|
||||
for name in ("height", "width"):
|
||||
value = DiffusionRequestBatch.get_prompt_field(prompt, name)
|
||||
if value is None and sampling is not None:
|
||||
value = getattr(sampling, name, None)
|
||||
if value is None:
|
||||
value = 1024
|
||||
try:
|
||||
dimensions.append(int(value))
|
||||
except (TypeError, ValueError, OverflowError) as exc:
|
||||
raise ValueError(f"Invalid image size {name}={value!r} for request {request_id}") from exc
|
||||
height, width = dimensions
|
||||
if height <= 0 or width <= 0:
|
||||
raise ValueError(f"Invalid image size: {height}x{width} for request {request_id}")
|
||||
if height % 16 != 0 or width % 16 != 0:
|
||||
raise ValueError(f"Image size must be multiples of 16, got {height}x{width} for request {request_id}")
|
||||
|
||||
info = prompt.get("additional_information")
|
||||
request_info = info if isinstance(info, dict) else {}
|
||||
extra_args = sampling.extra_args or {} if sampling else {}
|
||||
|
||||
guidance = extra_args.get("text_guidance_scale")
|
||||
if guidance is None and sampling is not None:
|
||||
guidance = sampling.guidance_scale if sampling.guidance_scale_provided else None
|
||||
if guidance is None:
|
||||
guidance = _first_request_value(request_info.get("text_guidance_scale"))
|
||||
if guidance is None:
|
||||
guidance = 9.0
|
||||
try:
|
||||
text_guidance_scale = float(guidance)
|
||||
except (TypeError, ValueError, OverflowError) as exc:
|
||||
raise ValueError(f"Invalid text_guidance_scale for request {request_id}") from exc
|
||||
|
||||
raw_num_inference_steps = extra_args.get("num_inference_steps")
|
||||
if raw_num_inference_steps is None and sampling is not None:
|
||||
raw_num_inference_steps = sampling.num_inference_steps
|
||||
if raw_num_inference_steps is None:
|
||||
raw_num_inference_steps = _first_request_value(request_info.get("num_inference_steps"))
|
||||
if raw_num_inference_steps is None:
|
||||
raw_num_inference_steps = 50
|
||||
try:
|
||||
num_inference_steps = int(raw_num_inference_steps)
|
||||
except (TypeError, ValueError, OverflowError) as exc:
|
||||
raise ValueError(f"Invalid num_inference_steps for request {request_id}") from exc
|
||||
if num_inference_steps <= 0:
|
||||
raise ValueError(f"num_inference_steps must be positive for request {request_id}")
|
||||
|
||||
cfg_range = extra_args.get("cfg_range")
|
||||
if cfg_range is None:
|
||||
cfg_range = request_info.get("cfg_range")
|
||||
if cfg_range is None:
|
||||
cfg_range = [0.0, 1.0]
|
||||
if not isinstance(cfg_range, (list, tuple)) or len(cfg_range) != 2:
|
||||
raise ValueError(f"cfg_range requires two values for request {request_id}")
|
||||
try:
|
||||
cfg_start, cfg_end = float(cfg_range[0]), float(cfg_range[1])
|
||||
except (TypeError, ValueError, OverflowError) as exc:
|
||||
raise ValueError(f"cfg_range requires two values convertible to floats for request {request_id}") from exc
|
||||
if not 0 <= cfg_start <= cfg_end <= 1:
|
||||
raise ValueError(f"cfg_range must satisfy 0 <= start <= end <= 1 for request {request_id}")
|
||||
|
||||
return height, width, text_guidance_scale, (cfg_start, cfg_end), num_inference_steps
|
||||
|
||||
|
||||
def _resolve_gen_vocab_start_index(od_config: OmniDiffusionConfig | None) -> int:
|
||||
"""Resolve the visual token threshold through the normalized model configuration."""
|
||||
if od_config is not None and getattr(od_config, "tf_model_config", None) is not None:
|
||||
try:
|
||||
cfg = _build_mammoth_config(od_config)
|
||||
llm_cfg = getattr(cfg, "llm_config", None)
|
||||
if llm_cfg is not None:
|
||||
val = getattr(llm_cfg, "gen_vocab_start_index", None)
|
||||
if val is not None:
|
||||
return int(val)
|
||||
text_cfg = getattr(llm_cfg, "text_config", None)
|
||||
if text_cfg is not None and getattr(text_cfg, "gen_vocab_start_index", None) is not None:
|
||||
return int(text_cfg.gen_vocab_start_index)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fallback inspection of raw dictionary if _build_mammoth_config fails (e.g. mock test objects)
|
||||
raw_cfg = od_config.tf_model_config.to_dict()
|
||||
if isinstance(raw_cfg, dict):
|
||||
llm_cfg = raw_cfg.get("llm_config")
|
||||
if isinstance(llm_cfg, dict):
|
||||
text_cfg = llm_cfg.get("text_config")
|
||||
if isinstance(text_cfg, dict) and text_cfg.get("gen_vocab_start_index") is not None:
|
||||
return int(text_cfg["gen_vocab_start_index"])
|
||||
if llm_cfg.get("gen_vocab_start_index") is not None:
|
||||
return int(llm_cfg["gen_vocab_start_index"])
|
||||
return 152064
|
||||
|
||||
|
||||
def _validate_request_for_admission(
|
||||
request: OmniDiffusionRequest,
|
||||
od_config: OmniDiffusionConfig | None = None,
|
||||
gen_vocab_start_index: int | None = None,
|
||||
) -> tuple[int, int, int]:
|
||||
"""Validate request at admission time to fail-fast before entering scheduler queue.
|
||||
|
||||
Returns (height, width, num_inference_steps).
|
||||
"""
|
||||
request_id = request.request_id
|
||||
prompt = request.prompt if isinstance(request.prompt, dict) else {}
|
||||
sampling = request.sampling_params
|
||||
if sampling is not None and getattr(sampling, "num_outputs_per_prompt", 1) != 1:
|
||||
raise ValueError(
|
||||
f"MammothModa2 requires num_outputs_per_prompt == 1, got {sampling.num_outputs_per_prompt} "
|
||||
f"for request {request_id}"
|
||||
)
|
||||
if not request.is_dummy_run():
|
||||
info = prompt.get("additional_information")
|
||||
if not isinstance(info, dict):
|
||||
raise ValueError(f"Missing additional_information AR conditions for request {request_id}")
|
||||
full_hidden_states = info.get("full_hidden_states")
|
||||
full_token_ids = info.get("full_token_ids")
|
||||
if not isinstance(full_hidden_states, torch.Tensor) or not isinstance(full_token_ids, list):
|
||||
raise ValueError(f"Expected full_hidden_states tensor and full_token_ids list for request {request_id}")
|
||||
try:
|
||||
answer_start_index = int(info.get("answer_start_index"))
|
||||
except (TypeError, ValueError, OverflowError) as exc:
|
||||
raise ValueError(f"Invalid answer_start_index for request {request_id}") from exc
|
||||
if full_hidden_states.ndim != 2:
|
||||
raise ValueError(f"Expected 2D full_hidden_states for request {request_id}")
|
||||
if full_hidden_states.shape[0] != len(full_token_ids):
|
||||
raise ValueError(f"AR hidden-state/token-count mismatch for request {request_id}")
|
||||
if not 0 <= answer_start_index <= len(full_token_ids):
|
||||
raise ValueError(f"answer_start_index outside token range for request {request_id}")
|
||||
try:
|
||||
int_tokens = [int(token_id) for token_id in full_token_ids]
|
||||
except (TypeError, ValueError, OverflowError) as exc:
|
||||
raise ValueError(f"Invalid full_token_ids for request {request_id}") from exc
|
||||
|
||||
if gen_vocab_start_index is None:
|
||||
gen_vocab_start_index = _resolve_gen_vocab_start_index(od_config)
|
||||
answer_tokens = int_tokens[answer_start_index:]
|
||||
if not any(token_id >= gen_vocab_start_index for token_id in answer_tokens):
|
||||
raise ValueError(
|
||||
f"MammothModa2 AR stage produced no visual-token hidden states for request {request_id}; "
|
||||
f"the DiT stage requires at least one generated visual token. Generated token ids: {answer_tokens[:32]}"
|
||||
)
|
||||
|
||||
height, width, _, _, num_inference_steps = _resolve_request_sampling_and_dims(prompt, sampling, request_id)
|
||||
return height, width, num_inference_steps
|
||||
|
||||
|
||||
def get_mammoth_moda2_pre_process_func(od_config: OmniDiffusionConfig | None = None):
|
||||
"""Admission preprocessor: fail-fast per-request validation and grouping.
|
||||
|
||||
Validates AR conditions, dimensions, and sampling knobs at admission so
|
||||
malformed requests are rejected individually with their request_id before
|
||||
entering the scheduler queue. Admitted requests are assigned a
|
||||
batch_compatibility_key based on output geometry and inference steps.
|
||||
"""
|
||||
gen_vocab_start_index = _resolve_gen_vocab_start_index(od_config)
|
||||
|
||||
def pre_process_func(request: OmniDiffusionRequest) -> OmniDiffusionRequest:
|
||||
height, width, num_inference_steps = _validate_request_for_admission(
|
||||
request, od_config=od_config, gen_vocab_start_index=gen_vocab_start_index
|
||||
)
|
||||
request.batch_compatibility_key = (
|
||||
"mammoth_moda2_dit",
|
||||
height,
|
||||
width,
|
||||
num_inference_steps,
|
||||
)
|
||||
return request
|
||||
|
||||
return pre_process_func
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _MammothRequest:
|
||||
index: int
|
||||
request_id: str
|
||||
full_hidden_states: torch.Tensor
|
||||
full_token_ids: list[int]
|
||||
@@ -88,6 +317,15 @@ class _MammothRequest:
|
||||
num_inference_steps: int
|
||||
seed: int | None
|
||||
generator: torch.Generator | list[torch.Generator] | None
|
||||
generator_device: torch.device | str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _RequestConditioning:
|
||||
text_embeds: torch.Tensor
|
||||
text_mask: torch.Tensor
|
||||
image_embeds: torch.Tensor
|
||||
image_mask: torch.Tensor
|
||||
|
||||
|
||||
class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
|
||||
@@ -103,7 +341,7 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
|
||||
_encoder_modules: ClassVar[list[str]] = ["gen_image_condition_refiner"]
|
||||
_vae_modules: ClassVar[list[str]] = ["gen_vae"]
|
||||
|
||||
supports_request_batch = False
|
||||
supports_request_batch = True
|
||||
supports_step_execution = False
|
||||
|
||||
# Load only gen_* weights; ignore llm_model.* to prevent loading the entire LLM backbone in the DiT stage.
|
||||
@@ -234,20 +472,42 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
|
||||
nn.Linear(in_features, out_features, bias=True),
|
||||
)
|
||||
|
||||
def _parse_request(self, req: DiffusionRequestBatch) -> _MammothRequest:
|
||||
if req.num_reqs != 1:
|
||||
raise ValueError("MammothModa2 diffusion requires exactly one request")
|
||||
request_id = req.request_id
|
||||
sampling = req.sampling_params
|
||||
if sampling.num_outputs_per_prompt != 1:
|
||||
raise ValueError(f"MammothModa2 requires num_outputs_per_prompt=1 for request {request_id}")
|
||||
@staticmethod
|
||||
def _group_requests(specs: list[_MammothRequest]) -> list[list[int]]:
|
||||
"""Group co-scheduled requests by (height, width, num_inference_steps).
|
||||
|
||||
prompt = req.prompts[0]
|
||||
prompt = prompt if isinstance(prompt, dict) else {}
|
||||
Preserves arrival order: the first-seen request of each bucket
|
||||
determines the group's execution order.
|
||||
"""
|
||||
buckets: dict[tuple[int, int, int], list[int]] = {}
|
||||
for idx, spec in enumerate(specs):
|
||||
key = (spec.height, spec.width, spec.num_inference_steps)
|
||||
buckets.setdefault(key, []).append(idx)
|
||||
return list(buckets.values())
|
||||
|
||||
def _parse_request(self, req: DiffusionRequestBatch, index: int = 0) -> _MammothRequest:
|
||||
request = req.requests[index]
|
||||
request_id = request.request_id
|
||||
prompt = request.prompt if isinstance(request.prompt, dict) else {}
|
||||
sampling = request.sampling_params
|
||||
if sampling is not None and getattr(sampling, "num_outputs_per_prompt", 1) != 1:
|
||||
raise ValueError(
|
||||
f"MammothModa2 requires num_outputs_per_prompt == 1, got {sampling.num_outputs_per_prompt} "
|
||||
f"for request {request_id}"
|
||||
)
|
||||
info = prompt.get("additional_information")
|
||||
if req.is_dummy_run():
|
||||
full_hidden_states = torch.zeros((2, self._llm_hidden_size), dtype=torch.float32, device="cpu")
|
||||
full_token_ids = [0, int(self.config.llm_config.gen_vocab_start_index)]
|
||||
if request.is_dummy_run():
|
||||
gen_start = 152064
|
||||
if hasattr(self, "config") and hasattr(self.config, "llm_config") and self.config.llm_config is not None:
|
||||
val = getattr(self.config.llm_config, "gen_vocab_start_index", None)
|
||||
if val is not None:
|
||||
gen_start = val
|
||||
else:
|
||||
text_cfg = getattr(self.config.llm_config, "text_config", None)
|
||||
if text_cfg is not None and getattr(text_cfg, "gen_vocab_start_index", None) is not None:
|
||||
gen_start = text_cfg.gen_vocab_start_index
|
||||
full_hidden_states = torch.zeros((2, self._llm_hidden_size), dtype=torch.float32)
|
||||
full_token_ids = [0, int(gen_start)]
|
||||
answer_start_index = 1
|
||||
else:
|
||||
if not isinstance(info, dict):
|
||||
@@ -271,70 +531,18 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
|
||||
except (TypeError, ValueError, OverflowError) as exc:
|
||||
raise ValueError(f"Invalid full_token_ids for request {request_id}") from exc
|
||||
|
||||
dimensions = []
|
||||
for name in ("height", "width"):
|
||||
value = DiffusionRequestBatch.get_prompt_field(prompt, name)
|
||||
if value is None:
|
||||
value = getattr(sampling, name)
|
||||
if value is None:
|
||||
value = 1024
|
||||
try:
|
||||
dimensions.append(int(value))
|
||||
except (TypeError, ValueError, OverflowError) as exc:
|
||||
raise ValueError(f"Invalid image size {name}={value!r} for request {request_id}") from exc
|
||||
height, width = dimensions
|
||||
if height <= 0 or width <= 0:
|
||||
raise ValueError(f"Invalid image size: {height}x{width} for request {request_id}")
|
||||
if height % 16 != 0 or width % 16 != 0:
|
||||
raise ValueError(f"Image size must be multiples of 16, got {height}x{width} for request {request_id}")
|
||||
height, width, text_guidance_scale, cfg_range, num_inference_steps = _resolve_request_sampling_and_dims(
|
||||
prompt, sampling, request_id
|
||||
)
|
||||
|
||||
request_info = info if isinstance(info, dict) else {}
|
||||
extra_args = sampling.extra_args or {}
|
||||
guidance = extra_args.get("text_guidance_scale")
|
||||
if guidance is None:
|
||||
guidance = sampling.guidance_scale if sampling.guidance_scale_provided else None
|
||||
if guidance is None:
|
||||
guidance = _first_request_value(request_info.get("text_guidance_scale"))
|
||||
if guidance is None:
|
||||
guidance = 9.0
|
||||
try:
|
||||
text_guidance_scale = float(guidance)
|
||||
except (TypeError, ValueError, OverflowError) as exc:
|
||||
raise ValueError(f"Invalid text_guidance_scale for request {request_id}") from exc
|
||||
raw_num_inference_steps = extra_args.get("num_inference_steps")
|
||||
if raw_num_inference_steps is None:
|
||||
raw_num_inference_steps = sampling.num_inference_steps
|
||||
if raw_num_inference_steps is None:
|
||||
raw_num_inference_steps = _first_request_value(request_info.get("num_inference_steps"))
|
||||
if raw_num_inference_steps is None:
|
||||
raw_num_inference_steps = 50
|
||||
try:
|
||||
num_inference_steps = int(raw_num_inference_steps)
|
||||
except (TypeError, ValueError, OverflowError) as exc:
|
||||
raise ValueError(f"Invalid num_inference_steps for request {request_id}") from exc
|
||||
if num_inference_steps <= 0:
|
||||
raise ValueError(f"num_inference_steps must be positive for request {request_id}")
|
||||
cfg_range = extra_args.get("cfg_range")
|
||||
if cfg_range is None:
|
||||
cfg_range = request_info.get("cfg_range")
|
||||
if cfg_range is None:
|
||||
cfg_range = [0.0, 1.0]
|
||||
if not isinstance(cfg_range, (list, tuple)) or len(cfg_range) != 2:
|
||||
raise ValueError(f"cfg_range requires two values for request {request_id}")
|
||||
try:
|
||||
cfg_start, cfg_end = float(cfg_range[0]), float(cfg_range[1])
|
||||
except (TypeError, ValueError, OverflowError) as exc:
|
||||
raise ValueError(f"cfg_range requires two values convertible to floats for request {request_id}") from exc
|
||||
if not 0 <= cfg_start <= cfg_end <= 1:
|
||||
raise ValueError(f"cfg_range must satisfy 0 <= start <= end <= 1 for request {request_id}")
|
||||
|
||||
generator = sampling.generator
|
||||
generator = sampling.generator if sampling else None
|
||||
if isinstance(generator, list) and len(generator) != 1:
|
||||
raise ValueError(
|
||||
f"MammothModa2 single-output request mode requires exactly one generator for request {request_id}"
|
||||
)
|
||||
|
||||
return _MammothRequest(
|
||||
index=index,
|
||||
request_id=request_id,
|
||||
full_hidden_states=full_hidden_states,
|
||||
full_token_ids=full_token_ids,
|
||||
@@ -342,10 +550,11 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
|
||||
height=height,
|
||||
width=width,
|
||||
text_guidance_scale=text_guidance_scale,
|
||||
cfg_range=(cfg_start, cfg_end),
|
||||
cfg_range=cfg_range,
|
||||
num_inference_steps=num_inference_steps,
|
||||
seed=sampling.seed,
|
||||
seed=sampling.seed if sampling else None,
|
||||
generator=generator,
|
||||
generator_device=getattr(sampling, "generator_device", None) if sampling else None,
|
||||
)
|
||||
|
||||
def _split_ar_conditions(
|
||||
@@ -363,13 +572,23 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
|
||||
caller no longer needs to pass them. Mirrors the masking the bespoke
|
||||
MammothModa2 example performed via ar2dit.
|
||||
"""
|
||||
gen_vocab_start_index = int(self.config.llm_config.gen_vocab_start_index)
|
||||
visual_ids = [
|
||||
int(self.config.image_token_id),
|
||||
int(self.config.video_token_id),
|
||||
int(self.config.vision_start_token_id),
|
||||
int(self.config.vision_end_token_id),
|
||||
]
|
||||
gen_vocab_start_index = getattr(self.config.llm_config, "gen_vocab_start_index", None)
|
||||
if gen_vocab_start_index is None:
|
||||
text_cfg = getattr(self.config.llm_config, "text_config", None)
|
||||
gen_vocab_start_index = getattr(text_cfg, "gen_vocab_start_index", 152064)
|
||||
gen_vocab_start_index = int(gen_vocab_start_index)
|
||||
cached_visual_ids = getattr(self, "_cached_visual_ids_tensor", None)
|
||||
if cached_visual_ids is None:
|
||||
cached_visual_ids = torch.tensor(
|
||||
[
|
||||
int(self.config.image_token_id),
|
||||
int(self.config.video_token_id),
|
||||
int(self.config.vision_start_token_id),
|
||||
int(self.config.vision_end_token_id),
|
||||
],
|
||||
dtype=torch.long,
|
||||
)
|
||||
self._cached_visual_ids_tensor = cached_visual_ids
|
||||
|
||||
device = full_hidden_states.device
|
||||
token_ids = torch.tensor(full_token_ids, dtype=torch.long, device=device)
|
||||
@@ -377,7 +596,8 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
|
||||
questions_mask = positions < answer_start_index
|
||||
answers_mask = ~questions_mask
|
||||
gen_token_mask = token_ids >= gen_vocab_start_index
|
||||
visual_token_mask = torch.isin(token_ids, torch.tensor(visual_ids, dtype=torch.long, device=device))
|
||||
visual_ids_device = cached_visual_ids.to(device=device, non_blocking=True)
|
||||
visual_token_mask = torch.isin(token_ids, visual_ids_device)
|
||||
text_mask = questions_mask & ~(visual_token_mask | gen_token_mask)
|
||||
image_mask = answers_mask & gen_token_mask
|
||||
|
||||
@@ -389,9 +609,8 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
|
||||
image_cond = full_hidden_states[image_mask].contiguous()
|
||||
return text_cond, image_cond
|
||||
|
||||
@torch.inference_mode()
|
||||
def forward(self, req: DiffusionRequestBatch) -> DiffusionOutput:
|
||||
request = self._parse_request(req)
|
||||
def _split_request_conditions(self, request: _MammothRequest) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Split AR hidden states and fail fast before touching model params."""
|
||||
text_cond, image_cond = self._split_ar_conditions(
|
||||
full_hidden_states=request.full_hidden_states,
|
||||
full_token_ids=request.full_token_ids,
|
||||
@@ -404,104 +623,181 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
|
||||
f"request {request.request_id}; the DiT stage requires at least one generated visual token. "
|
||||
f"Generated token ids: {answer_token_ids[:32]}"
|
||||
)
|
||||
return text_cond, image_cond
|
||||
|
||||
# Move to model device/dtype.
|
||||
model_device = next(self.parameters()).device
|
||||
def _build_conditioning(
|
||||
self,
|
||||
text_cond: torch.Tensor,
|
||||
image_cond: torch.Tensor,
|
||||
model_device: torch.device,
|
||||
target_dtype: torch.dtype,
|
||||
) -> _RequestConditioning:
|
||||
text_embeds = text_cond.to(device=model_device, dtype=target_dtype, non_blocking=True).contiguous().unsqueeze(0)
|
||||
text_mask = torch.ones((1, text_embeds.shape[1]), dtype=torch.bool, device=model_device)
|
||||
image_embeds = (
|
||||
image_cond.to(device=model_device, dtype=target_dtype, non_blocking=True).contiguous().unsqueeze(0)
|
||||
)
|
||||
image_mask = torch.ones((1, image_embeds.shape[1]), dtype=torch.bool, device=model_device)
|
||||
return _RequestConditioning(
|
||||
text_embeds=text_embeds,
|
||||
text_mask=text_mask,
|
||||
image_embeds=image_embeds,
|
||||
image_mask=image_mask,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _make_latent_generators(
|
||||
specs: list[_MammothRequest],
|
||||
fallback_device: torch.device,
|
||||
) -> list[torch.Generator] | None:
|
||||
generators: list[torch.Generator] = []
|
||||
for s in specs:
|
||||
if s.generator is not None:
|
||||
gen = s.generator[0] if isinstance(s.generator, list) else s.generator
|
||||
elif s.seed is not None:
|
||||
device = s.generator_device or fallback_device
|
||||
gen = torch.Generator(device=device).manual_seed(s.seed)
|
||||
else:
|
||||
return None
|
||||
generators.append(gen)
|
||||
return generators
|
||||
|
||||
def _denoise_group(
|
||||
self,
|
||||
specs: list[_MammothRequest],
|
||||
conds: list[_RequestConditioning],
|
||||
model_device: torch.device,
|
||||
) -> list[torch.Tensor]:
|
||||
batch = len(specs)
|
||||
height = specs[0].height
|
||||
width = specs[0].width
|
||||
num_inference_steps = specs[0].num_inference_steps
|
||||
target_dtype = conds[0].text_embeds.dtype
|
||||
|
||||
# Conditioning collation. Token order is text-then-image per request,
|
||||
# so concatenate first and right-pad the combined sequence.
|
||||
nested_image_embedder = getattr(self.gen_transformer.time_caption_embed, "image_embedder", None)
|
||||
if self.gen_image_condition_refiner is not None:
|
||||
target_dtype = next(self.gen_image_condition_refiner.parameters()).dtype
|
||||
else:
|
||||
target_dtype = next(self.gen_transformer.parameters()).dtype
|
||||
|
||||
text_cond = text_cond.to(device=model_device, dtype=target_dtype, non_blocking=True).contiguous()
|
||||
image_cond = image_cond.to(device=model_device, dtype=target_dtype, non_blocking=True).contiguous()
|
||||
|
||||
text_embeds = text_cond.unsqueeze(0) # [1, T_text, H]
|
||||
text_attention_mask = torch.ones(
|
||||
(1, text_embeds.shape[1]),
|
||||
dtype=torch.bool,
|
||||
device=text_embeds.device,
|
||||
)
|
||||
|
||||
image_embeds = image_cond.unsqueeze(0) # [1, T_img, H]
|
||||
image_attention_mask = torch.ones(
|
||||
(1, image_embeds.shape[1]),
|
||||
dtype=torch.bool,
|
||||
device=image_embeds.device,
|
||||
)
|
||||
|
||||
# Apply optional refiner ONLY on image condition tokens.
|
||||
if self.gen_image_condition_refiner is not None and image_embeds.shape[1] > 0:
|
||||
image_embeds = self.gen_image_condition_refiner(image_embeds, ~image_attention_mask.bool())
|
||||
image_attention_mask = torch.ones(
|
||||
image_embeds, image_mask = _pad_cond_sequence(
|
||||
[c.image_embeds for c in conds],
|
||||
[c.image_mask for c in conds],
|
||||
)
|
||||
# Apply optional refiner ONLY on image condition tokens.
|
||||
if image_embeds.shape[1] > 0:
|
||||
image_embeds = self.gen_image_condition_refiner(image_embeds, ~image_mask.bool())
|
||||
refined_image_mask = torch.ones(
|
||||
image_embeds.shape[:2],
|
||||
dtype=torch.bool,
|
||||
device=image_embeds.device,
|
||||
)
|
||||
|
||||
nested_image_embedder = getattr(self.gen_transformer.time_caption_embed, "image_embedder", None)
|
||||
if nested_image_embedder is None:
|
||||
prompt_embeds = torch.cat([text_embeds, image_embeds], dim=1)
|
||||
prompt_attention_mask = torch.cat([text_attention_mask, image_attention_mask], dim=1)
|
||||
seq_embeds = []
|
||||
seq_masks = []
|
||||
for i, c in enumerate(conds):
|
||||
cur_img_embed = image_embeds[i : i + 1]
|
||||
cur_img_mask = refined_image_mask[i : i + 1]
|
||||
seq_embeds.append(torch.cat([c.text_embeds, cur_img_embed], dim=1))
|
||||
seq_masks.append(torch.cat([c.text_mask, cur_img_mask], dim=1))
|
||||
prompt_embeds, prompt_attention_mask = _pad_cond_sequence(seq_embeds, seq_masks)
|
||||
ar_image_embeds = None
|
||||
ar_image_attention_mask = None
|
||||
elif nested_image_embedder is not None:
|
||||
prompt_embeds, prompt_attention_mask = _pad_cond_sequence(
|
||||
[c.text_embeds for c in conds],
|
||||
[c.text_mask for c in conds],
|
||||
)
|
||||
ar_image_embeds, ar_image_attention_mask = _pad_cond_sequence(
|
||||
[c.image_embeds for c in conds],
|
||||
[c.image_mask for c in conds],
|
||||
)
|
||||
else:
|
||||
prompt_embeds = text_embeds
|
||||
prompt_attention_mask = text_attention_mask
|
||||
ar_image_embeds = image_embeds
|
||||
ar_image_attention_mask = image_attention_mask
|
||||
prompt_embeds, prompt_attention_mask = _pad_cond_sequence(
|
||||
[torch.cat([c.text_embeds, c.image_embeds], dim=1) for c in conds],
|
||||
[torch.cat([c.text_mask, c.image_mask], dim=1) for c in conds],
|
||||
)
|
||||
ar_image_embeds = None
|
||||
ar_image_attention_mask = None
|
||||
|
||||
# Empty unconditional prompt for classifier-free guidance.
|
||||
# Empty unconditional prompt for classifier-free guidance, shared by
|
||||
# all rows; rows that never take the CFG branch select their cond
|
||||
# prediction directly via torch.where below.
|
||||
needs_uncond = any(s.text_guidance_scale > 1.0 for s in specs)
|
||||
negative_prompt_embeds = None
|
||||
negative_prompt_attention_mask = None
|
||||
if request.text_guidance_scale > 1.0:
|
||||
if needs_uncond:
|
||||
hidden_size = int(prompt_embeds.shape[-1])
|
||||
negative_prompt_embeds = torch.zeros(
|
||||
(1, 0, hidden_size),
|
||||
(batch, 0, hidden_size),
|
||||
dtype=target_dtype,
|
||||
device=prompt_embeds.device,
|
||||
device=model_device,
|
||||
)
|
||||
negative_prompt_attention_mask = torch.zeros(
|
||||
(1, 0),
|
||||
(batch, 0),
|
||||
dtype=torch.bool,
|
||||
device=prompt_embeds.device,
|
||||
device=model_device,
|
||||
)
|
||||
|
||||
generator = request.generator
|
||||
if generator is None and request.seed is not None:
|
||||
generator = torch.Generator(device=model_device).manual_seed(request.seed)
|
||||
|
||||
# Output image size (px), passed from stage input processor.
|
||||
height, width = request.height, request.width
|
||||
vae_scale_factor = 16
|
||||
|
||||
latent_channels = int(self.gen_transformer.config.in_channels)
|
||||
shape = (1, latent_channels, 2 * height // vae_scale_factor, 2 * width // vae_scale_factor)
|
||||
latents = randn_tensor(shape, generator=generator, device=prompt_embeds.device, dtype=prompt_embeds.dtype)
|
||||
shape = (batch, latent_channels, 2 * height // vae_scale_factor, 2 * width // vae_scale_factor)
|
||||
single_shape = (1, latent_channels, 2 * height // vae_scale_factor, 2 * width // vae_scale_factor)
|
||||
generators = self._make_latent_generators(specs, model_device)
|
||||
if generators is not None:
|
||||
noise_list = []
|
||||
for gen in generators:
|
||||
gen_device = gen.device if hasattr(gen, "device") else model_device
|
||||
n = randn_tensor(single_shape, generator=gen, device=gen_device, dtype=target_dtype)
|
||||
noise_list.append(n.to(device=model_device))
|
||||
latents = torch.cat(noise_list, dim=0)
|
||||
else:
|
||||
latents = randn_tensor(shape, device=model_device, dtype=target_dtype)
|
||||
|
||||
scheduler = FlowMatchEulerDiscreteScheduler()
|
||||
|
||||
scheduler.set_timesteps(
|
||||
num_inference_steps=request.num_inference_steps,
|
||||
device=prompt_embeds.device,
|
||||
num_inference_steps=num_inference_steps,
|
||||
device=model_device,
|
||||
num_tokens=latents.shape[-2] * latents.shape[-1],
|
||||
)
|
||||
|
||||
# Run diffusion loop (CFG supported when text_guidance_scale > 1.0)
|
||||
scale_vec = torch.tensor(
|
||||
[s.text_guidance_scale for s in specs],
|
||||
device=model_device,
|
||||
dtype=target_dtype,
|
||||
).view(batch, 1, 1, 1)
|
||||
cfg_specs = [(s.text_guidance_scale > 1.0, float(s.cfg_range[0]), float(s.cfg_range[1])) for s in specs]
|
||||
|
||||
total_steps = max(1, len(scheduler.timesteps))
|
||||
any_cfg_active = any(s.text_guidance_scale > 1.0 for s in specs)
|
||||
self._prepare_cache_dit_for_request(
|
||||
num_inference_steps=total_steps,
|
||||
cfg_active=request.text_guidance_scale > 1.0,
|
||||
cfg_active=any_cfg_active,
|
||||
)
|
||||
# cache-dit distinguishes the conditional vs unconditional passes
|
||||
# purely by transformer-forward parity, so when it is active the
|
||||
# uncond pass must run on every step. Outside cfg_range CFG is
|
||||
# neutralized via scale=1.0 (uncond + 1.0 * (cond - uncond) == cond),
|
||||
# matching the skip optimization's cond-only update.
|
||||
requires_paired_cfg = self._cache_dit_runtime.is_enabled and getattr(
|
||||
self, "_cache_dit_requires_paired_cfg", False
|
||||
)
|
||||
|
||||
# Precompute mixed-CFG active masks across timesteps to avoid CPU list
|
||||
# comprehensions and H2D copies in the step loop.
|
||||
precomputed_active_masks = None
|
||||
all_active_per_step = None
|
||||
any_active_per_step = None
|
||||
if needs_uncond:
|
||||
precomputed_active_masks = torch.empty((total_steps, batch, 1, 1, 1), device=model_device, dtype=torch.bool)
|
||||
all_active_per_step = []
|
||||
any_active_per_step = []
|
||||
for step_idx in range(total_steps):
|
||||
frac = step_idx / total_steps
|
||||
step_mask = [is_active and (lo <= frac <= hi) for is_active, lo, hi in cfg_specs]
|
||||
all_active_per_step.append(all(step_mask))
|
||||
any_active_per_step.append(any(step_mask))
|
||||
mask_t = torch.tensor(step_mask, device=model_device, dtype=torch.bool)
|
||||
precomputed_active_masks[step_idx] = mask_t.view(batch, 1, 1, 1)
|
||||
|
||||
# Precompute timesteps in latent dtype to eliminate per-step cast overhead
|
||||
timesteps_dtype = scheduler.timesteps.to(latents.dtype)
|
||||
|
||||
# Run diffusion loop (CFG supported when text_guidance_scale > 1.0)
|
||||
for i, t in enumerate(scheduler.timesteps):
|
||||
timestep = t.expand(latents.shape[0]).to(latents.dtype)
|
||||
timestep = timesteps_dtype[i].expand(batch)
|
||||
model_pred = self.gen_transformer(
|
||||
hidden_states=latents,
|
||||
timestep=timestep,
|
||||
@@ -512,10 +808,7 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
|
||||
ar_image_attention_mask=ar_image_attention_mask,
|
||||
freqs_cis=self.gen_freqs_cis,
|
||||
)
|
||||
guidance_scale = (
|
||||
request.text_guidance_scale if request.cfg_range[0] <= i / total_steps <= request.cfg_range[1] else 1.0
|
||||
)
|
||||
run_uncond = guidance_scale > 1.0 and negative_prompt_embeds is not None
|
||||
run_uncond = (any_active_per_step is not None and any_active_per_step[i]) if needs_uncond else False
|
||||
if requires_paired_cfg and negative_prompt_embeds is not None:
|
||||
run_uncond = True
|
||||
if run_uncond:
|
||||
@@ -527,18 +820,65 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
|
||||
ref_image_hidden_states=None,
|
||||
freqs_cis=self.gen_freqs_cis,
|
||||
)
|
||||
model_pred = model_pred_uncond + guidance_scale * (model_pred - model_pred_uncond)
|
||||
# Fused CFG blend: torch.lerp fuses (uncond + scale * (cond - uncond)) into 1 kernel
|
||||
blended = torch.lerp(model_pred_uncond, model_pred, scale_vec)
|
||||
if all_active_per_step is not None and all_active_per_step[i]:
|
||||
# Fast path: all requests in the batch are active.
|
||||
model_pred = blended
|
||||
else:
|
||||
# Inactive rows keep their conditional prediction exactly:
|
||||
# gating the blend with torch.where avoids propagating
|
||||
# uncond-branch NaN/rounding into CFG-free rows.
|
||||
active_tensor = precomputed_active_masks[i]
|
||||
model_pred = torch.where(active_tensor, blended, model_pred)
|
||||
latents = scheduler.step(model_pred, t, latents, return_dict=False)[0]
|
||||
latents = latents.to(dtype=prompt_embeds.dtype)
|
||||
latents = latents.to(dtype=target_dtype)
|
||||
|
||||
# VAE decode
|
||||
# VAE decode (in-place scaling to reduce peak VRAM before decode)
|
||||
if self.gen_vae.config.scaling_factor is not None:
|
||||
latents = latents / self.gen_vae.config.scaling_factor
|
||||
latents.div_(self.gen_vae.config.scaling_factor)
|
||||
if self.gen_vae.config.shift_factor is not None:
|
||||
latents = latents + self.gen_vae.config.shift_factor
|
||||
image = self.gen_vae.decode(latents, return_dict=False)[0]
|
||||
latents.add_(self.gen_vae.config.shift_factor)
|
||||
vae_dtype = next(self.gen_vae.parameters()).dtype
|
||||
image = self.gen_vae.decode(latents.to(dtype=vae_dtype), return_dict=False)[0] # [B, C, H, W]
|
||||
return [image[i : i + 1] for i in range(batch)]
|
||||
|
||||
return DiffusionOutput(output=image)
|
||||
@torch.inference_mode()
|
||||
def forward(self, req: DiffusionRequestBatch) -> list[DiffusionOutput]:
|
||||
specs = [self._parse_request(req, i) for i in range(req.num_reqs)]
|
||||
|
||||
# Validate AR conditions before touching model parameters so malformed
|
||||
# requests fail fast instead of surfacing device/dtype errors.
|
||||
raw_conds = [self._split_request_conditions(spec) for spec in specs]
|
||||
|
||||
model_device = next(self.parameters()).device
|
||||
if self.gen_image_condition_refiner is not None:
|
||||
target_dtype = next(self.gen_image_condition_refiner.parameters()).dtype
|
||||
else:
|
||||
target_dtype = next(self.gen_transformer.parameters()).dtype
|
||||
|
||||
conds = [
|
||||
self._build_conditioning(text_cond, image_cond, model_device, target_dtype)
|
||||
for text_cond, image_cond in raw_conds
|
||||
]
|
||||
|
||||
groups = self._group_requests(specs)
|
||||
if len(groups) == 1 and len(groups[0]) == len(specs):
|
||||
images = self._denoise_group(specs, conds, model_device)
|
||||
return [DiffusionOutput(output=img) for img in images]
|
||||
|
||||
outputs: list[DiffusionOutput | None] = [None] * len(specs)
|
||||
for group in groups:
|
||||
images = self._denoise_group(
|
||||
[specs[i] for i in group],
|
||||
[conds[i] for i in group],
|
||||
model_device,
|
||||
)
|
||||
for index, image in zip(group, images):
|
||||
outputs[index] = DiffusionOutput(output=image)
|
||||
if any(output is None for output in outputs):
|
||||
raise RuntimeError("DiT batching produced no image for at least one scheduled request")
|
||||
return [output for output in outputs if output is not None]
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
|
||||
Reference in new issue
Block a user