[Model] Add Tencent AuK speech generation and editing (encoder + diffusion pipeline) (#7385)
Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: Sy03 <1370724210@qq.com>
This commit is contained in:
1 parent
d3493384cf
commit
c111f3a2d6
28 files changed
+4986
No files matched your search
@@ -0,0 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
from vllm_omni.model_executor.models.auk.auk import AuKForConditionalGeneration
|
||||
|
||||
__all__ = ["AuKForConditionalGeneration"]
|
||||
@@ -0,0 +1,304 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""AuK (Tencent): instruction-driven speech generation and editing.
|
||||
|
||||
Stage 0 (``model_stage="encoder"``) runs the frozen Qwen2.5-Omni thinker as a
|
||||
prefill-only encoder. The outputs of all decoder layers are fused with learned
|
||||
softmax weights into one ``[tokens, hidden]`` text condition, emitted through
|
||||
``multimodal_outputs["hidden_states"]["output"]`` for the diffusion stage
|
||||
(``vllm_omni/diffusion/models/auk``). The stage samples exactly one token
|
||||
(``max_tokens=1``, forced to EOS); that token is discarded downstream.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Iterable, Mapping
|
||||
from functools import cached_property
|
||||
from itertools import islice
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from safetensors import safe_open
|
||||
from transformers import PretrainedConfig
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import SupportsMRoPE, SupportsMultiModal, SupportsPP
|
||||
from vllm.model_executor.models.utils import init_vllm_registered_model, maybe_prefix
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.sample.sampler import Sampler
|
||||
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni import Qwen2_5OmniForConditionalGeneration
|
||||
from vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker import (
|
||||
Qwen2_5OmniThinkerDummyInputsBuilder,
|
||||
Qwen2_5OmniThinkerForConditionalGeneration,
|
||||
Qwen2_5OmniThinkerMultiModalProcessor,
|
||||
Qwen2_5OmniThinkerProcessingInfo,
|
||||
)
|
||||
from vllm_omni.model_executor.models.utils import add_prefix_to_loaded_weights
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# File next to the thinker shards in the assembled checkpoint directory that
|
||||
# carries the DiT (``transformer.*``) and the two layer-fusion tensors.
|
||||
AUK_WEIGHTS_FILE = "auk.safetensors"
|
||||
LAYER_WEIGHTS_KEY = "layer_weights"
|
||||
LAYER_SCALE_KEY = "layer_scale"
|
||||
# The fused text condition travels as ``multimodal_outputs["hidden_states"]["output"]``
|
||||
# (``HiddenStatesStruct.output``), the same stage-wire slot MiniMax H3 uses.
|
||||
TEXT_COND_KEY = "output"
|
||||
# ``F.layer_norm`` epsilon used by the upstream fusion (torch default).
|
||||
_FUSION_LN_EPS = 1e-5
|
||||
|
||||
|
||||
class AuKProcessingInfo(Qwen2_5OmniThinkerProcessingInfo):
|
||||
"""Qwen2.5-Omni thinker processing restricted to AuK's single audio input.
|
||||
|
||||
AuK conditions on text plus at most one reference or source clip; images
|
||||
and video are not part of its contract, so they are not advertised (and the
|
||||
thinker then builds no vision tower).
|
||||
"""
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
return {"audio": 1}
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
Qwen2_5OmniThinkerMultiModalProcessor,
|
||||
info=AuKProcessingInfo,
|
||||
dummy_inputs=Qwen2_5OmniThinkerDummyInputsBuilder,
|
||||
)
|
||||
class AuKForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP, SupportsMRoPE):
|
||||
"""Encoder stage of AuK: Qwen2.5-Omni thinker with learned layer fusion.
|
||||
|
||||
The forward pass re-implements the thinker language model's layer loop so
|
||||
the per-layer outputs can be fused on the fly: vLLM's decoder layers carry
|
||||
the residual stream separately, and ``hidden + residual`` after layer ``i``
|
||||
is the HF ``output_hidden_states[i + 1]`` entry, while the last entry is
|
||||
the final-normed state. Only one accumulator lives at a time.
|
||||
"""
|
||||
|
||||
have_multimodal_outputs = True
|
||||
# The fused condition is the payload; the runner's default last-hidden
|
||||
# copy and prefix-cached hidden states are not needed (MiniMax H3 pattern).
|
||||
omni_pooler_payload_include_hidden = False
|
||||
requires_full_prefix_cached_hidden_states = False
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.has_preprocess = False
|
||||
self.vllm_config = vllm_config
|
||||
config = vllm_config.model_config.hf_config
|
||||
|
||||
self.model_stage = getattr(vllm_config.model_config, "model_stage", None) or "encoder"
|
||||
if self.model_stage != "encoder":
|
||||
raise ValueError(
|
||||
f"AuKForConditionalGeneration only hosts the 'encoder' stage, got model_stage={self.model_stage!r}; "
|
||||
"the DiT stage is the AuKPipeline diffusion model."
|
||||
)
|
||||
|
||||
thinker_config = config.thinker_config
|
||||
self.thinker_config = thinker_config
|
||||
self.thinker = init_vllm_registered_model(
|
||||
vllm_config=vllm_config,
|
||||
prefix=maybe_prefix(prefix, "thinker"),
|
||||
hf_config=thinker_config,
|
||||
architectures=["Qwen2_5OmniThinkerModel"],
|
||||
)
|
||||
# The audio-only mm limits keep the thinker from building its vision
|
||||
# tower; drop it explicitly as well, as the upstream implementation does,
|
||||
# so no vision weights are loaded or resident.
|
||||
if getattr(self.thinker, "visual", None) is not None:
|
||||
del self.thinker.visual
|
||||
self.thinker.visual = None
|
||||
self.model = self.thinker
|
||||
self.make_empty_intermediate_tensors = self.thinker.make_empty_intermediate_tensors
|
||||
|
||||
num_layers = thinker_config.text_config.num_hidden_layers
|
||||
# Filled by load_weights() from AUK_WEIGHTS_FILE; buffers so the vLLM
|
||||
# loader does not expect them among the thinker shards.
|
||||
self.register_buffer("layer_weights", torch.zeros(num_layers, dtype=torch.float32))
|
||||
self.register_buffer("layer_scale", torch.ones(1, dtype=torch.float32))
|
||||
self._fusion_loaded = False
|
||||
|
||||
# -------------------- delegations to the thinker --------------------
|
||||
|
||||
@classmethod
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
||||
return Qwen2_5OmniThinkerForConditionalGeneration.get_placeholder_str(modality, i)
|
||||
|
||||
def get_language_model(self) -> nn.Module:
|
||||
return self.thinker.get_language_model()
|
||||
|
||||
@cached_property
|
||||
def sampler(self):
|
||||
if hasattr(self.thinker, "sampler"):
|
||||
return self.thinker.sampler
|
||||
return Sampler()
|
||||
|
||||
def embed_input_ids(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
multimodal_embeddings=None,
|
||||
is_multimodal=None,
|
||||
) -> torch.Tensor:
|
||||
return self.thinker.embed_input_ids(
|
||||
input_ids=input_ids,
|
||||
multimodal_embeddings=multimodal_embeddings,
|
||||
is_multimodal=is_multimodal,
|
||||
)
|
||||
|
||||
def embed_multimodal(self, **kwargs):
|
||||
return self.thinker.embed_multimodal(**kwargs)
|
||||
|
||||
def get_mrope_input_positions(
|
||||
self,
|
||||
input_tokens: list[int],
|
||||
mm_features: list[MultiModalFeatureSpec] | None = None,
|
||||
*,
|
||||
hf_config: PretrainedConfig,
|
||||
image_grid_thw: list[list[int]] | torch.Tensor,
|
||||
video_grid_thw: list[list[int]] | torch.Tensor,
|
||||
second_per_grid_ts: list[float] | None = None,
|
||||
context_len: int = 0,
|
||||
seq_len: int | None = None,
|
||||
audio_feature_lengths: torch.Tensor | None = None,
|
||||
use_audio_in_video: bool = False,
|
||||
) -> tuple[torch.Tensor, int]:
|
||||
# The omni runner filters kwargs by this signature (the extended
|
||||
# Qwen2.5-Omni form); the unified Qwen2.5-Omni class implements the
|
||||
# audio-aware position math without touching instance state.
|
||||
return Qwen2_5OmniForConditionalGeneration.get_mrope_input_positions(
|
||||
self,
|
||||
input_tokens,
|
||||
mm_features,
|
||||
hf_config=hf_config,
|
||||
image_grid_thw=image_grid_thw,
|
||||
video_grid_thw=video_grid_thw,
|
||||
second_per_grid_ts=second_per_grid_ts,
|
||||
context_len=context_len,
|
||||
seq_len=seq_len,
|
||||
audio_feature_lengths=audio_feature_lengths,
|
||||
use_audio_in_video=use_audio_in_video,
|
||||
)
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor | OmniOutput, **kwargs: object) -> torch.Tensor | None:
|
||||
"""Emit EOS immediately; the useful result is the prefill text condition."""
|
||||
if isinstance(hidden_states, OmniOutput):
|
||||
hidden_states = hidden_states.text_hidden_states
|
||||
text_config = self.thinker_config.text_config
|
||||
logits = hidden_states.new_full(
|
||||
(hidden_states.shape[0], int(text_config.vocab_size)),
|
||||
torch.finfo(hidden_states.dtype).min,
|
||||
)
|
||||
logits[:, self._eos_token_id()] = 0
|
||||
return logits
|
||||
|
||||
def _eos_token_id(self) -> int:
|
||||
for cfg in (self.thinker_config.text_config, self.thinker_config, self.vllm_config.model_config.hf_config):
|
||||
eos = getattr(cfg, "eos_token_id", None)
|
||||
if isinstance(eos, (list, tuple)) and eos:
|
||||
eos = eos[0]
|
||||
if isinstance(eos, int):
|
||||
return eos
|
||||
# Qwen2.5-Omni tokenizers end turns with <|im_end|>.
|
||||
return 151645
|
||||
|
||||
# -------------------- forward --------------------
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**kwargs: object,
|
||||
) -> OmniOutput:
|
||||
if intermediate_tensors is not None:
|
||||
raise NotImplementedError("AuK encoder does not support pipeline parallelism")
|
||||
language_model = self.thinker.language_model.model
|
||||
num_layers = len(language_model.layers)
|
||||
if language_model.start_layer != 0 or language_model.end_layer != num_layers:
|
||||
raise NotImplementedError("AuK encoder needs all decoder layers on one rank")
|
||||
|
||||
if inputs_embeds is None:
|
||||
hidden_states = language_model.embed_input_ids(input_ids)
|
||||
else:
|
||||
hidden_states = inputs_embeds
|
||||
residual: torch.Tensor | None = None
|
||||
|
||||
weights = torch.softmax(self.layer_weights, dim=0)
|
||||
fused: torch.Tensor | None = None
|
||||
normed_shape = (hidden_states.shape[-1],)
|
||||
layers = islice(language_model.layers, language_model.start_layer, language_model.end_layer)
|
||||
for idx, layer in enumerate(layers):
|
||||
hidden_states, residual = layer(positions, hidden_states, residual)
|
||||
if idx + 1 < num_layers:
|
||||
layer_output = hidden_states + residual
|
||||
else:
|
||||
# HF's last hidden_states entry is post final norm.
|
||||
hidden_states, _ = language_model.norm(hidden_states, residual)
|
||||
layer_output = hidden_states
|
||||
normed = F.layer_norm(layer_output.float(), normed_shape, eps=_FUSION_LN_EPS)
|
||||
contribution = normed * weights[idx]
|
||||
fused = contribution if fused is None else fused + contribution
|
||||
|
||||
assert fused is not None
|
||||
text_cond = (fused * self.layer_scale).to(hidden_states.dtype)
|
||||
# Stage-wire layout: [tokens, features] under hidden_states.output so
|
||||
# the runner slices it per request (and concatenates across chunked
|
||||
# prefill steps); the stage-1 processor reads the same path.
|
||||
return OmniOutput(
|
||||
text_hidden_states=hidden_states,
|
||||
multimodal_outputs={"hidden_states": {"output": text_cond}},
|
||||
)
|
||||
|
||||
# -------------------- weights --------------------
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
thinker_weights: list[tuple[str, torch.Tensor]] = []
|
||||
skipped = 0
|
||||
for name, tensor in weights:
|
||||
if name.startswith("thinker."):
|
||||
thinker_weights.append((name, tensor))
|
||||
elif name.startswith(("talker.", "token2wav.", "transformer.")):
|
||||
skipped += 1
|
||||
else:
|
||||
logger.warning("AuK encoder: ignoring unexpected weight %s", name)
|
||||
if skipped:
|
||||
logger.debug("AuK encoder: skipped %d non-thinker weights", skipped)
|
||||
|
||||
loaded = self.thinker.load_weights(thinker_weights)
|
||||
loaded = add_prefix_to_loaded_weights(loaded, "thinker")
|
||||
self._load_fusion_params()
|
||||
return loaded
|
||||
|
||||
def _load_fusion_params(self) -> None:
|
||||
model_dir = self.vllm_config.model_config.model
|
||||
path = os.path.join(model_dir, AUK_WEIGHTS_FILE)
|
||||
if not os.path.isfile(path):
|
||||
raise FileNotFoundError(
|
||||
f"AuK encoder: {AUK_WEIGHTS_FILE} not found in {model_dir!r}; assemble the checkpoint with "
|
||||
"tools/prepare_auk_checkpoint.py"
|
||||
)
|
||||
with safe_open(path, framework="pt", device="cpu") as f:
|
||||
layer_weights = f.get_tensor(LAYER_WEIGHTS_KEY).float()
|
||||
layer_scale = f.get_tensor(LAYER_SCALE_KEY).float().reshape(1)
|
||||
if layer_weights.shape != self.layer_weights.shape:
|
||||
raise ValueError(
|
||||
f"AuK encoder: {LAYER_WEIGHTS_KEY} has {tuple(layer_weights.shape)} entries, "
|
||||
f"thinker has {tuple(self.layer_weights.shape)} layers"
|
||||
)
|
||||
self.layer_weights.copy_(layer_weights.to(self.layer_weights.device))
|
||||
self.layer_scale.copy_(layer_scale.to(self.layer_scale.device))
|
||||
self._fusion_loaded = True
|
||||
logger.info(
|
||||
"AuK encoder: loaded layer fusion (scale %.3f, top layer weight %.3f at layer %d)",
|
||||
float(layer_scale[0]),
|
||||
float(torch.softmax(layer_weights, 0).max()),
|
||||
int(torch.softmax(layer_weights, 0).argmax()) + 1,
|
||||
)
|
||||
@@ -0,0 +1,60 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""AuK pipeline topology (frozen).
|
||||
|
||||
Stage 0: Encoder — frozen Qwen2.5-Omni thinker, prefill only, emits the
|
||||
learned layer fusion as ``multimodal_outputs["hidden_states"]["output"]``
|
||||
Stage 1: DiT — rectified-flow transformer plus the BigVGAN-flow VAE,
|
||||
emits a 24 kHz waveform
|
||||
"""
|
||||
|
||||
from vllm_omni.config.stage_config import (
|
||||
PipelineConfig,
|
||||
StageExecutionType,
|
||||
StagePipelineConfig,
|
||||
)
|
||||
|
||||
_PROC = "vllm_omni.model_executor.stage_input_processors.auk"
|
||||
|
||||
AUK_PIPELINE = PipelineConfig(
|
||||
model_type="auk",
|
||||
default_deploy_config_name="auk.yaml",
|
||||
model_arch="AuKForConditionalGeneration",
|
||||
hf_architectures=("AuKForConditionalGeneration",),
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="encoder",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
final_output=False,
|
||||
owns_tokenizer=True,
|
||||
requires_multimodal_data=True,
|
||||
hf_config_name="thinker_config",
|
||||
engine_output_type="latent",
|
||||
model_arch="AuKForConditionalGeneration",
|
||||
# No model_subdir: tools/prepare_auk_checkpoint.py assembles one
|
||||
# flat directory, so the encoder finds the thinker shards, the
|
||||
# tokenizer, the audio processor and the layer-fusion tensors in
|
||||
# auk.safetensors all under the pipeline root.
|
||||
# One prefill step produces the text condition; the sampled token
|
||||
# is discarded, and detokenizing it would only cost latency.
|
||||
sampling_constraints={"max_tokens": 1, "temperature": 0.0, "detokenize": False},
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=1,
|
||||
model_stage="dit",
|
||||
execution_type=StageExecutionType.DIFFUSION,
|
||||
input_sources=(0,),
|
||||
requires_multimodal_data=True,
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
model_arch="AuKPipeline",
|
||||
custom_process_input_func=f"{_PROC}.encoder2dit",
|
||||
omni_kv_config={"need_recv_cache": False},
|
||||
# Single replica, and the whole ODE runs inside one forward, so
|
||||
# the stage stays in the orchestrator process.
|
||||
inline_diffusion=True,
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -87,6 +87,11 @@ _OMNI_MODELS = {
|
||||
"cosyvoice3",
|
||||
"CosyVoice3Model",
|
||||
),
|
||||
"AuKForConditionalGeneration": (
|
||||
"auk",
|
||||
"auk",
|
||||
"AuKForConditionalGeneration",
|
||||
),
|
||||
"NemotronDenseForCausalLM": (
|
||||
"audex",
|
||||
"audex_thinker",
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Stage input processor for AuK: thinker encoder -> diffusion transition."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from vllm.inputs import TextPrompt
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.inputs.data import OmniTokensPrompt
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# The encoder emits the fused condition as
|
||||
# ``multimodal_outputs["hidden_states"]["output"]`` (``HiddenStatesStruct.output``,
|
||||
# the same stage-wire slot MiniMax H3 uses). The payload flattener may deliver
|
||||
# it under the dotted spelling instead, and a plain ``text_cond`` key is
|
||||
# accepted so an encoder that emits the condition directly still works.
|
||||
HIDDEN_STATES_KEY = "hidden_states"
|
||||
TEXT_COND_KEY = "output"
|
||||
_FALLBACK_KEYS = ("hidden_states.output", "text_cond")
|
||||
|
||||
# Per-request knobs the prompt carries for stage 1. nfe/cfg/seed are not here:
|
||||
# they ride the stage-1 OmniDiffusionSamplingParams instead.
|
||||
_AUK_KNOBS = ("gen_seconds", "sway", "t_grid", "vae_sample")
|
||||
|
||||
|
||||
def _as_dict(prompt: Any) -> dict[str, Any]:
|
||||
"""Normalize the many prompt spellings the orchestrator may pass."""
|
||||
if isinstance(prompt, list):
|
||||
prompt = prompt[0] if prompt else {}
|
||||
if prompt is None:
|
||||
return {}
|
||||
if isinstance(prompt, dict):
|
||||
return prompt
|
||||
if hasattr(prompt, "_asdict"):
|
||||
return prompt._asdict()
|
||||
if hasattr(prompt, "__dict__"):
|
||||
return vars(prompt)
|
||||
return {}
|
||||
|
||||
|
||||
def _has_audio(mm_data: Any) -> bool:
|
||||
return isinstance(mm_data, Mapping) and mm_data.get("audio") is not None
|
||||
|
||||
|
||||
def _from_hidden_states(hidden: Any) -> Any:
|
||||
"""Read the fused condition out of the payload's hidden-states slot."""
|
||||
if isinstance(hidden, Mapping):
|
||||
return hidden.get(TEXT_COND_KEY)
|
||||
# A deserialized payload arrives as a msgspec struct rather than a dict.
|
||||
return getattr(hidden, TEXT_COND_KEY, None)
|
||||
|
||||
|
||||
def _extract_text_cond(ar_output: Any) -> Any:
|
||||
"""Read the fused text condition from the encoder output.
|
||||
|
||||
``multimodal_output`` is attached to the RequestOutput; the CompletionOutput
|
||||
is checked as a fallback for vLLM versions that attach it there.
|
||||
"""
|
||||
for holder in (ar_output, *(getattr(ar_output, "outputs", None) or ())):
|
||||
mm_output = getattr(holder, "multimodal_output", None)
|
||||
if not isinstance(mm_output, Mapping):
|
||||
continue
|
||||
text_cond = _from_hidden_states(mm_output.get(HIDDEN_STATES_KEY))
|
||||
if text_cond is not None:
|
||||
return text_cond
|
||||
for key in _FALLBACK_KEYS:
|
||||
if mm_output.get(key) is not None:
|
||||
return mm_output[key]
|
||||
return None
|
||||
|
||||
|
||||
def _to_cpu_tensor(text_cond: Any) -> torch.Tensor:
|
||||
"""Coerce the emitted condition to a single 2-D CPU tensor."""
|
||||
if isinstance(text_cond, list):
|
||||
if not text_cond:
|
||||
raise ValueError("AuK encoder emitted an empty text condition list")
|
||||
parts = [_to_cpu_tensor(part) for part in text_cond]
|
||||
return torch.cat(parts, dim=0) if len(parts) > 1 else parts[0]
|
||||
if isinstance(text_cond, torch.Tensor):
|
||||
tensor = text_cond
|
||||
else:
|
||||
# numpy array or any array-like the connector round-tripped.
|
||||
tensor = torch.as_tensor(text_cond)
|
||||
tensor = tensor.detach().cpu()
|
||||
if tensor.ndim != 2:
|
||||
raise ValueError(f"AuK text condition must be [tokens, hidden]; got shape {tuple(tensor.shape)}")
|
||||
return tensor
|
||||
|
||||
|
||||
def encoder2dit(
|
||||
source_outputs: list[Any],
|
||||
prompt: OmniTokensPrompt | TextPrompt | list | None = None,
|
||||
requires_multimodal_data: bool = False,
|
||||
streaming_context: Any | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Turn the AuK encoder stage's fused hidden states into a diffusion prompt.
|
||||
|
||||
The source clip is not decoded here: the original ``multi_modal_data`` is
|
||||
forwarded so stage 1 resamples and VAE-encodes it itself. It is forwarded
|
||||
unconditionally because the editing and enhancement tasks cannot run
|
||||
without it; ``requires_multimodal_data`` only records the stage's
|
||||
declaration.
|
||||
"""
|
||||
del streaming_context, requires_multimodal_data
|
||||
if not source_outputs:
|
||||
return None
|
||||
|
||||
ar_output = source_outputs[0]
|
||||
text_cond = _extract_text_cond(ar_output)
|
||||
if text_cond is None:
|
||||
raise ValueError(
|
||||
"AuK encoder stage produced no hidden_states.output multimodal payload; "
|
||||
"stage 1 cannot run without the fused thinker hidden states"
|
||||
)
|
||||
prompt_embeds = _to_cpu_tensor(text_cond)
|
||||
|
||||
original = _as_dict(prompt)
|
||||
original_knobs = original.get("additional_information") or {}
|
||||
if isinstance(original_knobs, Mapping):
|
||||
original_knobs = original_knobs.get("auk") or {}
|
||||
if not isinstance(original_knobs, Mapping):
|
||||
original_knobs = {}
|
||||
|
||||
mm_data = original.get("multi_modal_data")
|
||||
knobs: dict[str, Any] = {key: original_knobs.get(key) for key in _AUK_KNOBS}
|
||||
declared_audio = original_knobs.get("has_audio")
|
||||
knobs["has_audio"] = bool(declared_audio) if declared_audio is not None else _has_audio(mm_data)
|
||||
|
||||
logger.debug(
|
||||
"[encoder2dit] text_cond=%s dtype=%s has_audio=%s gen_seconds=%s",
|
||||
tuple(prompt_embeds.shape),
|
||||
prompt_embeds.dtype,
|
||||
knobs["has_audio"],
|
||||
knobs["gen_seconds"],
|
||||
)
|
||||
|
||||
return {
|
||||
"prompt": "",
|
||||
"prompt_embeds": prompt_embeds,
|
||||
"multi_modal_data": mm_data,
|
||||
"additional_information": {"auk": knobs},
|
||||
}
|
||||
Reference in new issue
Block a user