[Feature]: Support cfg kv-cache transfer in multi-stage (#1422)

Signed-off-by: princepride <wangzhipeng628@gmail.com>
Signed-off-by: 汪志鹏 <wangzhipeng628@gmail.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
汪志鹏andCursor authored and GitHub committed 2026-03-02 05:06:28 +00:00
1 parent 68764cc4f4
commit e37a89ffcb
21 files changed
+804 -29

No files matched your search

+3
View File
@@ -442,6 +442,9 @@ class OmniDiffusionConfig:
# Omni configuration (injected from stage config)
omni_kv_config: dict[str, Any] = field(default_factory=dict)
# Model-specific function for collecting CFG KV caches (set at runtime)
cfg_kv_collect_func: Any | None = None
# Quantization settings
# Supported methods: "fp8" (FP8 W8A8 on Ada/Hopper, weight-only on older GPUs)
quantization: str | None = None
@@ -327,16 +327,40 @@ class BagelPipeline(nn.Module):
gen_context["past_key_values"] = injected_kv
seq_len = injected_kv.key_cache[0].shape[0]
gen_context["kv_lens"] = [seq_len]
gen_context["ropes"] = [seq_len]
if req.sampling_params.kv_metadata and "ropes" in req.sampling_params.kv_metadata:
gen_context["ropes"] = req.sampling_params.kv_metadata["ropes"]
else:
gen_context["ropes"] = [seq_len]
# Disable CFG: single KV cache cannot support 3-branch CFG
logger.warning("CFG is disabled when using injected KV Cache")
gen_params = BagelGenParams(
num_timesteps=gen_params.num_timesteps,
timestep_shift=gen_params.timestep_shift,
cfg_text_scale=1.0,
cfg_img_scale=1.0,
)
cfg_text_kv = getattr(req.sampling_params, "cfg_text_past_key_values", None)
if cfg_text_kv is not None:
logger.info("CFG enabled with multi-KV: using injected cfg_text KV Cache")
cfg_text_seq_len = cfg_text_kv.key_cache[0].shape[0]
cfg_text_context["past_key_values"] = cfg_text_kv
cfg_text_context["kv_lens"] = [cfg_text_seq_len]
cfg_text_metadata = getattr(req.sampling_params, "cfg_text_kv_metadata", None)
if cfg_text_metadata and "ropes" in cfg_text_metadata:
cfg_text_context["ropes"] = cfg_text_metadata["ropes"]
else:
cfg_text_context["ropes"] = [cfg_text_seq_len]
cfg_img_kv = getattr(req.sampling_params, "cfg_img_past_key_values", None) or injected_kv
cfg_img_seq_len = cfg_img_kv.key_cache[0].shape[0]
cfg_img_context["past_key_values"] = cfg_img_kv
cfg_img_context["kv_lens"] = [cfg_img_seq_len]
cfg_img_metadata = getattr(req.sampling_params, "cfg_img_kv_metadata", None)
if cfg_img_metadata and "ropes" in cfg_img_metadata:
cfg_img_context["ropes"] = cfg_img_metadata["ropes"]
else:
cfg_img_context["ropes"] = [cfg_img_seq_len]
else:
logger.warning("CFG is disabled: only single KV cache available")
gen_params = BagelGenParams(
num_timesteps=gen_params.num_timesteps,
timestep_shift=gen_params.timestep_shift,
cfg_text_scale=1.0,
cfg_img_scale=1.0,
)
else:
image_input = (
@@ -191,7 +191,11 @@ class DiffusionModelRunner:
grad_context = torch.no_grad() if use_hsdp else torch.inference_mode()
with grad_context:
# The manager handles the check for need_recv_cache internally
self.kv_transfer_manager.receive_kv_cache(req, target_device=getattr(self.pipeline, "device", None))
self.kv_transfer_manager.receive_multi_kv_cache(
req,
cfg_kv_collect_func=getattr(self.od_config, "cfg_kv_collect_func", None),
target_device=getattr(self.pipeline, "device", None),
)
if req.sampling_params.generator is None and req.sampling_params.seed is not None:
if req.sampling_params.generator_device is not None: