[Bugfix][MiniCPM-o] Retire CFM DiT CUDA graphs a generation at a time (#6587)

Signed-off-by: BruceLoveDecimal <liuqihao970610@gmail.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
BruceLoveDecimalandamy-why-3459 authored and GitHub committed 2026-08-28 14:07:57 +08:00
1 parent f8f8aa8eb7
commit ab8d45d3cc
3 files changed
+419 -79

No files matched your search

@@ -1,7 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import lru_cache
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import torch
from torch.cuda import CUDAGraph
@@ -141,19 +139,44 @@ def _tensor_signature(value: torch.Tensor) -> tuple:
return tuple(value.shape), str(value.dtype), str(value.device)
_DTYPE_MAP = {
"torch.float32": torch.float32,
"torch.float16": torch.float16,
"torch.bfloat16": torch.bfloat16,
"torch.float64": torch.float64,
}
_DTYPE_MAP = {str(dtype): dtype for dtype in (torch.float32, torch.float16, torch.bfloat16, torch.float64)}
def _memory_snapshot(device: torch.device) -> tuple[int, int] | None:
"""(allocated, reserved) bytes, or None when the device cannot report them.
Capture draws on the caching allocator, so these are the numbers that say
what a capture cost. Free device memory is not: the allocator serves a
capture out of memory it has already reserved, which is most of the device
on a normally configured worker.
"""
if device.type != "cuda":
return None
try:
return int(torch.accelerator.memory_allocated(device)), int(torch.accelerator.memory_reserved(device))
except Exception:
return None
def _format_memory_delta(before: tuple[int, int] | None, after: tuple[int, int] | None) -> str:
if before is None or after is None:
return ""
mib = 1024 * 1024
return (
f" [allocated {after[0] / mib:.1f} MiB (+{(after[0] - before[0]) / mib:.1f}), "
f"reserved {after[1] / mib:.1f} MiB (+{(after[1] - before[1]) / mib:.1f})]"
)
def _tensors_from_key(key: tuple) -> tuple[torch.Tensor, ...]:
"""Rebuild zero tensors from a cache key (shape, dtype, device tuples)."""
"""Rebuild zero tensors from a cache key (shape, dtype, device tuples).
Raises ``KeyError`` for a dtype the key cannot round-trip, so the caller
can fall back to eager instead of capturing wrong-dtype static buffers.
"""
tensors = []
for shape, dtype_str, device_str in key[1:]:
dtype = _DTYPE_MAP.get(dtype_str, torch.float32)
dtype = _DTYPE_MAP[dtype_str]
device = torch.device(device_str)
tensors.append(torch.zeros(shape, dtype=dtype, device=device))
return tuple(tensors)
@@ -166,56 +189,121 @@ class CFMGraphWrapper:
as the graph target. The 10-step Euler loop stays in Python, replaying
the graph 10 times per decode.
Uses functools.lru_cache for automatic LRU eviction. Cache misses
trigger capture; capture failures fall back to eager. Outputs are
cloned after replay to prevent streaming cache corruption.
Graphs are retired a whole generation at a time rather than one at a time.
Every capture shares one private memory pool, so a retired graph's blocks
return to that pool while its live peers still hold those addresses in
their recorded kernel arguments, and the next replay reads memory that now
belongs to something else. Retiring the whole generation bounds the cache
without ever leaving a live graph behind a freed one.
Cache misses capture. A capture failure disables the wrapper for the rest
of the process, while a key whose static buffers cannot be rebuilt sends
only that one shape eager. Outputs are cloned after replay to prevent
streaming cache corruption.
"""
def __init__(self, graph_fn, *, max_graphs: int = 32) -> None:
def __init__(
self,
graph_fn,
*,
max_graphs: int = 32,
) -> None:
self.graph_fn = graph_fn
self.max_graphs = int(max_graphs)
self.device = next(graph_fn.__self__.parameters()).device
# A non-positive budget means "no graphs", the same as
# `enable_cfm_graph: false`. Clamping to 1 would instead build a
# one-entry cache that flushes on every new shape.
self.enabled = self.max_graphs > 0
self._cache: dict[tuple, tuple] = {}
# Shapes whose key cannot round-trip: eager for those, keep the rest.
self._unsupported: set[tuple] = set()
self._stats = {
"calls": 0,
"hits": 0,
"captures": 0,
"flushes": 0,
"eager": 0,
}
@lru_cache(maxsize=self.max_graphs)
def _capture_graph(key: tuple):
return self._capture(key)
self._capture_graph = _capture_graph
def stats_snapshot(self) -> dict[str, int]:
"""Bounded cumulative telemetry for the graph cache."""
return {**self._stats, "cache_size": len(self._cache)}
def _call_graph_fn(self, args: tuple[torch.Tensor, ...]) -> torch.Tensor:
return self.graph_fn(args[0], args[1], None, args[2], args[3], args[4], args[5])
def _eager(self, inputs: tuple[torch.Tensor, ...]) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
self._stats["eager"] += 1
with torch.no_grad():
result = self._call_graph_fn(inputs)
return result, inputs[4], inputs[5]
def _flush(self) -> None:
"""Retire every captured graph at once.
The sync keeps a replay from being in flight when the graphs go, and
the explicit ``reset()`` tears each one down here rather than whenever
Python drops the last reference. Nothing process-wide is touched: the
cuBLAS workspace is shared with graphs this wrapper does not own.
"""
if not self._cache:
return
torch.accelerator.synchronize(self.device)
for entry in self._cache.values():
entry[2].reset()
self._cache.clear()
self._stats["flushes"] += 1
logger.info("CFM graph cache flushed; stats=%s", self.stats_snapshot())
def _disable(self, reason: str, key: tuple) -> None:
logger.warning("Disabling CFM CUDA graphs (%s) for shape=%s; using eager", reason, key, exc_info=True)
self.enabled = False
self._flush()
def _capture(self, key: tuple) -> tuple | None:
"""Capture a CUDA graph for the given key. Returns None on failure."""
static_inputs = _tensors_from_key(key)
current_stream = torch.cuda.current_stream(self.device)
warmup_stream = torch.cuda.Stream(device=self.device)
warmup_stream.wait_stream(current_stream)
with torch.cuda.stream(warmup_stream), torch.no_grad():
for _ in range(3):
self._call_graph_fn(static_inputs)
current_stream.wait_stream(warmup_stream)
try:
static_inputs = _tensors_from_key(key)
except KeyError:
# An unsupported dtype is a property of this shape alone, so run it
# eager and keep the other shapes on graphs.
logger.warning("CFM graph key carries an unsupported dtype: %s; using eager", key)
self._unsupported.add(key)
return None
memory_before = _memory_snapshot(self.device)
try:
# Warmup runs the same kernels as the capture, so a fault here
# leaves the same dirty capture-stream and pool state, and must be
# handled the same way.
current_stream = torch.cuda.current_stream(self.device)
warmup_stream = torch.cuda.Stream(device=self.device)
warmup_stream.wait_stream(current_stream)
with torch.cuda.stream(warmup_stream), torch.no_grad():
for _ in range(3):
warmup_output = self._call_graph_fn(static_inputs)
current_stream.wait_stream(warmup_stream)
del warmup_output
graph = CUDAGraph()
with torch.no_grad(), torch.cuda.graph(graph, pool=current_platform.get_global_graph_pool()):
static_output = self._call_graph_fn(static_inputs)
except Exception:
logger.warning(
"CFM graph capture failed for shape=%s; using eager",
key,
exc_info=True,
)
# A failed capture can leave the capture stream current and the
# allocator still routing into the graph pool, so there is no safe
# way to keep capturing afterwards.
self._disable("capture failed", key)
return None
self._stats["captures"] += 1
logger.info(
"Captured CFM CUDA Graph for shape %s (cache=%d/%d, hits=%d, misses=%d)",
"Captured CFM CUDA Graph for shape %s (cache=%d/%d, stats=%s)%s",
key,
self._capture_graph.cache_info().currsize,
len(self._cache) + 1,
self.max_graphs,
self._capture_graph.cache_info().hits,
self._capture_graph.cache_info().misses,
self.stats_snapshot(),
_format_memory_delta(memory_before, _memory_snapshot(self.device)),
)
return (static_inputs, static_output, graph)
@@ -229,25 +317,25 @@ class CFMGraphWrapper:
att_out: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
inputs = (estimator_input, time_emb, cnn_cache, att_cache, cnn_out, att_out)
self._stats["calls"] += 1
if torch.cuda.is_current_stream_capturing() or estimator_input.device.type != "cuda":
with torch.no_grad():
result = self._call_graph_fn(inputs)
return result, inputs[4], inputs[5]
if not self.enabled or torch.cuda.is_current_stream_capturing() or estimator_input.device.type != "cuda":
return self._eager(inputs)
key = ("estimator_step",) + tuple(_tensor_signature(v) for v in inputs)
entry = self._capture_graph(key)
if key in self._unsupported:
return self._eager(inputs)
entry = self._cache.get(key)
if entry is None:
logger.debug(
"CFM graph eager fallback for shape=%s (hits=%d, misses=%d)",
key,
self._capture_graph.cache_info().hits,
self._capture_graph.cache_info().misses,
)
with torch.no_grad():
result = self._call_graph_fn(inputs)
return result, inputs[4], inputs[5]
if len(self._cache) >= self.max_graphs:
self._flush()
entry = self._capture(key)
if entry is None:
return self._eager(inputs)
self._cache[key] = entry
else:
self._stats["hits"] += 1
static_inputs, static_output, graph = entry
for static, current in zip(static_inputs, inputs, strict=True):