[Bugfix][MiniCPM-o] Retire CFM DiT CUDA graphs a generation at a time (#6587)
Signed-off-by: BruceLoveDecimal <liuqihao970610@gmail.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
1 parent
f8f8aa8eb7
commit
ab8d45d3cc
3 files changed
+419
-79
No files matched your search
@@ -1,7 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from functools import lru_cache
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import torch
|
||||
from torch.cuda import CUDAGraph
|
||||
@@ -141,19 +139,44 @@ def _tensor_signature(value: torch.Tensor) -> tuple:
|
||||
return tuple(value.shape), str(value.dtype), str(value.device)
|
||||
|
||||
|
||||
_DTYPE_MAP = {
|
||||
"torch.float32": torch.float32,
|
||||
"torch.float16": torch.float16,
|
||||
"torch.bfloat16": torch.bfloat16,
|
||||
"torch.float64": torch.float64,
|
||||
}
|
||||
_DTYPE_MAP = {str(dtype): dtype for dtype in (torch.float32, torch.float16, torch.bfloat16, torch.float64)}
|
||||
|
||||
|
||||
def _memory_snapshot(device: torch.device) -> tuple[int, int] | None:
|
||||
"""(allocated, reserved) bytes, or None when the device cannot report them.
|
||||
|
||||
Capture draws on the caching allocator, so these are the numbers that say
|
||||
what a capture cost. Free device memory is not: the allocator serves a
|
||||
capture out of memory it has already reserved, which is most of the device
|
||||
on a normally configured worker.
|
||||
"""
|
||||
if device.type != "cuda":
|
||||
return None
|
||||
try:
|
||||
return int(torch.accelerator.memory_allocated(device)), int(torch.accelerator.memory_reserved(device))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _format_memory_delta(before: tuple[int, int] | None, after: tuple[int, int] | None) -> str:
|
||||
if before is None or after is None:
|
||||
return ""
|
||||
mib = 1024 * 1024
|
||||
return (
|
||||
f" [allocated {after[0] / mib:.1f} MiB (+{(after[0] - before[0]) / mib:.1f}), "
|
||||
f"reserved {after[1] / mib:.1f} MiB (+{(after[1] - before[1]) / mib:.1f})]"
|
||||
)
|
||||
|
||||
|
||||
def _tensors_from_key(key: tuple) -> tuple[torch.Tensor, ...]:
|
||||
"""Rebuild zero tensors from a cache key (shape, dtype, device tuples)."""
|
||||
"""Rebuild zero tensors from a cache key (shape, dtype, device tuples).
|
||||
|
||||
Raises ``KeyError`` for a dtype the key cannot round-trip, so the caller
|
||||
can fall back to eager instead of capturing wrong-dtype static buffers.
|
||||
"""
|
||||
tensors = []
|
||||
for shape, dtype_str, device_str in key[1:]:
|
||||
dtype = _DTYPE_MAP.get(dtype_str, torch.float32)
|
||||
dtype = _DTYPE_MAP[dtype_str]
|
||||
device = torch.device(device_str)
|
||||
tensors.append(torch.zeros(shape, dtype=dtype, device=device))
|
||||
return tuple(tensors)
|
||||
@@ -166,56 +189,121 @@ class CFMGraphWrapper:
|
||||
as the graph target. The 10-step Euler loop stays in Python, replaying
|
||||
the graph 10 times per decode.
|
||||
|
||||
Uses functools.lru_cache for automatic LRU eviction. Cache misses
|
||||
trigger capture; capture failures fall back to eager. Outputs are
|
||||
cloned after replay to prevent streaming cache corruption.
|
||||
Graphs are retired a whole generation at a time rather than one at a time.
|
||||
Every capture shares one private memory pool, so a retired graph's blocks
|
||||
return to that pool while its live peers still hold those addresses in
|
||||
their recorded kernel arguments, and the next replay reads memory that now
|
||||
belongs to something else. Retiring the whole generation bounds the cache
|
||||
without ever leaving a live graph behind a freed one.
|
||||
|
||||
Cache misses capture. A capture failure disables the wrapper for the rest
|
||||
of the process, while a key whose static buffers cannot be rebuilt sends
|
||||
only that one shape eager. Outputs are cloned after replay to prevent
|
||||
streaming cache corruption.
|
||||
"""
|
||||
|
||||
def __init__(self, graph_fn, *, max_graphs: int = 32) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
graph_fn,
|
||||
*,
|
||||
max_graphs: int = 32,
|
||||
) -> None:
|
||||
self.graph_fn = graph_fn
|
||||
self.max_graphs = int(max_graphs)
|
||||
self.device = next(graph_fn.__self__.parameters()).device
|
||||
# A non-positive budget means "no graphs", the same as
|
||||
# `enable_cfm_graph: false`. Clamping to 1 would instead build a
|
||||
# one-entry cache that flushes on every new shape.
|
||||
self.enabled = self.max_graphs > 0
|
||||
self._cache: dict[tuple, tuple] = {}
|
||||
# Shapes whose key cannot round-trip: eager for those, keep the rest.
|
||||
self._unsupported: set[tuple] = set()
|
||||
self._stats = {
|
||||
"calls": 0,
|
||||
"hits": 0,
|
||||
"captures": 0,
|
||||
"flushes": 0,
|
||||
"eager": 0,
|
||||
}
|
||||
|
||||
@lru_cache(maxsize=self.max_graphs)
|
||||
def _capture_graph(key: tuple):
|
||||
return self._capture(key)
|
||||
|
||||
self._capture_graph = _capture_graph
|
||||
def stats_snapshot(self) -> dict[str, int]:
|
||||
"""Bounded cumulative telemetry for the graph cache."""
|
||||
return {**self._stats, "cache_size": len(self._cache)}
|
||||
|
||||
def _call_graph_fn(self, args: tuple[torch.Tensor, ...]) -> torch.Tensor:
|
||||
return self.graph_fn(args[0], args[1], None, args[2], args[3], args[4], args[5])
|
||||
|
||||
def _eager(self, inputs: tuple[torch.Tensor, ...]) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
self._stats["eager"] += 1
|
||||
with torch.no_grad():
|
||||
result = self._call_graph_fn(inputs)
|
||||
return result, inputs[4], inputs[5]
|
||||
|
||||
def _flush(self) -> None:
|
||||
"""Retire every captured graph at once.
|
||||
|
||||
The sync keeps a replay from being in flight when the graphs go, and
|
||||
the explicit ``reset()`` tears each one down here rather than whenever
|
||||
Python drops the last reference. Nothing process-wide is touched: the
|
||||
cuBLAS workspace is shared with graphs this wrapper does not own.
|
||||
"""
|
||||
if not self._cache:
|
||||
return
|
||||
torch.accelerator.synchronize(self.device)
|
||||
for entry in self._cache.values():
|
||||
entry[2].reset()
|
||||
self._cache.clear()
|
||||
self._stats["flushes"] += 1
|
||||
logger.info("CFM graph cache flushed; stats=%s", self.stats_snapshot())
|
||||
|
||||
def _disable(self, reason: str, key: tuple) -> None:
|
||||
logger.warning("Disabling CFM CUDA graphs (%s) for shape=%s; using eager", reason, key, exc_info=True)
|
||||
self.enabled = False
|
||||
self._flush()
|
||||
|
||||
def _capture(self, key: tuple) -> tuple | None:
|
||||
"""Capture a CUDA graph for the given key. Returns None on failure."""
|
||||
static_inputs = _tensors_from_key(key)
|
||||
|
||||
current_stream = torch.cuda.current_stream(self.device)
|
||||
warmup_stream = torch.cuda.Stream(device=self.device)
|
||||
warmup_stream.wait_stream(current_stream)
|
||||
with torch.cuda.stream(warmup_stream), torch.no_grad():
|
||||
for _ in range(3):
|
||||
self._call_graph_fn(static_inputs)
|
||||
current_stream.wait_stream(warmup_stream)
|
||||
|
||||
try:
|
||||
static_inputs = _tensors_from_key(key)
|
||||
except KeyError:
|
||||
# An unsupported dtype is a property of this shape alone, so run it
|
||||
# eager and keep the other shapes on graphs.
|
||||
logger.warning("CFM graph key carries an unsupported dtype: %s; using eager", key)
|
||||
self._unsupported.add(key)
|
||||
return None
|
||||
|
||||
memory_before = _memory_snapshot(self.device)
|
||||
try:
|
||||
# Warmup runs the same kernels as the capture, so a fault here
|
||||
# leaves the same dirty capture-stream and pool state, and must be
|
||||
# handled the same way.
|
||||
current_stream = torch.cuda.current_stream(self.device)
|
||||
warmup_stream = torch.cuda.Stream(device=self.device)
|
||||
warmup_stream.wait_stream(current_stream)
|
||||
with torch.cuda.stream(warmup_stream), torch.no_grad():
|
||||
for _ in range(3):
|
||||
warmup_output = self._call_graph_fn(static_inputs)
|
||||
current_stream.wait_stream(warmup_stream)
|
||||
del warmup_output
|
||||
|
||||
graph = CUDAGraph()
|
||||
with torch.no_grad(), torch.cuda.graph(graph, pool=current_platform.get_global_graph_pool()):
|
||||
static_output = self._call_graph_fn(static_inputs)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"CFM graph capture failed for shape=%s; using eager",
|
||||
key,
|
||||
exc_info=True,
|
||||
)
|
||||
# A failed capture can leave the capture stream current and the
|
||||
# allocator still routing into the graph pool, so there is no safe
|
||||
# way to keep capturing afterwards.
|
||||
self._disable("capture failed", key)
|
||||
return None
|
||||
|
||||
self._stats["captures"] += 1
|
||||
logger.info(
|
||||
"Captured CFM CUDA Graph for shape %s (cache=%d/%d, hits=%d, misses=%d)",
|
||||
"Captured CFM CUDA Graph for shape %s (cache=%d/%d, stats=%s)%s",
|
||||
key,
|
||||
self._capture_graph.cache_info().currsize,
|
||||
len(self._cache) + 1,
|
||||
self.max_graphs,
|
||||
self._capture_graph.cache_info().hits,
|
||||
self._capture_graph.cache_info().misses,
|
||||
self.stats_snapshot(),
|
||||
_format_memory_delta(memory_before, _memory_snapshot(self.device)),
|
||||
)
|
||||
return (static_inputs, static_output, graph)
|
||||
|
||||
@@ -229,25 +317,25 @@ class CFMGraphWrapper:
|
||||
att_out: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
inputs = (estimator_input, time_emb, cnn_cache, att_cache, cnn_out, att_out)
|
||||
self._stats["calls"] += 1
|
||||
|
||||
if torch.cuda.is_current_stream_capturing() or estimator_input.device.type != "cuda":
|
||||
with torch.no_grad():
|
||||
result = self._call_graph_fn(inputs)
|
||||
return result, inputs[4], inputs[5]
|
||||
if not self.enabled or torch.cuda.is_current_stream_capturing() or estimator_input.device.type != "cuda":
|
||||
return self._eager(inputs)
|
||||
|
||||
key = ("estimator_step",) + tuple(_tensor_signature(v) for v in inputs)
|
||||
entry = self._capture_graph(key)
|
||||
if key in self._unsupported:
|
||||
return self._eager(inputs)
|
||||
entry = self._cache.get(key)
|
||||
|
||||
if entry is None:
|
||||
logger.debug(
|
||||
"CFM graph eager fallback for shape=%s (hits=%d, misses=%d)",
|
||||
key,
|
||||
self._capture_graph.cache_info().hits,
|
||||
self._capture_graph.cache_info().misses,
|
||||
)
|
||||
with torch.no_grad():
|
||||
result = self._call_graph_fn(inputs)
|
||||
return result, inputs[4], inputs[5]
|
||||
if len(self._cache) >= self.max_graphs:
|
||||
self._flush()
|
||||
entry = self._capture(key)
|
||||
if entry is None:
|
||||
return self._eager(inputs)
|
||||
self._cache[key] = entry
|
||||
else:
|
||||
self._stats["hits"] += 1
|
||||
|
||||
static_inputs, static_output, graph = entry
|
||||
for static, current in zip(static_inputs, inputs, strict=True):
|
||||
|
||||
Reference in new issue
Block a user