feat(magi2): add fused SwiGLU7 activation (#7252)

Signed-off-by: Xiaodong Ye <xiaodong.ye@mthreads.com>
This commit is contained in:
R0CKSTAR authored and GitHub committed 2026-09-26 22:53:04 +08:00
1 parent e1303e5067
commit e75278dccc
5 files changed
+566 -6

No files matched your search

+194
View File
@@ -0,0 +1,194 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Cold-cache torch.cuda timing and correctness gates for MAGI-2 microbenchmarks."""
import argparse
import importlib
import importlib.metadata
import json
import logging
import math
import os
import subprocess
import sys
from collections.abc import Callable
from pathlib import Path
from types import ModuleType
from typing import Any
LOGGER = logging.getLogger(__name__)
def emit(**fields: Any) -> None:
LOGGER.info(json.dumps(fields, sort_keys=True))
def add_arguments(parser: argparse.ArgumentParser) -> None:
parser.add_argument("--tokens", type=int, nargs="+", default=[1, 4, 16, 64, 129, 4096])
parser.add_argument("--dtype", choices=["bfloat16", "float16", "float32"], default="bfloat16")
parser.add_argument("--num-tests", type=int, default=30, help="Measured samples per implementation.")
parser.add_argument("--warmup", type=int, default=10, help="Interleaved warmup calls before sampling.")
parser.add_argument("--seed", type=int, default=42)
def load_musa(args: argparse.Namespace, parser: argparse.ArgumentParser) -> Any:
if min(*args.tokens, args.num_tests) <= 0 or args.warmup < 0:
parser.error("tokens and num-tests must be positive; warmup must be nonnegative")
logging.basicConfig(level=logging.INFO, format="%(message)s", stream=sys.stdout)
torch = importlib.import_module("torch")
cuda = getattr(torch, "cuda")
# torchada is needed only on MUSA; after it is loaded, use torch.cuda for
# both CUDA and MUSA as documented by torchada.
if hasattr(torch.version, "musa") and torch.version.musa is not None:
try:
importlib.import_module("torchada")
except ModuleNotFoundError as error:
if error.name != "torchada":
raise
has_gpu = cuda.is_available() or (
hasattr(torch.version, "musa") and torch.version.musa is not None and torch.musa.is_available()
)
if not has_gpu:
emit(status="skip", reason="CUDA/MUSA device is unavailable", torch=torch.__version__)
return None
torch.manual_seed(args.seed)
cuda.manual_seed_all(args.seed)
return torch
def report_environment(torch: Any, args: argparse.Namespace, module: ModuleType) -> None:
root = Path(__file__).resolve().parents[2]
packages = {}
for name in ("torch", "torch_musa", "torchada", "triton", "vllm", "vllm-omni"):
try:
packages[name] = importlib.metadata.version(name)
except importlib.metadata.PackageNotFoundError:
packages[name] = None
cuda = getattr(torch, "cuda")
properties = cuda.get_device_properties(cuda.current_device())
try:
source_sha = subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"], text=True).strip()
source_dirty = bool(subprocess.check_output(["git", "-C", str(root), "status", "--porcelain"], text=True))
except (subprocess.CalledProcessError, FileNotFoundError):
source_sha = os.environ.get("MAGI2_BENCH_SOURCE_SHA", "unknown")
source_dirty = None
emit(
status="environment",
command=sys.argv,
source_root=str(root),
source_sha=source_sha,
source_dirty=source_dirty,
kernel_source=module.__file__,
device=cuda.get_device_name(cuda.current_device()),
sm_count=getattr(properties, "multi_processor_count", None),
packages=packages,
env={name: os.environ.get(name) for name in ("MUSA_VISIBLE_DEVICES", "TORCHDYNAMO_DISABLE")},
config=vars(args),
mode="eager operator diagnostic; no compile or graph capture",
comparison="forward_native versus the default CustomOp dispatch",
timing="torch.cuda.Event; alternating native/fused calls; vanilla 512 MiB flush outside timed regions",
timing_scope="whole callable, including device launch gaps; not isolated kernel duration",
l2_flush=True,
l2_flush_bytes=512 * 1024 * 1024,
)
class _LaunchProbe:
def __init__(self, kernel: Any) -> None:
self.kernel = kernel
self.count = 0
def __getitem__(self, grid: Any) -> Any:
self.count += 1
return self.kernel[grid]
def check_pair(
torch: Any,
reference: Callable,
fused: Callable,
module: ModuleType,
kernels: tuple[str, ...],
*,
rtol: float,
atol: float,
) -> dict:
cuda = getattr(torch, "cuda")
expected = reference()
probes = {name: _LaunchProbe(getattr(module, name)) for name in kernels}
try:
for name, probe in probes.items():
setattr(module, name, probe)
actual = fused()
cuda.synchronize()
finally:
for name, probe in probes.items():
setattr(module, name, probe.kernel)
if any(probe.count != 1 for probe in probes.values()):
raise RuntimeError(f"Expected one launch per fused kernel; got {[(k, p.count) for k, p in probes.items()]}")
expected = expected if isinstance(expected, tuple) else (expected,)
actual = actual if isinstance(actual, tuple) else (actual,)
errors = []
for result, baseline in zip(actual, expected, strict=True):
for value in (baseline, result):
if not torch.isfinite(value).all().item():
raise AssertionError("Nonfinite benchmark output")
if value.numel() > 1 and (value == value.flatten()[0]).all().item():
raise AssertionError("Constant benchmark output")
torch.testing.assert_close(result, baseline, rtol=rtol, atol=atol)
difference = (result.float() - baseline.float()).abs()
errors.append(
{
"max_abs": difference.max().item(),
"max_rel": (difference / baseline.float().abs().clamp_min(1e-6)).max().item(),
}
)
return {"parity": "pass", "errors": errors, "rtol": rtol, "atol": atol, "kernels": list(kernels)}
def _percentile(samples: list[float], quantile: float) -> float:
ordered = sorted(samples)
position = (len(ordered) - 1) * quantile
lower, upper = math.floor(position), math.ceil(position)
return ordered[lower] + (ordered[upper] - ordered[lower]) * (position - lower)
def measure_pair(torch: Any, reference: Callable, fused: Callable, args: argparse.Namespace) -> dict:
cuda = getattr(torch, "cuda")
functions = (reference, fused)
order: list[int] = []
def alternate() -> Any:
mode = len(order) % 2
order.append(mode)
return functions[mode]()
times_ms = []
for _ in range(args.warmup):
alternate()
cuda.synchronize()
order.clear()
flush = torch.empty(512 * 1024 * 1024, dtype=torch.uint8, device="cuda")
for _ in range(2 * args.num_tests):
flush.zero_()
cuda.synchronize()
start = cuda.Event(enable_timing=True)
end = cuda.Event(enable_timing=True)
start.record()
alternate()
end.record()
end.synchronize()
times_ms.append(start.elapsed_time(end))
samples: tuple[list[float], list[float]] = ([], [])
for mode, milliseconds in zip(order[-len(times_ms) :], times_ms, strict=True):
if not math.isfinite(milliseconds) or milliseconds <= 0:
raise RuntimeError(f"Invalid GPU event sample: {milliseconds}")
samples[mode].append(milliseconds * 1000)
stats = {}
for label, values in zip(("native", "fused"), samples, strict=True):
if len(values) != args.num_tests:
raise RuntimeError("GPU timing harness returned an unexpected sample count")
p50 = _percentile(values, 0.5)
stats[label] = {"median_us": p50, "p50_us": p50, "p90_us": _percentile(values, 0.9), "samples": len(values)}
stats["speedup_p50"] = stats["native"]["p50_us"] / stats["fused"]["p50_us"]
return stats
@@ -0,0 +1,87 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Compare native and fused MAGI-2 SwiGLU7 operators on CUDA or MUSA.
Run from the checkout root in an environment with PyTorch and the selected
backend dependencies. torchada is loaded automatically only on MUSA::
python -m benchmarks.kernels.benchmark_magi2_swiglu7 \
--tokens 1 4 16 64 129 4096 --intermediate-size 1280 --num-tests 30
The released MAGI-2 config supplies the 1280 intermediate width. Token counts
are a synthetic sweep; pass per-rank sizes from the target trace for production
comparisons. Input is contiguous interleaved gate/up [tokens, 2 * intermediate].
Both paths evaluate FP32 arithmetic with one cast to the requested output dtype.
This measures eager operator latency, not compiled serving performance.
No CUDA/MUSA device (or no PyTorch) produces a JSON skip record and exit status zero.
"""
import argparse
from functools import partial
from benchmarks.kernels._magi2_musa_bench import (
add_arguments,
check_pair,
emit,
load_musa,
measure_pair,
report_environment,
)
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
add_arguments(parser)
parser.add_argument("--intermediate-size", type=int, default=1280)
parser.add_argument("--out-dtype", choices=["bfloat16", "float16", "float32"], default=None)
args = parser.parse_args()
if args.intermediate_size <= 0:
parser.error("intermediate-size must be positive")
torch = load_musa(args, parser)
if torch is None:
return
cuda = getattr(torch, "cuda")
from vllm_omni.diffusion.layers import swiglu7
report_environment(torch, args, swiglu7)
dtype = getattr(torch, args.dtype)
out_dtype = getattr(torch, args.out_dtype or args.dtype)
device = torch.device("cuda")
rtol = {torch.bfloat16: 1.6e-2, torch.float16: 2e-3, torch.float32: 1e-5}[out_dtype]
atol = 1e-5 if out_dtype == torch.float32 else 2e-3
op = swiglu7.SwiGLU7()
with torch.inference_mode():
for tokens in args.tokens:
# Values outside both clamp boundaries exercise the released formula.
x = (torch.randn(tokens, 2 * args.intermediate_size, device=device) * 10).to(dtype)
reference = partial(op.forward_native, x, out_dtype=out_dtype)
fused = partial(op, x, out_dtype=out_dtype)
parity = check_pair(torch, reference, fused, swiglu7, ("_swiglu7_kernel",), rtol=rtol, atol=atol)
timing = measure_pair(torch, reference, fused, args)
output_elements = tokens * args.intermediate_size
minimum_io_bytes = (
x.numel() * x.element_size() + output_elements * torch.empty((), dtype=out_dtype).element_size()
)
emit(
status="pass",
op="swiglu7",
input_shape=list(x.shape),
input_stride=list(x.stride()),
dtype=args.dtype,
out_dtype=args.out_dtype or args.dtype,
alpha=1.702,
limit=7.0,
minimum_io_bytes=minimum_io_bytes,
# Six scalar arithmetic operations and one exp per output;
# clamps and transcendental costs are not modeled as FLOPs.
scalar_flops_excluding_exp_and_clamps=6 * output_elements,
effective_fused_gbps=minimum_io_bytes / timing["fused"]["p50_us"] / 1000,
**parity,
**timing,
)
del x, reference, fused
cuda.empty_cache()
if __name__ == "__main__":
main()