Add tool to configure gpu_memory_utilization for multi-stage pipelines (#1958)

Signed-off-by: linyueqian <linyueqian@outlook.com>
This commit is contained in:
Yueqian Lin authored and GitHub committed 2026-03-24 12:55:26 +08:00
1 parent 8b102bdac1
commit 4ac25287fd
1 file changed
+367
+367
View File
@@ -0,0 +1,367 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Interactive tool to configure multi-stage TTS/Omni pipelines.
Detects GPUs, shows available memory, and helps configure:
- GPU device assignment per stage
- gpu_memory_utilization per stage
- async_chunk (streaming vs non-streaming)
- enforce_eager vs CUDA graph compilation
- max_batch_size per stage
Usage:
python tools/configure_stage_memory.py --config qwen3_tts.yaml
python tools/configure_stage_memory.py --config qwen3_tts.yaml --auto
python tools/configure_stage_memory.py --config qwen3_tts.yaml --auto --streaming
"""
from __future__ import annotations
import argparse
import copy
import shutil
import sys
from pathlib import Path
from omegaconf import OmegaConf
def get_model_size_gib(model: str) -> float | None:
"""Get model weight size in GiB from HuggingFace model info."""
try:
from huggingface_hub import model_info
info = model_info(model)
if info.safetensors and info.safetensors.total:
# params * 2 bytes (bf16)
return info.safetensors.total * 2 / (1024**3)
except Exception:
pass
return None
def get_gpu_info() -> list[dict]:
"""Detect GPUs and return their memory info."""
gpus = []
try:
import torch
if not torch.cuda.is_available():
return []
for i in range(torch.cuda.device_count()):
free, total = torch.cuda.mem_get_info(i)
props = torch.cuda.get_device_properties(i)
gpus.append(
{
"id": i,
"name": props.name,
"total_gib": total / (1024**3),
"free_gib": free / (1024**3),
"used_gib": (total - free) / (1024**3),
"compute_capability": f"{props.major}.{props.minor}",
}
)
except Exception as e:
print(f"Warning: Could not detect GPUs: {e}", file=sys.stderr)
return gpus
def print_gpu_table(gpus: list[dict]) -> None:
"""Print GPU info table."""
print("\n Available GPUs:")
print(f" {'ID':>3} {'Name':<30} {'Free':>8} {'Total':>8} {'Used':>8} {'CC':>5}")
print(f" {'---':>3} {'----':<30} {'----':>8} {'-----':>8} {'----':>8} {'--':>5}")
for g in gpus:
print(
f" {g['id']:>3} {g['name']:<30} {g['free_gib']:>7.1f}G"
f" {g['total_gib']:>7.1f}G {g['used_gib']:>7.1f}G"
f" {g['compute_capability']:>5}"
)
print()
def print_config_summary(config: dict, stages: list[dict]) -> None:
"""Print full config summary."""
async_chunk = config.get("async_chunk", False)
print(f" async_chunk: {async_chunk}")
print(
f" {'Stage':>5} {'Model Stage':<15} {'Device':>6} {'GPU Mem':>8}"
f" {'Eager':>6} {'Async Sched':>11} {'Batch':>5}"
)
print(
f" {'-----':>5} {'-----------':<15} {'------':>6} {'-------':>8}"
f" {'-----':>6} {'-----------':>11} {'-----':>5}"
)
for s in stages:
print(
f" {s['stage_id']:>5} {s['model_stage']:<15} {s['device']:>6}"
f" {s['gpu_mem']:>7.3f} {'yes' if s['enforce_eager'] else 'no':>6}"
f" {'yes' if s['async_scheduling'] else 'no':>11}"
f" {s['max_batch_size']:>5}"
)
print()
def extract_stages(config: dict) -> list[dict]:
"""Extract stage info from config."""
stages = []
for stage_arg in config.get("stage_args", []):
ea = stage_arg.get("engine_args", {})
rt = stage_arg.get("runtime", {})
stages.append(
{
"stage_id": stage_arg.get("stage_id", 0),
"stage_type": stage_arg.get("stage_type", "llm"),
"model_stage": ea.get("model_stage", "unknown"),
"device": str(rt.get("devices", "0")),
"gpu_mem": ea.get("gpu_memory_utilization", 0.9),
"enforce_eager": ea.get("enforce_eager", False),
"async_scheduling": ea.get("async_scheduling", False),
"max_batch_size": rt.get("max_batch_size", 1),
"worker_type": ea.get("worker_type", "ar"),
}
)
return stages
def auto_configure(
config: dict,
stages: list[dict],
gpus: list[dict],
headroom_gib: float = 1.5,
model_size_gib: float | None = None,
streaming: bool | None = None,
latency_optimized: bool = False,
) -> tuple[dict, list[dict]]:
"""Auto-configure all settings."""
# async_chunk
if streaming is not None:
config["async_chunk"] = streaming
# GPU memory: use model size to compute what's actually needed,
# capped by available memory.
device_stages: dict[str, list[int]] = {}
for i, s in enumerate(stages):
device_stages.setdefault(s["device"], []).append(i)
for device, indices in device_stages.items():
# Handle multi-device strings like "0,1,2,3" (tensor-parallel).
# Use the first device for memory query; skip auto-sizing if invalid.
try:
gpu_id = int(device.split(",")[0])
except ValueError:
continue
if gpu_id >= len(gpus):
continue
gpu = gpus[gpu_id]
num = len(indices)
# What's available per stage
avail_per_stage = (gpu["free_gib"] - headroom_gib) / max(num, 1)
# What the model actually needs per stage (weights + KV cache headroom)
if model_size_gib is not None:
needed_per_stage = model_size_gib / max(num, 1) + 3.0 # +3G for KV cache
else:
needed_per_stage = avail_per_stage # no model info, use all available
# Take the smaller of available and needed
allocated = min(avail_per_stage, needed_per_stage)
util = round(max(allocated / gpu["total_gib"], 0.04), 3)
util = min(util, 0.95)
for idx in indices:
stages[idx]["gpu_mem"] = util
# Per-stage optimizations
for s in stages:
if latency_optimized:
# CUDA graphs for AR stages (lower latency)
if s["worker_type"] == "ar":
s["enforce_eager"] = False
s["async_scheduling"] = True
# Generation stages always eager (no KV cache)
if s["worker_type"] == "generation":
s["enforce_eager"] = True
s["async_scheduling"] = True
return config, stages
def interactive_configure(config: dict, stages: list[dict], gpus: list[dict]) -> tuple[dict, list[dict]]:
"""Interactive mode."""
gpu_ids = [str(g["id"]) for g in gpus]
# async_chunk
current_async = config.get("async_chunk", False)
val = input(f" Enable streaming (async_chunk)? [{'Y' if current_async else 'N'}]: ").strip().lower()
if val in ("y", "yes", "true", "1"):
config["async_chunk"] = True
elif val in ("n", "no", "false", "0"):
config["async_chunk"] = False
print()
for s in stages:
print(f" Stage {s['stage_id']} ({s['model_stage']}, {s['worker_type']}):")
# Device (accepts single id or comma-separated like "0,1,2")
default_dev = s["device"]
while True:
dev = input(f" GPU device [{default_dev}]: ").strip() or default_dev
dev_ids = [d.strip() for d in dev.split(",")]
if all(d in gpu_ids for d in dev_ids):
s["device"] = dev
break
print(f" Invalid. Choose from: {', '.join(gpu_ids)} (comma-separated for multi-GPU)")
# GPU memory (use first device for memory query)
first_dev = int(s["device"].split(",")[0])
gpu = gpus[first_dev]
same_device = sum(1 for st in stages if st["device"] == s["device"])
suggested = round((gpu["free_gib"] - 1.5) / same_device / gpu["total_gib"], 3)
suggested = max(suggested, 0.04)
suggested = min(suggested, 0.95)
while True:
val = input(f" gpu_memory_utilization [{suggested:.3f}]: ").strip()
if not val:
s["gpu_mem"] = suggested
break
try:
v = float(val)
if 0.01 <= v <= 0.99:
s["gpu_mem"] = round(v, 3)
break
print(" Must be between 0.01 and 0.99")
except ValueError:
print(" Invalid number")
# enforce_eager
if s["worker_type"] == "ar":
current = s["enforce_eager"]
hint = "no=CUDA graphs (faster), yes=eager (debug)" if not current else "yes=eager, no=CUDA graphs (faster)"
val = input(f" enforce_eager [{('yes' if current else 'no')}] ({hint}): ").strip().lower()
if val in ("y", "yes", "true", "1"):
s["enforce_eager"] = True
elif val in ("n", "no", "false", "0"):
s["enforce_eager"] = False
# max_batch_size
current_bs = s["max_batch_size"]
val = input(f" max_batch_size [{current_bs}]: ").strip()
if val:
try:
s["max_batch_size"] = int(val)
except ValueError:
pass
print()
return config, stages
def apply_to_config(config: dict, stages: list[dict]) -> dict:
"""Apply stage settings back to config dict."""
config = copy.deepcopy(config)
for stage_arg, s in zip(config["stage_args"], stages):
stage_arg.setdefault("runtime", {})["devices"] = s["device"]
stage_arg.setdefault("runtime", {})["max_batch_size"] = s["max_batch_size"]
ea = stage_arg.setdefault("engine_args", {})
ea["gpu_memory_utilization"] = s["gpu_mem"]
ea["enforce_eager"] = s["enforce_eager"]
ea["async_scheduling"] = s["async_scheduling"]
return config
def main():
parser = argparse.ArgumentParser(
description="Configure multi-stage TTS/Omni pipelines for your hardware",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
# Interactive mode - prompts for every setting
python tools/configure_stage_memory.py --config qwen3_tts.yaml
# Auto mode - detect GPUs and set optimal values
python tools/configure_stage_memory.py --config qwen3_tts.yaml --auto
# Auto mode optimized for low latency with streaming
python tools/configure_stage_memory.py --config qwen3_tts.yaml --auto --streaming --low-latency
# Save to a different file
python tools/configure_stage_memory.py --config qwen3_tts.yaml --auto -o my_config.yaml
""",
)
parser.add_argument("--config", required=True, help="Path to stage config YAML")
parser.add_argument("--model", help="HuggingFace model name (to query weight size for smart allocation)")
parser.add_argument("--auto", action="store_true", help="Auto-configure without prompts")
parser.add_argument("--output", "-o", help="Output path (default: overwrite input)")
parser.add_argument("--headroom", type=float, default=1.5, help="GiB headroom to leave free (default: 1.5)")
parser.add_argument("--streaming", action="store_true", default=None, help="Enable async_chunk streaming")
parser.add_argument("--no-streaming", action="store_true", help="Disable async_chunk streaming")
parser.add_argument(
"--low-latency",
action="store_true",
help="Optimize for latency (CUDA graphs for AR, async scheduling)",
)
args = parser.parse_args()
streaming = None
if args.streaming:
streaming = True
elif args.no_streaming:
streaming = False
config_path = Path(args.config)
if not config_path.exists():
print(f"Error: {config_path} not found", file=sys.stderr)
sys.exit(1)
config = OmegaConf.to_container(OmegaConf.load(config_path), resolve=True)
stages = extract_stages(config)
gpus = get_gpu_info()
if not gpus:
print("No GPUs detected. Cannot configure.", file=sys.stderr)
sys.exit(1)
# Query model size from HuggingFace
model_size_gib = None
if args.model:
model_size_gib = get_model_size_gib(args.model)
if model_size_gib:
print(f"\n Model: {args.model} ({model_size_gib:.1f} GiB in bf16)")
else:
print(f"\n Model: {args.model} (could not determine size)")
else:
print("\n Tip: pass --model <name> to auto-size based on HuggingFace weight info")
print(f" Config: {config_path}")
print_gpu_table(gpus)
print(" Before:")
print_config_summary(config, stages)
if args.auto:
config, stages = auto_configure(
config, stages, gpus, args.headroom, model_size_gib, streaming, args.low_latency
)
else:
config, stages = interactive_configure(config, stages, gpus)
config = apply_to_config(config, stages)
print(" After:")
print_config_summary(config, extract_stages(config))
output_path = Path(args.output) if args.output else config_path
if output_path == config_path:
backup = config_path.with_suffix(".yaml.bak")
shutil.copy2(config_path, backup)
print(f" Backup: {backup}")
OmegaConf.save(OmegaConf.create(config), output_path)
print(f" Saved: {output_path}")
if __name__ == "__main__":
main()