[Feat] support cache-dit for SenseNova-U1 (#3906)
Signed-off-by: hanzheli <hanzheli@kuaishou.com>
This commit is contained in:
11 files changed
+256
-6
No files matched your search
@@ -37,6 +37,10 @@ python end2end.py --modality text2text \
|
||||
# Custom resolution (image generation)
|
||||
python end2end.py --prompt "A futuristic cityscape at sunset" \
|
||||
--width 2048 --height 1024 --think
|
||||
|
||||
# Text-to-image with Cache-DiT acceleration
|
||||
python end2end.py --prompt "A cute cat" \
|
||||
--cache-backend cache_dit \
|
||||
```
|
||||
|
||||
> **Note**: Default configuration works on a single **NVIDIA A100 (80GB)** or **H100** GPU.
|
||||
@@ -135,6 +139,8 @@ python end2end.py \
|
||||
| `--tensor-parallel-size` | 1 | Number of GPUs for tensor parallelism |
|
||||
| `--enforce-eager` | False | Disable torch.compile |
|
||||
| `--enable-cpu-offload` | False | Enable module-wise (sequential) CPU offload to reduce peak VRAM |
|
||||
| `--cache-backend` | None | Set to `cache_dit` for Cache-DiT acceleration |
|
||||
| `--enable-cache-dit-summary` | False | Print Cache-DiT cache statistics after generation |
|
||||
|
||||
## Reducing GPU Memory Usage
|
||||
|
||||
|
||||
@@ -155,6 +155,18 @@ def parse_args():
|
||||
"of extra CPU<->GPU transfers."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cache-backend",
|
||||
type=str,
|
||||
default=None,
|
||||
choices=["cache_dit"],
|
||||
help="Cache backend to use for image generation acceleration.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--enable-cache-dit-summary",
|
||||
action="store_true",
|
||||
help="Enable cache-dit summary logging after diffusion forward passes.",
|
||||
)
|
||||
|
||||
nullify_stage_engine_defaults(parser)
|
||||
return parser.parse_args()
|
||||
@@ -182,6 +194,8 @@ def main():
|
||||
tensor_parallel_size=args.tensor_parallel_size,
|
||||
enforce_eager=args.enforce_eager,
|
||||
enable_cpu_offload=args.enable_cpu_offload,
|
||||
cache_backend=args.cache_backend,
|
||||
enable_cache_dit_summary=args.enable_cache_dit_summary,
|
||||
)
|
||||
|
||||
extra_args = {
|
||||
@@ -225,6 +239,7 @@ def main():
|
||||
print(f" Seed : {args.seed}")
|
||||
print(f" Think mode : {args.think}")
|
||||
print(f" TP size : {args.tensor_parallel_size}")
|
||||
print(f" Cache backend : {args.cache_backend or 'none'}")
|
||||
print(f"{'=' * 60}\n")
|
||||
|
||||
# Build prompt dict
|
||||
|
||||
@@ -6,11 +6,21 @@
|
||||
vllm serve SenseNova/SenseNova-U1-8B-MoT --omni --port 8091
|
||||
```
|
||||
|
||||
With Cache-DiT acceleration:
|
||||
|
||||
```bash
|
||||
vllm serve SenseNova/SenseNova-U1-8B-MoT --omni --port 8091 \
|
||||
--cache-backend cache_dit
|
||||
```
|
||||
|
||||
Or use the convenience script:
|
||||
|
||||
```bash
|
||||
cd examples/online_serving/sensenova_u1
|
||||
bash run_server.sh
|
||||
|
||||
# Cache-DiT
|
||||
CACHE_BACKEND=cache_dit bash run_server.sh
|
||||
```
|
||||
|
||||
### Tensor Parallelism (TP)
|
||||
|
||||
@@ -3,10 +3,18 @@
|
||||
|
||||
MODEL="${MODEL:-SenseNova/SenseNova-U1-8B-MoT}"
|
||||
PORT="${PORT:-8091}"
|
||||
CACHE_BACKEND="${CACHE_BACKEND:-none}"
|
||||
|
||||
echo "Starting SenseNova-U1 server..."
|
||||
echo "Model: $MODEL"
|
||||
echo "Port: $PORT"
|
||||
echo "Cache backend: $CACHE_BACKEND"
|
||||
|
||||
EXTRA_ARGS=()
|
||||
if [ "$CACHE_BACKEND" != "none" ]; then
|
||||
EXTRA_ARGS+=(--cache-backend "$CACHE_BACKEND")
|
||||
fi
|
||||
|
||||
vllm serve "$MODEL" --omni \
|
||||
--port "$PORT"
|
||||
--port "$PORT" \
|
||||
"${EXTRA_ARGS[@]}"
|
||||
Reference in new issue
Block a user