diff --git a/recipes/cosmos3/Cosmos3-Nano.md b/recipes/cosmos3/Cosmos3-Nano.md index e58a28623..c3282ba51 100644 --- a/recipes/cosmos3/Cosmos3-Nano.md +++ b/recipes/cosmos3/Cosmos3-Nano.md @@ -422,3 +422,185 @@ ffprobe -v error -show_entries stream=codec_type,nb_frames,width,height cosmos3_ `generate_sound`/`sound_duration`, `guardrails`, `action_*`, ...) are declared once in `vllm_omni/model_extras/cosmos3.py` and forwarded through `--extra-body`; unknown keys for the model are dropped. + +## NPU + +### 1x Ascend 910B / 910C (Atlas A2 / A3) — Online serving + +#### Environment + +- OS: Linux (aarch64) +- Python: 3.12+ +- Driver / runtime: CANN 8.5.1 + NNAL + Ascend 910B / 910C +- vLLM version: match the repository requirements from your current checkout +- vLLM-Ascend version: match the repository requirements from your current checkout +- vLLM-Omni version or commit: use the commit you are deploying from + +#### Command + +Requires the `vllm-omni` package (or the `quay.io/atlas-ci/vllm-ascend` A2 / A3 container), +which provides the `vllm serve … --omni` entrypoint used below. + +Safety guardrails are **on by default** (NVIDIA Open Model License). They load +the **gated** `nvidia/Cosmos-1.0-Guardrail` model, so to keep them on you must: + +1. `pip install cosmos-guardrail` +2. Accept the license at +3. Export a token with access: `export HF_TOKEN=hf_...` + +Then launch the recommended server: + +```bash +vllm serve nvidia/Cosmos3-Nano \ + --omni \ + --host 0.0.0.0 --port 8000 \ + --init-timeout 1800 +``` + +To run **without** guardrails (you are responsible for license compliance), +add `--no-guardrails` (no token/`cosmos-guardrail` needed). For tensor parallel +add `--tensor-parallel-size 8`. `--quantization fp8` and +`--enable-layerwise-offload` are not supported on NPU. +The pipeline auto-resolves from `model_index.json`; pass +`--model-class-name Cosmos3OmniDiffusersPipeline` to force it explicitly. + +#### Verification + +Best quality uses the JSON-upsampled prompts from `assets/` (download with +`hf download nvidia/Cosmos3-Nano assets/ --local-dir Cosmos3-Nano`). Minimal +self-contained examples: + +```bash +curl http://localhost:8000/v1/models + +# Text-to-image -> /v1/images/generations (1024x1024, 10 steps; base64 PNG) +curl -sS -X POST http://localhost:8000/v1/images/generations \ + -H "Content-Type: application/json" \ + -d '{ + "model": "nvidia/Cosmos3-Nano", + "prompt": "A photorealistic red sports car on a city street at golden hour, cinematic lighting.", + "negative_prompt": "blurry, distorted, low quality", + "size": "1024x1024", "n": 1, "response_format": "b64_json", + "num_inference_steps": 10, "guidance_scale": 7.0, "seed": 42 + }' | python -c "import sys,json,base64; open('cosmos3_t2i.png','wb').write(base64.b64decode(json.load(sys.stdin)['data'][0]['b64_json']))" + +# Text-to-video -> /v1/videos/sync (720p, 49 frames @ 24fps) +curl -sS -X POST http://localhost:8000/v1/videos/sync \ + -H "Accept: video/mp4" \ + -F "model=nvidia/Cosmos3-Nano" \ + -F "prompt=A robot arm is cleaning a plate in the kitchen" \ + -F "negative_prompt=blurry, distorted, low quality, jittery, deformed" \ + -F "size=1280x720" -F "num_frames=49" -F "fps=24" \ + -F "num_inference_steps=20" -F "guidance_scale=6.0" \ + -F "max_sequence_length=4096" -F "flow_shift=10.0" \ + -F "seed=123" \ + -o cosmos3_t2v.mp4 + +# Image-to-video -> /v1/videos/sync with an uploaded reference image +curl -sS -X POST http://localhost:8000/v1/videos/sync \ + -H "Accept: video/mp4" \ + -F "model=nvidia/Cosmos3-Nano" \ + -F "prompt=The scene comes to life with smooth, natural motion." \ + -F "size=1280x720" -F "num_frames=25" -F "fps=8" \ + -F "num_inference_steps=10" -F "guidance_scale=6.0" \ + -F "seed=42" \ + -F "input_reference=@reference.jpg;type=image/jpeg" \ + -o cosmos3_i2v.mp4 + +# Video-to-video -> /v1/videos/sync with an uploaded reference video +curl -sS -X POST http://localhost:8000/v1/videos/sync \ + -H "Accept: video/mp4" \ + -F "model=nvidia/Cosmos3-Nano" \ + -F "prompt=Continue the same scene with smooth natural motion and consistent subjects." \ + -F "size=1280x720" -F "num_frames=17" -F "fps=5" \ + -F "num_inference_steps=10" -F "guidance_scale=6.0" \ + -F "seed=42" \ + -F "input_reference=@reference.mp4;type=video/mp4" \ + -o cosmos3_v2v.mp4 +``` + +#### Notes + +- **Measured latency (1x Ascend 910B / 910C, bf16, guardrails off):** + - T2I 1024² — 10 steps → ~8 s + - T2V 1280×720 @ 20 steps — 49 frames → ~55 s + - I2V 1280×720 @ 10 steps — 25 frames → ~25 s + - V2V 480×320 @ 10 steps — 17 frames → ~12 s +- **Memory:** transformer ~17 GiB (bf16); peak ~46 GiB for 720p video on 1 NPU; + full repo (transformer + Wan VAE + Qwen3-VL vision encoder + audio tokenizer) + ~33 GB on disk. +- **Determinism:** identical seed reproduces identical output on the same + hardware; outputs are not bit-identical across different GPU/NPU types. +- **Supported sizes (per model card):** 256p / 480p / 720p at 16:9, 4:3, 1:1, + 3:4, 9:16. Defaults: T2I 1024², 50 steps, guidance 7.0; T2V/I2V/V2V + 1280×720, 35 steps, guidance 6.0, `flow_shift=10.0`. +- **Key flags / params:** `--no-guardrails` (optional, to disable guardrails), `--init-timeout 1800` + (for model loading), `--tensor-parallel-size 8` for multi-NPU, and + `--model-class-name Cosmos3OmniDiffusersPipeline` to force the pipeline class. +- **Known limitations:** + - Transfer V2V with `extra_params` (`edge`/`blur`/`depth`/`seg`/`wsm`) hits a + resolution-parsing bug; basic V2V without transfer hints works. + - FP8 online quantization and layerwise offload are not supported on NPU. + +### 1x Ascend 910B / 910C (Atlas A2 / A3) — Offline generation + +#### Environment + +- OS: Linux (aarch64) +- Python: 3.12+ +- Driver / runtime: CANN 8.5.1 + NNAL + Ascend 910B / 910C +- vLLM-Omni version or commit: use the commit you are deploying from + +#### Command + +The same offline task examples run on NPU; pass model-specific knobs via +`--extra-body`. Guardrails are on by default — pass `"guardrails": false` for a +quick local run (install `cosmos-guardrail` + accept the gated repo to enable them). + +```bash +# Text-to-image -> examples/offline_inference/text_to_image +python examples/offline_inference/text_to_image/text_to_image.py \ + --model nvidia/Cosmos3-Nano \ + --prompt "A photorealistic red sports car at golden hour, cinematic lighting." \ + --negative-prompt "blurry, distorted, low quality" \ + --height 1024 --width 1024 --num-inference-steps 50 --guidance-scale 7.0 \ + --extra-body '{"flow_shift": 3.0, "guardrails": false}' \ + --output cosmos3_t2i.png + +# Text-to-video -> examples/offline_inference/text_to_video +python examples/offline_inference/text_to_video/text_to_video.py \ + --model nvidia/Cosmos3-Nano \ + --prompt "A robot arm is cleaning a plate in the kitchen." \ + --negative-prompt "blurry, distorted, low quality, jittery, deformed" \ + --height 720 --width 1280 --num-frames 189 --fps 24 \ + --num-inference-steps 35 --guidance-scale 6.0 \ + --extra-body '{"flow_shift": 10.0, "max_sequence_length": 4096, "guardrails": false, + "use_resolution_template": false, "use_duration_template": false}' \ + --output cosmos3_t2v.mp4 + +# Image-to-video -> examples/offline_inference/image_to_video +# (Cosmos3 bundles example frames under assets/; any RGB image works too.) +python examples/offline_inference/image_to_video/image_to_video.py \ + --model nvidia/Cosmos3-Nano \ + --image /path/to/Cosmos3-Nano/assets/example_i2v_input.jpg \ + --prompt "The scene comes to life with smooth, natural motion." \ + --height 720 --width 1280 --num-frames 189 --fps 24 \ + --num-inference-steps 35 --guidance-scale 6.0 \ + --extra-body '{"flow_shift": 10.0, "max_sequence_length": 4096, "guardrails": false}' \ + --output cosmos3_i2v.mp4 +``` + +#### Verification + +```bash +python -c "from PIL import Image; im=Image.open('cosmos3_t2i.png'); print('image', im.size, im.mode)" +ffprobe -v error -show_entries stream=codec_type,nb_frames,width,height cosmos3_t2v.mp4 +``` + +#### Notes + +- Guardrails are on by default on NPU with `cosmos-guardrail` installed. Pass + `"guardrails": false` in `--extra-body` to disable them (there is no + `--no-guardrails` flag for the offline scripts). +- Video at the 189-frame default takes ~15 min/clip on 1 NPU; reduce + `--num-frames` for faster iteration. diff --git a/vllm_omni/diffusion/models/cosmos3/guardrails.py b/vllm_omni/diffusion/models/cosmos3/guardrails.py index e249d2826..3c6166e85 100644 --- a/vllm_omni/diffusion/models/cosmos3/guardrails.py +++ b/vllm_omni/diffusion/models/cosmos3/guardrails.py @@ -21,6 +21,7 @@ from vllm.logger import init_logger from vllm_omni.diffusion.models.progress_bar import _is_rank_zero from vllm_omni.errors import GuardrailViolationError +from vllm_omni.platforms import current_omni_platform if TYPE_CHECKING: from vllm_omni.diffusion.data import OmniDiffusionConfig @@ -28,6 +29,28 @@ if TYPE_CHECKING: logger = init_logger(__name__) +# --------------------------------------------------------------------------- +# NPU/XPU compatibility: cosmos_guardrail.cosmos_utils.load_model calls +# ``torch.load(path, weights_only=True)`` without ``map_location``. +# On NPU/XPU, ``torch.cuda.is_available()`` is False, so deserializing a +# CUDA checkpoint with ``weights_only=True`` raises an UnpicklingError. +# Work around by patching ``torch.load`` to default ``map_location="cpu"`` +# when CUDA is unavailable and no explicit map_location was given. +# --------------------------------------------------------------------------- +_original_torch_load = torch.load + + +def _patched_torch_load(*args, **kwargs): + if ( + "map_location" not in kwargs + and not torch.cuda.is_available() + and (current_omni_platform.is_npu() or current_omni_platform.is_xpu()) + ): + kwargs["map_location"] = "cpu" + return _original_torch_load(*args, **kwargs) + + +torch.load = _patched_torch_load # type: ignore[assignment] try: from cosmos_guardrail import CosmosSafetyChecker @@ -75,7 +98,7 @@ def _build_text_guardrail(checker: Any) -> TextGuardrailFn: def _build_video_guardrail(checker: Any, offload_to_cpu: bool) -> VideoGuardrailFn: video_models = _nn_models(checker.video_guardrail) - compute_device = "cuda" + compute_device = current_omni_platform.device_type def video_guardrail(frames: np.ndarray) -> np.ndarray: if offload_to_cpu: @@ -112,9 +135,9 @@ def _init_default_guardrails(offload_to_cpu: bool = False) -> None: checker = CosmosSafetyChecker() # Place text models on their resting device permanently. Video models - # idle on CPU when offload is on and move to GPU per-call (handled in + # idle on CPU when offload is on and move to compute device per-call (handled in # the video guardrail closure). - idle_device = "cpu" if offload_to_cpu else "cuda" + idle_device = "cpu" if offload_to_cpu else current_omni_platform.device_type for m in _nn_models(checker.text_guardrail): m.to(idle_device) for m in _nn_models(checker.video_guardrail):