Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0527aadf6b | ||
|
|
6b95de36a6 |
No files matched your search
@@ -12,7 +12,7 @@ implementations to reduce numerical drift and sampling divergence.
|
|||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
pip install kernels==0.14.1
|
pip install kernels==0.15.2
|
||||||
```
|
```
|
||||||
|
|
||||||
If `kernels` is unavailable, an explicit `FLASH_ATTN_HUB` or
|
If `kernels` is unavailable, an explicit `FLASH_ATTN_HUB` or
|
||||||
|
|||||||
@@ -21,7 +21,6 @@ from vllm.multimodal.media.audio import load_audio
|
|||||||
from vllm.sampling_params import SamplingParams
|
from vllm.sampling_params import SamplingParams
|
||||||
|
|
||||||
from vllm_omni.entrypoints.omni import Omni
|
from vllm_omni.entrypoints.omni import Omni
|
||||||
from vllm_omni.utils.profiler import add_profiler_config_arg
|
|
||||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
|
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
|
||||||
|
|
||||||
SEED = 42
|
SEED = 42
|
||||||
@@ -326,7 +325,6 @@ def main(args):
|
|||||||
else:
|
else:
|
||||||
query_result = query_func()
|
query_result = query_func()
|
||||||
args.quantization_config = quantization_config
|
args.quantization_config = quantization_config
|
||||||
# ``vars(args)`` below forwards profiler_config to Omni.
|
|
||||||
omni_kwargs = vars(args).copy()
|
omni_kwargs = vars(args).copy()
|
||||||
# Override CLI --model with the derived model_name.
|
# Override CLI --model with the derived model_name.
|
||||||
omni_kwargs["model"] = model_name
|
omni_kwargs["model"] = model_name
|
||||||
@@ -380,11 +378,11 @@ def main(args):
|
|||||||
for i, prompt in enumerate(prompts):
|
for i, prompt in enumerate(prompts):
|
||||||
prompt["modalities"] = output_modalities
|
prompt["modalities"] = output_modalities
|
||||||
|
|
||||||
profiler_enabled = args.profiler_config is not None
|
profiler_enabled = bool(os.getenv("VLLM_TORCH_PROFILER_DIR"))
|
||||||
if profiler_enabled and hasattr(omni, "start_profile"):
|
if profiler_enabled and hasattr(omni, "start_profile"):
|
||||||
omni.start_profile(stages=[0])
|
omni.start_profile(stages=[0])
|
||||||
elif profiler_enabled:
|
elif profiler_enabled:
|
||||||
print("[Warn] --profiler-config was provided, but current engine does not support profiler controls.")
|
print("[Warn] VLLM_TORCH_PROFILER_DIR is set, but current engine does not support profiler controls.")
|
||||||
omni_generator = omni.generate(prompts, sampling_params_list, py_generator=args.py_generator)
|
omni_generator = omni.generate(prompts, sampling_params_list, py_generator=args.py_generator)
|
||||||
|
|
||||||
# Determine output directory: prefer --output-dir; fallback to --output-wav
|
# Determine output directory: prefer --output-dir; fallback to --output-wav
|
||||||
@@ -460,7 +458,6 @@ def parse_args():
|
|||||||
default=False,
|
default=False,
|
||||||
help="Enable writing detailed statistics (default: disabled)",
|
help="Enable writing detailed statistics (default: disabled)",
|
||||||
)
|
)
|
||||||
add_profiler_config_arg(parser)
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--stage-init-timeout",
|
"--stage-init-timeout",
|
||||||
type=int,
|
type=int,
|
||||||
|
|||||||
@@ -14,7 +14,6 @@ from vllm_omni.entrypoints.omni import Omni
|
|||||||
from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokenizer
|
from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokenizer
|
||||||
from vllm_omni.model_executor.models.cosyvoice3.utils import extract_text_token
|
from vllm_omni.model_executor.models.cosyvoice3.utils import extract_text_token
|
||||||
from vllm_omni.transformers_utils.configs.cosyvoice3 import CosyVoice3Config
|
from vllm_omni.transformers_utils.configs.cosyvoice3 import CosyVoice3Config
|
||||||
from vllm_omni.utils.profiler import add_profiler_config_arg
|
|
||||||
|
|
||||||
# Upstream zero-shot reference clip
|
# Upstream zero-shot reference clip
|
||||||
ZERO_SHOT_PROMPT_URL = "https://raw.githubusercontent.com/FunAudioLLM/CosyVoice/main/asset/zero_shot_prompt.wav"
|
ZERO_SHOT_PROMPT_URL = "https://raw.githubusercontent.com/FunAudioLLM/CosyVoice/main/asset/zero_shot_prompt.wav"
|
||||||
@@ -46,7 +45,6 @@ def run_e2e():
|
|||||||
help="Override the deploy config path. If unset, auto-loads "
|
help="Override the deploy config path. If unset, auto-loads "
|
||||||
"vllm_omni/deploy/cosyvoice3.yaml based on the HF model_type.",
|
"vllm_omni/deploy/cosyvoice3.yaml based on the HF model_type.",
|
||||||
)
|
)
|
||||||
add_profiler_config_arg(parser)
|
|
||||||
parser.add_argument("--text", type=str, default="Hello, this is a test of the CosyVoice system capability.")
|
parser.add_argument("--text", type=str, default="Hello, this is a test of the CosyVoice system capability.")
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--prompt-text",
|
"--prompt-text",
|
||||||
@@ -81,7 +79,6 @@ def run_e2e():
|
|||||||
deploy_config=args.deploy_config,
|
deploy_config=args.deploy_config,
|
||||||
tokenizer=args.tokenizer,
|
tokenizer=args.tokenizer,
|
||||||
log_stats=True,
|
log_stats=True,
|
||||||
profiler_config=args.profiler_config,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
sampling_cfg = {"top_p": 0.8, "top_k": 25, "eos_token_id": 6561 + 1}
|
sampling_cfg = {"top_p": 0.8, "top_k": 25, "eos_token_id": 6561 + 1}
|
||||||
@@ -152,8 +149,8 @@ def run_e2e():
|
|||||||
|
|
||||||
sampling_params_list = [gpt_sampling, s2mel_sampling]
|
sampling_params_list = [gpt_sampling, s2mel_sampling]
|
||||||
|
|
||||||
profiler_enabled = args.profiler_config is not None
|
# Start profiling (requires VLLM_TORCH_PROFILER_DIR env var)
|
||||||
if profiler_enabled:
|
if os.environ.get("VLLM_TORCH_PROFILER_DIR"):
|
||||||
print("Starting profiler...")
|
print("Starting profiler...")
|
||||||
omni.start_profile()
|
omni.start_profile()
|
||||||
|
|
||||||
@@ -161,7 +158,7 @@ def run_e2e():
|
|||||||
outputs = list(omni.generate(prompts, sampling_params_list=sampling_params_list[:2]))
|
outputs = list(omni.generate(prompts, sampling_params_list=sampling_params_list[:2]))
|
||||||
|
|
||||||
# Stop profiling and get results
|
# Stop profiling and get results
|
||||||
if profiler_enabled:
|
if os.environ.get("VLLM_TORCH_PROFILER_DIR"):
|
||||||
print("Stopping profiler...")
|
print("Stopping profiler...")
|
||||||
profile_results = omni.stop_profile()
|
profile_results = omni.stop_profile()
|
||||||
print(f"Profile traces saved to: {profile_results}")
|
print(f"Profile traces saved to: {profile_results}")
|
||||||
|
|||||||
@@ -13,8 +13,9 @@ transformers >= 5.10.1, < 5.15
|
|||||||
av>=14.0.0
|
av>=14.0.0
|
||||||
omegaconf>=2.3.0
|
omegaconf>=2.3.0
|
||||||
diffusers==0.40.0
|
diffusers==0.40.0
|
||||||
# Hub kernels, including LTX-2.5 NATTEN
|
# Hub kernels, including LTX-2.5 NATTEN. Keep this pin in sync with the
|
||||||
kernels==0.14.1
|
# supported transformers range above.
|
||||||
|
kernels==0.15.2
|
||||||
safetensors>=0.8.0
|
safetensors>=0.8.0
|
||||||
accelerate==1.12.0
|
accelerate==1.12.0
|
||||||
soundfile>=0.13.1
|
soundfile>=0.13.1
|
||||||
|
|||||||
@@ -386,7 +386,7 @@ def test_ltx25_natten_failure_has_actionable_omni_remedy(monkeypatch):
|
|||||||
MissingNattenProcessor,
|
MissingNattenProcessor,
|
||||||
)
|
)
|
||||||
|
|
||||||
with pytest.raises(RuntimeError, match=r"kernels==0\.14\.1.*supported GPU.*allow Hub access"):
|
with pytest.raises(RuntimeError, match=r"kernels==0\.15\.2.*supported GPU.*allow Hub access"):
|
||||||
ltx2_components._create_ltx25_natten_processor()
|
ltx2_components._create_ltx25_natten_processor()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,23 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
|
||||||
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
|
||||||
|
|
||||||
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
||||||
_EXAMPLES = (
|
|
||||||
"examples/offline_inference/text_to_speech/cosyvoice3/end2end.py",
|
|
||||||
"examples/offline_inference/qwen2_5_omni/end2end.py",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("relative_path", _EXAMPLES)
|
|
||||||
def test_offline_examples_use_typed_profiler_config(relative_path: str):
|
|
||||||
source = (_REPO_ROOT / relative_path).read_text(encoding="utf-8")
|
|
||||||
|
|
||||||
assert "add_profiler_config_arg(parser)" in source
|
|
||||||
assert "profiler_enabled = args.profiler_config is not None" in source
|
|
||||||
assert "VLLM_TORCH_PROFILER_DIR" not in source
|
|
||||||
@@ -298,7 +298,7 @@ def _create_ltx25_natten_processor() -> LTX2VideoVaeNeighborhoodNattenProcessor:
|
|||||||
except (ImportError, OSError, RuntimeError, ValueError) as exc:
|
except (ImportError, OSError, RuntimeError, ValueError) as exc:
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
"LTX-2.5 DiffVAE requires the shi-labs/natten Hub kernel. "
|
"LTX-2.5 DiffVAE requires the shi-labs/natten Hub kernel. "
|
||||||
"Install kernels==0.14.1, use a supported GPU, leave "
|
"Install kernels==0.15.2, use a supported GPU, leave "
|
||||||
"DIFFUSERS_DISABLE_REMOTE_CODE unset, and allow Hub access during kernel initialization."
|
"DIFFUSERS_DISABLE_REMOTE_CODE unset, and allow Hub access during kernel initialization."
|
||||||
) from exc
|
) from exc
|
||||||
|
|
||||||
|
|||||||
@@ -1,37 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import json
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
|
|
||||||
def parse_json_object(value: str, flag_name: str = "argument") -> dict[str, Any]:
|
|
||||||
"""Parse a CLI value as a JSON object, attributing errors to ``flag_name``."""
|
|
||||||
try:
|
|
||||||
config = json.loads(value)
|
|
||||||
except json.JSONDecodeError as e:
|
|
||||||
raise argparse.ArgumentTypeError(f"{flag_name} must be valid JSON: {e}") from e
|
|
||||||
if not isinstance(config, dict):
|
|
||||||
raise argparse.ArgumentTypeError(f"{flag_name} must be a JSON object")
|
|
||||||
return config
|
|
||||||
|
|
||||||
|
|
||||||
def parse_profiler_config(value: str) -> dict[str, Any]:
|
|
||||||
"""Parse the JSON object passed to ``--profiler-config``."""
|
|
||||||
return parse_json_object(value, flag_name="--profiler-config")
|
|
||||||
|
|
||||||
|
|
||||||
def add_profiler_config_arg(parser: argparse.ArgumentParser) -> None:
|
|
||||||
"""Add the shared JSON profiler configuration option to ``parser``."""
|
|
||||||
parser.add_argument(
|
|
||||||
"--profiler-config",
|
|
||||||
type=parse_profiler_config,
|
|
||||||
default=None,
|
|
||||||
help=(
|
|
||||||
"JSON profiler config for torch/cuda profiling, e.g. "
|
|
||||||
'\'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.'
|
|
||||||
),
|
|
||||||
)
|
|
||||||
Reference in new issue
Block a user