Signed-off-by: lyj-jjj <liuyingjun5@huawei.com> Signed-off-by: gcanlin <canlinguosdu@gmail.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: gcanlin <canlinguosdu@gmail.com>
This commit is contained in:
20 files changed
+823
-25
No files matched your search
@@ -32,6 +32,54 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def parse_kv_cache_skip_selector(
|
||||
selector: str | list[int] | tuple[int, ...] | set[int] | None,
|
||||
) -> set[int] | None:
|
||||
"""Parse a non-negative index selector such as "0-9,20,25-30"."""
|
||||
if selector is None:
|
||||
return None
|
||||
if isinstance(selector, set):
|
||||
values = selector
|
||||
elif isinstance(selector, (list, tuple)):
|
||||
values = set(selector)
|
||||
elif isinstance(selector, str):
|
||||
text = selector.strip()
|
||||
if not text:
|
||||
return None
|
||||
values: set[int] = set()
|
||||
for chunk in text.split(","):
|
||||
token = chunk.strip()
|
||||
if not token:
|
||||
continue
|
||||
if "-" in token:
|
||||
start_str, end_str = token.split("-", 1)
|
||||
try:
|
||||
start = int(start_str.strip())
|
||||
end = int(end_str.strip())
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"Invalid range token '{token}' in selector '{selector}'.") from exc
|
||||
if start < 0 or end < 0 or start > end:
|
||||
raise ValueError(f"Invalid range token '{token}' in selector '{selector}'.")
|
||||
values.update(range(start, end + 1))
|
||||
else:
|
||||
try:
|
||||
index = int(token)
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"Invalid index token '{token}' in selector '{selector}'.") from exc
|
||||
if index < 0:
|
||||
raise ValueError(f"Negative index '{index}' is not allowed in selector '{selector}'.")
|
||||
values.add(index)
|
||||
else:
|
||||
raise TypeError(f"Unsupported selector type: {type(selector)!r}")
|
||||
|
||||
for idx in values:
|
||||
if not isinstance(idx, int):
|
||||
raise TypeError(f"Selector index must be int, got {type(idx)!r}")
|
||||
if idx < 0:
|
||||
raise ValueError("Selector indices must be non-negative.")
|
||||
return values
|
||||
|
||||
|
||||
@config
|
||||
@dataclass
|
||||
class DiffusionParallelConfig:
|
||||
@@ -519,6 +567,19 @@ class OmniDiffusionConfig:
|
||||
# has already resolved to vLLM's ModelOpt FP8 linear method.
|
||||
force_cutlass_fp8: bool = False
|
||||
|
||||
# KV cache dtype for attention. Aligned with upstream vLLM's --kv-cache-dtype.
|
||||
# None = native dtype (no quantization).
|
||||
# "fp8" = dynamic FP8 (float8_e4m3fn) quantization per forward pass.
|
||||
# On Hopper+FA3: native FP8 attention (memory + compute savings).
|
||||
# On other backends: no benefit, backends skip quantization.
|
||||
kv_cache_dtype: str | None = None
|
||||
# Optional skip selectors for KV-cache quantization. Format: "0-9,20,25-30".
|
||||
# Listed steps/layers skip quantization; others keep quantized execution.
|
||||
kv_cache_skip_steps: str | None = None
|
||||
kv_cache_skip_layers: str | None = None
|
||||
kv_cache_skip_step_indices: set[int] | None = None
|
||||
kv_cache_skip_layer_indices: set[int] | None = None
|
||||
|
||||
# Diffusion pipeline Profiling config
|
||||
enable_diffusion_pipeline_profiler: bool = False
|
||||
|
||||
@@ -527,6 +588,7 @@ class OmniDiffusionConfig:
|
||||
|
||||
# sleep mode
|
||||
enable_sleep_mode: bool = False
|
||||
|
||||
# Maximum number of sequences to generate in a batch
|
||||
max_num_seqs: int = 1
|
||||
|
||||
@@ -656,6 +718,8 @@ class OmniDiffusionConfig:
|
||||
# Match vLLM's config flow: parse entrypoint shorthands before the
|
||||
# config object is built, and keep a single runtime truth source.
|
||||
self.diffusion_attention_config = build_attention_config(self.diffusion_attention_config)
|
||||
self.kv_cache_skip_step_indices = parse_kv_cache_skip_selector(self.kv_cache_skip_steps)
|
||||
self.kv_cache_skip_layer_indices = parse_kv_cache_skip_selector(self.kv_cache_skip_layers)
|
||||
|
||||
if self.max_cpu_loras is None:
|
||||
self.max_cpu_loras = 1
|
||||
|
||||
Reference in new issue
Block a user