[Diffusion][Attention] Support per-role attention backend via CLI (#2681)
Signed-off-by: gcanlin <canlinguosdu@gmail.com> Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
26d481fc84
commit
11c4c7f0ff
29 files changed
+1262
-106
No files matched your search
@@ -370,6 +370,32 @@ class OmniServeCommand(CLISubcommand):
|
||||
help="Number of replica groups for HSDP. Each group holds a full sharded copy.",
|
||||
)
|
||||
|
||||
# Attention backend configuration
|
||||
omni_config_group.add_argument(
|
||||
"--diffusion-attention-backend",
|
||||
dest="diffusion_attention_backend",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Diffusion attention backend (shorthand). "
|
||||
"Sets the default backend for all diffusion attention roles, e.g. 'FLASH_ATTN'. "
|
||||
"May be combined with --diffusion-attention-config.per_role.* overrides, "
|
||||
"but mutually exclusive with --diffusion-attention-config.default.backend.",
|
||||
)
|
||||
omni_config_group.add_argument(
|
||||
"--diffusion-attention-config",
|
||||
"-dac",
|
||||
dest="diffusion_attention_config",
|
||||
type=json.loads,
|
||||
default=None,
|
||||
help="Diffusion attention config. Accepts JSON or vLLM-style dotted flags. "
|
||||
"Examples: "
|
||||
"--diffusion-attention-config.default.backend FLASH_ATTN, "
|
||||
"--diffusion-attention-config.per_role.self.backend SPARSE_BLOCK, "
|
||||
"--diffusion-attention-config.per_role.cross.backend SAGE_ATTN, "
|
||||
'--diffusion-attention-config \'{"default": {"backend": "FLASH_ATTN"}, '
|
||||
'"per_role": {"cross": {"backend": "SAGE_ATTN"}}}\'.',
|
||||
)
|
||||
|
||||
# Cache optimization parameters
|
||||
omni_config_group.add_argument(
|
||||
"--cache-backend",
|
||||
|
||||
Reference in new issue
Block a user