[Core] Add realtime AR-Diffusion tick sessions for LingBot World 2.0 (#5491)
Signed-off-by: zhangbin.327 <zhangbin.327@bytedance.com> Signed-off-by: Jack47 <scsvip@gmail.com> Co-authored-by: zhangbin.327 <zhangbin.327@bytedance.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
53 files changed
+10754
-129
No files matched your search
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"_class_name": "LingBotWorldCausalDMDPipeline",
|
||||
"_diffusers_version": "0.35.0.dev0",
|
||||
"boundary_ratio": 0.9,
|
||||
"image_encoder": [null, null],
|
||||
"image_processor": [null, null],
|
||||
"scheduler": ["diffusers", "UniPCMultistepScheduler"],
|
||||
"text_encoder": ["transformers", "UMT5EncoderModel"],
|
||||
"tokenizer": ["transformers", "T5TokenizerFast"],
|
||||
"transformer": ["diffusers", "CausalLingBotWorldTransformer3DModel"],
|
||||
"transformer_2": [null, null],
|
||||
"vae": ["diffusers", "AutoencoderKLWan"]
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"sources": {
|
||||
"config": "https://huggingface.co/robbyant/lingbot-world-v2-14b-causal-fast-diffusers/resolve/main/transformer/config.json",
|
||||
"shard_1_header": "https://huggingface.co/robbyant/lingbot-world-v2-14b-causal-fast-diffusers/resolve/main/transformer/diffusion_pytorch_model-00001-of-00008.safetensors",
|
||||
"shard_8_header": "https://huggingface.co/robbyant/lingbot-world-v2-14b-causal-fast-diffusers/resolve/main/transformer/diffusion_pytorch_model-00008-of-00008.safetensors"
|
||||
},
|
||||
"dtype": "F32",
|
||||
"shard_representatives": {
|
||||
"diffusion_pytorch_model-00001-of-00008.safetensors": "blocks.0.cam_injector_layer1.bias",
|
||||
"diffusion_pytorch_model-00002-of-00008.safetensors": "blocks.4.cam_injector_layer1.bias",
|
||||
"diffusion_pytorch_model-00003-of-00008.safetensors": "blocks.10.cam_injector_layer1.bias",
|
||||
"diffusion_pytorch_model-00004-of-00008.safetensors": "blocks.14.cam_scale_layer.bias",
|
||||
"diffusion_pytorch_model-00005-of-00008.safetensors": "blocks.20.cam_injector_layer1.bias",
|
||||
"diffusion_pytorch_model-00006-of-00008.safetensors": "blocks.25.cam_injector_layer1.bias",
|
||||
"diffusion_pytorch_model-00007-of-00008.safetensors": "blocks.30.cam_injector_layer1.bias",
|
||||
"diffusion_pytorch_model-00008-of-00008.safetensors": "blocks.35.cam_injector_layer1.bias"
|
||||
},
|
||||
"parameter_shapes": {
|
||||
"blocks.0.cam_injector_layer1.bias": [5120],
|
||||
"blocks.4.cam_injector_layer1.bias": [5120],
|
||||
"blocks.10.cam_injector_layer1.bias": [5120],
|
||||
"blocks.14.cam_scale_layer.bias": [5120],
|
||||
"blocks.20.cam_injector_layer1.bias": [5120],
|
||||
"blocks.25.cam_injector_layer1.bias": [5120],
|
||||
"blocks.30.cam_injector_layer1.bias": [5120],
|
||||
"blocks.35.cam_injector_layer1.bias": [5120],
|
||||
"patch_embedding.weight": [5120, 36, 1, 2, 2],
|
||||
"patch_embedding_wancamctrl.weight": [5120, 1536],
|
||||
"c2ws_hidden_states_layer1.weight": [5120, 5120],
|
||||
"c2ws_hidden_states_layer2.weight": [5120, 5120],
|
||||
"blocks.0.modulation": [1, 6, 5120],
|
||||
"blocks.0.self_attn.q.weight": [5120, 5120],
|
||||
"blocks.0.ffn.0.weight": [13824, 5120],
|
||||
"blocks.0.ffn.2.weight": [5120, 13824],
|
||||
"blocks.0.cam_injector_layer1.weight": [5120, 5120],
|
||||
"blocks.0.cam_injector_layer2.weight": [5120, 5120],
|
||||
"blocks.0.cam_scale_layer.weight": [5120, 5120],
|
||||
"blocks.0.cam_shift_layer.weight": [5120, 5120],
|
||||
"head.modulation": [1, 2, 5120],
|
||||
"head.head.weight": [64, 5120]
|
||||
}
|
||||
}
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"_class_name": "UniPCMultistepScheduler",
|
||||
"_diffusers_version": "0.35.0.dev0",
|
||||
"beta_end": 0.02,
|
||||
"beta_schedule": "linear",
|
||||
"beta_start": 0.0001,
|
||||
"disable_corrector": [],
|
||||
"dynamic_thresholding_ratio": 0.995,
|
||||
"final_sigmas_type": "zero",
|
||||
"flow_shift": 5.0,
|
||||
"lower_order_final": true,
|
||||
"num_train_timesteps": 1000,
|
||||
"predict_x0": true,
|
||||
"prediction_type": "flow_prediction",
|
||||
"rescale_betas_zero_snr": false,
|
||||
"sample_max_value": 1.0,
|
||||
"solver_order": 2,
|
||||
"solver_p": null,
|
||||
"solver_type": "bh2",
|
||||
"steps_offset": 0,
|
||||
"thresholding": false,
|
||||
"time_shift_type": "exponential",
|
||||
"timestep_spacing": "linspace",
|
||||
"trained_betas": null,
|
||||
"use_beta_sigmas": false,
|
||||
"use_dynamic_shifting": false,
|
||||
"use_exponential_sigmas": false,
|
||||
"use_flow_sigmas": true,
|
||||
"use_karras_sigmas": false
|
||||
}
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
{
|
||||
"source": "https://huggingface.co/robbyant/lingbot-world-v2-14b-causal-fast-diffusers/resolve/main/transformer/diffusion_pytorch_model.safetensors.index.json",
|
||||
"index_parameter_count": 1421,
|
||||
"canonical_name_set_sha256": "5d31b7f64fd5e5e271b789b2dbe50fb7bbf146bd8b750646bdc9abb4e5b8139d",
|
||||
"canonical_name_to_shard_sha256": "e1addc27d2b1fad6f10226a23a265ee3e3c61e74dce4d5859727bdf180a67992",
|
||||
"canonical_serialization": "sorted lines formatted as parameter_name=shard_filename with LF separators and no trailing LF",
|
||||
"shards": {
|
||||
"diffusion_pytorch_model-00001-of-00008.safetensors": {
|
||||
"parameter_count": 180,
|
||||
"parameter_names_sha256": "81a3d4778de604e75609aae7dac3d8dcbc081811e7748267fe953bbeb375e79d"
|
||||
},
|
||||
"diffusion_pytorch_model-00002-of-00008.safetensors": {
|
||||
"parameter_count": 180,
|
||||
"parameter_names_sha256": "4e36e2596a6468f29179fa5795396f2e9a7f7d8debc912b34a6f6c2c7c7c610d"
|
||||
},
|
||||
"diffusion_pytorch_model-00003-of-00008.safetensors": {
|
||||
"parameter_count": 180,
|
||||
"parameter_names_sha256": "6f74a7b2cf7444c2daa2b3fc32bd3787589b35543c230c5c13b89f0f0f2a0998"
|
||||
},
|
||||
"diffusion_pytorch_model-00004-of-00008.safetensors": {
|
||||
"parameter_count": 180,
|
||||
"parameter_names_sha256": "ec5c967ed5271ba7834337c4ee93a73f8ed0eead4492d4920624251c65c058ac"
|
||||
},
|
||||
"diffusion_pytorch_model-00005-of-00008.safetensors": {
|
||||
"parameter_count": 180,
|
||||
"parameter_names_sha256": "ad1fdab5830dd9ce43ea007f112b4be223327b7c8197c766d9cf9219e74d619b"
|
||||
},
|
||||
"diffusion_pytorch_model-00006-of-00008.safetensors": {
|
||||
"parameter_count": 180,
|
||||
"parameter_names_sha256": "fd58f0d86be42d8afcf55d3e07d8a22f7f6ef5064da931a600c890b71b6dfbae"
|
||||
},
|
||||
"diffusion_pytorch_model-00007-of-00008.safetensors": {
|
||||
"parameter_count": 180,
|
||||
"parameter_names_sha256": "305a1714b5f4fb54d5877c593b451273ab1490d90fd0896a299d96e140295047"
|
||||
},
|
||||
"diffusion_pytorch_model-00008-of-00008.safetensors": {
|
||||
"parameter_count": 161,
|
||||
"parameter_names_sha256": "4c18d94d96168682eb212f93e3cdf7313928776d8d9e6e4de0255fe899decb25"
|
||||
}
|
||||
},
|
||||
"top_level_modules": [
|
||||
"blocks",
|
||||
"c2ws_hidden_states_layer1",
|
||||
"c2ws_hidden_states_layer2",
|
||||
"head",
|
||||
"patch_embedding",
|
||||
"patch_embedding_wancamctrl",
|
||||
"text_embedding",
|
||||
"time_embedding",
|
||||
"time_projection"
|
||||
],
|
||||
"non_block_parameters": [
|
||||
"c2ws_hidden_states_layer1.bias",
|
||||
"c2ws_hidden_states_layer1.weight",
|
||||
"c2ws_hidden_states_layer2.bias",
|
||||
"c2ws_hidden_states_layer2.weight",
|
||||
"head.head.bias",
|
||||
"head.head.weight",
|
||||
"head.modulation",
|
||||
"patch_embedding.bias",
|
||||
"patch_embedding.weight",
|
||||
"patch_embedding_wancamctrl.bias",
|
||||
"patch_embedding_wancamctrl.weight",
|
||||
"text_embedding.0.bias",
|
||||
"text_embedding.0.weight",
|
||||
"text_embedding.2.bias",
|
||||
"text_embedding.2.weight",
|
||||
"time_embedding.0.bias",
|
||||
"time_embedding.0.weight",
|
||||
"time_embedding.2.bias",
|
||||
"time_embedding.2.weight",
|
||||
"time_projection.1.bias",
|
||||
"time_projection.1.weight"
|
||||
],
|
||||
"block_parameter_suffixes": [
|
||||
"cam_injector_layer1.bias",
|
||||
"cam_injector_layer1.weight",
|
||||
"cam_injector_layer2.bias",
|
||||
"cam_injector_layer2.weight",
|
||||
"cam_scale_layer.bias",
|
||||
"cam_scale_layer.weight",
|
||||
"cam_shift_layer.bias",
|
||||
"cam_shift_layer.weight",
|
||||
"cross_attn.k.bias",
|
||||
"cross_attn.k.weight",
|
||||
"cross_attn.norm_k.weight",
|
||||
"cross_attn.norm_q.weight",
|
||||
"cross_attn.o.bias",
|
||||
"cross_attn.o.weight",
|
||||
"cross_attn.q.bias",
|
||||
"cross_attn.q.weight",
|
||||
"cross_attn.v.bias",
|
||||
"cross_attn.v.weight",
|
||||
"ffn.0.bias",
|
||||
"ffn.0.weight",
|
||||
"ffn.2.bias",
|
||||
"ffn.2.weight",
|
||||
"modulation",
|
||||
"norm3.bias",
|
||||
"norm3.weight",
|
||||
"self_attn.k.bias",
|
||||
"self_attn.k.weight",
|
||||
"self_attn.norm_k.weight",
|
||||
"self_attn.norm_q.weight",
|
||||
"self_attn.o.bias",
|
||||
"self_attn.o.weight",
|
||||
"self_attn.q.bias",
|
||||
"self_attn.q.weight",
|
||||
"self_attn.v.bias",
|
||||
"self_attn.v.weight"
|
||||
]
|
||||
}
|
||||
Reference in new issue
Block a user