[Core] Add realtime AR-Diffusion tick sessions for LingBot World 2.0 (#5491)

Signed-off-by: zhangbin.327 <zhangbin.327@bytedance.com>
Signed-off-by: Jack47 <scsvip@gmail.com>
Co-authored-by: zhangbin.327 <zhangbin.327@bytedance.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
authored and GitHub committed 2026-08-16 09:11:20 +08:00
1 parent 9798bc3a8d
commit 3875f8ded4
53 files changed
+10754 -129

No files matched your search

@@ -0,0 +1,13 @@
{
"_class_name": "LingBotWorldCausalDMDPipeline",
"_diffusers_version": "0.35.0.dev0",
"boundary_ratio": 0.9,
"image_encoder": [null, null],
"image_processor": [null, null],
"scheduler": ["diffusers", "UniPCMultistepScheduler"],
"text_encoder": ["transformers", "UMT5EncoderModel"],
"tokenizer": ["transformers", "T5TokenizerFast"],
"transformer": ["diffusers", "CausalLingBotWorldTransformer3DModel"],
"transformer_2": [null, null],
"vae": ["diffusers", "AutoencoderKLWan"]
}
@@ -0,0 +1,42 @@
{
"sources": {
"config": "https://huggingface.co/robbyant/lingbot-world-v2-14b-causal-fast-diffusers/resolve/main/transformer/config.json",
"shard_1_header": "https://huggingface.co/robbyant/lingbot-world-v2-14b-causal-fast-diffusers/resolve/main/transformer/diffusion_pytorch_model-00001-of-00008.safetensors",
"shard_8_header": "https://huggingface.co/robbyant/lingbot-world-v2-14b-causal-fast-diffusers/resolve/main/transformer/diffusion_pytorch_model-00008-of-00008.safetensors"
},
"dtype": "F32",
"shard_representatives": {
"diffusion_pytorch_model-00001-of-00008.safetensors": "blocks.0.cam_injector_layer1.bias",
"diffusion_pytorch_model-00002-of-00008.safetensors": "blocks.4.cam_injector_layer1.bias",
"diffusion_pytorch_model-00003-of-00008.safetensors": "blocks.10.cam_injector_layer1.bias",
"diffusion_pytorch_model-00004-of-00008.safetensors": "blocks.14.cam_scale_layer.bias",
"diffusion_pytorch_model-00005-of-00008.safetensors": "blocks.20.cam_injector_layer1.bias",
"diffusion_pytorch_model-00006-of-00008.safetensors": "blocks.25.cam_injector_layer1.bias",
"diffusion_pytorch_model-00007-of-00008.safetensors": "blocks.30.cam_injector_layer1.bias",
"diffusion_pytorch_model-00008-of-00008.safetensors": "blocks.35.cam_injector_layer1.bias"
},
"parameter_shapes": {
"blocks.0.cam_injector_layer1.bias": [5120],
"blocks.4.cam_injector_layer1.bias": [5120],
"blocks.10.cam_injector_layer1.bias": [5120],
"blocks.14.cam_scale_layer.bias": [5120],
"blocks.20.cam_injector_layer1.bias": [5120],
"blocks.25.cam_injector_layer1.bias": [5120],
"blocks.30.cam_injector_layer1.bias": [5120],
"blocks.35.cam_injector_layer1.bias": [5120],
"patch_embedding.weight": [5120, 36, 1, 2, 2],
"patch_embedding_wancamctrl.weight": [5120, 1536],
"c2ws_hidden_states_layer1.weight": [5120, 5120],
"c2ws_hidden_states_layer2.weight": [5120, 5120],
"blocks.0.modulation": [1, 6, 5120],
"blocks.0.self_attn.q.weight": [5120, 5120],
"blocks.0.ffn.0.weight": [13824, 5120],
"blocks.0.ffn.2.weight": [5120, 13824],
"blocks.0.cam_injector_layer1.weight": [5120, 5120],
"blocks.0.cam_injector_layer2.weight": [5120, 5120],
"blocks.0.cam_scale_layer.weight": [5120, 5120],
"blocks.0.cam_shift_layer.weight": [5120, 5120],
"head.modulation": [1, 2, 5120],
"head.head.weight": [64, 5120]
}
}
@@ -0,0 +1,30 @@
{
"_class_name": "UniPCMultistepScheduler",
"_diffusers_version": "0.35.0.dev0",
"beta_end": 0.02,
"beta_schedule": "linear",
"beta_start": 0.0001,
"disable_corrector": [],
"dynamic_thresholding_ratio": 0.995,
"final_sigmas_type": "zero",
"flow_shift": 5.0,
"lower_order_final": true,
"num_train_timesteps": 1000,
"predict_x0": true,
"prediction_type": "flow_prediction",
"rescale_betas_zero_snr": false,
"sample_max_value": 1.0,
"solver_order": 2,
"solver_p": null,
"solver_type": "bh2",
"steps_offset": 0,
"thresholding": false,
"time_shift_type": "exponential",
"timestep_spacing": "linspace",
"trained_betas": null,
"use_beta_sigmas": false,
"use_dynamic_shifting": false,
"use_exponential_sigmas": false,
"use_flow_sigmas": true,
"use_karras_sigmas": false
}
@@ -0,0 +1,112 @@
{
"source": "https://huggingface.co/robbyant/lingbot-world-v2-14b-causal-fast-diffusers/resolve/main/transformer/diffusion_pytorch_model.safetensors.index.json",
"index_parameter_count": 1421,
"canonical_name_set_sha256": "5d31b7f64fd5e5e271b789b2dbe50fb7bbf146bd8b750646bdc9abb4e5b8139d",
"canonical_name_to_shard_sha256": "e1addc27d2b1fad6f10226a23a265ee3e3c61e74dce4d5859727bdf180a67992",
"canonical_serialization": "sorted lines formatted as parameter_name=shard_filename with LF separators and no trailing LF",
"shards": {
"diffusion_pytorch_model-00001-of-00008.safetensors": {
"parameter_count": 180,
"parameter_names_sha256": "81a3d4778de604e75609aae7dac3d8dcbc081811e7748267fe953bbeb375e79d"
},
"diffusion_pytorch_model-00002-of-00008.safetensors": {
"parameter_count": 180,
"parameter_names_sha256": "4e36e2596a6468f29179fa5795396f2e9a7f7d8debc912b34a6f6c2c7c7c610d"
},
"diffusion_pytorch_model-00003-of-00008.safetensors": {
"parameter_count": 180,
"parameter_names_sha256": "6f74a7b2cf7444c2daa2b3fc32bd3787589b35543c230c5c13b89f0f0f2a0998"
},
"diffusion_pytorch_model-00004-of-00008.safetensors": {
"parameter_count": 180,
"parameter_names_sha256": "ec5c967ed5271ba7834337c4ee93a73f8ed0eead4492d4920624251c65c058ac"
},
"diffusion_pytorch_model-00005-of-00008.safetensors": {
"parameter_count": 180,
"parameter_names_sha256": "ad1fdab5830dd9ce43ea007f112b4be223327b7c8197c766d9cf9219e74d619b"
},
"diffusion_pytorch_model-00006-of-00008.safetensors": {
"parameter_count": 180,
"parameter_names_sha256": "fd58f0d86be42d8afcf55d3e07d8a22f7f6ef5064da931a600c890b71b6dfbae"
},
"diffusion_pytorch_model-00007-of-00008.safetensors": {
"parameter_count": 180,
"parameter_names_sha256": "305a1714b5f4fb54d5877c593b451273ab1490d90fd0896a299d96e140295047"
},
"diffusion_pytorch_model-00008-of-00008.safetensors": {
"parameter_count": 161,
"parameter_names_sha256": "4c18d94d96168682eb212f93e3cdf7313928776d8d9e6e4de0255fe899decb25"
}
},
"top_level_modules": [
"blocks",
"c2ws_hidden_states_layer1",
"c2ws_hidden_states_layer2",
"head",
"patch_embedding",
"patch_embedding_wancamctrl",
"text_embedding",
"time_embedding",
"time_projection"
],
"non_block_parameters": [
"c2ws_hidden_states_layer1.bias",
"c2ws_hidden_states_layer1.weight",
"c2ws_hidden_states_layer2.bias",
"c2ws_hidden_states_layer2.weight",
"head.head.bias",
"head.head.weight",
"head.modulation",
"patch_embedding.bias",
"patch_embedding.weight",
"patch_embedding_wancamctrl.bias",
"patch_embedding_wancamctrl.weight",
"text_embedding.0.bias",
"text_embedding.0.weight",
"text_embedding.2.bias",
"text_embedding.2.weight",
"time_embedding.0.bias",
"time_embedding.0.weight",
"time_embedding.2.bias",
"time_embedding.2.weight",
"time_projection.1.bias",
"time_projection.1.weight"
],
"block_parameter_suffixes": [
"cam_injector_layer1.bias",
"cam_injector_layer1.weight",
"cam_injector_layer2.bias",
"cam_injector_layer2.weight",
"cam_scale_layer.bias",
"cam_scale_layer.weight",
"cam_shift_layer.bias",
"cam_shift_layer.weight",
"cross_attn.k.bias",
"cross_attn.k.weight",
"cross_attn.norm_k.weight",
"cross_attn.norm_q.weight",
"cross_attn.o.bias",
"cross_attn.o.weight",
"cross_attn.q.bias",
"cross_attn.q.weight",
"cross_attn.v.bias",
"cross_attn.v.weight",
"ffn.0.bias",
"ffn.0.weight",
"ffn.2.bias",
"ffn.2.weight",
"modulation",
"norm3.bias",
"norm3.weight",
"self_attn.k.bias",
"self_attn.k.weight",
"self_attn.norm_k.weight",
"self_attn.norm_q.weight",
"self_attn.o.bias",
"self_attn.o.weight",
"self_attn.q.bias",
"self_attn.q.weight",
"self_attn.v.bias",
"self_attn.v.weight"
]
}