[Tests] Config driven tiny model builder (#5090)
Signed-off-by: Nick Cao <ncao@redhat.com>
This commit is contained in:
Nick Cao authored and GitHub committed
1 parent
cc1939120f
commit
f10320f53f
18 files changed
+601
-60
No files matched your search
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"_class_name": "AutoencoderKLLTX2Audio",
|
||||
"_diffusers_version": "0.37.0.dev0",
|
||||
"attn_resolutions": null,
|
||||
"base_channels": 128,
|
||||
"causality_axis": "height",
|
||||
"ch_mult": [
|
||||
1,
|
||||
2,
|
||||
4
|
||||
],
|
||||
"double_z": true,
|
||||
"dropout": 0.0,
|
||||
"in_channels": 2,
|
||||
"is_causal": true,
|
||||
"latent_channels": 8,
|
||||
"mel_bins": 64,
|
||||
"mel_hop_length": 160,
|
||||
"mid_block_add_attention": false,
|
||||
"norm_type": "pixel",
|
||||
"num_res_blocks": 1,
|
||||
"output_channels": 2,
|
||||
"resolution": 256,
|
||||
"sample_rate": 16000
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"_class_name": "LTX2TextConnectors",
|
||||
"_diffusers_version": "0.37.0.dev0",
|
||||
"audio_connector_attention_head_dim": 16,
|
||||
"audio_connector_num_attention_heads": 2,
|
||||
"audio_connector_num_layers": 1,
|
||||
"audio_connector_num_learnable_registers": 4,
|
||||
"caption_channels": 32,
|
||||
"causal_temporal_positioning": false,
|
||||
"connector_rope_base_seq_len": 32,
|
||||
"rope_double_precision": true,
|
||||
"rope_theta": 10000.0,
|
||||
"rope_type": "split",
|
||||
"text_proj_in_factor": 3,
|
||||
"video_connector_attention_head_dim": 16,
|
||||
"video_connector_num_attention_heads": 2,
|
||||
"video_connector_num_layers": 1,
|
||||
"video_connector_num_learnable_registers": 4,
|
||||
"video_hidden_dim": 32,
|
||||
"audio_hidden_dim": 32
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"_class_name": "LTX2Pipeline",
|
||||
"_diffusers_version": "0.37.0.dev0",
|
||||
"audio_vae": [
|
||||
"diffusers",
|
||||
"AutoencoderKLLTX2Audio"
|
||||
],
|
||||
"connectors": [
|
||||
"ltx2",
|
||||
"LTX2TextConnectors"
|
||||
],
|
||||
"scheduler": [
|
||||
"diffusers",
|
||||
"FlowMatchEulerDiscreteScheduler"
|
||||
],
|
||||
"text_encoder": [
|
||||
"transformers",
|
||||
"Gemma3ForConditionalGeneration"
|
||||
],
|
||||
"tokenizer": [
|
||||
"transformers",
|
||||
"GemmaTokenizerFast"
|
||||
],
|
||||
"transformer": [
|
||||
"diffusers",
|
||||
"LTX2VideoTransformer3DModel"
|
||||
],
|
||||
"vae": [
|
||||
"diffusers",
|
||||
"AutoencoderKLLTX2Video"
|
||||
],
|
||||
"vocoder": [
|
||||
"ltx2",
|
||||
"LTX2Vocoder"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
{
|
||||
"architectures": [
|
||||
"Gemma3ForConditionalGeneration"
|
||||
],
|
||||
"boi_token_index": 255999,
|
||||
"dtype": "float32",
|
||||
"eoi_token_index": 256000,
|
||||
"eos_token_id": [
|
||||
1,
|
||||
106
|
||||
],
|
||||
"image_token_index": 262144,
|
||||
"initializer_range": 0.02,
|
||||
"mm_tokens_per_image": 256,
|
||||
"model_type": "gemma3",
|
||||
"text_config": {
|
||||
"_sliding_window_pattern": 6,
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"attn_logit_softcapping": null,
|
||||
"cache_implementation": "hybrid",
|
||||
"dtype": "float32",
|
||||
"final_logit_softcapping": null,
|
||||
"head_dim": 16,
|
||||
"hidden_activation": "gelu_pytorch_tanh",
|
||||
"hidden_size": 32,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 64,
|
||||
"layer_types": [
|
||||
"sliding_attention",
|
||||
"full_attention"
|
||||
],
|
||||
"max_position_embeddings": 512,
|
||||
"model_type": "gemma3_text",
|
||||
"num_attention_heads": 2,
|
||||
"num_hidden_layers": 2,
|
||||
"num_key_value_heads": 2,
|
||||
"query_pre_attn_scalar": 16,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_local_base_freq": 10000,
|
||||
"rope_scaling": {
|
||||
"factor": 8.0,
|
||||
"rope_type": "linear"
|
||||
},
|
||||
"rope_theta": 1000000,
|
||||
"sliding_window": 1024,
|
||||
"sliding_window_pattern": 6,
|
||||
"use_bidirectional_attention": false,
|
||||
"use_cache": true,
|
||||
"vocab_size": 262208
|
||||
},
|
||||
"transformers_version": "4.57.3",
|
||||
"vision_config": {
|
||||
"attention_dropout": 0.0,
|
||||
"dtype": "float32",
|
||||
"hidden_act": "gelu_pytorch_tanh",
|
||||
"hidden_size": 32,
|
||||
"image_size": 896,
|
||||
"intermediate_size": 64,
|
||||
"layer_norm_eps": 1e-06,
|
||||
"model_type": "siglip_vision_model",
|
||||
"num_attention_heads": 2,
|
||||
"num_channels": 3,
|
||||
"num_hidden_layers": 2,
|
||||
"patch_size": 14,
|
||||
"vision_use_head": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
{
|
||||
"_class_name": "LTX2VideoTransformer3DModel",
|
||||
"_diffusers_version": "0.37.0.dev0",
|
||||
"activation_fn": "gelu-approximate",
|
||||
"attention_bias": true,
|
||||
"attention_head_dim": 16,
|
||||
"attention_out_bias": true,
|
||||
"audio_attention_head_dim": 16,
|
||||
"audio_cross_attention_dim": 32,
|
||||
"audio_hop_length": 160,
|
||||
"audio_in_channels": 128,
|
||||
"audio_num_attention_heads": 2,
|
||||
"audio_out_channels": 128,
|
||||
"audio_patch_size": 1,
|
||||
"audio_patch_size_t": 1,
|
||||
"audio_pos_embed_max_pos": 20,
|
||||
"audio_sampling_rate": 16000,
|
||||
"audio_scale_factor": 4,
|
||||
"base_height": 2048,
|
||||
"base_width": 2048,
|
||||
"caption_channels": 32,
|
||||
"causal_offset": 1,
|
||||
"cross_attention_dim": 32,
|
||||
"cross_attn_timestep_scale_multiplier": 1000,
|
||||
"in_channels": 128,
|
||||
"norm_elementwise_affine": false,
|
||||
"norm_eps": 1e-06,
|
||||
"num_attention_heads": 2,
|
||||
"num_layers": 2,
|
||||
"out_channels": 128,
|
||||
"patch_size": 1,
|
||||
"patch_size_t": 1,
|
||||
"pos_embed_max_pos": 20,
|
||||
"qk_norm": "rms_norm_across_heads",
|
||||
"rope_double_precision": true,
|
||||
"rope_theta": 10000.0,
|
||||
"rope_type": "split",
|
||||
"timestep_scale_multiplier": 1000,
|
||||
"vae_scale_factors": [
|
||||
8,
|
||||
32,
|
||||
32
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
{
|
||||
"_class_name": "AutoencoderKLLTX2Video",
|
||||
"_diffusers_version": "0.37.0.dev0",
|
||||
"block_out_channels": [
|
||||
256,
|
||||
512,
|
||||
1024,
|
||||
2048
|
||||
],
|
||||
"decoder_block_out_channels": [
|
||||
256,
|
||||
512,
|
||||
1024
|
||||
],
|
||||
"decoder_causal": false,
|
||||
"decoder_inject_noise": [
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false
|
||||
],
|
||||
"decoder_layers_per_block": [
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1
|
||||
],
|
||||
"decoder_spatial_padding_mode": "reflect",
|
||||
"decoder_spatio_temporal_scaling": [
|
||||
true,
|
||||
true,
|
||||
true
|
||||
],
|
||||
"down_block_types": [
|
||||
"LTX2VideoDownBlock3D",
|
||||
"LTX2VideoDownBlock3D",
|
||||
"LTX2VideoDownBlock3D",
|
||||
"LTX2VideoDownBlock3D"
|
||||
],
|
||||
"downsample_type": [
|
||||
"spatial",
|
||||
"temporal",
|
||||
"spatiotemporal",
|
||||
"spatiotemporal"
|
||||
],
|
||||
"encoder_causal": true,
|
||||
"encoder_spatial_padding_mode": "zeros",
|
||||
"in_channels": 3,
|
||||
"latent_channels": 128,
|
||||
"layers_per_block": [
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1
|
||||
],
|
||||
"out_channels": 3,
|
||||
"patch_size": 4,
|
||||
"patch_size_t": 1,
|
||||
"resnet_norm_eps": 1e-06,
|
||||
"scaling_factor": 1.0,
|
||||
"spatial_compression_ratio": 32,
|
||||
"spatio_temporal_scaling": [
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true
|
||||
],
|
||||
"temporal_compression_ratio": 8,
|
||||
"timestep_conditioning": false,
|
||||
"upsample_factor": [
|
||||
2,
|
||||
2,
|
||||
2
|
||||
],
|
||||
"upsample_residual": [
|
||||
true,
|
||||
true,
|
||||
true
|
||||
]
|
||||
}
|
||||
Reference in new issue
Block a user