File size: 1,124 Bytes
72edb83 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 |
{
"distributed": {
"model_dtype": "bf16"
},
"model": {
"dim": 2048,
"n_layers": 16,
"n_heads": 32,
"n_kv_heads": 8,
"vocab_size": 128256,
"ffn_dim_multiplier": 1.5,
"multiple_of": 256,
"norm_eps": 1e-05,
"rope_theta": 500000.0,
"use_scaled_rope": true,
"rope_scale_factor": 32,
"high_freq_factor": 4,
"max_seqlen": 11520,
"pooling_ratio": 2,
"weight_tying": true,
"vision_model": {
"image_size": 448,
"patch_size": 14,
"width": 1024,
"layers": 23,
"heads": 16,
"use_cls_token": true,
"use_abs_posemb": true,
"ls_init_value": 0.1,
"drop_path": 0.1,
"mlp_ratio": 4.0,
"use_ln_post": false,
"pool_type": "none"
}
},
"data": {
"tokenizer_name": "plmchat",
"tokenizer_path": "tokenizer.model",
"vision_input_type": "thumb+tile",
"max_num_tiles": 36,
"max_video_frames": 32
}
}
|