| { | |
| "distributed": { | |
| "model_dtype": "bf16" | |
| }, | |
| "model": { | |
| "dim": 2048, | |
| "n_layers": 16, | |
| "n_heads": 32, | |
| "n_kv_heads": 8, | |
| "vocab_size": 128256, | |
| "ffn_dim_multiplier": 1.5, | |
| "multiple_of": 256, | |
| "norm_eps": 1e-05, | |
| "rope_theta": 500000.0, | |
| "use_scaled_rope": true, | |
| "rope_scale_factor": 32, | |
| "high_freq_factor": 4, | |
| "max_seqlen": 11520, | |
| "pooling_ratio": 2, | |
| "weight_tying": true, | |
| "vision_model": { | |
| "image_size": 448, | |
| "patch_size": 14, | |
| "width": 1024, | |
| "layers": 23, | |
| "heads": 16, | |
| "use_cls_token": true, | |
| "use_abs_posemb": true, | |
| "ls_init_value": 0.1, | |
| "drop_path": 0.1, | |
| "mlp_ratio": 4.0, | |
| "use_ln_post": false, | |
| "pool_type": "none" | |
| } | |
| }, | |
| "data": { | |
| "tokenizer_name": "plmchat", | |
| "tokenizer_path": "tokenizer.model", | |
| "vision_input_type": "thumb+tile", | |
| "max_num_tiles": 36, | |
| "max_video_frames": 32 | |
| } | |
| } | |