Hugging Face's logo

WINDop
/
qwen2_vl_convert

Safetensors
qqmm
custom_code
Model card Files
xet
Community
WINDop commited on
Commit
d48c0d1
·
verified ·
1 Parent(s): c516085

Upload folder using huggingface_hub

Browse files
Files changed (8) hide show
  1. config.json +258 -0
  2. configuration_qqmm.py +30 -0
  3. model-00001-of-00004.safetensors +3 -0
  4. model-00002-of-00004.safetensors +3 -0
  5. model-00003-of-00004.safetensors +3 -0
  6. model-00004-of-00004.safetensors +3 -0
  7. model.safetensors.index.json +737 -0
  8. modeling_qqmm.py +343 -0
config.json ADDED
@@ -0,0 +1,258 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "QQMMForCausalLM"
4
+ ],
5
+ "auto_map": {
6
+ "AutoConfig": "configuration_qqmm.QQMMConfig",
7
+ "AutoModelForCausalLM": "modeling_qqmm.QQMMForCausalLM"
8
+ },
9
+ "model_config": {
10
+ "_name_or_path": "Qwen/Qwen2-VL-7B-Instruct",
11
+ "architectures": [
12
+ "Qwen2VLForConditionalGeneration"
13
+ ],
14
+ "auto_map": {
15
+ "AutoConfig": "configuration_qqmm.QQMMConfig",
16
+ "AutoModelForCausalLM": "modeling_qqmm.QQMMForCausalLM"
17
+ },
18
+ "attention_dropout": 0.0,
19
+ "bos_token_id": 151643,
20
+ "eos_token_id": 151645,
21
+ "hidden_act": "silu",
22
+ "hidden_size": 3584,
23
+ "image_token_id": 151655,
24
+ "initializer_range": 0.02,
25
+ "intermediate_size": 18944,
26
+ "max_position_embeddings": 32768,
27
+ "max_window_layers": 28,
28
+ "model_type": "qwen2_vl",
29
+ "num_attention_heads": 28,
30
+ "num_hidden_layers": 28,
31
+ "num_key_value_heads": 4,
32
+ "rms_norm_eps": 1e-06,
33
+ "rope_scaling": {
34
+ "mrope_section": [
35
+ 16,
36
+ 24,
37
+ 24
38
+ ],
39
+ "rope_type": "default",
40
+ "type": "default"
41
+ },
42
+ "rope_theta": 1000000.0,
43
+ "sliding_window": 32768,
44
+ "text_config": {
45
+ "_name_or_path": "",
46
+ "add_cross_attention": false,
47
+ "architectures": [
48
+ "Qwen2VLForConditionalGeneration"
49
+ ],
50
+ "attention_dropout": 0.0,
51
+ "bad_words_ids": null,
52
+ "begin_suppress_tokens": null,
53
+ "bos_token_id": 151643,
54
+ "chunk_size_feed_forward": 0,
55
+ "cross_attention_hidden_size": null,
56
+ "decoder_start_token_id": null,
57
+ "diversity_penalty": 0.0,
58
+ "do_sample": false,
59
+ "early_stopping": false,
60
+ "encoder_no_repeat_ngram_size": 0,
61
+ "eos_token_id": 151645,
62
+ "exponential_decay_length_penalty": null,
63
+ "finetuning_task": null,
64
+ "forced_bos_token_id": null,
65
+ "forced_eos_token_id": null,
66
+ "hidden_act": "silu",
67
+ "hidden_size": 3584,
68
+ "id2label": {
69
+ "0": "LABEL_0",
70
+ "1": "LABEL_1"
71
+ },
72
+ "image_token_id": null,
73
+ "initializer_range": 0.02,
74
+ "intermediate_size": 18944,
75
+ "is_decoder": false,
76
+ "is_encoder_decoder": false,
77
+ "label2id": {
78
+ "LABEL_0": 0,
79
+ "LABEL_1": 1
80
+ },
81
+ "layer_types": [
82
+ "full_attention",
83
+ "full_attention",
84
+ "full_attention",
85
+ "full_attention",
86
+ "full_attention",
87
+ "full_attention",
88
+ "full_attention",
89
+ "full_attention",
90
+ "full_attention",
91
+ "full_attention",
92
+ "full_attention",
93
+ "full_attention",
94
+ "full_attention",
95
+ "full_attention",
96
+ "full_attention",
97
+ "full_attention",
98
+ "full_attention",
99
+ "full_attention",
100
+ "full_attention",
101
+ "full_attention",
102
+ "full_attention",
103
+ "full_attention",
104
+ "full_attention",
105
+ "full_attention",
106
+ "full_attention",
107
+ "full_attention",
108
+ "full_attention",
109
+ "full_attention"
110
+ ],
111
+ "length_penalty": 1.0,
112
+ "max_length": 20,
113
+ "max_position_embeddings": 32768,
114
+ "max_window_layers": 28,
115
+ "min_length": 0,
116
+ "model_type": "qwen2_vl_text",
117
+ "no_repeat_ngram_size": 0,
118
+ "num_attention_heads": 28,
119
+ "num_beam_groups": 1,
120
+ "num_beams": 1,
121
+ "num_hidden_layers": 28,
122
+ "num_key_value_heads": 4,
123
+ "num_return_sequences": 1,
124
+ "output_attentions": false,
125
+ "output_hidden_states": false,
126
+ "output_scores": false,
127
+ "pad_token_id": null,
128
+ "prefix": null,
129
+ "problem_type": null,
130
+ "pruned_heads": {},
131
+ "remove_invalid_values": false,
132
+ "repetition_penalty": 1.0,
133
+ "return_dict": true,
134
+ "return_dict_in_generate": false,
135
+ "rms_norm_eps": 1e-06,
136
+ "rope_scaling": {
137
+ "mrope_section": [
138
+ 16,
139
+ 24,
140
+ 24
141
+ ],
142
+ "rope_type": "default",
143
+ "type": "default"
144
+ },
145
+ "rope_theta": 1000000.0,
146
+ "sep_token_id": null,
147
+ "sliding_window": null,
148
+ "suppress_tokens": null,
149
+ "task_specific_params": null,
150
+ "temperature": 1.0,
151
+ "tf_legacy_loss": false,
152
+ "tie_encoder_decoder": false,
153
+ "tie_word_embeddings": false,
154
+ "tokenizer_class": null,
155
+ "top_k": 50,
156
+ "top_p": 1.0,
157
+ "torch_dtype": "bfloat16",
158
+ "torchscript": false,
159
+ "typical_p": 1.0,
160
+ "use_bfloat16": false,
161
+ "use_cache": true,
162
+ "use_sliding_window": false,
163
+ "video_token_id": null,
164
+ "vision_end_token_id": 151653,
165
+ "vision_start_token_id": 151652,
166
+ "vision_token_id": 151654,
167
+ "vocab_size": 152064
168
+ },
169
+ "tie_word_embeddings": false,
170
+ "torch_dtype": "bfloat16",
171
+ "use_cache": true,
172
+ "use_sliding_window": false,
173
+ "video_token_id": 151656,
174
+ "vision_config": {
175
+ "_name_or_path": "",
176
+ "add_cross_attention": false,
177
+ "architectures": null,
178
+ "bad_words_ids": null,
179
+ "begin_suppress_tokens": null,
180
+ "bos_token_id": null,
181
+ "chunk_size_feed_forward": 0,
182
+ "cross_attention_hidden_size": null,
183
+ "decoder_start_token_id": null,
184
+ "depth": 32,
185
+ "diversity_penalty": 0.0,
186
+ "do_sample": false,
187
+ "early_stopping": false,
188
+ "embed_dim": 1280,
189
+ "encoder_no_repeat_ngram_size": 0,
190
+ "eos_token_id": null,
191
+ "exponential_decay_length_penalty": null,
192
+ "finetuning_task": null,
193
+ "forced_bos_token_id": null,
194
+ "forced_eos_token_id": null,
195
+ "hidden_act": "quick_gelu",
196
+ "hidden_size": 3584,
197
+ "id2label": {
198
+ "0": "LABEL_0",
199
+ "1": "LABEL_1"
200
+ },
201
+ "in_channels": 3,
202
+ "in_chans": 3,
203
+ "initializer_range": 0.02,
204
+ "is_decoder": false,
205
+ "is_encoder_decoder": false,
206
+ "label2id": {
207
+ "LABEL_0": 0,
208
+ "LABEL_1": 1
209
+ },
210
+ "length_penalty": 1.0,
211
+ "max_length": 20,
212
+ "min_length": 0,
213
+ "mlp_ratio": 4,
214
+ "model_type": "qwen2_vl",
215
+ "no_repeat_ngram_size": 0,
216
+ "num_beam_groups": 1,
217
+ "num_beams": 1,
218
+ "num_heads": 16,
219
+ "num_return_sequences": 1,
220
+ "output_attentions": false,
221
+ "output_hidden_states": false,
222
+ "output_scores": false,
223
+ "pad_token_id": null,
224
+ "patch_size": 14,
225
+ "prefix": null,
226
+ "problem_type": null,
227
+ "pruned_heads": {},
228
+ "remove_invalid_values": false,
229
+ "repetition_penalty": 1.0,
230
+ "return_dict": true,
231
+ "return_dict_in_generate": false,
232
+ "sep_token_id": null,
233
+ "spatial_merge_size": 2,
234
+ "spatial_patch_size": 14,
235
+ "suppress_tokens": null,
236
+ "task_specific_params": null,
237
+ "temperature": 1.0,
238
+ "temporal_patch_size": 2,
239
+ "tf_legacy_loss": false,
240
+ "tie_encoder_decoder": false,
241
+ "tie_word_embeddings": true,
242
+ "tokenizer_class": null,
243
+ "top_k": 50,
244
+ "top_p": 1.0,
245
+ "torch_dtype": null,
246
+ "torchscript": false,
247
+ "typical_p": 1.0,
248
+ "use_bfloat16": false
249
+ },
250
+ "vision_end_token_id": 151653,
251
+ "vision_start_token_id": 151652,
252
+ "vision_token_id": 151654,
253
+ "vocab_size": 152064
254
+ },
255
+ "model_type": "qqmm",
256
+ "torch_dtype": "bfloat16",
257
+ "transformers_version": "4.53.3"
258
+ }
configuration_qqmm.py ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from transformers import PretrainedConfig, AutoConfig, CONFIG_MAPPING
2
+ from transformers.dynamic_module_utils import get_class_from_dynamic_module
3
+ from transformers import Qwen2_5_VLConfig, Qwen2_5_VLForConditionalGeneration
4
+
5
+ class QQMMConfig(PretrainedConfig):
6
+ model_type = "qqmm"
7
+ is_composition = True
8
+
9
+ def __init__(self,
10
+ model_config={},
11
+ **kwargs):
12
+
13
+ super().__init__(**kwargs)
14
+
15
+ model_type = model_config.get('model_type', '')
16
+ is_remote_code = '.' in model_config.get('auto_map', {}).get('AutoConfig', '')
17
+ if model_type in CONFIG_MAPPING and not is_remote_code:
18
+ self.model_config = AutoConfig.for_model(**model_config)
19
+ else:
20
+ self.model_config = model_config
21
+
22
+ @property
23
+ def hidden_size(self):
24
+ return self.model_config.hidden_size
25
+
26
+ @classmethod
27
+ def from_dict(cls, config_dict, **kwargs):
28
+ if 'name_or_path' in kwargs:
29
+ config_dict['_name_or_path'] = kwargs.pop('name_or_path')
30
+ return super().from_dict(config_dict, **kwargs)
model-00001-of-00004.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:678efadcee3ebbb025d2a47384e9e8fe05e360bc5627696b31cdf64f2b8982e4
3
+ size 4966671144
model-00002-of-00004.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:94fe343159ba2b80738323ca5b5550f4650091ac1d5c2b1d5b1358d040979f43
3
+ size 4991500008
model-00003-of-00004.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4653d3701973fa04449be75476dbdf6d4f792ec07d95c03eb954d9af1998b482
3
+ size 4932754944
model-00004-of-00004.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:42121a84dd5aaf1b6de092de81ff0793b144bf509830332523a10753c0a033ed
3
+ size 601930008
model.safetensors.index.json ADDED
@@ -0,0 +1,737 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "total_parameters": 7746378240,
4
+ "total_size": 15492756480
5
+ },
6
+ "weight_map": {
7
+ "qwen2_5_vl_model.model.language_model.embed_tokens.weight": "model-00001-of-00004.safetensors",
8
+ "qwen2_5_vl_model.model.language_model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
9
+ "qwen2_5_vl_model.model.language_model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
10
+ "qwen2_5_vl_model.model.language_model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
11
+ "qwen2_5_vl_model.model.language_model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
12
+ "qwen2_5_vl_model.model.language_model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
13
+ "qwen2_5_vl_model.model.language_model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
14
+ "qwen2_5_vl_model.model.language_model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
15
+ "qwen2_5_vl_model.model.language_model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
16
+ "qwen2_5_vl_model.model.language_model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
17
+ "qwen2_5_vl_model.model.language_model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
18
+ "qwen2_5_vl_model.model.language_model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
19
+ "qwen2_5_vl_model.model.language_model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
20
+ "qwen2_5_vl_model.model.language_model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
21
+ "qwen2_5_vl_model.model.language_model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
22
+ "qwen2_5_vl_model.model.language_model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
23
+ "qwen2_5_vl_model.model.language_model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
24
+ "qwen2_5_vl_model.model.language_model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
25
+ "qwen2_5_vl_model.model.language_model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
26
+ "qwen2_5_vl_model.model.language_model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
27
+ "qwen2_5_vl_model.model.language_model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
28
+ "qwen2_5_vl_model.model.language_model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
29
+ "qwen2_5_vl_model.model.language_model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
30
+ "qwen2_5_vl_model.model.language_model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
31
+ "qwen2_5_vl_model.model.language_model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
32
+ "qwen2_5_vl_model.model.language_model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
33
+ "qwen2_5_vl_model.model.language_model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
34
+ "qwen2_5_vl_model.model.language_model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
35
+ "qwen2_5_vl_model.model.language_model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
36
+ "qwen2_5_vl_model.model.language_model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
37
+ "qwen2_5_vl_model.model.language_model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
38
+ "qwen2_5_vl_model.model.language_model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
39
+ "qwen2_5_vl_model.model.language_model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
40
+ "qwen2_5_vl_model.model.language_model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
41
+ "qwen2_5_vl_model.model.language_model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
42
+ "qwen2_5_vl_model.model.language_model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
43
+ "qwen2_5_vl_model.model.language_model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
44
+ "qwen2_5_vl_model.model.language_model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
45
+ "qwen2_5_vl_model.model.language_model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
46
+ "qwen2_5_vl_model.model.language_model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
47
+ "qwen2_5_vl_model.model.language_model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
48
+ "qwen2_5_vl_model.model.language_model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
49
+ "qwen2_5_vl_model.model.language_model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
50
+ "qwen2_5_vl_model.model.language_model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
51
+ "qwen2_5_vl_model.model.language_model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
52
+ "qwen2_5_vl_model.model.language_model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
53
+ "qwen2_5_vl_model.model.language_model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
54
+ "qwen2_5_vl_model.model.language_model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
55
+ "qwen2_5_vl_model.model.language_model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
56
+ "qwen2_5_vl_model.model.language_model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
57
+ "qwen2_5_vl_model.model.language_model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
58
+ "qwen2_5_vl_model.model.language_model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
59
+ "qwen2_5_vl_model.model.language_model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
60
+ "qwen2_5_vl_model.model.language_model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
61
+ "qwen2_5_vl_model.model.language_model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
62
+ "qwen2_5_vl_model.model.language_model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
63
+ "qwen2_5_vl_model.model.language_model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
64
+ "qwen2_5_vl_model.model.language_model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
65
+ "qwen2_5_vl_model.model.language_model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
66
+ "qwen2_5_vl_model.model.language_model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
67
+ "qwen2_5_vl_model.model.language_model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
68
+ "qwen2_5_vl_model.model.language_model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
69
+ "qwen2_5_vl_model.model.language_model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
70
+ "qwen2_5_vl_model.model.language_model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
71
+ "qwen2_5_vl_model.model.language_model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
72
+ "qwen2_5_vl_model.model.language_model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
73
+ "qwen2_5_vl_model.model.language_model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
74
+ "qwen2_5_vl_model.model.language_model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
75
+ "qwen2_5_vl_model.model.language_model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
76
+ "qwen2_5_vl_model.model.language_model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
77
+ "qwen2_5_vl_model.model.language_model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
78
+ "qwen2_5_vl_model.model.language_model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
79
+ "qwen2_5_vl_model.model.language_model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
80
+ "qwen2_5_vl_model.model.language_model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
81
+ "qwen2_5_vl_model.model.language_model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
82
+ "qwen2_5_vl_model.model.language_model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
83
+ "qwen2_5_vl_model.model.language_model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
84
+ "qwen2_5_vl_model.model.language_model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
85
+ "qwen2_5_vl_model.model.language_model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
86
+ "qwen2_5_vl_model.model.language_model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
87
+ "qwen2_5_vl_model.model.language_model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
88
+ "qwen2_5_vl_model.model.language_model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
89
+ "qwen2_5_vl_model.model.language_model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
90
+ "qwen2_5_vl_model.model.language_model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
91
+ "qwen2_5_vl_model.model.language_model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
92
+ "qwen2_5_vl_model.model.language_model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
93
+ "qwen2_5_vl_model.model.language_model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
94
+ "qwen2_5_vl_model.model.language_model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
95
+ "qwen2_5_vl_model.model.language_model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
96
+ "qwen2_5_vl_model.model.language_model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
97
+ "qwen2_5_vl_model.model.language_model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
98
+ "qwen2_5_vl_model.model.language_model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
99
+ "qwen2_5_vl_model.model.language_model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
100
+ "qwen2_5_vl_model.model.language_model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
101
+ "qwen2_5_vl_model.model.language_model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
102
+ "qwen2_5_vl_model.model.language_model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
103
+ "qwen2_5_vl_model.model.language_model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
104
+ "qwen2_5_vl_model.model.language_model.layers.16.input_layernorm.weight": "model-00003-of-00004.safetensors",
105
+ "qwen2_5_vl_model.model.language_model.layers.16.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
106
+ "qwen2_5_vl_model.model.language_model.layers.16.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
107
+ "qwen2_5_vl_model.model.language_model.layers.16.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
108
+ "qwen2_5_vl_model.model.language_model.layers.16.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
109
+ "qwen2_5_vl_model.model.language_model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
110
+ "qwen2_5_vl_model.model.language_model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
111
+ "qwen2_5_vl_model.model.language_model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
112
+ "qwen2_5_vl_model.model.language_model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
113
+ "qwen2_5_vl_model.model.language_model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
114
+ "qwen2_5_vl_model.model.language_model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
115
+ "qwen2_5_vl_model.model.language_model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
116
+ "qwen2_5_vl_model.model.language_model.layers.17.input_layernorm.weight": "model-00003-of-00004.safetensors",
117
+ "qwen2_5_vl_model.model.language_model.layers.17.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
118
+ "qwen2_5_vl_model.model.language_model.layers.17.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
119
+ "qwen2_5_vl_model.model.language_model.layers.17.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
120
+ "qwen2_5_vl_model.model.language_model.layers.17.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
121
+ "qwen2_5_vl_model.model.language_model.layers.17.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
122
+ "qwen2_5_vl_model.model.language_model.layers.17.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
123
+ "qwen2_5_vl_model.model.language_model.layers.17.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
124
+ "qwen2_5_vl_model.model.language_model.layers.17.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
125
+ "qwen2_5_vl_model.model.language_model.layers.17.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
126
+ "qwen2_5_vl_model.model.language_model.layers.17.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
127
+ "qwen2_5_vl_model.model.language_model.layers.17.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
128
+ "qwen2_5_vl_model.model.language_model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
129
+ "qwen2_5_vl_model.model.language_model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
130
+ "qwen2_5_vl_model.model.language_model.layers.18.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
131
+ "qwen2_5_vl_model.model.language_model.layers.18.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
132
+ "qwen2_5_vl_model.model.language_model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
133
+ "qwen2_5_vl_model.model.language_model.layers.18.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
134
+ "qwen2_5_vl_model.model.language_model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
135
+ "qwen2_5_vl_model.model.language_model.layers.18.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
136
+ "qwen2_5_vl_model.model.language_model.layers.18.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
137
+ "qwen2_5_vl_model.model.language_model.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
138
+ "qwen2_5_vl_model.model.language_model.layers.18.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
139
+ "qwen2_5_vl_model.model.language_model.layers.18.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
140
+ "qwen2_5_vl_model.model.language_model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
141
+ "qwen2_5_vl_model.model.language_model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
142
+ "qwen2_5_vl_model.model.language_model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
143
+ "qwen2_5_vl_model.model.language_model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
144
+ "qwen2_5_vl_model.model.language_model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
145
+ "qwen2_5_vl_model.model.language_model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
146
+ "qwen2_5_vl_model.model.language_model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
147
+ "qwen2_5_vl_model.model.language_model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
148
+ "qwen2_5_vl_model.model.language_model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
149
+ "qwen2_5_vl_model.model.language_model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
150
+ "qwen2_5_vl_model.model.language_model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
151
+ "qwen2_5_vl_model.model.language_model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
152
+ "qwen2_5_vl_model.model.language_model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
153
+ "qwen2_5_vl_model.model.language_model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
154
+ "qwen2_5_vl_model.model.language_model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
155
+ "qwen2_5_vl_model.model.language_model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
156
+ "qwen2_5_vl_model.model.language_model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
157
+ "qwen2_5_vl_model.model.language_model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
158
+ "qwen2_5_vl_model.model.language_model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
159
+ "qwen2_5_vl_model.model.language_model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
160
+ "qwen2_5_vl_model.model.language_model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
161
+ "qwen2_5_vl_model.model.language_model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
162
+ "qwen2_5_vl_model.model.language_model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
163
+ "qwen2_5_vl_model.model.language_model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
164
+ "qwen2_5_vl_model.model.language_model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
165
+ "qwen2_5_vl_model.model.language_model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
166
+ "qwen2_5_vl_model.model.language_model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
167
+ "qwen2_5_vl_model.model.language_model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
168
+ "qwen2_5_vl_model.model.language_model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
169
+ "qwen2_5_vl_model.model.language_model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
170
+ "qwen2_5_vl_model.model.language_model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
171
+ "qwen2_5_vl_model.model.language_model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
172
+ "qwen2_5_vl_model.model.language_model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
173
+ "qwen2_5_vl_model.model.language_model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
174
+ "qwen2_5_vl_model.model.language_model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
175
+ "qwen2_5_vl_model.model.language_model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
176
+ "qwen2_5_vl_model.model.language_model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
177
+ "qwen2_5_vl_model.model.language_model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
178
+ "qwen2_5_vl_model.model.language_model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
179
+ "qwen2_5_vl_model.model.language_model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
180
+ "qwen2_5_vl_model.model.language_model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
181
+ "qwen2_5_vl_model.model.language_model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
182
+ "qwen2_5_vl_model.model.language_model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
183
+ "qwen2_5_vl_model.model.language_model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
184
+ "qwen2_5_vl_model.model.language_model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
185
+ "qwen2_5_vl_model.model.language_model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
186
+ "qwen2_5_vl_model.model.language_model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
187
+ "qwen2_5_vl_model.model.language_model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
188
+ "qwen2_5_vl_model.model.language_model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
189
+ "qwen2_5_vl_model.model.language_model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
190
+ "qwen2_5_vl_model.model.language_model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
191
+ "qwen2_5_vl_model.model.language_model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
192
+ "qwen2_5_vl_model.model.language_model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
193
+ "qwen2_5_vl_model.model.language_model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
194
+ "qwen2_5_vl_model.model.language_model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
195
+ "qwen2_5_vl_model.model.language_model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
196
+ "qwen2_5_vl_model.model.language_model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
197
+ "qwen2_5_vl_model.model.language_model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
198
+ "qwen2_5_vl_model.model.language_model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
199
+ "qwen2_5_vl_model.model.language_model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
200
+ "qwen2_5_vl_model.model.language_model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
201
+ "qwen2_5_vl_model.model.language_model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
202
+ "qwen2_5_vl_model.model.language_model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
203
+ "qwen2_5_vl_model.model.language_model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
204
+ "qwen2_5_vl_model.model.language_model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
205
+ "qwen2_5_vl_model.model.language_model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
206
+ "qwen2_5_vl_model.model.language_model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
207
+ "qwen2_5_vl_model.model.language_model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
208
+ "qwen2_5_vl_model.model.language_model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
209
+ "qwen2_5_vl_model.model.language_model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
210
+ "qwen2_5_vl_model.model.language_model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
211
+ "qwen2_5_vl_model.model.language_model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
212
+ "qwen2_5_vl_model.model.language_model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
213
+ "qwen2_5_vl_model.model.language_model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
214
+ "qwen2_5_vl_model.model.language_model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
215
+ "qwen2_5_vl_model.model.language_model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
216
+ "qwen2_5_vl_model.model.language_model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
217
+ "qwen2_5_vl_model.model.language_model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
218
+ "qwen2_5_vl_model.model.language_model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
219
+ "qwen2_5_vl_model.model.language_model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
220
+ "qwen2_5_vl_model.model.language_model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
221
+ "qwen2_5_vl_model.model.language_model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
222
+ "qwen2_5_vl_model.model.language_model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
223
+ "qwen2_5_vl_model.model.language_model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
224
+ "qwen2_5_vl_model.model.language_model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
225
+ "qwen2_5_vl_model.model.language_model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
226
+ "qwen2_5_vl_model.model.language_model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
227
+ "qwen2_5_vl_model.model.language_model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
228
+ "qwen2_5_vl_model.model.language_model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
229
+ "qwen2_5_vl_model.model.language_model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
230
+ "qwen2_5_vl_model.model.language_model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
231
+ "qwen2_5_vl_model.model.language_model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
232
+ "qwen2_5_vl_model.model.language_model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
233
+ "qwen2_5_vl_model.model.language_model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
234
+ "qwen2_5_vl_model.model.language_model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
235
+ "qwen2_5_vl_model.model.language_model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
236
+ "qwen2_5_vl_model.model.language_model.layers.26.input_layernorm.weight": "model-00004-of-00004.safetensors",
237
+ "qwen2_5_vl_model.model.language_model.layers.26.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
238
+ "qwen2_5_vl_model.model.language_model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
239
+ "qwen2_5_vl_model.model.language_model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
240
+ "qwen2_5_vl_model.model.language_model.layers.26.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
241
+ "qwen2_5_vl_model.model.language_model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
242
+ "qwen2_5_vl_model.model.language_model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
243
+ "qwen2_5_vl_model.model.language_model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
244
+ "qwen2_5_vl_model.model.language_model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
245
+ "qwen2_5_vl_model.model.language_model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
246
+ "qwen2_5_vl_model.model.language_model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
247
+ "qwen2_5_vl_model.model.language_model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
248
+ "qwen2_5_vl_model.model.language_model.layers.27.input_layernorm.weight": "model-00004-of-00004.safetensors",
249
+ "qwen2_5_vl_model.model.language_model.layers.27.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
250
+ "qwen2_5_vl_model.model.language_model.layers.27.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
251
+ "qwen2_5_vl_model.model.language_model.layers.27.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
252
+ "qwen2_5_vl_model.model.language_model.layers.27.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
253
+ "qwen2_5_vl_model.model.language_model.layers.27.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
254
+ "qwen2_5_vl_model.model.language_model.layers.27.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
255
+ "qwen2_5_vl_model.model.language_model.layers.27.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
256
+ "qwen2_5_vl_model.model.language_model.layers.27.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
257
+ "qwen2_5_vl_model.model.language_model.layers.27.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
258
+ "qwen2_5_vl_model.model.language_model.layers.27.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
259
+ "qwen2_5_vl_model.model.language_model.layers.27.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
260
+ "qwen2_5_vl_model.model.language_model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
261
+ "qwen2_5_vl_model.model.language_model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
262
+ "qwen2_5_vl_model.model.language_model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
263
+ "qwen2_5_vl_model.model.language_model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
264
+ "qwen2_5_vl_model.model.language_model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
265
+ "qwen2_5_vl_model.model.language_model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
266
+ "qwen2_5_vl_model.model.language_model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
267
+ "qwen2_5_vl_model.model.language_model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
268
+ "qwen2_5_vl_model.model.language_model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
269
+ "qwen2_5_vl_model.model.language_model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
270
+ "qwen2_5_vl_model.model.language_model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
271
+ "qwen2_5_vl_model.model.language_model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
272
+ "qwen2_5_vl_model.model.language_model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
273
+ "qwen2_5_vl_model.model.language_model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
274
+ "qwen2_5_vl_model.model.language_model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
275
+ "qwen2_5_vl_model.model.language_model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
276
+ "qwen2_5_vl_model.model.language_model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
277
+ "qwen2_5_vl_model.model.language_model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
278
+ "qwen2_5_vl_model.model.language_model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
279
+ "qwen2_5_vl_model.model.language_model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
280
+ "qwen2_5_vl_model.model.language_model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
281
+ "qwen2_5_vl_model.model.language_model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
282
+ "qwen2_5_vl_model.model.language_model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
283
+ "qwen2_5_vl_model.model.language_model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
284
+ "qwen2_5_vl_model.model.language_model.layers.5.input_layernorm.weight": "model-00002-of-00004.safetensors",
285
+ "qwen2_5_vl_model.model.language_model.layers.5.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
286
+ "qwen2_5_vl_model.model.language_model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
287
+ "qwen2_5_vl_model.model.language_model.layers.5.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
288
+ "qwen2_5_vl_model.model.language_model.layers.5.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
289
+ "qwen2_5_vl_model.model.language_model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
290
+ "qwen2_5_vl_model.model.language_model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
291
+ "qwen2_5_vl_model.model.language_model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
292
+ "qwen2_5_vl_model.model.language_model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
293
+ "qwen2_5_vl_model.model.language_model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
294
+ "qwen2_5_vl_model.model.language_model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
295
+ "qwen2_5_vl_model.model.language_model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
296
+ "qwen2_5_vl_model.model.language_model.layers.6.input_layernorm.weight": "model-00002-of-00004.safetensors",
297
+ "qwen2_5_vl_model.model.language_model.layers.6.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
298
+ "qwen2_5_vl_model.model.language_model.layers.6.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
299
+ "qwen2_5_vl_model.model.language_model.layers.6.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
300
+ "qwen2_5_vl_model.model.language_model.layers.6.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
301
+ "qwen2_5_vl_model.model.language_model.layers.6.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
302
+ "qwen2_5_vl_model.model.language_model.layers.6.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
303
+ "qwen2_5_vl_model.model.language_model.layers.6.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
304
+ "qwen2_5_vl_model.model.language_model.layers.6.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
305
+ "qwen2_5_vl_model.model.language_model.layers.6.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
306
+ "qwen2_5_vl_model.model.language_model.layers.6.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
307
+ "qwen2_5_vl_model.model.language_model.layers.6.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
308
+ "qwen2_5_vl_model.model.language_model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
309
+ "qwen2_5_vl_model.model.language_model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
310
+ "qwen2_5_vl_model.model.language_model.layers.7.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
311
+ "qwen2_5_vl_model.model.language_model.layers.7.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
312
+ "qwen2_5_vl_model.model.language_model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
313
+ "qwen2_5_vl_model.model.language_model.layers.7.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
314
+ "qwen2_5_vl_model.model.language_model.layers.7.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
315
+ "qwen2_5_vl_model.model.language_model.layers.7.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
316
+ "qwen2_5_vl_model.model.language_model.layers.7.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
317
+ "qwen2_5_vl_model.model.language_model.layers.7.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
318
+ "qwen2_5_vl_model.model.language_model.layers.7.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
319
+ "qwen2_5_vl_model.model.language_model.layers.7.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
320
+ "qwen2_5_vl_model.model.language_model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
321
+ "qwen2_5_vl_model.model.language_model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
322
+ "qwen2_5_vl_model.model.language_model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
323
+ "qwen2_5_vl_model.model.language_model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
324
+ "qwen2_5_vl_model.model.language_model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
325
+ "qwen2_5_vl_model.model.language_model.layers.8.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
326
+ "qwen2_5_vl_model.model.language_model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
327
+ "qwen2_5_vl_model.model.language_model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
328
+ "qwen2_5_vl_model.model.language_model.layers.8.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
329
+ "qwen2_5_vl_model.model.language_model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
330
+ "qwen2_5_vl_model.model.language_model.layers.8.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
331
+ "qwen2_5_vl_model.model.language_model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
332
+ "qwen2_5_vl_model.model.language_model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
333
+ "qwen2_5_vl_model.model.language_model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
334
+ "qwen2_5_vl_model.model.language_model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
335
+ "qwen2_5_vl_model.model.language_model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
336
+ "qwen2_5_vl_model.model.language_model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
337
+ "qwen2_5_vl_model.model.language_model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
338
+ "qwen2_5_vl_model.model.language_model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
339
+ "qwen2_5_vl_model.model.language_model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
340
+ "qwen2_5_vl_model.model.language_model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
341
+ "qwen2_5_vl_model.model.language_model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
342
+ "qwen2_5_vl_model.model.language_model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
343
+ "qwen2_5_vl_model.model.language_model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
344
+ "qwen2_5_vl_model.model.language_model.norm.weight": "model-00004-of-00004.safetensors",
345
+ "qwen2_5_vl_model.model.visual.blocks.0.attn.proj.bias": "model-00001-of-00004.safetensors",
346
+ "qwen2_5_vl_model.model.visual.blocks.0.attn.proj.weight": "model-00001-of-00004.safetensors",
347
+ "qwen2_5_vl_model.model.visual.blocks.0.attn.qkv.bias": "model-00001-of-00004.safetensors",
348
+ "qwen2_5_vl_model.model.visual.blocks.0.attn.qkv.weight": "model-00001-of-00004.safetensors",
349
+ "qwen2_5_vl_model.model.visual.blocks.0.mlp.fc1.bias": "model-00001-of-00004.safetensors",
350
+ "qwen2_5_vl_model.model.visual.blocks.0.mlp.fc1.weight": "model-00001-of-00004.safetensors",
351
+ "qwen2_5_vl_model.model.visual.blocks.0.mlp.fc2.bias": "model-00001-of-00004.safetensors",
352
+ "qwen2_5_vl_model.model.visual.blocks.0.mlp.fc2.weight": "model-00001-of-00004.safetensors",
353
+ "qwen2_5_vl_model.model.visual.blocks.0.norm1.bias": "model-00001-of-00004.safetensors",
354
+ "qwen2_5_vl_model.model.visual.blocks.0.norm1.weight": "model-00001-of-00004.safetensors",
355
+ "qwen2_5_vl_model.model.visual.blocks.0.norm2.bias": "model-00001-of-00004.safetensors",
356
+ "qwen2_5_vl_model.model.visual.blocks.0.norm2.weight": "model-00001-of-00004.safetensors",
357
+ "qwen2_5_vl_model.model.visual.blocks.1.attn.proj.bias": "model-00001-of-00004.safetensors",
358
+ "qwen2_5_vl_model.model.visual.blocks.1.attn.proj.weight": "model-00001-of-00004.safetensors",
359
+ "qwen2_5_vl_model.model.visual.blocks.1.attn.qkv.bias": "model-00001-of-00004.safetensors",
360
+ "qwen2_5_vl_model.model.visual.blocks.1.attn.qkv.weight": "model-00001-of-00004.safetensors",
361
+ "qwen2_5_vl_model.model.visual.blocks.1.mlp.fc1.bias": "model-00001-of-00004.safetensors",
362
+ "qwen2_5_vl_model.model.visual.blocks.1.mlp.fc1.weight": "model-00001-of-00004.safetensors",
363
+ "qwen2_5_vl_model.model.visual.blocks.1.mlp.fc2.bias": "model-00001-of-00004.safetensors",
364
+ "qwen2_5_vl_model.model.visual.blocks.1.mlp.fc2.weight": "model-00001-of-00004.safetensors",
365
+ "qwen2_5_vl_model.model.visual.blocks.1.norm1.bias": "model-00001-of-00004.safetensors",
366
+ "qwen2_5_vl_model.model.visual.blocks.1.norm1.weight": "model-00001-of-00004.safetensors",
367
+ "qwen2_5_vl_model.model.visual.blocks.1.norm2.bias": "model-00001-of-00004.safetensors",
368
+ "qwen2_5_vl_model.model.visual.blocks.1.norm2.weight": "model-00001-of-00004.safetensors",
369
+ "qwen2_5_vl_model.model.visual.blocks.10.attn.proj.bias": "model-00001-of-00004.safetensors",
370
+ "qwen2_5_vl_model.model.visual.blocks.10.attn.proj.weight": "model-00001-of-00004.safetensors",
371
+ "qwen2_5_vl_model.model.visual.blocks.10.attn.qkv.bias": "model-00001-of-00004.safetensors",
372
+ "qwen2_5_vl_model.model.visual.blocks.10.attn.qkv.weight": "model-00001-of-00004.safetensors",
373
+ "qwen2_5_vl_model.model.visual.blocks.10.mlp.fc1.bias": "model-00001-of-00004.safetensors",
374
+ "qwen2_5_vl_model.model.visual.blocks.10.mlp.fc1.weight": "model-00001-of-00004.safetensors",
375
+ "qwen2_5_vl_model.model.visual.blocks.10.mlp.fc2.bias": "model-00001-of-00004.safetensors",
376
+ "qwen2_5_vl_model.model.visual.blocks.10.mlp.fc2.weight": "model-00001-of-00004.safetensors",
377
+ "qwen2_5_vl_model.model.visual.blocks.10.norm1.bias": "model-00001-of-00004.safetensors",
378
+ "qwen2_5_vl_model.model.visual.blocks.10.norm1.weight": "model-00001-of-00004.safetensors",
379
+ "qwen2_5_vl_model.model.visual.blocks.10.norm2.bias": "model-00001-of-00004.safetensors",
380
+ "qwen2_5_vl_model.model.visual.blocks.10.norm2.weight": "model-00001-of-00004.safetensors",
381
+ "qwen2_5_vl_model.model.visual.blocks.11.attn.proj.bias": "model-00001-of-00004.safetensors",
382
+ "qwen2_5_vl_model.model.visual.blocks.11.attn.proj.weight": "model-00001-of-00004.safetensors",
383
+ "qwen2_5_vl_model.model.visual.blocks.11.attn.qkv.bias": "model-00001-of-00004.safetensors",
384
+ "qwen2_5_vl_model.model.visual.blocks.11.attn.qkv.weight": "model-00001-of-00004.safetensors",
385
+ "qwen2_5_vl_model.model.visual.blocks.11.mlp.fc1.bias": "model-00001-of-00004.safetensors",
386
+ "qwen2_5_vl_model.model.visual.blocks.11.mlp.fc1.weight": "model-00001-of-00004.safetensors",
387
+ "qwen2_5_vl_model.model.visual.blocks.11.mlp.fc2.bias": "model-00001-of-00004.safetensors",
388
+ "qwen2_5_vl_model.model.visual.blocks.11.mlp.fc2.weight": "model-00001-of-00004.safetensors",
389
+ "qwen2_5_vl_model.model.visual.blocks.11.norm1.bias": "model-00001-of-00004.safetensors",
390
+ "qwen2_5_vl_model.model.visual.blocks.11.norm1.weight": "model-00001-of-00004.safetensors",
391
+ "qwen2_5_vl_model.model.visual.blocks.11.norm2.bias": "model-00001-of-00004.safetensors",
392
+ "qwen2_5_vl_model.model.visual.blocks.11.norm2.weight": "model-00001-of-00004.safetensors",
393
+ "qwen2_5_vl_model.model.visual.blocks.12.attn.proj.bias": "model-00001-of-00004.safetensors",
394
+ "qwen2_5_vl_model.model.visual.blocks.12.attn.proj.weight": "model-00001-of-00004.safetensors",
395
+ "qwen2_5_vl_model.model.visual.blocks.12.attn.qkv.bias": "model-00001-of-00004.safetensors",
396
+ "qwen2_5_vl_model.model.visual.blocks.12.attn.qkv.weight": "model-00001-of-00004.safetensors",
397
+ "qwen2_5_vl_model.model.visual.blocks.12.mlp.fc1.bias": "model-00001-of-00004.safetensors",
398
+ "qwen2_5_vl_model.model.visual.blocks.12.mlp.fc1.weight": "model-00001-of-00004.safetensors",
399
+ "qwen2_5_vl_model.model.visual.blocks.12.mlp.fc2.bias": "model-00001-of-00004.safetensors",
400
+ "qwen2_5_vl_model.model.visual.blocks.12.mlp.fc2.weight": "model-00001-of-00004.safetensors",
401
+ "qwen2_5_vl_model.model.visual.blocks.12.norm1.bias": "model-00001-of-00004.safetensors",
402
+ "qwen2_5_vl_model.model.visual.blocks.12.norm1.weight": "model-00001-of-00004.safetensors",
403
+ "qwen2_5_vl_model.model.visual.blocks.12.norm2.bias": "model-00001-of-00004.safetensors",
404
+ "qwen2_5_vl_model.model.visual.blocks.12.norm2.weight": "model-00001-of-00004.safetensors",
405
+ "qwen2_5_vl_model.model.visual.blocks.13.attn.proj.bias": "model-00001-of-00004.safetensors",
406
+ "qwen2_5_vl_model.model.visual.blocks.13.attn.proj.weight": "model-00001-of-00004.safetensors",
407
+ "qwen2_5_vl_model.model.visual.blocks.13.attn.qkv.bias": "model-00001-of-00004.safetensors",
408
+ "qwen2_5_vl_model.model.visual.blocks.13.attn.qkv.weight": "model-00001-of-00004.safetensors",
409
+ "qwen2_5_vl_model.model.visual.blocks.13.mlp.fc1.bias": "model-00001-of-00004.safetensors",
410
+ "qwen2_5_vl_model.model.visual.blocks.13.mlp.fc1.weight": "model-00001-of-00004.safetensors",
411
+ "qwen2_5_vl_model.model.visual.blocks.13.mlp.fc2.bias": "model-00001-of-00004.safetensors",
412
+ "qwen2_5_vl_model.model.visual.blocks.13.mlp.fc2.weight": "model-00001-of-00004.safetensors",
413
+ "qwen2_5_vl_model.model.visual.blocks.13.norm1.bias": "model-00001-of-00004.safetensors",
414
+ "qwen2_5_vl_model.model.visual.blocks.13.norm1.weight": "model-00001-of-00004.safetensors",
415
+ "qwen2_5_vl_model.model.visual.blocks.13.norm2.bias": "model-00001-of-00004.safetensors",
416
+ "qwen2_5_vl_model.model.visual.blocks.13.norm2.weight": "model-00001-of-00004.safetensors",
417
+ "qwen2_5_vl_model.model.visual.blocks.14.attn.proj.bias": "model-00001-of-00004.safetensors",
418
+ "qwen2_5_vl_model.model.visual.blocks.14.attn.proj.weight": "model-00001-of-00004.safetensors",
419
+ "qwen2_5_vl_model.model.visual.blocks.14.attn.qkv.bias": "model-00001-of-00004.safetensors",
420
+ "qwen2_5_vl_model.model.visual.blocks.14.attn.qkv.weight": "model-00001-of-00004.safetensors",
421
+ "qwen2_5_vl_model.model.visual.blocks.14.mlp.fc1.bias": "model-00001-of-00004.safetensors",
422
+ "qwen2_5_vl_model.model.visual.blocks.14.mlp.fc1.weight": "model-00001-of-00004.safetensors",
423
+ "qwen2_5_vl_model.model.visual.blocks.14.mlp.fc2.bias": "model-00001-of-00004.safetensors",
424
+ "qwen2_5_vl_model.model.visual.blocks.14.mlp.fc2.weight": "model-00001-of-00004.safetensors",
425
+ "qwen2_5_vl_model.model.visual.blocks.14.norm1.bias": "model-00001-of-00004.safetensors",
426
+ "qwen2_5_vl_model.model.visual.blocks.14.norm1.weight": "model-00001-of-00004.safetensors",
427
+ "qwen2_5_vl_model.model.visual.blocks.14.norm2.bias": "model-00001-of-00004.safetensors",
428
+ "qwen2_5_vl_model.model.visual.blocks.14.norm2.weight": "model-00001-of-00004.safetensors",
429
+ "qwen2_5_vl_model.model.visual.blocks.15.attn.proj.bias": "model-00001-of-00004.safetensors",
430
+ "qwen2_5_vl_model.model.visual.blocks.15.attn.proj.weight": "model-00001-of-00004.safetensors",
431
+ "qwen2_5_vl_model.model.visual.blocks.15.attn.qkv.bias": "model-00001-of-00004.safetensors",
432
+ "qwen2_5_vl_model.model.visual.blocks.15.attn.qkv.weight": "model-00001-of-00004.safetensors",
433
+ "qwen2_5_vl_model.model.visual.blocks.15.mlp.fc1.bias": "model-00001-of-00004.safetensors",
434
+ "qwen2_5_vl_model.model.visual.blocks.15.mlp.fc1.weight": "model-00001-of-00004.safetensors",
435
+ "qwen2_5_vl_model.model.visual.blocks.15.mlp.fc2.bias": "model-00001-of-00004.safetensors",
436
+ "qwen2_5_vl_model.model.visual.blocks.15.mlp.fc2.weight": "model-00001-of-00004.safetensors",
437
+ "qwen2_5_vl_model.model.visual.blocks.15.norm1.bias": "model-00001-of-00004.safetensors",
438
+ "qwen2_5_vl_model.model.visual.blocks.15.norm1.weight": "model-00001-of-00004.safetensors",
439
+ "qwen2_5_vl_model.model.visual.blocks.15.norm2.bias": "model-00001-of-00004.safetensors",
440
+ "qwen2_5_vl_model.model.visual.blocks.15.norm2.weight": "model-00001-of-00004.safetensors",
441
+ "qwen2_5_vl_model.model.visual.blocks.16.attn.proj.bias": "model-00001-of-00004.safetensors",
442
+ "qwen2_5_vl_model.model.visual.blocks.16.attn.proj.weight": "model-00001-of-00004.safetensors",
443
+ "qwen2_5_vl_model.model.visual.blocks.16.attn.qkv.bias": "model-00001-of-00004.safetensors",
444
+ "qwen2_5_vl_model.model.visual.blocks.16.attn.qkv.weight": "model-00001-of-00004.safetensors",
445
+ "qwen2_5_vl_model.model.visual.blocks.16.mlp.fc1.bias": "model-00001-of-00004.safetensors",
446
+ "qwen2_5_vl_model.model.visual.blocks.16.mlp.fc1.weight": "model-00001-of-00004.safetensors",
447
+ "qwen2_5_vl_model.model.visual.blocks.16.mlp.fc2.bias": "model-00001-of-00004.safetensors",
448
+ "qwen2_5_vl_model.model.visual.blocks.16.mlp.fc2.weight": "model-00001-of-00004.safetensors",
449
+ "qwen2_5_vl_model.model.visual.blocks.16.norm1.bias": "model-00001-of-00004.safetensors",
450
+ "qwen2_5_vl_model.model.visual.blocks.16.norm1.weight": "model-00001-of-00004.safetensors",
451
+ "qwen2_5_vl_model.model.visual.blocks.16.norm2.bias": "model-00001-of-00004.safetensors",
452
+ "qwen2_5_vl_model.model.visual.blocks.16.norm2.weight": "model-00001-of-00004.safetensors",
453
+ "qwen2_5_vl_model.model.visual.blocks.17.attn.proj.bias": "model-00001-of-00004.safetensors",
454
+ "qwen2_5_vl_model.model.visual.blocks.17.attn.proj.weight": "model-00001-of-00004.safetensors",
455
+ "qwen2_5_vl_model.model.visual.blocks.17.attn.qkv.bias": "model-00001-of-00004.safetensors",
456
+ "qwen2_5_vl_model.model.visual.blocks.17.attn.qkv.weight": "model-00001-of-00004.safetensors",
457
+ "qwen2_5_vl_model.model.visual.blocks.17.mlp.fc1.bias": "model-00001-of-00004.safetensors",
458
+ "qwen2_5_vl_model.model.visual.blocks.17.mlp.fc1.weight": "model-00001-of-00004.safetensors",
459
+ "qwen2_5_vl_model.model.visual.blocks.17.mlp.fc2.bias": "model-00001-of-00004.safetensors",
460
+ "qwen2_5_vl_model.model.visual.blocks.17.mlp.fc2.weight": "model-00001-of-00004.safetensors",
461
+ "qwen2_5_vl_model.model.visual.blocks.17.norm1.bias": "model-00001-of-00004.safetensors",
462
+ "qwen2_5_vl_model.model.visual.blocks.17.norm1.weight": "model-00001-of-00004.safetensors",
463
+ "qwen2_5_vl_model.model.visual.blocks.17.norm2.bias": "model-00001-of-00004.safetensors",
464
+ "qwen2_5_vl_model.model.visual.blocks.17.norm2.weight": "model-00001-of-00004.safetensors",
465
+ "qwen2_5_vl_model.model.visual.blocks.18.attn.proj.bias": "model-00001-of-00004.safetensors",
466
+ "qwen2_5_vl_model.model.visual.blocks.18.attn.proj.weight": "model-00001-of-00004.safetensors",
467
+ "qwen2_5_vl_model.model.visual.blocks.18.attn.qkv.bias": "model-00001-of-00004.safetensors",
468
+ "qwen2_5_vl_model.model.visual.blocks.18.attn.qkv.weight": "model-00001-of-00004.safetensors",
469
+ "qwen2_5_vl_model.model.visual.blocks.18.mlp.fc1.bias": "model-00001-of-00004.safetensors",
470
+ "qwen2_5_vl_model.model.visual.blocks.18.mlp.fc1.weight": "model-00001-of-00004.safetensors",
471
+ "qwen2_5_vl_model.model.visual.blocks.18.mlp.fc2.bias": "model-00001-of-00004.safetensors",
472
+ "qwen2_5_vl_model.model.visual.blocks.18.mlp.fc2.weight": "model-00001-of-00004.safetensors",
473
+ "qwen2_5_vl_model.model.visual.blocks.18.norm1.bias": "model-00001-of-00004.safetensors",
474
+ "qwen2_5_vl_model.model.visual.blocks.18.norm1.weight": "model-00001-of-00004.safetensors",
475
+ "qwen2_5_vl_model.model.visual.blocks.18.norm2.bias": "model-00001-of-00004.safetensors",
476
+ "qwen2_5_vl_model.model.visual.blocks.18.norm2.weight": "model-00001-of-00004.safetensors",
477
+ "qwen2_5_vl_model.model.visual.blocks.19.attn.proj.bias": "model-00001-of-00004.safetensors",
478
+ "qwen2_5_vl_model.model.visual.blocks.19.attn.proj.weight": "model-00001-of-00004.safetensors",
479
+ "qwen2_5_vl_model.model.visual.blocks.19.attn.qkv.bias": "model-00001-of-00004.safetensors",
480
+ "qwen2_5_vl_model.model.visual.blocks.19.attn.qkv.weight": "model-00001-of-00004.safetensors",
481
+ "qwen2_5_vl_model.model.visual.blocks.19.mlp.fc1.bias": "model-00001-of-00004.safetensors",
482
+ "qwen2_5_vl_model.model.visual.blocks.19.mlp.fc1.weight": "model-00001-of-00004.safetensors",
483
+ "qwen2_5_vl_model.model.visual.blocks.19.mlp.fc2.bias": "model-00001-of-00004.safetensors",
484
+ "qwen2_5_vl_model.model.visual.blocks.19.mlp.fc2.weight": "model-00001-of-00004.safetensors",
485
+ "qwen2_5_vl_model.model.visual.blocks.19.norm1.bias": "model-00001-of-00004.safetensors",
486
+ "qwen2_5_vl_model.model.visual.blocks.19.norm1.weight": "model-00001-of-00004.safetensors",
487
+ "qwen2_5_vl_model.model.visual.blocks.19.norm2.bias": "model-00001-of-00004.safetensors",
488
+ "qwen2_5_vl_model.model.visual.blocks.19.norm2.weight": "model-00001-of-00004.safetensors",
489
+ "qwen2_5_vl_model.model.visual.blocks.2.attn.proj.bias": "model-00001-of-00004.safetensors",
490
+ "qwen2_5_vl_model.model.visual.blocks.2.attn.proj.weight": "model-00001-of-00004.safetensors",
491
+ "qwen2_5_vl_model.model.visual.blocks.2.attn.qkv.bias": "model-00001-of-00004.safetensors",
492
+ "qwen2_5_vl_model.model.visual.blocks.2.attn.qkv.weight": "model-00001-of-00004.safetensors",
493
+ "qwen2_5_vl_model.model.visual.blocks.2.mlp.fc1.bias": "model-00001-of-00004.safetensors",
494
+ "qwen2_5_vl_model.model.visual.blocks.2.mlp.fc1.weight": "model-00001-of-00004.safetensors",
495
+ "qwen2_5_vl_model.model.visual.blocks.2.mlp.fc2.bias": "model-00001-of-00004.safetensors",
496
+ "qwen2_5_vl_model.model.visual.blocks.2.mlp.fc2.weight": "model-00001-of-00004.safetensors",
497
+ "qwen2_5_vl_model.model.visual.blocks.2.norm1.bias": "model-00001-of-00004.safetensors",
498
+ "qwen2_5_vl_model.model.visual.blocks.2.norm1.weight": "model-00001-of-00004.safetensors",
499
+ "qwen2_5_vl_model.model.visual.blocks.2.norm2.bias": "model-00001-of-00004.safetensors",
500
+ "qwen2_5_vl_model.model.visual.blocks.2.norm2.weight": "model-00001-of-00004.safetensors",
501
+ "qwen2_5_vl_model.model.visual.blocks.20.attn.proj.bias": "model-00001-of-00004.safetensors",
502
+ "qwen2_5_vl_model.model.visual.blocks.20.attn.proj.weight": "model-00001-of-00004.safetensors",
503
+ "qwen2_5_vl_model.model.visual.blocks.20.attn.qkv.bias": "model-00001-of-00004.safetensors",
504
+ "qwen2_5_vl_model.model.visual.blocks.20.attn.qkv.weight": "model-00001-of-00004.safetensors",
505
+ "qwen2_5_vl_model.model.visual.blocks.20.mlp.fc1.bias": "model-00001-of-00004.safetensors",
506
+ "qwen2_5_vl_model.model.visual.blocks.20.mlp.fc1.weight": "model-00001-of-00004.safetensors",
507
+ "qwen2_5_vl_model.model.visual.blocks.20.mlp.fc2.bias": "model-00001-of-00004.safetensors",
508
+ "qwen2_5_vl_model.model.visual.blocks.20.mlp.fc2.weight": "model-00001-of-00004.safetensors",
509
+ "qwen2_5_vl_model.model.visual.blocks.20.norm1.bias": "model-00001-of-00004.safetensors",
510
+ "qwen2_5_vl_model.model.visual.blocks.20.norm1.weight": "model-00001-of-00004.safetensors",
511
+ "qwen2_5_vl_model.model.visual.blocks.20.norm2.bias": "model-00001-of-00004.safetensors",
512
+ "qwen2_5_vl_model.model.visual.blocks.20.norm2.weight": "model-00001-of-00004.safetensors",
513
+ "qwen2_5_vl_model.model.visual.blocks.21.attn.proj.bias": "model-00001-of-00004.safetensors",
514
+ "qwen2_5_vl_model.model.visual.blocks.21.attn.proj.weight": "model-00001-of-00004.safetensors",
515
+ "qwen2_5_vl_model.model.visual.blocks.21.attn.qkv.bias": "model-00001-of-00004.safetensors",
516
+ "qwen2_5_vl_model.model.visual.blocks.21.attn.qkv.weight": "model-00001-of-00004.safetensors",
517
+ "qwen2_5_vl_model.model.visual.blocks.21.mlp.fc1.bias": "model-00001-of-00004.safetensors",
518
+ "qwen2_5_vl_model.model.visual.blocks.21.mlp.fc1.weight": "model-00001-of-00004.safetensors",
519
+ "qwen2_5_vl_model.model.visual.blocks.21.mlp.fc2.bias": "model-00001-of-00004.safetensors",
520
+ "qwen2_5_vl_model.model.visual.blocks.21.mlp.fc2.weight": "model-00001-of-00004.safetensors",
521
+ "qwen2_5_vl_model.model.visual.blocks.21.norm1.bias": "model-00001-of-00004.safetensors",
522
+ "qwen2_5_vl_model.model.visual.blocks.21.norm1.weight": "model-00001-of-00004.safetensors",
523
+ "qwen2_5_vl_model.model.visual.blocks.21.norm2.bias": "model-00001-of-00004.safetensors",
524
+ "qwen2_5_vl_model.model.visual.blocks.21.norm2.weight": "model-00001-of-00004.safetensors",
525
+ "qwen2_5_vl_model.model.visual.blocks.22.attn.proj.bias": "model-00001-of-00004.safetensors",
526
+ "qwen2_5_vl_model.model.visual.blocks.22.attn.proj.weight": "model-00001-of-00004.safetensors",
527
+ "qwen2_5_vl_model.model.visual.blocks.22.attn.qkv.bias": "model-00001-of-00004.safetensors",
528
+ "qwen2_5_vl_model.model.visual.blocks.22.attn.qkv.weight": "model-00001-of-00004.safetensors",
529
+ "qwen2_5_vl_model.model.visual.blocks.22.mlp.fc1.bias": "model-00001-of-00004.safetensors",
530
+ "qwen2_5_vl_model.model.visual.blocks.22.mlp.fc1.weight": "model-00001-of-00004.safetensors",
531
+ "qwen2_5_vl_model.model.visual.blocks.22.mlp.fc2.bias": "model-00001-of-00004.safetensors",
532
+ "qwen2_5_vl_model.model.visual.blocks.22.mlp.fc2.weight": "model-00001-of-00004.safetensors",
533
+ "qwen2_5_vl_model.model.visual.blocks.22.norm1.bias": "model-00001-of-00004.safetensors",
534
+ "qwen2_5_vl_model.model.visual.blocks.22.norm1.weight": "model-00001-of-00004.safetensors",
535
+ "qwen2_5_vl_model.model.visual.blocks.22.norm2.bias": "model-00001-of-00004.safetensors",
536
+ "qwen2_5_vl_model.model.visual.blocks.22.norm2.weight": "model-00001-of-00004.safetensors",
537
+ "qwen2_5_vl_model.model.visual.blocks.23.attn.proj.bias": "model-00001-of-00004.safetensors",
538
+ "qwen2_5_vl_model.model.visual.blocks.23.attn.proj.weight": "model-00001-of-00004.safetensors",
539
+ "qwen2_5_vl_model.model.visual.blocks.23.attn.qkv.bias": "model-00001-of-00004.safetensors",
540
+ "qwen2_5_vl_model.model.visual.blocks.23.attn.qkv.weight": "model-00001-of-00004.safetensors",
541
+ "qwen2_5_vl_model.model.visual.blocks.23.mlp.fc1.bias": "model-00001-of-00004.safetensors",
542
+ "qwen2_5_vl_model.model.visual.blocks.23.mlp.fc1.weight": "model-00001-of-00004.safetensors",
543
+ "qwen2_5_vl_model.model.visual.blocks.23.mlp.fc2.bias": "model-00001-of-00004.safetensors",
544
+ "qwen2_5_vl_model.model.visual.blocks.23.mlp.fc2.weight": "model-00001-of-00004.safetensors",
545
+ "qwen2_5_vl_model.model.visual.blocks.23.norm1.bias": "model-00001-of-00004.safetensors",
546
+ "qwen2_5_vl_model.model.visual.blocks.23.norm1.weight": "model-00001-of-00004.safetensors",
547
+ "qwen2_5_vl_model.model.visual.blocks.23.norm2.bias": "model-00001-of-00004.safetensors",
548
+ "qwen2_5_vl_model.model.visual.blocks.23.norm2.weight": "model-00001-of-00004.safetensors",
549
+ "qwen2_5_vl_model.model.visual.blocks.24.attn.proj.bias": "model-00001-of-00004.safetensors",
550
+ "qwen2_5_vl_model.model.visual.blocks.24.attn.proj.weight": "model-00001-of-00004.safetensors",
551
+ "qwen2_5_vl_model.model.visual.blocks.24.attn.qkv.bias": "model-00001-of-00004.safetensors",
552
+ "qwen2_5_vl_model.model.visual.blocks.24.attn.qkv.weight": "model-00001-of-00004.safetensors",
553
+ "qwen2_5_vl_model.model.visual.blocks.24.mlp.fc1.bias": "model-00001-of-00004.safetensors",
554
+ "qwen2_5_vl_model.model.visual.blocks.24.mlp.fc1.weight": "model-00001-of-00004.safetensors",
555
+ "qwen2_5_vl_model.model.visual.blocks.24.mlp.fc2.bias": "model-00001-of-00004.safetensors",
556
+ "qwen2_5_vl_model.model.visual.blocks.24.mlp.fc2.weight": "model-00001-of-00004.safetensors",
557
+ "qwen2_5_vl_model.model.visual.blocks.24.norm1.bias": "model-00001-of-00004.safetensors",
558
+ "qwen2_5_vl_model.model.visual.blocks.24.norm1.weight": "model-00001-of-00004.safetensors",
559
+ "qwen2_5_vl_model.model.visual.blocks.24.norm2.bias": "model-00001-of-00004.safetensors",
560
+ "qwen2_5_vl_model.model.visual.blocks.24.norm2.weight": "model-00001-of-00004.safetensors",
561
+ "qwen2_5_vl_model.model.visual.blocks.25.attn.proj.bias": "model-00001-of-00004.safetensors",
562
+ "qwen2_5_vl_model.model.visual.blocks.25.attn.proj.weight": "model-00001-of-00004.safetensors",
563
+ "qwen2_5_vl_model.model.visual.blocks.25.attn.qkv.bias": "model-00001-of-00004.safetensors",
564
+ "qwen2_5_vl_model.model.visual.blocks.25.attn.qkv.weight": "model-00001-of-00004.safetensors",
565
+ "qwen2_5_vl_model.model.visual.blocks.25.mlp.fc1.bias": "model-00001-of-00004.safetensors",
566
+ "qwen2_5_vl_model.model.visual.blocks.25.mlp.fc1.weight": "model-00001-of-00004.safetensors",
567
+ "qwen2_5_vl_model.model.visual.blocks.25.mlp.fc2.bias": "model-00001-of-00004.safetensors",
568
+ "qwen2_5_vl_model.model.visual.blocks.25.mlp.fc2.weight": "model-00001-of-00004.safetensors",
569
+ "qwen2_5_vl_model.model.visual.blocks.25.norm1.bias": "model-00001-of-00004.safetensors",
570
+ "qwen2_5_vl_model.model.visual.blocks.25.norm1.weight": "model-00001-of-00004.safetensors",
571
+ "qwen2_5_vl_model.model.visual.blocks.25.norm2.bias": "model-00001-of-00004.safetensors",
572
+ "qwen2_5_vl_model.model.visual.blocks.25.norm2.weight": "model-00001-of-00004.safetensors",
573
+ "qwen2_5_vl_model.model.visual.blocks.26.attn.proj.bias": "model-00001-of-00004.safetensors",
574
+ "qwen2_5_vl_model.model.visual.blocks.26.attn.proj.weight": "model-00001-of-00004.safetensors",
575
+ "qwen2_5_vl_model.model.visual.blocks.26.attn.qkv.bias": "model-00001-of-00004.safetensors",
576
+ "qwen2_5_vl_model.model.visual.blocks.26.attn.qkv.weight": "model-00001-of-00004.safetensors",
577
+ "qwen2_5_vl_model.model.visual.blocks.26.mlp.fc1.bias": "model-00001-of-00004.safetensors",
578
+ "qwen2_5_vl_model.model.visual.blocks.26.mlp.fc1.weight": "model-00001-of-00004.safetensors",
579
+ "qwen2_5_vl_model.model.visual.blocks.26.mlp.fc2.bias": "model-00001-of-00004.safetensors",
580
+ "qwen2_5_vl_model.model.visual.blocks.26.mlp.fc2.weight": "model-00001-of-00004.safetensors",
581
+ "qwen2_5_vl_model.model.visual.blocks.26.norm1.bias": "model-00001-of-00004.safetensors",
582
+ "qwen2_5_vl_model.model.visual.blocks.26.norm1.weight": "model-00001-of-00004.safetensors",
583
+ "qwen2_5_vl_model.model.visual.blocks.26.norm2.bias": "model-00001-of-00004.safetensors",
584
+ "qwen2_5_vl_model.model.visual.blocks.26.norm2.weight": "model-00001-of-00004.safetensors",
585
+ "qwen2_5_vl_model.model.visual.blocks.27.attn.proj.bias": "model-00001-of-00004.safetensors",
586
+ "qwen2_5_vl_model.model.visual.blocks.27.attn.proj.weight": "model-00001-of-00004.safetensors",
587
+ "qwen2_5_vl_model.model.visual.blocks.27.attn.qkv.bias": "model-00001-of-00004.safetensors",
588
+ "qwen2_5_vl_model.model.visual.blocks.27.attn.qkv.weight": "model-00001-of-00004.safetensors",
589
+ "qwen2_5_vl_model.model.visual.blocks.27.mlp.fc1.bias": "model-00001-of-00004.safetensors",
590
+ "qwen2_5_vl_model.model.visual.blocks.27.mlp.fc1.weight": "model-00001-of-00004.safetensors",
591
+ "qwen2_5_vl_model.model.visual.blocks.27.mlp.fc2.bias": "model-00001-of-00004.safetensors",
592
+ "qwen2_5_vl_model.model.visual.blocks.27.mlp.fc2.weight": "model-00001-of-00004.safetensors",
593
+ "qwen2_5_vl_model.model.visual.blocks.27.norm1.bias": "model-00001-of-00004.safetensors",
594
+ "qwen2_5_vl_model.model.visual.blocks.27.norm1.weight": "model-00001-of-00004.safetensors",
595
+ "qwen2_5_vl_model.model.visual.blocks.27.norm2.bias": "model-00001-of-00004.safetensors",
596
+ "qwen2_5_vl_model.model.visual.blocks.27.norm2.weight": "model-00001-of-00004.safetensors",
597
+ "qwen2_5_vl_model.model.visual.blocks.28.attn.proj.bias": "model-00001-of-00004.safetensors",
598
+ "qwen2_5_vl_model.model.visual.blocks.28.attn.proj.weight": "model-00001-of-00004.safetensors",
599
+ "qwen2_5_vl_model.model.visual.blocks.28.attn.qkv.bias": "model-00001-of-00004.safetensors",
600
+ "qwen2_5_vl_model.model.visual.blocks.28.attn.qkv.weight": "model-00001-of-00004.safetensors",
601
+ "qwen2_5_vl_model.model.visual.blocks.28.mlp.fc1.bias": "model-00001-of-00004.safetensors",
602
+ "qwen2_5_vl_model.model.visual.blocks.28.mlp.fc1.weight": "model-00001-of-00004.safetensors",
603
+ "qwen2_5_vl_model.model.visual.blocks.28.mlp.fc2.bias": "model-00001-of-00004.safetensors",
604
+ "qwen2_5_vl_model.model.visual.blocks.28.mlp.fc2.weight": "model-00001-of-00004.safetensors",
605
+ "qwen2_5_vl_model.model.visual.blocks.28.norm1.bias": "model-00001-of-00004.safetensors",
606
+ "qwen2_5_vl_model.model.visual.blocks.28.norm1.weight": "model-00001-of-00004.safetensors",
607
+ "qwen2_5_vl_model.model.visual.blocks.28.norm2.bias": "model-00001-of-00004.safetensors",
608
+ "qwen2_5_vl_model.model.visual.blocks.28.norm2.weight": "model-00001-of-00004.safetensors",
609
+ "qwen2_5_vl_model.model.visual.blocks.29.attn.proj.bias": "model-00001-of-00004.safetensors",
610
+ "qwen2_5_vl_model.model.visual.blocks.29.attn.proj.weight": "model-00001-of-00004.safetensors",
611
+ "qwen2_5_vl_model.model.visual.blocks.29.attn.qkv.bias": "model-00001-of-00004.safetensors",
612
+ "qwen2_5_vl_model.model.visual.blocks.29.attn.qkv.weight": "model-00001-of-00004.safetensors",
613
+ "qwen2_5_vl_model.model.visual.blocks.29.mlp.fc1.bias": "model-00001-of-00004.safetensors",
614
+ "qwen2_5_vl_model.model.visual.blocks.29.mlp.fc1.weight": "model-00001-of-00004.safetensors",
615
+ "qwen2_5_vl_model.model.visual.blocks.29.mlp.fc2.bias": "model-00001-of-00004.safetensors",
616
+ "qwen2_5_vl_model.model.visual.blocks.29.mlp.fc2.weight": "model-00001-of-00004.safetensors",
617
+ "qwen2_5_vl_model.model.visual.blocks.29.norm1.bias": "model-00001-of-00004.safetensors",
618
+ "qwen2_5_vl_model.model.visual.blocks.29.norm1.weight": "model-00001-of-00004.safetensors",
619
+ "qwen2_5_vl_model.model.visual.blocks.29.norm2.bias": "model-00001-of-00004.safetensors",
620
+ "qwen2_5_vl_model.model.visual.blocks.29.norm2.weight": "model-00001-of-00004.safetensors",
621
+ "qwen2_5_vl_model.model.visual.blocks.3.attn.proj.bias": "model-00001-of-00004.safetensors",
622
+ "qwen2_5_vl_model.model.visual.blocks.3.attn.proj.weight": "model-00001-of-00004.safetensors",
623
+ "qwen2_5_vl_model.model.visual.blocks.3.attn.qkv.bias": "model-00001-of-00004.safetensors",
624
+ "qwen2_5_vl_model.model.visual.blocks.3.attn.qkv.weight": "model-00001-of-00004.safetensors",
625
+ "qwen2_5_vl_model.model.visual.blocks.3.mlp.fc1.bias": "model-00001-of-00004.safetensors",
626
+ "qwen2_5_vl_model.model.visual.blocks.3.mlp.fc1.weight": "model-00001-of-00004.safetensors",
627
+ "qwen2_5_vl_model.model.visual.blocks.3.mlp.fc2.bias": "model-00001-of-00004.safetensors",
628
+ "qwen2_5_vl_model.model.visual.blocks.3.mlp.fc2.weight": "model-00001-of-00004.safetensors",
629
+ "qwen2_5_vl_model.model.visual.blocks.3.norm1.bias": "model-00001-of-00004.safetensors",
630
+ "qwen2_5_vl_model.model.visual.blocks.3.norm1.weight": "model-00001-of-00004.safetensors",
631
+ "qwen2_5_vl_model.model.visual.blocks.3.norm2.bias": "model-00001-of-00004.safetensors",
632
+ "qwen2_5_vl_model.model.visual.blocks.3.norm2.weight": "model-00001-of-00004.safetensors",
633
+ "qwen2_5_vl_model.model.visual.blocks.30.attn.proj.bias": "model-00001-of-00004.safetensors",
634
+ "qwen2_5_vl_model.model.visual.blocks.30.attn.proj.weight": "model-00001-of-00004.safetensors",
635
+ "qwen2_5_vl_model.model.visual.blocks.30.attn.qkv.bias": "model-00001-of-00004.safetensors",
636
+ "qwen2_5_vl_model.model.visual.blocks.30.attn.qkv.weight": "model-00001-of-00004.safetensors",
637
+ "qwen2_5_vl_model.model.visual.blocks.30.mlp.fc1.bias": "model-00001-of-00004.safetensors",
638
+ "qwen2_5_vl_model.model.visual.blocks.30.mlp.fc1.weight": "model-00001-of-00004.safetensors",
639
+ "qwen2_5_vl_model.model.visual.blocks.30.mlp.fc2.bias": "model-00001-of-00004.safetensors",
640
+ "qwen2_5_vl_model.model.visual.blocks.30.mlp.fc2.weight": "model-00001-of-00004.safetensors",
641
+ "qwen2_5_vl_model.model.visual.blocks.30.norm1.bias": "model-00001-of-00004.safetensors",
642
+ "qwen2_5_vl_model.model.visual.blocks.30.norm1.weight": "model-00001-of-00004.safetensors",
643
+ "qwen2_5_vl_model.model.visual.blocks.30.norm2.bias": "model-00001-of-00004.safetensors",
644
+ "qwen2_5_vl_model.model.visual.blocks.30.norm2.weight": "model-00001-of-00004.safetensors",
645
+ "qwen2_5_vl_model.model.visual.blocks.31.attn.proj.bias": "model-00001-of-00004.safetensors",
646
+ "qwen2_5_vl_model.model.visual.blocks.31.attn.proj.weight": "model-00001-of-00004.safetensors",
647
+ "qwen2_5_vl_model.model.visual.blocks.31.attn.qkv.bias": "model-00001-of-00004.safetensors",
648
+ "qwen2_5_vl_model.model.visual.blocks.31.attn.qkv.weight": "model-00001-of-00004.safetensors",
649
+ "qwen2_5_vl_model.model.visual.blocks.31.mlp.fc1.bias": "model-00001-of-00004.safetensors",
650
+ "qwen2_5_vl_model.model.visual.blocks.31.mlp.fc1.weight": "model-00001-of-00004.safetensors",
651
+ "qwen2_5_vl_model.model.visual.blocks.31.mlp.fc2.bias": "model-00001-of-00004.safetensors",
652
+ "qwen2_5_vl_model.model.visual.blocks.31.mlp.fc2.weight": "model-00001-of-00004.safetensors",
653
+ "qwen2_5_vl_model.model.visual.blocks.31.norm1.bias": "model-00001-of-00004.safetensors",
654
+ "qwen2_5_vl_model.model.visual.blocks.31.norm1.weight": "model-00001-of-00004.safetensors",
655
+ "qwen2_5_vl_model.model.visual.blocks.31.norm2.bias": "model-00001-of-00004.safetensors",
656
+ "qwen2_5_vl_model.model.visual.blocks.31.norm2.weight": "model-00001-of-00004.safetensors",
657
+ "qwen2_5_vl_model.model.visual.blocks.4.attn.proj.bias": "model-00001-of-00004.safetensors",
658
+ "qwen2_5_vl_model.model.visual.blocks.4.attn.proj.weight": "model-00001-of-00004.safetensors",
659
+ "qwen2_5_vl_model.model.visual.blocks.4.attn.qkv.bias": "model-00001-of-00004.safetensors",
660
+ "qwen2_5_vl_model.model.visual.blocks.4.attn.qkv.weight": "model-00001-of-00004.safetensors",
661
+ "qwen2_5_vl_model.model.visual.blocks.4.mlp.fc1.bias": "model-00001-of-00004.safetensors",
662
+ "qwen2_5_vl_model.model.visual.blocks.4.mlp.fc1.weight": "model-00001-of-00004.safetensors",
663
+ "qwen2_5_vl_model.model.visual.blocks.4.mlp.fc2.bias": "model-00001-of-00004.safetensors",
664
+ "qwen2_5_vl_model.model.visual.blocks.4.mlp.fc2.weight": "model-00001-of-00004.safetensors",
665
+ "qwen2_5_vl_model.model.visual.blocks.4.norm1.bias": "model-00001-of-00004.safetensors",
666
+ "qwen2_5_vl_model.model.visual.blocks.4.norm1.weight": "model-00001-of-00004.safetensors",
667
+ "qwen2_5_vl_model.model.visual.blocks.4.norm2.bias": "model-00001-of-00004.safetensors",
668
+ "qwen2_5_vl_model.model.visual.blocks.4.norm2.weight": "model-00001-of-00004.safetensors",
669
+ "qwen2_5_vl_model.model.visual.blocks.5.attn.proj.bias": "model-00001-of-00004.safetensors",
670
+ "qwen2_5_vl_model.model.visual.blocks.5.attn.proj.weight": "model-00001-of-00004.safetensors",
671
+ "qwen2_5_vl_model.model.visual.blocks.5.attn.qkv.bias": "model-00001-of-00004.safetensors",
672
+ "qwen2_5_vl_model.model.visual.blocks.5.attn.qkv.weight": "model-00001-of-00004.safetensors",
673
+ "qwen2_5_vl_model.model.visual.blocks.5.mlp.fc1.bias": "model-00001-of-00004.safetensors",
674
+ "qwen2_5_vl_model.model.visual.blocks.5.mlp.fc1.weight": "model-00001-of-00004.safetensors",
675
+ "qwen2_5_vl_model.model.visual.blocks.5.mlp.fc2.bias": "model-00001-of-00004.safetensors",
676
+ "qwen2_5_vl_model.model.visual.blocks.5.mlp.fc2.weight": "model-00001-of-00004.safetensors",
677
+ "qwen2_5_vl_model.model.visual.blocks.5.norm1.bias": "model-00001-of-00004.safetensors",
678
+ "qwen2_5_vl_model.model.visual.blocks.5.norm1.weight": "model-00001-of-00004.safetensors",
679
+ "qwen2_5_vl_model.model.visual.blocks.5.norm2.bias": "model-00001-of-00004.safetensors",
680
+ "qwen2_5_vl_model.model.visual.blocks.5.norm2.weight": "model-00001-of-00004.safetensors",
681
+ "qwen2_5_vl_model.model.visual.blocks.6.attn.proj.bias": "model-00001-of-00004.safetensors",
682
+ "qwen2_5_vl_model.model.visual.blocks.6.attn.proj.weight": "model-00001-of-00004.safetensors",
683
+ "qwen2_5_vl_model.model.visual.blocks.6.attn.qkv.bias": "model-00001-of-00004.safetensors",
684
+ "qwen2_5_vl_model.model.visual.blocks.6.attn.qkv.weight": "model-00001-of-00004.safetensors",
685
+ "qwen2_5_vl_model.model.visual.blocks.6.mlp.fc1.bias": "model-00001-of-00004.safetensors",
686
+ "qwen2_5_vl_model.model.visual.blocks.6.mlp.fc1.weight": "model-00001-of-00004.safetensors",
687
+ "qwen2_5_vl_model.model.visual.blocks.6.mlp.fc2.bias": "model-00001-of-00004.safetensors",
688
+ "qwen2_5_vl_model.model.visual.blocks.6.mlp.fc2.weight": "model-00001-of-00004.safetensors",
689
+ "qwen2_5_vl_model.model.visual.blocks.6.norm1.bias": "model-00001-of-00004.safetensors",
690
+ "qwen2_5_vl_model.model.visual.blocks.6.norm1.weight": "model-00001-of-00004.safetensors",
691
+ "qwen2_5_vl_model.model.visual.blocks.6.norm2.bias": "model-00001-of-00004.safetensors",
692
+ "qwen2_5_vl_model.model.visual.blocks.6.norm2.weight": "model-00001-of-00004.safetensors",
693
+ "qwen2_5_vl_model.model.visual.blocks.7.attn.proj.bias": "model-00001-of-00004.safetensors",
694
+ "qwen2_5_vl_model.model.visual.blocks.7.attn.proj.weight": "model-00001-of-00004.safetensors",
695
+ "qwen2_5_vl_model.model.visual.blocks.7.attn.qkv.bias": "model-00001-of-00004.safetensors",
696
+ "qwen2_5_vl_model.model.visual.blocks.7.attn.qkv.weight": "model-00001-of-00004.safetensors",
697
+ "qwen2_5_vl_model.model.visual.blocks.7.mlp.fc1.bias": "model-00001-of-00004.safetensors",
698
+ "qwen2_5_vl_model.model.visual.blocks.7.mlp.fc1.weight": "model-00001-of-00004.safetensors",
699
+ "qwen2_5_vl_model.model.visual.blocks.7.mlp.fc2.bias": "model-00001-of-00004.safetensors",
700
+ "qwen2_5_vl_model.model.visual.blocks.7.mlp.fc2.weight": "model-00001-of-00004.safetensors",
701
+ "qwen2_5_vl_model.model.visual.blocks.7.norm1.bias": "model-00001-of-00004.safetensors",
702
+ "qwen2_5_vl_model.model.visual.blocks.7.norm1.weight": "model-00001-of-00004.safetensors",
703
+ "qwen2_5_vl_model.model.visual.blocks.7.norm2.bias": "model-00001-of-00004.safetensors",
704
+ "qwen2_5_vl_model.model.visual.blocks.7.norm2.weight": "model-00001-of-00004.safetensors",
705
+ "qwen2_5_vl_model.model.visual.blocks.8.attn.proj.bias": "model-00001-of-00004.safetensors",
706
+ "qwen2_5_vl_model.model.visual.blocks.8.attn.proj.weight": "model-00001-of-00004.safetensors",
707
+ "qwen2_5_vl_model.model.visual.blocks.8.attn.qkv.bias": "model-00001-of-00004.safetensors",
708
+ "qwen2_5_vl_model.model.visual.blocks.8.attn.qkv.weight": "model-00001-of-00004.safetensors",
709
+ "qwen2_5_vl_model.model.visual.blocks.8.mlp.fc1.bias": "model-00001-of-00004.safetensors",
710
+ "qwen2_5_vl_model.model.visual.blocks.8.mlp.fc1.weight": "model-00001-of-00004.safetensors",
711
+ "qwen2_5_vl_model.model.visual.blocks.8.mlp.fc2.bias": "model-00001-of-00004.safetensors",
712
+ "qwen2_5_vl_model.model.visual.blocks.8.mlp.fc2.weight": "model-00001-of-00004.safetensors",
713
+ "qwen2_5_vl_model.model.visual.blocks.8.norm1.bias": "model-00001-of-00004.safetensors",
714
+ "qwen2_5_vl_model.model.visual.blocks.8.norm1.weight": "model-00001-of-00004.safetensors",
715
+ "qwen2_5_vl_model.model.visual.blocks.8.norm2.bias": "model-00001-of-00004.safetensors",
716
+ "qwen2_5_vl_model.model.visual.blocks.8.norm2.weight": "model-00001-of-00004.safetensors",
717
+ "qwen2_5_vl_model.model.visual.blocks.9.attn.proj.bias": "model-00001-of-00004.safetensors",
718
+ "qwen2_5_vl_model.model.visual.blocks.9.attn.proj.weight": "model-00001-of-00004.safetensors",
719
+ "qwen2_5_vl_model.model.visual.blocks.9.attn.qkv.bias": "model-00001-of-00004.safetensors",
720
+ "qwen2_5_vl_model.model.visual.blocks.9.attn.qkv.weight": "model-00001-of-00004.safetensors",
721
+ "qwen2_5_vl_model.model.visual.blocks.9.mlp.fc1.bias": "model-00001-of-00004.safetensors",
722
+ "qwen2_5_vl_model.model.visual.blocks.9.mlp.fc1.weight": "model-00001-of-00004.safetensors",
723
+ "qwen2_5_vl_model.model.visual.blocks.9.mlp.fc2.bias": "model-00001-of-00004.safetensors",
724
+ "qwen2_5_vl_model.model.visual.blocks.9.mlp.fc2.weight": "model-00001-of-00004.safetensors",
725
+ "qwen2_5_vl_model.model.visual.blocks.9.norm1.bias": "model-00001-of-00004.safetensors",
726
+ "qwen2_5_vl_model.model.visual.blocks.9.norm1.weight": "model-00001-of-00004.safetensors",
727
+ "qwen2_5_vl_model.model.visual.blocks.9.norm2.bias": "model-00001-of-00004.safetensors",
728
+ "qwen2_5_vl_model.model.visual.blocks.9.norm2.weight": "model-00001-of-00004.safetensors",
729
+ "qwen2_5_vl_model.model.visual.merger.ln_q.bias": "model-00001-of-00004.safetensors",
730
+ "qwen2_5_vl_model.model.visual.merger.ln_q.weight": "model-00001-of-00004.safetensors",
731
+ "qwen2_5_vl_model.model.visual.merger.mlp.0.bias": "model-00001-of-00004.safetensors",
732
+ "qwen2_5_vl_model.model.visual.merger.mlp.0.weight": "model-00001-of-00004.safetensors",
733
+ "qwen2_5_vl_model.model.visual.merger.mlp.2.bias": "model-00001-of-00004.safetensors",
734
+ "qwen2_5_vl_model.model.visual.merger.mlp.2.weight": "model-00001-of-00004.safetensors",
735
+ "qwen2_5_vl_model.model.visual.patch_embed.proj.weight": "model-00001-of-00004.safetensors"
736
+ }
737
+ }
modeling_qqmm.py ADDED
@@ -0,0 +1,343 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Optional, List
2
+ import torch
3
+ from torch import nn
4
+ from torch.nn import functional as F
5
+ from transformers import PreTrainedModel, AutoModel, AutoModelForCausalLM, Qwen2_5_VLForConditionalGeneration
6
+ from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import Qwen2_5_VLCausalLMOutputWithPast
7
+ from typing import List, Optional, Tuple, Union, Dict
8
+ import torch
9
+ from torch.nn import CrossEntropyLoss
10
+
11
+ from transformers.modeling_outputs import CausalLMOutputWithPast
12
+ from transformers.generation.utils import GenerateOutput
13
+ from .configuration_qqmm import QQMMConfig
14
+
15
+ def _prepare_4d_causal_attention_mask_with_cache_position(
16
+ attention_mask: torch.Tensor,
17
+ sequence_length: int,
18
+ target_length: int,
19
+ dtype: torch.dtype,
20
+ device: torch.device,
21
+ min_dtype: float,
22
+ cache_position: torch.Tensor,
23
+ batch_size: int,
24
+ ):
25
+ """
26
+ Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
27
+ `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.
28
+
29
+ Args:
30
+ attention_mask (`torch.Tensor`):
31
+ A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape `(batch_size, 1, query_length, key_value_length)`.
32
+ sequence_length (`int`):
33
+ The sequence length being processed.
34
+ target_length (`int`):
35
+ The target length: when generating with static cache, the mask should be as long as the static cache, to account for the 0 padding, the part of the cache that is not filled yet.
36
+ dtype (`torch.dtype`):
37
+ The dtype to use for the 4D attention mask.
38
+ device (`torch.device`):
39
+ The device to plcae the 4D attention mask on.
40
+ min_dtype (`float`):
41
+ The minimum value representable with the dtype `dtype`.
42
+ cache_position (`torch.Tensor`):
43
+ Indices depicting the position of the input sequence tokens in the sequence.
44
+ batch_size (`torch.Tensor`):
45
+ Batch size.
46
+ """
47
+ if attention_mask is not None and attention_mask.dim() == 4:
48
+ # In this case we assume that the mask comes already in inverted form and requires no inversion or slicing.
49
+ causal_mask = attention_mask
50
+ else:
51
+ causal_mask = torch.full((sequence_length, target_length), fill_value=min_dtype, dtype=dtype, device=device)
52
+ if sequence_length != 1:
53
+ causal_mask = torch.triu(causal_mask, diagonal=1)
54
+ causal_mask *= torch.arange(target_length, device=device) > cache_position.reshape(-1, 1)
55
+ causal_mask = causal_mask[None, None, :, :].expand(batch_size, 1, -1, -1)
56
+ if attention_mask is not None:
57
+ causal_mask = causal_mask.clone() # copy to contiguous memory for in-place edit
58
+ mask_length = attention_mask.shape[-1]
59
+ padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[:, None, None, :]
60
+ padding_mask = padding_mask == 0
61
+ causal_mask[:, :, :, :mask_length] = causal_mask[:, :, :, :mask_length].masked_fill(
62
+ padding_mask, min_dtype
63
+ )
64
+
65
+ return causal_mask
66
+
67
+ def padcat_sequences(sequences, value=0, pad_side='right'):
68
+ if all(s is None for s in sequences):
69
+ return None
70
+ max_l = max(s.size(1) for s in sequences)
71
+ sequences_ = []
72
+ for seq in sequences:
73
+ if seq.size(1) != max_l:
74
+ pad_len = max_l - seq.size(1)
75
+ pad_len = (0, pad_len) if pad_side == 'right' else (pad_len, 0)
76
+ seq = F.pad(seq, pad_len, value=value)
77
+ sequences_.append(seq)
78
+
79
+ sequences = torch.cat(sequences_)
80
+
81
+ return sequences
82
+
83
+ class QQMMPreTrainedModel(PreTrainedModel):
84
+ config_class = QQMMConfig
85
+ supports_gradient_checkpointing = True
86
+ _skip_keys_device_placement = "past_key_values"
87
+ _supports_cache_class = True
88
+ _supports_flash_attn_2 = True
89
+ _supports_sdpa = True
90
+
91
+ class QQMMForCausalLM(QQMMPreTrainedModel):
92
+
93
+ def __init__(self,
94
+ config,
95
+ qwen2_5_vl_model=None):
96
+
97
+ super().__init__(config)
98
+ if qwen2_5_vl_model is None:
99
+ kwargs_ = {}
100
+ if config._attn_implementation_internal is not None:
101
+ kwargs_['attn_implementation'] = config._attn_implementation_internal
102
+ model = Qwen2_5_VLForConditionalGeneration(config.model_config)
103
+ # model = Qwen2_5_VLForConditionalGeneration.from_pretrained("/group/40048/windzhchen/pretrain_models/deepeyes_convert")
104
+
105
+ else:
106
+ model = qwen2_5_vl_model
107
+ self.qwen2_5_vl_model = model
108
+ self.post_init()
109
+
110
+ def make_diy_mask(self, input_ids, attention_mask, embed_token_id, im_start_id, im_end_id):
111
+ if len(attention_mask.shape) == 2:
112
+ sequence_length = attention_mask.shape[1]
113
+ target_length = attention_mask.shape[1]
114
+ dtype = torch.bfloat16
115
+ device = input_ids.device
116
+ min_dtype = torch.finfo(dtype).min
117
+ cache_position = torch.arange(0, sequence_length, device=attention_mask.device)
118
+ attention_mask = _prepare_4d_causal_attention_mask_with_cache_position(
119
+ attention_mask,
120
+ sequence_length=sequence_length,
121
+ target_length=target_length,
122
+ dtype=dtype,
123
+ device=device,
124
+ min_dtype=min_dtype,
125
+ cache_position=cache_position,
126
+ batch_size=attention_mask.shape[0],
127
+ )
128
+ else:
129
+ dtype = torch.bfloat16
130
+ min_dtype = torch.finfo(dtype).min
131
+ mask = input_ids == embed_token_id
132
+ embed_index = torch.argmax(mask.float(), dim=1)
133
+ embed_index[embed_index==0] = input_ids.shape[1]
134
+ embed_index = embed_index.view(-1, )
135
+ mask = input_ids == im_start_id
136
+ im_start_index_tmp = torch.argmax(mask.float(), dim=1).view(-1, 1)
137
+ mask = torch.scatter(mask, dim=1, index=im_start_index_tmp, value=False)
138
+ im_start_index = torch.argmax(mask.float(), dim=1).view(-1, )
139
+ mask = input_ids == im_end_id
140
+ im_end_index_tmp = torch.argmax(mask.float(), dim=1).view(-1, 1)
141
+ mask = torch.scatter(mask, dim=1, index=im_end_index_tmp, value=False)
142
+ im_end_index = torch.argmax(mask.float(), dim=1).view(-1, )
143
+ for b in range(attention_mask.shape[0]):
144
+ attention_mask[b, 0, embed_index[b]+1:, im_start_index[b]:im_end_index[b]+2] = min_dtype # <|im_start|>user\nxxxxx<|im_end|>\n
145
+ return attention_mask
146
+
147
+
148
+ def forward(
149
+ self,
150
+ input_ids: Optional[torch.LongTensor] = None,
151
+ attention_mask: Optional[torch.Tensor] = None,
152
+ position_ids: Optional[torch.LongTensor] = None,
153
+ past_key_values: Optional[List[torch.FloatTensor]] = None,
154
+ inputs_embeds: Optional[torch.FloatTensor] = None,
155
+ labels: Optional[torch.LongTensor] = None,
156
+ use_cache: Optional[bool] = None,
157
+ output_attentions: Optional[bool] = None,
158
+ output_hidden_states: Optional[bool] = None,
159
+ return_dict: Optional[bool] = None,
160
+ pixel_values: Optional[torch.Tensor] = None,
161
+ pixel_values_videos: Optional[torch.FloatTensor] = None,
162
+ image_grid_thw: Optional[torch.LongTensor] = None,
163
+ video_grid_thw: Optional[torch.LongTensor] = None,
164
+ rope_deltas: Optional[torch.LongTensor] = None,
165
+ cache_position: Optional[torch.LongTensor] = None,
166
+ second_per_grid_ts: Optional[torch.Tensor] = None,
167
+ embed_token_id: Optional[int] = None,
168
+ return_emb: Optional[bool] = False,
169
+ cal_loss: Optional[bool] = False
170
+ ) -> Union[Tuple, Qwen2_5_VLCausalLMOutputWithPast]:
171
+
172
+ if pixel_values is not None and pixel_values.shape[0] == 0:
173
+ pixel_values = None
174
+ image_grid_thw = None
175
+ output_attentions = output_attentions if output_attentions is not None else self.qwen2_5_vl_model.config.output_attentions
176
+ output_hidden_states = (
177
+ output_hidden_states if output_hidden_states is not None else self.qwen2_5_vl_model.config.output_hidden_states
178
+ )
179
+ return_dict = return_dict if return_dict is not None else self.qwen2_5_vl_model.config.use_return_dict
180
+
181
+ if inputs_embeds is None:
182
+ inputs_embeds = self.qwen2_5_vl_model.model.embed_tokens(input_ids)
183
+ if pixel_values is not None:
184
+ pixel_values = pixel_values.type(self.qwen2_5_vl_model.visual.dtype)
185
+ image_embeds = self.qwen2_5_vl_model.visual(pixel_values, grid_thw=image_grid_thw)
186
+ n_image_tokens = (input_ids == self.qwen2_5_vl_model.config.image_token_id).sum().item()
187
+ n_image_features = image_embeds.shape[0]
188
+ if n_image_tokens != n_image_features:
189
+ raise ValueError(
190
+ f"Image features and image tokens do not match: tokens: {n_image_tokens}, features {n_image_features}"
191
+ )
192
+
193
+ mask = input_ids == self.qwen2_5_vl_model.config.image_token_id
194
+ mask_unsqueezed = mask.unsqueeze(-1)
195
+ mask_expanded = mask_unsqueezed.expand_as(inputs_embeds)
196
+ image_mask = mask_expanded.to(inputs_embeds.device)
197
+
198
+ image_embeds = image_embeds.to(inputs_embeds.device, inputs_embeds.dtype)
199
+ inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds)
200
+
201
+ if pixel_values_videos is not None:
202
+ pixel_values_videos = pixel_values_videos.type(self.qwen2_5_vl_model.visual.dtype)
203
+ video_embeds = self.qwen2_5_vl_model.visual(pixel_values_videos, grid_thw=video_grid_thw)
204
+ n_video_tokens = (input_ids == self.qwen2_5_vl_model.config.video_token_id).sum().item()
205
+ n_video_features = video_embeds.shape[0]
206
+ if n_video_tokens != n_video_features:
207
+ raise ValueError(
208
+ f"Video features and video tokens do not match: tokens: {n_video_tokens}, features {n_video_features}"
209
+ )
210
+
211
+ mask = input_ids == self.qwen2_5_vl_model.config.video_token_id
212
+ mask_unsqueezed = mask.unsqueeze(-1)
213
+ mask_expanded = mask_unsqueezed.expand_as(inputs_embeds)
214
+ video_mask = mask_expanded.to(inputs_embeds.device)
215
+
216
+ video_embeds = video_embeds.to(inputs_embeds.device, inputs_embeds.dtype)
217
+ inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_embeds)
218
+
219
+ if attention_mask is not None:
220
+ attention_mask = attention_mask.to(inputs_embeds.device)
221
+
222
+ # if we get 4D attention mask we cannot calculate rope deltas anymore. TODO @raushan fixme
223
+ if position_ids is None and (attention_mask is None or attention_mask.ndim == 2):
224
+ # calculate RoPE index once per generation in the pre-fill stage only
225
+ if (
226
+ (cache_position is not None and cache_position[0] == 0)
227
+ or self.qwen2_5_vl_model.rope_deltas is None
228
+ or (past_key_values is None or past_key_values.get_seq_length() == 0)
229
+ ):
230
+ position_ids, rope_deltas = self.qwen2_5_vl_model.get_rope_index(
231
+ input_ids,
232
+ image_grid_thw,
233
+ video_grid_thw,
234
+ second_per_grid_ts,
235
+ attention_mask,
236
+ )
237
+ self.rope_deltas = rope_deltas
238
+ # then use the prev pre-calculated rope-deltas to get the correct position ids
239
+ else:
240
+ batch_size, seq_length, _ = inputs_embeds.shape
241
+ delta = (
242
+ (cache_position[0] + self.rope_deltas).to(inputs_embeds.device)
243
+ if cache_position is not None
244
+ else 0
245
+ )
246
+ position_ids = torch.arange(seq_length, device=inputs_embeds.device)
247
+ position_ids = position_ids.view(1, -1).expand(batch_size, -1)
248
+ if cache_position is not None: # otherwise `deltas` is an int `0`
249
+ delta = delta.repeat_interleave(batch_size // delta.shape[0], dim=0)
250
+ position_ids = position_ids.add(delta)
251
+ position_ids = position_ids.unsqueeze(0).expand(3, -1, -1)
252
+
253
+ outputs = self.qwen2_5_vl_model.model(
254
+ input_ids=None,
255
+ position_ids=position_ids,
256
+ attention_mask=attention_mask,
257
+ past_key_values=past_key_values,
258
+ inputs_embeds=inputs_embeds,
259
+ use_cache=use_cache,
260
+ output_attentions=output_attentions,
261
+ output_hidden_states=output_hidden_states,
262
+ return_dict=return_dict,
263
+ cache_position=cache_position,
264
+ )
265
+
266
+ hidden_states = outputs[0]
267
+
268
+ if labels is not None:
269
+ mask = labels == embed_token_id
270
+ labels[mask] = -100
271
+
272
+ logits = self.qwen2_5_vl_model.lm_head(hidden_states)
273
+
274
+ if return_emb:
275
+ assert labels is not None, 'labels must be provided to obtain embed'
276
+ hidden_index = torch.argmax(mask.float(), dim=1)
277
+ hidden_index[hidden_index==0] = labels.shape[1]
278
+ hidden_states = torch.gather(hidden_states, dim=1, index=(hidden_index-1).view(hidden_index.shape[0], 1, 1).repeat(1, 1, hidden_states.shape[-1]))
279
+ emb = hidden_states[:, 0, :].contiguous() # B, C
280
+ else:
281
+ emb = None
282
+
283
+ loss = None
284
+ if labels is not None and cal_loss:
285
+ # Upcast to float if we need to compute the loss to avoid potential precision issues
286
+ logits = logits.float()
287
+ # Shift so that tokens < n predict n
288
+ shift_logits = logits[..., :-1, :].contiguous()
289
+ shift_labels = labels[..., 1:].contiguous()
290
+ if (shift_labels < 0).all().item():
291
+ loss = 0.0
292
+ else:
293
+ # Flatten the tokens
294
+ loss_fct = CrossEntropyLoss()
295
+ shift_logits = shift_logits.view(-1, self.qwen2_5_vl_model.config.vocab_size)
296
+ shift_labels = shift_labels.view(-1)
297
+ # Enable model parallelism
298
+ shift_labels = shift_labels.to(shift_logits.device)
299
+ loss = loss_fct(shift_logits, shift_labels)
300
+
301
+ if not return_dict:
302
+ output = (logits,) + outputs[1:]
303
+ return (loss,) + output if loss is not None else output
304
+ else:
305
+ outputs = Qwen2_5_VLCausalLMOutputWithPast(
306
+ loss=loss,
307
+ logits=logits,
308
+ past_key_values=outputs.past_key_values,
309
+ hidden_states=outputs.hidden_states,
310
+ attentions=outputs.attentions,
311
+ rope_deltas=self.qwen2_5_vl_model.rope_deltas,
312
+ )
313
+ if emb is not None:
314
+ outputs['emb'] = emb
315
+
316
+ return outputs
317
+
318
+
319
+ @torch.no_grad()
320
+ def generate(self, input_ids, *args, **kwargs) -> Union[GenerateOutput, torch.LongTensor]:
321
+ return self.qwen2_5_vl_model.generate(input_ids, *args, **kwargs)
322
+
323
+ def gradient_checkpointing_enable(self, gradient_checkpointing_kwargs=None):
324
+ super().gradient_checkpointing_enable(gradient_checkpointing_kwargs)
325
+ self.qwen2_5_vl_model.model.enable_input_require_grads()
326
+
327
+ def get_input_embeddings(self):
328
+ return self.qwen2_5_vl_model.model.get_input_embeddings()
329
+
330
+ def set_input_embeddings(self, value):
331
+ self.qwen2_5_vl_model.model.set_input_embeddings(value)
332
+
333
+ def get_output_embeddings(self):
334
+ return self.qwen2_5_vl_model.lm_head
335
+
336
+ def set_output_embeddings(self, new_embeddings):
337
+ self.qwen2_5_vl_model.lm_head = new_embeddings
338
+
339
+ def set_decoder(self, decoder):
340
+ self.qwen2_5_vl_model.model = decoder
341
+
342
+ def get_decoder(self):
343
+ return self.qwen2_5_vl_model.model