Hugging Face's logo

mveroe
/
Qwen2.5-1.5B_lightr1_3_EN_1024_1p0_0p0_1p0_sft

Text Generation
Transformers
Safetensors
qwen2
Generated from Trainer
conversational
text-generation-inference
Model card Files
xet
Community
mveroe commited on
Commit
969358a
·
verified ·
1 Parent(s): e07f495

Training in progress, epoch 3, checkpoint

Browse files
Files changed (11) hide show
  1. checkpoint-63/config.json +59 -0
  2. checkpoint-63/generation_config.json +6 -0
  3. checkpoint-63/model.safetensors +3 -0
  4. checkpoint-63/optimizer.pt +3 -0
  5. checkpoint-63/rng_state_0.pth +3 -0
  6. checkpoint-63/rng_state_1.pth +3 -0
  7. checkpoint-63/rng_state_2.pth +3 -0
  8. checkpoint-63/rng_state_3.pth +3 -0
  9. checkpoint-63/scheduler.pt +3 -0
  10. checkpoint-63/trainer_state.json +475 -0
  11. checkpoint-63/training_args.bin +3 -0
checkpoint-63/config.json ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen2ForCausalLM"
4
+ ],
5
+ "attention_dropout": 0.0,
6
+ "bos_token_id": 151643,
7
+ "eos_token_id": 151643,
8
+ "hidden_act": "silu",
9
+ "hidden_size": 1536,
10
+ "initializer_range": 0.02,
11
+ "intermediate_size": 8960,
12
+ "layer_types": [
13
+ "full_attention",
14
+ "full_attention",
15
+ "full_attention",
16
+ "full_attention",
17
+ "full_attention",
18
+ "full_attention",
19
+ "full_attention",
20
+ "full_attention",
21
+ "full_attention",
22
+ "full_attention",
23
+ "full_attention",
24
+ "full_attention",
25
+ "full_attention",
26
+ "full_attention",
27
+ "full_attention",
28
+ "full_attention",
29
+ "full_attention",
30
+ "full_attention",
31
+ "full_attention",
32
+ "full_attention",
33
+ "full_attention",
34
+ "full_attention",
35
+ "full_attention",
36
+ "full_attention",
37
+ "full_attention",
38
+ "full_attention",
39
+ "full_attention",
40
+ "full_attention"
41
+ ],
42
+ "max_position_embeddings": 131072,
43
+ "max_window_layers": 28,
44
+ "model_type": "qwen2",
45
+ "num_attention_heads": 12,
46
+ "num_hidden_layers": 28,
47
+ "num_key_value_heads": 2,
48
+ "rms_norm_eps": 1e-06,
49
+ "rope_scaling": null,
50
+ "rope_theta": 1000000.0,
51
+ "sliding_window": null,
52
+ "tie_word_embeddings": true,
53
+ "torch_dtype": "bfloat16",
54
+ "transformers_version": "4.55.2",
55
+ "use_cache": true,
56
+ "use_mrope": false,
57
+ "use_sliding_window": false,
58
+ "vocab_size": 151667
59
+ }
checkpoint-63/generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 151643,
3
+ "eos_token_id": 151643,
4
+ "max_new_tokens": 2048,
5
+ "transformers_version": "4.55.2"
6
+ }
checkpoint-63/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:85f507b0d3859d5a42c970219587fa81011fa29adb74ba4ab5628dde56dafb73
3
+ size 3086640776
checkpoint-63/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4b640d9ef8b573c0450e41f463b65a9acae87a9a98f2afcfdd4f7a8d4d5dc601
3
+ size 6056331
checkpoint-63/rng_state_0.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aa0f95228341434b158308966669a313617960955e886ba19e972d72b58a1fc0
3
+ size 15365
checkpoint-63/rng_state_1.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8fb125336725f7741cb4daa1e3d06e225bbacfde8d41c4dcabb6762c222e62c6
3
+ size 15429
checkpoint-63/rng_state_2.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:153c865f77c7129ba565bded50f334683d51c80f20e3cfec39e62f8737b86f0d
3
+ size 15429
checkpoint-63/rng_state_3.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d017ce00fcebac7edc058ddd138f194eb0340f2d8ad0879bdab08f922ed0846e
3
+ size 15429
checkpoint-63/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3cc7c915bab2f07ef07670a36046156a10aac0b24022808e61bfd155d9d4b283
3
+ size 1465
checkpoint-63/trainer_state.json ADDED
@@ -0,0 +1,475 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 3.0,
6
+ "eval_steps": 500,
7
+ "global_step": 63,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.04878048780487805,
14
+ "grad_norm": 126.0,
15
+ "learning_rate": 0.0,
16
+ "loss": 6.3712,
17
+ "step": 1
18
+ },
19
+ {
20
+ "epoch": 0.0975609756097561,
21
+ "grad_norm": 139.0,
22
+ "learning_rate": 7.142857142857143e-06,
23
+ "loss": 6.9747,
24
+ "step": 2
25
+ },
26
+ {
27
+ "epoch": 0.14634146341463414,
28
+ "grad_norm": 141.0,
29
+ "learning_rate": 1.4285714285714285e-05,
30
+ "loss": 7.2169,
31
+ "step": 3
32
+ },
33
+ {
34
+ "epoch": 0.1951219512195122,
35
+ "grad_norm": 135.0,
36
+ "learning_rate": 2.1428571428571428e-05,
37
+ "loss": 6.3792,
38
+ "step": 4
39
+ },
40
+ {
41
+ "epoch": 0.24390243902439024,
42
+ "grad_norm": 356.0,
43
+ "learning_rate": 2.857142857142857e-05,
44
+ "loss": 5.1872,
45
+ "step": 5
46
+ },
47
+ {
48
+ "epoch": 0.2926829268292683,
49
+ "grad_norm": 158.0,
50
+ "learning_rate": 3.571428571428572e-05,
51
+ "loss": 5.7681,
52
+ "step": 6
53
+ },
54
+ {
55
+ "epoch": 0.34146341463414637,
56
+ "grad_norm": 149.0,
57
+ "learning_rate": 4.2857142857142856e-05,
58
+ "loss": 4.4005,
59
+ "step": 7
60
+ },
61
+ {
62
+ "epoch": 0.3902439024390244,
63
+ "grad_norm": 134.0,
64
+ "learning_rate": 5e-05,
65
+ "loss": 5.056,
66
+ "step": 8
67
+ },
68
+ {
69
+ "epoch": 0.43902439024390244,
70
+ "grad_norm": 94.5,
71
+ "learning_rate": 4.996067037544542e-05,
72
+ "loss": 4.2935,
73
+ "step": 9
74
+ },
75
+ {
76
+ "epoch": 0.4878048780487805,
77
+ "grad_norm": 212.0,
78
+ "learning_rate": 4.984280524733107e-05,
79
+ "loss": 8.9937,
80
+ "step": 10
81
+ },
82
+ {
83
+ "epoch": 0.5365853658536586,
84
+ "grad_norm": 302.0,
85
+ "learning_rate": 4.96467754629559e-05,
86
+ "loss": 4.9127,
87
+ "step": 11
88
+ },
89
+ {
90
+ "epoch": 0.5853658536585366,
91
+ "grad_norm": 44.5,
92
+ "learning_rate": 4.937319780454559e-05,
93
+ "loss": 4.4251,
94
+ "step": 12
95
+ },
96
+ {
97
+ "epoch": 0.6341463414634146,
98
+ "grad_norm": 39.25,
99
+ "learning_rate": 4.9022933048627496e-05,
100
+ "loss": 3.6691,
101
+ "step": 13
102
+ },
103
+ {
104
+ "epoch": 0.6829268292682927,
105
+ "grad_norm": 31.25,
106
+ "learning_rate": 4.8597083257709194e-05,
107
+ "loss": 3.232,
108
+ "step": 14
109
+ },
110
+ {
111
+ "epoch": 0.7317073170731707,
112
+ "grad_norm": 44.25,
113
+ "learning_rate": 4.8096988312782174e-05,
114
+ "loss": 3.1154,
115
+ "step": 15
116
+ },
117
+ {
118
+ "epoch": 0.7804878048780488,
119
+ "grad_norm": 43.75,
120
+ "learning_rate": 4.752422169756048e-05,
121
+ "loss": 2.6809,
122
+ "step": 16
123
+ },
124
+ {
125
+ "epoch": 0.8292682926829268,
126
+ "grad_norm": 41.0,
127
+ "learning_rate": 4.6880585547718845e-05,
128
+ "loss": 2.1384,
129
+ "step": 17
130
+ },
131
+ {
132
+ "epoch": 0.8780487804878049,
133
+ "grad_norm": 50.0,
134
+ "learning_rate": 4.6168104980707107e-05,
135
+ "loss": 2.0084,
136
+ "step": 18
137
+ },
138
+ {
139
+ "epoch": 0.926829268292683,
140
+ "grad_norm": 48.25,
141
+ "learning_rate": 4.538902172398151e-05,
142
+ "loss": 1.8335,
143
+ "step": 19
144
+ },
145
+ {
146
+ "epoch": 0.975609756097561,
147
+ "grad_norm": 50.0,
148
+ "learning_rate": 4.454578706170075e-05,
149
+ "loss": 1.6659,
150
+ "step": 20
151
+ },
152
+ {
153
+ "epoch": 1.0,
154
+ "grad_norm": 50.75,
155
+ "learning_rate": 4.364105412207914e-05,
156
+ "loss": 1.5889,
157
+ "step": 21
158
+ },
159
+ {
160
+ "epoch": 1.048780487804878,
161
+ "grad_norm": 44.0,
162
+ "learning_rate": 4.267766952966369e-05,
163
+ "loss": 1.4364,
164
+ "step": 22
165
+ },
166
+ {
167
+ "epoch": 1.0975609756097562,
168
+ "grad_norm": 44.25,
169
+ "learning_rate": 4.16586644488001e-05,
170
+ "loss": 1.3537,
171
+ "step": 23
172
+ },
173
+ {
174
+ "epoch": 1.146341463414634,
175
+ "grad_norm": 37.0,
176
+ "learning_rate": 4.058724504646834e-05,
177
+ "loss": 1.1017,
178
+ "step": 24
179
+ },
180
+ {
181
+ "epoch": 1.1951219512195121,
182
+ "grad_norm": 40.25,
183
+ "learning_rate": 3.946678240449515e-05,
184
+ "loss": 1.0232,
185
+ "step": 25
186
+ },
187
+ {
188
+ "epoch": 1.2439024390243902,
189
+ "grad_norm": 34.25,
190
+ "learning_rate": 3.830080191288342e-05,
191
+ "loss": 0.9218,
192
+ "step": 26
193
+ },
194
+ {
195
+ "epoch": 1.2926829268292683,
196
+ "grad_norm": 27.125,
197
+ "learning_rate": 3.7092972177631e-05,
198
+ "loss": 0.8892,
199
+ "step": 27
200
+ },
201
+ {
202
+ "epoch": 1.3414634146341464,
203
+ "grad_norm": 25.0,
204
+ "learning_rate": 3.5847093477938956e-05,
205
+ "loss": 0.9091,
206
+ "step": 28
207
+ },
208
+ {
209
+ "epoch": 1.3902439024390243,
210
+ "grad_norm": 22.5,
211
+ "learning_rate": 3.456708580912725e-05,
212
+ "loss": 0.9093,
213
+ "step": 29
214
+ },
215
+ {
216
+ "epoch": 1.4390243902439024,
217
+ "grad_norm": 23.625,
218
+ "learning_rate": 3.3256976548879184e-05,
219
+ "loss": 0.846,
220
+ "step": 30
221
+ },
222
+ {
223
+ "epoch": 1.4878048780487805,
224
+ "grad_norm": 19.375,
225
+ "learning_rate": 3.1920887785621235e-05,
226
+ "loss": 0.7321,
227
+ "step": 31
228
+ },
229
+ {
230
+ "epoch": 1.5365853658536586,
231
+ "grad_norm": 15.6875,
232
+ "learning_rate": 3.056302334890786e-05,
233
+ "loss": 0.7505,
234
+ "step": 32
235
+ },
236
+ {
237
+ "epoch": 1.5853658536585367,
238
+ "grad_norm": 16.5,
239
+ "learning_rate": 2.918765558261841e-05,
240
+ "loss": 0.7204,
241
+ "step": 33
242
+ },
243
+ {
244
+ "epoch": 1.6341463414634148,
245
+ "grad_norm": 13.25,
246
+ "learning_rate": 2.7799111902582696e-05,
247
+ "loss": 0.6878,
248
+ "step": 34
249
+ },
250
+ {
251
+ "epoch": 1.6829268292682928,
252
+ "grad_norm": 15.0,
253
+ "learning_rate": 2.6401761180929797e-05,
254
+ "loss": 0.6356,
255
+ "step": 35
256
+ },
257
+ {
258
+ "epoch": 1.7317073170731707,
259
+ "grad_norm": 14.3125,
260
+ "learning_rate": 2.5e-05,
261
+ "loss": 0.6332,
262
+ "step": 36
263
+ },
264
+ {
265
+ "epoch": 1.7804878048780488,
266
+ "grad_norm": 9.9375,
267
+ "learning_rate": 2.3598238819070202e-05,
268
+ "loss": 0.6806,
269
+ "step": 37
270
+ },
271
+ {
272
+ "epoch": 1.8292682926829267,
273
+ "grad_norm": 11.5,
274
+ "learning_rate": 2.2200888097417307e-05,
275
+ "loss": 0.6415,
276
+ "step": 38
277
+ },
278
+ {
279
+ "epoch": 1.8780487804878048,
280
+ "grad_norm": 9.75,
281
+ "learning_rate": 2.0812344417381595e-05,
282
+ "loss": 0.5846,
283
+ "step": 39
284
+ },
285
+ {
286
+ "epoch": 1.9268292682926829,
287
+ "grad_norm": 10.25,
288
+ "learning_rate": 1.9436976651092144e-05,
289
+ "loss": 0.5771,
290
+ "step": 40
291
+ },
292
+ {
293
+ "epoch": 1.975609756097561,
294
+ "grad_norm": 7.75,
295
+ "learning_rate": 1.8079112214378768e-05,
296
+ "loss": 0.6536,
297
+ "step": 41
298
+ },
299
+ {
300
+ "epoch": 2.0,
301
+ "grad_norm": 8.125,
302
+ "learning_rate": 1.6743023451120832e-05,
303
+ "loss": 0.6518,
304
+ "step": 42
305
+ },
306
+ {
307
+ "epoch": 2.048780487804878,
308
+ "grad_norm": 7.96875,
309
+ "learning_rate": 1.5432914190872757e-05,
310
+ "loss": 0.5361,
311
+ "step": 43
312
+ },
313
+ {
314
+ "epoch": 2.097560975609756,
315
+ "grad_norm": 8.0,
316
+ "learning_rate": 1.4152906522061048e-05,
317
+ "loss": 0.6016,
318
+ "step": 44
319
+ },
320
+ {
321
+ "epoch": 2.1463414634146343,
322
+ "grad_norm": 7.0625,
323
+ "learning_rate": 1.2907027822369005e-05,
324
+ "loss": 0.5721,
325
+ "step": 45
326
+ },
327
+ {
328
+ "epoch": 2.1951219512195124,
329
+ "grad_norm": 8.0,
330
+ "learning_rate": 1.1699198087116589e-05,
331
+ "loss": 0.5486,
332
+ "step": 46
333
+ },
334
+ {
335
+ "epoch": 2.2439024390243905,
336
+ "grad_norm": 7.65625,
337
+ "learning_rate": 1.0533217595504858e-05,
338
+ "loss": 0.5682,
339
+ "step": 47
340
+ },
341
+ {
342
+ "epoch": 2.292682926829268,
343
+ "grad_norm": 7.28125,
344
+ "learning_rate": 9.412754953531663e-06,
345
+ "loss": 0.54,
346
+ "step": 48
347
+ },
348
+ {
349
+ "epoch": 2.341463414634146,
350
+ "grad_norm": 7.25,
351
+ "learning_rate": 8.341335551199902e-06,
352
+ "loss": 0.5295,
353
+ "step": 49
354
+ },
355
+ {
356
+ "epoch": 2.3902439024390243,
357
+ "grad_norm": 7.53125,
358
+ "learning_rate": 7.3223304703363135e-06,
359
+ "loss": 0.5384,
360
+ "step": 50
361
+ },
362
+ {
363
+ "epoch": 2.4390243902439024,
364
+ "grad_norm": 6.40625,
365
+ "learning_rate": 6.358945877920861e-06,
366
+ "loss": 0.5395,
367
+ "step": 51
368
+ },
369
+ {
370
+ "epoch": 2.4878048780487805,
371
+ "grad_norm": 6.96875,
372
+ "learning_rate": 5.454212938299255e-06,
373
+ "loss": 0.5124,
374
+ "step": 52
375
+ },
376
+ {
377
+ "epoch": 2.5365853658536586,
378
+ "grad_norm": 6.875,
379
+ "learning_rate": 4.610978276018496e-06,
380
+ "loss": 0.4981,
381
+ "step": 53
382
+ },
383
+ {
384
+ "epoch": 2.5853658536585367,
385
+ "grad_norm": 6.78125,
386
+ "learning_rate": 3.831895019292897e-06,
387
+ "loss": 0.5473,
388
+ "step": 54
389
+ },
390
+ {
391
+ "epoch": 2.6341463414634148,
392
+ "grad_norm": 6.15625,
393
+ "learning_rate": 3.119414452281158e-06,
394
+ "loss": 0.527,
395
+ "step": 55
396
+ },
397
+ {
398
+ "epoch": 2.682926829268293,
399
+ "grad_norm": 6.75,
400
+ "learning_rate": 2.475778302439524e-06,
401
+ "loss": 0.5601,
402
+ "step": 56
403
+ },
404
+ {
405
+ "epoch": 2.7317073170731705,
406
+ "grad_norm": 6.15625,
407
+ "learning_rate": 1.9030116872178316e-06,
408
+ "loss": 0.5356,
409
+ "step": 57
410
+ },
411
+ {
412
+ "epoch": 2.7804878048780486,
413
+ "grad_norm": 6.28125,
414
+ "learning_rate": 1.4029167422908107e-06,
415
+ "loss": 0.5391,
416
+ "step": 58
417
+ },
418
+ {
419
+ "epoch": 2.8292682926829267,
420
+ "grad_norm": 6.625,
421
+ "learning_rate": 9.770669513725128e-07,
422
+ "loss": 0.5177,
423
+ "step": 59
424
+ },
425
+ {
426
+ "epoch": 2.8780487804878048,
427
+ "grad_norm": 6.625,
428
+ "learning_rate": 6.268021954544096e-07,
429
+ "loss": 0.5542,
430
+ "step": 60
431
+ },
432
+ {
433
+ "epoch": 2.926829268292683,
434
+ "grad_norm": 6.84375,
435
+ "learning_rate": 3.5322453704410286e-07,
436
+ "loss": 0.5601,
437
+ "step": 61
438
+ },
439
+ {
440
+ "epoch": 2.975609756097561,
441
+ "grad_norm": 6.1875,
442
+ "learning_rate": 1.571947526689349e-07,
443
+ "loss": 0.5629,
444
+ "step": 62
445
+ },
446
+ {
447
+ "epoch": 3.0,
448
+ "grad_norm": 8.375,
449
+ "learning_rate": 3.9329624554584884e-08,
450
+ "loss": 0.5202,
451
+ "step": 63
452
+ }
453
+ ],
454
+ "logging_steps": 1,
455
+ "max_steps": 63,
456
+ "num_input_tokens_seen": 0,
457
+ "num_train_epochs": 3,
458
+ "save_steps": 500,
459
+ "stateful_callbacks": {
460
+ "TrainerControl": {
461
+ "args": {
462
+ "should_epoch_stop": false,
463
+ "should_evaluate": false,
464
+ "should_log": false,
465
+ "should_save": true,
466
+ "should_training_stop": true
467
+ },
468
+ "attributes": {}
469
+ }
470
+ },
471
+ "total_flos": 3.168768388969267e+16,
472
+ "train_batch_size": 4,
473
+ "trial_name": null,
474
+ "trial_params": null
475
+ }
checkpoint-63/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:807175855d5dd743e81254531a1a0148b2ade3b9521c50fa1999cea9a77e0a70
3
+ size 5969