Upload folder using huggingface_hub

adapter_config.json

@@ -29,13 +29,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "down_proj",
-    "q_proj",
-    "up_proj",
     "v_proj",
-    "k_proj",
+    "gate_proj",
+    "up_proj",
+    "down_proj",
     "o_proj",
-    "gate_proj"
+    "k_proj",
+    "q_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dd2d08d826340d149aa5513b4dd0539e973bdec29c23d4a91364b7126c19c316
+oid sha256:a0656ea6a9b2b36d5759ece47a5b3eff68d20b48c1112eadf5ef4c6950d2cf55
 size 167832240

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f5c3f504088aead5b8c944df4ed47fcaef2fbcdf4678a6a6457ccbafe3f7bdcb
+oid sha256:6e98ec00b3d2f7e34b8479ef6eafbc1e1fa5efe3fb6e5d385f146df37257d394
 size 85733654

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:69b9dab5290c195fdacbf7884c40eeca52cafe3b7ddc5ee2280f8f048155a5ab
+oid sha256:07d2f57764cc2ea1a146a33cfaedbb1294d4dfbe7253eb0435c96b286b35141a
 size 14244

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.33641715727502103,
+  "epoch": 0.33651149747616377,
   "eval_steps": 179,
   "global_step": 300,
   "is_hyper_param_search": false,
@@ -10,603 +10,603 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "entropy": 1.696909672021866,
-      "epoch": 0.005606952621250351,
-      "grad_norm": 0.7663310170173645,
+      "entropy": 1.5857114374637604,
+      "epoch": 0.005608524957936063,
+      "grad_norm": 0.6155414581298828,
       "learning_rate": 0.00016,
-      "loss": 2.1596,
-      "mean_token_accuracy": 0.5812449663877487,
-      "num_tokens": 8218.0,
+      "loss": 1.9706,
+      "mean_token_accuracy": 0.6006834208965302,
+      "num_tokens": 8800.0,
       "step": 5
     },
     {
-      "entropy": 1.875484037399292,
-      "epoch": 0.011213905242500702,
-      "grad_norm": 0.8613530397415161,
+      "entropy": 1.9297356605529785,
+      "epoch": 0.011217049915872126,
+      "grad_norm": 0.7395716905593872,
       "learning_rate": 0.00019909808342728297,
-      "loss": 1.6298,
-      "mean_token_accuracy": 0.6346893429756164,
-      "num_tokens": 19584.0,
+      "loss": 1.7721,
+      "mean_token_accuracy": 0.6421857982873916,
+      "num_tokens": 17002.0,
       "step": 10
     },
     {
-      "entropy": 1.6492740571498872,
-      "epoch": 0.01682085786375105,
-      "grad_norm": 0.8438306450843811,
+      "entropy": 1.619718110561371,
+      "epoch": 0.01682557487380819,
+      "grad_norm": 0.6860098838806152,
       "learning_rate": 0.0001979706877113867,
-      "loss": 1.448,
-      "mean_token_accuracy": 0.667498791217804,
-      "num_tokens": 28392.0,
+      "loss": 1.4967,
+      "mean_token_accuracy": 0.6729415714740753,
+      "num_tokens": 25283.0,
       "step": 15
     },
     {
-      "entropy": 1.5424207150936127,
-      "epoch": 0.022427810485001403,
-      "grad_norm": 0.5755366086959839,
+      "entropy": 1.5023219525814056,
+      "epoch": 0.022434099831744252,
+      "grad_norm": 0.5842112898826599,
       "learning_rate": 0.00019684329199549043,
-      "loss": 1.5714,
-      "mean_token_accuracy": 0.6525402277708053,
-      "num_tokens": 40558.0,
+      "loss": 1.4969,
+      "mean_token_accuracy": 0.6710207283496856,
+      "num_tokens": 33505.0,
       "step": 20
     },
     {
-      "entropy": 1.6563467800617218,
-      "epoch": 0.02803476310625175,
-      "grad_norm": 0.640796422958374,
+      "entropy": 1.6429471731185914,
+      "epoch": 0.028042624789680313,
+      "grad_norm": 0.5911830067634583,
       "learning_rate": 0.00019571589627959414,
-      "loss": 1.4843,
-      "mean_token_accuracy": 0.6737909287214279,
-      "num_tokens": 50005.0,
+      "loss": 1.5174,
+      "mean_token_accuracy": 0.6597635358572006,
+      "num_tokens": 43397.0,
       "step": 25
     },
     {
-      "entropy": 1.477371919155121,
-      "epoch": 0.0336417157275021,
-      "grad_norm": 0.7493678331375122,
+      "entropy": 1.6084780812263488,
+      "epoch": 0.03365114974761638,
+      "grad_norm": 0.7406187653541565,
       "learning_rate": 0.00019458850056369787,
-      "loss": 1.3474,
-      "mean_token_accuracy": 0.6935703039169312,
-      "num_tokens": 58738.0,
+      "loss": 1.4757,
+      "mean_token_accuracy": 0.6691249191761017,
+      "num_tokens": 52556.0,
       "step": 30
     },
     {
-      "entropy": 1.464887660741806,
-      "epoch": 0.03924866834875245,
-      "grad_norm": 0.6396933794021606,
+      "entropy": 1.4238544702529907,
+      "epoch": 0.03925967470555244,
+      "grad_norm": 0.611213743686676,
       "learning_rate": 0.00019346110484780158,
-      "loss": 1.3963,
-      "mean_token_accuracy": 0.6689792603254319,
-      "num_tokens": 68528.0,
+      "loss": 1.4085,
+      "mean_token_accuracy": 0.692791685461998,
+      "num_tokens": 61579.0,
       "step": 35
     },
     {
-      "entropy": 1.4757700502872466,
-      "epoch": 0.044855620970002806,
-      "grad_norm": 0.5516763925552368,
+      "entropy": 1.3825733065605164,
+      "epoch": 0.044868199663488505,
+      "grad_norm": 0.6608020663261414,
       "learning_rate": 0.0001923337091319053,
-      "loss": 1.4236,
-      "mean_token_accuracy": 0.6675747632980347,
-      "num_tokens": 77986.0,
+      "loss": 1.3955,
+      "mean_token_accuracy": 0.6937497437000275,
+      "num_tokens": 68479.0,
       "step": 40
     },
     {
-      "entropy": 1.4118095993995667,
-      "epoch": 0.050462573591253154,
-      "grad_norm": 0.6395580172538757,
+      "entropy": 1.4096888184547425,
+      "epoch": 0.050476724621424565,
+      "grad_norm": 0.5221259593963623,
       "learning_rate": 0.00019120631341600902,
-      "loss": 1.2766,
-      "mean_token_accuracy": 0.6935016334056854,
-      "num_tokens": 86893.0,
+      "loss": 1.2979,
+      "mean_token_accuracy": 0.6925529271364212,
+      "num_tokens": 77911.0,
       "step": 45
     },
     {
-      "entropy": 1.4825987100601197,
-      "epoch": 0.0560695262125035,
-      "grad_norm": 0.7649742960929871,
+      "entropy": 1.3391252905130386,
+      "epoch": 0.056085249579360626,
+      "grad_norm": 0.6178808212280273,
       "learning_rate": 0.00019007891770011275,
-      "loss": 1.4255,
-      "mean_token_accuracy": 0.6750761657953263,
-      "num_tokens": 95692.0,
+      "loss": 1.3092,
+      "mean_token_accuracy": 0.704131829738617,
+      "num_tokens": 86382.0,
       "step": 50
     },
     {
-      "entropy": 1.3713403642177582,
-      "epoch": 0.06167647883375386,
-      "grad_norm": 0.6055657863616943,
+      "entropy": 1.3084194093942643,
+      "epoch": 0.06169377453729669,
+      "grad_norm": 0.570563554763794,
       "learning_rate": 0.00018895152198421646,
-      "loss": 1.2919,
-      "mean_token_accuracy": 0.6923940628767014,
-      "num_tokens": 104810.0,
+      "loss": 1.228,
+      "mean_token_accuracy": 0.6969290852546692,
+      "num_tokens": 94306.0,
       "step": 55
     },
     {
-      "entropy": 1.3107981920242309,
-      "epoch": 0.0672834314550042,
-      "grad_norm": 0.932307243347168,
+      "entropy": 1.418030035495758,
+      "epoch": 0.06730229949523275,
+      "grad_norm": 0.6073914766311646,
       "learning_rate": 0.0001878241262683202,
-      "loss": 1.2072,
-      "mean_token_accuracy": 0.7068106323480606,
-      "num_tokens": 112767.0,
+      "loss": 1.3252,
+      "mean_token_accuracy": 0.6798107504844666,
+      "num_tokens": 103567.0,
       "step": 60
     },
     {
-      "entropy": 1.290432232618332,
-      "epoch": 0.07289038407625456,
-      "grad_norm": 0.657538115978241,
+      "entropy": 1.5420262813568115,
+      "epoch": 0.07291082445316882,
+      "grad_norm": 0.4949992001056671,
       "learning_rate": 0.00018669673055242392,
-      "loss": 1.1683,
-      "mean_token_accuracy": 0.714971786737442,
-      "num_tokens": 122104.0,
+      "loss": 1.4262,
+      "mean_token_accuracy": 0.6678558409214019,
+      "num_tokens": 113923.0,
       "step": 65
     },
     {
-      "entropy": 1.3310194253921508,
-      "epoch": 0.0784973366975049,
-      "grad_norm": 0.5447025299072266,
+      "entropy": 1.3928685992956162,
+      "epoch": 0.07851934941110487,
+      "grad_norm": 0.5758721828460693,
       "learning_rate": 0.00018556933483652763,
-      "loss": 1.3566,
-      "mean_token_accuracy": 0.6949202805757523,
-      "num_tokens": 132347.0,
+      "loss": 1.3512,
+      "mean_token_accuracy": 0.6777496755123138,
+      "num_tokens": 126199.0,
       "step": 70
     },
     {
-      "entropy": 1.3567017048597336,
-      "epoch": 0.08410428931875526,
-      "grad_norm": 0.6126067042350769,
+      "entropy": 1.336549162864685,
+      "epoch": 0.08412787436904094,
+      "grad_norm": 0.678063154220581,
       "learning_rate": 0.00018444193912063134,
-      "loss": 1.2616,
-      "mean_token_accuracy": 0.6886427521705627,
-      "num_tokens": 140294.0,
+      "loss": 1.2387,
+      "mean_token_accuracy": 0.6992105931043625,
+      "num_tokens": 135737.0,
       "step": 75
     },
     {
-      "entropy": 1.3231679052114487,
-      "epoch": 0.08971124194000561,
-      "grad_norm": 0.5827459096908569,
+      "entropy": 1.5632237881422042,
+      "epoch": 0.08973639932697701,
+      "grad_norm": 0.5325204730033875,
       "learning_rate": 0.00018331454340473507,
-      "loss": 1.2312,
-      "mean_token_accuracy": 0.6998110383749008,
-      "num_tokens": 149796.0,
+      "loss": 1.479,
+      "mean_token_accuracy": 0.6516371637582778,
+      "num_tokens": 146847.0,
       "step": 80
     },
     {
-      "entropy": 1.3795920431613922,
-      "epoch": 0.09531819456125595,
-      "grad_norm": 0.6522558331489563,
+      "entropy": 1.4212194442749024,
+      "epoch": 0.09534492428491306,
+      "grad_norm": 0.8020451664924622,
       "learning_rate": 0.0001821871476888388,
-      "loss": 1.3163,
-      "mean_token_accuracy": 0.6797463029623032,
-      "num_tokens": 160094.0,
+      "loss": 1.3261,
+      "mean_token_accuracy": 0.6801748961210251,
+      "num_tokens": 154755.0,
       "step": 85
     },
     {
-      "entropy": 1.4354715049266815,
-      "epoch": 0.10092514718250631,
-      "grad_norm": 0.5437538623809814,
+      "entropy": 1.2850608110427857,
+      "epoch": 0.10095344924284913,
+      "grad_norm": 0.9955788254737854,
       "learning_rate": 0.0001810597519729425,
-      "loss": 1.4219,
-      "mean_token_accuracy": 0.6741667121648789,
-      "num_tokens": 167520.0,
+      "loss": 1.1832,
+      "mean_token_accuracy": 0.7192482769489288,
+      "num_tokens": 162857.0,
       "step": 90
     },
     {
-      "entropy": 1.3719047516584397,
-      "epoch": 0.10653209980375666,
-      "grad_norm": 0.6490810513496399,
+      "entropy": 1.24569151699543,
+      "epoch": 0.1065619742007852,
+      "grad_norm": 0.6132731437683105,
       "learning_rate": 0.00017993235625704624,
-      "loss": 1.3259,
-      "mean_token_accuracy": 0.69256811439991,
-      "num_tokens": 177631.0,
+      "loss": 1.1905,
+      "mean_token_accuracy": 0.7139606773853302,
+      "num_tokens": 171381.0,
       "step": 95
     },
     {
-      "entropy": 1.381699651479721,
-      "epoch": 0.112139052425007,
-      "grad_norm": 0.6738480925559998,
+      "entropy": 1.3500551611185074,
+      "epoch": 0.11217049915872125,
+      "grad_norm": 0.604263186454773,
       "learning_rate": 0.00017880496054114995,
-      "loss": 1.3181,
-      "mean_token_accuracy": 0.696441325545311,
-      "num_tokens": 186948.0,
+      "loss": 1.3683,
+      "mean_token_accuracy": 0.6883647471666337,
+      "num_tokens": 179672.0,
       "step": 100
     },
     {
-      "entropy": 1.271216405928135,
-      "epoch": 0.11774600504625736,
-      "grad_norm": 1.1620361804962158,
+      "entropy": 1.4277629852294922,
+      "epoch": 0.11777902411665732,
+      "grad_norm": 0.5616147518157959,
       "learning_rate": 0.00017767756482525365,
-      "loss": 1.2414,
-      "mean_token_accuracy": 0.7000553667545318,
-      "num_tokens": 196406.0,
+      "loss": 1.3609,
+      "mean_token_accuracy": 0.6817101955413818,
+      "num_tokens": 190223.0,
       "step": 105
     },
     {
-      "entropy": 1.3180717766284942,
-      "epoch": 0.12335295766750771,
-      "grad_norm": 0.5142589211463928,
+      "entropy": 1.3598074555397033,
+      "epoch": 0.12338754907459339,
+      "grad_norm": 0.49820777773857117,
       "learning_rate": 0.0001765501691093574,
-      "loss": 1.2179,
-      "mean_token_accuracy": 0.7069388717412949,
-      "num_tokens": 207134.0,
+      "loss": 1.3432,
+      "mean_token_accuracy": 0.6918457806110382,
+      "num_tokens": 200003.0,
       "step": 110
     },
     {
-      "entropy": 1.3922697693109511,
-      "epoch": 0.12895991028875806,
-      "grad_norm": 0.5426948666572571,
+      "entropy": 1.3189748495817184,
+      "epoch": 0.12899607403252944,
+      "grad_norm": 0.5357916951179504,
       "learning_rate": 0.00017542277339346112,
-      "loss": 1.3128,
-      "mean_token_accuracy": 0.6875284522771835,
-      "num_tokens": 218970.0,
+      "loss": 1.296,
+      "mean_token_accuracy": 0.6899977535009384,
+      "num_tokens": 210284.0,
       "step": 115
     },
     {
-      "entropy": 1.3745603621006013,
-      "epoch": 0.1345668629100084,
-      "grad_norm": 0.6224590539932251,
+      "entropy": 1.2477733016014099,
+      "epoch": 0.1346045989904655,
+      "grad_norm": 0.6869284510612488,
       "learning_rate": 0.00017429537767756482,
-      "loss": 1.3472,
-      "mean_token_accuracy": 0.6812066495418548,
-      "num_tokens": 227951.0,
+      "loss": 1.1922,
+      "mean_token_accuracy": 0.7168257743120193,
+      "num_tokens": 220374.0,
       "step": 120
     },
     {
-      "entropy": 1.2305602520704269,
-      "epoch": 0.14017381553125877,
-      "grad_norm": 0.6087414026260376,
+      "entropy": 1.30469251871109,
+      "epoch": 0.14021312394840157,
+      "grad_norm": 0.6618072986602783,
       "learning_rate": 0.00017316798196166856,
-      "loss": 1.1658,
-      "mean_token_accuracy": 0.7141091674566269,
-      "num_tokens": 237592.0,
+      "loss": 1.1881,
+      "mean_token_accuracy": 0.7009620904922486,
+      "num_tokens": 230717.0,
       "step": 125
     },
     {
-      "entropy": 1.138188686966896,
-      "epoch": 0.14578076815250912,
-      "grad_norm": 0.5676659345626831,
+      "entropy": 1.2923731699585914,
+      "epoch": 0.14582164890633764,
+      "grad_norm": 0.5542452931404114,
       "learning_rate": 0.0001720405862457723,
-      "loss": 1.1253,
-      "mean_token_accuracy": 0.7303309470415116,
-      "num_tokens": 246718.0,
+      "loss": 1.2582,
+      "mean_token_accuracy": 0.7028028458356858,
+      "num_tokens": 240440.0,
       "step": 130
     },
     {
-      "entropy": 1.390087616443634,
-      "epoch": 0.15138772077375945,
-      "grad_norm": 0.5323399901390076,
+      "entropy": 1.3109561294317245,
+      "epoch": 0.1514301738642737,
+      "grad_norm": 0.7251168489456177,
       "learning_rate": 0.000170913190529876,
-      "loss": 1.3956,
-      "mean_token_accuracy": 0.6867286145687104,
-      "num_tokens": 255586.0,
+      "loss": 1.2316,
+      "mean_token_accuracy": 0.6982571691274643,
+      "num_tokens": 248616.0,
       "step": 135
     },
     {
-      "entropy": 1.2312346011400224,
-      "epoch": 0.1569946733950098,
-      "grad_norm": 0.6513665318489075,
+      "entropy": 1.333225554227829,
+      "epoch": 0.15703869882220975,
+      "grad_norm": 0.6222459077835083,
       "learning_rate": 0.0001697857948139797,
-      "loss": 1.2024,
-      "mean_token_accuracy": 0.7194372028112411,
-      "num_tokens": 264356.0,
+      "loss": 1.2305,
+      "mean_token_accuracy": 0.7053877979516983,
+      "num_tokens": 259137.0,
       "step": 140
     },
     {
-      "entropy": 1.1984657406806947,
-      "epoch": 0.16260162601626016,
-      "grad_norm": 0.6147997379302979,
+      "entropy": 1.2306534215807914,
+      "epoch": 0.16264722378014582,
+      "grad_norm": 0.5644901394844055,
       "learning_rate": 0.00016865839909808344,
-      "loss": 1.2174,
-      "mean_token_accuracy": 0.7164656221866608,
-      "num_tokens": 272889.0,
+      "loss": 1.2349,
+      "mean_token_accuracy": 0.7150521993637085,
+      "num_tokens": 270269.0,
       "step": 145
     },
     {
-      "entropy": 1.2961942851543427,
-      "epoch": 0.16820857863751051,
-      "grad_norm": 0.6134310364723206,
+      "entropy": 1.2639500305056572,
+      "epoch": 0.16825574873808188,
+      "grad_norm": 0.6914640665054321,
       "learning_rate": 0.00016753100338218714,
-      "loss": 1.2279,
-      "mean_token_accuracy": 0.7087068349123001,
-      "num_tokens": 282106.0,
+      "loss": 1.2328,
+      "mean_token_accuracy": 0.7164399117231369,
+      "num_tokens": 278900.0,
       "step": 150
     },
     {
-      "entropy": 1.2217833191156386,
-      "epoch": 0.17381553125876087,
-      "grad_norm": 0.6766023635864258,
+      "entropy": 1.2492301687598228,
+      "epoch": 0.17386427369601795,
+      "grad_norm": 0.6064640879631042,
       "learning_rate": 0.00016640360766629087,
-      "loss": 1.2188,
-      "mean_token_accuracy": 0.7227874040603638,
-      "num_tokens": 291923.0,
+      "loss": 1.2309,
+      "mean_token_accuracy": 0.7067903488874435,
+      "num_tokens": 288895.0,
       "step": 155
     },
     {
-      "entropy": 1.2688794553279876,
-      "epoch": 0.17942248388001122,
-      "grad_norm": 0.666310727596283,
+      "entropy": 1.1464111924171447,
+      "epoch": 0.17947279865395402,
+      "grad_norm": 0.5932626724243164,
       "learning_rate": 0.0001652762119503946,
-      "loss": 1.2627,
-      "mean_token_accuracy": 0.7072140723466873,
-      "num_tokens": 299752.0,
+      "loss": 1.1398,
+      "mean_token_accuracy": 0.7307830601930618,
+      "num_tokens": 298947.0,
       "step": 160
     },
     {
-      "entropy": 1.3040128737688064,
-      "epoch": 0.18502943650126155,
-      "grad_norm": 0.581291913986206,
+      "entropy": 1.1858425721526147,
+      "epoch": 0.1850813236118901,
+      "grad_norm": 0.5144683122634888,
       "learning_rate": 0.0001641488162344983,
-      "loss": 1.2133,
-      "mean_token_accuracy": 0.6970360308885575,
-      "num_tokens": 308737.0,
+      "loss": 1.1402,
+      "mean_token_accuracy": 0.730936524271965,
+      "num_tokens": 307660.0,
       "step": 165
     },
     {
-      "entropy": 1.0922872066497802,
-      "epoch": 0.1906363891225119,
-      "grad_norm": 0.6826110482215881,
+      "entropy": 1.3814254194498061,
+      "epoch": 0.19068984856982613,
+      "grad_norm": 0.5210261940956116,
       "learning_rate": 0.00016302142051860202,
-      "loss": 1.0315,
-      "mean_token_accuracy": 0.7402381807565689,
-      "num_tokens": 315852.0,
+      "loss": 1.3247,
+      "mean_token_accuracy": 0.6925279140472412,
+      "num_tokens": 318346.0,
       "step": 170
     },
     {
-      "entropy": 1.184697662293911,
-      "epoch": 0.19624334174376226,
-      "grad_norm": 0.670078694820404,
+      "entropy": 1.0947081446647644,
+      "epoch": 0.1962983735277622,
+      "grad_norm": 0.5158228278160095,
       "learning_rate": 0.00016189402480270578,
-      "loss": 1.1541,
-      "mean_token_accuracy": 0.7335967868566513,
-      "num_tokens": 324430.0,
+      "loss": 1.0251,
+      "mean_token_accuracy": 0.7401407450437546,
+      "num_tokens": 327924.0,
       "step": 175
     },
     {
-      "entropy": 1.2350615233182907,
-      "epoch": 0.20185029436501262,
-      "grad_norm": 0.5114791989326477,
+      "entropy": 1.1991018429398537,
+      "epoch": 0.20190689848569826,
+      "grad_norm": 0.6816720366477966,
       "learning_rate": 0.00016076662908680949,
-      "loss": 1.1825,
-      "mean_token_accuracy": 0.7157844036817551,
-      "num_tokens": 331843.0,
+      "loss": 1.19,
+      "mean_token_accuracy": 0.7179287821054459,
+      "num_tokens": 337459.0,
       "step": 180
     },
     {
-      "entropy": 1.1912701576948166,
-      "epoch": 0.20745724698626297,
-      "grad_norm": 0.668006420135498,
+      "entropy": 1.2390994131565094,
+      "epoch": 0.20751542344363433,
+      "grad_norm": 0.5661265254020691,
       "learning_rate": 0.0001596392333709132,
-      "loss": 1.1067,
-      "mean_token_accuracy": 0.7309438616037369,
-      "num_tokens": 340933.0,
+      "loss": 1.1333,
+      "mean_token_accuracy": 0.7147542536258698,
+      "num_tokens": 345281.0,
       "step": 185
     },
     {
-      "entropy": 1.0719711840152741,
-      "epoch": 0.21306419960751333,
-      "grad_norm": 0.5813568234443665,
+      "entropy": 1.1696692734956742,
+      "epoch": 0.2131239484015704,
+      "grad_norm": 0.6144779324531555,
       "learning_rate": 0.00015851183765501692,
-      "loss": 1.0212,
-      "mean_token_accuracy": 0.7530986189842224,
-      "num_tokens": 349083.0,
+      "loss": 1.0103,
+      "mean_token_accuracy": 0.7357000023126602,
+      "num_tokens": 352958.0,
       "step": 190
     },
     {
-      "entropy": 1.3020119816064835,
-      "epoch": 0.21867115222876365,
-      "grad_norm": 0.6488296985626221,
+      "entropy": 1.0967671677470208,
+      "epoch": 0.21873247335950646,
+      "grad_norm": 0.724777102470398,
       "learning_rate": 0.00015738444193912063,
-      "loss": 1.2975,
-      "mean_token_accuracy": 0.6928794890642166,
-      "num_tokens": 358133.0,
+      "loss": 1.0985,
+      "mean_token_accuracy": 0.7314315021038056,
+      "num_tokens": 362012.0,
       "step": 195
     },
     {
-      "entropy": 1.1875290542840957,
-      "epoch": 0.224278104850014,
-      "grad_norm": 0.6314829587936401,
+      "entropy": 1.2297322571277618,
+      "epoch": 0.2243409983174425,
+      "grad_norm": 0.5694834589958191,
       "learning_rate": 0.00015625704622322436,
-      "loss": 1.2064,
-      "mean_token_accuracy": 0.7248553454875946,
-      "num_tokens": 367701.0,
+      "loss": 1.1923,
+      "mean_token_accuracy": 0.7140013068914414,
+      "num_tokens": 371661.0,
       "step": 200
     },
     {
-      "entropy": 1.2919130593538284,
-      "epoch": 0.22988505747126436,
-      "grad_norm": 0.5402503609657288,
+      "entropy": 1.1518067017197609,
+      "epoch": 0.22994952327537857,
+      "grad_norm": 0.5081655383110046,
       "learning_rate": 0.0001551296505073281,
-      "loss": 1.2512,
-      "mean_token_accuracy": 0.7068570107221603,
-      "num_tokens": 376039.0,
+      "loss": 1.0934,
+      "mean_token_accuracy": 0.7330572694540024,
+      "num_tokens": 380872.0,
       "step": 205
     },
     {
-      "entropy": 1.2941559731960297,
-      "epoch": 0.23549201009251472,
-      "grad_norm": 0.5794088244438171,
+      "entropy": 1.2965759575366973,
+      "epoch": 0.23555804823331464,
+      "grad_norm": 0.4979788362979889,
       "learning_rate": 0.0001540022547914318,
-      "loss": 1.2339,
-      "mean_token_accuracy": 0.7108895808458329,
-      "num_tokens": 384327.0,
+      "loss": 1.224,
+      "mean_token_accuracy": 0.702433243393898,
+      "num_tokens": 391828.0,
       "step": 210
     },
     {
-      "entropy": 1.1999757021665574,
-      "epoch": 0.24109896271376507,
-      "grad_norm": 0.5067386627197266,
+      "entropy": 1.3438188642263413,
+      "epoch": 0.2411665731912507,
+      "grad_norm": 0.5993033647537231,
       "learning_rate": 0.0001528748590755355,
-      "loss": 1.1892,
-      "mean_token_accuracy": 0.713716721534729,
-      "num_tokens": 394548.0,
+      "loss": 1.3083,
+      "mean_token_accuracy": 0.6839649230241776,
+      "num_tokens": 399939.0,
       "step": 215
     },
     {
-      "entropy": 1.1508339285850524,
-      "epoch": 0.24670591533501543,
-      "grad_norm": 0.7008835077285767,
+      "entropy": 1.18723217099905,
+      "epoch": 0.24677509814918677,
+      "grad_norm": 0.5683300495147705,
       "learning_rate": 0.00015174746335963924,
-      "loss": 1.1226,
-      "mean_token_accuracy": 0.7230166435241699,
-      "num_tokens": 403522.0,
+      "loss": 1.1751,
+      "mean_token_accuracy": 0.7297552257776261,
+      "num_tokens": 408814.0,
       "step": 220
     },
     {
-      "entropy": 1.196462707221508,
-      "epoch": 0.2523128679562658,
-      "grad_norm": 0.5998469591140747,
+      "entropy": 1.2557217076420784,
+      "epoch": 0.2523836231071228,
+      "grad_norm": 0.6166518926620483,
       "learning_rate": 0.00015062006764374297,
-      "loss": 1.2183,
-      "mean_token_accuracy": 0.7174128830432892,
-      "num_tokens": 414211.0,
+      "loss": 1.2271,
+      "mean_token_accuracy": 0.7061844110488892,
+      "num_tokens": 418295.0,
       "step": 225
     },
     {
-      "entropy": 1.044454263150692,
-      "epoch": 0.2579198205775161,
-      "grad_norm": 0.5503870844841003,
+      "entropy": 1.084635604918003,
+      "epoch": 0.2579921480650589,
+      "grad_norm": 0.5858753323554993,
       "learning_rate": 0.00014949267192784668,
-      "loss": 0.9978,
-      "mean_token_accuracy": 0.7547496408224106,
-      "num_tokens": 424482.0,
+      "loss": 1.0096,
+      "mean_token_accuracy": 0.7489422798156739,
+      "num_tokens": 426560.0,
       "step": 230
     },
     {
-      "entropy": 1.2794962912797927,
-      "epoch": 0.2635267731987665,
-      "grad_norm": 0.5634020566940308,
+      "entropy": 1.2373799532651901,
+      "epoch": 0.26360067302299495,
+      "grad_norm": 0.6692758202552795,
       "learning_rate": 0.00014836527621195039,
-      "loss": 1.2497,
-      "mean_token_accuracy": 0.701577228307724,
-      "num_tokens": 433911.0,
+      "loss": 1.2283,
+      "mean_token_accuracy": 0.7165287554264068,
+      "num_tokens": 435354.0,
       "step": 235
     },
     {
-      "entropy": 1.2079395592212676,
-      "epoch": 0.2691337258200168,
-      "grad_norm": 0.6672863364219666,
+      "entropy": 1.1109624326229095,
+      "epoch": 0.269209197980931,
+      "grad_norm": 0.5951205492019653,
       "learning_rate": 0.00014723788049605412,
-      "loss": 1.185,
-      "mean_token_accuracy": 0.717086723446846,
-      "num_tokens": 442974.0,
+      "loss": 1.0563,
+      "mean_token_accuracy": 0.732743826508522,
+      "num_tokens": 444810.0,
       "step": 240
     },
     {
-      "entropy": 1.1594905465841294,
-      "epoch": 0.27474067844126715,
-      "grad_norm": 0.6265320181846619,
+      "entropy": 1.2168930009007455,
+      "epoch": 0.2748177229388671,
+      "grad_norm": 0.4504969120025635,
       "learning_rate": 0.00014611048478015785,
-      "loss": 1.0919,
-      "mean_token_accuracy": 0.7310166716575622,
-      "num_tokens": 450850.0,
+      "loss": 1.2377,
+      "mean_token_accuracy": 0.7143894642591476,
+      "num_tokens": 454278.0,
       "step": 245
     },
     {
-      "entropy": 1.244988052546978,
-      "epoch": 0.28034763106251753,
-      "grad_norm": 0.5000255703926086,
+      "entropy": 1.123945553600788,
+      "epoch": 0.28042624789680315,
+      "grad_norm": 0.6897888779640198,
       "learning_rate": 0.00014498308906426156,
-      "loss": 1.2546,
-      "mean_token_accuracy": 0.7129500776529312,
-      "num_tokens": 461926.0,
+      "loss": 1.0974,
+      "mean_token_accuracy": 0.7368963181972503,
+      "num_tokens": 462577.0,
       "step": 250
     },
     {
-      "entropy": 1.0460072100162505,
-      "epoch": 0.28595458368376786,
-      "grad_norm": 0.6440132260322571,
+      "entropy": 1.141464115679264,
+      "epoch": 0.2860347728547392,
+      "grad_norm": 0.6041523218154907,
       "learning_rate": 0.0001438556933483653,
-      "loss": 1.0097,
-      "mean_token_accuracy": 0.7434480965137482,
-      "num_tokens": 469686.0,
+      "loss": 1.1427,
+      "mean_token_accuracy": 0.7364778339862823,
+      "num_tokens": 471168.0,
       "step": 255
     },
     {
-      "entropy": 1.1157130993902684,
-      "epoch": 0.29156153630501824,
-      "grad_norm": 0.6023146510124207,
+      "entropy": 1.2550072342157363,
+      "epoch": 0.2916432978126753,
+      "grad_norm": 0.7248488068580627,
       "learning_rate": 0.000142728297632469,
-      "loss": 1.0708,
-      "mean_token_accuracy": 0.7285511642694473,
-      "num_tokens": 479081.0,
+      "loss": 1.2154,
+      "mean_token_accuracy": 0.7032849937677383,
+      "num_tokens": 480745.0,
       "step": 260
     },
     {
-      "entropy": 1.2074461445212363,
-      "epoch": 0.29716848892626857,
-      "grad_norm": 0.6504403948783875,
+      "entropy": 1.1180053681135178,
+      "epoch": 0.29725182277061135,
+      "grad_norm": 0.6221792697906494,
       "learning_rate": 0.0001416009019165727,
-      "loss": 1.2055,
-      "mean_token_accuracy": 0.7167613714933395,
-      "num_tokens": 488026.0,
+      "loss": 1.1121,
+      "mean_token_accuracy": 0.7327967584133148,
+      "num_tokens": 489018.0,
       "step": 265
     },
     {
-      "entropy": 1.1629670545458795,
-      "epoch": 0.3027754415475189,
-      "grad_norm": 0.7767821550369263,
+      "entropy": 1.183423639833927,
+      "epoch": 0.3028603477285474,
+      "grad_norm": 0.5834987759590149,
       "learning_rate": 0.00014047350620067646,
-      "loss": 1.0925,
-      "mean_token_accuracy": 0.7313909947872161,
-      "num_tokens": 496200.0,
+      "loss": 1.1682,
+      "mean_token_accuracy": 0.7155926108360291,
+      "num_tokens": 498458.0,
       "step": 270
     },
     {
-      "entropy": 1.0763523548841476,
-      "epoch": 0.3083823941687693,
-      "grad_norm": 0.54071444272995,
+      "entropy": 1.2206986933946609,
+      "epoch": 0.30846887268648343,
+      "grad_norm": 0.6320741176605225,
       "learning_rate": 0.00013934611048478017,
-      "loss": 1.006,
-      "mean_token_accuracy": 0.7479322016239166,
-      "num_tokens": 506000.0,
+      "loss": 1.2206,
+      "mean_token_accuracy": 0.7106427907943725,
+      "num_tokens": 508278.0,
       "step": 275
     },
     {
-      "entropy": 1.2247862741351128,
-      "epoch": 0.3139893467900196,
-      "grad_norm": 0.5242252945899963,
+      "entropy": 1.3010928213596344,
+      "epoch": 0.3140773976444195,
+      "grad_norm": 0.8451023101806641,
       "learning_rate": 0.00013821871476888388,
-      "loss": 1.2591,
-      "mean_token_accuracy": 0.7159645825624465,
-      "num_tokens": 515345.0,
+      "loss": 1.2283,
+      "mean_token_accuracy": 0.7067124038934708,
+      "num_tokens": 518492.0,
       "step": 280
     },
     {
-      "entropy": 1.3475455969572068,
-      "epoch": 0.31959629941127,
-      "grad_norm": 0.5631939768791199,
+      "entropy": 1.2411514788866043,
+      "epoch": 0.31968592260235557,
+      "grad_norm": 0.5151481032371521,
       "learning_rate": 0.0001370913190529876,
-      "loss": 1.3375,
-      "mean_token_accuracy": 0.6826408416032791,
-      "num_tokens": 524644.0,
+      "loss": 1.1855,
+      "mean_token_accuracy": 0.7306610077619553,
+      "num_tokens": 528203.0,
       "step": 285
     },
     {
-      "entropy": 1.0502676755189895,
-      "epoch": 0.3252032520325203,
-      "grad_norm": 0.611739993095398,
+      "entropy": 1.094706454873085,
+      "epoch": 0.32529444756029163,
+      "grad_norm": 0.48817235231399536,
       "learning_rate": 0.00013596392333709131,
-      "loss": 0.9899,
-      "mean_token_accuracy": 0.7530880838632583,
-      "num_tokens": 532559.0,
+      "loss": 1.1335,
+      "mean_token_accuracy": 0.7377402186393738,
+      "num_tokens": 538068.0,
       "step": 290
     },
     {
-      "entropy": 1.1604034945368766,
-      "epoch": 0.3308102046537707,
-      "grad_norm": 0.4446961283683777,
+      "entropy": 1.0780605375766754,
+      "epoch": 0.3309029725182277,
+      "grad_norm": 0.5559823513031006,
       "learning_rate": 0.00013483652762119505,
-      "loss": 1.1227,
-      "mean_token_accuracy": 0.7217718571424484,
-      "num_tokens": 544302.0,
+      "loss": 1.1127,
+      "mean_token_accuracy": 0.7499582827091217,
+      "num_tokens": 545879.0,
       "step": 295
     },
     {
-      "entropy": 1.1056245781481266,
-      "epoch": 0.33641715727502103,
-      "grad_norm": 0.8371356725692749,
+      "entropy": 1.3355800449848174,
+      "epoch": 0.33651149747616377,
+      "grad_norm": 0.4629262685775757,
       "learning_rate": 0.00013370913190529878,
-      "loss": 1.1266,
-      "mean_token_accuracy": 0.7354450315237046,
-      "num_tokens": 553700.0,
+      "loss": 1.318,
+      "mean_token_accuracy": 0.6883445054292678,
+      "num_tokens": 555761.0,
       "step": 300
     }
   ],
@@ -627,7 +627,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2.50722039128064e+16,
+  "total_flos": 2.516552847893299e+16,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:79d41fa02013525705cb7a82d4f608a53737fdbc7baa1d76305c242ebd4e870e
+oid sha256:75cf331c13a33e1598e45e9013486c4013f3af1f377da7304e21a0d1c22d72cb
 size 5816