Upload folder using huggingface_hub

adapter_config.json

@@ -29,13 +29,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "down_proj",
     "up_proj",
-    "q_proj",
     "v_proj",
-    "gate_proj",
-    "down_proj",
     "k_proj",
-    "o_proj"
+    "gate_proj",
+    "o_proj",
+    "q_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a3db570fd27876879cc33103ca8933604745e61f4019c1d836c15c7fe2de9457
+oid sha256:01b1fff680a543298f99a02a59b07fdd20779d45d76050ae4657d7155dba7881
 size 167832240

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:beeda5e00e3f2a251a26931469526d6fefa8f6cc35fc3926826851fecb38c416
+oid sha256:d57468746ed225d196506f39da49e6e801e88bb9404cd3efc0e75e0398a685ef
 size 85728342

scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0738d3f7bc7a3d09895ccd1699afea3beb6076cacca6285d9dff68b3826864fb
+oid sha256:2955c699f5b1504e9840700f4d5dd5648f18dbf48dd8849c32859b9338c9c1b6
 size 988

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8bea4c55977be70b1134031e6a8b57e36f8f593b2249c6d9d6b94a16db34cae2
+oid sha256:af2fc7fde6a8810f2aaee4f51ad7b6ecee188721a4bad00d143c4300ff3122b5
 size 1064

trainer_state.json

@@ -10,1142 +10,1142 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "entropy": 1.333236712217331,
+      "entropy": 1.3178111910820007,
       "epoch": 0.008726003490401396,
-      "grad_norm": 1.05680251121521,
-      "learning_rate": 8e-05,
-      "loss": 1.3485,
-      "mean_token_accuracy": 0.6778763651847839,
+      "grad_norm": 1.3144103288650513,
+      "learning_rate": 2e-05,
+      "loss": 1.3891,
+      "mean_token_accuracy": 0.6749979019165039,
       "num_tokens": 4689.0,
       "step": 5
     },
     {
-      "entropy": 1.3305357813835144,
+      "entropy": 1.3977092504501343,
       "epoch": 0.017452006980802792,
-      "grad_norm": 0.872683584690094,
-      "learning_rate": 0.00018,
-      "loss": 1.2321,
-      "mean_token_accuracy": 0.6924330353736877,
+      "grad_norm": 0.9799396991729736,
+      "learning_rate": 4.5e-05,
+      "loss": 1.3155,
+      "mean_token_accuracy": 0.672690337896347,
       "num_tokens": 9076.0,
       "step": 10
     },
     {
-      "entropy": 1.1033322036266326,
+      "entropy": 1.1513380348682403,
       "epoch": 0.02617801047120419,
-      "grad_norm": 0.7987897992134094,
-      "learning_rate": 0.0001985790408525755,
-      "loss": 1.0149,
-      "mean_token_accuracy": 0.7376092612743378,
+      "grad_norm": 0.903432309627533,
+      "learning_rate": 7e-05,
+      "loss": 1.061,
+      "mean_token_accuracy": 0.728508323431015,
       "num_tokens": 13468.0,
       "step": 15
     },
     {
-      "entropy": 1.3769255757331849,
+      "entropy": 1.4041914224624634,
       "epoch": 0.034904013961605584,
-      "grad_norm": 0.5818138718605042,
-      "learning_rate": 0.00019680284191829486,
-      "loss": 1.3507,
-      "mean_token_accuracy": 0.6773534297943116,
+      "grad_norm": 0.6405840516090393,
+      "learning_rate": 9.5e-05,
+      "loss": 1.3689,
+      "mean_token_accuracy": 0.675159877538681,
       "num_tokens": 19687.0,
       "step": 20
     },
     {
-      "entropy": 1.3704544663429261,
+      "entropy": 1.3952594161033631,
       "epoch": 0.04363001745200698,
-      "grad_norm": 0.7997947335243225,
-      "learning_rate": 0.00019502664298401423,
-      "loss": 1.2011,
-      "mean_token_accuracy": 0.7000263512134552,
+      "grad_norm": 0.8190271854400635,
+      "learning_rate": 9.927667269439421e-05,
+      "loss": 1.2273,
+      "mean_token_accuracy": 0.6897247910499573,
       "num_tokens": 25321.0,
       "step": 25
     },
     {
-      "entropy": 1.1999043464660644,
+      "entropy": 1.2674797415733337,
       "epoch": 0.05235602094240838,
-      "grad_norm": 0.7836564779281616,
-      "learning_rate": 0.00019325044404973357,
-      "loss": 1.1702,
-      "mean_token_accuracy": 0.7157157480716705,
+      "grad_norm": 1.1967648267745972,
+      "learning_rate": 9.837251356238698e-05,
+      "loss": 1.199,
+      "mean_token_accuracy": 0.7063600897789002,
       "num_tokens": 29969.0,
       "step": 30
     },
     {
-      "entropy": 1.1485023498535156,
+      "entropy": 1.1735109210014343,
       "epoch": 0.06108202443280977,
-      "grad_norm": 0.8341999650001526,
-      "learning_rate": 0.00019147424511545294,
-      "loss": 1.0127,
-      "mean_token_accuracy": 0.736984384059906,
+      "grad_norm": 0.8456419706344604,
+      "learning_rate": 9.746835443037975e-05,
+      "loss": 1.0482,
+      "mean_token_accuracy": 0.7267675757408142,
       "num_tokens": 34779.0,
       "step": 35
     },
     {
-      "entropy": 1.175552135705948,
+      "entropy": 1.2172421634197235,
       "epoch": 0.06980802792321117,
-      "grad_norm": 0.8367146849632263,
-      "learning_rate": 0.0001896980461811723,
-      "loss": 1.081,
-      "mean_token_accuracy": 0.7145774185657501,
+      "grad_norm": 0.8677243590354919,
+      "learning_rate": 9.656419529837252e-05,
+      "loss": 1.1052,
+      "mean_token_accuracy": 0.7135568201541901,
       "num_tokens": 40454.0,
       "step": 40
     },
     {
-      "entropy": 1.2580669283866883,
+      "entropy": 1.2951707720756531,
       "epoch": 0.07853403141361257,
-      "grad_norm": 0.7274704575538635,
-      "learning_rate": 0.00018792184724689167,
-      "loss": 1.1622,
-      "mean_token_accuracy": 0.7037679016590118,
+      "grad_norm": 0.7614730596542358,
+      "learning_rate": 9.566003616636529e-05,
+      "loss": 1.1822,
+      "mean_token_accuracy": 0.6976100087165833,
       "num_tokens": 45435.0,
       "step": 45
     },
     {
-      "entropy": 1.3289404153823852,
+      "entropy": 1.369252896308899,
       "epoch": 0.08726003490401396,
-      "grad_norm": 0.9459134936332703,
-      "learning_rate": 0.00018614564831261103,
-      "loss": 1.2887,
-      "mean_token_accuracy": 0.7032808780670166,
+      "grad_norm": 1.0491780042648315,
+      "learning_rate": 9.475587703435806e-05,
+      "loss": 1.303,
+      "mean_token_accuracy": 0.6986614286899566,
       "num_tokens": 50161.0,
       "step": 50
     },
     {
-      "entropy": 1.206754869222641,
+      "entropy": 1.2288368880748748,
       "epoch": 0.09598603839441536,
-      "grad_norm": 0.8248263001441956,
-      "learning_rate": 0.00018436944937833037,
-      "loss": 1.0185,
-      "mean_token_accuracy": 0.7304032206535339,
+      "grad_norm": 0.8575630784034729,
+      "learning_rate": 9.385171790235083e-05,
+      "loss": 1.0351,
+      "mean_token_accuracy": 0.7264712870121002,
       "num_tokens": 55082.0,
       "step": 55
     },
     {
-      "entropy": 1.3467580556869507,
+      "entropy": 1.3860582947731017,
       "epoch": 0.10471204188481675,
-      "grad_norm": 0.7025023698806763,
-      "learning_rate": 0.00018259325044404974,
-      "loss": 1.3245,
-      "mean_token_accuracy": 0.6774280846118927,
+      "grad_norm": 0.7137913703918457,
+      "learning_rate": 9.29475587703436e-05,
+      "loss": 1.3276,
+      "mean_token_accuracy": 0.6737604200839996,
       "num_tokens": 61109.0,
       "step": 60
     },
     {
-      "entropy": 1.1657752752304078,
+      "entropy": 1.186289870738983,
       "epoch": 0.11343804537521815,
-      "grad_norm": 0.7866821885108948,
-      "learning_rate": 0.0001808170515097691,
-      "loss": 1.0342,
-      "mean_token_accuracy": 0.7379155635833741,
+      "grad_norm": 0.8168688416481018,
+      "learning_rate": 9.204339963833635e-05,
+      "loss": 1.0535,
+      "mean_token_accuracy": 0.7318423926830292,
       "num_tokens": 65130.0,
       "step": 65
     },
     {
-      "entropy": 1.3768277764320374,
+      "entropy": 1.3940527975559234,
       "epoch": 0.12216404886561955,
-      "grad_norm": 0.6452690958976746,
-      "learning_rate": 0.00017904085257548847,
-      "loss": 1.3499,
-      "mean_token_accuracy": 0.6878371357917785,
+      "grad_norm": 0.7069775462150574,
+      "learning_rate": 9.113924050632912e-05,
+      "loss": 1.3631,
+      "mean_token_accuracy": 0.6877448439598084,
       "num_tokens": 71720.0,
       "step": 70
     },
     {
-      "entropy": 1.2285258889198303,
+      "entropy": 1.248500007390976,
       "epoch": 0.13089005235602094,
-      "grad_norm": 0.8868134617805481,
-      "learning_rate": 0.00017726465364120784,
-      "loss": 1.1203,
-      "mean_token_accuracy": 0.7103672683238983,
+      "grad_norm": 0.9421133399009705,
+      "learning_rate": 9.023508137432188e-05,
+      "loss": 1.1336,
+      "mean_token_accuracy": 0.7071475625038147,
       "num_tokens": 76475.0,
       "step": 75
     },
     {
-      "entropy": 1.142468798160553,
+      "entropy": 1.1854017496109008,
       "epoch": 0.13961605584642234,
-      "grad_norm": 0.7537686228752136,
-      "learning_rate": 0.00017548845470692718,
-      "loss": 1.0207,
-      "mean_token_accuracy": 0.7329977452754974,
+      "grad_norm": 0.8254420757293701,
+      "learning_rate": 8.933092224231465e-05,
+      "loss": 1.0336,
+      "mean_token_accuracy": 0.7307356059551239,
       "num_tokens": 82239.0,
       "step": 80
     },
     {
-      "entropy": 1.30864217877388,
+      "entropy": 1.3216606438159944,
       "epoch": 0.14834205933682373,
-      "grad_norm": 0.9109086394309998,
-      "learning_rate": 0.00017371225577264654,
-      "loss": 1.2256,
-      "mean_token_accuracy": 0.6924388945102692,
+      "grad_norm": 0.9723417162895203,
+      "learning_rate": 8.842676311030742e-05,
+      "loss": 1.2206,
+      "mean_token_accuracy": 0.6910631775856018,
       "num_tokens": 86033.0,
       "step": 85
     },
     {
-      "entropy": 1.279932165145874,
+      "entropy": 1.2633710026741027,
       "epoch": 0.15706806282722513,
-      "grad_norm": 0.7983659505844116,
-      "learning_rate": 0.0001719360568383659,
-      "loss": 1.1764,
-      "mean_token_accuracy": 0.7101370930671692,
+      "grad_norm": 0.8575649857521057,
+      "learning_rate": 8.752260397830019e-05,
+      "loss": 1.1781,
+      "mean_token_accuracy": 0.711302649974823,
       "num_tokens": 90170.0,
       "step": 90
     },
     {
-      "entropy": 1.1692178070545196,
+      "entropy": 1.1930577754974365,
       "epoch": 0.16579406631762653,
-      "grad_norm": 0.8946067690849304,
-      "learning_rate": 0.00017015985790408525,
-      "loss": 1.0826,
-      "mean_token_accuracy": 0.7317939043045044,
+      "grad_norm": 0.9084628224372864,
+      "learning_rate": 8.661844484629296e-05,
+      "loss": 1.0925,
+      "mean_token_accuracy": 0.7296910464763642,
       "num_tokens": 95473.0,
       "step": 95
     },
     {
-      "entropy": 1.025848913192749,
+      "entropy": 1.097593402862549,
       "epoch": 0.17452006980802792,
-      "grad_norm": 0.8327645063400269,
-      "learning_rate": 0.00016838365896980464,
-      "loss": 0.9294,
-      "mean_token_accuracy": 0.7514408528804779,
+      "grad_norm": 0.9089357852935791,
+      "learning_rate": 8.571428571428571e-05,
+      "loss": 0.9541,
+      "mean_token_accuracy": 0.7425937652587891,
       "num_tokens": 99423.0,
       "step": 100
     },
     {
-      "entropy": 1.0799501717090607,
+      "entropy": 1.1105962693691254,
       "epoch": 0.18324607329842932,
-      "grad_norm": 0.7194784283638,
-      "learning_rate": 0.00016660746003552398,
-      "loss": 1.0222,
-      "mean_token_accuracy": 0.7337860226631164,
+      "grad_norm": 0.8107589483261108,
+      "learning_rate": 8.481012658227848e-05,
+      "loss": 1.0277,
+      "mean_token_accuracy": 0.7366280615329742,
       "num_tokens": 104249.0,
       "step": 105
     },
     {
-      "entropy": 1.1033223390579223,
+      "entropy": 1.121857112646103,
       "epoch": 0.19197207678883071,
-      "grad_norm": 0.7712328433990479,
-      "learning_rate": 0.00016483126110124335,
-      "loss": 0.9856,
-      "mean_token_accuracy": 0.7449049592018128,
+      "grad_norm": 0.8907290101051331,
+      "learning_rate": 8.390596745027125e-05,
+      "loss": 0.9944,
+      "mean_token_accuracy": 0.7407838046550751,
       "num_tokens": 109205.0,
       "step": 110
     },
     {
-      "entropy": 1.1388230919837952,
+      "entropy": 1.1668152272701264,
       "epoch": 0.2006980802792321,
-      "grad_norm": 0.6309220194816589,
-      "learning_rate": 0.00016305506216696272,
-      "loss": 1.1354,
-      "mean_token_accuracy": 0.724005150794983,
+      "grad_norm": 0.7008240222930908,
+      "learning_rate": 8.300180831826402e-05,
+      "loss": 1.1461,
+      "mean_token_accuracy": 0.7233073830604553,
       "num_tokens": 115207.0,
       "step": 115
     },
     {
-      "entropy": 1.0293731987476349,
+      "entropy": 1.0229185461997985,
       "epoch": 0.2094240837696335,
-      "grad_norm": 1.0027621984481812,
-      "learning_rate": 0.00016127886323268206,
-      "loss": 0.9218,
-      "mean_token_accuracy": 0.7559137165546417,
+      "grad_norm": 1.1023997068405151,
+      "learning_rate": 8.209764918625679e-05,
+      "loss": 0.9365,
+      "mean_token_accuracy": 0.7564353287220001,
       "num_tokens": 120323.0,
       "step": 120
     },
     {
-      "entropy": 1.1900119483470917,
+      "entropy": 1.205068576335907,
       "epoch": 0.2181500872600349,
-      "grad_norm": 0.8019612431526184,
-      "learning_rate": 0.00015950266429840145,
-      "loss": 1.106,
-      "mean_token_accuracy": 0.7178053438663483,
+      "grad_norm": 0.9460663199424744,
+      "learning_rate": 8.119349005424956e-05,
+      "loss": 1.1133,
+      "mean_token_accuracy": 0.7165132164955139,
       "num_tokens": 125253.0,
       "step": 125
     },
     {
-      "entropy": 1.0218496084213258,
+      "entropy": 1.0808123528957367,
       "epoch": 0.2268760907504363,
-      "grad_norm": 0.699367105960846,
-      "learning_rate": 0.0001577264653641208,
-      "loss": 0.931,
-      "mean_token_accuracy": 0.7488141357898712,
+      "grad_norm": 0.9484661221504211,
+      "learning_rate": 8.028933092224232e-05,
+      "loss": 0.9489,
+      "mean_token_accuracy": 0.7440634310245514,
       "num_tokens": 130360.0,
       "step": 130
     },
     {
-      "entropy": 1.1080122888088226,
+      "entropy": 1.152905023097992,
       "epoch": 0.2356020942408377,
-      "grad_norm": 0.7124127745628357,
-      "learning_rate": 0.00015595026642984015,
-      "loss": 1.0557,
-      "mean_token_accuracy": 0.7226514399051667,
+      "grad_norm": 0.8243975043296814,
+      "learning_rate": 7.938517179023509e-05,
+      "loss": 1.0735,
+      "mean_token_accuracy": 0.7156176209449768,
       "num_tokens": 135538.0,
       "step": 135
     },
     {
-      "entropy": 1.173432421684265,
+      "entropy": 1.2132245182991028,
       "epoch": 0.2443280977312391,
-      "grad_norm": 0.794236421585083,
-      "learning_rate": 0.00015417406749555952,
-      "loss": 1.056,
-      "mean_token_accuracy": 0.7334702372550964,
+      "grad_norm": 0.8641315698623657,
+      "learning_rate": 7.848101265822784e-05,
+      "loss": 1.0773,
+      "mean_token_accuracy": 0.7272770285606385,
       "num_tokens": 140532.0,
       "step": 140
     },
     {
-      "entropy": 1.0574114263057708,
+      "entropy": 1.0758103907108307,
       "epoch": 0.2530541012216405,
-      "grad_norm": 0.6696324944496155,
-      "learning_rate": 0.00015239786856127886,
-      "loss": 0.9361,
-      "mean_token_accuracy": 0.7482443630695343,
+      "grad_norm": 0.7532796859741211,
+      "learning_rate": 7.757685352622061e-05,
+      "loss": 0.9522,
+      "mean_token_accuracy": 0.7446901857852936,
       "num_tokens": 145908.0,
       "step": 145
     },
     {
-      "entropy": 1.086327201128006,
+      "entropy": 1.1179533541202544,
       "epoch": 0.2617801047120419,
-      "grad_norm": 0.5255310535430908,
-      "learning_rate": 0.00015062166962699825,
-      "loss": 1.0768,
-      "mean_token_accuracy": 0.7292326390743256,
+      "grad_norm": 0.5681455135345459,
+      "learning_rate": 7.667269439421338e-05,
+      "loss": 1.0841,
+      "mean_token_accuracy": 0.730886948108673,
       "num_tokens": 151148.0,
       "step": 150
     },
     {
-      "entropy": 1.092069786787033,
+      "entropy": 1.1392421841621398,
       "epoch": 0.2705061082024433,
-      "grad_norm": 0.6275709271430969,
-      "learning_rate": 0.0001488454706927176,
-      "loss": 1.0778,
-      "mean_token_accuracy": 0.7255069613456726,
+      "grad_norm": 0.6956934332847595,
+      "learning_rate": 7.576853526220615e-05,
+      "loss": 1.0957,
+      "mean_token_accuracy": 0.7271894216537476,
       "num_tokens": 157506.0,
       "step": 155
     },
     {
-      "entropy": 1.1596343219280243,
+      "entropy": 1.1840439975261687,
       "epoch": 0.2792321116928447,
-      "grad_norm": 0.9472619295120239,
-      "learning_rate": 0.00014706927175843693,
-      "loss": 1.1003,
-      "mean_token_accuracy": 0.7315803647041321,
+      "grad_norm": 1.076908826828003,
+      "learning_rate": 7.486437613019892e-05,
+      "loss": 1.1233,
+      "mean_token_accuracy": 0.7299613058567047,
       "num_tokens": 162992.0,
       "step": 160
     },
     {
-      "entropy": 1.0481273233890533,
+      "entropy": 1.0738611757755279,
       "epoch": 0.2879581151832461,
-      "grad_norm": 0.6921494007110596,
-      "learning_rate": 0.00014529307282415633,
-      "loss": 0.8895,
-      "mean_token_accuracy": 0.7529896676540375,
+      "grad_norm": 0.7862851023674011,
+      "learning_rate": 7.396021699819169e-05,
+      "loss": 0.9063,
+      "mean_token_accuracy": 0.7484164655208587,
       "num_tokens": 167640.0,
       "step": 165
     },
     {
-      "entropy": 1.0518691539764404,
+      "entropy": 1.0928088903427124,
       "epoch": 0.29668411867364747,
-      "grad_norm": 0.6654248237609863,
-      "learning_rate": 0.00014351687388987566,
-      "loss": 1.018,
-      "mean_token_accuracy": 0.7503870785236358,
+      "grad_norm": 0.7506266236305237,
+      "learning_rate": 7.305605786618446e-05,
+      "loss": 1.0254,
+      "mean_token_accuracy": 0.751311433315277,
       "num_tokens": 173423.0,
       "step": 170
     },
     {
-      "entropy": 1.1176642417907714,
+      "entropy": 1.1541666328907012,
       "epoch": 0.3054101221640489,
-      "grad_norm": 0.7743102312088013,
-      "learning_rate": 0.00014174067495559503,
-      "loss": 1.0807,
-      "mean_token_accuracy": 0.7225248873233795,
+      "grad_norm": 0.8343175053596497,
+      "learning_rate": 7.215189873417722e-05,
+      "loss": 1.0956,
+      "mean_token_accuracy": 0.7194818913936615,
       "num_tokens": 178986.0,
       "step": 175
     },
     {
-      "entropy": 0.9516431629657746,
+      "entropy": 0.9744765520095825,
       "epoch": 0.31413612565445026,
-      "grad_norm": 1.0389933586120605,
-      "learning_rate": 0.0001399644760213144,
-      "loss": 0.8189,
-      "mean_token_accuracy": 0.7752299129962921,
+      "grad_norm": 1.1534874439239502,
+      "learning_rate": 7.124773960216999e-05,
+      "loss": 0.8336,
+      "mean_token_accuracy": 0.772490268945694,
       "num_tokens": 183459.0,
       "step": 180
     },
     {
-      "entropy": 1.1684755861759186,
+      "entropy": 1.1847937881946564,
       "epoch": 0.3228621291448517,
-      "grad_norm": 1.4807476997375488,
-      "learning_rate": 0.00013818827708703374,
-      "loss": 1.1822,
-      "mean_token_accuracy": 0.7197710394859314,
+      "grad_norm": 1.7201915979385376,
+      "learning_rate": 7.034358047016275e-05,
+      "loss": 1.1898,
+      "mean_token_accuracy": 0.7138440608978271,
       "num_tokens": 187614.0,
       "step": 185
     },
     {
-      "entropy": 1.099220609664917,
+      "entropy": 1.1254307150840759,
       "epoch": 0.33158813263525305,
-      "grad_norm": 0.7266477346420288,
-      "learning_rate": 0.00013641207815275313,
-      "loss": 1.0095,
-      "mean_token_accuracy": 0.7297711133956909,
+      "grad_norm": 0.8325474262237549,
+      "learning_rate": 6.943942133815552e-05,
+      "loss": 1.0264,
+      "mean_token_accuracy": 0.7251651823520661,
       "num_tokens": 192316.0,
       "step": 190
     },
     {
-      "entropy": 1.0837588012218475,
+      "entropy": 1.1271761000156402,
       "epoch": 0.3403141361256545,
-      "grad_norm": 0.696660041809082,
-      "learning_rate": 0.00013463587921847247,
-      "loss": 0.9739,
-      "mean_token_accuracy": 0.7354932248592376,
+      "grad_norm": 0.7783677577972412,
+      "learning_rate": 6.85352622061483e-05,
+      "loss": 0.9858,
+      "mean_token_accuracy": 0.7338063836097717,
       "num_tokens": 197728.0,
       "step": 195
     },
     {
-      "entropy": 1.1696858763694764,
+      "entropy": 1.1959772825241088,
       "epoch": 0.34904013961605584,
-      "grad_norm": 0.5466914772987366,
-      "learning_rate": 0.00013285968028419184,
-      "loss": 1.1444,
-      "mean_token_accuracy": 0.7138558447360992,
+      "grad_norm": 0.6065697073936462,
+      "learning_rate": 6.763110307414105e-05,
+      "loss": 1.1579,
+      "mean_token_accuracy": 0.7148371398448944,
       "num_tokens": 204502.0,
       "step": 200
     },
     {
-      "entropy": 1.147382140159607,
+      "entropy": 1.1796980381011963,
       "epoch": 0.35776614310645727,
-      "grad_norm": 0.8311446905136108,
-      "learning_rate": 0.0001310834813499112,
-      "loss": 1.1093,
-      "mean_token_accuracy": 0.7309025764465332,
+      "grad_norm": 0.9461066722869873,
+      "learning_rate": 6.672694394213382e-05,
+      "loss": 1.1302,
+      "mean_token_accuracy": 0.7260191440582275,
       "num_tokens": 209069.0,
       "step": 205
     },
     {
-      "entropy": 1.2201330184936523,
+      "entropy": 1.2325600683689117,
       "epoch": 0.36649214659685864,
-      "grad_norm": 0.6816751956939697,
-      "learning_rate": 0.00012930728241563054,
-      "loss": 1.2094,
-      "mean_token_accuracy": 0.7130683898925781,
+      "grad_norm": 0.8394590020179749,
+      "learning_rate": 6.582278481012658e-05,
+      "loss": 1.2181,
+      "mean_token_accuracy": 0.7085430741310119,
       "num_tokens": 214185.0,
       "step": 210
     },
     {
-      "entropy": 1.152731454372406,
+      "entropy": 1.177025467157364,
       "epoch": 0.37521815008726006,
-      "grad_norm": 0.6387792825698853,
-      "learning_rate": 0.00012753108348134993,
-      "loss": 1.0565,
-      "mean_token_accuracy": 0.7268509924411773,
+      "grad_norm": 0.7099196910858154,
+      "learning_rate": 6.491862567811935e-05,
+      "loss": 1.0698,
+      "mean_token_accuracy": 0.7251012861728668,
       "num_tokens": 219312.0,
       "step": 215
     },
     {
-      "entropy": 1.1504864931106566,
+      "entropy": 1.1909004271030426,
       "epoch": 0.38394415357766143,
-      "grad_norm": 0.7773131728172302,
-      "learning_rate": 0.00012575488454706927,
-      "loss": 1.0913,
-      "mean_token_accuracy": 0.7241075754165649,
+      "grad_norm": 0.8594046831130981,
+      "learning_rate": 6.401446654611211e-05,
+      "loss": 1.1085,
+      "mean_token_accuracy": 0.7182255506515502,
       "num_tokens": 225616.0,
       "step": 220
     },
     {
-      "entropy": 1.0282553434371948,
+      "entropy": 1.0699391067028046,
       "epoch": 0.39267015706806285,
-      "grad_norm": 0.8763700723648071,
-      "learning_rate": 0.00012397868561278864,
-      "loss": 0.9342,
-      "mean_token_accuracy": 0.7502905786037445,
+      "grad_norm": 0.9555573463439941,
+      "learning_rate": 6.311030741410488e-05,
+      "loss": 0.9639,
+      "mean_token_accuracy": 0.7441750049591065,
       "num_tokens": 230696.0,
       "step": 225
     },
     {
-      "entropy": 1.0895283699035645,
+      "entropy": 1.1240986049175263,
       "epoch": 0.4013961605584642,
-      "grad_norm": 0.8293470740318298,
-      "learning_rate": 0.000122202486678508,
-      "loss": 1.067,
-      "mean_token_accuracy": 0.7364717125892639,
+      "grad_norm": 0.9631731510162354,
+      "learning_rate": 6.220614828209765e-05,
+      "loss": 1.083,
+      "mean_token_accuracy": 0.7323605120182037,
       "num_tokens": 236685.0,
       "step": 230
     },
     {
-      "entropy": 1.172694307565689,
+      "entropy": 1.1956682801246643,
       "epoch": 0.41012216404886565,
-      "grad_norm": 0.8818181753158569,
-      "learning_rate": 0.00012042628774422735,
-      "loss": 1.0149,
-      "mean_token_accuracy": 0.7262615323066711,
+      "grad_norm": 1.0035083293914795,
+      "learning_rate": 6.130198915009042e-05,
+      "loss": 1.0327,
+      "mean_token_accuracy": 0.7194140493869782,
       "num_tokens": 241211.0,
       "step": 235
     },
     {
-      "entropy": 1.2173514723777772,
+      "entropy": 1.2125337064266204,
       "epoch": 0.418848167539267,
-      "grad_norm": 0.5635867714881897,
-      "learning_rate": 0.00011865008880994673,
-      "loss": 1.1783,
-      "mean_token_accuracy": 0.7147055625915527,
+      "grad_norm": 0.6551855802536011,
+      "learning_rate": 6.039783001808319e-05,
+      "loss": 1.1878,
+      "mean_token_accuracy": 0.7105167448520661,
       "num_tokens": 246360.0,
       "step": 240
     },
     {
-      "entropy": 1.188833224773407,
+      "entropy": 1.1998135149478912,
       "epoch": 0.42757417102966844,
-      "grad_norm": 0.6060160398483276,
-      "learning_rate": 0.00011687388987566608,
-      "loss": 1.1545,
-      "mean_token_accuracy": 0.717083477973938,
+      "grad_norm": 0.6873531341552734,
+      "learning_rate": 5.949367088607595e-05,
+      "loss": 1.1622,
+      "mean_token_accuracy": 0.7172465562820435,
       "num_tokens": 252717.0,
       "step": 245
     },
     {
-      "entropy": 1.0905582129955291,
+      "entropy": 1.1167365849018096,
       "epoch": 0.4363001745200698,
-      "grad_norm": 0.6812947988510132,
-      "learning_rate": 0.00011509769094138544,
-      "loss": 0.9922,
-      "mean_token_accuracy": 0.7299255549907684,
+      "grad_norm": 0.8184595108032227,
+      "learning_rate": 5.858951175406872e-05,
+      "loss": 1.0069,
+      "mean_token_accuracy": 0.7279561281204223,
       "num_tokens": 257249.0,
       "step": 250
     },
     {
-      "entropy": 0.8695837318897247,
+      "entropy": 0.9268409907817841,
       "epoch": 0.44502617801047123,
-      "grad_norm": 0.8577454090118408,
-      "learning_rate": 0.0001133214920071048,
-      "loss": 0.8209,
-      "mean_token_accuracy": 0.7762204229831695,
+      "grad_norm": 1.001150131225586,
+      "learning_rate": 5.768535262206148e-05,
+      "loss": 0.8418,
+      "mean_token_accuracy": 0.7676967799663543,
       "num_tokens": 262381.0,
       "step": 255
     },
     {
-      "entropy": 0.9932888269424438,
+      "entropy": 1.0302452743053436,
       "epoch": 0.4537521815008726,
-      "grad_norm": 0.697665810585022,
-      "learning_rate": 0.00011154529307282415,
-      "loss": 1.0232,
-      "mean_token_accuracy": 0.7427519500255585,
+      "grad_norm": 0.8092007040977478,
+      "learning_rate": 5.678119349005425e-05,
+      "loss": 1.0422,
+      "mean_token_accuracy": 0.7399900496006012,
       "num_tokens": 267410.0,
       "step": 260
     },
     {
-      "entropy": 0.8414939880371094,
+      "entropy": 0.8591890454292297,
       "epoch": 0.462478184991274,
-      "grad_norm": 0.789999783039093,
-      "learning_rate": 0.00010976909413854353,
-      "loss": 0.7225,
-      "mean_token_accuracy": 0.7937956035137177,
+      "grad_norm": 0.9062550067901611,
+      "learning_rate": 5.587703435804702e-05,
+      "loss": 0.7452,
+      "mean_token_accuracy": 0.7874986886978149,
       "num_tokens": 272109.0,
       "step": 265
     },
     {
-      "entropy": 1.0776531934738158,
+      "entropy": 1.0930480301380157,
       "epoch": 0.4712041884816754,
-      "grad_norm": 0.6461851000785828,
-      "learning_rate": 0.00010799289520426288,
-      "loss": 1.0389,
-      "mean_token_accuracy": 0.7343196094036102,
+      "grad_norm": 0.7568405270576477,
+      "learning_rate": 5.497287522603979e-05,
+      "loss": 1.0479,
+      "mean_token_accuracy": 0.7330868363380432,
       "num_tokens": 276623.0,
       "step": 270
     },
     {
-      "entropy": 1.1227709293365478,
+      "entropy": 1.1516533672809601,
       "epoch": 0.4799301919720768,
-      "grad_norm": 0.6017542481422424,
-      "learning_rate": 0.00010621669626998225,
-      "loss": 1.0346,
-      "mean_token_accuracy": 0.7320161819458008,
+      "grad_norm": 0.6892510652542114,
+      "learning_rate": 5.406871609403256e-05,
+      "loss": 1.0581,
+      "mean_token_accuracy": 0.7289695620536805,
       "num_tokens": 283256.0,
       "step": 275
     },
     {
-      "entropy": 0.9767000675201416,
+      "entropy": 1.0084587812423706,
       "epoch": 0.4886561954624782,
-      "grad_norm": 0.7064502835273743,
-      "learning_rate": 0.0001044404973357016,
-      "loss": 0.9051,
-      "mean_token_accuracy": 0.7693962216377258,
+      "grad_norm": 0.8208179473876953,
+      "learning_rate": 5.3164556962025316e-05,
+      "loss": 0.9185,
+      "mean_token_accuracy": 0.7670749068260193,
       "num_tokens": 288780.0,
       "step": 280
     },
     {
-      "entropy": 0.9595549941062927,
+      "entropy": 0.9972082734107971,
       "epoch": 0.4973821989528796,
-      "grad_norm": 0.7622601985931396,
-      "learning_rate": 0.00010266429840142096,
-      "loss": 0.8922,
-      "mean_token_accuracy": 0.767174756526947,
+      "grad_norm": 0.917007565498352,
+      "learning_rate": 5.2260397830018085e-05,
+      "loss": 0.9135,
+      "mean_token_accuracy": 0.7594508528709412,
       "num_tokens": 293775.0,
       "step": 285
     },
     {
-      "entropy": 0.9456490218639374,
+      "entropy": 0.9781757235527039,
       "epoch": 0.506108202443281,
-      "grad_norm": 0.7910531163215637,
-      "learning_rate": 0.00010088809946714034,
-      "loss": 0.8845,
-      "mean_token_accuracy": 0.7625713229179383,
+      "grad_norm": 0.8585111498832703,
+      "learning_rate": 5.135623869801085e-05,
+      "loss": 0.9062,
+      "mean_token_accuracy": 0.7560348510742188,
       "num_tokens": 299667.0,
       "step": 290
     },
     {
-      "entropy": 0.9972454011440277,
+      "entropy": 1.0113820374011993,
       "epoch": 0.5148342059336823,
-      "grad_norm": 0.8077422976493835,
-      "learning_rate": 9.911190053285967e-05,
-      "loss": 0.9629,
-      "mean_token_accuracy": 0.7550196409225464,
+      "grad_norm": 0.9319034814834595,
+      "learning_rate": 5.045207956600362e-05,
+      "loss": 0.9816,
+      "mean_token_accuracy": 0.7550252437591553,
       "num_tokens": 304401.0,
       "step": 295
     },
     {
-      "entropy": 1.0132270872592926,
+      "entropy": 1.0391628086566924,
       "epoch": 0.5235602094240838,
-      "grad_norm": 0.5776278972625732,
-      "learning_rate": 9.733570159857904e-05,
-      "loss": 0.9083,
-      "mean_token_accuracy": 0.7645319044589997,
+      "grad_norm": 0.6635801196098328,
+      "learning_rate": 4.954792043399639e-05,
+      "loss": 0.921,
+      "mean_token_accuracy": 0.7602339625358582,
       "num_tokens": 310983.0,
       "step": 300
     },
     {
-      "entropy": 1.1321196973323822,
+      "entropy": 1.1690425276756287,
       "epoch": 0.5322862129144852,
-      "grad_norm": 0.765808641910553,
-      "learning_rate": 9.555950266429841e-05,
-      "loss": 1.0364,
-      "mean_token_accuracy": 0.7226320803165436,
+      "grad_norm": 0.8804797530174255,
+      "learning_rate": 4.864376130198916e-05,
+      "loss": 1.0541,
+      "mean_token_accuracy": 0.7173005819320679,
       "num_tokens": 315721.0,
       "step": 305
     },
     {
-      "entropy": 1.0132107377052306,
+      "entropy": 1.0556828916072845,
       "epoch": 0.5410122164048866,
-      "grad_norm": 0.5765398144721985,
-      "learning_rate": 9.378330373001777e-05,
-      "loss": 0.9858,
-      "mean_token_accuracy": 0.7562039911746978,
+      "grad_norm": 0.7090078592300415,
+      "learning_rate": 4.773960216998192e-05,
+      "loss": 1.0045,
+      "mean_token_accuracy": 0.7494379639625549,
       "num_tokens": 321834.0,
       "step": 310
     },
     {
-      "entropy": 1.097977089881897,
+      "entropy": 1.1323361456394196,
       "epoch": 0.5497382198952879,
-      "grad_norm": 0.7264753580093384,
-      "learning_rate": 9.200710479573713e-05,
-      "loss": 1.0686,
-      "mean_token_accuracy": 0.7291842579841614,
+      "grad_norm": 0.8676872253417969,
+      "learning_rate": 4.683544303797468e-05,
+      "loss": 1.0833,
+      "mean_token_accuracy": 0.7251208662986756,
       "num_tokens": 327063.0,
       "step": 315
     },
     {
-      "entropy": 1.2174109816551208,
+      "entropy": 1.2482255697250366,
       "epoch": 0.5584642233856894,
-      "grad_norm": 0.7541456818580627,
-      "learning_rate": 9.023090586145648e-05,
-      "loss": 1.1817,
-      "mean_token_accuracy": 0.7097965478897095,
+      "grad_norm": 0.8192646503448486,
+      "learning_rate": 4.593128390596745e-05,
+      "loss": 1.2016,
+      "mean_token_accuracy": 0.7059409141540527,
       "num_tokens": 332900.0,
       "step": 320
     },
     {
-      "entropy": 1.0044541895389556,
+      "entropy": 1.0314257562160491,
       "epoch": 0.5671902268760908,
-      "grad_norm": 0.5834890604019165,
-      "learning_rate": 8.845470692717585e-05,
-      "loss": 0.9467,
-      "mean_token_accuracy": 0.7500465452671051,
+      "grad_norm": 0.6856659650802612,
+      "learning_rate": 4.5027124773960215e-05,
+      "loss": 0.9671,
+      "mean_token_accuracy": 0.7470330059528351,
       "num_tokens": 337508.0,
       "step": 325
     },
     {
-      "entropy": 1.0295350253582,
+      "entropy": 1.060635393857956,
       "epoch": 0.5759162303664922,
-      "grad_norm": 0.8909983038902283,
-      "learning_rate": 8.667850799289521e-05,
-      "loss": 0.9113,
-      "mean_token_accuracy": 0.7476867496967315,
+      "grad_norm": 1.0749919414520264,
+      "learning_rate": 4.4122965641952984e-05,
+      "loss": 0.9374,
+      "mean_token_accuracy": 0.74344761967659,
       "num_tokens": 342644.0,
       "step": 330
     },
     {
-      "entropy": 1.0791299104690553,
+      "entropy": 1.1081909716129303,
       "epoch": 0.5846422338568935,
-      "grad_norm": 1.0385737419128418,
-      "learning_rate": 8.490230905861456e-05,
-      "loss": 1.1175,
-      "mean_token_accuracy": 0.7305109918117523,
+      "grad_norm": 1.2220737934112549,
+      "learning_rate": 4.3218806509945754e-05,
+      "loss": 1.1298,
+      "mean_token_accuracy": 0.7298382222652435,
       "num_tokens": 347547.0,
       "step": 335
     },
     {
-      "entropy": 1.0213176369667054,
+      "entropy": 1.0464669644832612,
       "epoch": 0.5933682373472949,
-      "grad_norm": 0.943204402923584,
-      "learning_rate": 8.312611012433393e-05,
-      "loss": 0.9055,
-      "mean_token_accuracy": 0.7596513092517853,
+      "grad_norm": 1.0865596532821655,
+      "learning_rate": 4.2314647377938523e-05,
+      "loss": 0.9198,
+      "mean_token_accuracy": 0.7569529414176941,
       "num_tokens": 351932.0,
       "step": 340
     },
     {
-      "entropy": 1.0257258594036103,
+      "entropy": 1.062287026643753,
       "epoch": 0.6020942408376964,
-      "grad_norm": 0.7949322462081909,
-      "learning_rate": 8.134991119005328e-05,
-      "loss": 0.9098,
-      "mean_token_accuracy": 0.7553630173206329,
+      "grad_norm": 0.9347847104072571,
+      "learning_rate": 4.1410488245931286e-05,
+      "loss": 0.9329,
+      "mean_token_accuracy": 0.7522122919559479,
       "num_tokens": 357045.0,
       "step": 345
     },
     {
-      "entropy": 1.0372248589992523,
+      "entropy": 1.0700653612613678,
       "epoch": 0.6108202443280978,
-      "grad_norm": 0.8405324220657349,
-      "learning_rate": 7.957371225577265e-05,
-      "loss": 0.9929,
-      "mean_token_accuracy": 0.7452831089496612,
+      "grad_norm": 0.9848562479019165,
+      "learning_rate": 4.050632911392405e-05,
+      "loss": 1.0067,
+      "mean_token_accuracy": 0.7463895082473755,
       "num_tokens": 362284.0,
       "step": 350
     },
     {
-      "entropy": 0.9565088748931885,
+      "entropy": 0.9772404789924621,
       "epoch": 0.6195462478184991,
-      "grad_norm": 0.6379778981208801,
-      "learning_rate": 7.779751332149202e-05,
-      "loss": 0.9219,
-      "mean_token_accuracy": 0.7565369844436646,
+      "grad_norm": 0.7483623623847961,
+      "learning_rate": 3.960216998191682e-05,
+      "loss": 0.9363,
+      "mean_token_accuracy": 0.7513622403144836,
       "num_tokens": 367217.0,
       "step": 355
     },
     {
-      "entropy": 1.0628814578056336,
+      "entropy": 1.0971436262130738,
       "epoch": 0.6282722513089005,
-      "grad_norm": 0.6335421204566956,
-      "learning_rate": 7.602131438721137e-05,
-      "loss": 1.0041,
-      "mean_token_accuracy": 0.7395376443862915,
+      "grad_norm": 0.7464447021484375,
+      "learning_rate": 3.869801084990959e-05,
+      "loss": 1.0344,
+      "mean_token_accuracy": 0.7349282145500183,
       "num_tokens": 372678.0,
       "step": 360
     },
     {
-      "entropy": 0.9448712587356567,
+      "entropy": 0.9754513740539551,
       "epoch": 0.6369982547993019,
-      "grad_norm": 0.737162172794342,
-      "learning_rate": 7.424511545293074e-05,
-      "loss": 0.8211,
-      "mean_token_accuracy": 0.771143788099289,
+      "grad_norm": 0.8585572838783264,
+      "learning_rate": 3.779385171790235e-05,
+      "loss": 0.8401,
+      "mean_token_accuracy": 0.7676230728626251,
       "num_tokens": 377750.0,
       "step": 365
     },
     {
-      "entropy": 0.9797238111495972,
+      "entropy": 1.0239232301712036,
       "epoch": 0.6457242582897034,
-      "grad_norm": 0.5577957034111023,
-      "learning_rate": 7.246891651865009e-05,
-      "loss": 0.9415,
-      "mean_token_accuracy": 0.7499814212322236,
+      "grad_norm": 0.6581646203994751,
+      "learning_rate": 3.688969258589512e-05,
+      "loss": 0.9598,
+      "mean_token_accuracy": 0.7424759864807129,
       "num_tokens": 383406.0,
       "step": 370
     },
     {
-      "entropy": 1.1891680419445039,
+      "entropy": 1.2168250560760498,
       "epoch": 0.6544502617801047,
-      "grad_norm": 0.48097750544548035,
-      "learning_rate": 7.069271758436945e-05,
-      "loss": 1.1327,
-      "mean_token_accuracy": 0.7193056166172027,
+      "grad_norm": 0.5685426592826843,
+      "learning_rate": 3.598553345388789e-05,
+      "loss": 1.1598,
+      "mean_token_accuracy": 0.7155482172966003,
       "num_tokens": 389696.0,
       "step": 375
     },
     {
-      "entropy": 1.0238433182239532,
+      "entropy": 1.0621464610099793,
       "epoch": 0.6631762652705061,
-      "grad_norm": 0.5823986530303955,
-      "learning_rate": 6.891651865008881e-05,
-      "loss": 0.9708,
-      "mean_token_accuracy": 0.7535522282123566,
+      "grad_norm": 0.6709677577018738,
+      "learning_rate": 3.508137432188065e-05,
+      "loss": 0.9943,
+      "mean_token_accuracy": 0.7490235090255737,
       "num_tokens": 394688.0,
       "step": 380
     },
     {
-      "entropy": 1.162860244512558,
+      "entropy": 1.1910510420799256,
       "epoch": 0.6719022687609075,
-      "grad_norm": 0.6299170255661011,
-      "learning_rate": 6.714031971580817e-05,
-      "loss": 1.1866,
-      "mean_token_accuracy": 0.710206264257431,
+      "grad_norm": 0.7342821359634399,
+      "learning_rate": 3.4177215189873416e-05,
+      "loss": 1.2074,
+      "mean_token_accuracy": 0.7063271880149842,
       "num_tokens": 400319.0,
       "step": 385
     },
     {
-      "entropy": 1.0206872344017028,
+      "entropy": 1.053422886133194,
       "epoch": 0.680628272251309,
-      "grad_norm": 0.7722362875938416,
-      "learning_rate": 6.536412078152754e-05,
-      "loss": 0.9289,
-      "mean_token_accuracy": 0.7554251432418824,
+      "grad_norm": 0.9014110565185547,
+      "learning_rate": 3.3273056057866185e-05,
+      "loss": 0.957,
+      "mean_token_accuracy": 0.7507923722267151,
       "num_tokens": 404918.0,
       "step": 390
     },
     {
-      "entropy": 1.0980794131755829,
+      "entropy": 1.12480788230896,
       "epoch": 0.6893542757417103,
-      "grad_norm": 0.9234552979469299,
-      "learning_rate": 6.358792184724689e-05,
-      "loss": 0.9551,
-      "mean_token_accuracy": 0.7426558673381806,
+      "grad_norm": 1.1246176958084106,
+      "learning_rate": 3.2368896925858955e-05,
+      "loss": 0.9808,
+      "mean_token_accuracy": 0.735921996831894,
       "num_tokens": 410635.0,
       "step": 395
     },
     {
-      "entropy": 1.0166767477989196,
+      "entropy": 1.0456868290901185,
       "epoch": 0.6980802792321117,
-      "grad_norm": 0.9343558549880981,
-      "learning_rate": 6.181172291296625e-05,
-      "loss": 0.9624,
-      "mean_token_accuracy": 0.7539155185222626,
+      "grad_norm": 1.0994240045547485,
+      "learning_rate": 3.146473779385172e-05,
+      "loss": 0.9792,
+      "mean_token_accuracy": 0.7470909178256988,
       "num_tokens": 415005.0,
       "step": 400
     },
     {
-      "entropy": 1.0832793176174165,
+      "entropy": 1.1365525662899016,
       "epoch": 0.7068062827225131,
-      "grad_norm": 0.7815644145011902,
-      "learning_rate": 6.003552397868561e-05,
-      "loss": 1.0316,
-      "mean_token_accuracy": 0.7289174854755401,
+      "grad_norm": 0.9170509576797485,
+      "learning_rate": 3.056057866184449e-05,
+      "loss": 1.0618,
+      "mean_token_accuracy": 0.7218761801719665,
       "num_tokens": 419347.0,
       "step": 405
     },
     {
-      "entropy": 1.0699054658412934,
+      "entropy": 1.1176020860671998,
       "epoch": 0.7155322862129145,
-      "grad_norm": 0.7760159373283386,
-      "learning_rate": 5.825932504440498e-05,
-      "loss": 1.0357,
-      "mean_token_accuracy": 0.7321902751922608,
+      "grad_norm": 0.9267513155937195,
+      "learning_rate": 2.9656419529837253e-05,
+      "loss": 1.057,
+      "mean_token_accuracy": 0.7263721942901611,
       "num_tokens": 424588.0,
       "step": 410
     },
     {
-      "entropy": 0.966323298215866,
+      "entropy": 0.997305154800415,
       "epoch": 0.7242582897033158,
-      "grad_norm": 0.805746853351593,
-      "learning_rate": 5.648312611012434e-05,
-      "loss": 0.9306,
-      "mean_token_accuracy": 0.7569182515144348,
+      "grad_norm": 1.025118350982666,
+      "learning_rate": 2.8752260397830023e-05,
+      "loss": 0.9536,
+      "mean_token_accuracy": 0.7515883207321167,
       "num_tokens": 428943.0,
       "step": 415
     },
     {
-      "entropy": 0.9721911072731018,
+      "entropy": 1.0053665816783905,
       "epoch": 0.7329842931937173,
-      "grad_norm": 0.6620533466339111,
-      "learning_rate": 5.470692717584369e-05,
-      "loss": 0.9465,
-      "mean_token_accuracy": 0.7597197592258453,
+      "grad_norm": 0.7526717782020569,
+      "learning_rate": 2.7848101265822786e-05,
+      "loss": 0.9632,
+      "mean_token_accuracy": 0.7552886366844177,
       "num_tokens": 435326.0,
       "step": 420
     },
     {
-      "entropy": 0.9292757451534271,
+      "entropy": 0.967602401971817,
       "epoch": 0.7417102966841187,
-      "grad_norm": 0.7177068591117859,
-      "learning_rate": 5.293072824156306e-05,
-      "loss": 0.858,
-      "mean_token_accuracy": 0.7738080501556397,
+      "grad_norm": 0.8893188834190369,
+      "learning_rate": 2.6943942133815552e-05,
+      "loss": 0.8872,
+      "mean_token_accuracy": 0.7692794561386108,
       "num_tokens": 441702.0,
       "step": 425
     },
     {
-      "entropy": 1.0638712823390961,
+      "entropy": 1.099077934026718,
       "epoch": 0.7504363001745201,
-      "grad_norm": 0.5912255048751831,
-      "learning_rate": 5.115452930728242e-05,
-      "loss": 1.0654,
-      "mean_token_accuracy": 0.747636479139328,
+      "grad_norm": 0.7207973599433899,
+      "learning_rate": 2.603978300180832e-05,
+      "loss": 1.0975,
+      "mean_token_accuracy": 0.7474483132362366,
       "num_tokens": 446862.0,
       "step": 430
     },
     {
-      "entropy": 0.9203409194946289,
+      "entropy": 0.9622289180755615,
       "epoch": 0.7591623036649214,
-      "grad_norm": 0.8877400159835815,
-      "learning_rate": 4.9378330373001777e-05,
-      "loss": 0.8225,
-      "mean_token_accuracy": 0.7788766026496887,
+      "grad_norm": 1.0675289630889893,
+      "learning_rate": 2.5135623869801084e-05,
+      "loss": 0.8535,
+      "mean_token_accuracy": 0.7668149411678314,
       "num_tokens": 451024.0,
       "step": 435
     },
     {
-      "entropy": 1.0310194969177247,
+      "entropy": 1.0761532068252564,
       "epoch": 0.7678883071553229,
-      "grad_norm": 0.593137800693512,
-      "learning_rate": 4.7602131438721136e-05,
-      "loss": 1.0058,
-      "mean_token_accuracy": 0.7474644720554352,
+      "grad_norm": 0.7296017408370972,
+      "learning_rate": 2.423146473779385e-05,
+      "loss": 1.0343,
+      "mean_token_accuracy": 0.744589650630951,
       "num_tokens": 457528.0,
       "step": 440
     },
     {
-      "entropy": 0.9218507647514343,
+      "entropy": 0.9684066355228425,
       "epoch": 0.7766143106457243,
-      "grad_norm": 0.8034109473228455,
-      "learning_rate": 4.58259325044405e-05,
-      "loss": 0.8161,
-      "mean_token_accuracy": 0.773482757806778,
+      "grad_norm": 0.931486189365387,
+      "learning_rate": 2.332730560578662e-05,
+      "loss": 0.8462,
+      "mean_token_accuracy": 0.7654486835002899,
       "num_tokens": 462267.0,
       "step": 445
     },
     {
-      "entropy": 1.0368493318557739,
+      "entropy": 1.082474511861801,
       "epoch": 0.7853403141361257,
-      "grad_norm": 0.9129230380058289,
-      "learning_rate": 4.404973357015986e-05,
-      "loss": 1.0042,
-      "mean_token_accuracy": 0.7518712699413299,
+      "grad_norm": 0.95744389295578,
+      "learning_rate": 2.2423146473779386e-05,
+      "loss": 1.0284,
+      "mean_token_accuracy": 0.7476698577404022,
       "num_tokens": 467337.0,
       "step": 450
     },
     {
-      "entropy": 0.8776600241661072,
+      "entropy": 0.9113831341266632,
       "epoch": 0.794066317626527,
-      "grad_norm": 0.5392698645591736,
-      "learning_rate": 4.227353463587922e-05,
-      "loss": 0.7964,
-      "mean_token_accuracy": 0.773613715171814,
+      "grad_norm": 0.6706241965293884,
+      "learning_rate": 2.1518987341772153e-05,
+      "loss": 0.8162,
+      "mean_token_accuracy": 0.7696994006633758,
       "num_tokens": 472361.0,
       "step": 455
     },
     {
-      "entropy": 0.9013674080371856,
+      "entropy": 0.932946115732193,
       "epoch": 0.8027923211169284,
-      "grad_norm": 0.731060266494751,
-      "learning_rate": 4.049733570159858e-05,
-      "loss": 0.9098,
-      "mean_token_accuracy": 0.7663923025131225,
+      "grad_norm": 0.8883486390113831,
+      "learning_rate": 2.061482820976492e-05,
+      "loss": 0.9311,
+      "mean_token_accuracy": 0.761142897605896,
       "num_tokens": 477324.0,
       "step": 460
     },
     {
-      "entropy": 1.0141965687274932,
+      "entropy": 1.0308326423168181,
       "epoch": 0.8115183246073299,
-      "grad_norm": 0.6941847205162048,
-      "learning_rate": 3.872113676731794e-05,
-      "loss": 1.0052,
-      "mean_token_accuracy": 0.747931432723999,
+      "grad_norm": 0.8458265066146851,
+      "learning_rate": 1.971066907775769e-05,
+      "loss": 1.0271,
+      "mean_token_accuracy": 0.74349946975708,
       "num_tokens": 483192.0,
       "step": 465
     },
     {
-      "entropy": 0.9370434999465942,
+      "entropy": 0.957154580950737,
       "epoch": 0.8202443280977313,
-      "grad_norm": 0.7024611830711365,
-      "learning_rate": 3.69449378330373e-05,
-      "loss": 0.9472,
-      "mean_token_accuracy": 0.7648843646049499,
+      "grad_norm": 0.8496681451797485,
+      "learning_rate": 1.8806509945750454e-05,
+      "loss": 0.9683,
+      "mean_token_accuracy": 0.7599529087543487,
       "num_tokens": 488771.0,
       "step": 470
     },
     {
-      "entropy": 1.2138389825820923,
+      "entropy": 1.2366232931613923,
       "epoch": 0.8289703315881326,
-      "grad_norm": 0.6181853413581848,
-      "learning_rate": 3.516873889875667e-05,
-      "loss": 1.1913,
-      "mean_token_accuracy": 0.7199933648109436,
+      "grad_norm": 0.736077606678009,
+      "learning_rate": 1.7902350813743217e-05,
+      "loss": 1.2093,
+      "mean_token_accuracy": 0.7199373066425323,
       "num_tokens": 495594.0,
       "step": 475
     },
     {
-      "entropy": 0.9946802318096161,
+      "entropy": 1.0375137269496917,
       "epoch": 0.837696335078534,
-      "grad_norm": 0.8392300009727478,
-      "learning_rate": 3.339253996447602e-05,
-      "loss": 0.8846,
-      "mean_token_accuracy": 0.7601681053638458,
+      "grad_norm": 1.0149139165878296,
+      "learning_rate": 1.6998191681735987e-05,
+      "loss": 0.9121,
+      "mean_token_accuracy": 0.7566475987434387,
       "num_tokens": 501431.0,
       "step": 480
     },
     {
-      "entropy": 1.0851561069488525,
+      "entropy": 1.119648665189743,
       "epoch": 0.8464223385689355,
-      "grad_norm": 0.7538084983825684,
-      "learning_rate": 3.1616341030195386e-05,
-      "loss": 1.0112,
-      "mean_token_accuracy": 0.7339279770851135,
+      "grad_norm": 0.9202610850334167,
+      "learning_rate": 1.6094032549728753e-05,
+      "loss": 1.0352,
+      "mean_token_accuracy": 0.7305955648422241,
       "num_tokens": 506603.0,
       "step": 485
     },
     {
-      "entropy": 0.9791876435279846,
+      "entropy": 1.023408180475235,
       "epoch": 0.8551483420593369,
-      "grad_norm": 0.6512478590011597,
-      "learning_rate": 2.9840142095914742e-05,
-      "loss": 0.9047,
-      "mean_token_accuracy": 0.7637781441211701,
+      "grad_norm": 0.793682873249054,
+      "learning_rate": 1.5189873417721521e-05,
+      "loss": 0.9317,
+      "mean_token_accuracy": 0.7577860534191132,
       "num_tokens": 511657.0,
       "step": 490
     },
     {
-      "entropy": 0.8807009816169739,
+      "entropy": 0.9122473716735839,
       "epoch": 0.8638743455497382,
-      "grad_norm": 1.0381275415420532,
-      "learning_rate": 2.8063943161634105e-05,
-      "loss": 0.7989,
-      "mean_token_accuracy": 0.778300940990448,
+      "grad_norm": 1.3430960178375244,
+      "learning_rate": 1.4285714285714285e-05,
+      "loss": 0.829,
+      "mean_token_accuracy": 0.7760975360870361,
       "num_tokens": 516346.0,
       "step": 495
     },
     {
-      "entropy": 0.9706099390983581,
+      "entropy": 1.0150888204574584,
       "epoch": 0.8726003490401396,
-      "grad_norm": 0.7503977417945862,
-      "learning_rate": 2.6287744227353468e-05,
-      "loss": 0.8633,
-      "mean_token_accuracy": 0.7602749288082122,
+      "grad_norm": 0.8981541395187378,
+      "learning_rate": 1.3381555153707053e-05,
+      "loss": 0.8899,
+      "mean_token_accuracy": 0.7577051818370819,
       "num_tokens": 521118.0,
       "step": 500
     },
     {
-      "entropy": 0.9830702662467956,
+      "entropy": 1.0423671543598174,
       "epoch": 0.881326352530541,
-      "grad_norm": 0.7824010252952576,
-      "learning_rate": 2.4511545293072824e-05,
-      "loss": 0.8701,
-      "mean_token_accuracy": 0.7697367370128632,
+      "grad_norm": 0.9293156862258911,
+      "learning_rate": 1.247739602169982e-05,
+      "loss": 0.9083,
+      "mean_token_accuracy": 0.7593627750873566,
       "num_tokens": 525785.0,
       "step": 505
     },
     {
-      "entropy": 1.0895603597164154,
+      "entropy": 1.1302937150001526,
       "epoch": 0.8900523560209425,
-      "grad_norm": 0.6201509237289429,
-      "learning_rate": 2.2735346358792187e-05,
-      "loss": 0.999,
-      "mean_token_accuracy": 0.7415844857692718,
+      "grad_norm": 0.7647919058799744,
+      "learning_rate": 1.1573236889692586e-05,
+      "loss": 1.0402,
+      "mean_token_accuracy": 0.7308323442935943,
       "num_tokens": 531296.0,
       "step": 510
     },
     {
-      "entropy": 1.0094242215156555,
+      "entropy": 1.0633261859416963,
       "epoch": 0.8987783595113438,
-      "grad_norm": 0.6755935549736023,
-      "learning_rate": 2.0959147424511547e-05,
-      "loss": 0.9283,
-      "mean_token_accuracy": 0.7551429510116577,
+      "grad_norm": 0.8269041776657104,
+      "learning_rate": 1.0669077757685354e-05,
+      "loss": 0.9716,
+      "mean_token_accuracy": 0.7478925228118897,
       "num_tokens": 536703.0,
       "step": 515
     },
     {
-      "entropy": 0.9846092760562897,
+      "entropy": 1.024457675218582,
       "epoch": 0.9075043630017452,
-      "grad_norm": 1.0709046125411987,
-      "learning_rate": 1.9182948490230906e-05,
-      "loss": 0.9426,
-      "mean_token_accuracy": 0.7431533575057984,
+      "grad_norm": 1.324475646018982,
+      "learning_rate": 9.76491862567812e-06,
+      "loss": 0.9741,
+      "mean_token_accuracy": 0.7305623233318329,
       "num_tokens": 541044.0,
       "step": 520
     },
     {
-      "entropy": 0.9527219116687775,
+      "entropy": 1.0003005802631377,
       "epoch": 0.9162303664921466,
-      "grad_norm": 0.6978484392166138,
-      "learning_rate": 1.7406749555950266e-05,
-      "loss": 0.8911,
-      "mean_token_accuracy": 0.7653971970081329,
+      "grad_norm": 0.9192453622817993,
+      "learning_rate": 8.860759493670886e-06,
+      "loss": 0.9253,
+      "mean_token_accuracy": 0.7594391465187073,
       "num_tokens": 546836.0,
       "step": 525
     },
     {
-      "entropy": 0.8855733275413513,
+      "entropy": 0.9267178893089294,
       "epoch": 0.924956369982548,
-      "grad_norm": 0.9127820134162903,
-      "learning_rate": 1.563055062166963e-05,
-      "loss": 0.8139,
-      "mean_token_accuracy": 0.7775610208511352,
+      "grad_norm": 1.1192840337753296,
+      "learning_rate": 7.956600361663654e-06,
+      "loss": 0.849,
+      "mean_token_accuracy": 0.7724036395549774,
       "num_tokens": 551666.0,
       "step": 530
     },
     {
-      "entropy": 0.9590709805488586,
+      "entropy": 0.9956534147262573,
       "epoch": 0.9336823734729494,
-      "grad_norm": 0.7010323405265808,
-      "learning_rate": 1.3854351687388988e-05,
-      "loss": 0.9334,
-      "mean_token_accuracy": 0.759455144405365,
+      "grad_norm": 0.8383934497833252,
+      "learning_rate": 7.05244122965642e-06,
+      "loss": 0.9602,
+      "mean_token_accuracy": 0.7577574133872986,
       "num_tokens": 556932.0,
       "step": 535
     },
     {
-      "entropy": 0.9646609544754028,
+      "entropy": 1.0035830855369567,
       "epoch": 0.9424083769633508,
-      "grad_norm": 0.5711817145347595,
-      "learning_rate": 1.2078152753108348e-05,
-      "loss": 0.9678,
-      "mean_token_accuracy": 0.7603480279445648,
+      "grad_norm": 0.6870363354682922,
+      "learning_rate": 6.148282097649186e-06,
+      "loss": 0.9922,
+      "mean_token_accuracy": 0.753506338596344,
       "num_tokens": 562608.0,
       "step": 540
     },
     {
-      "entropy": 1.0106851994991302,
+      "entropy": 1.0436702132225038,
       "epoch": 0.9511343804537522,
-      "grad_norm": 0.7159616947174072,
-      "learning_rate": 1.030195381882771e-05,
-      "loss": 0.9285,
-      "mean_token_accuracy": 0.7571455597877502,
+      "grad_norm": 0.8812151551246643,
+      "learning_rate": 5.244122965641953e-06,
+      "loss": 0.9626,
+      "mean_token_accuracy": 0.7489932656288147,
       "num_tokens": 568591.0,
       "step": 545
     },
     {
-      "entropy": 1.0865988105535507,
+      "entropy": 1.123832467198372,
       "epoch": 0.9598603839441536,
-      "grad_norm": 0.7819423079490662,
-      "learning_rate": 8.52575488454707e-06,
-      "loss": 1.1628,
-      "mean_token_accuracy": 0.7484253525733948,
+      "grad_norm": 0.9656490683555603,
+      "learning_rate": 4.33996383363472e-06,
+      "loss": 1.2141,
+      "mean_token_accuracy": 0.7377211570739746,
       "num_tokens": 572932.0,
       "step": 550
     },
     {
-      "entropy": 0.8808701932430267,
+      "entropy": 0.9230763018131256,
       "epoch": 0.9685863874345549,
-      "grad_norm": 0.6782775521278381,
-      "learning_rate": 6.74955595026643e-06,
-      "loss": 0.7661,
-      "mean_token_accuracy": 0.7774575710296631,
+      "grad_norm": 0.8448044657707214,
+      "learning_rate": 3.435804701627487e-06,
+      "loss": 0.8004,
+      "mean_token_accuracy": 0.7740457057952881,
       "num_tokens": 577818.0,
       "step": 555
     },
     {
-      "entropy": 0.9287233471870422,
+      "entropy": 0.9738658726215362,
       "epoch": 0.9773123909249564,
-      "grad_norm": 0.8206584453582764,
-      "learning_rate": 4.973357015985791e-06,
-      "loss": 0.7572,
-      "mean_token_accuracy": 0.7789205074310303,
+      "grad_norm": 1.040029764175415,
+      "learning_rate": 2.531645569620253e-06,
+      "loss": 0.7951,
+      "mean_token_accuracy": 0.7707339942455291,
       "num_tokens": 581899.0,
       "step": 560
     },
     {
-      "entropy": 0.8616821765899658,
+      "entropy": 0.9029529750347137,
       "epoch": 0.9860383944153578,
-      "grad_norm": 0.6403858661651611,
-      "learning_rate": 3.197158081705151e-06,
-      "loss": 0.7855,
-      "mean_token_accuracy": 0.7908896625041961,
+      "grad_norm": 0.777828574180603,
+      "learning_rate": 1.62748643761302e-06,
+      "loss": 0.8196,
+      "mean_token_accuracy": 0.7824884414672851,
       "num_tokens": 587864.0,
       "step": 565
     },
     {
-      "entropy": 1.0378393054008483,
+      "entropy": 1.0749013006687165,
       "epoch": 0.9947643979057592,
-      "grad_norm": 0.7347800731658936,
-      "learning_rate": 1.4209591474245117e-06,
-      "loss": 1.0399,
-      "mean_token_accuracy": 0.7526734173297882,
+      "grad_norm": 0.9190181493759155,
+      "learning_rate": 7.233273056057866e-07,
+      "loss": 1.0676,
+      "mean_token_accuracy": 0.7512221932411194,
       "num_tokens": 592990.0,
       "step": 570
     }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d49ed87fd1007ddba65a781a7a824d4db6222aa26b1008b2e988302b8cec8fab
+oid sha256:c4982b9ed04996dd5f6e1133823637f4dc00aad549bfd88090393c4ca029c70b
 size 5816