{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 3750,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.04,
      "grad_norm": 39.1501579284668,
      "learning_rate": 1.3066666666666667e-06,
      "logits/chosen": 2.069579839706421,
      "logits/rejected": 2.1242542266845703,
      "logps/chosen": -73.9653091430664,
      "logps/rejected": -88.28418731689453,
      "loss": 4.8463,
      "nll_loss": 0.46437376737594604,
      "rewards/accuracies": 0.7049999833106995,
      "rewards/chosen": -7.396530628204346,
      "rewards/margins": 1.431888461112976,
      "rewards/rejected": -8.828418731689453,
      "step": 50
    },
    {
      "epoch": 0.08,
      "grad_norm": 23.26362419128418,
      "learning_rate": 2.64e-06,
      "logits/chosen": 2.039077043533325,
      "logits/rejected": 2.1060538291931152,
      "logps/chosen": -57.755859375,
      "logps/rejected": -72.76531982421875,
      "loss": 3.8951,
      "nll_loss": 0.35950180888175964,
      "rewards/accuracies": 0.7112500071525574,
      "rewards/chosen": -5.77558708190918,
      "rewards/margins": 1.5009456872940063,
      "rewards/rejected": -7.276532173156738,
      "step": 100
    },
    {
      "epoch": 0.12,
      "grad_norm": 29.71898078918457,
      "learning_rate": 3.973333333333333e-06,
      "logits/chosen": 2.017021894454956,
      "logits/rejected": 2.063059091567993,
      "logps/chosen": -52.110809326171875,
      "logps/rejected": -63.39885711669922,
      "loss": 3.5484,
      "nll_loss": 0.3122263252735138,
      "rewards/accuracies": 0.6600000262260437,
      "rewards/chosen": -5.211080551147461,
      "rewards/margins": 1.1288050413131714,
      "rewards/rejected": -6.339885711669922,
      "step": 150
    },
    {
      "epoch": 0.16,
      "grad_norm": 21.129802703857422,
      "learning_rate": 5.306666666666667e-06,
      "logits/chosen": 2.0352795124053955,
      "logits/rejected": 2.09993052482605,
      "logps/chosen": -46.72736740112305,
      "logps/rejected": -59.81150436401367,
      "loss": 3.1762,
      "nll_loss": 0.2814980447292328,
      "rewards/accuracies": 0.7087500095367432,
      "rewards/chosen": -4.672736644744873,
      "rewards/margins": 1.308414340019226,
      "rewards/rejected": -5.9811506271362305,
      "step": 200
    },
    {
      "epoch": 0.2,
      "grad_norm": 25.128114700317383,
      "learning_rate": 6.640000000000001e-06,
      "logits/chosen": 2.0161619186401367,
      "logits/rejected": 2.080204725265503,
      "logps/chosen": -46.92048263549805,
      "logps/rejected": -59.91488265991211,
      "loss": 3.0645,
      "nll_loss": 0.2713809609413147,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -4.6920485496521,
      "rewards/margins": 1.2994396686553955,
      "rewards/rejected": -5.991488456726074,
      "step": 250
    },
    {
      "epoch": 0.24,
      "grad_norm": 22.990211486816406,
      "learning_rate": 7.973333333333334e-06,
      "logits/chosen": 2.006951332092285,
      "logits/rejected": 2.0681934356689453,
      "logps/chosen": -43.61228942871094,
      "logps/rejected": -58.394710540771484,
      "loss": 2.8512,
      "nll_loss": 0.2565268874168396,
      "rewards/accuracies": 0.7512500286102295,
      "rewards/chosen": -4.361229419708252,
      "rewards/margins": 1.4782426357269287,
      "rewards/rejected": -5.839471817016602,
      "step": 300
    },
    {
      "epoch": 0.28,
      "grad_norm": 17.802993774414062,
      "learning_rate": 9.306666666666667e-06,
      "logits/chosen": 1.9981739521026611,
      "logits/rejected": 2.056286334991455,
      "logps/chosen": -42.770790100097656,
      "logps/rejected": -53.638092041015625,
      "loss": 3.0075,
      "nll_loss": 0.25887373089790344,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -4.2770795822143555,
      "rewards/margins": 1.0867302417755127,
      "rewards/rejected": -5.363808631896973,
      "step": 350
    },
    {
      "epoch": 0.32,
      "grad_norm": 21.133516311645508,
      "learning_rate": 9.998752338940612e-06,
      "logits/chosen": 1.876654028892517,
      "logits/rejected": 1.9367033243179321,
      "logps/chosen": -38.542076110839844,
      "logps/rejected": -48.9306755065918,
      "loss": 2.6798,
      "nll_loss": 0.22952309250831604,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -3.8542075157165527,
      "rewards/margins": 1.0388593673706055,
      "rewards/rejected": -4.893066883087158,
      "step": 400
    },
    {
      "epoch": 0.36,
      "grad_norm": 23.868408203125,
      "learning_rate": 9.988142751731797e-06,
      "logits/chosen": 1.7912520170211792,
      "logits/rejected": 1.8582316637039185,
      "logps/chosen": -38.205177307128906,
      "logps/rejected": -51.677650451660156,
      "loss": 2.6625,
      "nll_loss": 0.23552022874355316,
      "rewards/accuracies": 0.7412499785423279,
      "rewards/chosen": -3.8205177783966064,
      "rewards/margins": 1.3472480773925781,
      "rewards/rejected": -5.167766094207764,
      "step": 450
    },
    {
      "epoch": 0.4,
      "grad_norm": 19.167991638183594,
      "learning_rate": 9.966729958067638e-06,
      "logits/chosen": 1.7662477493286133,
      "logits/rejected": 1.8194670677185059,
      "logps/chosen": -38.83460998535156,
      "logps/rejected": -50.7568359375,
      "loss": 2.7149,
      "nll_loss": 0.23406948149204254,
      "rewards/accuracies": 0.7137500047683716,
      "rewards/chosen": -3.8834612369537354,
      "rewards/margins": 1.1922229528427124,
      "rewards/rejected": -5.075684070587158,
      "step": 500
    },
    {
      "epoch": 0.44,
      "grad_norm": 22.746564865112305,
      "learning_rate": 9.934560333291077e-06,
      "logits/chosen": 1.6905887126922607,
      "logits/rejected": 1.756126046180725,
      "logps/chosen": -38.07200241088867,
      "logps/rejected": -50.685115814208984,
      "loss": 2.6286,
      "nll_loss": 0.22550682723522186,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -3.8072004318237305,
      "rewards/margins": 1.2613120079040527,
      "rewards/rejected": -5.068512439727783,
      "step": 550
    },
    {
      "epoch": 0.48,
      "grad_norm": 19.4133358001709,
      "learning_rate": 9.89170354964594e-06,
      "logits/chosen": 1.7051059007644653,
      "logits/rejected": 1.7545236349105835,
      "logps/chosen": -35.82815933227539,
      "logps/rejected": -47.575111389160156,
      "loss": 2.5393,
      "nll_loss": 0.21860088407993317,
      "rewards/accuracies": 0.7112500071525574,
      "rewards/chosen": -3.5828158855438232,
      "rewards/margins": 1.174695611000061,
      "rewards/rejected": -4.757511138916016,
      "step": 600
    },
    {
      "epoch": 0.52,
      "grad_norm": 16.378013610839844,
      "learning_rate": 9.83825242538238e-06,
      "logits/chosen": 1.6559092998504639,
      "logits/rejected": 1.7158703804016113,
      "logps/chosen": -36.47942352294922,
      "logps/rejected": -49.875587463378906,
      "loss": 2.5252,
      "nll_loss": 0.21864959597587585,
      "rewards/accuracies": 0.7337499856948853,
      "rewards/chosen": -3.647942304611206,
      "rewards/margins": 1.3396165370941162,
      "rewards/rejected": -4.987558364868164,
      "step": 650
    },
    {
      "epoch": 0.56,
      "grad_norm": 15.000406265258789,
      "learning_rate": 9.774322723733216e-06,
      "logits/chosen": 1.6370524168014526,
      "logits/rejected": 1.6942112445831299,
      "logps/chosen": -34.71821212768555,
      "logps/rejected": -48.40507507324219,
      "loss": 2.4278,
      "nll_loss": 0.21341566741466522,
      "rewards/accuracies": 0.7549999952316284,
      "rewards/chosen": -3.471820592880249,
      "rewards/margins": 1.368686556816101,
      "rewards/rejected": -4.8405070304870605,
      "step": 700
    },
    {
      "epoch": 0.6,
      "grad_norm": 17.42306900024414,
      "learning_rate": 9.700052902196541e-06,
      "logits/chosen": 1.606693148612976,
      "logits/rejected": 1.6712034940719604,
      "logps/chosen": -34.60812759399414,
      "logps/rejected": -48.792850494384766,
      "loss": 2.4102,
      "nll_loss": 0.21256333589553833,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.460812568664551,
      "rewards/margins": 1.4184722900390625,
      "rewards/rejected": -4.879284858703613,
      "step": 750
    },
    {
      "epoch": 0.64,
      "grad_norm": 18.26099967956543,
      "learning_rate": 9.615603812667618e-06,
      "logits/chosen": 1.574639916419983,
      "logits/rejected": 1.6343886852264404,
      "logps/chosen": -34.159088134765625,
      "logps/rejected": -48.842376708984375,
      "loss": 2.3734,
      "nll_loss": 0.20658308267593384,
      "rewards/accuracies": 0.7512500286102295,
      "rewards/chosen": -3.4159088134765625,
      "rewards/margins": 1.4683291912078857,
      "rewards/rejected": -4.884237766265869,
      "step": 800
    },
    {
      "epoch": 0.68,
      "grad_norm": 19.284168243408203,
      "learning_rate": 9.521158353069494e-06,
      "logits/chosen": 1.5315083265304565,
      "logits/rejected": 1.5860241651535034,
      "logps/chosen": -35.90006637573242,
      "logps/rejected": -51.2423095703125,
      "loss": 2.4185,
      "nll_loss": 0.2139773815870285,
      "rewards/accuracies": 0.7487499713897705,
      "rewards/chosen": -3.5900063514709473,
      "rewards/margins": 1.5342237949371338,
      "rewards/rejected": -5.124230861663818,
      "step": 850
    },
    {
      "epoch": 0.72,
      "grad_norm": 28.010725021362305,
      "learning_rate": 9.416921071236821e-06,
      "logits/chosen": 1.562249779701233,
      "logits/rejected": 1.6213380098342896,
      "logps/chosen": -34.88312530517578,
      "logps/rejected": -49.12727355957031,
      "loss": 2.4402,
      "nll_loss": 0.20900359749794006,
      "rewards/accuracies": 0.7362499833106995,
      "rewards/chosen": -3.4883129596710205,
      "rewards/margins": 1.4244145154953003,
      "rewards/rejected": -4.912726879119873,
      "step": 900
    },
    {
      "epoch": 0.76,
      "grad_norm": 22.14881134033203,
      "learning_rate": 9.303117721910801e-06,
      "logits/chosen": 1.5611748695373535,
      "logits/rejected": 1.613703966140747,
      "logps/chosen": -33.08067321777344,
      "logps/rejected": -46.956138610839844,
      "loss": 2.2957,
      "nll_loss": 0.19786672294139862,
      "rewards/accuracies": 0.7524999976158142,
      "rewards/chosen": -3.3080673217773438,
      "rewards/margins": 1.3875468969345093,
      "rewards/rejected": -4.695613861083984,
      "step": 950
    },
    {
      "epoch": 0.8,
      "grad_norm": 18.468719482421875,
      "learning_rate": 9.179994777804677e-06,
      "logits/chosen": 1.532340407371521,
      "logits/rejected": 1.595700979232788,
      "logps/chosen": -33.659690856933594,
      "logps/rejected": -48.1495475769043,
      "loss": 2.3428,
      "nll_loss": 0.2023465782403946,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.365969657897949,
      "rewards/margins": 1.4489855766296387,
      "rewards/rejected": -4.81495475769043,
      "step": 1000
    },
    {
      "epoch": 0.84,
      "grad_norm": 19.125810623168945,
      "learning_rate": 9.04781889579873e-06,
      "logits/chosen": 1.4953616857528687,
      "logits/rejected": 1.5516945123672485,
      "logps/chosen": -33.457584381103516,
      "logps/rejected": -48.823360443115234,
      "loss": 2.2845,
      "nll_loss": 0.2007952332496643,
      "rewards/accuracies": 0.7612500190734863,
      "rewards/chosen": -3.3457586765289307,
      "rewards/margins": 1.5365774631500244,
      "rewards/rejected": -4.882336139678955,
      "step": 1050
    },
    {
      "epoch": 0.88,
      "grad_norm": 18.376596450805664,
      "learning_rate": 8.906876339420851e-06,
      "logits/chosen": 1.452122449874878,
      "logits/rejected": 1.519229531288147,
      "logps/chosen": -31.664844512939453,
      "logps/rejected": -47.539306640625,
      "loss": 2.1647,
      "nll_loss": 0.19097968935966492,
      "rewards/accuracies": 0.7925000190734863,
      "rewards/chosen": -3.1664845943450928,
      "rewards/margins": 1.5874464511871338,
      "rewards/rejected": -4.753930568695068,
      "step": 1100
    },
    {
      "epoch": 0.92,
      "grad_norm": 20.535789489746094,
      "learning_rate": 8.757472358863481e-06,
      "logits/chosen": 1.4528400897979736,
      "logits/rejected": 1.5169428586959839,
      "logps/chosen": -33.37004089355469,
      "logps/rejected": -47.855865478515625,
      "loss": 2.2511,
      "nll_loss": 0.19547508656978607,
      "rewards/accuracies": 0.7574999928474426,
      "rewards/chosen": -3.337003707885742,
      "rewards/margins": 1.4485827684402466,
      "rewards/rejected": -4.785586357116699,
      "step": 1150
    },
    {
      "epoch": 0.96,
      "grad_norm": 22.283950805664062,
      "learning_rate": 8.599930529879669e-06,
      "logits/chosen": 1.4502493143081665,
      "logits/rejected": 1.5124887228012085,
      "logps/chosen": -31.987987518310547,
      "logps/rejected": -47.01323699951172,
      "loss": 2.1816,
      "nll_loss": 0.19316434860229492,
      "rewards/accuracies": 0.7774999737739563,
      "rewards/chosen": -3.198798418045044,
      "rewards/margins": 1.5025256872177124,
      "rewards/rejected": -4.701324462890625,
      "step": 1200
    },
    {
      "epoch": 1.0,
      "grad_norm": 19.24060821533203,
      "learning_rate": 8.434592052990044e-06,
      "logits/chosen": 1.4031856060028076,
      "logits/rejected": 1.462039589881897,
      "logps/chosen": -30.157163619995117,
      "logps/rejected": -45.0657844543457,
      "loss": 2.1647,
      "nll_loss": 0.18531297147274017,
      "rewards/accuracies": 0.7662500143051147,
      "rewards/chosen": -3.015716314315796,
      "rewards/margins": 1.4908628463745117,
      "rewards/rejected": -4.5065789222717285,
      "step": 1250
    },
    {
      "epoch": 1.04,
      "grad_norm": 16.792695999145508,
      "learning_rate": 8.261815014518465e-06,
      "logits/chosen": 1.334591031074524,
      "logits/rejected": 1.3955891132354736,
      "logps/chosen": -29.710214614868164,
      "logps/rejected": -49.487152099609375,
      "loss": 1.9604,
      "nll_loss": 0.17893625795841217,
      "rewards/accuracies": 0.8450000286102295,
      "rewards/chosen": -2.9710214138031006,
      "rewards/margins": 1.977694034576416,
      "rewards/rejected": -4.948715686798096,
      "step": 1300
    },
    {
      "epoch": 1.08,
      "grad_norm": 21.147741317749023,
      "learning_rate": 8.081973611056784e-06,
      "logits/chosen": 1.257203221321106,
      "logits/rejected": 1.3112421035766602,
      "logps/chosen": -28.43024444580078,
      "logps/rejected": -48.68648910522461,
      "loss": 1.872,
      "nll_loss": 0.16993314027786255,
      "rewards/accuracies": 0.8662499785423279,
      "rewards/chosen": -2.843024492263794,
      "rewards/margins": 2.0256245136260986,
      "rewards/rejected": -4.868649482727051,
      "step": 1350
    },
    {
      "epoch": 1.12,
      "grad_norm": 18.095643997192383,
      "learning_rate": 7.89545733903834e-06,
      "logits/chosen": 1.268505573272705,
      "logits/rejected": 1.3283963203430176,
      "logps/chosen": -29.838397979736328,
      "logps/rejected": -48.807472229003906,
      "loss": 1.9653,
      "nll_loss": 0.18002687394618988,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.983839988708496,
      "rewards/margins": 1.8969072103500366,
      "rewards/rejected": -4.880747318267822,
      "step": 1400
    },
    {
      "epoch": 1.16,
      "grad_norm": 20.254247665405273,
      "learning_rate": 7.702670151175435e-06,
      "logits/chosen": 1.3044127225875854,
      "logits/rejected": 1.368518352508545,
      "logps/chosen": -29.120044708251953,
      "logps/rejected": -47.06825637817383,
      "loss": 1.9557,
      "nll_loss": 0.1742786169052124,
      "rewards/accuracies": 0.8387500047683716,
      "rewards/chosen": -2.9120047092437744,
      "rewards/margins": 1.7948211431503296,
      "rewards/rejected": -4.706826210021973,
      "step": 1450
    },
    {
      "epoch": 1.2,
      "grad_norm": 15.50015640258789,
      "learning_rate": 7.5040295815877e-06,
      "logits/chosen": 1.2754545211791992,
      "logits/rejected": 1.33790922164917,
      "logps/chosen": -28.515518188476562,
      "logps/rejected": -49.134769439697266,
      "loss": 1.8568,
      "nll_loss": 0.16948945820331573,
      "rewards/accuracies": 0.8737499713897705,
      "rewards/chosen": -2.8515517711639404,
      "rewards/margins": 2.061924934387207,
      "rewards/rejected": -4.913476943969727,
      "step": 1500
    },
    {
      "epoch": 1.24,
      "grad_norm": 18.263025283813477,
      "learning_rate": 7.299965841516164e-06,
      "logits/chosen": 1.2721493244171143,
      "logits/rejected": 1.3371402025222778,
      "logps/chosen": -29.20949935913086,
      "logps/rejected": -48.70906066894531,
      "loss": 1.9402,
      "nll_loss": 0.17609195411205292,
      "rewards/accuracies": 0.8550000190734863,
      "rewards/chosen": -2.920949935913086,
      "rewards/margins": 1.9499554634094238,
      "rewards/rejected": -4.87090539932251,
      "step": 1550
    },
    {
      "epoch": 1.28,
      "grad_norm": 17.24570655822754,
      "learning_rate": 7.090920887581507e-06,
      "logits/chosen": 1.2389811277389526,
      "logits/rejected": 1.2887934446334839,
      "logps/chosen": -28.364063262939453,
      "logps/rejected": -46.97511291503906,
      "loss": 1.8952,
      "nll_loss": 0.1697738617658615,
      "rewards/accuracies": 0.8362500071525574,
      "rewards/chosen": -2.8364062309265137,
      "rewards/margins": 1.8611053228378296,
      "rewards/rejected": -4.697511196136475,
      "step": 1600
    },
    {
      "epoch": 1.32,
      "grad_norm": 20.073883056640625,
      "learning_rate": 6.877347464604446e-06,
      "logits/chosen": 1.2145293951034546,
      "logits/rejected": 1.2653400897979736,
      "logps/chosen": -28.30023765563965,
      "logps/rejected": -47.83637619018555,
      "loss": 1.8857,
      "nll_loss": 0.16985949873924255,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.830024003982544,
      "rewards/margins": 1.9536142349243164,
      "rewards/rejected": -4.7836384773254395,
      "step": 1650
    },
    {
      "epoch": 1.3599999999999999,
      "grad_norm": 20.44426727294922,
      "learning_rate": 6.659708125061242e-06,
      "logits/chosen": 1.210806131362915,
      "logits/rejected": 1.2653027772903442,
      "logps/chosen": -28.112794876098633,
      "logps/rejected": -47.89250183105469,
      "loss": 1.8333,
      "nll_loss": 0.16796012222766876,
      "rewards/accuracies": 0.8637499809265137,
      "rewards/chosen": -2.811279535293579,
      "rewards/margins": 1.9779702425003052,
      "rewards/rejected": -4.789250373840332,
      "step": 1700
    },
    {
      "epoch": 1.4,
      "grad_norm": 15.871622085571289,
      "learning_rate": 6.438474227298065e-06,
      "logits/chosen": 1.2420759201049805,
      "logits/rejected": 1.296012282371521,
      "logps/chosen": -28.223691940307617,
      "logps/rejected": -47.52497863769531,
      "loss": 1.8911,
      "nll_loss": 0.1713765263557434,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.82236909866333,
      "rewards/margins": 1.9301284551620483,
      "rewards/rejected": -4.752498149871826,
      "step": 1750
    },
    {
      "epoch": 1.44,
      "grad_norm": 17.20113182067871,
      "learning_rate": 6.2141249146737545e-06,
      "logits/chosen": 1.2283880710601807,
      "logits/rejected": 1.2828854322433472,
      "logps/chosen": -26.78079605102539,
      "logps/rejected": -45.25880813598633,
      "loss": 1.8039,
      "nll_loss": 0.15980492532253265,
      "rewards/accuracies": 0.8450000286102295,
      "rewards/chosen": -2.678079605102539,
      "rewards/margins": 1.8478012084960938,
      "rewards/rejected": -4.525880813598633,
      "step": 1800
    },
    {
      "epoch": 1.48,
      "grad_norm": 20.9070987701416,
      "learning_rate": 5.987146077842015e-06,
      "logits/chosen": 1.2112573385238647,
      "logits/rejected": 1.2712033987045288,
      "logps/chosen": -28.28427505493164,
      "logps/rejected": -48.554771423339844,
      "loss": 1.8766,
      "nll_loss": 0.1684311032295227,
      "rewards/accuracies": 0.8387500047683716,
      "rewards/chosen": -2.828427314758301,
      "rewards/margins": 2.0270493030548096,
      "rewards/rejected": -4.855477333068848,
      "step": 1850
    },
    {
      "epoch": 1.52,
      "grad_norm": 16.220539093017578,
      "learning_rate": 5.7580293024204455e-06,
      "logits/chosen": 1.2373614311218262,
      "logits/rejected": 1.2950869798660278,
      "logps/chosen": -27.206884384155273,
      "logps/rejected": -46.864036560058594,
      "loss": 1.7983,
      "nll_loss": 0.16223596036434174,
      "rewards/accuracies": 0.8712499737739563,
      "rewards/chosen": -2.720688819885254,
      "rewards/margins": 1.9657151699066162,
      "rewards/rejected": -4.686403751373291,
      "step": 1900
    },
    {
      "epoch": 1.56,
      "grad_norm": 19.87531852722168,
      "learning_rate": 5.5272708043255605e-06,
      "logits/chosen": 1.2204946279525757,
      "logits/rejected": 1.266890048980713,
      "logps/chosen": -26.85431671142578,
      "logps/rejected": -45.090606689453125,
      "loss": 1.8634,
      "nll_loss": 0.1644485890865326,
      "rewards/accuracies": 0.8387500047683716,
      "rewards/chosen": -2.685431718826294,
      "rewards/margins": 1.8236281871795654,
      "rewards/rejected": -4.509060382843018,
      "step": 1950
    },
    {
      "epoch": 1.6,
      "grad_norm": 20.69000244140625,
      "learning_rate": 5.295370355079615e-06,
      "logits/chosen": 1.1966464519500732,
      "logits/rejected": 1.2531131505966187,
      "logps/chosen": -26.715871810913086,
      "logps/rejected": -46.64927673339844,
      "loss": 1.7931,
      "nll_loss": 0.16148775815963745,
      "rewards/accuracies": 0.8424999713897705,
      "rewards/chosen": -2.6715869903564453,
      "rewards/margins": 1.993341088294983,
      "rewards/rejected": -4.664927959442139,
      "step": 2000
    },
    {
      "epoch": 1.6400000000000001,
      "grad_norm": 20.352148056030273,
      "learning_rate": 5.062830199416764e-06,
      "logits/chosen": 1.2095577716827393,
      "logits/rejected": 1.2642172574996948,
      "logps/chosen": -27.680116653442383,
      "logps/rejected": -47.270565032958984,
      "loss": 1.8336,
      "nll_loss": 0.1655382215976715,
      "rewards/accuracies": 0.8487499952316284,
      "rewards/chosen": -2.768012046813965,
      "rewards/margins": 1.9590446949005127,
      "rewards/rejected": -4.727056980133057,
      "step": 2050
    },
    {
      "epoch": 1.6800000000000002,
      "grad_norm": 20.977998733520508,
      "learning_rate": 4.8301539675328205e-06,
      "logits/chosen": 1.1827168464660645,
      "logits/rejected": 1.2348774671554565,
      "logps/chosen": -26.652387619018555,
      "logps/rejected": -46.84341812133789,
      "loss": 1.758,
      "nll_loss": 0.15646891295909882,
      "rewards/accuracies": 0.8512499928474426,
      "rewards/chosen": -2.665238857269287,
      "rewards/margins": 2.0191030502319336,
      "rewards/rejected": -4.684341907501221,
      "step": 2100
    },
    {
      "epoch": 1.72,
      "grad_norm": 18.503433227539062,
      "learning_rate": 4.597845584334387e-06,
      "logits/chosen": 1.1708920001983643,
      "logits/rejected": 1.2160160541534424,
      "logps/chosen": -27.61713409423828,
      "logps/rejected": -47.54500961303711,
      "loss": 1.8257,
      "nll_loss": 0.1635328233242035,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.7617132663726807,
      "rewards/margins": 1.9927879571914673,
      "rewards/rejected": -4.7545013427734375,
      "step": 2150
    },
    {
      "epoch": 1.76,
      "grad_norm": 17.69391441345215,
      "learning_rate": 4.366408178049728e-06,
      "logits/chosen": 1.184067964553833,
      "logits/rejected": 1.2318971157073975,
      "logps/chosen": -25.899795532226562,
      "logps/rejected": -45.311283111572266,
      "loss": 1.7548,
      "nll_loss": 0.15687091648578644,
      "rewards/accuracies": 0.8550000190734863,
      "rewards/chosen": -2.589979648590088,
      "rewards/margins": 1.9411489963531494,
      "rewards/rejected": -4.531128406524658,
      "step": 2200
    },
    {
      "epoch": 1.8,
      "grad_norm": 21.093082427978516,
      "learning_rate": 4.136342990565055e-06,
      "logits/chosen": 1.165001630783081,
      "logits/rejected": 1.2157142162322998,
      "logps/chosen": -26.217182159423828,
      "logps/rejected": -47.43375015258789,
      "loss": 1.7791,
      "nll_loss": 0.16027042269706726,
      "rewards/accuracies": 0.8675000071525574,
      "rewards/chosen": -2.621718406677246,
      "rewards/margins": 2.121656894683838,
      "rewards/rejected": -4.743375301361084,
      "step": 2250
    },
    {
      "epoch": 1.8399999999999999,
      "grad_norm": 22.38056182861328,
      "learning_rate": 3.908148291846225e-06,
      "logits/chosen": 1.1481218338012695,
      "logits/rejected": 1.1908549070358276,
      "logps/chosen": -24.328386306762695,
      "logps/rejected": -43.8892707824707,
      "loss": 1.683,
      "nll_loss": 0.1506115198135376,
      "rewards/accuracies": 0.8700000047683716,
      "rewards/chosen": -2.4328386783599854,
      "rewards/margins": 1.9560893774032593,
      "rewards/rejected": -4.388927459716797,
      "step": 2300
    },
    {
      "epoch": 1.88,
      "grad_norm": 38.030303955078125,
      "learning_rate": 3.6823183007969375e-06,
      "logits/chosen": 1.1535931825637817,
      "logits/rejected": 1.2029131650924683,
      "logps/chosen": -25.072856903076172,
      "logps/rejected": -44.75934982299805,
      "loss": 1.7215,
      "nll_loss": 0.15159070491790771,
      "rewards/accuracies": 0.8424999713897705,
      "rewards/chosen": -2.5072858333587646,
      "rewards/margins": 1.9686492681503296,
      "rewards/rejected": -4.475934982299805,
      "step": 2350
    },
    {
      "epoch": 1.92,
      "grad_norm": 16.922834396362305,
      "learning_rate": 3.4593421148906523e-06,
      "logits/chosen": 1.150818943977356,
      "logits/rejected": 1.20121431350708,
      "logps/chosen": -27.409215927124023,
      "logps/rejected": -46.66661071777344,
      "loss": 1.7911,
      "nll_loss": 0.15817226469516754,
      "rewards/accuracies": 0.8450000286102295,
      "rewards/chosen": -2.740921974182129,
      "rewards/margins": 1.9257392883300781,
      "rewards/rejected": -4.666661262512207,
      "step": 2400
    },
    {
      "epoch": 1.96,
      "grad_norm": 15.894891738891602,
      "learning_rate": 3.239702650894364e-06,
      "logits/chosen": 1.1605645418167114,
      "logits/rejected": 1.2133049964904785,
      "logps/chosen": -25.806852340698242,
      "logps/rejected": -46.503700256347656,
      "loss": 1.739,
      "nll_loss": 0.15509046614170074,
      "rewards/accuracies": 0.8550000190734863,
      "rewards/chosen": -2.5806851387023926,
      "rewards/margins": 2.0696845054626465,
      "rewards/rejected": -4.650369644165039,
      "step": 2450
    },
    {
      "epoch": 2.0,
      "grad_norm": 17.814403533935547,
      "learning_rate": 3.023875598978419e-06,
      "logits/chosen": 1.1574615240097046,
      "logits/rejected": 1.2072219848632812,
      "logps/chosen": -25.269817352294922,
      "logps/rejected": -47.318443298339844,
      "loss": 1.6896,
      "nll_loss": 0.15325431525707245,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.5269815921783447,
      "rewards/margins": 2.2048630714416504,
      "rewards/rejected": -4.731843948364258,
      "step": 2500
    },
    {
      "epoch": 2.04,
      "grad_norm": 17.23746109008789,
      "learning_rate": 2.812328392477536e-06,
      "logits/chosen": 1.106400489807129,
      "logits/rejected": 1.1545926332473755,
      "logps/chosen": -23.484037399291992,
      "logps/rejected": -48.1876106262207,
      "loss": 1.499,
      "nll_loss": 0.14059405028820038,
      "rewards/accuracies": 0.9449999928474426,
      "rewards/chosen": -2.3484036922454834,
      "rewards/margins": 2.470357656478882,
      "rewards/rejected": -4.818761825561523,
      "step": 2550
    },
    {
      "epoch": 2.08,
      "grad_norm": 14.634954452514648,
      "learning_rate": 2.6055191955342886e-06,
      "logits/chosen": 1.0940419435501099,
      "logits/rejected": 1.1334995031356812,
      "logps/chosen": -24.663402557373047,
      "logps/rejected": -49.43788146972656,
      "loss": 1.5717,
      "nll_loss": 0.1473802775144577,
      "rewards/accuracies": 0.9275000095367432,
      "rewards/chosen": -2.4663405418395996,
      "rewards/margins": 2.477447509765625,
      "rewards/rejected": -4.943788051605225,
      "step": 2600
    },
    {
      "epoch": 2.12,
      "grad_norm": 17.942777633666992,
      "learning_rate": 2.403895910817593e-06,
      "logits/chosen": 1.0923943519592285,
      "logits/rejected": 1.1312522888183594,
      "logps/chosen": -23.995264053344727,
      "logps/rejected": -47.21249008178711,
      "loss": 1.5767,
      "nll_loss": 0.14557117223739624,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.399526357650757,
      "rewards/margins": 2.3217225074768066,
      "rewards/rejected": -4.721249103546143,
      "step": 2650
    },
    {
      "epoch": 2.16,
      "grad_norm": 16.142436981201172,
      "learning_rate": 2.2078952094652705e-06,
      "logits/chosen": 1.0400832891464233,
      "logits/rejected": 1.1018368005752563,
      "logps/chosen": -24.179841995239258,
      "logps/rejected": -50.50990295410156,
      "loss": 1.5025,
      "nll_loss": 0.14027239382266998,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.4179842472076416,
      "rewards/margins": 2.6330063343048096,
      "rewards/rejected": -5.050990104675293,
      "step": 2700
    },
    {
      "epoch": 2.2,
      "grad_norm": 19.593612670898438,
      "learning_rate": 2.017941585351591e-06,
      "logits/chosen": 1.0729751586914062,
      "logits/rejected": 1.1096735000610352,
      "logps/chosen": -24.066631317138672,
      "logps/rejected": -47.574344635009766,
      "loss": 1.5804,
      "nll_loss": 0.14690618216991425,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.406662940979004,
      "rewards/margins": 2.3507721424102783,
      "rewards/rejected": -4.757434844970703,
      "step": 2750
    },
    {
      "epoch": 2.24,
      "grad_norm": 14.078289985656738,
      "learning_rate": 1.8344464357280722e-06,
      "logits/chosen": 1.0587913990020752,
      "logits/rejected": 1.0995149612426758,
      "logps/chosen": -23.153541564941406,
      "logps/rejected": -47.50178527832031,
      "loss": 1.4804,
      "nll_loss": 0.13591693341732025,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.315354347229004,
      "rewards/margins": 2.4348244667053223,
      "rewards/rejected": -4.750178813934326,
      "step": 2800
    },
    {
      "epoch": 2.2800000000000002,
      "grad_norm": 15.281038284301758,
      "learning_rate": 1.6578071702286396e-06,
      "logits/chosen": 1.0582990646362305,
      "logits/rejected": 1.0928741693496704,
      "logps/chosen": -23.392786026000977,
      "logps/rejected": -47.97736358642578,
      "loss": 1.5417,
      "nll_loss": 0.1428249180316925,
      "rewards/accuracies": 0.9175000190734863,
      "rewards/chosen": -2.33927845954895,
      "rewards/margins": 2.4584577083587646,
      "rewards/rejected": -4.797736167907715,
      "step": 2850
    },
    {
      "epoch": 2.32,
      "grad_norm": 15.694136619567871,
      "learning_rate": 1.4884063501688539e-06,
      "logits/chosen": 1.064022183418274,
      "logits/rejected": 1.1182899475097656,
      "logps/chosen": -23.681140899658203,
      "logps/rejected": -48.91339874267578,
      "loss": 1.4978,
      "nll_loss": 0.1389184594154358,
      "rewards/accuracies": 0.9287499785423279,
      "rewards/chosen": -2.3681137561798096,
      "rewards/margins": 2.523225784301758,
      "rewards/rejected": -4.8913397789001465,
      "step": 2900
    },
    {
      "epoch": 2.36,
      "grad_norm": 19.51872444152832,
      "learning_rate": 1.3266108600032928e-06,
      "logits/chosen": 1.0682127475738525,
      "logits/rejected": 1.1111141443252563,
      "logps/chosen": -24.015024185180664,
      "logps/rejected": -47.481693267822266,
      "loss": 1.5333,
      "nll_loss": 0.14073985815048218,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.4015023708343506,
      "rewards/margins": 2.3466668128967285,
      "rewards/rejected": -4.748169422149658,
      "step": 2950
    },
    {
      "epoch": 2.4,
      "grad_norm": 21.052757263183594,
      "learning_rate": 1.1727711127355118e-06,
      "logits/chosen": 1.054885745048523,
      "logits/rejected": 1.0945353507995605,
      "logps/chosen": -22.908018112182617,
      "logps/rejected": -46.528038024902344,
      "loss": 1.4819,
      "nll_loss": 0.13584649562835693,
      "rewards/accuracies": 0.9100000262260437,
      "rewards/chosen": -2.290801525115967,
      "rewards/margins": 2.362002372741699,
      "rewards/rejected": -4.652803897857666,
      "step": 3000
    },
    {
      "epoch": 2.44,
      "grad_norm": 16.752124786376953,
      "learning_rate": 1.0272202910015083e-06,
      "logits/chosen": 1.0628950595855713,
      "logits/rejected": 1.1132755279541016,
      "logps/chosen": -23.901992797851562,
      "logps/rejected": -48.75297927856445,
      "loss": 1.5343,
      "nll_loss": 0.14184221625328064,
      "rewards/accuracies": 0.9175000190734863,
      "rewards/chosen": -2.390199661254883,
      "rewards/margins": 2.485098361968994,
      "rewards/rejected": -4.875298023223877,
      "step": 3050
    },
    {
      "epoch": 2.48,
      "grad_norm": 20.135562896728516,
      "learning_rate": 8.902736254703347e-07,
      "logits/chosen": 1.0642120838165283,
      "logits/rejected": 1.0948975086212158,
      "logps/chosen": -22.418310165405273,
      "logps/rejected": -44.414913177490234,
      "loss": 1.4982,
      "nll_loss": 0.13570523262023926,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.241831064224243,
      "rewards/margins": 2.199659824371338,
      "rewards/rejected": -4.441490650177002,
      "step": 3100
    },
    {
      "epoch": 2.52,
      "grad_norm": 17.3437557220459,
      "learning_rate": 7.622277121246513e-07,
      "logits/chosen": 1.0698988437652588,
      "logits/rejected": 1.1142799854278564,
      "logps/chosen": -23.52094078063965,
      "logps/rejected": -47.37723159790039,
      "loss": 1.5275,
      "nll_loss": 0.13999156653881073,
      "rewards/accuracies": 0.9137499928474426,
      "rewards/chosen": -2.3520936965942383,
      "rewards/margins": 2.385629415512085,
      "rewards/rejected": -4.737722873687744,
      "step": 3150
    },
    {
      "epoch": 2.56,
      "grad_norm": 19.78534698486328,
      "learning_rate": 6.433598698998766e-07,
      "logits/chosen": 1.065538763999939,
      "logits/rejected": 1.118047833442688,
      "logps/chosen": -23.858076095581055,
      "logps/rejected": -46.57390594482422,
      "loss": 1.5673,
      "nll_loss": 0.1432953178882599,
      "rewards/accuracies": 0.9024999737739563,
      "rewards/chosen": -2.385807752609253,
      "rewards/margins": 2.271583318710327,
      "rewards/rejected": -4.657390594482422,
      "step": 3200
    },
    {
      "epoch": 2.6,
      "grad_norm": 20.778789520263672,
      "learning_rate": 5.339275400731331e-07,
      "logits/chosen": 1.064239263534546,
      "logits/rejected": 1.1029696464538574,
      "logps/chosen": -23.444625854492188,
      "logps/rejected": -45.59990692138672,
      "loss": 1.5517,
      "nll_loss": 0.14206433296203613,
      "rewards/accuracies": 0.9150000214576721,
      "rewards/chosen": -2.3444626331329346,
      "rewards/margins": 2.2155280113220215,
      "rewards/rejected": -4.559990882873535,
      "step": 3250
    },
    {
      "epoch": 2.64,
      "grad_norm": 13.808253288269043,
      "learning_rate": 4.3416772870275295e-07,
      "logits/chosen": 1.0500396490097046,
      "logits/rejected": 1.0961949825286865,
      "logps/chosen": -23.120912551879883,
      "logps/rejected": -47.897361755371094,
      "loss": 1.479,
      "nll_loss": 0.13585034012794495,
      "rewards/accuracies": 0.9275000095367432,
      "rewards/chosen": -2.31209135055542,
      "rewards/margins": 2.4776451587677,
      "rewards/rejected": -4.789736270904541,
      "step": 3300
    },
    {
      "epoch": 2.68,
      "grad_norm": 13.967582702636719,
      "learning_rate": 3.442964933259474e-07,
      "logits/chosen": 1.0531511306762695,
      "logits/rejected": 1.1031429767608643,
      "logps/chosen": -23.176462173461914,
      "logps/rejected": -48.2183723449707,
      "loss": 1.4983,
      "nll_loss": 0.138520285487175,
      "rewards/accuracies": 0.9137499928474426,
      "rewards/chosen": -2.3176462650299072,
      "rewards/margins": 2.5041911602020264,
      "rewards/rejected": -4.821837425231934,
      "step": 3350
    },
    {
      "epoch": 2.7199999999999998,
      "grad_norm": 18.23953628540039,
      "learning_rate": 2.6450847502627883e-07,
      "logits/chosen": 1.038626790046692,
      "logits/rejected": 1.0861948728561401,
      "logps/chosen": -23.72449493408203,
      "logps/rejected": -49.453460693359375,
      "loss": 1.4878,
      "nll_loss": 0.13710632920265198,
      "rewards/accuracies": 0.9175000190734863,
      "rewards/chosen": -2.3724493980407715,
      "rewards/margins": 2.572896957397461,
      "rewards/rejected": -4.945346832275391,
      "step": 3400
    },
    {
      "epoch": 2.76,
      "grad_norm": 19.383655548095703,
      "learning_rate": 1.9497647688442478e-07,
      "logits/chosen": 1.0585306882858276,
      "logits/rejected": 1.1055423021316528,
      "logps/chosen": -22.849088668823242,
      "logps/rejected": -46.4496955871582,
      "loss": 1.5048,
      "nll_loss": 0.13851623237133026,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.2849090099334717,
      "rewards/margins": 2.360060214996338,
      "rewards/rejected": -4.6449689865112305,
      "step": 3450
    },
    {
      "epoch": 2.8,
      "grad_norm": 18.49037742614746,
      "learning_rate": 1.358510897251808e-07,
      "logits/chosen": 1.0519126653671265,
      "logits/rejected": 1.0990923643112183,
      "logps/chosen": -22.153770446777344,
      "logps/rejected": -47.41790008544922,
      "loss": 1.4731,
      "nll_loss": 0.13626331090927124,
      "rewards/accuracies": 0.9212499856948853,
      "rewards/chosen": -2.21537709236145,
      "rewards/margins": 2.5264134407043457,
      "rewards/rejected": -4.741790294647217,
      "step": 3500
    },
    {
      "epoch": 2.84,
      "grad_norm": 16.536218643188477,
      "learning_rate": 8.726036597126619e-08,
      "logits/chosen": 1.0618226528167725,
      "logits/rejected": 1.1148923635482788,
      "logps/chosen": -22.412979125976562,
      "logps/rejected": -45.536277770996094,
      "loss": 1.4864,
      "nll_loss": 0.1364334225654602,
      "rewards/accuracies": 0.9162499904632568,
      "rewards/chosen": -2.241297960281372,
      "rewards/margins": 2.3123297691345215,
      "rewards/rejected": -4.5536274909973145,
      "step": 3550
    },
    {
      "epoch": 2.88,
      "grad_norm": 20.392322540283203,
      "learning_rate": 4.93095423102935e-08,
      "logits/chosen": 1.0614595413208008,
      "logits/rejected": 1.1021995544433594,
      "logps/chosen": -24.838594436645508,
      "logps/rejected": -49.431583404541016,
      "loss": 1.5749,
      "nll_loss": 0.14667731523513794,
      "rewards/accuracies": 0.9049999713897705,
      "rewards/chosen": -2.4838597774505615,
      "rewards/margins": 2.459298610687256,
      "rewards/rejected": -4.943158149719238,
      "step": 3600
    },
    {
      "epoch": 2.92,
      "grad_norm": 17.081296920776367,
      "learning_rate": 2.2080811775535006e-08,
      "logits/chosen": 1.0633502006530762,
      "logits/rejected": 1.1077544689178467,
      "logps/chosen": -24.01654052734375,
      "logps/rejected": -49.075321197509766,
      "loss": 1.5384,
      "nll_loss": 0.1433144360780716,
      "rewards/accuracies": 0.9212499856948853,
      "rewards/chosen": -2.401654005050659,
      "rewards/margins": 2.505878210067749,
      "rewards/rejected": -4.907532215118408,
      "step": 3650
    },
    {
      "epoch": 2.96,
      "grad_norm": 19.85245132446289,
      "learning_rate": 5.633145734114665e-09,
      "logits/chosen": 1.0758558511734009,
      "logits/rejected": 1.114052653312683,
      "logps/chosen": -23.172121047973633,
      "logps/rejected": -46.996238708496094,
      "loss": 1.5239,
      "nll_loss": 0.14142683148384094,
      "rewards/accuracies": 0.9262499809265137,
      "rewards/chosen": -2.3172123432159424,
      "rewards/margins": 2.3824121952056885,
      "rewards/rejected": -4.699624538421631,
      "step": 3700
    },
    {
      "epoch": 3.0,
      "grad_norm": 19.575937271118164,
      "learning_rate": 2.1661681620654963e-12,
      "logits/chosen": 1.0582020282745361,
      "logits/rejected": 1.1024978160858154,
      "logps/chosen": -22.676530838012695,
      "logps/rejected": -48.214569091796875,
      "loss": 1.4608,
      "nll_loss": 0.13524094223976135,
      "rewards/accuracies": 0.9287499785423279,
      "rewards/chosen": -2.267652988433838,
      "rewards/margins": 2.5538036823272705,
      "rewards/rejected": -4.821456432342529,
      "step": 3750
    }
  ],
  "logging_steps": 50,
  "max_steps": 3750,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 200,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}