{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 4.0,
  "eval_steps": 500,
  "global_step": 7588,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.005271481286241434,
      "grad_norm": 123.59642931847907,
      "learning_rate": 9.988139167105956e-07,
      "logits/chosen": -1.994140625,
      "logits/rejected": -1.942968726158142,
      "logps/chosen": -343.32501220703125,
      "logps/rejected": -309.2749938964844,
      "loss": 0.6958,
      "rewards/accuracies": 0.34375,
      "rewards/chosen": 0.00821685791015625,
      "rewards/margins": 0.01567840576171875,
      "rewards/rejected": -0.0074562071822583675,
      "step": 10
    },
    {
      "epoch": 0.010542962572482868,
      "grad_norm": 103.05643609924492,
      "learning_rate": 9.974960463890353e-07,
      "logits/chosen": -2.009960889816284,
      "logits/rejected": -1.974609375,
      "logps/chosen": -368.4375,
      "logps/rejected": -346.61248779296875,
      "loss": 0.6915,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": 0.02215271070599556,
      "rewards/margins": 0.020787810906767845,
      "rewards/rejected": 0.00146484375,
      "step": 20
    },
    {
      "epoch": 0.0158144438587243,
      "grad_norm": 101.31333207934405,
      "learning_rate": 9.96178176067475e-07,
      "logits/chosen": -2.0220704078674316,
      "logits/rejected": -1.9289062023162842,
      "logps/chosen": -369.4750061035156,
      "logps/rejected": -325.07501220703125,
      "loss": 0.6448,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": 0.2802902162075043,
      "rewards/margins": 0.13973084092140198,
      "rewards/rejected": 0.14059142768383026,
      "step": 30
    },
    {
      "epoch": 0.021085925144965736,
      "grad_norm": 110.19749779310162,
      "learning_rate": 9.948603057459145e-07,
      "logits/chosen": -2.022265672683716,
      "logits/rejected": -1.9548828601837158,
      "logps/chosen": -350.04998779296875,
      "logps/rejected": -339.7875061035156,
      "loss": 0.6438,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.45526123046875,
      "rewards/margins": 0.17221984267234802,
      "rewards/rejected": 0.2831268310546875,
      "step": 40
    },
    {
      "epoch": 0.02635740643120717,
      "grad_norm": 94.42467630968116,
      "learning_rate": 9.935424354243542e-07,
      "logits/chosen": -2.1400389671325684,
      "logits/rejected": -1.9470703601837158,
      "logps/chosen": -302.29376220703125,
      "logps/rejected": -301.13751220703125,
      "loss": 0.6392,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.514941394329071,
      "rewards/margins": 0.19003906846046448,
      "rewards/rejected": 0.3249038755893707,
      "step": 50
    },
    {
      "epoch": 0.0316288877174486,
      "grad_norm": 111.56042840186295,
      "learning_rate": 9.922245651027939e-07,
      "logits/chosen": -2.0550780296325684,
      "logits/rejected": -2.0386719703674316,
      "logps/chosen": -325.7250061035156,
      "logps/rejected": -324.67498779296875,
      "loss": 0.6513,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": 0.3443054258823395,
      "rewards/margins": 0.16503600776195526,
      "rewards/rejected": 0.17918090522289276,
      "step": 60
    },
    {
      "epoch": 0.03690036900369004,
      "grad_norm": 108.59320711020872,
      "learning_rate": 9.909066947812334e-07,
      "logits/chosen": -2.1832032203674316,
      "logits/rejected": -2.1474609375,
      "logps/chosen": -377.61248779296875,
      "logps/rejected": -358.1499938964844,
      "loss": 0.6135,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.20591430366039276,
      "rewards/margins": 0.2781051695346832,
      "rewards/rejected": -0.07223205268383026,
      "step": 70
    },
    {
      "epoch": 0.04217185028993147,
      "grad_norm": 94.96275136620257,
      "learning_rate": 9.895888244596733e-07,
      "logits/chosen": -2.0660157203674316,
      "logits/rejected": -2.0804686546325684,
      "logps/chosen": -347.11248779296875,
      "logps/rejected": -298.38751220703125,
      "loss": 0.6364,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.22720947861671448,
      "rewards/margins": 0.24664917588233948,
      "rewards/rejected": -0.01948242262005806,
      "step": 80
    },
    {
      "epoch": 0.047443331576172906,
      "grad_norm": 100.51516530042538,
      "learning_rate": 9.882709541381128e-07,
      "logits/chosen": -2.123046875,
      "logits/rejected": -2.169140577316284,
      "logps/chosen": -355.79998779296875,
      "logps/rejected": -321.57501220703125,
      "loss": 0.615,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": 0.506152331829071,
      "rewards/margins": 0.3186401426792145,
      "rewards/rejected": 0.18773499131202698,
      "step": 90
    },
    {
      "epoch": 0.05271481286241434,
      "grad_norm": 134.71954932121406,
      "learning_rate": 9.869530838165525e-07,
      "logits/chosen": -2.086132764816284,
      "logits/rejected": -2.0492186546325684,
      "logps/chosen": -377.625,
      "logps/rejected": -361.1499938964844,
      "loss": 0.6341,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.41334229707717896,
      "rewards/margins": 0.318389892578125,
      "rewards/rejected": 0.0948486328125,
      "step": 100
    },
    {
      "epoch": 0.05798629414865577,
      "grad_norm": 114.18858896507273,
      "learning_rate": 9.85635213494992e-07,
      "logits/chosen": -2.1519532203674316,
      "logits/rejected": -2.0771484375,
      "logps/chosen": -427.5249938964844,
      "logps/rejected": -375.57501220703125,
      "loss": 0.6313,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": 0.21003112196922302,
      "rewards/margins": 0.3476318418979645,
      "rewards/rejected": -0.13771668076515198,
      "step": 110
    },
    {
      "epoch": 0.0632577754348972,
      "grad_norm": 126.61067425655155,
      "learning_rate": 9.843173431734316e-07,
      "logits/chosen": -2.01953125,
      "logits/rejected": -2.0015625953674316,
      "logps/chosen": -364.42498779296875,
      "logps/rejected": -334.42498779296875,
      "loss": 0.6818,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": 0.09673156589269638,
      "rewards/margins": 0.22440795600414276,
      "rewards/rejected": -0.12760619819164276,
      "step": 120
    },
    {
      "epoch": 0.06852925672113865,
      "grad_norm": 128.95188775973085,
      "learning_rate": 9.829994728518713e-07,
      "logits/chosen": -2.0894532203674316,
      "logits/rejected": -2.0757813453674316,
      "logps/chosen": -355.4624938964844,
      "logps/rejected": -320.3999938964844,
      "loss": 0.6157,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.29109495878219604,
      "rewards/margins": 0.350433349609375,
      "rewards/rejected": -0.05954284593462944,
      "step": 130
    },
    {
      "epoch": 0.07380073800738007,
      "grad_norm": 106.48220211528671,
      "learning_rate": 9.81681602530311e-07,
      "logits/chosen": -2.039843797683716,
      "logits/rejected": -2.0093750953674316,
      "logps/chosen": -373.1499938964844,
      "logps/rejected": -341.1875,
      "loss": 0.6314,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.22719116508960724,
      "rewards/margins": 0.35310059785842896,
      "rewards/rejected": -0.12624207139015198,
      "step": 140
    },
    {
      "epoch": 0.0790722192936215,
      "grad_norm": 122.8891839818806,
      "learning_rate": 9.803637322087505e-07,
      "logits/chosen": -2.1123046875,
      "logits/rejected": -2.052734375,
      "logps/chosen": -384.7250061035156,
      "logps/rejected": -350.8500061035156,
      "loss": 0.5816,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.06817626953125,
      "rewards/margins": 0.4328064024448395,
      "rewards/rejected": -0.365081787109375,
      "step": 150
    },
    {
      "epoch": 0.08434370057986294,
      "grad_norm": 97.07810586572502,
      "learning_rate": 9.790458618871902e-07,
      "logits/chosen": -2.0458984375,
      "logits/rejected": -2.080859422683716,
      "logps/chosen": -378.0375061035156,
      "logps/rejected": -330.2875061035156,
      "loss": 0.5979,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.1933189332485199,
      "rewards/margins": 0.412445068359375,
      "rewards/rejected": -0.6058410406112671,
      "step": 160
    },
    {
      "epoch": 0.08961518186610437,
      "grad_norm": 126.1520249734552,
      "learning_rate": 9.7772799156563e-07,
      "logits/chosen": -2.174609422683716,
      "logits/rejected": -2.086132764816284,
      "logps/chosen": -394.20001220703125,
      "logps/rejected": -367.04998779296875,
      "loss": 0.6648,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.3048339784145355,
      "rewards/margins": 0.30616456270217896,
      "rewards/rejected": -0.610888659954071,
      "step": 170
    },
    {
      "epoch": 0.09488666315234581,
      "grad_norm": 121.68835008729005,
      "learning_rate": 9.764101212440694e-07,
      "logits/chosen": -2.091015577316284,
      "logits/rejected": -2.022656202316284,
      "logps/chosen": -379.7250061035156,
      "logps/rejected": -341.25,
      "loss": 0.5835,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.12272033840417862,
      "rewards/margins": 0.4968933165073395,
      "rewards/rejected": -0.3742309510707855,
      "step": 180
    },
    {
      "epoch": 0.10015814443858724,
      "grad_norm": 98.80851216154285,
      "learning_rate": 9.750922509225091e-07,
      "logits/chosen": -2.2083983421325684,
      "logits/rejected": -2.1748046875,
      "logps/chosen": -354.9750061035156,
      "logps/rejected": -336.38751220703125,
      "loss": 0.582,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.23486328125,
      "rewards/margins": 0.46875,
      "rewards/rejected": -0.23398132622241974,
      "step": 190
    },
    {
      "epoch": 0.10542962572482868,
      "grad_norm": 114.72712318227737,
      "learning_rate": 9.737743806009488e-07,
      "logits/chosen": -2.175976514816284,
      "logits/rejected": -2.1748046875,
      "logps/chosen": -379.07501220703125,
      "logps/rejected": -338.67498779296875,
      "loss": 0.6115,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.415200799703598,
      "rewards/margins": 0.4037948548793793,
      "rewards/rejected": 0.01155242882668972,
      "step": 200
    },
    {
      "epoch": 0.11070110701107011,
      "grad_norm": 85.39038197572455,
      "learning_rate": 9.724565102793885e-07,
      "logits/chosen": -2.2164063453674316,
      "logits/rejected": -2.121875047683716,
      "logps/chosen": -376.32501220703125,
      "logps/rejected": -356.63751220703125,
      "loss": 0.5698,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.6265503168106079,
      "rewards/margins": 0.543444812297821,
      "rewards/rejected": 0.0831298828125,
      "step": 210
    },
    {
      "epoch": 0.11597258829731154,
      "grad_norm": 154.40066488948165,
      "learning_rate": 9.71138639957828e-07,
      "logits/chosen": -2.199023485183716,
      "logits/rejected": -2.174023389816284,
      "logps/chosen": -410.95001220703125,
      "logps/rejected": -387.75,
      "loss": 0.6622,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.529309093952179,
      "rewards/margins": 0.2861572206020355,
      "rewards/rejected": 0.24324341118335724,
      "step": 220
    },
    {
      "epoch": 0.12124406958355298,
      "grad_norm": 113.59093862540607,
      "learning_rate": 9.698207696362677e-07,
      "logits/chosen": -2.1810545921325684,
      "logits/rejected": -2.133984327316284,
      "logps/chosen": -405.4375,
      "logps/rejected": -390.70001220703125,
      "loss": 0.5619,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.6037842035293579,
      "rewards/margins": 0.5692504644393921,
      "rewards/rejected": 0.03437500074505806,
      "step": 230
    },
    {
      "epoch": 0.1265155508697944,
      "grad_norm": 97.24837217247598,
      "learning_rate": 9.685028993147074e-07,
      "logits/chosen": -2.1050782203674316,
      "logits/rejected": -2.136523485183716,
      "logps/chosen": -357.0,
      "logps/rejected": -338.8125,
      "loss": 0.6476,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.521228015422821,
      "rewards/margins": 0.43751221895217896,
      "rewards/rejected": 0.08436278998851776,
      "step": 240
    },
    {
      "epoch": 0.13178703215603585,
      "grad_norm": 111.30703332298172,
      "learning_rate": 9.671850289931471e-07,
      "logits/chosen": -2.0757813453674316,
      "logits/rejected": -2.028125047683716,
      "logps/chosen": -383.8999938964844,
      "logps/rejected": -328.54998779296875,
      "loss": 0.6035,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.42667847871780396,
      "rewards/margins": 0.4700256288051605,
      "rewards/rejected": -0.0432586669921875,
      "step": 250
    },
    {
      "epoch": 0.1370585134422773,
      "grad_norm": 110.50864617903044,
      "learning_rate": 9.658671586715866e-07,
      "logits/chosen": -2.2269530296325684,
      "logits/rejected": -2.1285157203674316,
      "logps/chosen": -451.6499938964844,
      "logps/rejected": -398.29998779296875,
      "loss": 0.5471,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.2375228852033615,
      "rewards/margins": 0.6149047613143921,
      "rewards/rejected": -0.37718504667282104,
      "step": 260
    },
    {
      "epoch": 0.1423299947285187,
      "grad_norm": 71.54222933771291,
      "learning_rate": 9.645492883500263e-07,
      "logits/chosen": -2.177734375,
      "logits/rejected": -2.1763672828674316,
      "logps/chosen": -364.51251220703125,
      "logps/rejected": -327.4624938964844,
      "loss": 0.5886,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.1816360503435135,
      "rewards/margins": 0.5497192144393921,
      "rewards/rejected": -0.730926513671875,
      "step": 270
    },
    {
      "epoch": 0.14760147601476015,
      "grad_norm": 150.37776446545666,
      "learning_rate": 9.63231418028466e-07,
      "logits/chosen": -2.1298828125,
      "logits/rejected": -2.100390672683716,
      "logps/chosen": -368.1875,
      "logps/rejected": -338.1499938964844,
      "loss": 0.6093,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.05333862453699112,
      "rewards/margins": 0.541552722454071,
      "rewards/rejected": -0.48811644315719604,
      "step": 280
    },
    {
      "epoch": 0.1528729573010016,
      "grad_norm": 106.67844762585126,
      "learning_rate": 9.619135477069055e-07,
      "logits/chosen": -2.090625047683716,
      "logits/rejected": -2.070507764816284,
      "logps/chosen": -373.1625061035156,
      "logps/rejected": -336.5375061035156,
      "loss": 0.5214,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.4401412904262543,
      "rewards/margins": 0.72772216796875,
      "rewards/rejected": -0.2877670228481293,
      "step": 290
    },
    {
      "epoch": 0.158144438587243,
      "grad_norm": 130.31423388842992,
      "learning_rate": 9.605956773853452e-07,
      "logits/chosen": -2.1070313453674316,
      "logits/rejected": -2.020800828933716,
      "logps/chosen": -347.23748779296875,
      "logps/rejected": -336.42498779296875,
      "loss": 0.6021,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.2332916259765625,
      "rewards/margins": 0.533496081829071,
      "rewards/rejected": -0.29980164766311646,
      "step": 300
    },
    {
      "epoch": 0.16341591987348444,
      "grad_norm": 75.33013100273158,
      "learning_rate": 9.59277807063785e-07,
      "logits/chosen": -2.1513671875,
      "logits/rejected": -2.1318359375,
      "logps/chosen": -437.8500061035156,
      "logps/rejected": -400.5249938964844,
      "loss": 0.5415,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.04572753980755806,
      "rewards/margins": 0.6412903070449829,
      "rewards/rejected": -0.596331775188446,
      "step": 310
    },
    {
      "epoch": 0.16868740115972589,
      "grad_norm": 105.4574194867471,
      "learning_rate": 9.579599367422246e-07,
      "logits/chosen": -1.9958984851837158,
      "logits/rejected": -2.054882764816284,
      "logps/chosen": -410.73748779296875,
      "logps/rejected": -342.7749938964844,
      "loss": 0.5328,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.19996948540210724,
      "rewards/margins": 0.74224853515625,
      "rewards/rejected": -0.941986083984375,
      "step": 320
    },
    {
      "epoch": 0.17395888244596733,
      "grad_norm": 130.68923526073365,
      "learning_rate": 9.56642066420664e-07,
      "logits/chosen": -2.1473631858825684,
      "logits/rejected": -2.014453172683716,
      "logps/chosen": -388.7250061035156,
      "logps/rejected": -370.82501220703125,
      "loss": 0.6207,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.3050537109375,
      "rewards/margins": 0.630603015422821,
      "rewards/rejected": -0.9354248046875,
      "step": 330
    },
    {
      "epoch": 0.17923036373220874,
      "grad_norm": 120.24309478108884,
      "learning_rate": 9.553241960991038e-07,
      "logits/chosen": -2.150585889816284,
      "logits/rejected": -2.1021485328674316,
      "logps/chosen": -365.04998779296875,
      "logps/rejected": -338.95001220703125,
      "loss": 0.5686,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06582260131835938,
      "rewards/margins": 0.6565917730331421,
      "rewards/rejected": -0.722735583782196,
      "step": 340
    },
    {
      "epoch": 0.18450184501845018,
      "grad_norm": 93.06443179537182,
      "learning_rate": 9.540063257775435e-07,
      "logits/chosen": -2.046679735183716,
      "logits/rejected": -2.037304639816284,
      "logps/chosen": -382.86248779296875,
      "logps/rejected": -351.73748779296875,
      "loss": 0.6208,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.21924133598804474,
      "rewards/margins": 0.5395873785018921,
      "rewards/rejected": -0.319509893655777,
      "step": 350
    },
    {
      "epoch": 0.18977332630469163,
      "grad_norm": 99.00481928672006,
      "learning_rate": 9.526884554559831e-07,
      "logits/chosen": -2.1500000953674316,
      "logits/rejected": -2.123046875,
      "logps/chosen": -413.4750061035156,
      "logps/rejected": -357.9125061035156,
      "loss": 0.5824,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.3424316346645355,
      "rewards/margins": 0.6312500238418579,
      "rewards/rejected": -0.28862303495407104,
      "step": 360
    },
    {
      "epoch": 0.19504480759093304,
      "grad_norm": 147.488919923455,
      "learning_rate": 9.513705851344227e-07,
      "logits/chosen": -2.1390624046325684,
      "logits/rejected": -2.0572266578674316,
      "logps/chosen": -402.4750061035156,
      "logps/rejected": -365.125,
      "loss": 0.6291,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.163726806640625,
      "rewards/margins": 0.49139404296875,
      "rewards/rejected": -0.3268798887729645,
      "step": 370
    },
    {
      "epoch": 0.20031628887717448,
      "grad_norm": 111.22807149819624,
      "learning_rate": 9.500527148128624e-07,
      "logits/chosen": -2.051562547683716,
      "logits/rejected": -2.051953077316284,
      "logps/chosen": -463.79998779296875,
      "logps/rejected": -364.4750061035156,
      "loss": 0.5221,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.01689758338034153,
      "rewards/margins": 0.7472778558731079,
      "rewards/rejected": -0.7649596929550171,
      "step": 380
    },
    {
      "epoch": 0.20558777016341592,
      "grad_norm": 129.17682575204753,
      "learning_rate": 9.487348444913021e-07,
      "logits/chosen": -2.0638670921325684,
      "logits/rejected": -1.9744141101837158,
      "logps/chosen": -390.0249938964844,
      "logps/rejected": -339.3999938964844,
      "loss": 0.5956,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.17184142768383026,
      "rewards/margins": 0.618896484375,
      "rewards/rejected": -0.790252685546875,
      "step": 390
    },
    {
      "epoch": 0.21085925144965736,
      "grad_norm": 142.5751022261305,
      "learning_rate": 9.474169741697417e-07,
      "logits/chosen": -2.0933594703674316,
      "logits/rejected": -2.015625,
      "logps/chosen": -408.1499938964844,
      "logps/rejected": -379.92498779296875,
      "loss": 0.6411,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.154632568359375,
      "rewards/margins": 0.511853039264679,
      "rewards/rejected": -0.666638195514679,
      "step": 400
    },
    {
      "epoch": 0.21613073273589878,
      "grad_norm": 131.5564616229884,
      "learning_rate": 9.460991038481813e-07,
      "logits/chosen": -2.0775389671325684,
      "logits/rejected": -2.077929735183716,
      "logps/chosen": -395.9624938964844,
      "logps/rejected": -363.54998779296875,
      "loss": 0.5888,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.19798584282398224,
      "rewards/margins": 0.6507934331893921,
      "rewards/rejected": -0.849108874797821,
      "step": 410
    },
    {
      "epoch": 0.22140221402214022,
      "grad_norm": 118.14534928610438,
      "learning_rate": 9.44781233526621e-07,
      "logits/chosen": -2.079296827316284,
      "logits/rejected": -2.043164014816284,
      "logps/chosen": -416.08123779296875,
      "logps/rejected": -365.3500061035156,
      "loss": 0.5909,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.2545104920864105,
      "rewards/margins": 0.6281982660293579,
      "rewards/rejected": -0.8827148675918579,
      "step": 420
    },
    {
      "epoch": 0.22667369530838166,
      "grad_norm": 86.73408763531067,
      "learning_rate": 9.434633632050606e-07,
      "logits/chosen": -2.0521483421325684,
      "logits/rejected": -2.073046922683716,
      "logps/chosen": -362.125,
      "logps/rejected": -354.375,
      "loss": 0.6375,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.47081297636032104,
      "rewards/margins": 0.46430665254592896,
      "rewards/rejected": -0.934191882610321,
      "step": 430
    },
    {
      "epoch": 0.23194517659462308,
      "grad_norm": 143.37928082755286,
      "learning_rate": 9.421454928835002e-07,
      "logits/chosen": -2.0472655296325684,
      "logits/rejected": -1.984960913658142,
      "logps/chosen": -391.5625,
      "logps/rejected": -337.875,
      "loss": 0.578,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.10753174126148224,
      "rewards/margins": 0.5483642816543579,
      "rewards/rejected": -0.6554321050643921,
      "step": 440
    },
    {
      "epoch": 0.23721665788086452,
      "grad_norm": 138.6483078878509,
      "learning_rate": 9.408276225619399e-07,
      "logits/chosen": -1.9992187023162842,
      "logits/rejected": -1.996191382408142,
      "logps/chosen": -420.76873779296875,
      "logps/rejected": -371.54998779296875,
      "loss": 0.5251,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.07990112155675888,
      "rewards/margins": 0.76324462890625,
      "rewards/rejected": -0.6834350824356079,
      "step": 450
    },
    {
      "epoch": 0.24248813916710596,
      "grad_norm": 110.00997255707976,
      "learning_rate": 9.395097522403796e-07,
      "logits/chosen": -2.0013670921325684,
      "logits/rejected": -1.9572265148162842,
      "logps/chosen": -382.76251220703125,
      "logps/rejected": -340.9375,
      "loss": 0.6031,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.04163818433880806,
      "rewards/margins": 0.570935070514679,
      "rewards/rejected": -0.61285400390625,
      "step": 460
    },
    {
      "epoch": 0.2477596204533474,
      "grad_norm": 132.32461757276212,
      "learning_rate": 9.381918819188192e-07,
      "logits/chosen": -1.9689452648162842,
      "logits/rejected": -1.9187500476837158,
      "logps/chosen": -383.7875061035156,
      "logps/rejected": -361.5874938964844,
      "loss": 0.6285,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.335479736328125,
      "rewards/margins": 0.5303710699081421,
      "rewards/rejected": -0.8660522699356079,
      "step": 470
    },
    {
      "epoch": 0.2530311017395888,
      "grad_norm": 119.80220651019971,
      "learning_rate": 9.368740115972587e-07,
      "logits/chosen": -2.036914110183716,
      "logits/rejected": -1.9796874523162842,
      "logps/chosen": -381.98748779296875,
      "logps/rejected": -359.45001220703125,
      "loss": 0.6154,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.150848388671875,
      "rewards/margins": 0.553234875202179,
      "rewards/rejected": -0.7035156488418579,
      "step": 480
    },
    {
      "epoch": 0.25830258302583026,
      "grad_norm": 124.92515030289202,
      "learning_rate": 9.355561412756983e-07,
      "logits/chosen": -2.0341796875,
      "logits/rejected": -2.048046827316284,
      "logps/chosen": -450.3999938964844,
      "logps/rejected": -389.875,
      "loss": 0.609,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.4446777403354645,
      "rewards/margins": 0.5486694574356079,
      "rewards/rejected": -0.103302001953125,
      "step": 490
    },
    {
      "epoch": 0.2635740643120717,
      "grad_norm": 129.50230614998048,
      "learning_rate": 9.34238270954138e-07,
      "logits/chosen": -1.9755859375,
      "logits/rejected": -1.9822266101837158,
      "logps/chosen": -396.8125,
      "logps/rejected": -377.79998779296875,
      "loss": 0.6146,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": 0.4496826231479645,
      "rewards/margins": 0.4946655333042145,
      "rewards/rejected": -0.0455322265625,
      "step": 500
    },
    {
      "epoch": 0.26884554559831314,
      "grad_norm": 85.26588671544069,
      "learning_rate": 9.329204006325777e-07,
      "logits/chosen": -1.9425780773162842,
      "logits/rejected": -1.9582030773162842,
      "logps/chosen": -401.2749938964844,
      "logps/rejected": -331.70001220703125,
      "loss": 0.5371,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.625445544719696,
      "rewards/margins": 0.6556640863418579,
      "rewards/rejected": -0.03047485277056694,
      "step": 510
    },
    {
      "epoch": 0.2741170268845546,
      "grad_norm": 105.23092955778965,
      "learning_rate": 9.316025303110173e-07,
      "logits/chosen": -1.941015601158142,
      "logits/rejected": -1.9978516101837158,
      "logps/chosen": -435.29998779296875,
      "logps/rejected": -374.7749938964844,
      "loss": 0.5975,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.31364744901657104,
      "rewards/margins": 0.6072021722793579,
      "rewards/rejected": -0.29353028535842896,
      "step": 520
    },
    {
      "epoch": 0.27938850817079597,
      "grad_norm": 144.5473161075403,
      "learning_rate": 9.30284659989457e-07,
      "logits/chosen": -2.043164014816284,
      "logits/rejected": -1.935156226158142,
      "logps/chosen": -407.07501220703125,
      "logps/rejected": -354.70001220703125,
      "loss": 0.5689,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.10639343410730362,
      "rewards/margins": 0.7123001217842102,
      "rewards/rejected": -0.6069915890693665,
      "step": 530
    },
    {
      "epoch": 0.2846599894570374,
      "grad_norm": 91.77376675532808,
      "learning_rate": 9.289667896678966e-07,
      "logits/chosen": -1.9597656726837158,
      "logits/rejected": -1.875390648841858,
      "logps/chosen": -421.25,
      "logps/rejected": -354.2749938964844,
      "loss": 0.5368,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.16525879502296448,
      "rewards/margins": 0.809619128704071,
      "rewards/rejected": -0.644848644733429,
      "step": 540
    },
    {
      "epoch": 0.28993147074327885,
      "grad_norm": 112.39626880115516,
      "learning_rate": 9.276489193463362e-07,
      "logits/chosen": -2.0263671875,
      "logits/rejected": -1.939843773841858,
      "logps/chosen": -429.125,
      "logps/rejected": -384.67498779296875,
      "loss": 0.5285,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.02728424035012722,
      "rewards/margins": 0.8555663824081421,
      "rewards/rejected": -0.8282715082168579,
      "step": 550
    },
    {
      "epoch": 0.2952029520295203,
      "grad_norm": 97.54817876080061,
      "learning_rate": 9.263310490247759e-07,
      "logits/chosen": -2.1343750953674316,
      "logits/rejected": -2.1224608421325684,
      "logps/chosen": -403.7250061035156,
      "logps/rejected": -347.6499938964844,
      "loss": 0.558,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.18235473334789276,
      "rewards/margins": 0.611499011516571,
      "rewards/rejected": -0.794146716594696,
      "step": 560
    },
    {
      "epoch": 0.30047443331576174,
      "grad_norm": 103.7205949339287,
      "learning_rate": 9.250131787032156e-07,
      "logits/chosen": -1.981542944908142,
      "logits/rejected": -1.979101538658142,
      "logps/chosen": -397.7749938964844,
      "logps/rejected": -365.67498779296875,
      "loss": 0.5564,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.09096679836511612,
      "rewards/margins": 0.6309814453125,
      "rewards/rejected": -0.7213989496231079,
      "step": 570
    },
    {
      "epoch": 0.3057459146020032,
      "grad_norm": 89.11686565761096,
      "learning_rate": 9.236953083816552e-07,
      "logits/chosen": -2.0572266578674316,
      "logits/rejected": -1.98828125,
      "logps/chosen": -375.51251220703125,
      "logps/rejected": -363.0249938964844,
      "loss": 0.5438,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02515563927590847,
      "rewards/margins": 0.682788074016571,
      "rewards/rejected": -0.707629382610321,
      "step": 580
    },
    {
      "epoch": 0.3110173958882446,
      "grad_norm": 152.33913313152195,
      "learning_rate": 9.223774380600948e-07,
      "logits/chosen": -2.1361327171325684,
      "logits/rejected": -2.169140577316284,
      "logps/chosen": -441.95001220703125,
      "logps/rejected": -387.5,
      "loss": 0.5891,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.225677490234375,
      "rewards/margins": 0.6501220464706421,
      "rewards/rejected": -0.8760010004043579,
      "step": 590
    },
    {
      "epoch": 0.316288877174486,
      "grad_norm": 190.90924995440156,
      "learning_rate": 9.210595677385344e-07,
      "logits/chosen": -2.02490234375,
      "logits/rejected": -2.004687547683716,
      "logps/chosen": -404.75,
      "logps/rejected": -365.20001220703125,
      "loss": 0.6317,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06562499701976776,
      "rewards/margins": 0.48554688692092896,
      "rewards/rejected": -0.55126953125,
      "step": 600
    },
    {
      "epoch": 0.32156035846072745,
      "grad_norm": 121.28819743267138,
      "learning_rate": 9.197416974169741e-07,
      "logits/chosen": -2.0218749046325684,
      "logits/rejected": -2.0306639671325684,
      "logps/chosen": -419.7875061035156,
      "logps/rejected": -386.95001220703125,
      "loss": 0.5993,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.15645751357078552,
      "rewards/margins": 0.619140625,
      "rewards/rejected": -0.46252137422561646,
      "step": 610
    },
    {
      "epoch": 0.3268318397469689,
      "grad_norm": 90.69188677406852,
      "learning_rate": 9.184238270954138e-07,
      "logits/chosen": -2.024609327316284,
      "logits/rejected": -2.033984422683716,
      "logps/chosen": -409.5249938964844,
      "logps/rejected": -364.0249938964844,
      "loss": 0.5699,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.3362793028354645,
      "rewards/margins": 0.6728515625,
      "rewards/rejected": -0.33657073974609375,
      "step": 620
    },
    {
      "epoch": 0.33210332103321033,
      "grad_norm": 114.04196275959744,
      "learning_rate": 9.171059567738534e-07,
      "logits/chosen": -2.0972657203674316,
      "logits/rejected": -2.0667967796325684,
      "logps/chosen": -394.1000061035156,
      "logps/rejected": -339.5249938964844,
      "loss": 0.5429,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.14747925102710724,
      "rewards/margins": 0.679638683795929,
      "rewards/rejected": -0.531890869140625,
      "step": 630
    },
    {
      "epoch": 0.33737480231945177,
      "grad_norm": 111.92501824960604,
      "learning_rate": 9.157880864522931e-07,
      "logits/chosen": -2.004101514816284,
      "logits/rejected": -1.917382836341858,
      "logps/chosen": -391.0375061035156,
      "logps/rejected": -355.7250061035156,
      "loss": 0.5909,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.26812744140625,
      "rewards/margins": 0.6800171136856079,
      "rewards/rejected": -0.4115844666957855,
      "step": 640
    },
    {
      "epoch": 0.3426462836056932,
      "grad_norm": 106.38813893101805,
      "learning_rate": 9.144702161307327e-07,
      "logits/chosen": -2.035937547683716,
      "logits/rejected": -2.048046827316284,
      "logps/chosen": -428.04998779296875,
      "logps/rejected": -391.79998779296875,
      "loss": 0.5029,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.3949218690395355,
      "rewards/margins": 0.8203185796737671,
      "rewards/rejected": -0.42549437284469604,
      "step": 650
    },
    {
      "epoch": 0.34791776489193466,
      "grad_norm": 120.41937282093383,
      "learning_rate": 9.131523458091723e-07,
      "logits/chosen": -2.023242235183716,
      "logits/rejected": -2.049999952316284,
      "logps/chosen": -423.3374938964844,
      "logps/rejected": -374.45001220703125,
      "loss": 0.6312,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.14521484076976776,
      "rewards/margins": 0.576794445514679,
      "rewards/rejected": -0.432180792093277,
      "step": 660
    },
    {
      "epoch": 0.35318924617817604,
      "grad_norm": 112.27854317164666,
      "learning_rate": 9.118344754876119e-07,
      "logits/chosen": -2.072265625,
      "logits/rejected": -2.033398389816284,
      "logps/chosen": -436.04998779296875,
      "logps/rejected": -381.7250061035156,
      "loss": 0.4914,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.21813812851905823,
      "rewards/margins": 1.02294921875,
      "rewards/rejected": -0.804943859577179,
      "step": 670
    },
    {
      "epoch": 0.3584607274644175,
      "grad_norm": 89.33782064200511,
      "learning_rate": 9.105166051660517e-07,
      "logits/chosen": -2.1275391578674316,
      "logits/rejected": -2.074414014816284,
      "logps/chosen": -409.0,
      "logps/rejected": -391.8500061035156,
      "loss": 0.5695,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.16028442978858948,
      "rewards/margins": 0.855517566204071,
      "rewards/rejected": -0.695385754108429,
      "step": 680
    },
    {
      "epoch": 0.3637322087506589,
      "grad_norm": 79.36281643697679,
      "learning_rate": 9.091987348444913e-07,
      "logits/chosen": -2.209179639816284,
      "logits/rejected": -2.1214842796325684,
      "logps/chosen": -397.6625061035156,
      "logps/rejected": -350.25,
      "loss": 0.5358,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.09719695895910263,
      "rewards/margins": 0.8172851800918579,
      "rewards/rejected": -0.720874011516571,
      "step": 690
    },
    {
      "epoch": 0.36900369003690037,
      "grad_norm": 107.92273061105814,
      "learning_rate": 9.078808645229309e-07,
      "logits/chosen": -2.154296875,
      "logits/rejected": -2.1796875,
      "logps/chosen": -427.29998779296875,
      "logps/rejected": -369.1000061035156,
      "loss": 0.5667,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.13243713974952698,
      "rewards/margins": 0.66546630859375,
      "rewards/rejected": -0.7978271245956421,
      "step": 700
    },
    {
      "epoch": 0.3742751713231418,
      "grad_norm": 130.01277912938113,
      "learning_rate": 9.065629942013705e-07,
      "logits/chosen": -2.200390577316284,
      "logits/rejected": -2.084765672683716,
      "logps/chosen": -371.8500061035156,
      "logps/rejected": -357.04376220703125,
      "loss": 0.5699,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.4560546875,
      "rewards/margins": 0.646484375,
      "rewards/rejected": -1.1027343273162842,
      "step": 710
    },
    {
      "epoch": 0.37954665260938325,
      "grad_norm": 90.74467154501966,
      "learning_rate": 9.052451238798102e-07,
      "logits/chosen": -2.1142578125,
      "logits/rejected": -2.091796875,
      "logps/chosen": -392.1875,
      "logps/rejected": -354.04998779296875,
      "loss": 0.5877,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.575665295124054,
      "rewards/margins": 0.778332531452179,
      "rewards/rejected": -1.3542969226837158,
      "step": 720
    },
    {
      "epoch": 0.3848181338956247,
      "grad_norm": 110.65783976498547,
      "learning_rate": 9.039272535582499e-07,
      "logits/chosen": -2.091992139816284,
      "logits/rejected": -2.0814452171325684,
      "logps/chosen": -419.2250061035156,
      "logps/rejected": -385.25,
      "loss": 0.5755,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.6411803960800171,
      "rewards/margins": 0.6994994878768921,
      "rewards/rejected": -1.3402831554412842,
      "step": 730
    },
    {
      "epoch": 0.3900896151818661,
      "grad_norm": 116.98549186192578,
      "learning_rate": 9.026093832366895e-07,
      "logits/chosen": -2.151171922683716,
      "logits/rejected": -2.0947265625,
      "logps/chosen": -374.125,
      "logps/rejected": -359.4750061035156,
      "loss": 0.5016,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.5179382562637329,
      "rewards/margins": 0.8989197015762329,
      "rewards/rejected": -1.4166991710662842,
      "step": 740
    },
    {
      "epoch": 0.3953610964681075,
      "grad_norm": 144.11429200893863,
      "learning_rate": 9.012915129151291e-07,
      "logits/chosen": -2.091992139816284,
      "logits/rejected": -2.070507764816284,
      "logps/chosen": -425.1000061035156,
      "logps/rejected": -370.375,
      "loss": 0.6018,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.9186156988143921,
      "rewards/margins": 0.67181396484375,
      "rewards/rejected": -1.58984375,
      "step": 750
    },
    {
      "epoch": 0.40063257775434896,
      "grad_norm": 89.98147746112576,
      "learning_rate": 8.999736425935688e-07,
      "logits/chosen": -2.1302733421325684,
      "logits/rejected": -2.185742139816284,
      "logps/chosen": -387.4624938964844,
      "logps/rejected": -340.2749938964844,
      "loss": 0.5603,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.677136242389679,
      "rewards/margins": 0.8334289789199829,
      "rewards/rejected": -1.5107421875,
      "step": 760
    },
    {
      "epoch": 0.4059040590405904,
      "grad_norm": 113.99281914843293,
      "learning_rate": 8.986557722720084e-07,
      "logits/chosen": -2.0210938453674316,
      "logits/rejected": -2.047070264816284,
      "logps/chosen": -366.4375,
      "logps/rejected": -356.3500061035156,
      "loss": 0.4959,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.15304259955883026,
      "rewards/margins": 1.0658782720565796,
      "rewards/rejected": -1.2184569835662842,
      "step": 770
    },
    {
      "epoch": 0.41117554032683185,
      "grad_norm": 125.58864088734396,
      "learning_rate": 8.97337901950448e-07,
      "logits/chosen": -2.0347657203674316,
      "logits/rejected": -2.042187452316284,
      "logps/chosen": -390.5,
      "logps/rejected": -347.67498779296875,
      "loss": 0.5192,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.13743285834789276,
      "rewards/margins": 0.945025622844696,
      "rewards/rejected": -1.082678198814392,
      "step": 780
    },
    {
      "epoch": 0.4164470216130733,
      "grad_norm": 102.24864364738528,
      "learning_rate": 8.960200316288878e-07,
      "logits/chosen": -1.9523437023162842,
      "logits/rejected": -1.896093726158142,
      "logps/chosen": -387.2749938964844,
      "logps/rejected": -374.625,
      "loss": 0.5316,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07953186333179474,
      "rewards/margins": 0.9917358160018921,
      "rewards/rejected": -1.0704224109649658,
      "step": 790
    },
    {
      "epoch": 0.42171850289931473,
      "grad_norm": 96.11787572230425,
      "learning_rate": 8.947021613073274e-07,
      "logits/chosen": -2.119921922683716,
      "logits/rejected": -2.0326170921325684,
      "logps/chosen": -385.11248779296875,
      "logps/rejected": -372.01251220703125,
      "loss": 0.5356,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.01309814490377903,
      "rewards/margins": 0.880291759967804,
      "rewards/rejected": -0.867449939250946,
      "step": 800
    },
    {
      "epoch": 0.4269899841855561,
      "grad_norm": 82.7037769568601,
      "learning_rate": 8.93384290985767e-07,
      "logits/chosen": -2.183789014816284,
      "logits/rejected": -2.1708984375,
      "logps/chosen": -403.375,
      "logps/rejected": -333.82501220703125,
      "loss": 0.6012,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.27265626192092896,
      "rewards/margins": 0.7476867437362671,
      "rewards/rejected": -0.47436219453811646,
      "step": 810
    },
    {
      "epoch": 0.43226146547179756,
      "grad_norm": 90.73650896490028,
      "learning_rate": 8.920664206642066e-07,
      "logits/chosen": -2.1041016578674316,
      "logits/rejected": -2.0888671875,
      "logps/chosen": -397.3999938964844,
      "logps/rejected": -348.4750061035156,
      "loss": 0.5655,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.5527709722518921,
      "rewards/margins": 0.738751232624054,
      "rewards/rejected": -0.18546143174171448,
      "step": 820
    },
    {
      "epoch": 0.437532946758039,
      "grad_norm": 111.97673327111477,
      "learning_rate": 8.907485503426463e-07,
      "logits/chosen": -2.1011719703674316,
      "logits/rejected": -2.1060547828674316,
      "logps/chosen": -437.42498779296875,
      "logps/rejected": -386.45001220703125,
      "loss": 0.5625,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.6082671880722046,
      "rewards/margins": 0.7829955816268921,
      "rewards/rejected": -0.17441406846046448,
      "step": 830
    },
    {
      "epoch": 0.44280442804428044,
      "grad_norm": 115.0530070594459,
      "learning_rate": 8.894306800210858e-07,
      "logits/chosen": -2.053906202316284,
      "logits/rejected": -2.0,
      "logps/chosen": -349.42498779296875,
      "logps/rejected": -336.1000061035156,
      "loss": 0.6232,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.351470947265625,
      "rewards/margins": 0.590222179889679,
      "rewards/rejected": -0.23827210068702698,
      "step": 840
    },
    {
      "epoch": 0.4480759093305219,
      "grad_norm": 94.38953331596515,
      "learning_rate": 8.881128096995255e-07,
      "logits/chosen": -2.033007860183716,
      "logits/rejected": -2.029296875,
      "logps/chosen": -419.70001220703125,
      "logps/rejected": -365.7250061035156,
      "loss": 0.5066,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.496896356344223,
      "rewards/margins": 0.9481872320175171,
      "rewards/rejected": -0.451028436422348,
      "step": 850
    },
    {
      "epoch": 0.4533473906167633,
      "grad_norm": 154.7372764324159,
      "learning_rate": 8.867949393779651e-07,
      "logits/chosen": -2.0316405296325684,
      "logits/rejected": -2.03515625,
      "logps/chosen": -402.7749938964844,
      "logps/rejected": -384.0249938964844,
      "loss": 0.6206,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.26762694120407104,
      "rewards/margins": 0.6639404296875,
      "rewards/rejected": -0.396200567483902,
      "step": 860
    },
    {
      "epoch": 0.45861887190300477,
      "grad_norm": 157.1427246594179,
      "learning_rate": 8.854770690564048e-07,
      "logits/chosen": -2.007031202316284,
      "logits/rejected": -2.0472655296325684,
      "logps/chosen": -379.75,
      "logps/rejected": -353.9125061035156,
      "loss": 0.5031,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.3580261170864105,
      "rewards/margins": 0.876721203327179,
      "rewards/rejected": -0.51800537109375,
      "step": 870
    },
    {
      "epoch": 0.46389035318924615,
      "grad_norm": 118.97692154117517,
      "learning_rate": 8.841591987348444e-07,
      "logits/chosen": -2.0908203125,
      "logits/rejected": -2.0521483421325684,
      "logps/chosen": -392.82501220703125,
      "logps/rejected": -371.9750061035156,
      "loss": 0.5804,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.2904724180698395,
      "rewards/margins": 0.7400146722793579,
      "rewards/rejected": -0.44976806640625,
      "step": 880
    },
    {
      "epoch": 0.4691618344754876,
      "grad_norm": 77.24764539029883,
      "learning_rate": 8.82841328413284e-07,
      "logits/chosen": -2.072265625,
      "logits/rejected": -2.085156202316284,
      "logps/chosen": -414.61248779296875,
      "logps/rejected": -354.125,
      "loss": 0.521,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.33781129121780396,
      "rewards/margins": 0.8510986566543579,
      "rewards/rejected": -0.5130981206893921,
      "step": 890
    },
    {
      "epoch": 0.47443331576172904,
      "grad_norm": 87.07453755381584,
      "learning_rate": 8.815234580917237e-07,
      "logits/chosen": -2.117968797683716,
      "logits/rejected": -2.105273485183716,
      "logps/chosen": -368.4375,
      "logps/rejected": -347.9750061035156,
      "loss": 0.5974,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": 0.14063720405101776,
      "rewards/margins": 0.6573241949081421,
      "rewards/rejected": -0.5172363519668579,
      "step": 900
    },
    {
      "epoch": 0.4797047970479705,
      "grad_norm": 64.56351613260088,
      "learning_rate": 8.802055877701634e-07,
      "logits/chosen": -2.0674805641174316,
      "logits/rejected": -2.0414061546325684,
      "logps/chosen": -375.4624938964844,
      "logps/rejected": -353.0,
      "loss": 0.6078,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.17872008681297302,
      "rewards/margins": 0.630719006061554,
      "rewards/rejected": -0.451324462890625,
      "step": 910
    },
    {
      "epoch": 0.4849762783342119,
      "grad_norm": 128.62361257188385,
      "learning_rate": 8.78887717448603e-07,
      "logits/chosen": -2.0982422828674316,
      "logits/rejected": -2.0406250953674316,
      "logps/chosen": -376.0874938964844,
      "logps/rejected": -371.2250061035156,
      "loss": 0.5874,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.31085205078125,
      "rewards/margins": 0.718859851360321,
      "rewards/rejected": -0.40812987089157104,
      "step": 920
    },
    {
      "epoch": 0.49024775962045336,
      "grad_norm": 113.07425677618353,
      "learning_rate": 8.775698471270426e-07,
      "logits/chosen": -2.1214842796325684,
      "logits/rejected": -2.066210985183716,
      "logps/chosen": -328.4750061035156,
      "logps/rejected": -313.57501220703125,
      "loss": 0.5677,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.12265624850988388,
      "rewards/margins": 0.6986328363418579,
      "rewards/rejected": -0.5755615234375,
      "step": 930
    },
    {
      "epoch": 0.4955192409066948,
      "grad_norm": 102.88377422390226,
      "learning_rate": 8.762519768054823e-07,
      "logits/chosen": -2.1519532203674316,
      "logits/rejected": -2.0943360328674316,
      "logps/chosen": -368.57501220703125,
      "logps/rejected": -357.9750061035156,
      "loss": 0.5858,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.12027587741613388,
      "rewards/margins": 0.6277221441268921,
      "rewards/rejected": -0.5073181390762329,
      "step": 940
    },
    {
      "epoch": 0.5007907221929362,
      "grad_norm": 92.60149117401427,
      "learning_rate": 8.749341064839219e-07,
      "logits/chosen": -2.1758790016174316,
      "logits/rejected": -2.189257860183716,
      "logps/chosen": -423.3999938964844,
      "logps/rejected": -397.125,
      "loss": 0.5591,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.039947509765625,
      "rewards/margins": 0.811230480670929,
      "rewards/rejected": -0.7712157964706421,
      "step": 950
    },
    {
      "epoch": 0.5060622034791776,
      "grad_norm": 92.74831510948387,
      "learning_rate": 8.736162361623616e-07,
      "logits/chosen": -2.0931639671325684,
      "logits/rejected": -2.081249952316284,
      "logps/chosen": -425.3500061035156,
      "logps/rejected": -390.875,
      "loss": 0.511,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.20562744140625,
      "rewards/margins": 0.87255859375,
      "rewards/rejected": -1.0796997547149658,
      "step": 960
    },
    {
      "epoch": 0.5113336847654191,
      "grad_norm": 107.28019086876711,
      "learning_rate": 8.722983658408012e-07,
      "logits/chosen": -2.136523485183716,
      "logits/rejected": -2.091796875,
      "logps/chosen": -374.9624938964844,
      "logps/rejected": -352.1875,
      "loss": 0.581,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.197052001953125,
      "rewards/margins": 0.6382995843887329,
      "rewards/rejected": -0.835705578327179,
      "step": 970
    },
    {
      "epoch": 0.5166051660516605,
      "grad_norm": 75.490491165349,
      "learning_rate": 8.709804955192409e-07,
      "logits/chosen": -2.1302733421325684,
      "logits/rejected": -2.167187452316284,
      "logps/chosen": -394.20001220703125,
      "logps/rejected": -352.0874938964844,
      "loss": 0.5434,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.01966247521340847,
      "rewards/margins": 0.851550281047821,
      "rewards/rejected": -0.831494152545929,
      "step": 980
    },
    {
      "epoch": 0.521876647337902,
      "grad_norm": 118.38994084918315,
      "learning_rate": 8.696626251976805e-07,
      "logits/chosen": -2.123046875,
      "logits/rejected": -2.042187452316284,
      "logps/chosen": -354.70001220703125,
      "logps/rejected": -340.42498779296875,
      "loss": 0.4913,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07682494819164276,
      "rewards/margins": 1.009033203125,
      "rewards/rejected": -1.084570288658142,
      "step": 990
    },
    {
      "epoch": 0.5271481286241434,
      "grad_norm": 105.76577340001352,
      "learning_rate": 8.683447548761201e-07,
      "logits/chosen": -2.131640672683716,
      "logits/rejected": -2.0328125953674316,
      "logps/chosen": -384.7875061035156,
      "logps/rejected": -382.375,
      "loss": 0.6369,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.4345336854457855,
      "rewards/margins": 0.7616943120956421,
      "rewards/rejected": -1.197149634361267,
      "step": 1000
    },
    {
      "epoch": 0.5324196099103848,
      "grad_norm": 83.18615572971608,
      "learning_rate": 8.670268845545597e-07,
      "logits/chosen": -2.1800780296325684,
      "logits/rejected": -2.1205077171325684,
      "logps/chosen": -426.92498779296875,
      "logps/rejected": -369.5,
      "loss": 0.563,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.871630847454071,
      "rewards/margins": 0.809863269329071,
      "rewards/rejected": -1.680273413658142,
      "step": 1010
    },
    {
      "epoch": 0.5376910911966263,
      "grad_norm": 115.23782298990187,
      "learning_rate": 8.657090142329995e-07,
      "logits/chosen": -2.145312547683716,
      "logits/rejected": -2.0204100608825684,
      "logps/chosen": -367.0249938964844,
      "logps/rejected": -321.1875,
      "loss": 0.5203,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.89947509765625,
      "rewards/margins": 0.8373779058456421,
      "rewards/rejected": -1.7365233898162842,
      "step": 1020
    },
    {
      "epoch": 0.5429625724828677,
      "grad_norm": 83.73144442554737,
      "learning_rate": 8.643911439114391e-07,
      "logits/chosen": -2.167187452316284,
      "logits/rejected": -2.1484375,
      "logps/chosen": -426.2749938964844,
      "logps/rejected": -391.625,
      "loss": 0.5835,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.861053466796875,
      "rewards/margins": 0.847399890422821,
      "rewards/rejected": -1.7077147960662842,
      "step": 1030
    },
    {
      "epoch": 0.5482340537691092,
      "grad_norm": 97.96584177153517,
      "learning_rate": 8.630732735898787e-07,
      "logits/chosen": -2.0712890625,
      "logits/rejected": -2.0308594703674316,
      "logps/chosen": -387.0249938964844,
      "logps/rejected": -372.0874938964844,
      "loss": 0.5842,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.9025024175643921,
      "rewards/margins": 0.7254638671875,
      "rewards/rejected": -1.6281249523162842,
      "step": 1040
    },
    {
      "epoch": 0.5535055350553506,
      "grad_norm": 67.41789305046882,
      "learning_rate": 8.617554032683183e-07,
      "logits/chosen": -2.097851514816284,
      "logits/rejected": -2.0873045921325684,
      "logps/chosen": -348.2124938964844,
      "logps/rejected": -339.92498779296875,
      "loss": 0.6575,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.0059630870819092,
      "rewards/margins": 0.6885741949081421,
      "rewards/rejected": -1.6950194835662842,
      "step": 1050
    },
    {
      "epoch": 0.5587770163415919,
      "grad_norm": 148.6441117773589,
      "learning_rate": 8.60437532946758e-07,
      "logits/chosen": -2.095703125,
      "logits/rejected": -2.0810546875,
      "logps/chosen": -396.57501220703125,
      "logps/rejected": -336.73748779296875,
      "loss": 0.6339,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.6324828863143921,
      "rewards/margins": 0.6246887445449829,
      "rewards/rejected": -1.257568359375,
      "step": 1060
    },
    {
      "epoch": 0.5640484976278334,
      "grad_norm": 95.62387640678155,
      "learning_rate": 8.591196626251977e-07,
      "logits/chosen": -2.1624999046325684,
      "logits/rejected": -2.074023485183716,
      "logps/chosen": -411.9125061035156,
      "logps/rejected": -382.9125061035156,
      "loss": 0.5529,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.572265625,
      "rewards/margins": 0.7801879644393921,
      "rewards/rejected": -1.352941870689392,
      "step": 1070
    },
    {
      "epoch": 0.5693199789140748,
      "grad_norm": 108.78417955389374,
      "learning_rate": 8.578017923036373e-07,
      "logits/chosen": -2.0689454078674316,
      "logits/rejected": -2.009570360183716,
      "logps/chosen": -407.0249938964844,
      "logps/rejected": -406.8999938964844,
      "loss": 0.5153,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.46522217988967896,
      "rewards/margins": 0.889086902141571,
      "rewards/rejected": -1.354089379310608,
      "step": 1080
    },
    {
      "epoch": 0.5745914602003163,
      "grad_norm": 93.52336721192782,
      "learning_rate": 8.56483921982077e-07,
      "logits/chosen": -2.091015577316284,
      "logits/rejected": -2.0726561546325684,
      "logps/chosen": -364.3500061035156,
      "logps/rejected": -358.07501220703125,
      "loss": 0.5891,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.690521240234375,
      "rewards/margins": 0.6814819574356079,
      "rewards/rejected": -1.371789574623108,
      "step": 1090
    },
    {
      "epoch": 0.5798629414865577,
      "grad_norm": 120.27548598955995,
      "learning_rate": 8.551660516605166e-07,
      "logits/chosen": -2.090625047683716,
      "logits/rejected": -2.0628905296325684,
      "logps/chosen": -408.07501220703125,
      "logps/rejected": -364.375,
      "loss": 0.6102,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.756518542766571,
      "rewards/margins": 0.671704113483429,
      "rewards/rejected": -1.4287109375,
      "step": 1100
    },
    {
      "epoch": 0.5851344227727991,
      "grad_norm": 106.55462836971667,
      "learning_rate": 8.538481813389562e-07,
      "logits/chosen": -2.1187500953674316,
      "logits/rejected": -2.170117139816284,
      "logps/chosen": -396.3500061035156,
      "logps/rejected": -351.6875,
      "loss": 0.6435,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.580554187297821,
      "rewards/margins": 0.6236206293106079,
      "rewards/rejected": -1.2035644054412842,
      "step": 1110
    },
    {
      "epoch": 0.5904059040590406,
      "grad_norm": 116.57400237794332,
      "learning_rate": 8.525303110173958e-07,
      "logits/chosen": -2.0859375,
      "logits/rejected": -2.0101561546325684,
      "logps/chosen": -367.1875,
      "logps/rejected": -341.88751220703125,
      "loss": 0.5755,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.42731934785842896,
      "rewards/margins": 0.672686755657196,
      "rewards/rejected": -1.100122094154358,
      "step": 1120
    },
    {
      "epoch": 0.595677385345282,
      "grad_norm": 78.47097911906096,
      "learning_rate": 8.512124406958356e-07,
      "logits/chosen": -1.9480469226837158,
      "logits/rejected": -2.015429735183716,
      "logps/chosen": -400.26251220703125,
      "logps/rejected": -339.70001220703125,
      "loss": 0.6124,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.05307617038488388,
      "rewards/margins": 0.753161609172821,
      "rewards/rejected": -0.700146496295929,
      "step": 1130
    },
    {
      "epoch": 0.6009488666315235,
      "grad_norm": 103.78595791992363,
      "learning_rate": 8.498945703742752e-07,
      "logits/chosen": -1.976171851158142,
      "logits/rejected": -1.9402344226837158,
      "logps/chosen": -391.6875,
      "logps/rejected": -370.5375061035156,
      "loss": 0.6403,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.27708739042282104,
      "rewards/margins": 0.655261218547821,
      "rewards/rejected": -0.37904053926467896,
      "step": 1140
    },
    {
      "epoch": 0.6062203479177649,
      "grad_norm": 120.03313774123582,
      "learning_rate": 8.485767000527148e-07,
      "logits/chosen": -2.092578172683716,
      "logits/rejected": -2.06640625,
      "logps/chosen": -357.4750061035156,
      "logps/rejected": -365.45001220703125,
      "loss": 0.5967,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.3886474668979645,
      "rewards/margins": 0.6645752191543579,
      "rewards/rejected": -0.2770523130893707,
      "step": 1150
    },
    {
      "epoch": 0.6114918292040064,
      "grad_norm": 108.3332207699811,
      "learning_rate": 8.472588297311544e-07,
      "logits/chosen": -2.0189452171325684,
      "logits/rejected": -2.0980467796325684,
      "logps/chosen": -422.5,
      "logps/rejected": -369.0249938964844,
      "loss": 0.5588,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.30269163846969604,
      "rewards/margins": 0.908923327922821,
      "rewards/rejected": -0.60540771484375,
      "step": 1160
    },
    {
      "epoch": 0.6167633104902478,
      "grad_norm": 91.43827148165173,
      "learning_rate": 8.459409594095941e-07,
      "logits/chosen": -2.071484327316284,
      "logits/rejected": -1.999414086341858,
      "logps/chosen": -380.3999938964844,
      "logps/rejected": -366.875,
      "loss": 0.5427,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.48768311738967896,
      "rewards/margins": 0.9385620355606079,
      "rewards/rejected": -0.4516845643520355,
      "step": 1170
    },
    {
      "epoch": 0.6220347917764892,
      "grad_norm": 92.0245425806093,
      "learning_rate": 8.446230890880337e-07,
      "logits/chosen": -2.101757764816284,
      "logits/rejected": -2.088671922683716,
      "logps/chosen": -391.4624938964844,
      "logps/rejected": -415.2250061035156,
      "loss": 0.6417,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.5240722894668579,
      "rewards/margins": 0.708300769329071,
      "rewards/rejected": -0.18428954482078552,
      "step": 1180
    },
    {
      "epoch": 0.6273062730627307,
      "grad_norm": 104.10789918302594,
      "learning_rate": 8.433052187664734e-07,
      "logits/chosen": -1.975000023841858,
      "logits/rejected": -2.017578125,
      "logps/chosen": -385.375,
      "logps/rejected": -326.2749938964844,
      "loss": 0.4851,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.6592773199081421,
      "rewards/margins": 0.984875500202179,
      "rewards/rejected": -0.32501220703125,
      "step": 1190
    },
    {
      "epoch": 0.632577754348972,
      "grad_norm": 163.76011253760683,
      "learning_rate": 8.419873484449131e-07,
      "logits/chosen": -1.974218726158142,
      "logits/rejected": -1.977929711341858,
      "logps/chosen": -420.20001220703125,
      "logps/rejected": -364.63751220703125,
      "loss": 0.6132,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.315765380859375,
      "rewards/margins": 0.752185046672821,
      "rewards/rejected": -0.43597412109375,
      "step": 1200
    },
    {
      "epoch": 0.6378492356352135,
      "grad_norm": 97.93225388840814,
      "learning_rate": 8.406694781233526e-07,
      "logits/chosen": -2.091992139816284,
      "logits/rejected": -2.044921875,
      "logps/chosen": -382.45001220703125,
      "logps/rejected": -402.2749938964844,
      "loss": 0.5772,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.19417724013328552,
      "rewards/margins": 0.7640136480331421,
      "rewards/rejected": -0.569720447063446,
      "step": 1210
    },
    {
      "epoch": 0.6431207169214549,
      "grad_norm": 114.07496150951249,
      "learning_rate": 8.393516078017922e-07,
      "logits/chosen": -1.96142578125,
      "logits/rejected": -1.9246094226837158,
      "logps/chosen": -384.38751220703125,
      "logps/rejected": -352.5625,
      "loss": 0.5142,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.2664856016635895,
      "rewards/margins": 0.9546264410018921,
      "rewards/rejected": -0.6881774663925171,
      "step": 1220
    },
    {
      "epoch": 0.6483921982076963,
      "grad_norm": 115.23738039971484,
      "learning_rate": 8.380337374802318e-07,
      "logits/chosen": -2.0775389671325684,
      "logits/rejected": -2.0005860328674316,
      "logps/chosen": -435.5375061035156,
      "logps/rejected": -372.375,
      "loss": 0.6109,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0036117553245276213,
      "rewards/margins": 0.7600342035293579,
      "rewards/rejected": -0.7566589117050171,
      "step": 1230
    },
    {
      "epoch": 0.6536636794939378,
      "grad_norm": 121.01623688573979,
      "learning_rate": 8.367158671586716e-07,
      "logits/chosen": -2.1181640625,
      "logits/rejected": -2.0582032203674316,
      "logps/chosen": -393.8500061035156,
      "logps/rejected": -341.79998779296875,
      "loss": 0.6124,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.14473876357078552,
      "rewards/margins": 0.69891357421875,
      "rewards/rejected": -0.843518078327179,
      "step": 1240
    },
    {
      "epoch": 0.6589351607801792,
      "grad_norm": 136.28262519391006,
      "learning_rate": 8.353979968371112e-07,
      "logits/chosen": -2.0355467796325684,
      "logits/rejected": -2.077929735183716,
      "logps/chosen": -418.4375,
      "logps/rejected": -346.11248779296875,
      "loss": 0.5627,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.24388733506202698,
      "rewards/margins": 0.7860962152481079,
      "rewards/rejected": -1.029882788658142,
      "step": 1250
    },
    {
      "epoch": 0.6642066420664207,
      "grad_norm": 132.746298487088,
      "learning_rate": 8.340801265155508e-07,
      "logits/chosen": -2.0824217796325684,
      "logits/rejected": -1.9578125476837158,
      "logps/chosen": -355.8374938964844,
      "logps/rejected": -328.13751220703125,
      "loss": 0.5965,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.04725341871380806,
      "rewards/margins": 0.8168090581893921,
      "rewards/rejected": -0.8638671636581421,
      "step": 1260
    },
    {
      "epoch": 0.6694781233526621,
      "grad_norm": 114.24972853985538,
      "learning_rate": 8.327622561939904e-07,
      "logits/chosen": -2.163281202316284,
      "logits/rejected": -2.080273389816284,
      "logps/chosen": -388.4624938964844,
      "logps/rejected": -394.8374938964844,
      "loss": 0.5981,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.04628295823931694,
      "rewards/margins": 0.6263488531112671,
      "rewards/rejected": -0.580126941204071,
      "step": 1270
    },
    {
      "epoch": 0.6747496046389035,
      "grad_norm": 107.8860825119244,
      "learning_rate": 8.314443858724301e-07,
      "logits/chosen": -2.0824217796325684,
      "logits/rejected": -2.070117235183716,
      "logps/chosen": -374.6875,
      "logps/rejected": -370.875,
      "loss": 0.6004,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.10522155463695526,
      "rewards/margins": 0.592602550983429,
      "rewards/rejected": -0.48795777559280396,
      "step": 1280
    },
    {
      "epoch": 0.680021085925145,
      "grad_norm": 103.40194037247932,
      "learning_rate": 8.301265155508697e-07,
      "logits/chosen": -2.099414110183716,
      "logits/rejected": -2.102734327316284,
      "logps/chosen": -395.125,
      "logps/rejected": -335.1000061035156,
      "loss": 0.5605,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.27686768770217896,
      "rewards/margins": 0.790771484375,
      "rewards/rejected": -0.513958752155304,
      "step": 1290
    },
    {
      "epoch": 0.6852925672113864,
      "grad_norm": 145.09047379643027,
      "learning_rate": 8.288086452293094e-07,
      "logits/chosen": -2.0894532203674316,
      "logits/rejected": -2.0572266578674316,
      "logps/chosen": -384.38751220703125,
      "logps/rejected": -343.8374938964844,
      "loss": 0.548,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.224456787109375,
      "rewards/margins": 0.7983154058456421,
      "rewards/rejected": -0.574200451374054,
      "step": 1300
    },
    {
      "epoch": 0.6905640484976279,
      "grad_norm": 104.14594142672861,
      "learning_rate": 8.27490774907749e-07,
      "logits/chosen": -2.119921922683716,
      "logits/rejected": -2.1636719703674316,
      "logps/chosen": -420.98748779296875,
      "logps/rejected": -374.7749938964844,
      "loss": 0.579,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.2577453553676605,
      "rewards/margins": 0.6711212396621704,
      "rewards/rejected": -0.41307371854782104,
      "step": 1310
    },
    {
      "epoch": 0.6958355297838693,
      "grad_norm": 99.45405430735264,
      "learning_rate": 8.261729045861887e-07,
      "logits/chosen": -2.0882811546325684,
      "logits/rejected": -2.080078125,
      "logps/chosen": -408.25,
      "logps/rejected": -369.3999938964844,
      "loss": 0.5833,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.18046264350414276,
      "rewards/margins": 0.756420910358429,
      "rewards/rejected": -0.5757080316543579,
      "step": 1320
    },
    {
      "epoch": 0.7011070110701108,
      "grad_norm": 105.30205408675101,
      "learning_rate": 8.248550342646283e-07,
      "logits/chosen": -2.1302733421325684,
      "logits/rejected": -2.0162110328674316,
      "logps/chosen": -372.9750061035156,
      "logps/rejected": -380.20001220703125,
      "loss": 0.483,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.09693603217601776,
      "rewards/margins": 0.926684558391571,
      "rewards/rejected": -0.8288208246231079,
      "step": 1330
    },
    {
      "epoch": 0.7063784923563521,
      "grad_norm": 75.49800447737638,
      "learning_rate": 8.235371639430679e-07,
      "logits/chosen": -2.220703125,
      "logits/rejected": -2.133593797683716,
      "logps/chosen": -420.11248779296875,
      "logps/rejected": -341.73748779296875,
      "loss": 0.501,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.01763000525534153,
      "rewards/margins": 0.949475109577179,
      "rewards/rejected": -0.968518078327179,
      "step": 1340
    },
    {
      "epoch": 0.7116499736425935,
      "grad_norm": 133.44292694735134,
      "learning_rate": 8.222192936215076e-07,
      "logits/chosen": -2.1410155296325684,
      "logits/rejected": -2.0667967796325684,
      "logps/chosen": -403.9750061035156,
      "logps/rejected": -361.2124938964844,
      "loss": 0.6547,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.324859619140625,
      "rewards/margins": 0.577392578125,
      "rewards/rejected": -0.90234375,
      "step": 1350
    },
    {
      "epoch": 0.716921454928835,
      "grad_norm": 107.77720443519331,
      "learning_rate": 8.209014232999473e-07,
      "logits/chosen": -2.0380859375,
      "logits/rejected": -2.0982422828674316,
      "logps/chosen": -380.17498779296875,
      "logps/rejected": -362.92498779296875,
      "loss": 0.551,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.3225341737270355,
      "rewards/margins": 0.890515148639679,
      "rewards/rejected": -1.2135131359100342,
      "step": 1360
    },
    {
      "epoch": 0.7221929362150764,
      "grad_norm": 85.73178955352851,
      "learning_rate": 8.195835529783869e-07,
      "logits/chosen": -2.037304639816284,
      "logits/rejected": -2.0435547828674316,
      "logps/chosen": -400.3999938964844,
      "logps/rejected": -389.8500061035156,
      "loss": 0.4673,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.0953369140625,
      "rewards/margins": 1.125158667564392,
      "rewards/rejected": -1.2209961414337158,
      "step": 1370
    },
    {
      "epoch": 0.7274644175013179,
      "grad_norm": 86.98901637292381,
      "learning_rate": 8.182656826568265e-07,
      "logits/chosen": -2.1060547828674316,
      "logits/rejected": -2.177539110183716,
      "logps/chosen": -402.17498779296875,
      "logps/rejected": -359.82501220703125,
      "loss": 0.4948,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.20682373642921448,
      "rewards/margins": 0.9398437738418579,
      "rewards/rejected": -1.1459228992462158,
      "step": 1380
    },
    {
      "epoch": 0.7327358987875593,
      "grad_norm": 89.07665452574099,
      "learning_rate": 8.169478123352662e-07,
      "logits/chosen": -2.149609327316284,
      "logits/rejected": -2.137500047683716,
      "logps/chosen": -349.92498779296875,
      "logps/rejected": -354.8999938964844,
      "loss": 0.5266,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4056396484375,
      "rewards/margins": 1.0520751476287842,
      "rewards/rejected": -1.457006812095642,
      "step": 1390
    },
    {
      "epoch": 0.7380073800738007,
      "grad_norm": 129.6173769203683,
      "learning_rate": 8.156299420137058e-07,
      "logits/chosen": -2.151171922683716,
      "logits/rejected": -2.156445264816284,
      "logps/chosen": -412.1499938964844,
      "logps/rejected": -380.1875,
      "loss": 0.4966,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.3201538026332855,
      "rewards/margins": 1.0529906749725342,
      "rewards/rejected": -1.373144507408142,
      "step": 1400
    },
    {
      "epoch": 0.7432788613600422,
      "grad_norm": 126.42136150155862,
      "learning_rate": 8.143120716921455e-07,
      "logits/chosen": -2.1939454078674316,
      "logits/rejected": -2.173632860183716,
      "logps/chosen": -446.79998779296875,
      "logps/rejected": -384.625,
      "loss": 0.6403,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.2601684629917145,
      "rewards/margins": 0.8611084222793579,
      "rewards/rejected": -1.122094750404358,
      "step": 1410
    },
    {
      "epoch": 0.7485503426462836,
      "grad_norm": 72.85840601432446,
      "learning_rate": 8.129942013705851e-07,
      "logits/chosen": -2.178906202316284,
      "logits/rejected": -2.1724610328674316,
      "logps/chosen": -361.79998779296875,
      "logps/rejected": -303.7875061035156,
      "loss": 0.5893,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.6039062738418579,
      "rewards/margins": 0.787182629108429,
      "rewards/rejected": -1.391259789466858,
      "step": 1420
    },
    {
      "epoch": 0.7538218239325251,
      "grad_norm": 118.62446402326742,
      "learning_rate": 8.116763310490248e-07,
      "logits/chosen": -2.2626953125,
      "logits/rejected": -2.2347655296325684,
      "logps/chosen": -383.54998779296875,
      "logps/rejected": -358.11248779296875,
      "loss": 0.5841,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2632202208042145,
      "rewards/margins": 0.9173218011856079,
      "rewards/rejected": -1.181884765625,
      "step": 1430
    },
    {
      "epoch": 0.7590933052187665,
      "grad_norm": 92.61991802066771,
      "learning_rate": 8.103584607274644e-07,
      "logits/chosen": -2.0384764671325684,
      "logits/rejected": -2.04296875,
      "logps/chosen": -402.25,
      "logps/rejected": -343.51251220703125,
      "loss": 0.5632,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.08253173530101776,
      "rewards/margins": 0.805163562297821,
      "rewards/rejected": -0.8876708745956421,
      "step": 1440
    },
    {
      "epoch": 0.7643647865050079,
      "grad_norm": 89.91223595864741,
      "learning_rate": 8.09040590405904e-07,
      "logits/chosen": -2.1435546875,
      "logits/rejected": -2.1937499046325684,
      "logps/chosen": -380.3374938964844,
      "logps/rejected": -343.2250061035156,
      "loss": 0.5699,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.199951171875,
      "rewards/margins": 0.828625500202179,
      "rewards/rejected": -1.0277283191680908,
      "step": 1450
    },
    {
      "epoch": 0.7696362677912494,
      "grad_norm": 93.51985124227528,
      "learning_rate": 8.077227200843436e-07,
      "logits/chosen": -2.1435546875,
      "logits/rejected": -2.1005859375,
      "logps/chosen": -426.5249938964844,
      "logps/rejected": -362.6000061035156,
      "loss": 0.5685,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.4215942323207855,
      "rewards/margins": 0.858105480670929,
      "rewards/rejected": -1.278833031654358,
      "step": 1460
    },
    {
      "epoch": 0.7749077490774908,
      "grad_norm": 127.23849849857486,
      "learning_rate": 8.064048497627834e-07,
      "logits/chosen": -2.092968702316284,
      "logits/rejected": -2.0005860328674316,
      "logps/chosen": -382.3125,
      "logps/rejected": -369.6499938964844,
      "loss": 0.5857,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.6350463628768921,
      "rewards/margins": 0.7241455316543579,
      "rewards/rejected": -1.3598144054412842,
      "step": 1470
    },
    {
      "epoch": 0.7801792303637322,
      "grad_norm": 124.25032951041737,
      "learning_rate": 8.05086979441223e-07,
      "logits/chosen": -2.0962891578674316,
      "logits/rejected": -2.135937452316284,
      "logps/chosen": -446.9125061035156,
      "logps/rejected": -368.9750061035156,
      "loss": 0.5538,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.5923095941543579,
      "rewards/margins": 0.8409789800643921,
      "rewards/rejected": -1.4337646961212158,
      "step": 1480
    },
    {
      "epoch": 0.7854507116499736,
      "grad_norm": 97.51898983160439,
      "learning_rate": 8.037691091196626e-07,
      "logits/chosen": -2.2230467796325684,
      "logits/rejected": -2.217968702316284,
      "logps/chosen": -403.29998779296875,
      "logps/rejected": -358.57501220703125,
      "loss": 0.5513,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.548229992389679,
      "rewards/margins": 0.7825378179550171,
      "rewards/rejected": -1.330810546875,
      "step": 1490
    },
    {
      "epoch": 0.790722192936215,
      "grad_norm": 121.37567115034034,
      "learning_rate": 8.024512387981023e-07,
      "logits/chosen": -2.150585889816284,
      "logits/rejected": -2.173046827316284,
      "logps/chosen": -445.4750061035156,
      "logps/rejected": -383.57501220703125,
      "loss": 0.5318,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.3999267518520355,
      "rewards/margins": 0.8344970941543579,
      "rewards/rejected": -1.234185814857483,
      "step": 1500
    },
    {
      "epoch": 0.7959936742224565,
      "grad_norm": 96.28918271899973,
      "learning_rate": 8.011333684765419e-07,
      "logits/chosen": -2.0806641578674316,
      "logits/rejected": -2.1078124046325684,
      "logps/chosen": -408.6000061035156,
      "logps/rejected": -366.0,
      "loss": 0.5702,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.35160523653030396,
      "rewards/margins": 0.8395751714706421,
      "rewards/rejected": -1.1915252208709717,
      "step": 1510
    },
    {
      "epoch": 0.8012651555086979,
      "grad_norm": 122.89401549968632,
      "learning_rate": 7.998154981549815e-07,
      "logits/chosen": -2.1166014671325684,
      "logits/rejected": -2.091992139816284,
      "logps/chosen": -389.0625,
      "logps/rejected": -357.38751220703125,
      "loss": 0.5383,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.03493652492761612,
      "rewards/margins": 0.9966796636581421,
      "rewards/rejected": -1.032446265220642,
      "step": 1520
    },
    {
      "epoch": 0.8065366367949394,
      "grad_norm": 86.68084464273358,
      "learning_rate": 7.984976278334212e-07,
      "logits/chosen": -2.096874952316284,
      "logits/rejected": -2.1109375953674316,
      "logps/chosen": -369.92498779296875,
      "logps/rejected": -336.3999938964844,
      "loss": 0.5932,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.29932403564453125,
      "rewards/margins": 0.7828124761581421,
      "rewards/rejected": -1.082696557044983,
      "step": 1530
    },
    {
      "epoch": 0.8118081180811808,
      "grad_norm": 75.21196317035866,
      "learning_rate": 7.971797575118609e-07,
      "logits/chosen": -2.0755858421325684,
      "logits/rejected": -2.1014647483825684,
      "logps/chosen": -361.6625061035156,
      "logps/rejected": -334.0874938964844,
      "loss": 0.5739,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07426147162914276,
      "rewards/margins": 0.804461658000946,
      "rewards/rejected": -0.8795105218887329,
      "step": 1540
    },
    {
      "epoch": 0.8170795993674222,
      "grad_norm": 81.09639466139058,
      "learning_rate": 7.958618871903005e-07,
      "logits/chosen": -2.128124952316284,
      "logits/rejected": -2.077343702316284,
      "logps/chosen": -376.875,
      "logps/rejected": -352.0874938964844,
      "loss": 0.5954,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.16710814833641052,
      "rewards/margins": 0.6969665288925171,
      "rewards/rejected": -0.8638671636581421,
      "step": 1550
    },
    {
      "epoch": 0.8223510806536637,
      "grad_norm": 88.5504915240659,
      "learning_rate": 7.9454401686874e-07,
      "logits/chosen": -2.1826171875,
      "logits/rejected": -2.163867235183716,
      "logps/chosen": -392.0249938964844,
      "logps/rejected": -344.23748779296875,
      "loss": 0.5103,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 0.09522094577550888,
      "rewards/margins": 0.8916870355606079,
      "rewards/rejected": -0.796307384967804,
      "step": 1560
    },
    {
      "epoch": 0.8276225619399051,
      "grad_norm": 129.68795846595032,
      "learning_rate": 7.932261465471796e-07,
      "logits/chosen": -2.1312499046325684,
      "logits/rejected": -2.169726610183716,
      "logps/chosen": -368.07501220703125,
      "logps/rejected": -351.42498779296875,
      "loss": 0.5382,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.04502563551068306,
      "rewards/margins": 0.86614990234375,
      "rewards/rejected": -0.821154773235321,
      "step": 1570
    },
    {
      "epoch": 0.8328940432261466,
      "grad_norm": 113.62732329439716,
      "learning_rate": 7.919082762256194e-07,
      "logits/chosen": -2.083203077316284,
      "logits/rejected": -2.1136717796325684,
      "logps/chosen": -380.5249938964844,
      "logps/rejected": -352.63751220703125,
      "loss": 0.6027,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.10304565727710724,
      "rewards/margins": 0.815948486328125,
      "rewards/rejected": -0.917773425579071,
      "step": 1580
    },
    {
      "epoch": 0.838165524512388,
      "grad_norm": 105.4961355674116,
      "learning_rate": 7.90590405904059e-07,
      "logits/chosen": -2.089648485183716,
      "logits/rejected": -2.064257860183716,
      "logps/chosen": -363.29998779296875,
      "logps/rejected": -345.51251220703125,
      "loss": 0.5684,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.10136108100414276,
      "rewards/margins": 0.862011730670929,
      "rewards/rejected": -0.9634338617324829,
      "step": 1590
    },
    {
      "epoch": 0.8434370057986295,
      "grad_norm": 144.89798807320187,
      "learning_rate": 7.892725355824986e-07,
      "logits/chosen": -2.0859375,
      "logits/rejected": -2.038281202316284,
      "logps/chosen": -366.88751220703125,
      "logps/rejected": -320.67498779296875,
      "loss": 0.5669,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.011569214053452015,
      "rewards/margins": 0.88037109375,
      "rewards/rejected": -0.891552746295929,
      "step": 1600
    },
    {
      "epoch": 0.8487084870848709,
      "grad_norm": 99.63229251212486,
      "learning_rate": 7.879546652609382e-07,
      "logits/chosen": -2.0693359375,
      "logits/rejected": -2.0101561546325684,
      "logps/chosen": -385.9750061035156,
      "logps/rejected": -344.45001220703125,
      "loss": 0.5936,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.16979369521141052,
      "rewards/margins": 0.689849853515625,
      "rewards/rejected": -0.8597046136856079,
      "step": 1610
    },
    {
      "epoch": 0.8539799683711122,
      "grad_norm": 80.56037526587473,
      "learning_rate": 7.866367949393779e-07,
      "logits/chosen": -2.0833983421325684,
      "logits/rejected": -2.0464844703674316,
      "logps/chosen": -387.9624938964844,
      "logps/rejected": -372.6000061035156,
      "loss": 0.6061,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.019989013671875,
      "rewards/margins": 0.785388171672821,
      "rewards/rejected": -0.805065929889679,
      "step": 1620
    },
    {
      "epoch": 0.8592514496573537,
      "grad_norm": 74.36423453633968,
      "learning_rate": 7.853189246178175e-07,
      "logits/chosen": -2.1791014671325684,
      "logits/rejected": -2.1070313453674316,
      "logps/chosen": -374.7875061035156,
      "logps/rejected": -337.1499938964844,
      "loss": 0.4744,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.08785095065832138,
      "rewards/margins": 0.9624878168106079,
      "rewards/rejected": -0.8753906488418579,
      "step": 1630
    },
    {
      "epoch": 0.8645229309435951,
      "grad_norm": 110.25284852542443,
      "learning_rate": 7.840010542962572e-07,
      "logits/chosen": -2.099609375,
      "logits/rejected": -2.0501952171325684,
      "logps/chosen": -400.79998779296875,
      "logps/rejected": -375.0,
      "loss": 0.5893,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06566162407398224,
      "rewards/margins": 0.7565673589706421,
      "rewards/rejected": -0.822583019733429,
      "step": 1640
    },
    {
      "epoch": 0.8697944122298366,
      "grad_norm": 110.99033796341996,
      "learning_rate": 7.826831839746969e-07,
      "logits/chosen": -2.2279295921325684,
      "logits/rejected": -2.261523485183716,
      "logps/chosen": -380.7124938964844,
      "logps/rejected": -381.17498779296875,
      "loss": 0.5782,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.3184570372104645,
      "rewards/margins": 0.822558581829071,
      "rewards/rejected": -1.140478491783142,
      "step": 1650
    },
    {
      "epoch": 0.875065893516078,
      "grad_norm": 123.8547109049773,
      "learning_rate": 7.813653136531365e-07,
      "logits/chosen": -2.1998047828674316,
      "logits/rejected": -2.1416015625,
      "logps/chosen": -380.0,
      "logps/rejected": -355.25,
      "loss": 0.5502,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.4677185118198395,
      "rewards/margins": 1.0126221179962158,
      "rewards/rejected": -1.4789307117462158,
      "step": 1660
    },
    {
      "epoch": 0.8803373748023194,
      "grad_norm": 93.57963511818957,
      "learning_rate": 7.800474433315761e-07,
      "logits/chosen": -2.148242235183716,
      "logits/rejected": -2.1058592796325684,
      "logps/chosen": -390.17498779296875,
      "logps/rejected": -392.82501220703125,
      "loss": 0.5094,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.35755616426467896,
      "rewards/margins": 0.955981433391571,
      "rewards/rejected": -1.313720703125,
      "step": 1670
    },
    {
      "epoch": 0.8856088560885609,
      "grad_norm": 104.39245856816378,
      "learning_rate": 7.787295730100157e-07,
      "logits/chosen": -2.118359327316284,
      "logits/rejected": -2.133984327316284,
      "logps/chosen": -426.54998779296875,
      "logps/rejected": -375.32501220703125,
      "loss": 0.5534,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.41619569063186646,
      "rewards/margins": 1.0113403797149658,
      "rewards/rejected": -1.4289977550506592,
      "step": 1680
    },
    {
      "epoch": 0.8908803373748023,
      "grad_norm": 119.6501735853056,
      "learning_rate": 7.774117026884554e-07,
      "logits/chosen": -2.1148438453674316,
      "logits/rejected": -2.142578125,
      "logps/chosen": -387.5,
      "logps/rejected": -332.73748779296875,
      "loss": 0.5026,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15385131537914276,
      "rewards/margins": 1.0687744617462158,
      "rewards/rejected": -1.223046898841858,
      "step": 1690
    },
    {
      "epoch": 0.8961518186610438,
      "grad_norm": 112.0278771803744,
      "learning_rate": 7.760938323668951e-07,
      "logits/chosen": -2.12890625,
      "logits/rejected": -2.2138671875,
      "logps/chosen": -408.5,
      "logps/rejected": -379.75,
      "loss": 0.6008,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.39039307832717896,
      "rewards/margins": 0.9409424066543579,
      "rewards/rejected": -1.3315002918243408,
      "step": 1700
    },
    {
      "epoch": 0.9014232999472852,
      "grad_norm": 98.49201142501627,
      "learning_rate": 7.747759620453347e-07,
      "logits/chosen": -2.212890625,
      "logits/rejected": -2.2417969703674316,
      "logps/chosen": -396.6499938964844,
      "logps/rejected": -359.75,
      "loss": 0.588,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.22793884575366974,
      "rewards/margins": 0.7932494878768921,
      "rewards/rejected": -1.021124243736267,
      "step": 1710
    },
    {
      "epoch": 0.9066947812335266,
      "grad_norm": 80.9271199995487,
      "learning_rate": 7.734580917237743e-07,
      "logits/chosen": -2.1346678733825684,
      "logits/rejected": -2.0445313453674316,
      "logps/chosen": -363.1000061035156,
      "logps/rejected": -335.5249938964844,
      "loss": 0.5383,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.18114013969898224,
      "rewards/margins": 0.874584972858429,
      "rewards/rejected": -1.0556151866912842,
      "step": 1720
    },
    {
      "epoch": 0.9119662625197681,
      "grad_norm": 108.243883724463,
      "learning_rate": 7.72140221402214e-07,
      "logits/chosen": -2.166796922683716,
      "logits/rejected": -2.1294922828674316,
      "logps/chosen": -413.95001220703125,
      "logps/rejected": -343.95001220703125,
      "loss": 0.4811,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 0.12735596299171448,
      "rewards/margins": 1.213037133216858,
      "rewards/rejected": -1.086084008216858,
      "step": 1730
    },
    {
      "epoch": 0.9172377438060095,
      "grad_norm": 80.63328588374654,
      "learning_rate": 7.708223510806536e-07,
      "logits/chosen": -2.0728516578674316,
      "logits/rejected": -2.12548828125,
      "logps/chosen": -392.9312438964844,
      "logps/rejected": -349.57501220703125,
      "loss": 0.5296,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.05423278734087944,
      "rewards/margins": 0.9995971918106079,
      "rewards/rejected": -0.9458587765693665,
      "step": 1740
    },
    {
      "epoch": 0.922509225092251,
      "grad_norm": 69.279959033007,
      "learning_rate": 7.695044807590932e-07,
      "logits/chosen": -2.1830077171325684,
      "logits/rejected": -2.146484375,
      "logps/chosen": -361.75,
      "logps/rejected": -363.51251220703125,
      "loss": 0.5889,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.102203369140625,
      "rewards/margins": 0.84271240234375,
      "rewards/rejected": -0.9449462890625,
      "step": 1750
    },
    {
      "epoch": 0.9277807063784923,
      "grad_norm": 106.3791116530763,
      "learning_rate": 7.68186610437533e-07,
      "logits/chosen": -2.173046827316284,
      "logits/rejected": -2.097851514816284,
      "logps/chosen": -385.98748779296875,
      "logps/rejected": -388.75,
      "loss": 0.6173,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0631256103515625,
      "rewards/margins": 0.653674304485321,
      "rewards/rejected": -0.589892566204071,
      "step": 1760
    },
    {
      "epoch": 0.9330521876647337,
      "grad_norm": 88.67864813170485,
      "learning_rate": 7.668687401159726e-07,
      "logits/chosen": -2.163281202316284,
      "logits/rejected": -2.114453077316284,
      "logps/chosen": -362.0,
      "logps/rejected": -338.3500061035156,
      "loss": 0.5518,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.22886963188648224,
      "rewards/margins": 0.866809070110321,
      "rewards/rejected": -0.637469470500946,
      "step": 1770
    },
    {
      "epoch": 0.9383236689509752,
      "grad_norm": 96.21797474549373,
      "learning_rate": 7.655508697944122e-07,
      "logits/chosen": -2.0562500953674316,
      "logits/rejected": -2.041796922683716,
      "logps/chosen": -421.8374938964844,
      "logps/rejected": -353.36248779296875,
      "loss": 0.5323,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.4207214415073395,
      "rewards/margins": 0.950207531452179,
      "rewards/rejected": -0.529553234577179,
      "step": 1780
    },
    {
      "epoch": 0.9435951502372166,
      "grad_norm": 79.42838330972548,
      "learning_rate": 7.642329994728518e-07,
      "logits/chosen": -2.192187547683716,
      "logits/rejected": -2.133007764816284,
      "logps/chosen": -394.95001220703125,
      "logps/rejected": -379.17498779296875,
      "loss": 0.5667,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.3866516053676605,
      "rewards/margins": 0.785839855670929,
      "rewards/rejected": -0.3994812071323395,
      "step": 1790
    },
    {
      "epoch": 0.9488666315234581,
      "grad_norm": 104.14203631032716,
      "learning_rate": 7.629151291512915e-07,
      "logits/chosen": -2.16015625,
      "logits/rejected": -2.056445360183716,
      "logps/chosen": -345.92498779296875,
      "logps/rejected": -358.3125,
      "loss": 0.5119,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.305380254983902,
      "rewards/margins": 0.9326416254043579,
      "rewards/rejected": -0.627185046672821,
      "step": 1800
    },
    {
      "epoch": 0.9541381128096995,
      "grad_norm": 88.62494541482786,
      "learning_rate": 7.615972588297312e-07,
      "logits/chosen": -2.1263670921325684,
      "logits/rejected": -2.096484422683716,
      "logps/chosen": -370.2749938964844,
      "logps/rejected": -333.98748779296875,
      "loss": 0.5655,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.2645202577114105,
      "rewards/margins": 0.8829101324081421,
      "rewards/rejected": -0.6181274652481079,
      "step": 1810
    },
    {
      "epoch": 0.959409594095941,
      "grad_norm": 104.26950461728805,
      "learning_rate": 7.602793885081708e-07,
      "logits/chosen": -2.102734327316284,
      "logits/rejected": -2.1572265625,
      "logps/chosen": -393.875,
      "logps/rejected": -358.7250061035156,
      "loss": 0.5398,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.27092283964157104,
      "rewards/margins": 0.8744751214981079,
      "rewards/rejected": -0.6034790277481079,
      "step": 1820
    },
    {
      "epoch": 0.9646810753821824,
      "grad_norm": 72.06726128834688,
      "learning_rate": 7.589615181866104e-07,
      "logits/chosen": -2.0697264671325684,
      "logits/rejected": -1.990820288658142,
      "logps/chosen": -376.0874938964844,
      "logps/rejected": -339.04998779296875,
      "loss": 0.5436,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.1420242339372635,
      "rewards/margins": 1.037805199623108,
      "rewards/rejected": -0.8954437375068665,
      "step": 1830
    },
    {
      "epoch": 0.9699525566684238,
      "grad_norm": 103.06919239763764,
      "learning_rate": 7.576436478650501e-07,
      "logits/chosen": -2.121289014816284,
      "logits/rejected": -2.0980467796325684,
      "logps/chosen": -411.75,
      "logps/rejected": -353.20001220703125,
      "loss": 0.5949,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.08437500149011612,
      "rewards/margins": 0.7623840570449829,
      "rewards/rejected": -0.67791748046875,
      "step": 1840
    },
    {
      "epoch": 0.9752240379546653,
      "grad_norm": 98.98261674682453,
      "learning_rate": 7.563257775434897e-07,
      "logits/chosen": -2.1044921875,
      "logits/rejected": -2.0830078125,
      "logps/chosen": -389.1499938964844,
      "logps/rejected": -355.3500061035156,
      "loss": 0.5808,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.23350830376148224,
      "rewards/margins": 0.96575927734375,
      "rewards/rejected": -1.198510766029358,
      "step": 1850
    },
    {
      "epoch": 0.9804955192409067,
      "grad_norm": 98.99648829145005,
      "learning_rate": 7.550079072219293e-07,
      "logits/chosen": -2.1568360328674316,
      "logits/rejected": -2.1234374046325684,
      "logps/chosen": -378.3125,
      "logps/rejected": -371.29998779296875,
      "loss": 0.5829,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07951965183019638,
      "rewards/margins": 0.956896960735321,
      "rewards/rejected": -1.036901831626892,
      "step": 1860
    },
    {
      "epoch": 0.9857670005271482,
      "grad_norm": 110.66780509950561,
      "learning_rate": 7.53690036900369e-07,
      "logits/chosen": -2.0601563453674316,
      "logits/rejected": -2.031445264816284,
      "logps/chosen": -368.17498779296875,
      "logps/rejected": -356.95001220703125,
      "loss": 0.5244,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.01149902306497097,
      "rewards/margins": 0.927966296672821,
      "rewards/rejected": -0.91607666015625,
      "step": 1870
    },
    {
      "epoch": 0.9910384818133896,
      "grad_norm": 82.48047671581215,
      "learning_rate": 7.523721665788087e-07,
      "logits/chosen": -2.1011719703674316,
      "logits/rejected": -2.05859375,
      "logps/chosen": -361.04998779296875,
      "logps/rejected": -333.04998779296875,
      "loss": 0.564,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.2342071533203125,
      "rewards/margins": 0.7314697504043579,
      "rewards/rejected": -0.49772948026657104,
      "step": 1880
    },
    {
      "epoch": 0.996309963099631,
      "grad_norm": 119.45104100069418,
      "learning_rate": 7.510542962572483e-07,
      "logits/chosen": -2.105273485183716,
      "logits/rejected": -2.0494141578674316,
      "logps/chosen": -357.0625,
      "logps/rejected": -334.4750061035156,
      "loss": 0.5179,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.165313720703125,
      "rewards/margins": 0.925872802734375,
      "rewards/rejected": -0.7606231570243835,
      "step": 1890
    },
    {
      "epoch": 1.0015814443858724,
      "grad_norm": 42.157718819396216,
      "learning_rate": 7.497364259356879e-07,
      "logits/chosen": -2.103710889816284,
      "logits/rejected": -2.155468702316284,
      "logps/chosen": -385.9125061035156,
      "logps/rejected": -349.6625061035156,
      "loss": 0.4505,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 0.06606445461511612,
      "rewards/margins": 1.344580054283142,
      "rewards/rejected": -1.278680443763733,
      "step": 1900
    },
    {
      "epoch": 1.006852925672114,
      "grad_norm": 49.848882366309965,
      "learning_rate": 7.484185556141276e-07,
      "logits/chosen": -2.0263671875,
      "logits/rejected": -1.9462890625,
      "logps/chosen": -377.17498779296875,
      "logps/rejected": -373.5,
      "loss": 0.1633,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": 0.548596203327179,
      "rewards/margins": 2.500781297683716,
      "rewards/rejected": -1.9523437023162842,
      "step": 1910
    },
    {
      "epoch": 1.0121244069583553,
      "grad_norm": 27.559414185813285,
      "learning_rate": 7.471006852925671e-07,
      "logits/chosen": -2.268359422683716,
      "logits/rejected": -2.219531297683716,
      "logps/chosen": -365.1625061035156,
      "logps/rejected": -382.88751220703125,
      "loss": 0.191,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.4683471620082855,
      "rewards/margins": 2.4681639671325684,
      "rewards/rejected": -1.998144507408142,
      "step": 1920
    },
    {
      "epoch": 1.0173958882445968,
      "grad_norm": 32.023042348872586,
      "learning_rate": 7.457828149710068e-07,
      "logits/chosen": -2.216601610183716,
      "logits/rejected": -2.2552733421325684,
      "logps/chosen": -371.375,
      "logps/rejected": -369.4750061035156,
      "loss": 0.1953,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.502729058265686,
      "rewards/margins": 2.4564452171325684,
      "rewards/rejected": -1.9538085460662842,
      "step": 1930
    },
    {
      "epoch": 1.0226673695308381,
      "grad_norm": 25.806533680181637,
      "learning_rate": 7.444649446494464e-07,
      "logits/chosen": -2.2787108421325684,
      "logits/rejected": -2.2281250953674316,
      "logps/chosen": -435.625,
      "logps/rejected": -396.7749938964844,
      "loss": 0.1328,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.738757312297821,
      "rewards/margins": 2.837597608566284,
      "rewards/rejected": -2.097949266433716,
      "step": 1940
    },
    {
      "epoch": 1.0279388508170797,
      "grad_norm": 25.588164882441134,
      "learning_rate": 7.431470743278861e-07,
      "logits/chosen": -2.2933592796325684,
      "logits/rejected": -2.2964844703674316,
      "logps/chosen": -403.17498779296875,
      "logps/rejected": -380.5625,
      "loss": 0.2007,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 0.5788513422012329,
      "rewards/margins": 2.6011719703674316,
      "rewards/rejected": -2.0230469703674316,
      "step": 1950
    },
    {
      "epoch": 1.033210332103321,
      "grad_norm": 49.50819103080627,
      "learning_rate": 7.418292040063257e-07,
      "logits/chosen": -2.235546827316284,
      "logits/rejected": -2.232421875,
      "logps/chosen": -384.4750061035156,
      "logps/rejected": -394.45001220703125,
      "loss": 0.1887,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.6348022222518921,
      "rewards/margins": 2.829296827316284,
      "rewards/rejected": -2.193652391433716,
      "step": 1960
    },
    {
      "epoch": 1.0384818133895624,
      "grad_norm": 35.32320048600686,
      "learning_rate": 7.405113336847653e-07,
      "logits/chosen": -2.21484375,
      "logits/rejected": -2.2232422828674316,
      "logps/chosen": -336.9125061035156,
      "logps/rejected": -339.2749938964844,
      "loss": 0.1889,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.792816162109375,
      "rewards/margins": 2.5234375,
      "rewards/rejected": -1.72998046875,
      "step": 1970
    },
    {
      "epoch": 1.043753294675804,
      "grad_norm": 32.880265188941266,
      "learning_rate": 7.39193463363205e-07,
      "logits/chosen": -2.144824266433716,
      "logits/rejected": -2.0947265625,
      "logps/chosen": -398.20001220703125,
      "logps/rejected": -378.625,
      "loss": 0.1428,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.745288074016571,
      "rewards/margins": 2.9751954078674316,
      "rewards/rejected": -2.2284178733825684,
      "step": 1980
    },
    {
      "epoch": 1.0490247759620452,
      "grad_norm": 29.78970062715657,
      "learning_rate": 7.378755930416447e-07,
      "logits/chosen": -2.215039014816284,
      "logits/rejected": -2.1644530296325684,
      "logps/chosen": -373.11248779296875,
      "logps/rejected": -364.23748779296875,
      "loss": 0.1711,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.4493041932582855,
      "rewards/margins": 2.688671827316284,
      "rewards/rejected": -2.2398438453674316,
      "step": 1990
    },
    {
      "epoch": 1.0542962572482868,
      "grad_norm": 35.83959554513254,
      "learning_rate": 7.365577227200843e-07,
      "logits/chosen": -2.348437547683716,
      "logits/rejected": -2.244921922683716,
      "logps/chosen": -376.20001220703125,
      "logps/rejected": -367.1499938964844,
      "loss": 0.1779,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.2052001953125,
      "rewards/margins": 2.709667921066284,
      "rewards/rejected": -2.505859375,
      "step": 2000
    },
    {
      "epoch": 1.0595677385345281,
      "grad_norm": 58.42982328979334,
      "learning_rate": 7.352398523985239e-07,
      "logits/chosen": -2.2955079078674316,
      "logits/rejected": -2.2203125953674316,
      "logps/chosen": -367.3999938964844,
      "logps/rejected": -368.57501220703125,
      "loss": 0.1325,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.4880737364292145,
      "rewards/margins": 2.950976610183716,
      "rewards/rejected": -2.4618163108825684,
      "step": 2010
    },
    {
      "epoch": 1.0648392198207697,
      "grad_norm": 39.181447160868,
      "learning_rate": 7.339219820769635e-07,
      "logits/chosen": -2.346874952316284,
      "logits/rejected": -2.3837890625,
      "logps/chosen": -369.3999938964844,
      "logps/rejected": -337.4624938964844,
      "loss": 0.186,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.3142456114292145,
      "rewards/margins": 2.5355467796325684,
      "rewards/rejected": -2.2198243141174316,
      "step": 2020
    },
    {
      "epoch": 1.070110701107011,
      "grad_norm": 63.727243370506336,
      "learning_rate": 7.326041117554032e-07,
      "logits/chosen": -2.334179639816284,
      "logits/rejected": -2.289257764816284,
      "logps/chosen": -376.8999938964844,
      "logps/rejected": -369.79998779296875,
      "loss": 0.1722,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.406076043844223,
      "rewards/margins": 2.9439454078674316,
      "rewards/rejected": -2.537304639816284,
      "step": 2030
    },
    {
      "epoch": 1.0753821823932526,
      "grad_norm": 64.80133248298888,
      "learning_rate": 7.312862414338429e-07,
      "logits/chosen": -2.380078077316284,
      "logits/rejected": -2.3990235328674316,
      "logps/chosen": -428.0874938964844,
      "logps/rejected": -361.17498779296875,
      "loss": 0.1736,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.47474366426467896,
      "rewards/margins": 2.705273389816284,
      "rewards/rejected": -2.228466749191284,
      "step": 2040
    },
    {
      "epoch": 1.080653663679494,
      "grad_norm": 87.58856633081436,
      "learning_rate": 7.299683711122825e-07,
      "logits/chosen": -2.3984375,
      "logits/rejected": -2.383984327316284,
      "logps/chosen": -366.42498779296875,
      "logps/rejected": -386.25,
      "loss": 0.2114,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.21229247748851776,
      "rewards/margins": 2.742480516433716,
      "rewards/rejected": -2.528271436691284,
      "step": 2050
    },
    {
      "epoch": 1.0859251449657354,
      "grad_norm": 64.58446988314415,
      "learning_rate": 7.286505007907222e-07,
      "logits/chosen": -2.283203125,
      "logits/rejected": -2.3896484375,
      "logps/chosen": -353.8999938964844,
      "logps/rejected": -345.51251220703125,
      "loss": 0.1895,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 0.166778564453125,
      "rewards/margins": 2.7162108421325684,
      "rewards/rejected": -2.5499024391174316,
      "step": 2060
    },
    {
      "epoch": 1.0911966262519768,
      "grad_norm": 39.3576651101669,
      "learning_rate": 7.273326304691618e-07,
      "logits/chosen": -2.3140625953674316,
      "logits/rejected": -2.385546922683716,
      "logps/chosen": -385.0,
      "logps/rejected": -364.375,
      "loss": 0.164,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3548950254917145,
      "rewards/margins": 2.939648389816284,
      "rewards/rejected": -2.586621046066284,
      "step": 2070
    },
    {
      "epoch": 1.0964681075382183,
      "grad_norm": 52.14520093494933,
      "learning_rate": 7.260147601476014e-07,
      "logits/chosen": -2.331249952316284,
      "logits/rejected": -2.3277344703674316,
      "logps/chosen": -352.45001220703125,
      "logps/rejected": -364.17498779296875,
      "loss": 0.2028,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.45946043729782104,
      "rewards/margins": 2.819140672683716,
      "rewards/rejected": -2.359130859375,
      "step": 2080
    },
    {
      "epoch": 1.1017395888244597,
      "grad_norm": 26.61463059651111,
      "learning_rate": 7.24696889826041e-07,
      "logits/chosen": -2.313281297683716,
      "logits/rejected": -2.292187452316284,
      "logps/chosen": -370.0874938964844,
      "logps/rejected": -335.6499938964844,
      "loss": 0.1675,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": 0.606677234172821,
      "rewards/margins": 2.6820311546325684,
      "rewards/rejected": -2.0757813453674316,
      "step": 2090
    },
    {
      "epoch": 1.1070110701107012,
      "grad_norm": 31.50547398267179,
      "learning_rate": 7.233790195044808e-07,
      "logits/chosen": -2.3939452171325684,
      "logits/rejected": -2.389453172683716,
      "logps/chosen": -310.13751220703125,
      "logps/rejected": -314.95001220703125,
      "loss": 0.1807,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.630480945110321,
      "rewards/margins": 2.8414063453674316,
      "rewards/rejected": -2.2098631858825684,
      "step": 2100
    },
    {
      "epoch": 1.1122825513969425,
      "grad_norm": 19.131018356523544,
      "learning_rate": 7.220611491829204e-07,
      "logits/chosen": -2.2822265625,
      "logits/rejected": -2.3597655296325684,
      "logps/chosen": -334.17498779296875,
      "logps/rejected": -364.1000061035156,
      "loss": 0.1698,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.784191906452179,
      "rewards/margins": 2.9161133766174316,
      "rewards/rejected": -2.1337890625,
      "step": 2110
    },
    {
      "epoch": 1.1175540326831839,
      "grad_norm": 32.85684620163968,
      "learning_rate": 7.2074327886136e-07,
      "logits/chosen": -2.295703172683716,
      "logits/rejected": -2.287304639816284,
      "logps/chosen": -375.57501220703125,
      "logps/rejected": -339.0249938964844,
      "loss": 0.1753,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.24854736030101776,
      "rewards/margins": 2.791015625,
      "rewards/rejected": -2.5430665016174316,
      "step": 2120
    },
    {
      "epoch": 1.1228255139694254,
      "grad_norm": 42.74910169733137,
      "learning_rate": 7.194254085397996e-07,
      "logits/chosen": -2.3076171875,
      "logits/rejected": -2.313281297683716,
      "logps/chosen": -417.3687438964844,
      "logps/rejected": -431.1499938964844,
      "loss": 0.1371,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.29880982637405396,
      "rewards/margins": 3.303906202316284,
      "rewards/rejected": -3.0069336891174316,
      "step": 2130
    },
    {
      "epoch": 1.1280969952556668,
      "grad_norm": 28.30580001825563,
      "learning_rate": 7.181075382182393e-07,
      "logits/chosen": -2.4087891578674316,
      "logits/rejected": -2.3564453125,
      "logps/chosen": -334.3999938964844,
      "logps/rejected": -330.67498779296875,
      "loss": 0.2663,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.669573962688446,
      "rewards/margins": 2.7525391578674316,
      "rewards/rejected": -3.424511671066284,
      "step": 2140
    },
    {
      "epoch": 1.1333684765419083,
      "grad_norm": 43.66433754815898,
      "learning_rate": 7.16789667896679e-07,
      "logits/chosen": -2.425976514816284,
      "logits/rejected": -2.385937452316284,
      "logps/chosen": -362.92498779296875,
      "logps/rejected": -345.625,
      "loss": 0.182,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.526110827922821,
      "rewards/margins": 2.834667921066284,
      "rewards/rejected": -3.3626952171325684,
      "step": 2150
    },
    {
      "epoch": 1.1386399578281496,
      "grad_norm": 73.64019913501049,
      "learning_rate": 7.154717975751186e-07,
      "logits/chosen": -2.3759765625,
      "logits/rejected": -2.4193358421325684,
      "logps/chosen": -387.45001220703125,
      "logps/rejected": -366.88751220703125,
      "loss": 0.1918,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.6525329351425171,
      "rewards/margins": 2.7979979515075684,
      "rewards/rejected": -3.449023485183716,
      "step": 2160
    },
    {
      "epoch": 1.1439114391143912,
      "grad_norm": 50.5798260851398,
      "learning_rate": 7.141539272535582e-07,
      "logits/chosen": -2.3970704078674316,
      "logits/rejected": -2.329296827316284,
      "logps/chosen": -356.67498779296875,
      "logps/rejected": -337.0,
      "loss": 0.2096,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.1229248046875,
      "rewards/margins": 2.8252930641174316,
      "rewards/rejected": -2.949023485183716,
      "step": 2170
    },
    {
      "epoch": 1.1491829204006325,
      "grad_norm": 42.76072933230189,
      "learning_rate": 7.128360569319979e-07,
      "logits/chosen": -2.346484422683716,
      "logits/rejected": -2.3636717796325684,
      "logps/chosen": -408.2124938964844,
      "logps/rejected": -349.98748779296875,
      "loss": 0.1629,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.5777832269668579,
      "rewards/margins": 2.9546875953674316,
      "rewards/rejected": -2.3765625953674316,
      "step": 2180
    },
    {
      "epoch": 1.154454401686874,
      "grad_norm": 39.66319817362194,
      "learning_rate": 7.115181866104375e-07,
      "logits/chosen": -2.3246092796325684,
      "logits/rejected": -2.3212890625,
      "logps/chosen": -417.1000061035156,
      "logps/rejected": -414.79998779296875,
      "loss": 0.1372,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.5675750970840454,
      "rewards/margins": 3.3349609375,
      "rewards/rejected": -2.765625,
      "step": 2190
    },
    {
      "epoch": 1.1597258829731154,
      "grad_norm": 30.29776337272455,
      "learning_rate": 7.102003162888771e-07,
      "logits/chosen": -2.2802734375,
      "logits/rejected": -2.331249952316284,
      "logps/chosen": -389.3374938964844,
      "logps/rejected": -355.5375061035156,
      "loss": 0.1644,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.33880615234375,
      "rewards/margins": 3.006054639816284,
      "rewards/rejected": -2.665234327316284,
      "step": 2200
    },
    {
      "epoch": 1.164997364259357,
      "grad_norm": 33.42556201661973,
      "learning_rate": 7.088824459673169e-07,
      "logits/chosen": -2.445507764816284,
      "logits/rejected": -2.3919920921325684,
      "logps/chosen": -366.9750061035156,
      "logps/rejected": -377.125,
      "loss": 0.1575,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.16268309950828552,
      "rewards/margins": 2.893749952316284,
      "rewards/rejected": -2.7298827171325684,
      "step": 2210
    },
    {
      "epoch": 1.1702688455455983,
      "grad_norm": 51.71918939878104,
      "learning_rate": 7.075645756457565e-07,
      "logits/chosen": -2.4312500953674316,
      "logits/rejected": -2.337109327316284,
      "logps/chosen": -346.07501220703125,
      "logps/rejected": -351.54998779296875,
      "loss": 0.194,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.15362548828125,
      "rewards/margins": 2.799609422683716,
      "rewards/rejected": -2.645312547683716,
      "step": 2220
    },
    {
      "epoch": 1.1755403268318398,
      "grad_norm": 49.45183742615248,
      "learning_rate": 7.062467053241961e-07,
      "logits/chosen": -2.466992139816284,
      "logits/rejected": -2.4544920921325684,
      "logps/chosen": -385.11248779296875,
      "logps/rejected": -363.92498779296875,
      "loss": 0.1968,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.176422119140625,
      "rewards/margins": 2.743457078933716,
      "rewards/rejected": -2.56640625,
      "step": 2230
    },
    {
      "epoch": 1.1808118081180812,
      "grad_norm": 56.103169294593386,
      "learning_rate": 7.049288350026357e-07,
      "logits/chosen": -2.4449219703674316,
      "logits/rejected": -2.446484327316284,
      "logps/chosen": -400.75,
      "logps/rejected": -382.7749938964844,
      "loss": 0.1905,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.04129638522863388,
      "rewards/margins": 2.8099122047424316,
      "rewards/rejected": -2.849609375,
      "step": 2240
    },
    {
      "epoch": 1.1860832894043227,
      "grad_norm": 25.785535637801612,
      "learning_rate": 7.036109646810754e-07,
      "logits/chosen": -2.301953077316284,
      "logits/rejected": -2.3287110328674316,
      "logps/chosen": -397.3999938964844,
      "logps/rejected": -368.79998779296875,
      "loss": 0.1972,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.05502929538488388,
      "rewards/margins": 2.670117139816284,
      "rewards/rejected": -2.6156249046325684,
      "step": 2250
    },
    {
      "epoch": 1.191354770690564,
      "grad_norm": 54.95007581044649,
      "learning_rate": 7.02293094359515e-07,
      "logits/chosen": -2.3882813453674316,
      "logits/rejected": -2.3837890625,
      "logps/chosen": -339.0,
      "logps/rejected": -325.82501220703125,
      "loss": 0.2018,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.07403373718261719,
      "rewards/margins": 2.523730516433716,
      "rewards/rejected": -2.5977540016174316,
      "step": 2260
    },
    {
      "epoch": 1.1966262519768054,
      "grad_norm": 67.6485894676327,
      "learning_rate": 7.009752240379547e-07,
      "logits/chosen": -2.322265625,
      "logits/rejected": -2.331835985183716,
      "logps/chosen": -397.1000061035156,
      "logps/rejected": -373.79998779296875,
      "loss": 0.1515,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.39190673828125,
      "rewards/margins": 3.049609422683716,
      "rewards/rejected": -2.6587891578674316,
      "step": 2270
    },
    {
      "epoch": 1.201897733263047,
      "grad_norm": 31.398716526951034,
      "learning_rate": 6.996573537163942e-07,
      "logits/chosen": -2.2044920921325684,
      "logits/rejected": -2.2964844703674316,
      "logps/chosen": -388.5249938964844,
      "logps/rejected": -356.6625061035156,
      "loss": 0.1765,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.5289398431777954,
      "rewards/margins": 2.7314453125,
      "rewards/rejected": -2.203564405441284,
      "step": 2280
    },
    {
      "epoch": 1.2071692145492883,
      "grad_norm": 26.834421597507433,
      "learning_rate": 6.98339483394834e-07,
      "logits/chosen": -2.3873047828674316,
      "logits/rejected": -2.466796875,
      "logps/chosen": -359.82501220703125,
      "logps/rejected": -371.5,
      "loss": 0.1819,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.6697753667831421,
      "rewards/margins": 2.8231201171875,
      "rewards/rejected": -2.153125047683716,
      "step": 2290
    },
    {
      "epoch": 1.2124406958355298,
      "grad_norm": 23.03804380146357,
      "learning_rate": 6.970216130732735e-07,
      "logits/chosen": -2.341796875,
      "logits/rejected": -2.3314452171325684,
      "logps/chosen": -398.6000061035156,
      "logps/rejected": -374.4750061035156,
      "loss": 0.1566,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.820727527141571,
      "rewards/margins": 2.9761719703674316,
      "rewards/rejected": -2.155468702316284,
      "step": 2300
    },
    {
      "epoch": 1.2177121771217712,
      "grad_norm": 53.261641976818304,
      "learning_rate": 6.957037427517131e-07,
      "logits/chosen": -2.4039063453674316,
      "logits/rejected": -2.4302735328674316,
      "logps/chosen": -367.17498779296875,
      "logps/rejected": -330.36248779296875,
      "loss": 0.1719,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.5959106683731079,
      "rewards/margins": 2.7890625,
      "rewards/rejected": -2.1932616233825684,
      "step": 2310
    },
    {
      "epoch": 1.2229836584080127,
      "grad_norm": 62.374688439398014,
      "learning_rate": 6.943858724301529e-07,
      "logits/chosen": -2.4300780296325684,
      "logits/rejected": -2.4076170921325684,
      "logps/chosen": -407.92498779296875,
      "logps/rejected": -345.2749938964844,
      "loss": 0.1932,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 0.00811767578125,
      "rewards/margins": 2.845703125,
      "rewards/rejected": -2.838671922683716,
      "step": 2320
    },
    {
      "epoch": 1.228255139694254,
      "grad_norm": 32.802474856413674,
      "learning_rate": 6.930680021085925e-07,
      "logits/chosen": -2.533984422683716,
      "logits/rejected": -2.464648485183716,
      "logps/chosen": -356.3500061035156,
      "logps/rejected": -343.54998779296875,
      "loss": 0.1963,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.0182037353515625,
      "rewards/margins": 2.76904296875,
      "rewards/rejected": -2.7874999046325684,
      "step": 2330
    },
    {
      "epoch": 1.2335266209804956,
      "grad_norm": 35.29582309696523,
      "learning_rate": 6.917501317870321e-07,
      "logits/chosen": -2.3470702171325684,
      "logits/rejected": -2.4423828125,
      "logps/chosen": -378.29998779296875,
      "logps/rejected": -369.2250061035156,
      "loss": 0.1699,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.15971679985523224,
      "rewards/margins": 3.017578125,
      "rewards/rejected": -2.8578124046325684,
      "step": 2340
    },
    {
      "epoch": 1.238798102266737,
      "grad_norm": 35.906609595060154,
      "learning_rate": 6.904322614654717e-07,
      "logits/chosen": -2.373046875,
      "logits/rejected": -2.411914110183716,
      "logps/chosen": -397.8500061035156,
      "logps/rejected": -402.875,
      "loss": 0.1561,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.04046630859375,
      "rewards/margins": 2.905468702316284,
      "rewards/rejected": -2.946093797683716,
      "step": 2350
    },
    {
      "epoch": 1.2440695835529785,
      "grad_norm": 57.24809444923897,
      "learning_rate": 6.891143911439114e-07,
      "logits/chosen": -2.358593702316284,
      "logits/rejected": -2.384765625,
      "logps/chosen": -418.88751220703125,
      "logps/rejected": -419.0,
      "loss": 0.1546,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.008227539248764515,
      "rewards/margins": 3.039843797683716,
      "rewards/rejected": -3.0308594703674316,
      "step": 2360
    },
    {
      "epoch": 1.2493410648392198,
      "grad_norm": 35.824780948309225,
      "learning_rate": 6.87796520822351e-07,
      "logits/chosen": -2.2845702171325684,
      "logits/rejected": -2.369140625,
      "logps/chosen": -375.67498779296875,
      "logps/rejected": -340.3125,
      "loss": 0.2233,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.2787414491176605,
      "rewards/margins": 2.725781202316284,
      "rewards/rejected": -3.0048828125,
      "step": 2370
    },
    {
      "epoch": 1.2546125461254611,
      "grad_norm": 40.79206812885821,
      "learning_rate": 6.864786505007907e-07,
      "logits/chosen": -2.419726610183716,
      "logits/rejected": -2.442578077316284,
      "logps/chosen": -448.29998779296875,
      "logps/rejected": -391.67498779296875,
      "loss": 0.1863,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.06229247897863388,
      "rewards/margins": 2.8150391578674316,
      "rewards/rejected": -2.753124952316284,
      "step": 2380
    },
    {
      "epoch": 1.2598840274117027,
      "grad_norm": 32.48668566636298,
      "learning_rate": 6.851607801792303e-07,
      "logits/chosen": -2.3359375,
      "logits/rejected": -2.3287110328674316,
      "logps/chosen": -403.86248779296875,
      "logps/rejected": -402.26251220703125,
      "loss": 0.1555,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.24499817192554474,
      "rewards/margins": 2.9427733421325684,
      "rewards/rejected": -2.697265625,
      "step": 2390
    },
    {
      "epoch": 1.2651555086979442,
      "grad_norm": 40.45138567184018,
      "learning_rate": 6.8384290985767e-07,
      "logits/chosen": -2.385546922683716,
      "logits/rejected": -2.343945264816284,
      "logps/chosen": -338.70001220703125,
      "logps/rejected": -358.73748779296875,
      "loss": 0.1738,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.06630249321460724,
      "rewards/margins": 2.8779296875,
      "rewards/rejected": -2.810742139816284,
      "step": 2400
    },
    {
      "epoch": 1.2704269899841856,
      "grad_norm": 25.671104200202343,
      "learning_rate": 6.825250395361096e-07,
      "logits/chosen": -2.4144530296325684,
      "logits/rejected": -2.3638672828674316,
      "logps/chosen": -414.92498779296875,
      "logps/rejected": -389.9375,
      "loss": 0.1217,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": 0.03261413425207138,
      "rewards/margins": 3.0541014671325684,
      "rewards/rejected": -3.0208983421325684,
      "step": 2410
    },
    {
      "epoch": 1.275698471270427,
      "grad_norm": 59.97543447905003,
      "learning_rate": 6.812071692145492e-07,
      "logits/chosen": -2.3798828125,
      "logits/rejected": -2.4371094703674316,
      "logps/chosen": -385.0874938964844,
      "logps/rejected": -322.375,
      "loss": 0.1725,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.15858764946460724,
      "rewards/margins": 2.948437452316284,
      "rewards/rejected": -2.789355516433716,
      "step": 2420
    },
    {
      "epoch": 1.2809699525566685,
      "grad_norm": 24.274117242874404,
      "learning_rate": 6.798892988929888e-07,
      "logits/chosen": -2.384960889816284,
      "logits/rejected": -2.354296922683716,
      "logps/chosen": -460.25,
      "logps/rejected": -431.0,
      "loss": 0.1127,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.01875610277056694,
      "rewards/margins": 3.56640625,
      "rewards/rejected": -3.584765672683716,
      "step": 2430
    },
    {
      "epoch": 1.2862414338429098,
      "grad_norm": 51.80320094224363,
      "learning_rate": 6.785714285714286e-07,
      "logits/chosen": -2.3970704078674316,
      "logits/rejected": -2.381054639816284,
      "logps/chosen": -364.23748779296875,
      "logps/rejected": -358.01251220703125,
      "loss": 0.1938,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.05949706956744194,
      "rewards/margins": 3.1214842796325684,
      "rewards/rejected": -3.0625,
      "step": 2440
    },
    {
      "epoch": 1.2915129151291513,
      "grad_norm": 32.29448819322456,
      "learning_rate": 6.772535582498682e-07,
      "logits/chosen": -2.3101563453674316,
      "logits/rejected": -2.345703125,
      "logps/chosen": -358.88751220703125,
      "logps/rejected": -328.57501220703125,
      "loss": 0.1664,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.18281249701976776,
      "rewards/margins": 2.9737305641174316,
      "rewards/rejected": -2.792187452316284,
      "step": 2450
    },
    {
      "epoch": 1.2967843964153927,
      "grad_norm": 31.676889645241637,
      "learning_rate": 6.759356879283078e-07,
      "logits/chosen": -2.439257860183716,
      "logits/rejected": -2.3759765625,
      "logps/chosen": -411.75,
      "logps/rejected": -420.125,
      "loss": 0.1562,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.31729429960250854,
      "rewards/margins": 2.990039110183716,
      "rewards/rejected": -2.6724610328674316,
      "step": 2460
    },
    {
      "epoch": 1.3020558777016342,
      "grad_norm": 43.73603885420168,
      "learning_rate": 6.746178176067475e-07,
      "logits/chosen": -2.377148389816284,
      "logits/rejected": -2.313671827316284,
      "logps/chosen": -427.32501220703125,
      "logps/rejected": -398.32501220703125,
      "loss": 0.1284,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.14015503227710724,
      "rewards/margins": 3.3355469703674316,
      "rewards/rejected": -3.195117235183716,
      "step": 2470
    },
    {
      "epoch": 1.3073273589878756,
      "grad_norm": 36.9254471409548,
      "learning_rate": 6.732999472851871e-07,
      "logits/chosen": -2.3226561546325684,
      "logits/rejected": -2.3550782203674316,
      "logps/chosen": -412.42498779296875,
      "logps/rejected": -384.5,
      "loss": 0.1635,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.3875183165073395,
      "rewards/margins": 3.1375975608825684,
      "rewards/rejected": -2.751220703125,
      "step": 2480
    },
    {
      "epoch": 1.312598840274117,
      "grad_norm": 57.84250765644717,
      "learning_rate": 6.719820769636268e-07,
      "logits/chosen": -2.3677735328674316,
      "logits/rejected": -2.479687452316284,
      "logps/chosen": -389.2250061035156,
      "logps/rejected": -410.45001220703125,
      "loss": 0.1716,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.22697754204273224,
      "rewards/margins": 2.980761766433716,
      "rewards/rejected": -2.7529296875,
      "step": 2490
    },
    {
      "epoch": 1.3178703215603584,
      "grad_norm": 44.33185257034434,
      "learning_rate": 6.706642066420664e-07,
      "logits/chosen": -2.476367235183716,
      "logits/rejected": -2.417187452316284,
      "logps/chosen": -369.0874938964844,
      "logps/rejected": -350.79998779296875,
      "loss": 0.2013,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.05342712253332138,
      "rewards/margins": 2.8119139671325684,
      "rewards/rejected": -2.866015672683716,
      "step": 2500
    },
    {
      "epoch": 1.3231418028466,
      "grad_norm": 27.684583872259072,
      "learning_rate": 6.693463363205061e-07,
      "logits/chosen": -2.454296827316284,
      "logits/rejected": -2.423046827316284,
      "logps/chosen": -403.0,
      "logps/rejected": -397.8500061035156,
      "loss": 0.1517,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.12498779594898224,
      "rewards/margins": 3.2330079078674316,
      "rewards/rejected": -3.357421875,
      "step": 2510
    },
    {
      "epoch": 1.3284132841328413,
      "grad_norm": 41.392978703013554,
      "learning_rate": 6.680284659989457e-07,
      "logits/chosen": -2.3824219703674316,
      "logits/rejected": -2.3744139671325684,
      "logps/chosen": -402.04998779296875,
      "logps/rejected": -438.6499938964844,
      "loss": 0.2029,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.36607664823532104,
      "rewards/margins": 3.1142578125,
      "rewards/rejected": -3.4837889671325684,
      "step": 2520
    },
    {
      "epoch": 1.3336847654190827,
      "grad_norm": 53.753329337927866,
      "learning_rate": 6.667105956773853e-07,
      "logits/chosen": -2.4140625,
      "logits/rejected": -2.456249952316284,
      "logps/chosen": -470.3125,
      "logps/rejected": -442.0874938964844,
      "loss": 0.1559,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.264251708984375,
      "rewards/margins": 3.2108397483825684,
      "rewards/rejected": -3.475390672683716,
      "step": 2530
    },
    {
      "epoch": 1.3389562467053242,
      "grad_norm": 48.4470616116594,
      "learning_rate": 6.653927253558249e-07,
      "logits/chosen": -2.4984374046325684,
      "logits/rejected": -2.5404295921325684,
      "logps/chosen": -410.82501220703125,
      "logps/rejected": -375.07501220703125,
      "loss": 0.1805,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.4241394102573395,
      "rewards/margins": 2.951367139816284,
      "rewards/rejected": -3.3746094703674316,
      "step": 2540
    },
    {
      "epoch": 1.3442277279915658,
      "grad_norm": 24.615001253807744,
      "learning_rate": 6.640748550342647e-07,
      "logits/chosen": -2.456249952316284,
      "logits/rejected": -2.4300780296325684,
      "logps/chosen": -413.875,
      "logps/rejected": -396.8999938964844,
      "loss": 0.1497,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.4702392518520355,
      "rewards/margins": 2.8451170921325684,
      "rewards/rejected": -3.3150391578674316,
      "step": 2550
    },
    {
      "epoch": 1.349499209277807,
      "grad_norm": 59.80634099168327,
      "learning_rate": 6.627569847127043e-07,
      "logits/chosen": -2.4771485328674316,
      "logits/rejected": -2.4658203125,
      "logps/chosen": -444.1000061035156,
      "logps/rejected": -378.20001220703125,
      "loss": 0.142,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.5243164300918579,
      "rewards/margins": 3.034960985183716,
      "rewards/rejected": -3.5601563453674316,
      "step": 2560
    },
    {
      "epoch": 1.3547706905640484,
      "grad_norm": 50.47518444019526,
      "learning_rate": 6.614391143911439e-07,
      "logits/chosen": -2.494335889816284,
      "logits/rejected": -2.5746092796325684,
      "logps/chosen": -418.82501220703125,
      "logps/rejected": -365.8999938964844,
      "loss": 0.1501,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.10500488430261612,
      "rewards/margins": 3.066210985183716,
      "rewards/rejected": -3.171093702316284,
      "step": 2570
    },
    {
      "epoch": 1.36004217185029,
      "grad_norm": 28.08848841653263,
      "learning_rate": 6.601212440695835e-07,
      "logits/chosen": -2.428515672683716,
      "logits/rejected": -2.4976563453674316,
      "logps/chosen": -397.5,
      "logps/rejected": -370.4125061035156,
      "loss": 0.1844,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.43190306425094604,
      "rewards/margins": 3.023730516433716,
      "rewards/rejected": -3.458789110183716,
      "step": 2580
    },
    {
      "epoch": 1.3653136531365313,
      "grad_norm": 56.339887161993694,
      "learning_rate": 6.588033737480232e-07,
      "logits/chosen": -2.384570360183716,
      "logits/rejected": -2.4466795921325684,
      "logps/chosen": -414.32501220703125,
      "logps/rejected": -415.45001220703125,
      "loss": 0.1436,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.48515623807907104,
      "rewards/margins": 3.3255858421325684,
      "rewards/rejected": -3.8101563453674316,
      "step": 2590
    },
    {
      "epoch": 1.3705851344227729,
      "grad_norm": 35.91649810110443,
      "learning_rate": 6.574855034264628e-07,
      "logits/chosen": -2.593554735183716,
      "logits/rejected": -2.506640672683716,
      "logps/chosen": -341.625,
      "logps/rejected": -353.54998779296875,
      "loss": 0.2242,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.4096435606479645,
      "rewards/margins": 2.6937499046325684,
      "rewards/rejected": -3.101367235183716,
      "step": 2600
    },
    {
      "epoch": 1.3758566157090142,
      "grad_norm": 48.098521023009035,
      "learning_rate": 6.561676331049025e-07,
      "logits/chosen": -2.509960889816284,
      "logits/rejected": -2.4798827171325684,
      "logps/chosen": -446.54998779296875,
      "logps/rejected": -437.07501220703125,
      "loss": 0.1479,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.07086791843175888,
      "rewards/margins": 3.1064453125,
      "rewards/rejected": -3.035937547683716,
      "step": 2610
    },
    {
      "epoch": 1.3811280969952557,
      "grad_norm": 14.41752311530744,
      "learning_rate": 6.548497627833422e-07,
      "logits/chosen": -2.4384765625,
      "logits/rejected": -2.524609327316284,
      "logps/chosen": -403.70001220703125,
      "logps/rejected": -383.86248779296875,
      "loss": 0.179,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.004425048828125,
      "rewards/margins": 3.016406297683716,
      "rewards/rejected": -3.0123047828674316,
      "step": 2620
    },
    {
      "epoch": 1.386399578281497,
      "grad_norm": 49.765918180308546,
      "learning_rate": 6.535318924617818e-07,
      "logits/chosen": -2.4974608421325684,
      "logits/rejected": -2.5205078125,
      "logps/chosen": -397.07501220703125,
      "logps/rejected": -389.70001220703125,
      "loss": 0.1443,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.033203125,
      "rewards/margins": 3.2496094703674316,
      "rewards/rejected": -3.2164063453674316,
      "step": 2630
    },
    {
      "epoch": 1.3916710595677384,
      "grad_norm": 91.93382250827338,
      "learning_rate": 6.522140221402213e-07,
      "logits/chosen": -2.4517579078674316,
      "logits/rejected": -2.368359327316284,
      "logps/chosen": -418.1000061035156,
      "logps/rejected": -400.1499938964844,
      "loss": 0.1872,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.01831665076315403,
      "rewards/margins": 3.3154296875,
      "rewards/rejected": -3.297070264816284,
      "step": 2640
    },
    {
      "epoch": 1.39694254085398,
      "grad_norm": 68.25294337470777,
      "learning_rate": 6.508961518186609e-07,
      "logits/chosen": -2.512500047683716,
      "logits/rejected": -2.4576172828674316,
      "logps/chosen": -432.75,
      "logps/rejected": -390.92498779296875,
      "loss": 0.1828,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.09624023735523224,
      "rewards/margins": 3.180957078933716,
      "rewards/rejected": -3.0835938453674316,
      "step": 2650
    },
    {
      "epoch": 1.4022140221402215,
      "grad_norm": 64.04064697401049,
      "learning_rate": 6.495782814971007e-07,
      "logits/chosen": -2.44140625,
      "logits/rejected": -2.4712891578674316,
      "logps/chosen": -336.1875,
      "logps/rejected": -336.70001220703125,
      "loss": 0.1482,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.15718384087085724,
      "rewards/margins": 3.300097703933716,
      "rewards/rejected": -3.4564452171325684,
      "step": 2660
    },
    {
      "epoch": 1.4074855034264628,
      "grad_norm": 21.330639793165297,
      "learning_rate": 6.482604111755403e-07,
      "logits/chosen": -2.471484422683716,
      "logits/rejected": -2.4935545921325684,
      "logps/chosen": -390.875,
      "logps/rejected": -394.92498779296875,
      "loss": 0.1639,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.36253660917282104,
      "rewards/margins": 3.1646485328674316,
      "rewards/rejected": -3.5308594703674316,
      "step": 2670
    },
    {
      "epoch": 1.4127569847127042,
      "grad_norm": 18.1961474939078,
      "learning_rate": 6.469425408539799e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.572460889816284,
      "logps/chosen": -406.8500061035156,
      "logps/rejected": -372.125,
      "loss": 0.1873,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.702587902545929,
      "rewards/margins": 3.0814452171325684,
      "rewards/rejected": -3.7855467796325684,
      "step": 2680
    },
    {
      "epoch": 1.4180284659989457,
      "grad_norm": 70.56281719092125,
      "learning_rate": 6.456246705324195e-07,
      "logits/chosen": -2.3863282203674316,
      "logits/rejected": -2.3910155296325684,
      "logps/chosen": -388.65625,
      "logps/rejected": -412.04998779296875,
      "loss": 0.2006,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.533374011516571,
      "rewards/margins": 3.092480421066284,
      "rewards/rejected": -3.624804735183716,
      "step": 2690
    },
    {
      "epoch": 1.4232999472851873,
      "grad_norm": 51.00233459136705,
      "learning_rate": 6.443068002108592e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.455273389816284,
      "logps/chosen": -391.1499938964844,
      "logps/rejected": -410.125,
      "loss": 0.1526,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.3272338807582855,
      "rewards/margins": 3.4380860328674316,
      "rewards/rejected": -3.7642579078674316,
      "step": 2700
    },
    {
      "epoch": 1.4285714285714286,
      "grad_norm": 76.46679175081505,
      "learning_rate": 6.429889298892988e-07,
      "logits/chosen": -2.3408203125,
      "logits/rejected": -2.393749952316284,
      "logps/chosen": -400.9750061035156,
      "logps/rejected": -405.32501220703125,
      "loss": 0.1629,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.35792845487594604,
      "rewards/margins": 3.166015625,
      "rewards/rejected": -3.5240235328674316,
      "step": 2710
    },
    {
      "epoch": 1.43384290985767,
      "grad_norm": 46.2743193223281,
      "learning_rate": 6.416710595677385e-07,
      "logits/chosen": -2.4078125953674316,
      "logits/rejected": -2.517382860183716,
      "logps/chosen": -393.5249938964844,
      "logps/rejected": -364.625,
      "loss": 0.2163,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.5914551019668579,
      "rewards/margins": 2.938281297683716,
      "rewards/rejected": -3.530468702316284,
      "step": 2720
    },
    {
      "epoch": 1.4391143911439115,
      "grad_norm": 50.75205126666711,
      "learning_rate": 6.403531892461781e-07,
      "logits/chosen": -2.3501954078674316,
      "logits/rejected": -2.454296827316284,
      "logps/chosen": -444.8125,
      "logps/rejected": -406.8374938964844,
      "loss": 0.175,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.536853015422821,
      "rewards/margins": 3.00390625,
      "rewards/rejected": -3.5439453125,
      "step": 2730
    },
    {
      "epoch": 1.4443858724301528,
      "grad_norm": 26.817470969555632,
      "learning_rate": 6.390353189246178e-07,
      "logits/chosen": -2.398632764816284,
      "logits/rejected": -2.4664063453674316,
      "logps/chosen": -406.0625,
      "logps/rejected": -381.1000061035156,
      "loss": 0.1669,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.24737548828125,
      "rewards/margins": 3.053906202316284,
      "rewards/rejected": -3.298632860183716,
      "step": 2740
    },
    {
      "epoch": 1.4496573537163944,
      "grad_norm": 48.5095379634166,
      "learning_rate": 6.377174486030574e-07,
      "logits/chosen": -2.479687452316284,
      "logits/rejected": -2.4140625,
      "logps/chosen": -390.3999938964844,
      "logps/rejected": -414.8500061035156,
      "loss": 0.178,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.3340698182582855,
      "rewards/margins": 3.123242139816284,
      "rewards/rejected": -3.456835985183716,
      "step": 2750
    },
    {
      "epoch": 1.4549288350026357,
      "grad_norm": 32.02892025801427,
      "learning_rate": 6.36399578281497e-07,
      "logits/chosen": -2.4408202171325684,
      "logits/rejected": -2.4228515625,
      "logps/chosen": -422.57501220703125,
      "logps/rejected": -406.7250061035156,
      "loss": 0.1147,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.20692138373851776,
      "rewards/margins": 3.352734327316284,
      "rewards/rejected": -3.55859375,
      "step": 2760
    },
    {
      "epoch": 1.4602003162888773,
      "grad_norm": 33.52982734692608,
      "learning_rate": 6.350817079599367e-07,
      "logits/chosen": -2.431835889816284,
      "logits/rejected": -2.564453125,
      "logps/chosen": -386.0249938964844,
      "logps/rejected": -368.20001220703125,
      "loss": 0.1409,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.192108154296875,
      "rewards/margins": 3.0254883766174316,
      "rewards/rejected": -3.219531297683716,
      "step": 2770
    },
    {
      "epoch": 1.4654717975751186,
      "grad_norm": 89.86759202940793,
      "learning_rate": 6.337638376383764e-07,
      "logits/chosen": -2.450976610183716,
      "logits/rejected": -2.463671922683716,
      "logps/chosen": -400.61248779296875,
      "logps/rejected": -402.04998779296875,
      "loss": 0.144,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.2295379638671875,
      "rewards/margins": 3.2113280296325684,
      "rewards/rejected": -3.4400391578674316,
      "step": 2780
    },
    {
      "epoch": 1.47074327886136,
      "grad_norm": 101.83171529935747,
      "learning_rate": 6.32445967316816e-07,
      "logits/chosen": -2.4609375,
      "logits/rejected": -2.423046827316284,
      "logps/chosen": -402.04998779296875,
      "logps/rejected": -383.5,
      "loss": 0.1653,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.09714965522289276,
      "rewards/margins": 3.461962938308716,
      "rewards/rejected": -3.559765577316284,
      "step": 2790
    },
    {
      "epoch": 1.4760147601476015,
      "grad_norm": 30.92253561429161,
      "learning_rate": 6.311280969952556e-07,
      "logits/chosen": -2.4847655296325684,
      "logits/rejected": -2.4921875,
      "logps/chosen": -414.7749938964844,
      "logps/rejected": -404.7250061035156,
      "loss": 0.1572,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.47839051485061646,
      "rewards/margins": 3.393847703933716,
      "rewards/rejected": -3.8734374046325684,
      "step": 2800
    },
    {
      "epoch": 1.481286241433843,
      "grad_norm": 34.20069526252577,
      "learning_rate": 6.298102266736953e-07,
      "logits/chosen": -2.5732421875,
      "logits/rejected": -2.5023436546325684,
      "logps/chosen": -396.9375,
      "logps/rejected": -403.2749938964844,
      "loss": 0.1656,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.05074463039636612,
      "rewards/margins": 3.3036131858825684,
      "rewards/rejected": -3.353320360183716,
      "step": 2810
    },
    {
      "epoch": 1.4865577227200844,
      "grad_norm": 65.55201289196853,
      "learning_rate": 6.284923563521349e-07,
      "logits/chosen": -2.482421875,
      "logits/rejected": -2.511914014816284,
      "logps/chosen": -387.3500061035156,
      "logps/rejected": -366.0625,
      "loss": 0.1552,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.09255371242761612,
      "rewards/margins": 2.982421875,
      "rewards/rejected": -2.889453172683716,
      "step": 2820
    },
    {
      "epoch": 1.4918292040063257,
      "grad_norm": 47.636581177961496,
      "learning_rate": 6.271744860305746e-07,
      "logits/chosen": -2.352734327316284,
      "logits/rejected": -2.399609327316284,
      "logps/chosen": -418.7124938964844,
      "logps/rejected": -406.1000061035156,
      "loss": 0.1433,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.17411498725414276,
      "rewards/margins": 3.035351514816284,
      "rewards/rejected": -3.208984375,
      "step": 2830
    },
    {
      "epoch": 1.4971006852925672,
      "grad_norm": 43.55769453879038,
      "learning_rate": 6.258566157090142e-07,
      "logits/chosen": -2.4957032203674316,
      "logits/rejected": -2.5277342796325684,
      "logps/chosen": -380.98748779296875,
      "logps/rejected": -394.38751220703125,
      "loss": 0.1791,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.05557861179113388,
      "rewards/margins": 3.065624952316284,
      "rewards/rejected": -3.12109375,
      "step": 2840
    },
    {
      "epoch": 1.5023721665788088,
      "grad_norm": 89.64463304643245,
      "learning_rate": 6.245387453874539e-07,
      "logits/chosen": -2.424609422683716,
      "logits/rejected": -2.435351610183716,
      "logps/chosen": -421.75,
      "logps/rejected": -403.5,
      "loss": 0.1719,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.46464842557907104,
      "rewards/margins": 3.1465821266174316,
      "rewards/rejected": -3.6123046875,
      "step": 2850
    },
    {
      "epoch": 1.5076436478650501,
      "grad_norm": 56.57503946966515,
      "learning_rate": 6.232208750658935e-07,
      "logits/chosen": -2.50390625,
      "logits/rejected": -2.465039014816284,
      "logps/chosen": -372.1000061035156,
      "logps/rejected": -365.61248779296875,
      "loss": 0.2029,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.3837524354457855,
      "rewards/margins": 3.0147461891174316,
      "rewards/rejected": -3.3974609375,
      "step": 2860
    },
    {
      "epoch": 1.5129151291512914,
      "grad_norm": 61.38486607183011,
      "learning_rate": 6.219030047443331e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.5703125,
      "logps/chosen": -405.375,
      "logps/rejected": -369.45001220703125,
      "loss": 0.1544,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.49023133516311646,
      "rewards/margins": 3.081738233566284,
      "rewards/rejected": -3.570507764816284,
      "step": 2870
    },
    {
      "epoch": 1.518186610437533,
      "grad_norm": 59.27938114441121,
      "learning_rate": 6.205851344227727e-07,
      "logits/chosen": -2.437304735183716,
      "logits/rejected": -2.395703077316284,
      "logps/chosen": -410.8500061035156,
      "logps/rejected": -415.57501220703125,
      "loss": 0.2007,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.3519043028354645,
      "rewards/margins": 3.260937452316284,
      "rewards/rejected": -3.613476514816284,
      "step": 2880
    },
    {
      "epoch": 1.5234580917237743,
      "grad_norm": 34.83119886433461,
      "learning_rate": 6.192672641012125e-07,
      "logits/chosen": -2.576171875,
      "logits/rejected": -2.6156249046325684,
      "logps/chosen": -367.45001220703125,
      "logps/rejected": -372.54998779296875,
      "loss": 0.1929,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.22935180366039276,
      "rewards/margins": 2.914257764816284,
      "rewards/rejected": -3.143749952316284,
      "step": 2890
    },
    {
      "epoch": 1.5287295730100157,
      "grad_norm": 31.28823350803994,
      "learning_rate": 6.179493937796521e-07,
      "logits/chosen": -2.4847655296325684,
      "logits/rejected": -2.490039110183716,
      "logps/chosen": -414.9624938964844,
      "logps/rejected": -388.1000061035156,
      "loss": 0.1599,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.3113952577114105,
      "rewards/margins": 3.169140577316284,
      "rewards/rejected": -3.4814453125,
      "step": 2900
    },
    {
      "epoch": 1.5340010542962572,
      "grad_norm": 36.94264056734878,
      "learning_rate": 6.166315234580917e-07,
      "logits/chosen": -2.509570360183716,
      "logits/rejected": -2.4828124046325684,
      "logps/chosen": -400.8374938964844,
      "logps/rejected": -365.125,
      "loss": 0.1747,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.02947998046875,
      "rewards/margins": 3.047656297683716,
      "rewards/rejected": -3.0794920921325684,
      "step": 2910
    },
    {
      "epoch": 1.5392725355824988,
      "grad_norm": 58.7343576612861,
      "learning_rate": 6.153136531365314e-07,
      "logits/chosen": -2.520703077316284,
      "logits/rejected": -2.456249952316284,
      "logps/chosen": -353.4624938964844,
      "logps/rejected": -412.92498779296875,
      "loss": 0.1927,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.16368408501148224,
      "rewards/margins": 3.0728516578674316,
      "rewards/rejected": -3.234375,
      "step": 2920
    },
    {
      "epoch": 1.54454401686874,
      "grad_norm": 33.20372551105343,
      "learning_rate": 6.13995782814971e-07,
      "logits/chosen": -2.508007764816284,
      "logits/rejected": -2.4541015625,
      "logps/chosen": -392.4125061035156,
      "logps/rejected": -394.7875061035156,
      "loss": 0.1215,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.2769103944301605,
      "rewards/margins": 3.2508788108825684,
      "rewards/rejected": -3.5296874046325684,
      "step": 2930
    },
    {
      "epoch": 1.5498154981549814,
      "grad_norm": 42.440125269326124,
      "learning_rate": 6.126779124934106e-07,
      "logits/chosen": -2.5458984375,
      "logits/rejected": -2.5132813453674316,
      "logps/chosen": -396.0375061035156,
      "logps/rejected": -361.125,
      "loss": 0.155,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -4.57763671875e-05,
      "rewards/margins": 3.169921875,
      "rewards/rejected": -3.171093702316284,
      "step": 2940
    },
    {
      "epoch": 1.555086979441223,
      "grad_norm": 62.62891960905312,
      "learning_rate": 6.113600421718503e-07,
      "logits/chosen": -2.527148485183716,
      "logits/rejected": -2.4417967796325684,
      "logps/chosen": -410.125,
      "logps/rejected": -416.375,
      "loss": 0.163,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.01523437537252903,
      "rewards/margins": 3.2574219703674316,
      "rewards/rejected": -3.274609327316284,
      "step": 2950
    },
    {
      "epoch": 1.5603584607274645,
      "grad_norm": 44.79204347210229,
      "learning_rate": 6.1004217185029e-07,
      "logits/chosen": -2.403515577316284,
      "logits/rejected": -2.4458985328674316,
      "logps/chosen": -376.45001220703125,
      "logps/rejected": -372.29998779296875,
      "loss": 0.1577,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.2550598084926605,
      "rewards/margins": 3.279296875,
      "rewards/rejected": -3.533398389816284,
      "step": 2960
    },
    {
      "epoch": 1.5656299420137059,
      "grad_norm": 39.20486138833542,
      "learning_rate": 6.087243015287296e-07,
      "logits/chosen": -2.482617139816284,
      "logits/rejected": -2.4447264671325684,
      "logps/chosen": -396.5249938964844,
      "logps/rejected": -370.2749938964844,
      "loss": 0.1842,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.22868041694164276,
      "rewards/margins": 3.091796875,
      "rewards/rejected": -3.321093797683716,
      "step": 2970
    },
    {
      "epoch": 1.5709014232999472,
      "grad_norm": 31.660946918762175,
      "learning_rate": 6.074064312071692e-07,
      "logits/chosen": -2.4535155296325684,
      "logits/rejected": -2.4224610328674316,
      "logps/chosen": -390.3999938964844,
      "logps/rejected": -362.0249938964844,
      "loss": 0.1152,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.03634033352136612,
      "rewards/margins": 3.3794922828674316,
      "rewards/rejected": -3.340039014816284,
      "step": 2980
    },
    {
      "epoch": 1.5761729045861887,
      "grad_norm": 30.022232865508087,
      "learning_rate": 6.060885608856087e-07,
      "logits/chosen": -2.42138671875,
      "logits/rejected": -2.4095702171325684,
      "logps/chosen": -383.6000061035156,
      "logps/rejected": -369.82501220703125,
      "loss": 0.1578,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.018463134765625,
      "rewards/margins": 3.468701124191284,
      "rewards/rejected": -3.4498047828674316,
      "step": 2990
    },
    {
      "epoch": 1.5814443858724303,
      "grad_norm": 27.840599981213508,
      "learning_rate": 6.047706905640486e-07,
      "logits/chosen": -2.3462891578674316,
      "logits/rejected": -2.3882813453674316,
      "logps/chosen": -386.1875,
      "logps/rejected": -421.04998779296875,
      "loss": 0.1198,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.06932983547449112,
      "rewards/margins": 3.5689454078674316,
      "rewards/rejected": -3.498828172683716,
      "step": 3000
    },
    {
      "epoch": 1.5867158671586716,
      "grad_norm": 23.261527412698662,
      "learning_rate": 6.034528202424881e-07,
      "logits/chosen": -2.6142578125,
      "logits/rejected": -2.686328172683716,
      "logps/chosen": -408.5249938964844,
      "logps/rejected": -383.125,
      "loss": 0.1959,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.32452392578125,
      "rewards/margins": 3.123339891433716,
      "rewards/rejected": -3.4458985328674316,
      "step": 3010
    },
    {
      "epoch": 1.591987348444913,
      "grad_norm": 107.70808998995616,
      "learning_rate": 6.021349499209277e-07,
      "logits/chosen": -2.528515577316284,
      "logits/rejected": -2.5318360328674316,
      "logps/chosen": -394.8999938964844,
      "logps/rejected": -352.4750061035156,
      "loss": 0.1941,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.18596191704273224,
      "rewards/margins": 3.182812452316284,
      "rewards/rejected": -3.3707032203674316,
      "step": 3020
    },
    {
      "epoch": 1.5972588297311545,
      "grad_norm": 29.57239476269693,
      "learning_rate": 6.008170795993674e-07,
      "logits/chosen": -2.4789061546325684,
      "logits/rejected": -2.5845704078674316,
      "logps/chosen": -379.3374938964844,
      "logps/rejected": -344.3500061035156,
      "loss": 0.1759,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.13730469346046448,
      "rewards/margins": 3.173046827316284,
      "rewards/rejected": -3.310351610183716,
      "step": 3030
    },
    {
      "epoch": 1.6025303110173958,
      "grad_norm": 50.241676433720194,
      "learning_rate": 5.99499209277807e-07,
      "logits/chosen": -2.4496092796325684,
      "logits/rejected": -2.49609375,
      "logps/chosen": -409.1499938964844,
      "logps/rejected": -424.67498779296875,
      "loss": 0.1574,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.15676268935203552,
      "rewards/margins": 3.411914110183716,
      "rewards/rejected": -3.569531202316284,
      "step": 3040
    },
    {
      "epoch": 1.6078017923036372,
      "grad_norm": 70.06241867058623,
      "learning_rate": 5.981813389562466e-07,
      "logits/chosen": -2.4677734375,
      "logits/rejected": -2.413281202316284,
      "logps/chosen": -392.17498779296875,
      "logps/rejected": -373.25,
      "loss": 0.1788,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.260009765625,
      "rewards/margins": 3.0962891578674316,
      "rewards/rejected": -3.353515625,
      "step": 3050
    },
    {
      "epoch": 1.6130732735898787,
      "grad_norm": 54.38952380589992,
      "learning_rate": 5.968634686346863e-07,
      "logits/chosen": -2.503710985183716,
      "logits/rejected": -2.545117139816284,
      "logps/chosen": -411.54998779296875,
      "logps/rejected": -359.42498779296875,
      "loss": 0.1775,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.7015136480331421,
      "rewards/margins": 2.8311524391174316,
      "rewards/rejected": -3.5318360328674316,
      "step": 3060
    },
    {
      "epoch": 1.6183447548761203,
      "grad_norm": 25.491931745154105,
      "learning_rate": 5.95545598313126e-07,
      "logits/chosen": -2.463671922683716,
      "logits/rejected": -2.419140577316284,
      "logps/chosen": -403.2250061035156,
      "logps/rejected": -430.3500061035156,
      "loss": 0.1479,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.4617919921875,
      "rewards/margins": 3.3404297828674316,
      "rewards/rejected": -3.7992186546325684,
      "step": 3070
    },
    {
      "epoch": 1.6236162361623616,
      "grad_norm": 48.13507092693223,
      "learning_rate": 5.942277279915656e-07,
      "logits/chosen": -2.4351563453674316,
      "logits/rejected": -2.4609375,
      "logps/chosen": -421.0375061035156,
      "logps/rejected": -399.2749938964844,
      "loss": 0.1616,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.33758544921875,
      "rewards/margins": 3.223437547683716,
      "rewards/rejected": -3.5619139671325684,
      "step": 3080
    },
    {
      "epoch": 1.628887717448603,
      "grad_norm": 67.93176247757793,
      "learning_rate": 5.929098576700052e-07,
      "logits/chosen": -2.5322265625,
      "logits/rejected": -2.528515577316284,
      "logps/chosen": -340.5,
      "logps/rejected": -348.70001220703125,
      "loss": 0.1832,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.261474609375,
      "rewards/margins": 2.753124952316284,
      "rewards/rejected": -3.013476610183716,
      "step": 3090
    },
    {
      "epoch": 1.6341591987348445,
      "grad_norm": 20.16777562036025,
      "learning_rate": 5.915919873484448e-07,
      "logits/chosen": -2.4755859375,
      "logits/rejected": -2.5123047828674316,
      "logps/chosen": -346.2124938964844,
      "logps/rejected": -342.8374938964844,
      "loss": 0.1533,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.126953125,
      "rewards/margins": 3.1078124046325684,
      "rewards/rejected": -2.9820313453674316,
      "step": 3100
    },
    {
      "epoch": 1.639430680021086,
      "grad_norm": 63.655093895312795,
      "learning_rate": 5.902741170268845e-07,
      "logits/chosen": -2.42578125,
      "logits/rejected": -2.468945264816284,
      "logps/chosen": -370.38751220703125,
      "logps/rejected": -378.17498779296875,
      "loss": 0.1571,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.06853027641773224,
      "rewards/margins": 2.981640577316284,
      "rewards/rejected": -2.912890672683716,
      "step": 3110
    },
    {
      "epoch": 1.6447021613073274,
      "grad_norm": 51.644522562334934,
      "learning_rate": 5.889562467053242e-07,
      "logits/chosen": -2.416796922683716,
      "logits/rejected": -2.443554639816284,
      "logps/chosen": -408.29998779296875,
      "logps/rejected": -408.95001220703125,
      "loss": 0.1539,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.29864501953125,
      "rewards/margins": 3.322558641433716,
      "rewards/rejected": -3.024218797683716,
      "step": 3120
    },
    {
      "epoch": 1.6499736425935687,
      "grad_norm": 21.833230775855508,
      "learning_rate": 5.876383763837638e-07,
      "logits/chosen": -2.4974608421325684,
      "logits/rejected": -2.4482421875,
      "logps/chosen": -396.2749938964844,
      "logps/rejected": -408.20001220703125,
      "loss": 0.1541,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.06207885593175888,
      "rewards/margins": 3.5367188453674316,
      "rewards/rejected": -3.472851514816284,
      "step": 3130
    },
    {
      "epoch": 1.6552451238798103,
      "grad_norm": 61.62558183565713,
      "learning_rate": 5.863205060622034e-07,
      "logits/chosen": -2.398242235183716,
      "logits/rejected": -2.3857421875,
      "logps/chosen": -370.2875061035156,
      "logps/rejected": -402.45001220703125,
      "loss": 0.1786,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.28167420625686646,
      "rewards/margins": 3.291796922683716,
      "rewards/rejected": -3.57421875,
      "step": 3140
    },
    {
      "epoch": 1.6605166051660518,
      "grad_norm": 17.97674798543096,
      "learning_rate": 5.850026357406431e-07,
      "logits/chosen": -2.5501952171325684,
      "logits/rejected": -2.580859422683716,
      "logps/chosen": -403.1000061035156,
      "logps/rejected": -384.75,
      "loss": 0.1546,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.13291016221046448,
      "rewards/margins": 3.412109375,
      "rewards/rejected": -3.54296875,
      "step": 3150
    },
    {
      "epoch": 1.6657880864522931,
      "grad_norm": 69.4336287208856,
      "learning_rate": 5.836847654190827e-07,
      "logits/chosen": -2.5048828125,
      "logits/rejected": -2.5521483421325684,
      "logps/chosen": -402.92498779296875,
      "logps/rejected": -426.54998779296875,
      "loss": 0.1608,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.18848876655101776,
      "rewards/margins": 3.407031297683716,
      "rewards/rejected": -3.595507860183716,
      "step": 3160
    },
    {
      "epoch": 1.6710595677385345,
      "grad_norm": 70.02343919172361,
      "learning_rate": 5.823668950975223e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.4574217796325684,
      "logps/chosen": -413.9375,
      "logps/rejected": -381.6000061035156,
      "loss": 0.1853,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.8201538324356079,
      "rewards/margins": 3.0475096702575684,
      "rewards/rejected": -3.866015672683716,
      "step": 3170
    },
    {
      "epoch": 1.676331049024776,
      "grad_norm": 29.94767750629102,
      "learning_rate": 5.810490247759621e-07,
      "logits/chosen": -2.4447264671325684,
      "logits/rejected": -2.3935546875,
      "logps/chosen": -370.29998779296875,
      "logps/rejected": -382.1499938964844,
      "loss": 0.1311,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.7084716558456421,
      "rewards/margins": 3.4234375953674316,
      "rewards/rejected": -4.1318359375,
      "step": 3180
    },
    {
      "epoch": 1.6816025303110174,
      "grad_norm": 45.340700605200794,
      "learning_rate": 5.797311544544017e-07,
      "logits/chosen": -2.468554735183716,
      "logits/rejected": -2.413281202316284,
      "logps/chosen": -414.3125,
      "logps/rejected": -406.625,
      "loss": 0.1759,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.39289551973342896,
      "rewards/margins": 3.2974610328674316,
      "rewards/rejected": -3.690624952316284,
      "step": 3190
    },
    {
      "epoch": 1.6868740115972587,
      "grad_norm": 46.29698691802266,
      "learning_rate": 5.784132841328413e-07,
      "logits/chosen": -2.536328077316284,
      "logits/rejected": -2.5054688453674316,
      "logps/chosen": -366.0249938964844,
      "logps/rejected": -382.92498779296875,
      "loss": 0.1685,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.393240362405777,
      "rewards/margins": 3.088085889816284,
      "rewards/rejected": -3.4828124046325684,
      "step": 3200
    },
    {
      "epoch": 1.6921454928835002,
      "grad_norm": 55.59466182290307,
      "learning_rate": 5.770954138112809e-07,
      "logits/chosen": -2.469921827316284,
      "logits/rejected": -2.455078125,
      "logps/chosen": -357.6499938964844,
      "logps/rejected": -362.2250061035156,
      "loss": 0.1727,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.371551513671875,
      "rewards/margins": 2.9710936546325684,
      "rewards/rejected": -3.342578172683716,
      "step": 3210
    },
    {
      "epoch": 1.6974169741697418,
      "grad_norm": 66.03413211470915,
      "learning_rate": 5.757775434897206e-07,
      "logits/chosen": -2.515820264816284,
      "logits/rejected": -2.5298829078674316,
      "logps/chosen": -365.0562438964844,
      "logps/rejected": -345.3374938964844,
      "loss": 0.2089,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.732666015625,
      "rewards/margins": 2.994335889816284,
      "rewards/rejected": -3.7281250953674316,
      "step": 3220
    },
    {
      "epoch": 1.7026884554559831,
      "grad_norm": 45.69404645378638,
      "learning_rate": 5.744596731681603e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.460156202316284,
      "logps/chosen": -422.4750061035156,
      "logps/rejected": -393.6499938964844,
      "loss": 0.1103,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.4084716737270355,
      "rewards/margins": 3.4515624046325684,
      "rewards/rejected": -3.8599610328674316,
      "step": 3230
    },
    {
      "epoch": 1.7079599367422245,
      "grad_norm": 64.03764413554721,
      "learning_rate": 5.731418028465999e-07,
      "logits/chosen": -2.511523485183716,
      "logits/rejected": -2.5210938453674316,
      "logps/chosen": -405.2124938964844,
      "logps/rejected": -378.98748779296875,
      "loss": 0.1677,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.2855896055698395,
      "rewards/margins": 3.292187452316284,
      "rewards/rejected": -3.5748047828674316,
      "step": 3240
    },
    {
      "epoch": 1.713231418028466,
      "grad_norm": 59.4422866236046,
      "learning_rate": 5.718239325250395e-07,
      "logits/chosen": -2.602734327316284,
      "logits/rejected": -2.473437547683716,
      "logps/chosen": -327.2250061035156,
      "logps/rejected": -328.5,
      "loss": 0.1408,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.539349377155304,
      "rewards/margins": 3.251757860183716,
      "rewards/rejected": -3.790234327316284,
      "step": 3250
    },
    {
      "epoch": 1.7185028993147076,
      "grad_norm": 81.61425002942862,
      "learning_rate": 5.705060622034792e-07,
      "logits/chosen": -2.5042967796325684,
      "logits/rejected": -2.5181641578674316,
      "logps/chosen": -354.6000061035156,
      "logps/rejected": -376.17498779296875,
      "loss": 0.1785,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.4279846251010895,
      "rewards/margins": 3.382519483566284,
      "rewards/rejected": -3.8125,
      "step": 3260
    },
    {
      "epoch": 1.723774380600949,
      "grad_norm": 46.6786800389872,
      "learning_rate": 5.691881918819188e-07,
      "logits/chosen": -2.5492186546325684,
      "logits/rejected": -2.5035157203674316,
      "logps/chosen": -391.6499938964844,
      "logps/rejected": -383.6000061035156,
      "loss": 0.2066,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.5324341058731079,
      "rewards/margins": 2.8744139671325684,
      "rewards/rejected": -3.4072265625,
      "step": 3270
    },
    {
      "epoch": 1.7290458618871902,
      "grad_norm": 24.33242480312147,
      "learning_rate": 5.678703215603584e-07,
      "logits/chosen": -2.366503953933716,
      "logits/rejected": -2.4091796875,
      "logps/chosen": -418.9375,
      "logps/rejected": -417.07501220703125,
      "loss": 0.1392,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.3327880799770355,
      "rewards/margins": 3.4564452171325684,
      "rewards/rejected": -3.7884764671325684,
      "step": 3280
    },
    {
      "epoch": 1.7343173431734318,
      "grad_norm": 127.55140791883957,
      "learning_rate": 5.665524512387981e-07,
      "logits/chosen": -2.4173827171325684,
      "logits/rejected": -2.437304735183716,
      "logps/chosen": -378.2875061035156,
      "logps/rejected": -414.57501220703125,
      "loss": 0.1506,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.16776733100414276,
      "rewards/margins": 3.3349609375,
      "rewards/rejected": -3.5013670921325684,
      "step": 3290
    },
    {
      "epoch": 1.7395888244596733,
      "grad_norm": 20.131380668702043,
      "learning_rate": 5.652345809172378e-07,
      "logits/chosen": -2.468554735183716,
      "logits/rejected": -2.5542969703674316,
      "logps/chosen": -505.57501220703125,
      "logps/rejected": -456.82501220703125,
      "loss": 0.1676,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.13450928032398224,
      "rewards/margins": 3.7037110328674316,
      "rewards/rejected": -3.571093797683716,
      "step": 3300
    },
    {
      "epoch": 1.7448603057459144,
      "grad_norm": 44.95233310308116,
      "learning_rate": 5.639167105956774e-07,
      "logits/chosen": -2.533203125,
      "logits/rejected": -2.575390577316284,
      "logps/chosen": -408.625,
      "logps/rejected": -382.875,
      "loss": 0.1608,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.305642694234848,
      "rewards/margins": 2.9976563453674316,
      "rewards/rejected": -3.303906202316284,
      "step": 3310
    },
    {
      "epoch": 1.750131787032156,
      "grad_norm": 53.72662932582384,
      "learning_rate": 5.62598840274117e-07,
      "logits/chosen": -2.417187452316284,
      "logits/rejected": -2.421093702316284,
      "logps/chosen": -375.57501220703125,
      "logps/rejected": -395.6000061035156,
      "loss": 0.1546,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.32490235567092896,
      "rewards/margins": 3.3414063453674316,
      "rewards/rejected": -3.6673827171325684,
      "step": 3320
    },
    {
      "epoch": 1.7554032683183975,
      "grad_norm": 47.96885721283558,
      "learning_rate": 5.612809699525567e-07,
      "logits/chosen": -2.651562452316284,
      "logits/rejected": -2.5250000953674316,
      "logps/chosen": -367.875,
      "logps/rejected": -371.25,
      "loss": 0.2068,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.3871398866176605,
      "rewards/margins": 2.9924073219299316,
      "rewards/rejected": -3.3792967796325684,
      "step": 3330
    },
    {
      "epoch": 1.7606747496046389,
      "grad_norm": 36.36657623138649,
      "learning_rate": 5.599630996309963e-07,
      "logits/chosen": -2.4535155296325684,
      "logits/rejected": -2.485156297683716,
      "logps/chosen": -423.98748779296875,
      "logps/rejected": -399.1499938964844,
      "loss": 0.1484,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.24193724989891052,
      "rewards/margins": 3.58203125,
      "rewards/rejected": -3.8238282203674316,
      "step": 3340
    },
    {
      "epoch": 1.7659462308908802,
      "grad_norm": 34.67794842662678,
      "learning_rate": 5.58645229309436e-07,
      "logits/chosen": -2.4302735328674316,
      "logits/rejected": -2.443164110183716,
      "logps/chosen": -404.125,
      "logps/rejected": -414.2875061035156,
      "loss": 0.2279,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.21809081733226776,
      "rewards/margins": 3.099316358566284,
      "rewards/rejected": -3.31689453125,
      "step": 3350
    },
    {
      "epoch": 1.7712177121771218,
      "grad_norm": 47.061634089374515,
      "learning_rate": 5.573273589878755e-07,
      "logits/chosen": -2.369140625,
      "logits/rejected": -2.4330077171325684,
      "logps/chosen": -446.17498779296875,
      "logps/rejected": -424.20001220703125,
      "loss": 0.1626,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.23947754502296448,
      "rewards/margins": 3.419140577316284,
      "rewards/rejected": -3.658007860183716,
      "step": 3360
    },
    {
      "epoch": 1.7764891934633633,
      "grad_norm": 21.220528898291526,
      "learning_rate": 5.560094886663152e-07,
      "logits/chosen": -2.500195264816284,
      "logits/rejected": -2.4437499046325684,
      "logps/chosen": -372.1875,
      "logps/rejected": -374.42498779296875,
      "loss": 0.151,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.3306823670864105,
      "rewards/margins": 3.359179735183716,
      "rewards/rejected": -3.688769578933716,
      "step": 3370
    },
    {
      "epoch": 1.7817606747496046,
      "grad_norm": 28.66128170502656,
      "learning_rate": 5.546916183447548e-07,
      "logits/chosen": -2.4437499046325684,
      "logits/rejected": -2.4693360328674316,
      "logps/chosen": -380.54998779296875,
      "logps/rejected": -380.0,
      "loss": 0.2037,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.34449464082717896,
      "rewards/margins": 3.106250047683716,
      "rewards/rejected": -3.4515624046325684,
      "step": 3380
    },
    {
      "epoch": 1.787032156035846,
      "grad_norm": 40.291636430635826,
      "learning_rate": 5.533737480231944e-07,
      "logits/chosen": -2.541210889816284,
      "logits/rejected": -2.5361328125,
      "logps/chosen": -343.8500061035156,
      "logps/rejected": -366.04998779296875,
      "loss": 0.1562,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.3771423399448395,
      "rewards/margins": 3.021289110183716,
      "rewards/rejected": -3.396484375,
      "step": 3390
    },
    {
      "epoch": 1.7923036373220875,
      "grad_norm": 46.34594061957881,
      "learning_rate": 5.520558777016341e-07,
      "logits/chosen": -2.468945264816284,
      "logits/rejected": -2.536328077316284,
      "logps/chosen": -432.3999938964844,
      "logps/rejected": -389.32501220703125,
      "loss": 0.1519,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.28559571504592896,
      "rewards/margins": 3.1451172828674316,
      "rewards/rejected": -3.431445360183716,
      "step": 3400
    },
    {
      "epoch": 1.797575118608329,
      "grad_norm": 37.616132254221405,
      "learning_rate": 5.507380073800738e-07,
      "logits/chosen": -2.4634766578674316,
      "logits/rejected": -2.3900389671325684,
      "logps/chosen": -384.0,
      "logps/rejected": -343.7749938964844,
      "loss": 0.1822,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.40772706270217896,
      "rewards/margins": 3.1078124046325684,
      "rewards/rejected": -3.5152344703674316,
      "step": 3410
    },
    {
      "epoch": 1.8028465998945704,
      "grad_norm": 41.66754051674586,
      "learning_rate": 5.494201370585134e-07,
      "logits/chosen": -2.474414110183716,
      "logits/rejected": -2.477734327316284,
      "logps/chosen": -416.57501220703125,
      "logps/rejected": -418.4750061035156,
      "loss": 0.1537,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.4169875979423523,
      "rewards/margins": 3.2906250953674316,
      "rewards/rejected": -3.707226514816284,
      "step": 3420
    },
    {
      "epoch": 1.8081180811808117,
      "grad_norm": 16.11256104856305,
      "learning_rate": 5.48102266736953e-07,
      "logits/chosen": -2.420703172683716,
      "logits/rejected": -2.479687452316284,
      "logps/chosen": -400.0,
      "logps/rejected": -396.625,
      "loss": 0.1633,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.500170886516571,
      "rewards/margins": 3.266406297683716,
      "rewards/rejected": -3.7685546875,
      "step": 3430
    },
    {
      "epoch": 1.8133895624670533,
      "grad_norm": 56.45709997257399,
      "learning_rate": 5.467843964153926e-07,
      "logits/chosen": -2.455859422683716,
      "logits/rejected": -2.4613280296325684,
      "logps/chosen": -356.875,
      "logps/rejected": -379.45001220703125,
      "loss": 0.1794,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.3890380859375,
      "rewards/margins": 3.064257860183716,
      "rewards/rejected": -3.4535155296325684,
      "step": 3440
    },
    {
      "epoch": 1.8186610437532946,
      "grad_norm": 55.76548370269663,
      "learning_rate": 5.454665260938323e-07,
      "logits/chosen": -2.5445313453674316,
      "logits/rejected": -2.474609375,
      "logps/chosen": -423.79998779296875,
      "logps/rejected": -398.0,
      "loss": 0.1729,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.08301391452550888,
      "rewards/margins": 3.2867188453674316,
      "rewards/rejected": -3.37109375,
      "step": 3450
    },
    {
      "epoch": 1.823932525039536,
      "grad_norm": 91.70322371421054,
      "learning_rate": 5.44148655772272e-07,
      "logits/chosen": -2.5699219703674316,
      "logits/rejected": -2.434765577316284,
      "logps/chosen": -360.3125,
      "logps/rejected": -383.32501220703125,
      "loss": 0.1877,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.08069457858800888,
      "rewards/margins": 2.997851610183716,
      "rewards/rejected": -3.079296827316284,
      "step": 3460
    },
    {
      "epoch": 1.8292040063257775,
      "grad_norm": 32.64555501075105,
      "learning_rate": 5.428307854507116e-07,
      "logits/chosen": -2.502734422683716,
      "logits/rejected": -2.5013670921325684,
      "logps/chosen": -377.875,
      "logps/rejected": -324.375,
      "loss": 0.2259,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.22203369438648224,
      "rewards/margins": 2.799023389816284,
      "rewards/rejected": -2.576855421066284,
      "step": 3470
    },
    {
      "epoch": 1.834475487612019,
      "grad_norm": 57.77772354650779,
      "learning_rate": 5.415129151291513e-07,
      "logits/chosen": -2.4716796875,
      "logits/rejected": -2.5083985328674316,
      "logps/chosen": -388.79998779296875,
      "logps/rejected": -378.82501220703125,
      "loss": 0.1688,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.23048095405101776,
      "rewards/margins": 3.4608397483825684,
      "rewards/rejected": -3.2310547828674316,
      "step": 3480
    },
    {
      "epoch": 1.8397469688982604,
      "grad_norm": 330.41042099283607,
      "learning_rate": 5.401950448075909e-07,
      "logits/chosen": -2.505859375,
      "logits/rejected": -2.4419922828674316,
      "logps/chosen": -362.36248779296875,
      "logps/rejected": -381.7749938964844,
      "loss": 0.2231,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.090789794921875,
      "rewards/margins": 2.810546875,
      "rewards/rejected": -2.903125047683716,
      "step": 3490
    },
    {
      "epoch": 1.8450184501845017,
      "grad_norm": 63.06746947496312,
      "learning_rate": 5.388771744860305e-07,
      "logits/chosen": -2.4332032203674316,
      "logits/rejected": -2.403515577316284,
      "logps/chosen": -420.2749938964844,
      "logps/rejected": -401.25,
      "loss": 0.134,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.16762694716453552,
      "rewards/margins": 3.1474609375,
      "rewards/rejected": -2.98046875,
      "step": 3500
    },
    {
      "epoch": 1.8502899314707433,
      "grad_norm": 50.273848875994275,
      "learning_rate": 5.375593041644701e-07,
      "logits/chosen": -2.4203124046325684,
      "logits/rejected": -2.425976514816284,
      "logps/chosen": -379.3374938964844,
      "logps/rejected": -394.82501220703125,
      "loss": 0.1807,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.34158629179000854,
      "rewards/margins": 3.0501952171325684,
      "rewards/rejected": -3.3931641578674316,
      "step": 3510
    },
    {
      "epoch": 1.8555614127569848,
      "grad_norm": 61.57244838783358,
      "learning_rate": 5.362414338429099e-07,
      "logits/chosen": -2.554492235183716,
      "logits/rejected": -2.5435547828674316,
      "logps/chosen": -372.3999938964844,
      "logps/rejected": -374.375,
      "loss": 0.1792,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.4486450254917145,
      "rewards/margins": 2.9644532203674316,
      "rewards/rejected": -3.4124999046325684,
      "step": 3520
    },
    {
      "epoch": 1.8608328940432262,
      "grad_norm": 25.2013050603074,
      "learning_rate": 5.349235635213495e-07,
      "logits/chosen": -2.6253905296325684,
      "logits/rejected": -2.608593702316284,
      "logps/chosen": -367.67498779296875,
      "logps/rejected": -365.67498779296875,
      "loss": 0.1796,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.3004516661167145,
      "rewards/margins": 3.0433592796325684,
      "rewards/rejected": -3.343945264816284,
      "step": 3530
    },
    {
      "epoch": 1.8661043753294675,
      "grad_norm": 51.68538345409033,
      "learning_rate": 5.336056931997891e-07,
      "logits/chosen": -2.5523438453674316,
      "logits/rejected": -2.5785155296325684,
      "logps/chosen": -411.4624938964844,
      "logps/rejected": -391.7749938964844,
      "loss": 0.1502,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.5265136957168579,
      "rewards/margins": 3.1898436546325684,
      "rewards/rejected": -3.719531297683716,
      "step": 3540
    },
    {
      "epoch": 1.871375856615709,
      "grad_norm": 58.1071625622398,
      "learning_rate": 5.322878228782287e-07,
      "logits/chosen": -2.514843702316284,
      "logits/rejected": -2.5902342796325684,
      "logps/chosen": -380.07501220703125,
      "logps/rejected": -369.5,
      "loss": 0.2092,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.3656249940395355,
      "rewards/margins": 2.900390625,
      "rewards/rejected": -3.265918016433716,
      "step": 3550
    },
    {
      "epoch": 1.8766473379019506,
      "grad_norm": 35.13741250759249,
      "learning_rate": 5.309699525566684e-07,
      "logits/chosen": -2.51953125,
      "logits/rejected": -2.5101561546325684,
      "logps/chosen": -417.42498779296875,
      "logps/rejected": -397.6499938964844,
      "loss": 0.1607,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.3770507872104645,
      "rewards/margins": 3.249316453933716,
      "rewards/rejected": -3.6263670921325684,
      "step": 3560
    },
    {
      "epoch": 1.881918819188192,
      "grad_norm": 34.01476709921728,
      "learning_rate": 5.296520822351081e-07,
      "logits/chosen": -2.5521483421325684,
      "logits/rejected": -2.5570311546325684,
      "logps/chosen": -359.82501220703125,
      "logps/rejected": -374.5,
      "loss": 0.1445,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.3307647705078125,
      "rewards/margins": 3.366992235183716,
      "rewards/rejected": -3.697460889816284,
      "step": 3570
    },
    {
      "epoch": 1.8871903004744333,
      "grad_norm": 22.250595790037405,
      "learning_rate": 5.283342119135477e-07,
      "logits/chosen": -2.518359422683716,
      "logits/rejected": -2.448437452316284,
      "logps/chosen": -452.42498779296875,
      "logps/rejected": -401.9750061035156,
      "loss": 0.1661,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.14729614555835724,
      "rewards/margins": 3.119335889816284,
      "rewards/rejected": -3.2679686546325684,
      "step": 3580
    },
    {
      "epoch": 1.8924617817606748,
      "grad_norm": 56.64921712668238,
      "learning_rate": 5.270163415919874e-07,
      "logits/chosen": -2.560351610183716,
      "logits/rejected": -2.5888671875,
      "logps/chosen": -389.17498779296875,
      "logps/rejected": -383.3500061035156,
      "loss": 0.1502,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.2564330995082855,
      "rewards/margins": 3.473437547683716,
      "rewards/rejected": -3.730664014816284,
      "step": 3590
    },
    {
      "epoch": 1.8977332630469161,
      "grad_norm": 60.04633389879101,
      "learning_rate": 5.25698471270427e-07,
      "logits/chosen": -2.5796875953674316,
      "logits/rejected": -2.5726561546325684,
      "logps/chosen": -450.3500061035156,
      "logps/rejected": -435.17498779296875,
      "loss": 0.1603,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.36838990449905396,
      "rewards/margins": 3.4820313453674316,
      "rewards/rejected": -3.8499999046325684,
      "step": 3600
    },
    {
      "epoch": 1.9030047443331575,
      "grad_norm": 28.441677781865327,
      "learning_rate": 5.243806009488666e-07,
      "logits/chosen": -2.570507764816284,
      "logits/rejected": -2.5453124046325684,
      "logps/chosen": -387.42498779296875,
      "logps/rejected": -433.1000061035156,
      "loss": 0.1517,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.32780760526657104,
      "rewards/margins": 3.376171827316284,
      "rewards/rejected": -3.7041015625,
      "step": 3610
    },
    {
      "epoch": 1.908276225619399,
      "grad_norm": 38.76188274431356,
      "learning_rate": 5.230627306273062e-07,
      "logits/chosen": -2.466796875,
      "logits/rejected": -2.512500047683716,
      "logps/chosen": -422.0874938964844,
      "logps/rejected": -379.125,
      "loss": 0.1302,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.2791900634765625,
      "rewards/margins": 3.2748045921325684,
      "rewards/rejected": -3.5531249046325684,
      "step": 3620
    },
    {
      "epoch": 1.9135477069056406,
      "grad_norm": 77.69649326169264,
      "learning_rate": 5.21744860305746e-07,
      "logits/chosen": -2.5025391578674316,
      "logits/rejected": -2.467968702316284,
      "logps/chosen": -425.4375,
      "logps/rejected": -413.29998779296875,
      "loss": 0.1905,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.33085936307907104,
      "rewards/margins": 3.299023389816284,
      "rewards/rejected": -3.6292967796325684,
      "step": 3630
    },
    {
      "epoch": 1.918819188191882,
      "grad_norm": 64.91545224481476,
      "learning_rate": 5.204269899841856e-07,
      "logits/chosen": -2.4984374046325684,
      "logits/rejected": -2.396484375,
      "logps/chosen": -373.9125061035156,
      "logps/rejected": -379.4750061035156,
      "loss": 0.1708,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.11816711723804474,
      "rewards/margins": 3.1318359375,
      "rewards/rejected": -3.252148389816284,
      "step": 3640
    },
    {
      "epoch": 1.9240906694781232,
      "grad_norm": 62.78156330523507,
      "learning_rate": 5.191091196626252e-07,
      "logits/chosen": -2.548632860183716,
      "logits/rejected": -2.491015672683716,
      "logps/chosen": -380.42498779296875,
      "logps/rejected": -394.88751220703125,
      "loss": 0.197,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.10475464165210724,
      "rewards/margins": 3.225830078125,
      "rewards/rejected": -3.328125,
      "step": 3650
    },
    {
      "epoch": 1.9293621507643648,
      "grad_norm": 42.738092397938566,
      "learning_rate": 5.177912493410648e-07,
      "logits/chosen": -2.5658202171325684,
      "logits/rejected": -2.5794920921325684,
      "logps/chosen": -414.8999938964844,
      "logps/rejected": -415.20001220703125,
      "loss": 0.1199,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.009204101748764515,
      "rewards/margins": 3.4554686546325684,
      "rewards/rejected": -3.4662108421325684,
      "step": 3660
    },
    {
      "epoch": 1.9346336320506063,
      "grad_norm": 64.99112896124977,
      "learning_rate": 5.164733790195045e-07,
      "logits/chosen": -2.58984375,
      "logits/rejected": -2.600781202316284,
      "logps/chosen": -368.9750061035156,
      "logps/rejected": -387.36248779296875,
      "loss": 0.1647,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.21293945610523224,
      "rewards/margins": 3.167187452316284,
      "rewards/rejected": -3.37890625,
      "step": 3670
    },
    {
      "epoch": 1.9399051133368477,
      "grad_norm": 52.37032991042282,
      "learning_rate": 5.151555086979441e-07,
      "logits/chosen": -2.5746092796325684,
      "logits/rejected": -2.59375,
      "logps/chosen": -395.54998779296875,
      "logps/rejected": -391.32501220703125,
      "loss": 0.1488,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.25761717557907104,
      "rewards/margins": 3.3016600608825684,
      "rewards/rejected": -3.5582032203674316,
      "step": 3680
    },
    {
      "epoch": 1.945176594623089,
      "grad_norm": 43.71229453363993,
      "learning_rate": 5.138376383763838e-07,
      "logits/chosen": -2.6177735328674316,
      "logits/rejected": -2.6478514671325684,
      "logps/chosen": -380.1875,
      "logps/rejected": -386.0,
      "loss": 0.2012,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.514782726764679,
      "rewards/margins": 2.958984375,
      "rewards/rejected": -3.4740233421325684,
      "step": 3690
    },
    {
      "epoch": 1.9504480759093306,
      "grad_norm": 69.51553420582842,
      "learning_rate": 5.125197680548234e-07,
      "logits/chosen": -2.5126953125,
      "logits/rejected": -2.484179735183716,
      "logps/chosen": -319.13751220703125,
      "logps/rejected": -347.5,
      "loss": 0.2194,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.7349853515625,
      "rewards/margins": 3.1380858421325684,
      "rewards/rejected": -3.8726563453674316,
      "step": 3700
    },
    {
      "epoch": 1.9557195571955721,
      "grad_norm": 35.28093333776043,
      "learning_rate": 5.11201897733263e-07,
      "logits/chosen": -2.5693359375,
      "logits/rejected": -2.54296875,
      "logps/chosen": -407.6499938964844,
      "logps/rejected": -426.79998779296875,
      "loss": 0.1637,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.30112916231155396,
      "rewards/margins": 3.265429735183716,
      "rewards/rejected": -3.5697264671325684,
      "step": 3710
    },
    {
      "epoch": 1.9609910384818134,
      "grad_norm": 40.80858757697696,
      "learning_rate": 5.098840274117026e-07,
      "logits/chosen": -2.4330077171325684,
      "logits/rejected": -2.476757764816284,
      "logps/chosen": -396.20001220703125,
      "logps/rejected": -383.0,
      "loss": 0.1518,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.490814208984375,
      "rewards/margins": 3.2548828125,
      "rewards/rejected": -3.744140625,
      "step": 3720
    },
    {
      "epoch": 1.9662625197680548,
      "grad_norm": 63.21383353754104,
      "learning_rate": 5.085661570901422e-07,
      "logits/chosen": -2.450000047683716,
      "logits/rejected": -2.522656202316284,
      "logps/chosen": -405.63751220703125,
      "logps/rejected": -416.1000061035156,
      "loss": 0.1447,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5095764398574829,
      "rewards/margins": 3.344531297683716,
      "rewards/rejected": -3.852734327316284,
      "step": 3730
    },
    {
      "epoch": 1.9715340010542963,
      "grad_norm": 55.249185921253705,
      "learning_rate": 5.07248286768582e-07,
      "logits/chosen": -2.4267578125,
      "logits/rejected": -2.466015577316284,
      "logps/chosen": -370.3125,
      "logps/rejected": -361.42498779296875,
      "loss": 0.1679,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8729308843612671,
      "rewards/margins": 3.083789110183716,
      "rewards/rejected": -3.9564452171325684,
      "step": 3740
    },
    {
      "epoch": 1.9768054823405377,
      "grad_norm": 23.108036129946978,
      "learning_rate": 5.059304164470216e-07,
      "logits/chosen": -2.4613280296325684,
      "logits/rejected": -2.41796875,
      "logps/chosen": -369.7124938964844,
      "logps/rejected": -374.04998779296875,
      "loss": 0.1453,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.678497314453125,
      "rewards/margins": 3.016796827316284,
      "rewards/rejected": -3.6929688453674316,
      "step": 3750
    },
    {
      "epoch": 1.982076963626779,
      "grad_norm": 33.56362185933789,
      "learning_rate": 5.046125461254612e-07,
      "logits/chosen": -2.4925780296325684,
      "logits/rejected": -2.4935545921325684,
      "logps/chosen": -392.4125061035156,
      "logps/rejected": -368.5249938964844,
      "loss": 0.1446,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.37164306640625,
      "rewards/margins": 3.628710985183716,
      "rewards/rejected": -3.9935545921325684,
      "step": 3760
    },
    {
      "epoch": 1.9873484449130205,
      "grad_norm": 30.837244149619593,
      "learning_rate": 5.032946758039008e-07,
      "logits/chosen": -2.499804735183716,
      "logits/rejected": -2.5234375,
      "logps/chosen": -392.5375061035156,
      "logps/rejected": -386.92498779296875,
      "loss": 0.1531,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.20664367079734802,
      "rewards/margins": 3.115429639816284,
      "rewards/rejected": -3.3236327171325684,
      "step": 3770
    },
    {
      "epoch": 1.992619926199262,
      "grad_norm": 45.53730549922731,
      "learning_rate": 5.019768054823405e-07,
      "logits/chosen": -2.5269532203674316,
      "logits/rejected": -2.453320264816284,
      "logps/chosen": -418.6000061035156,
      "logps/rejected": -384.5,
      "loss": 0.1389,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.18779906630516052,
      "rewards/margins": 3.3794922828674316,
      "rewards/rejected": -3.5667967796325684,
      "step": 3780
    },
    {
      "epoch": 1.9978914074855034,
      "grad_norm": 62.09546891446375,
      "learning_rate": 5.006589351607801e-07,
      "logits/chosen": -2.4251952171325684,
      "logits/rejected": -2.5091795921325684,
      "logps/chosen": -460.20001220703125,
      "logps/rejected": -419.5,
      "loss": 0.1423,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.12371826171875,
      "rewards/margins": 3.348828077316284,
      "rewards/rejected": -3.471874952316284,
      "step": 3790
    },
    {
      "epoch": 2.0031628887717448,
      "grad_norm": 18.0801935195064,
      "learning_rate": 4.993410648392198e-07,
      "logits/chosen": -2.591796875,
      "logits/rejected": -2.5873045921325684,
      "logps/chosen": -392.79998779296875,
      "logps/rejected": -394.3999938964844,
      "loss": 0.0888,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.00689697265625,
      "rewards/margins": 3.629101514816284,
      "rewards/rejected": -3.6376953125,
      "step": 3800
    },
    {
      "epoch": 2.0084343700579863,
      "grad_norm": 11.768842432124458,
      "learning_rate": 4.980231945176594e-07,
      "logits/chosen": -2.5277342796325684,
      "logits/rejected": -2.6029295921325684,
      "logps/chosen": -380.23126220703125,
      "logps/rejected": -371.2250061035156,
      "loss": 0.0547,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.26139527559280396,
      "rewards/margins": 4.291796684265137,
      "rewards/rejected": -4.032031059265137,
      "step": 3810
    },
    {
      "epoch": 2.013705851344228,
      "grad_norm": 18.05304061933159,
      "learning_rate": 4.96705324196099e-07,
      "logits/chosen": -2.614062547683716,
      "logits/rejected": -2.5904297828674316,
      "logps/chosen": -399.04998779296875,
      "logps/rejected": -365.8999938964844,
      "loss": 0.0566,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10681762546300888,
      "rewards/margins": 4.170702934265137,
      "rewards/rejected": -4.276562690734863,
      "step": 3820
    },
    {
      "epoch": 2.018977332630469,
      "grad_norm": 10.760028104494145,
      "learning_rate": 4.953874538745387e-07,
      "logits/chosen": -2.4664063453674316,
      "logits/rejected": -2.6205077171325684,
      "logps/chosen": -450.7749938964844,
      "logps/rejected": -438.95001220703125,
      "loss": 0.0339,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.42022705078125,
      "rewards/margins": 4.806640625,
      "rewards/rejected": -5.226953029632568,
      "step": 3830
    },
    {
      "epoch": 2.0242488139167105,
      "grad_norm": 16.564479609975134,
      "learning_rate": 4.940695835529783e-07,
      "logits/chosen": -2.588085889816284,
      "logits/rejected": -2.5601563453674316,
      "logps/chosen": -439.4624938964844,
      "logps/rejected": -424.17498779296875,
      "loss": 0.0719,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.765673816204071,
      "rewards/margins": 4.518359184265137,
      "rewards/rejected": -5.283984184265137,
      "step": 3840
    },
    {
      "epoch": 2.029520295202952,
      "grad_norm": 6.341523601238165,
      "learning_rate": 4.92751713231418e-07,
      "logits/chosen": -2.4833984375,
      "logits/rejected": -2.5517578125,
      "logps/chosen": -413.3500061035156,
      "logps/rejected": -438.3500061035156,
      "loss": 0.0397,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4145141541957855,
      "rewards/margins": 4.860547065734863,
      "rewards/rejected": -5.27734375,
      "step": 3850
    },
    {
      "epoch": 2.0347917764891936,
      "grad_norm": 41.22826871128923,
      "learning_rate": 4.914338429098576e-07,
      "logits/chosen": -2.5615234375,
      "logits/rejected": -2.742968797683716,
      "logps/chosen": -448.7749938964844,
      "logps/rejected": -402.3999938964844,
      "loss": 0.0633,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.4004272520542145,
      "rewards/margins": 4.593359470367432,
      "rewards/rejected": -4.994531154632568,
      "step": 3860
    },
    {
      "epoch": 2.0400632577754347,
      "grad_norm": 21.079708525237887,
      "learning_rate": 4.901159725882973e-07,
      "logits/chosen": -2.7181639671325684,
      "logits/rejected": -2.6988282203674316,
      "logps/chosen": -389.61248779296875,
      "logps/rejected": -407.57501220703125,
      "loss": 0.0447,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.4341979920864105,
      "rewards/margins": 4.717187404632568,
      "rewards/rejected": -5.150000095367432,
      "step": 3870
    },
    {
      "epoch": 2.0453347390616763,
      "grad_norm": 6.635219112878441,
      "learning_rate": 4.88798102266737e-07,
      "logits/chosen": -2.7183594703674316,
      "logits/rejected": -2.7330079078674316,
      "logps/chosen": -412.79998779296875,
      "logps/rejected": -408.1499938964844,
      "loss": 0.0427,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.600415050983429,
      "rewards/margins": 5.020703315734863,
      "rewards/rejected": -5.617578029632568,
      "step": 3880
    },
    {
      "epoch": 2.050606220347918,
      "grad_norm": 46.06118289099573,
      "learning_rate": 4.874802319451766e-07,
      "logits/chosen": -2.769726514816284,
      "logits/rejected": -2.771484375,
      "logps/chosen": -382.4750061035156,
      "logps/rejected": -379.57501220703125,
      "loss": 0.0913,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.549700915813446,
      "rewards/margins": 4.485937595367432,
      "rewards/rejected": -5.035546779632568,
      "step": 3890
    },
    {
      "epoch": 2.0558777016341594,
      "grad_norm": 9.370722448285374,
      "learning_rate": 4.861623616236162e-07,
      "logits/chosen": -2.6253905296325684,
      "logits/rejected": -2.6298828125,
      "logps/chosen": -379.01251220703125,
      "logps/rejected": -431.8374938964844,
      "loss": 0.0546,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.39984130859375,
      "rewards/margins": 5.026562690734863,
      "rewards/rejected": -5.425390720367432,
      "step": 3900
    },
    {
      "epoch": 2.0611491829204005,
      "grad_norm": 15.967052867424943,
      "learning_rate": 4.848444913020559e-07,
      "logits/chosen": -2.7757811546325684,
      "logits/rejected": -2.716796875,
      "logps/chosen": -385.29998779296875,
      "logps/rejected": -409.29998779296875,
      "loss": 0.0599,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.8996337652206421,
      "rewards/margins": 5.081250190734863,
      "rewards/rejected": -5.980859279632568,
      "step": 3910
    },
    {
      "epoch": 2.066420664206642,
      "grad_norm": 22.71493218601555,
      "learning_rate": 4.835266209804955e-07,
      "logits/chosen": -2.704296827316284,
      "logits/rejected": -2.807812452316284,
      "logps/chosen": -408.79998779296875,
      "logps/rejected": -424.82501220703125,
      "loss": 0.0438,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.1700439453125,
      "rewards/margins": 5.031640529632568,
      "rewards/rejected": -6.203125,
      "step": 3920
    },
    {
      "epoch": 2.0716921454928836,
      "grad_norm": 11.091633487496273,
      "learning_rate": 4.822087506589351e-07,
      "logits/chosen": -2.697265625,
      "logits/rejected": -2.752734422683716,
      "logps/chosen": -391.8999938964844,
      "logps/rejected": -447.9750061035156,
      "loss": 0.0416,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.126397728919983,
      "rewards/margins": 5.319140434265137,
      "rewards/rejected": -6.448828220367432,
      "step": 3930
    },
    {
      "epoch": 2.0769636267791247,
      "grad_norm": 20.559642272629922,
      "learning_rate": 4.808908803373748e-07,
      "logits/chosen": -2.703906297683716,
      "logits/rejected": -2.682421922683716,
      "logps/chosen": -374.63751220703125,
      "logps/rejected": -392.92498779296875,
      "loss": 0.0554,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.013818383216858,
      "rewards/margins": 4.584765434265137,
      "rewards/rejected": -5.59765625,
      "step": 3940
    },
    {
      "epoch": 2.0822351080653663,
      "grad_norm": 3.6186235662444846,
      "learning_rate": 4.795730100158144e-07,
      "logits/chosen": -2.6763672828674316,
      "logits/rejected": -2.7992186546325684,
      "logps/chosen": -414.82501220703125,
      "logps/rejected": -379.29998779296875,
      "loss": 0.0474,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.989489734172821,
      "rewards/margins": 4.73046875,
      "rewards/rejected": -5.720312595367432,
      "step": 3950
    },
    {
      "epoch": 2.087506589351608,
      "grad_norm": 15.937790980604726,
      "learning_rate": 4.782551396942541e-07,
      "logits/chosen": -2.693554639816284,
      "logits/rejected": -2.748242139816284,
      "logps/chosen": -440.25,
      "logps/rejected": -424.8500061035156,
      "loss": 0.0549,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.071008324623108,
      "rewards/margins": 4.860156059265137,
      "rewards/rejected": -5.9296875,
      "step": 3960
    },
    {
      "epoch": 2.0927780706378494,
      "grad_norm": 32.4171721810916,
      "learning_rate": 4.769372693726937e-07,
      "logits/chosen": -2.696093797683716,
      "logits/rejected": -2.7632813453674316,
      "logps/chosen": -400.75,
      "logps/rejected": -396.45001220703125,
      "loss": 0.0358,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9964965581893921,
      "rewards/margins": 5.046484470367432,
      "rewards/rejected": -6.038281440734863,
      "step": 3970
    },
    {
      "epoch": 2.0980495519240905,
      "grad_norm": 6.636740872339192,
      "learning_rate": 4.756193990511334e-07,
      "logits/chosen": -2.4964842796325684,
      "logits/rejected": -2.6195311546325684,
      "logps/chosen": -385.42498779296875,
      "logps/rejected": -444.6000061035156,
      "loss": 0.0352,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.966552734375,
      "rewards/margins": 5.078515529632568,
      "rewards/rejected": -6.044531345367432,
      "step": 3980
    },
    {
      "epoch": 2.103321033210332,
      "grad_norm": 16.018703605436514,
      "learning_rate": 4.7430152872957297e-07,
      "logits/chosen": -2.6996092796325684,
      "logits/rejected": -2.785937547683716,
      "logps/chosen": -414.85626220703125,
      "logps/rejected": -398.5625,
      "loss": 0.0508,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.0158812999725342,
      "rewards/margins": 4.864843845367432,
      "rewards/rejected": -5.880468845367432,
      "step": 3990
    },
    {
      "epoch": 2.1085925144965736,
      "grad_norm": 5.468970459783248,
      "learning_rate": 4.729836584080126e-07,
      "logits/chosen": -2.656054735183716,
      "logits/rejected": -2.6830077171325684,
      "logps/chosen": -389.70001220703125,
      "logps/rejected": -380.32501220703125,
      "loss": 0.0449,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.949694812297821,
      "rewards/margins": 4.957812309265137,
      "rewards/rejected": -5.907422065734863,
      "step": 4000
    },
    {
      "epoch": 2.113863995782815,
      "grad_norm": 6.480403967309898,
      "learning_rate": 4.716657880864523e-07,
      "logits/chosen": -2.6947264671325684,
      "logits/rejected": -2.770703077316284,
      "logps/chosen": -434.2749938964844,
      "logps/rejected": -411.57501220703125,
      "loss": 0.027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.567840576171875,
      "rewards/margins": 5.138281345367432,
      "rewards/rejected": -5.706640720367432,
      "step": 4010
    },
    {
      "epoch": 2.1191354770690563,
      "grad_norm": 21.968934677159613,
      "learning_rate": 4.703479177648919e-07,
      "logits/chosen": -2.781445264816284,
      "logits/rejected": -2.668164014816284,
      "logps/chosen": -409.70001220703125,
      "logps/rejected": -418.1000061035156,
      "loss": 0.0454,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.070556640625,
      "rewards/margins": 4.873827934265137,
      "rewards/rejected": -5.947265625,
      "step": 4020
    },
    {
      "epoch": 2.124406958355298,
      "grad_norm": 10.962075596063821,
      "learning_rate": 4.6903004744333156e-07,
      "logits/chosen": -2.798632860183716,
      "logits/rejected": -2.781054735183716,
      "logps/chosen": -407.6875,
      "logps/rejected": -433.25,
      "loss": 0.0431,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.985107421875,
      "rewards/margins": 4.897656440734863,
      "rewards/rejected": -5.884375095367432,
      "step": 4030
    },
    {
      "epoch": 2.1296784396415394,
      "grad_norm": 28.598346305648707,
      "learning_rate": 4.677121771217712e-07,
      "logits/chosen": -2.7294921875,
      "logits/rejected": -2.774218797683716,
      "logps/chosen": -386.57501220703125,
      "logps/rejected": -399.125,
      "loss": 0.0521,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.8633056879043579,
      "rewards/margins": 4.842187404632568,
      "rewards/rejected": -5.701562404632568,
      "step": 4040
    },
    {
      "epoch": 2.134949920927781,
      "grad_norm": 17.48391845186459,
      "learning_rate": 4.6639430680021086e-07,
      "logits/chosen": -2.7630858421325684,
      "logits/rejected": -2.7984375953674316,
      "logps/chosen": -332.1000061035156,
      "logps/rejected": -362.5249938964844,
      "loss": 0.0571,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.5360625982284546,
      "rewards/margins": 4.63671875,
      "rewards/rejected": -5.170312404632568,
      "step": 4050
    },
    {
      "epoch": 2.140221402214022,
      "grad_norm": 11.94327189229535,
      "learning_rate": 4.6507643647865045e-07,
      "logits/chosen": -2.7880859375,
      "logits/rejected": -2.846874952316284,
      "logps/chosen": -395.125,
      "logps/rejected": -439.375,
      "loss": 0.038,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.4588867127895355,
      "rewards/margins": 5.001172065734863,
      "rewards/rejected": -5.4609375,
      "step": 4060
    },
    {
      "epoch": 2.1454928835002636,
      "grad_norm": 4.7564728919211925,
      "learning_rate": 4.6375856615709015e-07,
      "logits/chosen": -2.7847657203674316,
      "logits/rejected": -2.7652344703674316,
      "logps/chosen": -377.23748779296875,
      "logps/rejected": -410.95001220703125,
      "loss": 0.0439,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.8698486089706421,
      "rewards/margins": 5.228515625,
      "rewards/rejected": -6.097265720367432,
      "step": 4070
    },
    {
      "epoch": 2.150764364786505,
      "grad_norm": 46.34938238724925,
      "learning_rate": 4.6244069583552975e-07,
      "logits/chosen": -2.6812500953674316,
      "logits/rejected": -2.752148389816284,
      "logps/chosen": -382.4624938964844,
      "logps/rejected": -401.6000061035156,
      "loss": 0.0598,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.7953246831893921,
      "rewards/margins": 5.222265720367432,
      "rewards/rejected": -6.021874904632568,
      "step": 4080
    },
    {
      "epoch": 2.1560358460727462,
      "grad_norm": 5.167882835657518,
      "learning_rate": 4.611228255139694e-07,
      "logits/chosen": -2.866015672683716,
      "logits/rejected": -2.844531297683716,
      "logps/chosen": -427.45001220703125,
      "logps/rejected": -421.2749938964844,
      "loss": 0.053,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.188012719154358,
      "rewards/margins": 5.258593559265137,
      "rewards/rejected": -6.442187309265137,
      "step": 4090
    },
    {
      "epoch": 2.161307327358988,
      "grad_norm": 31.76899137757702,
      "learning_rate": 4.5980495519240904e-07,
      "logits/chosen": -2.726757764816284,
      "logits/rejected": -2.7134766578674316,
      "logps/chosen": -478.95001220703125,
      "logps/rejected": -434.4750061035156,
      "loss": 0.0367,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2642822265625,
      "rewards/margins": 4.87109375,
      "rewards/rejected": -6.139843940734863,
      "step": 4100
    },
    {
      "epoch": 2.1665788086452293,
      "grad_norm": 52.60483847608397,
      "learning_rate": 4.584870848708487e-07,
      "logits/chosen": -2.758984327316284,
      "logits/rejected": -2.7171874046325684,
      "logps/chosen": -444.9375,
      "logps/rejected": -437.2250061035156,
      "loss": 0.0389,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.3018310070037842,
      "rewards/margins": 4.959374904632568,
      "rewards/rejected": -6.260937690734863,
      "step": 4110
    },
    {
      "epoch": 2.171850289931471,
      "grad_norm": 23.650838382524523,
      "learning_rate": 4.571692145492883e-07,
      "logits/chosen": -2.666796922683716,
      "logits/rejected": -2.7613282203674316,
      "logps/chosen": -468.54998779296875,
      "logps/rejected": -440.3999938964844,
      "loss": 0.0376,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.9091796875,
      "rewards/margins": 5.049609184265137,
      "rewards/rejected": -5.961718559265137,
      "step": 4120
    },
    {
      "epoch": 2.177121771217712,
      "grad_norm": 15.279682803230667,
      "learning_rate": 4.55851344227728e-07,
      "logits/chosen": -2.7835936546325684,
      "logits/rejected": -2.707226514816284,
      "logps/chosen": -346.45001220703125,
      "logps/rejected": -395.6000061035156,
      "loss": 0.0421,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.917041003704071,
      "rewards/margins": 4.885156154632568,
      "rewards/rejected": -5.803515434265137,
      "step": 4130
    },
    {
      "epoch": 2.1823932525039536,
      "grad_norm": 7.704017071403733,
      "learning_rate": 4.545334739061676e-07,
      "logits/chosen": -2.7001953125,
      "logits/rejected": -2.932812452316284,
      "logps/chosen": -432.29998779296875,
      "logps/rejected": -418.375,
      "loss": 0.0464,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.226318359375,
      "rewards/margins": 5.067578315734863,
      "rewards/rejected": -6.291406154632568,
      "step": 4140
    },
    {
      "epoch": 2.187664733790195,
      "grad_norm": 14.101597986331114,
      "learning_rate": 4.532156035846073e-07,
      "logits/chosen": -2.831249952316284,
      "logits/rejected": -2.887500047683716,
      "logps/chosen": -432.79998779296875,
      "logps/rejected": -411.17498779296875,
      "loss": 0.0427,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.2539551258087158,
      "rewards/margins": 4.918749809265137,
      "rewards/rejected": -6.17578125,
      "step": 4150
    },
    {
      "epoch": 2.1929362150764367,
      "grad_norm": 10.33033379072858,
      "learning_rate": 4.5189773326304693e-07,
      "logits/chosen": -2.888671875,
      "logits/rejected": -2.846874952316284,
      "logps/chosen": -381.04998779296875,
      "logps/rejected": -412.04998779296875,
      "loss": 0.0411,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4890625476837158,
      "rewards/margins": 5.022265434265137,
      "rewards/rejected": -6.514062404632568,
      "step": 4160
    },
    {
      "epoch": 2.1982076963626778,
      "grad_norm": 9.069811411816035,
      "learning_rate": 4.505798629414865e-07,
      "logits/chosen": -2.736132860183716,
      "logits/rejected": -2.7611327171325684,
      "logps/chosen": -428.45001220703125,
      "logps/rejected": -428.375,
      "loss": 0.0484,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.2312500476837158,
      "rewards/margins": 5.383984565734863,
      "rewards/rejected": -6.619140625,
      "step": 4170
    },
    {
      "epoch": 2.2034791776489193,
      "grad_norm": 28.867239001599447,
      "learning_rate": 4.492619926199262e-07,
      "logits/chosen": -2.809375047683716,
      "logits/rejected": -2.818164110183716,
      "logps/chosen": -374.54998779296875,
      "logps/rejected": -417.0,
      "loss": 0.0718,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.6823546886444092,
      "rewards/margins": 4.877343654632568,
      "rewards/rejected": -6.557812690734863,
      "step": 4180
    },
    {
      "epoch": 2.208750658935161,
      "grad_norm": 32.101518125985606,
      "learning_rate": 4.479441222983658e-07,
      "logits/chosen": -2.807421922683716,
      "logits/rejected": -2.7779297828674316,
      "logps/chosen": -391.375,
      "logps/rejected": -445.17498779296875,
      "loss": 0.0476,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.521081566810608,
      "rewards/margins": 4.986718654632568,
      "rewards/rejected": -6.499218940734863,
      "step": 4190
    },
    {
      "epoch": 2.2140221402214024,
      "grad_norm": 39.992480189160894,
      "learning_rate": 4.4662625197680546e-07,
      "logits/chosen": -2.6966795921325684,
      "logits/rejected": -2.7562499046325684,
      "logps/chosen": -384.4750061035156,
      "logps/rejected": -399.29998779296875,
      "loss": 0.0431,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.0579650402069092,
      "rewards/margins": 4.854296684265137,
      "rewards/rejected": -5.91015625,
      "step": 4200
    },
    {
      "epoch": 2.2192936215076435,
      "grad_norm": 59.29659961461139,
      "learning_rate": 4.453083816552451e-07,
      "logits/chosen": -2.708789110183716,
      "logits/rejected": -2.731640577316284,
      "logps/chosen": -416.0249938964844,
      "logps/rejected": -477.75,
      "loss": 0.0495,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.2369873523712158,
      "rewards/margins": 5.204297065734863,
      "rewards/rejected": -6.442968845367432,
      "step": 4210
    },
    {
      "epoch": 2.224565102793885,
      "grad_norm": 57.45587726075804,
      "learning_rate": 4.4399051133368476e-07,
      "logits/chosen": -2.7056641578674316,
      "logits/rejected": -2.721874952316284,
      "logps/chosen": -386.8999938964844,
      "logps/rejected": -425.4750061035156,
      "loss": 0.0416,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.163732886314392,
      "rewards/margins": 5.160937309265137,
      "rewards/rejected": -6.324999809265137,
      "step": 4220
    },
    {
      "epoch": 2.2298365840801266,
      "grad_norm": 4.70137900230389,
      "learning_rate": 4.4267264101212435e-07,
      "logits/chosen": -2.731640577316284,
      "logits/rejected": -2.751171827316284,
      "logps/chosen": -404.0249938964844,
      "logps/rejected": -395.0249938964844,
      "loss": 0.0199,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7367187738418579,
      "rewards/margins": 5.823046684265137,
      "rewards/rejected": -6.558203220367432,
      "step": 4230
    },
    {
      "epoch": 2.2351080653663677,
      "grad_norm": 42.53552343748656,
      "learning_rate": 4.4135477069056405e-07,
      "logits/chosen": -2.7671875953674316,
      "logits/rejected": -2.822460889816284,
      "logps/chosen": -431.8500061035156,
      "logps/rejected": -445.73748779296875,
      "loss": 0.0468,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.331701636314392,
      "rewards/margins": 5.224218845367432,
      "rewards/rejected": -6.559765815734863,
      "step": 4240
    },
    {
      "epoch": 2.2403795466526093,
      "grad_norm": 18.344446076055924,
      "learning_rate": 4.4003690036900365e-07,
      "logits/chosen": -2.75390625,
      "logits/rejected": -2.805468797683716,
      "logps/chosen": -411.6000061035156,
      "logps/rejected": -406.32501220703125,
      "loss": 0.0545,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.48248291015625,
      "rewards/margins": 5.043359279632568,
      "rewards/rejected": -6.524218559265137,
      "step": 4250
    },
    {
      "epoch": 2.245651027938851,
      "grad_norm": 17.911672942777873,
      "learning_rate": 4.387190300474433e-07,
      "logits/chosen": -2.714062452316284,
      "logits/rejected": -2.740234375,
      "logps/chosen": -405.7124938964844,
      "logps/rejected": -405.375,
      "loss": 0.051,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.269506812095642,
      "rewards/margins": 5.224609375,
      "rewards/rejected": -6.491406440734863,
      "step": 4260
    },
    {
      "epoch": 2.2509225092250924,
      "grad_norm": 69.32447744428822,
      "learning_rate": 4.3740115972588294e-07,
      "logits/chosen": -2.776562452316284,
      "logits/rejected": -2.783007860183716,
      "logps/chosen": -396.25,
      "logps/rejected": -392.2250061035156,
      "loss": 0.0498,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.5925781726837158,
      "rewards/margins": 4.775000095367432,
      "rewards/rejected": -6.366406440734863,
      "step": 4270
    },
    {
      "epoch": 2.2561939905113335,
      "grad_norm": 12.273672137432174,
      "learning_rate": 4.360832894043226e-07,
      "logits/chosen": -2.821093797683716,
      "logits/rejected": -2.7681641578674316,
      "logps/chosen": -442.7250061035156,
      "logps/rejected": -479.2749938964844,
      "loss": 0.0445,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.6326904296875,
      "rewards/margins": 5.387890815734863,
      "rewards/rejected": -7.024218559265137,
      "step": 4280
    },
    {
      "epoch": 2.261465471797575,
      "grad_norm": 21.842390609836315,
      "learning_rate": 4.3476541908276224e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.6832032203674316,
      "logps/chosen": -445.92498779296875,
      "logps/rejected": -434.375,
      "loss": 0.0269,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6288940906524658,
      "rewards/margins": 5.430859565734863,
      "rewards/rejected": -7.057812690734863,
      "step": 4290
    },
    {
      "epoch": 2.2667369530838166,
      "grad_norm": 10.824031435442759,
      "learning_rate": 4.334475487612019e-07,
      "logits/chosen": -2.756640672683716,
      "logits/rejected": -2.813281297683716,
      "logps/chosen": -390.625,
      "logps/rejected": -413.1499938964844,
      "loss": 0.0394,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.6769897937774658,
      "rewards/margins": 5.272265434265137,
      "rewards/rejected": -6.950781345367432,
      "step": 4300
    },
    {
      "epoch": 2.272008434370058,
      "grad_norm": 44.9181435738699,
      "learning_rate": 4.3212967843964153e-07,
      "logits/chosen": -2.6351561546325684,
      "logits/rejected": -2.705273389816284,
      "logps/chosen": -401.5,
      "logps/rejected": -424.75,
      "loss": 0.0663,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.4843871593475342,
      "rewards/margins": 5.413671970367432,
      "rewards/rejected": -6.900000095367432,
      "step": 4310
    },
    {
      "epoch": 2.2772799156562993,
      "grad_norm": 5.924611433635076,
      "learning_rate": 4.3081180811808113e-07,
      "logits/chosen": -2.742968797683716,
      "logits/rejected": -2.759765625,
      "logps/chosen": -424.57501220703125,
      "logps/rejected": -423.20001220703125,
      "loss": 0.0398,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.4293944835662842,
      "rewards/margins": 5.354687690734863,
      "rewards/rejected": -6.787890434265137,
      "step": 4320
    },
    {
      "epoch": 2.282551396942541,
      "grad_norm": 17.923032268257668,
      "learning_rate": 4.2949393779652083e-07,
      "logits/chosen": -2.8226561546325684,
      "logits/rejected": -2.8511719703674316,
      "logps/chosen": -399.0375061035156,
      "logps/rejected": -402.42498779296875,
      "loss": 0.0358,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.873559594154358,
      "rewards/margins": 5.094922065734863,
      "rewards/rejected": -6.969531059265137,
      "step": 4330
    },
    {
      "epoch": 2.2878228782287824,
      "grad_norm": 10.778945673188154,
      "learning_rate": 4.281760674749604e-07,
      "logits/chosen": -2.7330079078674316,
      "logits/rejected": -2.804882764816284,
      "logps/chosen": -401.45001220703125,
      "logps/rejected": -417.54998779296875,
      "loss": 0.0357,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.967382788658142,
      "rewards/margins": 5.075390815734863,
      "rewards/rejected": -7.041406154632568,
      "step": 4340
    },
    {
      "epoch": 2.293094359515024,
      "grad_norm": 25.373541516129084,
      "learning_rate": 4.268581971534001e-07,
      "logits/chosen": -2.873828172683716,
      "logits/rejected": -2.844531297683716,
      "logps/chosen": -387.3500061035156,
      "logps/rejected": -402.6000061035156,
      "loss": 0.0367,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.702111840248108,
      "rewards/margins": 5.386328220367432,
      "rewards/rejected": -7.088281154632568,
      "step": 4350
    },
    {
      "epoch": 2.298365840801265,
      "grad_norm": 17.501961205882832,
      "learning_rate": 4.255403268318397e-07,
      "logits/chosen": -2.7919921875,
      "logits/rejected": -2.851367235183716,
      "logps/chosen": -413.57501220703125,
      "logps/rejected": -392.3999938964844,
      "loss": 0.0471,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.07080078125,
      "rewards/margins": 5.142187595367432,
      "rewards/rejected": -7.216406345367432,
      "step": 4360
    },
    {
      "epoch": 2.3036373220875066,
      "grad_norm": 36.35057656414172,
      "learning_rate": 4.2422245651027937e-07,
      "logits/chosen": -2.866406202316284,
      "logits/rejected": -2.916015625,
      "logps/chosen": -406.5249938964844,
      "logps/rejected": -438.32501220703125,
      "loss": 0.0369,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.81982421875,
      "rewards/margins": 5.515234470367432,
      "rewards/rejected": -7.33984375,
      "step": 4370
    },
    {
      "epoch": 2.308908803373748,
      "grad_norm": 22.298120583049673,
      "learning_rate": 4.22904586188719e-07,
      "logits/chosen": -2.7457032203674316,
      "logits/rejected": -2.798828125,
      "logps/chosen": -399.75,
      "logps/rejected": -456.29998779296875,
      "loss": 0.0606,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.2889647483825684,
      "rewards/margins": 5.435546875,
      "rewards/rejected": -7.72265625,
      "step": 4380
    },
    {
      "epoch": 2.3141802846599893,
      "grad_norm": 9.558283916627024,
      "learning_rate": 4.2158671586715866e-07,
      "logits/chosen": -2.8460936546325684,
      "logits/rejected": -2.8636717796325684,
      "logps/chosen": -452.0,
      "logps/rejected": -487.5,
      "loss": 0.0288,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.224365234375,
      "rewards/margins": 5.46484375,
      "rewards/rejected": -7.689062595367432,
      "step": 4390
    },
    {
      "epoch": 2.319451765946231,
      "grad_norm": 19.598822949850184,
      "learning_rate": 4.2026884554559826e-07,
      "logits/chosen": -2.794921875,
      "logits/rejected": -2.8226561546325684,
      "logps/chosen": -396.7875061035156,
      "logps/rejected": -413.82501220703125,
      "loss": 0.0535,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.340246558189392,
      "rewards/margins": 5.103906154632568,
      "rewards/rejected": -6.443359375,
      "step": 4400
    },
    {
      "epoch": 2.3247232472324724,
      "grad_norm": 17.6518809206023,
      "learning_rate": 4.1895097522403796e-07,
      "logits/chosen": -2.740234375,
      "logits/rejected": -2.721874952316284,
      "logps/chosen": -416.67498779296875,
      "logps/rejected": -420.2250061035156,
      "loss": 0.0472,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.211859107017517,
      "rewards/margins": 5.086328029632568,
      "rewards/rejected": -6.298437595367432,
      "step": 4410
    },
    {
      "epoch": 2.329994728518714,
      "grad_norm": 20.330571764273607,
      "learning_rate": 4.1763310490247755e-07,
      "logits/chosen": -2.908203125,
      "logits/rejected": -2.8666014671325684,
      "logps/chosen": -403.9750061035156,
      "logps/rejected": -426.125,
      "loss": 0.0487,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.2411620616912842,
      "rewards/margins": 5.064062595367432,
      "rewards/rejected": -6.306250095367432,
      "step": 4420
    },
    {
      "epoch": 2.335266209804955,
      "grad_norm": 179.5247250338662,
      "learning_rate": 4.163152345809172e-07,
      "logits/chosen": -2.7300782203674316,
      "logits/rejected": -2.792187452316284,
      "logps/chosen": -395.04998779296875,
      "logps/rejected": -404.32501220703125,
      "loss": 0.0677,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.3921020030975342,
      "rewards/margins": 4.857031345367432,
      "rewards/rejected": -6.253125190734863,
      "step": 4430
    },
    {
      "epoch": 2.3405376910911966,
      "grad_norm": 8.874433737509996,
      "learning_rate": 4.149973642593569e-07,
      "logits/chosen": -2.8755860328674316,
      "logits/rejected": -2.9097657203674316,
      "logps/chosen": -369.29998779296875,
      "logps/rejected": -385.8999938964844,
      "loss": 0.0365,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.699346899986267,
      "rewards/margins": 5.094140529632568,
      "rewards/rejected": -6.793554782867432,
      "step": 4440
    },
    {
      "epoch": 2.345809172377438,
      "grad_norm": 17.075394437127578,
      "learning_rate": 4.136794939377965e-07,
      "logits/chosen": -2.8851561546325684,
      "logits/rejected": -2.8363280296325684,
      "logps/chosen": -409.82501220703125,
      "logps/rejected": -445.20001220703125,
      "loss": 0.0376,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3125,
      "rewards/margins": 5.153906345367432,
      "rewards/rejected": -6.467577934265137,
      "step": 4450
    },
    {
      "epoch": 2.3510806536636797,
      "grad_norm": 26.917169330971586,
      "learning_rate": 4.1236162361623614e-07,
      "logits/chosen": -2.781445264816284,
      "logits/rejected": -2.85546875,
      "logps/chosen": -423.6000061035156,
      "logps/rejected": -436.8500061035156,
      "loss": 0.0611,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.651879906654358,
      "rewards/margins": 5.32421875,
      "rewards/rejected": -6.9765625,
      "step": 4460
    },
    {
      "epoch": 2.356352134949921,
      "grad_norm": 11.822548398967221,
      "learning_rate": 4.110437532946758e-07,
      "logits/chosen": -2.687695264816284,
      "logits/rejected": -2.796875,
      "logps/chosen": -418.125,
      "logps/rejected": -401.2250061035156,
      "loss": 0.0428,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.9477050304412842,
      "rewards/margins": 5.217187404632568,
      "rewards/rejected": -7.166406154632568,
      "step": 4470
    },
    {
      "epoch": 2.3616236162361623,
      "grad_norm": 36.76247587627958,
      "learning_rate": 4.0972588297311544e-07,
      "logits/chosen": -2.5933594703674316,
      "logits/rejected": -2.6302733421325684,
      "logps/chosen": -402.07501220703125,
      "logps/rejected": -372.88751220703125,
      "loss": 0.0518,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.225830078125,
      "rewards/margins": 5.19140625,
      "rewards/rejected": -7.415625095367432,
      "step": 4480
    },
    {
      "epoch": 2.366895097522404,
      "grad_norm": 60.09239329565684,
      "learning_rate": 4.0840801265155503e-07,
      "logits/chosen": -2.7203125953674316,
      "logits/rejected": -2.754687547683716,
      "logps/chosen": -416.32501220703125,
      "logps/rejected": -429.0249938964844,
      "loss": 0.052,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.0836424827575684,
      "rewards/margins": 5.33203125,
      "rewards/rejected": -7.411328315734863,
      "step": 4490
    },
    {
      "epoch": 2.3721665788086455,
      "grad_norm": 26.240433843639646,
      "learning_rate": 4.0709014232999473e-07,
      "logits/chosen": -2.8330078125,
      "logits/rejected": -2.840625047683716,
      "logps/chosen": -348.92498779296875,
      "logps/rejected": -388.7749938964844,
      "loss": 0.036,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.89678955078125,
      "rewards/margins": 5.457812309265137,
      "rewards/rejected": -7.3515625,
      "step": 4500
    },
    {
      "epoch": 2.3774380600948866,
      "grad_norm": 23.65366206992632,
      "learning_rate": 4.0577227200843433e-07,
      "logits/chosen": -2.687695264816284,
      "logits/rejected": -2.751953125,
      "logps/chosen": -405.3500061035156,
      "logps/rejected": -431.25,
      "loss": 0.0493,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.333471655845642,
      "rewards/margins": 5.321875095367432,
      "rewards/rejected": -6.659375190734863,
      "step": 4510
    },
    {
      "epoch": 2.382709541381128,
      "grad_norm": 43.7175785109272,
      "learning_rate": 4.0445440168687403e-07,
      "logits/chosen": -2.7437500953674316,
      "logits/rejected": -2.8623046875,
      "logps/chosen": -390.1499938964844,
      "logps/rejected": -379.2250061035156,
      "loss": 0.0553,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.8172485828399658,
      "rewards/margins": 5.190625190734863,
      "rewards/rejected": -7.005468845367432,
      "step": 4520
    },
    {
      "epoch": 2.3879810226673697,
      "grad_norm": 76.93683192433622,
      "learning_rate": 4.031365313653136e-07,
      "logits/chosen": -2.8238282203674316,
      "logits/rejected": -2.7886719703674316,
      "logps/chosen": -450.70001220703125,
      "logps/rejected": -460.7250061035156,
      "loss": 0.039,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.1966795921325684,
      "rewards/margins": 5.277539253234863,
      "rewards/rejected": -7.4765625,
      "step": 4530
    },
    {
      "epoch": 2.3932525039536108,
      "grad_norm": 18.89194432305766,
      "learning_rate": 4.0181866104375327e-07,
      "logits/chosen": -2.7652344703674316,
      "logits/rejected": -2.768749952316284,
      "logps/chosen": -355.8999938964844,
      "logps/rejected": -395.7749938964844,
      "loss": 0.0622,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.687890648841858,
      "rewards/margins": 5.1796875,
      "rewards/rejected": -6.865624904632568,
      "step": 4540
    },
    {
      "epoch": 2.3985239852398523,
      "grad_norm": 8.911323514163024,
      "learning_rate": 4.005007907221929e-07,
      "logits/chosen": -2.715039014816284,
      "logits/rejected": -2.874218702316284,
      "logps/chosen": -439.63751220703125,
      "logps/rejected": -420.54998779296875,
      "loss": 0.0438,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.895330786705017,
      "rewards/margins": 5.214453220367432,
      "rewards/rejected": -7.109375,
      "step": 4550
    },
    {
      "epoch": 2.403795466526094,
      "grad_norm": 11.389772955984098,
      "learning_rate": 3.9918292040063256e-07,
      "logits/chosen": -2.7593750953674316,
      "logits/rejected": -2.7802734375,
      "logps/chosen": -408.79998779296875,
      "logps/rejected": -414.0,
      "loss": 0.0304,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.412109375,
      "rewards/margins": 5.196093559265137,
      "rewards/rejected": -6.608593940734863,
      "step": 4560
    },
    {
      "epoch": 2.4090669478123354,
      "grad_norm": 32.289610041441456,
      "learning_rate": 3.978650500790722e-07,
      "logits/chosen": -2.633984327316284,
      "logits/rejected": -2.6654295921325684,
      "logps/chosen": -461.5625,
      "logps/rejected": -488.2250061035156,
      "loss": 0.0397,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.487280249595642,
      "rewards/margins": 5.631249904632568,
      "rewards/rejected": -7.118750095367432,
      "step": 4570
    },
    {
      "epoch": 2.4143384290985765,
      "grad_norm": 29.875454671637325,
      "learning_rate": 3.9654717975751186e-07,
      "logits/chosen": -2.723437547683716,
      "logits/rejected": -2.783203125,
      "logps/chosen": -371.07501220703125,
      "logps/rejected": -426.2250061035156,
      "loss": 0.0842,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.5448181629180908,
      "rewards/margins": 5.147656440734863,
      "rewards/rejected": -6.689843654632568,
      "step": 4580
    },
    {
      "epoch": 2.419609910384818,
      "grad_norm": 23.040727702079764,
      "learning_rate": 3.952293094359515e-07,
      "logits/chosen": -2.752734422683716,
      "logits/rejected": -2.7724609375,
      "logps/chosen": -403.11248779296875,
      "logps/rejected": -402.82501220703125,
      "loss": 0.047,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.5705139636993408,
      "rewards/margins": 5.375390529632568,
      "rewards/rejected": -6.946875095367432,
      "step": 4590
    },
    {
      "epoch": 2.4248813916710596,
      "grad_norm": 24.83581088789688,
      "learning_rate": 3.939114391143911e-07,
      "logits/chosen": -2.7835936546325684,
      "logits/rejected": -2.847460985183716,
      "logps/chosen": -453.3999938964844,
      "logps/rejected": -417.3500061035156,
      "loss": 0.0396,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.4081909656524658,
      "rewards/margins": 5.287109375,
      "rewards/rejected": -6.692578315734863,
      "step": 4600
    },
    {
      "epoch": 2.4301528729573008,
      "grad_norm": 64.00396846048803,
      "learning_rate": 3.925935687928308e-07,
      "logits/chosen": -2.7095704078674316,
      "logits/rejected": -2.7021484375,
      "logps/chosen": -400.48748779296875,
      "logps/rejected": -414.0249938964844,
      "loss": 0.0413,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6485106945037842,
      "rewards/margins": 5.365624904632568,
      "rewards/rejected": -7.013281345367432,
      "step": 4610
    },
    {
      "epoch": 2.4354243542435423,
      "grad_norm": 11.569731171110421,
      "learning_rate": 3.912756984712704e-07,
      "logits/chosen": -2.809375047683716,
      "logits/rejected": -2.84375,
      "logps/chosen": -387.57501220703125,
      "logps/rejected": -415.17498779296875,
      "loss": 0.0271,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5994384288787842,
      "rewards/margins": 5.372656345367432,
      "rewards/rejected": -6.971875190734863,
      "step": 4620
    },
    {
      "epoch": 2.440695835529784,
      "grad_norm": 3.767140517288781,
      "learning_rate": 3.8995782814971004e-07,
      "logits/chosen": -2.755859375,
      "logits/rejected": -2.7652344703674316,
      "logps/chosen": -423.25,
      "logps/rejected": -406.7250061035156,
      "loss": 0.0334,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.0860352516174316,
      "rewards/margins": 5.604296684265137,
      "rewards/rejected": -7.689843654632568,
      "step": 4630
    },
    {
      "epoch": 2.4459673168160254,
      "grad_norm": 39.61505778115417,
      "learning_rate": 3.886399578281497e-07,
      "logits/chosen": -2.746875047683716,
      "logits/rejected": -2.883593797683716,
      "logps/chosen": -438.92498779296875,
      "logps/rejected": -435.625,
      "loss": 0.039,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.5899169445037842,
      "rewards/margins": 5.482031345367432,
      "rewards/rejected": -7.069531440734863,
      "step": 4640
    },
    {
      "epoch": 2.451238798102267,
      "grad_norm": 48.69601472155945,
      "learning_rate": 3.8732208750658934e-07,
      "logits/chosen": -2.8726563453674316,
      "logits/rejected": -3.002734422683716,
      "logps/chosen": -437.73748779296875,
      "logps/rejected": -409.95001220703125,
      "loss": 0.0325,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.6618163585662842,
      "rewards/margins": 5.3125,
      "rewards/rejected": -6.971875190734863,
      "step": 4650
    },
    {
      "epoch": 2.456510279388508,
      "grad_norm": 13.31980477652077,
      "learning_rate": 3.8600421718502893e-07,
      "logits/chosen": -2.725781202316284,
      "logits/rejected": -2.7750000953674316,
      "logps/chosen": -412.9750061035156,
      "logps/rejected": -426.67498779296875,
      "loss": 0.0365,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.647558569908142,
      "rewards/margins": 5.428124904632568,
      "rewards/rejected": -7.081250190734863,
      "step": 4660
    },
    {
      "epoch": 2.4617817606747496,
      "grad_norm": 11.7811437928705,
      "learning_rate": 3.8468634686346863e-07,
      "logits/chosen": -2.777539014816284,
      "logits/rejected": -2.8199219703674316,
      "logps/chosen": -399.2875061035156,
      "logps/rejected": -428.2250061035156,
      "loss": 0.0484,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.108642578125,
      "rewards/margins": 5.683984279632568,
      "rewards/rejected": -7.793749809265137,
      "step": 4670
    },
    {
      "epoch": 2.467053241960991,
      "grad_norm": 12.027356220827837,
      "learning_rate": 3.8336847654190823e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.9105467796325684,
      "logps/chosen": -402.4750061035156,
      "logps/rejected": -407.8500061035156,
      "loss": 0.0333,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.5788452625274658,
      "rewards/margins": 5.669921875,
      "rewards/rejected": -7.246874809265137,
      "step": 4680
    },
    {
      "epoch": 2.4723247232472323,
      "grad_norm": 10.814018951139355,
      "learning_rate": 3.8205060622034793e-07,
      "logits/chosen": -2.7388672828674316,
      "logits/rejected": -2.846484422683716,
      "logps/chosen": -405.42498779296875,
      "logps/rejected": -406.1000061035156,
      "loss": 0.0424,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.507238745689392,
      "rewards/margins": 5.189062595367432,
      "rewards/rejected": -6.694531440734863,
      "step": 4690
    },
    {
      "epoch": 2.477596204533474,
      "grad_norm": 17.265413869806224,
      "learning_rate": 3.807327358987875e-07,
      "logits/chosen": -2.766406297683716,
      "logits/rejected": -2.8207030296325684,
      "logps/chosen": -397.4750061035156,
      "logps/rejected": -418.1000061035156,
      "loss": 0.0364,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.643212914466858,
      "rewards/margins": 5.197265625,
      "rewards/rejected": -6.83984375,
      "step": 4700
    },
    {
      "epoch": 2.4828676858197154,
      "grad_norm": 28.749110042621055,
      "learning_rate": 3.7941486557722717e-07,
      "logits/chosen": -2.753124952316284,
      "logits/rejected": -2.9291014671325684,
      "logps/chosen": -400.3999938964844,
      "logps/rejected": -403.1499938964844,
      "loss": 0.0437,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.566503882408142,
      "rewards/margins": 5.249609470367432,
      "rewards/rejected": -6.811718940734863,
      "step": 4710
    },
    {
      "epoch": 2.488139167105957,
      "grad_norm": 32.13505997195597,
      "learning_rate": 3.7809699525566687e-07,
      "logits/chosen": -2.746875047683716,
      "logits/rejected": -2.78125,
      "logps/chosen": -429.17498779296875,
      "logps/rejected": -433.67498779296875,
      "loss": 0.0381,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.362451195716858,
      "rewards/margins": 5.418359279632568,
      "rewards/rejected": -6.780859470367432,
      "step": 4720
    },
    {
      "epoch": 2.493410648392198,
      "grad_norm": 19.7596190778966,
      "learning_rate": 3.7677912493410647e-07,
      "logits/chosen": -2.855273485183716,
      "logits/rejected": -2.9183592796325684,
      "logps/chosen": -409.8999938964844,
      "logps/rejected": -391.2749938964844,
      "loss": 0.0379,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.6887085437774658,
      "rewards/margins": 5.421875,
      "rewards/rejected": -7.10546875,
      "step": 4730
    },
    {
      "epoch": 2.4986821296784396,
      "grad_norm": 44.464380525241424,
      "learning_rate": 3.754612546125461e-07,
      "logits/chosen": -2.706835985183716,
      "logits/rejected": -2.7953124046325684,
      "logps/chosen": -353.4750061035156,
      "logps/rejected": -361.95001220703125,
      "loss": 0.1027,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.0008788108825684,
      "rewards/margins": 5.223046779632568,
      "rewards/rejected": -7.224218845367432,
      "step": 4740
    },
    {
      "epoch": 2.503953610964681,
      "grad_norm": 8.391678720180327,
      "learning_rate": 3.7414338429098576e-07,
      "logits/chosen": -2.862499952316284,
      "logits/rejected": -2.9195313453674316,
      "logps/chosen": -443.4375,
      "logps/rejected": -436.7749938964844,
      "loss": 0.0698,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.938134789466858,
      "rewards/margins": 5.407031059265137,
      "rewards/rejected": -7.341406345367432,
      "step": 4750
    },
    {
      "epoch": 2.5092250922509223,
      "grad_norm": 26.77834237620794,
      "learning_rate": 3.728255139694254e-07,
      "logits/chosen": -2.8109374046325684,
      "logits/rejected": -2.87109375,
      "logps/chosen": -407.70001220703125,
      "logps/rejected": -403.7250061035156,
      "loss": 0.0452,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.8184082508087158,
      "rewards/margins": 5.385546684265137,
      "rewards/rejected": -7.201562404632568,
      "step": 4760
    },
    {
      "epoch": 2.514496573537164,
      "grad_norm": 36.23430926670236,
      "learning_rate": 3.71507643647865e-07,
      "logits/chosen": -2.797656297683716,
      "logits/rejected": -2.8402342796325684,
      "logps/chosen": -422.70001220703125,
      "logps/rejected": -420.25,
      "loss": 0.0384,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7670166492462158,
      "rewards/margins": 5.206640720367432,
      "rewards/rejected": -6.969531059265137,
      "step": 4770
    },
    {
      "epoch": 2.5197680548234054,
      "grad_norm": 138.7269619719819,
      "learning_rate": 3.701897733263047e-07,
      "logits/chosen": -2.784374952316284,
      "logits/rejected": -2.815234422683716,
      "logps/chosen": -369.7749938964844,
      "logps/rejected": -391.25,
      "loss": 0.0768,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.5557372570037842,
      "rewards/margins": 4.822265625,
      "rewards/rejected": -6.381249904632568,
      "step": 4780
    },
    {
      "epoch": 2.525039536109647,
      "grad_norm": 10.137880051909976,
      "learning_rate": 3.688719030047443e-07,
      "logits/chosen": -2.8011717796325684,
      "logits/rejected": -2.8525390625,
      "logps/chosen": -362.01251220703125,
      "logps/rejected": -392.1000061035156,
      "loss": 0.0478,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.37286376953125,
      "rewards/margins": 4.803515434265137,
      "rewards/rejected": -6.178124904632568,
      "step": 4790
    },
    {
      "epoch": 2.5303110173958885,
      "grad_norm": 18.444926700115523,
      "learning_rate": 3.6755403268318395e-07,
      "logits/chosen": -2.6820311546325684,
      "logits/rejected": -2.819531202316284,
      "logps/chosen": -465.17498779296875,
      "logps/rejected": -409.375,
      "loss": 0.025,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1975586414337158,
      "rewards/margins": 5.441015720367432,
      "rewards/rejected": -6.639452934265137,
      "step": 4800
    },
    {
      "epoch": 2.5355824986821296,
      "grad_norm": 34.67053817282632,
      "learning_rate": 3.662361623616236e-07,
      "logits/chosen": -2.7757811546325684,
      "logits/rejected": -2.803906202316284,
      "logps/chosen": -355.7124938964844,
      "logps/rejected": -409.8999938964844,
      "loss": 0.0379,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.308203101158142,
      "rewards/margins": 5.436327934265137,
      "rewards/rejected": -6.744531154632568,
      "step": 4810
    },
    {
      "epoch": 2.540853979968371,
      "grad_norm": 10.733937314661805,
      "learning_rate": 3.6491829204006324e-07,
      "logits/chosen": -2.755664110183716,
      "logits/rejected": -2.810742139816284,
      "logps/chosen": -409.25,
      "logps/rejected": -446.32501220703125,
      "loss": 0.0435,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.586572289466858,
      "rewards/margins": 5.267968654632568,
      "rewards/rejected": -6.850781440734863,
      "step": 4820
    },
    {
      "epoch": 2.5461254612546127,
      "grad_norm": 3.5250033831052447,
      "learning_rate": 3.6360042171850284e-07,
      "logits/chosen": -2.774609327316284,
      "logits/rejected": -2.878124952316284,
      "logps/chosen": -384.375,
      "logps/rejected": -423.04998779296875,
      "loss": 0.046,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.787542700767517,
      "rewards/margins": 5.153906345367432,
      "rewards/rejected": -6.94140625,
      "step": 4830
    },
    {
      "epoch": 2.551396942540854,
      "grad_norm": 8.487622719147256,
      "learning_rate": 3.6228255139694254e-07,
      "logits/chosen": -2.852343797683716,
      "logits/rejected": -2.834765672683716,
      "logps/chosen": -399.07501220703125,
      "logps/rejected": -421.75,
      "loss": 0.035,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.596337914466858,
      "rewards/margins": 5.166015625,
      "rewards/rejected": -6.762109279632568,
      "step": 4840
    },
    {
      "epoch": 2.5566684238270954,
      "grad_norm": 57.5370547482681,
      "learning_rate": 3.609646810753822e-07,
      "logits/chosen": -2.775390625,
      "logits/rejected": -2.903515577316284,
      "logps/chosen": -391.75,
      "logps/rejected": -385.8500061035156,
      "loss": 0.048,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.765112280845642,
      "rewards/margins": 5.078125,
      "rewards/rejected": -6.841406345367432,
      "step": 4850
    },
    {
      "epoch": 2.561939905113337,
      "grad_norm": 18.258445742231107,
      "learning_rate": 3.5964681075382183e-07,
      "logits/chosen": -2.797070264816284,
      "logits/rejected": -2.833789110183716,
      "logps/chosen": -367.8999938964844,
      "logps/rejected": -373.125,
      "loss": 0.1077,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.792883276939392,
      "rewards/margins": 4.815234184265137,
      "rewards/rejected": -6.605078220367432,
      "step": 4860
    },
    {
      "epoch": 2.5672113863995785,
      "grad_norm": 9.484364344439397,
      "learning_rate": 3.583289404322615e-07,
      "logits/chosen": -2.834765672683716,
      "logits/rejected": -2.8531250953674316,
      "logps/chosen": -379.42498779296875,
      "logps/rejected": -391.07501220703125,
      "loss": 0.0491,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.337695360183716,
      "rewards/margins": 5.221093654632568,
      "rewards/rejected": -7.55078125,
      "step": 4870
    },
    {
      "epoch": 2.5724828676858196,
      "grad_norm": 24.8010178645133,
      "learning_rate": 3.570110701107011e-07,
      "logits/chosen": -2.8779296875,
      "logits/rejected": -2.855664014816284,
      "logps/chosen": -393.42498779296875,
      "logps/rejected": -394.29998779296875,
      "loss": 0.0594,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.381420850753784,
      "rewards/margins": 4.847851753234863,
      "rewards/rejected": -7.229687690734863,
      "step": 4880
    },
    {
      "epoch": 2.577754348972061,
      "grad_norm": 28.10302450844873,
      "learning_rate": 3.556931997891408e-07,
      "logits/chosen": -2.7455077171325684,
      "logits/rejected": -2.843554735183716,
      "logps/chosen": -385.07501220703125,
      "logps/rejected": -402.7250061035156,
      "loss": 0.032,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.151660203933716,
      "rewards/margins": 5.508984565734863,
      "rewards/rejected": -7.664843559265137,
      "step": 4890
    },
    {
      "epoch": 2.5830258302583027,
      "grad_norm": 10.430233945352581,
      "learning_rate": 3.5437532946758037e-07,
      "logits/chosen": -2.850781202316284,
      "logits/rejected": -2.9058594703674316,
      "logps/chosen": -446.8999938964844,
      "logps/rejected": -462.5,
      "loss": 0.028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.77197265625,
      "rewards/margins": 5.535937309265137,
      "rewards/rejected": -7.306250095367432,
      "step": 4900
    },
    {
      "epoch": 2.588297311544544,
      "grad_norm": 36.524493147791425,
      "learning_rate": 3.5305745914602e-07,
      "logits/chosen": -2.768359422683716,
      "logits/rejected": -2.8724608421325684,
      "logps/chosen": -423.5,
      "logps/rejected": -419.0,
      "loss": 0.0469,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.274670362472534,
      "rewards/margins": 5.110547065734863,
      "rewards/rejected": -7.381249904632568,
      "step": 4910
    },
    {
      "epoch": 2.5935687928307853,
      "grad_norm": 18.01082915716174,
      "learning_rate": 3.5173958882445966e-07,
      "logits/chosen": -2.821484327316284,
      "logits/rejected": -2.876953125,
      "logps/chosen": -449.70001220703125,
      "logps/rejected": -450.04998779296875,
      "loss": 0.0599,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.405139207839966,
      "rewards/margins": 5.069140434265137,
      "rewards/rejected": -7.473046779632568,
      "step": 4920
    },
    {
      "epoch": 2.598840274117027,
      "grad_norm": 59.02064103868156,
      "learning_rate": 3.504217185028993e-07,
      "logits/chosen": -2.7408204078674316,
      "logits/rejected": -2.709179639816284,
      "logps/chosen": -420.86248779296875,
      "logps/rejected": -466.67498779296875,
      "loss": 0.0578,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.2270264625549316,
      "rewards/margins": 5.285937309265137,
      "rewards/rejected": -7.509765625,
      "step": 4930
    },
    {
      "epoch": 2.6041117554032684,
      "grad_norm": 32.02078832315668,
      "learning_rate": 3.491038481813389e-07,
      "logits/chosen": -2.7230467796325684,
      "logits/rejected": -2.799609422683716,
      "logps/chosen": -430.5249938964844,
      "logps/rejected": -429.8500061035156,
      "loss": 0.0528,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.0000977516174316,
      "rewards/margins": 5.392187595367432,
      "rewards/rejected": -7.396093845367432,
      "step": 4940
    },
    {
      "epoch": 2.60938323668951,
      "grad_norm": 21.188017486138428,
      "learning_rate": 3.477859778597786e-07,
      "logits/chosen": -2.8212890625,
      "logits/rejected": -2.876953125,
      "logps/chosen": -441.92498779296875,
      "logps/rejected": -442.25,
      "loss": 0.0512,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.119128465652466,
      "rewards/margins": 5.33203125,
      "rewards/rejected": -7.451171875,
      "step": 4950
    },
    {
      "epoch": 2.614654717975751,
      "grad_norm": 7.352149895265314,
      "learning_rate": 3.464681075382182e-07,
      "logits/chosen": -2.882617235183716,
      "logits/rejected": -2.9292969703674316,
      "logps/chosen": -393.7749938964844,
      "logps/rejected": -395.25,
      "loss": 0.0593,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.168774366378784,
      "rewards/margins": 5.267968654632568,
      "rewards/rejected": -7.4375,
      "step": 4960
    },
    {
      "epoch": 2.6199261992619927,
      "grad_norm": 18.319564671184285,
      "learning_rate": 3.4515023721665785e-07,
      "logits/chosen": -2.805468797683716,
      "logits/rejected": -2.8033204078674316,
      "logps/chosen": -403.3999938964844,
      "logps/rejected": -408.9750061035156,
      "loss": 0.0385,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.9845092296600342,
      "rewards/margins": 5.810546875,
      "rewards/rejected": -7.794921875,
      "step": 4970
    },
    {
      "epoch": 2.625197680548234,
      "grad_norm": 69.18225485444006,
      "learning_rate": 3.438323668950975e-07,
      "logits/chosen": -2.7955079078674316,
      "logits/rejected": -2.8306641578674316,
      "logps/chosen": -443.8500061035156,
      "logps/rejected": -432.79998779296875,
      "loss": 0.0571,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.380175828933716,
      "rewards/margins": 5.204297065734863,
      "rewards/rejected": -7.580468654632568,
      "step": 4980
    },
    {
      "epoch": 2.6304691618344753,
      "grad_norm": 77.49144893913405,
      "learning_rate": 3.4251449657353714e-07,
      "logits/chosen": -2.860546827316284,
      "logits/rejected": -2.8949217796325684,
      "logps/chosen": -373.57501220703125,
      "logps/rejected": -393.6499938964844,
      "loss": 0.044,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.0152344703674316,
      "rewards/margins": 5.517578125,
      "rewards/rejected": -7.53125,
      "step": 4990
    },
    {
      "epoch": 2.635740643120717,
      "grad_norm": 18.814383310237233,
      "learning_rate": 3.411966262519768e-07,
      "logits/chosen": -2.889843702316284,
      "logits/rejected": -2.9906249046325684,
      "logps/chosen": -382.375,
      "logps/rejected": -378.07501220703125,
      "loss": 0.0341,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5949218273162842,
      "rewards/margins": 5.363671779632568,
      "rewards/rejected": -6.960156440734863,
      "step": 5000
    },
    {
      "epoch": 2.6410121244069584,
      "grad_norm": 4.881124435785823,
      "learning_rate": 3.3987875593041644e-07,
      "logits/chosen": -2.881640672683716,
      "logits/rejected": -2.895312547683716,
      "logps/chosen": -408.07501220703125,
      "logps/rejected": -454.29998779296875,
      "loss": 0.0306,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.8216949701309204,
      "rewards/margins": 5.723437309265137,
      "rewards/rejected": -7.547265529632568,
      "step": 5010
    },
    {
      "epoch": 2.6462836056932,
      "grad_norm": 32.487313849078674,
      "learning_rate": 3.385608856088561e-07,
      "logits/chosen": -2.819531202316284,
      "logits/rejected": -2.921875,
      "logps/chosen": -457.75,
      "logps/rejected": -432.20001220703125,
      "loss": 0.0381,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.7522399425506592,
      "rewards/margins": 5.223437309265137,
      "rewards/rejected": -6.975781440734863,
      "step": 5020
    },
    {
      "epoch": 2.651555086979441,
      "grad_norm": 32.567744375145686,
      "learning_rate": 3.372430152872957e-07,
      "logits/chosen": -2.795703172683716,
      "logits/rejected": -2.9984374046325684,
      "logps/chosen": -438.29998779296875,
      "logps/rejected": -427.7250061035156,
      "loss": 0.0443,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.673669457435608,
      "rewards/margins": 5.295312404632568,
      "rewards/rejected": -6.967577934265137,
      "step": 5030
    },
    {
      "epoch": 2.6568265682656826,
      "grad_norm": 31.142144062500588,
      "learning_rate": 3.359251449657354e-07,
      "logits/chosen": -2.8832030296325684,
      "logits/rejected": -2.9664063453674316,
      "logps/chosen": -397.8999938964844,
      "logps/rejected": -377.2250061035156,
      "loss": 0.0545,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.6665527820587158,
      "rewards/margins": 4.961718559265137,
      "rewards/rejected": -6.626562595367432,
      "step": 5040
    },
    {
      "epoch": 2.662098049551924,
      "grad_norm": 16.68068141327151,
      "learning_rate": 3.34607274644175e-07,
      "logits/chosen": -2.858593702316284,
      "logits/rejected": -2.871875047683716,
      "logps/chosen": -388.0249938964844,
      "logps/rejected": -417.04998779296875,
      "loss": 0.0519,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.639318823814392,
      "rewards/margins": 5.310937404632568,
      "rewards/rejected": -6.950781345367432,
      "step": 5050
    },
    {
      "epoch": 2.6673695308381653,
      "grad_norm": 9.946766143114333,
      "learning_rate": 3.332894043226147e-07,
      "logits/chosen": -2.77734375,
      "logits/rejected": -2.797656297683716,
      "logps/chosen": -436.9750061035156,
      "logps/rejected": -423.6000061035156,
      "loss": 0.0391,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.8922607898712158,
      "rewards/margins": 5.549218654632568,
      "rewards/rejected": -7.442968845367432,
      "step": 5060
    },
    {
      "epoch": 2.672641012124407,
      "grad_norm": 27.57110558742089,
      "learning_rate": 3.3197153400105427e-07,
      "logits/chosen": -2.95703125,
      "logits/rejected": -2.9535155296325684,
      "logps/chosen": -354.36248779296875,
      "logps/rejected": -372.0249938964844,
      "loss": 0.0418,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.7880370616912842,
      "rewards/margins": 5.237500190734863,
      "rewards/rejected": -7.025781154632568,
      "step": 5070
    },
    {
      "epoch": 2.6779124934106484,
      "grad_norm": 70.11567166194769,
      "learning_rate": 3.306536636794939e-07,
      "logits/chosen": -2.775390625,
      "logits/rejected": -2.8316407203674316,
      "logps/chosen": -425.17498779296875,
      "logps/rejected": -407.25,
      "loss": 0.0535,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.964135766029358,
      "rewards/margins": 5.045312404632568,
      "rewards/rejected": -7.009375095367432,
      "step": 5080
    },
    {
      "epoch": 2.68318397469689,
      "grad_norm": 16.2016559387298,
      "learning_rate": 3.2933579335793357e-07,
      "logits/chosen": -2.8343749046325684,
      "logits/rejected": -2.901171922683716,
      "logps/chosen": -414.20001220703125,
      "logps/rejected": -408.29998779296875,
      "loss": 0.0448,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.9125854969024658,
      "rewards/margins": 5.114062309265137,
      "rewards/rejected": -7.021874904632568,
      "step": 5090
    },
    {
      "epoch": 2.6884554559831315,
      "grad_norm": 7.693446217075289,
      "learning_rate": 3.280179230363732e-07,
      "logits/chosen": -2.783203125,
      "logits/rejected": -2.895312547683716,
      "logps/chosen": -418.4750061035156,
      "logps/rejected": -451.1499938964844,
      "loss": 0.0311,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.81121826171875,
      "rewards/margins": 5.272265434265137,
      "rewards/rejected": -7.08203125,
      "step": 5100
    },
    {
      "epoch": 2.6937269372693726,
      "grad_norm": 26.800345235015648,
      "learning_rate": 3.267000527148128e-07,
      "logits/chosen": -2.792773485183716,
      "logits/rejected": -2.907421827316284,
      "logps/chosen": -360.61248779296875,
      "logps/rejected": -454.54998779296875,
      "loss": 0.0469,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.193920850753784,
      "rewards/margins": 5.263671875,
      "rewards/rejected": -7.464062690734863,
      "step": 5110
    },
    {
      "epoch": 2.698998418555614,
      "grad_norm": 16.333584287858926,
      "learning_rate": 3.253821823932525e-07,
      "logits/chosen": -2.80078125,
      "logits/rejected": -2.8695311546325684,
      "logps/chosen": -418.7749938964844,
      "logps/rejected": -403.79998779296875,
      "loss": 0.0495,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.1407713890075684,
      "rewards/margins": 5.252343654632568,
      "rewards/rejected": -7.398046970367432,
      "step": 5120
    },
    {
      "epoch": 2.7042698998418553,
      "grad_norm": 5.382754470955361,
      "learning_rate": 3.2406431207169216e-07,
      "logits/chosen": -2.719531297683716,
      "logits/rejected": -2.891406297683716,
      "logps/chosen": -458.1000061035156,
      "logps/rejected": -428.125,
      "loss": 0.0319,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.5369170904159546,
      "rewards/margins": 5.540625095367432,
      "rewards/rejected": -7.079687595367432,
      "step": 5130
    },
    {
      "epoch": 2.709541381128097,
      "grad_norm": 20.78683429772268,
      "learning_rate": 3.2274644175013175e-07,
      "logits/chosen": -2.7357420921325684,
      "logits/rejected": -2.8480467796325684,
      "logps/chosen": -407.5249938964844,
      "logps/rejected": -404.3500061035156,
      "loss": 0.0656,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.918603539466858,
      "rewards/margins": 5.194140434265137,
      "rewards/rejected": -7.110937595367432,
      "step": 5140
    },
    {
      "epoch": 2.7148128624143384,
      "grad_norm": 4.233208376807893,
      "learning_rate": 3.2142857142857145e-07,
      "logits/chosen": -2.83984375,
      "logits/rejected": -2.9625000953674316,
      "logps/chosen": -402.95001220703125,
      "logps/rejected": -419.92498779296875,
      "loss": 0.0228,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.1376633644104004,
      "rewards/margins": 5.303906440734863,
      "rewards/rejected": -7.442968845367432,
      "step": 5150
    },
    {
      "epoch": 2.72008434370058,
      "grad_norm": 23.577248683920303,
      "learning_rate": 3.2011070110701105e-07,
      "logits/chosen": -2.780468702316284,
      "logits/rejected": -2.829882860183716,
      "logps/chosen": -466.875,
      "logps/rejected": -450.7250061035156,
      "loss": 0.0483,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.1195311546325684,
      "rewards/margins": 5.563281059265137,
      "rewards/rejected": -7.685937404632568,
      "step": 5160
    },
    {
      "epoch": 2.7253558249868215,
      "grad_norm": 24.70950705737043,
      "learning_rate": 3.187928307854507e-07,
      "logits/chosen": -2.873828172683716,
      "logits/rejected": -2.857421875,
      "logps/chosen": -402.25,
      "logps/rejected": -459.6000061035156,
      "loss": 0.0261,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.0940184593200684,
      "rewards/margins": 5.709765434265137,
      "rewards/rejected": -7.801562309265137,
      "step": 5170
    },
    {
      "epoch": 2.7306273062730626,
      "grad_norm": 43.42093070356716,
      "learning_rate": 3.1747496046389034e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.8984375,
      "logps/chosen": -384.4750061035156,
      "logps/rejected": -397.5249938964844,
      "loss": 0.0686,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.8822753429412842,
      "rewards/margins": 5.308984279632568,
      "rewards/rejected": -7.1875,
      "step": 5180
    },
    {
      "epoch": 2.735898787559304,
      "grad_norm": 12.777836492556917,
      "learning_rate": 3.1615709014233e-07,
      "logits/chosen": -2.8265624046325684,
      "logits/rejected": -2.963085889816284,
      "logps/chosen": -430.92498779296875,
      "logps/rejected": -425.3999938964844,
      "loss": 0.073,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.131787061691284,
      "rewards/margins": 5.629687309265137,
      "rewards/rejected": -7.767187595367432,
      "step": 5190
    },
    {
      "epoch": 2.7411702688455457,
      "grad_norm": 19.03117336451652,
      "learning_rate": 3.148392198207696e-07,
      "logits/chosen": -2.8707032203674316,
      "logits/rejected": -3.063281297683716,
      "logps/chosen": -427.17498779296875,
      "logps/rejected": -388.17498779296875,
      "loss": 0.0481,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.744653344154358,
      "rewards/margins": 5.344140529632568,
      "rewards/rejected": -7.088281154632568,
      "step": 5200
    },
    {
      "epoch": 2.746441750131787,
      "grad_norm": 16.831712459777258,
      "learning_rate": 3.135213494992093e-07,
      "logits/chosen": -2.8384766578674316,
      "logits/rejected": -2.874218702316284,
      "logps/chosen": -363.57501220703125,
      "logps/rejected": -387.29998779296875,
      "loss": 0.0577,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.69476318359375,
      "rewards/margins": 5.543749809265137,
      "rewards/rejected": -7.236718654632568,
      "step": 5210
    },
    {
      "epoch": 2.7517132314180284,
      "grad_norm": 24.206176037294792,
      "learning_rate": 3.122034791776489e-07,
      "logits/chosen": -2.790820360183716,
      "logits/rejected": -2.932812452316284,
      "logps/chosen": -446.8999938964844,
      "logps/rejected": -412.07501220703125,
      "loss": 0.0364,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.8209717273712158,
      "rewards/margins": 5.521874904632568,
      "rewards/rejected": -7.34375,
      "step": 5220
    },
    {
      "epoch": 2.75698471270427,
      "grad_norm": 82.49032965105545,
      "learning_rate": 3.108856088560886e-07,
      "logits/chosen": -2.8873047828674316,
      "logits/rejected": -2.8656249046325684,
      "logps/chosen": -362.5874938964844,
      "logps/rejected": -400.7250061035156,
      "loss": 0.0442,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.0470948219299316,
      "rewards/margins": 5.484375,
      "rewards/rejected": -7.53125,
      "step": 5230
    },
    {
      "epoch": 2.7622561939905115,
      "grad_norm": 7.510774517997259,
      "learning_rate": 3.095677385345282e-07,
      "logits/chosen": -2.8265624046325684,
      "logits/rejected": -2.801953077316284,
      "logps/chosen": -423.5,
      "logps/rejected": -441.79998779296875,
      "loss": 0.0372,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.278125047683716,
      "rewards/margins": 5.440234184265137,
      "rewards/rejected": -7.721093654632568,
      "step": 5240
    },
    {
      "epoch": 2.767527675276753,
      "grad_norm": 7.724275771157441,
      "learning_rate": 3.082498682129678e-07,
      "logits/chosen": -2.815624952316284,
      "logits/rejected": -2.880859375,
      "logps/chosen": -410.13751220703125,
      "logps/rejected": -394.54998779296875,
      "loss": 0.0438,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.004589796066284,
      "rewards/margins": 5.446093559265137,
      "rewards/rejected": -7.456250190734863,
      "step": 5250
    },
    {
      "epoch": 2.772799156562994,
      "grad_norm": 3.628187865812853,
      "learning_rate": 3.0693199789140747e-07,
      "logits/chosen": -2.88671875,
      "logits/rejected": -2.866015672683716,
      "logps/chosen": -447.4750061035156,
      "logps/rejected": -484.20001220703125,
      "loss": 0.0284,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.115478515625,
      "rewards/margins": 5.871484279632568,
      "rewards/rejected": -7.987500190734863,
      "step": 5260
    },
    {
      "epoch": 2.7780706378492357,
      "grad_norm": 5.178911758843689,
      "learning_rate": 3.056141275698471e-07,
      "logits/chosen": -2.9749999046325684,
      "logits/rejected": -2.9671874046325684,
      "logps/chosen": -424.2749938964844,
      "logps/rejected": -480.1499938964844,
      "loss": 0.0456,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.722747802734375,
      "rewards/margins": 5.194921970367432,
      "rewards/rejected": -6.916406154632568,
      "step": 5270
    },
    {
      "epoch": 2.783342119135477,
      "grad_norm": 19.753226127604254,
      "learning_rate": 3.0429625724828676e-07,
      "logits/chosen": -2.891406297683716,
      "logits/rejected": -2.933398485183716,
      "logps/chosen": -413.5,
      "logps/rejected": -377.8374938964844,
      "loss": 0.0385,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.666357398033142,
      "rewards/margins": 5.481640815734863,
      "rewards/rejected": -7.145312309265137,
      "step": 5280
    },
    {
      "epoch": 2.7886136004217184,
      "grad_norm": 14.040257903113075,
      "learning_rate": 3.029783869267264e-07,
      "logits/chosen": -2.767773389816284,
      "logits/rejected": -2.852343797683716,
      "logps/chosen": -418.82501220703125,
      "logps/rejected": -432.3500061035156,
      "loss": 0.0547,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.9697265625,
      "rewards/margins": 5.248437404632568,
      "rewards/rejected": -7.217968940734863,
      "step": 5290
    },
    {
      "epoch": 2.79388508170796,
      "grad_norm": 14.112150646525643,
      "learning_rate": 3.0166051660516606e-07,
      "logits/chosen": -2.826953172683716,
      "logits/rejected": -2.906445264816284,
      "logps/chosen": -401.54998779296875,
      "logps/rejected": -452.70001220703125,
      "loss": 0.0668,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.193530321121216,
      "rewards/margins": 5.379296779632568,
      "rewards/rejected": -7.571093559265137,
      "step": 5300
    },
    {
      "epoch": 2.7991565629942015,
      "grad_norm": 13.89316057375122,
      "learning_rate": 3.0034264628360565e-07,
      "logits/chosen": -2.916015625,
      "logits/rejected": -2.906054735183716,
      "logps/chosen": -429.29998779296875,
      "logps/rejected": -454.2749938964844,
      "loss": 0.0485,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.2206053733825684,
      "rewards/margins": 5.174218654632568,
      "rewards/rejected": -7.395312309265137,
      "step": 5310
    },
    {
      "epoch": 2.804428044280443,
      "grad_norm": 45.070138732022045,
      "learning_rate": 2.9902477596204535e-07,
      "logits/chosen": -2.894335985183716,
      "logits/rejected": -2.999804735183716,
      "logps/chosen": -421.0,
      "logps/rejected": -414.1499938964844,
      "loss": 0.046,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.109130859375,
      "rewards/margins": 5.463281154632568,
      "rewards/rejected": -7.578125,
      "step": 5320
    },
    {
      "epoch": 2.809699525566684,
      "grad_norm": 62.00710642851612,
      "learning_rate": 2.9770690564048495e-07,
      "logits/chosen": -2.830273389816284,
      "logits/rejected": -2.935546875,
      "logps/chosen": -405.1000061035156,
      "logps/rejected": -435.125,
      "loss": 0.0425,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.2493653297424316,
      "rewards/margins": 5.074999809265137,
      "rewards/rejected": -7.325781345367432,
      "step": 5330
    },
    {
      "epoch": 2.8149710068529257,
      "grad_norm": 8.40088799762744,
      "learning_rate": 2.963890353189246e-07,
      "logits/chosen": -2.728515625,
      "logits/rejected": -2.8656249046325684,
      "logps/chosen": -417.2749938964844,
      "logps/rejected": -395.54998779296875,
      "loss": 0.0572,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.384570360183716,
      "rewards/margins": 5.421875,
      "rewards/rejected": -7.809374809265137,
      "step": 5340
    },
    {
      "epoch": 2.8202424881391672,
      "grad_norm": 8.66513079220955,
      "learning_rate": 2.9507116499736424e-07,
      "logits/chosen": -2.729687452316284,
      "logits/rejected": -2.806640625,
      "logps/chosen": -409.0,
      "logps/rejected": -435.5249938964844,
      "loss": 0.0391,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.885717749595642,
      "rewards/margins": 5.642773628234863,
      "rewards/rejected": -7.530468940734863,
      "step": 5350
    },
    {
      "epoch": 2.8255139694254083,
      "grad_norm": 45.05265398403296,
      "learning_rate": 2.937532946758039e-07,
      "logits/chosen": -2.8072266578674316,
      "logits/rejected": -2.8570313453674316,
      "logps/chosen": -343.92498779296875,
      "logps/rejected": -370.32501220703125,
      "loss": 0.1061,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.3097290992736816,
      "rewards/margins": 5.005859375,
      "rewards/rejected": -7.315625190734863,
      "step": 5360
    },
    {
      "epoch": 2.83078545071165,
      "grad_norm": 23.258921810472057,
      "learning_rate": 2.924354243542435e-07,
      "logits/chosen": -2.8578124046325684,
      "logits/rejected": -2.940234422683716,
      "logps/chosen": -415.11248779296875,
      "logps/rejected": -452.70001220703125,
      "loss": 0.0231,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.1617798805236816,
      "rewards/margins": 5.732812404632568,
      "rewards/rejected": -7.897656440734863,
      "step": 5370
    },
    {
      "epoch": 2.8360569319978914,
      "grad_norm": 8.145801135046169,
      "learning_rate": 2.911175540326832e-07,
      "logits/chosen": -2.8316407203674316,
      "logits/rejected": -2.8851561546325684,
      "logps/chosen": -424.82501220703125,
      "logps/rejected": -441.70001220703125,
      "loss": 0.0289,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.153515577316284,
      "rewards/margins": 5.717968940734863,
      "rewards/rejected": -7.877343654632568,
      "step": 5380
    },
    {
      "epoch": 2.841328413284133,
      "grad_norm": 32.59207457816861,
      "learning_rate": 2.897996837111228e-07,
      "logits/chosen": -2.9927735328674316,
      "logits/rejected": -2.956249952316284,
      "logps/chosen": -411.29998779296875,
      "logps/rejected": -407.6499938964844,
      "loss": 0.0447,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.2417235374450684,
      "rewards/margins": 5.069921970367432,
      "rewards/rejected": -7.312890529632568,
      "step": 5390
    },
    {
      "epoch": 2.8465998945703745,
      "grad_norm": 52.5358338061678,
      "learning_rate": 2.884818133895625e-07,
      "logits/chosen": -2.734179735183716,
      "logits/rejected": -2.887500047683716,
      "logps/chosen": -371.6875,
      "logps/rejected": -425.7749938964844,
      "loss": 0.0374,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.0027832984924316,
      "rewards/margins": 5.819531440734863,
      "rewards/rejected": -7.828906059265137,
      "step": 5400
    },
    {
      "epoch": 2.8518713758566157,
      "grad_norm": 66.11711623612116,
      "learning_rate": 2.871639430680021e-07,
      "logits/chosen": -2.8330078125,
      "logits/rejected": -2.9117188453674316,
      "logps/chosen": -406.45001220703125,
      "logps/rejected": -405.25,
      "loss": 0.041,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.031543016433716,
      "rewards/margins": 4.98046875,
      "rewards/rejected": -7.013281345367432,
      "step": 5410
    },
    {
      "epoch": 2.857142857142857,
      "grad_norm": 36.95321896606931,
      "learning_rate": 2.858460727464417e-07,
      "logits/chosen": -2.866406202316284,
      "logits/rejected": -2.9164061546325684,
      "logps/chosen": -382.88751220703125,
      "logps/rejected": -407.625,
      "loss": 0.0288,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.946044921875,
      "rewards/margins": 5.337109565734863,
      "rewards/rejected": -7.282031059265137,
      "step": 5420
    },
    {
      "epoch": 2.8624143384290983,
      "grad_norm": 15.568449150348481,
      "learning_rate": 2.845282024248814e-07,
      "logits/chosen": -2.861132860183716,
      "logits/rejected": -2.8656249046325684,
      "logps/chosen": -426.9750061035156,
      "logps/rejected": -442.25,
      "loss": 0.0546,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.145800828933716,
      "rewards/margins": 5.264843940734863,
      "rewards/rejected": -7.408593654632568,
      "step": 5430
    },
    {
      "epoch": 2.86768581971534,
      "grad_norm": 8.504406493313956,
      "learning_rate": 2.83210332103321e-07,
      "logits/chosen": -2.78515625,
      "logits/rejected": -2.908007860183716,
      "logps/chosen": -424.29998779296875,
      "logps/rejected": -431.95001220703125,
      "loss": 0.0367,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.0493531227111816,
      "rewards/margins": 5.844140529632568,
      "rewards/rejected": -7.891406059265137,
      "step": 5440
    },
    {
      "epoch": 2.8729573010015814,
      "grad_norm": 31.51909881613036,
      "learning_rate": 2.8189246178176067e-07,
      "logits/chosen": -2.852343797683716,
      "logits/rejected": -2.9339842796325684,
      "logps/chosen": -452.95001220703125,
      "logps/rejected": -456.7749938964844,
      "loss": 0.031,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.0555176734924316,
      "rewards/margins": 5.727343559265137,
      "rewards/rejected": -7.779687404632568,
      "step": 5450
    },
    {
      "epoch": 2.878228782287823,
      "grad_norm": 16.619694276631424,
      "learning_rate": 2.805745914602003e-07,
      "logits/chosen": -2.885937452316284,
      "logits/rejected": -2.908203125,
      "logps/chosen": -417.9750061035156,
      "logps/rejected": -450.3999938964844,
      "loss": 0.047,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.197216749191284,
      "rewards/margins": 5.460546970367432,
      "rewards/rejected": -7.658593654632568,
      "step": 5460
    },
    {
      "epoch": 2.8835002635740645,
      "grad_norm": 36.57610264314854,
      "learning_rate": 2.7925672113863996e-07,
      "logits/chosen": -2.8267579078674316,
      "logits/rejected": -2.8441405296325684,
      "logps/chosen": -397.875,
      "logps/rejected": -380.67498779296875,
      "loss": 0.0486,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.161328077316284,
      "rewards/margins": 5.3828125,
      "rewards/rejected": -7.541406154632568,
      "step": 5470
    },
    {
      "epoch": 2.8887717448603056,
      "grad_norm": 4.090907596598056,
      "learning_rate": 2.7793885081707956e-07,
      "logits/chosen": -2.825000047683716,
      "logits/rejected": -2.939453125,
      "logps/chosen": -477.25,
      "logps/rejected": -457.625,
      "loss": 0.0448,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.2237305641174316,
      "rewards/margins": 5.744336128234863,
      "rewards/rejected": -7.965624809265137,
      "step": 5480
    },
    {
      "epoch": 2.894043226146547,
      "grad_norm": 23.83427029112007,
      "learning_rate": 2.7662098049551926e-07,
      "logits/chosen": -2.8275389671325684,
      "logits/rejected": -2.859570264816284,
      "logps/chosen": -409.1000061035156,
      "logps/rejected": -416.45001220703125,
      "loss": 0.0627,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.4622559547424316,
      "rewards/margins": 5.335156440734863,
      "rewards/rejected": -7.793749809265137,
      "step": 5490
    },
    {
      "epoch": 2.8993147074327887,
      "grad_norm": 10.029137897944123,
      "learning_rate": 2.7530311017395885e-07,
      "logits/chosen": -2.8095703125,
      "logits/rejected": -2.919921875,
      "logps/chosen": -391.79998779296875,
      "logps/rejected": -412.5,
      "loss": 0.0434,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.3560547828674316,
      "rewards/margins": 5.439453125,
      "rewards/rejected": -7.794531345367432,
      "step": 5500
    },
    {
      "epoch": 2.90458618871903,
      "grad_norm": 14.612330106191875,
      "learning_rate": 2.739852398523985e-07,
      "logits/chosen": -2.7826170921325684,
      "logits/rejected": -2.849414110183716,
      "logps/chosen": -418.3999938964844,
      "logps/rejected": -429.4750061035156,
      "loss": 0.0403,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.7232177257537842,
      "rewards/margins": 5.738671779632568,
      "rewards/rejected": -7.4609375,
      "step": 5510
    },
    {
      "epoch": 2.9098576700052714,
      "grad_norm": 23.628829766590574,
      "learning_rate": 2.7266736953083815e-07,
      "logits/chosen": -2.8822264671325684,
      "logits/rejected": -2.9085936546325684,
      "logps/chosen": -368.29998779296875,
      "logps/rejected": -432.0,
      "loss": 0.0728,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.6197021007537842,
      "rewards/margins": 5.387499809265137,
      "rewards/rejected": -7.007031440734863,
      "step": 5520
    },
    {
      "epoch": 2.915129151291513,
      "grad_norm": 37.87695428007151,
      "learning_rate": 2.713494992092778e-07,
      "logits/chosen": -2.6507811546325684,
      "logits/rejected": -2.734375,
      "logps/chosen": -391.2250061035156,
      "logps/rejected": -421.7749938964844,
      "loss": 0.051,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.755957007408142,
      "rewards/margins": 5.677734375,
      "rewards/rejected": -7.432031154632568,
      "step": 5530
    },
    {
      "epoch": 2.9204006325777545,
      "grad_norm": 4.406395744343251,
      "learning_rate": 2.700316288877174e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.9013671875,
      "logps/chosen": -394.70001220703125,
      "logps/rejected": -438.70001220703125,
      "loss": 0.0215,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.3104491233825684,
      "rewards/margins": 5.864843845367432,
      "rewards/rejected": -8.178125381469727,
      "step": 5540
    },
    {
      "epoch": 2.925672113863996,
      "grad_norm": 58.86879349023922,
      "learning_rate": 2.687137585661571e-07,
      "logits/chosen": -2.961718797683716,
      "logits/rejected": -2.9117188453674316,
      "logps/chosen": -392.8999938964844,
      "logps/rejected": -422.92498779296875,
      "loss": 0.0684,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.252734422683716,
      "rewards/margins": 5.360547065734863,
      "rewards/rejected": -7.611718654632568,
      "step": 5550
    },
    {
      "epoch": 2.930943595150237,
      "grad_norm": 36.89655724001556,
      "learning_rate": 2.6739588824459674e-07,
      "logits/chosen": -2.9189453125,
      "logits/rejected": -2.891406297683716,
      "logps/chosen": -398.6000061035156,
      "logps/rejected": -442.2250061035156,
      "loss": 0.0616,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.971704125404358,
      "rewards/margins": 5.250390529632568,
      "rewards/rejected": -7.217968940734863,
      "step": 5560
    },
    {
      "epoch": 2.9362150764364787,
      "grad_norm": 7.761495235795378,
      "learning_rate": 2.660780179230364e-07,
      "logits/chosen": -2.811718702316284,
      "logits/rejected": -2.8851561546325684,
      "logps/chosen": -389.2749938964844,
      "logps/rejected": -414.4750061035156,
      "loss": 0.058,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.9677855968475342,
      "rewards/margins": 5.564062595367432,
      "rewards/rejected": -7.532812595367432,
      "step": 5570
    },
    {
      "epoch": 2.94148655772272,
      "grad_norm": 11.02168060254254,
      "learning_rate": 2.6476014760147603e-07,
      "logits/chosen": -2.8310546875,
      "logits/rejected": -2.9332032203674316,
      "logps/chosen": -378.36248779296875,
      "logps/rejected": -377.6499938964844,
      "loss": 0.0336,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.2839598655700684,
      "rewards/margins": 5.454297065734863,
      "rewards/rejected": -7.739843845367432,
      "step": 5580
    },
    {
      "epoch": 2.9467580390089614,
      "grad_norm": 62.8192565971031,
      "learning_rate": 2.634422772799156e-07,
      "logits/chosen": -2.837695360183716,
      "logits/rejected": -2.8863282203674316,
      "logps/chosen": -374.82501220703125,
      "logps/rejected": -420.70001220703125,
      "loss": 0.0642,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.404980421066284,
      "rewards/margins": 5.421093940734863,
      "rewards/rejected": -7.827343940734863,
      "step": 5590
    },
    {
      "epoch": 2.952029520295203,
      "grad_norm": 29.472278228724505,
      "learning_rate": 2.6212440695835533e-07,
      "logits/chosen": -2.9039063453674316,
      "logits/rejected": -2.914257764816284,
      "logps/chosen": -374.9125061035156,
      "logps/rejected": -440.4750061035156,
      "loss": 0.0328,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.175048828125,
      "rewards/margins": 5.620312690734863,
      "rewards/rejected": -7.795312404632568,
      "step": 5600
    },
    {
      "epoch": 2.9573010015814445,
      "grad_norm": 40.71876369828337,
      "learning_rate": 2.608065366367949e-07,
      "logits/chosen": -2.810546875,
      "logits/rejected": -2.9203124046325684,
      "logps/chosen": -382.8125,
      "logps/rejected": -424.0249938964844,
      "loss": 0.0369,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.1965088844299316,
      "rewards/margins": 5.49609375,
      "rewards/rejected": -7.697656154632568,
      "step": 5610
    },
    {
      "epoch": 2.962572482867686,
      "grad_norm": 2.4696030094032633,
      "learning_rate": 2.5948866631523457e-07,
      "logits/chosen": -2.7261719703674316,
      "logits/rejected": -2.8623046875,
      "logps/chosen": -433.17498779296875,
      "logps/rejected": -434.3500061035156,
      "loss": 0.0675,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.986883521080017,
      "rewards/margins": 5.608593940734863,
      "rewards/rejected": -7.599218845367432,
      "step": 5620
    },
    {
      "epoch": 2.967843964153927,
      "grad_norm": 32.40943537244626,
      "learning_rate": 2.581707959936742e-07,
      "logits/chosen": -2.8714842796325684,
      "logits/rejected": -2.9097657203674316,
      "logps/chosen": -403.29998779296875,
      "logps/rejected": -448.1499938964844,
      "loss": 0.0439,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.9969604015350342,
      "rewards/margins": 5.356640815734863,
      "rewards/rejected": -7.360156059265137,
      "step": 5630
    },
    {
      "epoch": 2.9731154454401687,
      "grad_norm": 61.830106160896435,
      "learning_rate": 2.5685292567211386e-07,
      "logits/chosen": -2.69921875,
      "logits/rejected": -2.873828172683716,
      "logps/chosen": -435.73748779296875,
      "logps/rejected": -467.07501220703125,
      "loss": 0.0323,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.318798780441284,
      "rewards/margins": 5.81640625,
      "rewards/rejected": -8.132031440734863,
      "step": 5640
    },
    {
      "epoch": 2.9783869267264103,
      "grad_norm": 56.036818806147096,
      "learning_rate": 2.5553505535055346e-07,
      "logits/chosen": -2.992968797683716,
      "logits/rejected": -3.059375047683716,
      "logps/chosen": -344.375,
      "logps/rejected": -388.95001220703125,
      "loss": 0.0489,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.588366746902466,
      "rewards/margins": 5.369140625,
      "rewards/rejected": -7.953125,
      "step": 5650
    },
    {
      "epoch": 2.9836584080126514,
      "grad_norm": 16.869976532750265,
      "learning_rate": 2.5421718502899316e-07,
      "logits/chosen": -2.8349609375,
      "logits/rejected": -2.861523389816284,
      "logps/chosen": -400.73748779296875,
      "logps/rejected": -404.6499938964844,
      "loss": 0.0442,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.982177734375,
      "rewards/margins": 5.3828125,
      "rewards/rejected": -7.365234375,
      "step": 5660
    },
    {
      "epoch": 2.988929889298893,
      "grad_norm": 12.012776714004579,
      "learning_rate": 2.5289931470743275e-07,
      "logits/chosen": -2.905468702316284,
      "logits/rejected": -2.887500047683716,
      "logps/chosen": -409.8999938964844,
      "logps/rejected": -432.6000061035156,
      "loss": 0.043,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.885107398033142,
      "rewards/margins": 5.619531154632568,
      "rewards/rejected": -7.5,
      "step": 5670
    },
    {
      "epoch": 2.9942013705851345,
      "grad_norm": 12.011273774738266,
      "learning_rate": 2.515814443858724e-07,
      "logits/chosen": -2.695507764816284,
      "logits/rejected": -2.8257813453674316,
      "logps/chosen": -420.7749938964844,
      "logps/rejected": -470.17498779296875,
      "loss": 0.0806,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.0364747047424316,
      "rewards/margins": 6.056640625,
      "rewards/rejected": -8.099218368530273,
      "step": 5680
    },
    {
      "epoch": 2.999472851871376,
      "grad_norm": 17.162320099489012,
      "learning_rate": 2.5026357406431205e-07,
      "logits/chosen": -2.737499952316284,
      "logits/rejected": -2.771679639816284,
      "logps/chosen": -414.2250061035156,
      "logps/rejected": -463.7749938964844,
      "loss": 0.0308,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.905236840248108,
      "rewards/margins": 5.603906154632568,
      "rewards/rejected": -7.510937690734863,
      "step": 5690
    },
    {
      "epoch": 3.004744333157617,
      "grad_norm": 9.56513792043428,
      "learning_rate": 2.489457037427517e-07,
      "logits/chosen": -2.969531297683716,
      "logits/rejected": -2.9867186546325684,
      "logps/chosen": -430.45001220703125,
      "logps/rejected": -475.45001220703125,
      "loss": 0.0115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.739953637123108,
      "rewards/margins": 5.98828125,
      "rewards/rejected": -7.73046875,
      "step": 5700
    },
    {
      "epoch": 3.0100158144438587,
      "grad_norm": 8.559804643583002,
      "learning_rate": 2.4762783342119134e-07,
      "logits/chosen": -2.9150390625,
      "logits/rejected": -2.9781250953674316,
      "logps/chosen": -393.70001220703125,
      "logps/rejected": -432.7250061035156,
      "loss": 0.013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.0222535133361816,
      "rewards/margins": 6.151562690734863,
      "rewards/rejected": -8.171093940734863,
      "step": 5710
    },
    {
      "epoch": 3.0152872957301002,
      "grad_norm": 16.924020675057438,
      "learning_rate": 2.46309963099631e-07,
      "logits/chosen": -2.874218702316284,
      "logits/rejected": -2.955078125,
      "logps/chosen": -404.20001220703125,
      "logps/rejected": -438.70001220703125,
      "loss": 0.0102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.6884522438049316,
      "rewards/margins": 6.109765529632568,
      "rewards/rejected": -8.796093940734863,
      "step": 5720
    },
    {
      "epoch": 3.020558777016342,
      "grad_norm": 3.149237415252699,
      "learning_rate": 2.4499209277807064e-07,
      "logits/chosen": -2.7935547828674316,
      "logits/rejected": -2.8871092796325684,
      "logps/chosen": -457.25,
      "logps/rejected": -471.7250061035156,
      "loss": 0.0139,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.26953125,
      "rewards/margins": 6.530468940734863,
      "rewards/rejected": -8.798437118530273,
      "step": 5730
    },
    {
      "epoch": 3.025830258302583,
      "grad_norm": 7.33490079952825,
      "learning_rate": 2.4367422245651023e-07,
      "logits/chosen": -2.837109327316284,
      "logits/rejected": -3.0250000953674316,
      "logps/chosen": -485.29998779296875,
      "logps/rejected": -512.1749877929688,
      "loss": 0.0063,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.358245849609375,
      "rewards/margins": 6.534375190734863,
      "rewards/rejected": -8.889062881469727,
      "step": 5740
    },
    {
      "epoch": 3.0311017395888245,
      "grad_norm": 2.304112728874411,
      "learning_rate": 2.423563521349499e-07,
      "logits/chosen": -2.967968702316284,
      "logits/rejected": -2.9457030296325684,
      "logps/chosen": -438.92498779296875,
      "logps/rejected": -468.8500061035156,
      "loss": 0.0159,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9187500476837158,
      "rewards/margins": 6.8359375,
      "rewards/rejected": -8.753125190734863,
      "step": 5750
    },
    {
      "epoch": 3.036373220875066,
      "grad_norm": 2.9910962775638272,
      "learning_rate": 2.4103848181338953e-07,
      "logits/chosen": -2.9124999046325684,
      "logits/rejected": -3.0140624046325684,
      "logps/chosen": -409.7749938964844,
      "logps/rejected": -415.375,
      "loss": 0.015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.6714844703674316,
      "rewards/margins": 6.506249904632568,
      "rewards/rejected": -9.181249618530273,
      "step": 5760
    },
    {
      "epoch": 3.041644702161307,
      "grad_norm": 6.662782117937364,
      "learning_rate": 2.3972061149182923e-07,
      "logits/chosen": -2.877734422683716,
      "logits/rejected": -2.9105467796325684,
      "logps/chosen": -430.67498779296875,
      "logps/rejected": -455.92498779296875,
      "loss": 0.0089,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.542773485183716,
      "rewards/margins": 6.543749809265137,
      "rewards/rejected": -9.08984375,
      "step": 5770
    },
    {
      "epoch": 3.0469161834475487,
      "grad_norm": 4.132370302135704,
      "learning_rate": 2.3840274117026885e-07,
      "logits/chosen": -2.860156297683716,
      "logits/rejected": -2.919140577316284,
      "logps/chosen": -468.2749938964844,
      "logps/rejected": -481.29998779296875,
      "loss": 0.0093,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.7984375953674316,
      "rewards/margins": 6.579687595367432,
      "rewards/rejected": -9.380468368530273,
      "step": 5780
    },
    {
      "epoch": 3.05218766473379,
      "grad_norm": 35.125835065657924,
      "learning_rate": 2.370848708487085e-07,
      "logits/chosen": -2.9107422828674316,
      "logits/rejected": -3.0455079078674316,
      "logps/chosen": -401.82501220703125,
      "logps/rejected": -423.45001220703125,
      "loss": 0.0109,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.657031297683716,
      "rewards/margins": 6.5390625,
      "rewards/rejected": -9.196093559265137,
      "step": 5790
    },
    {
      "epoch": 3.0574591460200318,
      "grad_norm": 13.694783466006315,
      "learning_rate": 2.3576700052714812e-07,
      "logits/chosen": -2.943359375,
      "logits/rejected": -3.057812452316284,
      "logps/chosen": -401.0249938964844,
      "logps/rejected": -399.82501220703125,
      "loss": 0.0109,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.3919920921325684,
      "rewards/margins": 6.669531345367432,
      "rewards/rejected": -9.064062118530273,
      "step": 5800
    },
    {
      "epoch": 3.062730627306273,
      "grad_norm": 134.91854458911718,
      "learning_rate": 2.3444913020558777e-07,
      "logits/chosen": -2.900390625,
      "logits/rejected": -3.0374999046325684,
      "logps/chosen": -420.95001220703125,
      "logps/rejected": -424.625,
      "loss": 0.0172,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.9945311546325684,
      "rewards/margins": 6.632031440734863,
      "rewards/rejected": -9.62890625,
      "step": 5810
    },
    {
      "epoch": 3.0680021085925144,
      "grad_norm": 3.9875259549017734,
      "learning_rate": 2.3313125988402741e-07,
      "logits/chosen": -2.987109422683716,
      "logits/rejected": -3.0794920921325684,
      "logps/chosen": -417.82501220703125,
      "logps/rejected": -438.875,
      "loss": 0.0085,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.5050597190856934,
      "rewards/margins": 6.803906440734863,
      "rewards/rejected": -9.310937881469727,
      "step": 5820
    },
    {
      "epoch": 3.073273589878756,
      "grad_norm": 8.931824768410003,
      "learning_rate": 2.3181338956246703e-07,
      "logits/chosen": -2.9078125953674316,
      "logits/rejected": -2.987109422683716,
      "logps/chosen": -380.57501220703125,
      "logps/rejected": -436.1499938964844,
      "loss": 0.0123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.9391846656799316,
      "rewards/margins": 6.735937595367432,
      "rewards/rejected": -9.674219131469727,
      "step": 5830
    },
    {
      "epoch": 3.0785450711649975,
      "grad_norm": 2.8109461442794363,
      "learning_rate": 2.3049551924090668e-07,
      "logits/chosen": -2.8701171875,
      "logits/rejected": -3.0230469703674316,
      "logps/chosen": -424.375,
      "logps/rejected": -417.42498779296875,
      "loss": 0.0168,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.469653367996216,
      "rewards/margins": 6.567187309265137,
      "rewards/rejected": -9.043749809265137,
      "step": 5840
    },
    {
      "epoch": 3.0838165524512386,
      "grad_norm": 6.0916461964039375,
      "learning_rate": 2.2917764891934633e-07,
      "logits/chosen": -2.9619140625,
      "logits/rejected": -3.0550780296325684,
      "logps/chosen": -425.92498779296875,
      "logps/rejected": -442.75,
      "loss": 0.0112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.251025438308716,
      "rewards/margins": 6.474218845367432,
      "rewards/rejected": -8.725781440734863,
      "step": 5850
    },
    {
      "epoch": 3.08908803373748,
      "grad_norm": 15.801999367200871,
      "learning_rate": 2.2785977859778595e-07,
      "logits/chosen": -2.9833984375,
      "logits/rejected": -2.9765625,
      "logps/chosen": -418.75,
      "logps/rejected": -470.92498779296875,
      "loss": 0.0336,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.119824171066284,
      "rewards/margins": 6.895898342132568,
      "rewards/rejected": -10.020312309265137,
      "step": 5860
    },
    {
      "epoch": 3.0943595150237218,
      "grad_norm": 2.957445721389348,
      "learning_rate": 2.265419082762256e-07,
      "logits/chosen": -2.8246092796325684,
      "logits/rejected": -2.907031297683716,
      "logps/chosen": -400.875,
      "logps/rejected": -441.42498779296875,
      "loss": 0.0186,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.834765672683716,
      "rewards/margins": 6.789843559265137,
      "rewards/rejected": -9.621874809265137,
      "step": 5870
    },
    {
      "epoch": 3.0996309963099633,
      "grad_norm": 15.87599536502702,
      "learning_rate": 2.2522403795466525e-07,
      "logits/chosen": -2.9398436546325684,
      "logits/rejected": -2.967578172683716,
      "logps/chosen": -436.3500061035156,
      "logps/rejected": -466.17498779296875,
      "loss": 0.008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.76953125,
      "rewards/margins": 6.889843940734863,
      "rewards/rejected": -9.666406631469727,
      "step": 5880
    },
    {
      "epoch": 3.1049024775962044,
      "grad_norm": 10.795925509816033,
      "learning_rate": 2.239061676331049e-07,
      "logits/chosen": -2.9634766578674316,
      "logits/rejected": -3.0107421875,
      "logps/chosen": -373.6499938964844,
      "logps/rejected": -424.04998779296875,
      "loss": 0.0122,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.783886671066284,
      "rewards/margins": 6.42578125,
      "rewards/rejected": -9.210156440734863,
      "step": 5890
    },
    {
      "epoch": 3.110173958882446,
      "grad_norm": 14.135667058596123,
      "learning_rate": 2.2258829731154451e-07,
      "logits/chosen": -2.963671922683716,
      "logits/rejected": -2.9917969703674316,
      "logps/chosen": -439.17498779296875,
      "logps/rejected": -454.7749938964844,
      "loss": 0.0159,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.71099853515625,
      "rewards/margins": 6.5703125,
      "rewards/rejected": -9.28515625,
      "step": 5900
    },
    {
      "epoch": 3.1154454401686875,
      "grad_norm": 2.65778117251665,
      "learning_rate": 2.212704269899842e-07,
      "logits/chosen": -3.1175780296325684,
      "logits/rejected": -3.1292967796325684,
      "logps/chosen": -435.75,
      "logps/rejected": -464.45001220703125,
      "loss": 0.0095,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.959179639816284,
      "rewards/margins": 6.888281345367432,
      "rewards/rejected": -9.842968940734863,
      "step": 5910
    },
    {
      "epoch": 3.1207169214549286,
      "grad_norm": 4.290248470912423,
      "learning_rate": 2.1995255666842384e-07,
      "logits/chosen": -2.9384765625,
      "logits/rejected": -2.9710936546325684,
      "logps/chosen": -427.4375,
      "logps/rejected": -462.42498779296875,
      "loss": 0.0136,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.58984375,
      "rewards/margins": 6.659375190734863,
      "rewards/rejected": -9.247655868530273,
      "step": 5920
    },
    {
      "epoch": 3.12598840274117,
      "grad_norm": 4.857983595410003,
      "learning_rate": 2.1863468634686346e-07,
      "logits/chosen": -2.7734375,
      "logits/rejected": -3.0093750953674316,
      "logps/chosen": -433.92498779296875,
      "logps/rejected": -470.25,
      "loss": 0.0147,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.820507764816284,
      "rewards/margins": 6.81640625,
      "rewards/rejected": -9.631250381469727,
      "step": 5930
    },
    {
      "epoch": 3.1312598840274117,
      "grad_norm": 22.87166403617284,
      "learning_rate": 2.173168160253031e-07,
      "logits/chosen": -2.9253907203674316,
      "logits/rejected": -2.998828172683716,
      "logps/chosen": -402.29998779296875,
      "logps/rejected": -436.9750061035156,
      "loss": 0.0109,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.92529296875,
      "rewards/margins": 6.733593940734863,
      "rewards/rejected": -9.650781631469727,
      "step": 5940
    },
    {
      "epoch": 3.1365313653136533,
      "grad_norm": 5.366949590775416,
      "learning_rate": 2.1599894570374275e-07,
      "logits/chosen": -3.039843797683716,
      "logits/rejected": -3.0542969703674316,
      "logps/chosen": -394.20001220703125,
      "logps/rejected": -460.0249938964844,
      "loss": 0.0081,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.3072752952575684,
      "rewards/margins": 6.599218845367432,
      "rewards/rejected": -8.903905868530273,
      "step": 5950
    },
    {
      "epoch": 3.1418028465998944,
      "grad_norm": 11.451203147622229,
      "learning_rate": 2.1468107538218237e-07,
      "logits/chosen": -2.801562547683716,
      "logits/rejected": -2.9417967796325684,
      "logps/chosen": -398.3999938964844,
      "logps/rejected": -441.9750061035156,
      "loss": 0.0116,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.759082078933716,
      "rewards/margins": 6.447656154632568,
      "rewards/rejected": -9.202343940734863,
      "step": 5960
    },
    {
      "epoch": 3.147074327886136,
      "grad_norm": 3.4533565099524828,
      "learning_rate": 2.1336320506062202e-07,
      "logits/chosen": -3.036328077316284,
      "logits/rejected": -3.157031297683716,
      "logps/chosen": -393.95001220703125,
      "logps/rejected": -431.4750061035156,
      "loss": 0.01,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.075756788253784,
      "rewards/margins": 6.6875,
      "rewards/rejected": -9.767187118530273,
      "step": 5970
    },
    {
      "epoch": 3.1523458091723775,
      "grad_norm": 9.398085449640558,
      "learning_rate": 2.1204533473906167e-07,
      "logits/chosen": -2.9390625953674316,
      "logits/rejected": -3.006640672683716,
      "logps/chosen": -420.8500061035156,
      "logps/rejected": -451.6499938964844,
      "loss": 0.0113,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.8148436546325684,
      "rewards/margins": 6.425000190734863,
      "rewards/rejected": -9.241796493530273,
      "step": 5980
    },
    {
      "epoch": 3.157617290458619,
      "grad_norm": 3.592166980420666,
      "learning_rate": 2.1072746441750132e-07,
      "logits/chosen": -2.975390672683716,
      "logits/rejected": -3.0884766578674316,
      "logps/chosen": -409.9750061035156,
      "logps/rejected": -447.95001220703125,
      "loss": 0.0066,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.4693360328674316,
      "rewards/margins": 7.009375095367432,
      "rewards/rejected": -9.477343559265137,
      "step": 5990
    },
    {
      "epoch": 3.16288877174486,
      "grad_norm": 29.51572633030105,
      "learning_rate": 2.0940959409594094e-07,
      "logits/chosen": -2.8798828125,
      "logits/rejected": -2.8646483421325684,
      "logps/chosen": -398.7749938964844,
      "logps/rejected": -425.67498779296875,
      "loss": 0.0109,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.6276612281799316,
      "rewards/margins": 6.467187404632568,
      "rewards/rejected": -9.09375,
      "step": 6000
    },
    {
      "epoch": 3.1681602530311017,
      "grad_norm": 8.541939966388064,
      "learning_rate": 2.0809172377438058e-07,
      "logits/chosen": -2.8033204078674316,
      "logits/rejected": -2.955273389816284,
      "logps/chosen": -463.8999938964844,
      "logps/rejected": -424.125,
      "loss": 0.0108,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.741943359375,
      "rewards/margins": 6.265625,
      "rewards/rejected": -9.01171875,
      "step": 6010
    },
    {
      "epoch": 3.1734317343173433,
      "grad_norm": 4.702772560833678,
      "learning_rate": 2.0677385345282023e-07,
      "logits/chosen": -3.034374952316284,
      "logits/rejected": -3.049609422683716,
      "logps/chosen": -360.3500061035156,
      "logps/rejected": -428.4750061035156,
      "loss": 0.0114,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.644482374191284,
      "rewards/margins": 6.956250190734863,
      "rewards/rejected": -9.607030868530273,
      "step": 6020
    },
    {
      "epoch": 3.1787032156035844,
      "grad_norm": 7.279762631031457,
      "learning_rate": 2.0545598313125985e-07,
      "logits/chosen": -2.9917969703674316,
      "logits/rejected": -3.018749952316284,
      "logps/chosen": -391.2250061035156,
      "logps/rejected": -444.9750061035156,
      "loss": 0.0204,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.624523878097534,
      "rewards/margins": 6.573437690734863,
      "rewards/rejected": -9.200780868530273,
      "step": 6030
    },
    {
      "epoch": 3.183974696889826,
      "grad_norm": 1.7700427707422017,
      "learning_rate": 2.041381128096995e-07,
      "logits/chosen": -2.977734327316284,
      "logits/rejected": -3.0726561546325684,
      "logps/chosen": -420.5249938964844,
      "logps/rejected": -431.79998779296875,
      "loss": 0.0297,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.7789063453674316,
      "rewards/margins": 7.207812309265137,
      "rewards/rejected": -9.985937118530273,
      "step": 6040
    },
    {
      "epoch": 3.1892461781760675,
      "grad_norm": 5.248861663741007,
      "learning_rate": 2.0282024248813917e-07,
      "logits/chosen": -2.9623045921325684,
      "logits/rejected": -3.0609374046325684,
      "logps/chosen": -384.57501220703125,
      "logps/rejected": -421.5,
      "loss": 0.0139,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.8956542015075684,
      "rewards/margins": 6.510937690734863,
      "rewards/rejected": -9.404687881469727,
      "step": 6050
    },
    {
      "epoch": 3.194517659462309,
      "grad_norm": 1.6536983063221848,
      "learning_rate": 2.0150237216657882e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.0386719703674316,
      "logps/chosen": -423.88751220703125,
      "logps/rejected": -430.32501220703125,
      "loss": 0.0054,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.56787109375,
      "rewards/margins": 6.839062690734863,
      "rewards/rejected": -9.407812118530273,
      "step": 6060
    },
    {
      "epoch": 3.19978914074855,
      "grad_norm": 4.465482925650517,
      "learning_rate": 2.0018450184501844e-07,
      "logits/chosen": -2.891796827316284,
      "logits/rejected": -3.0648436546325684,
      "logps/chosen": -449.36248779296875,
      "logps/rejected": -452.5,
      "loss": 0.0072,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.0429930686950684,
      "rewards/margins": 6.830468654632568,
      "rewards/rejected": -9.875781059265137,
      "step": 6070
    },
    {
      "epoch": 3.2050606220347917,
      "grad_norm": 6.500555010590176,
      "learning_rate": 1.988666315234581e-07,
      "logits/chosen": -3.003124952316284,
      "logits/rejected": -3.0953125953674316,
      "logps/chosen": -434.2749938964844,
      "logps/rejected": -439.6000061035156,
      "loss": 0.0103,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.68359375,
      "rewards/margins": 6.411328315734863,
      "rewards/rejected": -9.094531059265137,
      "step": 6080
    },
    {
      "epoch": 3.2103321033210332,
      "grad_norm": 43.33375357544977,
      "learning_rate": 1.9754876120189774e-07,
      "logits/chosen": -2.953906297683716,
      "logits/rejected": -2.9898438453674316,
      "logps/chosen": -408.875,
      "logps/rejected": -483.20001220703125,
      "loss": 0.017,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.025341749191284,
      "rewards/margins": 6.9375,
      "rewards/rejected": -9.967968940734863,
      "step": 6090
    },
    {
      "epoch": 3.215603584607275,
      "grad_norm": 4.598245949171116,
      "learning_rate": 1.9623089088033736e-07,
      "logits/chosen": -2.9292969703674316,
      "logits/rejected": -3.0478515625,
      "logps/chosen": -382.32501220703125,
      "logps/rejected": -431.7250061035156,
      "loss": 0.0104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.1470704078674316,
      "rewards/margins": 6.274218559265137,
      "rewards/rejected": -9.414843559265137,
      "step": 6100
    },
    {
      "epoch": 3.220875065893516,
      "grad_norm": 6.950726002375337,
      "learning_rate": 1.94913020558777e-07,
      "logits/chosen": -2.927734375,
      "logits/rejected": -3.056640625,
      "logps/chosen": -440.17498779296875,
      "logps/rejected": -438.1499938964844,
      "loss": 0.0103,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.153515577316284,
      "rewards/margins": 6.632031440734863,
      "rewards/rejected": -9.78515625,
      "step": 6110
    },
    {
      "epoch": 3.2261465471797575,
      "grad_norm": 20.12063972279568,
      "learning_rate": 1.9359515023721665e-07,
      "logits/chosen": -2.9310545921325684,
      "logits/rejected": -3.114453077316284,
      "logps/chosen": -430.42498779296875,
      "logps/rejected": -463.8500061035156,
      "loss": 0.0149,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.2777342796325684,
      "rewards/margins": 6.528906345367432,
      "rewards/rejected": -9.8046875,
      "step": 6120
    },
    {
      "epoch": 3.231418028465999,
      "grad_norm": 1.557779193923754,
      "learning_rate": 1.9227727991565628e-07,
      "logits/chosen": -2.9144530296325684,
      "logits/rejected": -2.9794921875,
      "logps/chosen": -485.7749938964844,
      "logps/rejected": -497.7250061035156,
      "loss": 0.0145,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.2586913108825684,
      "rewards/margins": 7.452343940734863,
      "rewards/rejected": -10.713281631469727,
      "step": 6130
    },
    {
      "epoch": 3.2366895097522406,
      "grad_norm": 21.72539579307352,
      "learning_rate": 1.9095940959409592e-07,
      "logits/chosen": -3.0433592796325684,
      "logits/rejected": -3.071484327316284,
      "logps/chosen": -418.2250061035156,
      "logps/rejected": -432.25,
      "loss": 0.0107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4649415016174316,
      "rewards/margins": 6.828125,
      "rewards/rejected": -10.292187690734863,
      "step": 6140
    },
    {
      "epoch": 3.2419609910384817,
      "grad_norm": 15.243366344543587,
      "learning_rate": 1.8964153927253557e-07,
      "logits/chosen": -2.9515624046325684,
      "logits/rejected": -3.019335985183716,
      "logps/chosen": -436.82501220703125,
      "logps/rejected": -444.6000061035156,
      "loss": 0.0092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.582470655441284,
      "rewards/margins": 6.8125,
      "rewards/rejected": -10.396875381469727,
      "step": 6150
    },
    {
      "epoch": 3.2472324723247232,
      "grad_norm": 15.461843892212697,
      "learning_rate": 1.8832366895097522e-07,
      "logits/chosen": -2.950000047683716,
      "logits/rejected": -3.0130858421325684,
      "logps/chosen": -438.79998779296875,
      "logps/rejected": -467.25,
      "loss": 0.0119,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4564452171325684,
      "rewards/margins": 6.91015625,
      "rewards/rejected": -10.370312690734863,
      "step": 6160
    },
    {
      "epoch": 3.252503953610965,
      "grad_norm": 8.02124314451471,
      "learning_rate": 1.8700579862941484e-07,
      "logits/chosen": -2.943554639816284,
      "logits/rejected": -3.0289063453674316,
      "logps/chosen": -410.2124938964844,
      "logps/rejected": -427.45001220703125,
      "loss": 0.0177,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.0235838890075684,
      "rewards/margins": 6.896093845367432,
      "rewards/rejected": -9.921875,
      "step": 6170
    },
    {
      "epoch": 3.257775434897206,
      "grad_norm": 20.781845891550894,
      "learning_rate": 1.856879283078545e-07,
      "logits/chosen": -3.0609374046325684,
      "logits/rejected": -3.0933594703674316,
      "logps/chosen": -430.1499938964844,
      "logps/rejected": -454.45001220703125,
      "loss": 0.0154,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.137890577316284,
      "rewards/margins": 6.209374904632568,
      "rewards/rejected": -9.337499618530273,
      "step": 6180
    },
    {
      "epoch": 3.2630469161834474,
      "grad_norm": 11.723859158437953,
      "learning_rate": 1.8437005798629416e-07,
      "logits/chosen": -2.976367235183716,
      "logits/rejected": -3.072265625,
      "logps/chosen": -382.8374938964844,
      "logps/rejected": -414.7749938964844,
      "loss": 0.0085,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.9231200218200684,
      "rewards/margins": 7.291406154632568,
      "rewards/rejected": -10.210156440734863,
      "step": 6190
    },
    {
      "epoch": 3.268318397469689,
      "grad_norm": 10.596931138954517,
      "learning_rate": 1.8305218766473378e-07,
      "logits/chosen": -3.01953125,
      "logits/rejected": -3.16015625,
      "logps/chosen": -413.2250061035156,
      "logps/rejected": -449.70001220703125,
      "loss": 0.0124,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.522656202316284,
      "rewards/margins": 6.875781059265137,
      "rewards/rejected": -10.399999618530273,
      "step": 6200
    },
    {
      "epoch": 3.2735898787559305,
      "grad_norm": 10.258303783557968,
      "learning_rate": 1.8173431734317343e-07,
      "logits/chosen": -2.8707032203674316,
      "logits/rejected": -2.9388670921325684,
      "logps/chosen": -415.79998779296875,
      "logps/rejected": -426.45001220703125,
      "loss": 0.0114,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.5285401344299316,
      "rewards/margins": 7.052343845367432,
      "rewards/rejected": -9.587499618530273,
      "step": 6210
    },
    {
      "epoch": 3.2788613600421717,
      "grad_norm": 6.357972405936254,
      "learning_rate": 1.8041644702161308e-07,
      "logits/chosen": -3.0396485328674316,
      "logits/rejected": -3.151171922683716,
      "logps/chosen": -422.04998779296875,
      "logps/rejected": -456.07501220703125,
      "loss": 0.0279,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.880859375,
      "rewards/margins": 6.551562309265137,
      "rewards/rejected": -9.434374809265137,
      "step": 6220
    },
    {
      "epoch": 3.284132841328413,
      "grad_norm": 1.4736643678255623,
      "learning_rate": 1.790985767000527e-07,
      "logits/chosen": -2.923046827316284,
      "logits/rejected": -3.0210938453674316,
      "logps/chosen": -421.4125061035156,
      "logps/rejected": -437.9750061035156,
      "loss": 0.0135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.019604444503784,
      "rewards/margins": 6.907812595367432,
      "rewards/rejected": -9.924219131469727,
      "step": 6230
    },
    {
      "epoch": 3.2894043226146548,
      "grad_norm": 2.8373593315865593,
      "learning_rate": 1.7778070637849235e-07,
      "logits/chosen": -2.9722657203674316,
      "logits/rejected": -2.9580078125,
      "logps/chosen": -458.7749938964844,
      "logps/rejected": -461.3500061035156,
      "loss": 0.0101,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.85009765625,
      "rewards/margins": 6.47265625,
      "rewards/rejected": -9.321093559265137,
      "step": 6240
    },
    {
      "epoch": 3.2946758039008963,
      "grad_norm": 6.667313299296099,
      "learning_rate": 1.76462836056932e-07,
      "logits/chosen": -2.9330077171325684,
      "logits/rejected": -3.0,
      "logps/chosen": -424.17498779296875,
      "logps/rejected": -446.3500061035156,
      "loss": 0.0114,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.2505860328674316,
      "rewards/margins": 6.778906345367432,
      "rewards/rejected": -10.028905868530273,
      "step": 6250
    },
    {
      "epoch": 3.2999472851871374,
      "grad_norm": 12.756970296837107,
      "learning_rate": 1.7514496573537164e-07,
      "logits/chosen": -2.9625000953674316,
      "logits/rejected": -3.107421875,
      "logps/chosen": -451.7250061035156,
      "logps/rejected": -473.95001220703125,
      "loss": 0.007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.2638182640075684,
      "rewards/margins": 7.133593559265137,
      "rewards/rejected": -10.392187118530273,
      "step": 6260
    },
    {
      "epoch": 3.305218766473379,
      "grad_norm": 4.3128828808026105,
      "learning_rate": 1.7382709541381126e-07,
      "logits/chosen": -2.930859327316284,
      "logits/rejected": -3.110546827316284,
      "logps/chosen": -456.20001220703125,
      "logps/rejected": -433.6499938964844,
      "loss": 0.0119,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.18701171875,
      "rewards/margins": 6.5703125,
      "rewards/rejected": -9.751562118530273,
      "step": 6270
    },
    {
      "epoch": 3.3104902477596205,
      "grad_norm": 20.983743787902704,
      "learning_rate": 1.725092250922509e-07,
      "logits/chosen": -3.028125047683716,
      "logits/rejected": -3.0445313453674316,
      "logps/chosen": -411.36248779296875,
      "logps/rejected": -447.29998779296875,
      "loss": 0.0277,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.99462890625,
      "rewards/margins": 7.389843940734863,
      "rewards/rejected": -10.3828125,
      "step": 6280
    },
    {
      "epoch": 3.315761729045862,
      "grad_norm": 23.814920999565356,
      "learning_rate": 1.7119135477069056e-07,
      "logits/chosen": -2.9996094703674316,
      "logits/rejected": -3.1039061546325684,
      "logps/chosen": -397.625,
      "logps/rejected": -455.6499938964844,
      "loss": 0.0128,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.994708299636841,
      "rewards/margins": 7.264062404632568,
      "rewards/rejected": -10.260937690734863,
      "step": 6290
    },
    {
      "epoch": 3.321033210332103,
      "grad_norm": 4.588604297481822,
      "learning_rate": 1.6987348444913018e-07,
      "logits/chosen": -3.024609327316284,
      "logits/rejected": -3.0824217796325684,
      "logps/chosen": -409.45001220703125,
      "logps/rejected": -455.8500061035156,
      "loss": 0.0075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.0736327171325684,
      "rewards/margins": 7.334374904632568,
      "rewards/rejected": -10.40625,
      "step": 6300
    },
    {
      "epoch": 3.3263046916183447,
      "grad_norm": 18.516424506163123,
      "learning_rate": 1.6855561412756983e-07,
      "logits/chosen": -3.0326170921325684,
      "logits/rejected": -3.078320264816284,
      "logps/chosen": -404.1000061035156,
      "logps/rejected": -498.3999938964844,
      "loss": 0.0073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.3602538108825684,
      "rewards/margins": 7.155468940734863,
      "rewards/rejected": -10.516406059265137,
      "step": 6310
    },
    {
      "epoch": 3.3315761729045863,
      "grad_norm": 4.585623286100726,
      "learning_rate": 1.6723774380600947e-07,
      "logits/chosen": -2.889843702316284,
      "logits/rejected": -3.010546922683716,
      "logps/chosen": -367.0249938964844,
      "logps/rejected": -426.625,
      "loss": 0.0097,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.168774366378784,
      "rewards/margins": 6.834374904632568,
      "rewards/rejected": -9.997655868530273,
      "step": 6320
    },
    {
      "epoch": 3.3368476541908274,
      "grad_norm": 19.912647452570656,
      "learning_rate": 1.6591987348444915e-07,
      "logits/chosen": -2.897265672683716,
      "logits/rejected": -3.024609327316284,
      "logps/chosen": -458.82501220703125,
      "logps/rejected": -469.54998779296875,
      "loss": 0.0169,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.047168016433716,
      "rewards/margins": 7.083593845367432,
      "rewards/rejected": -10.135156631469727,
      "step": 6330
    },
    {
      "epoch": 3.342119135477069,
      "grad_norm": 6.007714348684326,
      "learning_rate": 1.6460200316288877e-07,
      "logits/chosen": -2.96484375,
      "logits/rejected": -3.108593702316284,
      "logps/chosen": -408.67498779296875,
      "logps/rejected": -421.82501220703125,
      "loss": 0.0171,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.684863328933716,
      "rewards/margins": 6.975390434265137,
      "rewards/rejected": -9.660937309265137,
      "step": 6340
    },
    {
      "epoch": 3.3473906167633105,
      "grad_norm": 15.422109711678566,
      "learning_rate": 1.6328413284132842e-07,
      "logits/chosen": -3.0003905296325684,
      "logits/rejected": -3.0347657203674316,
      "logps/chosen": -408.1875,
      "logps/rejected": -442.1000061035156,
      "loss": 0.0185,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.8282227516174316,
      "rewards/margins": 6.651562690734863,
      "rewards/rejected": -9.475000381469727,
      "step": 6350
    },
    {
      "epoch": 3.352662098049552,
      "grad_norm": 3.4406848013978197,
      "learning_rate": 1.6196626251976806e-07,
      "logits/chosen": -3.0113282203674316,
      "logits/rejected": -3.05859375,
      "logps/chosen": -394.9624938964844,
      "logps/rejected": -449.04998779296875,
      "loss": 0.0085,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.0743165016174316,
      "rewards/margins": 6.887109279632568,
      "rewards/rejected": -9.965624809265137,
      "step": 6360
    },
    {
      "epoch": 3.357933579335793,
      "grad_norm": 3.248903440527941,
      "learning_rate": 1.6064839219820768e-07,
      "logits/chosen": -3.021289110183716,
      "logits/rejected": -3.155468702316284,
      "logps/chosen": -472.82501220703125,
      "logps/rejected": -454.17498779296875,
      "loss": 0.0095,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.10791015625,
      "rewards/margins": 6.891406059265137,
      "rewards/rejected": -10.003125190734863,
      "step": 6370
    },
    {
      "epoch": 3.3632050606220347,
      "grad_norm": 1.5988479874640122,
      "learning_rate": 1.5933052187664733e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -3.0816407203674316,
      "logps/chosen": -481.0,
      "logps/rejected": -477.67498779296875,
      "loss": 0.0083,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.133984327316284,
      "rewards/margins": 7.057812690734863,
      "rewards/rejected": -10.192968368530273,
      "step": 6380
    },
    {
      "epoch": 3.3684765419082763,
      "grad_norm": 6.078817637645651,
      "learning_rate": 1.5801265155508698e-07,
      "logits/chosen": -3.1363282203674316,
      "logits/rejected": -3.2562499046325684,
      "logps/chosen": -407.2124938964844,
      "logps/rejected": -438.1000061035156,
      "loss": 0.015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.5543212890625,
      "rewards/margins": 6.973437309265137,
      "rewards/rejected": -10.524218559265137,
      "step": 6390
    },
    {
      "epoch": 3.373748023194518,
      "grad_norm": 2.1305740252892362,
      "learning_rate": 1.566947812335266e-07,
      "logits/chosen": -3.1058592796325684,
      "logits/rejected": -3.0875000953674316,
      "logps/chosen": -440.92498779296875,
      "logps/rejected": -482.0,
      "loss": 0.0122,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.431884765625,
      "rewards/margins": 6.705468654632568,
      "rewards/rejected": -10.137499809265137,
      "step": 6400
    },
    {
      "epoch": 3.379019504480759,
      "grad_norm": 3.167990415553431,
      "learning_rate": 1.5537691091196625e-07,
      "logits/chosen": -3.096874952316284,
      "logits/rejected": -3.139843702316284,
      "logps/chosen": -412.57501220703125,
      "logps/rejected": -424.79998779296875,
      "loss": 0.0073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.7357420921325684,
      "rewards/margins": 7.134375095367432,
      "rewards/rejected": -10.864843368530273,
      "step": 6410
    },
    {
      "epoch": 3.3842909857670005,
      "grad_norm": 9.160463371253021,
      "learning_rate": 1.540590405904059e-07,
      "logits/chosen": -2.9798827171325684,
      "logits/rejected": -3.08984375,
      "logps/chosen": -433.75,
      "logps/rejected": -477.6499938964844,
      "loss": 0.0064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.519726514816284,
      "rewards/margins": 7.138281345367432,
      "rewards/rejected": -10.66015625,
      "step": 6420
    },
    {
      "epoch": 3.389562467053242,
      "grad_norm": 4.503123217776247,
      "learning_rate": 1.5274117026884554e-07,
      "logits/chosen": -3.0199217796325684,
      "logits/rejected": -3.049999952316284,
      "logps/chosen": -470.2749938964844,
      "logps/rejected": -498.25,
      "loss": 0.0156,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.1192383766174316,
      "rewards/margins": 6.612890720367432,
      "rewards/rejected": -9.728124618530273,
      "step": 6430
    },
    {
      "epoch": 3.3948339483394836,
      "grad_norm": 3.668397567238529,
      "learning_rate": 1.5142329994728516e-07,
      "logits/chosen": -2.9847655296325684,
      "logits/rejected": -3.099609375,
      "logps/chosen": -426.70001220703125,
      "logps/rejected": -451.8500061035156,
      "loss": 0.0083,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.7041015625,
      "rewards/margins": 6.912499904632568,
      "rewards/rejected": -9.6171875,
      "step": 6440
    },
    {
      "epoch": 3.4001054296257247,
      "grad_norm": 2.395429416482089,
      "learning_rate": 1.501054296257248e-07,
      "logits/chosen": -3.007031202316284,
      "logits/rejected": -3.0746092796325684,
      "logps/chosen": -424.6000061035156,
      "logps/rejected": -445.9750061035156,
      "loss": 0.008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.295605421066284,
      "rewards/margins": 6.701562404632568,
      "rewards/rejected": -9.991406440734863,
      "step": 6450
    },
    {
      "epoch": 3.4053769109119663,
      "grad_norm": 2.2150945929354156,
      "learning_rate": 1.4878755930416446e-07,
      "logits/chosen": -2.9828124046325684,
      "logits/rejected": -3.1546874046325684,
      "logps/chosen": -455.04998779296875,
      "logps/rejected": -451.42498779296875,
      "loss": 0.0088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.268359422683716,
      "rewards/margins": 6.982812404632568,
      "rewards/rejected": -10.250781059265137,
      "step": 6460
    },
    {
      "epoch": 3.410648392198208,
      "grad_norm": 34.1946099520775,
      "learning_rate": 1.474696889826041e-07,
      "logits/chosen": -3.073437452316284,
      "logits/rejected": -3.201953172683716,
      "logps/chosen": -408.4750061035156,
      "logps/rejected": -431.0,
      "loss": 0.0169,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.216015577316284,
      "rewards/margins": 6.51171875,
      "rewards/rejected": -9.739843368530273,
      "step": 6470
    },
    {
      "epoch": 3.415919873484449,
      "grad_norm": 64.88307664539212,
      "learning_rate": 1.4615181866104375e-07,
      "logits/chosen": -3.0054688453674316,
      "logits/rejected": -3.0582032203674316,
      "logps/chosen": -414.3999938964844,
      "logps/rejected": -469.82501220703125,
      "loss": 0.0165,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.215625047683716,
      "rewards/margins": 7.060937404632568,
      "rewards/rejected": -10.278124809265137,
      "step": 6480
    },
    {
      "epoch": 3.4211913547706905,
      "grad_norm": 4.141083797788313,
      "learning_rate": 1.448339483394834e-07,
      "logits/chosen": -3.0210938453674316,
      "logits/rejected": -3.1197266578674316,
      "logps/chosen": -430.7250061035156,
      "logps/rejected": -457.5249938964844,
      "loss": 0.0122,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.440185546875,
      "rewards/margins": 6.984375,
      "rewards/rejected": -10.426562309265137,
      "step": 6490
    },
    {
      "epoch": 3.426462836056932,
      "grad_norm": 7.594440506878496,
      "learning_rate": 1.4351607801792305e-07,
      "logits/chosen": -3.034374952316284,
      "logits/rejected": -3.186718702316284,
      "logps/chosen": -415.625,
      "logps/rejected": -447.2250061035156,
      "loss": 0.0098,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.408496141433716,
      "rewards/margins": 6.663281440734863,
      "rewards/rejected": -10.071093559265137,
      "step": 6500
    },
    {
      "epoch": 3.4317343173431736,
      "grad_norm": 2.6163624586779592,
      "learning_rate": 1.4219820769636267e-07,
      "logits/chosen": -2.8515625,
      "logits/rejected": -2.995312452316284,
      "logps/chosen": -422.0249938964844,
      "logps/rejected": -456.0249938964844,
      "loss": 0.0096,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.0133299827575684,
      "rewards/margins": 7.057812690734863,
      "rewards/rejected": -10.065625190734863,
      "step": 6510
    },
    {
      "epoch": 3.4370057986294147,
      "grad_norm": 12.180273645702465,
      "learning_rate": 1.4088033737480232e-07,
      "logits/chosen": -2.9808592796325684,
      "logits/rejected": -3.0640625953674316,
      "logps/chosen": -413.01251220703125,
      "logps/rejected": -396.82501220703125,
      "loss": 0.0114,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.1649413108825684,
      "rewards/margins": 7.061718940734863,
      "rewards/rejected": -10.23046875,
      "step": 6520
    },
    {
      "epoch": 3.4422772799156562,
      "grad_norm": 25.376820163988075,
      "learning_rate": 1.3956246705324197e-07,
      "logits/chosen": -3.072265625,
      "logits/rejected": -3.067578077316284,
      "logps/chosen": -404.4750061035156,
      "logps/rejected": -441.5,
      "loss": 0.01,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.2127928733825684,
      "rewards/margins": 6.857812404632568,
      "rewards/rejected": -10.071093559265137,
      "step": 6530
    },
    {
      "epoch": 3.447548761201898,
      "grad_norm": 29.26019420559389,
      "learning_rate": 1.382445967316816e-07,
      "logits/chosen": -2.983593702316284,
      "logits/rejected": -3.07421875,
      "logps/chosen": -460.75,
      "logps/rejected": -447.9750061035156,
      "loss": 0.0093,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.02294921875,
      "rewards/margins": 7.001562595367432,
      "rewards/rejected": -10.028124809265137,
      "step": 6540
    },
    {
      "epoch": 3.4528202424881393,
      "grad_norm": 3.842987529562621,
      "learning_rate": 1.3692672641012123e-07,
      "logits/chosen": -3.005078077316284,
      "logits/rejected": -3.0835938453674316,
      "logps/chosen": -436.3999938964844,
      "logps/rejected": -450.42498779296875,
      "loss": 0.0069,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.3564453125,
      "rewards/margins": 6.8671875,
      "rewards/rejected": -10.227343559265137,
      "step": 6550
    },
    {
      "epoch": 3.4580917237743805,
      "grad_norm": 5.447185919510633,
      "learning_rate": 1.3560885608856088e-07,
      "logits/chosen": -2.912890672683716,
      "logits/rejected": -3.073437452316284,
      "logps/chosen": -404.36248779296875,
      "logps/rejected": -425.57501220703125,
      "loss": 0.0064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.462939500808716,
      "rewards/margins": 6.967187404632568,
      "rewards/rejected": -10.425000190734863,
      "step": 6560
    },
    {
      "epoch": 3.463363205060622,
      "grad_norm": 8.815903777254167,
      "learning_rate": 1.342909857670005e-07,
      "logits/chosen": -2.896484375,
      "logits/rejected": -2.966796875,
      "logps/chosen": -449.875,
      "logps/rejected": -479.8999938964844,
      "loss": 0.0119,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.468066453933716,
      "rewards/margins": 6.885156154632568,
      "rewards/rejected": -10.354687690734863,
      "step": 6570
    },
    {
      "epoch": 3.4686346863468636,
      "grad_norm": 13.542544467918121,
      "learning_rate": 1.3297311544544015e-07,
      "logits/chosen": -2.9554686546325684,
      "logits/rejected": -2.951953172683716,
      "logps/chosen": -425.70001220703125,
      "logps/rejected": -465.57501220703125,
      "loss": 0.0118,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.6101317405700684,
      "rewards/margins": 7.354687690734863,
      "rewards/rejected": -10.96484375,
      "step": 6580
    },
    {
      "epoch": 3.473906167633105,
      "grad_norm": 28.375106836077393,
      "learning_rate": 1.316552451238798e-07,
      "logits/chosen": -3.0648436546325684,
      "logits/rejected": -3.196484327316284,
      "logps/chosen": -395.82501220703125,
      "logps/rejected": -434.32501220703125,
      "loss": 0.0138,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.895312547683716,
      "rewards/margins": 6.8203125,
      "rewards/rejected": -10.70703125,
      "step": 6590
    },
    {
      "epoch": 3.479177648919346,
      "grad_norm": 9.723050539551247,
      "learning_rate": 1.3033737480231945e-07,
      "logits/chosen": -2.990234375,
      "logits/rejected": -3.0621094703674316,
      "logps/chosen": -400.5,
      "logps/rejected": -456.2749938964844,
      "loss": 0.0224,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.2783203125,
      "rewards/margins": 6.953906059265137,
      "rewards/rejected": -10.225000381469727,
      "step": 6600
    },
    {
      "epoch": 3.4844491302055878,
      "grad_norm": 5.929432642296313,
      "learning_rate": 1.290195044807591e-07,
      "logits/chosen": -3.0347657203674316,
      "logits/rejected": -3.1021485328674316,
      "logps/chosen": -421.7749938964844,
      "logps/rejected": -438.5,
      "loss": 0.0164,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4541015625,
      "rewards/margins": 6.958593845367432,
      "rewards/rejected": -10.414843559265137,
      "step": 6610
    },
    {
      "epoch": 3.4897206114918293,
      "grad_norm": 1.0390087133138441,
      "learning_rate": 1.2770163415919874e-07,
      "logits/chosen": -2.923046827316284,
      "logits/rejected": -2.9849610328674316,
      "logps/chosen": -422.38751220703125,
      "logps/rejected": -483.42498779296875,
      "loss": 0.0095,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.096484422683716,
      "rewards/margins": 7.259375095367432,
      "rewards/rejected": -10.364062309265137,
      "step": 6620
    },
    {
      "epoch": 3.4949920927780704,
      "grad_norm": 9.557970979910342,
      "learning_rate": 1.263837638376384e-07,
      "logits/chosen": -3.083984375,
      "logits/rejected": -3.1949219703674316,
      "logps/chosen": -444.2250061035156,
      "logps/rejected": -442.6499938964844,
      "loss": 0.0075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4058594703674316,
      "rewards/margins": 6.672656059265137,
      "rewards/rejected": -10.078906059265137,
      "step": 6630
    },
    {
      "epoch": 3.500263574064312,
      "grad_norm": 129.42825536537885,
      "learning_rate": 1.25065893516078e-07,
      "logits/chosen": -2.991406202316284,
      "logits/rejected": -3.095703125,
      "logps/chosen": -446.9750061035156,
      "logps/rejected": -468.0,
      "loss": 0.0274,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.4150390625,
      "rewards/margins": 6.751562595367432,
      "rewards/rejected": -10.165624618530273,
      "step": 6640
    },
    {
      "epoch": 3.5055350553505535,
      "grad_norm": 13.407980679609196,
      "learning_rate": 1.2374802319451766e-07,
      "logits/chosen": -2.975390672683716,
      "logits/rejected": -3.044921875,
      "logps/chosen": -457.25,
      "logps/rejected": -485.54998779296875,
      "loss": 0.0065,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.0874266624450684,
      "rewards/margins": 7.253125190734863,
      "rewards/rejected": -10.342187881469727,
      "step": 6650
    },
    {
      "epoch": 3.510806536636795,
      "grad_norm": 7.152159338756636,
      "learning_rate": 1.224301528729573e-07,
      "logits/chosen": -3.0472655296325684,
      "logits/rejected": -3.1753907203674316,
      "logps/chosen": -415.07501220703125,
      "logps/rejected": -421.7250061035156,
      "loss": 0.0148,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.27325439453125,
      "rewards/margins": 6.633593559265137,
      "rewards/rejected": -9.907031059265137,
      "step": 6660
    },
    {
      "epoch": 3.5160780179230366,
      "grad_norm": 30.87624297415897,
      "learning_rate": 1.2111228255139693e-07,
      "logits/chosen": -3.071093797683716,
      "logits/rejected": -3.13671875,
      "logps/chosen": -388.4375,
      "logps/rejected": -468.95001220703125,
      "loss": 0.0116,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.047314405441284,
      "rewards/margins": 7.0703125,
      "rewards/rejected": -10.117968559265137,
      "step": 6670
    },
    {
      "epoch": 3.5213494992092778,
      "grad_norm": 12.801486992472116,
      "learning_rate": 1.1979441222983657e-07,
      "logits/chosen": -2.9730467796325684,
      "logits/rejected": -3.094921827316284,
      "logps/chosen": -412.2749938964844,
      "logps/rejected": -448.67498779296875,
      "loss": 0.0185,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.025390625,
      "rewards/margins": 7.043749809265137,
      "rewards/rejected": -10.071874618530273,
      "step": 6680
    },
    {
      "epoch": 3.5266209804955193,
      "grad_norm": 15.313332943405523,
      "learning_rate": 1.1847654190827622e-07,
      "logits/chosen": -2.943554639816284,
      "logits/rejected": -3.0218749046325684,
      "logps/chosen": -401.70001220703125,
      "logps/rejected": -476.0,
      "loss": 0.006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.873461961746216,
      "rewards/margins": 7.561718940734863,
      "rewards/rejected": -10.43359375,
      "step": 6690
    },
    {
      "epoch": 3.5318924617817604,
      "grad_norm": 2.46890597706297,
      "learning_rate": 1.1715867158671585e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -3.104687452316284,
      "logps/chosen": -429.2749938964844,
      "logps/rejected": -460.6000061035156,
      "loss": 0.0116,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.3985838890075684,
      "rewards/margins": 7.030859470367432,
      "rewards/rejected": -10.42578125,
      "step": 6700
    },
    {
      "epoch": 3.537163943068002,
      "grad_norm": 5.606056665632293,
      "learning_rate": 1.158408012651555e-07,
      "logits/chosen": -3.084765672683716,
      "logits/rejected": -3.1781249046325684,
      "logps/chosen": -431.70001220703125,
      "logps/rejected": -452.1499938964844,
      "loss": 0.0121,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.3638672828674316,
      "rewards/margins": 6.90625,
      "rewards/rejected": -10.263280868530273,
      "step": 6710
    },
    {
      "epoch": 3.5424354243542435,
      "grad_norm": 9.464756743646253,
      "learning_rate": 1.1452293094359515e-07,
      "logits/chosen": -3.0660157203674316,
      "logits/rejected": -3.133984327316284,
      "logps/chosen": -398.54998779296875,
      "logps/rejected": -426.29998779296875,
      "loss": 0.0094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.2671875953674316,
      "rewards/margins": 6.7890625,
      "rewards/rejected": -10.056249618530273,
      "step": 6720
    },
    {
      "epoch": 3.547706905640485,
      "grad_norm": 8.884703962148397,
      "learning_rate": 1.1320506062203478e-07,
      "logits/chosen": -2.9833984375,
      "logits/rejected": -3.0171875953674316,
      "logps/chosen": -449.6499938964844,
      "logps/rejected": -483.04998779296875,
      "loss": 0.016,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.2855467796325684,
      "rewards/margins": 6.721875190734863,
      "rewards/rejected": -10.009374618530273,
      "step": 6730
    },
    {
      "epoch": 3.5529783869267266,
      "grad_norm": 19.730679020592746,
      "learning_rate": 1.1188719030047443e-07,
      "logits/chosen": -2.973828077316284,
      "logits/rejected": -3.159374952316284,
      "logps/chosen": -445.29998779296875,
      "logps/rejected": -451.125,
      "loss": 0.0196,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.4146485328674316,
      "rewards/margins": 6.741406440734863,
      "rewards/rejected": -10.157812118530273,
      "step": 6740
    },
    {
      "epoch": 3.5582498682129677,
      "grad_norm": 69.21029511053685,
      "learning_rate": 1.1056931997891407e-07,
      "logits/chosen": -3.1265625953674316,
      "logits/rejected": -3.2320313453674316,
      "logps/chosen": -415.42498779296875,
      "logps/rejected": -449.67498779296875,
      "loss": 0.0166,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.404296875,
      "rewards/margins": 7.121874809265137,
      "rewards/rejected": -10.517187118530273,
      "step": 6750
    },
    {
      "epoch": 3.5635213494992093,
      "grad_norm": 1.5148192176301725,
      "learning_rate": 1.0925144965735371e-07,
      "logits/chosen": -3.0234375,
      "logits/rejected": -3.032421827316284,
      "logps/chosen": -444.01251220703125,
      "logps/rejected": -476.5,
      "loss": 0.0134,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.1573729515075684,
      "rewards/margins": 7.170312404632568,
      "rewards/rejected": -10.325780868530273,
      "step": 6760
    },
    {
      "epoch": 3.568792830785451,
      "grad_norm": 2.374943897996071,
      "learning_rate": 1.0793357933579335e-07,
      "logits/chosen": -2.866406202316284,
      "logits/rejected": -2.967968702316284,
      "logps/chosen": -393.3374938964844,
      "logps/rejected": -432.82501220703125,
      "loss": 0.0302,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.058886766433716,
      "rewards/margins": 6.834374904632568,
      "rewards/rejected": -9.89453125,
      "step": 6770
    },
    {
      "epoch": 3.574064312071692,
      "grad_norm": 28.313308313495558,
      "learning_rate": 1.0661570901423298e-07,
      "logits/chosen": -3.024218797683716,
      "logits/rejected": -3.102343797683716,
      "logps/chosen": -498.0,
      "logps/rejected": -501.0,
      "loss": 0.0105,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.925244092941284,
      "rewards/margins": 7.165625095367432,
      "rewards/rejected": -10.087499618530273,
      "step": 6780
    },
    {
      "epoch": 3.5793357933579335,
      "grad_norm": 1.120999327329201,
      "learning_rate": 1.0529783869267264e-07,
      "logits/chosen": -3.0234375,
      "logits/rejected": -3.0902342796325684,
      "logps/chosen": -419.7875061035156,
      "logps/rejected": -440.1499938964844,
      "loss": 0.0093,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.46246337890625,
      "rewards/margins": 7.177343845367432,
      "rewards/rejected": -10.643750190734863,
      "step": 6790
    },
    {
      "epoch": 3.584607274644175,
      "grad_norm": 13.00346705294843,
      "learning_rate": 1.0397996837111228e-07,
      "logits/chosen": -3.0570311546325684,
      "logits/rejected": -3.1109375953674316,
      "logps/chosen": -412.3125,
      "logps/rejected": -466.3999938964844,
      "loss": 0.0107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4476561546325684,
      "rewards/margins": 6.965624809265137,
      "rewards/rejected": -10.413281440734863,
      "step": 6800
    },
    {
      "epoch": 3.5898787559304166,
      "grad_norm": 17.663121332216125,
      "learning_rate": 1.0266209804955192e-07,
      "logits/chosen": -3.043750047683716,
      "logits/rejected": -3.150390625,
      "logps/chosen": -426.45001220703125,
      "logps/rejected": -434.45001220703125,
      "loss": 0.0115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.569531202316284,
      "rewards/margins": 6.6953125,
      "rewards/rejected": -10.276562690734863,
      "step": 6810
    },
    {
      "epoch": 3.5951502372166577,
      "grad_norm": 6.493416870497475,
      "learning_rate": 1.0134422772799156e-07,
      "logits/chosen": -3.0824217796325684,
      "logits/rejected": -3.1449217796325684,
      "logps/chosen": -470.88751220703125,
      "logps/rejected": -474.25,
      "loss": 0.0084,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.8756346702575684,
      "rewards/margins": 6.993750095367432,
      "rewards/rejected": -10.868749618530273,
      "step": 6820
    },
    {
      "epoch": 3.6004217185028993,
      "grad_norm": 15.680782761795186,
      "learning_rate": 1.000263574064312e-07,
      "logits/chosen": -3.0406250953674316,
      "logits/rejected": -3.1103515625,
      "logps/chosen": -418.17498779296875,
      "logps/rejected": -431.79998779296875,
      "loss": 0.0137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.7132811546325684,
      "rewards/margins": 7.318749904632568,
      "rewards/rejected": -11.028905868530273,
      "step": 6830
    },
    {
      "epoch": 3.605693199789141,
      "grad_norm": 3.8662554457533607,
      "learning_rate": 9.870848708487084e-08,
      "logits/chosen": -2.943359375,
      "logits/rejected": -3.041796922683716,
      "logps/chosen": -384.6000061035156,
      "logps/rejected": -451.5,
      "loss": 0.0135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.744335889816284,
      "rewards/margins": 6.8671875,
      "rewards/rejected": -10.609375,
      "step": 6840
    },
    {
      "epoch": 3.610964681075382,
      "grad_norm": 8.658445717726753,
      "learning_rate": 9.739061676331048e-08,
      "logits/chosen": -2.9886717796325684,
      "logits/rejected": -3.1187500953674316,
      "logps/chosen": -429.2250061035156,
      "logps/rejected": -444.9750061035156,
      "loss": 0.01,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.82080078125,
      "rewards/margins": 7.139062404632568,
      "rewards/rejected": -10.958593368530273,
      "step": 6850
    },
    {
      "epoch": 3.6162361623616235,
      "grad_norm": 3.853140936532241,
      "learning_rate": 9.607274644175014e-08,
      "logits/chosen": -2.9964842796325684,
      "logits/rejected": -3.1796875,
      "logps/chosen": -443.32501220703125,
      "logps/rejected": -443.5249938964844,
      "loss": 0.0135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4662108421325684,
      "rewards/margins": 6.824999809265137,
      "rewards/rejected": -10.285937309265137,
      "step": 6860
    },
    {
      "epoch": 3.621507643647865,
      "grad_norm": 12.99925309957599,
      "learning_rate": 9.475487612018977e-08,
      "logits/chosen": -2.966015577316284,
      "logits/rejected": -3.0160155296325684,
      "logps/chosen": -436.63751220703125,
      "logps/rejected": -449.20001220703125,
      "loss": 0.0079,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.270770311355591,
      "rewards/margins": 7.372656345367432,
      "rewards/rejected": -10.64453125,
      "step": 6870
    },
    {
      "epoch": 3.6267791249341066,
      "grad_norm": 4.21947936282834,
      "learning_rate": 9.343700579862942e-08,
      "logits/chosen": -2.9847655296325684,
      "logits/rejected": -3.119140625,
      "logps/chosen": -415.67498779296875,
      "logps/rejected": -463.1499938964844,
      "loss": 0.0071,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.848925828933716,
      "rewards/margins": 7.260937690734863,
      "rewards/rejected": -10.118749618530273,
      "step": 6880
    },
    {
      "epoch": 3.632050606220348,
      "grad_norm": 2.804799172459493,
      "learning_rate": 9.211913547706905e-08,
      "logits/chosen": -3.002148389816284,
      "logits/rejected": -3.161328077316284,
      "logps/chosen": -427.6499938964844,
      "logps/rejected": -444.25,
      "loss": 0.0085,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.9658203125,
      "rewards/margins": 6.875,
      "rewards/rejected": -9.841405868530273,
      "step": 6890
    },
    {
      "epoch": 3.6373220875065893,
      "grad_norm": 3.7956874463605645,
      "learning_rate": 9.080126515550869e-08,
      "logits/chosen": -2.921679735183716,
      "logits/rejected": -3.0140624046325684,
      "logps/chosen": -408.875,
      "logps/rejected": -451.125,
      "loss": 0.0097,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.5458984375,
      "rewards/margins": 6.74609375,
      "rewards/rejected": -10.2890625,
      "step": 6900
    },
    {
      "epoch": 3.642593568792831,
      "grad_norm": 3.9030966676460155,
      "learning_rate": 8.948339483394833e-08,
      "logits/chosen": -2.9595704078674316,
      "logits/rejected": -3.098828077316284,
      "logps/chosen": -430.75,
      "logps/rejected": -430.29998779296875,
      "loss": 0.0121,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.488476514816284,
      "rewards/margins": 6.9375,
      "rewards/rejected": -10.420312881469727,
      "step": 6910
    },
    {
      "epoch": 3.6478650500790724,
      "grad_norm": 2.594447610668766,
      "learning_rate": 8.816552451238797e-08,
      "logits/chosen": -3.008984327316284,
      "logits/rejected": -3.125,
      "logps/chosen": -440.125,
      "logps/rejected": -452.7250061035156,
      "loss": 0.0157,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4544920921325684,
      "rewards/margins": 6.696093559265137,
      "rewards/rejected": -10.15234375,
      "step": 6920
    },
    {
      "epoch": 3.6531365313653135,
      "grad_norm": 12.405137995305399,
      "learning_rate": 8.684765419082763e-08,
      "logits/chosen": -2.9749999046325684,
      "logits/rejected": -3.072265625,
      "logps/chosen": -437.125,
      "logps/rejected": -466.25,
      "loss": 0.0163,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.350048780441284,
      "rewards/margins": 6.743750095367432,
      "rewards/rejected": -10.095312118530273,
      "step": 6930
    },
    {
      "epoch": 3.658408012651555,
      "grad_norm": 4.666709904572924,
      "learning_rate": 8.552978386926726e-08,
      "logits/chosen": -3.046093702316284,
      "logits/rejected": -3.0015625953674316,
      "logps/chosen": -386.9750061035156,
      "logps/rejected": -444.25,
      "loss": 0.0106,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.362499952316284,
      "rewards/margins": 7.172656059265137,
      "rewards/rejected": -10.52734375,
      "step": 6940
    },
    {
      "epoch": 3.6636794939377966,
      "grad_norm": 22.173821505003524,
      "learning_rate": 8.42119135477069e-08,
      "logits/chosen": -2.967578172683716,
      "logits/rejected": -3.114453077316284,
      "logps/chosen": -439.7124938964844,
      "logps/rejected": -470.04998779296875,
      "loss": 0.0074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.2613525390625,
      "rewards/margins": 7.157812595367432,
      "rewards/rejected": -10.418749809265137,
      "step": 6950
    },
    {
      "epoch": 3.668950975224038,
      "grad_norm": 12.506661166459358,
      "learning_rate": 8.289404322614655e-08,
      "logits/chosen": -3.020312547683716,
      "logits/rejected": -3.159374952316284,
      "logps/chosen": -428.4750061035156,
      "logps/rejected": -464.57501220703125,
      "loss": 0.0237,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.356640577316284,
      "rewards/margins": 6.802734375,
      "rewards/rejected": -10.158594131469727,
      "step": 6960
    },
    {
      "epoch": 3.6742224565102792,
      "grad_norm": 21.881475839099696,
      "learning_rate": 8.157617290458618e-08,
      "logits/chosen": -3.098828077316284,
      "logits/rejected": -3.236328125,
      "logps/chosen": -397.3125,
      "logps/rejected": -438.92498779296875,
      "loss": 0.013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4683594703674316,
      "rewards/margins": 6.660937309265137,
      "rewards/rejected": -10.129687309265137,
      "step": 6970
    },
    {
      "epoch": 3.679493937796521,
      "grad_norm": 19.894383643590142,
      "learning_rate": 8.025830258302583e-08,
      "logits/chosen": -3.1089844703674316,
      "logits/rejected": -3.221484422683716,
      "logps/chosen": -437.04998779296875,
      "logps/rejected": -445.70001220703125,
      "loss": 0.0098,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.8746094703674316,
      "rewards/margins": 7.121874809265137,
      "rewards/rejected": -10.999218940734863,
      "step": 6980
    },
    {
      "epoch": 3.6847654190827623,
      "grad_norm": 1.9876857243270154,
      "learning_rate": 7.894043226146546e-08,
      "logits/chosen": -2.879687547683716,
      "logits/rejected": -3.045703172683716,
      "logps/chosen": -445.45001220703125,
      "logps/rejected": -474.7250061035156,
      "loss": 0.0048,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.095752000808716,
      "rewards/margins": 7.47265625,
      "rewards/rejected": -10.5625,
      "step": 6990
    },
    {
      "epoch": 3.6900369003690034,
      "grad_norm": 3.748749090762218,
      "learning_rate": 7.762256193990511e-08,
      "logits/chosen": -2.8775391578674316,
      "logits/rejected": -3.083203077316284,
      "logps/chosen": -464.54998779296875,
      "logps/rejected": -452.17498779296875,
      "loss": 0.0098,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.5,
      "rewards/margins": 6.94140625,
      "rewards/rejected": -10.439062118530273,
      "step": 7000
    },
    {
      "epoch": 3.695308381655245,
      "grad_norm": 23.085497335562316,
      "learning_rate": 7.630469161834476e-08,
      "logits/chosen": -3.044726610183716,
      "logits/rejected": -3.082812547683716,
      "logps/chosen": -412.45001220703125,
      "logps/rejected": -453.875,
      "loss": 0.0142,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.2841796875,
      "rewards/margins": 7.032031059265137,
      "rewards/rejected": -10.32421875,
      "step": 7010
    },
    {
      "epoch": 3.7005798629414866,
      "grad_norm": 2.604081124417128,
      "learning_rate": 7.498682129678439e-08,
      "logits/chosen": -3.055468797683716,
      "logits/rejected": -3.0679688453674316,
      "logps/chosen": -467.20001220703125,
      "logps/rejected": -507.6499938964844,
      "loss": 0.0076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.411669969558716,
      "rewards/margins": 7.206250190734863,
      "rewards/rejected": -10.615625381469727,
      "step": 7020
    },
    {
      "epoch": 3.705851344227728,
      "grad_norm": 10.05537906014251,
      "learning_rate": 7.366895097522404e-08,
      "logits/chosen": -3.0816407203674316,
      "logits/rejected": -3.0611329078674316,
      "logps/chosen": -422.57501220703125,
      "logps/rejected": -465.25,
      "loss": 0.0198,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4833984375,
      "rewards/margins": 6.669531345367432,
      "rewards/rejected": -10.157812118530273,
      "step": 7030
    },
    {
      "epoch": 3.7111228255139697,
      "grad_norm": 4.223249794484176,
      "learning_rate": 7.235108065366367e-08,
      "logits/chosen": -3.071484327316284,
      "logits/rejected": -3.2289061546325684,
      "logps/chosen": -413.76251220703125,
      "logps/rejected": -444.54998779296875,
      "loss": 0.0068,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.260546922683716,
      "rewards/margins": 7.30078125,
      "rewards/rejected": -10.564844131469727,
      "step": 7040
    },
    {
      "epoch": 3.7163943068002108,
      "grad_norm": 8.665737712866381,
      "learning_rate": 7.103321033210331e-08,
      "logits/chosen": -3.0335936546325684,
      "logits/rejected": -3.126953125,
      "logps/chosen": -390.5249938964844,
      "logps/rejected": -411.5249938964844,
      "loss": 0.0087,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.416210889816284,
      "rewards/margins": 6.837500095367432,
      "rewards/rejected": -10.256250381469727,
      "step": 7050
    },
    {
      "epoch": 3.7216657880864523,
      "grad_norm": 12.761760517586598,
      "learning_rate": 6.971534001054295e-08,
      "logits/chosen": -3.116015672683716,
      "logits/rejected": -3.164843797683716,
      "logps/chosen": -485.45001220703125,
      "logps/rejected": -500.54998779296875,
      "loss": 0.007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.476269483566284,
      "rewards/margins": 7.109375,
      "rewards/rejected": -10.589062690734863,
      "step": 7060
    },
    {
      "epoch": 3.726937269372694,
      "grad_norm": 4.642803253599,
      "learning_rate": 6.83974696889826e-08,
      "logits/chosen": -3.0699219703674316,
      "logits/rejected": -3.1792969703674316,
      "logps/chosen": -429.20001220703125,
      "logps/rejected": -440.2250061035156,
      "loss": 0.0095,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.774218797683716,
      "rewards/margins": 7.17578125,
      "rewards/rejected": -10.951562881469727,
      "step": 7070
    },
    {
      "epoch": 3.732208750658935,
      "grad_norm": 23.504649640420038,
      "learning_rate": 6.707959936742225e-08,
      "logits/chosen": -3.1714844703674316,
      "logits/rejected": -3.229296922683716,
      "logps/chosen": -390.125,
      "logps/rejected": -446.0249938964844,
      "loss": 0.0088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.095117092132568,
      "rewards/margins": 7.4453125,
      "rewards/rejected": -11.543749809265137,
      "step": 7080
    },
    {
      "epoch": 3.7374802319451765,
      "grad_norm": 15.366082432950115,
      "learning_rate": 6.576172904586188e-08,
      "logits/chosen": -2.9878907203674316,
      "logits/rejected": -3.0902342796325684,
      "logps/chosen": -390.1000061035156,
      "logps/rejected": -421.375,
      "loss": 0.0166,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.26318359375,
      "rewards/margins": 6.70703125,
      "rewards/rejected": -9.971094131469727,
      "step": 7090
    },
    {
      "epoch": 3.742751713231418,
      "grad_norm": 14.393617502062604,
      "learning_rate": 6.444385872430153e-08,
      "logits/chosen": -3.0146484375,
      "logits/rejected": -3.0386719703674316,
      "logps/chosen": -422.25,
      "logps/rejected": -466.04998779296875,
      "loss": 0.0073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.712646484375,
      "rewards/margins": 7.265234470367432,
      "rewards/rejected": -10.983593940734863,
      "step": 7100
    },
    {
      "epoch": 3.7480231945176596,
      "grad_norm": 7.9118894317441955,
      "learning_rate": 6.312598840274117e-08,
      "logits/chosen": -3.0746092796325684,
      "logits/rejected": -3.223437547683716,
      "logps/chosen": -389.29998779296875,
      "logps/rejected": -432.32501220703125,
      "loss": 0.0119,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.2862792015075684,
      "rewards/margins": 7.375,
      "rewards/rejected": -10.657812118530273,
      "step": 7110
    },
    {
      "epoch": 3.7532946758039007,
      "grad_norm": 74.7036104119729,
      "learning_rate": 6.180811808118081e-08,
      "logits/chosen": -3.1050782203674316,
      "logits/rejected": -3.1617188453674316,
      "logps/chosen": -424.32501220703125,
      "logps/rejected": -477.0,
      "loss": 0.0117,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.821093797683716,
      "rewards/margins": 7.022656440734863,
      "rewards/rejected": -10.841405868530273,
      "step": 7120
    },
    {
      "epoch": 3.7585661570901423,
      "grad_norm": 9.532593252523613,
      "learning_rate": 6.049024775962045e-08,
      "logits/chosen": -3.021484375,
      "logits/rejected": -3.1214842796325684,
      "logps/chosen": -434.20001220703125,
      "logps/rejected": -441.04998779296875,
      "loss": 0.0094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.1419920921325684,
      "rewards/margins": 7.005468845367432,
      "rewards/rejected": -10.146875381469727,
      "step": 7130
    },
    {
      "epoch": 3.763837638376384,
      "grad_norm": 6.01157669759323,
      "learning_rate": 5.917237743806009e-08,
      "logits/chosen": -3.0960936546325684,
      "logits/rejected": -3.173046827316284,
      "logps/chosen": -396.79998779296875,
      "logps/rejected": -428.6000061035156,
      "loss": 0.0153,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.2326416969299316,
      "rewards/margins": 6.950781345367432,
      "rewards/rejected": -10.182812690734863,
      "step": 7140
    },
    {
      "epoch": 3.769109119662625,
      "grad_norm": 7.660303488035486,
      "learning_rate": 5.7854507116499736e-08,
      "logits/chosen": -2.9498047828674316,
      "logits/rejected": -3.0972657203674316,
      "logps/chosen": -455.0,
      "logps/rejected": -456.75,
      "loss": 0.0201,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.582812547683716,
      "rewards/margins": 6.971093654632568,
      "rewards/rejected": -10.557812690734863,
      "step": 7150
    },
    {
      "epoch": 3.7743806009488665,
      "grad_norm": 3.6916165020163456,
      "learning_rate": 5.653663679493938e-08,
      "logits/chosen": -3.005859375,
      "logits/rejected": -3.088671922683716,
      "logps/chosen": -390.79998779296875,
      "logps/rejected": -452.57501220703125,
      "loss": 0.0114,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4117188453674316,
      "rewards/margins": 7.58984375,
      "rewards/rejected": -10.998437881469727,
      "step": 7160
    },
    {
      "epoch": 3.779652082235108,
      "grad_norm": 13.15335383630665,
      "learning_rate": 5.521876647337902e-08,
      "logits/chosen": -2.966015577316284,
      "logits/rejected": -3.053906202316284,
      "logps/chosen": -449.0249938964844,
      "logps/rejected": -462.75,
      "loss": 0.0098,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.0858397483825684,
      "rewards/margins": 7.143750190734863,
      "rewards/rejected": -10.228906631469727,
      "step": 7170
    },
    {
      "epoch": 3.7849235635213496,
      "grad_norm": 7.097556126922211,
      "learning_rate": 5.390089615181866e-08,
      "logits/chosen": -3.004687547683716,
      "logits/rejected": -3.0394530296325684,
      "logps/chosen": -454.375,
      "logps/rejected": -479.82501220703125,
      "loss": 0.0138,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.728320360183716,
      "rewards/margins": 7.196875095367432,
      "rewards/rejected": -10.92578125,
      "step": 7180
    },
    {
      "epoch": 3.790195044807591,
      "grad_norm": 29.17323463590735,
      "learning_rate": 5.25830258302583e-08,
      "logits/chosen": -2.9671874046325684,
      "logits/rejected": -3.037890672683716,
      "logps/chosen": -449.8500061035156,
      "logps/rejected": -456.82501220703125,
      "loss": 0.0091,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.2093749046325684,
      "rewards/margins": 7.095312595367432,
      "rewards/rejected": -10.309374809265137,
      "step": 7190
    },
    {
      "epoch": 3.7954665260938323,
      "grad_norm": 1.8873178096088177,
      "learning_rate": 5.126515550869794e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -3.075390577316284,
      "logps/chosen": -395.2250061035156,
      "logps/rejected": -456.20001220703125,
      "loss": 0.0084,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.5126953125,
      "rewards/margins": 7.185937404632568,
      "rewards/rejected": -10.699999809265137,
      "step": 7200
    },
    {
      "epoch": 3.800738007380074,
      "grad_norm": 30.261127342763437,
      "learning_rate": 4.994728518713758e-08,
      "logits/chosen": -3.099609375,
      "logits/rejected": -3.1820311546325684,
      "logps/chosen": -444.3999938964844,
      "logps/rejected": -496.75,
      "loss": 0.0107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.561962842941284,
      "rewards/margins": 7.212500095367432,
      "rewards/rejected": -10.7734375,
      "step": 7210
    },
    {
      "epoch": 3.8060094886663154,
      "grad_norm": 8.10020219707972,
      "learning_rate": 4.862941486557722e-08,
      "logits/chosen": -2.880078077316284,
      "logits/rejected": -2.991015672683716,
      "logps/chosen": -464.625,
      "logps/rejected": -475.57501220703125,
      "loss": 0.0257,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.4046874046325684,
      "rewards/margins": 7.336718559265137,
      "rewards/rejected": -10.73828125,
      "step": 7220
    },
    {
      "epoch": 3.8112809699525565,
      "grad_norm": 27.056554617132775,
      "learning_rate": 4.731154454401687e-08,
      "logits/chosen": -2.973828077316284,
      "logits/rejected": -3.023242235183716,
      "logps/chosen": -396.8500061035156,
      "logps/rejected": -475.32501220703125,
      "loss": 0.0111,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.353759765625,
      "rewards/margins": 7.2734375,
      "rewards/rejected": -10.631250381469727,
      "step": 7230
    },
    {
      "epoch": 3.816552451238798,
      "grad_norm": 0.9536819958881432,
      "learning_rate": 4.5993674222456505e-08,
      "logits/chosen": -3.006640672683716,
      "logits/rejected": -3.053515672683716,
      "logps/chosen": -381.5,
      "logps/rejected": -450.45001220703125,
      "loss": 0.0081,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.088085889816284,
      "rewards/margins": 7.38671875,
      "rewards/rejected": -10.471094131469727,
      "step": 7240
    },
    {
      "epoch": 3.8218239325250396,
      "grad_norm": 12.835345687524974,
      "learning_rate": 4.4675803900896145e-08,
      "logits/chosen": -3.102734327316284,
      "logits/rejected": -3.186328172683716,
      "logps/chosen": -401.9750061035156,
      "logps/rejected": -425.1499938964844,
      "loss": 0.0102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.116991996765137,
      "rewards/margins": 6.922265529632568,
      "rewards/rejected": -11.045312881469727,
      "step": 7250
    },
    {
      "epoch": 3.827095413811281,
      "grad_norm": 2.491750719855027,
      "learning_rate": 4.335793357933579e-08,
      "logits/chosen": -3.080859422683716,
      "logits/rejected": -3.125,
      "logps/chosen": -430.625,
      "logps/rejected": -480.67498779296875,
      "loss": 0.007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.7232422828674316,
      "rewards/margins": 7.102343559265137,
      "rewards/rejected": -10.827343940734863,
      "step": 7260
    },
    {
      "epoch": 3.8323668950975223,
      "grad_norm": 5.91213727403538,
      "learning_rate": 4.2040063257775434e-08,
      "logits/chosen": -3.0267577171325684,
      "logits/rejected": -3.151562452316284,
      "logps/chosen": -424.32501220703125,
      "logps/rejected": -480.6499938964844,
      "loss": 0.0058,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.295214891433716,
      "rewards/margins": 7.305468559265137,
      "rewards/rejected": -10.603124618530273,
      "step": 7270
    },
    {
      "epoch": 3.837638376383764,
      "grad_norm": 7.143544300525695,
      "learning_rate": 4.0722192936215075e-08,
      "logits/chosen": -3.0628905296325684,
      "logits/rejected": -3.155468702316284,
      "logps/chosen": -383.7250061035156,
      "logps/rejected": -428.17498779296875,
      "loss": 0.0075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.2505860328674316,
      "rewards/margins": 7.181250095367432,
      "rewards/rejected": -10.431249618530273,
      "step": 7280
    },
    {
      "epoch": 3.8429098576700054,
      "grad_norm": 17.585546342057476,
      "learning_rate": 3.9404322614654716e-08,
      "logits/chosen": -3.029101610183716,
      "logits/rejected": -3.173046827316284,
      "logps/chosen": -432.54998779296875,
      "logps/rejected": -470.25,
      "loss": 0.0109,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.861328125,
      "rewards/margins": 7.309374809265137,
      "rewards/rejected": -11.171875,
      "step": 7290
    },
    {
      "epoch": 3.8481813389562465,
      "grad_norm": 1.7655628458741963,
      "learning_rate": 3.808645229309436e-08,
      "logits/chosen": -3.116406202316284,
      "logits/rejected": -3.172656297683716,
      "logps/chosen": -404.5,
      "logps/rejected": -433.70001220703125,
      "loss": 0.0105,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.306640625,
      "rewards/margins": 7.313281059265137,
      "rewards/rejected": -10.620312690734863,
      "step": 7300
    },
    {
      "epoch": 3.853452820242488,
      "grad_norm": 10.719408061023445,
      "learning_rate": 3.6768581971534e-08,
      "logits/chosen": -2.899218797683716,
      "logits/rejected": -3.0980467796325684,
      "logps/chosen": -459.1000061035156,
      "logps/rejected": -452.8500061035156,
      "loss": 0.0123,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.747851610183716,
      "rewards/margins": 6.984375,
      "rewards/rejected": -10.727343559265137,
      "step": 7310
    },
    {
      "epoch": 3.8587243015287296,
      "grad_norm": 3.3273436580657334,
      "learning_rate": 3.545071164997364e-08,
      "logits/chosen": -2.9599609375,
      "logits/rejected": -3.0816407203674316,
      "logps/chosen": -434.4750061035156,
      "logps/rejected": -479.20001220703125,
      "loss": 0.0081,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.2664794921875,
      "rewards/margins": 7.193749904632568,
      "rewards/rejected": -10.463281631469727,
      "step": 7320
    },
    {
      "epoch": 3.863995782814971,
      "grad_norm": 2.4450506954703894,
      "learning_rate": 3.4132841328413286e-08,
      "logits/chosen": -2.9390625953674316,
      "logits/rejected": -3.173828125,
      "logps/chosen": -435.1000061035156,
      "logps/rejected": -421.125,
      "loss": 0.0103,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.04425048828125,
      "rewards/margins": 6.9140625,
      "rewards/rejected": -9.955469131469727,
      "step": 7330
    },
    {
      "epoch": 3.8692672641012127,
      "grad_norm": 7.759473220984968,
      "learning_rate": 3.281497100685293e-08,
      "logits/chosen": -3.057812452316284,
      "logits/rejected": -3.106250047683716,
      "logps/chosen": -452.57501220703125,
      "logps/rejected": -472.7250061035156,
      "loss": 0.0099,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4390625953674316,
      "rewards/margins": 7.350781440734863,
      "rewards/rejected": -10.796093940734863,
      "step": 7340
    },
    {
      "epoch": 3.874538745387454,
      "grad_norm": 63.33762983333093,
      "learning_rate": 3.149710068529256e-08,
      "logits/chosen": -2.987499952316284,
      "logits/rejected": -3.158984422683716,
      "logps/chosen": -405.3999938964844,
      "logps/rejected": -432.7749938964844,
      "loss": 0.0147,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.336132764816284,
      "rewards/margins": 7.103906154632568,
      "rewards/rejected": -10.4453125,
      "step": 7350
    },
    {
      "epoch": 3.8798102266736954,
      "grad_norm": 2.7706465214248013,
      "learning_rate": 3.017923036373221e-08,
      "logits/chosen": -3.0140624046325684,
      "logits/rejected": -3.0824217796325684,
      "logps/chosen": -451.2250061035156,
      "logps/rejected": -510.125,
      "loss": 0.0089,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.3650879859924316,
      "rewards/margins": 7.39453125,
      "rewards/rejected": -10.760156631469727,
      "step": 7360
    },
    {
      "epoch": 3.885081707959937,
      "grad_norm": 2.119382032633305,
      "learning_rate": 2.886136004217185e-08,
      "logits/chosen": -3.0103516578674316,
      "logits/rejected": -3.186718702316284,
      "logps/chosen": -424.67498779296875,
      "logps/rejected": -444.6000061035156,
      "loss": 0.0056,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.389843702316284,
      "rewards/margins": 7.2109375,
      "rewards/rejected": -10.6015625,
      "step": 7370
    },
    {
      "epoch": 3.890353189246178,
      "grad_norm": 9.434102493527863,
      "learning_rate": 2.754348972061149e-08,
      "logits/chosen": -3.0205078125,
      "logits/rejected": -3.1246094703674316,
      "logps/chosen": -422.75,
      "logps/rejected": -446.82501220703125,
      "loss": 0.0076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.268798828125,
      "rewards/margins": 7.311718940734863,
      "rewards/rejected": -10.582812309265137,
      "step": 7380
    },
    {
      "epoch": 3.8956246705324196,
      "grad_norm": 4.83816112144879,
      "learning_rate": 2.6225619399051132e-08,
      "logits/chosen": -3.096484422683716,
      "logits/rejected": -3.1996092796325684,
      "logps/chosen": -354.875,
      "logps/rejected": -390.3999938964844,
      "loss": 0.0086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.2154297828674316,
      "rewards/margins": 6.950781345367432,
      "rewards/rejected": -10.162500381469727,
      "step": 7390
    },
    {
      "epoch": 3.900896151818661,
      "grad_norm": 11.006635989436868,
      "learning_rate": 2.4907749077490773e-08,
      "logits/chosen": -2.943359375,
      "logits/rejected": -3.076953172683716,
      "logps/chosen": -385.51251220703125,
      "logps/rejected": -394.57501220703125,
      "loss": 0.0129,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4227538108825684,
      "rewards/margins": 6.978125095367432,
      "rewards/rejected": -10.392969131469727,
      "step": 7400
    },
    {
      "epoch": 3.9061676331049027,
      "grad_norm": 20.786535938700155,
      "learning_rate": 2.3589878755930417e-08,
      "logits/chosen": -2.9017577171325684,
      "logits/rejected": -3.1597657203674316,
      "logps/chosen": -449.4750061035156,
      "logps/rejected": -440.79998779296875,
      "loss": 0.0134,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.177539110183716,
      "rewards/margins": 7.090624809265137,
      "rewards/rejected": -10.264062881469727,
      "step": 7410
    },
    {
      "epoch": 3.911439114391144,
      "grad_norm": 3.304918108911277,
      "learning_rate": 2.2272008434370054e-08,
      "logits/chosen": -2.9449219703674316,
      "logits/rejected": -3.0335936546325684,
      "logps/chosen": -483.45001220703125,
      "logps/rejected": -506.75,
      "loss": 0.0131,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.359179735183716,
      "rewards/margins": 6.957812309265137,
      "rewards/rejected": -10.321874618530273,
      "step": 7420
    },
    {
      "epoch": 3.9167105956773853,
      "grad_norm": 6.964788693395953,
      "learning_rate": 2.09541381128097e-08,
      "logits/chosen": -3.0703125,
      "logits/rejected": -3.1265625953674316,
      "logps/chosen": -400.2124938964844,
      "logps/rejected": -438.125,
      "loss": 0.0296,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.4916014671325684,
      "rewards/margins": 6.479687690734863,
      "rewards/rejected": -9.975781440734863,
      "step": 7430
    },
    {
      "epoch": 3.921982076963627,
      "grad_norm": 4.6232841621384075,
      "learning_rate": 1.9636267791249343e-08,
      "logits/chosen": -3.012500047683716,
      "logits/rejected": -3.1332030296325684,
      "logps/chosen": -447.625,
      "logps/rejected": -468.5,
      "loss": 0.0078,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.394848585128784,
      "rewards/margins": 6.911718845367432,
      "rewards/rejected": -10.300000190734863,
      "step": 7440
    },
    {
      "epoch": 3.927253558249868,
      "grad_norm": 63.86648353427456,
      "learning_rate": 1.831839746968898e-08,
      "logits/chosen": -3.0503907203674316,
      "logits/rejected": -3.1253905296325684,
      "logps/chosen": -382.32501220703125,
      "logps/rejected": -420.20001220703125,
      "loss": 0.0164,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.1684813499450684,
      "rewards/margins": 7.696093559265137,
      "rewards/rejected": -10.8671875,
      "step": 7450
    },
    {
      "epoch": 3.9325250395361095,
      "grad_norm": 3.1752627281882497,
      "learning_rate": 1.7000527148128625e-08,
      "logits/chosen": -3.135546922683716,
      "logits/rejected": -3.194531202316284,
      "logps/chosen": -401.125,
      "logps/rejected": -412.4750061035156,
      "loss": 0.0136,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.5562500953674316,
      "rewards/margins": 6.801562309265137,
      "rewards/rejected": -10.360156059265137,
      "step": 7460
    },
    {
      "epoch": 3.937796520822351,
      "grad_norm": 2.97687193745803,
      "learning_rate": 1.5682656826568266e-08,
      "logits/chosen": -3.0425782203674316,
      "logits/rejected": -3.075000047683716,
      "logps/chosen": -415.32501220703125,
      "logps/rejected": -495.04998779296875,
      "loss": 0.0201,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.5824217796325684,
      "rewards/margins": 7.126953125,
      "rewards/rejected": -10.705469131469727,
      "step": 7470
    },
    {
      "epoch": 3.9430680021085927,
      "grad_norm": 9.685133701759685,
      "learning_rate": 1.4364786505007907e-08,
      "logits/chosen": -3.0082030296325684,
      "logits/rejected": -3.119921922683716,
      "logps/chosen": -411.92498779296875,
      "logps/rejected": -469.7250061035156,
      "loss": 0.0106,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4634766578674316,
      "rewards/margins": 7.573437690734863,
      "rewards/rejected": -11.03515625,
      "step": 7480
    },
    {
      "epoch": 3.948339483394834,
      "grad_norm": 33.83351555151406,
      "learning_rate": 1.304691618344755e-08,
      "logits/chosen": -3.059375047683716,
      "logits/rejected": -3.150390625,
      "logps/chosen": -424.625,
      "logps/rejected": -415.7250061035156,
      "loss": 0.0138,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.251953125,
      "rewards/margins": 6.644140720367432,
      "rewards/rejected": -9.896875381469727,
      "step": 7490
    },
    {
      "epoch": 3.9536109646810753,
      "grad_norm": 2.924314414845222,
      "learning_rate": 1.172904586188719e-08,
      "logits/chosen": -2.952343702316284,
      "logits/rejected": -3.0570311546325684,
      "logps/chosen": -418.2875061035156,
      "logps/rejected": -477.6000061035156,
      "loss": 0.0119,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.3941407203674316,
      "rewards/margins": 7.489843845367432,
      "rewards/rejected": -10.883593559265137,
      "step": 7500
    },
    {
      "epoch": 3.958882445967317,
      "grad_norm": 5.645230945926707,
      "learning_rate": 1.0411175540326831e-08,
      "logits/chosen": -3.076953172683716,
      "logits/rejected": -3.153125047683716,
      "logps/chosen": -404.92498779296875,
      "logps/rejected": -423.6000061035156,
      "loss": 0.012,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.5517578125,
      "rewards/margins": 6.769921779632568,
      "rewards/rejected": -10.323437690734863,
      "step": 7510
    },
    {
      "epoch": 3.964153927253558,
      "grad_norm": 6.525905042484368,
      "learning_rate": 9.093305218766472e-09,
      "logits/chosen": -3.045703172683716,
      "logits/rejected": -3.126953125,
      "logps/chosen": -401.6000061035156,
      "logps/rejected": -462.92498779296875,
      "loss": 0.0079,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4510741233825684,
      "rewards/margins": 7.159375190734863,
      "rewards/rejected": -10.611719131469727,
      "step": 7520
    },
    {
      "epoch": 3.9694254085397995,
      "grad_norm": 23.6386317008188,
      "learning_rate": 7.775434897206115e-09,
      "logits/chosen": -3.028515577316284,
      "logits/rejected": -3.011523485183716,
      "logps/chosen": -388.125,
      "logps/rejected": -464.625,
      "loss": 0.0226,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.310546875,
      "rewards/margins": 7.421093940734863,
      "rewards/rejected": -10.721875190734863,
      "step": 7530
    },
    {
      "epoch": 3.974696889826041,
      "grad_norm": 5.567187272210403,
      "learning_rate": 6.457564575645756e-09,
      "logits/chosen": -3.052734375,
      "logits/rejected": -3.0816407203674316,
      "logps/chosen": -403.6625061035156,
      "logps/rejected": -467.79998779296875,
      "loss": 0.0068,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.371563673019409,
      "rewards/margins": 6.829687595367432,
      "rewards/rejected": -10.204687118530273,
      "step": 7540
    },
    {
      "epoch": 3.9799683711122826,
      "grad_norm": 12.04343853461702,
      "learning_rate": 5.139694254085398e-09,
      "logits/chosen": -3.026562452316284,
      "logits/rejected": -3.083984375,
      "logps/chosen": -410.8999938964844,
      "logps/rejected": -443.6000061035156,
      "loss": 0.0081,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.081591844558716,
      "rewards/margins": 7.346093654632568,
      "rewards/rejected": -10.4296875,
      "step": 7550
    },
    {
      "epoch": 3.985239852398524,
      "grad_norm": 4.671586634762327,
      "learning_rate": 3.82182393252504e-09,
      "logits/chosen": -3.0552735328674316,
      "logits/rejected": -3.211718797683716,
      "logps/chosen": -385.82501220703125,
      "logps/rejected": -432.79998779296875,
      "loss": 0.0077,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.369433641433716,
      "rewards/margins": 7.14453125,
      "rewards/rejected": -10.509374618530273,
      "step": 7560
    },
    {
      "epoch": 3.9905113336847653,
      "grad_norm": 2.7065165815531347,
      "learning_rate": 2.5039536109646808e-09,
      "logits/chosen": -2.963671922683716,
      "logits/rejected": -3.116406202316284,
      "logps/chosen": -446.32501220703125,
      "logps/rejected": -475.9750061035156,
      "loss": 0.0115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.3176026344299316,
      "rewards/margins": 7.301562309265137,
      "rewards/rejected": -10.615625381469727,
      "step": 7570
    },
    {
      "epoch": 3.995782814971007,
      "grad_norm": 3.8792961859290584,
      "learning_rate": 1.1860832894043225e-09,
      "logits/chosen": -3.024218797683716,
      "logits/rejected": -3.1656250953674316,
      "logps/chosen": -431.4750061035156,
      "logps/rejected": -425.6499938964844,
      "loss": 0.0094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.47412109375,
      "rewards/margins": 6.748437404632568,
      "rewards/rejected": -10.214062690734863,
      "step": 7580
    }
  ],
  "logging_steps": 10,
  "max_steps": 7588,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 4,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}