{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.997144739732045,
  "eval_steps": 500,
  "global_step": 4552,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.008785416209092905,
      "grad_norm": 155.8822891200712,
      "learning_rate": 9.980228471001756e-07,
      "logits/chosen": -1.96337890625,
      "logits/rejected": -1.9060547351837158,
      "logps/chosen": -375.375,
      "logps/rejected": -335.5874938964844,
      "loss": 0.6861,
      "rewards/accuracies": 0.3187499940395355,
      "rewards/chosen": 0.04362335056066513,
      "rewards/margins": 0.01884918287396431,
      "rewards/rejected": 0.024869536980986595,
      "step": 10
    },
    {
      "epoch": 0.01757083241818581,
      "grad_norm": 99.15102123223278,
      "learning_rate": 9.958260105448154e-07,
      "logits/chosen": -2.0732421875,
      "logits/rejected": -2.0283203125,
      "logps/chosen": -407.2749938964844,
      "logps/rejected": -333.98748779296875,
      "loss": 0.6705,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.15559081733226776,
      "rewards/margins": 0.05898284912109375,
      "rewards/rejected": 0.09674148261547089,
      "step": 20
    },
    {
      "epoch": 0.026356248627278717,
      "grad_norm": 113.63458698517307,
      "learning_rate": 9.936291739894551e-07,
      "logits/chosen": -1.892578125,
      "logits/rejected": -1.851953148841858,
      "logps/chosen": -398.9750061035156,
      "logps/rejected": -345.7875061035156,
      "loss": 0.6363,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.3360961973667145,
      "rewards/margins": 0.15620116889476776,
      "rewards/rejected": 0.18000182509422302,
      "step": 30
    },
    {
      "epoch": 0.03514166483637162,
      "grad_norm": 150.67652798030218,
      "learning_rate": 9.91432337434095e-07,
      "logits/chosen": -1.9860351085662842,
      "logits/rejected": -1.943750023841858,
      "logps/chosen": -377.3500061035156,
      "logps/rejected": -339.4750061035156,
      "loss": 0.661,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.43072509765625,
      "rewards/margins": 0.149586483836174,
      "rewards/rejected": 0.28121185302734375,
      "step": 40
    },
    {
      "epoch": 0.04392708104546453,
      "grad_norm": 94.22063873035276,
      "learning_rate": 9.892355008787344e-07,
      "logits/chosen": -1.864843726158142,
      "logits/rejected": -1.895117163658142,
      "logps/chosen": -399.86248779296875,
      "logps/rejected": -335.3999938964844,
      "loss": 0.6226,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.3837219178676605,
      "rewards/margins": 0.26385194063186646,
      "rewards/rejected": 0.11977691948413849,
      "step": 50
    },
    {
      "epoch": 0.052712497254557435,
      "grad_norm": 137.31335272322622,
      "learning_rate": 9.870386643233744e-07,
      "logits/chosen": -2.096484422683716,
      "logits/rejected": -2.0777344703674316,
      "logps/chosen": -387.95001220703125,
      "logps/rejected": -321.0249938964844,
      "loss": 0.6685,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": 0.3160156309604645,
      "rewards/margins": 0.20939941704273224,
      "rewards/rejected": 0.1069183349609375,
      "step": 60
    },
    {
      "epoch": 0.06149791346365034,
      "grad_norm": 109.98678924550337,
      "learning_rate": 9.84841827768014e-07,
      "logits/chosen": -1.9767577648162842,
      "logits/rejected": -1.964453101158142,
      "logps/chosen": -386.7749938964844,
      "logps/rejected": -348.67498779296875,
      "loss": 0.5969,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.443991094827652,
      "rewards/margins": 0.3212173581123352,
      "rewards/rejected": 0.12285461276769638,
      "step": 70
    },
    {
      "epoch": 0.07028332967274324,
      "grad_norm": 126.58179814156911,
      "learning_rate": 9.826449912126537e-07,
      "logits/chosen": -1.9796874523162842,
      "logits/rejected": -1.9929687976837158,
      "logps/chosen": -388.2749938964844,
      "logps/rejected": -332.67498779296875,
      "loss": 0.6464,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": 0.30174559354782104,
      "rewards/margins": 0.201995849609375,
      "rewards/rejected": 0.09957122802734375,
      "step": 80
    },
    {
      "epoch": 0.07906874588183616,
      "grad_norm": 105.97981073296539,
      "learning_rate": 9.804481546572935e-07,
      "logits/chosen": -2.0374999046325684,
      "logits/rejected": -1.9666016101837158,
      "logps/chosen": -366.70001220703125,
      "logps/rejected": -327.26251220703125,
      "loss": 0.6092,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.2299247682094574,
      "rewards/margins": 0.29453736543655396,
      "rewards/rejected": -0.06458739936351776,
      "step": 90
    },
    {
      "epoch": 0.08785416209092906,
      "grad_norm": 118.5220151144916,
      "learning_rate": 9.782513181019332e-07,
      "logits/chosen": -2.0390625,
      "logits/rejected": -2.0306639671325684,
      "logps/chosen": -401.29998779296875,
      "logps/rejected": -368.6875,
      "loss": 0.6138,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.3008575439453125,
      "rewards/margins": 0.34840089082717896,
      "rewards/rejected": -0.04737548902630806,
      "step": 100
    },
    {
      "epoch": 0.09663957830002197,
      "grad_norm": 96.76392627299862,
      "learning_rate": 9.760544815465728e-07,
      "logits/chosen": -1.970312476158142,
      "logits/rejected": -1.95703125,
      "logps/chosen": -378.70001220703125,
      "logps/rejected": -338.8999938964844,
      "loss": 0.5657,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.5574310421943665,
      "rewards/margins": 0.4928832948207855,
      "rewards/rejected": 0.06468963623046875,
      "step": 110
    },
    {
      "epoch": 0.10542499450911487,
      "grad_norm": 96.45099870820317,
      "learning_rate": 9.738576449912126e-07,
      "logits/chosen": -2.095898389816284,
      "logits/rejected": -2.060351610183716,
      "logps/chosen": -324.8500061035156,
      "logps/rejected": -355.11248779296875,
      "loss": 0.632,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.6447693109512329,
      "rewards/margins": 0.3291015625,
      "rewards/rejected": 0.31571656465530396,
      "step": 120
    },
    {
      "epoch": 0.11421041071820777,
      "grad_norm": 110.45886601671714,
      "learning_rate": 9.716608084358523e-07,
      "logits/chosen": -2.060546875,
      "logits/rejected": -2.071484327316284,
      "logps/chosen": -383.79998779296875,
      "logps/rejected": -335.07501220703125,
      "loss": 0.5482,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.674755871295929,
      "rewards/margins": 0.5454651117324829,
      "rewards/rejected": 0.12957763671875,
      "step": 130
    },
    {
      "epoch": 0.12299582692730068,
      "grad_norm": 134.30589950388924,
      "learning_rate": 9.69463971880492e-07,
      "logits/chosen": -2.159374952316284,
      "logits/rejected": -2.1294922828674316,
      "logps/chosen": -410.45001220703125,
      "logps/rejected": -378.17498779296875,
      "loss": 0.6599,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": 0.2630981504917145,
      "rewards/margins": 0.3323913514614105,
      "rewards/rejected": -0.06897582858800888,
      "step": 140
    },
    {
      "epoch": 0.13178124313639358,
      "grad_norm": 108.28511048394968,
      "learning_rate": 9.672671353251316e-07,
      "logits/chosen": -2.1412110328674316,
      "logits/rejected": -2.210742235183716,
      "logps/chosen": -456.75,
      "logps/rejected": -366.1499938964844,
      "loss": 0.5712,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.682598888874054,
      "rewards/margins": 0.54315185546875,
      "rewards/rejected": 0.1385147124528885,
      "step": 150
    },
    {
      "epoch": 0.14056665934548648,
      "grad_norm": 80.68383019979049,
      "learning_rate": 9.650702987697716e-07,
      "logits/chosen": -2.0654296875,
      "logits/rejected": -2.0966796875,
      "logps/chosen": -351.9624938964844,
      "logps/rejected": -309.6499938964844,
      "loss": 0.5958,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.752331554889679,
      "rewards/margins": 0.46577149629592896,
      "rewards/rejected": 0.28666383028030396,
      "step": 160
    },
    {
      "epoch": 0.1493520755545794,
      "grad_norm": 153.01315456863145,
      "learning_rate": 9.628734622144111e-07,
      "logits/chosen": -2.091992139816284,
      "logits/rejected": -2.025390625,
      "logps/chosen": -378.2124938964844,
      "logps/rejected": -316.82501220703125,
      "loss": 0.6078,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.830065906047821,
      "rewards/margins": 0.4075866639614105,
      "rewards/rejected": 0.42235106229782104,
      "step": 170
    },
    {
      "epoch": 0.1581374917636723,
      "grad_norm": 113.87848162502131,
      "learning_rate": 9.60676625659051e-07,
      "logits/chosen": -2.1097655296325684,
      "logits/rejected": -2.143749952316284,
      "logps/chosen": -370.0375061035156,
      "logps/rejected": -306.3374938964844,
      "loss": 0.5798,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.755053699016571,
      "rewards/margins": 0.46387940645217896,
      "rewards/rejected": 0.291778564453125,
      "step": 180
    },
    {
      "epoch": 0.1669229079727652,
      "grad_norm": 129.97305621440046,
      "learning_rate": 9.584797891036907e-07,
      "logits/chosen": -2.092968702316284,
      "logits/rejected": -2.0746092796325684,
      "logps/chosen": -333.1875,
      "logps/rejected": -319.20001220703125,
      "loss": 0.603,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.5179687738418579,
      "rewards/margins": 0.44950562715530396,
      "rewards/rejected": 0.06821747124195099,
      "step": 190
    },
    {
      "epoch": 0.1757083241818581,
      "grad_norm": 102.62209167870962,
      "learning_rate": 9.562829525483304e-07,
      "logits/chosen": -2.132031202316284,
      "logits/rejected": -2.0855469703674316,
      "logps/chosen": -391.92498779296875,
      "logps/rejected": -309.5,
      "loss": 0.5422,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.3085784912109375,
      "rewards/margins": 0.6611572504043579,
      "rewards/rejected": -0.35221558809280396,
      "step": 200
    },
    {
      "epoch": 0.184493740390951,
      "grad_norm": 166.29450490107476,
      "learning_rate": 9.5408611599297e-07,
      "logits/chosen": -2.0517578125,
      "logits/rejected": -2.0599608421325684,
      "logps/chosen": -420.79998779296875,
      "logps/rejected": -385.125,
      "loss": 0.62,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.03372955322265625,
      "rewards/margins": 0.5146728754043579,
      "rewards/rejected": -0.48126220703125,
      "step": 210
    },
    {
      "epoch": 0.19327915660004394,
      "grad_norm": 86.2733796760713,
      "learning_rate": 9.518892794376097e-07,
      "logits/chosen": -2.2222657203674316,
      "logits/rejected": -2.1080079078674316,
      "logps/chosen": -379.61248779296875,
      "logps/rejected": -373.20001220703125,
      "loss": 0.5878,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.0010253905784338713,
      "rewards/margins": 0.496429443359375,
      "rewards/rejected": -0.49617308378219604,
      "step": 220
    },
    {
      "epoch": 0.20206457280913684,
      "grad_norm": 83.57098850484103,
      "learning_rate": 9.496924428822495e-07,
      "logits/chosen": -2.0728516578674316,
      "logits/rejected": -2.062304735183716,
      "logps/chosen": -385.13751220703125,
      "logps/rejected": -377.4750061035156,
      "loss": 0.524,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.5042785406112671,
      "rewards/margins": 0.7062438726425171,
      "rewards/rejected": -0.2021484375,
      "step": 230
    },
    {
      "epoch": 0.21084998901822974,
      "grad_norm": 118.6376927991011,
      "learning_rate": 9.474956063268892e-07,
      "logits/chosen": -2.1488280296325684,
      "logits/rejected": -2.128124952316284,
      "logps/chosen": -403.4375,
      "logps/rejected": -361.3125,
      "loss": 0.6033,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.3468368649482727,
      "rewards/margins": 0.5249878168106079,
      "rewards/rejected": -0.17822265625,
      "step": 240
    },
    {
      "epoch": 0.21963540522732264,
      "grad_norm": 135.93606996598305,
      "learning_rate": 9.45298769771529e-07,
      "logits/chosen": -2.156445264816284,
      "logits/rejected": -2.0328125953674316,
      "logps/chosen": -378.57501220703125,
      "logps/rejected": -373.5,
      "loss": 0.639,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.482086181640625,
      "rewards/margins": 0.49391478300094604,
      "rewards/rejected": -0.011807250790297985,
      "step": 250
    },
    {
      "epoch": 0.22842082143641554,
      "grad_norm": 103.38942201514023,
      "learning_rate": 9.431019332161687e-07,
      "logits/chosen": -2.0005860328674316,
      "logits/rejected": -2.065234422683716,
      "logps/chosen": -412.3999938964844,
      "logps/rejected": -384.54998779296875,
      "loss": 0.6107,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.815234363079071,
      "rewards/margins": 0.5205078125,
      "rewards/rejected": 0.2941955626010895,
      "step": 260
    },
    {
      "epoch": 0.23720623764550847,
      "grad_norm": 106.2230161589957,
      "learning_rate": 9.409050966608084e-07,
      "logits/chosen": -2.055859327316284,
      "logits/rejected": -2.116015672683716,
      "logps/chosen": -370.04998779296875,
      "logps/rejected": -347.2875061035156,
      "loss": 0.5352,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 1.095556616783142,
      "rewards/margins": 0.6816161870956421,
      "rewards/rejected": 0.41414183378219604,
      "step": 270
    },
    {
      "epoch": 0.24599165385460137,
      "grad_norm": 103.7692752948406,
      "learning_rate": 9.387082601054481e-07,
      "logits/chosen": -2.1396484375,
      "logits/rejected": -2.105664014816284,
      "logps/chosen": -375.9125061035156,
      "logps/rejected": -329.2250061035156,
      "loss": 0.5234,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 1.33740234375,
      "rewards/margins": 0.695874035358429,
      "rewards/rejected": 0.642712414264679,
      "step": 280
    },
    {
      "epoch": 0.25477707006369427,
      "grad_norm": 107.89597465200681,
      "learning_rate": 9.365114235500879e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -2.161914110183716,
      "logps/chosen": -423.92498779296875,
      "logps/rejected": -362.2250061035156,
      "loss": 0.6198,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.9814453125,
      "rewards/margins": 0.4621826112270355,
      "rewards/rejected": 0.5203186273574829,
      "step": 290
    },
    {
      "epoch": 0.26356248627278717,
      "grad_norm": 113.7049078822622,
      "learning_rate": 9.343145869947275e-07,
      "logits/chosen": -2.241015672683716,
      "logits/rejected": -2.2357420921325684,
      "logps/chosen": -388.29998779296875,
      "logps/rejected": -331.375,
      "loss": 0.5723,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.610302746295929,
      "rewards/margins": 0.6314452886581421,
      "rewards/rejected": -0.02096252515912056,
      "step": 300
    },
    {
      "epoch": 0.27234790248188007,
      "grad_norm": 98.77130025990422,
      "learning_rate": 9.321177504393673e-07,
      "logits/chosen": -2.0687499046325684,
      "logits/rejected": -2.023632764816284,
      "logps/chosen": -395.0249938964844,
      "logps/rejected": -371.375,
      "loss": 0.6251,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.120697021484375,
      "rewards/margins": 0.42295533418655396,
      "rewards/rejected": -0.543560802936554,
      "step": 310
    },
    {
      "epoch": 0.28113331869097297,
      "grad_norm": 120.32453332957613,
      "learning_rate": 9.299209138840069e-07,
      "logits/chosen": -2.158007860183716,
      "logits/rejected": -2.093554735183716,
      "logps/chosen": -381.11248779296875,
      "logps/rejected": -323.2250061035156,
      "loss": 0.5947,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.3796325623989105,
      "rewards/margins": 0.586779773235321,
      "rewards/rejected": -0.9680541753768921,
      "step": 320
    },
    {
      "epoch": 0.28991873490006587,
      "grad_norm": 72.85176703758339,
      "learning_rate": 9.277240773286467e-07,
      "logits/chosen": -2.107226610183716,
      "logits/rejected": -2.032910108566284,
      "logps/chosen": -393.0375061035156,
      "logps/rejected": -357.875,
      "loss": 0.5552,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.5272003412246704,
      "rewards/margins": 0.6527770757675171,
      "rewards/rejected": -0.12534180283546448,
      "step": 330
    },
    {
      "epoch": 0.2987041511091588,
      "grad_norm": 128.42722392942485,
      "learning_rate": 9.255272407732864e-07,
      "logits/chosen": -2.1273436546325684,
      "logits/rejected": -2.108593702316284,
      "logps/chosen": -369.9750061035156,
      "logps/rejected": -351.6625061035156,
      "loss": 0.583,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 1.068811058998108,
      "rewards/margins": 0.5328735113143921,
      "rewards/rejected": 0.535980224609375,
      "step": 340
    },
    {
      "epoch": 0.3074895673182517,
      "grad_norm": 83.15376930684808,
      "learning_rate": 9.233304042179262e-07,
      "logits/chosen": -2.1109375953674316,
      "logits/rejected": -2.127148389816284,
      "logps/chosen": -397.7250061035156,
      "logps/rejected": -329.8374938964844,
      "loss": 0.5398,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 1.279541015625,
      "rewards/margins": 0.677172839641571,
      "rewards/rejected": 0.6024749875068665,
      "step": 350
    },
    {
      "epoch": 0.3162749835273446,
      "grad_norm": 150.29420551654687,
      "learning_rate": 9.211335676625659e-07,
      "logits/chosen": -2.1646485328674316,
      "logits/rejected": -2.198046922683716,
      "logps/chosen": -394.0625,
      "logps/rejected": -336.57501220703125,
      "loss": 0.555,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 1.1234619617462158,
      "rewards/margins": 0.620837390422821,
      "rewards/rejected": 0.503662109375,
      "step": 360
    },
    {
      "epoch": 0.3250603997364375,
      "grad_norm": 109.31976346391468,
      "learning_rate": 9.189367311072056e-07,
      "logits/chosen": -2.1128907203674316,
      "logits/rejected": -2.126953125,
      "logps/chosen": -336.34375,
      "logps/rejected": -332.375,
      "loss": 0.6079,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.9482666254043579,
      "rewards/margins": 0.6097046136856079,
      "rewards/rejected": 0.33937376737594604,
      "step": 370
    },
    {
      "epoch": 0.3338458159455304,
      "grad_norm": 118.93843162220737,
      "learning_rate": 9.167398945518453e-07,
      "logits/chosen": -2.111132860183716,
      "logits/rejected": -2.1089844703674316,
      "logps/chosen": -395.38751220703125,
      "logps/rejected": -334.0,
      "loss": 0.5578,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.9377197027206421,
      "rewards/margins": 0.656964123249054,
      "rewards/rejected": 0.28004151582717896,
      "step": 380
    },
    {
      "epoch": 0.3426312321546233,
      "grad_norm": 111.72116141316867,
      "learning_rate": 9.14543057996485e-07,
      "logits/chosen": -2.1568360328674316,
      "logits/rejected": -2.0894532203674316,
      "logps/chosen": -382.0249938964844,
      "logps/rejected": -363.125,
      "loss": 0.5161,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 1.04705810546875,
      "rewards/margins": 0.790148913860321,
      "rewards/rejected": 0.2553558349609375,
      "step": 390
    },
    {
      "epoch": 0.3514166483637162,
      "grad_norm": 128.04901575399117,
      "learning_rate": 9.123462214411247e-07,
      "logits/chosen": -2.0478515625,
      "logits/rejected": -2.044726610183716,
      "logps/chosen": -380.9125061035156,
      "logps/rejected": -340.7562561035156,
      "loss": 0.6175,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.7431091070175171,
      "rewards/margins": 0.6383911371231079,
      "rewards/rejected": 0.10472412407398224,
      "step": 400
    },
    {
      "epoch": 0.3602020645728091,
      "grad_norm": 132.78635909304225,
      "learning_rate": 9.101493848857645e-07,
      "logits/chosen": -1.939843773841858,
      "logits/rejected": -1.968164086341858,
      "logps/chosen": -392.2749938964844,
      "logps/rejected": -372.9750061035156,
      "loss": 0.599,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.954663097858429,
      "rewards/margins": 0.689868152141571,
      "rewards/rejected": 0.265228271484375,
      "step": 410
    },
    {
      "epoch": 0.368987480781902,
      "grad_norm": 91.81400586479155,
      "learning_rate": 9.079525483304041e-07,
      "logits/chosen": -2.1703124046325684,
      "logits/rejected": -2.1259765625,
      "logps/chosen": -400.0,
      "logps/rejected": -343.3374938964844,
      "loss": 0.5366,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 1.2410156726837158,
      "rewards/margins": 0.6505371332168579,
      "rewards/rejected": 0.590136706829071,
      "step": 420
    },
    {
      "epoch": 0.3777728969909949,
      "grad_norm": 68.4390670597341,
      "learning_rate": 9.057557117750439e-07,
      "logits/chosen": -2.0902342796325684,
      "logits/rejected": -2.084765672683716,
      "logps/chosen": -380.125,
      "logps/rejected": -360.0,
      "loss": 0.5512,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 1.069580078125,
      "rewards/margins": 0.6681762933731079,
      "rewards/rejected": 0.4018615782260895,
      "step": 430
    },
    {
      "epoch": 0.3865583132000879,
      "grad_norm": 98.93758160665001,
      "learning_rate": 9.035588752196836e-07,
      "logits/chosen": -2.1576170921325684,
      "logits/rejected": -2.101367235183716,
      "logps/chosen": -363.88751220703125,
      "logps/rejected": -331.5,
      "loss": 0.5684,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.776171863079071,
      "rewards/margins": 0.646044909954071,
      "rewards/rejected": 0.130584716796875,
      "step": 440
    },
    {
      "epoch": 0.3953437294091808,
      "grad_norm": 156.73935275285933,
      "learning_rate": 9.013620386643234e-07,
      "logits/chosen": -2.201367139816284,
      "logits/rejected": -2.23046875,
      "logps/chosen": -394.13751220703125,
      "logps/rejected": -347.95001220703125,
      "loss": 0.5314,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.608563244342804,
      "rewards/margins": 0.7679443359375,
      "rewards/rejected": -0.16030272841453552,
      "step": 450
    },
    {
      "epoch": 0.4041291456182737,
      "grad_norm": 92.69532915617263,
      "learning_rate": 8.99165202108963e-07,
      "logits/chosen": -2.083203077316284,
      "logits/rejected": -2.031054735183716,
      "logps/chosen": -443.2250061035156,
      "logps/rejected": -370.625,
      "loss": 0.5702,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.492706298828125,
      "rewards/margins": 0.660205066204071,
      "rewards/rejected": -0.16776123642921448,
      "step": 460
    },
    {
      "epoch": 0.4129145618273666,
      "grad_norm": 117.52112926683493,
      "learning_rate": 8.969683655536028e-07,
      "logits/chosen": -2.0667967796325684,
      "logits/rejected": -2.09765625,
      "logps/chosen": -360.95001220703125,
      "logps/rejected": -362.29998779296875,
      "loss": 0.5827,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.692242443561554,
      "rewards/margins": 0.7540283203125,
      "rewards/rejected": -0.061767578125,
      "step": 470
    },
    {
      "epoch": 0.4216999780364595,
      "grad_norm": 121.18202528312374,
      "learning_rate": 8.947715289982425e-07,
      "logits/chosen": -2.124804735183716,
      "logits/rejected": -2.1675782203674316,
      "logps/chosen": -361.7749938964844,
      "logps/rejected": -321.32501220703125,
      "loss": 0.553,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.600939929485321,
      "rewards/margins": 0.6930297613143921,
      "rewards/rejected": -0.09183044731616974,
      "step": 480
    },
    {
      "epoch": 0.4304853942455524,
      "grad_norm": 112.68336670679064,
      "learning_rate": 8.925746924428822e-07,
      "logits/chosen": -2.143749952316284,
      "logits/rejected": -2.189453125,
      "logps/chosen": -367.25,
      "logps/rejected": -316.4375,
      "loss": 0.6104,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.47169798612594604,
      "rewards/margins": 0.56298828125,
      "rewards/rejected": -0.09134826809167862,
      "step": 490
    },
    {
      "epoch": 0.4392708104546453,
      "grad_norm": 115.48736360532297,
      "learning_rate": 8.903778558875219e-07,
      "logits/chosen": -2.1019530296325684,
      "logits/rejected": -2.043164014816284,
      "logps/chosen": -394.76251220703125,
      "logps/rejected": -398.8500061035156,
      "loss": 0.5352,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.47856444120407104,
      "rewards/margins": 0.7632507085800171,
      "rewards/rejected": -0.28432005643844604,
      "step": 500
    },
    {
      "epoch": 0.4480562266637382,
      "grad_norm": 128.90248479736061,
      "learning_rate": 8.881810193321616e-07,
      "logits/chosen": -2.087109327316284,
      "logits/rejected": -2.068554639816284,
      "logps/chosen": -446.48748779296875,
      "logps/rejected": -361.82501220703125,
      "loss": 0.5473,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.4374633729457855,
      "rewards/margins": 0.7466186285018921,
      "rewards/rejected": -0.3085784912109375,
      "step": 510
    },
    {
      "epoch": 0.4568416428728311,
      "grad_norm": 120.86256763033802,
      "learning_rate": 8.859841827768013e-07,
      "logits/chosen": -2.196093797683716,
      "logits/rejected": -2.169140577316284,
      "logps/chosen": -403.29998779296875,
      "logps/rejected": -326.73748779296875,
      "loss": 0.5007,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.3766937255859375,
      "rewards/margins": 0.905834972858429,
      "rewards/rejected": -0.528643786907196,
      "step": 520
    },
    {
      "epoch": 0.46562705908192403,
      "grad_norm": 98.23411604113977,
      "learning_rate": 8.837873462214412e-07,
      "logits/chosen": -2.2251954078674316,
      "logits/rejected": -2.200976610183716,
      "logps/chosen": -385.8374938964844,
      "logps/rejected": -368.8374938964844,
      "loss": 0.6381,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.11692504584789276,
      "rewards/margins": 0.5814574956893921,
      "rewards/rejected": -0.4654296934604645,
      "step": 530
    },
    {
      "epoch": 0.47441247529101693,
      "grad_norm": 88.09944115054553,
      "learning_rate": 8.815905096660808e-07,
      "logits/chosen": -2.1357421875,
      "logits/rejected": -2.0902342796325684,
      "logps/chosen": -396.2124938964844,
      "logps/rejected": -349.5375061035156,
      "loss": 0.5506,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.18062743544578552,
      "rewards/margins": 0.8259216547012329,
      "rewards/rejected": -0.6450866460800171,
      "step": 540
    },
    {
      "epoch": 0.48319789150010983,
      "grad_norm": 110.78670109642715,
      "learning_rate": 8.793936731107206e-07,
      "logits/chosen": -2.130664110183716,
      "logits/rejected": -2.117382764816284,
      "logps/chosen": -411.6625061035156,
      "logps/rejected": -372.1875,
      "loss": 0.5813,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.3484741151332855,
      "rewards/margins": 0.7659912109375,
      "rewards/rejected": -0.4173339903354645,
      "step": 550
    },
    {
      "epoch": 0.49198330770920273,
      "grad_norm": 117.51225751184334,
      "learning_rate": 8.771968365553602e-07,
      "logits/chosen": -2.086132764816284,
      "logits/rejected": -2.029296875,
      "logps/chosen": -374.3125,
      "logps/rejected": -374.1499938964844,
      "loss": 0.6091,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.26878660917282104,
      "rewards/margins": 0.683764636516571,
      "rewards/rejected": -0.41448974609375,
      "step": 560
    },
    {
      "epoch": 0.5007687239182956,
      "grad_norm": 130.53812311436175,
      "learning_rate": 8.75e-07,
      "logits/chosen": -2.137500047683716,
      "logits/rejected": -2.0677733421325684,
      "logps/chosen": -386.8125,
      "logps/rejected": -354.42498779296875,
      "loss": 0.5863,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.192108154296875,
      "rewards/margins": 0.6750122308731079,
      "rewards/rejected": -0.482421875,
      "step": 570
    },
    {
      "epoch": 0.5095541401273885,
      "grad_norm": 92.07438198767338,
      "learning_rate": 8.728031634446396e-07,
      "logits/chosen": -2.101757764816284,
      "logits/rejected": -2.050976514816284,
      "logps/chosen": -349.82501220703125,
      "logps/rejected": -330.2875061035156,
      "loss": 0.5205,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.16904601454734802,
      "rewards/margins": 0.909802258014679,
      "rewards/rejected": -0.7400878667831421,
      "step": 580
    },
    {
      "epoch": 0.5183395563364814,
      "grad_norm": 100.39131733801983,
      "learning_rate": 8.706063268892794e-07,
      "logits/chosen": -2.092578172683716,
      "logits/rejected": -2.0425782203674316,
      "logps/chosen": -415.95001220703125,
      "logps/rejected": -382.6625061035156,
      "loss": 0.6137,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.02144775353372097,
      "rewards/margins": 0.6675170660018921,
      "rewards/rejected": -0.645678699016571,
      "step": 590
    },
    {
      "epoch": 0.5271249725455743,
      "grad_norm": 136.85854649823818,
      "learning_rate": 8.684094903339191e-07,
      "logits/chosen": -2.0035157203674316,
      "logits/rejected": -2.0707030296325684,
      "logps/chosen": -397.38751220703125,
      "logps/rejected": -356.32501220703125,
      "loss": 0.6622,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.19396361708641052,
      "rewards/margins": 0.519287109375,
      "rewards/rejected": -0.32563477754592896,
      "step": 600
    },
    {
      "epoch": 0.5359103887546672,
      "grad_norm": 72.19238064793001,
      "learning_rate": 8.662126537785588e-07,
      "logits/chosen": -2.080859422683716,
      "logits/rejected": -2.0250000953674316,
      "logps/chosen": -341.7250061035156,
      "logps/rejected": -403.5,
      "loss": 0.6065,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.03489379957318306,
      "rewards/margins": 0.590197741985321,
      "rewards/rejected": -0.624249279499054,
      "step": 610
    },
    {
      "epoch": 0.5446958049637601,
      "grad_norm": 120.80669957627049,
      "learning_rate": 8.640158172231986e-07,
      "logits/chosen": -2.197265625,
      "logits/rejected": -2.2164063453674316,
      "logps/chosen": -425.7250061035156,
      "logps/rejected": -394.63751220703125,
      "loss": 0.5235,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.183634951710701,
      "rewards/margins": 0.723724365234375,
      "rewards/rejected": -0.5400024652481079,
      "step": 620
    },
    {
      "epoch": 0.553481221172853,
      "grad_norm": 112.43554568022826,
      "learning_rate": 8.618189806678383e-07,
      "logits/chosen": -2.166210889816284,
      "logits/rejected": -2.0875000953674316,
      "logps/chosen": -390.07501220703125,
      "logps/rejected": -352.29998779296875,
      "loss": 0.5671,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.05703125149011612,
      "rewards/margins": 0.66845703125,
      "rewards/rejected": -0.6116577386856079,
      "step": 630
    },
    {
      "epoch": 0.5622666373819459,
      "grad_norm": 131.79391521780173,
      "learning_rate": 8.59622144112478e-07,
      "logits/chosen": -2.1171875,
      "logits/rejected": -2.078906297683716,
      "logps/chosen": -370.7250061035156,
      "logps/rejected": -357.70001220703125,
      "loss": 0.5256,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.05456848070025444,
      "rewards/margins": 0.811999499797821,
      "rewards/rejected": -0.7571655511856079,
      "step": 640
    },
    {
      "epoch": 0.5710520535910388,
      "grad_norm": 146.99805250040816,
      "learning_rate": 8.574253075571178e-07,
      "logits/chosen": -2.1001954078674316,
      "logits/rejected": -2.092578172683716,
      "logps/chosen": -381.32501220703125,
      "logps/rejected": -380.42498779296875,
      "loss": 0.5342,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.081512451171875,
      "rewards/margins": 0.814038097858429,
      "rewards/rejected": -0.73236083984375,
      "step": 650
    },
    {
      "epoch": 0.5798374698001317,
      "grad_norm": 95.93084771448207,
      "learning_rate": 8.552284710017574e-07,
      "logits/chosen": -2.136914014816284,
      "logits/rejected": -2.0595703125,
      "logps/chosen": -420.0,
      "logps/rejected": -375.63751220703125,
      "loss": 0.5222,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.082916259765625,
      "rewards/margins": 0.7411864995956421,
      "rewards/rejected": -0.6586059331893921,
      "step": 660
    },
    {
      "epoch": 0.5886228860092246,
      "grad_norm": 104.72176290889409,
      "learning_rate": 8.530316344463972e-07,
      "logits/chosen": -2.216992139816284,
      "logits/rejected": -2.1412110328674316,
      "logps/chosen": -429.20001220703125,
      "logps/rejected": -410.5249938964844,
      "loss": 0.5671,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.07985839992761612,
      "rewards/margins": 0.760986328125,
      "rewards/rejected": -0.6802428960800171,
      "step": 670
    },
    {
      "epoch": 0.5974083022183176,
      "grad_norm": 136.575051034984,
      "learning_rate": 8.508347978910368e-07,
      "logits/chosen": -2.0562500953674316,
      "logits/rejected": -2.025390625,
      "logps/chosen": -378.0249938964844,
      "logps/rejected": -347.625,
      "loss": 0.6313,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.02207336388528347,
      "rewards/margins": 0.6147705316543579,
      "rewards/rejected": -0.592053234577179,
      "step": 680
    },
    {
      "epoch": 0.6061937184274105,
      "grad_norm": 117.4028154804877,
      "learning_rate": 8.486379613356766e-07,
      "logits/chosen": -1.9912109375,
      "logits/rejected": -2.015429735183716,
      "logps/chosen": -401.3999938964844,
      "logps/rejected": -354.7124938964844,
      "loss": 0.5352,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.17271117866039276,
      "rewards/margins": 0.785296618938446,
      "rewards/rejected": -0.612622082233429,
      "step": 690
    },
    {
      "epoch": 0.6149791346365034,
      "grad_norm": 105.98460288056658,
      "learning_rate": 8.464411247803162e-07,
      "logits/chosen": -2.122265577316284,
      "logits/rejected": -2.0601563453674316,
      "logps/chosen": -405.7749938964844,
      "logps/rejected": -349.6499938964844,
      "loss": 0.5313,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.18923339247703552,
      "rewards/margins": 0.969372570514679,
      "rewards/rejected": -0.7802581787109375,
      "step": 700
    },
    {
      "epoch": 0.6237645508455963,
      "grad_norm": 98.0592762182037,
      "learning_rate": 8.442442882249561e-07,
      "logits/chosen": -2.0537109375,
      "logits/rejected": -2.0927734375,
      "logps/chosen": -418.54998779296875,
      "logps/rejected": -360.26251220703125,
      "loss": 0.6708,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.18865355849266052,
      "rewards/margins": 0.65802001953125,
      "rewards/rejected": -0.8472900390625,
      "step": 710
    },
    {
      "epoch": 0.6325499670546892,
      "grad_norm": 118.97726130656062,
      "learning_rate": 8.420474516695958e-07,
      "logits/chosen": -2.1548829078674316,
      "logits/rejected": -2.19140625,
      "logps/chosen": -401.42498779296875,
      "logps/rejected": -378.125,
      "loss": 0.6195,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.14882048964500427,
      "rewards/margins": 0.6210082769393921,
      "rewards/rejected": -0.769635021686554,
      "step": 720
    },
    {
      "epoch": 0.6413353832637821,
      "grad_norm": 151.70707078295658,
      "learning_rate": 8.398506151142355e-07,
      "logits/chosen": -2.003124952316284,
      "logits/rejected": -2.0826172828674316,
      "logps/chosen": -353.42498779296875,
      "logps/rejected": -339.70001220703125,
      "loss": 0.5876,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.12838134169578552,
      "rewards/margins": 0.698437511920929,
      "rewards/rejected": -0.8260726928710938,
      "step": 730
    },
    {
      "epoch": 0.650120799472875,
      "grad_norm": 128.21855301675993,
      "learning_rate": 8.376537785588752e-07,
      "logits/chosen": -2.0718750953674316,
      "logits/rejected": -2.0279297828674316,
      "logps/chosen": -384.67498779296875,
      "logps/rejected": -357.7250061035156,
      "loss": 0.6344,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.3074096739292145,
      "rewards/margins": 0.521209716796875,
      "rewards/rejected": -0.8286377191543579,
      "step": 740
    },
    {
      "epoch": 0.6589062156819679,
      "grad_norm": 112.69618504324573,
      "learning_rate": 8.354569420035149e-07,
      "logits/chosen": -2.058789014816284,
      "logits/rejected": -2.054882764816284,
      "logps/chosen": -339.11248779296875,
      "logps/rejected": -311.2250061035156,
      "loss": 0.6078,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.35819703340530396,
      "rewards/margins": 0.592822253704071,
      "rewards/rejected": -0.9501708745956421,
      "step": 750
    },
    {
      "epoch": 0.6676916318910608,
      "grad_norm": 147.36346074420834,
      "learning_rate": 8.332601054481546e-07,
      "logits/chosen": -2.133984327316284,
      "logits/rejected": -2.125781297683716,
      "logps/chosen": -392.67498779296875,
      "logps/rejected": -358.04998779296875,
      "loss": 0.6294,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.02570495568215847,
      "rewards/margins": 0.5409911870956421,
      "rewards/rejected": -0.5667358636856079,
      "step": 760
    },
    {
      "epoch": 0.6764770481001537,
      "grad_norm": 81.95265519480763,
      "learning_rate": 8.310632688927944e-07,
      "logits/chosen": -2.103515625,
      "logits/rejected": -2.0580077171325684,
      "logps/chosen": -374.82501220703125,
      "logps/rejected": -351.875,
      "loss": 0.5717,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.09528808295726776,
      "rewards/margins": 0.65655517578125,
      "rewards/rejected": -0.561145007610321,
      "step": 770
    },
    {
      "epoch": 0.6852624643092466,
      "grad_norm": 112.32787689305337,
      "learning_rate": 8.28866432337434e-07,
      "logits/chosen": -2.041210889816284,
      "logits/rejected": -2.01171875,
      "logps/chosen": -429.92498779296875,
      "logps/rejected": -377.3374938964844,
      "loss": 0.6631,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.0026977539528161287,
      "rewards/margins": 0.49822998046875,
      "rewards/rejected": -0.5006729364395142,
      "step": 780
    },
    {
      "epoch": 0.6940478805183395,
      "grad_norm": 101.79578397653445,
      "learning_rate": 8.266695957820738e-07,
      "logits/chosen": -1.9773437976837158,
      "logits/rejected": -2.051562547683716,
      "logps/chosen": -405.2749938964844,
      "logps/rejected": -362.5249938964844,
      "loss": 0.5686,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.14421692490577698,
      "rewards/margins": 0.66485595703125,
      "rewards/rejected": -0.520031750202179,
      "step": 790
    },
    {
      "epoch": 0.7028332967274324,
      "grad_norm": 112.31085910264315,
      "learning_rate": 8.244727592267134e-07,
      "logits/chosen": -2.017773389816284,
      "logits/rejected": -1.9812500476837158,
      "logps/chosen": -358.98748779296875,
      "logps/rejected": -349.6000061035156,
      "loss": 0.5541,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.006317138671875,
      "rewards/margins": 0.7072967290878296,
      "rewards/rejected": -0.7143188714981079,
      "step": 800
    },
    {
      "epoch": 0.7116187129365253,
      "grad_norm": 82.96219900441324,
      "learning_rate": 8.222759226713533e-07,
      "logits/chosen": -2.0384764671325684,
      "logits/rejected": -2.025195360183716,
      "logps/chosen": -403.7250061035156,
      "logps/rejected": -361.0874938964844,
      "loss": 0.5284,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.05264892429113388,
      "rewards/margins": 0.70965576171875,
      "rewards/rejected": -0.7616607546806335,
      "step": 810
    },
    {
      "epoch": 0.7204041291456182,
      "grad_norm": 104.04615789286976,
      "learning_rate": 8.200790861159929e-07,
      "logits/chosen": -2.0960936546325684,
      "logits/rejected": -2.0882811546325684,
      "logps/chosen": -406.82501220703125,
      "logps/rejected": -350.625,
      "loss": 0.544,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.19548645615577698,
      "rewards/margins": 0.8200439214706421,
      "rewards/rejected": -1.0159790515899658,
      "step": 820
    },
    {
      "epoch": 0.7291895453547111,
      "grad_norm": 95.8425665979854,
      "learning_rate": 8.178822495606327e-07,
      "logits/chosen": -2.136914014816284,
      "logits/rejected": -2.061328172683716,
      "logps/chosen": -350.7250061035156,
      "logps/rejected": -313.0625,
      "loss": 0.5179,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2125244140625,
      "rewards/margins": 0.8913818597793579,
      "rewards/rejected": -1.1043701171875,
      "step": 830
    },
    {
      "epoch": 0.737974961563804,
      "grad_norm": 97.1655165425616,
      "learning_rate": 8.156854130052724e-07,
      "logits/chosen": -2.0302734375,
      "logits/rejected": -2.008984327316284,
      "logps/chosen": -392.8125,
      "logps/rejected": -371.625,
      "loss": 0.5551,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.21661682426929474,
      "rewards/margins": 0.8555542230606079,
      "rewards/rejected": -1.072241187095642,
      "step": 840
    },
    {
      "epoch": 0.7467603777728969,
      "grad_norm": 67.87831213979665,
      "learning_rate": 8.134885764499121e-07,
      "logits/chosen": -2.0888671875,
      "logits/rejected": -2.0757813453674316,
      "logps/chosen": -391.1000061035156,
      "logps/rejected": -342.5375061035156,
      "loss": 0.5483,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.24292603135108948,
      "rewards/margins": 0.7552551031112671,
      "rewards/rejected": -0.9986206293106079,
      "step": 850
    },
    {
      "epoch": 0.7555457939819898,
      "grad_norm": 138.74616300263608,
      "learning_rate": 8.112917398945518e-07,
      "logits/chosen": -2.128124952316284,
      "logits/rejected": -2.143359422683716,
      "logps/chosen": -369.3500061035156,
      "logps/rejected": -335.9375,
      "loss": 0.5894,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.17897339165210724,
      "rewards/margins": 0.6103881597518921,
      "rewards/rejected": -0.789746105670929,
      "step": 860
    },
    {
      "epoch": 0.7643312101910829,
      "grad_norm": 100.52165451810066,
      "learning_rate": 8.090949033391915e-07,
      "logits/chosen": -2.0404295921325684,
      "logits/rejected": -1.951171875,
      "logps/chosen": -385.7875061035156,
      "logps/rejected": -334.42498779296875,
      "loss": 0.4991,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3601440489292145,
      "rewards/margins": 0.911755383014679,
      "rewards/rejected": -0.5523681640625,
      "step": 870
    },
    {
      "epoch": 0.7731166264001758,
      "grad_norm": 86.10705888302455,
      "learning_rate": 8.068980667838312e-07,
      "logits/chosen": -2.0533204078674316,
      "logits/rejected": -2.011523485183716,
      "logps/chosen": -387.1499938964844,
      "logps/rejected": -341.7250061035156,
      "loss": 0.5633,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.29811400175094604,
      "rewards/margins": 0.7901855707168579,
      "rewards/rejected": -0.49169921875,
      "step": 880
    },
    {
      "epoch": 0.7819020426092687,
      "grad_norm": 85.8151170293382,
      "learning_rate": 8.04701230228471e-07,
      "logits/chosen": -2.1597657203674316,
      "logits/rejected": -2.0414061546325684,
      "logps/chosen": -342.0625,
      "logps/rejected": -331.6000061035156,
      "loss": 0.528,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.5947510004043579,
      "rewards/margins": 0.840832531452179,
      "rewards/rejected": -0.24583740532398224,
      "step": 890
    },
    {
      "epoch": 0.7906874588183616,
      "grad_norm": 113.53741513551641,
      "learning_rate": 8.025043936731107e-07,
      "logits/chosen": -2.1888670921325684,
      "logits/rejected": -2.0376954078674316,
      "logps/chosen": -343.20001220703125,
      "logps/rejected": -377.20001220703125,
      "loss": 0.5235,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.53912353515625,
      "rewards/margins": 0.9484008550643921,
      "rewards/rejected": -0.41002655029296875,
      "step": 900
    },
    {
      "epoch": 0.7994728750274545,
      "grad_norm": 100.98050038343825,
      "learning_rate": 8.003075571177505e-07,
      "logits/chosen": -2.1117186546325684,
      "logits/rejected": -2.109570264816284,
      "logps/chosen": -417.76251220703125,
      "logps/rejected": -339.20001220703125,
      "loss": 0.5941,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.584057629108429,
      "rewards/margins": 0.637988269329071,
      "rewards/rejected": -0.053497314453125,
      "step": 910
    },
    {
      "epoch": 0.8082582912365474,
      "grad_norm": 126.06989864812985,
      "learning_rate": 7.981107205623901e-07,
      "logits/chosen": -2.155468702316284,
      "logits/rejected": -2.1273436546325684,
      "logps/chosen": -400.5,
      "logps/rejected": -324.1625061035156,
      "loss": 0.6608,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.30823975801467896,
      "rewards/margins": 0.5610740780830383,
      "rewards/rejected": -0.2526001036167145,
      "step": 920
    },
    {
      "epoch": 0.8170437074456403,
      "grad_norm": 131.89979970077715,
      "learning_rate": 7.959138840070299e-07,
      "logits/chosen": -2.140625,
      "logits/rejected": -2.1337890625,
      "logps/chosen": -418.2749938964844,
      "logps/rejected": -366.2250061035156,
      "loss": 0.5994,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.16143493354320526,
      "rewards/margins": 0.6745971441268921,
      "rewards/rejected": -0.5129852294921875,
      "step": 930
    },
    {
      "epoch": 0.8258291236547332,
      "grad_norm": 145.56923873152502,
      "learning_rate": 7.937170474516695e-07,
      "logits/chosen": -2.0853514671325684,
      "logits/rejected": -2.1312499046325684,
      "logps/chosen": -377.20001220703125,
      "logps/rejected": -324.42498779296875,
      "loss": 0.5249,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.21050414443016052,
      "rewards/margins": 0.9142395257949829,
      "rewards/rejected": -0.7032257318496704,
      "step": 940
    },
    {
      "epoch": 0.834614539863826,
      "grad_norm": 130.97654213172422,
      "learning_rate": 7.915202108963093e-07,
      "logits/chosen": -2.0999999046325684,
      "logits/rejected": -2.058398485183716,
      "logps/chosen": -391.8500061035156,
      "logps/rejected": -333.9750061035156,
      "loss": 0.5332,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.116973876953125,
      "rewards/margins": 0.992749035358429,
      "rewards/rejected": -0.8756774663925171,
      "step": 950
    },
    {
      "epoch": 0.843399956072919,
      "grad_norm": 89.01778626323322,
      "learning_rate": 7.89323374340949e-07,
      "logits/chosen": -1.936914086341858,
      "logits/rejected": -1.937109351158142,
      "logps/chosen": -439.8500061035156,
      "logps/rejected": -376.54998779296875,
      "loss": 0.6068,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.11349792778491974,
      "rewards/margins": 0.6944580078125,
      "rewards/rejected": -0.80816650390625,
      "step": 960
    },
    {
      "epoch": 0.8521853722820119,
      "grad_norm": 63.53933418147523,
      "learning_rate": 7.871265377855887e-07,
      "logits/chosen": -2.0054688453674316,
      "logits/rejected": -1.9480469226837158,
      "logps/chosen": -346.73748779296875,
      "logps/rejected": -342.57501220703125,
      "loss": 0.5366,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.05663909763097763,
      "rewards/margins": 0.8056640625,
      "rewards/rejected": -0.862963855266571,
      "step": 970
    },
    {
      "epoch": 0.8609707884911048,
      "grad_norm": 92.29318930799836,
      "learning_rate": 7.849297012302284e-07,
      "logits/chosen": -2.074414014816284,
      "logits/rejected": -2.115234375,
      "logps/chosen": -414.54998779296875,
      "logps/rejected": -383.3999938964844,
      "loss": 0.5441,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.16046142578125,
      "rewards/margins": 0.855395495891571,
      "rewards/rejected": -1.015527367591858,
      "step": 980
    },
    {
      "epoch": 0.8697562047001977,
      "grad_norm": 106.95030751297574,
      "learning_rate": 7.827328646748682e-07,
      "logits/chosen": -2.0843749046325684,
      "logits/rejected": -2.0687499046325684,
      "logps/chosen": -386.67498779296875,
      "logps/rejected": -334.70001220703125,
      "loss": 0.4968,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.01910705491900444,
      "rewards/margins": 0.986712634563446,
      "rewards/rejected": -1.006677269935608,
      "step": 990
    },
    {
      "epoch": 0.8785416209092906,
      "grad_norm": 106.60083969896942,
      "learning_rate": 7.805360281195079e-07,
      "logits/chosen": -2.0804686546325684,
      "logits/rejected": -2.01953125,
      "logps/chosen": -388.3999938964844,
      "logps/rejected": -350.0,
      "loss": 0.5155,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.10001220554113388,
      "rewards/margins": 0.995220959186554,
      "rewards/rejected": -0.8949218988418579,
      "step": 1000
    },
    {
      "epoch": 0.8873270371183835,
      "grad_norm": 78.10094454089598,
      "learning_rate": 7.783391915641477e-07,
      "logits/chosen": -2.147265672683716,
      "logits/rejected": -2.082226514816284,
      "logps/chosen": -419.25,
      "logps/rejected": -359.0625,
      "loss": 0.594,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.05129699781537056,
      "rewards/margins": 0.730334460735321,
      "rewards/rejected": -0.678814709186554,
      "step": 1010
    },
    {
      "epoch": 0.8961124533274764,
      "grad_norm": 109.60176535908208,
      "learning_rate": 7.761423550087873e-07,
      "logits/chosen": -2.1031250953674316,
      "logits/rejected": -2.011914014816284,
      "logps/chosen": -363.04998779296875,
      "logps/rejected": -348.20623779296875,
      "loss": 0.563,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.132781982421875,
      "rewards/margins": 0.7556396722793579,
      "rewards/rejected": -0.8883697390556335,
      "step": 1020
    },
    {
      "epoch": 0.9048978695365693,
      "grad_norm": 60.655352444236314,
      "learning_rate": 7.739455184534271e-07,
      "logits/chosen": -2.089062452316284,
      "logits/rejected": -2.067187547683716,
      "logps/chosen": -388.29998779296875,
      "logps/rejected": -365.7749938964844,
      "loss": 0.5257,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07167968899011612,
      "rewards/margins": 0.9446777105331421,
      "rewards/rejected": -1.016638159751892,
      "step": 1030
    },
    {
      "epoch": 0.9136832857456622,
      "grad_norm": 153.06511602750464,
      "learning_rate": 7.717486818980667e-07,
      "logits/chosen": -2.0816407203674316,
      "logits/rejected": -2.057812452316284,
      "logps/chosen": -348.8999938964844,
      "logps/rejected": -330.7124938964844,
      "loss": 0.615,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.37891846895217896,
      "rewards/margins": 0.7215331792831421,
      "rewards/rejected": -1.100610375404358,
      "step": 1040
    },
    {
      "epoch": 0.922468701954755,
      "grad_norm": 110.32398735720535,
      "learning_rate": 7.695518453427065e-07,
      "logits/chosen": -2.1357421875,
      "logits/rejected": -2.0960936546325684,
      "logps/chosen": -470.1000061035156,
      "logps/rejected": -402.7250061035156,
      "loss": 0.5925,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.17001953721046448,
      "rewards/margins": 0.854418933391571,
      "rewards/rejected": -1.0235106945037842,
      "step": 1050
    },
    {
      "epoch": 0.9312541181638481,
      "grad_norm": 82.77722907767057,
      "learning_rate": 7.673550087873461e-07,
      "logits/chosen": -2.0980467796325684,
      "logits/rejected": -2.033007860183716,
      "logps/chosen": -435.8125,
      "logps/rejected": -381.63751220703125,
      "loss": 0.6151,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.13768920302391052,
      "rewards/margins": 0.7027953863143921,
      "rewards/rejected": -0.84027099609375,
      "step": 1060
    },
    {
      "epoch": 0.940039534372941,
      "grad_norm": 143.0734450297781,
      "learning_rate": 7.651581722319859e-07,
      "logits/chosen": -2.0669922828674316,
      "logits/rejected": -2.1460938453674316,
      "logps/chosen": -387.88751220703125,
      "logps/rejected": -335.2250061035156,
      "loss": 0.623,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.2704834043979645,
      "rewards/margins": 0.6748291254043579,
      "rewards/rejected": -0.945086658000946,
      "step": 1070
    },
    {
      "epoch": 0.9488249505820339,
      "grad_norm": 122.48362112434941,
      "learning_rate": 7.629613356766256e-07,
      "logits/chosen": -2.1869139671325684,
      "logits/rejected": -2.052539110183716,
      "logps/chosen": -404.1000061035156,
      "logps/rejected": -343.875,
      "loss": 0.6064,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.23363342881202698,
      "rewards/margins": 0.793121337890625,
      "rewards/rejected": -1.0263183116912842,
      "step": 1080
    },
    {
      "epoch": 0.9576103667911268,
      "grad_norm": 87.70622581889572,
      "learning_rate": 7.607644991212654e-07,
      "logits/chosen": -2.040234327316284,
      "logits/rejected": -2.0416016578674316,
      "logps/chosen": -387.67498779296875,
      "logps/rejected": -340.13751220703125,
      "loss": 0.549,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.08520813286304474,
      "rewards/margins": 0.76043701171875,
      "rewards/rejected": -0.845745861530304,
      "step": 1090
    },
    {
      "epoch": 0.9663957830002197,
      "grad_norm": 138.65541296737018,
      "learning_rate": 7.585676625659051e-07,
      "logits/chosen": -2.080078125,
      "logits/rejected": -2.091796875,
      "logps/chosen": -411.88751220703125,
      "logps/rejected": -368.2749938964844,
      "loss": 0.5596,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.09306640923023224,
      "rewards/margins": 0.6867614984512329,
      "rewards/rejected": -0.779571533203125,
      "step": 1100
    },
    {
      "epoch": 0.9751811992093126,
      "grad_norm": 114.10613718167174,
      "learning_rate": 7.563708260105448e-07,
      "logits/chosen": -2.075000047683716,
      "logits/rejected": -2.039843797683716,
      "logps/chosen": -451.9750061035156,
      "logps/rejected": -412.6499938964844,
      "loss": 0.5428,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.0005310058477334678,
      "rewards/margins": 0.8461059331893921,
      "rewards/rejected": -0.845013439655304,
      "step": 1110
    },
    {
      "epoch": 0.9839666154184055,
      "grad_norm": 108.91882009889896,
      "learning_rate": 7.541739894551845e-07,
      "logits/chosen": -1.975976586341858,
      "logits/rejected": -1.9783203601837158,
      "logps/chosen": -397.125,
      "logps/rejected": -360.82501220703125,
      "loss": 0.6276,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.26954346895217896,
      "rewards/margins": 0.6716674566268921,
      "rewards/rejected": -0.941815197467804,
      "step": 1120
    },
    {
      "epoch": 0.9927520316274984,
      "grad_norm": 112.4377143624681,
      "learning_rate": 7.519771528998243e-07,
      "logits/chosen": -2.1039061546325684,
      "logits/rejected": -2.1146483421325684,
      "logps/chosen": -373.9750061035156,
      "logps/rejected": -357.04998779296875,
      "loss": 0.5777,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.30066221952438354,
      "rewards/margins": 0.8022521734237671,
      "rewards/rejected": -1.10296630859375,
      "step": 1130
    },
    {
      "epoch": 1.0008785416209094,
      "grad_norm": 45.49926394966473,
      "learning_rate": 7.497803163444639e-07,
      "logits/chosen": -2.1045186519622803,
      "logits/rejected": -2.1262669563293457,
      "logps/chosen": -398.3513488769531,
      "logps/rejected": -372.2432556152344,
      "loss": 0.4258,
      "rewards/accuracies": 0.7567567825317383,
      "rewards/chosen": -0.09911449253559113,
      "rewards/margins": 1.0961781740188599,
      "rewards/rejected": -1.1948637962341309,
      "step": 1140
    },
    {
      "epoch": 1.0096639578300022,
      "grad_norm": 45.879263990151465,
      "learning_rate": 7.475834797891037e-07,
      "logits/chosen": -2.074414014816284,
      "logits/rejected": -2.0443358421325684,
      "logps/chosen": -409.42498779296875,
      "logps/rejected": -367.125,
      "loss": 0.2343,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.34564208984375,
      "rewards/margins": 2.1449217796325684,
      "rewards/rejected": -1.799169898033142,
      "step": 1150
    },
    {
      "epoch": 1.0184493740390952,
      "grad_norm": 39.46538375202206,
      "learning_rate": 7.453866432337433e-07,
      "logits/chosen": -2.065624952316284,
      "logits/rejected": -2.1455078125,
      "logps/chosen": -376.2250061035156,
      "logps/rejected": -385.0249938964844,
      "loss": 0.2316,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.522918701171875,
      "rewards/margins": 2.1470704078674316,
      "rewards/rejected": -1.6243164539337158,
      "step": 1160
    },
    {
      "epoch": 1.027234790248188,
      "grad_norm": 33.14759653216138,
      "learning_rate": 7.431898066783831e-07,
      "logits/chosen": -2.0884766578674316,
      "logits/rejected": -2.0384764671325684,
      "logps/chosen": -371.6875,
      "logps/rejected": -356.63751220703125,
      "loss": 0.2084,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.2580703794956207,
      "rewards/margins": 2.140429735183716,
      "rewards/rejected": -1.880615234375,
      "step": 1170
    },
    {
      "epoch": 1.036020206457281,
      "grad_norm": 40.28458954645533,
      "learning_rate": 7.409929701230228e-07,
      "logits/chosen": -2.1080079078674316,
      "logits/rejected": -2.1390624046325684,
      "logps/chosen": -396.8999938964844,
      "logps/rejected": -387.4624938964844,
      "loss": 0.1754,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.21111145615577698,
      "rewards/margins": 2.4263672828674316,
      "rewards/rejected": -2.216015577316284,
      "step": 1180
    },
    {
      "epoch": 1.0448056226663738,
      "grad_norm": 39.98554691461192,
      "learning_rate": 7.387961335676626e-07,
      "logits/chosen": -2.19921875,
      "logits/rejected": -2.129687547683716,
      "logps/chosen": -408.125,
      "logps/rejected": -390.04998779296875,
      "loss": 0.2199,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.07682953029870987,
      "rewards/margins": 2.3114256858825684,
      "rewards/rejected": -2.232714891433716,
      "step": 1190
    },
    {
      "epoch": 1.0535910388754668,
      "grad_norm": 29.451485502036107,
      "learning_rate": 7.365992970123023e-07,
      "logits/chosen": -2.2222657203674316,
      "logits/rejected": -2.1953125,
      "logps/chosen": -378.3125,
      "logps/rejected": -350.98748779296875,
      "loss": 0.196,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.6161254644393921,
      "rewards/margins": 2.556201219558716,
      "rewards/rejected": -1.939111351966858,
      "step": 1200
    },
    {
      "epoch": 1.0623764550845596,
      "grad_norm": 30.683359180677627,
      "learning_rate": 7.34402460456942e-07,
      "logits/chosen": -2.258593797683716,
      "logits/rejected": -2.199414014816284,
      "logps/chosen": -396.48748779296875,
      "logps/rejected": -344.0375061035156,
      "loss": 0.1851,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 1.0160949230194092,
      "rewards/margins": 2.47509765625,
      "rewards/rejected": -1.4597809314727783,
      "step": 1210
    },
    {
      "epoch": 1.0711618712936526,
      "grad_norm": 36.675670173159645,
      "learning_rate": 7.322056239015817e-07,
      "logits/chosen": -2.2359375953674316,
      "logits/rejected": -2.302734375,
      "logps/chosen": -410.32501220703125,
      "logps/rejected": -355.42498779296875,
      "loss": 0.146,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.9383300542831421,
      "rewards/margins": 2.5707030296325684,
      "rewards/rejected": -1.63330078125,
      "step": 1220
    },
    {
      "epoch": 1.0799472875027454,
      "grad_norm": 42.12125391163783,
      "learning_rate": 7.300087873462214e-07,
      "logits/chosen": -2.3251953125,
      "logits/rejected": -2.3291015625,
      "logps/chosen": -335.0625,
      "logps/rejected": -359.20001220703125,
      "loss": 0.2017,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.7184203863143921,
      "rewards/margins": 2.393359422683716,
      "rewards/rejected": -1.6750977039337158,
      "step": 1230
    },
    {
      "epoch": 1.0887327037118384,
      "grad_norm": 35.70745184522949,
      "learning_rate": 7.278119507908611e-07,
      "logits/chosen": -2.300585985183716,
      "logits/rejected": -2.2748045921325684,
      "logps/chosen": -375.73748779296875,
      "logps/rejected": -400.32501220703125,
      "loss": 0.1899,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.39692384004592896,
      "rewards/margins": 2.6249022483825684,
      "rewards/rejected": -2.2274413108825684,
      "step": 1240
    },
    {
      "epoch": 1.0975181199209312,
      "grad_norm": 37.84951936631745,
      "learning_rate": 7.256151142355009e-07,
      "logits/chosen": -2.304882764816284,
      "logits/rejected": -2.2802734375,
      "logps/chosen": -464.6000061035156,
      "logps/rejected": -398.42498779296875,
      "loss": 0.1776,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.473663330078125,
      "rewards/margins": 2.585253953933716,
      "rewards/rejected": -2.1104493141174316,
      "step": 1250
    },
    {
      "epoch": 1.1063035361300242,
      "grad_norm": 60.48329505889704,
      "learning_rate": 7.234182776801405e-07,
      "logits/chosen": -2.2953124046325684,
      "logits/rejected": -2.33203125,
      "logps/chosen": -453.13751220703125,
      "logps/rejected": -400.1499938964844,
      "loss": 0.1647,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.610729992389679,
      "rewards/margins": 2.5663084983825684,
      "rewards/rejected": -1.955078125,
      "step": 1260
    },
    {
      "epoch": 1.115088952339117,
      "grad_norm": 36.14810169254854,
      "learning_rate": 7.212214411247804e-07,
      "logits/chosen": -2.333984375,
      "logits/rejected": -2.2783203125,
      "logps/chosen": -402.9624938964844,
      "logps/rejected": -420.125,
      "loss": 0.1425,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.40679931640625,
      "rewards/margins": 2.9708008766174316,
      "rewards/rejected": -2.5660157203674316,
      "step": 1270
    },
    {
      "epoch": 1.12387436854821,
      "grad_norm": 34.34907761107589,
      "learning_rate": 7.1902460456942e-07,
      "logits/chosen": -2.4339842796325684,
      "logits/rejected": -2.380859375,
      "logps/chosen": -373.51251220703125,
      "logps/rejected": -335.7250061035156,
      "loss": 0.2076,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 0.5068267583847046,
      "rewards/margins": 2.560253858566284,
      "rewards/rejected": -2.0546875,
      "step": 1280
    },
    {
      "epoch": 1.132659784757303,
      "grad_norm": 25.3242562833613,
      "learning_rate": 7.168277680140598e-07,
      "logits/chosen": -2.3570313453674316,
      "logits/rejected": -2.270703077316284,
      "logps/chosen": -349.6499938964844,
      "logps/rejected": -347.2250061035156,
      "loss": 0.1762,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.6113525629043579,
      "rewards/margins": 2.5992188453674316,
      "rewards/rejected": -1.98974609375,
      "step": 1290
    },
    {
      "epoch": 1.1414452009663958,
      "grad_norm": 33.56968817804464,
      "learning_rate": 7.146309314586994e-07,
      "logits/chosen": -2.4205079078674316,
      "logits/rejected": -2.4730467796325684,
      "logps/chosen": -388.79998779296875,
      "logps/rejected": -385.8500061035156,
      "loss": 0.2012,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.6247924566268921,
      "rewards/margins": 2.6661133766174316,
      "rewards/rejected": -2.04150390625,
      "step": 1300
    },
    {
      "epoch": 1.1502306171754886,
      "grad_norm": 35.603707263671645,
      "learning_rate": 7.124340949033392e-07,
      "logits/chosen": -2.294726610183716,
      "logits/rejected": -2.380859375,
      "logps/chosen": -377.42498779296875,
      "logps/rejected": -391.7749938964844,
      "loss": 0.1964,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.592376708984375,
      "rewards/margins": 2.8114256858825684,
      "rewards/rejected": -2.2188477516174316,
      "step": 1310
    },
    {
      "epoch": 1.1590160333845816,
      "grad_norm": 52.597351069196144,
      "learning_rate": 7.102372583479789e-07,
      "logits/chosen": -2.4349608421325684,
      "logits/rejected": -2.3617186546325684,
      "logps/chosen": -412.1875,
      "logps/rejected": -404.20001220703125,
      "loss": 0.1584,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.6376708745956421,
      "rewards/margins": 2.8228516578674316,
      "rewards/rejected": -2.1849122047424316,
      "step": 1320
    },
    {
      "epoch": 1.1678014495936746,
      "grad_norm": 29.056161402606275,
      "learning_rate": 7.080404217926186e-07,
      "logits/chosen": -2.359570264816284,
      "logits/rejected": -2.3589844703674316,
      "logps/chosen": -390.9624938964844,
      "logps/rejected": -344.4750061035156,
      "loss": 0.172,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.20881500840187073,
      "rewards/margins": 2.73828125,
      "rewards/rejected": -2.528515577316284,
      "step": 1330
    },
    {
      "epoch": 1.1765868658027674,
      "grad_norm": 68.58660565985217,
      "learning_rate": 7.058435852372583e-07,
      "logits/chosen": -2.297656297683716,
      "logits/rejected": -2.306445360183716,
      "logps/chosen": -388.92498779296875,
      "logps/rejected": -356.3500061035156,
      "loss": 0.1678,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.20996704697608948,
      "rewards/margins": 2.7606444358825684,
      "rewards/rejected": -2.550488233566284,
      "step": 1340
    },
    {
      "epoch": 1.1853722820118604,
      "grad_norm": 51.15039006907345,
      "learning_rate": 7.03646748681898e-07,
      "logits/chosen": -2.390429735183716,
      "logits/rejected": -2.3597655296325684,
      "logps/chosen": -397.5375061035156,
      "logps/rejected": -408.70001220703125,
      "loss": 0.1517,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.359130859375,
      "rewards/margins": 3.1146483421325684,
      "rewards/rejected": -2.756542921066284,
      "step": 1350
    },
    {
      "epoch": 1.1941576982209532,
      "grad_norm": 59.15770902436017,
      "learning_rate": 7.014499121265377e-07,
      "logits/chosen": -2.404101610183716,
      "logits/rejected": -2.3998045921325684,
      "logps/chosen": -422.20001220703125,
      "logps/rejected": -395.1000061035156,
      "loss": 0.2197,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.22677001357078552,
      "rewards/margins": 2.8414063453674316,
      "rewards/rejected": -2.6128907203674316,
      "step": 1360
    },
    {
      "epoch": 1.2029431144300462,
      "grad_norm": 30.640754338098088,
      "learning_rate": 6.992530755711776e-07,
      "logits/chosen": -2.4541015625,
      "logits/rejected": -2.438281297683716,
      "logps/chosen": -393.2124938964844,
      "logps/rejected": -393.625,
      "loss": 0.2004,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.24766846001148224,
      "rewards/margins": 2.6537108421325684,
      "rewards/rejected": -2.407519578933716,
      "step": 1370
    },
    {
      "epoch": 1.211728530639139,
      "grad_norm": 55.32826203991818,
      "learning_rate": 6.970562390158172e-07,
      "logits/chosen": -2.41015625,
      "logits/rejected": -2.3892579078674316,
      "logps/chosen": -370.4624938964844,
      "logps/rejected": -385.4375,
      "loss": 0.1952,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.2937866151332855,
      "rewards/margins": 2.4693360328674316,
      "rewards/rejected": -2.174511671066284,
      "step": 1380
    },
    {
      "epoch": 1.220513946848232,
      "grad_norm": 43.50412453765831,
      "learning_rate": 6.94859402460457e-07,
      "logits/chosen": -2.344531297683716,
      "logits/rejected": -2.2568359375,
      "logps/chosen": -350.70001220703125,
      "logps/rejected": -360.3500061035156,
      "loss": 0.1574,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.3028808534145355,
      "rewards/margins": 2.730761766433716,
      "rewards/rejected": -2.4263672828674316,
      "step": 1390
    },
    {
      "epoch": 1.2292993630573248,
      "grad_norm": 57.196214763625264,
      "learning_rate": 6.926625659050966e-07,
      "logits/chosen": -2.3885741233825684,
      "logits/rejected": -2.574414014816284,
      "logps/chosen": -472.875,
      "logps/rejected": -405.3125,
      "loss": 0.1668,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.21923828125,
      "rewards/margins": 3.0135741233825684,
      "rewards/rejected": -2.796191453933716,
      "step": 1400
    },
    {
      "epoch": 1.2380847792664178,
      "grad_norm": 40.89167782871062,
      "learning_rate": 6.904657293497364e-07,
      "logits/chosen": -2.3707032203674316,
      "logits/rejected": -2.3626952171325684,
      "logps/chosen": -365.4750061035156,
      "logps/rejected": -366.9750061035156,
      "loss": 0.15,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.24525146186351776,
      "rewards/margins": 2.8013672828674316,
      "rewards/rejected": -2.5572266578674316,
      "step": 1410
    },
    {
      "epoch": 1.2468701954755106,
      "grad_norm": 43.0889717213335,
      "learning_rate": 6.88268892794376e-07,
      "logits/chosen": -2.349414110183716,
      "logits/rejected": -2.3949217796325684,
      "logps/chosen": -417.04998779296875,
      "logps/rejected": -373.2250061035156,
      "loss": 0.1677,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3618423342704773,
      "rewards/margins": 2.8714842796325684,
      "rewards/rejected": -2.507031202316284,
      "step": 1420
    },
    {
      "epoch": 1.2556556116846036,
      "grad_norm": 62.819733596749145,
      "learning_rate": 6.860720562390158e-07,
      "logits/chosen": -2.365429639816284,
      "logits/rejected": -2.338085889816284,
      "logps/chosen": -412.54998779296875,
      "logps/rejected": -402.70001220703125,
      "loss": 0.1752,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 0.563610851764679,
      "rewards/margins": 2.874804735183716,
      "rewards/rejected": -2.311962842941284,
      "step": 1430
    },
    {
      "epoch": 1.2644410278936964,
      "grad_norm": 37.3124831459758,
      "learning_rate": 6.838752196836555e-07,
      "logits/chosen": -2.4091796875,
      "logits/rejected": -2.447460889816284,
      "logps/chosen": -374.98748779296875,
      "logps/rejected": -356.125,
      "loss": 0.1604,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.972790539264679,
      "rewards/margins": 2.835156202316284,
      "rewards/rejected": -1.8624999523162842,
      "step": 1440
    },
    {
      "epoch": 1.2732264441027894,
      "grad_norm": 85.32466817982576,
      "learning_rate": 6.816783831282952e-07,
      "logits/chosen": -2.373828172683716,
      "logits/rejected": -2.450390577316284,
      "logps/chosen": -374.8500061035156,
      "logps/rejected": -328.63751220703125,
      "loss": 0.1966,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.798571765422821,
      "rewards/margins": 2.6548829078674316,
      "rewards/rejected": -1.854956030845642,
      "step": 1450
    },
    {
      "epoch": 1.2820118603118824,
      "grad_norm": 44.59623315335978,
      "learning_rate": 6.79481546572935e-07,
      "logits/chosen": -2.3128905296325684,
      "logits/rejected": -2.3330078125,
      "logps/chosen": -393.01251220703125,
      "logps/rejected": -356.29998779296875,
      "loss": 0.1768,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.38041990995407104,
      "rewards/margins": 2.7752928733825684,
      "rewards/rejected": -2.394726514816284,
      "step": 1460
    },
    {
      "epoch": 1.2907972765209752,
      "grad_norm": 57.12009503374566,
      "learning_rate": 6.772847100175747e-07,
      "logits/chosen": -2.3519530296325684,
      "logits/rejected": -2.327929735183716,
      "logps/chosen": -388.79998779296875,
      "logps/rejected": -381.0249938964844,
      "loss": 0.1353,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.5368896722793579,
      "rewards/margins": 3.2427735328674316,
      "rewards/rejected": -2.7066407203674316,
      "step": 1470
    },
    {
      "epoch": 1.299582692730068,
      "grad_norm": 55.40776270981302,
      "learning_rate": 6.750878734622144e-07,
      "logits/chosen": -2.424609422683716,
      "logits/rejected": -2.4150390625,
      "logps/chosen": -337.3125,
      "logps/rejected": -358.9750061035156,
      "loss": 0.1995,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 0.33441162109375,
      "rewards/margins": 2.6190428733825684,
      "rewards/rejected": -2.2857422828674316,
      "step": 1480
    },
    {
      "epoch": 1.308368108939161,
      "grad_norm": 61.94776851201695,
      "learning_rate": 6.728910369068542e-07,
      "logits/chosen": -2.4037108421325684,
      "logits/rejected": -2.402539014816284,
      "logps/chosen": -407.51251220703125,
      "logps/rejected": -352.0249938964844,
      "loss": 0.1805,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.4716247618198395,
      "rewards/margins": 2.671093702316284,
      "rewards/rejected": -2.2005372047424316,
      "step": 1490
    },
    {
      "epoch": 1.317153525148254,
      "grad_norm": 42.73631263659781,
      "learning_rate": 6.706942003514938e-07,
      "logits/chosen": -2.3951172828674316,
      "logits/rejected": -2.3753905296325684,
      "logps/chosen": -373.33123779296875,
      "logps/rejected": -339.65625,
      "loss": 0.1873,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.5482025146484375,
      "rewards/margins": 2.777148485183716,
      "rewards/rejected": -2.229248046875,
      "step": 1500
    },
    {
      "epoch": 1.3259389413573468,
      "grad_norm": 75.91719298531234,
      "learning_rate": 6.684973637961336e-07,
      "logits/chosen": -2.3148436546325684,
      "logits/rejected": -2.4634766578674316,
      "logps/chosen": -419.8999938964844,
      "logps/rejected": -364.57501220703125,
      "loss": 0.1634,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.3204406797885895,
      "rewards/margins": 3.002734422683716,
      "rewards/rejected": -2.682324171066284,
      "step": 1510
    },
    {
      "epoch": 1.3347243575664396,
      "grad_norm": 67.8910347493484,
      "learning_rate": 6.663005272407732e-07,
      "logits/chosen": -2.4032225608825684,
      "logits/rejected": -2.5067381858825684,
      "logps/chosen": -389.4375,
      "logps/rejected": -359.54998779296875,
      "loss": 0.1668,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.10951538383960724,
      "rewards/margins": 2.80078125,
      "rewards/rejected": -2.9097657203674316,
      "step": 1520
    },
    {
      "epoch": 1.3435097737755326,
      "grad_norm": 81.6491697427617,
      "learning_rate": 6.64103690685413e-07,
      "logits/chosen": -2.4488282203674316,
      "logits/rejected": -2.338671922683716,
      "logps/chosen": -369.7875061035156,
      "logps/rejected": -378.2250061035156,
      "loss": 0.1898,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.11117859184741974,
      "rewards/margins": 2.9930663108825684,
      "rewards/rejected": -2.880175828933716,
      "step": 1530
    },
    {
      "epoch": 1.3522951899846256,
      "grad_norm": 49.78423494143571,
      "learning_rate": 6.619068541300526e-07,
      "logits/chosen": -2.321093797683716,
      "logits/rejected": -2.286328077316284,
      "logps/chosen": -382.45001220703125,
      "logps/rejected": -380.3500061035156,
      "loss": 0.1689,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.27852171659469604,
      "rewards/margins": 2.9439454078674316,
      "rewards/rejected": -2.6651368141174316,
      "step": 1540
    },
    {
      "epoch": 1.3610806061937184,
      "grad_norm": 44.77568937606941,
      "learning_rate": 6.597100175746925e-07,
      "logits/chosen": -2.310546875,
      "logits/rejected": -2.3160157203674316,
      "logps/chosen": -375.6499938964844,
      "logps/rejected": -343.4750061035156,
      "loss": 0.1723,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.43156737089157104,
      "rewards/margins": 2.652148485183716,
      "rewards/rejected": -2.2220702171325684,
      "step": 1550
    },
    {
      "epoch": 1.3698660224028114,
      "grad_norm": 59.13971659543953,
      "learning_rate": 6.575131810193322e-07,
      "logits/chosen": -2.4501953125,
      "logits/rejected": -2.4117188453674316,
      "logps/chosen": -352.4125061035156,
      "logps/rejected": -371.625,
      "loss": 0.1847,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.4675659239292145,
      "rewards/margins": 2.553515672683716,
      "rewards/rejected": -2.087597608566284,
      "step": 1560
    },
    {
      "epoch": 1.3786514386119042,
      "grad_norm": 52.95042312281107,
      "learning_rate": 6.553163444639719e-07,
      "logits/chosen": -2.3753905296325684,
      "logits/rejected": -2.4677734375,
      "logps/chosen": -332.125,
      "logps/rejected": -301.6499938964844,
      "loss": 0.1991,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.04233093187212944,
      "rewards/margins": 2.5679688453674316,
      "rewards/rejected": -2.610546827316284,
      "step": 1570
    },
    {
      "epoch": 1.3874368548209972,
      "grad_norm": 63.50494836125058,
      "learning_rate": 6.531195079086116e-07,
      "logits/chosen": -2.4302735328674316,
      "logits/rejected": -2.4468750953674316,
      "logps/chosen": -350.6625061035156,
      "logps/rejected": -369.60626220703125,
      "loss": 0.1799,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.2577270567417145,
      "rewards/margins": 2.811816453933716,
      "rewards/rejected": -2.556469678878784,
      "step": 1580
    },
    {
      "epoch": 1.39622227103009,
      "grad_norm": 29.348663516289367,
      "learning_rate": 6.509226713532513e-07,
      "logits/chosen": -2.4136719703674316,
      "logits/rejected": -2.3646483421325684,
      "logps/chosen": -437.9750061035156,
      "logps/rejected": -411.125,
      "loss": 0.1657,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.49693602323532104,
      "rewards/margins": 2.7867188453674316,
      "rewards/rejected": -2.292187452316284,
      "step": 1590
    },
    {
      "epoch": 1.405007687239183,
      "grad_norm": 43.99558094332415,
      "learning_rate": 6.48725834797891e-07,
      "logits/chosen": -2.3929686546325684,
      "logits/rejected": -2.435351610183716,
      "logps/chosen": -368.2875061035156,
      "logps/rejected": -360.8500061035156,
      "loss": 0.175,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.3564605712890625,
      "rewards/margins": 2.6952147483825684,
      "rewards/rejected": -2.336718797683716,
      "step": 1600
    },
    {
      "epoch": 1.4137931034482758,
      "grad_norm": 62.17878655423189,
      "learning_rate": 6.465289982425308e-07,
      "logits/chosen": -2.3519530296325684,
      "logits/rejected": -2.396484375,
      "logps/chosen": -365.4750061035156,
      "logps/rejected": -362.7749938964844,
      "loss": 0.1717,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.33867186307907104,
      "rewards/margins": 2.7919921875,
      "rewards/rejected": -2.452929735183716,
      "step": 1610
    },
    {
      "epoch": 1.4225785196573688,
      "grad_norm": 19.97163186289689,
      "learning_rate": 6.443321616871704e-07,
      "logits/chosen": -2.401562452316284,
      "logits/rejected": -2.489062547683716,
      "logps/chosen": -372.75,
      "logps/rejected": -350.3374938964844,
      "loss": 0.1758,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.23663941025733948,
      "rewards/margins": 2.7083983421325684,
      "rewards/rejected": -2.473437547683716,
      "step": 1620
    },
    {
      "epoch": 1.4313639358664616,
      "grad_norm": 39.536345443038456,
      "learning_rate": 6.421353251318102e-07,
      "logits/chosen": -2.3521485328674316,
      "logits/rejected": -2.3267579078674316,
      "logps/chosen": -384.2875061035156,
      "logps/rejected": -360.04998779296875,
      "loss": 0.1379,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.24154052138328552,
      "rewards/margins": 3.0087890625,
      "rewards/rejected": -2.7669920921325684,
      "step": 1630
    },
    {
      "epoch": 1.4401493520755546,
      "grad_norm": 47.16146701914266,
      "learning_rate": 6.399384885764498e-07,
      "logits/chosen": -2.3216795921325684,
      "logits/rejected": -2.313281297683716,
      "logps/chosen": -346.7250061035156,
      "logps/rejected": -335.57501220703125,
      "loss": 0.1948,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.2913879454135895,
      "rewards/margins": 2.865429639816284,
      "rewards/rejected": -2.5718750953674316,
      "step": 1640
    },
    {
      "epoch": 1.4489347682846474,
      "grad_norm": 64.45854608018932,
      "learning_rate": 6.377416520210897e-07,
      "logits/chosen": -2.453906297683716,
      "logits/rejected": -2.459179639816284,
      "logps/chosen": -380.92498779296875,
      "logps/rejected": -398.625,
      "loss": 0.1941,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.21044921875,
      "rewards/margins": 2.8701171875,
      "rewards/rejected": -3.079882860183716,
      "step": 1650
    },
    {
      "epoch": 1.4577201844937404,
      "grad_norm": 29.29267711774568,
      "learning_rate": 6.355448154657293e-07,
      "logits/chosen": -2.4222655296325684,
      "logits/rejected": -2.4730467796325684,
      "logps/chosen": -432.42498779296875,
      "logps/rejected": -392.0249938964844,
      "loss": 0.1593,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.49049073457717896,
      "rewards/margins": 3.145312547683716,
      "rewards/rejected": -3.6361327171325684,
      "step": 1660
    },
    {
      "epoch": 1.4665056007028334,
      "grad_norm": 46.709102268660075,
      "learning_rate": 6.333479789103691e-07,
      "logits/chosen": -2.470703125,
      "logits/rejected": -2.5150389671325684,
      "logps/chosen": -366.48748779296875,
      "logps/rejected": -362.1499938964844,
      "loss": 0.1958,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.7540527582168579,
      "rewards/margins": 2.8013672828674316,
      "rewards/rejected": -3.556640625,
      "step": 1670
    },
    {
      "epoch": 1.4752910169119262,
      "grad_norm": 58.741664885723154,
      "learning_rate": 6.311511423550088e-07,
      "logits/chosen": -2.4917969703674316,
      "logits/rejected": -2.4027342796325684,
      "logps/chosen": -398.7250061035156,
      "logps/rejected": -389.45001220703125,
      "loss": 0.165,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.703503429889679,
      "rewards/margins": 2.77392578125,
      "rewards/rejected": -3.476367235183716,
      "step": 1680
    },
    {
      "epoch": 1.484076433121019,
      "grad_norm": 45.41746481780642,
      "learning_rate": 6.289543057996485e-07,
      "logits/chosen": -2.4966797828674316,
      "logits/rejected": -2.5257811546325684,
      "logps/chosen": -445.98748779296875,
      "logps/rejected": -436.17498779296875,
      "loss": 0.1469,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.36415404081344604,
      "rewards/margins": 3.0511717796325684,
      "rewards/rejected": -3.413281202316284,
      "step": 1690
    },
    {
      "epoch": 1.492861849330112,
      "grad_norm": 33.16755434874732,
      "learning_rate": 6.267574692442882e-07,
      "logits/chosen": -2.4443359375,
      "logits/rejected": -2.4453125,
      "logps/chosen": -411.125,
      "logps/rejected": -405.625,
      "loss": 0.1751,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.26177978515625,
      "rewards/margins": 3.107617139816284,
      "rewards/rejected": -3.3690428733825684,
      "step": 1700
    },
    {
      "epoch": 1.501647265539205,
      "grad_norm": 82.26440079830088,
      "learning_rate": 6.245606326889279e-07,
      "logits/chosen": -2.5230469703674316,
      "logits/rejected": -2.4916014671325684,
      "logps/chosen": -389.5,
      "logps/rejected": -355.8500061035156,
      "loss": 0.185,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.14260253310203552,
      "rewards/margins": 2.962695360183716,
      "rewards/rejected": -3.1064453125,
      "step": 1710
    },
    {
      "epoch": 1.5104326817482978,
      "grad_norm": 68.75691132842842,
      "learning_rate": 6.223637961335676e-07,
      "logits/chosen": -2.514843702316284,
      "logits/rejected": -2.490234375,
      "logps/chosen": -390.6000061035156,
      "logps/rejected": -385.79998779296875,
      "loss": 0.2132,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.26591795682907104,
      "rewards/margins": 2.8080077171325684,
      "rewards/rejected": -3.072949171066284,
      "step": 1720
    },
    {
      "epoch": 1.5192180979573906,
      "grad_norm": 37.97618413685639,
      "learning_rate": 6.201669595782074e-07,
      "logits/chosen": -2.5667967796325684,
      "logits/rejected": -2.5921874046325684,
      "logps/chosen": -339.86248779296875,
      "logps/rejected": -343.76251220703125,
      "loss": 0.1545,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.07120361179113388,
      "rewards/margins": 2.7925782203674316,
      "rewards/rejected": -2.722460985183716,
      "step": 1730
    },
    {
      "epoch": 1.5280035141664836,
      "grad_norm": 65.73135378020557,
      "learning_rate": 6.179701230228471e-07,
      "logits/chosen": -2.4712891578674316,
      "logits/rejected": -2.543750047683716,
      "logps/chosen": -370.79998779296875,
      "logps/rejected": -356.4375,
      "loss": 0.2449,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.004223632626235485,
      "rewards/margins": 2.477343797683716,
      "rewards/rejected": -2.4732422828674316,
      "step": 1740
    },
    {
      "epoch": 1.5367889303755766,
      "grad_norm": 50.42871757656171,
      "learning_rate": 6.157732864674869e-07,
      "logits/chosen": -2.3326172828674316,
      "logits/rejected": -2.383007764816284,
      "logps/chosen": -417.2250061035156,
      "logps/rejected": -399.125,
      "loss": 0.1487,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.07333068549633026,
      "rewards/margins": 2.98046875,
      "rewards/rejected": -2.905468702316284,
      "step": 1750
    },
    {
      "epoch": 1.5455743465846694,
      "grad_norm": 22.32544952916617,
      "learning_rate": 6.135764499121265e-07,
      "logits/chosen": -2.30859375,
      "logits/rejected": -2.4017577171325684,
      "logps/chosen": -402.11248779296875,
      "logps/rejected": -389.1000061035156,
      "loss": 0.164,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.12346801906824112,
      "rewards/margins": 2.86328125,
      "rewards/rejected": -2.986328125,
      "step": 1760
    },
    {
      "epoch": 1.5543597627937622,
      "grad_norm": 26.71446236518599,
      "learning_rate": 6.113796133567663e-07,
      "logits/chosen": -2.435351610183716,
      "logits/rejected": -2.462109327316284,
      "logps/chosen": -442.875,
      "logps/rejected": -405.7250061035156,
      "loss": 0.1417,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.3649658262729645,
      "rewards/margins": 3.136425733566284,
      "rewards/rejected": -2.7718749046325684,
      "step": 1770
    },
    {
      "epoch": 1.5631451790028552,
      "grad_norm": 40.21359140879732,
      "learning_rate": 6.091827768014059e-07,
      "logits/chosen": -2.400585889816284,
      "logits/rejected": -2.4580078125,
      "logps/chosen": -383.95001220703125,
      "logps/rejected": -359.3500061035156,
      "loss": 0.164,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.4265991151332855,
      "rewards/margins": 2.9410157203674316,
      "rewards/rejected": -2.5145020484924316,
      "step": 1780
    },
    {
      "epoch": 1.5719305952119482,
      "grad_norm": 55.04936568920536,
      "learning_rate": 6.069859402460457e-07,
      "logits/chosen": -2.4408202171325684,
      "logits/rejected": -2.451171875,
      "logps/chosen": -413.4750061035156,
      "logps/rejected": -386.32501220703125,
      "loss": 0.1811,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.44279783964157104,
      "rewards/margins": 2.8882813453674316,
      "rewards/rejected": -2.4462890625,
      "step": 1790
    },
    {
      "epoch": 1.5807160114210412,
      "grad_norm": 47.530207783872164,
      "learning_rate": 6.047891036906854e-07,
      "logits/chosen": -2.444140672683716,
      "logits/rejected": -2.4765625,
      "logps/chosen": -376.73748779296875,
      "logps/rejected": -363.04998779296875,
      "loss": 0.1868,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.02094116248190403,
      "rewards/margins": 2.854931592941284,
      "rewards/rejected": -2.836230516433716,
      "step": 1800
    },
    {
      "epoch": 1.589501427630134,
      "grad_norm": 31.576429254594764,
      "learning_rate": 6.025922671353251e-07,
      "logits/chosen": -2.3583984375,
      "logits/rejected": -2.3609375953674316,
      "logps/chosen": -403.0375061035156,
      "logps/rejected": -350.0249938964844,
      "loss": 0.1855,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.26211243867874146,
      "rewards/margins": 2.8675780296325684,
      "rewards/rejected": -2.6064453125,
      "step": 1810
    },
    {
      "epoch": 1.5982868438392268,
      "grad_norm": 34.349749595805086,
      "learning_rate": 6.003954305799648e-07,
      "logits/chosen": -2.3814454078674316,
      "logits/rejected": -2.344921827316284,
      "logps/chosen": -408.2749938964844,
      "logps/rejected": -363.1000061035156,
      "loss": 0.1701,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.4264068603515625,
      "rewards/margins": 3.138671875,
      "rewards/rejected": -2.7118163108825684,
      "step": 1820
    },
    {
      "epoch": 1.6070722600483198,
      "grad_norm": 42.184910694449705,
      "learning_rate": 5.981985940246046e-07,
      "logits/chosen": -2.3794922828674316,
      "logits/rejected": -2.3306641578674316,
      "logps/chosen": -370.79998779296875,
      "logps/rejected": -360.6499938964844,
      "loss": 0.2199,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.338522344827652,
      "rewards/margins": 2.5775389671325684,
      "rewards/rejected": -2.238964796066284,
      "step": 1830
    },
    {
      "epoch": 1.6158576762574128,
      "grad_norm": 52.107493940905215,
      "learning_rate": 5.960017574692443e-07,
      "logits/chosen": -2.325976610183716,
      "logits/rejected": -2.280078172683716,
      "logps/chosen": -358.11248779296875,
      "logps/rejected": -348.9125061035156,
      "loss": 0.1587,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.2571472227573395,
      "rewards/margins": 3.08251953125,
      "rewards/rejected": -2.8250975608825684,
      "step": 1840
    },
    {
      "epoch": 1.6246430924665056,
      "grad_norm": 30.468451539435726,
      "learning_rate": 5.938049209138841e-07,
      "logits/chosen": -2.394335985183716,
      "logits/rejected": -2.35546875,
      "logps/chosen": -358.4750061035156,
      "logps/rejected": -382.2749938964844,
      "loss": 0.159,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.4044189453125,
      "rewards/margins": 2.9798827171325684,
      "rewards/rejected": -2.576464891433716,
      "step": 1850
    },
    {
      "epoch": 1.6334285086755984,
      "grad_norm": 34.337683046448355,
      "learning_rate": 5.916080843585237e-07,
      "logits/chosen": -2.370898485183716,
      "logits/rejected": -2.440624952316284,
      "logps/chosen": -429.25,
      "logps/rejected": -365.3999938964844,
      "loss": 0.1586,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.23833008110523224,
      "rewards/margins": 2.8648438453674316,
      "rewards/rejected": -2.628710985183716,
      "step": 1860
    },
    {
      "epoch": 1.6422139248846914,
      "grad_norm": 35.94494201793303,
      "learning_rate": 5.894112478031635e-07,
      "logits/chosen": -2.3267579078674316,
      "logits/rejected": -2.375195264816284,
      "logps/chosen": -368.5249938964844,
      "logps/rejected": -378.7749938964844,
      "loss": 0.1744,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.37999266386032104,
      "rewards/margins": 2.9969725608825684,
      "rewards/rejected": -2.6195311546325684,
      "step": 1870
    },
    {
      "epoch": 1.6509993410937844,
      "grad_norm": 37.86891171206751,
      "learning_rate": 5.872144112478031e-07,
      "logits/chosen": -2.385937452316284,
      "logits/rejected": -2.389453172683716,
      "logps/chosen": -359.8125,
      "logps/rejected": -365.17498779296875,
      "loss": 0.1877,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.1273193359375,
      "rewards/margins": 2.7833495140075684,
      "rewards/rejected": -2.9105467796325684,
      "step": 1880
    },
    {
      "epoch": 1.6597847573028772,
      "grad_norm": 43.37272322889731,
      "learning_rate": 5.850175746924429e-07,
      "logits/chosen": -2.3759765625,
      "logits/rejected": -2.3984375,
      "logps/chosen": -362.61248779296875,
      "logps/rejected": -355.01251220703125,
      "loss": 0.2062,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.01629028283059597,
      "rewards/margins": 2.90625,
      "rewards/rejected": -2.887500047683716,
      "step": 1890
    },
    {
      "epoch": 1.66857017351197,
      "grad_norm": 72.91655950690685,
      "learning_rate": 5.828207381370825e-07,
      "logits/chosen": -2.410937547683716,
      "logits/rejected": -2.3744139671325684,
      "logps/chosen": -388.07501220703125,
      "logps/rejected": -368.1000061035156,
      "loss": 0.1888,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.14750976860523224,
      "rewards/margins": 2.7412109375,
      "rewards/rejected": -2.595996141433716,
      "step": 1900
    },
    {
      "epoch": 1.677355589721063,
      "grad_norm": 59.79990662713488,
      "learning_rate": 5.806239015817222e-07,
      "logits/chosen": -2.355273485183716,
      "logits/rejected": -2.3775391578674316,
      "logps/chosen": -386.54998779296875,
      "logps/rejected": -356.45001220703125,
      "loss": 0.1589,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.13657227158546448,
      "rewards/margins": 2.8490233421325684,
      "rewards/rejected": -2.7113280296325684,
      "step": 1910
    },
    {
      "epoch": 1.686141005930156,
      "grad_norm": 43.224381325466936,
      "learning_rate": 5.78427065026362e-07,
      "logits/chosen": -2.4775390625,
      "logits/rejected": -2.381640672683716,
      "logps/chosen": -420.1499938964844,
      "logps/rejected": -392.2749938964844,
      "loss": 0.1627,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.02564544603228569,
      "rewards/margins": 2.9853515625,
      "rewards/rejected": -3.009082078933716,
      "step": 1920
    },
    {
      "epoch": 1.6949264221392488,
      "grad_norm": 46.32005762116804,
      "learning_rate": 5.762302284710018e-07,
      "logits/chosen": -2.383984327316284,
      "logits/rejected": -2.4136719703674316,
      "logps/chosen": -352.9125061035156,
      "logps/rejected": -343.95001220703125,
      "loss": 0.1917,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.157135009765625,
      "rewards/margins": 2.6976561546325684,
      "rewards/rejected": -2.8525390625,
      "step": 1930
    },
    {
      "epoch": 1.7037118383483416,
      "grad_norm": 48.08937547019117,
      "learning_rate": 5.740333919156415e-07,
      "logits/chosen": -2.392382860183716,
      "logits/rejected": -2.3560547828674316,
      "logps/chosen": -416.67498779296875,
      "logps/rejected": -390.5249938964844,
      "loss": 0.1502,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.49298095703125,
      "rewards/margins": 2.9847655296325684,
      "rewards/rejected": -3.477734327316284,
      "step": 1940
    },
    {
      "epoch": 1.7124972545574346,
      "grad_norm": 43.78919819247465,
      "learning_rate": 5.718365553602812e-07,
      "logits/chosen": -2.449023485183716,
      "logits/rejected": -2.3505859375,
      "logps/chosen": -369.95001220703125,
      "logps/rejected": -391.625,
      "loss": 0.194,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.6126464605331421,
      "rewards/margins": 3.114062547683716,
      "rewards/rejected": -3.729296922683716,
      "step": 1950
    },
    {
      "epoch": 1.7212826707665276,
      "grad_norm": 65.61375603915934,
      "learning_rate": 5.696397188049209e-07,
      "logits/chosen": -2.4150390625,
      "logits/rejected": -2.3232421875,
      "logps/chosen": -397.7749938964844,
      "logps/rejected": -393.8500061035156,
      "loss": 0.17,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.42356568574905396,
      "rewards/margins": 2.8477540016174316,
      "rewards/rejected": -3.271289110183716,
      "step": 1960
    },
    {
      "epoch": 1.7300680869756204,
      "grad_norm": 33.9542395173659,
      "learning_rate": 5.674428822495607e-07,
      "logits/chosen": -2.3529295921325684,
      "logits/rejected": -2.352734327316284,
      "logps/chosen": -402.4375,
      "logps/rejected": -386.5,
      "loss": 0.1856,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.4585205018520355,
      "rewards/margins": 3.03076171875,
      "rewards/rejected": -3.487597703933716,
      "step": 1970
    },
    {
      "epoch": 1.7388535031847132,
      "grad_norm": 26.826054904675612,
      "learning_rate": 5.652460456942003e-07,
      "logits/chosen": -2.4775390625,
      "logits/rejected": -2.419140577316284,
      "logps/chosen": -375.32501220703125,
      "logps/rejected": -401.7250061035156,
      "loss": 0.1945,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.44355469942092896,
      "rewards/margins": 2.981396436691284,
      "rewards/rejected": -3.423828125,
      "step": 1980
    },
    {
      "epoch": 1.7476389193938062,
      "grad_norm": 103.36565177308432,
      "learning_rate": 5.6304920913884e-07,
      "logits/chosen": -2.408007860183716,
      "logits/rejected": -2.411914110183716,
      "logps/chosen": -383.625,
      "logps/rejected": -388.29998779296875,
      "loss": 0.1777,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.3983154296875,
      "rewards/margins": 2.9144530296325684,
      "rewards/rejected": -3.3121094703674316,
      "step": 1990
    },
    {
      "epoch": 1.7564243356028992,
      "grad_norm": 89.99547345288165,
      "learning_rate": 5.608523725834797e-07,
      "logits/chosen": -2.392773389816284,
      "logits/rejected": -2.294726610183716,
      "logps/chosen": -404.07501220703125,
      "logps/rejected": -405.7749938964844,
      "loss": 0.2106,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.43522340059280396,
      "rewards/margins": 2.9189453125,
      "rewards/rejected": -3.352734327316284,
      "step": 2000
    },
    {
      "epoch": 1.7652097518119922,
      "grad_norm": 82.15788136879195,
      "learning_rate": 5.586555360281194e-07,
      "logits/chosen": -2.355761766433716,
      "logits/rejected": -2.3980469703674316,
      "logps/chosen": -453.04998779296875,
      "logps/rejected": -421.8999938964844,
      "loss": 0.1394,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.25574952363967896,
      "rewards/margins": 3.120312452316284,
      "rewards/rejected": -3.375,
      "step": 2010
    },
    {
      "epoch": 1.773995168021085,
      "grad_norm": 33.57695659607092,
      "learning_rate": 5.564586994727593e-07,
      "logits/chosen": -2.486523389816284,
      "logits/rejected": -2.4566407203674316,
      "logps/chosen": -331.375,
      "logps/rejected": -356.17498779296875,
      "loss": 0.1791,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.610809326171875,
      "rewards/margins": 2.8662109375,
      "rewards/rejected": -3.476367235183716,
      "step": 2020
    },
    {
      "epoch": 1.7827805842301778,
      "grad_norm": 16.45433272753634,
      "learning_rate": 5.54261862917399e-07,
      "logits/chosen": -2.4488282203674316,
      "logits/rejected": -2.322460889816284,
      "logps/chosen": -430.13751220703125,
      "logps/rejected": -433.7250061035156,
      "loss": 0.1577,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.24506835639476776,
      "rewards/margins": 3.3101563453674316,
      "rewards/rejected": -3.5541014671325684,
      "step": 2030
    },
    {
      "epoch": 1.7915660004392708,
      "grad_norm": 49.43452319468024,
      "learning_rate": 5.520650263620387e-07,
      "logits/chosen": -2.4935545921325684,
      "logits/rejected": -2.4693360328674316,
      "logps/chosen": -439.95001220703125,
      "logps/rejected": -438.17498779296875,
      "loss": 0.1763,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.3330322206020355,
      "rewards/margins": 3.026171922683716,
      "rewards/rejected": -3.3583984375,
      "step": 2040
    },
    {
      "epoch": 1.8003514166483638,
      "grad_norm": 49.9261057656198,
      "learning_rate": 5.498681898066783e-07,
      "logits/chosen": -2.4017577171325684,
      "logits/rejected": -2.4664063453674316,
      "logps/chosen": -432.7749938964844,
      "logps/rejected": -425.20001220703125,
      "loss": 0.1537,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.2880493104457855,
      "rewards/margins": 3.1739258766174316,
      "rewards/rejected": -3.460742235183716,
      "step": 2050
    },
    {
      "epoch": 1.8091368328574566,
      "grad_norm": 58.6073685918449,
      "learning_rate": 5.476713532513181e-07,
      "logits/chosen": -2.447070360183716,
      "logits/rejected": -2.388867139816284,
      "logps/chosen": -394.7875061035156,
      "logps/rejected": -365.70001220703125,
      "loss": 0.2107,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.3677978515625,
      "rewards/margins": 2.8511719703674316,
      "rewards/rejected": -3.21875,
      "step": 2060
    },
    {
      "epoch": 1.8179222490665494,
      "grad_norm": 40.4101827276486,
      "learning_rate": 5.454745166959577e-07,
      "logits/chosen": -2.440234422683716,
      "logits/rejected": -2.4462890625,
      "logps/chosen": -407.1875,
      "logps/rejected": -406.2749938964844,
      "loss": 0.1473,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.287994384765625,
      "rewards/margins": 3.131640672683716,
      "rewards/rejected": -3.4195313453674316,
      "step": 2070
    },
    {
      "epoch": 1.8267076652756424,
      "grad_norm": 44.75182411851845,
      "learning_rate": 5.432776801405975e-07,
      "logits/chosen": -2.4681639671325684,
      "logits/rejected": -2.523632764816284,
      "logps/chosen": -370.76251220703125,
      "logps/rejected": -337.6499938964844,
      "loss": 0.1767,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.04229431226849556,
      "rewards/margins": 2.709277391433716,
      "rewards/rejected": -2.666699171066284,
      "step": 2080
    },
    {
      "epoch": 1.8354930814847354,
      "grad_norm": 26.030317549545604,
      "learning_rate": 5.410808435852372e-07,
      "logits/chosen": -2.432812452316284,
      "logits/rejected": -2.4478516578674316,
      "logps/chosen": -396.67498779296875,
      "logps/rejected": -370.82501220703125,
      "loss": 0.1528,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.7069152593612671,
      "rewards/margins": 3.0625,
      "rewards/rejected": -2.354443311691284,
      "step": 2090
    },
    {
      "epoch": 1.8442784976938282,
      "grad_norm": 51.7558065520137,
      "learning_rate": 5.388840070298769e-07,
      "logits/chosen": -2.3687500953674316,
      "logits/rejected": -2.341601610183716,
      "logps/chosen": -387.8812561035156,
      "logps/rejected": -368.875,
      "loss": 0.1626,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.664508044719696,
      "rewards/margins": 3.00341796875,
      "rewards/rejected": -2.3397459983825684,
      "step": 2100
    },
    {
      "epoch": 1.853063913902921,
      "grad_norm": 43.28299588108422,
      "learning_rate": 5.366871704745168e-07,
      "logits/chosen": -2.518359422683716,
      "logits/rejected": -2.4732422828674316,
      "logps/chosen": -357.54998779296875,
      "logps/rejected": -349.54998779296875,
      "loss": 0.1819,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.659228503704071,
      "rewards/margins": 2.8306641578674316,
      "rewards/rejected": -2.17333984375,
      "step": 2110
    },
    {
      "epoch": 1.861849330112014,
      "grad_norm": 55.535820898333604,
      "learning_rate": 5.344903339191564e-07,
      "logits/chosen": -2.541015625,
      "logits/rejected": -2.4755859375,
      "logps/chosen": -350.9750061035156,
      "logps/rejected": -354.0249938964844,
      "loss": 0.1737,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.3622985780239105,
      "rewards/margins": 2.774609327316284,
      "rewards/rejected": -2.4117674827575684,
      "step": 2120
    },
    {
      "epoch": 1.870634746321107,
      "grad_norm": 46.771507560489,
      "learning_rate": 5.322934973637961e-07,
      "logits/chosen": -2.4986329078674316,
      "logits/rejected": -2.4828124046325684,
      "logps/chosen": -401.5,
      "logps/rejected": -371.6499938964844,
      "loss": 0.1521,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.4620361328125,
      "rewards/margins": 2.903515577316284,
      "rewards/rejected": -2.438671827316284,
      "step": 2130
    },
    {
      "epoch": 1.8794201625301998,
      "grad_norm": 38.79593223605629,
      "learning_rate": 5.300966608084358e-07,
      "logits/chosen": -2.480273485183716,
      "logits/rejected": -2.420117139816284,
      "logps/chosen": -427.54998779296875,
      "logps/rejected": -390.875,
      "loss": 0.1677,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.41336363554000854,
      "rewards/margins": 3.093945264816284,
      "rewards/rejected": -2.6812500953674316,
      "step": 2140
    },
    {
      "epoch": 1.8882055787392926,
      "grad_norm": 85.77698415941728,
      "learning_rate": 5.278998242530755e-07,
      "logits/chosen": -2.5132813453674316,
      "logits/rejected": -2.501953125,
      "logps/chosen": -395.67498779296875,
      "logps/rejected": -345.70001220703125,
      "loss": 0.1996,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.03468627855181694,
      "rewards/margins": 2.868457078933716,
      "rewards/rejected": -2.9013671875,
      "step": 2150
    },
    {
      "epoch": 1.8969909949483856,
      "grad_norm": 45.743054832187404,
      "learning_rate": 5.257029876977153e-07,
      "logits/chosen": -2.376171827316284,
      "logits/rejected": -2.3382811546325684,
      "logps/chosen": -392.8500061035156,
      "logps/rejected": -407.6000061035156,
      "loss": 0.155,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.3066467344760895,
      "rewards/margins": 3.000781297683716,
      "rewards/rejected": -2.6917967796325684,
      "step": 2160
    },
    {
      "epoch": 1.9057764111574786,
      "grad_norm": 30.621124715602033,
      "learning_rate": 5.235061511423549e-07,
      "logits/chosen": -2.483203172683716,
      "logits/rejected": -2.488085985183716,
      "logps/chosen": -410.13751220703125,
      "logps/rejected": -403.82501220703125,
      "loss": 0.1454,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.23139342665672302,
      "rewards/margins": 3.330273389816284,
      "rewards/rejected": -3.1001954078674316,
      "step": 2170
    },
    {
      "epoch": 1.9145618273665717,
      "grad_norm": 52.51505350594621,
      "learning_rate": 5.213093145869947e-07,
      "logits/chosen": -2.5287108421325684,
      "logits/rejected": -2.547070264816284,
      "logps/chosen": -410.8687438964844,
      "logps/rejected": -387.42498779296875,
      "loss": 0.1746,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 0.19741210341453552,
      "rewards/margins": 2.901562452316284,
      "rewards/rejected": -2.703418016433716,
      "step": 2180
    },
    {
      "epoch": 1.9233472435756644,
      "grad_norm": 99.6050299385793,
      "learning_rate": 5.191124780316343e-07,
      "logits/chosen": -2.408398389816284,
      "logits/rejected": -2.4214844703674316,
      "logps/chosen": -395.04998779296875,
      "logps/rejected": -352.9750061035156,
      "loss": 0.1894,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.538525402545929,
      "rewards/margins": 2.762500047683716,
      "rewards/rejected": -2.2242188453674316,
      "step": 2190
    },
    {
      "epoch": 1.9321326597847572,
      "grad_norm": 53.102679638694006,
      "learning_rate": 5.169156414762741e-07,
      "logits/chosen": -2.3792967796325684,
      "logits/rejected": -2.3451170921325684,
      "logps/chosen": -400.4750061035156,
      "logps/rejected": -396.6499938964844,
      "loss": 0.1603,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.606658935546875,
      "rewards/margins": 3.002734422683716,
      "rewards/rejected": -2.397509813308716,
      "step": 2200
    },
    {
      "epoch": 1.9409180759938502,
      "grad_norm": 28.61594904512125,
      "learning_rate": 5.147188049209139e-07,
      "logits/chosen": -2.408007860183716,
      "logits/rejected": -2.399609327316284,
      "logps/chosen": -394.82501220703125,
      "logps/rejected": -359.625,
      "loss": 0.1576,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.48228150606155396,
      "rewards/margins": 2.8558592796325684,
      "rewards/rejected": -2.3736329078674316,
      "step": 2210
    },
    {
      "epoch": 1.9497034922029433,
      "grad_norm": 31.272987080647297,
      "learning_rate": 5.125219683655536e-07,
      "logits/chosen": -2.4623045921325684,
      "logits/rejected": -2.4466795921325684,
      "logps/chosen": -390.2250061035156,
      "logps/rejected": -373.29998779296875,
      "loss": 0.1587,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.3359222412109375,
      "rewards/margins": 3.146484375,
      "rewards/rejected": -2.8089842796325684,
      "step": 2220
    },
    {
      "epoch": 1.958488908412036,
      "grad_norm": 72.38616942057737,
      "learning_rate": 5.103251318101933e-07,
      "logits/chosen": -2.4427733421325684,
      "logits/rejected": -2.456249952316284,
      "logps/chosen": -364.9750061035156,
      "logps/rejected": -356.45001220703125,
      "loss": 0.1732,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.08791504055261612,
      "rewards/margins": 2.90283203125,
      "rewards/rejected": -2.9912109375,
      "step": 2230
    },
    {
      "epoch": 1.9672743246211288,
      "grad_norm": 43.956074915367665,
      "learning_rate": 5.08128295254833e-07,
      "logits/chosen": -2.4837889671325684,
      "logits/rejected": -2.4916014671325684,
      "logps/chosen": -372.5249938964844,
      "logps/rejected": -375.9750061035156,
      "loss": 0.1666,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.22761841118335724,
      "rewards/margins": 2.883984327316284,
      "rewards/rejected": -3.1128907203674316,
      "step": 2240
    },
    {
      "epoch": 1.9760597408302218,
      "grad_norm": 37.37593249853592,
      "learning_rate": 5.059314586994727e-07,
      "logits/chosen": -2.4330077171325684,
      "logits/rejected": -2.335742235183716,
      "logps/chosen": -371.42498779296875,
      "logps/rejected": -363.5249938964844,
      "loss": 0.1497,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.2511047422885895,
      "rewards/margins": 3.072265625,
      "rewards/rejected": -3.325390577316284,
      "step": 2250
    },
    {
      "epoch": 1.9848451570393149,
      "grad_norm": 52.443482630455804,
      "learning_rate": 5.037346221441124e-07,
      "logits/chosen": -2.4554686546325684,
      "logits/rejected": -2.409374952316284,
      "logps/chosen": -389.8374938964844,
      "logps/rejected": -346.92498779296875,
      "loss": 0.1651,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.05180663987994194,
      "rewards/margins": 2.967968702316284,
      "rewards/rejected": -3.020703077316284,
      "step": 2260
    },
    {
      "epoch": 1.9936305732484076,
      "grad_norm": 44.56529783938918,
      "learning_rate": 5.015377855887521e-07,
      "logits/chosen": -2.447265625,
      "logits/rejected": -2.511523485183716,
      "logps/chosen": -441.1000061035156,
      "logps/rejected": -410.57501220703125,
      "loss": 0.133,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.01636962965130806,
      "rewards/margins": 3.2386717796325684,
      "rewards/rejected": -3.2591795921325684,
      "step": 2270
    },
    {
      "epoch": 2.001757083241819,
      "grad_norm": 15.184373206656998,
      "learning_rate": 4.993409490333919e-07,
      "logits/chosen": -2.410684108734131,
      "logits/rejected": -2.4556586742401123,
      "logps/chosen": -387.9459533691406,
      "logps/rejected": -388.8648681640625,
      "loss": 0.1133,
      "rewards/accuracies": 0.9797297120094299,
      "rewards/chosen": -0.04624815285205841,
      "rewards/margins": 3.2440879344940186,
      "rewards/rejected": -3.2911739349365234,
      "step": 2280
    },
    {
      "epoch": 2.0105424994509113,
      "grad_norm": 14.291580357294745,
      "learning_rate": 4.971441124780316e-07,
      "logits/chosen": -2.5474610328674316,
      "logits/rejected": -2.513867139816284,
      "logps/chosen": -375.7875061035156,
      "logps/rejected": -409.3500061035156,
      "loss": 0.0553,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03370361402630806,
      "rewards/margins": 4.026562690734863,
      "rewards/rejected": -4.059765815734863,
      "step": 2290
    },
    {
      "epoch": 2.0193279156600044,
      "grad_norm": 14.647599544242166,
      "learning_rate": 4.949472759226713e-07,
      "logits/chosen": -2.4033203125,
      "logits/rejected": -2.4830079078674316,
      "logps/chosen": -399.5874938964844,
      "logps/rejected": -394.13751220703125,
      "loss": 0.0557,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": 0.18491820991039276,
      "rewards/margins": 4.3798828125,
      "rewards/rejected": -4.199023246765137,
      "step": 2300
    },
    {
      "epoch": 2.0281133318690974,
      "grad_norm": 34.39023984818986,
      "learning_rate": 4.92750439367311e-07,
      "logits/chosen": -2.5208983421325684,
      "logits/rejected": -2.5835938453674316,
      "logps/chosen": -383.875,
      "logps/rejected": -349.625,
      "loss": 0.0465,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.05908813327550888,
      "rewards/margins": 4.152734279632568,
      "rewards/rejected": -4.214453220367432,
      "step": 2310
    },
    {
      "epoch": 2.0368987480781904,
      "grad_norm": 11.09249900840731,
      "learning_rate": 4.905536028119508e-07,
      "logits/chosen": -2.572265625,
      "logits/rejected": -2.5296874046325684,
      "logps/chosen": -390.04998779296875,
      "logps/rejected": -385.17498779296875,
      "loss": 0.042,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11704101413488388,
      "rewards/margins": 4.263671875,
      "rewards/rejected": -4.383203029632568,
      "step": 2320
    },
    {
      "epoch": 2.045684164287283,
      "grad_norm": 10.672371998223756,
      "learning_rate": 4.883567662565905e-07,
      "logits/chosen": -2.59765625,
      "logits/rejected": -2.5634765625,
      "logps/chosen": -372.61248779296875,
      "logps/rejected": -384.2250061035156,
      "loss": 0.0525,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.0966033935546875,
      "rewards/margins": 4.502734184265137,
      "rewards/rejected": -4.598046779632568,
      "step": 2330
    },
    {
      "epoch": 2.054469580496376,
      "grad_norm": 43.06390115854267,
      "learning_rate": 4.861599297012302e-07,
      "logits/chosen": -2.638671875,
      "logits/rejected": -2.668750047683716,
      "logps/chosen": -379.48748779296875,
      "logps/rejected": -369.5,
      "loss": 0.0383,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.34791260957717896,
      "rewards/margins": 4.609375,
      "rewards/rejected": -4.263671875,
      "step": 2340
    },
    {
      "epoch": 2.063254996705469,
      "grad_norm": 7.830484718228644,
      "learning_rate": 4.839630931458699e-07,
      "logits/chosen": -2.611132860183716,
      "logits/rejected": -2.651171922683716,
      "logps/chosen": -406.82501220703125,
      "logps/rejected": -402.0,
      "loss": 0.0337,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13344725966453552,
      "rewards/margins": 4.834765434265137,
      "rewards/rejected": -4.968359470367432,
      "step": 2350
    },
    {
      "epoch": 2.072040412914562,
      "grad_norm": 6.499640699746776,
      "learning_rate": 4.817662565905096e-07,
      "logits/chosen": -2.691210985183716,
      "logits/rejected": -2.6558594703674316,
      "logps/chosen": -382.9750061035156,
      "logps/rejected": -367.07501220703125,
      "loss": 0.0479,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.41387939453125,
      "rewards/margins": 4.496484279632568,
      "rewards/rejected": -4.912109375,
      "step": 2360
    },
    {
      "epoch": 2.0808258291236545,
      "grad_norm": 22.087808770845328,
      "learning_rate": 4.795694200351494e-07,
      "logits/chosen": -2.6015625,
      "logits/rejected": -2.6820311546325684,
      "logps/chosen": -361.63751220703125,
      "logps/rejected": -373.92498779296875,
      "loss": 0.0513,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.535144031047821,
      "rewards/margins": 4.251953125,
      "rewards/rejected": -4.786718845367432,
      "step": 2370
    },
    {
      "epoch": 2.0896112453327476,
      "grad_norm": 22.487583920185834,
      "learning_rate": 4.77372583479789e-07,
      "logits/chosen": -2.6722655296325684,
      "logits/rejected": -2.696093797683716,
      "logps/chosen": -408.6875,
      "logps/rejected": -453.07501220703125,
      "loss": 0.0648,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.525726318359375,
      "rewards/margins": 5.102734565734863,
      "rewards/rejected": -5.630859375,
      "step": 2380
    },
    {
      "epoch": 2.0983966615418406,
      "grad_norm": 60.12245075232196,
      "learning_rate": 4.751757469244288e-07,
      "logits/chosen": -2.569140672683716,
      "logits/rejected": -2.640820264816284,
      "logps/chosen": -410.95001220703125,
      "logps/rejected": -398.9750061035156,
      "loss": 0.0431,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.5954955816268921,
      "rewards/margins": 4.828515529632568,
      "rewards/rejected": -5.42578125,
      "step": 2390
    },
    {
      "epoch": 2.1071820777509336,
      "grad_norm": 22.4715045315629,
      "learning_rate": 4.729789103690685e-07,
      "logits/chosen": -2.5875000953674316,
      "logits/rejected": -2.6126952171325684,
      "logps/chosen": -405.70001220703125,
      "logps/rejected": -410.9750061035156,
      "loss": 0.0486,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.5542236566543579,
      "rewards/margins": 4.743359565734863,
      "rewards/rejected": -5.294531345367432,
      "step": 2400
    },
    {
      "epoch": 2.115967493960026,
      "grad_norm": 20.07465260780732,
      "learning_rate": 4.707820738137082e-07,
      "logits/chosen": -2.576367139816284,
      "logits/rejected": -2.7261719703674316,
      "logps/chosen": -397.20001220703125,
      "logps/rejected": -380.98748779296875,
      "loss": 0.0495,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.3844238221645355,
      "rewards/margins": 4.822265625,
      "rewards/rejected": -5.210156440734863,
      "step": 2410
    },
    {
      "epoch": 2.124752910169119,
      "grad_norm": 9.971262901875129,
      "learning_rate": 4.68585237258348e-07,
      "logits/chosen": -2.6539063453674316,
      "logits/rejected": -2.6337890625,
      "logps/chosen": -365.25,
      "logps/rejected": -394.1000061035156,
      "loss": 0.046,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.30754393339157104,
      "rewards/margins": 4.51171875,
      "rewards/rejected": -4.821093559265137,
      "step": 2420
    },
    {
      "epoch": 2.133538326378212,
      "grad_norm": 25.523620830729755,
      "learning_rate": 4.663884007029877e-07,
      "logits/chosen": -2.583984375,
      "logits/rejected": -2.681640625,
      "logps/chosen": -391.42498779296875,
      "logps/rejected": -416.8500061035156,
      "loss": 0.0425,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.719952404499054,
      "rewards/margins": 4.693359375,
      "rewards/rejected": -5.416406154632568,
      "step": 2430
    },
    {
      "epoch": 2.142323742587305,
      "grad_norm": 19.185683299195773,
      "learning_rate": 4.641915641476274e-07,
      "logits/chosen": -2.748242139816284,
      "logits/rejected": -2.8666014671325684,
      "logps/chosen": -364.2250061035156,
      "logps/rejected": -374.7749938964844,
      "loss": 0.0606,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.7808486819267273,
      "rewards/margins": 4.608984470367432,
      "rewards/rejected": -5.389062404632568,
      "step": 2440
    },
    {
      "epoch": 2.151109158796398,
      "grad_norm": 16.846983128535907,
      "learning_rate": 4.619947275922671e-07,
      "logits/chosen": -2.6626954078674316,
      "logits/rejected": -2.8179688453674316,
      "logps/chosen": -425.875,
      "logps/rejected": -433.625,
      "loss": 0.0529,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.2686889171600342,
      "rewards/margins": 4.783984184265137,
      "rewards/rejected": -6.055078029632568,
      "step": 2450
    },
    {
      "epoch": 2.1598945750054908,
      "grad_norm": 49.46230292418592,
      "learning_rate": 4.5979789103690687e-07,
      "logits/chosen": -2.7197265625,
      "logits/rejected": -2.7509765625,
      "logps/chosen": -406.8500061035156,
      "logps/rejected": -413.4750061035156,
      "loss": 0.0446,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.36591798067092896,
      "rewards/margins": 5.030859470367432,
      "rewards/rejected": -5.397265434265137,
      "step": 2460
    },
    {
      "epoch": 2.1686799912145838,
      "grad_norm": 8.363228899190744,
      "learning_rate": 4.576010544815466e-07,
      "logits/chosen": -2.6322264671325684,
      "logits/rejected": -2.6439452171325684,
      "logps/chosen": -412.57501220703125,
      "logps/rejected": -395.07501220703125,
      "loss": 0.032,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.37773436307907104,
      "rewards/margins": 4.747656345367432,
      "rewards/rejected": -5.125781059265137,
      "step": 2470
    },
    {
      "epoch": 2.177465407423677,
      "grad_norm": 10.506955944769931,
      "learning_rate": 4.554042179261863e-07,
      "logits/chosen": -2.7535157203674316,
      "logits/rejected": -2.855664014816284,
      "logps/chosen": -379.79998779296875,
      "logps/rejected": -400.25,
      "loss": 0.0638,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.43397217988967896,
      "rewards/margins": 4.967187404632568,
      "rewards/rejected": -5.402734279632568,
      "step": 2480
    },
    {
      "epoch": 2.18625082363277,
      "grad_norm": 16.46807513409746,
      "learning_rate": 4.53207381370826e-07,
      "logits/chosen": -2.565624952316284,
      "logits/rejected": -2.620312452316284,
      "logps/chosen": -387.07501220703125,
      "logps/rejected": -364.9750061035156,
      "loss": 0.0607,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.759350597858429,
      "rewards/margins": 4.755078315734863,
      "rewards/rejected": -5.513671875,
      "step": 2490
    },
    {
      "epoch": 2.1950362398418624,
      "grad_norm": 32.72026774779755,
      "learning_rate": 4.510105448154657e-07,
      "logits/chosen": -2.7269530296325684,
      "logits/rejected": -2.746875047683716,
      "logps/chosen": -388.4750061035156,
      "logps/rejected": -355.32501220703125,
      "loss": 0.0655,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.0762207508087158,
      "rewards/margins": 4.471875190734863,
      "rewards/rejected": -5.545702934265137,
      "step": 2500
    },
    {
      "epoch": 2.2038216560509554,
      "grad_norm": 25.91122200767686,
      "learning_rate": 4.4881370826010546e-07,
      "logits/chosen": -2.716796875,
      "logits/rejected": -2.7376952171325684,
      "logps/chosen": -450.1875,
      "logps/rejected": -407.2749938964844,
      "loss": 0.047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.236352562904358,
      "rewards/margins": 4.680468559265137,
      "rewards/rejected": -5.916015625,
      "step": 2510
    },
    {
      "epoch": 2.2126070722600484,
      "grad_norm": 17.11610574917694,
      "learning_rate": 4.4661687170474517e-07,
      "logits/chosen": -2.7015624046325684,
      "logits/rejected": -2.7025389671325684,
      "logps/chosen": -401.6000061035156,
      "logps/rejected": -403.75,
      "loss": 0.0361,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9945617914199829,
      "rewards/margins": 4.807031154632568,
      "rewards/rejected": -5.804296970367432,
      "step": 2520
    },
    {
      "epoch": 2.2213924884691414,
      "grad_norm": 15.002634919056705,
      "learning_rate": 4.444200351493849e-07,
      "logits/chosen": -2.8285155296325684,
      "logits/rejected": -2.8472657203674316,
      "logps/chosen": -426.375,
      "logps/rejected": -433.2250061035156,
      "loss": 0.0437,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.9117187261581421,
      "rewards/margins": 4.806250095367432,
      "rewards/rejected": -5.718359470367432,
      "step": 2530
    },
    {
      "epoch": 2.230177904678234,
      "grad_norm": 42.3319187264704,
      "learning_rate": 4.422231985940246e-07,
      "logits/chosen": -2.6519532203674316,
      "logits/rejected": -2.6669921875,
      "logps/chosen": -409.9750061035156,
      "logps/rejected": -423.1000061035156,
      "loss": 0.0522,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.924084484577179,
      "rewards/margins": 4.939062595367432,
      "rewards/rejected": -5.864062309265137,
      "step": 2540
    },
    {
      "epoch": 2.238963320887327,
      "grad_norm": 22.644186752720554,
      "learning_rate": 4.400263620386643e-07,
      "logits/chosen": -2.6869139671325684,
      "logits/rejected": -2.709179639816284,
      "logps/chosen": -353.8125,
      "logps/rejected": -341.75,
      "loss": 0.0524,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.5467468500137329,
      "rewards/margins": 4.492578029632568,
      "rewards/rejected": -5.041796684265137,
      "step": 2550
    },
    {
      "epoch": 2.24774873709642,
      "grad_norm": 23.262334078031042,
      "learning_rate": 4.3782952548330405e-07,
      "logits/chosen": -2.697265625,
      "logits/rejected": -2.603320360183716,
      "logps/chosen": -376.01251220703125,
      "logps/rejected": -393.125,
      "loss": 0.0507,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.537915050983429,
      "rewards/margins": 4.614843845367432,
      "rewards/rejected": -5.15234375,
      "step": 2560
    },
    {
      "epoch": 2.256534153305513,
      "grad_norm": 26.723862264253114,
      "learning_rate": 4.3563268892794376e-07,
      "logits/chosen": -2.744335889816284,
      "logits/rejected": -2.72265625,
      "logps/chosen": -363.75,
      "logps/rejected": -362.45001220703125,
      "loss": 0.046,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.6469360589981079,
      "rewards/margins": 4.748437404632568,
      "rewards/rejected": -5.396874904632568,
      "step": 2570
    },
    {
      "epoch": 2.265319569514606,
      "grad_norm": 34.516139722876794,
      "learning_rate": 4.3343585237258347e-07,
      "logits/chosen": -2.8062500953674316,
      "logits/rejected": -2.8382811546325684,
      "logps/chosen": -356.42498779296875,
      "logps/rejected": -367.70001220703125,
      "loss": 0.0574,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.5832763910293579,
      "rewards/margins": 4.70703125,
      "rewards/rejected": -5.294531345367432,
      "step": 2580
    },
    {
      "epoch": 2.2741049857236986,
      "grad_norm": 19.160255311987573,
      "learning_rate": 4.312390158172232e-07,
      "logits/chosen": -2.6314454078674316,
      "logits/rejected": -2.6484375,
      "logps/chosen": -381.88751220703125,
      "logps/rejected": -391.82501220703125,
      "loss": 0.0701,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.5466552972793579,
      "rewards/margins": 4.482812404632568,
      "rewards/rejected": -5.030077934265137,
      "step": 2590
    },
    {
      "epoch": 2.2828904019327916,
      "grad_norm": 11.017312673907828,
      "learning_rate": 4.2904217926186293e-07,
      "logits/chosen": -2.6986327171325684,
      "logits/rejected": -2.751171827316284,
      "logps/chosen": -406.7875061035156,
      "logps/rejected": -388.25,
      "loss": 0.0385,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.803057849407196,
      "rewards/margins": 4.793359279632568,
      "rewards/rejected": -5.598437309265137,
      "step": 2600
    },
    {
      "epoch": 2.2916758181418846,
      "grad_norm": 9.409411437595013,
      "learning_rate": 4.2684534270650264e-07,
      "logits/chosen": -2.662890672683716,
      "logits/rejected": -2.721874952316284,
      "logps/chosen": -426.82501220703125,
      "logps/rejected": -423.625,
      "loss": 0.0326,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.4748596251010895,
      "rewards/margins": 4.9453125,
      "rewards/rejected": -5.423828125,
      "step": 2610
    },
    {
      "epoch": 2.300461234350977,
      "grad_norm": 11.568370878534967,
      "learning_rate": 4.2464850615114235e-07,
      "logits/chosen": -2.6060547828674316,
      "logits/rejected": -2.8119139671325684,
      "logps/chosen": -431.3125,
      "logps/rejected": -377.6499938964844,
      "loss": 0.0351,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.3773437440395355,
      "rewards/margins": 4.812109470367432,
      "rewards/rejected": -5.189843654632568,
      "step": 2620
    },
    {
      "epoch": 2.30924665056007,
      "grad_norm": 11.938900536525669,
      "learning_rate": 4.2245166959578206e-07,
      "logits/chosen": -2.7134766578674316,
      "logits/rejected": -2.748046875,
      "logps/chosen": -388.8999938964844,
      "logps/rejected": -401.375,
      "loss": 0.0459,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.801098644733429,
      "rewards/margins": 5.166211128234863,
      "rewards/rejected": -5.970703125,
      "step": 2630
    },
    {
      "epoch": 2.318032066769163,
      "grad_norm": 21.18099988028898,
      "learning_rate": 4.2025483304042177e-07,
      "logits/chosen": -2.7523436546325684,
      "logits/rejected": -2.758593797683716,
      "logps/chosen": -403.6000061035156,
      "logps/rejected": -387.82501220703125,
      "loss": 0.0482,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.5114196538925171,
      "rewards/margins": 4.822656154632568,
      "rewards/rejected": -5.333203315734863,
      "step": 2640
    },
    {
      "epoch": 2.326817482978256,
      "grad_norm": 16.1724733711894,
      "learning_rate": 4.180579964850615e-07,
      "logits/chosen": -2.5669922828674316,
      "logits/rejected": -2.6841797828674316,
      "logps/chosen": -427.32501220703125,
      "logps/rejected": -406.17498779296875,
      "loss": 0.0401,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7243713140487671,
      "rewards/margins": 4.901171684265137,
      "rewards/rejected": -5.625781059265137,
      "step": 2650
    },
    {
      "epoch": 2.335602899187349,
      "grad_norm": 18.723424850069016,
      "learning_rate": 4.1586115992970123e-07,
      "logits/chosen": -2.6669921875,
      "logits/rejected": -2.6265625953674316,
      "logps/chosen": -407.6625061035156,
      "logps/rejected": -465.75,
      "loss": 0.0497,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.38238525390625,
      "rewards/margins": 4.845312595367432,
      "rewards/rejected": -5.231249809265137,
      "step": 2660
    },
    {
      "epoch": 2.3443883153964418,
      "grad_norm": 35.877605752385676,
      "learning_rate": 4.1366432337434094e-07,
      "logits/chosen": -2.66796875,
      "logits/rejected": -2.750781297683716,
      "logps/chosen": -420.5687561035156,
      "logps/rejected": -422.3125,
      "loss": 0.0571,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.2551513612270355,
      "rewards/margins": 4.534765720367432,
      "rewards/rejected": -4.787499904632568,
      "step": 2670
    },
    {
      "epoch": 2.353173731605535,
      "grad_norm": 43.3816326722385,
      "learning_rate": 4.1146748681898065e-07,
      "logits/chosen": -2.6615233421325684,
      "logits/rejected": -2.6314454078674316,
      "logps/chosen": -415.0249938964844,
      "logps/rejected": -405.42498779296875,
      "loss": 0.0386,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.593823254108429,
      "rewards/margins": 4.898046970367432,
      "rewards/rejected": -5.491015434265137,
      "step": 2680
    },
    {
      "epoch": 2.361959147814628,
      "grad_norm": 15.125806757802135,
      "learning_rate": 4.0927065026362036e-07,
      "logits/chosen": -2.645703077316284,
      "logits/rejected": -2.758593797683716,
      "logps/chosen": -364.67498779296875,
      "logps/rejected": -387.79998779296875,
      "loss": 0.045,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6376953125,
      "rewards/margins": 4.74609375,
      "rewards/rejected": -6.382031440734863,
      "step": 2690
    },
    {
      "epoch": 2.370744564023721,
      "grad_norm": 33.20852802321471,
      "learning_rate": 4.070738137082601e-07,
      "logits/chosen": -2.7017579078674316,
      "logits/rejected": -2.6490235328674316,
      "logps/chosen": -460.3999938964844,
      "logps/rejected": -460.875,
      "loss": 0.0437,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.209924340248108,
      "rewards/margins": 5.010937690734863,
      "rewards/rejected": -6.218359470367432,
      "step": 2700
    },
    {
      "epoch": 2.3795299802328134,
      "grad_norm": 23.092384706048207,
      "learning_rate": 4.048769771528998e-07,
      "logits/chosen": -2.734179735183716,
      "logits/rejected": -2.712109327316284,
      "logps/chosen": -445.7749938964844,
      "logps/rejected": -464.8500061035156,
      "loss": 0.0428,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.517602562904358,
      "rewards/margins": 5.106249809265137,
      "rewards/rejected": -6.622656345367432,
      "step": 2710
    },
    {
      "epoch": 2.3883153964419064,
      "grad_norm": 9.616055917614172,
      "learning_rate": 4.0268014059753953e-07,
      "logits/chosen": -2.5953125953674316,
      "logits/rejected": -2.591015577316284,
      "logps/chosen": -401.75,
      "logps/rejected": -419.42498779296875,
      "loss": 0.0322,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2724120616912842,
      "rewards/margins": 5.057421684265137,
      "rewards/rejected": -6.329297065734863,
      "step": 2720
    },
    {
      "epoch": 2.3971008126509994,
      "grad_norm": 16.46133003580557,
      "learning_rate": 4.0048330404217924e-07,
      "logits/chosen": -2.7261719703674316,
      "logits/rejected": -2.7533202171325684,
      "logps/chosen": -436.75,
      "logps/rejected": -429.67498779296875,
      "loss": 0.0351,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.912945568561554,
      "rewards/margins": 4.973437309265137,
      "rewards/rejected": -5.885156154632568,
      "step": 2730
    },
    {
      "epoch": 2.4058862288600924,
      "grad_norm": 69.17610942447253,
      "learning_rate": 3.98286467486819e-07,
      "logits/chosen": -2.5986328125,
      "logits/rejected": -2.7037110328674316,
      "logps/chosen": -436.6499938964844,
      "logps/rejected": -427.6000061035156,
      "loss": 0.0415,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.6945556402206421,
      "rewards/margins": 5.03515625,
      "rewards/rejected": -5.727734565734863,
      "step": 2740
    },
    {
      "epoch": 2.414671645069185,
      "grad_norm": 5.299541236219124,
      "learning_rate": 3.960896309314587e-07,
      "logits/chosen": -2.6546874046325684,
      "logits/rejected": -2.711718797683716,
      "logps/chosen": -385.3125,
      "logps/rejected": -401.17498779296875,
      "loss": 0.0297,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7743285894393921,
      "rewards/margins": 4.955468654632568,
      "rewards/rejected": -5.729296684265137,
      "step": 2750
    },
    {
      "epoch": 2.423457061278278,
      "grad_norm": 13.550851289570062,
      "learning_rate": 3.938927943760984e-07,
      "logits/chosen": -2.739062547683716,
      "logits/rejected": -2.784960985183716,
      "logps/chosen": -394.76251220703125,
      "logps/rejected": -419.875,
      "loss": 0.0458,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.39862060546875,
      "rewards/margins": 5.252734184265137,
      "rewards/rejected": -5.650781154632568,
      "step": 2760
    },
    {
      "epoch": 2.432242477487371,
      "grad_norm": 38.71907004988617,
      "learning_rate": 3.916959578207381e-07,
      "logits/chosen": -2.6107420921325684,
      "logits/rejected": -2.6167969703674316,
      "logps/chosen": -398.26251220703125,
      "logps/rejected": -389.42498779296875,
      "loss": 0.0533,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.5092010498046875,
      "rewards/margins": 5.214062690734863,
      "rewards/rejected": -5.724999904632568,
      "step": 2770
    },
    {
      "epoch": 2.441027893696464,
      "grad_norm": 59.19132382311149,
      "learning_rate": 3.8949912126537783e-07,
      "logits/chosen": -2.6714844703674316,
      "logits/rejected": -2.6500000953674316,
      "logps/chosen": -384.38751220703125,
      "logps/rejected": -406.0,
      "loss": 0.0478,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.719464123249054,
      "rewards/margins": 4.885156154632568,
      "rewards/rejected": -5.604687690734863,
      "step": 2780
    },
    {
      "epoch": 2.449813309905557,
      "grad_norm": 18.013187619561155,
      "learning_rate": 3.873022847100176e-07,
      "logits/chosen": -2.636523485183716,
      "logits/rejected": -2.6763672828674316,
      "logps/chosen": -348.79998779296875,
      "logps/rejected": -379.57501220703125,
      "loss": 0.0659,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.2472991943359375,
      "rewards/margins": 4.451952934265137,
      "rewards/rejected": -5.695703029632568,
      "step": 2790
    },
    {
      "epoch": 2.4585987261146496,
      "grad_norm": 51.437322123015306,
      "learning_rate": 3.851054481546573e-07,
      "logits/chosen": -2.632617235183716,
      "logits/rejected": -2.6800780296325684,
      "logps/chosen": -451.82501220703125,
      "logps/rejected": -434.6000061035156,
      "loss": 0.0599,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.404541015625,
      "rewards/margins": 4.536718845367432,
      "rewards/rejected": -5.938672065734863,
      "step": 2800
    },
    {
      "epoch": 2.4673841423237426,
      "grad_norm": 12.380136051100894,
      "learning_rate": 3.82908611599297e-07,
      "logits/chosen": -2.7972655296325684,
      "logits/rejected": -2.7750000953674316,
      "logps/chosen": -367.3999938964844,
      "logps/rejected": -402.04998779296875,
      "loss": 0.0404,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.36090087890625,
      "rewards/margins": 5.0625,
      "rewards/rejected": -6.421875,
      "step": 2810
    },
    {
      "epoch": 2.4761695585328356,
      "grad_norm": 30.86678455122048,
      "learning_rate": 3.807117750439367e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.7457032203674316,
      "logps/chosen": -362.5249938964844,
      "logps/rejected": -384.82501220703125,
      "loss": 0.0442,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.364892601966858,
      "rewards/margins": 4.896093845367432,
      "rewards/rejected": -6.262499809265137,
      "step": 2820
    },
    {
      "epoch": 2.484954974741928,
      "grad_norm": 44.03151808165407,
      "learning_rate": 3.785149384885764e-07,
      "logits/chosen": -2.6714844703674316,
      "logits/rejected": -2.739062547683716,
      "logps/chosen": -421.92498779296875,
      "logps/rejected": -425.67498779296875,
      "loss": 0.0319,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.7667602300643921,
      "rewards/margins": 5.266797065734863,
      "rewards/rejected": -6.035546779632568,
      "step": 2830
    },
    {
      "epoch": 2.493740390951021,
      "grad_norm": 20.145167535010252,
      "learning_rate": 3.763181019332162e-07,
      "logits/chosen": -2.576171875,
      "logits/rejected": -2.601367235183716,
      "logps/chosen": -351.2875061035156,
      "logps/rejected": -410.7749938964844,
      "loss": 0.0403,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0316650867462158,
      "rewards/margins": 5.071093559265137,
      "rewards/rejected": -6.104687690734863,
      "step": 2840
    },
    {
      "epoch": 2.502525807160114,
      "grad_norm": 12.077918386736718,
      "learning_rate": 3.741212653778559e-07,
      "logits/chosen": -2.7992186546325684,
      "logits/rejected": -2.7455077171325684,
      "logps/chosen": -388.29998779296875,
      "logps/rejected": -410.04998779296875,
      "loss": 0.0423,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.75726318359375,
      "rewards/margins": 4.87890625,
      "rewards/rejected": -5.639843940734863,
      "step": 2850
    },
    {
      "epoch": 2.511311223369207,
      "grad_norm": 16.143867749980863,
      "learning_rate": 3.719244288224956e-07,
      "logits/chosen": -2.8597655296325684,
      "logits/rejected": -2.825000047683716,
      "logps/chosen": -389.2250061035156,
      "logps/rejected": -423.9750061035156,
      "loss": 0.0379,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.6396087408065796,
      "rewards/margins": 4.991406440734863,
      "rewards/rejected": -5.633593559265137,
      "step": 2860
    },
    {
      "epoch": 2.5200966395783,
      "grad_norm": 5.782134279964404,
      "learning_rate": 3.697275922671353e-07,
      "logits/chosen": -2.766796827316284,
      "logits/rejected": -2.727734327316284,
      "logps/chosen": -382.1000061035156,
      "logps/rejected": -431.0,
      "loss": 0.0454,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.7990478277206421,
      "rewards/margins": 4.989843845367432,
      "rewards/rejected": -5.787109375,
      "step": 2870
    },
    {
      "epoch": 2.528882055787393,
      "grad_norm": 22.52328793488939,
      "learning_rate": 3.6753075571177507e-07,
      "logits/chosen": -2.7994141578674316,
      "logits/rejected": -2.852343797683716,
      "logps/chosen": -346.48748779296875,
      "logps/rejected": -357.7749938964844,
      "loss": 0.0598,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.912628173828125,
      "rewards/margins": 4.460546970367432,
      "rewards/rejected": -5.37109375,
      "step": 2880
    },
    {
      "epoch": 2.537667471996486,
      "grad_norm": 70.09378094280417,
      "learning_rate": 3.653339191564148e-07,
      "logits/chosen": -2.698437452316284,
      "logits/rejected": -2.8798828125,
      "logps/chosen": -409.0249938964844,
      "logps/rejected": -363.32501220703125,
      "loss": 0.0547,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.930908203125,
      "rewards/margins": 4.943749904632568,
      "rewards/rejected": -5.877343654632568,
      "step": 2890
    },
    {
      "epoch": 2.546452888205579,
      "grad_norm": 49.61426934944271,
      "learning_rate": 3.631370826010545e-07,
      "logits/chosen": -2.698437452316284,
      "logits/rejected": -2.866406202316284,
      "logps/chosen": -396.3999938964844,
      "logps/rejected": -391.3999938964844,
      "loss": 0.0481,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.314477562904358,
      "rewards/margins": 4.940625190734863,
      "rewards/rejected": -6.256249904632568,
      "step": 2900
    },
    {
      "epoch": 2.5552383044146714,
      "grad_norm": 39.943322808985535,
      "learning_rate": 3.609402460456942e-07,
      "logits/chosen": -2.680859327316284,
      "logits/rejected": -2.79296875,
      "logps/chosen": -441.29998779296875,
      "logps/rejected": -448.29998779296875,
      "loss": 0.0559,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.63739013671875,
      "rewards/margins": 5.074609279632568,
      "rewards/rejected": -6.7109375,
      "step": 2910
    },
    {
      "epoch": 2.564023720623765,
      "grad_norm": 15.675409728738032,
      "learning_rate": 3.587434094903339e-07,
      "logits/chosen": -2.80078125,
      "logits/rejected": -2.857617139816284,
      "logps/chosen": -433.82501220703125,
      "logps/rejected": -413.4750061035156,
      "loss": 0.0512,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.395654320716858,
      "rewards/margins": 5.4140625,
      "rewards/rejected": -6.810937404632568,
      "step": 2920
    },
    {
      "epoch": 2.5728091368328574,
      "grad_norm": 2.8974448427101533,
      "learning_rate": 3.5654657293497366e-07,
      "logits/chosen": -2.7582030296325684,
      "logits/rejected": -2.809375047683716,
      "logps/chosen": -425.6000061035156,
      "logps/rejected": -441.6000061035156,
      "loss": 0.0347,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7211425304412842,
      "rewards/margins": 5.20703125,
      "rewards/rejected": -6.92578125,
      "step": 2930
    },
    {
      "epoch": 2.5815945530419504,
      "grad_norm": 11.730269760539837,
      "learning_rate": 3.5434973637961337e-07,
      "logits/chosen": -2.73046875,
      "logits/rejected": -2.71484375,
      "logps/chosen": -405.9375,
      "logps/rejected": -468.32501220703125,
      "loss": 0.0426,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.5612061023712158,
      "rewards/margins": 5.005468845367432,
      "rewards/rejected": -6.564062595367432,
      "step": 2940
    },
    {
      "epoch": 2.5903799692510434,
      "grad_norm": 8.66121984309592,
      "learning_rate": 3.521528998242531e-07,
      "logits/chosen": -2.789843797683716,
      "logits/rejected": -2.828320264816284,
      "logps/chosen": -393.01251220703125,
      "logps/rejected": -398.6000061035156,
      "loss": 0.0509,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.289636254310608,
      "rewards/margins": 4.876562595367432,
      "rewards/rejected": -6.16796875,
      "step": 2950
    },
    {
      "epoch": 2.599165385460136,
      "grad_norm": 9.122808047643048,
      "learning_rate": 3.499560632688928e-07,
      "logits/chosen": -2.759765625,
      "logits/rejected": -2.8511719703674316,
      "logps/chosen": -458.29998779296875,
      "logps/rejected": -421.3999938964844,
      "loss": 0.0326,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3362548351287842,
      "rewards/margins": 5.202343940734863,
      "rewards/rejected": -6.539843559265137,
      "step": 2960
    },
    {
      "epoch": 2.607950801669229,
      "grad_norm": 50.23379225777122,
      "learning_rate": 3.477592267135325e-07,
      "logits/chosen": -2.710742235183716,
      "logits/rejected": -2.8388671875,
      "logps/chosen": -425.625,
      "logps/rejected": -412.61248779296875,
      "loss": 0.073,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.6255004405975342,
      "rewards/margins": 4.746484279632568,
      "rewards/rejected": -6.37109375,
      "step": 2970
    },
    {
      "epoch": 2.616736217878322,
      "grad_norm": 71.29793370702616,
      "learning_rate": 3.4556239015817225e-07,
      "logits/chosen": -2.7398438453674316,
      "logits/rejected": -2.8111329078674316,
      "logps/chosen": -390.67498779296875,
      "logps/rejected": -368.0249938964844,
      "loss": 0.0494,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.0258057117462158,
      "rewards/margins": 4.970703125,
      "rewards/rejected": -5.996484279632568,
      "step": 2980
    },
    {
      "epoch": 2.625521634087415,
      "grad_norm": 37.60487091732075,
      "learning_rate": 3.4336555360281196e-07,
      "logits/chosen": -2.6259765625,
      "logits/rejected": -2.5619139671325684,
      "logps/chosen": -447.2749938964844,
      "logps/rejected": -455.2250061035156,
      "loss": 0.0468,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.791430652141571,
      "rewards/margins": 5.051953315734863,
      "rewards/rejected": -5.841406345367432,
      "step": 2990
    },
    {
      "epoch": 2.634307050296508,
      "grad_norm": 15.551261211600231,
      "learning_rate": 3.4116871704745167e-07,
      "logits/chosen": -2.720703125,
      "logits/rejected": -2.679882764816284,
      "logps/chosen": -391.1499938964844,
      "logps/rejected": -435.4750061035156,
      "loss": 0.0313,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.610949695110321,
      "rewards/margins": 5.117968559265137,
      "rewards/rejected": -5.730859279632568,
      "step": 3000
    },
    {
      "epoch": 2.6430924665056006,
      "grad_norm": 22.331223871807886,
      "learning_rate": 3.389718804920914e-07,
      "logits/chosen": -2.685351610183716,
      "logits/rejected": -2.703125,
      "logps/chosen": -411.45001220703125,
      "logps/rejected": -443.17498779296875,
      "loss": 0.0403,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.471716284751892,
      "rewards/margins": 5.400000095367432,
      "rewards/rejected": -6.874218940734863,
      "step": 3010
    },
    {
      "epoch": 2.6518778827146936,
      "grad_norm": 7.998792049337297,
      "learning_rate": 3.3677504393673114e-07,
      "logits/chosen": -2.762890577316284,
      "logits/rejected": -2.779296875,
      "logps/chosen": -408.9624938964844,
      "logps/rejected": -401.5,
      "loss": 0.0419,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5364196300506592,
      "rewards/margins": 5.205078125,
      "rewards/rejected": -6.741406440734863,
      "step": 3020
    },
    {
      "epoch": 2.6606632989237866,
      "grad_norm": 15.104998427027214,
      "learning_rate": 3.3457820738137084e-07,
      "logits/chosen": -2.685351610183716,
      "logits/rejected": -2.7392578125,
      "logps/chosen": -382.25,
      "logps/rejected": -438.20001220703125,
      "loss": 0.0471,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.509423851966858,
      "rewards/margins": 5.114843845367432,
      "rewards/rejected": -6.627343654632568,
      "step": 3030
    },
    {
      "epoch": 2.669448715132879,
      "grad_norm": 29.63290893811304,
      "learning_rate": 3.3238137082601055e-07,
      "logits/chosen": -2.7796874046325684,
      "logits/rejected": -2.759960889816284,
      "logps/chosen": -378.51251220703125,
      "logps/rejected": -401.5,
      "loss": 0.0486,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.538934350013733,
      "rewards/margins": 5.37890625,
      "rewards/rejected": -6.922656059265137,
      "step": 3040
    },
    {
      "epoch": 2.678234131341972,
      "grad_norm": 44.69897092699583,
      "learning_rate": 3.3018453427065026e-07,
      "logits/chosen": -2.7242188453674316,
      "logits/rejected": -2.8453125953674316,
      "logps/chosen": -399.2250061035156,
      "logps/rejected": -366.20001220703125,
      "loss": 0.0547,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.3238646984100342,
      "rewards/margins": 4.682031154632568,
      "rewards/rejected": -6.003125190734863,
      "step": 3050
    },
    {
      "epoch": 2.687019547551065,
      "grad_norm": 26.543315751187205,
      "learning_rate": 3.2798769771528997e-07,
      "logits/chosen": -2.757031202316284,
      "logits/rejected": -2.737109422683716,
      "logps/chosen": -389.51251220703125,
      "logps/rejected": -418.67498779296875,
      "loss": 0.0642,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.993115246295929,
      "rewards/margins": 4.991991996765137,
      "rewards/rejected": -5.982031345367432,
      "step": 3060
    },
    {
      "epoch": 2.695804963760158,
      "grad_norm": 14.018106882602998,
      "learning_rate": 3.2579086115992973e-07,
      "logits/chosen": -2.6822266578674316,
      "logits/rejected": -2.645703077316284,
      "logps/chosen": -365.29998779296875,
      "logps/rejected": -409.92498779296875,
      "loss": 0.0566,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.38104248046875,
      "rewards/margins": 5.340234279632568,
      "rewards/rejected": -6.721093654632568,
      "step": 3070
    },
    {
      "epoch": 2.7045903799692512,
      "grad_norm": 20.553847654800308,
      "learning_rate": 3.2359402460456944e-07,
      "logits/chosen": -2.719531297683716,
      "logits/rejected": -2.7679686546325684,
      "logps/chosen": -408.2749938964844,
      "logps/rejected": -398.82501220703125,
      "loss": 0.0493,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.053466796875,
      "rewards/margins": 5.108593940734863,
      "rewards/rejected": -6.1640625,
      "step": 3080
    },
    {
      "epoch": 2.713375796178344,
      "grad_norm": 9.044981625525146,
      "learning_rate": 3.2139718804920914e-07,
      "logits/chosen": -2.776171922683716,
      "logits/rejected": -2.8203125,
      "logps/chosen": -411.125,
      "logps/rejected": -420.29998779296875,
      "loss": 0.0516,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.6521255373954773,
      "rewards/margins": 5.012890815734863,
      "rewards/rejected": -5.666796684265137,
      "step": 3090
    },
    {
      "epoch": 2.722161212387437,
      "grad_norm": 87.60389202749188,
      "learning_rate": 3.1920035149384885e-07,
      "logits/chosen": -2.701171875,
      "logits/rejected": -2.7162108421325684,
      "logps/chosen": -426.5249938964844,
      "logps/rejected": -416.1499938964844,
      "loss": 0.0394,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.977893054485321,
      "rewards/margins": 4.882421970367432,
      "rewards/rejected": -5.859375,
      "step": 3100
    },
    {
      "epoch": 2.73094662859653,
      "grad_norm": 43.13679529326943,
      "learning_rate": 3.1700351493848856e-07,
      "logits/chosen": -2.7623047828674316,
      "logits/rejected": -2.801953077316284,
      "logps/chosen": -381.20001220703125,
      "logps/rejected": -409.32501220703125,
      "loss": 0.0624,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.951708972454071,
      "rewards/margins": 5.101953029632568,
      "rewards/rejected": -6.053515434265137,
      "step": 3110
    },
    {
      "epoch": 2.739732044805623,
      "grad_norm": 45.109635036091476,
      "learning_rate": 3.148066783831283e-07,
      "logits/chosen": -2.639843702316284,
      "logits/rejected": -2.675976514816284,
      "logps/chosen": -437.67498779296875,
      "logps/rejected": -440.3999938964844,
      "loss": 0.0375,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.116064429283142,
      "rewards/margins": 5.263281345367432,
      "rewards/rejected": -6.380468845367432,
      "step": 3120
    },
    {
      "epoch": 2.748517461014716,
      "grad_norm": 29.105294983279684,
      "learning_rate": 3.1260984182776803e-07,
      "logits/chosen": -2.6900391578674316,
      "logits/rejected": -2.8199219703674316,
      "logps/chosen": -443.32501220703125,
      "logps/rejected": -382.75,
      "loss": 0.0326,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.3930175304412842,
      "rewards/margins": 5.244921684265137,
      "rewards/rejected": -6.640625,
      "step": 3130
    },
    {
      "epoch": 2.7573028772238084,
      "grad_norm": 29.946010829550332,
      "learning_rate": 3.1041300527240773e-07,
      "logits/chosen": -2.6470704078674316,
      "logits/rejected": -2.6939454078674316,
      "logps/chosen": -485.75,
      "logps/rejected": -436.32501220703125,
      "loss": 0.0336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.318823218345642,
      "rewards/margins": 5.344922065734863,
      "rewards/rejected": -6.665234565734863,
      "step": 3140
    },
    {
      "epoch": 2.7660882934329014,
      "grad_norm": 33.39553204204725,
      "learning_rate": 3.0821616871704744e-07,
      "logits/chosen": -2.7046875953674316,
      "logits/rejected": -2.7845702171325684,
      "logps/chosen": -381.7749938964844,
      "logps/rejected": -397.3500061035156,
      "loss": 0.0459,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.434777855873108,
      "rewards/margins": 4.907031059265137,
      "rewards/rejected": -6.342968940734863,
      "step": 3150
    },
    {
      "epoch": 2.7748737096419944,
      "grad_norm": 19.1605025424591,
      "learning_rate": 3.060193321616872e-07,
      "logits/chosen": -2.7349610328674316,
      "logits/rejected": -2.7730469703674316,
      "logps/chosen": -422.54998779296875,
      "logps/rejected": -455.125,
      "loss": 0.0461,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.508056640625,
      "rewards/margins": 5.016406059265137,
      "rewards/rejected": -6.5234375,
      "step": 3160
    },
    {
      "epoch": 2.783659125851087,
      "grad_norm": 13.31371372313935,
      "learning_rate": 3.038224956063269e-07,
      "logits/chosen": -2.7265625,
      "logits/rejected": -2.6888670921325684,
      "logps/chosen": -398.48748779296875,
      "logps/rejected": -381.6000061035156,
      "loss": 0.0429,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.484948754310608,
      "rewards/margins": 5.115234375,
      "rewards/rejected": -6.597265720367432,
      "step": 3170
    },
    {
      "epoch": 2.79244454206018,
      "grad_norm": 14.536292883845247,
      "learning_rate": 3.016256590509666e-07,
      "logits/chosen": -2.829296827316284,
      "logits/rejected": -2.915234327316284,
      "logps/chosen": -391.1000061035156,
      "logps/rejected": -420.25,
      "loss": 0.0496,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.234838843345642,
      "rewards/margins": 5.154687404632568,
      "rewards/rejected": -6.392187595367432,
      "step": 3180
    },
    {
      "epoch": 2.801229958269273,
      "grad_norm": 10.9728977514854,
      "learning_rate": 2.994288224956063e-07,
      "logits/chosen": -2.752148389816284,
      "logits/rejected": -2.8863282203674316,
      "logps/chosen": -418.8374938964844,
      "logps/rejected": -405.42498779296875,
      "loss": 0.0313,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.494970679283142,
      "rewards/margins": 5.36328125,
      "rewards/rejected": -6.859375,
      "step": 3190
    },
    {
      "epoch": 2.810015374478366,
      "grad_norm": 13.638256200294474,
      "learning_rate": 2.9723198594024603e-07,
      "logits/chosen": -2.7744140625,
      "logits/rejected": -2.8394532203674316,
      "logps/chosen": -353.6000061035156,
      "logps/rejected": -365.7749938964844,
      "loss": 0.0585,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.4029114246368408,
      "rewards/margins": 4.899609565734863,
      "rewards/rejected": -6.298828125,
      "step": 3200
    },
    {
      "epoch": 2.818800790687459,
      "grad_norm": 42.04698046018047,
      "learning_rate": 2.950351493848858e-07,
      "logits/chosen": -2.793164014816284,
      "logits/rejected": -2.829296827316284,
      "logps/chosen": -394.45001220703125,
      "logps/rejected": -410.82501220703125,
      "loss": 0.039,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.3771483898162842,
      "rewards/margins": 5.044921875,
      "rewards/rejected": -6.421093940734863,
      "step": 3210
    },
    {
      "epoch": 2.8275862068965516,
      "grad_norm": 17.536217302081866,
      "learning_rate": 2.928383128295255e-07,
      "logits/chosen": -2.8714842796325684,
      "logits/rejected": -2.8101563453674316,
      "logps/chosen": -352.07501220703125,
      "logps/rejected": -411.79998779296875,
      "loss": 0.0327,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.1515624523162842,
      "rewards/margins": 4.768750190734863,
      "rewards/rejected": -5.921875,
      "step": 3220
    },
    {
      "epoch": 2.8363716231056446,
      "grad_norm": 9.881099306910587,
      "learning_rate": 2.906414762741652e-07,
      "logits/chosen": -2.792773485183716,
      "logits/rejected": -2.872265577316284,
      "logps/chosen": -423.125,
      "logps/rejected": -403.92498779296875,
      "loss": 0.0556,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.4213135242462158,
      "rewards/margins": 5.029296875,
      "rewards/rejected": -6.449609279632568,
      "step": 3230
    },
    {
      "epoch": 2.8451570393147376,
      "grad_norm": 83.4920122580327,
      "learning_rate": 2.884446397188049e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.8226561546325684,
      "logps/chosen": -416.01251220703125,
      "logps/rejected": -451.625,
      "loss": 0.0368,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3373534679412842,
      "rewards/margins": 5.317187309265137,
      "rewards/rejected": -6.653906345367432,
      "step": 3240
    },
    {
      "epoch": 2.85394245552383,
      "grad_norm": 21.933468360845836,
      "learning_rate": 2.862478031634446e-07,
      "logits/chosen": -2.795117139816284,
      "logits/rejected": -2.8929686546325684,
      "logps/chosen": -453.7250061035156,
      "logps/rejected": -390.1499938964844,
      "loss": 0.056,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.9015624523162842,
      "rewards/margins": 5.044921875,
      "rewards/rejected": -6.946875095367432,
      "step": 3250
    },
    {
      "epoch": 2.862727871732923,
      "grad_norm": 16.986129320652044,
      "learning_rate": 2.840509666080844e-07,
      "logits/chosen": -2.8720703125,
      "logits/rejected": -2.905078172683716,
      "logps/chosen": -380.67498779296875,
      "logps/rejected": -420.5,
      "loss": 0.0427,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5536178350448608,
      "rewards/margins": 5.063281059265137,
      "rewards/rejected": -6.615624904632568,
      "step": 3260
    },
    {
      "epoch": 2.871513287942016,
      "grad_norm": 52.66072637706807,
      "learning_rate": 2.818541300527241e-07,
      "logits/chosen": -2.7847657203674316,
      "logits/rejected": -2.9400391578674316,
      "logps/chosen": -402.875,
      "logps/rejected": -402.9624938964844,
      "loss": 0.043,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.70916748046875,
      "rewards/margins": 5.33984375,
      "rewards/rejected": -7.047656059265137,
      "step": 3270
    },
    {
      "epoch": 2.8802987041511092,
      "grad_norm": 5.0012609757092985,
      "learning_rate": 2.796572934973638e-07,
      "logits/chosen": -2.745312452316284,
      "logits/rejected": -2.819140672683716,
      "logps/chosen": -421.5625,
      "logps/rejected": -434.7250061035156,
      "loss": 0.0386,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.4912109375,
      "rewards/margins": 5.598828315734863,
      "rewards/rejected": -7.089062690734863,
      "step": 3280
    },
    {
      "epoch": 2.8890841203602022,
      "grad_norm": 9.664810502562544,
      "learning_rate": 2.774604569420035e-07,
      "logits/chosen": -2.9156250953674316,
      "logits/rejected": -2.8929686546325684,
      "logps/chosen": -435.95001220703125,
      "logps/rejected": -432.375,
      "loss": 0.0348,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.474755883216858,
      "rewards/margins": 5.571093559265137,
      "rewards/rejected": -7.042187690734863,
      "step": 3290
    },
    {
      "epoch": 2.897869536569295,
      "grad_norm": 31.71618131960354,
      "learning_rate": 2.7526362038664327e-07,
      "logits/chosen": -2.776562452316284,
      "logits/rejected": -2.9027342796325684,
      "logps/chosen": -408.2562561035156,
      "logps/rejected": -416.625,
      "loss": 0.0344,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9384765625,
      "rewards/margins": 5.226953029632568,
      "rewards/rejected": -7.163281440734863,
      "step": 3300
    },
    {
      "epoch": 2.906654952778388,
      "grad_norm": 21.793097634919665,
      "learning_rate": 2.73066783831283e-07,
      "logits/chosen": -2.6724610328674316,
      "logits/rejected": -2.8746094703674316,
      "logps/chosen": -457.8125,
      "logps/rejected": -406.82501220703125,
      "loss": 0.0351,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.762426733970642,
      "rewards/margins": 5.191015720367432,
      "rewards/rejected": -6.955468654632568,
      "step": 3310
    },
    {
      "epoch": 2.915440368987481,
      "grad_norm": 17.318980080793725,
      "learning_rate": 2.708699472759227e-07,
      "logits/chosen": -2.840625047683716,
      "logits/rejected": -2.885546922683716,
      "logps/chosen": -387.04998779296875,
      "logps/rejected": -376.57501220703125,
      "loss": 0.0626,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.1016602516174316,
      "rewards/margins": 4.987890720367432,
      "rewards/rejected": -7.090624809265137,
      "step": 3320
    },
    {
      "epoch": 2.924225785196574,
      "grad_norm": 21.084019466466707,
      "learning_rate": 2.686731107205624e-07,
      "logits/chosen": -2.860546827316284,
      "logits/rejected": -2.852734327316284,
      "logps/chosen": -384.20001220703125,
      "logps/rejected": -397.54998779296875,
      "loss": 0.0386,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4193236827850342,
      "rewards/margins": 4.787109375,
      "rewards/rejected": -6.205859184265137,
      "step": 3330
    },
    {
      "epoch": 2.933011201405667,
      "grad_norm": 13.430067258552999,
      "learning_rate": 2.664762741652021e-07,
      "logits/chosen": -2.7611327171325684,
      "logits/rejected": -2.88671875,
      "logps/chosen": -398.04998779296875,
      "logps/rejected": -391.2749938964844,
      "loss": 0.0415,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.2623779773712158,
      "rewards/margins": 5.0234375,
      "rewards/rejected": -6.283203125,
      "step": 3340
    },
    {
      "epoch": 2.9417966176147594,
      "grad_norm": 44.41713731360837,
      "learning_rate": 2.6427943760984186e-07,
      "logits/chosen": -2.6878905296325684,
      "logits/rejected": -2.743359327316284,
      "logps/chosen": -423.67498779296875,
      "logps/rejected": -421.79998779296875,
      "loss": 0.0607,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.1527862548828125,
      "rewards/margins": 5.090624809265137,
      "rewards/rejected": -6.24609375,
      "step": 3350
    },
    {
      "epoch": 2.9505820338238524,
      "grad_norm": 14.228977050821788,
      "learning_rate": 2.6208260105448157e-07,
      "logits/chosen": -2.7085938453674316,
      "logits/rejected": -2.7796874046325684,
      "logps/chosen": -419.9750061035156,
      "logps/rejected": -424.17498779296875,
      "loss": 0.0495,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.32421875,
      "rewards/margins": 5.024218559265137,
      "rewards/rejected": -6.345703125,
      "step": 3360
    },
    {
      "epoch": 2.9593674500329454,
      "grad_norm": 13.998607772467793,
      "learning_rate": 2.598857644991213e-07,
      "logits/chosen": -2.8134765625,
      "logits/rejected": -2.849609375,
      "logps/chosen": -404.3999938964844,
      "logps/rejected": -426.375,
      "loss": 0.0523,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.1676146984100342,
      "rewards/margins": 4.8828125,
      "rewards/rejected": -6.053515434265137,
      "step": 3370
    },
    {
      "epoch": 2.968152866242038,
      "grad_norm": 23.18200455219762,
      "learning_rate": 2.57688927943761e-07,
      "logits/chosen": -2.7578125,
      "logits/rejected": -2.809375047683716,
      "logps/chosen": -440.4624938964844,
      "logps/rejected": -467.2749938964844,
      "loss": 0.0309,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.3624267578125,
      "rewards/margins": 5.358593940734863,
      "rewards/rejected": -6.71875,
      "step": 3380
    },
    {
      "epoch": 2.976938282451131,
      "grad_norm": 7.559402046018458,
      "learning_rate": 2.5549209138840064e-07,
      "logits/chosen": -2.817578077316284,
      "logits/rejected": -2.8531250953674316,
      "logps/chosen": -392.5375061035156,
      "logps/rejected": -416.0,
      "loss": 0.0405,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.423925757408142,
      "rewards/margins": 5.223046779632568,
      "rewards/rejected": -6.645703315734863,
      "step": 3390
    },
    {
      "epoch": 2.985723698660224,
      "grad_norm": 11.879826949685755,
      "learning_rate": 2.5329525483304045e-07,
      "logits/chosen": -2.8177733421325684,
      "logits/rejected": -2.886914014816284,
      "logps/chosen": -464.7250061035156,
      "logps/rejected": -442.2749938964844,
      "loss": 0.0577,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.505273461341858,
      "rewards/margins": 5.055468559265137,
      "rewards/rejected": -6.559374809265137,
      "step": 3400
    },
    {
      "epoch": 2.994509114869317,
      "grad_norm": 11.92856536311904,
      "learning_rate": 2.5109841827768016e-07,
      "logits/chosen": -2.837109327316284,
      "logits/rejected": -2.844921827316284,
      "logps/chosen": -403.95001220703125,
      "logps/rejected": -448.45001220703125,
      "loss": 0.0443,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.380834937095642,
      "rewards/margins": 5.041796684265137,
      "rewards/rejected": -6.421875,
      "step": 3410
    },
    {
      "epoch": 3.002635624862728,
      "grad_norm": 3.414167219866603,
      "learning_rate": 2.489015817223198e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.870354652404785,
      "logps/chosen": -375.270263671875,
      "logps/rejected": -414.3513488769531,
      "loss": 0.0331,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6340266466140747,
      "rewards/margins": 5.407094478607178,
      "rewards/rejected": -7.038851261138916,
      "step": 3420
    },
    {
      "epoch": 3.011421041071821,
      "grad_norm": 6.289020201694963,
      "learning_rate": 2.467047451669596e-07,
      "logits/chosen": -2.688281297683716,
      "logits/rejected": -2.8125,
      "logps/chosen": -422.25,
      "logps/rejected": -402.07501220703125,
      "loss": 0.0166,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8986450433731079,
      "rewards/margins": 5.842968940734863,
      "rewards/rejected": -6.745312690734863,
      "step": 3430
    },
    {
      "epoch": 3.0202064572809135,
      "grad_norm": 4.260053064288435,
      "learning_rate": 2.445079086115993e-07,
      "logits/chosen": -2.846874952316284,
      "logits/rejected": -2.885546922683716,
      "logps/chosen": -389.92498779296875,
      "logps/rejected": -434.375,
      "loss": 0.0098,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2520294189453125,
      "rewards/margins": 6.409375190734863,
      "rewards/rejected": -7.663281440734863,
      "step": 3440
    },
    {
      "epoch": 3.0289918734900065,
      "grad_norm": 4.699249305357544,
      "learning_rate": 2.42311072056239e-07,
      "logits/chosen": -2.8414063453674316,
      "logits/rejected": -2.9351563453674316,
      "logps/chosen": -359.1625061035156,
      "logps/rejected": -407.32501220703125,
      "loss": 0.0146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4541504383087158,
      "rewards/margins": 5.882031440734863,
      "rewards/rejected": -7.339062690734863,
      "step": 3450
    },
    {
      "epoch": 3.0377772896990995,
      "grad_norm": 3.926629678211093,
      "learning_rate": 2.401142355008787e-07,
      "logits/chosen": -2.8460936546325684,
      "logits/rejected": -2.8990235328674316,
      "logps/chosen": -389.61248779296875,
      "logps/rejected": -421.6000061035156,
      "loss": 0.0185,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.4955322742462158,
      "rewards/margins": 6.086718559265137,
      "rewards/rejected": -7.584374904632568,
      "step": 3460
    },
    {
      "epoch": 3.0465627059081926,
      "grad_norm": 11.544351045225936,
      "learning_rate": 2.3791739894551843e-07,
      "logits/chosen": -2.820117235183716,
      "logits/rejected": -2.9164061546325684,
      "logps/chosen": -432.3125,
      "logps/rejected": -468.95001220703125,
      "loss": 0.0207,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.033575415611267,
      "rewards/margins": 5.828515529632568,
      "rewards/rejected": -6.858593940734863,
      "step": 3470
    },
    {
      "epoch": 3.055348122117285,
      "grad_norm": 3.5989835439335045,
      "learning_rate": 2.3572056239015817e-07,
      "logits/chosen": -2.953906297683716,
      "logits/rejected": -2.9273438453674316,
      "logps/chosen": -383.7250061035156,
      "logps/rejected": -441.8500061035156,
      "loss": 0.0085,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.488037109375,
      "rewards/margins": 6.077343940734863,
      "rewards/rejected": -7.56640625,
      "step": 3480
    },
    {
      "epoch": 3.064133538326378,
      "grad_norm": 5.333461761872321,
      "learning_rate": 2.3352372583479788e-07,
      "logits/chosen": -2.942187547683716,
      "logits/rejected": -2.891406297683716,
      "logps/chosen": -417.17498779296875,
      "logps/rejected": -452.125,
      "loss": 0.0097,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6548583507537842,
      "rewards/margins": 6.189453125,
      "rewards/rejected": -7.842187404632568,
      "step": 3490
    },
    {
      "epoch": 3.072918954535471,
      "grad_norm": 4.595031603617876,
      "learning_rate": 2.313268892794376e-07,
      "logits/chosen": -2.873046875,
      "logits/rejected": -2.944140672683716,
      "logps/chosen": -375.6875,
      "logps/rejected": -414.95001220703125,
      "loss": 0.013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0995972156524658,
      "rewards/margins": 5.989843845367432,
      "rewards/rejected": -7.090624809265137,
      "step": 3500
    },
    {
      "epoch": 3.081704370744564,
      "grad_norm": 1.2388179221504463,
      "learning_rate": 2.2913005272407732e-07,
      "logits/chosen": -2.883593797683716,
      "logits/rejected": -3.076953172683716,
      "logps/chosen": -417.42498779296875,
      "logps/rejected": -425.7749938964844,
      "loss": 0.013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.247717261314392,
      "rewards/margins": 6.253125190734863,
      "rewards/rejected": -7.5,
      "step": 3510
    },
    {
      "epoch": 3.0904897869536567,
      "grad_norm": 9.772687423571334,
      "learning_rate": 2.2693321616871705e-07,
      "logits/chosen": -2.9251952171325684,
      "logits/rejected": -2.9585938453674316,
      "logps/chosen": -385.75,
      "logps/rejected": -378.8500061035156,
      "loss": 0.012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6288330554962158,
      "rewards/margins": 6.332812309265137,
      "rewards/rejected": -7.962500095367432,
      "step": 3520
    },
    {
      "epoch": 3.0992752031627497,
      "grad_norm": 5.716298938087962,
      "learning_rate": 2.2473637961335676e-07,
      "logits/chosen": -2.86328125,
      "logits/rejected": -2.915820360183716,
      "logps/chosen": -410.20001220703125,
      "logps/rejected": -420.29998779296875,
      "loss": 0.015,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.88818359375,
      "rewards/margins": 6.285937309265137,
      "rewards/rejected": -8.172656059265137,
      "step": 3530
    },
    {
      "epoch": 3.1080606193718427,
      "grad_norm": 13.604375684759066,
      "learning_rate": 2.2253954305799647e-07,
      "logits/chosen": -2.904296875,
      "logits/rejected": -3.041796922683716,
      "logps/chosen": -404.1499938964844,
      "logps/rejected": -425.625,
      "loss": 0.008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.758935570716858,
      "rewards/margins": 6.492968559265137,
      "rewards/rejected": -8.252344131469727,
      "step": 3540
    },
    {
      "epoch": 3.1168460355809358,
      "grad_norm": 6.29966255400896,
      "learning_rate": 2.203427065026362e-07,
      "logits/chosen": -2.8773436546325684,
      "logits/rejected": -2.94921875,
      "logps/chosen": -438.04998779296875,
      "logps/rejected": -446.45001220703125,
      "loss": 0.0113,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.972143530845642,
      "rewards/margins": 6.47265625,
      "rewards/rejected": -8.447656631469727,
      "step": 3550
    },
    {
      "epoch": 3.1256314517900288,
      "grad_norm": 4.241334863854754,
      "learning_rate": 2.181458699472759e-07,
      "logits/chosen": -2.8578124046325684,
      "logits/rejected": -2.9886717796325684,
      "logps/chosen": -468.5625,
      "logps/rejected": -456.04998779296875,
      "loss": 0.0111,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7622559070587158,
      "rewards/margins": 6.196093559265137,
      "rewards/rejected": -7.958593845367432,
      "step": 3560
    },
    {
      "epoch": 3.1344168679991213,
      "grad_norm": 5.03709696320225,
      "learning_rate": 2.1594903339191564e-07,
      "logits/chosen": -2.7724609375,
      "logits/rejected": -2.8490233421325684,
      "logps/chosen": -405.0249938964844,
      "logps/rejected": -425.7749938964844,
      "loss": 0.0104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9119141101837158,
      "rewards/margins": 6.514843940734863,
      "rewards/rejected": -8.430468559265137,
      "step": 3570
    },
    {
      "epoch": 3.1432022842082143,
      "grad_norm": 12.862553685016508,
      "learning_rate": 2.1375219683655535e-07,
      "logits/chosen": -2.91796875,
      "logits/rejected": -2.948437452316284,
      "logps/chosen": -395.3500061035156,
      "logps/rejected": -406.7250061035156,
      "loss": 0.0219,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.904394507408142,
      "rewards/margins": 6.072656154632568,
      "rewards/rejected": -7.975781440734863,
      "step": 3580
    },
    {
      "epoch": 3.1519877004173074,
      "grad_norm": 14.681117053466906,
      "learning_rate": 2.1155536028119509e-07,
      "logits/chosen": -2.872851610183716,
      "logits/rejected": -3.01953125,
      "logps/chosen": -391.17498779296875,
      "logps/rejected": -406.82501220703125,
      "loss": 0.0161,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.8270995616912842,
      "rewards/margins": 6.517187595367432,
      "rewards/rejected": -8.342968940734863,
      "step": 3590
    },
    {
      "epoch": 3.1607731166264004,
      "grad_norm": 10.125331769087346,
      "learning_rate": 2.093585237258348e-07,
      "logits/chosen": -2.941601514816284,
      "logits/rejected": -2.952343702316284,
      "logps/chosen": -379.5249938964844,
      "logps/rejected": -432.79998779296875,
      "loss": 0.0187,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.103320360183716,
      "rewards/margins": 6.426953315734863,
      "rewards/rejected": -8.53515625,
      "step": 3600
    },
    {
      "epoch": 3.169558532835493,
      "grad_norm": 3.5498481736157808,
      "learning_rate": 2.071616871704745e-07,
      "logits/chosen": -2.909374952316284,
      "logits/rejected": -2.931445360183716,
      "logps/chosen": -400.4750061035156,
      "logps/rejected": -412.6499938964844,
      "loss": 0.0137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7677733898162842,
      "rewards/margins": 6.185156345367432,
      "rewards/rejected": -7.948437690734863,
      "step": 3610
    },
    {
      "epoch": 3.178343949044586,
      "grad_norm": 8.99115337492826,
      "learning_rate": 2.0496485061511424e-07,
      "logits/chosen": -2.9693360328674316,
      "logits/rejected": -2.953906297683716,
      "logps/chosen": -361.8500061035156,
      "logps/rejected": -365.95001220703125,
      "loss": 0.0119,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.4791502952575684,
      "rewards/margins": 6.103515625,
      "rewards/rejected": -8.5859375,
      "step": 3620
    },
    {
      "epoch": 3.187129365253679,
      "grad_norm": 12.529872613897242,
      "learning_rate": 2.0276801405975394e-07,
      "logits/chosen": -2.931640625,
      "logits/rejected": -3.005859375,
      "logps/chosen": -391.88751220703125,
      "logps/rejected": -448.6000061035156,
      "loss": 0.0146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.300549268722534,
      "rewards/margins": 6.403906345367432,
      "rewards/rejected": -8.703125,
      "step": 3630
    },
    {
      "epoch": 3.195914781462772,
      "grad_norm": 3.1285598074374854,
      "learning_rate": 2.0057117750439368e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.8759765625,
      "logps/chosen": -416.92498779296875,
      "logps/rejected": -423.0,
      "loss": 0.0086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5449707508087158,
      "rewards/margins": 6.640625,
      "rewards/rejected": -8.186718940734863,
      "step": 3640
    },
    {
      "epoch": 3.2047001976718645,
      "grad_norm": 2.1255927180815677,
      "learning_rate": 1.9837434094903339e-07,
      "logits/chosen": -2.775195360183716,
      "logits/rejected": -2.859375,
      "logps/chosen": -421.9750061035156,
      "logps/rejected": -418.125,
      "loss": 0.0077,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9755859375,
      "rewards/margins": 6.64453125,
      "rewards/rejected": -8.618749618530273,
      "step": 3650
    },
    {
      "epoch": 3.2134856138809575,
      "grad_norm": 1.8954942162621815,
      "learning_rate": 1.9617750439367312e-07,
      "logits/chosen": -2.928515672683716,
      "logits/rejected": -3.068359375,
      "logps/chosen": -419.6625061035156,
      "logps/rejected": -396.0,
      "loss": 0.0223,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.1656494140625,
      "rewards/margins": 6.200781345367432,
      "rewards/rejected": -8.364843368530273,
      "step": 3660
    },
    {
      "epoch": 3.2222710300900506,
      "grad_norm": 4.6284479414824,
      "learning_rate": 1.9398066783831283e-07,
      "logits/chosen": -2.806640625,
      "logits/rejected": -2.9722657203674316,
      "logps/chosen": -428.2124938964844,
      "logps/rejected": -449.3500061035156,
      "loss": 0.0173,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.0701661109924316,
      "rewards/margins": 6.727734565734863,
      "rewards/rejected": -8.793749809265137,
      "step": 3670
    },
    {
      "epoch": 3.2310564462991436,
      "grad_norm": 6.832045018202729,
      "learning_rate": 1.9178383128295253e-07,
      "logits/chosen": -3.007031202316284,
      "logits/rejected": -3.07421875,
      "logps/chosen": -424.79998779296875,
      "logps/rejected": -469.45001220703125,
      "loss": 0.0094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.7050414085388184,
      "rewards/margins": 6.456250190734863,
      "rewards/rejected": -9.161718368530273,
      "step": 3680
    },
    {
      "epoch": 3.239841862508236,
      "grad_norm": 14.760961824109547,
      "learning_rate": 1.8958699472759227e-07,
      "logits/chosen": -2.903125047683716,
      "logits/rejected": -2.935546875,
      "logps/chosen": -416.5,
      "logps/rejected": -416.0249938964844,
      "loss": 0.0109,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.148120164871216,
      "rewards/margins": 6.365624904632568,
      "rewards/rejected": -8.514843940734863,
      "step": 3690
    },
    {
      "epoch": 3.248627278717329,
      "grad_norm": 8.05248202178939,
      "learning_rate": 1.8739015817223198e-07,
      "logits/chosen": -2.820117235183716,
      "logits/rejected": -2.9115233421325684,
      "logps/chosen": -417.92498779296875,
      "logps/rejected": -444.0,
      "loss": 0.014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.2945799827575684,
      "rewards/margins": 6.801562309265137,
      "rewards/rejected": -9.09765625,
      "step": 3700
    },
    {
      "epoch": 3.257412694926422,
      "grad_norm": 6.978450349612142,
      "learning_rate": 1.851933216168717e-07,
      "logits/chosen": -2.886914014816284,
      "logits/rejected": -2.950390577316284,
      "logps/chosen": -421.0249938964844,
      "logps/rejected": -436.20001220703125,
      "loss": 0.0137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.368603467941284,
      "rewards/margins": 6.510156154632568,
      "rewards/rejected": -8.87890625,
      "step": 3710
    },
    {
      "epoch": 3.266198111135515,
      "grad_norm": 29.74071686461494,
      "learning_rate": 1.8299648506151142e-07,
      "logits/chosen": -2.844531297683716,
      "logits/rejected": -3.0042967796325684,
      "logps/chosen": -468.0249938964844,
      "logps/rejected": -465.07501220703125,
      "loss": 0.0161,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9578368663787842,
      "rewards/margins": 6.37890625,
      "rewards/rejected": -8.339062690734863,
      "step": 3720
    },
    {
      "epoch": 3.2749835273446077,
      "grad_norm": 6.110397355445672,
      "learning_rate": 1.8079964850615115e-07,
      "logits/chosen": -2.952343702316284,
      "logits/rejected": -3.004687547683716,
      "logps/chosen": -380.79998779296875,
      "logps/rejected": -392.6499938964844,
      "loss": 0.0116,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.8979980945587158,
      "rewards/margins": 6.548437595367432,
      "rewards/rejected": -8.444531440734863,
      "step": 3730
    },
    {
      "epoch": 3.2837689435537007,
      "grad_norm": 6.6451137142718455,
      "learning_rate": 1.7860281195079086e-07,
      "logits/chosen": -2.779101610183716,
      "logits/rejected": -2.9554686546325684,
      "logps/chosen": -417.42498779296875,
      "logps/rejected": -398.1499938964844,
      "loss": 0.009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.0990967750549316,
      "rewards/margins": 6.416015625,
      "rewards/rejected": -8.518750190734863,
      "step": 3740
    },
    {
      "epoch": 3.2925543597627938,
      "grad_norm": 8.896042632173886,
      "learning_rate": 1.7640597539543057e-07,
      "logits/chosen": -2.936328172683716,
      "logits/rejected": -2.9839844703674316,
      "logps/chosen": -360.38751220703125,
      "logps/rejected": -404.25,
      "loss": 0.0138,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.3913207054138184,
      "rewards/margins": 6.459374904632568,
      "rewards/rejected": -8.8515625,
      "step": 3750
    },
    {
      "epoch": 3.3013397759718868,
      "grad_norm": 20.002846956095095,
      "learning_rate": 1.742091388400703e-07,
      "logits/chosen": -2.9136719703674316,
      "logits/rejected": -2.885546922683716,
      "logps/chosen": -398.7250061035156,
      "logps/rejected": -424.25,
      "loss": 0.0137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.4557127952575684,
      "rewards/margins": 6.453515529632568,
      "rewards/rejected": -8.91015625,
      "step": 3760
    },
    {
      "epoch": 3.31012519218098,
      "grad_norm": 4.5148052868126705,
      "learning_rate": 1.7201230228471e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.9957032203674316,
      "logps/chosen": -395.79998779296875,
      "logps/rejected": -437.7749938964844,
      "loss": 0.0219,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.179980516433716,
      "rewards/margins": 6.603906154632568,
      "rewards/rejected": -8.783594131469727,
      "step": 3770
    },
    {
      "epoch": 3.3189106083900723,
      "grad_norm": 2.9357094309582292,
      "learning_rate": 1.6981546572934974e-07,
      "logits/chosen": -2.873242139816284,
      "logits/rejected": -2.8207030296325684,
      "logps/chosen": -437.70001220703125,
      "logps/rejected": -470.07501220703125,
      "loss": 0.0092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.273193359375,
      "rewards/margins": 6.735937595367432,
      "rewards/rejected": -9.010156631469727,
      "step": 3780
    },
    {
      "epoch": 3.3276960245991654,
      "grad_norm": 9.381844546771463,
      "learning_rate": 1.6761862917398945e-07,
      "logits/chosen": -2.8167967796325684,
      "logits/rejected": -2.956249952316284,
      "logps/chosen": -396.0,
      "logps/rejected": -393.70001220703125,
      "loss": 0.0102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.0227904319763184,
      "rewards/margins": 6.432031154632568,
      "rewards/rejected": -8.454687118530273,
      "step": 3790
    },
    {
      "epoch": 3.3364814408082584,
      "grad_norm": 6.415496207405547,
      "learning_rate": 1.6542179261862919e-07,
      "logits/chosen": -2.797656297683716,
      "logits/rejected": -2.8988280296325684,
      "logps/chosen": -413.6875,
      "logps/rejected": -464.20001220703125,
      "loss": 0.0111,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.0936522483825684,
      "rewards/margins": 6.2890625,
      "rewards/rejected": -8.384374618530273,
      "step": 3800
    },
    {
      "epoch": 3.3452668570173514,
      "grad_norm": 4.492741488682358,
      "learning_rate": 1.632249560632689e-07,
      "logits/chosen": -2.878124952316284,
      "logits/rejected": -2.9681639671325684,
      "logps/chosen": -492.04998779296875,
      "logps/rejected": -488.5249938964844,
      "loss": 0.0125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.1321778297424316,
      "rewards/margins": 6.8671875,
      "rewards/rejected": -9.003125190734863,
      "step": 3810
    },
    {
      "epoch": 3.354052273226444,
      "grad_norm": 1.7388551687550062,
      "learning_rate": 1.610281195079086e-07,
      "logits/chosen": -2.943554639816284,
      "logits/rejected": -2.9525389671325684,
      "logps/chosen": -422.5249938964844,
      "logps/rejected": -419.375,
      "loss": 0.0125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.35693359375,
      "rewards/margins": 6.696875095367432,
      "rewards/rejected": -9.053906440734863,
      "step": 3820
    },
    {
      "epoch": 3.362837689435537,
      "grad_norm": 1.753775377156349,
      "learning_rate": 1.5883128295254834e-07,
      "logits/chosen": -2.844921827316284,
      "logits/rejected": -2.9228515625,
      "logps/chosen": -460.125,
      "logps/rejected": -481.42498779296875,
      "loss": 0.0084,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.228564500808716,
      "rewards/margins": 6.469531059265137,
      "rewards/rejected": -8.6953125,
      "step": 3830
    },
    {
      "epoch": 3.37162310564463,
      "grad_norm": 2.0392796743406865,
      "learning_rate": 1.5663444639718804e-07,
      "logits/chosen": -2.8404297828674316,
      "logits/rejected": -2.958203077316284,
      "logps/chosen": -418.63751220703125,
      "logps/rejected": -428.25,
      "loss": 0.0088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.3414063453674316,
      "rewards/margins": 6.467187404632568,
      "rewards/rejected": -8.814844131469727,
      "step": 3840
    },
    {
      "epoch": 3.380408521853723,
      "grad_norm": 23.652229813085096,
      "learning_rate": 1.5443760984182778e-07,
      "logits/chosen": -2.8921875953674316,
      "logits/rejected": -3.0394530296325684,
      "logps/chosen": -405.98748779296875,
      "logps/rejected": -410.82501220703125,
      "loss": 0.0156,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.6180663108825684,
      "rewards/margins": 6.499218940734863,
      "rewards/rejected": -9.116406440734863,
      "step": 3850
    },
    {
      "epoch": 3.3891939380628155,
      "grad_norm": 2.8885719582268052,
      "learning_rate": 1.5224077328646749e-07,
      "logits/chosen": -2.901562452316284,
      "logits/rejected": -3.044140577316284,
      "logps/chosen": -438.3500061035156,
      "logps/rejected": -432.8999938964844,
      "loss": 0.011,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.806054711341858,
      "rewards/margins": 6.732031345367432,
      "rewards/rejected": -8.534375190734863,
      "step": 3860
    },
    {
      "epoch": 3.3979793542719086,
      "grad_norm": 3.9422963662675348,
      "learning_rate": 1.5004393673110722e-07,
      "logits/chosen": -2.919140577316284,
      "logits/rejected": -2.948046922683716,
      "logps/chosen": -396.86248779296875,
      "logps/rejected": -462.5,
      "loss": 0.0119,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.2069334983825684,
      "rewards/margins": 6.598437309265137,
      "rewards/rejected": -8.80859375,
      "step": 3870
    },
    {
      "epoch": 3.4067647704810016,
      "grad_norm": 84.18141029989187,
      "learning_rate": 1.4784710017574693e-07,
      "logits/chosen": -2.8173828125,
      "logits/rejected": -2.9281249046325684,
      "logps/chosen": -423.1000061035156,
      "logps/rejected": -431.7250061035156,
      "loss": 0.0217,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.394970655441284,
      "rewards/margins": 6.682031154632568,
      "rewards/rejected": -9.078125,
      "step": 3880
    },
    {
      "epoch": 3.4155501866900946,
      "grad_norm": 11.239899866740663,
      "learning_rate": 1.4565026362038664e-07,
      "logits/chosen": -2.8173828125,
      "logits/rejected": -2.9828124046325684,
      "logps/chosen": -424.0249938964844,
      "logps/rejected": -423.6499938964844,
      "loss": 0.0115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.294921875,
      "rewards/margins": 6.548437595367432,
      "rewards/rejected": -8.83984375,
      "step": 3890
    },
    {
      "epoch": 3.4243356028991876,
      "grad_norm": 7.719857916271868,
      "learning_rate": 1.4345342706502637e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.9937500953674316,
      "logps/chosen": -453.0375061035156,
      "logps/rejected": -452.7250061035156,
      "loss": 0.0104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.360644578933716,
      "rewards/margins": 6.51953125,
      "rewards/rejected": -8.881250381469727,
      "step": 3900
    },
    {
      "epoch": 3.43312101910828,
      "grad_norm": 5.60664600185375,
      "learning_rate": 1.4125659050966608e-07,
      "logits/chosen": -2.867968797683716,
      "logits/rejected": -2.998046875,
      "logps/chosen": -445.92498779296875,
      "logps/rejected": -482.79998779296875,
      "loss": 0.0138,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.4117674827575684,
      "rewards/margins": 6.739062309265137,
      "rewards/rejected": -9.154687881469727,
      "step": 3910
    },
    {
      "epoch": 3.441906435317373,
      "grad_norm": 1.3962351841471203,
      "learning_rate": 1.390597539543058e-07,
      "logits/chosen": -2.8490233421325684,
      "logits/rejected": -2.903125047683716,
      "logps/chosen": -380.7250061035156,
      "logps/rejected": -451.8999938964844,
      "loss": 0.0088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.214160203933716,
      "rewards/margins": 7.141406059265137,
      "rewards/rejected": -9.353906631469727,
      "step": 3920
    },
    {
      "epoch": 3.450691851526466,
      "grad_norm": 1.6577982576791694,
      "learning_rate": 1.3686291739894552e-07,
      "logits/chosen": -2.8251953125,
      "logits/rejected": -2.9339842796325684,
      "logps/chosen": -417.54998779296875,
      "logps/rejected": -444.6499938964844,
      "loss": 0.0118,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.944848656654358,
      "rewards/margins": 6.640625,
      "rewards/rejected": -8.586718559265137,
      "step": 3930
    },
    {
      "epoch": 3.4594772677355587,
      "grad_norm": 25.281563484698918,
      "learning_rate": 1.3466608084358525e-07,
      "logits/chosen": -2.91796875,
      "logits/rejected": -3.001171827316284,
      "logps/chosen": -354.2250061035156,
      "logps/rejected": -412.01251220703125,
      "loss": 0.0159,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.9141356945037842,
      "rewards/margins": 6.53125,
      "rewards/rejected": -8.444531440734863,
      "step": 3940
    },
    {
      "epoch": 3.4682626839446518,
      "grad_norm": 13.46279073529727,
      "learning_rate": 1.3246924428822496e-07,
      "logits/chosen": -2.9458985328674316,
      "logits/rejected": -3.017578125,
      "logps/chosen": -414.54998779296875,
      "logps/rejected": -442.3500061035156,
      "loss": 0.0103,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.3888182640075684,
      "rewards/margins": 6.607812404632568,
      "rewards/rejected": -8.99609375,
      "step": 3950
    },
    {
      "epoch": 3.4770481001537448,
      "grad_norm": 13.283840017210744,
      "learning_rate": 1.3027240773286467e-07,
      "logits/chosen": -2.9566407203674316,
      "logits/rejected": -3.0472655296325684,
      "logps/chosen": -438.61248779296875,
      "logps/rejected": -442.70001220703125,
      "loss": 0.011,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.306640625,
      "rewards/margins": 6.69921875,
      "rewards/rejected": -9.005468368530273,
      "step": 3960
    },
    {
      "epoch": 3.485833516362838,
      "grad_norm": 13.98021260788341,
      "learning_rate": 1.280755711775044e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.999218702316284,
      "logps/chosen": -448.29998779296875,
      "logps/rejected": -440.07501220703125,
      "loss": 0.0144,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.4383301734924316,
      "rewards/margins": 6.318749904632568,
      "rewards/rejected": -8.759374618530273,
      "step": 3970
    },
    {
      "epoch": 3.494618932571931,
      "grad_norm": 5.294826123320576,
      "learning_rate": 1.258787346221441e-07,
      "logits/chosen": -3.0679688453674316,
      "logits/rejected": -3.1722655296325684,
      "logps/chosen": -403.6000061035156,
      "logps/rejected": -426.17498779296875,
      "loss": 0.0092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.487597703933716,
      "rewards/margins": 6.410937309265137,
      "rewards/rejected": -8.896875381469727,
      "step": 3980
    },
    {
      "epoch": 3.5034043487810234,
      "grad_norm": 9.290736019427845,
      "learning_rate": 1.2368189806678382e-07,
      "logits/chosen": -2.930468797683716,
      "logits/rejected": -3.123828172683716,
      "logps/chosen": -456.63751220703125,
      "logps/rejected": -449.54998779296875,
      "loss": 0.0145,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.302685499191284,
      "rewards/margins": 6.6328125,
      "rewards/rejected": -8.936718940734863,
      "step": 3990
    },
    {
      "epoch": 3.5121897649901164,
      "grad_norm": 3.545596978830052,
      "learning_rate": 1.2148506151142355e-07,
      "logits/chosen": -2.948046922683716,
      "logits/rejected": -2.9732422828674316,
      "logps/chosen": -386.4375,
      "logps/rejected": -440.6000061035156,
      "loss": 0.017,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.4395508766174316,
      "rewards/margins": 6.213671684265137,
      "rewards/rejected": -8.64453125,
      "step": 4000
    },
    {
      "epoch": 3.5209751811992094,
      "grad_norm": 7.429504580264419,
      "learning_rate": 1.1928822495606326e-07,
      "logits/chosen": -2.97265625,
      "logits/rejected": -2.998046875,
      "logps/chosen": -425.48748779296875,
      "logps/rejected": -456.92498779296875,
      "loss": 0.0094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.03778076171875,
      "rewards/margins": 6.51953125,
      "rewards/rejected": -8.557031631469727,
      "step": 4010
    },
    {
      "epoch": 3.529760597408302,
      "grad_norm": 5.741887296154292,
      "learning_rate": 1.1709138840070298e-07,
      "logits/chosen": -2.876953125,
      "logits/rejected": -3.038281202316284,
      "logps/chosen": -437.4750061035156,
      "logps/rejected": -432.9750061035156,
      "loss": 0.0132,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.098010301589966,
      "rewards/margins": 6.250781059265137,
      "rewards/rejected": -8.344531059265137,
      "step": 4020
    },
    {
      "epoch": 3.538546013617395,
      "grad_norm": 11.782304090539055,
      "learning_rate": 1.148945518453427e-07,
      "logits/chosen": -2.845507860183716,
      "logits/rejected": -2.9156250953674316,
      "logps/chosen": -378.1499938964844,
      "logps/rejected": -407.9750061035156,
      "loss": 0.0261,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.1287841796875,
      "rewards/margins": 6.394140720367432,
      "rewards/rejected": -8.526562690734863,
      "step": 4030
    },
    {
      "epoch": 3.547331429826488,
      "grad_norm": 18.683475749091517,
      "learning_rate": 1.1269771528998242e-07,
      "logits/chosen": -2.9312500953674316,
      "logits/rejected": -3.083203077316284,
      "logps/chosen": -356.2250061035156,
      "logps/rejected": -405.92498779296875,
      "loss": 0.0071,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.0382080078125,
      "rewards/margins": 6.729687690734863,
      "rewards/rejected": -8.770312309265137,
      "step": 4040
    },
    {
      "epoch": 3.556116846035581,
      "grad_norm": 4.881669812619082,
      "learning_rate": 1.1050087873462213e-07,
      "logits/chosen": -2.9183592796325684,
      "logits/rejected": -2.951367139816284,
      "logps/chosen": -375.67498779296875,
      "logps/rejected": -473.32501220703125,
      "loss": 0.0108,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.174060106277466,
      "rewards/margins": 6.457812309265137,
      "rewards/rejected": -8.639062881469727,
      "step": 4050
    },
    {
      "epoch": 3.564902262244674,
      "grad_norm": 5.228020584922703,
      "learning_rate": 1.0830404217926185e-07,
      "logits/chosen": -2.9535155296325684,
      "logits/rejected": -3.003124952316284,
      "logps/chosen": -449.6499938964844,
      "logps/rejected": -457.92498779296875,
      "loss": 0.0077,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.5882325172424316,
      "rewards/margins": 6.579687595367432,
      "rewards/rejected": -9.166406631469727,
      "step": 4060
    },
    {
      "epoch": 3.5736876784537666,
      "grad_norm": 9.120020223430558,
      "learning_rate": 1.0610720562390157e-07,
      "logits/chosen": -2.975390672683716,
      "logits/rejected": -2.960156202316284,
      "logps/chosen": -392.51251220703125,
      "logps/rejected": -429.3500061035156,
      "loss": 0.0092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.981689453125,
      "rewards/margins": 6.592968940734863,
      "rewards/rejected": -8.574999809265137,
      "step": 4070
    },
    {
      "epoch": 3.5824730946628596,
      "grad_norm": 3.348276560872841,
      "learning_rate": 1.039103690685413e-07,
      "logits/chosen": -2.950000047683716,
      "logits/rejected": -3.065234422683716,
      "logps/chosen": -487.95001220703125,
      "logps/rejected": -500.2250061035156,
      "loss": 0.0066,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.258862257003784,
      "rewards/margins": 6.715624809265137,
      "rewards/rejected": -8.974218368530273,
      "step": 4080
    },
    {
      "epoch": 3.5912585108719526,
      "grad_norm": 31.524756488191674,
      "learning_rate": 1.0171353251318102e-07,
      "logits/chosen": -2.994921922683716,
      "logits/rejected": -2.994140625,
      "logps/chosen": -390.6000061035156,
      "logps/rejected": -456.95001220703125,
      "loss": 0.0099,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.4727783203125,
      "rewards/margins": 6.291406154632568,
      "rewards/rejected": -8.764843940734863,
      "step": 4090
    },
    {
      "epoch": 3.6000439270810456,
      "grad_norm": 2.4671935117017245,
      "learning_rate": 9.951669595782074e-08,
      "logits/chosen": -2.884765625,
      "logits/rejected": -3.0199217796325684,
      "logps/chosen": -389.75,
      "logps/rejected": -395.1499938964844,
      "loss": 0.0088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.1677613258361816,
      "rewards/margins": 6.639843940734863,
      "rewards/rejected": -8.807812690734863,
      "step": 4100
    },
    {
      "epoch": 3.6088293432901386,
      "grad_norm": 4.563502896404394,
      "learning_rate": 9.731985940246046e-08,
      "logits/chosen": -3.0394530296325684,
      "logits/rejected": -3.028125047683716,
      "logps/chosen": -402.32501220703125,
      "logps/rejected": -459.3500061035156,
      "loss": 0.0128,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.673840284347534,
      "rewards/margins": 6.317578315734863,
      "rewards/rejected": -8.996874809265137,
      "step": 4110
    },
    {
      "epoch": 3.617614759499231,
      "grad_norm": 3.3496554519079518,
      "learning_rate": 9.512302284710017e-08,
      "logits/chosen": -2.9554686546325684,
      "logits/rejected": -3.033203125,
      "logps/chosen": -480.54998779296875,
      "logps/rejected": -436.82501220703125,
      "loss": 0.0127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.597705125808716,
      "rewards/margins": 6.335156440734863,
      "rewards/rejected": -8.9296875,
      "step": 4120
    },
    {
      "epoch": 3.626400175708324,
      "grad_norm": 6.2145508517135015,
      "learning_rate": 9.292618629173989e-08,
      "logits/chosen": -2.8515625,
      "logits/rejected": -3.005664110183716,
      "logps/chosen": -401.6499938964844,
      "logps/rejected": -400.75,
      "loss": 0.0102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.223681688308716,
      "rewards/margins": 6.604687690734863,
      "rewards/rejected": -8.832812309265137,
      "step": 4130
    },
    {
      "epoch": 3.635185591917417,
      "grad_norm": 4.1625150065100645,
      "learning_rate": 9.072934973637961e-08,
      "logits/chosen": -2.958984375,
      "logits/rejected": -2.961718797683716,
      "logps/chosen": -413.125,
      "logps/rejected": -433.20001220703125,
      "loss": 0.0108,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.1835570335388184,
      "rewards/margins": 6.200390815734863,
      "rewards/rejected": -8.380468368530273,
      "step": 4140
    },
    {
      "epoch": 3.6439710081265098,
      "grad_norm": 10.029568158841256,
      "learning_rate": 8.853251318101933e-08,
      "logits/chosen": -3.0171875953674316,
      "logits/rejected": -2.998828172683716,
      "logps/chosen": -397.2749938964844,
      "logps/rejected": -446.82501220703125,
      "loss": 0.0086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.2596678733825684,
      "rewards/margins": 6.573437690734863,
      "rewards/rejected": -8.8359375,
      "step": 4150
    },
    {
      "epoch": 3.6527564243356028,
      "grad_norm": 4.5199297955982765,
      "learning_rate": 8.633567662565905e-08,
      "logits/chosen": -3.005078077316284,
      "logits/rejected": -2.96875,
      "logps/chosen": -386.11248779296875,
      "logps/rejected": -424.0249938964844,
      "loss": 0.0104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.464062452316284,
      "rewards/margins": 6.662499904632568,
      "rewards/rejected": -9.125,
      "step": 4160
    },
    {
      "epoch": 3.661541840544696,
      "grad_norm": 11.150755889455755,
      "learning_rate": 8.413884007029877e-08,
      "logits/chosen": -2.986328125,
      "logits/rejected": -3.0199217796325684,
      "logps/chosen": -384.04998779296875,
      "logps/rejected": -396.3999938964844,
      "loss": 0.0137,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.092041015625,
      "rewards/margins": 6.4765625,
      "rewards/rejected": -8.563281059265137,
      "step": 4170
    },
    {
      "epoch": 3.670327256753789,
      "grad_norm": 5.723530788328881,
      "learning_rate": 8.194200351493849e-08,
      "logits/chosen": -2.986328125,
      "logits/rejected": -3.07421875,
      "logps/chosen": -405.01251220703125,
      "logps/rejected": -415.6000061035156,
      "loss": 0.0122,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.4269165992736816,
      "rewards/margins": 7.194531440734863,
      "rewards/rejected": -9.624218940734863,
      "step": 4180
    },
    {
      "epoch": 3.679112672962882,
      "grad_norm": 3.335175121884927,
      "learning_rate": 7.97451669595782e-08,
      "logits/chosen": -2.9234375953674316,
      "logits/rejected": -3.061718702316284,
      "logps/chosen": -430.0,
      "logps/rejected": -427.6000061035156,
      "loss": 0.0082,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.419726610183716,
      "rewards/margins": 6.5390625,
      "rewards/rejected": -8.95703125,
      "step": 4190
    },
    {
      "epoch": 3.6878980891719744,
      "grad_norm": 6.841196350155745,
      "learning_rate": 7.754833040421792e-08,
      "logits/chosen": -2.974609375,
      "logits/rejected": -3.0078125,
      "logps/chosen": -452.2749938964844,
      "logps/rejected": -442.32501220703125,
      "loss": 0.0104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.2720704078674316,
      "rewards/margins": 7.213281154632568,
      "rewards/rejected": -9.482812881469727,
      "step": 4200
    },
    {
      "epoch": 3.6966835053810674,
      "grad_norm": 6.00029841347423,
      "learning_rate": 7.535149384885764e-08,
      "logits/chosen": -2.9214844703674316,
      "logits/rejected": -2.948437452316284,
      "logps/chosen": -427.625,
      "logps/rejected": -455.5249938964844,
      "loss": 0.0138,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.570507764816284,
      "rewards/margins": 6.546093940734863,
      "rewards/rejected": -9.1171875,
      "step": 4210
    },
    {
      "epoch": 3.7054689215901604,
      "grad_norm": 2.574295193415827,
      "learning_rate": 7.315465729349736e-08,
      "logits/chosen": -2.9769530296325684,
      "logits/rejected": -3.048046827316284,
      "logps/chosen": -419.5,
      "logps/rejected": -445.45001220703125,
      "loss": 0.0128,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.3653931617736816,
      "rewards/margins": 6.615624904632568,
      "rewards/rejected": -8.98046875,
      "step": 4220
    },
    {
      "epoch": 3.7142543377992534,
      "grad_norm": 8.879383011953129,
      "learning_rate": 7.095782073813708e-08,
      "logits/chosen": -3.001953125,
      "logits/rejected": -3.008593797683716,
      "logps/chosen": -395.625,
      "logps/rejected": -424.4750061035156,
      "loss": 0.0102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.3426756858825684,
      "rewards/margins": 6.328906059265137,
      "rewards/rejected": -8.672656059265137,
      "step": 4230
    },
    {
      "epoch": 3.7230397540083464,
      "grad_norm": 3.2546772825649093,
      "learning_rate": 6.87609841827768e-08,
      "logits/chosen": -2.9273438453674316,
      "logits/rejected": -2.9957032203674316,
      "logps/chosen": -420.9125061035156,
      "logps/rejected": -426.20001220703125,
      "loss": 0.0087,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.1610107421875,
      "rewards/margins": 6.596875190734863,
      "rewards/rejected": -8.754687309265137,
      "step": 4240
    },
    {
      "epoch": 3.731825170217439,
      "grad_norm": 5.250002306370726,
      "learning_rate": 6.656414762741652e-08,
      "logits/chosen": -2.87890625,
      "logits/rejected": -2.8667969703674316,
      "logps/chosen": -400.8125,
      "logps/rejected": -417.17498779296875,
      "loss": 0.0147,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.0886383056640625,
      "rewards/margins": 6.582812309265137,
      "rewards/rejected": -8.672656059265137,
      "step": 4250
    },
    {
      "epoch": 3.740610586426532,
      "grad_norm": 2.960788817836322,
      "learning_rate": 6.436731107205623e-08,
      "logits/chosen": -2.884765625,
      "logits/rejected": -3.084765672683716,
      "logps/chosen": -456.6000061035156,
      "logps/rejected": -405.32501220703125,
      "loss": 0.0098,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.474804639816284,
      "rewards/margins": 6.671093940734863,
      "rewards/rejected": -9.145312309265137,
      "step": 4260
    },
    {
      "epoch": 3.749396002635625,
      "grad_norm": 6.973510642120954,
      "learning_rate": 6.217047451669595e-08,
      "logits/chosen": -2.9496092796325684,
      "logits/rejected": -2.957812547683716,
      "logps/chosen": -404.95001220703125,
      "logps/rejected": -453.07501220703125,
      "loss": 0.0094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.207080125808716,
      "rewards/margins": 6.5546875,
      "rewards/rejected": -8.765625,
      "step": 4270
    },
    {
      "epoch": 3.7581814188447176,
      "grad_norm": 7.071302412681325,
      "learning_rate": 5.997363796133567e-08,
      "logits/chosen": -2.863476514816284,
      "logits/rejected": -2.982617139816284,
      "logps/chosen": -393.125,
      "logps/rejected": -409.57501220703125,
      "loss": 0.014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.145703077316284,
      "rewards/margins": 6.34765625,
      "rewards/rejected": -8.495312690734863,
      "step": 4280
    },
    {
      "epoch": 3.7669668350538106,
      "grad_norm": 11.247264295982855,
      "learning_rate": 5.7776801405975395e-08,
      "logits/chosen": -2.9320311546325684,
      "logits/rejected": -2.9761719703674316,
      "logps/chosen": -385.45001220703125,
      "logps/rejected": -426.3999938964844,
      "loss": 0.0107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.969531297683716,
      "rewards/margins": 6.588281154632568,
      "rewards/rejected": -9.561718940734863,
      "step": 4290
    },
    {
      "epoch": 3.7757522512629036,
      "grad_norm": 4.129046597878558,
      "learning_rate": 5.5579964850615116e-08,
      "logits/chosen": -2.932812452316284,
      "logits/rejected": -3.0572266578674316,
      "logps/chosen": -459.45001220703125,
      "logps/rejected": -475.32501220703125,
      "loss": 0.0065,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.6207032203674316,
      "rewards/margins": 6.748437404632568,
      "rewards/rejected": -9.37109375,
      "step": 4300
    },
    {
      "epoch": 3.7845376674719966,
      "grad_norm": 2.974905956866584,
      "learning_rate": 5.338312829525484e-08,
      "logits/chosen": -2.888476610183716,
      "logits/rejected": -3.0152344703674316,
      "logps/chosen": -445.7749938964844,
      "logps/rejected": -426.32501220703125,
      "loss": 0.0173,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.535400390625,
      "rewards/margins": 6.443749904632568,
      "rewards/rejected": -8.979687690734863,
      "step": 4310
    },
    {
      "epoch": 3.7933230836810896,
      "grad_norm": 5.15823852652524,
      "learning_rate": 5.1186291739894545e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.989453077316284,
      "logps/chosen": -404.3500061035156,
      "logps/rejected": -457.2250061035156,
      "loss": 0.0083,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.1978759765625,
      "rewards/margins": 7.23828125,
      "rewards/rejected": -9.4375,
      "step": 4320
    },
    {
      "epoch": 3.802108499890182,
      "grad_norm": 4.129395128227712,
      "learning_rate": 4.8989455184534266e-08,
      "logits/chosen": -2.855273485183716,
      "logits/rejected": -2.9117188453674316,
      "logps/chosen": -450.20001220703125,
      "logps/rejected": -461.5249938964844,
      "loss": 0.0078,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.667285203933716,
      "rewards/margins": 6.509375095367432,
      "rewards/rejected": -9.175000190734863,
      "step": 4330
    },
    {
      "epoch": 3.810893916099275,
      "grad_norm": 16.239449145385244,
      "learning_rate": 4.679261862917399e-08,
      "logits/chosen": -2.9810547828674316,
      "logits/rejected": -3.139843702316284,
      "logps/chosen": -442.9750061035156,
      "logps/rejected": -458.1000061035156,
      "loss": 0.0404,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.283032178878784,
      "rewards/margins": 6.3828125,
      "rewards/rejected": -8.665624618530273,
      "step": 4340
    },
    {
      "epoch": 3.819679332308368,
      "grad_norm": 6.314421923517643,
      "learning_rate": 4.45957820738137e-08,
      "logits/chosen": -2.887500047683716,
      "logits/rejected": -2.996875047683716,
      "logps/chosen": -436.6499938964844,
      "logps/rejected": -452.1000061035156,
      "loss": 0.0085,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.3412108421325684,
      "rewards/margins": 6.763281345367432,
      "rewards/rejected": -9.102343559265137,
      "step": 4350
    },
    {
      "epoch": 3.8284647485174608,
      "grad_norm": 1.292047012435659,
      "learning_rate": 4.239894551845342e-08,
      "logits/chosen": -3.01953125,
      "logits/rejected": -3.186718702316284,
      "logps/chosen": -458.57501220703125,
      "logps/rejected": -468.25,
      "loss": 0.0077,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.5463500022888184,
      "rewards/margins": 6.655468940734863,
      "rewards/rejected": -9.197656631469727,
      "step": 4360
    },
    {
      "epoch": 3.837250164726554,
      "grad_norm": 11.792544787460898,
      "learning_rate": 4.020210896309314e-08,
      "logits/chosen": -2.865039110183716,
      "logits/rejected": -3.039843797683716,
      "logps/chosen": -418.75,
      "logps/rejected": -462.7250061035156,
      "loss": 0.0087,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.3321776390075684,
      "rewards/margins": 6.626562595367432,
      "rewards/rejected": -8.956250190734863,
      "step": 4370
    },
    {
      "epoch": 3.846035580935647,
      "grad_norm": 13.6264372851017,
      "learning_rate": 3.8005272407732864e-08,
      "logits/chosen": -2.9703125953674316,
      "logits/rejected": -3.147265672683716,
      "logps/chosen": -415.45001220703125,
      "logps/rejected": -437.79998779296875,
      "loss": 0.0128,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.2011871337890625,
      "rewards/margins": 6.735156059265137,
      "rewards/rejected": -8.9375,
      "step": 4380
    },
    {
      "epoch": 3.85482099714474,
      "grad_norm": 22.389556623210236,
      "learning_rate": 3.580843585237258e-08,
      "logits/chosen": -2.9478516578674316,
      "logits/rejected": -3.040820360183716,
      "logps/chosen": -427.9750061035156,
      "logps/rejected": -446.75,
      "loss": 0.0206,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.336352586746216,
      "rewards/margins": 6.365624904632568,
      "rewards/rejected": -8.697656631469727,
      "step": 4390
    },
    {
      "epoch": 3.863606413353833,
      "grad_norm": 26.416140034942558,
      "learning_rate": 3.36115992970123e-08,
      "logits/chosen": -3.0078125,
      "logits/rejected": -3.0078125,
      "logps/chosen": -401.82501220703125,
      "logps/rejected": -444.6499938964844,
      "loss": 0.0115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.6388182640075684,
      "rewards/margins": 6.575781345367432,
      "rewards/rejected": -9.213281631469727,
      "step": 4400
    },
    {
      "epoch": 3.8723918295629254,
      "grad_norm": 12.62830827903792,
      "learning_rate": 3.141476274165202e-08,
      "logits/chosen": -2.993945360183716,
      "logits/rejected": -2.977343797683716,
      "logps/chosen": -432.57501220703125,
      "logps/rejected": -450.7749938964844,
      "loss": 0.0207,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.479199171066284,
      "rewards/margins": 6.370312690734863,
      "rewards/rejected": -8.852343559265137,
      "step": 4410
    },
    {
      "epoch": 3.8811772457720184,
      "grad_norm": 19.288372541559557,
      "learning_rate": 2.9217926186291738e-08,
      "logits/chosen": -2.9427733421325684,
      "logits/rejected": -3.0625,
      "logps/chosen": -453.6499938964844,
      "logps/rejected": -466.75,
      "loss": 0.0162,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.220996141433716,
      "rewards/margins": 6.371484279632568,
      "rewards/rejected": -8.591405868530273,
      "step": 4420
    },
    {
      "epoch": 3.8899626619811114,
      "grad_norm": 3.2891694847409605,
      "learning_rate": 2.7021089630931456e-08,
      "logits/chosen": -2.9287109375,
      "logits/rejected": -3.0033202171325684,
      "logps/chosen": -404.25,
      "logps/rejected": -405.79998779296875,
      "loss": 0.01,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.349780321121216,
      "rewards/margins": 6.497265815734863,
      "rewards/rejected": -8.846094131469727,
      "step": 4430
    },
    {
      "epoch": 3.8987480781902044,
      "grad_norm": 2.1062599881263995,
      "learning_rate": 2.4824253075571177e-08,
      "logits/chosen": -2.9857420921325684,
      "logits/rejected": -3.067187547683716,
      "logps/chosen": -394.20001220703125,
      "logps/rejected": -408.3999938964844,
      "loss": 0.0084,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.5130858421325684,
      "rewards/margins": 6.6953125,
      "rewards/rejected": -9.209375381469727,
      "step": 4440
    },
    {
      "epoch": 3.9075334943992974,
      "grad_norm": 13.126526740116192,
      "learning_rate": 2.2627416520210894e-08,
      "logits/chosen": -2.981250047683716,
      "logits/rejected": -3.0306639671325684,
      "logps/chosen": -408.6499938964844,
      "logps/rejected": -449.125,
      "loss": 0.0117,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.3833985328674316,
      "rewards/margins": 6.375,
      "rewards/rejected": -8.759374618530273,
      "step": 4450
    },
    {
      "epoch": 3.91631891060839,
      "grad_norm": 10.292362489150445,
      "learning_rate": 2.0430579964850612e-08,
      "logits/chosen": -2.9263672828674316,
      "logits/rejected": -3.0679688453674316,
      "logps/chosen": -448.2749938964844,
      "logps/rejected": -455.92498779296875,
      "loss": 0.0114,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.397656202316284,
      "rewards/margins": 6.407812595367432,
      "rewards/rejected": -8.80859375,
      "step": 4460
    },
    {
      "epoch": 3.925104326817483,
      "grad_norm": 55.74091872215601,
      "learning_rate": 1.8233743409490333e-08,
      "logits/chosen": -2.889843702316284,
      "logits/rejected": -2.964062452316284,
      "logps/chosen": -381.5249938964844,
      "logps/rejected": -406.75,
      "loss": 0.0208,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.169921875,
      "rewards/margins": 6.541406154632568,
      "rewards/rejected": -8.711718559265137,
      "step": 4470
    },
    {
      "epoch": 3.933889743026576,
      "grad_norm": 17.81712281583376,
      "learning_rate": 1.603690685413005e-08,
      "logits/chosen": -2.9095702171325684,
      "logits/rejected": -2.9361329078674316,
      "logps/chosen": -435.1499938964844,
      "logps/rejected": -440.1499938964844,
      "loss": 0.0123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.449462890625,
      "rewards/margins": 6.614062309265137,
      "rewards/rejected": -9.064844131469727,
      "step": 4480
    },
    {
      "epoch": 3.9426751592356686,
      "grad_norm": 32.88882405729128,
      "learning_rate": 1.3840070298769772e-08,
      "logits/chosen": -2.967968702316284,
      "logits/rejected": -3.0093750953674316,
      "logps/chosen": -395.2124938964844,
      "logps/rejected": -418.4375,
      "loss": 0.0136,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.110107421875,
      "rewards/margins": 6.424609184265137,
      "rewards/rejected": -8.534375190734863,
      "step": 4490
    },
    {
      "epoch": 3.9514605754447616,
      "grad_norm": 6.012617616489225,
      "learning_rate": 1.164323374340949e-08,
      "logits/chosen": -3.0121092796325684,
      "logits/rejected": -3.014453172683716,
      "logps/chosen": -401.4750061035156,
      "logps/rejected": -429.625,
      "loss": 0.0094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.702685594558716,
      "rewards/margins": 6.481249809265137,
      "rewards/rejected": -9.19140625,
      "step": 4500
    },
    {
      "epoch": 3.9602459916538546,
      "grad_norm": 3.7457791865262027,
      "learning_rate": 9.446397188049209e-09,
      "logits/chosen": -2.935742139816284,
      "logits/rejected": -2.9574217796325684,
      "logps/chosen": -435.625,
      "logps/rejected": -468.375,
      "loss": 0.0139,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.423388719558716,
      "rewards/margins": 6.62890625,
      "rewards/rejected": -9.050000190734863,
      "step": 4510
    },
    {
      "epoch": 3.9690314078629476,
      "grad_norm": 8.972886063672679,
      "learning_rate": 7.249560632688927e-09,
      "logits/chosen": -2.90234375,
      "logits/rejected": -2.993359327316284,
      "logps/chosen": -405.5,
      "logps/rejected": -441.0,
      "loss": 0.0125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.3037962913513184,
      "rewards/margins": 6.493750095367432,
      "rewards/rejected": -8.798437118530273,
      "step": 4520
    },
    {
      "epoch": 3.9778168240720406,
      "grad_norm": 7.360946037715389,
      "learning_rate": 5.0527240773286466e-09,
      "logits/chosen": -2.8824219703674316,
      "logits/rejected": -2.9253907203674316,
      "logps/chosen": -422.92498779296875,
      "logps/rejected": -500.45001220703125,
      "loss": 0.0107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.414746046066284,
      "rewards/margins": 7.224999904632568,
      "rewards/rejected": -9.642187118530273,
      "step": 4530
    },
    {
      "epoch": 3.986602240281133,
      "grad_norm": 3.3393018457240293,
      "learning_rate": 2.8558875219683655e-09,
      "logits/chosen": -2.865429639816284,
      "logits/rejected": -3.000781297683716,
      "logps/chosen": -401.70001220703125,
      "logps/rejected": -424.2749938964844,
      "loss": 0.0133,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.425463914871216,
      "rewards/margins": 6.544140815734863,
      "rewards/rejected": -8.96875,
      "step": 4540
    },
    {
      "epoch": 3.995387656490226,
      "grad_norm": 20.37707629766171,
      "learning_rate": 6.590509666080844e-10,
      "logits/chosen": -2.9820313453674316,
      "logits/rejected": -3.0859375,
      "logps/chosen": -414.25,
      "logps/rejected": -446.75,
      "loss": 0.0092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.125927686691284,
      "rewards/margins": 6.505468845367432,
      "rewards/rejected": -8.6328125,
      "step": 4550
    }
  ],
  "logging_steps": 10,
  "max_steps": 4552,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 4,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}